@memberjunction/ng-dashboards 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/AI/components/vectors/sensitive-fields.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Fields that must never be embedded into a vector index.
+ *
+ * Vectorizing a column copies its values into a third-party vector database, where they
+ * live outside the entity permissions that normally guard them and are retrievable by
+ * nearest-neighbour search rather than by an authorized query. For a password hash, a
+ * national insurance or social security number, a bank account or a salary, that is not a
+ * feature with a caveat — it is a disclosure.
+ *
+ * The field picker's suggestions come from a language model, which has no notion of which
+ * columns are sensitive and will happily propose `PasswordHash` if the use case seems to
+ * call for identity matching. So the refusal cannot live in the prompt: it has to be a
+ * property of the code on both sides of the model.
+ *
+ * Two layers, deliberately:
+ *
+ *   1. The model never sees these fields, on the entity or on its related entities, so it
+ *      cannot suggest them.
+ *   2. The saved template is re-checked, because the template is hand-editable after the
+ *      model returns and a user can type a field name the picker never offered.
+ *
+ * Layer 1 alone would be a prompt-shaped guarantee, which is no guarantee at all.
+ */
+/**
+ * True when this field name must not be embedded.
+ *
+ * Note `hash` and `auth` are deliberately broad. A column called `AuthorName` normalises to
+ * `authorname`, which contains `auth` — so it is refused. That is the trade accepted above:
+ * a visible false refusal the user can work around beats a silent disclosure they cannot.
+ */
+export declare function IsSensitiveFieldName(name: string | null | undefined): boolean;
+/** Drops every sensitive field from a list, preserving order. */
+export declare function WithoutSensitiveFields<T extends {
+    Name: string;
+}>(fields: ReadonlyArray<T>): T[];
+/**
+ * Sensitive field names referenced by a template's `{{Placeholder}}` tokens.
+ *
+ * This is the second layer. The template is hand-editable after the model returns, so a
+ * user can type a field the picker never offered. Returns the offending names so the
+ * refusal can say which ones, rather than only that something was wrong.
+ */
+export declare function SensitiveFieldsInTemplate(templateText: string | null | undefined): string[];
+/**
+ * Field names referenced by a template's `{{Placeholder}}` tokens, in order of appearance
+ * and including repeats.
+ *
+ * Exported so the field-count cap in `vector-document-rules.ts` parses placeholders the same
+ * way this refusal does. Two parsers that drift apart would mean a template that satisfies
+ * one guard and evades the other.
+ */
+export declare function TemplatePlaceholderNames(templateText: string | null | undefined): string[];
+//# sourceMappingURL=sensitive-fields.d.ts.map

package/dist/AI/components/vectors/sensitive-fields.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sensitive-fields.d.ts","sourceRoot":"","sources":["../../../../src/AI/components/vectors/sensitive-fields.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAuJH;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAW7E;AAED,iEAAiE;AACjE,wBAAgB,sBAAsB,CAAC,CAAC,SAAS;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAEhG;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,EAAE,CAQ3F;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,EAAE,CAa1F"}

package/dist/AI/components/vectors/sensitive-fields.js

@@ -0,0 +1,226 @@
+/**
+ * Fields that must never be embedded into a vector index.
+ *
+ * Vectorizing a column copies its values into a third-party vector database, where they
+ * live outside the entity permissions that normally guard them and are retrievable by
+ * nearest-neighbour search rather than by an authorized query. For a password hash, a
+ * national insurance or social security number, a bank account or a salary, that is not a
+ * feature with a caveat — it is a disclosure.
+ *
+ * The field picker's suggestions come from a language model, which has no notion of which
+ * columns are sensitive and will happily propose `PasswordHash` if the use case seems to
+ * call for identity matching. So the refusal cannot live in the prompt: it has to be a
+ * property of the code on both sides of the model.
+ *
+ * Two layers, deliberately:
+ *
+ *   1. The model never sees these fields, on the entity or on its related entities, so it
+ *      cannot suggest them.
+ *   2. The saved template is re-checked, because the template is hand-editable after the
+ *      model returns and a user can type a field name the picker never offered.
+ *
+ * Layer 1 alone would be a prompt-shaped guarantee, which is no guarantee at all.
+ */
+/**
+ * Substrings that mark a column as unembeddable, matched case-insensitively against the
+ * field name with separators ignored — so `password_hash`, `PasswordHash` and `PASSWORD`
+ * all match the same rule.
+ *
+ * These are substrings rather than whole names on purpose. Real schemas spell the same
+ * concept a dozen ways (`SSNumber`, `employee_ssn`, `ClientSecretEncrypted`), and the cost
+ * of the two kinds of mistake is not symmetric: refusing a harmless column produces a
+ * visible, correctable annoyance, while embedding a sensitive one is silent and, once
+ * written to the index, not fully reversible.
+ *
+ * That asymmetry justifies over-refusing a column that RESEMBLES the concept — `AuthorName`
+ * contains `auth`, and `auth` really is the prefix of authentication, so the resemblance
+ * carries some signal and refusing it is a trade we accept. It does not justify a match
+ * with no semantic relationship at all. `Rating` is not refused because it looks like a tax
+ * identifier; it is refused because the letters `t-i-n` happen to sit inside an unrelated
+ * English word, which trades away a usable column for no signal whatsoever. Patterns of
+ * that shape belong in {@link SENSITIVE_FIELD_WORD_PATTERNS} or nowhere — see
+ * `vector-sensitive-fields.test.ts`, which pins the refusal count over MJ's own field names
+ * so that re-adding one is a red test rather than a quiet doubling of the blast radius.
+ */
+const SENSITIVE_FIELD_PATTERNS = [
+    // Credentials and secrets. `token` is NOT here on its own: MJ alone has 36 columns that
+    // are token *counts* (`ContextWindowMaxTokens`, `InputTokenLimit`, `FirstTokenTime`),
+    // and a count is not a secret. The compounds below keep the credentials; `AuthToken`
+    // and `OAuthToken` are still caught by `auth`.
+    'password', 'passwd', 'pwd', 'secret', 'apikey', 'accesskey', 'privatekey',
+    'accesstoken', 'refreshtoken', 'apitoken', 'bearertoken', 'sessiontoken',
+    'salt', 'hash', 'credential', 'auth', 'connectionstring',
+    // Government and tax identifiers. `tin` is deliberately absent: `taxid` already covers
+    // the concept, while `t-i-n` is a substring of ordinary English (Ra-tin-g, Marke-tin-g,
+    // Set-tin-gs, Mee-tin-g) and matched 43 MJ columns, none of them a tax identifier.
+    // A whole-word `tin` is in SENSITIVE_FIELD_WHOLE_WORD_PATTERNS below.
+    'socialsec', 'nationalinsurance', 'taxid', 'passport', 'driverlicense', 'driverslicense',
+    // Financial
+    'bank', 'routing', 'iban', 'swift', 'accountnumber', 'cardnumber', 'cardnum', 'cvv',
+    'cvc', 'creditcard', 'salary', 'compensation', 'income',
+    // Protected characteristics and other special-category data
+    'ethnic', 'religion', 'political', 'sexualorientation', 'disability',
+    'healthcondition', 'diagnosis', 'medicalrecord', 'dateofbirth', 'birthdate',
+];
+/**
+ * Patterns that must BEGIN A WORD in the field name rather than appear anywhere in it.
+ *
+ * These three are short enough that as bare substrings they collide with unrelated words
+ * and catch nothing real: `ssn` is inside `cla-ssn-ame` (every `*ClassName` column in MJ),
+ * `dob` is inside `recor-dob-ject`, `race` is inside `StackT-race` and `T-race-ID`. Each of
+ * those matches carries zero information about sensitivity, and because the template check
+ * throws, a user who needs `{{Rating}}` or `{{StackTrace}}` has no workaround at all — it
+ * is a feature they cannot use, not an annoyance they route around.
+ *
+ * Anchoring loses no true positives, because all three are whole words wherever they are
+ * meant: `ssn`, `employee_ssn`, `EmployeeSSN`, `SSNumber`, `dob`, `DOB`, `race`,
+ * `EmployeeRace`. (`date_of_birth` and `DateOfBirth` are covered by `dateofbirth` above.)
+ *
+ * `idtoken` is here for the same reason: as a substring it sits inside `Pa-idToken-s` and
+ * `Val-idToken-Count`, which are counts, while `IdToken`, `IDToken` and `id_token` all begin
+ * a word.
+ */
+const SENSITIVE_FIELD_WORD_PATTERNS = ['ssn', 'dob', 'race', 'idtoken'];
+/**
+ * Patterns that must be a WHOLE WORD: begin a word and end at the end of the name or where
+ * the next word begins.
+ *
+ * `tin` (tax identification number) is too short even for a word start: `Tint`, `TinyInt`
+ * and `Tinker` all begin with it. As a word of its own it means one thing: `TIN`,
+ * `VendorTIN`, `vendor_tin`, `TINNumber`.
+ */
+const SENSITIVE_FIELD_WHOLE_WORD_PATTERNS = ['tin'];
+/** Characters dropped by normalisation. The two spellings must stay in step. */
+const SEPARATORS_GLOBAL = /[\s_\-.]/g;
+const SEPARATOR = /[\s_\-.]/;
+const LOWER_OR_DIGIT = /[a-z0-9]/;
+const UPPER = /[A-Z]/;
+const LOWER = /[a-z]/;
+/** Normalises a field name for matching: lower-cased, separators and spaces removed. */
+function normalizeFieldName(name) {
+    return name.toLowerCase().replace(SEPARATORS_GLOBAL, '');
+}
+/**
+ * The offsets in `normalizeFieldName(name)` at which a word begins.
+ *
+ * Normalisation *removes* separators rather than replacing them, so by the time a pattern is
+ * matched there is no separator left to anchor a `\b` against — `employee_ssn` and
+ * `className` are both one unbroken run of letters (`employeessn`, `classname`). A regex word
+ * boundary would therefore match at offset 0 and nowhere else, which is not the boundary we
+ * need. So the word starts are computed from the ORIGINAL spelling, where the evidence still
+ * exists, and carried as offsets into the normalised string.
+ *
+ * A word starts at the beginning, after a separator, and at the two camelCase transitions:
+ * lower-or-digit followed by upper (`employee|SSN`), and the tail of an acronym run that
+ * begins a capitalised word (`SS|Number`, so `SSNumber` still matches `ssn`).
+ */
+function wordStartOffsets(name) {
+    const starts = new Set();
+    let offset = 0;
+    let startsWord = true;
+    for (let i = 0; i < name.length; i++) {
+        const ch = name[i];
+        if (SEPARATOR.test(ch)) {
+            // Dropped by normalizeFieldName, so it consumes no offset — but whatever
+            // survives next begins a word.
+            startsWord = true;
+            continue;
+        }
+        const prev = i > 0 ? name[i - 1] : '';
+        const next = i + 1 < name.length ? name[i + 1] : '';
+        if (UPPER.test(ch) && (LOWER_OR_DIGIT.test(prev) || (UPPER.test(prev) && LOWER.test(next)))) {
+            startsWord = true;
+        }
+        if (startsWord) {
+            starts.add(offset);
+        }
+        startsWord = false;
+        offset++;
+    }
+    return starts;
+}
+/** True when `pattern` occurs in `normalized` starting at one of `wordStarts`. */
+function matchesAtWordStart(normalized, wordStarts, pattern) {
+    let at = normalized.indexOf(pattern);
+    while (at !== -1) {
+        if (wordStarts.has(at)) {
+            return true;
+        }
+        at = normalized.indexOf(pattern, at + 1);
+    }
+    return false;
+}
+/** True when `pattern` occurs in `normalized` as a whole word, by the offsets in `wordStarts`. */
+function matchesWholeWord(normalized, wordStarts, pattern) {
+    let at = normalized.indexOf(pattern);
+    while (at !== -1) {
+        const end = at + pattern.length;
+        if (wordStarts.has(at) && (end === normalized.length || wordStarts.has(end))) {
+            return true;
+        }
+        at = normalized.indexOf(pattern, at + 1);
+    }
+    return false;
+}
+/**
+ * True when this field name must not be embedded.
+ *
+ * Note `hash` and `auth` are deliberately broad. A column called `AuthorName` normalises to
+ * `authorname`, which contains `auth` — so it is refused. That is the trade accepted above:
+ * a visible false refusal the user can work around beats a silent disclosure they cannot.
+ */
+export function IsSensitiveFieldName(name) {
+    if (!name) {
+        return false;
+    }
+    const normalized = normalizeFieldName(name);
+    if (SENSITIVE_FIELD_PATTERNS.some((pattern) => normalized.includes(pattern))) {
+        return true;
+    }
+    const wordStarts = wordStartOffsets(name);
+    return SENSITIVE_FIELD_WORD_PATTERNS.some((pattern) => matchesAtWordStart(normalized, wordStarts, pattern))
+        || SENSITIVE_FIELD_WHOLE_WORD_PATTERNS.some((pattern) => matchesWholeWord(normalized, wordStarts, pattern));
+}
+/** Drops every sensitive field from a list, preserving order. */
+export function WithoutSensitiveFields(fields) {
+    return fields.filter((f) => !IsSensitiveFieldName(f.Name));
+}
+/**
+ * Sensitive field names referenced by a template's `{{Placeholder}}` tokens.
+ *
+ * This is the second layer. The template is hand-editable after the model returns, so a
+ * user can type a field the picker never offered. Returns the offending names so the
+ * refusal can say which ones, rather than only that something was wrong.
+ */
+export function SensitiveFieldsInTemplate(templateText) {
+    const found = new Set();
+    for (const token of TemplatePlaceholderNames(templateText)) {
+        if (IsSensitiveFieldName(token)) {
+            found.add(token);
+        }
+    }
+    return [...found].sort();
+}
+/**
+ * Field names referenced by a template's `{{Placeholder}}` tokens, in order of appearance
+ * and including repeats.
+ *
+ * Exported so the field-count cap in `vector-document-rules.ts` parses placeholders the same
+ * way this refusal does. Two parsers that drift apart would mean a template that satisfies
+ * one guard and evades the other.
+ */
+export function TemplatePlaceholderNames(templateText) {
+    if (!templateText) {
+        return [];
+    }
+    // Nunjucks/Handlebars-style placeholders. The field name is the first identifier
+    // inside the braces; filters and whitespace after it are ignored.
+    const placeholder = /\{\{\s*([A-Za-z0-9_.\-]+)/g;
+    const names = [];
+    let match;
+    while ((match = placeholder.exec(templateText)) !== null) {
+        names.push(match[1]);
+    }
+    return names;
+}
+//# sourceMappingURL=sensitive-fields.js.map

package/dist/AI/components/vectors/sensitive-fields.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sensitive-fields.js","sourceRoot":"","sources":["../../../../src/AI/components/vectors/sensitive-fields.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,wBAAwB,GAA0B;IACpD,wFAAwF;IACxF,sFAAsF;IACtF,qFAAqF;IACrF,+CAA+C;IAC/C,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY;IAC1E,aAAa,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc;IACxE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,kBAAkB;IACxD,uFAAuF;IACvF,wFAAwF;IACxF,mFAAmF;IACnF,sEAAsE;IACtE,WAAW,EAAE,mBAAmB,EAAE,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,gBAAgB;IACxF,YAAY;IACZ,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK;IACnF,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ;IACvD,4DAA4D;IAC5D,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,mBAAmB,EAAE,YAAY;IACpE,iBAAiB,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW;CAC9E,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,6BAA6B,GAA0B,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAE/F;;;;;;;GAOG;AACH,MAAM,mCAAmC,GAA0B,CAAC,KAAK,CAAC,CAAC;AAE3E,gFAAgF;AAChF,MAAM,iBAAiB,GAAG,WAAW,CAAC;AACtC,MAAM,SAAS,GAAG,UAAU,CAAC;AAC7B,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,KAAK,GAAG,OAAO,CAAC;AACtB,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB,wFAAwF;AACxF,SAAS,kBAAkB,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,gBAAgB,CAAC,IAAY;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACrB,yEAAyE;YACzE,+BAA+B;YAC/B,UAAU,GAAG,IAAI,CAAC;YAClB,SAAS;QACb,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1F,UAAU,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,UAAU,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QACD,UAAU,GAAG,KAAK,CAAC;QACnB,MAAM,EAAE,CAAC;IACb,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,kFAAkF;AAClF,SAAS,kBAAkB,CAAC,UAAkB,EAAE,UAA+B,EAAE,OAAe;IAC5F,IAAI,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,kGAAkG;AAClG,SAAS,gBAAgB,CAAC,UAAkB,EAAE,UAA+B,EAAE,OAAe;IAC1F,IAAI,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAChC,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAA+B;IAChE,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC1C,OAAO,6BAA6B,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;WACpG,mCAAmC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,gBAAgB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AACpH,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,sBAAsB,CAA6B,MAAwB;IACvF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CAAC,YAAuC;IAC7E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,KAAK,IAAI,wBAAwB,CAAC,YAAY,CAAC,EAAE,CAAC;QACzD,IAAI,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CAAC,YAAuC;IAC5E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,iFAAiF;IACjF,kEAAkE;IAClE,MAAM,WAAW,GAAG,4BAA4B,CAAC;IACjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC","sourcesContent":["/**\n * Fields that must never be embedded into a vector index.\n *\n * Vectorizing a column copies its values into a third-party vector database, where they\n * live outside the entity permissions that normally guard them and are retrievable by\n * nearest-neighbour search rather than by an authorized query. For a password hash, a\n * national insurance or social security number, a bank account or a salary, that is not a\n * feature with a caveat — it is a disclosure.\n *\n * The field picker's suggestions come from a language model, which has no notion of which\n * columns are sensitive and will happily propose `PasswordHash` if the use case seems to\n * call for identity matching. So the refusal cannot live in the prompt: it has to be a\n * property of the code on both sides of the model.\n *\n * Two layers, deliberately:\n *\n *   1. The model never sees these fields, on the entity or on its related entities, so it\n *      cannot suggest them.\n *   2. The saved template is re-checked, because the template is hand-editable after the\n *      model returns and a user can type a field name the picker never offered.\n *\n * Layer 1 alone would be a prompt-shaped guarantee, which is no guarantee at all.\n */\n\n/**\n * Substrings that mark a column as unembeddable, matched case-insensitively against the\n * field name with separators ignored — so `password_hash`, `PasswordHash` and `PASSWORD`\n * all match the same rule.\n *\n * These are substrings rather than whole names on purpose. Real schemas spell the same\n * concept a dozen ways (`SSNumber`, `employee_ssn`, `ClientSecretEncrypted`), and the cost\n * of the two kinds of mistake is not symmetric: refusing a harmless column produces a\n * visible, correctable annoyance, while embedding a sensitive one is silent and, once\n * written to the index, not fully reversible.\n *\n * That asymmetry justifies over-refusing a column that RESEMBLES the concept — `AuthorName`\n * contains `auth`, and `auth` really is the prefix of authentication, so the resemblance\n * carries some signal and refusing it is a trade we accept. It does not justify a match\n * with no semantic relationship at all. `Rating` is not refused because it looks like a tax\n * identifier; it is refused because the letters `t-i-n` happen to sit inside an unrelated\n * English word, which trades away a usable column for no signal whatsoever. Patterns of\n * that shape belong in {@link SENSITIVE_FIELD_WORD_PATTERNS} or nowhere — see\n * `vector-sensitive-fields.test.ts`, which pins the refusal count over MJ's own field names\n * so that re-adding one is a red test rather than a quiet doubling of the blast radius.\n */\nconst SENSITIVE_FIELD_PATTERNS: ReadonlyArray<string> = [\n    // Credentials and secrets. `token` is NOT here on its own: MJ alone has 36 columns that\n    // are token *counts* (`ContextWindowMaxTokens`, `InputTokenLimit`, `FirstTokenTime`),\n    // and a count is not a secret. The compounds below keep the credentials; `AuthToken`\n    // and `OAuthToken` are still caught by `auth`.\n    'password', 'passwd', 'pwd', 'secret', 'apikey', 'accesskey', 'privatekey',\n    'accesstoken', 'refreshtoken', 'apitoken', 'bearertoken', 'sessiontoken',\n    'salt', 'hash', 'credential', 'auth', 'connectionstring',\n    // Government and tax identifiers. `tin` is deliberately absent: `taxid` already covers\n    // the concept, while `t-i-n` is a substring of ordinary English (Ra-tin-g, Marke-tin-g,\n    // Set-tin-gs, Mee-tin-g) and matched 43 MJ columns, none of them a tax identifier.\n    // A whole-word `tin` is in SENSITIVE_FIELD_WHOLE_WORD_PATTERNS below.\n    'socialsec', 'nationalinsurance', 'taxid', 'passport', 'driverlicense', 'driverslicense',\n    // Financial\n    'bank', 'routing', 'iban', 'swift', 'accountnumber', 'cardnumber', 'cardnum', 'cvv',\n    'cvc', 'creditcard', 'salary', 'compensation', 'income',\n    // Protected characteristics and other special-category data\n    'ethnic', 'religion', 'political', 'sexualorientation', 'disability',\n    'healthcondition', 'diagnosis', 'medicalrecord', 'dateofbirth', 'birthdate',\n];\n\n/**\n * Patterns that must BEGIN A WORD in the field name rather than appear anywhere in it.\n *\n * These three are short enough that as bare substrings they collide with unrelated words\n * and catch nothing real: `ssn` is inside `cla-ssn-ame` (every `*ClassName` column in MJ),\n * `dob` is inside `recor-dob-ject`, `race` is inside `StackT-race` and `T-race-ID`. Each of\n * those matches carries zero information about sensitivity, and because the template check\n * throws, a user who needs `{{Rating}}` or `{{StackTrace}}` has no workaround at all — it\n * is a feature they cannot use, not an annoyance they route around.\n *\n * Anchoring loses no true positives, because all three are whole words wherever they are\n * meant: `ssn`, `employee_ssn`, `EmployeeSSN`, `SSNumber`, `dob`, `DOB`, `race`,\n * `EmployeeRace`. (`date_of_birth` and `DateOfBirth` are covered by `dateofbirth` above.)\n *\n * `idtoken` is here for the same reason: as a substring it sits inside `Pa-idToken-s` and\n * `Val-idToken-Count`, which are counts, while `IdToken`, `IDToken` and `id_token` all begin\n * a word.\n */\nconst SENSITIVE_FIELD_WORD_PATTERNS: ReadonlyArray<string> = ['ssn', 'dob', 'race', 'idtoken'];\n\n/**\n * Patterns that must be a WHOLE WORD: begin a word and end at the end of the name or where\n * the next word begins.\n *\n * `tin` (tax identification number) is too short even for a word start: `Tint`, `TinyInt`\n * and `Tinker` all begin with it. As a word of its own it means one thing: `TIN`,\n * `VendorTIN`, `vendor_tin`, `TINNumber`.\n */\nconst SENSITIVE_FIELD_WHOLE_WORD_PATTERNS: ReadonlyArray<string> = ['tin'];\n\n/** Characters dropped by normalisation. The two spellings must stay in step. */\nconst SEPARATORS_GLOBAL = /[\\s_\\-.]/g;\nconst SEPARATOR = /[\\s_\\-.]/;\nconst LOWER_OR_DIGIT = /[a-z0-9]/;\nconst UPPER = /[A-Z]/;\nconst LOWER = /[a-z]/;\n\n/** Normalises a field name for matching: lower-cased, separators and spaces removed. */\nfunction normalizeFieldName(name: string): string {\n    return name.toLowerCase().replace(SEPARATORS_GLOBAL, '');\n}\n\n/**\n * The offsets in `normalizeFieldName(name)` at which a word begins.\n *\n * Normalisation *removes* separators rather than replacing them, so by the time a pattern is\n * matched there is no separator left to anchor a `\\b` against — `employee_ssn` and\n * `className` are both one unbroken run of letters (`employeessn`, `classname`). A regex word\n * boundary would therefore match at offset 0 and nowhere else, which is not the boundary we\n * need. So the word starts are computed from the ORIGINAL spelling, where the evidence still\n * exists, and carried as offsets into the normalised string.\n *\n * A word starts at the beginning, after a separator, and at the two camelCase transitions:\n * lower-or-digit followed by upper (`employee|SSN`), and the tail of an acronym run that\n * begins a capitalised word (`SS|Number`, so `SSNumber` still matches `ssn`).\n */\nfunction wordStartOffsets(name: string): ReadonlySet<number> {\n    const starts = new Set<number>();\n    let offset = 0;\n    let startsWord = true;\n    for (let i = 0; i < name.length; i++) {\n        const ch = name[i];\n        if (SEPARATOR.test(ch)) {\n            // Dropped by normalizeFieldName, so it consumes no offset — but whatever\n            // survives next begins a word.\n            startsWord = true;\n            continue;\n        }\n        const prev = i > 0 ? name[i - 1] : '';\n        const next = i + 1 < name.length ? name[i + 1] : '';\n        if (UPPER.test(ch) && (LOWER_OR_DIGIT.test(prev) || (UPPER.test(prev) && LOWER.test(next)))) {\n            startsWord = true;\n        }\n        if (startsWord) {\n            starts.add(offset);\n        }\n        startsWord = false;\n        offset++;\n    }\n    return starts;\n}\n\n/** True when `pattern` occurs in `normalized` starting at one of `wordStarts`. */\nfunction matchesAtWordStart(normalized: string, wordStarts: ReadonlySet<number>, pattern: string): boolean {\n    let at = normalized.indexOf(pattern);\n    while (at !== -1) {\n        if (wordStarts.has(at)) {\n            return true;\n        }\n        at = normalized.indexOf(pattern, at + 1);\n    }\n    return false;\n}\n\n/** True when `pattern` occurs in `normalized` as a whole word, by the offsets in `wordStarts`. */\nfunction matchesWholeWord(normalized: string, wordStarts: ReadonlySet<number>, pattern: string): boolean {\n    let at = normalized.indexOf(pattern);\n    while (at !== -1) {\n        const end = at + pattern.length;\n        if (wordStarts.has(at) && (end === normalized.length || wordStarts.has(end))) {\n            return true;\n        }\n        at = normalized.indexOf(pattern, at + 1);\n    }\n    return false;\n}\n\n/**\n * True when this field name must not be embedded.\n *\n * Note `hash` and `auth` are deliberately broad. A column called `AuthorName` normalises to\n * `authorname`, which contains `auth` — so it is refused. That is the trade accepted above:\n * a visible false refusal the user can work around beats a silent disclosure they cannot.\n */\nexport function IsSensitiveFieldName(name: string | null | undefined): boolean {\n    if (!name) {\n        return false;\n    }\n    const normalized = normalizeFieldName(name);\n    if (SENSITIVE_FIELD_PATTERNS.some((pattern) => normalized.includes(pattern))) {\n        return true;\n    }\n    const wordStarts = wordStartOffsets(name);\n    return SENSITIVE_FIELD_WORD_PATTERNS.some((pattern) => matchesAtWordStart(normalized, wordStarts, pattern))\n        || SENSITIVE_FIELD_WHOLE_WORD_PATTERNS.some((pattern) => matchesWholeWord(normalized, wordStarts, pattern));\n}\n\n/** Drops every sensitive field from a list, preserving order. */\nexport function WithoutSensitiveFields<T extends { Name: string }>(fields: ReadonlyArray<T>): T[] {\n    return fields.filter((f) => !IsSensitiveFieldName(f.Name));\n}\n\n/**\n * Sensitive field names referenced by a template's `{{Placeholder}}` tokens.\n *\n * This is the second layer. The template is hand-editable after the model returns, so a\n * user can type a field the picker never offered. Returns the offending names so the\n * refusal can say which ones, rather than only that something was wrong.\n */\nexport function SensitiveFieldsInTemplate(templateText: string | null | undefined): string[] {\n    const found = new Set<string>();\n    for (const token of TemplatePlaceholderNames(templateText)) {\n        if (IsSensitiveFieldName(token)) {\n            found.add(token);\n        }\n    }\n    return [...found].sort();\n}\n\n/**\n * Field names referenced by a template's `{{Placeholder}}` tokens, in order of appearance\n * and including repeats.\n *\n * Exported so the field-count cap in `vector-document-rules.ts` parses placeholders the same\n * way this refusal does. Two parsers that drift apart would mean a template that satisfies\n * one guard and evades the other.\n */\nexport function TemplatePlaceholderNames(templateText: string | null | undefined): string[] {\n    if (!templateText) {\n        return [];\n    }\n    // Nunjucks/Handlebars-style placeholders. The field name is the first identifier\n    // inside the braces; filters and whitespace after it are ignored.\n    const placeholder = /\\{\\{\\s*([A-Za-z0-9_.\\-]+)/g;\n    const names: string[] = [];\n    let match: RegExpExecArray | null;\n    while ((match = placeholder.exec(templateText)) !== null) {\n        names.push(match[1]);\n    }\n    return names;\n}\n"]}

package/dist/AI/components/vectors/vector-document-rules.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Rules the Vectors tab applies when it turns a model suggestion into real metadata.
+ *
+ * Three separate refusals live here, and they share a theme with `sensitive-fields.ts`: the
+ * suggestion comes from a language model and the template stays hand-editable afterwards, so
+ * anything that matters has to be a property of the code rather than of the prompt.
+ *
+ *   1. The Entity Document Type is resolved from the use case the user actually picked,
+ *      instead of always writing a duplicate-detection document.
+ *   2. The number of interpolated fields is bounded, because an unbounded list is how a
+ *      template balloons and how columns nobody chose end up embedded.
+ *   3. MemberJunction's own internal entities are not offered as vectorization targets.
+ *
+ * Kept out of the component so each rule can be tested as a function rather than through a
+ * TestBed, matching how `sensitive-fields.ts` is structured.
+ */
+/**
+ * The Entity Document Type each use case writes.
+ *
+ * The dialog offers these use cases, and until now every one of them produced a
+ * `Record Duplicate` document — the type name was hardcoded in the save path, so choosing
+ * "search" silently discarded the choice and wrote the wrong type. A vector pool is typed:
+ * `Provider.SearchEntity` and the Search Entity action read `Search`-typed documents, so a
+ * search request stored as `Record Duplicate` is not merely mislabelled, it is not found by
+ * the thing that wanted it.
+ *
+ * The two names here are MJ's own, not invented: both are seeded Entity Document Types, and
+ * the Vectorize Entity action's `EntityDocumentType` parameter is documented as defaulting
+ * to `Record Duplicate` and accepting `Search`.
+ *
+ * The dialog's Use Case buttons are exactly this map's keys. There is no `classification`
+ * entry: MJ seeds no `Classification` type, so that button failed on every stock install and
+ * was removed until the type exists (#5630).
+ */
+export declare const ENTITY_DOCUMENT_TYPE_BY_USE_CASE: Readonly<Record<string, string>>;
+/**
+ * The Entity Document Type name for a use case, or null when the use case is unrecognised.
+ *
+ * Returns null rather than falling back to a default: a use case this code does not know
+ * about must not quietly become a duplicate-detection document.
+ */
+export declare function EntityDocumentTypeForUseCase(useCase: string | null | undefined): string | null;
+/**
+ * The use case the suggestion prompt is given for an existing document of this type, or null
+ * when the type has none (e.g. `Context`).
+ *
+ * The edit panel's "Regenerate template" uses it, so a regenerated template still matches the
+ * document's type. It reads the same map as {@link EntityDocumentTypeForUseCase}, so the two
+ * directions cannot drift. Callers refuse on null rather than substitute a use case.
+ */
+export declare function SuggestionUseCaseForDocumentType(typeName: string | null | undefined): string | null;
+/**
+ * The most distinct fields one generated template may interpolate.
+ *
+ * Derived from the suggestion prompt's own contract rather than copied from elsewhere.
+ * `metadata/prompts/templates/system/entity-document-suggestion.template.md` requires
+ * "**Output must be 1-4 natural language sentences**" and explains at length why a
+ * key-value dump embeds badly compared with English prose.
+ *
+ * So: 4 sentences — the prompt's own ceiling — times 6 interpolations per sentence, past
+ * which a sentence stops being the natural English the prompt demands and becomes the dump
+ * it explicitly forbids. That gives 24.
+ *
+ * Cross-check against the prompt's three worked examples, which interpolate 8, 6 and 4
+ * distinct placeholders: 24 is three times the richest thing the prompt itself demonstrates,
+ * so the cap cannot fire on anything the prompt teaches a model to produce. It fires when a
+ * model has ignored the sentence contract and reached for the whole table.
+ */
+export declare const MAX_TEMPLATE_FIELDS = 24;
+/** Distinct `{{Placeholder}}` field names a template interpolates. */
+export declare function TemplateFieldNames(templateText: string | null | undefined): string[];
+/**
+ * Truncates a model-supplied field list to the cap.
+ *
+ * Truncation rather than refusal, because this list is the model's own description of what
+ * it selected and is shown as chips — dropping the tail and saying so is more useful than
+ * discarding the whole suggestion. The authoritative refusal is on the template, below,
+ * since the template is what actually gets embedded.
+ */
+export declare function CapSelectedFields(fields: ReadonlyArray<string> | null | undefined): {
+    fields: string[];
+    dropped: number;
+};
+/**
+ * The refusal message when a template interpolates more fields than the cap allows, or null
+ * when it is within the cap.
+ *
+ * A template cannot be truncated without mangling its prose, so this one refuses rather than
+ * trimming — and the message carries both numbers, because "too many fields" without the
+ * count is not actionable. The template is editable in the dialog, so the user can act on it
+ * without losing the suggestion.
+ */
+export declare function TemplateFieldCapRefusal(templateText: string | null | undefined): string | null;
+/**
+ * Schemas whose entities are not offered in the picker.
+ *
+ * Follows the existing frontend precedent — `FRONTEND_BLOCKED_SCHEMAS` in
+ * `DatabaseDesigner/services/database-designer.engine.ts` and in
+ * `DatabaseDesigner/components/create-wizard/steps/step-relationships.component.ts`, both
+ * `new Set(['__mj'])` — rather than inventing a rule.
+ */
+export declare const VECTOR_BLOCKED_SCHEMAS: ReadonlySet<string>;
+/**
+ * True when an entity may be offered as a vectorization target.
+ *
+ * The picker previously listed every entity in metadata, so MemberJunction's own ~378
+ * internal entities — prompt runs, audit logs, record changes, credentials — were offered
+ * alongside the user's business tables, both as noise and as embedding targets.
+ *
+ * Two rules, and they currently pick out the same set: every `MJ:`-prefixed entity is in the
+ * `__mj` schema and vice versa (378 and 378, exactly coextensive, in the 5.46 baseline). The
+ * name-prefix rule is therefore defence in depth against an MJ entity registered into some
+ * other schema, not extra coverage today. Said plainly so nobody later reads it as load
+ * bearing and is surprised.
+ *
+ * Note MJ does seed two `Search`-typed documents over `__mj` entities (`AI Agents Search`,
+ * `AI Prompts Search`), so framework entities are not unvectorizable in principle. Those
+ * arrive through metadata sync, which is a different authoring channel with a different
+ * audience; this dialog is where a business user configures vectorization of their own data,
+ * and burying their tables under the framework's is the defect being fixed.
+ */
+export declare function IsVectorizableEntity(entity: {
+    Name: string;
+    SchemaName?: string | null;
+}): boolean;
+//# sourceMappingURL=vector-document-rules.d.ts.map

package/dist/AI/components/vectors/vector-document-rules.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-document-rules.d.ts","sourceRoot":"","sources":["../../../../src/AI/components/vectors/vector-document-rules.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,gCAAgC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAG7E,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAK9F;AAED;;;;;;;GAOG;AACH,wBAAgB,gCAAgC,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAQnG;AAID;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,sEAAsE;AACtE,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,EAAE,CAEpF;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC7B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,GACjD;IAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAYvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAW9F;AAID;;;;;;;GAOG;AACH,eAAO,MAAM,sBAAsB,EAAE,WAAW,CAAC,MAAM,CAAqB,CAAC;AAE7E;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,oBAAoB,CAChC,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GACrD,OAAO,CAKT"}

package/dist/AI/components/vectors/vector-document-rules.js

@@ -0,0 +1,168 @@
+/**
+ * Rules the Vectors tab applies when it turns a model suggestion into real metadata.
+ *
+ * Three separate refusals live here, and they share a theme with `sensitive-fields.ts`: the
+ * suggestion comes from a language model and the template stays hand-editable afterwards, so
+ * anything that matters has to be a property of the code rather than of the prompt.
+ *
+ *   1. The Entity Document Type is resolved from the use case the user actually picked,
+ *      instead of always writing a duplicate-detection document.
+ *   2. The number of interpolated fields is bounded, because an unbounded list is how a
+ *      template balloons and how columns nobody chose end up embedded.
+ *   3. MemberJunction's own internal entities are not offered as vectorization targets.
+ *
+ * Kept out of the component so each rule can be tested as a function rather than through a
+ * TestBed, matching how `sensitive-fields.ts` is structured.
+ */
+import { TemplatePlaceholderNames } from './sensitive-fields';
+// ─── 1. Entity Document Type per use case ────────────────────────────────────
+/**
+ * The Entity Document Type each use case writes.
+ *
+ * The dialog offers these use cases, and until now every one of them produced a
+ * `Record Duplicate` document — the type name was hardcoded in the save path, so choosing
+ * "search" silently discarded the choice and wrote the wrong type. A vector pool is typed:
+ * `Provider.SearchEntity` and the Search Entity action read `Search`-typed documents, so a
+ * search request stored as `Record Duplicate` is not merely mislabelled, it is not found by
+ * the thing that wanted it.
+ *
+ * The two names here are MJ's own, not invented: both are seeded Entity Document Types, and
+ * the Vectorize Entity action's `EntityDocumentType` parameter is documented as defaulting
+ * to `Record Duplicate` and accepting `Search`.
+ *
+ * The dialog's Use Case buttons are exactly this map's keys. There is no `classification`
+ * entry: MJ seeds no `Classification` type, so that button failed on every stock install and
+ * was removed until the type exists (#5630).
+ */
+export const ENTITY_DOCUMENT_TYPE_BY_USE_CASE = {
+    'duplicate detection': 'Record Duplicate',
+    'search': 'Search',
+};
+/**
+ * The Entity Document Type name for a use case, or null when the use case is unrecognised.
+ *
+ * Returns null rather than falling back to a default: a use case this code does not know
+ * about must not quietly become a duplicate-detection document.
+ */
+export function EntityDocumentTypeForUseCase(useCase) {
+    if (!useCase) {
+        return null;
+    }
+    return ENTITY_DOCUMENT_TYPE_BY_USE_CASE[useCase.trim().toLowerCase()] ?? null;
+}
+/**
+ * The use case the suggestion prompt is given for an existing document of this type, or null
+ * when the type has none (e.g. `Context`).
+ *
+ * The edit panel's "Regenerate template" uses it, so a regenerated template still matches the
+ * document's type. It reads the same map as {@link EntityDocumentTypeForUseCase}, so the two
+ * directions cannot drift. Callers refuse on null rather than substitute a use case.
+ */
+export function SuggestionUseCaseForDocumentType(typeName) {
+    if (!typeName) {
+        return null;
+    }
+    const wanted = typeName.trim().toLowerCase();
+    const match = Object.entries(ENTITY_DOCUMENT_TYPE_BY_USE_CASE)
+        .find(([, documentType]) => documentType.toLowerCase() === wanted);
+    return match ? match[0] : null;
+}
+// ─── 2. How many fields a template may interpolate ───────────────────────────
+/**
+ * The most distinct fields one generated template may interpolate.
+ *
+ * Derived from the suggestion prompt's own contract rather than copied from elsewhere.
+ * `metadata/prompts/templates/system/entity-document-suggestion.template.md` requires
+ * "**Output must be 1-4 natural language sentences**" and explains at length why a
+ * key-value dump embeds badly compared with English prose.
+ *
+ * So: 4 sentences — the prompt's own ceiling — times 6 interpolations per sentence, past
+ * which a sentence stops being the natural English the prompt demands and becomes the dump
+ * it explicitly forbids. That gives 24.
+ *
+ * Cross-check against the prompt's three worked examples, which interpolate 8, 6 and 4
+ * distinct placeholders: 24 is three times the richest thing the prompt itself demonstrates,
+ * so the cap cannot fire on anything the prompt teaches a model to produce. It fires when a
+ * model has ignored the sentence contract and reached for the whole table.
+ */
+export const MAX_TEMPLATE_FIELDS = 24;
+/** Distinct `{{Placeholder}}` field names a template interpolates. */
+export function TemplateFieldNames(templateText) {
+    return [...new Set(TemplatePlaceholderNames(templateText))];
+}
+/**
+ * Truncates a model-supplied field list to the cap.
+ *
+ * Truncation rather than refusal, because this list is the model's own description of what
+ * it selected and is shown as chips — dropping the tail and saying so is more useful than
+ * discarding the whole suggestion. The authoritative refusal is on the template, below,
+ * since the template is what actually gets embedded.
+ */
+export function CapSelectedFields(fields) {
+    if (!fields || fields.length === 0) {
+        return { fields: [], dropped: 0 };
+    }
+    const unique = [...new Set(fields)];
+    if (unique.length <= MAX_TEMPLATE_FIELDS) {
+        return { fields: unique, dropped: 0 };
+    }
+    return {
+        fields: unique.slice(0, MAX_TEMPLATE_FIELDS),
+        dropped: unique.length - MAX_TEMPLATE_FIELDS,
+    };
+}
+/**
+ * The refusal message when a template interpolates more fields than the cap allows, or null
+ * when it is within the cap.
+ *
+ * A template cannot be truncated without mangling its prose, so this one refuses rather than
+ * trimming — and the message carries both numbers, because "too many fields" without the
+ * count is not actionable. The template is editable in the dialog, so the user can act on it
+ * without losing the suggestion.
+ */
+export function TemplateFieldCapRefusal(templateText) {
+    const names = TemplateFieldNames(templateText);
+    if (names.length <= MAX_TEMPLATE_FIELDS) {
+        return null;
+    }
+    return (`This template interpolates ${names.length} fields, and at most ${MAX_TEMPLATE_FIELDS} ` +
+        `may be embedded. A template this wide stops reading like the natural language that ` +
+        `embeds well, and pulls in columns nobody chose. Remove ` +
+        `${names.length - MAX_TEMPLATE_FIELDS} field(s) from the template and save again.`);
+}
+// ─── 3. Which entities may be offered as vectorization targets ───────────────
+/**
+ * Schemas whose entities are not offered in the picker.
+ *
+ * Follows the existing frontend precedent — `FRONTEND_BLOCKED_SCHEMAS` in
+ * `DatabaseDesigner/services/database-designer.engine.ts` and in
+ * `DatabaseDesigner/components/create-wizard/steps/step-relationships.component.ts`, both
+ * `new Set(['__mj'])` — rather than inventing a rule.
+ */
+export const VECTOR_BLOCKED_SCHEMAS = new Set(['__mj']);
+/**
+ * True when an entity may be offered as a vectorization target.
+ *
+ * The picker previously listed every entity in metadata, so MemberJunction's own ~378
+ * internal entities — prompt runs, audit logs, record changes, credentials — were offered
+ * alongside the user's business tables, both as noise and as embedding targets.
+ *
+ * Two rules, and they currently pick out the same set: every `MJ:`-prefixed entity is in the
+ * `__mj` schema and vice versa (378 and 378, exactly coextensive, in the 5.46 baseline). The
+ * name-prefix rule is therefore defence in depth against an MJ entity registered into some
+ * other schema, not extra coverage today. Said plainly so nobody later reads it as load
+ * bearing and is surprised.
+ *
+ * Note MJ does seed two `Search`-typed documents over `__mj` entities (`AI Agents Search`,
+ * `AI Prompts Search`), so framework entities are not unvectorizable in principle. Those
+ * arrive through metadata sync, which is a different authoring channel with a different
+ * audience; this dialog is where a business user configures vectorization of their own data,
+ * and burying their tables under the framework's is the defect being fixed.
+ */
+export function IsVectorizableEntity(entity) {
+    if (entity.SchemaName && VECTOR_BLOCKED_SCHEMAS.has(entity.SchemaName)) {
+        return false;
+    }
+    return !entity.Name.startsWith('MJ:');
+}
+//# sourceMappingURL=vector-document-rules.js.map

package/dist/AI/components/vectors/vector-document-rules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-document-rules.js","sourceRoot":"","sources":["../../../../src/AI/components/vectors/vector-document-rules.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,gFAAgF;AAEhF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAqC;IAC9E,qBAAqB,EAAE,kBAAkB;IACzC,QAAQ,EAAE,QAAQ;CACrB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAAC,OAAkC;IAC3E,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,gCAAgC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAC;AAClF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gCAAgC,CAAC,QAAmC;IAChF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC;SACzD,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,CAAC;IACvE,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACnC,CAAC;AAED,gFAAgF;AAEhF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAEtC,sEAAsE;AACtE,MAAM,UAAU,kBAAkB,CAAC,YAAuC;IACtE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,wBAAwB,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC7B,MAAgD;IAEhD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACtC,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE,CAAC;QACvC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC1C,CAAC;IACD,OAAO;QACH,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC;QAC5C,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,mBAAmB;KAC/C,CAAC;AACN,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,uBAAuB,CAAC,YAAuC;IAC3E,MAAM,KAAK,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;IAC/C,IAAI,KAAK,CAAC,MAAM,IAAI,mBAAmB,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,CACH,8BAA8B,KAAK,CAAC,MAAM,wBAAwB,mBAAmB,GAAG;QACxF,qFAAqF;QACrF,yDAAyD;QACzD,GAAG,KAAK,CAAC,MAAM,GAAG,mBAAmB,6CAA6C,CACrF,CAAC;AACN,CAAC;AAED,gFAAgF;AAEhF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAwB,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,oBAAoB,CAChC,MAAoD;IAEpD,IAAI,MAAM,CAAC,UAAU,IAAI,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACrE,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AAC1C,CAAC","sourcesContent":["/**\n * Rules the Vectors tab applies when it turns a model suggestion into real metadata.\n *\n * Three separate refusals live here, and they share a theme with `sensitive-fields.ts`: the\n * suggestion comes from a language model and the template stays hand-editable afterwards, so\n * anything that matters has to be a property of the code rather than of the prompt.\n *\n *   1. The Entity Document Type is resolved from the use case the user actually picked,\n *      instead of always writing a duplicate-detection document.\n *   2. The number of interpolated fields is bounded, because an unbounded list is how a\n *      template balloons and how columns nobody chose end up embedded.\n *   3. MemberJunction's own internal entities are not offered as vectorization targets.\n *\n * Kept out of the component so each rule can be tested as a function rather than through a\n * TestBed, matching how `sensitive-fields.ts` is structured.\n */\n\nimport { TemplatePlaceholderNames } from './sensitive-fields';\n\n// ─── 1. Entity Document Type per use case ────────────────────────────────────\n\n/**\n * The Entity Document Type each use case writes.\n *\n * The dialog offers these use cases, and until now every one of them produced a\n * `Record Duplicate` document — the type name was hardcoded in the save path, so choosing\n * \"search\" silently discarded the choice and wrote the wrong type. A vector pool is typed:\n * `Provider.SearchEntity` and the Search Entity action read `Search`-typed documents, so a\n * search request stored as `Record Duplicate` is not merely mislabelled, it is not found by\n * the thing that wanted it.\n *\n * The two names here are MJ's own, not invented: both are seeded Entity Document Types, and\n * the Vectorize Entity action's `EntityDocumentType` parameter is documented as defaulting\n * to `Record Duplicate` and accepting `Search`.\n *\n * The dialog's Use Case buttons are exactly this map's keys. There is no `classification`\n * entry: MJ seeds no `Classification` type, so that button failed on every stock install and\n * was removed until the type exists (#5630).\n */\nexport const ENTITY_DOCUMENT_TYPE_BY_USE_CASE: Readonly<Record<string, string>> = {\n    'duplicate detection': 'Record Duplicate',\n    'search': 'Search',\n};\n\n/**\n * The Entity Document Type name for a use case, or null when the use case is unrecognised.\n *\n * Returns null rather than falling back to a default: a use case this code does not know\n * about must not quietly become a duplicate-detection document.\n */\nexport function EntityDocumentTypeForUseCase(useCase: string | null | undefined): string | null {\n    if (!useCase) {\n        return null;\n    }\n    return ENTITY_DOCUMENT_TYPE_BY_USE_CASE[useCase.trim().toLowerCase()] ?? null;\n}\n\n/**\n * The use case the suggestion prompt is given for an existing document of this type, or null\n * when the type has none (e.g. `Context`).\n *\n * The edit panel's \"Regenerate template\" uses it, so a regenerated template still matches the\n * document's type. It reads the same map as {@link EntityDocumentTypeForUseCase}, so the two\n * directions cannot drift. Callers refuse on null rather than substitute a use case.\n */\nexport function SuggestionUseCaseForDocumentType(typeName: string | null | undefined): string | null {\n    if (!typeName) {\n        return null;\n    }\n    const wanted = typeName.trim().toLowerCase();\n    const match = Object.entries(ENTITY_DOCUMENT_TYPE_BY_USE_CASE)\n        .find(([, documentType]) => documentType.toLowerCase() === wanted);\n    return match ? match[0] : null;\n}\n\n// ─── 2. How many fields a template may interpolate ───────────────────────────\n\n/**\n * The most distinct fields one generated template may interpolate.\n *\n * Derived from the suggestion prompt's own contract rather than copied from elsewhere.\n * `metadata/prompts/templates/system/entity-document-suggestion.template.md` requires\n * \"**Output must be 1-4 natural language sentences**\" and explains at length why a\n * key-value dump embeds badly compared with English prose.\n *\n * So: 4 sentences — the prompt's own ceiling — times 6 interpolations per sentence, past\n * which a sentence stops being the natural English the prompt demands and becomes the dump\n * it explicitly forbids. That gives 24.\n *\n * Cross-check against the prompt's three worked examples, which interpolate 8, 6 and 4\n * distinct placeholders: 24 is three times the richest thing the prompt itself demonstrates,\n * so the cap cannot fire on anything the prompt teaches a model to produce. It fires when a\n * model has ignored the sentence contract and reached for the whole table.\n */\nexport const MAX_TEMPLATE_FIELDS = 24;\n\n/** Distinct `{{Placeholder}}` field names a template interpolates. */\nexport function TemplateFieldNames(templateText: string | null | undefined): string[] {\n    return [...new Set(TemplatePlaceholderNames(templateText))];\n}\n\n/**\n * Truncates a model-supplied field list to the cap.\n *\n * Truncation rather than refusal, because this list is the model's own description of what\n * it selected and is shown as chips — dropping the tail and saying so is more useful than\n * discarding the whole suggestion. The authoritative refusal is on the template, below,\n * since the template is what actually gets embedded.\n */\nexport function CapSelectedFields(\n    fields: ReadonlyArray<string> | null | undefined,\n): { fields: string[]; dropped: number } {\n    if (!fields || fields.length === 0) {\n        return { fields: [], dropped: 0 };\n    }\n    const unique = [...new Set(fields)];\n    if (unique.length <= MAX_TEMPLATE_FIELDS) {\n        return { fields: unique, dropped: 0 };\n    }\n    return {\n        fields: unique.slice(0, MAX_TEMPLATE_FIELDS),\n        dropped: unique.length - MAX_TEMPLATE_FIELDS,\n    };\n}\n\n/**\n * The refusal message when a template interpolates more fields than the cap allows, or null\n * when it is within the cap.\n *\n * A template cannot be truncated without mangling its prose, so this one refuses rather than\n * trimming — and the message carries both numbers, because \"too many fields\" without the\n * count is not actionable. The template is editable in the dialog, so the user can act on it\n * without losing the suggestion.\n */\nexport function TemplateFieldCapRefusal(templateText: string | null | undefined): string | null {\n    const names = TemplateFieldNames(templateText);\n    if (names.length <= MAX_TEMPLATE_FIELDS) {\n        return null;\n    }\n    return (\n        `This template interpolates ${names.length} fields, and at most ${MAX_TEMPLATE_FIELDS} ` +\n        `may be embedded. A template this wide stops reading like the natural language that ` +\n        `embeds well, and pulls in columns nobody chose. Remove ` +\n        `${names.length - MAX_TEMPLATE_FIELDS} field(s) from the template and save again.`\n    );\n}\n\n// ─── 3. Which entities may be offered as vectorization targets ───────────────\n\n/**\n * Schemas whose entities are not offered in the picker.\n *\n * Follows the existing frontend precedent — `FRONTEND_BLOCKED_SCHEMAS` in\n * `DatabaseDesigner/services/database-designer.engine.ts` and in\n * `DatabaseDesigner/components/create-wizard/steps/step-relationships.component.ts`, both\n * `new Set(['__mj'])` — rather than inventing a rule.\n */\nexport const VECTOR_BLOCKED_SCHEMAS: ReadonlySet<string> = new Set(['__mj']);\n\n/**\n * True when an entity may be offered as a vectorization target.\n *\n * The picker previously listed every entity in metadata, so MemberJunction's own ~378\n * internal entities — prompt runs, audit logs, record changes, credentials — were offered\n * alongside the user's business tables, both as noise and as embedding targets.\n *\n * Two rules, and they currently pick out the same set: every `MJ:`-prefixed entity is in the\n * `__mj` schema and vice versa (378 and 378, exactly coextensive, in the 5.46 baseline). The\n * name-prefix rule is therefore defence in depth against an MJ entity registered into some\n * other schema, not extra coverage today. Said plainly so nobody later reads it as load\n * bearing and is surprised.\n *\n * Note MJ does seed two `Search`-typed documents over `__mj` entities (`AI Agents Search`,\n * `AI Prompts Search`), so framework entities are not unvectorizable in principle. Those\n * arrive through metadata sync, which is a different authoring channel with a different\n * audience; this dialog is where a business user configures vectorization of their own data,\n * and burying their tables under the framework's is the defect being fixed.\n */\nexport function IsVectorizableEntity(\n    entity: { Name: string; SchemaName?: string | null },\n): boolean {\n    if (entity.SchemaName && VECTOR_BLOCKED_SCHEMAS.has(entity.SchemaName)) {\n        return false;\n    }\n    return !entity.Name.startsWith('MJ:');\n}\n"]}