@juspay/neurolink 10.3.0 → 10.4.0

package/dist/lib/knowledge/knowledgeIndex.js

@@ -0,0 +1,205 @@
+/**
+ * In-memory knowledge index construction.
+ *
+ * Index construction is an application-process lifecycle operation, not a
+ * per-session one. A complete `KnowledgeIndexSnapshot` is built once from the
+ * normalized entries and then queried by every turn.
+ *
+ * The lexical scorer is a compact field-aware BM25 (BM25F-style): each of the
+ * five text fields is scored independently and combined with per-field
+ * weights, retaining the per-field breakdown so a retrieval decision is
+ * explainable in traces. No embedding model or vector store is involved.
+ */
+import { normalizePhrases, normalizeText, tokenize } from "./normalize.js";
+const FIELD_NAMES = [
+    "title",
+    "aliases",
+    "keywords",
+    "summary",
+    "body",
+];
+/** Okapi BM25 saturation and length-normalization parameters. */
+const BM25_K1 = 1.5;
+const BM25_B = 0.75;
+/**
+ * Field-aware BM25 over the document set. Query text is scored per field and
+ * the weighted per-field scores are summed. Satisfies the structural
+ * `KnowledgeLexicalSearcher` type held by a snapshot.
+ */
+export class KnowledgeLexicalIndex {
+    weights;
+    // field -> (term -> (docId -> term frequency))
+    postings = new Map();
+    // field -> (docId -> field length in tokens)
+    fieldLengths = new Map();
+    // field -> total tokens across all docs (for average length)
+    fieldTokenTotals = new Map();
+    // field -> average field length (computed in finalize)
+    avgFieldLength = new Map();
+    docCount = 0;
+    constructor(weights) {
+        this.weights = weights;
+        for (const field of FIELD_NAMES) {
+            this.postings.set(field, new Map());
+            this.fieldLengths.set(field, new Map());
+            this.fieldTokenTotals.set(field, 0);
+        }
+    }
+    add(document) {
+        this.docCount += 1;
+        for (const field of FIELD_NAMES) {
+            const tokens = document.fields[field];
+            const postingsForField = this.postings.get(field);
+            const lengthsForField = this.fieldLengths.get(field);
+            if (!postingsForField || !lengthsForField) {
+                continue;
+            }
+            lengthsForField.set(document.id, tokens.length);
+            this.fieldTokenTotals.set(field, (this.fieldTokenTotals.get(field) ?? 0) + tokens.length);
+            const termFrequency = new Map();
+            for (const token of tokens) {
+                termFrequency.set(token, (termFrequency.get(token) ?? 0) + 1);
+            }
+            for (const [term, frequency] of termFrequency) {
+                const docMap = postingsForField.get(term);
+                if (docMap) {
+                    docMap.set(document.id, frequency);
+                }
+                else {
+                    postingsForField.set(term, new Map([[document.id, frequency]]));
+                }
+            }
+        }
+    }
+    finalize() {
+        for (const field of FIELD_NAMES) {
+            const total = this.fieldTokenTotals.get(field) ?? 0;
+            this.avgFieldLength.set(field, this.docCount > 0 ? total / this.docCount : 0);
+        }
+    }
+    search(queryTokens, topK, eligibleEntryIds) {
+        if (this.docCount === 0 || queryTokens.length === 0) {
+            return [];
+        }
+        const uniqueTerms = Array.from(new Set(queryTokens));
+        const totals = new Map();
+        const breakdowns = new Map();
+        for (const field of FIELD_NAMES) {
+            const weight = this.weights[field];
+            const avgdl = this.avgFieldLength.get(field) ?? 0;
+            const postingsForField = this.postings.get(field);
+            const lengthsForField = this.fieldLengths.get(field);
+            if (weight <= 0 || avgdl <= 0 || !postingsForField || !lengthsForField) {
+                continue;
+            }
+            for (const term of uniqueTerms) {
+                const docMap = postingsForField.get(term);
+                if (!docMap) {
+                    continue;
+                }
+                const documentFrequency = docMap.size;
+                const idf = Math.log(1 +
+                    (this.docCount - documentFrequency + 0.5) /
+                        (documentFrequency + 0.5));
+                for (const [docId, frequency] of docMap) {
+                    if (eligibleEntryIds && !eligibleEntryIds.has(docId)) {
+                        continue;
+                    }
+                    const length = lengthsForField.get(docId) ?? 0;
+                    const denominator = frequency + BM25_K1 * (1 - BM25_B + BM25_B * (length / avgdl));
+                    const contribution = ((idf * (frequency * (BM25_K1 + 1))) / denominator) * weight;
+                    totals.set(docId, (totals.get(docId) ?? 0) + contribution);
+                    const breakdown = breakdowns.get(docId) ?? {};
+                    breakdown[field] = (breakdown[field] ?? 0) + contribution;
+                    breakdowns.set(docId, breakdown);
+                }
+            }
+        }
+        const matches = [];
+        for (const [id, score] of totals) {
+            matches.push({ id, score, fieldScores: breakdowns.get(id) ?? {} });
+        }
+        matches.sort((left, right) => right.score - left.score || compareIds(left.id, right.id));
+        return matches.slice(0, topK);
+    }
+}
+/** Deterministic tiebreak so equal scores produce a stable order. */
+const compareIds = (left, right) => {
+    if (left < right) {
+        return -1;
+    }
+    if (left > right) {
+        return 1;
+    }
+    return 0;
+};
+const addToIndex = (index, key, id) => {
+    const bucket = index.get(key);
+    if (bucket) {
+        bucket.add(id);
+    }
+    else {
+        index.set(key, new Set([id]));
+    }
+};
+/**
+ * Build the internal search document from a normalized entry. `exactKeys`
+ * carry whole-phrase identifiers (entry id and title); each
+ * text field is tokenized for the field-aware scorer.
+ */
+export const buildDocument = (entry) => {
+    const exactKeys = normalizePhrases([entry.id, entry.title]);
+    return {
+        id: entry.id,
+        exactKeys,
+        fields: {
+            title: tokenize(entry.title),
+            aliases: entry.aliases.flatMap((alias) => tokenize(alias)),
+            keywords: entry.keywords.flatMap((keyword) => tokenize(keyword)),
+            summary: tokenize(entry.summary),
+            body: tokenize(entry.body),
+        },
+    };
+};
+/**
+ * Build the complete immutable snapshot: entry map, exact/alias/relation
+ * indexes, and the field-aware lexical index.
+ */
+export const buildIndexSnapshot = (entries, weights) => {
+    const entriesById = new Map();
+    const exactIndex = new Map();
+    const aliasIndex = new Map();
+    const relationIndex = new Map();
+    const lexical = new KnowledgeLexicalIndex(weights);
+    for (const entry of entries) {
+        entriesById.set(entry.id, entry);
+        const document = buildDocument(entry);
+        for (const key of document.exactKeys) {
+            addToIndex(exactIndex, key, entry.id);
+        }
+        for (const alias of entry.aliases) {
+            const key = normalizeText(alias);
+            if (key) {
+                addToIndex(aliasIndex, key, entry.id);
+            }
+        }
+        const relations = [...entry.relatedEntryIds];
+        if (entry.parentEntryId) {
+            relations.push(entry.parentEntryId);
+        }
+        if (relations.length > 0) {
+            relationIndex.set(entry.id, relations);
+        }
+        lexical.add(document);
+    }
+    lexical.finalize();
+    return {
+        entriesById,
+        exactIndex,
+        aliasIndex,
+        relationIndex,
+        lexical,
+        entryCount: entries.length,
+    };
+};
+//# sourceMappingURL=knowledgeIndex.js.map

package/dist/lib/knowledge/normalize.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Deterministic text normalization for knowledge grounding.
+ *
+ * The SAME normalizer runs at index-build time and query time so that a
+ * configuration key, a technical identifier, and a user's phrasing all
+ * collapse to the same comparable token stream:
+ *
+ *   "Multi-Step Flow"        -> "multi step flow"
+ *   "enableMultiStepFlow"    -> "enable multi step flow"
+ *   "ACCOUNT_STATUS"         -> "account status"
+ *   "  Access   Policy  "    -> "access policy"
+ *
+ * The pipeline is intentionally conservative: it splits identifiers and
+ * separators and folds case/diacritics, but it NEVER invents synonyms.
+ * Reviewed acronym and spelling expansions belong in an entry's `aliases`,
+ * not in this function.
+ */
+/**
+ * Normalize an arbitrary string into a single space-delimited lowercase
+ * phrase. Idempotent: `normalizeText(normalizeText(x)) === normalizeText(x)`.
+ */
+export declare const normalizeText: (input: string) => string;
+/**
+ * Normalize then split into tokens. Empty input yields an empty array.
+ * Numbers and alphanumeric identifiers are preserved as their own tokens.
+ */
+export declare const tokenize: (input: string) => string[];
+/**
+ * Normalize each candidate phrase and drop blanks/duplicates while preserving
+ * first-seen order. Used to build the exact-key and alias key sets.
+ */
+export declare const normalizePhrases: (phrases: string[]) => string[];

package/dist/lib/knowledge/normalize.js

@@ -0,0 +1,75 @@
+/**
+ * Deterministic text normalization for knowledge grounding.
+ *
+ * The SAME normalizer runs at index-build time and query time so that a
+ * configuration key, a technical identifier, and a user's phrasing all
+ * collapse to the same comparable token stream:
+ *
+ *   "Multi-Step Flow"        -> "multi step flow"
+ *   "enableMultiStepFlow"    -> "enable multi step flow"
+ *   "ACCOUNT_STATUS"         -> "account status"
+ *   "  Access   Policy  "    -> "access policy"
+ *
+ * The pipeline is intentionally conservative: it splits identifiers and
+ * separators and folds case/diacritics, but it NEVER invents synonyms.
+ * Reviewed acronym and spelling expansions belong in an entry's `aliases`,
+ * not in this function.
+ */
+/**
+ * Insert spaces at camelCase / PascalCase / acronym boundaries. Must run
+ * BEFORE lowercasing because it relies on case information.
+ *
+ *   "enableURLReading"            -> "enable URL Reading"
+ *   "customAccessOptionsV2"       -> "custom Access Options V2"
+ */
+const splitIdentifierBoundaries = (input) => input
+    // lower/number followed by an uppercase: "enableTwo" -> "enable Two"
+    .replace(/([\p{Ll}\p{N}])(\p{Lu})/gu, "$1 $2")
+    // acronym run followed by a Word: "URLReading" -> "URL Reading"
+    .replace(/(\p{Lu}+)(\p{Lu}\p{Ll})/gu, "$1 $2");
+/**
+ * Normalize an arbitrary string into a single space-delimited lowercase
+ * phrase. Idempotent: `normalizeText(normalizeText(x)) === normalizeText(x)`.
+ */
+export const normalizeText = (input) => {
+    if (!input) {
+        return "";
+    }
+    const boundaries = splitIdentifierBoundaries(
+    // NFKC folds compatibility forms; NFD + mark-strip removes diacritics.
+    input
+        .normalize("NFKC")
+        .normalize("NFD")
+        .replace(/\p{M}+/gu, ""));
+    return (boundaries
+        // any run of non-letter/non-number (snake_case, kebab-case, dots, slashes,
+        // punctuation, and existing whitespace) collapses to a single space.
+        .replace(/[^\p{L}\p{N}]+/gu, " ")
+        .trim()
+        .toLowerCase());
+};
+/**
+ * Normalize then split into tokens. Empty input yields an empty array.
+ * Numbers and alphanumeric identifiers are preserved as their own tokens.
+ */
+export const tokenize = (input) => {
+    const normalized = normalizeText(input);
+    return normalized ? normalized.split(" ") : [];
+};
+/**
+ * Normalize each candidate phrase and drop blanks/duplicates while preserving
+ * first-seen order. Used to build the exact-key and alias key sets.
+ */
+export const normalizePhrases = (phrases) => {
+    const seen = new Set();
+    const out = [];
+    for (const phrase of phrases) {
+        const normalized = normalizeText(phrase);
+        if (normalized && !seen.has(normalized)) {
+            seen.add(normalized);
+            out.push(normalized);
+        }
+    }
+    return out;
+};
+//# sourceMappingURL=normalize.js.map

package/dist/lib/knowledge/resolve.d.ts

@@ -0,0 +1,18 @@
+import type { KnowledgeEntryInput, KnowledgeManifest, KnowledgeNormalizeOptions, KnowledgeNormalizeResult, KnowledgeSource, NormalizedKnowledgeEntry } from "../types/index.js";
+/**
+ * Resolve one authored entry into a complete `NormalizedKnowledgeEntry`,
+ * materializing every omitted optional (optional arrays to `[]`, `body` to "",
+ * `kind` to "text", `status` to "active") so downstream code never re-checks
+ * it. Authored arrays are cloned so the normalized snapshot is isolated from
+ * caller-owned references.
+ */
+export declare const resolveEntry: (input: KnowledgeEntryInput, version: string) => NormalizedKnowledgeEntry;
+/** Expand a build manifest into sources — one per catalog. */
+export declare const manifestToSources: (manifest: KnowledgeManifest) => KnowledgeSource[];
+/**
+ * Resolve and validate every source into normalized entries. `validation.ok`
+ * is false when any error-level issue was found; the caller (index builder)
+ * must not swap in an index built from an invalid set. Async so the public
+ * contract is stable if a future source kind needs asynchronous loading.
+ */
+export declare const normalizeAndValidate: (sources: KnowledgeSource[], options: KnowledgeNormalizeOptions) => Promise<KnowledgeNormalizeResult>;

package/dist/lib/knowledge/resolve.js

@@ -0,0 +1,157 @@
+/**
+ * Source loading, default resolution, and generic validation.
+ *
+ * Required fields (id/title/summary/domain/integrations) come straight
+ * from the authored entry; omitted optionals fall back to SDK defaults. This
+ * covers only provider-neutral invariants (ids, required fields, enum values,
+ * relationship integrity). Host-specific rules — configuration-key coverage,
+ * taxonomy conflicts — are enforced by the host build BEFORE the manifest
+ * reaches NeuroLink. Errors block indexing; warnings are surfaced but do not.
+ */
+import { decodeArray, decodeString } from "type-decoder/dist/index.js";
+import { normalizeText } from "./normalize.js";
+const SDK_DEFAULT_KIND = "text";
+const SDK_DEFAULT_STATUS = "active";
+const STATUSES = new Set(["draft", "active", "deprecated"]);
+const KINDS = new Set([
+    "concept",
+    "text",
+    "configuration",
+    "tool",
+    "procedure",
+    "policy",
+    "troubleshooting",
+]);
+/**
+ * Resolve one authored entry into a complete `NormalizedKnowledgeEntry`,
+ * materializing every omitted optional (optional arrays to `[]`, `body` to "",
+ * `kind` to "text", `status` to "active") so downstream code never re-checks
+ * it. Authored arrays are cloned so the normalized snapshot is isolated from
+ * caller-owned references.
+ */
+export const resolveEntry = (input, version) => ({
+    id: input.id,
+    title: input.title,
+    summary: input.summary,
+    domain: input.domain,
+    integrations: decodeArray(input.integrations, decodeString) ?? [],
+    kind: input.kind ?? SDK_DEFAULT_KIND,
+    status: input.status ?? SDK_DEFAULT_STATUS,
+    body: input.body ?? "",
+    aliases: [...(input.aliases ?? [])],
+    keywords: [...(input.keywords ?? [])],
+    relatedEntryIds: [...(input.relatedEntryIds ?? [])],
+    parentEntryId: input.parentEntryId,
+    version,
+});
+/** Expand a build manifest into sources — one per catalog. */
+export const manifestToSources = (manifest) => manifest.catalogs.map((catalog) => ({
+    id: catalog.id,
+    version: manifest.contentVersion,
+    entries: catalog.entries,
+}));
+/** Resolve a source's entries and its effective version. */
+const loadSourceEntries = (source, manifestVersion) => ({
+    id: source.id,
+    version: source.version ?? manifestVersion,
+    entries: source.entries,
+});
+const error = (code, message, extra) => ({ level: "error", code, message, ...extra });
+const warn = (code, message, extra) => ({ level: "warning", code, message, ...extra });
+/**
+ * Resolve and validate every source into normalized entries. `validation.ok`
+ * is false when any error-level issue was found; the caller (index builder)
+ * must not swap in an index built from an invalid set. Async so the public
+ * contract is stable if a future source kind needs asynchronous loading.
+ */
+export const normalizeAndValidate = async (sources, options) => {
+    const issues = [];
+    const entries = [];
+    const seenIds = new Set();
+    const aliasOwner = new Map();
+    for (const source of sources) {
+        const loaded = loadSourceEntries(source, options.manifestVersion);
+        if (!loaded.id) {
+            issues.push(error("missing-source-id", "Source has no id"));
+        }
+        for (const input of loaded.entries) {
+            if (!input.id) {
+                issues.push(error("missing-entry-id", "Entry has no id", { sourceId: loaded.id }));
+                continue;
+            }
+            if (seenIds.has(input.id)) {
+                issues.push(error("duplicate-entry-id", `Duplicate entry id "${input.id}"`, {
+                    entryId: input.id,
+                }));
+                continue;
+            }
+            seenIds.add(input.id);
+            if (!input.title || !input.title.trim()) {
+                issues.push(error("missing-title", "Entry is missing a title", {
+                    entryId: input.id,
+                }));
+            }
+            if (!input.summary || !input.summary.trim()) {
+                issues.push(error("missing-summary", "Entry is missing a summary", {
+                    entryId: input.id,
+                }));
+            }
+            if (!input.domain || !input.domain.trim()) {
+                issues.push(error("missing-domain", "Entry is missing a domain", {
+                    entryId: input.id,
+                }));
+            }
+            if (input.integrations === undefined) {
+                issues.push(error("missing-integrations", "Entry is missing integrations", {
+                    entryId: input.id,
+                }));
+            }
+            else if (decodeArray(input.integrations, decodeString) === null) {
+                issues.push(error("bad-integrations", "Entry integrations must be an array of strings", { entryId: input.id }));
+            }
+            const normalized = resolveEntry(input, loaded.version);
+            if (!KINDS.has(normalized.kind)) {
+                issues.push(error("bad-kind", `Unsupported kind "${normalized.kind}"`, {
+                    entryId: normalized.id,
+                }));
+            }
+            if (!STATUSES.has(normalized.status)) {
+                issues.push(error("bad-status", `Unsupported status "${normalized.status}"`, {
+                    entryId: normalized.id,
+                }));
+            }
+            for (const alias of normalized.aliases) {
+                const key = normalizeText(alias);
+                if (!key) {
+                    issues.push(warn("empty-alias", `Alias "${alias}" is empty after normalization`, { entryId: normalized.id }));
+                    continue;
+                }
+                const owner = aliasOwner.get(key);
+                if (owner && owner !== normalized.id) {
+                    issues.push(warn("duplicate-alias", `Alias "${alias}" also used by "${owner}"`, { entryId: normalized.id }));
+                }
+                else {
+                    aliasOwner.set(key, normalized.id);
+                }
+            }
+            entries.push(normalized);
+        }
+    }
+    // Relationship integrity — resolved after all entries are collected.
+    const idSet = new Set(entries.map((entry) => entry.id));
+    for (const entry of entries) {
+        for (const related of entry.relatedEntryIds) {
+            if (!idSet.has(related)) {
+                issues.push(warn("broken-relation", `relatedEntryId "${related}" not found`, {
+                    entryId: entry.id,
+                }));
+            }
+        }
+        if (entry.parentEntryId && !idSet.has(entry.parentEntryId)) {
+            issues.push(warn("broken-parent", `parentEntryId "${entry.parentEntryId}" not found`, { entryId: entry.id }));
+        }
+    }
+    const ok = !issues.some((issue) => issue.level === "error");
+    return { entries, validation: { ok, issues } };
+};
+//# sourceMappingURL=resolve.js.map

package/dist/lib/knowledge/retrieval.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Lexical-first retrieval pipeline.
+ *
+ * For one turn: authorize/filter BEFORE ranking, resolve exact identifiers and
+ * reviewed aliases via query n-gram lookup, add field-aware BM25 over a bounded
+ * query+history window, combine the deterministic signals, take the top
+ * candidates, expand a bounded set of directly-related entries, and classify
+ * confidence. No LLM, no embeddings, no vector store.
+ */
+import type { KnowledgeIndexSnapshot, KnowledgeResolvedRetrieval, KnowledgeRetrievalRequest, KnowledgeSelection } from "../types/index.js";
+/**
+ * Run retrieval against a ready snapshot. Returns the primary selection, a
+ * bounded relationship expansion, the scored candidate list (diagnostics), and
+ * a confidence class. Context assembly is a separate, later step.
+ */
+export declare const retrieve: (snapshot: KnowledgeIndexSnapshot, request: KnowledgeRetrievalRequest, config: KnowledgeResolvedRetrieval, blockedDomains?: string[]) => KnowledgeSelection;