@futdevpro/nts-dynamo 1.15.151 → 1.15.154

package/src/_modules/local-vector-search/_services/lvs-bm25.util.ts

@@ -1,206 +1,206 @@
-/**
- * BM25 text-search ranking util a LVS hybrid search-hez (FR-004).
- *
- * Pure TypeScript, dependency-free. In-memory corpus alapjan szamol score-okat,
- * NEM perzisztal indexet (a hybrid hivasonkent ujraepiti a corpus-t a candidate
- * dokumentumokon — kis (~100..10000 dokumentum) LVS-corpus eseten ez gyors es
- * egyszeru).
- *
- * Canonical params: k1=1.2, b=0.75 (industry standard a "lucene-szeru"
- * implementaciokban). NEM expose-oltak — ha kell, FR-002 kovetkezo iteracioban
- * tehetjuk parameterizalhatova.
- *
- * Tokenizer: `text.toLowerCase().match(/\w+/g) || []`. Case-insensitive,
- * alphanumeric+underscore boundary-k. `UserController` egy token marad (jo az
- * identifier match-re), `auth-flow` ket tokenre esik (auth + flow).
- *
- * IDF formula (BM25+): `log((N - df + 0.5) / (df + 0.5) + 1)`. A +1 a logon
- * belul garantalja, hogy a kozos szavak is pozitiv (kicsi) IDF-et kapjanak,
- * NEM negativ-t — fontos a hybrid score-merge-nel hogy ne huzzon le dokumentumot
- * ahol kozos szo szerepel.
- */
-
-
-/** BM25 k1 parameter (term saturation control). Canonical default. */
-const BM25_K1: number = 1.2;
-
-/** BM25 b parameter (length normalization weight, 0=off, 1=full). Canonical default. */
-const BM25_B: number = 0.75;
-
-/** Token regex — alphanumeric + underscore. */
-const TOKEN_REGEX: RegExp = /\w+/g;
-
-
-/**
- * Egy dokumentum BM25-score-ja egy query ellen, egy elore-felepitett corpus
- * konteztusaban.
- */
-export interface DyNTS_LVS_BM25_DocScore {
-  /** Dokumentum azonosito (mint az LVS_SearchResult `id`-jaben). */
-  id: string;
-  /** Nyers BM25 score (0..∞). NEM normalizalt. */
-  score: number;
-}
-
-
-/**
- * Felepitett BM25 corpus — egy adott dokumentumhalmaz indexe. A `score()` az
- * indexen kerdez le egy query-t es minden dokumentumra ad egy score-t.
- *
- * Egy corpus egyszer-hasznalatos a hybrid search hivasban — NEM kell cache-elni,
- * a felepites O(N * |doc|) ami pici N-re elhanyagolhato.
- */
-export class DyNTS_LVS_BM25_Corpus {
-
-  /** Tokenizalt dokumentumok: id -> tokens. */
-  private readonly docTokens: Map<string, string[]> = new Map<string, string[]>();
-  /** Doc-length: id -> token count. */
-  private readonly docLengths: Map<string, number> = new Map<string, number>();
-  /** Term -> doc-frequency (hany docban szerepel az adott term, legalabb 1x). */
-  private readonly termDocFreq: Map<string, number> = new Map<string, number>();
-  /** Term -> id -> term-frequency a docban. */
-  private readonly termFreqByDoc: Map<string, Map<string, number>> = new Map<string, Map<string, number>>();
-  /** Atlagos dokumentum-hossz (token count). */
-  private avgDocLength: number = 0;
-  /** Total doc count. */
-  private docCount: number = 0;
-
-  /**
-   * Letrehoz egy uj corpus-t a megadott id->text parok-bol.
-   * NEM dob hibat ures input-ra — ures corpus ervenyes (minden score = 0).
-   */
-  constructor(docs: { id: string; text: string }[]) {
-    if (!Array.isArray(docs) || docs.length === 0) { return; }
-
-    let totalLength: number = 0;
-    for (const doc of docs) {
-      if (!doc || typeof doc.id !== 'string' || typeof doc.text !== 'string') { continue; }
-      const tokens: string[] = DyNTS_LVS_BM25_Corpus.tokenize(doc.text);
-      this.docTokens.set(doc.id, tokens);
-      this.docLengths.set(doc.id, tokens.length);
-      totalLength += tokens.length;
-
-      // Term-frequency a docban
-      const localTf: Map<string, number> = new Map<string, number>();
-      for (const tok of tokens) {
-        localTf.set(tok, (localTf.get(tok) ?? 0) + 1);
-      }
-
-      for (const [term, tf] of localTf) {
-        // Doc-frequency: +1 per term, per doc
-        this.termDocFreq.set(term, (this.termDocFreq.get(term) ?? 0) + 1);
-        // Term-freq-by-doc reverse index
-        let perDoc: Map<string, number> | undefined = this.termFreqByDoc.get(term);
-        if (!perDoc) {
-          perDoc = new Map<string, number>();
-          this.termFreqByDoc.set(term, perDoc);
-        }
-        perDoc.set(doc.id, tf);
-      }
-    }
-
-    this.docCount = this.docTokens.size;
-    this.avgDocLength = this.docCount > 0 ? totalLength / this.docCount : 0;
-  }
-
-
-  /**
-   * Public tokenizer — exportalt, hogy spec-ek + hivok ugyanazt a normalizalast
-   * tudjak hasznalni mint a corpus.
-   */
-  static tokenize(text: string): string[] {
-    if (typeof text !== 'string' || text.length === 0) { return []; }
-    return text.toLowerCase().match(TOKEN_REGEX) ?? [];
-  }
-
-
-  /**
-   * Visszaadja a corpus dokumentum-szamat (NEM-ures docok).
-   */
-  size(): number {
-    return this.docCount;
-  }
-
-
-  /**
-   * BM25 score minden dokumentumra a query-re.
-   *
-   * Ures query → minden score 0 (degenerate case; a hivo kezelje ha kell).
-   * Ures corpus → ures array.
-   */
-  score(query: string): DyNTS_LVS_BM25_DocScore[] {
-    if (this.docCount === 0) { return []; }
-    const queryTokens: string[] = DyNTS_LVS_BM25_Corpus.tokenize(query);
-    if (queryTokens.length === 0) {
-      // Minden doc 0 score-t kap
-      const results: DyNTS_LVS_BM25_DocScore[] = [];
-      for (const id of this.docTokens.keys()) {
-        results.push({ id: id, score: 0 });
-      }
-      return results;
-    }
-
-    // Egyedi query-termek halmaza (ismetlodes nem ad nagyobb IDF-et)
-    const uniqueTerms: string[] = Array.from(new Set<string>(queryTokens));
-
-    // Pre-compute IDF a query-termekre
-    const idfMap: Map<string, number> = new Map<string, number>();
-    for (const term of uniqueTerms) {
-      const df: number = this.termDocFreq.get(term) ?? 0;
-      // BM25+ IDF: log((N - df + 0.5) / (df + 0.5) + 1)
-      const idf: number = Math.log((this.docCount - df + 0.5) / (df + 0.5) + 1);
-      idfMap.set(term, idf);
-    }
-
-    // Per-doc BM25 score
-    const results: DyNTS_LVS_BM25_DocScore[] = [];
-    for (const [docId, docLen] of this.docLengths) {
-      let score: number = 0;
-      for (const term of uniqueTerms) {
-        const tf: number = this.termFreqByDoc.get(term)?.get(docId) ?? 0;
-        if (tf === 0) { continue; }
-        const idf: number = idfMap.get(term) ?? 0;
-        const norm: number = 1 - BM25_B + BM25_B * (docLen / (this.avgDocLength || 1));
-        score += idf * (tf * (BM25_K1 + 1)) / (tf + BM25_K1 * norm);
-      }
-      results.push({ id: docId, score: score });
-    }
-
-    return results;
-  }
-}
-
-
-/**
- * Min-max normalizalas [0,1] tartomanyra. A hybrid score-merge-hez kell a BM25
- * score-okat a candidate-szetten 0..1 sav-ba hozni (a cosine mar 0..1).
- *
- * Edge case-ek:
- *  - Ures array → [].
- *  - Minden score azonos (max-min === 0) → minden 0.0 (NEM 0.5 vagy 1.0; ha
- *    nincs diszkriminacio, ne tegyunk hozza signal-t).
- *  - Negativ score-ok (BM25+IDF garantal pozitivat, de defensive): a min-max
- *    ugyanugy mukodik.
- */
-export function dyNTS_LVS_BM25_minMaxNormalize(
-  scores: DyNTS_LVS_BM25_DocScore[],
-): DyNTS_LVS_BM25_DocScore[] {
-  if (!Array.isArray(scores) || scores.length === 0) { return []; }
-  let min: number = Infinity;
-  let max: number = -Infinity;
-  for (const s of scores) {
-    if (s.score < min) { min = s.score; }
-    if (s.score > max) { max = s.score; }
-  }
-  const range: number = max - min;
-  if (range === 0) {
-    // Nincs diszkriminacio — minden 0.0 (NEM huzzon le, NEM huzzon fel)
-    return scores.map((s: DyNTS_LVS_BM25_DocScore): DyNTS_LVS_BM25_DocScore => ({
-      id: s.id, score: 0,
-    }));
-  }
-  return scores.map((s: DyNTS_LVS_BM25_DocScore): DyNTS_LVS_BM25_DocScore => ({
-    id: s.id,
-    score: (s.score - min) / range,
-  }));
-}
+/**
+ * BM25 text-search ranking util a LVS hybrid search-hez (FR-004).
+ *
+ * Pure TypeScript, dependency-free. In-memory corpus alapjan szamol score-okat,
+ * NEM perzisztal indexet (a hybrid hivasonkent ujraepiti a corpus-t a candidate
+ * dokumentumokon — kis (~100..10000 dokumentum) LVS-corpus eseten ez gyors es
+ * egyszeru).
+ *
+ * Canonical params: k1=1.2, b=0.75 (industry standard a "lucene-szeru"
+ * implementaciokban). NEM expose-oltak — ha kell, FR-002 kovetkezo iteracioban
+ * tehetjuk parameterizalhatova.
+ *
+ * Tokenizer: `text.toLowerCase().match(/\w+/g) || []`. Case-insensitive,
+ * alphanumeric+underscore boundary-k. `UserController` egy token marad (jo az
+ * identifier match-re), `auth-flow` ket tokenre esik (auth + flow).
+ *
+ * IDF formula (BM25+): `log((N - df + 0.5) / (df + 0.5) + 1)`. A +1 a logon
+ * belul garantalja, hogy a kozos szavak is pozitiv (kicsi) IDF-et kapjanak,
+ * NEM negativ-t — fontos a hybrid score-merge-nel hogy ne huzzon le dokumentumot
+ * ahol kozos szo szerepel.
+ */
+
+
+/** BM25 k1 parameter (term saturation control). Canonical default. */
+const BM25_K1: number = 1.2;
+
+/** BM25 b parameter (length normalization weight, 0=off, 1=full). Canonical default. */
+const BM25_B: number = 0.75;
+
+/** Token regex — alphanumeric + underscore. */
+const TOKEN_REGEX: RegExp = /\w+/g;
+
+
+/**
+ * Egy dokumentum BM25-score-ja egy query ellen, egy elore-felepitett corpus
+ * konteztusaban.
+ */
+export interface DyNTS_LVS_BM25_DocScore {
+  /** Dokumentum azonosito (mint az LVS_SearchResult `id`-jaben). */
+  id: string;
+  /** Nyers BM25 score (0..∞). NEM normalizalt. */
+  score: number;
+}
+
+
+/**
+ * Felepitett BM25 corpus — egy adott dokumentumhalmaz indexe. A `score()` az
+ * indexen kerdez le egy query-t es minden dokumentumra ad egy score-t.
+ *
+ * Egy corpus egyszer-hasznalatos a hybrid search hivasban — NEM kell cache-elni,
+ * a felepites O(N * |doc|) ami pici N-re elhanyagolhato.
+ */
+export class DyNTS_LVS_BM25_Corpus {
+
+  /** Tokenizalt dokumentumok: id -> tokens. */
+  private readonly docTokens: Map<string, string[]> = new Map<string, string[]>();
+  /** Doc-length: id -> token count. */
+  private readonly docLengths: Map<string, number> = new Map<string, number>();
+  /** Term -> doc-frequency (hany docban szerepel az adott term, legalabb 1x). */
+  private readonly termDocFreq: Map<string, number> = new Map<string, number>();
+  /** Term -> id -> term-frequency a docban. */
+  private readonly termFreqByDoc: Map<string, Map<string, number>> = new Map<string, Map<string, number>>();
+  /** Atlagos dokumentum-hossz (token count). */
+  private avgDocLength: number = 0;
+  /** Total doc count. */
+  private docCount: number = 0;
+
+  /**
+   * Letrehoz egy uj corpus-t a megadott id->text parok-bol.
+   * NEM dob hibat ures input-ra — ures corpus ervenyes (minden score = 0).
+   */
+  constructor(docs: { id: string; text: string }[]) {
+    if (!Array.isArray(docs) || docs.length === 0) { return; }
+
+    let totalLength: number = 0;
+    for (const doc of docs) {
+      if (!doc || typeof doc.id !== 'string' || typeof doc.text !== 'string') { continue; }
+      const tokens: string[] = DyNTS_LVS_BM25_Corpus.tokenize(doc.text);
+      this.docTokens.set(doc.id, tokens);
+      this.docLengths.set(doc.id, tokens.length);
+      totalLength += tokens.length;
+
+      // Term-frequency a docban
+      const localTf: Map<string, number> = new Map<string, number>();
+      for (const tok of tokens) {
+        localTf.set(tok, (localTf.get(tok) ?? 0) + 1);
+      }
+
+      for (const [term, tf] of localTf) {
+        // Doc-frequency: +1 per term, per doc
+        this.termDocFreq.set(term, (this.termDocFreq.get(term) ?? 0) + 1);
+        // Term-freq-by-doc reverse index
+        let perDoc: Map<string, number> | undefined = this.termFreqByDoc.get(term);
+        if (!perDoc) {
+          perDoc = new Map<string, number>();
+          this.termFreqByDoc.set(term, perDoc);
+        }
+        perDoc.set(doc.id, tf);
+      }
+    }
+
+    this.docCount = this.docTokens.size;
+    this.avgDocLength = this.docCount > 0 ? totalLength / this.docCount : 0;
+  }
+
+
+  /**
+   * Public tokenizer — exportalt, hogy spec-ek + hivok ugyanazt a normalizalast
+   * tudjak hasznalni mint a corpus.
+   */
+  static tokenize(text: string): string[] {
+    if (typeof text !== 'string' || text.length === 0) { return []; }
+    return text.toLowerCase().match(TOKEN_REGEX) ?? [];
+  }
+
+
+  /**
+   * Visszaadja a corpus dokumentum-szamat (NEM-ures docok).
+   */
+  size(): number {
+    return this.docCount;
+  }
+
+
+  /**
+   * BM25 score minden dokumentumra a query-re.
+   *
+   * Ures query → minden score 0 (degenerate case; a hivo kezelje ha kell).
+   * Ures corpus → ures array.
+   */
+  score(query: string): DyNTS_LVS_BM25_DocScore[] {
+    if (this.docCount === 0) { return []; }
+    const queryTokens: string[] = DyNTS_LVS_BM25_Corpus.tokenize(query);
+    if (queryTokens.length === 0) {
+      // Minden doc 0 score-t kap
+      const results: DyNTS_LVS_BM25_DocScore[] = [];
+      for (const id of this.docTokens.keys()) {
+        results.push({ id: id, score: 0 });
+      }
+      return results;
+    }
+
+    // Egyedi query-termek halmaza (ismetlodes nem ad nagyobb IDF-et)
+    const uniqueTerms: string[] = Array.from(new Set<string>(queryTokens));
+
+    // Pre-compute IDF a query-termekre
+    const idfMap: Map<string, number> = new Map<string, number>();
+    for (const term of uniqueTerms) {
+      const df: number = this.termDocFreq.get(term) ?? 0;
+      // BM25+ IDF: log((N - df + 0.5) / (df + 0.5) + 1)
+      const idf: number = Math.log((this.docCount - df + 0.5) / (df + 0.5) + 1);
+      idfMap.set(term, idf);
+    }
+
+    // Per-doc BM25 score
+    const results: DyNTS_LVS_BM25_DocScore[] = [];
+    for (const [docId, docLen] of this.docLengths) {
+      let score: number = 0;
+      for (const term of uniqueTerms) {
+        const tf: number = this.termFreqByDoc.get(term)?.get(docId) ?? 0;
+        if (tf === 0) { continue; }
+        const idf: number = idfMap.get(term) ?? 0;
+        const norm: number = 1 - BM25_B + BM25_B * (docLen / (this.avgDocLength || 1));
+        score += idf * (tf * (BM25_K1 + 1)) / (tf + BM25_K1 * norm);
+      }
+      results.push({ id: docId, score: score });
+    }
+
+    return results;
+  }
+}
+
+
+/**
+ * Min-max normalizalas [0,1] tartomanyra. A hybrid score-merge-hez kell a BM25
+ * score-okat a candidate-szetten 0..1 sav-ba hozni (a cosine mar 0..1).
+ *
+ * Edge case-ek:
+ *  - Ures array → [].
+ *  - Minden score azonos (max-min === 0) → minden 0.0 (NEM 0.5 vagy 1.0; ha
+ *    nincs diszkriminacio, ne tegyunk hozza signal-t).
+ *  - Negativ score-ok (BM25+IDF garantal pozitivat, de defensive): a min-max
+ *    ugyanugy mukodik.
+ */
+export function dyNTS_LVS_BM25_minMaxNormalize(
+  scores: DyNTS_LVS_BM25_DocScore[],
+): DyNTS_LVS_BM25_DocScore[] {
+  if (!Array.isArray(scores) || scores.length === 0) { return []; }
+  let min: number = Infinity;
+  let max: number = -Infinity;
+  for (const s of scores) {
+    if (s.score < min) { min = s.score; }
+    if (s.score > max) { max = s.score; }
+  }
+  const range: number = max - min;
+  if (range === 0) {
+    // Nincs diszkriminacio — minden 0.0 (NEM huzzon le, NEM huzzon fel)
+    return scores.map((s: DyNTS_LVS_BM25_DocScore): DyNTS_LVS_BM25_DocScore => ({
+      id: s.id, score: 0,
+    }));
+  }
+  return scores.map((s: DyNTS_LVS_BM25_DocScore): DyNTS_LVS_BM25_DocScore => ({
+    id: s.id,
+    score: (s.score - min) / range,
+  }));
+}