@absolutejs/absolute 0.19.0-beta.493 → 0.19.0-beta.494

package/dist/ai/index.js

@@ -2149,6 +2149,8 @@ var STOP_WORDS = new Set([
   "why"
 ]);
 var tokenize = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => !STOP_WORDS.has(token)).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("ing") && token.length > 5 ? token.slice(0, -3) : token.endsWith("ed") && token.length > 4 ? token.slice(0, -2) : token.endsWith("es") && token.length > 4 ? token.slice(0, -2) : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).filter((token) => token.length > 1);
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
 var collectMetadataStrings = (value) => {
   if (typeof value === "string" || typeof value === "number") {
     return [String(value)];
@@ -2164,7 +2166,7 @@ var collectMetadataStrings = (value) => {
 var normalizeSourceForLexical = (source) => source.replace(/[#/_.-]+/g, " ").replace(/\bmd\b/g, "markdown").replace(/\bpptx\b/g, "presentation").replace(/\bxlsx\b/g, "spreadsheet workbook sheet").replace(/\bmp3\b/g, "audio transcript media").replace(/\bmp4\b/g, "video transcript media").replace(/\bzip\b/g, "archive bundle");
 var toFieldText = (value) => collectMetadataStrings(value).filter(Boolean).join(" ");
 var scoreTokenCoverage = (queryTokens, text) => {
-  const normalizedText = text.toLowerCase();
+  const normalizedText = (text ?? "").toLowerCase();
   if (normalizedText.length === 0) {
     return 0;
   }
@@ -2178,7 +2180,7 @@ var scoreTokenCoverage = (queryTokens, text) => {
 };
 var scorePhraseMatch = (query, text) => {
   const normalizedQuery = tokenize(query).join(" ");
-  const normalizedText = tokenize(text).join(" ");
+  const normalizedText = tokenize(text ?? "").join(" ");
   if (normalizedQuery.length === 0 || normalizedText.length === 0) {
     return 0;
   }
@@ -2190,7 +2192,7 @@ var scoreWeightedField = ({
   query,
   queryTokens,
   text
-}) => scoreTokenCoverage(queryTokens, text) * coverageWeight + scorePhraseMatch(query, text) * phraseWeight;
+}) => scoreTokenCoverage(queryTokens, text ?? "") * coverageWeight + scorePhraseMatch(query, text ?? "") * phraseWeight;
 var extractWeightedLexicalFields = (result) => {
   const metadata = result.metadata ?? {};
   const source = result.source ?? "";
@@ -2209,7 +2211,7 @@ var extractWeightedLexicalFields = (result) => {
   ].flatMap((value) => collectMetadataStrings(value)).join(" ");
   return {
     archivePath,
-    chunkText: result.chunkText,
+    chunkText: result.text,
     mediaSegments,
     metadataFocus,
     metadataText: toFieldText(metadata),
@@ -2217,6 +2219,34 @@ var extractWeightedLexicalFields = (result) => {
     title: result.title ?? ""
   };
 };
+var FIELD_WEIGHTS = {
+  archivePath: 4.2,
+  chunkText: 1,
+  mediaSegments: 3.8,
+  metadataFocus: 3.2,
+  metadataText: 1.4,
+  source: 3.4,
+  title: 2
+};
+var getWeightedFieldTokens = (result) => {
+  const fields = extractWeightedLexicalFields({
+    metadata: result.metadata,
+    source: result.source,
+    text: result.text,
+    title: result.title
+  });
+  return {
+    archivePath: tokenize(fields.archivePath ?? ""),
+    chunkText: tokenize(fields.chunkText ?? ""),
+    mediaSegments: tokenize(fields.mediaSegments ?? ""),
+    metadataFocus: tokenize(fields.metadataFocus ?? ""),
+    metadataText: tokenize(fields.metadataText ?? ""),
+    source: tokenize(fields.source ?? ""),
+    title: tokenize(fields.title ?? "")
+  };
+};
+var countWeightedTermFrequency = (fieldTokens, token) => Object.keys(FIELD_WEIGHTS).reduce((total, fieldName) => total + fieldTokens[fieldName].filter((value) => value === token).length * FIELD_WEIGHTS[fieldName], 0);
+var computeWeightedDocumentLength = (fieldTokens) => Object.keys(FIELD_WEIGHTS).reduce((total, fieldName) => total + fieldTokens[fieldName].length * FIELD_WEIGHTS[fieldName], 0);
 var buildRAGLexicalHaystack = (result) => [
   result.title,
   result.source,
@@ -2229,7 +2259,12 @@ var scoreRAGLexicalMatch = (query, result) => {
   if (queryTokens.length === 0) {
     return 0;
   }
-  const fields = extractWeightedLexicalFields(result);
+  const fields = extractWeightedLexicalFields({
+    metadata: result.metadata,
+    source: result.source,
+    text: result.chunkText,
+    title: result.title
+  });
   const haystack = buildRAGLexicalHaystack(result).toLowerCase();
   const overallCoverage = scoreTokenCoverage(queryTokens, haystack);
   if (overallCoverage === 0) {
@@ -2291,6 +2326,65 @@ var scoreRAGLexicalMatch = (query, result) => {
   const archiveBoost = resolveArchiveBoost(queryTokens, result);
   return titleScore + sourceScore + metadataFocusScore + archivePathScore + mediaSegmentScore + metadataScore + chunkScore + coverageBoost + exactPhraseBoost + fileKindBoost + transcriptBoost + archiveBoost;
 };
+var rankRAGLexicalMatches = (query, results) => {
+  const queryTokens = tokenize(query);
+  if (queryTokens.length === 0 || results.length === 0) {
+    return [];
+  }
+  const candidates = results.map((result) => {
+    const fieldTokens = getWeightedFieldTokens(result);
+    return {
+      fieldTokens,
+      length: computeWeightedDocumentLength(fieldTokens),
+      result
+    };
+  });
+  const averageDocumentLength = candidates.reduce((total, candidate) => total + candidate.length, 0) / Math.max(1, candidates.length);
+  const uniqueQueryTokens = [...new Set(queryTokens)];
+  const documentFrequency = new Map;
+  for (const token of uniqueQueryTokens) {
+    let seen = 0;
+    for (const candidate of candidates) {
+      const tf = countWeightedTermFrequency(candidate.fieldTokens, token);
+      if (tf > 0) {
+        seen += 1;
+      }
+    }
+    documentFrequency.set(token, seen);
+  }
+  return candidates.map((candidate, index) => {
+    let bm25Score = 0;
+    for (const token of uniqueQueryTokens) {
+      const termFrequency = countWeightedTermFrequency(candidate.fieldTokens, token);
+      if (termFrequency <= 0) {
+        continue;
+      }
+      const df = documentFrequency.get(token) ?? 0;
+      const idf = Math.log(1 + (candidates.length - df + 0.5) / (df + 0.5));
+      const denominator = termFrequency + BM25_K1 * (1 - BM25_B + BM25_B * (candidate.length / Math.max(1, averageDocumentLength)));
+      bm25Score += idf * (termFrequency * (BM25_K1 + 1) / Math.max(0.000000001, denominator));
+    }
+    const heuristicScore = scoreRAGLexicalMatch(query, {
+      chunkText: candidate.result.text,
+      metadata: candidate.result.metadata,
+      source: candidate.result.source,
+      title: candidate.result.title
+    });
+    return {
+      index,
+      result: candidate.result,
+      score: bm25Score + heuristicScore * 0.35
+    };
+  }).filter((entry) => entry.score > 0).sort((left, right) => {
+    if (right.score !== left.score) {
+      return right.score - left.score;
+    }
+    return left.index - right.index;
+  }).map(({ result, score }) => ({
+    result,
+    score
+  }));
+};
 var hasAnyToken = (tokens, values) => values.some((value) => tokens.includes(value));
 var resolveFileKindBoost = (queryTokens, metadata) => {
   const fileKind = typeof metadata?.fileKind === "string" ? metadata.fileKind : "";
@@ -6860,27 +6954,15 @@ var createInMemoryRAGStore = (options = {}) => {
     }));
   };
   const queryLexical = async (input) => {
-    const results = chunks.map((chunk) => ({
-      chunk,
-      score: scoreRAGLexicalMatch(input.query, {
-        chunkText: chunk.text,
-        metadata: chunk.metadata,
-        source: chunk.source,
-        title: chunk.title
-      })
-    })).filter(({ chunk }) => matchesFilter(chunk, input.filter)).filter(({ score }) => score > 0).sort((left, right) => {
-      if (right.score !== left.score) {
-        return right.score - left.score;
-      }
-      return left.chunk.chunkId.localeCompare(right.chunk.chunkId);
-    });
-    return results.slice(0, input.topK).map((entry) => ({
-      chunkId: entry.chunk.chunkId,
-      chunkText: entry.chunk.text,
-      metadata: entry.chunk.metadata,
-      score: entry.score,
-      source: entry.chunk.source,
-      title: entry.chunk.title
+    const filtered = chunks.filter((chunk) => matchesFilter(chunk, input.filter));
+    const ranked = rankRAGLexicalMatches(input.query, filtered);
+    return ranked.slice(0, input.topK).map(({ result, score }) => ({
+      chunkId: result.chunkId,
+      chunkText: result.text,
+      metadata: result.metadata,
+      score,
+      source: result.source,
+      title: result.title
     }));
   };
   const upsert = async (input) => {
@@ -7511,27 +7593,15 @@ var createSQLiteRAGStore = (options = {}) => {
   };
   const queryLexical = async (input) => {
     const rawRows = toStoredRows(jsonStatements.query.all());
-    const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter(chunk, input.filter)).map((chunk) => ({
-      chunk,
-      score: scoreRAGLexicalMatch(input.query, {
-        chunkText: chunk.text,
-        metadata: chunk.metadata,
-        source: chunk.source,
-        title: chunk.title
-      })
-    })).filter(({ score }) => score > 0).sort((left, right) => {
-      if (right.score !== left.score) {
-        return right.score - left.score;
-      }
-      return left.chunk.chunkId.localeCompare(right.chunk.chunkId);
-    });
-    return chunks.slice(0, input.topK).map(({ chunk, score }) => ({
-      chunkId: chunk.chunkId,
-      chunkText: chunk.text,
-      metadata: chunk.metadata,
+    const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter(chunk, input.filter));
+    const ranked = rankRAGLexicalMatches(input.query, chunks);
+    return ranked.slice(0, input.topK).map(({ result, score }) => ({
+      chunkId: result.chunkId,
+      chunkText: result.text,
+      metadata: result.metadata,
       score,
-      source: chunk.source,
-      title: chunk.title
+      source: result.source,
+      title: result.title
     }));
   };
   const upsert = async (input) => {
@@ -8703,5 +8773,5 @@ export {
   aiChat
 };
 
-//# debugId=36F6407CE8163A4F64756E2164756E21
+//# debugId=F37A373F20F3691864756E2164756E21
 //# sourceMappingURL=index.js.map