@absolutejs/absolute 0.19.0-beta.492 → 0.19.0-beta.494

package/dist/ai/index.js

@@ -2149,6 +2149,8 @@ var STOP_WORDS = new Set([
   "why"
 ]);
 var tokenize = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => !STOP_WORDS.has(token)).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("ing") && token.length > 5 ? token.slice(0, -3) : token.endsWith("ed") && token.length > 4 ? token.slice(0, -2) : token.endsWith("es") && token.length > 4 ? token.slice(0, -2) : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).filter((token) => token.length > 1);
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
 var collectMetadataStrings = (value) => {
   if (typeof value === "string" || typeof value === "number") {
     return [String(value)];
@@ -2161,10 +2163,94 @@ var collectMetadataStrings = (value) => {
   }
   return [];
 };
+var normalizeSourceForLexical = (source) => source.replace(/[#/_.-]+/g, " ").replace(/\bmd\b/g, "markdown").replace(/\bpptx\b/g, "presentation").replace(/\bxlsx\b/g, "spreadsheet workbook sheet").replace(/\bmp3\b/g, "audio transcript media").replace(/\bmp4\b/g, "video transcript media").replace(/\bzip\b/g, "archive bundle");
+var toFieldText = (value) => collectMetadataStrings(value).filter(Boolean).join(" ");
+var scoreTokenCoverage = (queryTokens, text) => {
+  const normalizedText = (text ?? "").toLowerCase();
+  if (normalizedText.length === 0) {
+    return 0;
+  }
+  const tokens = tokenize(normalizedText);
+  if (tokens.length === 0) {
+    return 0;
+  }
+  const tokenSet = new Set(tokens);
+  const overlap = queryTokens.filter((token) => tokenSet.has(token)).length;
+  return overlap / Math.max(1, queryTokens.length);
+};
+var scorePhraseMatch = (query, text) => {
+  const normalizedQuery = tokenize(query).join(" ");
+  const normalizedText = tokenize(text ?? "").join(" ");
+  if (normalizedQuery.length === 0 || normalizedText.length === 0) {
+    return 0;
+  }
+  return normalizedText.includes(normalizedQuery) ? 1 : 0;
+};
+var scoreWeightedField = ({
+  coverageWeight,
+  phraseWeight,
+  query,
+  queryTokens,
+  text
+}) => scoreTokenCoverage(queryTokens, text ?? "") * coverageWeight + scorePhraseMatch(query, text ?? "") * phraseWeight;
+var extractWeightedLexicalFields = (result) => {
+  const metadata = result.metadata ?? {};
+  const source = result.source ?? "";
+  const archivePath = typeof metadata.archivePath === "string" ? metadata.archivePath : source.includes("#") ? source.split("#")[1] ?? "" : "";
+  const mediaSegments = Array.isArray(metadata.mediaSegments) ? metadata.mediaSegments.map((segment) => segment && typeof segment === "object" ? toFieldText(segment) : "").filter(Boolean).join(" ") : "";
+  const metadataFocus = [
+    metadata.sheetName,
+    metadata.sheetNames,
+    metadata.slideTitle,
+    metadata.slideTitles,
+    metadata.threadTopic,
+    metadata.speaker,
+    metadata.fileKind,
+    metadata.transcriptSource,
+    metadata.archiveType
+  ].flatMap((value) => collectMetadataStrings(value)).join(" ");
+  return {
+    archivePath,
+    chunkText: result.text,
+    mediaSegments,
+    metadataFocus,
+    metadataText: toFieldText(metadata),
+    source: source ? normalizeSourceForLexical(source) : "",
+    title: result.title ?? ""
+  };
+};
+var FIELD_WEIGHTS = {
+  archivePath: 4.2,
+  chunkText: 1,
+  mediaSegments: 3.8,
+  metadataFocus: 3.2,
+  metadataText: 1.4,
+  source: 3.4,
+  title: 2
+};
+var getWeightedFieldTokens = (result) => {
+  const fields = extractWeightedLexicalFields({
+    metadata: result.metadata,
+    source: result.source,
+    text: result.text,
+    title: result.title
+  });
+  return {
+    archivePath: tokenize(fields.archivePath ?? ""),
+    chunkText: tokenize(fields.chunkText ?? ""),
+    mediaSegments: tokenize(fields.mediaSegments ?? ""),
+    metadataFocus: tokenize(fields.metadataFocus ?? ""),
+    metadataText: tokenize(fields.metadataText ?? ""),
+    source: tokenize(fields.source ?? ""),
+    title: tokenize(fields.title ?? "")
+  };
+};
+var countWeightedTermFrequency = (fieldTokens, token) => Object.keys(FIELD_WEIGHTS).reduce((total, fieldName) => total + fieldTokens[fieldName].filter((value) => value === token).length * FIELD_WEIGHTS[fieldName], 0);
+var computeWeightedDocumentLength = (fieldTokens) => Object.keys(FIELD_WEIGHTS).reduce((total, fieldName) => total + fieldTokens[fieldName].length * FIELD_WEIGHTS[fieldName], 0);
 var buildRAGLexicalHaystack = (result) => [
   result.title,
   result.source,
-  typeof result.source === "string" ? result.source.replace(/[#/_.-]+/g, " ").replace(/\bmd\b/g, "markdown").replace(/\bpptx\b/g, "presentation").replace(/\bxlsx\b/g, "spreadsheet workbook sheet").replace(/\bmp3\b/g, "audio transcript media").replace(/\bmp4\b/g, "video transcript media").replace(/\bzip\b/g, "archive bundle") : undefined,
+  typeof result.source === "string" ? normalizeSourceForLexical(result.source) : undefined,
   result.chunkText,
   ...collectMetadataStrings(result.metadata)
 ].filter((value) => Boolean(value)).join(" ");
@@ -2173,20 +2259,131 @@ var scoreRAGLexicalMatch = (query, result) => {
   if (queryTokens.length === 0) {
     return 0;
   }
+  const fields = extractWeightedLexicalFields({
+    metadata: result.metadata,
+    source: result.source,
+    text: result.chunkText,
+    title: result.title
+  });
   const haystack = buildRAGLexicalHaystack(result).toLowerCase();
-  const haystackTokens = tokenize(haystack);
-  const haystackSet = new Set(haystackTokens);
-  const overlap = queryTokens.filter((token) => haystackSet.has(token)).length;
-  if (overlap === 0) {
+  const overallCoverage = scoreTokenCoverage(queryTokens, haystack);
+  if (overallCoverage === 0) {
     return 0;
   }
-  const exactPhraseBoost = haystack.includes(query.toLowerCase()) ? 1 : 0;
-  const sourceBoost = typeof result.source === "string" && queryTokens.some((token) => result.source?.toLowerCase().includes(token)) ? 0.5 : 0;
-  const coverageBoost = overlap / queryTokens.length;
+  const titleScore = scoreWeightedField({
+    coverageWeight: 1.8,
+    phraseWeight: 1.2,
+    query,
+    queryTokens,
+    text: fields.title
+  });
+  const sourceScore = scoreWeightedField({
+    coverageWeight: 2.6,
+    phraseWeight: 1.4,
+    query,
+    queryTokens,
+    text: fields.source
+  });
+  const metadataFocusScore = scoreWeightedField({
+    coverageWeight: 2.8,
+    phraseWeight: 1.6,
+    query,
+    queryTokens,
+    text: fields.metadataFocus
+  });
+  const archivePathScore = scoreWeightedField({
+    coverageWeight: 3.2,
+    phraseWeight: 2.2,
+    query,
+    queryTokens,
+    text: fields.archivePath
+  });
+  const mediaSegmentScore = scoreWeightedField({
+    coverageWeight: 3,
+    phraseWeight: 1.8,
+    query,
+    queryTokens,
+    text: fields.mediaSegments
+  });
+  const metadataScore = scoreWeightedField({
+    coverageWeight: 1.2,
+    phraseWeight: 0.8,
+    query,
+    queryTokens,
+    text: fields.metadataText
+  });
+  const chunkScore = scoreWeightedField({
+    coverageWeight: 0.9,
+    phraseWeight: 0.6,
+    query,
+    queryTokens,
+    text: fields.chunkText
+  });
+  const exactPhraseBoost = scorePhraseMatch(query, haystack);
+  const coverageBoost = overallCoverage;
   const fileKindBoost = resolveFileKindBoost(queryTokens, result.metadata);
   const transcriptBoost = resolveTranscriptBoost(queryTokens, result.metadata);
   const archiveBoost = resolveArchiveBoost(queryTokens, result);
-  return coverageBoost + exactPhraseBoost + sourceBoost + fileKindBoost + transcriptBoost + archiveBoost;
+  return titleScore + sourceScore + metadataFocusScore + archivePathScore + mediaSegmentScore + metadataScore + chunkScore + coverageBoost + exactPhraseBoost + fileKindBoost + transcriptBoost + archiveBoost;
+};
+var rankRAGLexicalMatches = (query, results) => {
+  const queryTokens = tokenize(query);
+  if (queryTokens.length === 0 || results.length === 0) {
+    return [];
+  }
+  const candidates = results.map((result) => {
+    const fieldTokens = getWeightedFieldTokens(result);
+    return {
+      fieldTokens,
+      length: computeWeightedDocumentLength(fieldTokens),
+      result
+    };
+  });
+  const averageDocumentLength = candidates.reduce((total, candidate) => total + candidate.length, 0) / Math.max(1, candidates.length);
+  const uniqueQueryTokens = [...new Set(queryTokens)];
+  const documentFrequency = new Map;
+  for (const token of uniqueQueryTokens) {
+    let seen = 0;
+    for (const candidate of candidates) {
+      const tf = countWeightedTermFrequency(candidate.fieldTokens, token);
+      if (tf > 0) {
+        seen += 1;
+      }
+    }
+    documentFrequency.set(token, seen);
+  }
+  return candidates.map((candidate, index) => {
+    let bm25Score = 0;
+    for (const token of uniqueQueryTokens) {
+      const termFrequency = countWeightedTermFrequency(candidate.fieldTokens, token);
+      if (termFrequency <= 0) {
+        continue;
+      }
+      const df = documentFrequency.get(token) ?? 0;
+      const idf = Math.log(1 + (candidates.length - df + 0.5) / (df + 0.5));
+      const denominator = termFrequency + BM25_K1 * (1 - BM25_B + BM25_B * (candidate.length / Math.max(1, averageDocumentLength)));
+      bm25Score += idf * (termFrequency * (BM25_K1 + 1) / Math.max(0.000000001, denominator));
+    }
+    const heuristicScore = scoreRAGLexicalMatch(query, {
+      chunkText: candidate.result.text,
+      metadata: candidate.result.metadata,
+      source: candidate.result.source,
+      title: candidate.result.title
+    });
+    return {
+      index,
+      result: candidate.result,
+      score: bm25Score + heuristicScore * 0.35
+    };
+  }).filter((entry) => entry.score > 0).sort((left, right) => {
+    if (right.score !== left.score) {
+      return right.score - left.score;
+    }
+    return left.index - right.index;
+  }).map(({ result, score }) => ({
+    result,
+    score
+  }));
 };
 var hasAnyToken = (tokens, values) => values.some((value) => tokens.includes(value));
 var resolveFileKindBoost = (queryTokens, metadata) => {
@@ -6757,27 +6954,15 @@ var createInMemoryRAGStore = (options = {}) => {
     }));
   };
   const queryLexical = async (input) => {
-    const results = chunks.map((chunk) => ({
-      chunk,
-      score: scoreRAGLexicalMatch(input.query, {
-        chunkText: chunk.text,
-        metadata: chunk.metadata,
-        source: chunk.source,
-        title: chunk.title
-      })
-    })).filter(({ chunk }) => matchesFilter(chunk, input.filter)).filter(({ score }) => score > 0).sort((left, right) => {
-      if (right.score !== left.score) {
-        return right.score - left.score;
-      }
-      return left.chunk.chunkId.localeCompare(right.chunk.chunkId);
-    });
-    return results.slice(0, input.topK).map((entry) => ({
-      chunkId: entry.chunk.chunkId,
-      chunkText: entry.chunk.text,
-      metadata: entry.chunk.metadata,
-      score: entry.score,
-      source: entry.chunk.source,
-      title: entry.chunk.title
+    const filtered = chunks.filter((chunk) => matchesFilter(chunk, input.filter));
+    const ranked = rankRAGLexicalMatches(input.query, filtered);
+    return ranked.slice(0, input.topK).map(({ result, score }) => ({
+      chunkId: result.chunkId,
+      chunkText: result.text,
+      metadata: result.metadata,
+      score,
+      source: result.source,
+      title: result.title
     }));
   };
   const upsert = async (input) => {
@@ -7408,27 +7593,15 @@ var createSQLiteRAGStore = (options = {}) => {
   };
   const queryLexical = async (input) => {
     const rawRows = toStoredRows(jsonStatements.query.all());
-    const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter(chunk, input.filter)).map((chunk) => ({
-      chunk,
-      score: scoreRAGLexicalMatch(input.query, {
-        chunkText: chunk.text,
-        metadata: chunk.metadata,
-        source: chunk.source,
-        title: chunk.title
-      })
-    })).filter(({ score }) => score > 0).sort((left, right) => {
-      if (right.score !== left.score) {
-        return right.score - left.score;
-      }
-      return left.chunk.chunkId.localeCompare(right.chunk.chunkId);
-    });
-    return chunks.slice(0, input.topK).map(({ chunk, score }) => ({
-      chunkId: chunk.chunkId,
-      chunkText: chunk.text,
-      metadata: chunk.metadata,
+    const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter(chunk, input.filter));
+    const ranked = rankRAGLexicalMatches(input.query, chunks);
+    return ranked.slice(0, input.topK).map(({ result, score }) => ({
+      chunkId: result.chunkId,
+      chunkText: result.text,
+      metadata: result.metadata,
       score,
-      source: chunk.source,
-      title: chunk.title
+      source: result.source,
+      title: result.title
     }));
   };
   const upsert = async (input) => {
@@ -8600,5 +8773,5 @@ export {
   aiChat
 };
 
-//# debugId=8B383E0793D06CEF64756E2164756E21
+//# debugId=F37A373F20F3691864756E2164756E21
 //# sourceMappingURL=index.js.map