@absolutejs/absolute 0.19.0-beta.490 → 0.19.0-beta.492

package/dist/ai/index.js

@@ -2115,7 +2115,40 @@ import { Elysia as Elysia2 } from "elysia";
 
 // src/ai/rag/lexical.ts
 var DEFAULT_FUSION_CONSTANT = 60;
-var tokenize = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => token.length > 1);
+var STOP_WORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "does",
+  "every",
+  "explain",
+  "explains",
+  "for",
+  "how",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "say",
+  "says",
+  "should",
+  "stay",
+  "the",
+  "this",
+  "to",
+  "track",
+  "what",
+  "which",
+  "why"
+]);
+var tokenize = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => !STOP_WORDS.has(token)).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("ing") && token.length > 5 ? token.slice(0, -3) : token.endsWith("ed") && token.length > 4 ? token.slice(0, -2) : token.endsWith("es") && token.length > 4 ? token.slice(0, -2) : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).filter((token) => token.length > 1);
 var collectMetadataStrings = (value) => {
   if (typeof value === "string" || typeof value === "number") {
     return [String(value)];
@@ -2131,6 +2164,7 @@ var collectMetadataStrings = (value) => {
 var buildRAGLexicalHaystack = (result) => [
   result.title,
   result.source,
+  typeof result.source === "string" ? result.source.replace(/[#/_.-]+/g, " ").replace(/\bmd\b/g, "markdown").replace(/\bpptx\b/g, "presentation").replace(/\bxlsx\b/g, "spreadsheet workbook sheet").replace(/\bmp3\b/g, "audio transcript media").replace(/\bmp4\b/g, "video transcript media").replace(/\bzip\b/g, "archive bundle") : undefined,
   result.chunkText,
   ...collectMetadataStrings(result.metadata)
 ].filter((value) => Boolean(value)).join(" ");
@@ -2149,7 +2183,66 @@ var scoreRAGLexicalMatch = (query, result) => {
   const exactPhraseBoost = haystack.includes(query.toLowerCase()) ? 1 : 0;
   const sourceBoost = typeof result.source === "string" && queryTokens.some((token) => result.source?.toLowerCase().includes(token)) ? 0.5 : 0;
   const coverageBoost = overlap / queryTokens.length;
-  return coverageBoost + exactPhraseBoost + sourceBoost;
+  const fileKindBoost = resolveFileKindBoost(queryTokens, result.metadata);
+  const transcriptBoost = resolveTranscriptBoost(queryTokens, result.metadata);
+  const archiveBoost = resolveArchiveBoost(queryTokens, result);
+  return coverageBoost + exactPhraseBoost + sourceBoost + fileKindBoost + transcriptBoost + archiveBoost;
+};
+var hasAnyToken = (tokens, values) => values.some((value) => tokens.includes(value));
+var resolveFileKindBoost = (queryTokens, metadata) => {
+  const fileKind = typeof metadata?.fileKind === "string" ? metadata.fileKind : "";
+  if (fileKind === "office" && hasAnyToken(queryTokens, [
+    "sheet",
+    "worksheet",
+    "workbook",
+    "spreadsheet"
+  ])) {
+    return 0.75;
+  }
+  if (fileKind === "archive" && hasAnyToken(queryTokens, [
+    "archive",
+    "bundle",
+    "entry",
+    "runbook",
+    "recovery"
+  ])) {
+    return 0.85;
+  }
+  if (fileKind === "media" && hasAnyToken(queryTokens, [
+    "frontend",
+    "framework",
+    "transcript",
+    "audio",
+    "video"
+  ])) {
+    return 0.75;
+  }
+  return 0;
+};
+var resolveTranscriptBoost = (queryTokens, metadata) => {
+  const segments = Array.isArray(metadata?.mediaSegments) ? metadata.mediaSegments : [];
+  if (segments.length === 0) {
+    return 0;
+  }
+  const segmentText = segments.map((segment) => segment && typeof segment === "object" && ("text" in segment) ? String(segment.text ?? "") : "").join(" ").toLowerCase();
+  if (segmentText.length === 0) {
+    return 0;
+  }
+  const overlap = queryTokens.filter((token) => segmentText.includes(token)).length;
+  return overlap / Math.max(1, queryTokens.length);
+};
+var resolveArchiveBoost = (queryTokens, result) => {
+  const archivePath = typeof result.metadata?.archivePath === "string" ? result.metadata.archivePath.toLowerCase() : typeof result.source === "string" && result.source.includes("#") ? result.source.split("#")[1]?.toLowerCase() ?? "" : "";
+  if (!archivePath) {
+    return 0;
+  }
+  if (queryTokens.includes("recovery") && archivePath.includes("recovery")) {
+    return 1;
+  }
+  if (queryTokens.includes("runbook") && archivePath.includes("runbook")) {
+    return 0.8;
+  }
+  return 0;
 };
 var rankResults = (results) => results.map((result, index) => ({
   rank: index + 1,
@@ -2231,17 +2324,113 @@ var resolveRAGHybridSearchOptions = (retrieval) => {
 };
 
 // src/ai/rag/queryTransforms.ts
-var tokenize2 = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).filter((token) => token.length > 1);
+var tokenize2 = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => !STOP_WORDS2.has(token)).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("ing") && token.length > 5 ? token.slice(0, -3) : token.endsWith("ed") && token.length > 4 ? token.slice(0, -2) : token.endsWith("es") && token.length > 4 ? token.slice(0, -2) : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).map((token) => token.endsWith("ck") && token.length > 4 ? token.slice(0, -1) : token).map((token) => token.endsWith("ay") && token.length > 4 ? `${token.slice(0, -2)}i` : token).filter((token) => token.length > 1);
+var STOP_WORDS2 = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "does",
+  "every",
+  "explain",
+  "explains",
+  "for",
+  "how",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "say",
+  "says",
+  "should",
+  "stay",
+  "the",
+  "this",
+  "to",
+  "track",
+  "what",
+  "which",
+  "why"
+]);
 var DOMAIN_EXPANSIONS = {
-  archive: ["archive", "zip", "bundle", "entry", "runbook", "recovery"],
-  audio: ["audio", "media", "recording", "speaker", "transcript"],
-  deck: ["slide", "deck", "presentation", "pptx"],
+  archive: [
+    "archive",
+    "zip",
+    "bundle",
+    "entry",
+    "runbook",
+    "recovery",
+    "procedure",
+    "runbooks/recovery"
+  ],
+  audio: [
+    "audio",
+    "media",
+    "recording",
+    "speaker",
+    "transcript",
+    "standup",
+    "mp3"
+  ],
+  deck: ["slide", "deck", "presentation", "pptx", "roadmap"],
   email: ["email", "mail", "thread", "message", "attachment"],
-  image: ["image", "ocr", "scan", "screenshot"],
+  frontend: [
+    "frontend",
+    "framework",
+    "react",
+    "vue",
+    "svelte",
+    "angular",
+    "html",
+    "htmx"
+  ],
+  image: ["image", "ocr", "scan", "screenshot", "receipt"],
   pdf: ["pdf", "document", "page", "ocr", "scan"],
-  spreadsheet: ["sheet", "worksheet", "workbook", "spreadsheet", "xlsx"],
+  spreadsheet: [
+    "sheet",
+    "worksheet",
+    "workbook",
+    "spreadsheet",
+    "xlsx",
+    "regional",
+    "growth"
+  ],
   video: ["video", "media", "recording", "transcript", "timestamp"]
 };
+var TERM_EXPANSIONS = {
+  frontend: [
+    "frontend",
+    "framework",
+    "react",
+    "vue",
+    "svelte",
+    "angular",
+    "html",
+    "htmx"
+  ],
+  framework: [
+    "frontend",
+    "framework",
+    "react",
+    "vue",
+    "svelte",
+    "angular",
+    "html",
+    "htmx"
+  ],
+  procedure: ["recovery", "runbook"],
+  procedur: ["recovery", "runbook"],
+  receipt: ["invoice", "ocr", "pdf"],
+  sheet: ["worksheet", "workbook", "xlsx"],
+  transcript: ["audio", "video", "media"],
+  workbook: ["sheet", "spreadsheet", "xlsx"]
+};
 var detectDomains = (tokens) => {
   const tokenSet = new Set(tokens);
   const domains = new Set;
@@ -2264,6 +2453,9 @@ var detectDomains = (tokens) => {
     if (token === "slide" || token === "deck" || token === "presentation") {
       domains.add("deck");
     }
+    if (token === "frontend" || token === "framework" || token === "react" || token === "vue" || token === "svelte" || token === "angular" || token === "html" || token === "htmx") {
+      domains.add("frontend");
+    }
     if (token === "email" || token === "mail" || token === "thread") {
       domains.add("email");
     }
@@ -2284,13 +2476,16 @@ var createHeuristicRAGQueryTransform = (options = {}) => createRAGQueryTransform
       return { query };
     }
     const expandedTerms = domains.flatMap((domain) => DOMAIN_EXPANSIONS[domain] ?? []);
+    const tokenExpansions = tokens.flatMap((token) => TERM_EXPANSIONS[token] ?? []);
     const rewrittenQuery = uniqueQueryStrings([
       ...tokens,
-      ...expandedTerms
+      ...expandedTerms,
+      ...tokenExpansions
     ]).join(" ");
     const variants = domains.map((domain) => uniqueQueryStrings([
       ...tokens,
-      ...DOMAIN_EXPANSIONS[domain] ?? []
+      ...DOMAIN_EXPANSIONS[domain] ?? [],
+      ...tokenExpansions
     ]).join(" "));
     return {
       query: rewrittenQuery,
@@ -2334,7 +2529,40 @@ var applyRAGQueryTransform = async ({
 };
 
 // src/ai/rag/reranking.ts
-var tokenize3 = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).filter((token) => token.length > 1);
+var tokenize3 = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => !STOP_WORDS3.has(token)).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("ing") && token.length > 5 ? token.slice(0, -3) : token.endsWith("ed") && token.length > 4 ? token.slice(0, -2) : token.endsWith("es") && token.length > 4 ? token.slice(0, -2) : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).map((token) => token.endsWith("ck") && token.length > 4 ? token.slice(0, -1) : token).map((token) => token.endsWith("ay") && token.length > 4 ? `${token.slice(0, -2)}i` : token).filter((token) => token.length > 1);
+var STOP_WORDS3 = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "does",
+  "every",
+  "explain",
+  "explains",
+  "for",
+  "how",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "say",
+  "says",
+  "should",
+  "stay",
+  "the",
+  "this",
+  "to",
+  "track",
+  "what",
+  "which",
+  "why"
+]);
 var collectMetadataStrings2 = (value) => {
   if (typeof value === "string" || typeof value === "number") {
     return [String(value)];
@@ -3649,8 +3877,10 @@ var createRAGCollection = (options) => {
     const model = input.model ?? options.defaultModel;
     const topK = input.topK ?? defaultTopK;
     const hasReranker = Boolean(input.rerank ?? options.rerank);
-    const candidateTopK = Math.max(topK, Math.floor(input.candidateTopK ?? (hasReranker ? topK * defaultCandidateMultiplier : topK)));
     const retrieval = resolveRAGHybridSearchOptions(input.retrieval);
+    const hasQueryTransform = Boolean(input.queryTransform ?? options.queryTransform);
+    const shouldExpandCandidates = hasReranker || hasQueryTransform || retrieval.mode !== "vector";
+    const candidateTopK = Math.max(topK, Math.floor(input.candidateTopK ?? (shouldExpandCandidates ? topK * defaultCandidateMultiplier : topK)));
     const transformed = await applyRAGQueryTransform({
       input: {
         candidateTopK,
@@ -8370,5 +8600,5 @@ export {
   aiChat
 };
 
-//# debugId=4FC06DD026EF238A64756E2164756E21
+//# debugId=8B383E0793D06CEF64756E2164756E21
 //# sourceMappingURL=index.js.map