@absolutejs/absolute 0.19.0-beta.553 → 0.19.0-beta.555

package/dist/ai/index.js

@@ -2551,7 +2551,9 @@ var resolveRAGHybridSearchOptions = (retrieval) => {
       fusionConstant: DEFAULT_FUSION_CONSTANT,
       lexicalTopK: undefined,
       lexicalWeight: 2,
+      mmrLambda: 0.7,
       sourceBalanceStrategy: "cap",
+      noveltyStrategy: "none",
       mode: "vector",
       vectorWeight: 1
     };
@@ -2562,7 +2564,9 @@ var resolveRAGHybridSearchOptions = (retrieval) => {
       fusionConstant: DEFAULT_FUSION_CONSTANT,
       lexicalTopK: undefined,
       lexicalWeight: 2,
+      mmrLambda: 0.7,
       sourceBalanceStrategy: "cap",
+      noveltyStrategy: "none",
       mode: retrieval,
       vectorWeight: 1
     };
@@ -2573,7 +2577,9 @@ var resolveRAGHybridSearchOptions = (retrieval) => {
     lexicalTopK: retrieval.lexicalTopK,
     maxResultsPerSource: typeof retrieval.maxResultsPerSource === "number" ? Math.max(1, Math.floor(retrieval.maxResultsPerSource)) : undefined,
     lexicalWeight: Math.max(0, retrieval.lexicalWeight ?? 2),
+    mmrLambda: Math.min(1, Math.max(0, retrieval.mmrLambda ?? 0.7)),
     mode: retrieval.mode ?? "vector",
+    noveltyStrategy: retrieval.noveltyStrategy === "mmr" ? "mmr" : "none",
     sourceBalanceStrategy: retrieval.sourceBalanceStrategy === "round_robin" ? "round_robin" : "cap",
     vectorWeight: Math.max(0, retrieval.vectorWeight ?? 1)
   };
@@ -4376,6 +4382,47 @@ var applyRAGSourceDiversity = (results, maxResultsPerSource, strategy = "cap") =
   }
   return limited;
 };
+var dotProduct = (left, right) => {
+  const length = Math.min(left.length, right.length);
+  let total = 0;
+  for (let index = 0;index < length; index += 1) {
+    total += (left[index] ?? 0) * (right[index] ?? 0);
+  }
+  return total;
+};
+var applyRAGMMRDiversity = (results, queryVector, lambda) => {
+  const withEmbeddings = results.filter((result) => Array.isArray(result.embedding) && result.embedding.length > 0);
+  if (withEmbeddings.length < 2 || queryVector.length === 0) {
+    return results;
+  }
+  const selected = [];
+  const remaining = [...withEmbeddings];
+  while (remaining.length > 0) {
+    let bestIndex = 0;
+    let bestScore = Number.NEGATIVE_INFINITY;
+    for (let index = 0;index < remaining.length; index += 1) {
+      const candidate = remaining[index];
+      if (!candidate) {
+        continue;
+      }
+      const relevance = dotProduct(queryVector, candidate.embedding);
+      const redundancy = selected.length > 0 ? Math.max(...selected.map((entry) => dotProduct(entry.embedding, candidate.embedding))) : 0;
+      const mmrScore = lambda * relevance - (1 - lambda) * redundancy;
+      if (mmrScore > bestScore) {
+        bestScore = mmrScore;
+        bestIndex = index;
+      }
+    }
+    const next = remaining.splice(bestIndex, 1)[0];
+    if (!next) {
+      break;
+    }
+    selected.push(next);
+  }
+  const selectedIds = new Set(selected.map((entry) => entry.chunkId));
+  const tail = results.filter((entry) => !selectedIds.has(entry.chunkId));
+  return [...selected, ...tail];
+};
 var weightQueryResults = (results, queryIndex) => {
   if (queryIndex === 0) {
     return results;
@@ -4432,7 +4479,9 @@ var createRAGCollection = (options) => {
           hasQueryTransform,
           hasReranker,
           maxResultsPerSource: retrieval.maxResultsPerSource ?? null,
+          mmrLambda: retrieval.mmrLambda,
           mode: retrieval.mode,
+          noveltyStrategy: retrieval.noveltyStrategy,
           runLexical,
           runVector,
           sourceBalanceStrategy: retrieval.sourceBalanceStrategy,
@@ -4550,7 +4599,19 @@ var createRAGCollection = (options) => {
       },
       stage: "rerank"
     });
-    const diversified = applyRAGSourceDiversity(reranked, retrieval.maxResultsPerSource, retrieval.sourceBalanceStrategy);
+    const noveltyAdjusted = retrieval.noveltyStrategy === "mmr" ? applyRAGMMRDiversity(reranked, queryVector, retrieval.mmrLambda) : reranked;
+    if (retrieval.noveltyStrategy === "mmr") {
+      steps.push({
+        count: noveltyAdjusted.length,
+        label: "Applied MMR novelty reordering",
+        metadata: {
+          applied: queryVector.length > 0 && noveltyAdjusted.some((entry) => Array.isArray(entry.embedding)),
+          mmrLambda: retrieval.mmrLambda
+        },
+        stage: "novelty"
+      });
+    }
+    const diversified = applyRAGSourceDiversity(noveltyAdjusted, retrieval.maxResultsPerSource, retrieval.sourceBalanceStrategy);
     if (typeof retrieval.maxResultsPerSource === "number") {
       steps.push({
         count: diversified.length,
@@ -4578,7 +4639,9 @@ var createRAGCollection = (options) => {
           candidateTopK,
           lexicalTopK,
           maxResultsPerSource: retrieval.maxResultsPerSource,
+          mmrLambda: retrieval.mmrLambda,
           mode: retrieval.mode,
+          noveltyStrategy: retrieval.noveltyStrategy,
           query: input.query,
           resultCounts: {
             final: limited.length,
@@ -4621,7 +4684,9 @@ var createRAGCollection = (options) => {
         candidateTopK,
         lexicalTopK,
         maxResultsPerSource: retrieval.maxResultsPerSource,
+        mmrLambda: retrieval.mmrLambda,
         mode: retrieval.mode,
+        noveltyStrategy: retrieval.noveltyStrategy,
         query: input.query,
         resultCounts: {
           final: filtered.length,
@@ -5809,8 +5874,11 @@ var summarizeRetrievalTraces = (traces) => {
   }
   const totalCases = traces.length;
   const modeSet = new Set;
+  const sourceBalanceStrategySet = new Set;
   let vectorCases = 0;
   let lexicalCases = 0;
+  let balancedCases = 0;
+  let roundRobinCases = 0;
   let transformedCases = 0;
   let variantCases = 0;
   let finalCountSum = 0;
@@ -5820,12 +5888,19 @@ var summarizeRetrievalTraces = (traces) => {
   let lexicalTopKSum = 0;
   for (const trace of traces) {
     modeSet.add(trace.mode);
+    sourceBalanceStrategySet.add(trace.sourceBalanceStrategy ?? "cap");
     if (trace.runVector) {
       vectorCases += 1;
     }
     if (trace.runLexical) {
       lexicalCases += 1;
     }
+    if (typeof trace.maxResultsPerSource === "number") {
+      balancedCases += 1;
+      if (trace.sourceBalanceStrategy === "round_robin") {
+        roundRobinCases += 1;
+      }
+    }
     if (trace.transformedQuery !== trace.query) {
       transformedCases += 1;
     }
@@ -5843,9 +5918,12 @@ var summarizeRetrievalTraces = (traces) => {
     averageFinalCount: roundTraceAverage(finalCountSum, totalCases),
     averageLexicalCount: roundTraceAverage(lexicalCountSum, totalCases),
     averageLexicalTopK: roundTraceAverage(lexicalTopKSum, totalCases),
+    balancedCases,
     averageVectorCount: roundTraceAverage(vectorCountSum, totalCases),
     lexicalCases,
     modes: Array.from(modeSet),
+    roundRobinCases,
+    sourceBalanceStrategies: Array.from(sourceBalanceStrategySet),
     stageCounts: buildTraceStageCounts(traces),
     totalCases,
     transformedCases,
@@ -6181,7 +6259,7 @@ var buildEvaluationCaseTraceSnapshot = ({
 }) => {
   const stageCounts = currentTrace ? buildTraceStageCounts([currentTrace]) : {};
   const previousStageCounts = previousTrace?.stageCounts ?? {};
-  const traceChange = !previousTrace ? currentTrace ? "new" : "unchanged" : previousTrace.traceMode !== currentTrace?.mode || previousTrace.transformedQuery !== (currentTrace?.transformedQuery || undefined) || previousTrace.variantQueries.join("|") !== (currentTrace?.variantQueries ?? []).join("|") || previousTrace.finalCount !== (currentTrace?.resultCounts.final ?? 0) || previousTrace.vectorCount !== (currentTrace?.resultCounts.vector ?? 0) || previousTrace.lexicalCount !== (currentTrace?.resultCounts.lexical ?? 0) || previousTrace.candidateTopK !== (currentTrace?.candidateTopK ?? 0) || previousTrace.lexicalTopK !== (currentTrace?.lexicalTopK ?? 0) || !areStageCountsEqual(previousStageCounts, stageCounts) ? "changed" : "unchanged";
+  const traceChange = !previousTrace ? currentTrace ? "new" : "unchanged" : previousTrace.traceMode !== currentTrace?.mode || previousTrace.sourceBalanceStrategy !== currentTrace?.sourceBalanceStrategy || previousTrace.transformedQuery !== (currentTrace?.transformedQuery || undefined) || previousTrace.variantQueries.join("|") !== (currentTrace?.variantQueries ?? []).join("|") || previousTrace.finalCount !== (currentTrace?.resultCounts.final ?? 0) || previousTrace.vectorCount !== (currentTrace?.resultCounts.vector ?? 0) || previousTrace.lexicalCount !== (currentTrace?.resultCounts.lexical ?? 0) || previousTrace.candidateTopK !== (currentTrace?.candidateTopK ?? 0) || previousTrace.lexicalTopK !== (currentTrace?.lexicalTopK ?? 0) || !areStageCountsEqual(previousStageCounts, stageCounts) ? "changed" : "unchanged";
   return {
     candidateTopK: currentTrace?.candidateTopK ?? 0,
     caseId: caseResult.caseId,
@@ -6193,6 +6271,7 @@ var buildEvaluationCaseTraceSnapshot = ({
     previousFinalCount: previousTrace?.finalCount,
     previousLexicalCount: previousTrace?.lexicalCount,
     previousLexicalTopK: previousTrace?.lexicalTopK,
+    previousSourceBalanceStrategy: previousTrace?.sourceBalanceStrategy,
     previousStageCounts,
     previousTraceMode: previousTrace?.traceMode,
     previousTransformedQuery: previousTrace?.transformedQuery,
@@ -6201,6 +6280,7 @@ var buildEvaluationCaseTraceSnapshot = ({
     query: caseResult.query,
     stageCounts,
     status: caseResult.status,
+    sourceBalanceStrategy: currentTrace?.sourceBalanceStrategy,
     traceChange,
     traceMode: currentTrace?.mode,
     transformedQuery: currentTrace?.transformedQuery || undefined,
@@ -6245,6 +6325,7 @@ var buildEvaluationCaseTraceSnapshots = ({
         },
         runLexical: currentSnapshot.lexicalCount > 0,
         runVector: currentSnapshot.vectorCount > 0,
+        sourceBalanceStrategy: currentSnapshot.sourceBalanceStrategy ?? "cap",
         steps: [],
         topK: caseResult.topK,
         transformedQuery: currentSnapshot.transformedQuery ?? caseResult.query,
@@ -6275,6 +6356,7 @@ var formatRetrievalComparisonHeadline = (entry) => [
   `latency ${entry.response.summary.averageLatencyMs.toFixed(1)}ms`
 ].join(" \xB7 ");
 var formatTraceModes = (modes) => modes.length > 0 ? modes.join(" / ") : "n/a";
+var formatSourceBalanceStrategies = (strategies) => strategies.length > 0 ? strategies.join(" / ") : "n/a";
 var formatTraceStageSummary = (stageCounts) => {
   const topStages = Object.entries(stageCounts).sort((left, right) => right[1] - left[1]).slice(0, 3);
   return topStages.length > 0 ? topStages.map(([stage, count]) => `${stage} ${count}`).join(" \xB7 ") : "n/a";
@@ -6310,9 +6392,21 @@ var buildRAGComparisonTraceSummaryRows = (entry) => {
   }
   return [
     { label: "Modes", value: formatTraceModes(trace.modes) },
+    {
+      label: "Balance",
+      value: formatSourceBalanceStrategies(trace.sourceBalanceStrategies)
+    },
     { label: "Avg final", value: trace.averageFinalCount.toFixed(1) },
     { label: "Avg vector", value: trace.averageVectorCount.toFixed(1) },
     { label: "Avg lexical", value: trace.averageLexicalCount.toFixed(1) },
+    {
+      label: "Balanced",
+      value: formatTraceRatio(trace.balancedCases, trace.totalCases)
+    },
+    {
+      label: "Round robin",
+      value: formatTraceRatio(trace.roundRobinCases, trace.totalCases)
+    },
     {
       label: "Transforms",
       value: formatTraceRatio(trace.transformedCases, trace.totalCases)
@@ -6360,6 +6454,12 @@ var buildRAGComparisonTraceDiffRows = (entry, leader) => {
       value: `${formatTraceModes(trace.modes)} vs ${formatTraceModes(leaderTrace.modes)}`
     });
   }
+  if (formatSourceBalanceStrategies(trace.sourceBalanceStrategies) !== formatSourceBalanceStrategies(leaderTrace.sourceBalanceStrategies)) {
+    rows.push({
+      label: "Balance vs leader",
+      value: `${formatSourceBalanceStrategies(trace.sourceBalanceStrategies)} vs ${formatSourceBalanceStrategies(leaderTrace.sourceBalanceStrategies)}`
+    });
+  }
   rows.push({
     label: "Final delta",
     value: formatSignedDelta(trace.averageFinalCount - leaderTrace.averageFinalCount, 1)
@@ -6372,6 +6472,12 @@ var buildRAGComparisonTraceDiffRows = (entry, leader) => {
   }, {
     label: "Transform delta",
     value: formatTraceCountDelta(trace.transformedCases - leaderTrace.transformedCases)
+  }, {
+    label: "Balanced delta",
+    value: formatTraceCountDelta(trace.balancedCases - leaderTrace.balancedCases)
+  }, {
+    label: "Round robin delta",
+    value: formatTraceCountDelta(trace.roundRobinCases - leaderTrace.roundRobinCases)
   });
   if (stageDelta) {
     rows.push({ label: "Stage delta", value: stageDelta });
@@ -6535,7 +6641,7 @@ var buildRAGEvaluationHistoryRows = (history) => {
   if (history.latestRun.traceSummary) {
     rows.push({
       label: "Latest trace",
-      value: `${formatTraceModes(history.latestRun.traceSummary.modes)} \xB7 final ${history.latestRun.traceSummary.averageFinalCount.toFixed(1)} \xB7 vector ${history.latestRun.traceSummary.averageVectorCount.toFixed(1)} \xB7 lexical ${history.latestRun.traceSummary.averageLexicalCount.toFixed(1)}`
+      value: `${formatTraceModes(history.latestRun.traceSummary.modes)} \xB7 balance ${formatSourceBalanceStrategies(history.latestRun.traceSummary.sourceBalanceStrategies)} \xB7 final ${history.latestRun.traceSummary.averageFinalCount.toFixed(1)} \xB7 vector ${history.latestRun.traceSummary.averageVectorCount.toFixed(1)} \xB7 lexical ${history.latestRun.traceSummary.averageLexicalCount.toFixed(1)}`
     });
   }
   if (history.previousRun) {
@@ -6571,6 +6677,9 @@ var buildRAGEvaluationHistoryRows = (history) => {
     rows.push({
       label: "Trace mode shift",
       value: history.diff.traceSummaryDelta.modesChanged ? "changed" : "stable"
+    }, {
+      label: "Trace balance shift",
+      value: history.diff.traceSummaryDelta.sourceBalanceStrategiesChanged ? "changed" : "stable"
     }, {
       label: "Trace final delta",
       value: formatSignedDelta(history.diff.traceSummaryDelta.averageFinalCount, 1)
@@ -6580,6 +6689,12 @@ var buildRAGEvaluationHistoryRows = (history) => {
     }, {
       label: "Trace lexical delta",
       value: formatSignedDelta(history.diff.traceSummaryDelta.averageLexicalCount, 1)
+    }, {
+      label: "Trace balanced delta",
+      value: formatTraceCountDelta(history.diff.traceSummaryDelta.balancedCases)
+    }, {
+      label: "Trace round robin delta",
+      value: formatTraceCountDelta(history.diff.traceSummaryDelta.roundRobinCases)
     }, {
       label: "Trace transform delta",
       value: formatTraceCountDelta(history.diff.traceSummaryDelta.transformedCases)
@@ -6609,6 +6724,8 @@ var buildRAGEvaluationCaseTracePresentations = (history) => {
     const label = entry.label ?? entry.caseId;
     const currentMode = entry.traceMode ?? "no-trace";
     const previousMode = entry.previousTraceMode ?? "n/a";
+    const currentBalance = entry.sourceBalanceStrategy ?? "cap";
+    const previousBalance = entry.previousSourceBalanceStrategy ?? "n/a";
     const currentVariants = entry.variantQueries.length > 0 ? entry.variantQueries.join(", ") : "none";
     const previousVariants = entry.previousVariantQueries.length > 0 ? entry.previousVariantQueries.join(", ") : "none";
     const currentStages = Object.keys(entry.stageCounts).length > 0 ? Object.entries(entry.stageCounts).map(([stage, count]) => `${stage} ${count}`).join(", ") : "none";
@@ -6616,12 +6733,16 @@ var buildRAGEvaluationCaseTracePresentations = (history) => {
     return {
       caseId: entry.caseId,
       label,
-      summary: `${entry.traceChange} \xB7 ${previousMode}\u2192${currentMode} \xB7 final ${entry.previousFinalCount ?? 0}\u2192${entry.finalCount}`,
+      summary: `${entry.traceChange} \xB7 ${previousMode}\u2192${currentMode} \xB7 ${previousBalance}\u2192${currentBalance} \xB7 final ${entry.previousFinalCount ?? 0}\u2192${entry.finalCount}`,
       traceChange: entry.traceChange,
       rows: [
         { label: "Query", value: entry.query },
         { label: "Trace change", value: entry.traceChange },
         { label: "Mode", value: `${previousMode}\u2192${currentMode}` },
+        {
+          label: "Balance",
+          value: `${previousBalance}\u2192${currentBalance}`
+        },
         {
           label: "Transformed query",
           value: `${entry.previousTransformedQuery?.trim() || "n/a"}\u2192${entry.transformedQuery?.trim() || "n/a"}`
@@ -6693,12 +6814,15 @@ var buildRAGEvaluationRunDiff = ({
       averageLexicalCount: (current.traceSummary?.averageLexicalCount ?? 0) - (previous?.traceSummary?.averageLexicalCount ?? 0),
       averageLexicalTopK: (current.traceSummary?.averageLexicalTopK ?? 0) - (previous?.traceSummary?.averageLexicalTopK ?? 0),
       averageVectorCount: (current.traceSummary?.averageVectorCount ?? 0) - (previous?.traceSummary?.averageVectorCount ?? 0),
+      balancedCases: (current.traceSummary?.balancedCases ?? 0) - (previous?.traceSummary?.balancedCases ?? 0),
       lexicalCases: (current.traceSummary?.lexicalCases ?? 0) - (previous?.traceSummary?.lexicalCases ?? 0),
       modesChanged: (current.traceSummary?.modes ?? []).join("|") !== (previous?.traceSummary?.modes ?? []).join("|"),
+      roundRobinCases: (current.traceSummary?.roundRobinCases ?? 0) - (previous?.traceSummary?.roundRobinCases ?? 0),
       stageCounts: diffTraceStageCounts({
         current: current.traceSummary?.stageCounts ?? {},
         previous: previous?.traceSummary?.stageCounts ?? {}
       }),
+      sourceBalanceStrategiesChanged: (current.traceSummary?.sourceBalanceStrategies ?? []).join("|") !== (previous?.traceSummary?.sourceBalanceStrategies ?? []).join("|"),
       transformedCases: (current.traceSummary?.transformedCases ?? 0) - (previous?.traceSummary?.transformedCases ?? 0),
       variantCases: (current.traceSummary?.variantCases ?? 0) - (previous?.traceSummary?.variantCases ?? 0),
       vectorCases: (current.traceSummary?.vectorCases ?? 0) - (previous?.traceSummary?.vectorCases ?? 0)
@@ -10645,6 +10769,7 @@ var createInMemoryRAGStore = (options = {}) => {
     return results.slice(0, input.topK).map((entry) => ({
       chunkId: entry.chunk.chunkId,
       chunkText: entry.chunk.text,
+      embedding: entry.chunk.vector,
       metadata: entry.chunk.metadata,
       score: entry.score,
       source: entry.chunk.source,
@@ -10797,7 +10922,7 @@ var IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
 var isParsedMetadata = (value) => Boolean(value) && typeof value === "object";
 var isObjectRecord2 = (value) => Boolean(value) && typeof value === "object";
 var isStoredRow = (value) => isObjectRecord2(value) && typeof value.chunk_id === "string" && typeof value.text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string";
-var isNativeStoredRow = (value) => isObjectRecord2(value) && typeof value.chunk_id === "string" && typeof value.chunk_text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.distance === "number";
+var isNativeStoredRow = (value) => isObjectRecord2(value) && typeof value.chunk_id === "string" && typeof value.chunk_text === "string" && (typeof value.title === "string" || value.title === null) && (typeof value.source === "string" || value.source === null) && (typeof value.metadata === "string" || value.metadata === null) && typeof value.embedding === "string" && typeof value.distance === "number";
 var toStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isStoredRow(row)) : [];
 var toNativeStoredRows = (value) => Array.isArray(value) ? value.filter((row) => isNativeStoredRow(row)) : [];
 var createSQLiteStatus = (dimensions, nativeDiagnostics, useNative) => ({
@@ -10961,6 +11086,7 @@ var createNativeVec0Statements = (db, tableName) => {
   const querySql = `
 		SELECT
 			chunk_id,
+			embedding,
 			chunk_text,
 			title,
 			source,
@@ -11238,6 +11364,7 @@ var createSQLiteRAGStore = (options = {}) => {
     return filtered.slice(0, input.topK).map(({ chunk, score }) => ({
       chunkId: chunk.chunkId,
       chunkText: chunk.text,
+      embedding: chunk.vector,
       metadata: chunk.metadata,
       score,
       source: chunk.source,
@@ -11259,7 +11386,7 @@ var createSQLiteRAGStore = (options = {}) => {
         source: row.source ?? undefined,
         text: row.chunk_text,
         title: row.title ?? undefined,
-        vector: []
+        vector: parseVector(row.embedding)
       };
       return {
         chunk,
@@ -11268,6 +11395,7 @@ var createSQLiteRAGStore = (options = {}) => {
     }).filter(({ chunk }) => matchesFilter(chunk, input.filter)).map((entry) => ({
       chunkId: entry.chunk.chunkId,
       chunkText: entry.chunk.text,
+      embedding: entry.chunk.vector,
       metadata: entry.chunk.metadata,
       score: entry.score,
       source: entry.chunk.source,
@@ -12585,5 +12713,5 @@ export {
   aiChat
 };
 
-//# debugId=BD4F3B903BF6366564756E2164756E21
+//# debugId=115CEF45D67205BC64756E2164756E21
 //# sourceMappingURL=index.js.map