@absolutejs/absolute 0.19.0-beta.531 → 0.19.0-beta.533

package/dist/src/vue/ai/useRAG.d.ts

@@ -307,6 +307,30 @@ export declare const useRAG: (path: string, options?: UseRAGOptions) => {
                 totalCases: number;
                 passingRate: number;
             };
+            traceSummary?: {
+                totalCases: number;
+                modes: import("../..").RAGHybridRetrievalMode[];
+                vectorCases: number;
+                lexicalCases: number;
+                transformedCases: number;
+                variantCases: number;
+                averageFinalCount: number;
+                averageVectorCount: number;
+                averageLexicalCount: number;
+                averageCandidateTopK: number;
+                averageLexicalTopK: number;
+                stageCounts: {
+                    input?: number | undefined;
+                    embed?: number | undefined;
+                    query_transform?: number | undefined;
+                    vector_search?: number | undefined;
+                    lexical_search?: number | undefined;
+                    fusion?: number | undefined;
+                    rerank?: number | undefined;
+                    score_filter?: number | undefined;
+                    finalize?: number | undefined;
+                };
+            } | undefined;
             metadata?: Record<string, unknown> | undefined;
         }[], import("../..").RAGEvaluationSuiteRun[] | {
             id: string;
@@ -351,6 +375,30 @@ export declare const useRAG: (path: string, options?: UseRAGOptions) => {
                 totalCases: number;
                 passingRate: number;
             };
+            traceSummary?: {
+                totalCases: number;
+                modes: import("../..").RAGHybridRetrievalMode[];
+                vectorCases: number;
+                lexicalCases: number;
+                transformedCases: number;
+                variantCases: number;
+                averageFinalCount: number;
+                averageVectorCount: number;
+                averageLexicalCount: number;
+                averageCandidateTopK: number;
+                averageLexicalTopK: number;
+                stageCounts: {
+                    input?: number | undefined;
+                    embed?: number | undefined;
+                    query_transform?: number | undefined;
+                    vector_search?: number | undefined;
+                    lexical_search?: number | undefined;
+                    fusion?: number | undefined;
+                    rerank?: number | undefined;
+                    score_filter?: number | undefined;
+                    finalize?: number | undefined;
+                };
+            } | undefined;
             metadata?: Record<string, unknown> | undefined;
         }[]>;
         suites: import("vue").Ref<{

package/dist/src/vue/ai/useRAGEvaluate.d.ts

@@ -171,6 +171,30 @@ export declare const useRAGEvaluate: (path: string) => {
             totalCases: number;
             passingRate: number;
         };
+        traceSummary?: {
+            totalCases: number;
+            modes: import("../..").RAGHybridRetrievalMode[];
+            vectorCases: number;
+            lexicalCases: number;
+            transformedCases: number;
+            variantCases: number;
+            averageFinalCount: number;
+            averageVectorCount: number;
+            averageLexicalCount: number;
+            averageCandidateTopK: number;
+            averageLexicalTopK: number;
+            stageCounts: {
+                input?: number | undefined;
+                embed?: number | undefined;
+                query_transform?: number | undefined;
+                vector_search?: number | undefined;
+                lexical_search?: number | undefined;
+                fusion?: number | undefined;
+                rerank?: number | undefined;
+                score_filter?: number | undefined;
+                finalize?: number | undefined;
+            };
+        } | undefined;
         metadata?: Record<string, unknown> | undefined;
     }[], RAGEvaluationSuiteRun[] | {
         id: string;
@@ -215,6 +239,30 @@ export declare const useRAGEvaluate: (path: string) => {
             totalCases: number;
             passingRate: number;
         };
+        traceSummary?: {
+            totalCases: number;
+            modes: import("../..").RAGHybridRetrievalMode[];
+            vectorCases: number;
+            lexicalCases: number;
+            transformedCases: number;
+            variantCases: number;
+            averageFinalCount: number;
+            averageVectorCount: number;
+            averageLexicalCount: number;
+            averageCandidateTopK: number;
+            averageLexicalTopK: number;
+            stageCounts: {
+                input?: number | undefined;
+                embed?: number | undefined;
+                query_transform?: number | undefined;
+                vector_search?: number | undefined;
+                lexical_search?: number | undefined;
+                fusion?: number | undefined;
+                rerank?: number | undefined;
+                score_filter?: number | undefined;
+                finalize?: number | undefined;
+            };
+        } | undefined;
         metadata?: Record<string, unknown> | undefined;
     }[]>;
     suites: import("vue").Ref<{

package/dist/svelte/ai/index.js

@@ -1771,6 +1771,131 @@ var buildRAGAnswerGroundingEvaluationLeaderboard = (runs) => {
     totalCases: run.response.totalCases
   }));
 };
+var buildTraceStageCounts = (traces) => {
+  const counts = {};
+  for (const trace of traces) {
+    for (const step of trace.steps) {
+      counts[step.stage] = (counts[step.stage] ?? 0) + 1;
+    }
+  }
+  return counts;
+};
+var diffTraceStageCounts = ({
+  current,
+  previous
+}) => {
+  const next = {};
+  const stages = new Set([
+    ...Object.keys(current),
+    ...Object.keys(previous)
+  ]);
+  for (const stage of stages) {
+    const delta = (current[stage] ?? 0) - (previous[stage] ?? 0);
+    if (delta !== 0) {
+      next[stage] = delta;
+    }
+  }
+  return next;
+};
+var roundTraceAverage = (value, total) => total > 0 ? Number((value / total).toFixed(2)) : 0;
+var summarizeRetrievalTraces = (traces) => {
+  if (traces.length === 0) {
+    return;
+  }
+  const totalCases = traces.length;
+  const modeSet = new Set;
+  let vectorCases = 0;
+  let lexicalCases = 0;
+  let transformedCases = 0;
+  let variantCases = 0;
+  let finalCountSum = 0;
+  let vectorCountSum = 0;
+  let lexicalCountSum = 0;
+  let candidateTopKSum = 0;
+  let lexicalTopKSum = 0;
+  for (const trace of traces) {
+    modeSet.add(trace.mode);
+    if (trace.runVector) {
+      vectorCases += 1;
+    }
+    if (trace.runLexical) {
+      lexicalCases += 1;
+    }
+    if (trace.transformedQuery !== trace.query) {
+      transformedCases += 1;
+    }
+    if (trace.variantQueries.length > 0) {
+      variantCases += 1;
+    }
+    finalCountSum += trace.resultCounts.final;
+    vectorCountSum += trace.resultCounts.vector;
+    lexicalCountSum += trace.resultCounts.lexical;
+    candidateTopKSum += trace.candidateTopK;
+    lexicalTopKSum += trace.lexicalTopK;
+  }
+  return {
+    averageCandidateTopK: roundTraceAverage(candidateTopKSum, totalCases),
+    averageFinalCount: roundTraceAverage(finalCountSum, totalCases),
+    averageLexicalCount: roundTraceAverage(lexicalCountSum, totalCases),
+    averageLexicalTopK: roundTraceAverage(lexicalTopKSum, totalCases),
+    averageVectorCount: roundTraceAverage(vectorCountSum, totalCases),
+    lexicalCases,
+    modes: Array.from(modeSet),
+    stageCounts: buildTraceStageCounts(traces),
+    totalCases,
+    transformedCases,
+    variantCases,
+    vectorCases
+  };
+};
+var evaluateRAGCollectionCases = async ({
+  collection,
+  input,
+  defaultTopK = DEFAULT_TOP_K,
+  rerank,
+  includeTrace = false
+}) => {
+  if (input.dryRun) {
+    return executeDryRunRAGEvaluation(input, defaultTopK).map((caseResult) => ({
+      caseResult,
+      trace: undefined
+    }));
+  }
+  return Promise.all(input.cases.map(async (caseInput, caseIndex) => {
+    const startedAt = Date.now();
+    const mode = resolveEvaluationMode(caseInput);
+    const query = caseInput.query.trim();
+    const expectedIds = normalizeExpectedIds(mode === "chunkId" ? caseInput.expectedChunkIds ?? [] : mode === "source" ? caseInput.expectedSources ?? [] : caseInput.expectedDocumentIds ?? []);
+    const topK = typeof caseInput.topK === "number" ? caseInput.topK : typeof input.topK === "number" ? input.topK : defaultTopK;
+    const searchInput = {
+      filter: typeof caseInput.filter === "object" ? caseInput.filter : input.filter,
+      model: caseInput.model ?? input.model,
+      query,
+      rerank,
+      scoreThreshold: typeof caseInput.scoreThreshold === "number" ? caseInput.scoreThreshold : input.scoreThreshold,
+      topK
+    };
+    const searchOutcome = includeTrace ? await collection.searchWithTrace(searchInput) : {
+      results: await collection.search(searchInput),
+      trace: undefined
+    };
+    const sources = buildSources(searchOutcome.results);
+    const elapsedMs = Date.now() - startedAt;
+    const retrievedIds = normalizeExpectedIds(sources.map((source) => extractExpectedId(source, mode)));
+    return {
+      caseResult: summarizeRAGEvaluationCase({
+        caseIndex,
+        caseInput: { ...caseInput, topK },
+        elapsedMs,
+        expectedIds,
+        mode,
+        query,
+        retrievedIds
+      }),
+      trace: searchOutcome.trace
+    };
+  }));
+};
 var buildRAGAnswerGroundingCaseDifficultyLeaderboard = (entries) => {
   const grouped = new Map;
   for (const entry of entries) {
@@ -2056,6 +2181,22 @@ var buildRAGEvaluationRunDiff = ({
       passingRate: current.response.passingRate - (previous?.response.passingRate ?? 0),
       partialCases: current.response.summary.partialCases - (previous?.response.summary.partialCases ?? 0)
     },
+    traceSummaryDelta: current.traceSummary || previous?.traceSummary ? {
+      averageCandidateTopK: (current.traceSummary?.averageCandidateTopK ?? 0) - (previous?.traceSummary?.averageCandidateTopK ?? 0),
+      averageFinalCount: (current.traceSummary?.averageFinalCount ?? 0) - (previous?.traceSummary?.averageFinalCount ?? 0),
+      averageLexicalCount: (current.traceSummary?.averageLexicalCount ?? 0) - (previous?.traceSummary?.averageLexicalCount ?? 0),
+      averageLexicalTopK: (current.traceSummary?.averageLexicalTopK ?? 0) - (previous?.traceSummary?.averageLexicalTopK ?? 0),
+      averageVectorCount: (current.traceSummary?.averageVectorCount ?? 0) - (previous?.traceSummary?.averageVectorCount ?? 0),
+      lexicalCases: (current.traceSummary?.lexicalCases ?? 0) - (previous?.traceSummary?.lexicalCases ?? 0),
+      modesChanged: (current.traceSummary?.modes ?? []).join("|") !== (previous?.traceSummary?.modes ?? []).join("|"),
+      stageCounts: diffTraceStageCounts({
+        current: current.traceSummary?.stageCounts ?? {},
+        previous: previous?.traceSummary?.stageCounts ?? {}
+      }),
+      transformedCases: (current.traceSummary?.transformedCases ?? 0) - (previous?.traceSummary?.transformedCases ?? 0),
+      variantCases: (current.traceSummary?.variantCases ?? 0) - (previous?.traceSummary?.variantCases ?? 0),
+      vectorCases: (current.traceSummary?.vectorCases ?? 0) - (previous?.traceSummary?.vectorCases ?? 0)
+    } : undefined,
     unchangedCases
   };
 };
@@ -2406,17 +2547,20 @@ var compareRAGRerankers = async ({
   defaultTopK = DEFAULT_TOP_K
 }) => {
   const entries = await Promise.all(rerankers.map(async (candidate) => {
-    const response = await evaluateRAGCollection({
+    const evaluated = await evaluateRAGCollectionCases({
       collection,
       defaultTopK,
       input: suite.input,
+      includeTrace: true,
       rerank: candidate.rerank
     });
+    const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
       label: candidate.label ?? candidate.id,
       providerName: typeof candidate.rerank === "function" ? undefined : candidate.rerank?.providerName,
       response,
-      rerankerId: candidate.id
+      rerankerId: candidate.id,
+      traceSummary: summarizeRetrievalTraces(evaluated.map((entry) => entry.trace).filter((trace) => Boolean(trace)))
     };
   }));
   const leaderboard = buildRAGEvaluationLeaderboard(entries.map((entry) => ({
@@ -2426,7 +2570,8 @@ var compareRAGRerankers = async ({
     label: entry.label,
     response: entry.response,
     startedAt: 0,
-    suiteId: suite.id
+    suiteId: suite.id,
+    traceSummary: entry.traceSummary
   })));
   return {
     entries,
@@ -2471,25 +2616,37 @@ var compareRAGRetrievalStrategies = async ({
   defaultTopK = DEFAULT_TOP_K
 }) => {
   const entries = await Promise.all(retrievals.map(async (candidate) => {
-    const response = await evaluateRAGCollection({
+    const tracedCollection = {
+      ...collection,
+      search: (input) => collection.search({
+        ...input,
+        queryTransform: candidate.queryTransform ?? input.queryTransform,
+        rerank: candidate.rerank ?? input.rerank,
+        retrieval: candidate.retrieval ?? input.retrieval
+      }),
+      searchWithTrace: (input) => collection.searchWithTrace({
+        ...input,
+        queryTransform: candidate.queryTransform ?? input.queryTransform,
+        rerank: candidate.rerank ?? input.rerank,
+        retrieval: candidate.retrieval ?? input.retrieval
+      })
+    };
+    const evaluated = await evaluateRAGCollectionCases({
       collection: {
-        ...collection,
-        search: (input) => collection.search({
-          ...input,
-          queryTransform: candidate.queryTransform ?? input.queryTransform,
-          rerank: candidate.rerank ?? input.rerank,
-          retrieval: candidate.retrieval ?? input.retrieval
-        })
+        ...tracedCollection
       },
       defaultTopK,
       input: suite.input,
+      includeTrace: true,
       rerank: candidate.rerank
     });
+    const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
       label: candidate.label ?? candidate.id,
       response,
       retrievalId: candidate.id,
-      retrievalMode: resolveRetrievalMode(candidate)
+      retrievalMode: resolveRetrievalMode(candidate),
+      traceSummary: summarizeRetrievalTraces(evaluated.map((entry) => entry.trace).filter((trace) => Boolean(trace)))
     };
   }));
   const leaderboard = buildRAGEvaluationLeaderboard(entries.map((entry) => ({
@@ -2499,7 +2656,8 @@ var compareRAGRetrievalStrategies = async ({
     label: entry.label,
     response: entry.response,
     startedAt: 0,
-    suiteId: suite.id
+    suiteId: suite.id,
+    traceSummary: entry.traceSummary
   })));
   return {
     entries,
@@ -2516,37 +2674,14 @@ var evaluateRAGCollection = async ({
   defaultTopK = DEFAULT_TOP_K,
   rerank
 }) => {
-  if (input.dryRun) {
-    return buildRAGEvaluationResponse(executeDryRunRAGEvaluation(input, defaultTopK));
-  }
-  const evaluated = await Promise.all(input.cases.map(async (caseInput, caseIndex) => {
-    const startedAt = Date.now();
-    const mode = resolveEvaluationMode(caseInput);
-    const query = caseInput.query.trim();
-    const expectedIds = normalizeExpectedIds(mode === "chunkId" ? caseInput.expectedChunkIds ?? [] : mode === "source" ? caseInput.expectedSources ?? [] : caseInput.expectedDocumentIds ?? []);
-    const topK = typeof caseInput.topK === "number" ? caseInput.topK : typeof input.topK === "number" ? input.topK : defaultTopK;
-    const searchResults = await collection.search({
-      filter: typeof caseInput.filter === "object" ? caseInput.filter : input.filter,
-      model: caseInput.model ?? input.model,
-      query,
-      rerank,
-      scoreThreshold: typeof caseInput.scoreThreshold === "number" ? caseInput.scoreThreshold : input.scoreThreshold,
-      topK
-    });
-    const sources = buildSources(searchResults);
-    const elapsedMs = Date.now() - startedAt;
-    const retrievedIds = normalizeExpectedIds(sources.map((source) => extractExpectedId(source, mode)));
-    return summarizeRAGEvaluationCase({
-      caseIndex,
-      caseInput: { ...caseInput, topK },
-      elapsedMs,
-      expectedIds,
-      mode,
-      query,
-      retrievedIds
-    });
-  }));
-  return buildRAGEvaluationResponse(evaluated);
+  const evaluated = await evaluateRAGCollectionCases({
+    collection,
+    defaultTopK,
+    includeTrace: false,
+    input,
+    rerank
+  });
+  return buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
 };
 var executeDryRunRAGEvaluation = (input, defaultTopK = DEFAULT_TOP_K) => input.cases.map((caseInput, caseIndex) => {
   const mode = resolveEvaluationMode(caseInput);
@@ -3319,5 +3454,5 @@ export {
   createAIStream
 };
 
-//# debugId=037EF89CF3E8405464756E2164756E21
+//# debugId=4CDEE3E3E09A9FC864756E2164756E21
 //# sourceMappingURL=index.js.map