@absolutejs/absolute 0.19.0-beta.531 → 0.19.0-beta.533

package/dist/ai/client/index.js

@@ -1744,6 +1744,131 @@ var buildRAGAnswerGroundingEvaluationLeaderboard = (runs) => {
     totalCases: run.response.totalCases
   }));
 };
+var buildTraceStageCounts = (traces) => {
+  const counts = {};
+  for (const trace of traces) {
+    for (const step of trace.steps) {
+      counts[step.stage] = (counts[step.stage] ?? 0) + 1;
+    }
+  }
+  return counts;
+};
+var diffTraceStageCounts = ({
+  current,
+  previous
+}) => {
+  const next = {};
+  const stages = new Set([
+    ...Object.keys(current),
+    ...Object.keys(previous)
+  ]);
+  for (const stage of stages) {
+    const delta = (current[stage] ?? 0) - (previous[stage] ?? 0);
+    if (delta !== 0) {
+      next[stage] = delta;
+    }
+  }
+  return next;
+};
+var roundTraceAverage = (value, total) => total > 0 ? Number((value / total).toFixed(2)) : 0;
+var summarizeRetrievalTraces = (traces) => {
+  if (traces.length === 0) {
+    return;
+  }
+  const totalCases = traces.length;
+  const modeSet = new Set;
+  let vectorCases = 0;
+  let lexicalCases = 0;
+  let transformedCases = 0;
+  let variantCases = 0;
+  let finalCountSum = 0;
+  let vectorCountSum = 0;
+  let lexicalCountSum = 0;
+  let candidateTopKSum = 0;
+  let lexicalTopKSum = 0;
+  for (const trace of traces) {
+    modeSet.add(trace.mode);
+    if (trace.runVector) {
+      vectorCases += 1;
+    }
+    if (trace.runLexical) {
+      lexicalCases += 1;
+    }
+    if (trace.transformedQuery !== trace.query) {
+      transformedCases += 1;
+    }
+    if (trace.variantQueries.length > 0) {
+      variantCases += 1;
+    }
+    finalCountSum += trace.resultCounts.final;
+    vectorCountSum += trace.resultCounts.vector;
+    lexicalCountSum += trace.resultCounts.lexical;
+    candidateTopKSum += trace.candidateTopK;
+    lexicalTopKSum += trace.lexicalTopK;
+  }
+  return {
+    averageCandidateTopK: roundTraceAverage(candidateTopKSum, totalCases),
+    averageFinalCount: roundTraceAverage(finalCountSum, totalCases),
+    averageLexicalCount: roundTraceAverage(lexicalCountSum, totalCases),
+    averageLexicalTopK: roundTraceAverage(lexicalTopKSum, totalCases),
+    averageVectorCount: roundTraceAverage(vectorCountSum, totalCases),
+    lexicalCases,
+    modes: Array.from(modeSet),
+    stageCounts: buildTraceStageCounts(traces),
+    totalCases,
+    transformedCases,
+    variantCases,
+    vectorCases
+  };
+};
+var evaluateRAGCollectionCases = async ({
+  collection,
+  input,
+  defaultTopK = DEFAULT_TOP_K,
+  rerank,
+  includeTrace = false
+}) => {
+  if (input.dryRun) {
+    return executeDryRunRAGEvaluation(input, defaultTopK).map((caseResult) => ({
+      caseResult,
+      trace: undefined
+    }));
+  }
+  return Promise.all(input.cases.map(async (caseInput, caseIndex) => {
+    const startedAt = Date.now();
+    const mode = resolveEvaluationMode(caseInput);
+    const query = caseInput.query.trim();
+    const expectedIds = normalizeExpectedIds(mode === "chunkId" ? caseInput.expectedChunkIds ?? [] : mode === "source" ? caseInput.expectedSources ?? [] : caseInput.expectedDocumentIds ?? []);
+    const topK = typeof caseInput.topK === "number" ? caseInput.topK : typeof input.topK === "number" ? input.topK : defaultTopK;
+    const searchInput = {
+      filter: typeof caseInput.filter === "object" ? caseInput.filter : input.filter,
+      model: caseInput.model ?? input.model,
+      query,
+      rerank,
+      scoreThreshold: typeof caseInput.scoreThreshold === "number" ? caseInput.scoreThreshold : input.scoreThreshold,
+      topK
+    };
+    const searchOutcome = includeTrace ? await collection.searchWithTrace(searchInput) : {
+      results: await collection.search(searchInput),
+      trace: undefined
+    };
+    const sources = buildSources(searchOutcome.results);
+    const elapsedMs = Date.now() - startedAt;
+    const retrievedIds = normalizeExpectedIds(sources.map((source) => extractExpectedId(source, mode)));
+    return {
+      caseResult: summarizeRAGEvaluationCase({
+        caseIndex,
+        caseInput: { ...caseInput, topK },
+        elapsedMs,
+        expectedIds,
+        mode,
+        query,
+        retrievedIds
+      }),
+      trace: searchOutcome.trace
+    };
+  }));
+};
 var buildRAGAnswerGroundingCaseDifficultyLeaderboard = (entries) => {
   const grouped = new Map;
   for (const entry of entries) {
@@ -2029,6 +2154,22 @@ var buildRAGEvaluationRunDiff = ({
       passingRate: current.response.passingRate - (previous?.response.passingRate ?? 0),
       partialCases: current.response.summary.partialCases - (previous?.response.summary.partialCases ?? 0)
     },
+    traceSummaryDelta: current.traceSummary || previous?.traceSummary ? {
+      averageCandidateTopK: (current.traceSummary?.averageCandidateTopK ?? 0) - (previous?.traceSummary?.averageCandidateTopK ?? 0),
+      averageFinalCount: (current.traceSummary?.averageFinalCount ?? 0) - (previous?.traceSummary?.averageFinalCount ?? 0),
+      averageLexicalCount: (current.traceSummary?.averageLexicalCount ?? 0) - (previous?.traceSummary?.averageLexicalCount ?? 0),
+      averageLexicalTopK: (current.traceSummary?.averageLexicalTopK ?? 0) - (previous?.traceSummary?.averageLexicalTopK ?? 0),
+      averageVectorCount: (current.traceSummary?.averageVectorCount ?? 0) - (previous?.traceSummary?.averageVectorCount ?? 0),
+      lexicalCases: (current.traceSummary?.lexicalCases ?? 0) - (previous?.traceSummary?.lexicalCases ?? 0),
+      modesChanged: (current.traceSummary?.modes ?? []).join("|") !== (previous?.traceSummary?.modes ?? []).join("|"),
+      stageCounts: diffTraceStageCounts({
+        current: current.traceSummary?.stageCounts ?? {},
+        previous: previous?.traceSummary?.stageCounts ?? {}
+      }),
+      transformedCases: (current.traceSummary?.transformedCases ?? 0) - (previous?.traceSummary?.transformedCases ?? 0),
+      variantCases: (current.traceSummary?.variantCases ?? 0) - (previous?.traceSummary?.variantCases ?? 0),
+      vectorCases: (current.traceSummary?.vectorCases ?? 0) - (previous?.traceSummary?.vectorCases ?? 0)
+    } : undefined,
     unchangedCases
   };
 };
@@ -2379,17 +2520,20 @@ var compareRAGRerankers = async ({
   defaultTopK = DEFAULT_TOP_K
 }) => {
   const entries = await Promise.all(rerankers.map(async (candidate) => {
-    const response = await evaluateRAGCollection({
+    const evaluated = await evaluateRAGCollectionCases({
       collection,
       defaultTopK,
       input: suite.input,
+      includeTrace: true,
       rerank: candidate.rerank
     });
+    const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
       label: candidate.label ?? candidate.id,
       providerName: typeof candidate.rerank === "function" ? undefined : candidate.rerank?.providerName,
       response,
-      rerankerId: candidate.id
+      rerankerId: candidate.id,
+      traceSummary: summarizeRetrievalTraces(evaluated.map((entry) => entry.trace).filter((trace) => Boolean(trace)))
     };
   }));
   const leaderboard = buildRAGEvaluationLeaderboard(entries.map((entry) => ({
@@ -2399,7 +2543,8 @@ var compareRAGRerankers = async ({
     label: entry.label,
     response: entry.response,
     startedAt: 0,
-    suiteId: suite.id
+    suiteId: suite.id,
+    traceSummary: entry.traceSummary
   })));
   return {
     entries,
@@ -2444,25 +2589,37 @@ var compareRAGRetrievalStrategies = async ({
   defaultTopK = DEFAULT_TOP_K
 }) => {
   const entries = await Promise.all(retrievals.map(async (candidate) => {
-    const response = await evaluateRAGCollection({
+    const tracedCollection = {
+      ...collection,
+      search: (input) => collection.search({
+        ...input,
+        queryTransform: candidate.queryTransform ?? input.queryTransform,
+        rerank: candidate.rerank ?? input.rerank,
+        retrieval: candidate.retrieval ?? input.retrieval
+      }),
+      searchWithTrace: (input) => collection.searchWithTrace({
+        ...input,
+        queryTransform: candidate.queryTransform ?? input.queryTransform,
+        rerank: candidate.rerank ?? input.rerank,
+        retrieval: candidate.retrieval ?? input.retrieval
+      })
+    };
+    const evaluated = await evaluateRAGCollectionCases({
       collection: {
-        ...collection,
-        search: (input) => collection.search({
-          ...input,
-          queryTransform: candidate.queryTransform ?? input.queryTransform,
-          rerank: candidate.rerank ?? input.rerank,
-          retrieval: candidate.retrieval ?? input.retrieval
-        })
+        ...tracedCollection
       },
       defaultTopK,
       input: suite.input,
+      includeTrace: true,
       rerank: candidate.rerank
     });
+    const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
       label: candidate.label ?? candidate.id,
       response,
       retrievalId: candidate.id,
-      retrievalMode: resolveRetrievalMode(candidate)
+      retrievalMode: resolveRetrievalMode(candidate),
+      traceSummary: summarizeRetrievalTraces(evaluated.map((entry) => entry.trace).filter((trace) => Boolean(trace)))
     };
   }));
   const leaderboard = buildRAGEvaluationLeaderboard(entries.map((entry) => ({
@@ -2472,7 +2629,8 @@ var compareRAGRetrievalStrategies = async ({
     label: entry.label,
     response: entry.response,
     startedAt: 0,
-    suiteId: suite.id
+    suiteId: suite.id,
+    traceSummary: entry.traceSummary
   })));
   return {
     entries,
@@ -2489,37 +2647,14 @@ var evaluateRAGCollection = async ({
   defaultTopK = DEFAULT_TOP_K,
   rerank
 }) => {
-  if (input.dryRun) {
-    return buildRAGEvaluationResponse(executeDryRunRAGEvaluation(input, defaultTopK));
-  }
-  const evaluated = await Promise.all(input.cases.map(async (caseInput, caseIndex) => {
-    const startedAt = Date.now();
-    const mode = resolveEvaluationMode(caseInput);
-    const query = caseInput.query.trim();
-    const expectedIds = normalizeExpectedIds(mode === "chunkId" ? caseInput.expectedChunkIds ?? [] : mode === "source" ? caseInput.expectedSources ?? [] : caseInput.expectedDocumentIds ?? []);
-    const topK = typeof caseInput.topK === "number" ? caseInput.topK : typeof input.topK === "number" ? input.topK : defaultTopK;
-    const searchResults = await collection.search({
-      filter: typeof caseInput.filter === "object" ? caseInput.filter : input.filter,
-      model: caseInput.model ?? input.model,
-      query,
-      rerank,
-      scoreThreshold: typeof caseInput.scoreThreshold === "number" ? caseInput.scoreThreshold : input.scoreThreshold,
-      topK
-    });
-    const sources = buildSources(searchResults);
-    const elapsedMs = Date.now() - startedAt;
-    const retrievedIds = normalizeExpectedIds(sources.map((source) => extractExpectedId(source, mode)));
-    return summarizeRAGEvaluationCase({
-      caseIndex,
-      caseInput: { ...caseInput, topK },
-      elapsedMs,
-      expectedIds,
-      mode,
-      query,
-      retrievedIds
-    });
-  }));
-  return buildRAGEvaluationResponse(evaluated);
+  const evaluated = await evaluateRAGCollectionCases({
+    collection,
+    defaultTopK,
+    includeTrace: false,
+    input,
+    rerank
+  });
+  return buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
 };
 var executeDryRunRAGEvaluation = (input, defaultTopK = DEFAULT_TOP_K) => input.cases.map((caseInput, caseIndex) => {
   const mode = resolveEvaluationMode(caseInput);
@@ -2639,5 +2774,5 @@ export {
   buildRAGAnswerWorkflowState
 };
 
-//# debugId=CBECE8C7471759AC64756E2164756E21
+//# debugId=BCBFBA9A1CF59F3A64756E2164756E21
 //# sourceMappingURL=index.js.map