@absolutejs/absolute 0.19.0-beta.531 → 0.19.0-beta.532

package/dist/ai/client/index.js

@@ -1744,6 +1744,114 @@ var buildRAGAnswerGroundingEvaluationLeaderboard = (runs) => {
     totalCases: run.response.totalCases
   }));
 };
+var buildTraceStageCounts = (traces) => {
+  const counts = {};
+  for (const trace of traces) {
+    for (const step of trace.steps) {
+      counts[step.stage] = (counts[step.stage] ?? 0) + 1;
+    }
+  }
+  return counts;
+};
+var roundTraceAverage = (value, total) => total > 0 ? Number((value / total).toFixed(2)) : 0;
+var summarizeRetrievalTraces = (traces) => {
+  if (traces.length === 0) {
+    return;
+  }
+  const totalCases = traces.length;
+  const modeSet = new Set;
+  let vectorCases = 0;
+  let lexicalCases = 0;
+  let transformedCases = 0;
+  let variantCases = 0;
+  let finalCountSum = 0;
+  let vectorCountSum = 0;
+  let lexicalCountSum = 0;
+  let candidateTopKSum = 0;
+  let lexicalTopKSum = 0;
+  for (const trace of traces) {
+    modeSet.add(trace.mode);
+    if (trace.runVector) {
+      vectorCases += 1;
+    }
+    if (trace.runLexical) {
+      lexicalCases += 1;
+    }
+    if (trace.transformedQuery !== trace.query) {
+      transformedCases += 1;
+    }
+    if (trace.variantQueries.length > 0) {
+      variantCases += 1;
+    }
+    finalCountSum += trace.resultCounts.final;
+    vectorCountSum += trace.resultCounts.vector;
+    lexicalCountSum += trace.resultCounts.lexical;
+    candidateTopKSum += trace.candidateTopK;
+    lexicalTopKSum += trace.lexicalTopK;
+  }
+  return {
+    averageCandidateTopK: roundTraceAverage(candidateTopKSum, totalCases),
+    averageFinalCount: roundTraceAverage(finalCountSum, totalCases),
+    averageLexicalCount: roundTraceAverage(lexicalCountSum, totalCases),
+    averageLexicalTopK: roundTraceAverage(lexicalTopKSum, totalCases),
+    averageVectorCount: roundTraceAverage(vectorCountSum, totalCases),
+    lexicalCases,
+    modes: Array.from(modeSet),
+    stageCounts: buildTraceStageCounts(traces),
+    totalCases,
+    transformedCases,
+    variantCases,
+    vectorCases
+  };
+};
+var evaluateRAGCollectionCases = async ({
+  collection,
+  input,
+  defaultTopK = DEFAULT_TOP_K,
+  rerank,
+  includeTrace = false
+}) => {
+  if (input.dryRun) {
+    return executeDryRunRAGEvaluation(input, defaultTopK).map((caseResult) => ({
+      caseResult,
+      trace: undefined
+    }));
+  }
+  return Promise.all(input.cases.map(async (caseInput, caseIndex) => {
+    const startedAt = Date.now();
+    const mode = resolveEvaluationMode(caseInput);
+    const query = caseInput.query.trim();
+    const expectedIds = normalizeExpectedIds(mode === "chunkId" ? caseInput.expectedChunkIds ?? [] : mode === "source" ? caseInput.expectedSources ?? [] : caseInput.expectedDocumentIds ?? []);
+    const topK = typeof caseInput.topK === "number" ? caseInput.topK : typeof input.topK === "number" ? input.topK : defaultTopK;
+    const searchInput = {
+      filter: typeof caseInput.filter === "object" ? caseInput.filter : input.filter,
+      model: caseInput.model ?? input.model,
+      query,
+      rerank,
+      scoreThreshold: typeof caseInput.scoreThreshold === "number" ? caseInput.scoreThreshold : input.scoreThreshold,
+      topK
+    };
+    const searchOutcome = includeTrace ? await collection.searchWithTrace(searchInput) : {
+      results: await collection.search(searchInput),
+      trace: undefined
+    };
+    const sources = buildSources(searchOutcome.results);
+    const elapsedMs = Date.now() - startedAt;
+    const retrievedIds = normalizeExpectedIds(sources.map((source) => extractExpectedId(source, mode)));
+    return {
+      caseResult: summarizeRAGEvaluationCase({
+        caseIndex,
+        caseInput: { ...caseInput, topK },
+        elapsedMs,
+        expectedIds,
+        mode,
+        query,
+        retrievedIds
+      }),
+      trace: searchOutcome.trace
+    };
+  }));
+};
 var buildRAGAnswerGroundingCaseDifficultyLeaderboard = (entries) => {
   const grouped = new Map;
   for (const entry of entries) {
@@ -2379,17 +2487,20 @@ var compareRAGRerankers = async ({
   defaultTopK = DEFAULT_TOP_K
 }) => {
   const entries = await Promise.all(rerankers.map(async (candidate) => {
-    const response = await evaluateRAGCollection({
+    const evaluated = await evaluateRAGCollectionCases({
       collection,
       defaultTopK,
       input: suite.input,
+      includeTrace: true,
       rerank: candidate.rerank
     });
+    const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
       label: candidate.label ?? candidate.id,
       providerName: typeof candidate.rerank === "function" ? undefined : candidate.rerank?.providerName,
       response,
-      rerankerId: candidate.id
+      rerankerId: candidate.id,
+      traceSummary: summarizeRetrievalTraces(evaluated.map((entry) => entry.trace).filter((trace) => Boolean(trace)))
     };
   }));
   const leaderboard = buildRAGEvaluationLeaderboard(entries.map((entry) => ({
@@ -2444,25 +2555,37 @@ var compareRAGRetrievalStrategies = async ({
   defaultTopK = DEFAULT_TOP_K
 }) => {
   const entries = await Promise.all(retrievals.map(async (candidate) => {
-    const response = await evaluateRAGCollection({
+    const tracedCollection = {
+      ...collection,
+      search: (input) => collection.search({
+        ...input,
+        queryTransform: candidate.queryTransform ?? input.queryTransform,
+        rerank: candidate.rerank ?? input.rerank,
+        retrieval: candidate.retrieval ?? input.retrieval
+      }),
+      searchWithTrace: (input) => collection.searchWithTrace({
+        ...input,
+        queryTransform: candidate.queryTransform ?? input.queryTransform,
+        rerank: candidate.rerank ?? input.rerank,
+        retrieval: candidate.retrieval ?? input.retrieval
+      })
+    };
+    const evaluated = await evaluateRAGCollectionCases({
       collection: {
-        ...collection,
-        search: (input) => collection.search({
-          ...input,
-          queryTransform: candidate.queryTransform ?? input.queryTransform,
-          rerank: candidate.rerank ?? input.rerank,
-          retrieval: candidate.retrieval ?? input.retrieval
-        })
+        ...tracedCollection
       },
       defaultTopK,
       input: suite.input,
+      includeTrace: true,
       rerank: candidate.rerank
     });
+    const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
       label: candidate.label ?? candidate.id,
       response,
       retrievalId: candidate.id,
-      retrievalMode: resolveRetrievalMode(candidate)
+      retrievalMode: resolveRetrievalMode(candidate),
+      traceSummary: summarizeRetrievalTraces(evaluated.map((entry) => entry.trace).filter((trace) => Boolean(trace)))
     };
   }));
   const leaderboard = buildRAGEvaluationLeaderboard(entries.map((entry) => ({
@@ -2489,37 +2612,14 @@ var evaluateRAGCollection = async ({
   defaultTopK = DEFAULT_TOP_K,
   rerank
 }) => {
-  if (input.dryRun) {
-    return buildRAGEvaluationResponse(executeDryRunRAGEvaluation(input, defaultTopK));
-  }
-  const evaluated = await Promise.all(input.cases.map(async (caseInput, caseIndex) => {
-    const startedAt = Date.now();
-    const mode = resolveEvaluationMode(caseInput);
-    const query = caseInput.query.trim();
-    const expectedIds = normalizeExpectedIds(mode === "chunkId" ? caseInput.expectedChunkIds ?? [] : mode === "source" ? caseInput.expectedSources ?? [] : caseInput.expectedDocumentIds ?? []);
-    const topK = typeof caseInput.topK === "number" ? caseInput.topK : typeof input.topK === "number" ? input.topK : defaultTopK;
-    const searchResults = await collection.search({
-      filter: typeof caseInput.filter === "object" ? caseInput.filter : input.filter,
-      model: caseInput.model ?? input.model,
-      query,
-      rerank,
-      scoreThreshold: typeof caseInput.scoreThreshold === "number" ? caseInput.scoreThreshold : input.scoreThreshold,
-      topK
-    });
-    const sources = buildSources(searchResults);
-    const elapsedMs = Date.now() - startedAt;
-    const retrievedIds = normalizeExpectedIds(sources.map((source) => extractExpectedId(source, mode)));
-    return summarizeRAGEvaluationCase({
-      caseIndex,
-      caseInput: { ...caseInput, topK },
-      elapsedMs,
-      expectedIds,
-      mode,
-      query,
-      retrievedIds
-    });
-  }));
-  return buildRAGEvaluationResponse(evaluated);
+  const evaluated = await evaluateRAGCollectionCases({
+    collection,
+    defaultTopK,
+    includeTrace: false,
+    input,
+    rerank
+  });
+  return buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
 };
 var executeDryRunRAGEvaluation = (input, defaultTopK = DEFAULT_TOP_K) => input.cases.map((caseInput, caseIndex) => {
   const mode = resolveEvaluationMode(caseInput);
@@ -2639,5 +2739,5 @@ export {
   buildRAGAnswerWorkflowState
 };
 
-//# debugId=CBECE8C7471759AC64756E2164756E21
+//# debugId=9271E9C7FE8F8B9964756E2164756E21
 //# sourceMappingURL=index.js.map