@absolutejs/absolute 0.19.0-beta.533 → 0.19.0-beta.535

package/dist/ai/client/index.js

@@ -2130,7 +2130,266 @@ var buildGroundingCaseSnapshots = ({
     };
   });
 };
+var areStageCountsEqual = (left, right) => {
+  const keys = new Set([
+    ...Object.keys(left),
+    ...Object.keys(right)
+  ]);
+  for (const key of keys) {
+    if ((left[key] ?? 0) !== (right[key] ?? 0)) {
+      return false;
+    }
+  }
+  return true;
+};
+var buildEvaluationCaseTraceSnapshot = ({
+  caseResult,
+  currentTrace,
+  previousTrace
+}) => {
+  const stageCounts = currentTrace ? buildTraceStageCounts([currentTrace]) : {};
+  const previousStageCounts = previousTrace?.stageCounts ?? {};
+  const traceChange = !previousTrace ? currentTrace ? "new" : "unchanged" : previousTrace.traceMode !== currentTrace?.mode || previousTrace.transformedQuery !== (currentTrace?.transformedQuery || undefined) || previousTrace.variantQueries.join("|") !== (currentTrace?.variantQueries ?? []).join("|") || previousTrace.finalCount !== (currentTrace?.resultCounts.final ?? 0) || previousTrace.vectorCount !== (currentTrace?.resultCounts.vector ?? 0) || previousTrace.lexicalCount !== (currentTrace?.resultCounts.lexical ?? 0) || previousTrace.candidateTopK !== (currentTrace?.candidateTopK ?? 0) || previousTrace.lexicalTopK !== (currentTrace?.lexicalTopK ?? 0) || !areStageCountsEqual(previousStageCounts, stageCounts) ? "changed" : "unchanged";
+  return {
+    candidateTopK: currentTrace?.candidateTopK ?? 0,
+    caseId: caseResult.caseId,
+    finalCount: currentTrace?.resultCounts.final ?? 0,
+    label: caseResult.label,
+    lexicalCount: currentTrace?.resultCounts.lexical ?? 0,
+    lexicalTopK: currentTrace?.lexicalTopK ?? 0,
+    previousCandidateTopK: previousTrace?.candidateTopK,
+    previousFinalCount: previousTrace?.finalCount,
+    previousLexicalCount: previousTrace?.lexicalCount,
+    previousLexicalTopK: previousTrace?.lexicalTopK,
+    previousStageCounts,
+    previousTraceMode: previousTrace?.traceMode,
+    previousTransformedQuery: previousTrace?.transformedQuery,
+    previousVariantQueries: previousTrace?.variantQueries ?? [],
+    previousVectorCount: previousTrace?.vectorCount,
+    query: caseResult.query,
+    stageCounts,
+    status: caseResult.status,
+    traceChange,
+    traceMode: currentTrace?.mode,
+    transformedQuery: currentTrace?.transformedQuery || undefined,
+    variantQueries: currentTrace?.variantQueries ?? [],
+    vectorCount: currentTrace?.resultCounts.vector ?? 0
+  };
+};
+var buildEvaluationCaseTraceSnapshotsFromEvaluated = (evaluated) => evaluated.map(({ caseResult, trace }) => buildEvaluationCaseTraceSnapshot({
+  caseResult,
+  currentTrace: trace
+}));
+var buildEvaluationCaseTraceSnapshots = ({
+  current,
+  previous
+}) => {
+  if (!current) {
+    return [];
+  }
+  const currentTraces = new Map((current.caseTraceSnapshots ?? []).map((entry) => [entry.caseId, entry]));
+  const previousTraces = new Map((previous?.caseTraceSnapshots ?? []).map((entry) => [
+    entry.caseId,
+    entry
+  ]));
+  return current.response.cases.map((caseResult) => buildEvaluationCaseTraceSnapshot({
+    caseResult,
+    currentTrace: (() => {
+      const currentSnapshot = currentTraces.get(caseResult.caseId);
+      if (!currentSnapshot) {
+        return;
+      }
+      return {
+        candidateTopK: currentSnapshot.candidateTopK,
+        lexicalTopK: currentSnapshot.lexicalTopK,
+        mode: currentSnapshot.traceMode ?? "vector",
+        query: caseResult.query,
+        resultCounts: {
+          final: currentSnapshot.finalCount,
+          fused: currentSnapshot.finalCount,
+          lexical: currentSnapshot.lexicalCount,
+          reranked: currentSnapshot.finalCount,
+          vector: currentSnapshot.vectorCount
+        },
+        runLexical: currentSnapshot.lexicalCount > 0,
+        runVector: currentSnapshot.vectorCount > 0,
+        steps: [],
+        topK: caseResult.topK,
+        transformedQuery: currentSnapshot.transformedQuery ?? caseResult.query,
+        variantQueries: currentSnapshot.variantQueries
+      };
+    })(),
+    previousTrace: previousTraces.get(caseResult.caseId)
+  }));
+};
 var getStatusRank = (status) => status === "pass" ? 2 : status === "partial" ? 1 : 0;
+var formatSignedDelta = (value, decimals = 0, suffix = "") => `${value >= 0 ? "+" : ""}${value.toFixed(decimals)}${suffix}`;
+var formatEvaluationSummary = (response) => `${response.summary.passedCases}/${response.totalCases} pass \xB7 f1 ${response.summary.averageF1.toFixed(3)} \xB7 latency ${response.summary.averageLatencyMs.toFixed(1)}ms`;
+var formatHistoryCaseLabels = (cases) => cases.length > 0 ? cases.map((entry) => entry.label ?? entry.caseId).join(", ") : "none";
+var formatTraceModes = (modes) => modes.length > 0 ? modes.join(" / ") : "n/a";
+var formatTraceStageSummary = (stageCounts) => {
+  const topStages = Object.entries(stageCounts).sort((left, right) => right[1] - left[1]).slice(0, 3);
+  return topStages.length > 0 ? topStages.map(([stage, count]) => `${stage} ${count}`).join(" \xB7 ") : "n/a";
+};
+var formatTraceRatio = (count, total) => `${count}/${total}`;
+var formatTraceCountDelta = (value) => `${value >= 0 ? "+" : ""}${value}`;
+var buildRAGComparisonTraceSummaryRows = (entry) => {
+  const trace = entry.traceSummary;
+  if (!trace) {
+    return [{ label: "Trace", value: "Unavailable" }];
+  }
+  return [
+    { label: "Modes", value: formatTraceModes(trace.modes) },
+    { label: "Avg final", value: trace.averageFinalCount.toFixed(1) },
+    { label: "Avg vector", value: trace.averageVectorCount.toFixed(1) },
+    { label: "Avg lexical", value: trace.averageLexicalCount.toFixed(1) },
+    {
+      label: "Transforms",
+      value: formatTraceRatio(trace.transformedCases, trace.totalCases)
+    },
+    {
+      label: "Variants",
+      value: formatTraceRatio(trace.variantCases, trace.totalCases)
+    },
+    {
+      label: "TopK",
+      value: `${trace.averageCandidateTopK.toFixed(1)} / ${trace.averageLexicalTopK.toFixed(1)}`
+    },
+    {
+      label: "Stages",
+      value: formatTraceStageSummary(trace.stageCounts)
+    }
+  ];
+};
+var buildRAGComparisonTraceDiffRows = (entry, leader) => {
+  const trace = entry.traceSummary;
+  if (!trace) {
+    return [{ label: "Trace", value: "Unavailable for comparison" }];
+  }
+  const leaderTrace = leader?.traceSummary;
+  if (!leaderTrace) {
+    return [{ label: "Baseline", value: "Leader trace unavailable" }];
+  }
+  if (entry === leader) {
+    return [{ label: "Baseline", value: "Leader strategy" }];
+  }
+  const stageDelta = Object.keys({
+    ...leaderTrace.stageCounts,
+    ...trace.stageCounts
+  }).map((stage) => {
+    const typedStage = stage;
+    const delta = (trace.stageCounts[typedStage] ?? 0) - (leaderTrace.stageCounts[typedStage] ?? 0);
+    return delta === 0 ? null : `${typedStage} ${formatTraceCountDelta(delta)}`;
+  }).filter((value) => Boolean(value)).slice(0, 3).join(" \xB7 ");
+  const rows = [
+    { label: "Baseline", value: leader.label }
+  ];
+  if (formatTraceModes(trace.modes) !== formatTraceModes(leaderTrace.modes)) {
+    rows.push({
+      label: "Modes vs leader",
+      value: `${formatTraceModes(trace.modes)} vs ${formatTraceModes(leaderTrace.modes)}`
+    });
+  }
+  rows.push({
+    label: "Final delta",
+    value: formatSignedDelta(trace.averageFinalCount - leaderTrace.averageFinalCount, 1)
+  }, {
+    label: "Vector delta",
+    value: formatSignedDelta(trace.averageVectorCount - leaderTrace.averageVectorCount, 1)
+  }, {
+    label: "Lexical delta",
+    value: formatSignedDelta(trace.averageLexicalCount - leaderTrace.averageLexicalCount, 1)
+  }, {
+    label: "Transform delta",
+    value: formatTraceCountDelta(trace.transformedCases - leaderTrace.transformedCases)
+  });
+  if (stageDelta) {
+    rows.push({ label: "Stage delta", value: stageDelta });
+  }
+  return rows;
+};
+var buildRAGEvaluationHistoryRows = (history) => {
+  if (!history?.latestRun) {
+    return [
+      { label: "History", value: "No persisted benchmark runs yet." }
+    ];
+  }
+  const rows = [
+    { label: "Runs recorded", value: String(history.runs.length) },
+    {
+      label: "Latest",
+      value: `${history.latestRun.label} \xB7 ${formatEvaluationSummary(history.latestRun.response)}`
+    }
+  ];
+  if (history.latestRun.traceSummary) {
+    rows.push({
+      label: "Latest trace",
+      value: `${formatTraceModes(history.latestRun.traceSummary.modes)} \xB7 final ${history.latestRun.traceSummary.averageFinalCount.toFixed(1)} \xB7 vector ${history.latestRun.traceSummary.averageVectorCount.toFixed(1)} \xB7 lexical ${history.latestRun.traceSummary.averageLexicalCount.toFixed(1)}`
+    });
+  }
+  if (history.previousRun) {
+    rows.push({
+      label: "Previous",
+      value: `${history.previousRun.label} \xB7 ${formatEvaluationSummary(history.previousRun.response)}`
+    });
+  }
+  if (!history.diff) {
+    rows.push({
+      label: "History diff",
+      value: "Run the benchmark again to diff regressions over time."
+    });
+    return rows;
+  }
+  rows.push({
+    label: "Passing delta",
+    value: formatSignedDelta(history.diff.summaryDelta.passingRate, 1, "%")
+  }, {
+    label: "Average F1 delta",
+    value: formatSignedDelta(history.diff.summaryDelta.averageF1, 3)
+  }, {
+    label: "Latency delta",
+    value: formatSignedDelta(history.diff.summaryDelta.averageLatencyMs, 1, "ms")
+  }, {
+    label: "Improved",
+    value: formatHistoryCaseLabels(history.diff.improvedCases)
+  }, {
+    label: "Regressed",
+    value: formatHistoryCaseLabels(history.diff.regressedCases)
+  });
+  if (history.diff.traceSummaryDelta) {
+    rows.push({
+      label: "Trace mode shift",
+      value: history.diff.traceSummaryDelta.modesChanged ? "changed" : "stable"
+    }, {
+      label: "Trace final delta",
+      value: formatSignedDelta(history.diff.traceSummaryDelta.averageFinalCount, 1)
+    }, {
+      label: "Trace vector delta",
+      value: formatSignedDelta(history.diff.traceSummaryDelta.averageVectorCount, 1)
+    }, {
+      label: "Trace lexical delta",
+      value: formatSignedDelta(history.diff.traceSummaryDelta.averageLexicalCount, 1)
+    }, {
+      label: "Trace transform delta",
+      value: formatTraceCountDelta(history.diff.traceSummaryDelta.transformedCases)
+    }, {
+      label: "Trace variant delta",
+      value: formatTraceCountDelta(history.diff.traceSummaryDelta.variantCases)
+    });
+    const stageDelta = Object.entries(history.diff.traceSummaryDelta.stageCounts ?? {}).map(([stage, count]) => `${stage} ${formatTraceCountDelta(count)}`).join(", ");
+    if (stageDelta) {
+      rows.push({ label: "Trace stage delta", value: stageDelta });
+    }
+  }
+  if (history.caseTraceSnapshots.length > 0) {
+    const changedCases = history.caseTraceSnapshots.filter((entry) => entry.traceChange === "changed");
+    rows.push({
+      label: "Trace drift cases",
+      value: changedCases.length > 0 ? changedCases.map((entry) => entry.label ?? entry.caseId).slice(0, 4).join(", ") : "none"
+    });
+  }
+  return rows;
+};
 var buildRAGEvaluationRunDiff = ({
   current,
   previous
@@ -2318,6 +2577,10 @@ var loadRAGEvaluationHistory = async ({
   const latestRun = runs[0];
   const previousRun = runs[1];
   return {
+    caseTraceSnapshots: buildEvaluationCaseTraceSnapshots({
+      current: latestRun,
+      previous: previousRun
+    }),
     diff: latestRun && previousRun ? buildRAGEvaluationRunDiff({
       current: latestRun,
       previous: previousRun
@@ -2529,6 +2792,7 @@ var compareRAGRerankers = async ({
     });
     const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
+      caseTraceSnapshots: buildEvaluationCaseTraceSnapshotsFromEvaluated(evaluated),
       label: candidate.label ?? candidate.id,
       providerName: typeof candidate.rerank === "function" ? undefined : candidate.rerank?.providerName,
       response,
@@ -2615,6 +2879,7 @@ var compareRAGRetrievalStrategies = async ({
     });
     const response = buildRAGEvaluationResponse(evaluated.map((entry) => entry.caseResult));
     return {
+      caseTraceSnapshots: buildEvaluationCaseTraceSnapshotsFromEvaluated(evaluated),
       label: candidate.label ?? candidate.id,
       response,
       retrievalId: candidate.id,
@@ -2683,7 +2948,8 @@ var executeDryRunRAGEvaluation = (input, defaultTopK = DEFAULT_TOP_K) => input.c
 var runRAGEvaluationSuite = async ({
   suite,
   evaluate,
-  overrides
+  overrides,
+  artifacts
 }) => {
   const startedAt = Date.now();
   const response = await evaluate({
@@ -2693,6 +2959,7 @@ var runRAGEvaluationSuite = async ({
   });
   const finishedAt = Date.now();
   return {
+    caseTraceSnapshots: artifacts?.caseTraceSnapshots,
     elapsedMs: finishedAt - startedAt,
     finishedAt,
     id: generateId(),
@@ -2700,7 +2967,8 @@ var runRAGEvaluationSuite = async ({
     metadata: suite.metadata,
     response,
     startedAt,
-    suiteId: suite.id
+    suiteId: suite.id,
+    traceSummary: artifacts?.traceSummary
   };
 };
 var summarizeRAGEvaluationCase = ({
@@ -2774,5 +3042,5 @@ export {
   buildRAGAnswerWorkflowState
 };
 
-//# debugId=BCBFBA9A1CF59F3A64756E2164756E21
+//# debugId=EF37DD99ACE1DD0D64756E2164756E21
 //# sourceMappingURL=index.js.map