@absolutejs/absolute 0.19.0-beta.517 → 0.19.0-beta.519

package/dist/types/ai.d.ts

@@ -919,6 +919,71 @@ export type RAGAnswerGroundingEvaluationResponse = {
     totalCases: number;
     passingRate: number;
 };
+export type RAGAnswerGroundingEvaluationRun = {
+    id: string;
+    suiteId: string;
+    label: string;
+    startedAt: number;
+    finishedAt: number;
+    elapsedMs: number;
+    response: RAGAnswerGroundingEvaluationResponse;
+    metadata?: Record<string, unknown>;
+};
+export type RAGAnswerGroundingEvaluationHistoryStore = {
+    saveRun: (run: RAGAnswerGroundingEvaluationRun) => Promise<void> | void;
+    listRuns: (input?: {
+        suiteId?: string;
+        limit?: number;
+    }) => Promise<RAGAnswerGroundingEvaluationRun[]> | RAGAnswerGroundingEvaluationRun[];
+};
+export type RAGAnswerGroundingEvaluationLeaderboardEntry = {
+    runId: string;
+    suiteId: string;
+    label: string;
+    passingRate: number;
+    averageCitationF1: number;
+    averageResolvedCitationRate: number;
+    rank: number;
+    totalCases: number;
+};
+export type RAGAnswerGroundingEvaluationCaseDiff = {
+    caseId: string;
+    label?: string;
+    query?: string;
+    previousStatus?: RAGAnswerGroundingEvaluationCaseResult['status'];
+    currentStatus: RAGAnswerGroundingEvaluationCaseResult['status'];
+    previousCitationF1?: number;
+    currentCitationF1: number;
+    previousMatchedIds: string[];
+    currentMatchedIds: string[];
+    previousMissingIds: string[];
+    currentMissingIds: string[];
+};
+export type RAGAnswerGroundingEvaluationRunDiff = {
+    suiteId: string;
+    currentRunId: string;
+    previousRunId?: string;
+    regressedCases: RAGAnswerGroundingEvaluationCaseDiff[];
+    improvedCases: RAGAnswerGroundingEvaluationCaseDiff[];
+    unchangedCases: RAGAnswerGroundingEvaluationCaseDiff[];
+    summaryDelta: {
+        passingRate: number;
+        averageCitationF1: number;
+        averageResolvedCitationRate: number;
+        passedCases: number;
+        failedCases: number;
+        partialCases: number;
+    };
+};
+export type RAGAnswerGroundingEvaluationHistory = {
+    suiteId: string;
+    suiteLabel?: string;
+    runs: RAGAnswerGroundingEvaluationRun[];
+    leaderboard: RAGAnswerGroundingEvaluationLeaderboardEntry[];
+    latestRun?: RAGAnswerGroundingEvaluationRun;
+    previousRun?: RAGAnswerGroundingEvaluationRun;
+    diff?: RAGAnswerGroundingEvaluationRunDiff;
+};
 export type RAGEvaluationInput = {
     cases: RAGEvaluationCase[];
     topK?: number;

package/dist/vue/ai/index.js

@@ -1716,8 +1716,34 @@ var buildRAGEvaluationLeaderboard = (runs) => {
     totalCases: run.response.totalCases
   }));
 };
+var buildRAGAnswerGroundingEvaluationLeaderboard = (runs) => {
+  const sorted = [...runs].sort((left, right) => {
+    if (right.response.passingRate !== left.response.passingRate) {
+      return right.response.passingRate - left.response.passingRate;
+    }
+    if (right.response.summary.averageCitationF1 !== left.response.summary.averageCitationF1) {
+      return right.response.summary.averageCitationF1 - left.response.summary.averageCitationF1;
+    }
+    if (right.response.summary.averageResolvedCitationRate !== left.response.summary.averageResolvedCitationRate) {
+      return right.response.summary.averageResolvedCitationRate - left.response.summary.averageResolvedCitationRate;
+    }
+    return left.elapsedMs - right.elapsedMs;
+  });
+  return sorted.map((run, index) => ({
+    averageCitationF1: run.response.summary.averageCitationF1,
+    averageResolvedCitationRate: run.response.summary.averageResolvedCitationRate,
+    label: run.label,
+    passingRate: run.response.passingRate,
+    rank: index + 1,
+    runId: run.id,
+    suiteId: run.suiteId,
+    totalCases: run.response.totalCases
+  }));
+};
 var toHistorySortOrder = (left, right) => right.finishedAt - left.finishedAt;
 var normalizeHistoryRuns = (runs) => [...runs].sort(toHistorySortOrder);
+var toGroundingHistorySortOrder = (left, right) => right.finishedAt - left.finishedAt;
+var normalizeGroundingHistoryRuns = (runs) => [...runs].sort(toGroundingHistorySortOrder);
 var buildCaseDiff = (currentCase, previousCase) => ({
   caseId: currentCase.caseId,
   currentF1: currentCase.f1,
@@ -1731,6 +1757,19 @@ var buildCaseDiff = (currentCase, previousCase) => ({
   previousStatus: previousCase?.status,
   query: currentCase.query
 });
+var buildGroundingCaseDiff = (currentCase, previousCase) => ({
+  caseId: currentCase.caseId,
+  currentCitationF1: currentCase.citationF1,
+  currentMatchedIds: currentCase.matchedIds,
+  currentMissingIds: currentCase.missingIds,
+  currentStatus: currentCase.status,
+  label: currentCase.label,
+  previousCitationF1: previousCase?.citationF1,
+  previousMatchedIds: previousCase?.matchedIds ?? [],
+  previousMissingIds: previousCase?.missingIds ?? [],
+  previousStatus: previousCase?.status,
+  query: currentCase.query
+});
 var getStatusRank = (status) => status === "pass" ? 2 : status === "partial" ? 1 : 0;
 var buildRAGEvaluationRunDiff = ({
   current,
@@ -1758,6 +1797,32 @@ var buildRAGEvaluationRunDiff = ({
     unchangedCases
   };
 };
+var buildRAGAnswerGroundingEvaluationRunDiff = ({
+  current,
+  previous
+}) => {
+  const previousCases = new Map((previous?.response.cases ?? []).map((entry) => [entry.caseId, entry]));
+  const diffs = current.response.cases.map((entry) => buildGroundingCaseDiff(entry, previousCases.get(entry.caseId)));
+  const regressedCases = diffs.filter((entry) => getStatusRank(entry.currentStatus) < getStatusRank(entry.previousStatus ?? "fail"));
+  const improvedCases = diffs.filter((entry) => getStatusRank(entry.currentStatus) > getStatusRank(entry.previousStatus ?? "fail"));
+  const unchangedCases = diffs.filter((entry) => getStatusRank(entry.currentStatus) === getStatusRank(entry.previousStatus ?? "fail"));
+  return {
+    currentRunId: current.id,
+    improvedCases,
+    previousRunId: previous?.id,
+    regressedCases,
+    suiteId: current.suiteId,
+    summaryDelta: {
+      averageCitationF1: current.response.summary.averageCitationF1 - (previous?.response.summary.averageCitationF1 ?? 0),
+      averageResolvedCitationRate: current.response.summary.averageResolvedCitationRate - (previous?.response.summary.averageResolvedCitationRate ?? 0),
+      failedCases: current.response.summary.failedCases - (previous?.response.summary.failedCases ?? 0),
+      passedCases: current.response.summary.passedCases - (previous?.response.summary.passedCases ?? 0),
+      passingRate: current.response.passingRate - (previous?.response.passingRate ?? 0),
+      partialCases: current.response.summary.partialCases - (previous?.response.summary.partialCases ?? 0)
+    },
+    unchangedCases
+  };
+};
 var createRAGFileEvaluationHistoryStore = (path) => ({
   listRuns: async ({ limit, suiteId } = {}) => {
     let parsed = [];
@@ -1796,6 +1861,42 @@ var createRAGFileEvaluationHistoryStore = (path) => ({
 `, "utf8");
   }
 });
+var createRAGFileAnswerGroundingEvaluationHistoryStore = (path) => ({
+  async listRuns(input) {
+    try {
+      const raw = await readFile(path, "utf8");
+      const data = JSON.parse(raw);
+      const runs = Array.isArray(data.runs) ? data.runs : [];
+      const filtered = input?.suiteId ? runs.filter((run) => run.suiteId === input.suiteId) : runs;
+      return filtered.sort(toGroundingHistorySortOrder).slice(0, input?.limit ?? DEFAULT_HISTORY_LIMIT);
+    } catch (error) {
+      if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
+        return [];
+      }
+      throw error;
+    }
+  },
+  async saveRun(run) {
+    let runs = [];
+    try {
+      const raw = await readFile(path, "utf8");
+      const data = JSON.parse(raw);
+      runs = Array.isArray(data.runs) ? data.runs : [];
+    } catch (error) {
+      if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") {
+        throw error;
+      }
+    }
+    const nextRuns = normalizeGroundingHistoryRuns([
+      run,
+      ...runs.filter((entry) => entry.id !== run.id)
+    ]);
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, JSON.stringify({
+      runs: nextRuns
+    }, null, 2));
+  }
+});
 var loadRAGEvaluationHistory = async ({
   store,
   suite,
@@ -1817,6 +1918,30 @@ var loadRAGEvaluationHistory = async ({
     suiteLabel: suite.label ?? suite.id
   };
 };
+var loadRAGAnswerGroundingEvaluationHistory = async ({
+  store,
+  suite,
+  limit = DEFAULT_HISTORY_LIMIT
+}) => {
+  const runs = normalizeGroundingHistoryRuns(await Promise.resolve(store.listRuns({
+    limit,
+    suiteId: suite.id
+  })));
+  const latestRun = runs[0];
+  const previousRun = runs[1];
+  return {
+    diff: latestRun && previousRun ? buildRAGAnswerGroundingEvaluationRunDiff({
+      current: latestRun,
+      previous: previousRun
+    }) : undefined,
+    latestRun,
+    leaderboard: buildRAGAnswerGroundingEvaluationLeaderboard(runs),
+    previousRun,
+    runs,
+    suiteId: suite.id,
+    suiteLabel: suite.label ?? suite.id
+  };
+};
 var persistRAGEvaluationSuiteRun = async ({
   store,
   run
@@ -1824,6 +1949,13 @@ var persistRAGEvaluationSuiteRun = async ({
   await Promise.resolve(store.saveRun(run));
   return run;
 };
+var persistRAGAnswerGroundingEvaluationRun = async ({
+  store,
+  run
+}) => {
+  await Promise.resolve(store.saveRun(run));
+  return run;
+};
 var buildRAGEvaluationResponse = (cases) => {
   const totalCases = cases.length;
   const passedCases = cases.filter((entry) => entry.status === "pass").length;
@@ -2840,5 +2972,5 @@ export {
   AIStreamKey
 };
 
-//# debugId=EEFE8418E86C288B64756E2164756E21
+//# debugId=CE780A331BCCEACE64756E2164756E21
 //# sourceMappingURL=index.js.map