@absolutejs/absolute 0.19.0-beta.520 → 0.19.0-beta.521

package/dist/src/ai/index.d.ts

@@ -1,7 +1,7 @@
 export { aiChat } from '../plugins/aiChat';
 export { streamAI } from './streamAI';
 export { ragChat } from './rag';
-export { buildRAGAnswerGroundingEvaluationResponse, buildRAGAnswerGroundingEvaluationLeaderboard, buildRAGAnswerGroundingEvaluationRunDiff, buildRAGContext, buildRAGEvaluationLeaderboard, buildRAGEvaluationResponse, buildRAGEvaluationRunDiff, compareRAGRetrievalStrategies, compareRAGRerankers, createRAGFileAnswerGroundingEvaluationHistoryStore, createRAGBunS3SyncClient, createRAGFileEvaluationHistoryStore, createRAGEvaluationSuite, createRAGDirectorySyncSource, createRAGEmailSyncSource, createRAGGmailEmailSyncClient, createRAGFileSyncStateStore, createRAGGraphEmailSyncClient, createRAGIMAPEmailSyncClient, createRAGStaticEmailSyncClient, createRAGStorageSyncSource, evaluateRAGAnswerGrounding, evaluateRAGAnswerGroundingCase, evaluateRAGCollection, executeDryRunRAGEvaluation, loadRAGAnswerGroundingEvaluationHistory, loadRAGEvaluationHistory, persistRAGAnswerGroundingEvaluationRun, persistRAGEvaluationSuiteRun, runRAGEvaluationSuite, summarizeRAGEvaluationCase, summarizeRAGRerankerComparison, buildRAGCitations, buildRAGCitationReferenceMap, buildRAGLexicalHaystack, buildRAGGroundedAnswer, buildRAGGroundingReferences, buildRAGSourceGroups, buildRAGSourceSummaries, buildRAGUpsertInputFromDirectory, buildRAGUpsertInputFromDocuments, createBuiltinArchiveExpander, createEmailExtractor, createEPUBExtractor, createLegacyDocumentExtractor, createRAGPDFOCRExtractor, createRAGArchiveExpander, createRAGArchiveFileExtractor, createOfficeDocumentExtractor, createPDFFileExtractor, createRAGCollection, createRAGEmbeddingProvider, createRAGFileExtractor, createRAGImageOCRExtractor, createHeuristicRAGReranker, createHeuristicRAGQueryTransform, createRAGHTMXConfig, createRAGMediaFileExtractor, createRAGMediaTranscriber, createRAGSyncManager, createRAGSyncScheduler, createRAGOCRProvider, createRAGUrlSyncSource, createRAGQueryTransform, createRAGReranker, fuseRAGQueryResults, createTextFileExtractor, createRAGHTMXWorkflowRenderConfig, alibabaEmbeddings, anthropicOCR, deepseekEmbeddings, geminiEmbeddings, geminiOCR, getLatestAssistantMessage, getLatestRAGSources, buildRAGTransportProgress, googleEmbeddings, ingestDocuments, ingestRAGDocuments, loadRAGDocumentFile, loadRAGDocumentsFromDirectory, loadRAGDocumentFromURL, loadRAGDocumentsFromURLs, metaEmbeddings, mistralaiEmbeddings, moonshotEmbeddings, ollamaEmbeddings, ollamaOCR, ollamaTranscriber, openaiCompatibleEmbeddings, openaiCompatibleOCR, openaiCompatibleTranscriber, openaiEmbeddings, openaiOCR, openaiTranscriber, prepareRAGDirectoryDocuments, buildRAGUpsertInputFromUploads, loadRAGDocumentUpload, loadRAGDocumentsFromUploads, buildRAGUpsertInputFromURLs, prepareRAGDocument, prepareRAGDocumentFile, prepareRAGDocuments, ragPlugin, resolveRAGQueryTransform, resolveRAGEmbeddingProvider, resolveRAGHybridSearchOptions, resolveRAGReranker, resolveRAGStreamStage, searchDocuments, scoreRAGLexicalMatch, applyRAGReranking, applyRAGQueryTransform, validateRAGEmbeddingDimensions, xaiEmbeddings } from './rag';
+export { buildRAGAnswerGroundingCaseDifficultyLeaderboard, buildRAGAnswerGroundingEvaluationResponse, buildRAGAnswerGroundingEvaluationLeaderboard, buildRAGAnswerGroundingEvaluationRunDiff, buildRAGContext, buildRAGEvaluationLeaderboard, buildRAGEvaluationResponse, buildRAGEvaluationRunDiff, compareRAGRetrievalStrategies, compareRAGRerankers, createRAGFileAnswerGroundingEvaluationHistoryStore, createRAGBunS3SyncClient, createRAGFileEvaluationHistoryStore, createRAGEvaluationSuite, createRAGDirectorySyncSource, createRAGEmailSyncSource, createRAGGmailEmailSyncClient, createRAGFileSyncStateStore, createRAGGraphEmailSyncClient, createRAGIMAPEmailSyncClient, createRAGStaticEmailSyncClient, createRAGStorageSyncSource, evaluateRAGAnswerGrounding, evaluateRAGAnswerGroundingCase, evaluateRAGCollection, executeDryRunRAGEvaluation, loadRAGAnswerGroundingEvaluationHistory, loadRAGEvaluationHistory, persistRAGAnswerGroundingEvaluationRun, persistRAGEvaluationSuiteRun, runRAGEvaluationSuite, summarizeRAGEvaluationCase, summarizeRAGRerankerComparison, buildRAGCitations, buildRAGCitationReferenceMap, buildRAGLexicalHaystack, buildRAGGroundedAnswer, buildRAGGroundingReferences, buildRAGSourceGroups, buildRAGSourceSummaries, buildRAGUpsertInputFromDirectory, buildRAGUpsertInputFromDocuments, createBuiltinArchiveExpander, createEmailExtractor, createEPUBExtractor, createLegacyDocumentExtractor, createRAGPDFOCRExtractor, createRAGArchiveExpander, createRAGArchiveFileExtractor, createOfficeDocumentExtractor, createPDFFileExtractor, createRAGCollection, createRAGEmbeddingProvider, createRAGFileExtractor, createRAGImageOCRExtractor, createHeuristicRAGReranker, createHeuristicRAGQueryTransform, createRAGHTMXConfig, createRAGMediaFileExtractor, createRAGMediaTranscriber, createRAGSyncManager, createRAGSyncScheduler, createRAGOCRProvider, createRAGUrlSyncSource, createRAGQueryTransform, createRAGReranker, fuseRAGQueryResults, createTextFileExtractor, createRAGHTMXWorkflowRenderConfig, alibabaEmbeddings, anthropicOCR, deepseekEmbeddings, geminiEmbeddings, geminiOCR, getLatestAssistantMessage, getLatestRAGSources, buildRAGTransportProgress, googleEmbeddings, ingestDocuments, ingestRAGDocuments, loadRAGDocumentFile, loadRAGDocumentsFromDirectory, loadRAGDocumentFromURL, loadRAGDocumentsFromURLs, metaEmbeddings, mistralaiEmbeddings, moonshotEmbeddings, ollamaEmbeddings, ollamaOCR, ollamaTranscriber, openaiCompatibleEmbeddings, openaiCompatibleOCR, openaiCompatibleTranscriber, openaiEmbeddings, openaiOCR, openaiTranscriber, prepareRAGDirectoryDocuments, buildRAGUpsertInputFromUploads, loadRAGDocumentUpload, loadRAGDocumentsFromUploads, buildRAGUpsertInputFromURLs, prepareRAGDocument, prepareRAGDocumentFile, prepareRAGDocuments, ragPlugin, resolveRAGQueryTransform, resolveRAGEmbeddingProvider, resolveRAGHybridSearchOptions, resolveRAGReranker, resolveRAGStreamStage, searchDocuments, scoreRAGLexicalMatch, applyRAGReranking, applyRAGQueryTransform, validateRAGEmbeddingDimensions, xaiEmbeddings } from './rag';
 export { createInMemoryRAGStore } from './rag/adapters/inMemory';
 export { createSQLiteRAGStore } from './rag/adapters/sqlite';
 export { resolveAbsoluteSQLiteVec, resolveAbsoluteSQLiteVecExtensionPath } from './rag/resolveAbsoluteSQLiteVec';

package/dist/src/ai/rag/index.d.ts

@@ -10,7 +10,7 @@ export { createRAGGmailEmailSyncClient, createRAGGraphEmailSyncClient, createRAG
 export { buildRAGUpsertInputFromDirectory, buildRAGUpsertInputFromDocuments, buildRAGUpsertInputFromUploads, createBuiltinArchiveExpander, createEmailExtractor, createEPUBExtractor, createLegacyDocumentExtractor, createRAGPDFOCRExtractor, createRAGArchiveExpander, createRAGArchiveFileExtractor, createOfficeDocumentExtractor, createPDFFileExtractor, createRAGFileExtractor, createRAGImageOCRExtractor, createRAGMediaFileExtractor, createRAGMediaTranscriber, createRAGOCRProvider, createTextFileExtractor, loadRAGDocumentFromURL, loadRAGDocumentUpload, loadRAGDocumentsFromUploads, buildRAGUpsertInputFromURLs, loadRAGDocumentFile, loadRAGDocumentsFromDirectory, loadRAGDocumentsFromURLs, prepareRAGDirectoryDocuments, prepareRAGDocument, prepareRAGDocumentFile, prepareRAGDocuments } from './ingestion';
 export { buildRAGAnswerWorkflowState, buildRAGCitations, buildRAGCitationReferenceMap, buildRAGGroundedAnswer, buildRAGGroundingReferences, buildRAGSourceGroups, buildRAGSourceSummaries, buildRAGStreamProgress as buildRAGTransportProgress, getLatestAssistantMessage, getLatestRAGSources, resolveRAGStreamStage } from './presentation';
 export { buildRAGContext } from './types';
-export { buildRAGAnswerGroundingEvaluationResponse, buildRAGAnswerGroundingEvaluationLeaderboard, buildRAGAnswerGroundingEvaluationRunDiff, buildRAGEvaluationLeaderboard, buildRAGEvaluationResponse, buildRAGEvaluationRunDiff, compareRAGRetrievalStrategies, compareRAGRerankers, createRAGFileAnswerGroundingEvaluationHistoryStore, createRAGFileEvaluationHistoryStore, createRAGEvaluationSuite, evaluateRAGAnswerGrounding, evaluateRAGAnswerGroundingCase, evaluateRAGCollection, executeDryRunRAGEvaluation, loadRAGAnswerGroundingEvaluationHistory, loadRAGEvaluationHistory, persistRAGAnswerGroundingEvaluationRun, persistRAGEvaluationSuiteRun, runRAGEvaluationSuite, summarizeRAGEvaluationCase, summarizeRAGRerankerComparison } from './quality';
+export { buildRAGAnswerGroundingCaseDifficultyLeaderboard, buildRAGAnswerGroundingEvaluationResponse, buildRAGAnswerGroundingEvaluationLeaderboard, buildRAGAnswerGroundingEvaluationRunDiff, buildRAGEvaluationLeaderboard, buildRAGEvaluationResponse, buildRAGEvaluationRunDiff, compareRAGRetrievalStrategies, compareRAGRerankers, createRAGFileAnswerGroundingEvaluationHistoryStore, createRAGFileEvaluationHistoryStore, createRAGEvaluationSuite, evaluateRAGAnswerGrounding, evaluateRAGAnswerGroundingCase, evaluateRAGCollection, executeDryRunRAGEvaluation, loadRAGAnswerGroundingEvaluationHistory, loadRAGEvaluationHistory, persistRAGAnswerGroundingEvaluationRun, persistRAGEvaluationSuiteRun, runRAGEvaluationSuite, summarizeRAGEvaluationCase, summarizeRAGRerankerComparison } from './quality';
 export { createRAGBunS3SyncClient, createRAGDirectorySyncSource, createRAGEmailSyncSource, createRAGFileSyncStateStore, createRAGStaticEmailSyncClient, createRAGStorageSyncSource, createRAGSyncManager, createRAGSyncScheduler, createRAGUrlSyncSource } from './sync';
 export type { RAGStreamProgress as RAGTransportProgress, RAGStreamProgressState as RAGTransportProgressState } from './presentation';
 export { createRAGCollection, ingestDocuments, ingestRAGDocuments, searchDocuments } from './collection';

package/dist/src/ai/rag/quality.d.ts

@@ -1,6 +1,10 @@
-import type { RAGAnswerGroundingEvaluationCase, RAGAnswerGroundingEvaluationCaseResult, RAGAnswerGroundingEvaluationHistory, RAGAnswerGroundingEvaluationLeaderboardEntry, RAGAnswerGroundingEvaluationHistoryStore, RAGAnswerGroundingEvaluationInput, RAGAnswerGroundingEvaluationResponse, RAGAnswerGroundingEvaluationRun, RAGAnswerGroundingEvaluationRunDiff, RAGCollection, RAGEvaluationCase, RAGEvaluationCaseResult, RAGEvaluationHistory, RAGEvaluationHistoryStore, RAGEvaluationInput, RAGEvaluationLeaderboardEntry, RAGEvaluationResponse, RAGEvaluationRunDiff, RAGEvaluationSuite, RAGEvaluationSuiteRun, RAGRetrievalCandidate, RAGRetrievalComparison, RAGRetrievalComparisonEntry, RAGRetrievalComparisonSummary, RAGRerankerCandidate, RAGRerankerComparison, RAGRerankerComparisonEntry, RAGRerankerComparisonSummary, RAGRerankerProviderLike } from '../../../types/ai';
+import type { RAGAnswerGroundingEvaluationCase, RAGAnswerGroundingEvaluationCaseDifficultyEntry, RAGAnswerGroundingEvaluationCaseResult, RAGAnswerGroundingEvaluationHistory, RAGAnswerGroundingEvaluationLeaderboardEntry, RAGAnswerGroundingEvaluationHistoryStore, RAGAnswerGroundingEvaluationInput, RAGAnswerGroundingEvaluationResponse, RAGAnswerGroundingEvaluationRun, RAGAnswerGroundingEvaluationRunDiff, RAGCollection, RAGEvaluationCase, RAGEvaluationCaseResult, RAGEvaluationHistory, RAGEvaluationHistoryStore, RAGEvaluationInput, RAGEvaluationLeaderboardEntry, RAGEvaluationResponse, RAGEvaluationRunDiff, RAGEvaluationSuite, RAGEvaluationSuiteRun, RAGRetrievalCandidate, RAGRetrievalComparison, RAGRetrievalComparisonEntry, RAGRetrievalComparisonSummary, RAGRerankerCandidate, RAGRerankerComparison, RAGRerankerComparisonEntry, RAGRerankerComparisonSummary, RAGRerankerProviderLike } from '../../../types/ai';
 export declare const buildRAGEvaluationLeaderboard: (runs: RAGEvaluationSuiteRun[]) => RAGEvaluationLeaderboardEntry[];
 export declare const buildRAGAnswerGroundingEvaluationLeaderboard: (runs: RAGAnswerGroundingEvaluationRun[]) => RAGAnswerGroundingEvaluationLeaderboardEntry[];
+export declare const buildRAGAnswerGroundingCaseDifficultyLeaderboard: (entries: Array<{
+    label: string;
+    response: RAGAnswerGroundingEvaluationResponse;
+}>) => RAGAnswerGroundingEvaluationCaseDifficultyEntry[];
 export declare const buildRAGEvaluationRunDiff: ({ current, previous }: {
     current: RAGEvaluationSuiteRun;
     previous?: RAGEvaluationSuiteRun;

package/dist/svelte/ai/index.js

@@ -1743,6 +1743,72 @@ var buildRAGAnswerGroundingEvaluationLeaderboard = (runs) => {
     totalCases: run.response.totalCases
   }));
 };
+var buildRAGAnswerGroundingCaseDifficultyLeaderboard = (entries) => {
+  const grouped = new Map;
+  for (const entry of entries) {
+    for (const result of entry.response.cases) {
+      const current = grouped.get(result.caseId) ?? {
+        caseId: result.caseId,
+        failCount: 0,
+        groundedCount: 0,
+        label: result.label,
+        passCount: 0,
+        partialCount: 0,
+        query: result.query,
+        totalCitationF1: 0,
+        totalEvaluations: 0,
+        totalResolvedCitationRate: 0
+      };
+      current.label ??= result.label;
+      current.query ??= result.query;
+      current.totalEvaluations += 1;
+      current.totalCitationF1 += result.citationF1;
+      current.totalResolvedCitationRate += result.resolvedCitationRate;
+      if (result.status === "pass") {
+        current.passCount += 1;
+      } else if (result.status === "partial") {
+        current.partialCount += 1;
+      } else {
+        current.failCount += 1;
+      }
+      if (result.coverage === "grounded") {
+        current.groundedCount += 1;
+      }
+      grouped.set(result.caseId, current);
+    }
+  }
+  const ranked = Array.from(grouped.values()).sort((left, right) => {
+    const leftPassRate = left.passCount / left.totalEvaluations;
+    const rightPassRate = right.passCount / right.totalEvaluations;
+    if (leftPassRate !== rightPassRate) {
+      return leftPassRate - rightPassRate;
+    }
+    const leftCitationF1 = left.totalCitationF1 / left.totalEvaluations;
+    const rightCitationF1 = right.totalCitationF1 / right.totalEvaluations;
+    if (leftCitationF1 !== rightCitationF1) {
+      return leftCitationF1 - rightCitationF1;
+    }
+    const leftResolved = left.totalResolvedCitationRate / left.totalEvaluations;
+    const rightResolved = right.totalResolvedCitationRate / right.totalEvaluations;
+    if (leftResolved !== rightResolved) {
+      return leftResolved - rightResolved;
+    }
+    return left.caseId.localeCompare(right.caseId);
+  });
+  return ranked.map((entry, index) => ({
+    averageCitationF1: entry.totalCitationF1 / entry.totalEvaluations,
+    averageResolvedCitationRate: entry.totalResolvedCitationRate / entry.totalEvaluations,
+    caseId: entry.caseId,
+    failRate: entry.failCount / entry.totalEvaluations * 100,
+    groundedRate: entry.groundedCount / entry.totalEvaluations * 100,
+    label: entry.label,
+    passRate: entry.passCount / entry.totalEvaluations * 100,
+    partialRate: entry.partialCount / entry.totalEvaluations * 100,
+    query: entry.query,
+    rank: index + 1,
+    totalEvaluations: entry.totalEvaluations
+  }));
+};
 var toHistorySortOrder = (left, right) => right.finishedAt - left.finishedAt;
 var normalizeHistoryRuns = (runs) => [...runs].sort(toHistorySortOrder);
 var toGroundingHistorySortOrder = (left, right) => right.finishedAt - left.finishedAt;
@@ -3025,5 +3091,5 @@ export {
   createAIStream
 };
 
-//# debugId=AD2F134AA387AD8E64756E2164756E21
+//# debugId=C9E7C122AD802CB564756E2164756E21
 //# sourceMappingURL=index.js.map