@absolutejs/absolute 0.19.0-beta.623 → 0.19.0-beta.625
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/client/index.js +8751 -6935
- package/dist/ai/client/index.js.map +5 -5
- package/dist/ai/client/ui.js +6402 -42
- package/dist/ai/client/ui.js.map +7 -4
- package/dist/ai/index.js +12994 -8132
- package/dist/ai/index.js.map +15 -13
- package/dist/ai/rag/quality.js +4194 -37
- package/dist/ai/rag/quality.js.map +6 -5
- package/dist/ai/rag/ui.js +6407 -42
- package/dist/ai/rag/ui.js.map +7 -4
- package/dist/ai-client/angular/ai/index.js +401 -4
- package/dist/ai-client/react/ai/index.js +537 -511
- package/dist/ai-client/vue/ai/index.js +743 -720
- package/dist/angular/ai/index.js +9400 -3140
- package/dist/angular/ai/index.js.map +7 -6
- package/dist/react/ai/index.js +8187 -6372
- package/dist/react/ai/index.js.map +5 -5
- package/dist/src/ai/index.d.ts +2 -2
- package/dist/src/ai/rag/adapters/filtering.d.ts +2 -0
- package/dist/src/ai/rag/adapters/queryPlanning.d.ts +11 -0
- package/dist/src/ai/rag/index.d.ts +4 -4
- package/dist/src/ai/rag/ingestion.d.ts +3 -0
- package/dist/src/ai/rag/presentation.d.ts +6 -1
- package/dist/src/ai/rag/quality.d.ts +95 -2
- package/dist/src/ai/rag/sync.d.ts +20 -1
- package/dist/src/ai/rag/types.d.ts +1 -1
- package/dist/src/ai/rag/ui.d.ts +1 -1
- package/dist/src/react/ai/useRAG.d.ts +1 -1
- package/dist/src/react/ai/useRAGIndexAdmin.d.ts +2 -2
- package/dist/src/vue/ai/useRAG.d.ts +408 -0
- package/dist/src/vue/ai/useRAGEvaluate.d.ts +84 -0
- package/dist/src/vue/ai/useRAGIndexAdmin.d.ts +222 -0
- package/dist/src/vue/ai/useRAGOps.d.ts +102 -0
- package/dist/src/vue/components/Image.d.ts +74 -0
- package/dist/src/vue/components/index.d.ts +1 -1
- package/dist/src/vue/index.d.ts +1 -1
- package/dist/svelte/ai/index.js +8186 -6371
- package/dist/svelte/ai/index.js.map +4 -4
- package/dist/types/ai.d.ts +389 -1
- package/dist/vue/ai/index.js +8186 -6371
- package/dist/vue/ai/index.js.map +4 -4
- package/dist/vue/components/Image.js +253 -0
- package/dist/vue/components/Image.js.map +11 -0
- package/dist/vue/components/index.js +181 -16
- package/dist/vue/components/index.js.map +5 -3
- package/dist/vue/index.js +195 -30
- package/dist/vue/index.js.map +6 -4
- package/package.json +11 -11
- package/dist/Image-0pe96k20.vue +0 -186
- package/dist/vue/components/Image.vue +0 -186
- package/dist/vue/components/Image.vue.d.ts +0 -4
package/dist/ai/rag/quality.js
CHANGED
|
@@ -653,10 +653,3046 @@ var buildRAGGroundingReferences = (sources) => {
|
|
|
653
653
|
});
|
|
654
654
|
};
|
|
655
655
|
|
|
656
|
+
// src/ai/rag/presentation.ts
|
|
657
|
+
var buildSourceGroupKey = (source) => source.source ?? source.title ?? source.chunkId;
|
|
658
|
+
var buildSourceLabel2 = (source) => source.source ?? source.title ?? source.chunkId;
|
|
659
|
+
var getContextNumber2 = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
660
|
+
var getContextString2 = (value) => typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
|
|
661
|
+
var isRAGRetrievalTrace = (value) => {
|
|
662
|
+
if (!value || typeof value !== "object") {
|
|
663
|
+
return false;
|
|
664
|
+
}
|
|
665
|
+
const candidate = value;
|
|
666
|
+
return typeof candidate.query === "string" && typeof candidate.transformedQuery === "string" && Array.isArray(candidate.variantQueries) && Array.isArray(candidate.steps);
|
|
667
|
+
};
|
|
668
|
+
var formatTimestampLabel2 = (value) => {
|
|
669
|
+
const timestamp = typeof value === "number" && Number.isFinite(value) ? value : typeof value === "string" ? Date.parse(value) : Number.NaN;
|
|
670
|
+
if (!Number.isFinite(timestamp)) {
|
|
671
|
+
return;
|
|
672
|
+
}
|
|
673
|
+
return new Date(timestamp).toLocaleString("en-US", {
|
|
674
|
+
dateStyle: "medium",
|
|
675
|
+
timeStyle: "short"
|
|
676
|
+
});
|
|
677
|
+
};
|
|
678
|
+
var formatRAGTraceValue = (value) => {
|
|
679
|
+
if (typeof value === "string")
|
|
680
|
+
return value;
|
|
681
|
+
if (typeof value === "number" || typeof value === "boolean") {
|
|
682
|
+
return String(value);
|
|
683
|
+
}
|
|
684
|
+
if (Array.isArray(value)) {
|
|
685
|
+
return value.join(", ");
|
|
686
|
+
}
|
|
687
|
+
if (value && typeof value === "object") {
|
|
688
|
+
return JSON.stringify(value);
|
|
689
|
+
}
|
|
690
|
+
return "n/a";
|
|
691
|
+
};
|
|
692
|
+
var formatLeadContinuityCue = (value) => {
|
|
693
|
+
const cue = getContextString2(value);
|
|
694
|
+
switch (cue) {
|
|
695
|
+
case "immediate_follow_up":
|
|
696
|
+
return "Immediate follow-up";
|
|
697
|
+
case "near_follow_up":
|
|
698
|
+
return "Near follow-up";
|
|
699
|
+
case "close_follow_up":
|
|
700
|
+
return "Close follow-up";
|
|
701
|
+
case "delayed_follow_up":
|
|
702
|
+
return "Delayed follow-up";
|
|
703
|
+
case "immediate_prior":
|
|
704
|
+
return "Immediate prior segment";
|
|
705
|
+
case "near_prior":
|
|
706
|
+
return "Near prior segment";
|
|
707
|
+
case "close_prior":
|
|
708
|
+
return "Close prior segment";
|
|
709
|
+
case "delayed_prior":
|
|
710
|
+
return "Delayed prior segment";
|
|
711
|
+
default:
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
};
|
|
715
|
+
var formatLeadSpeakerAttributionCue = (value) => {
|
|
716
|
+
const cue = getContextString2(value);
|
|
717
|
+
switch (cue) {
|
|
718
|
+
case "quoted_match":
|
|
719
|
+
return "Quoted speaker match";
|
|
720
|
+
default:
|
|
721
|
+
return;
|
|
722
|
+
}
|
|
723
|
+
};
|
|
724
|
+
var formatLeadChannelAttributionCue = (value) => {
|
|
725
|
+
const cue = getContextString2(value);
|
|
726
|
+
switch (cue) {
|
|
727
|
+
case "quoted_match":
|
|
728
|
+
return "Quoted channel match";
|
|
729
|
+
default:
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
};
|
|
733
|
+
var formatLeadMediaCueSummary = (input) => {
|
|
734
|
+
const parts = [
|
|
735
|
+
input?.leadSpeakerCue ? `speaker ${input.leadSpeakerCue}` : undefined,
|
|
736
|
+
input?.leadSpeakerAttributionCue === "quoted_match" ? "quoted speaker match" : undefined,
|
|
737
|
+
input?.leadChannelCue ? `channel ${input.leadChannelCue}` : undefined,
|
|
738
|
+
input?.leadChannelAttributionCue === "quoted_match" ? "quoted channel match" : undefined,
|
|
739
|
+
formatLeadContinuityCue(input?.leadContinuityCue)
|
|
740
|
+
].filter((value) => typeof value === "string" && value.length > 0);
|
|
741
|
+
return parts.length > 0 ? parts.join(" \xB7 ") : "none";
|
|
742
|
+
};
|
|
743
|
+
var formatRAGTraceMetadataRow = (key, value) => ({
|
|
744
|
+
label: key === "sqliteQueryMode" ? "SQLite query mode" : key === "sqliteQueryPushdownApplied" ? "SQLite pushdown applied" : key === "sqliteQueryPushdownClauseCount" ? "SQLite pushdown clauses" : key === "sqliteQueryFilteredCandidates" ? "SQLite filtered candidates" : key === "sqliteQueryInitialSearchK" ? "SQLite initial searchK" : key === "sqliteQueryFinalSearchK" ? "SQLite final searchK" : key === "sqliteQueryBackfillCount" ? "SQLite backfill count" : key === "leadSpeakerCue" ? "Lead speaker cue" : key === "leadSpeakerAttributionCue" ? "Lead speaker attribution" : key === "leadChannelAttributionCue" ? "Lead channel attribution" : key === "leadChannelCue" ? "Lead channel cue" : key === "leadContinuityCue" ? "Lead continuity cue" : key,
|
|
745
|
+
value: key === "sourceAwareChunkReason" ? formatSourceAwareChunkReason(value) ?? formatRAGTraceValue(value) : key === "leadSpeakerAttributionCue" ? formatLeadSpeakerAttributionCue(value) ?? formatRAGTraceValue(value) : key === "leadChannelAttributionCue" ? formatLeadChannelAttributionCue(value) ?? formatRAGTraceValue(value) : key === "leadContinuityCue" ? formatLeadContinuityCue(value) ?? formatRAGTraceValue(value) : formatRAGTraceValue(value)
|
|
746
|
+
});
|
|
747
|
+
var buildRAGRetrievalTracePresentation = (trace) => {
|
|
748
|
+
if (!trace) {
|
|
749
|
+
return {
|
|
750
|
+
details: [],
|
|
751
|
+
stats: [],
|
|
752
|
+
steps: []
|
|
753
|
+
};
|
|
754
|
+
}
|
|
755
|
+
const stats = [
|
|
756
|
+
{ label: "Mode", value: trace.mode },
|
|
757
|
+
{ label: "Final Results", value: String(trace.resultCounts.final) },
|
|
758
|
+
{
|
|
759
|
+
label: "Vector Candidates",
|
|
760
|
+
value: String(trace.resultCounts.vector)
|
|
761
|
+
},
|
|
762
|
+
{
|
|
763
|
+
label: "Lexical Candidates",
|
|
764
|
+
value: String(trace.resultCounts.lexical)
|
|
765
|
+
}
|
|
766
|
+
];
|
|
767
|
+
const details = [
|
|
768
|
+
{ label: "Transformed query", value: trace.transformedQuery },
|
|
769
|
+
...trace.queryTransformLabel || trace.queryTransformProvider ? [
|
|
770
|
+
{
|
|
771
|
+
label: "Query transform",
|
|
772
|
+
value: trace.queryTransformLabel ?? trace.queryTransformProvider ?? "configured"
|
|
773
|
+
},
|
|
774
|
+
...trace.queryTransformReason ? [
|
|
775
|
+
{
|
|
776
|
+
label: "Query transform reason",
|
|
777
|
+
value: trace.queryTransformReason
|
|
778
|
+
}
|
|
779
|
+
] : []
|
|
780
|
+
] : [],
|
|
781
|
+
{
|
|
782
|
+
label: "Variant queries",
|
|
783
|
+
value: trace.variantQueries.length > 0 ? trace.variantQueries.join(" \xB7 ") : "none"
|
|
784
|
+
},
|
|
785
|
+
...trace.requestedMode && trace.requestedMode !== trace.mode ? [{ label: "Requested mode", value: trace.requestedMode }] : [],
|
|
786
|
+
...trace.routingLabel || trace.routingProvider ? [
|
|
787
|
+
{
|
|
788
|
+
label: "Routing decision",
|
|
789
|
+
value: trace.routingLabel ?? trace.routingProvider ?? "configured"
|
|
790
|
+
},
|
|
791
|
+
...trace.routingReason ? [
|
|
792
|
+
{
|
|
793
|
+
label: "Routing reason",
|
|
794
|
+
value: trace.routingReason
|
|
795
|
+
}
|
|
796
|
+
] : []
|
|
797
|
+
] : [],
|
|
798
|
+
{ label: "Candidate topK", value: String(trace.candidateTopK) },
|
|
799
|
+
{ label: "Lexical topK", value: String(trace.lexicalTopK) }
|
|
800
|
+
];
|
|
801
|
+
const steps = trace.steps.map((step) => ({
|
|
802
|
+
count: step.count,
|
|
803
|
+
label: step.label,
|
|
804
|
+
rows: [
|
|
805
|
+
{ label: "stage", value: step.stage },
|
|
806
|
+
...typeof step.count === "number" ? [{ label: "count", value: String(step.count) }] : [],
|
|
807
|
+
...typeof step.durationMs === "number" ? [{ label: "durationMs", value: String(step.durationMs) }] : [],
|
|
808
|
+
...Object.entries(step.metadata ?? {}).map(([key, value]) => formatRAGTraceMetadataRow(key, value))
|
|
809
|
+
],
|
|
810
|
+
stage: step.stage
|
|
811
|
+
}));
|
|
812
|
+
return {
|
|
813
|
+
details,
|
|
814
|
+
stats,
|
|
815
|
+
steps
|
|
816
|
+
};
|
|
817
|
+
};
|
|
818
|
+
var formatCompactList = (values) => values && values.length > 0 ? values.join(", ") : "none";
|
|
819
|
+
var formatCoverageMap = (entries) => {
|
|
820
|
+
if (!entries) {
|
|
821
|
+
return "none";
|
|
822
|
+
}
|
|
823
|
+
const values = Object.entries(entries);
|
|
824
|
+
return values.length > 0 ? values.map(([key, value]) => `${key} ${value}`).join(" \xB7 ") : "none";
|
|
825
|
+
};
|
|
826
|
+
var formatDurationLabel = (value) => {
|
|
827
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
828
|
+
return "n/a";
|
|
829
|
+
}
|
|
830
|
+
if (value < 1000) {
|
|
831
|
+
return `${value}ms`;
|
|
832
|
+
}
|
|
833
|
+
if (value < 60000) {
|
|
834
|
+
return `${(value / 1000).toFixed(value >= 1e4 ? 0 : 1)}s`;
|
|
835
|
+
}
|
|
836
|
+
if (value < 3600000) {
|
|
837
|
+
return `${(value / 60000).toFixed(value >= 600000 ? 0 : 1)}m`;
|
|
838
|
+
}
|
|
839
|
+
return `${(value / 3600000).toFixed(value >= 36000000 ? 0 : 1)}h`;
|
|
840
|
+
};
|
|
841
|
+
var formatDateLabel = (value) => typeof value === "number" && Number.isFinite(value) ? new Date(value).toLocaleString("en-US") : "n/a";
|
|
842
|
+
var formatAgeLabel = (value) => {
|
|
843
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
844
|
+
return "n/a";
|
|
845
|
+
}
|
|
846
|
+
if (value < 1000) {
|
|
847
|
+
return `${Math.round(value)}ms`;
|
|
848
|
+
}
|
|
849
|
+
if (value < 60000) {
|
|
850
|
+
return `${(value / 1000).toFixed(value >= 1e4 ? 0 : 1)}s`;
|
|
851
|
+
}
|
|
852
|
+
if (value < 3600000) {
|
|
853
|
+
return `${(value / 60000).toFixed(value >= 600000 ? 0 : 1)}m`;
|
|
854
|
+
}
|
|
855
|
+
if (value < 86400000) {
|
|
856
|
+
return `${(value / 3600000).toFixed(value >= 36000000 ? 0 : 1)}h`;
|
|
857
|
+
}
|
|
858
|
+
return `${(value / 86400000).toFixed(value >= 864000000 ? 0 : 1)}d`;
|
|
859
|
+
};
|
|
860
|
+
var formatSourceAwareChunkReason = (value) => {
|
|
861
|
+
const reason = getContextString2(value);
|
|
862
|
+
if (reason === "section_boundary") {
|
|
863
|
+
return "Chunk boundary section";
|
|
864
|
+
}
|
|
865
|
+
if (reason === "size_limit") {
|
|
866
|
+
return "Chunk boundary size limit";
|
|
867
|
+
}
|
|
868
|
+
if (reason === "source_native_unit") {
|
|
869
|
+
return "Chunk boundary source-native unit";
|
|
870
|
+
}
|
|
871
|
+
return;
|
|
872
|
+
};
|
|
873
|
+
var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
874
|
+
if (!metadata) {
|
|
875
|
+
return;
|
|
876
|
+
}
|
|
877
|
+
const sectionKind = getContextString2(metadata.sectionKind);
|
|
878
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
879
|
+
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
880
|
+
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
881
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
882
|
+
const sheetName = getContextString2(metadata.sheetName);
|
|
883
|
+
const spreadsheetTableLabel = formatSpreadsheetTableLabel(getContextNumber2(metadata.spreadsheetTableIndex), getContextNumber2(metadata.spreadsheetTableCount));
|
|
884
|
+
const slideTitle = getContextString2(metadata.slideTitle);
|
|
885
|
+
const slideNumber = getContextNumber2(metadata.slideNumber) ?? (typeof metadata.slideIndex === "number" ? metadata.slideIndex + 1 : undefined);
|
|
886
|
+
if (sectionPath.length > 0 && (sectionKind === "markdown_heading" || sectionKind === "html_heading" || sectionKind === "office_heading" || sectionKind === undefined)) {
|
|
887
|
+
return `Source-aware section ${sectionPath.join(" > ")}`;
|
|
888
|
+
}
|
|
889
|
+
if (sectionKind === "pdf_block") {
|
|
890
|
+
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
891
|
+
return `Source-aware PDF table block ${sectionTitle}`;
|
|
892
|
+
}
|
|
893
|
+
if (sectionTitle) {
|
|
894
|
+
return `Source-aware PDF block ${sectionTitle}`;
|
|
895
|
+
}
|
|
896
|
+
return "Source-aware PDF block";
|
|
897
|
+
}
|
|
898
|
+
if (sectionKind === "office_block") {
|
|
899
|
+
if (officeBlockKind && sectionTitle) {
|
|
900
|
+
return `Source-aware office ${officeBlockKind} block ${sectionTitle}`;
|
|
901
|
+
}
|
|
902
|
+
if (sectionTitle) {
|
|
903
|
+
return `Source-aware office block ${sectionTitle}`;
|
|
904
|
+
}
|
|
905
|
+
return "Source-aware office block";
|
|
906
|
+
}
|
|
907
|
+
if (sectionKind === "spreadsheet_rows" || sectionKind === undefined && (sheetName || spreadsheetTableLabel || getContextNumber2(metadata.spreadsheetRowStart) !== undefined || getContextNumber2(metadata.spreadsheetRowEnd) !== undefined)) {
|
|
908
|
+
if (sheetName && spreadsheetTableLabel) {
|
|
909
|
+
return `Source-aware spreadsheet ${sheetName} ${spreadsheetTableLabel}`;
|
|
910
|
+
}
|
|
911
|
+
if (sheetName) {
|
|
912
|
+
return `Source-aware spreadsheet ${sheetName}`;
|
|
913
|
+
}
|
|
914
|
+
return "Source-aware spreadsheet";
|
|
915
|
+
}
|
|
916
|
+
if (sectionKind === "presentation_slide") {
|
|
917
|
+
if (slideNumber && slideTitle) {
|
|
918
|
+
return `Source-aware slide ${slideNumber} ${slideTitle}`;
|
|
919
|
+
}
|
|
920
|
+
if (slideTitle) {
|
|
921
|
+
return `Source-aware slide ${slideTitle}`;
|
|
922
|
+
}
|
|
923
|
+
if (slideNumber) {
|
|
924
|
+
return `Source-aware slide ${slideNumber}`;
|
|
925
|
+
}
|
|
926
|
+
return "Source-aware slide";
|
|
927
|
+
}
|
|
928
|
+
return;
|
|
929
|
+
};
|
|
930
|
+
var buildSyncOverviewLatestRow = (sources) => {
|
|
931
|
+
const latest = [...sources].filter((record) => typeof record.lastSuccessfulSyncAt === "number" || typeof record.lastSyncedAt === "number").sort((left, right) => (right.lastSuccessfulSyncAt ?? right.lastSyncedAt ?? 0) - (left.lastSuccessfulSyncAt ?? left.lastSyncedAt ?? 0))[0];
|
|
932
|
+
if (!latest) {
|
|
933
|
+
return {
|
|
934
|
+
label: "Latest sync",
|
|
935
|
+
value: "No completed run yet"
|
|
936
|
+
};
|
|
937
|
+
}
|
|
938
|
+
return {
|
|
939
|
+
label: "Latest sync",
|
|
940
|
+
value: [
|
|
941
|
+
latest.label,
|
|
942
|
+
typeof latest.documentCount === "number" ? `${latest.documentCount} docs` : "",
|
|
943
|
+
typeof latest.chunkCount === "number" ? `${latest.chunkCount} chunks` : "",
|
|
944
|
+
typeof latest.lastSyncDurationMs === "number" ? formatDurationLabel(latest.lastSyncDurationMs) : "",
|
|
945
|
+
typeof latest.lastSuccessfulSyncAt === "number" ? formatDateLabel(latest.lastSuccessfulSyncAt) : typeof latest.lastSyncedAt === "number" ? formatDateLabel(latest.lastSyncedAt) : ""
|
|
946
|
+
].filter(Boolean).join(" \xB7 ")
|
|
947
|
+
};
|
|
948
|
+
};
|
|
949
|
+
var formatAdminActionLabel = (value) => value.replaceAll("_", " ");
|
|
950
|
+
var buildAdminTimingLabel = (input) => typeof input.elapsedMs === "number" ? formatDurationLabel(input.elapsedMs) : formatDateLabel(input.startedAt);
|
|
951
|
+
var buildRAGAdminJobPresentation = (job) => ({
|
|
952
|
+
id: job.id,
|
|
953
|
+
action: job.action,
|
|
954
|
+
status: job.status,
|
|
955
|
+
summary: [
|
|
956
|
+
job.status.toUpperCase(),
|
|
957
|
+
formatAdminActionLabel(job.action),
|
|
958
|
+
job.target,
|
|
959
|
+
buildAdminTimingLabel(job)
|
|
960
|
+
].filter(Boolean).join(" \xB7 "),
|
|
961
|
+
rows: [
|
|
962
|
+
{ label: "Action", value: formatAdminActionLabel(job.action) },
|
|
963
|
+
{ label: "Status", value: job.status },
|
|
964
|
+
...job.target ? [{ label: "Target", value: job.target }] : [],
|
|
965
|
+
{ label: "Timing", value: buildAdminTimingLabel(job) },
|
|
966
|
+
...job.error ? [{ label: "Error", value: job.error }] : []
|
|
967
|
+
]
|
|
968
|
+
});
|
|
969
|
+
var buildRAGAdminJobPresentations = (jobs) => (jobs ?? []).slice(0, 3).map(buildRAGAdminJobPresentation);
|
|
970
|
+
var buildRAGAdminActionPresentation = (action) => ({
|
|
971
|
+
id: action.id,
|
|
972
|
+
action: action.action,
|
|
973
|
+
status: action.status,
|
|
974
|
+
summary: [
|
|
975
|
+
action.status.toUpperCase(),
|
|
976
|
+
formatAdminActionLabel(action.action),
|
|
977
|
+
action.documentId ?? action.target,
|
|
978
|
+
buildAdminTimingLabel(action)
|
|
979
|
+
].filter(Boolean).join(" \xB7 "),
|
|
980
|
+
rows: [
|
|
981
|
+
{ label: "Action", value: formatAdminActionLabel(action.action) },
|
|
982
|
+
{ label: "Status", value: action.status },
|
|
983
|
+
...action.documentId ? [{ label: "Document", value: action.documentId }] : action.target ? [{ label: "Target", value: action.target }] : [],
|
|
984
|
+
{ label: "Timing", value: buildAdminTimingLabel(action) },
|
|
985
|
+
...action.error ? [{ label: "Error", value: action.error }] : []
|
|
986
|
+
]
|
|
987
|
+
});
|
|
988
|
+
var buildRAGAdminActionPresentations = (actions) => (actions ?? []).slice(0, 3).map(buildRAGAdminActionPresentation);
|
|
989
|
+
var buildRAGSyncSourceRunPresentations = (source) => {
|
|
990
|
+
if (!Array.isArray(source.metadata?.recentRuns)) {
|
|
991
|
+
return [];
|
|
992
|
+
}
|
|
993
|
+
return source.metadata.recentRuns.slice(0, 3).map((entry, index) => {
|
|
994
|
+
const trigger = typeof entry.trigger === "string" ? entry.trigger : "sync";
|
|
995
|
+
const status = typeof entry.status === "string" ? entry.status : "unknown";
|
|
996
|
+
const finishedAt = typeof entry.finishedAt === "number" ? formatDateLabel(entry.finishedAt) : "n/a";
|
|
997
|
+
const duration = typeof entry.durationMs === "number" ? formatDurationLabel(entry.durationMs) : "n/a";
|
|
998
|
+
const docs = typeof entry.documentCount === "number" ? `${entry.documentCount} docs` : "n/a";
|
|
999
|
+
const chunks = typeof entry.chunkCount === "number" ? `${entry.chunkCount} chunks` : "n/a";
|
|
1000
|
+
const error = typeof entry.error === "string" && entry.error.length > 0 ? entry.error : undefined;
|
|
1001
|
+
return {
|
|
1002
|
+
label: `Run ${index + 1}`,
|
|
1003
|
+
status,
|
|
1004
|
+
summary: `${trigger} \xB7 ${status} \xB7 ${docs} \xB7 ${chunks} \xB7 ${duration} \xB7 ${finishedAt}`,
|
|
1005
|
+
rows: [
|
|
1006
|
+
{ label: "Trigger", value: trigger },
|
|
1007
|
+
{ label: "Status", value: status },
|
|
1008
|
+
{ label: "Output", value: `${docs} \xB7 ${chunks}` },
|
|
1009
|
+
{ label: "Duration", value: duration },
|
|
1010
|
+
{ label: "Finished", value: finishedAt },
|
|
1011
|
+
...error ? [{ label: "Error", value: error }] : []
|
|
1012
|
+
]
|
|
1013
|
+
};
|
|
1014
|
+
});
|
|
1015
|
+
};
|
|
1016
|
+
var buildRAGSyncSourcePresentation = (source) => {
|
|
1017
|
+
const provider = typeof source.metadata?.provider === "string" ? source.metadata.provider : undefined;
|
|
1018
|
+
const accountMode = typeof source.metadata?.accountMode === "string" ? source.metadata.accountMode : undefined;
|
|
1019
|
+
const schedule = typeof source.metadata?.schedule === "string" ? source.metadata.schedule : undefined;
|
|
1020
|
+
const lastTrigger = typeof source.metadata?.lastTrigger === "string" ? source.metadata.lastTrigger : undefined;
|
|
1021
|
+
const liveReady = typeof source.metadata?.liveReady === "string" ? source.metadata.liveReady : undefined;
|
|
1022
|
+
const diagnosticSummary = source.diagnostics?.summary;
|
|
1023
|
+
const diagnosticTags = source.diagnostics?.entries.map((entry) => entry.code.replace(/_/g, " ").replace(/\b\w/g, (letter) => letter.toUpperCase())) ?? [];
|
|
1024
|
+
const retryGuidance = source.diagnostics?.retryGuidance;
|
|
1025
|
+
return {
|
|
1026
|
+
id: source.id,
|
|
1027
|
+
label: source.label,
|
|
1028
|
+
kind: source.kind,
|
|
1029
|
+
status: source.status,
|
|
1030
|
+
summary: [
|
|
1031
|
+
source.kind,
|
|
1032
|
+
source.status,
|
|
1033
|
+
typeof source.documentCount === "number" ? `${source.documentCount} docs` : "",
|
|
1034
|
+
typeof source.chunkCount === "number" ? `${source.chunkCount} chunks` : "",
|
|
1035
|
+
typeof source.lastSyncDurationMs === "number" ? formatDurationLabel(source.lastSyncDurationMs) : "",
|
|
1036
|
+
typeof source.lastSuccessfulSyncAt === "number" ? `last success ${formatDateLabel(source.lastSuccessfulSyncAt)}` : typeof source.lastSyncedAt === "number" ? `last sync ${formatDateLabel(source.lastSyncedAt)}` : ""
|
|
1037
|
+
].filter(Boolean).join(" \xB7 "),
|
|
1038
|
+
rows: [
|
|
1039
|
+
...source.target ? [{ label: "Target", value: source.target }] : [],
|
|
1040
|
+
...provider ? [{ label: "Provider", value: provider }] : [],
|
|
1041
|
+
...accountMode ? [{ label: "Account mode", value: accountMode }] : [],
|
|
1042
|
+
...schedule ? [{ label: "Schedule", value: schedule }] : [],
|
|
1043
|
+
...lastTrigger ? [{ label: "Last trigger", value: lastTrigger }] : [],
|
|
1044
|
+
...typeof source.lastSuccessfulSyncAt === "number" ? [
|
|
1045
|
+
{
|
|
1046
|
+
label: "Last success",
|
|
1047
|
+
value: formatDateLabel(source.lastSuccessfulSyncAt)
|
|
1048
|
+
}
|
|
1049
|
+
] : [],
|
|
1050
|
+
...typeof source.nextRetryAt === "number" ? [
|
|
1051
|
+
{
|
|
1052
|
+
label: "Next retry",
|
|
1053
|
+
value: formatDateLabel(source.nextRetryAt)
|
|
1054
|
+
}
|
|
1055
|
+
] : [],
|
|
1056
|
+
...source.lastError ? [{ label: "Last error", value: source.lastError }] : [],
|
|
1057
|
+
...diagnosticSummary ? [{ label: "Diagnostics", value: diagnosticSummary }] : [],
|
|
1058
|
+
...retryGuidance ? [{ label: "Retry guidance", value: retryGuidance.reason }] : []
|
|
1059
|
+
],
|
|
1060
|
+
tags: [
|
|
1061
|
+
provider,
|
|
1062
|
+
accountMode ? `mode ${accountMode}` : undefined,
|
|
1063
|
+
liveReady,
|
|
1064
|
+
...diagnosticTags
|
|
1065
|
+
].filter((value) => Boolean(value)),
|
|
1066
|
+
extendedSummary: diagnosticSummary ?? source.description ?? liveReady,
|
|
1067
|
+
runs: buildRAGSyncSourceRunPresentations(source)
|
|
1068
|
+
};
|
|
1069
|
+
};
|
|
1070
|
+
var buildRAGSyncSourcePresentations = (sources) => (sources ?? []).map(buildRAGSyncSourcePresentation);
|
|
1071
|
+
var buildRAGReadinessPresentation = (readiness) => {
|
|
1072
|
+
if (!readiness) {
|
|
1073
|
+
return {
|
|
1074
|
+
sections: [
|
|
1075
|
+
{
|
|
1076
|
+
label: "Provider",
|
|
1077
|
+
title: "Unavailable",
|
|
1078
|
+
summary: "Readiness data is not available yet."
|
|
1079
|
+
}
|
|
1080
|
+
]
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
return {
|
|
1084
|
+
sections: [
|
|
1085
|
+
{
|
|
1086
|
+
label: "Provider",
|
|
1087
|
+
title: readiness.providerConfigured ? readiness.providerName ?? "Runtime provider routing" : "Not configured",
|
|
1088
|
+
summary: readiness.providerConfigured ? readiness.model ? `Requests route through ${readiness.providerName ?? "the runtime provider registry"} with default model ${readiness.model}.` : `Requests route through ${readiness.providerName ?? "the runtime provider registry"}.` : "Provider-backed retrieval is not configured yet."
|
|
1089
|
+
},
|
|
1090
|
+
{
|
|
1091
|
+
label: "Embeddings",
|
|
1092
|
+
title: readiness.embeddingConfigured ? readiness.embeddingModel === "collection-managed embeddings" ? "Collection-managed" : "Configured" : "Missing",
|
|
1093
|
+
summary: readiness.embeddingConfigured ? readiness.embeddingModel === "collection-managed embeddings" ? "Embeddings come from the collection and vector store layer, so retrieval stays vector-backed without a separate top-level embedding provider." : readiness.embeddingModel ?? "Embedding model configured." : "Embeddings are not configured yet."
|
|
1094
|
+
},
|
|
1095
|
+
{
|
|
1096
|
+
label: "Retrieval Stack",
|
|
1097
|
+
title: readiness.rerankerConfigured ? "Reranker ready" : "Vector only",
|
|
1098
|
+
summary: readiness.indexManagerConfigured ? "Index manager configured." : "Index manager not configured."
|
|
1099
|
+
},
|
|
1100
|
+
{
|
|
1101
|
+
label: "Extractors",
|
|
1102
|
+
title: readiness.extractorsConfigured ? `${readiness.extractorNames.length} configured` : "None configured",
|
|
1103
|
+
summary: readiness.extractorsConfigured ? `Configured extractors: ${formatCompactList(readiness.extractorNames)}` : "No extractors configured.",
|
|
1104
|
+
tags: readiness.extractorNames.length > 0 ? readiness.extractorNames : ["No extractors configured"]
|
|
1105
|
+
}
|
|
1106
|
+
]
|
|
1107
|
+
};
|
|
1108
|
+
};
|
|
1109
|
+
var buildRAGCorpusHealthPresentation = (health) => {
|
|
1110
|
+
if (!health) {
|
|
1111
|
+
return {
|
|
1112
|
+
sections: [
|
|
1113
|
+
{
|
|
1114
|
+
label: "Corpus health",
|
|
1115
|
+
title: "Unavailable",
|
|
1116
|
+
summary: "Corpus health is not available yet."
|
|
1117
|
+
}
|
|
1118
|
+
]
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
1121
|
+
return {
|
|
1122
|
+
sections: [
|
|
1123
|
+
{
|
|
1124
|
+
label: "Corpus coverage",
|
|
1125
|
+
title: `Formats: ${formatCoverageMap(health.coverageByFormat)}`,
|
|
1126
|
+
summary: `Kinds: ${formatCoverageMap(health.coverageByKind)}`,
|
|
1127
|
+
rows: [
|
|
1128
|
+
{
|
|
1129
|
+
label: "Average chunks per document",
|
|
1130
|
+
value: health.averageChunksPerDocument.toFixed(2)
|
|
1131
|
+
}
|
|
1132
|
+
]
|
|
1133
|
+
},
|
|
1134
|
+
{
|
|
1135
|
+
label: "Chunk quality",
|
|
1136
|
+
title: `${health.averageChunksPerDocument.toFixed(2)} avg chunks/doc`,
|
|
1137
|
+
summary: `Empty docs ${health.emptyDocuments} \xB7 empty chunks ${health.emptyChunks} \xB7 low signal ${health.lowSignalChunks}`,
|
|
1138
|
+
rows: [
|
|
1139
|
+
{
|
|
1140
|
+
label: "Missing source",
|
|
1141
|
+
value: String(health.documentsMissingSource)
|
|
1142
|
+
},
|
|
1143
|
+
{
|
|
1144
|
+
label: "Missing title",
|
|
1145
|
+
value: String(health.documentsMissingTitle)
|
|
1146
|
+
},
|
|
1147
|
+
{
|
|
1148
|
+
label: "Missing metadata",
|
|
1149
|
+
value: String(health.documentsMissingMetadata)
|
|
1150
|
+
}
|
|
1151
|
+
]
|
|
1152
|
+
},
|
|
1153
|
+
{
|
|
1154
|
+
label: "Freshness",
|
|
1155
|
+
title: `${health.staleDocuments.length} stale docs`,
|
|
1156
|
+
summary: `Stale threshold ${formatAgeLabel(health.staleAfterMs)}`,
|
|
1157
|
+
rows: [
|
|
1158
|
+
{
|
|
1159
|
+
label: "Oldest age",
|
|
1160
|
+
value: formatAgeLabel(health.oldestDocumentAgeMs)
|
|
1161
|
+
},
|
|
1162
|
+
{
|
|
1163
|
+
label: "Newest age",
|
|
1164
|
+
value: formatAgeLabel(health.newestDocumentAgeMs)
|
|
1165
|
+
}
|
|
1166
|
+
]
|
|
1167
|
+
},
|
|
1168
|
+
{
|
|
1169
|
+
label: "Failures",
|
|
1170
|
+
title: `${health.failedIngestJobs} ingest \xB7 ${health.failedAdminJobs} admin`,
|
|
1171
|
+
summary: `Duplicate sources ${health.duplicateSourceGroups.length} \xB7 duplicate ids ${health.duplicateDocumentIdGroups.length}`,
|
|
1172
|
+
rows: [
|
|
1173
|
+
{
|
|
1174
|
+
label: "Failures by input",
|
|
1175
|
+
value: formatCoverageMap(health.failuresByInputKind)
|
|
1176
|
+
},
|
|
1177
|
+
{
|
|
1178
|
+
label: "Failures by extractor",
|
|
1179
|
+
value: formatCoverageMap(health.failuresByExtractor)
|
|
1180
|
+
},
|
|
1181
|
+
{
|
|
1182
|
+
label: "Failures by admin action",
|
|
1183
|
+
value: formatCoverageMap(health.failuresByAdminAction)
|
|
1184
|
+
}
|
|
1185
|
+
]
|
|
1186
|
+
}
|
|
1187
|
+
]
|
|
1188
|
+
};
|
|
1189
|
+
};
|
|
1190
|
+
var buildRAGSyncOverviewPresentation = (sources) => {
|
|
1191
|
+
const records = sources ?? [];
|
|
1192
|
+
if (records.length === 0) {
|
|
1193
|
+
return {
|
|
1194
|
+
rows: [
|
|
1195
|
+
{ label: "Configured sync sources", value: "0" },
|
|
1196
|
+
{
|
|
1197
|
+
label: "Latest sync",
|
|
1198
|
+
value: "No sync sources configured yet."
|
|
1199
|
+
}
|
|
1200
|
+
],
|
|
1201
|
+
sections: [
|
|
1202
|
+
{
|
|
1203
|
+
label: "Sync overview",
|
|
1204
|
+
title: "No sync sources configured",
|
|
1205
|
+
summary: "Add sync sources to monitor directories, URLs, storage, or mailboxes."
|
|
1206
|
+
}
|
|
1207
|
+
]
|
|
1208
|
+
};
|
|
1209
|
+
}
|
|
1210
|
+
const countByStatus = (status) => records.filter((record) => record.status === status).length;
|
|
1211
|
+
return {
|
|
1212
|
+
rows: [
|
|
1213
|
+
{ label: "Configured sync sources", value: String(records.length) },
|
|
1214
|
+
{ label: "Completed", value: String(countByStatus("completed")) },
|
|
1215
|
+
{ label: "Running", value: String(countByStatus("running")) },
|
|
1216
|
+
{
|
|
1217
|
+
label: "Failed",
|
|
1218
|
+
value: String(countByStatus("failed"))
|
|
1219
|
+
},
|
|
1220
|
+
buildSyncOverviewLatestRow(records)
|
|
1221
|
+
],
|
|
1222
|
+
sections: [
|
|
1223
|
+
{
|
|
1224
|
+
label: "Sync overview",
|
|
1225
|
+
title: `${records.length} configured`,
|
|
1226
|
+
summary: `${countByStatus("completed")} completed \xB7 ${countByStatus("running")} running \xB7 ${countByStatus("failed")} failed`
|
|
1227
|
+
},
|
|
1228
|
+
{
|
|
1229
|
+
label: "Latest sync",
|
|
1230
|
+
title: buildSyncOverviewLatestRow(records).value,
|
|
1231
|
+
summary: "Most recent completed or last-known sync activity."
|
|
1232
|
+
}
|
|
1233
|
+
]
|
|
1234
|
+
};
|
|
1235
|
+
};
|
|
1236
|
+
var formatMediaTimestamp2 = (value) => {
|
|
1237
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
const totalSeconds = Math.floor(value / 1000);
|
|
1241
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
1242
|
+
const seconds = totalSeconds % 60;
|
|
1243
|
+
const milliseconds = Math.floor(value % 1000);
|
|
1244
|
+
return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`;
|
|
1245
|
+
};
|
|
1246
|
+
var getAttachmentName2 = (source, title) => {
|
|
1247
|
+
const sourceAttachment = source?.split("/").at(-1);
|
|
1248
|
+
if (sourceAttachment && sourceAttachment.includes(".")) {
|
|
1249
|
+
return sourceAttachment;
|
|
1250
|
+
}
|
|
1251
|
+
const titleAttachment = title?.split(" \xB7 ").at(-1);
|
|
1252
|
+
if (titleAttachment && titleAttachment.includes(".")) {
|
|
1253
|
+
return titleAttachment;
|
|
1254
|
+
}
|
|
1255
|
+
return;
|
|
1256
|
+
};
|
|
1257
|
+
var getSpreadsheetHeaders = (metadata) => Array.isArray(metadata?.spreadsheetHeaders) ? metadata.spreadsheetHeaders.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1258
|
+
var formatSpreadsheetColumnRange = (columnStart, columnEnd) => {
|
|
1259
|
+
if (typeof columnStart !== "string" || columnStart.length === 0) {
|
|
1260
|
+
return;
|
|
1261
|
+
}
|
|
1262
|
+
if (typeof columnEnd !== "string" || columnEnd.length === 0) {
|
|
1263
|
+
return `Columns ${columnStart}`;
|
|
1264
|
+
}
|
|
1265
|
+
if (columnStart === columnEnd) {
|
|
1266
|
+
return `Columns ${columnStart}`;
|
|
1267
|
+
}
|
|
1268
|
+
return `Columns ${columnStart}-${columnEnd}`;
|
|
1269
|
+
};
|
|
1270
|
+
var formatSpreadsheetRowRange = (rowStart, rowEnd) => {
|
|
1271
|
+
if (typeof rowStart !== "number" || !Number.isFinite(rowStart)) {
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
if (typeof rowEnd !== "number" && typeof rowStart === "number" && Number.isFinite(rowStart)) {
|
|
1275
|
+
return `Rows ${rowStart}`;
|
|
1276
|
+
}
|
|
1277
|
+
if (rowStart === rowEnd) {
|
|
1278
|
+
return `Rows ${rowStart}`;
|
|
1279
|
+
}
|
|
1280
|
+
return `Rows ${rowStart}-${rowEnd}`;
|
|
1281
|
+
};
|
|
1282
|
+
var formatSpreadsheetTableLabel = (tableIndex, tableCount) => {
|
|
1283
|
+
if (typeof tableIndex !== "number" || !Number.isFinite(tableIndex) || tableIndex < 1) {
|
|
1284
|
+
return;
|
|
1285
|
+
}
|
|
1286
|
+
if (typeof tableCount === "number" && Number.isFinite(tableCount) && tableCount >= tableIndex) {
|
|
1287
|
+
return `Table ${tableIndex} of ${tableCount}`;
|
|
1288
|
+
}
|
|
1289
|
+
return `Table ${tableIndex}`;
|
|
1290
|
+
};
|
|
1291
|
+
var formatMediaDurationLabel2 = (value) => {
|
|
1292
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1293
|
+
return;
|
|
1294
|
+
}
|
|
1295
|
+
return formatMediaTimestamp2(value);
|
|
1296
|
+
};
|
|
1297
|
+
var buildContextLabel2 = (metadata) => {
|
|
1298
|
+
if (!metadata) {
|
|
1299
|
+
return;
|
|
1300
|
+
}
|
|
1301
|
+
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1302
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1303
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1304
|
+
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
1305
|
+
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
1306
|
+
return `PDF table block ${sectionTitle}`;
|
|
1307
|
+
}
|
|
1308
|
+
if (pdfTextKind === "paragraph" && sectionTitle) {
|
|
1309
|
+
return `PDF text block ${sectionTitle}`;
|
|
1310
|
+
}
|
|
1311
|
+
if (officeBlockKind === "table" && sectionTitle) {
|
|
1312
|
+
return `Office table block ${sectionTitle}`;
|
|
1313
|
+
}
|
|
1314
|
+
if (officeBlockKind === "list" && sectionTitle) {
|
|
1315
|
+
return `Office list block ${sectionTitle}`;
|
|
1316
|
+
}
|
|
1317
|
+
if (officeBlockKind === "paragraph" && sectionTitle) {
|
|
1318
|
+
return `Office paragraph block ${sectionTitle}`;
|
|
1319
|
+
}
|
|
1320
|
+
const emailKind = getContextString2(metadata.emailKind);
|
|
1321
|
+
if (emailKind === "attachment") {
|
|
1322
|
+
const attachmentName = getContextString2(metadata.attachmentName);
|
|
1323
|
+
const threadTopic2 = getContextString2(metadata.threadTopic);
|
|
1324
|
+
return attachmentName ? threadTopic2 ? `Attachment evidence ${attachmentName} in ${threadTopic2}` : `Attachment evidence ${attachmentName}` : "Attachment evidence";
|
|
1325
|
+
}
|
|
1326
|
+
if (emailKind === "message") {
|
|
1327
|
+
const threadTopic2 = getContextString2(metadata.threadTopic);
|
|
1328
|
+
const from = getContextString2(metadata.from);
|
|
1329
|
+
if (threadTopic2) {
|
|
1330
|
+
return from ? `Message in ${threadTopic2} from ${from}` : `Message in ${threadTopic2}`;
|
|
1331
|
+
}
|
|
1332
|
+
return from ? `Message from ${from}` : "Message evidence";
|
|
1333
|
+
}
|
|
1334
|
+
const page = getContextNumber2(metadata.page) ?? getContextNumber2(metadata.pageNumber) ?? (typeof metadata.pageIndex === "number" ? metadata.pageIndex + 1 : undefined);
|
|
1335
|
+
const region = getContextNumber2(metadata.regionNumber) ?? (typeof metadata.regionIndex === "number" ? metadata.regionIndex + 1 : undefined);
|
|
1336
|
+
const hasOCRTrace = typeof metadata.ocrRegionConfidence === "number" || typeof metadata.ocrConfidence === "number" || getContextString2(metadata.pdfTextMode) === "ocr" || typeof metadata.ocrRegionCount === "number";
|
|
1337
|
+
const ocrPageStart = getContextNumber2(metadata.ocrPageStart);
|
|
1338
|
+
const ocrPageEnd = getContextNumber2(metadata.ocrPageEnd);
|
|
1339
|
+
if (page && region) {
|
|
1340
|
+
if (hasOCRTrace) {
|
|
1341
|
+
return `OCR page ${page} region ${region}`;
|
|
1342
|
+
}
|
|
1343
|
+
return `Page ${page} region ${region}`;
|
|
1344
|
+
}
|
|
1345
|
+
if (page) {
|
|
1346
|
+
if (hasOCRTrace) {
|
|
1347
|
+
return `OCR page ${page}`;
|
|
1348
|
+
}
|
|
1349
|
+
return `Page ${page}`;
|
|
1350
|
+
}
|
|
1351
|
+
if (hasOCRTrace && typeof ocrPageStart === "number" && typeof ocrPageEnd === "number") {
|
|
1352
|
+
return ocrPageStart === ocrPageEnd ? `OCR page ${ocrPageStart}` : `OCR pages ${ocrPageStart}-${ocrPageEnd}`;
|
|
1353
|
+
}
|
|
1354
|
+
const sheet = getContextString2(metadata.sheetName) ?? (Array.isArray(metadata.sheetNames) ? getContextString2(metadata.sheetNames[0]) : undefined);
|
|
1355
|
+
if (sheet) {
|
|
1356
|
+
const tableLabel = formatSpreadsheetTableLabel(getContextNumber2(metadata.spreadsheetTableIndex), getContextNumber2(metadata.spreadsheetTableCount));
|
|
1357
|
+
const columnRange = formatSpreadsheetColumnRange(getContextString2(metadata.spreadsheetColumnStart), getContextString2(metadata.spreadsheetColumnEnd));
|
|
1358
|
+
const rowRange = formatSpreadsheetRowRange(getContextNumber2(metadata.spreadsheetRowStart), getContextNumber2(metadata.spreadsheetRowEnd));
|
|
1359
|
+
const headers = getSpreadsheetHeaders(metadata);
|
|
1360
|
+
if (tableLabel && rowRange && columnRange) {
|
|
1361
|
+
return `Sheet ${sheet} ${tableLabel} ${rowRange} ${columnRange}`;
|
|
1362
|
+
}
|
|
1363
|
+
if (tableLabel && rowRange) {
|
|
1364
|
+
return `Sheet ${sheet} ${tableLabel} ${rowRange}`;
|
|
1365
|
+
}
|
|
1366
|
+
if (tableLabel && columnRange) {
|
|
1367
|
+
return `Sheet ${sheet} ${tableLabel} ${columnRange}`;
|
|
1368
|
+
}
|
|
1369
|
+
if (tableLabel) {
|
|
1370
|
+
return `Sheet ${sheet} ${tableLabel}`;
|
|
1371
|
+
}
|
|
1372
|
+
if (rowRange && columnRange) {
|
|
1373
|
+
return `Sheet ${sheet} ${rowRange} ${columnRange}`;
|
|
1374
|
+
}
|
|
1375
|
+
if (rowRange) {
|
|
1376
|
+
return `Sheet ${sheet} ${rowRange}`;
|
|
1377
|
+
}
|
|
1378
|
+
if (columnRange) {
|
|
1379
|
+
return `Sheet ${sheet} ${columnRange}`;
|
|
1380
|
+
}
|
|
1381
|
+
if (headers.length > 0) {
|
|
1382
|
+
return `Sheet ${sheet} by ${headers.slice(0, 2).join(", ")}`;
|
|
1383
|
+
}
|
|
1384
|
+
return `Sheet ${sheet}`;
|
|
1385
|
+
}
|
|
1386
|
+
const slide = getContextNumber2(metadata.slide) ?? getContextNumber2(metadata.slideNumber) ?? (typeof metadata.slideIndex === "number" ? metadata.slideIndex + 1 : undefined);
|
|
1387
|
+
const slideTitle = getContextString2(metadata.slideTitle);
|
|
1388
|
+
if (slide) {
|
|
1389
|
+
if (slideTitle) {
|
|
1390
|
+
return `Slide ${slide} ${slideTitle}`;
|
|
1391
|
+
}
|
|
1392
|
+
return `Slide ${slide}`;
|
|
1393
|
+
}
|
|
1394
|
+
const archiveEntry = getContextString2(metadata.archiveFullPath) ?? getContextString2(metadata.archivePath) ?? getContextString2(metadata.archiveEntryPath) ?? getContextString2(metadata.entryPath);
|
|
1395
|
+
if (archiveEntry) {
|
|
1396
|
+
return `Archive entry ${archiveEntry}`;
|
|
1397
|
+
}
|
|
1398
|
+
const threadTopic = getContextString2(metadata.threadTopic);
|
|
1399
|
+
if (threadTopic) {
|
|
1400
|
+
return `Thread ${threadTopic}`;
|
|
1401
|
+
}
|
|
1402
|
+
const speaker = getContextString2(metadata.speaker);
|
|
1403
|
+
if (speaker) {
|
|
1404
|
+
return `Speaker ${speaker}`;
|
|
1405
|
+
}
|
|
1406
|
+
if (sectionTitle) {
|
|
1407
|
+
return `Section ${sectionTitle}`;
|
|
1408
|
+
}
|
|
1409
|
+
return;
|
|
1410
|
+
};
|
|
1411
|
+
var buildLocatorLabel2 = (metadata, source, title) => {
|
|
1412
|
+
if (!metadata) {
|
|
1413
|
+
return;
|
|
1414
|
+
}
|
|
1415
|
+
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1416
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1417
|
+
const pdfBlockNumber = getContextNumber2(metadata.pdfBlockNumber);
|
|
1418
|
+
const officeBlockNumber = getContextNumber2(metadata.officeBlockNumber);
|
|
1419
|
+
const spreadsheetRowStart = getContextNumber2(metadata.spreadsheetRowStart);
|
|
1420
|
+
const spreadsheetRowEnd = getContextNumber2(metadata.spreadsheetRowEnd);
|
|
1421
|
+
const slideTitle = getContextString2(metadata.slideTitle);
|
|
1422
|
+
const page = getContextNumber2(metadata.page) ?? getContextNumber2(metadata.pageNumber) ?? (typeof metadata.pageIndex === "number" ? metadata.pageIndex + 1 : undefined);
|
|
1423
|
+
const region = getContextNumber2(metadata.regionNumber) ?? (typeof metadata.regionIndex === "number" ? metadata.regionIndex + 1 : undefined);
|
|
1424
|
+
const ocrPageStart = getContextNumber2(metadata.ocrPageStart);
|
|
1425
|
+
const ocrPageEnd = getContextNumber2(metadata.ocrPageEnd);
|
|
1426
|
+
if (page && region) {
|
|
1427
|
+
return `Page ${page} \xB7 Region ${region}`;
|
|
1428
|
+
}
|
|
1429
|
+
if (page && pdfBlockNumber && pdfTextKind === "table_like") {
|
|
1430
|
+
return `Page ${page} \xB7 Table Block ${pdfBlockNumber}`;
|
|
1431
|
+
}
|
|
1432
|
+
if (page && pdfBlockNumber) {
|
|
1433
|
+
return `Page ${page} \xB7 Text Block ${pdfBlockNumber}`;
|
|
1434
|
+
}
|
|
1435
|
+
if (page) {
|
|
1436
|
+
return `Page ${page}`;
|
|
1437
|
+
}
|
|
1438
|
+
if (typeof ocrPageStart === "number" && typeof ocrPageEnd === "number") {
|
|
1439
|
+
return ocrPageStart === ocrPageEnd ? `Page ${ocrPageStart}` : `Pages ${ocrPageStart}-${ocrPageEnd}`;
|
|
1440
|
+
}
|
|
1441
|
+
const sheet = getContextString2(metadata.sheetName) ?? (Array.isArray(metadata.sheetNames) ? getContextString2(metadata.sheetNames[0]) : undefined);
|
|
1442
|
+
if (sheet) {
|
|
1443
|
+
const tableLabel = formatSpreadsheetTableLabel(getContextNumber2(metadata.spreadsheetTableIndex), getContextNumber2(metadata.spreadsheetTableCount));
|
|
1444
|
+
const columnRange = formatSpreadsheetColumnRange(getContextString2(metadata.spreadsheetColumnStart), getContextString2(metadata.spreadsheetColumnEnd));
|
|
1445
|
+
const rowRange = formatSpreadsheetRowRange(spreadsheetRowStart, spreadsheetRowEnd);
|
|
1446
|
+
if (tableLabel && rowRange && columnRange) {
|
|
1447
|
+
return `Sheet ${sheet} \xB7 ${tableLabel} \xB7 ${rowRange} \xB7 ${columnRange}`;
|
|
1448
|
+
}
|
|
1449
|
+
if (tableLabel && rowRange) {
|
|
1450
|
+
return `Sheet ${sheet} \xB7 ${tableLabel} \xB7 ${rowRange}`;
|
|
1451
|
+
}
|
|
1452
|
+
if (tableLabel && columnRange) {
|
|
1453
|
+
return `Sheet ${sheet} \xB7 ${tableLabel} \xB7 ${columnRange}`;
|
|
1454
|
+
}
|
|
1455
|
+
if (tableLabel) {
|
|
1456
|
+
return `Sheet ${sheet} \xB7 ${tableLabel}`;
|
|
1457
|
+
}
|
|
1458
|
+
if (rowRange && columnRange) {
|
|
1459
|
+
return `Sheet ${sheet} \xB7 ${rowRange} \xB7 ${columnRange}`;
|
|
1460
|
+
}
|
|
1461
|
+
if (rowRange) {
|
|
1462
|
+
return `Sheet ${sheet} \xB7 ${rowRange}`;
|
|
1463
|
+
}
|
|
1464
|
+
return columnRange ? `Sheet ${sheet} \xB7 ${columnRange}` : `Sheet ${sheet}`;
|
|
1465
|
+
}
|
|
1466
|
+
const slide = getContextNumber2(metadata.slide) ?? getContextNumber2(metadata.slideNumber) ?? (typeof metadata.slideIndex === "number" ? metadata.slideIndex + 1 : undefined);
|
|
1467
|
+
if (slide) {
|
|
1468
|
+
return slideTitle ? `Slide ${slide} \xB7 ${slideTitle}` : `Slide ${slide}`;
|
|
1469
|
+
}
|
|
1470
|
+
const archiveEntry = getContextString2(metadata.archiveFullPath) ?? getContextString2(metadata.archivePath) ?? getContextString2(metadata.archiveEntryPath) ?? getContextString2(metadata.entryPath);
|
|
1471
|
+
if (archiveEntry) {
|
|
1472
|
+
return `Archive entry ${archiveEntry}`;
|
|
1473
|
+
}
|
|
1474
|
+
const emailKind = getContextString2(metadata.emailKind);
|
|
1475
|
+
if (emailKind === "attachment") {
|
|
1476
|
+
const attachmentName = getContextString2(metadata.attachmentName) ?? getAttachmentName2(source, title);
|
|
1477
|
+
const replyDepth = getContextNumber2(metadata.replyDepth);
|
|
1478
|
+
if (attachmentName && replyDepth && replyDepth > 0) {
|
|
1479
|
+
return `Attachment ${attachmentName} \xB7 Reply depth ${replyDepth}`;
|
|
1480
|
+
}
|
|
1481
|
+
return attachmentName ? `Attachment ${attachmentName}` : "Attachment";
|
|
1482
|
+
}
|
|
1483
|
+
const mediaStart = formatMediaTimestamp2(metadata.startMs);
|
|
1484
|
+
const mediaEnd = formatMediaTimestamp2(metadata.endMs);
|
|
1485
|
+
if (mediaStart && mediaEnd) {
|
|
1486
|
+
return `Timestamp ${mediaStart} - ${mediaEnd}`;
|
|
1487
|
+
}
|
|
1488
|
+
if (mediaStart) {
|
|
1489
|
+
return `Timestamp ${mediaStart}`;
|
|
1490
|
+
}
|
|
1491
|
+
if (officeBlockNumber && officeBlockKind === "table") {
|
|
1492
|
+
return `Office table block ${officeBlockNumber}`;
|
|
1493
|
+
}
|
|
1494
|
+
if (officeBlockNumber && officeBlockKind === "list") {
|
|
1495
|
+
return `Office list block ${officeBlockNumber}`;
|
|
1496
|
+
}
|
|
1497
|
+
if (officeBlockNumber && officeBlockKind === "paragraph") {
|
|
1498
|
+
return `Office paragraph block ${officeBlockNumber}`;
|
|
1499
|
+
}
|
|
1500
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1501
|
+
if (sectionPath.length > 0) {
|
|
1502
|
+
return `Section ${sectionPath.join(" > ")}`;
|
|
1503
|
+
}
|
|
1504
|
+
return;
|
|
1505
|
+
};
|
|
1506
|
+
var buildProvenanceLabel2 = (metadata) => {
|
|
1507
|
+
if (!metadata) {
|
|
1508
|
+
return;
|
|
1509
|
+
}
|
|
1510
|
+
const threadTopic = getContextString2(metadata.threadTopic);
|
|
1511
|
+
const replyDepth = getContextNumber2(metadata.replyDepth);
|
|
1512
|
+
const threadMessageCount = getContextNumber2(metadata.threadMessageCount);
|
|
1513
|
+
const threadRootMessageId = getContextString2(metadata.threadRootMessageId);
|
|
1514
|
+
const from = getContextString2(metadata.from);
|
|
1515
|
+
const sentAt = formatTimestampLabel2(metadata.sentAt) ?? formatTimestampLabel2(metadata.receivedAt);
|
|
1516
|
+
const speaker = getContextString2(metadata.speaker);
|
|
1517
|
+
const mediaKind = getContextString2(metadata.mediaKind);
|
|
1518
|
+
const transcriptSource = getContextString2(metadata.transcriptSource);
|
|
1519
|
+
const mediaSpeakerCount = getContextNumber2(metadata.mediaSpeakerCount);
|
|
1520
|
+
const mediaSegmentCount = getContextNumber2(metadata.mediaSegmentCount);
|
|
1521
|
+
const mediaSegmentGroupSize = getContextNumber2(metadata.mediaSegmentGroupSize);
|
|
1522
|
+
const mediaSegmentGroupIndex = getContextNumber2(metadata.mediaSegmentGroupIndex);
|
|
1523
|
+
const mediaChannel = getContextString2(metadata.mediaChannel);
|
|
1524
|
+
const mediaDurationLabel = formatMediaDurationLabel2(metadata.mediaDurationMs);
|
|
1525
|
+
const mediaSegmentWindowDurationLabel = formatMediaDurationLabel2(metadata.mediaSegmentGroupDurationMs);
|
|
1526
|
+
const mediaSegmentGapLabel = formatMediaDurationLabel2(metadata.mediaSegmentGapFromPreviousMs);
|
|
1527
|
+
const spreadsheetHeaders = getSpreadsheetHeaders(metadata);
|
|
1528
|
+
const spreadsheetColumnRange = formatSpreadsheetColumnRange(getContextString2(metadata.spreadsheetColumnStart), getContextString2(metadata.spreadsheetColumnEnd));
|
|
1529
|
+
const slideNotesText = getContextString2(metadata.slideNotesText);
|
|
1530
|
+
const pdfTextMode = getContextString2(metadata.pdfTextMode);
|
|
1531
|
+
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1532
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1533
|
+
const ocrEngine = getContextString2(metadata.ocrEngine);
|
|
1534
|
+
const extractorRegistryMatch = getContextString2(metadata.extractorRegistryMatch);
|
|
1535
|
+
const chunkingProfile = getContextString2(metadata.chunkingProfile);
|
|
1536
|
+
const archiveDepth = getContextNumber2(metadata.archiveDepth);
|
|
1537
|
+
const archiveNestedDepth = getContextNumber2(metadata.archiveNestedDepth);
|
|
1538
|
+
const archiveContainerPath = getContextString2(metadata.archiveContainerPath);
|
|
1539
|
+
const archiveRootName = getContextString2(metadata.archiveRootName);
|
|
1540
|
+
const sourceAwareChunkReason = formatSourceAwareChunkReason(metadata.sourceAwareChunkReason);
|
|
1541
|
+
const sourceAwareUnitScope = buildSourceAwareUnitScopeLabel(metadata);
|
|
1542
|
+
const spreadsheetTableLabel = formatSpreadsheetTableLabel(getContextNumber2(metadata.spreadsheetTableIndex), getContextNumber2(metadata.spreadsheetTableCount));
|
|
1543
|
+
const ocrConfidence = getContextNumber2(metadata.ocrRegionConfidence) ?? getContextNumber2(metadata.ocrConfidence);
|
|
1544
|
+
const ocrAverageConfidence = getContextNumber2(metadata.ocrPageAverageConfidence) ?? getContextNumber2(metadata.ocrAverageConfidence);
|
|
1545
|
+
const ocrMinConfidence = getContextNumber2(metadata.ocrPageMinConfidence) ?? getContextNumber2(metadata.ocrMinConfidence);
|
|
1546
|
+
const ocrMaxConfidence = getContextNumber2(metadata.ocrPageMaxConfidence) ?? getContextNumber2(metadata.ocrMaxConfidence);
|
|
1547
|
+
const ocrRegionCount = getContextNumber2(metadata.ocrRegionCount);
|
|
1548
|
+
const labels = [
|
|
1549
|
+
pdfTextMode ? `PDF ${pdfTextMode}` : "",
|
|
1550
|
+
pdfTextKind === "table_like" ? "PDF table block" : pdfTextKind === "paragraph" ? "PDF text block" : "",
|
|
1551
|
+
officeBlockKind ? `Office ${officeBlockKind}` : "",
|
|
1552
|
+
ocrEngine ? `OCR ${ocrEngine}` : "",
|
|
1553
|
+
extractorRegistryMatch ? `Extractor ${extractorRegistryMatch}` : "",
|
|
1554
|
+
chunkingProfile ? `Chunking ${chunkingProfile}` : "",
|
|
1555
|
+
sourceAwareChunkReason ?? "",
|
|
1556
|
+
sourceAwareUnitScope ?? "",
|
|
1557
|
+
typeof ocrConfidence === "number" ? `Confidence ${ocrConfidence.toFixed(2)}` : "",
|
|
1558
|
+
typeof ocrAverageConfidence === "number" && ocrAverageConfidence !== ocrConfidence ? `Average ${ocrAverageConfidence.toFixed(2)}` : "",
|
|
1559
|
+
typeof ocrMinConfidence === "number" && typeof ocrMaxConfidence === "number" && ocrMinConfidence !== ocrMaxConfidence ? `Range ${ocrMinConfidence.toFixed(2)}-${ocrMaxConfidence.toFixed(2)}` : "",
|
|
1560
|
+
typeof ocrRegionCount === "number" ? `${ocrRegionCount} regions` : "",
|
|
1561
|
+
spreadsheetHeaders.length > 0 ? `Spreadsheet ${spreadsheetHeaders.join(", ")}` : "",
|
|
1562
|
+
spreadsheetColumnRange ? `Spreadsheet ${spreadsheetColumnRange}` : "",
|
|
1563
|
+
spreadsheetTableLabel ? `Spreadsheet ${spreadsheetTableLabel}` : "",
|
|
1564
|
+
mediaKind ? `Media ${mediaKind}` : "",
|
|
1565
|
+
mediaSegmentCount ? `${mediaSegmentCount} segments` : "",
|
|
1566
|
+
mediaSegmentGroupSize ? `${mediaSegmentGroupSize} grouped segments` : "",
|
|
1567
|
+
mediaSegmentGroupIndex !== undefined ? `Segment group ${mediaSegmentGroupIndex + 1}` : "",
|
|
1568
|
+
mediaChannel ? `Channel ${mediaChannel}` : "",
|
|
1569
|
+
mediaSpeakerCount ? `${mediaSpeakerCount} speakers` : "",
|
|
1570
|
+
mediaDurationLabel ? `Duration ${mediaDurationLabel}` : "",
|
|
1571
|
+
mediaSegmentWindowDurationLabel ? `Segment window ${mediaSegmentWindowDurationLabel}` : "",
|
|
1572
|
+
mediaSegmentGapLabel ? `Gap ${mediaSegmentGapLabel} from previous window` : "",
|
|
1573
|
+
transcriptSource ? `Transcript ${transcriptSource}` : "",
|
|
1574
|
+
threadTopic ? `Thread ${threadTopic}` : "",
|
|
1575
|
+
threadRootMessageId ? `Thread root ${threadRootMessageId}` : "",
|
|
1576
|
+
threadMessageCount ? `${threadMessageCount} thread messages` : "",
|
|
1577
|
+
replyDepth ? `Reply depth ${replyDepth}` : "",
|
|
1578
|
+
slideNotesText ? "Speaker notes" : "",
|
|
1579
|
+
archiveDepth ? `Archive depth ${archiveDepth}` : "",
|
|
1580
|
+
archiveNestedDepth ? `Archive nested depth ${archiveNestedDepth}` : "",
|
|
1581
|
+
archiveContainerPath ? `Archive container ${archiveContainerPath}` : "",
|
|
1582
|
+
archiveRootName ? `Archive root ${archiveRootName}` : "",
|
|
1583
|
+
speaker ? `Speaker ${speaker}` : "",
|
|
1584
|
+
from ? `Sender ${from}` : "",
|
|
1585
|
+
sentAt ? `Sent ${sentAt}` : ""
|
|
1586
|
+
].filter((value) => value.length > 0);
|
|
1587
|
+
return labels.length > 0 ? labels.join(" \xB7 ") : undefined;
|
|
1588
|
+
};
|
|
1589
|
+
var buildRAGSourceLabels = ({
|
|
1590
|
+
metadata,
|
|
1591
|
+
source,
|
|
1592
|
+
title
|
|
1593
|
+
}) => {
|
|
1594
|
+
const contextLabel = buildContextLabel2(metadata);
|
|
1595
|
+
const locatorLabel = buildLocatorLabel2(metadata, source, title);
|
|
1596
|
+
const provenanceLabel = buildProvenanceLabel2(metadata);
|
|
1597
|
+
if (!contextLabel && !locatorLabel && !provenanceLabel) {
|
|
1598
|
+
return;
|
|
1599
|
+
}
|
|
1600
|
+
return {
|
|
1601
|
+
contextLabel,
|
|
1602
|
+
locatorLabel,
|
|
1603
|
+
provenanceLabel
|
|
1604
|
+
};
|
|
1605
|
+
};
|
|
1606
|
+
var buildRAGChunkStructure = (metadata) => {
|
|
1607
|
+
if (!metadata) {
|
|
1608
|
+
return;
|
|
1609
|
+
}
|
|
1610
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.filter((value) => typeof value === "string" && value.trim().length > 0) : undefined;
|
|
1611
|
+
const sectionKind = metadata.sectionKind === "markdown_heading" || metadata.sectionKind === "html_heading" || metadata.sectionKind === "office_heading" || metadata.sectionKind === "office_block" || metadata.sectionKind === "pdf_block" || metadata.sectionKind === "spreadsheet_rows" || metadata.sectionKind === "presentation_slide" ? metadata.sectionKind : undefined;
|
|
1612
|
+
const section = {
|
|
1613
|
+
depth: getContextNumber2(metadata.sectionDepth),
|
|
1614
|
+
kind: sectionKind,
|
|
1615
|
+
path: sectionPath && sectionPath.length > 0 ? sectionPath : undefined,
|
|
1616
|
+
title: getContextString2(metadata.sectionTitle)
|
|
1617
|
+
};
|
|
1618
|
+
const sequence = {
|
|
1619
|
+
nextChunkId: getContextString2(metadata.nextChunkId),
|
|
1620
|
+
previousChunkId: getContextString2(metadata.previousChunkId),
|
|
1621
|
+
sectionChunkCount: getContextNumber2(metadata.sectionChunkCount),
|
|
1622
|
+
sectionChunkId: getContextString2(metadata.sectionChunkId),
|
|
1623
|
+
sectionChunkIndex: getContextNumber2(metadata.sectionChunkIndex)
|
|
1624
|
+
};
|
|
1625
|
+
if (!section.title && (!section.path || section.path.length === 0) && typeof section.depth !== "number" && !section.kind && !sequence.nextChunkId && !sequence.previousChunkId && typeof sequence.sectionChunkCount !== "number" && !sequence.sectionChunkId && typeof sequence.sectionChunkIndex !== "number") {
|
|
1626
|
+
return;
|
|
1627
|
+
}
|
|
1628
|
+
return {
|
|
1629
|
+
section: section.title || section.path && section.path.length > 0 || typeof section.depth === "number" || section.kind ? section : undefined,
|
|
1630
|
+
sequence: sequence.nextChunkId || sequence.previousChunkId || typeof sequence.sectionChunkCount === "number" || sequence.sectionChunkId || typeof sequence.sectionChunkIndex === "number" ? sequence : undefined
|
|
1631
|
+
};
|
|
1632
|
+
};
|
|
1633
|
+
var buildExcerpt2 = (text, maxLength = 160) => {
|
|
1634
|
+
const normalized = text.replaceAll(/\s+/g, " ").trim();
|
|
1635
|
+
if (normalized.length <= maxLength) {
|
|
1636
|
+
return normalized;
|
|
1637
|
+
}
|
|
1638
|
+
return `${normalized.slice(0, Math.max(0, maxLength - 1)).trimEnd()}\u2026`;
|
|
1639
|
+
};
|
|
1640
|
+
var buildRAGChunkExcerpts = (chunks, activeChunkId) => {
|
|
1641
|
+
if (chunks.length === 0) {
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
const graph = buildRAGChunkGraph(chunks.map((chunk) => ({
|
|
1645
|
+
chunkId: chunk.chunkId,
|
|
1646
|
+
metadata: chunk.metadata,
|
|
1647
|
+
structure: chunk.structure
|
|
1648
|
+
})));
|
|
1649
|
+
const navigation = buildRAGChunkGraphNavigation(graph, activeChunkId);
|
|
1650
|
+
const activeChunk = chunks.find((chunk) => chunk.chunkId === navigation.activeChunkId) ?? chunks[0];
|
|
1651
|
+
if (!activeChunk) {
|
|
1652
|
+
return;
|
|
1653
|
+
}
|
|
1654
|
+
const chunkMap = new Map(chunks.map((chunk) => [chunk.chunkId, chunk]));
|
|
1655
|
+
const orderedSectionIds = navigation.sectionNodes.length > 0 ? navigation.sectionNodes.map((node) => node.chunkId) : [activeChunk.chunkId];
|
|
1656
|
+
const orderedWindowIds = navigation.sectionNodes.length > 0 ? (() => {
|
|
1657
|
+
const activeIndex = navigation.sectionNodes.findIndex((node) => node.chunkId === activeChunk.chunkId);
|
|
1658
|
+
const startIndex = Math.max(0, activeIndex - 2);
|
|
1659
|
+
const endIndex = Math.min(navigation.sectionNodes.length, activeIndex + 3);
|
|
1660
|
+
return navigation.sectionNodes.slice(startIndex, endIndex).map((node) => node.chunkId);
|
|
1661
|
+
})() : [
|
|
1662
|
+
navigation.previousNode?.chunkId,
|
|
1663
|
+
activeChunk.chunkId,
|
|
1664
|
+
navigation.nextNode?.chunkId
|
|
1665
|
+
].filter((chunkId, index, ids) => Boolean(chunkId) && ids.indexOf(chunkId) === index);
|
|
1666
|
+
const collectText = (chunkIds) => chunkIds.map((chunkId) => chunkMap.get(chunkId)?.text).filter((text) => typeof text === "string").join(`
|
|
1667
|
+
|
|
1668
|
+
`);
|
|
1669
|
+
return {
|
|
1670
|
+
chunkExcerpt: buildExcerpt2(activeChunk.text, 160),
|
|
1671
|
+
sectionExcerpt: buildExcerpt2(collectText(orderedSectionIds), 320),
|
|
1672
|
+
windowExcerpt: buildExcerpt2(collectText(orderedWindowIds), 240)
|
|
1673
|
+
};
|
|
1674
|
+
};
|
|
1675
|
+
var buildRAGExcerptSelection = (excerpts, structure) => {
|
|
1676
|
+
if (!excerpts) {
|
|
1677
|
+
return {
|
|
1678
|
+
excerpt: "",
|
|
1679
|
+
mode: "chunk",
|
|
1680
|
+
reason: "single_chunk"
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
const chunkLength = excerpts.chunkExcerpt.trim().length;
|
|
1684
|
+
const sectionChunkCount = structure?.sequence?.sectionChunkCount ?? 1;
|
|
1685
|
+
if (sectionChunkCount > 1 && chunkLength > 0 && chunkLength < 72) {
|
|
1686
|
+
if (sectionChunkCount <= 3 && excerpts.sectionExcerpt.trim().length > 0) {
|
|
1687
|
+
return {
|
|
1688
|
+
excerpt: excerpts.sectionExcerpt,
|
|
1689
|
+
mode: "section",
|
|
1690
|
+
reason: "section_small_enough"
|
|
1691
|
+
};
|
|
1692
|
+
}
|
|
1693
|
+
if (excerpts.windowExcerpt.trim().length > 0) {
|
|
1694
|
+
return {
|
|
1695
|
+
excerpt: excerpts.windowExcerpt,
|
|
1696
|
+
mode: "window",
|
|
1697
|
+
reason: "section_too_large_use_window"
|
|
1698
|
+
};
|
|
1699
|
+
}
|
|
1700
|
+
return {
|
|
1701
|
+
excerpt: excerpts.chunkExcerpt,
|
|
1702
|
+
mode: "chunk",
|
|
1703
|
+
reason: "chunk_too_narrow"
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
return {
|
|
1707
|
+
excerpt: excerpts.chunkExcerpt,
|
|
1708
|
+
mode: "chunk",
|
|
1709
|
+
reason: sectionChunkCount > 1 ? "chunk_too_narrow" : "single_chunk"
|
|
1710
|
+
};
|
|
1711
|
+
};
|
|
1712
|
+
var buildRAGExcerptModeCounts = (selections) => selections.reduce((counts, selection) => {
|
|
1713
|
+
if (selection) {
|
|
1714
|
+
counts[selection.mode] += 1;
|
|
1715
|
+
}
|
|
1716
|
+
return counts;
|
|
1717
|
+
}, { chunk: 0, section: 0, window: 0 });
|
|
1718
|
+
var buildRAGChunkGraph = (chunks) => {
|
|
1719
|
+
const nodes = [];
|
|
1720
|
+
const edges = [];
|
|
1721
|
+
const edgeKeys = new Set;
|
|
1722
|
+
const sections = new Map;
|
|
1723
|
+
for (const chunk of chunks) {
|
|
1724
|
+
const labels = chunk.labels ?? buildRAGSourceLabels({
|
|
1725
|
+
metadata: chunk.metadata,
|
|
1726
|
+
source: chunk.source,
|
|
1727
|
+
title: chunk.title
|
|
1728
|
+
});
|
|
1729
|
+
const structure = chunk.structure ?? buildRAGChunkStructure(chunk.metadata);
|
|
1730
|
+
nodes.push({
|
|
1731
|
+
chunkId: chunk.chunkId,
|
|
1732
|
+
contextLabel: labels?.contextLabel,
|
|
1733
|
+
label: chunk.source ?? chunk.title ?? chunk.chunkId,
|
|
1734
|
+
locatorLabel: labels?.locatorLabel,
|
|
1735
|
+
provenanceLabel: labels?.provenanceLabel,
|
|
1736
|
+
score: chunk.score,
|
|
1737
|
+
source: chunk.source,
|
|
1738
|
+
structure,
|
|
1739
|
+
title: chunk.title
|
|
1740
|
+
});
|
|
1741
|
+
const previousChunkId = structure?.sequence?.previousChunkId;
|
|
1742
|
+
if (previousChunkId) {
|
|
1743
|
+
const key = `previous:${previousChunkId}:${chunk.chunkId}`;
|
|
1744
|
+
if (!edgeKeys.has(key)) {
|
|
1745
|
+
edgeKeys.add(key);
|
|
1746
|
+
edges.push({
|
|
1747
|
+
fromChunkId: previousChunkId,
|
|
1748
|
+
relation: "previous",
|
|
1749
|
+
toChunkId: chunk.chunkId
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
const nextChunkId = structure?.sequence?.nextChunkId;
|
|
1754
|
+
if (nextChunkId) {
|
|
1755
|
+
const key = `next:${chunk.chunkId}:${nextChunkId}`;
|
|
1756
|
+
if (!edgeKeys.has(key)) {
|
|
1757
|
+
edgeKeys.add(key);
|
|
1758
|
+
edges.push({
|
|
1759
|
+
fromChunkId: chunk.chunkId,
|
|
1760
|
+
relation: "next",
|
|
1761
|
+
toChunkId: nextChunkId
|
|
1762
|
+
});
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
const sectionId = structure?.sequence?.sectionChunkId;
|
|
1766
|
+
if (sectionId) {
|
|
1767
|
+
const existing = sections.get(sectionId);
|
|
1768
|
+
if (!existing) {
|
|
1769
|
+
sections.set(sectionId, {
|
|
1770
|
+
childSectionIds: [],
|
|
1771
|
+
chunkCount: structure.sequence?.sectionChunkCount ?? 1,
|
|
1772
|
+
chunkIds: [chunk.chunkId],
|
|
1773
|
+
depth: structure.section?.depth,
|
|
1774
|
+
id: sectionId,
|
|
1775
|
+
kind: structure.section?.kind,
|
|
1776
|
+
leadChunkId: chunk.chunkId,
|
|
1777
|
+
path: structure.section?.path,
|
|
1778
|
+
title: structure.section?.title
|
|
1779
|
+
});
|
|
1780
|
+
continue;
|
|
1781
|
+
}
|
|
1782
|
+
if (!existing.chunkIds.includes(chunk.chunkId)) {
|
|
1783
|
+
existing.chunkIds.push(chunk.chunkId);
|
|
1784
|
+
}
|
|
1785
|
+
existing.chunkCount = Math.max(existing.chunkCount, structure.sequence?.sectionChunkCount ?? existing.chunkCount);
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
for (const section of sections.values()) {
|
|
1789
|
+
section.chunkIds.sort((left, right) => {
|
|
1790
|
+
const leftNode = nodes.find((node) => node.chunkId === left);
|
|
1791
|
+
const rightNode = nodes.find((node) => node.chunkId === right);
|
|
1792
|
+
const leftIndex = leftNode?.structure?.sequence?.sectionChunkIndex ?? Number.MAX_SAFE_INTEGER;
|
|
1793
|
+
const rightIndex = rightNode?.structure?.sequence?.sectionChunkIndex ?? Number.MAX_SAFE_INTEGER;
|
|
1794
|
+
if (leftIndex !== rightIndex) {
|
|
1795
|
+
return leftIndex - rightIndex;
|
|
1796
|
+
}
|
|
1797
|
+
return left.localeCompare(right);
|
|
1798
|
+
});
|
|
1799
|
+
section.leadChunkId = section.chunkIds[0];
|
|
1800
|
+
}
|
|
1801
|
+
const sectionPathIndex = new Map;
|
|
1802
|
+
for (const section of sections.values()) {
|
|
1803
|
+
const path = section.path && section.path.length > 0 ? section.path : section.title ? [section.title] : undefined;
|
|
1804
|
+
if (path && path.length > 0) {
|
|
1805
|
+
sectionPathIndex.set(path.join("\x00"), section);
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
for (const section of sections.values()) {
|
|
1809
|
+
const path = section.path && section.path.length > 0 ? section.path : section.title ? [section.title] : undefined;
|
|
1810
|
+
if (!path || path.length < 2) {
|
|
1811
|
+
continue;
|
|
1812
|
+
}
|
|
1813
|
+
const parent = sectionPathIndex.get(path.slice(0, -1).join("\x00"));
|
|
1814
|
+
if (!parent || parent.id === section.id) {
|
|
1815
|
+
continue;
|
|
1816
|
+
}
|
|
1817
|
+
section.parentSectionId = parent.id;
|
|
1818
|
+
if (!parent.childSectionIds.includes(section.id)) {
|
|
1819
|
+
parent.childSectionIds.push(section.id);
|
|
1820
|
+
}
|
|
1821
|
+
if (parent.leadChunkId && section.leadChunkId) {
|
|
1822
|
+
const parentKey = `section_parent:${section.leadChunkId}:${parent.leadChunkId}`;
|
|
1823
|
+
if (!edgeKeys.has(parentKey)) {
|
|
1824
|
+
edgeKeys.add(parentKey);
|
|
1825
|
+
edges.push({
|
|
1826
|
+
fromChunkId: section.leadChunkId,
|
|
1827
|
+
relation: "section_parent",
|
|
1828
|
+
toChunkId: parent.leadChunkId
|
|
1829
|
+
});
|
|
1830
|
+
}
|
|
1831
|
+
const childKey = `section_child:${parent.leadChunkId}:${section.leadChunkId}`;
|
|
1832
|
+
if (!edgeKeys.has(childKey)) {
|
|
1833
|
+
edgeKeys.add(childKey);
|
|
1834
|
+
edges.push({
|
|
1835
|
+
fromChunkId: parent.leadChunkId,
|
|
1836
|
+
relation: "section_child",
|
|
1837
|
+
toChunkId: section.leadChunkId
|
|
1838
|
+
});
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
nodes.sort((left, right) => {
|
|
1843
|
+
const leftSection = left.structure?.sequence?.sectionChunkIndex ?? Number.MAX_SAFE_INTEGER;
|
|
1844
|
+
const rightSection = right.structure?.sequence?.sectionChunkIndex ?? Number.MAX_SAFE_INTEGER;
|
|
1845
|
+
if (leftSection !== rightSection) {
|
|
1846
|
+
return leftSection - rightSection;
|
|
1847
|
+
}
|
|
1848
|
+
const leftScore = left.score ?? Number.NEGATIVE_INFINITY;
|
|
1849
|
+
const rightScore = right.score ?? Number.NEGATIVE_INFINITY;
|
|
1850
|
+
if (leftScore !== rightScore) {
|
|
1851
|
+
return rightScore - leftScore;
|
|
1852
|
+
}
|
|
1853
|
+
return left.label.localeCompare(right.label);
|
|
1854
|
+
});
|
|
1855
|
+
return {
|
|
1856
|
+
edges,
|
|
1857
|
+
nodes,
|
|
1858
|
+
sections: [...sections.values()].sort((left, right) => (left.title ?? left.id).localeCompare(right.title ?? right.id))
|
|
1859
|
+
};
|
|
1860
|
+
};
|
|
1861
|
+
var buildRAGChunkPreviewGraph = (preview) => buildRAGChunkGraph(preview.chunks.map((chunk) => ({
|
|
1862
|
+
chunkId: chunk.chunkId,
|
|
1863
|
+
labels: chunk.labels,
|
|
1864
|
+
metadata: chunk.metadata,
|
|
1865
|
+
source: chunk.source ?? preview.document.source,
|
|
1866
|
+
structure: chunk.structure,
|
|
1867
|
+
title: chunk.title ?? preview.document.title
|
|
1868
|
+
})));
|
|
1869
|
+
var buildRAGChunkPreviewNavigation = (preview, activeChunkId) => buildRAGChunkGraphNavigation(buildRAGChunkPreviewGraph(preview), activeChunkId);
|
|
1870
|
+
var buildRAGChunkGraphNavigation = (graph, activeChunkId) => {
|
|
1871
|
+
if (graph.nodes.length === 0) {
|
|
1872
|
+
return {
|
|
1873
|
+
activeChunkId,
|
|
1874
|
+
childSections: [],
|
|
1875
|
+
siblingSections: [],
|
|
1876
|
+
sectionNodes: []
|
|
1877
|
+
};
|
|
1878
|
+
}
|
|
1879
|
+
const activeNode = (activeChunkId ? graph.nodes.find((node) => node.chunkId === activeChunkId) : undefined) ?? graph.nodes[0];
|
|
1880
|
+
const resolvedActiveChunkId = activeNode?.chunkId;
|
|
1881
|
+
const previousNode = activeNode?.structure?.sequence?.previousChunkId ? graph.nodes.find((node) => node.chunkId === activeNode.structure?.sequence?.previousChunkId) : undefined;
|
|
1882
|
+
const nextNode = activeNode?.structure?.sequence?.nextChunkId ? graph.nodes.find((node) => node.chunkId === activeNode.structure?.sequence?.nextChunkId) : undefined;
|
|
1883
|
+
const section = activeNode?.structure?.sequence?.sectionChunkId ? graph.sections.find((entry) => entry.id === activeNode.structure?.sequence?.sectionChunkId) : undefined;
|
|
1884
|
+
const parentSection = section?.parentSectionId ? graph.sections.find((entry) => entry.id === section.parentSectionId) : undefined;
|
|
1885
|
+
const childSections = section ? section.childSectionIds.map((sectionId) => graph.sections.find((entry) => entry.id === sectionId)).filter((entry) => Boolean(entry)) : [];
|
|
1886
|
+
const siblingSections = section?.parentSectionId ? graph.sections.filter((entry) => entry.parentSectionId === section.parentSectionId && entry.id !== section.id) : [];
|
|
1887
|
+
const sectionNodes = section ? section.chunkIds.map((chunkId) => graph.nodes.find((node) => node.chunkId === chunkId)).filter((node) => Boolean(node)) : activeNode ? [activeNode] : [];
|
|
1888
|
+
return {
|
|
1889
|
+
activeChunkId: resolvedActiveChunkId,
|
|
1890
|
+
activeNode,
|
|
1891
|
+
childSections,
|
|
1892
|
+
nextNode,
|
|
1893
|
+
parentSection,
|
|
1894
|
+
previousNode,
|
|
1895
|
+
section,
|
|
1896
|
+
siblingSections,
|
|
1897
|
+
sectionNodes
|
|
1898
|
+
};
|
|
1899
|
+
};
|
|
1900
|
+
var buildRAGRetrievedState = (messages) => {
|
|
1901
|
+
const message = getLatestRetrievedMessage(messages);
|
|
1902
|
+
if (!message) {
|
|
1903
|
+
return null;
|
|
1904
|
+
}
|
|
1905
|
+
const sources = message.sources ?? [];
|
|
1906
|
+
const citations = buildRAGCitations(sources);
|
|
1907
|
+
const sectionDiagnostics = buildRAGSectionRetrievalDiagnostics(sources, isRAGRetrievalTrace(message.retrievalTrace) ? message.retrievalTrace : undefined);
|
|
1908
|
+
const sourceSummaries = buildRAGSourceSummaries(sources);
|
|
1909
|
+
const groundedAnswer = buildRAGGroundedAnswer(message.content, sources);
|
|
1910
|
+
return {
|
|
1911
|
+
citationReferenceMap: buildRAGCitationReferenceMap(citations),
|
|
1912
|
+
citations,
|
|
1913
|
+
conversationId: message.conversationId,
|
|
1914
|
+
excerptModeCounts: buildRAGExcerptModeCounts([
|
|
1915
|
+
...citations.map((citation) => citation.excerptSelection),
|
|
1916
|
+
...sourceSummaries.map((summary) => summary.excerptSelection)
|
|
1917
|
+
]),
|
|
1918
|
+
groundedAnswer,
|
|
1919
|
+
messageId: message.id,
|
|
1920
|
+
retrievalDurationMs: message.retrievalDurationMs,
|
|
1921
|
+
retrievalStartedAt: message.retrievalStartedAt,
|
|
1922
|
+
retrievedAt: message.retrievedAt,
|
|
1923
|
+
trace: isRAGRetrievalTrace(message.retrievalTrace) ? message.retrievalTrace : undefined,
|
|
1924
|
+
sectionDiagnostics,
|
|
1925
|
+
sourceGroups: buildRAGSourceGroups(sources),
|
|
1926
|
+
sourceSummaries,
|
|
1927
|
+
sources
|
|
1928
|
+
};
|
|
1929
|
+
};
|
|
1930
|
+
var buildRAGSourceSummaries = (sources) => {
|
|
1931
|
+
const sourceGroups = buildRAGSourceGroups(sources);
|
|
1932
|
+
const citations = buildRAGCitations(sources);
|
|
1933
|
+
const citationReferenceMap = buildRAGCitationReferenceMap(citations);
|
|
1934
|
+
return sourceGroups.map((group) => {
|
|
1935
|
+
const groupCitations = citations.filter((citation) => group.chunks.some((chunk) => chunk.chunkId === citation.chunkId));
|
|
1936
|
+
const leadChunk = getPreferredSourceLeadChunk(group.chunks);
|
|
1937
|
+
const excerpts = leadChunk ? buildRAGChunkExcerpts(group.chunks, leadChunk.chunkId) : undefined;
|
|
1938
|
+
const structure = leadChunk?.structure ?? buildRAGChunkStructure(leadChunk?.metadata);
|
|
1939
|
+
const excerptSelection = buildRAGExcerptSelection(excerpts, structure);
|
|
1940
|
+
return {
|
|
1941
|
+
bestScore: group.bestScore,
|
|
1942
|
+
citationNumbers: groupCitations.map((citation) => citationReferenceMap[citation.chunkId] ?? 0),
|
|
1943
|
+
citations: groupCitations,
|
|
1944
|
+
chunkIds: group.chunks.map((chunk) => chunk.chunkId),
|
|
1945
|
+
contextLabel: leadChunk?.labels?.contextLabel ?? buildContextLabel2(leadChunk?.metadata),
|
|
1946
|
+
count: group.count,
|
|
1947
|
+
excerpt: excerptSelection.excerpt || buildExcerpt2(leadChunk?.text ?? ""),
|
|
1948
|
+
excerpts,
|
|
1949
|
+
excerptSelection,
|
|
1950
|
+
key: group.key,
|
|
1951
|
+
label: group.label,
|
|
1952
|
+
locatorLabel: leadChunk?.labels?.locatorLabel ?? buildLocatorLabel2(leadChunk?.metadata, leadChunk?.source, leadChunk?.title),
|
|
1953
|
+
provenanceLabel: leadChunk?.labels?.provenanceLabel ?? buildProvenanceLabel2(leadChunk?.metadata),
|
|
1954
|
+
structure,
|
|
1955
|
+
source: group.source,
|
|
1956
|
+
title: group.title
|
|
1957
|
+
};
|
|
1958
|
+
});
|
|
1959
|
+
};
|
|
1960
|
+
var getSectionPathFromSource = (source) => {
|
|
1961
|
+
const path = source.structure?.section?.path ?? (Array.isArray(source.metadata?.sectionPath) ? source.metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : []);
|
|
1962
|
+
return path.length > 0 ? path : undefined;
|
|
1963
|
+
};
|
|
1964
|
+
var isBlockAwareContextLabel = (value) => typeof value === "string" && (value.startsWith("PDF ") || value.startsWith("Office ") || value.startsWith("Slide "));
|
|
1965
|
+
var getStructuredSectionScoreWeight = (metadata) => {
|
|
1966
|
+
if (!metadata) {
|
|
1967
|
+
return 1;
|
|
1968
|
+
}
|
|
1969
|
+
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1970
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1971
|
+
const sectionKind = getContextString2(metadata.sectionKind);
|
|
1972
|
+
const slideTitle = getContextString2(metadata.slideTitle);
|
|
1973
|
+
const slideNotesText = getContextString2(metadata.slideNotesText);
|
|
1974
|
+
if (pdfTextKind === "table_like") {
|
|
1975
|
+
return 1.28;
|
|
1976
|
+
}
|
|
1977
|
+
if (officeBlockKind === "table" || officeBlockKind === "list") {
|
|
1978
|
+
return 1.24;
|
|
1979
|
+
}
|
|
1980
|
+
if (sectionKind === "pdf_block" || sectionKind === "office_block" || officeBlockKind === "paragraph" || pdfTextKind === "paragraph") {
|
|
1981
|
+
return 1.12;
|
|
1982
|
+
}
|
|
1983
|
+
if (sectionKind === "presentation_slide" && slideNotesText) {
|
|
1984
|
+
return 1.2;
|
|
1985
|
+
}
|
|
1986
|
+
if (sectionKind === "presentation_slide" && slideTitle) {
|
|
1987
|
+
return 1.14;
|
|
1988
|
+
}
|
|
1989
|
+
return 1;
|
|
1990
|
+
};
|
|
1991
|
+
var getStructuredSourceLeadScore = (source) => source.score * getStructuredSectionScoreWeight(source.metadata);
|
|
1992
|
+
var getPreferredSourceLeadChunk = (chunks) => chunks.slice().sort((left, right) => {
|
|
1993
|
+
const leftWeightedScore = getStructuredSourceLeadScore(left);
|
|
1994
|
+
const rightWeightedScore = getStructuredSourceLeadScore(right);
|
|
1995
|
+
if (rightWeightedScore !== leftWeightedScore) {
|
|
1996
|
+
return rightWeightedScore - leftWeightedScore;
|
|
1997
|
+
}
|
|
1998
|
+
if (right.score !== left.score) {
|
|
1999
|
+
return right.score - left.score;
|
|
2000
|
+
}
|
|
2001
|
+
return left.chunkId.localeCompare(right.chunkId);
|
|
2002
|
+
})[0];
|
|
2003
|
+
var buildRAGSectionRetrievalDiagnostics = (sources, trace) => {
|
|
2004
|
+
const totalScore = sources.reduce((sum, source) => sum + source.score * getStructuredSectionScoreWeight(source.metadata), 0);
|
|
2005
|
+
if (sources.length === 0 || totalScore <= 0) {
|
|
2006
|
+
return [];
|
|
2007
|
+
}
|
|
2008
|
+
const sections = new Map;
|
|
2009
|
+
for (const source of sources) {
|
|
2010
|
+
const structuredScore = source.score * getStructuredSectionScoreWeight(source.metadata);
|
|
2011
|
+
const path = getSectionPathFromSource(source);
|
|
2012
|
+
if (!path) {
|
|
2013
|
+
continue;
|
|
2014
|
+
}
|
|
2015
|
+
const key = path.join(" > ");
|
|
2016
|
+
const label = path.at(-1) ?? key;
|
|
2017
|
+
const parentLabel = path.length > 1 ? path.slice(0, -1).join(" > ") : undefined;
|
|
2018
|
+
const existing = sections.get(key);
|
|
2019
|
+
const channels = Array.isArray(source.metadata?.retrievalChannels) ? source.metadata.retrievalChannels.filter((value) => value === "vector" || value === "lexical") : [];
|
|
2020
|
+
const isHybrid = channels.includes("vector") && channels.includes("lexical");
|
|
2021
|
+
const vectorHits = channels.includes("vector") ? 1 : 0;
|
|
2022
|
+
const lexicalHits = channels.includes("lexical") ? 1 : 0;
|
|
2023
|
+
const hybridHits = isHybrid ? 1 : 0;
|
|
2024
|
+
const queryOrigins = Array.isArray(source.metadata?.retrievalQueryOrigins) ? source.metadata.retrievalQueryOrigins.filter((value) => value === "primary" || value === "transformed" || value === "variant") : source.metadata?.retrievalQueryOrigin === "primary" || source.metadata?.retrievalQueryOrigin === "transformed" || source.metadata?.retrievalQueryOrigin === "variant" ? [source.metadata.retrievalQueryOrigin] : [];
|
|
2025
|
+
const primaryHits = queryOrigins.includes("primary") ? 1 : 0;
|
|
2026
|
+
const transformedHits = queryOrigins.includes("transformed") ? 1 : 0;
|
|
2027
|
+
const variantHits = queryOrigins.includes("variant") ? 1 : 0;
|
|
2028
|
+
if (!existing) {
|
|
2029
|
+
sections.set(key, {
|
|
2030
|
+
bestScore: source.score,
|
|
2031
|
+
count: 1,
|
|
2032
|
+
hybridHits,
|
|
2033
|
+
key,
|
|
2034
|
+
label,
|
|
2035
|
+
lexicalHits,
|
|
2036
|
+
parentLabel,
|
|
2037
|
+
path,
|
|
2038
|
+
primaryHits,
|
|
2039
|
+
sourceSet: new Set(source.source ? [source.source] : []),
|
|
2040
|
+
topChunkId: source.chunkId,
|
|
2041
|
+
topSource: source.source,
|
|
2042
|
+
totalScore: structuredScore,
|
|
2043
|
+
transformedHits,
|
|
2044
|
+
variantHits,
|
|
2045
|
+
vectorHits
|
|
2046
|
+
});
|
|
2047
|
+
continue;
|
|
2048
|
+
}
|
|
2049
|
+
existing.count += 1;
|
|
2050
|
+
existing.totalScore += structuredScore;
|
|
2051
|
+
if (source.source) {
|
|
2052
|
+
existing.sourceSet.add(source.source);
|
|
2053
|
+
}
|
|
2054
|
+
existing.vectorHits += vectorHits;
|
|
2055
|
+
existing.lexicalHits += lexicalHits;
|
|
2056
|
+
existing.hybridHits += hybridHits;
|
|
2057
|
+
existing.primaryHits += primaryHits;
|
|
2058
|
+
existing.transformedHits += transformedHits;
|
|
2059
|
+
existing.variantHits += variantHits;
|
|
2060
|
+
if (source.score > existing.bestScore) {
|
|
2061
|
+
existing.bestScore = source.score;
|
|
2062
|
+
existing.topChunkId = source.chunkId;
|
|
2063
|
+
existing.topSource = source.source;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
const diagnostics = [...sections.values()];
|
|
2067
|
+
const strongestBestHit = diagnostics.reduce((highest, section) => Math.max(highest, section.bestScore), 0);
|
|
2068
|
+
const parentLabelByKey = new Map(diagnostics.map((section) => [section.key, section.parentLabel]));
|
|
2069
|
+
const stageSectionCounts = new Map((trace?.steps ?? []).filter((step) => Array.isArray(step.sectionCounts) && step.sectionCounts.length > 0).map((step) => [step.stage, step.sectionCounts ?? []]));
|
|
2070
|
+
const stageSectionScores = new Map((trace?.steps ?? []).filter((step) => Array.isArray(step.sectionScores) && step.sectionScores.length > 0).map((step) => [step.stage, step.sectionScores ?? []]));
|
|
2071
|
+
return diagnostics.map((section) => {
|
|
2072
|
+
const siblingPool = diagnostics.filter((entry) => entry.parentLabel === section.parentLabel);
|
|
2073
|
+
const siblings = siblingPool.filter((entry) => entry.key !== section.key);
|
|
2074
|
+
const strongestSibling = siblings.slice().sort((left, right) => right.totalScore - left.totalScore)[0];
|
|
2075
|
+
const parentTotal = siblingPool.reduce((sum, entry) => sum + entry.totalScore, 0);
|
|
2076
|
+
const scoreShare = section.totalScore / totalScore;
|
|
2077
|
+
const parentShare = parentTotal > 0 ? section.totalScore / parentTotal : undefined;
|
|
2078
|
+
const topChunk = sources.find((source) => source.chunkId === section.topChunkId);
|
|
2079
|
+
const topContextLabel = topChunk?.labels?.contextLabel ?? buildContextLabel2(topChunk?.metadata);
|
|
2080
|
+
const topLocatorLabel = topChunk?.labels?.locatorLabel ?? buildLocatorLabel2(topChunk?.metadata, topChunk?.source, topChunk?.title);
|
|
2081
|
+
const sourceAwareChunkReason = formatSourceAwareChunkReason(topChunk?.metadata?.sourceAwareChunkReason);
|
|
2082
|
+
const sourceAwareUnitScopeLabel = buildSourceAwareUnitScopeLabel(topChunk?.metadata);
|
|
2083
|
+
const parentDistribution = parentTotal > 0 ? siblingPool.map((entry) => ({
|
|
2084
|
+
count: entry.count,
|
|
2085
|
+
isActive: entry.key === section.key,
|
|
2086
|
+
key: entry.key,
|
|
2087
|
+
label: entry.label,
|
|
2088
|
+
parentShare: entry.totalScore / parentTotal,
|
|
2089
|
+
totalScore: entry.totalScore
|
|
2090
|
+
})).sort((left, right) => right.totalScore - left.totalScore) : [];
|
|
2091
|
+
const reasons = [];
|
|
2092
|
+
const stageCounts = trace?.steps.map((step) => ({
|
|
2093
|
+
count: step.sectionCounts?.find((entry) => entry.key === section.key)?.count ?? 0,
|
|
2094
|
+
stage: step.stage
|
|
2095
|
+
})).filter((entry) => entry.count > 0) ?? [];
|
|
2096
|
+
const stageWeights = stageCounts.map((entry) => {
|
|
2097
|
+
const previousStageEntry = stageCounts[stageCounts.findIndex((candidate) => candidate.stage === entry.stage) - 1];
|
|
2098
|
+
const stageEntries = stageSectionCounts.get(entry.stage)?.filter((candidate) => candidate.count > 0) ?? [];
|
|
2099
|
+
const stageScoreEntries = stageSectionScores.get(entry.stage)?.filter((candidate) => candidate.totalScore > 0) ?? [];
|
|
2100
|
+
const stageTotal = stageEntries.reduce((sum, candidate) => sum + candidate.count, 0);
|
|
2101
|
+
const stageScoreTotal = stageScoreEntries.reduce((sum, candidate) => sum + candidate.totalScore, 0);
|
|
2102
|
+
const siblingStageEntries = stageEntries.filter((candidate) => candidate.key !== section.key && parentLabelByKey.get(candidate.key) === section.parentLabel);
|
|
2103
|
+
const parentStageEntries = stageEntries.filter((candidate) => parentLabelByKey.get(candidate.key) === section.parentLabel);
|
|
2104
|
+
const siblingStageScoreEntries = stageScoreEntries.filter((candidate) => candidate.key !== section.key && parentLabelByKey.get(candidate.key) === section.parentLabel);
|
|
2105
|
+
const parentStageScoreEntries = stageScoreEntries.filter((candidate) => parentLabelByKey.get(candidate.key) === section.parentLabel);
|
|
2106
|
+
const strongestStageSibling = siblingStageEntries.slice().sort((left, right) => right.count - left.count)[0];
|
|
2107
|
+
const parentStageTotal = parentStageEntries.reduce((sum, candidate) => sum + candidate.count, 0);
|
|
2108
|
+
const activeStageScore = stageScoreEntries.find((candidate) => candidate.key === section.key)?.totalScore;
|
|
2109
|
+
const strongestStageScoreSibling = siblingStageScoreEntries.slice().sort((left, right) => right.totalScore - left.totalScore)[0];
|
|
2110
|
+
const parentStageScoreTotal = parentStageScoreEntries.reduce((sum, candidate) => sum + candidate.totalScore, 0);
|
|
2111
|
+
const stageShare = stageTotal > 0 ? entry.count / stageTotal : 0;
|
|
2112
|
+
const retentionRate = typeof previousStageEntry?.count === "number" && previousStageEntry.count > 0 ? entry.count / previousStageEntry.count : undefined;
|
|
2113
|
+
const countDelta = typeof previousStageEntry?.count === "number" ? entry.count - previousStageEntry.count : undefined;
|
|
2114
|
+
const parentStageShare = parentStageTotal > 0 ? entry.count / parentStageTotal : undefined;
|
|
2115
|
+
const stageScoreShare = typeof activeStageScore === "number" && stageScoreTotal > 0 ? activeStageScore / stageScoreTotal : undefined;
|
|
2116
|
+
const parentStageScoreShare = typeof activeStageScore === "number" && parentStageScoreTotal > 0 ? activeStageScore / parentStageScoreTotal : undefined;
|
|
2117
|
+
const stageShareGap = stageTotal > 0 && strongestStageSibling ? entry.count / stageTotal - strongestStageSibling.count / stageTotal : undefined;
|
|
2118
|
+
const parentStageShareGap = parentStageTotal > 0 && strongestStageSibling ? entry.count / parentStageTotal - strongestStageSibling.count / parentStageTotal : undefined;
|
|
2119
|
+
const stageScoreShareGap = typeof activeStageScore === "number" && stageScoreTotal > 0 && strongestStageScoreSibling ? activeStageScore / stageScoreTotal - strongestStageScoreSibling.totalScore / stageScoreTotal : undefined;
|
|
2120
|
+
const parentStageScoreShareGap = typeof activeStageScore === "number" && parentStageScoreTotal > 0 && strongestStageScoreSibling ? activeStageScore / parentStageScoreTotal - strongestStageScoreSibling.totalScore / parentStageScoreTotal : undefined;
|
|
2121
|
+
const reasons2 = [];
|
|
2122
|
+
if (entry.stage === "rerank" && stageShare > 0.5 && (typeof stageShareGap !== "number" || stageShareGap > 0)) {
|
|
2123
|
+
reasons2.push("rerank_preserved_lead");
|
|
2124
|
+
}
|
|
2125
|
+
if (entry.stage === "finalize" && stageShare >= 0.5) {
|
|
2126
|
+
reasons2.push("final_stage_concentration");
|
|
2127
|
+
}
|
|
2128
|
+
if (entry.stage === "finalize" && typeof parentStageShare === "number" && parentStageShare >= 0.6 && (typeof parentStageShareGap !== "number" || parentStageShareGap > 0)) {
|
|
2129
|
+
reasons2.push("final_stage_dominant_within_parent");
|
|
2130
|
+
}
|
|
2131
|
+
if (strongestStageSibling && (typeof stageShareGap === "number" && stageShareGap <= 0.1 || typeof parentStageShareGap === "number" && parentStageShareGap <= 0.1)) {
|
|
2132
|
+
reasons2.push("stage_runner_up_pressure");
|
|
2133
|
+
}
|
|
2134
|
+
if (typeof countDelta === "number") {
|
|
2135
|
+
if (countDelta > 0) {
|
|
2136
|
+
reasons2.push("stage_expanded");
|
|
2137
|
+
} else if (countDelta < 0) {
|
|
2138
|
+
reasons2.push("stage_narrowed");
|
|
2139
|
+
} else {
|
|
2140
|
+
reasons2.push("stage_held");
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
return {
|
|
2144
|
+
count: entry.count,
|
|
2145
|
+
countDelta,
|
|
2146
|
+
parentStageScoreShare,
|
|
2147
|
+
parentStageShare,
|
|
2148
|
+
parentStageShareGap,
|
|
2149
|
+
previousCount: previousStageEntry?.count,
|
|
2150
|
+
previousStage: previousStageEntry?.stage,
|
|
2151
|
+
reasons: reasons2,
|
|
2152
|
+
retentionRate,
|
|
2153
|
+
stage: entry.stage,
|
|
2154
|
+
stageScoreShare,
|
|
2155
|
+
stageScoreShareGap,
|
|
2156
|
+
stageShare,
|
|
2157
|
+
stageShareGap,
|
|
2158
|
+
totalScore: activeStageScore,
|
|
2159
|
+
strongestSiblingCount: strongestStageSibling?.count,
|
|
2160
|
+
strongestSiblingLabel: strongestStageSibling ? diagnostics.find((candidate) => candidate.key === strongestStageSibling.key)?.label ?? strongestStageSibling.key : undefined
|
|
2161
|
+
};
|
|
2162
|
+
});
|
|
2163
|
+
const firstSeenStage = stageCounts[0]?.stage;
|
|
2164
|
+
const lastSeenStage = stageCounts.at(-1)?.stage;
|
|
2165
|
+
const peakStageEntry = stageCounts.reduce((highest, entry) => !highest || entry.count > highest.count ? entry : highest, undefined);
|
|
2166
|
+
const finalStageEntry = stageCounts.at(-1);
|
|
2167
|
+
const peakCount = peakStageEntry?.count ?? section.count;
|
|
2168
|
+
const finalCount = finalStageEntry?.count;
|
|
2169
|
+
const finalRetentionRate = typeof finalCount === "number" && peakCount > 0 ? finalCount / peakCount : undefined;
|
|
2170
|
+
const dropFromPeak = typeof finalCount === "number" ? peakCount - finalCount : undefined;
|
|
2171
|
+
const queryAttributionReasons = [];
|
|
2172
|
+
const queryAttributionMode = section.primaryHits > 0 && section.transformedHits === 0 && section.variantHits === 0 ? "primary" : section.transformedHits > 0 && section.primaryHits === 0 && section.variantHits === 0 ? "transformed" : section.variantHits > 0 && section.primaryHits === 0 && section.transformedHits === 0 ? "variant" : "mixed";
|
|
2173
|
+
if (queryAttributionMode === "primary") {
|
|
2174
|
+
queryAttributionReasons.push("base_query_only");
|
|
2175
|
+
}
|
|
2176
|
+
if (queryAttributionMode === "transformed") {
|
|
2177
|
+
queryAttributionReasons.push("transformed_query_only");
|
|
2178
|
+
queryAttributionReasons.push("transform_introduced");
|
|
2179
|
+
}
|
|
2180
|
+
if (queryAttributionMode === "variant") {
|
|
2181
|
+
queryAttributionReasons.push("variant_only");
|
|
2182
|
+
queryAttributionReasons.push("variant_supported");
|
|
2183
|
+
}
|
|
2184
|
+
if (queryAttributionMode === "mixed") {
|
|
2185
|
+
queryAttributionReasons.push("mixed_query_sources");
|
|
2186
|
+
if (section.variantHits > 0) {
|
|
2187
|
+
queryAttributionReasons.push("variant_supported");
|
|
2188
|
+
}
|
|
2189
|
+
if (section.transformedHits > 0 && section.primaryHits === 0) {
|
|
2190
|
+
queryAttributionReasons.push("transform_introduced");
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
if (section.bestScore >= strongestBestHit) {
|
|
2194
|
+
reasons.push("best_hit");
|
|
2195
|
+
}
|
|
2196
|
+
if (section.count > 1) {
|
|
2197
|
+
reasons.push("multi_hit_section");
|
|
2198
|
+
}
|
|
2199
|
+
if (siblings.length === 0) {
|
|
2200
|
+
reasons.push("only_section_in_parent");
|
|
2201
|
+
} else if (!strongestSibling || section.totalScore >= strongestSibling.totalScore) {
|
|
2202
|
+
reasons.push("dominant_within_parent");
|
|
2203
|
+
}
|
|
2204
|
+
if (scoreShare >= 0.5 || (parentShare ?? 0) >= 0.6) {
|
|
2205
|
+
reasons.push("concentrated_evidence");
|
|
2206
|
+
}
|
|
2207
|
+
const summaryParts = [
|
|
2208
|
+
isBlockAwareContextLabel(topContextLabel) ? topContextLabel : "",
|
|
2209
|
+
`${section.count} hit${section.count === 1 ? "" : "s"}`,
|
|
2210
|
+
`${(scoreShare * 100).toFixed(0)}% score share`,
|
|
2211
|
+
`vector ${section.vectorHits} \xB7 lexical ${section.lexicalHits} \xB7 hybrid ${section.hybridHits}`,
|
|
2212
|
+
typeof parentShare === "number" ? `${(parentShare * 100).toFixed(0)}% of parent section set` : "",
|
|
2213
|
+
sourceAwareChunkReason ? `boundary ${sourceAwareChunkReason}` : "",
|
|
2214
|
+
sourceAwareUnitScopeLabel ? `scope ${sourceAwareUnitScopeLabel}` : "",
|
|
2215
|
+
strongestSibling ? `ahead of ${strongestSibling.label} by ${(section.totalScore - strongestSibling.totalScore).toFixed(2)}` : "no sibling competition"
|
|
2216
|
+
].filter(Boolean);
|
|
2217
|
+
return {
|
|
2218
|
+
averageScore: section.totalScore / section.count,
|
|
2219
|
+
bestScore: section.bestScore,
|
|
2220
|
+
count: section.count,
|
|
2221
|
+
key: section.key,
|
|
2222
|
+
label: section.label,
|
|
2223
|
+
parentLabel: section.parentLabel,
|
|
2224
|
+
parentDistribution,
|
|
2225
|
+
parentShare,
|
|
2226
|
+
parentShareGap: typeof parentShare === "number" && strongestSibling && parentTotal > 0 ? parentShare - strongestSibling.totalScore / parentTotal : undefined,
|
|
2227
|
+
path: section.path,
|
|
2228
|
+
firstSeenStage,
|
|
2229
|
+
finalCount,
|
|
2230
|
+
finalRetentionRate,
|
|
2231
|
+
lastSeenStage,
|
|
2232
|
+
dropFromPeak,
|
|
2233
|
+
peakCount,
|
|
2234
|
+
peakStage: peakStageEntry?.stage,
|
|
2235
|
+
queryAttribution: {
|
|
2236
|
+
mode: queryAttributionMode,
|
|
2237
|
+
primaryHits: section.primaryHits,
|
|
2238
|
+
reasons: queryAttributionReasons,
|
|
2239
|
+
transformedHits: section.transformedHits,
|
|
2240
|
+
variantHits: section.variantHits
|
|
2241
|
+
},
|
|
2242
|
+
requestedMode: trace?.requestedMode,
|
|
2243
|
+
retrievalMode: trace?.mode,
|
|
2244
|
+
routingLabel: trace?.routingLabel,
|
|
2245
|
+
routingProvider: trace?.routingProvider,
|
|
2246
|
+
routingReason: trace?.routingReason,
|
|
2247
|
+
queryTransformLabel: trace?.queryTransformLabel,
|
|
2248
|
+
queryTransformProvider: trace?.queryTransformProvider,
|
|
2249
|
+
queryTransformReason: trace?.queryTransformReason,
|
|
2250
|
+
reasons,
|
|
2251
|
+
rerankApplied: trace?.steps.some((step) => step.stage === "rerank" && step.metadata?.applied === true),
|
|
2252
|
+
scoreShare,
|
|
2253
|
+
scoreThresholdApplied: trace?.steps.some((step) => step.stage === "score_filter"),
|
|
2254
|
+
stageCounts,
|
|
2255
|
+
stageWeights,
|
|
2256
|
+
siblingCount: siblings.length,
|
|
2257
|
+
siblingScoreGap: strongestSibling ? section.totalScore - strongestSibling.totalScore : undefined,
|
|
2258
|
+
sourceCount: section.sourceSet.size,
|
|
2259
|
+
sourceAwareChunkReasonLabel: sourceAwareChunkReason,
|
|
2260
|
+
sourceAwareUnitScopeLabel,
|
|
2261
|
+
topContextLabel,
|
|
2262
|
+
topLocatorLabel,
|
|
2263
|
+
sourceBalanceApplied: trace?.steps.some((step) => step.stage === "source_balance"),
|
|
2264
|
+
strongestSiblingLabel: strongestSibling?.label,
|
|
2265
|
+
strongestSiblingScore: strongestSibling?.totalScore,
|
|
2266
|
+
summary: summaryParts.join(" \xB7 "),
|
|
2267
|
+
topChunkId: section.topChunkId,
|
|
2268
|
+
topSource: section.topSource,
|
|
2269
|
+
totalScore: section.totalScore,
|
|
2270
|
+
hybridHits: section.hybridHits,
|
|
2271
|
+
lexicalHits: section.lexicalHits,
|
|
2272
|
+
vectorHits: section.vectorHits
|
|
2273
|
+
};
|
|
2274
|
+
}).sort((left, right) => {
|
|
2275
|
+
if (right.totalScore !== left.totalScore) {
|
|
2276
|
+
return right.totalScore - left.totalScore;
|
|
2277
|
+
}
|
|
2278
|
+
if (right.bestScore !== left.bestScore) {
|
|
2279
|
+
return right.bestScore - left.bestScore;
|
|
2280
|
+
}
|
|
2281
|
+
return left.label.localeCompare(right.label);
|
|
2282
|
+
});
|
|
2283
|
+
};
|
|
2284
|
+
var buildStreamProgressState = (messages) => {
|
|
2285
|
+
const latestMessage = getLatestAssistantMessage(messages);
|
|
2286
|
+
const retrieved = latestMessage ? buildRAGRetrievedState(messages) : undefined;
|
|
2287
|
+
return {
|
|
2288
|
+
conversationId: latestMessage?.conversationId,
|
|
2289
|
+
latestMessage,
|
|
2290
|
+
messageId: latestMessage?.id,
|
|
2291
|
+
retrieved,
|
|
2292
|
+
sourceCount: retrieved?.sources.length ?? latestMessage?.sources?.length ?? 0
|
|
2293
|
+
};
|
|
2294
|
+
};
|
|
2295
|
+
var buildRAGStreamProgress = ({
|
|
2296
|
+
error,
|
|
2297
|
+
isStreaming,
|
|
2298
|
+
messages
|
|
2299
|
+
}) => {
|
|
2300
|
+
const stage = resolveRAGStreamStage({
|
|
2301
|
+
error,
|
|
2302
|
+
isStreaming,
|
|
2303
|
+
messages
|
|
2304
|
+
});
|
|
2305
|
+
const state = buildStreamProgressState(messages);
|
|
2306
|
+
const hasSources = state.sourceCount > 0;
|
|
2307
|
+
const hasRetrieved = stage === "retrieved" || state.retrieved !== undefined || state.latestMessage?.retrievedAt !== undefined;
|
|
2308
|
+
const hasThinking = typeof state.latestMessage?.thinking === "string" && state.latestMessage.thinking.length > 0;
|
|
2309
|
+
const hasToolCalls = (state.latestMessage?.toolCalls?.length ?? 0) > 0;
|
|
2310
|
+
return {
|
|
2311
|
+
conversationId: state.conversationId,
|
|
2312
|
+
hasContent: typeof state.latestMessage?.content === "string" && state.latestMessage.content.length > 0,
|
|
2313
|
+
hasRetrieved,
|
|
2314
|
+
hasSources,
|
|
2315
|
+
hasThinking,
|
|
2316
|
+
hasToolCalls,
|
|
2317
|
+
isComplete: stage === "complete",
|
|
2318
|
+
isError: stage === "error",
|
|
2319
|
+
isIdle: stage === "idle",
|
|
2320
|
+
isRetrieved: stage === "retrieved",
|
|
2321
|
+
isRetrieving: stage === "submitting" || stage === "retrieving",
|
|
2322
|
+
isStreaming: stage === "streaming",
|
|
2323
|
+
isSubmitting: stage === "submitting",
|
|
2324
|
+
latestMessage: state.latestMessage,
|
|
2325
|
+
messageId: state.messageId,
|
|
2326
|
+
retrievalDurationMs: state.retrieved?.retrievalDurationMs,
|
|
2327
|
+
retrievalStartedAt: state.retrieved?.retrievalStartedAt,
|
|
2328
|
+
retrievedAt: state.retrieved?.retrievedAt,
|
|
2329
|
+
sourceCount: state.sourceCount,
|
|
2330
|
+
stage
|
|
2331
|
+
};
|
|
2332
|
+
};
|
|
2333
|
+
var buildRAGAnswerWorkflowState = ({
|
|
2334
|
+
error,
|
|
2335
|
+
isStreaming,
|
|
2336
|
+
messages
|
|
2337
|
+
}) => {
|
|
2338
|
+
const latestAssistantMessage = getLatestAssistantMessage(messages);
|
|
2339
|
+
const sources = getLatestRAGSources(messages);
|
|
2340
|
+
const sourceGroups = buildRAGSourceGroups(sources);
|
|
2341
|
+
const citations = buildRAGCitations(sources);
|
|
2342
|
+
const citationReferenceMap = buildRAGCitationReferenceMap(citations);
|
|
2343
|
+
const sourceSummaries = buildRAGSourceSummaries(sources);
|
|
2344
|
+
const groundingReferences = buildRAGGroundingReferences(sources);
|
|
2345
|
+
const groundedAnswer = buildRAGGroundedAnswer(latestAssistantMessage?.content ?? "", sources);
|
|
2346
|
+
const retrieval = buildRAGRetrievedState(messages);
|
|
2347
|
+
const sectionDiagnostics = buildRAGSectionRetrievalDiagnostics(sources, retrieval?.trace);
|
|
2348
|
+
const progress = buildRAGStreamProgress({
|
|
2349
|
+
error,
|
|
2350
|
+
isStreaming,
|
|
2351
|
+
messages
|
|
2352
|
+
});
|
|
2353
|
+
return {
|
|
2354
|
+
excerptModeCounts: buildRAGExcerptModeCounts([
|
|
2355
|
+
...citations.map((citation) => citation.excerptSelection),
|
|
2356
|
+
...sourceSummaries.map((summary) => summary.excerptSelection),
|
|
2357
|
+
...groundingReferences.map((reference) => reference.excerptSelection),
|
|
2358
|
+
...groundedAnswer.sectionSummaries.map((summary) => summary.excerptSelection)
|
|
2359
|
+
]),
|
|
2360
|
+
citationReferenceMap,
|
|
2361
|
+
citations,
|
|
2362
|
+
coverage: groundedAnswer.coverage,
|
|
2363
|
+
error,
|
|
2364
|
+
groundedAnswer,
|
|
2365
|
+
groundingReferences,
|
|
2366
|
+
hasCitations: groundedAnswer.hasCitations,
|
|
2367
|
+
hasGrounding: groundingReferences.length > 0,
|
|
2368
|
+
hasRetrieved: progress.hasRetrieved,
|
|
2369
|
+
hasSources: sources.length > 0,
|
|
2370
|
+
isAnswerStreaming: progress.isStreaming,
|
|
2371
|
+
isComplete: progress.isComplete,
|
|
2372
|
+
isError: progress.isError,
|
|
2373
|
+
isIdle: progress.isIdle,
|
|
2374
|
+
isRetrieved: progress.isRetrieved,
|
|
2375
|
+
isRetrieving: progress.isRetrieving,
|
|
2376
|
+
isRunning: progress.isSubmitting || progress.isRetrieving || progress.isStreaming,
|
|
2377
|
+
isSubmitting: progress.isSubmitting,
|
|
2378
|
+
latestAssistantMessage,
|
|
2379
|
+
messages,
|
|
2380
|
+
retrieval,
|
|
2381
|
+
retrievalDurationMs: retrieval?.retrievalDurationMs,
|
|
2382
|
+
retrievalStartedAt: retrieval?.retrievalStartedAt,
|
|
2383
|
+
retrievedAt: retrieval?.retrievedAt,
|
|
2384
|
+
sectionDiagnostics,
|
|
2385
|
+
sourceGroups,
|
|
2386
|
+
sourceSummaries,
|
|
2387
|
+
sources,
|
|
2388
|
+
stage: progress.stage,
|
|
2389
|
+
ungroundedReferenceNumbers: groundedAnswer.ungroundedReferenceNumbers
|
|
2390
|
+
};
|
|
2391
|
+
};
|
|
2392
|
+
var buildRAGSourceGroups = (sources) => {
|
|
2393
|
+
const groups = new Map;
|
|
2394
|
+
for (const source of sources) {
|
|
2395
|
+
updateSourceGroup(groups, source);
|
|
2396
|
+
}
|
|
2397
|
+
return [...groups.values()].sort((left, right) => {
|
|
2398
|
+
if (right.bestScore !== left.bestScore) {
|
|
2399
|
+
return right.bestScore - left.bestScore;
|
|
2400
|
+
}
|
|
2401
|
+
return left.label.localeCompare(right.label);
|
|
2402
|
+
});
|
|
2403
|
+
};
|
|
2404
|
+
var buildSourceGroup = (source, key) => ({
|
|
2405
|
+
bestScore: source.score,
|
|
2406
|
+
chunks: [source],
|
|
2407
|
+
count: 1,
|
|
2408
|
+
key,
|
|
2409
|
+
label: buildSourceLabel2(source),
|
|
2410
|
+
labels: source.labels ?? buildRAGSourceLabels({
|
|
2411
|
+
metadata: source.metadata,
|
|
2412
|
+
source: source.source,
|
|
2413
|
+
title: source.title
|
|
2414
|
+
}),
|
|
2415
|
+
structure: source.structure ?? buildRAGChunkStructure(source.metadata),
|
|
2416
|
+
source: source.source,
|
|
2417
|
+
title: source.title
|
|
2418
|
+
});
|
|
2419
|
+
var updateSourceGroup = (groups, source) => {
|
|
2420
|
+
const key = buildSourceGroupKey(source);
|
|
2421
|
+
const existing = groups.get(key);
|
|
2422
|
+
if (!existing) {
|
|
2423
|
+
groups.set(key, buildSourceGroup(source, key));
|
|
2424
|
+
return;
|
|
2425
|
+
}
|
|
2426
|
+
existing.bestScore = Math.max(existing.bestScore, source.score);
|
|
2427
|
+
existing.count += 1;
|
|
2428
|
+
existing.chunks.push(source);
|
|
2429
|
+
const leadChunk = getPreferredSourceLeadChunk(existing.chunks);
|
|
2430
|
+
if (leadChunk) {
|
|
2431
|
+
existing.label = buildSourceLabel2(leadChunk);
|
|
2432
|
+
existing.labels = leadChunk.labels ?? buildRAGSourceLabels({
|
|
2433
|
+
metadata: leadChunk.metadata,
|
|
2434
|
+
source: leadChunk.source,
|
|
2435
|
+
title: leadChunk.title
|
|
2436
|
+
});
|
|
2437
|
+
existing.structure = leadChunk.structure ?? buildRAGChunkStructure(leadChunk.metadata);
|
|
2438
|
+
existing.source = leadChunk.source;
|
|
2439
|
+
existing.title = leadChunk.title;
|
|
2440
|
+
}
|
|
2441
|
+
};
|
|
2442
|
+
var getLatestAssistantMessage = (messages) => {
|
|
2443
|
+
for (let index = messages.length - 1;index >= 0; index -= 1) {
|
|
2444
|
+
const message = messages[index];
|
|
2445
|
+
if (message?.role === "assistant") {
|
|
2446
|
+
return message;
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
return;
|
|
2450
|
+
};
|
|
2451
|
+
var getLatestRAGSources = (messages) => getLatestAssistantMessage(messages)?.sources ?? [];
|
|
2452
|
+
var getLatestRetrievedMessage = (messages) => {
|
|
2453
|
+
for (let index = messages.length - 1;index >= 0; index -= 1) {
|
|
2454
|
+
const message = messages[index];
|
|
2455
|
+
if (message?.role === "assistant" && (typeof message.retrievedAt === "number" || (message.sources?.length ?? 0) > 0)) {
|
|
2456
|
+
return message;
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2459
|
+
return;
|
|
2460
|
+
};
|
|
2461
|
+
var resolveRAGStreamStage = ({
|
|
2462
|
+
error,
|
|
2463
|
+
isStreaming,
|
|
2464
|
+
messages
|
|
2465
|
+
}) => {
|
|
2466
|
+
if (error) {
|
|
2467
|
+
return "error";
|
|
2468
|
+
}
|
|
2469
|
+
const assistantMessage = getLatestAssistantMessage(messages);
|
|
2470
|
+
if (!assistantMessage) {
|
|
2471
|
+
return isStreaming ? "submitting" : "idle";
|
|
2472
|
+
}
|
|
2473
|
+
const isRetrieving = typeof assistantMessage.retrievalStartedAt === "number" && typeof assistantMessage.retrievedAt !== "number";
|
|
2474
|
+
if (isRetrieving) {
|
|
2475
|
+
return "retrieving";
|
|
2476
|
+
}
|
|
2477
|
+
if (!isStreaming) {
|
|
2478
|
+
return "complete";
|
|
2479
|
+
}
|
|
2480
|
+
const hasRetrieved = typeof assistantMessage.retrievedAt === "number";
|
|
2481
|
+
const hasContent = assistantMessage.content.trim().length > 0 || assistantMessage.thinking?.trim().length || (assistantMessage.toolCalls?.length ?? 0) > 0 || (assistantMessage.images?.length ?? 0) > 0;
|
|
2482
|
+
if (hasRetrieved && !hasContent) {
|
|
2483
|
+
return "retrieved";
|
|
2484
|
+
}
|
|
2485
|
+
return "streaming";
|
|
2486
|
+
};
|
|
2487
|
+
var formatSignedDelta = (value, decimals = 0, suffix = "") => `${value >= 0 ? "+" : ""}${value.toFixed(decimals)}${suffix}`;
|
|
2488
|
+
var formatEvaluationPassingRate = (value) => `${value.toFixed(1)}%`;
|
|
2489
|
+
var formatEvaluationSummary = (response) => `${response.summary.totalCases} total \xB7 f1 ${response.summary.averageF1.toFixed(3)} \xB7 latency ${response.summary.averageLatencyMs.toFixed(1)}ms`;
|
|
2490
|
+
var formatGroundingHistorySummaryValue = (response) => `${response.summary.passedCases}/${response.summary.totalCases} pass \xB7 grounded ${response.summary.groundedCases} \xB7 partial ${response.summary.partialCases} \xB7 ungrounded ${response.summary.ungroundedCases} \xB7 resolved citations ${(response.summary.averageResolvedCitationRate * 100).toFixed(1)}% \xB7 citation f1 ${response.summary.averageCitationF1.toFixed(3)}`;
|
|
2491
|
+
var formatHistoryCaseLabels = (cases) => cases.length > 0 ? cases.map((entry) => entry.label ?? entry.caseId).join(", ") : "none";
|
|
2492
|
+
var buildComparisonCaseLeadLabel = (snapshot) => snapshot?.topLocatorLabel ?? snapshot?.topContextLabel ?? snapshot?.sourceAwareUnitScopeLabel ?? snapshot?.sourceAwareChunkReasonLabel;
|
|
2493
|
+
var formatComparisonLeadCueSummary = (entry) => {
|
|
2494
|
+
const labels = Array.from(new Set((entry.caseTraceSnapshots ?? []).map((snapshot) => buildComparisonCaseLeadLabel(snapshot)).filter((value) => typeof value === "string"))).slice(0, 3);
|
|
2495
|
+
return labels.length > 0 ? labels.join(" \xB7 ") : "none";
|
|
2496
|
+
};
|
|
2497
|
+
var formatComparisonLeadMediaCueSummary = (entry) => {
|
|
2498
|
+
const labels = Array.from(new Set((entry.caseTraceSnapshots ?? []).map((snapshot) => formatLeadMediaCueSummary({
|
|
2499
|
+
leadSpeakerCue: snapshot.leadSpeakerCue,
|
|
2500
|
+
leadSpeakerAttributionCue: snapshot.leadSpeakerAttributionCue,
|
|
2501
|
+
leadChannelCue: snapshot.leadChannelCue,
|
|
2502
|
+
leadChannelAttributionCue: snapshot.leadChannelAttributionCue,
|
|
2503
|
+
leadContinuityCue: snapshot.leadContinuityCue
|
|
2504
|
+
})).filter((value) => typeof value === "string" && value !== "none"))).slice(0, 3);
|
|
2505
|
+
return labels.length > 0 ? labels.join(" \xB7 ") : "none";
|
|
2506
|
+
};
|
|
2507
|
+
var formatSQLitePlannerCueSummary = (input) => {
|
|
2508
|
+
const parts = [
|
|
2509
|
+
input?.sqliteQueryMode ? `mode ${input.sqliteQueryMode}` : undefined,
|
|
2510
|
+
typeof input?.sqliteQueryPushdownApplied === "boolean" ? input.sqliteQueryPushdownApplied ? "pushdown applied" : "pushdown skipped" : undefined,
|
|
2511
|
+
typeof input?.sqliteQueryPushdownClauseCount === "number" ? `clauses ${input.sqliteQueryPushdownClauseCount}` : undefined,
|
|
2512
|
+
typeof input?.sqliteQueryFilteredCandidates === "number" ? `filtered ${input.sqliteQueryFilteredCandidates}` : undefined,
|
|
2513
|
+
typeof input?.sqliteQueryInitialSearchK === "number" && typeof input?.sqliteQueryFinalSearchK === "number" ? `searchK ${input.sqliteQueryInitialSearchK}->${input.sqliteQueryFinalSearchK}` : typeof input?.sqliteQueryFinalSearchK === "number" ? `searchK ${input.sqliteQueryFinalSearchK}` : undefined,
|
|
2514
|
+
typeof input?.sqliteQueryBackfillCount === "number" ? `backfill ${input.sqliteQueryBackfillCount}` : undefined
|
|
2515
|
+
].filter((value) => typeof value === "string");
|
|
2516
|
+
return parts.length > 0 ? parts.join(" \xB7 ") : "none";
|
|
2517
|
+
};
|
|
2518
|
+
var formatComparisonSQLitePlannerCueSummary = (entry) => {
|
|
2519
|
+
const labels = Array.from(new Set((entry.caseTraceSnapshots ?? []).map((snapshot) => formatSQLitePlannerCueSummary({
|
|
2520
|
+
sqliteQueryBackfillCount: snapshot.sqliteQueryBackfillCount,
|
|
2521
|
+
sqliteQueryFilteredCandidates: snapshot.sqliteQueryFilteredCandidates,
|
|
2522
|
+
sqliteQueryFinalSearchK: snapshot.sqliteQueryFinalSearchK,
|
|
2523
|
+
sqliteQueryInitialSearchK: snapshot.sqliteQueryInitialSearchK,
|
|
2524
|
+
sqliteQueryMode: snapshot.sqliteQueryMode,
|
|
2525
|
+
sqliteQueryPushdownApplied: snapshot.sqliteQueryPushdownApplied,
|
|
2526
|
+
sqliteQueryPushdownClauseCount: snapshot.sqliteQueryPushdownClauseCount
|
|
2527
|
+
})).filter((value) => typeof value === "string" && value !== "none"))).slice(0, 3);
|
|
2528
|
+
return labels.length > 0 ? labels.join(" \xB7 ") : "none";
|
|
2529
|
+
};
|
|
2530
|
+
var formatComparisonLeadDrift = (entry, leader) => {
|
|
2531
|
+
if (!leader?.caseTraceSnapshots?.length || !entry.caseTraceSnapshots?.length) {
|
|
2532
|
+
return "none";
|
|
2533
|
+
}
|
|
2534
|
+
const leaderCases = new Map(leader.caseTraceSnapshots.map((snapshot) => [snapshot.caseId, snapshot]));
|
|
2535
|
+
const drift = entry.caseTraceSnapshots.map((snapshot) => {
|
|
2536
|
+
const leaderSnapshot = leaderCases.get(snapshot.caseId);
|
|
2537
|
+
const currentLead = buildComparisonCaseLeadLabel(snapshot);
|
|
2538
|
+
const leaderLead = buildComparisonCaseLeadLabel(leaderSnapshot);
|
|
2539
|
+
if (!currentLead || !leaderLead || currentLead === leaderLead) {
|
|
2540
|
+
return;
|
|
2541
|
+
}
|
|
2542
|
+
return `${snapshot.label ?? snapshot.caseId} ${leaderLead}\u2192${currentLead}`;
|
|
2543
|
+
}).filter((value) => typeof value === "string").slice(0, 3);
|
|
2544
|
+
return drift.length > 0 ? drift.join(" \xB7 ") : "none";
|
|
2545
|
+
};
|
|
2546
|
+
var formatComparisonLeadMediaDrift = (entry, leader) => {
|
|
2547
|
+
if (!leader?.caseTraceSnapshots?.length || !entry.caseTraceSnapshots?.length) {
|
|
2548
|
+
return "none";
|
|
2549
|
+
}
|
|
2550
|
+
const leaderCases = new Map(leader.caseTraceSnapshots.map((snapshot) => [snapshot.caseId, snapshot]));
|
|
2551
|
+
const drift = entry.caseTraceSnapshots.map((snapshot) => {
|
|
2552
|
+
const leaderSnapshot = leaderCases.get(snapshot.caseId);
|
|
2553
|
+
const currentMediaCues = formatLeadMediaCueSummary({
|
|
2554
|
+
leadSpeakerCue: snapshot.leadSpeakerCue,
|
|
2555
|
+
leadSpeakerAttributionCue: snapshot.leadSpeakerAttributionCue,
|
|
2556
|
+
leadChannelCue: snapshot.leadChannelCue,
|
|
2557
|
+
leadChannelAttributionCue: snapshot.leadChannelAttributionCue,
|
|
2558
|
+
leadContinuityCue: snapshot.leadContinuityCue
|
|
2559
|
+
});
|
|
2560
|
+
const leaderMediaCues = formatLeadMediaCueSummary({
|
|
2561
|
+
leadSpeakerCue: leaderSnapshot?.leadSpeakerCue,
|
|
2562
|
+
leadSpeakerAttributionCue: leaderSnapshot?.leadSpeakerAttributionCue,
|
|
2563
|
+
leadChannelCue: leaderSnapshot?.leadChannelCue,
|
|
2564
|
+
leadChannelAttributionCue: leaderSnapshot?.leadChannelAttributionCue,
|
|
2565
|
+
leadContinuityCue: leaderSnapshot?.leadContinuityCue
|
|
2566
|
+
});
|
|
2567
|
+
if (currentMediaCues === "none" || leaderMediaCues === "none" || currentMediaCues === leaderMediaCues) {
|
|
2568
|
+
return;
|
|
2569
|
+
}
|
|
2570
|
+
return `${snapshot.label ?? snapshot.caseId} ${leaderMediaCues}\u2192${currentMediaCues}`;
|
|
2571
|
+
}).filter((value) => typeof value === "string").slice(0, 3);
|
|
2572
|
+
return drift.length > 0 ? drift.join(" \xB7 ") : "none";
|
|
2573
|
+
};
|
|
2574
|
+
var formatComparisonSQLitePlannerDrift = (entry, leader) => {
|
|
2575
|
+
if (!leader?.caseTraceSnapshots?.length || !entry.caseTraceSnapshots?.length) {
|
|
2576
|
+
return "none";
|
|
2577
|
+
}
|
|
2578
|
+
const leaderCases = new Map(leader.caseTraceSnapshots.map((snapshot) => [snapshot.caseId, snapshot]));
|
|
2579
|
+
const drift = entry.caseTraceSnapshots.map((snapshot) => {
|
|
2580
|
+
const leaderSnapshot = leaderCases.get(snapshot.caseId);
|
|
2581
|
+
const currentPlanner = formatSQLitePlannerCueSummary(snapshot);
|
|
2582
|
+
const leaderPlanner = formatSQLitePlannerCueSummary(leaderSnapshot);
|
|
2583
|
+
if (currentPlanner === "none" || leaderPlanner === "none" || currentPlanner === leaderPlanner) {
|
|
2584
|
+
return;
|
|
2585
|
+
}
|
|
2586
|
+
return `${snapshot.label ?? snapshot.caseId} ${leaderPlanner}\u2192${currentPlanner}`;
|
|
2587
|
+
}).filter((value) => typeof value === "string").slice(0, 3);
|
|
2588
|
+
return drift.length > 0 ? drift.join(" \xB7 ") : "none";
|
|
2589
|
+
};
|
|
2590
|
+
var formatComparisonRoutingCueSummary = (entry) => {
|
|
2591
|
+
const trace = entry.traceSummary;
|
|
2592
|
+
if (!trace) {
|
|
2593
|
+
return "none";
|
|
2594
|
+
}
|
|
2595
|
+
const parts = [
|
|
2596
|
+
`modes ${formatTraceModes(trace.modes)}`,
|
|
2597
|
+
`balance ${formatSourceBalanceStrategies(trace.sourceBalanceStrategies)}`,
|
|
2598
|
+
`rewrites ${formatTraceRatio(trace.transformedCases, trace.totalCases)}`,
|
|
2599
|
+
`variants ${formatTraceRatio(trace.variantCases, trace.totalCases)}`
|
|
2600
|
+
].filter((value) => !value.endsWith("n/a"));
|
|
2601
|
+
return parts.length > 0 ? parts.join(" \xB7 ") : "none";
|
|
2602
|
+
};
|
|
2603
|
+
var formatComparisonRoutingDrift = (entry, leader) => {
|
|
2604
|
+
const trace = entry.traceSummary;
|
|
2605
|
+
const leaderTrace = leader?.traceSummary;
|
|
2606
|
+
if (!trace || !leaderTrace || entry === leader) {
|
|
2607
|
+
return "none";
|
|
2608
|
+
}
|
|
2609
|
+
const deltas = [];
|
|
2610
|
+
const currentModes = formatTraceModes(trace.modes);
|
|
2611
|
+
const previousModes = formatTraceModes(leaderTrace.modes);
|
|
2612
|
+
if (currentModes !== previousModes) {
|
|
2613
|
+
deltas.push(`modes ${previousModes}\u2192${currentModes}`);
|
|
2614
|
+
}
|
|
2615
|
+
const currentBalance = formatSourceBalanceStrategies(trace.sourceBalanceStrategies);
|
|
2616
|
+
const previousBalance = formatSourceBalanceStrategies(leaderTrace.sourceBalanceStrategies);
|
|
2617
|
+
if (currentBalance !== previousBalance) {
|
|
2618
|
+
deltas.push(`balance ${previousBalance}\u2192${currentBalance}`);
|
|
2619
|
+
}
|
|
2620
|
+
const transformedDelta = trace.transformedCases - leaderTrace.transformedCases;
|
|
2621
|
+
if (transformedDelta !== 0) {
|
|
2622
|
+
deltas.push(`rewrites ${formatTraceCountDelta(transformedDelta)}`);
|
|
2623
|
+
}
|
|
2624
|
+
const variantDelta = trace.variantCases - leaderTrace.variantCases;
|
|
2625
|
+
if (variantDelta !== 0) {
|
|
2626
|
+
deltas.push(`variants ${formatTraceCountDelta(variantDelta)}`);
|
|
2627
|
+
}
|
|
2628
|
+
return deltas.length > 0 ? deltas.slice(0, 3).join(" \xB7 ") : "none";
|
|
2629
|
+
};
|
|
2630
|
+
var formatHistoryRoutingShift = (history) => {
|
|
2631
|
+
const delta = history?.diff?.traceSummaryDelta;
|
|
2632
|
+
if (!delta) {
|
|
2633
|
+
return "none";
|
|
2634
|
+
}
|
|
2635
|
+
const shifts = [];
|
|
2636
|
+
if (delta.modesChanged) {
|
|
2637
|
+
shifts.push("modes changed");
|
|
2638
|
+
}
|
|
2639
|
+
if (delta.sourceBalanceStrategiesChanged) {
|
|
2640
|
+
shifts.push("balance changed");
|
|
2641
|
+
}
|
|
2642
|
+
if (delta.transformedCases !== 0) {
|
|
2643
|
+
shifts.push(`rewrites ${formatTraceCountDelta(delta.transformedCases)}`);
|
|
2644
|
+
}
|
|
2645
|
+
if (delta.variantCases !== 0) {
|
|
2646
|
+
shifts.push(`variants ${formatTraceCountDelta(delta.variantCases)}`);
|
|
2647
|
+
}
|
|
2648
|
+
return shifts.length > 0 ? shifts.join(" \xB7 ") : "none";
|
|
2649
|
+
};
|
|
2650
|
+
var formatHistorySQLitePlannerShift = (history) => {
|
|
2651
|
+
if (!history?.caseTraceSnapshots.length) {
|
|
2652
|
+
return "none";
|
|
2653
|
+
}
|
|
2654
|
+
const shifts = history.caseTraceSnapshots.map((entry) => {
|
|
2655
|
+
const previousPlanner = formatSQLitePlannerCueSummary({
|
|
2656
|
+
sqliteQueryBackfillCount: entry.previousSqliteQueryBackfillCount,
|
|
2657
|
+
sqliteQueryFilteredCandidates: entry.previousSqliteQueryFilteredCandidates,
|
|
2658
|
+
sqliteQueryFinalSearchK: entry.previousSqliteQueryFinalSearchK,
|
|
2659
|
+
sqliteQueryInitialSearchK: entry.previousSqliteQueryInitialSearchK,
|
|
2660
|
+
sqliteQueryMode: entry.previousSqliteQueryMode,
|
|
2661
|
+
sqliteQueryPushdownApplied: entry.previousSqliteQueryPushdownApplied,
|
|
2662
|
+
sqliteQueryPushdownClauseCount: entry.previousSqliteQueryPushdownClauseCount
|
|
2663
|
+
});
|
|
2664
|
+
const currentPlanner = formatSQLitePlannerCueSummary(entry);
|
|
2665
|
+
if (previousPlanner === "none" || currentPlanner === "none" || previousPlanner === currentPlanner) {
|
|
2666
|
+
return;
|
|
2667
|
+
}
|
|
2668
|
+
return `${entry.label ?? entry.caseId} ${previousPlanner}\u2192${currentPlanner}`;
|
|
2669
|
+
}).filter((value) => typeof value === "string").slice(0, 3);
|
|
2670
|
+
return shifts.length > 0 ? shifts.join(" \xB7 ") : "none";
|
|
2671
|
+
};
|
|
2672
|
+
var formatTraceLeadChanges = (changes) => changes && changes.length > 0 ? changes.slice(0, 3).map((entry) => {
|
|
2673
|
+
const label = entry.label ?? entry.caseId;
|
|
2674
|
+
const previousLead = entry.previousLead ?? "n/a";
|
|
2675
|
+
return `${label} ${previousLead}\u2192${entry.currentLead}`;
|
|
2676
|
+
}).join(" \xB7 ") : "none";
|
|
2677
|
+
var formatGroundingHistoryCaseLabels = (cases) => cases.length > 0 ? cases.map((entry) => entry.label ?? entry.caseId).join(", ") : "none";
|
|
2678
|
+
var formatRerankerComparisonHeadline = (entry) => [
|
|
2679
|
+
entry.label,
|
|
2680
|
+
`passing ${formatEvaluationPassingRate(entry.response.passingRate)}`,
|
|
2681
|
+
`f1 ${entry.response.summary.averageF1.toFixed(3)}`,
|
|
2682
|
+
`latency ${entry.response.summary.averageLatencyMs.toFixed(1)}ms`
|
|
2683
|
+
].join(" \xB7 ");
|
|
2684
|
+
var formatRetrievalComparisonHeadline = (entry) => [
|
|
2685
|
+
entry.label,
|
|
2686
|
+
`mode ${entry.retrievalMode}`,
|
|
2687
|
+
`passing ${formatEvaluationPassingRate(entry.response.passingRate)}`,
|
|
2688
|
+
`f1 ${entry.response.summary.averageF1.toFixed(3)}`,
|
|
2689
|
+
`latency ${entry.response.summary.averageLatencyMs.toFixed(1)}ms`
|
|
2690
|
+
].join(" \xB7 ");
|
|
2691
|
+
var formatTraceModes = (modes) => modes.length > 0 ? modes.join(" / ") : "n/a";
|
|
2692
|
+
var formatSourceBalanceStrategies = (strategies) => strategies.length > 0 ? strategies.join(" / ") : "n/a";
|
|
2693
|
+
var formatTraceStageSummary = (stageCounts) => {
|
|
2694
|
+
const topStages = Object.entries(stageCounts).sort((left, right) => right[1] - left[1]).slice(0, 3);
|
|
2695
|
+
return topStages.length > 0 ? topStages.map(([stage, count]) => `${stage} ${count}`).join(" \xB7 ") : "n/a";
|
|
2696
|
+
};
|
|
2697
|
+
var formatTraceRatio = (count, total) => `${count}/${total}`;
|
|
2698
|
+
var formatTraceCountDelta = (value) => `${value >= 0 ? "+" : ""}${value}`;
|
|
2699
|
+
var buildComparisonOverviewPresentation = (input) => {
|
|
2700
|
+
const winnerLabel = input.resolveLabel(input.summary.bestByPassingRate);
|
|
2701
|
+
const winnerEntry = input.resolveEntry(input.summary.bestByPassingRate);
|
|
2702
|
+
return {
|
|
2703
|
+
rows: [
|
|
2704
|
+
{
|
|
2705
|
+
label: "Best passing rate",
|
|
2706
|
+
value: input.resolveLabel(input.summary.bestByPassingRate)
|
|
2707
|
+
},
|
|
2708
|
+
{
|
|
2709
|
+
label: "Best average F1",
|
|
2710
|
+
value: input.resolveLabel(input.summary.bestByAverageF1)
|
|
2711
|
+
},
|
|
2712
|
+
{
|
|
2713
|
+
label: "Fastest",
|
|
2714
|
+
value: input.resolveLabel(input.summary.fastest)
|
|
2715
|
+
}
|
|
2716
|
+
],
|
|
2717
|
+
winnerLabel,
|
|
2718
|
+
summary: winnerEntry ? `passing ${formatEvaluationPassingRate(winnerEntry.response.passingRate)} \xB7 f1 ${winnerEntry.response.summary.averageF1.toFixed(3)} \xB7 latency ${winnerEntry.response.summary.averageLatencyMs.toFixed(1)}ms` : "Stored benchmark comparison"
|
|
2719
|
+
};
|
|
2720
|
+
};
|
|
2721
|
+
var buildRAGComparisonTraceSummaryRows = (entry) => {
|
|
2722
|
+
const trace = entry.traceSummary;
|
|
2723
|
+
const rows = [
|
|
2724
|
+
{
|
|
2725
|
+
label: "Lead cues",
|
|
2726
|
+
value: formatComparisonLeadCueSummary(entry)
|
|
2727
|
+
},
|
|
2728
|
+
{
|
|
2729
|
+
label: "Lead media cues",
|
|
2730
|
+
value: formatComparisonLeadMediaCueSummary(entry)
|
|
2731
|
+
},
|
|
2732
|
+
{
|
|
2733
|
+
label: "SQLite planner cues",
|
|
2734
|
+
value: formatComparisonSQLitePlannerCueSummary(entry)
|
|
2735
|
+
},
|
|
2736
|
+
{
|
|
2737
|
+
label: "Routing cues",
|
|
2738
|
+
value: formatComparisonRoutingCueSummary(entry)
|
|
2739
|
+
}
|
|
2740
|
+
];
|
|
2741
|
+
if (!trace) {
|
|
2742
|
+
rows.unshift({ label: "Trace", value: "Unavailable" });
|
|
2743
|
+
return rows;
|
|
2744
|
+
}
|
|
2745
|
+
rows.push({ label: "Modes", value: formatTraceModes(trace.modes) }, {
|
|
2746
|
+
label: "Balance",
|
|
2747
|
+
value: formatSourceBalanceStrategies(trace.sourceBalanceStrategies)
|
|
2748
|
+
}, { label: "Avg final", value: trace.averageFinalCount.toFixed(1) }, { label: "Avg vector", value: trace.averageVectorCount.toFixed(1) }, { label: "Avg lexical", value: trace.averageLexicalCount.toFixed(1) }, {
|
|
2749
|
+
label: "Balanced",
|
|
2750
|
+
value: formatTraceRatio(trace.balancedCases, trace.totalCases)
|
|
2751
|
+
}, {
|
|
2752
|
+
label: "Round robin",
|
|
2753
|
+
value: formatTraceRatio(trace.roundRobinCases, trace.totalCases)
|
|
2754
|
+
}, {
|
|
2755
|
+
label: "Transforms",
|
|
2756
|
+
value: formatTraceRatio(trace.transformedCases, trace.totalCases)
|
|
2757
|
+
}, {
|
|
2758
|
+
label: "Variants",
|
|
2759
|
+
value: formatTraceRatio(trace.variantCases, trace.totalCases)
|
|
2760
|
+
}, {
|
|
2761
|
+
label: "TopK",
|
|
2762
|
+
value: `${trace.averageCandidateTopK.toFixed(1)} / ${trace.averageLexicalTopK.toFixed(1)}`
|
|
2763
|
+
}, {
|
|
2764
|
+
label: "Stages",
|
|
2765
|
+
value: formatTraceStageSummary(trace.stageCounts)
|
|
2766
|
+
});
|
|
2767
|
+
return rows;
|
|
2768
|
+
};
|
|
2769
|
+
var buildRAGComparisonTraceDiffRows = (entry, leader) => {
|
|
2770
|
+
const trace = entry.traceSummary;
|
|
2771
|
+
const leadDriftRow = {
|
|
2772
|
+
label: "Lead drift vs leader",
|
|
2773
|
+
value: formatComparisonLeadDrift(entry, leader)
|
|
2774
|
+
};
|
|
2775
|
+
const leadMediaDriftRow = {
|
|
2776
|
+
label: "Lead media shift vs leader",
|
|
2777
|
+
value: formatComparisonLeadMediaDrift(entry, leader)
|
|
2778
|
+
};
|
|
2779
|
+
const sqlitePlannerDriftRow = {
|
|
2780
|
+
label: "SQLite planner shift vs leader",
|
|
2781
|
+
value: formatComparisonSQLitePlannerDrift(entry, leader)
|
|
2782
|
+
};
|
|
2783
|
+
const routingDriftRow = {
|
|
2784
|
+
label: "Routing shift vs leader",
|
|
2785
|
+
value: formatComparisonRoutingDrift(entry, leader)
|
|
2786
|
+
};
|
|
2787
|
+
if (!trace) {
|
|
2788
|
+
return [
|
|
2789
|
+
{ label: "Trace", value: "Unavailable for comparison" },
|
|
2790
|
+
leadDriftRow,
|
|
2791
|
+
leadMediaDriftRow,
|
|
2792
|
+
sqlitePlannerDriftRow,
|
|
2793
|
+
routingDriftRow
|
|
2794
|
+
];
|
|
2795
|
+
}
|
|
2796
|
+
const leaderTrace = leader?.traceSummary;
|
|
2797
|
+
if (!leaderTrace) {
|
|
2798
|
+
return [
|
|
2799
|
+
{ label: "Baseline", value: "Leader trace unavailable" },
|
|
2800
|
+
leadDriftRow,
|
|
2801
|
+
leadMediaDriftRow,
|
|
2802
|
+
sqlitePlannerDriftRow,
|
|
2803
|
+
routingDriftRow
|
|
2804
|
+
];
|
|
2805
|
+
}
|
|
2806
|
+
if (entry === leader) {
|
|
2807
|
+
return [
|
|
2808
|
+
{ label: "Baseline", value: "Leader strategy" },
|
|
2809
|
+
leadDriftRow,
|
|
2810
|
+
leadMediaDriftRow,
|
|
2811
|
+
sqlitePlannerDriftRow,
|
|
2812
|
+
routingDriftRow
|
|
2813
|
+
];
|
|
2814
|
+
}
|
|
2815
|
+
const stageDelta = Object.keys({
|
|
2816
|
+
...leaderTrace.stageCounts,
|
|
2817
|
+
...trace.stageCounts
|
|
2818
|
+
}).map((stage) => {
|
|
2819
|
+
const typedStage = stage;
|
|
2820
|
+
const delta = (trace.stageCounts[typedStage] ?? 0) - (leaderTrace.stageCounts[typedStage] ?? 0);
|
|
2821
|
+
return delta === 0 ? null : `${typedStage} ${formatTraceCountDelta(delta)}`;
|
|
2822
|
+
}).filter((value) => Boolean(value)).slice(0, 3).join(" \xB7 ");
|
|
2823
|
+
const rows = [
|
|
2824
|
+
{ label: "Baseline", value: leader.label }
|
|
2825
|
+
];
|
|
2826
|
+
if (formatTraceModes(trace.modes) !== formatTraceModes(leaderTrace.modes)) {
|
|
2827
|
+
rows.push({
|
|
2828
|
+
label: "Modes vs leader",
|
|
2829
|
+
value: `${formatTraceModes(trace.modes)} vs ${formatTraceModes(leaderTrace.modes)}`
|
|
2830
|
+
});
|
|
2831
|
+
}
|
|
2832
|
+
if (formatSourceBalanceStrategies(trace.sourceBalanceStrategies) !== formatSourceBalanceStrategies(leaderTrace.sourceBalanceStrategies)) {
|
|
2833
|
+
rows.push({
|
|
2834
|
+
label: "Balance vs leader",
|
|
2835
|
+
value: `${formatSourceBalanceStrategies(trace.sourceBalanceStrategies)} vs ${formatSourceBalanceStrategies(leaderTrace.sourceBalanceStrategies)}`
|
|
2836
|
+
});
|
|
2837
|
+
}
|
|
2838
|
+
rows.push(leadDriftRow, leadMediaDriftRow, sqlitePlannerDriftRow, routingDriftRow);
|
|
2839
|
+
rows.push({
|
|
2840
|
+
label: "Final delta",
|
|
2841
|
+
value: formatSignedDelta(trace.averageFinalCount - leaderTrace.averageFinalCount, 1)
|
|
2842
|
+
}, {
|
|
2843
|
+
label: "Vector delta",
|
|
2844
|
+
value: formatSignedDelta(trace.averageVectorCount - leaderTrace.averageVectorCount, 1)
|
|
2845
|
+
}, {
|
|
2846
|
+
label: "Lexical delta",
|
|
2847
|
+
value: formatSignedDelta(trace.averageLexicalCount - leaderTrace.averageLexicalCount, 1)
|
|
2848
|
+
}, {
|
|
2849
|
+
label: "Transform delta",
|
|
2850
|
+
value: formatTraceCountDelta(trace.transformedCases - leaderTrace.transformedCases)
|
|
2851
|
+
}, {
|
|
2852
|
+
label: "Balanced delta",
|
|
2853
|
+
value: formatTraceCountDelta(trace.balancedCases - leaderTrace.balancedCases)
|
|
2854
|
+
}, {
|
|
2855
|
+
label: "Round robin delta",
|
|
2856
|
+
value: formatTraceCountDelta(trace.roundRobinCases - leaderTrace.roundRobinCases)
|
|
2857
|
+
});
|
|
2858
|
+
if (stageDelta) {
|
|
2859
|
+
rows.push({ label: "Stage delta", value: stageDelta });
|
|
2860
|
+
}
|
|
2861
|
+
return rows;
|
|
2862
|
+
};
|
|
2863
|
+
var buildRAGRetrievalComparisonPresentations = (comparison) => {
|
|
2864
|
+
const leader = comparison.entries[0];
|
|
2865
|
+
return comparison.entries.map((entry) => ({
|
|
2866
|
+
diffLabel: leader?.label ?? "Leader",
|
|
2867
|
+
diffRows: buildRAGComparisonTraceDiffRows(entry, leader),
|
|
2868
|
+
summary: formatRetrievalComparisonHeadline(entry),
|
|
2869
|
+
id: entry.retrievalId,
|
|
2870
|
+
label: entry.label,
|
|
2871
|
+
traceSummaryRows: buildRAGComparisonTraceSummaryRows(entry)
|
|
2872
|
+
}));
|
|
2873
|
+
};
|
|
2874
|
+
var buildRAGRetrievalComparisonOverviewPresentation = (comparison) => buildComparisonOverviewPresentation({
|
|
2875
|
+
entries: comparison.entries,
|
|
2876
|
+
resolveEntry: (id) => comparison.entries.find((entry) => entry.retrievalId === id),
|
|
2877
|
+
resolveLabel: (id) => comparison.entries.find((entry) => entry.retrievalId === id)?.label ?? id ?? "n/a",
|
|
2878
|
+
summary: comparison.summary
|
|
2879
|
+
});
|
|
2880
|
+
var buildRAGRerankerComparisonPresentations = (comparison) => {
|
|
2881
|
+
const leader = comparison.entries[0];
|
|
2882
|
+
return comparison.entries.map((entry) => ({
|
|
2883
|
+
diffLabel: leader?.label ?? "Leader",
|
|
2884
|
+
diffRows: buildRAGComparisonTraceDiffRows(entry, leader),
|
|
2885
|
+
summary: formatRerankerComparisonHeadline(entry),
|
|
2886
|
+
id: entry.rerankerId,
|
|
2887
|
+
label: entry.label,
|
|
2888
|
+
traceSummaryRows: buildRAGComparisonTraceSummaryRows(entry)
|
|
2889
|
+
}));
|
|
2890
|
+
};
|
|
2891
|
+
var buildRAGRerankerComparisonOverviewPresentation = (comparison) => buildComparisonOverviewPresentation({
|
|
2892
|
+
entries: comparison.entries,
|
|
2893
|
+
resolveEntry: (id) => comparison.entries.find((entry) => entry.rerankerId === id),
|
|
2894
|
+
resolveLabel: (id) => comparison.entries.find((entry) => entry.rerankerId === id)?.label ?? id ?? "n/a",
|
|
2895
|
+
summary: comparison.summary
|
|
2896
|
+
});
|
|
2897
|
+
var buildRAGGroundingProviderPresentations = (entries) => entries.map((entry) => ({
|
|
2898
|
+
summary: [
|
|
2899
|
+
entry.label,
|
|
2900
|
+
`passing ${formatEvaluationPassingRate(entry.response.passingRate)}`,
|
|
2901
|
+
`citation f1 ${entry.response.summary.averageCitationF1.toFixed(3)}`,
|
|
2902
|
+
`resolved ${formatEvaluationPassingRate(entry.response.summary.averageResolvedCitationRate)}`,
|
|
2903
|
+
`latency ${entry.elapsedMs.toFixed(1)}ms`
|
|
2904
|
+
].join(" \xB7 "),
|
|
2905
|
+
id: entry.providerKey,
|
|
2906
|
+
label: entry.label
|
|
2907
|
+
}));
|
|
2908
|
+
var buildRAGGroundingProviderOverviewPresentation = (input) => {
|
|
2909
|
+
const resolveLabel = (key) => input.entries.find((entry) => entry.providerKey === key)?.label ?? key ?? "n/a";
|
|
2910
|
+
const winnerLabel = resolveLabel(input.summary.bestByPassingRate);
|
|
2911
|
+
const winnerEntry = input.entries.find((entry) => entry.providerKey === input.summary.bestByPassingRate);
|
|
2912
|
+
return {
|
|
2913
|
+
rows: [
|
|
2914
|
+
{
|
|
2915
|
+
label: "Best passing rate",
|
|
2916
|
+
value: resolveLabel(input.summary.bestByPassingRate)
|
|
2917
|
+
},
|
|
2918
|
+
{
|
|
2919
|
+
label: "Best citation F1",
|
|
2920
|
+
value: resolveLabel(input.summary.bestByAverageCitationF1)
|
|
2921
|
+
},
|
|
2922
|
+
{
|
|
2923
|
+
label: "Best resolved citations",
|
|
2924
|
+
value: resolveLabel(input.summary.bestByResolvedCitationRate)
|
|
2925
|
+
},
|
|
2926
|
+
{
|
|
2927
|
+
label: "Fastest",
|
|
2928
|
+
value: resolveLabel(input.summary.fastest)
|
|
2929
|
+
}
|
|
2930
|
+
],
|
|
2931
|
+
winnerLabel,
|
|
2932
|
+
summary: winnerEntry ? `passing ${formatEvaluationPassingRate(winnerEntry.response.passingRate)} \xB7 citation f1 ${winnerEntry.response.summary.averageCitationF1.toFixed(3)} \xB7 resolved ${formatEvaluationPassingRate(winnerEntry.response.summary.averageResolvedCitationRate)}` : "Stored workflow evaluation"
|
|
2933
|
+
};
|
|
2934
|
+
};
|
|
2935
|
+
var buildRAGRetrievalOverviewPresentation = (comparison) => ({
|
|
2936
|
+
rows: buildRAGRetrievalComparisonOverviewPresentation(comparison).rows
|
|
2937
|
+
});
|
|
2938
|
+
var buildRAGRerankerOverviewPresentation = (comparison) => ({
|
|
2939
|
+
rows: buildRAGRerankerComparisonOverviewPresentation(comparison).rows
|
|
2940
|
+
});
|
|
2941
|
+
var buildRAGGroundingOverviewPresentation = (input) => ({
|
|
2942
|
+
rows: [
|
|
2943
|
+
{
|
|
2944
|
+
label: "Grounding",
|
|
2945
|
+
value: formatGroundingHistorySummaryValue(input.groundingEvaluation)
|
|
2946
|
+
},
|
|
2947
|
+
...input.groundingProviderOverview?.rows ?? [
|
|
2948
|
+
{
|
|
2949
|
+
label: "Grounding providers",
|
|
2950
|
+
value: "Configure an AI provider to compare grounded answers."
|
|
2951
|
+
}
|
|
2952
|
+
]
|
|
2953
|
+
]
|
|
2954
|
+
});
|
|
2955
|
+
var buildRAGQualityOverviewPresentation = (input) => ({
|
|
2956
|
+
rows: [
|
|
2957
|
+
...buildRAGRetrievalOverviewPresentation(input.retrievalComparison).rows,
|
|
2958
|
+
...buildRAGRerankerOverviewPresentation(input.rerankerComparison).rows,
|
|
2959
|
+
...buildRAGGroundingOverviewPresentation({
|
|
2960
|
+
groundingEvaluation: input.groundingEvaluation,
|
|
2961
|
+
groundingProviderOverview: input.groundingProviderOverview
|
|
2962
|
+
}).rows
|
|
2963
|
+
]
|
|
2964
|
+
});
|
|
2965
|
+
var buildRAGGroundingProviderCaseComparisonPresentations = (comparisons) => comparisons.map((comparison) => {
|
|
2966
|
+
const resolveLabel = (key) => comparison.entries.find((entry) => entry.providerKey === key)?.label ?? key ?? "n/a";
|
|
2967
|
+
return {
|
|
2968
|
+
caseId: comparison.caseId,
|
|
2969
|
+
label: comparison.label,
|
|
2970
|
+
rows: [
|
|
2971
|
+
{
|
|
2972
|
+
label: "Best grounded",
|
|
2973
|
+
value: resolveLabel(comparison.summary.bestByStatus)
|
|
2974
|
+
},
|
|
2975
|
+
{
|
|
2976
|
+
label: "Best citation F1",
|
|
2977
|
+
value: resolveLabel(comparison.summary.bestByCitationF1)
|
|
2978
|
+
},
|
|
2979
|
+
{
|
|
2980
|
+
label: "Best resolved citations",
|
|
2981
|
+
value: resolveLabel(comparison.summary.bestByResolvedCitationRate)
|
|
2982
|
+
},
|
|
2983
|
+
...comparison.entries.map((entry) => ({
|
|
2984
|
+
label: entry.label,
|
|
2985
|
+
value: [
|
|
2986
|
+
entry.status.toUpperCase(),
|
|
2987
|
+
`coverage ${entry.coverage}`,
|
|
2988
|
+
`f1 ${entry.citationF1.toFixed(3)}`,
|
|
2989
|
+
`resolved ${formatEvaluationPassingRate(entry.resolvedCitationRate)}`,
|
|
2990
|
+
`matched ${entry.matchedIds.join(", ") || "none"}`,
|
|
2991
|
+
`missing ${entry.missingIds.join(", ") || "none"}`,
|
|
2992
|
+
`extra ${entry.extraIds.join(", ") || "none"}`,
|
|
2993
|
+
`answer ${entry.answerExcerpt || "n/a"}`
|
|
2994
|
+
].join(" \xB7 ")
|
|
2995
|
+
}))
|
|
2996
|
+
],
|
|
2997
|
+
summary: [
|
|
2998
|
+
`Best grounded: ${resolveLabel(comparison.summary.bestByStatus)}`,
|
|
2999
|
+
`Best citation F1: ${resolveLabel(comparison.summary.bestByCitationF1)}`,
|
|
3000
|
+
`Best resolved citations: ${resolveLabel(comparison.summary.bestByResolvedCitationRate)}`
|
|
3001
|
+
].join(" \xB7 ")
|
|
3002
|
+
};
|
|
3003
|
+
});
|
|
3004
|
+
var buildRAGEvaluationHistoryRows = (history) => {
|
|
3005
|
+
if (!history?.latestRun) {
|
|
3006
|
+
return [
|
|
3007
|
+
{ label: "History", value: "No persisted benchmark runs yet." }
|
|
3008
|
+
];
|
|
3009
|
+
}
|
|
3010
|
+
const rows = [
|
|
3011
|
+
{ label: "Runs recorded", value: String(history.runs.length) },
|
|
3012
|
+
{
|
|
3013
|
+
label: "Latest",
|
|
3014
|
+
value: `${history.latestRun.label} \xB7 ${formatEvaluationSummary(history.latestRun.response)}`
|
|
3015
|
+
}
|
|
3016
|
+
];
|
|
3017
|
+
if (history.latestRun.traceSummary) {
|
|
3018
|
+
rows.push({
|
|
3019
|
+
label: "Latest trace",
|
|
3020
|
+
value: `${formatTraceModes(history.latestRun.traceSummary.modes)} \xB7 balance ${formatSourceBalanceStrategies(history.latestRun.traceSummary.sourceBalanceStrategies)} \xB7 final ${history.latestRun.traceSummary.averageFinalCount.toFixed(1)} \xB7 vector ${history.latestRun.traceSummary.averageVectorCount.toFixed(1)} \xB7 lexical ${history.latestRun.traceSummary.averageLexicalCount.toFixed(1)}`
|
|
3021
|
+
});
|
|
3022
|
+
}
|
|
3023
|
+
if (history.previousRun) {
|
|
3024
|
+
rows.push({
|
|
3025
|
+
label: "Previous",
|
|
3026
|
+
value: `${history.previousRun.label} \xB7 ${formatEvaluationSummary(history.previousRun.response)}`
|
|
3027
|
+
});
|
|
3028
|
+
}
|
|
3029
|
+
if (!history.diff) {
|
|
3030
|
+
rows.push({
|
|
3031
|
+
label: "History diff",
|
|
3032
|
+
value: "Run the benchmark again to diff regressions over time."
|
|
3033
|
+
});
|
|
3034
|
+
return rows;
|
|
3035
|
+
}
|
|
3036
|
+
rows.push({
|
|
3037
|
+
label: "Passing delta",
|
|
3038
|
+
value: formatSignedDelta(history.diff.summaryDelta.passingRate, 1, "%")
|
|
3039
|
+
}, {
|
|
3040
|
+
label: "Average F1 delta",
|
|
3041
|
+
value: formatSignedDelta(history.diff.summaryDelta.averageF1, 3)
|
|
3042
|
+
}, {
|
|
3043
|
+
label: "Latency delta",
|
|
3044
|
+
value: formatSignedDelta(history.diff.summaryDelta.averageLatencyMs, 1, "ms")
|
|
3045
|
+
}, {
|
|
3046
|
+
label: "Improved",
|
|
3047
|
+
value: formatHistoryCaseLabels(history.diff.improvedCases)
|
|
3048
|
+
}, {
|
|
3049
|
+
label: "Regressed",
|
|
3050
|
+
value: formatHistoryCaseLabels(history.diff.regressedCases)
|
|
3051
|
+
}, {
|
|
3052
|
+
label: "Lead drift",
|
|
3053
|
+
value: formatTraceLeadChanges(history.diff.traceLeadChanges)
|
|
3054
|
+
}, {
|
|
3055
|
+
label: "SQLite planner shift",
|
|
3056
|
+
value: formatHistorySQLitePlannerShift(history)
|
|
3057
|
+
}, {
|
|
3058
|
+
label: "Trace routing shift",
|
|
3059
|
+
value: formatHistoryRoutingShift(history)
|
|
3060
|
+
});
|
|
3061
|
+
if (history.previousRun) {
|
|
3062
|
+
const currentEntityView = buildRAGEvaluationEntityQualityView(history.latestRun.response);
|
|
3063
|
+
const previousEntityView = buildRAGEvaluationEntityQualityView(history.previousRun.response);
|
|
3064
|
+
rows.push({
|
|
3065
|
+
label: "Source regression hotspots",
|
|
3066
|
+
value: formatEntityRegressionHotspots(currentEntityView.bySource, previousEntityView.bySource, (entry) => entry.averageF1, "f1")
|
|
3067
|
+
}, {
|
|
3068
|
+
label: "Document regression hotspots",
|
|
3069
|
+
value: formatEntityRegressionHotspots(currentEntityView.byDocument, previousEntityView.byDocument, (entry) => entry.averageF1, "f1")
|
|
3070
|
+
});
|
|
3071
|
+
}
|
|
3072
|
+
if (history.diff.traceSummaryDelta) {
|
|
3073
|
+
rows.push({
|
|
3074
|
+
label: "Trace mode shift",
|
|
3075
|
+
value: history.diff.traceSummaryDelta.modesChanged ? "changed" : "stable"
|
|
3076
|
+
}, {
|
|
3077
|
+
label: "Trace balance shift",
|
|
3078
|
+
value: history.diff.traceSummaryDelta.sourceBalanceStrategiesChanged ? "changed" : "stable"
|
|
3079
|
+
}, {
|
|
3080
|
+
label: "Trace final delta",
|
|
3081
|
+
value: formatSignedDelta(history.diff.traceSummaryDelta.averageFinalCount, 1)
|
|
3082
|
+
}, {
|
|
3083
|
+
label: "Trace vector delta",
|
|
3084
|
+
value: formatSignedDelta(history.diff.traceSummaryDelta.averageVectorCount, 1)
|
|
3085
|
+
}, {
|
|
3086
|
+
label: "Trace lexical delta",
|
|
3087
|
+
value: formatSignedDelta(history.diff.traceSummaryDelta.averageLexicalCount, 1)
|
|
3088
|
+
}, {
|
|
3089
|
+
label: "Trace balanced delta",
|
|
3090
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.balancedCases)
|
|
3091
|
+
}, {
|
|
3092
|
+
label: "Trace round robin delta",
|
|
3093
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.roundRobinCases)
|
|
3094
|
+
}, {
|
|
3095
|
+
label: "Trace transform delta",
|
|
3096
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.transformedCases)
|
|
3097
|
+
}, {
|
|
3098
|
+
label: "Trace variant delta",
|
|
3099
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.variantCases)
|
|
3100
|
+
});
|
|
3101
|
+
const stageDelta = Object.entries(history.diff.traceSummaryDelta.stageCounts ?? {}).map(([stage, count]) => `${stage} ${formatTraceCountDelta(count)}`).join(", ");
|
|
3102
|
+
if (stageDelta) {
|
|
3103
|
+
rows.push({ label: "Trace stage delta", value: stageDelta });
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
if (history.caseTraceSnapshots.length > 0) {
|
|
3107
|
+
const changedCases = history.caseTraceSnapshots.filter((entry) => entry.traceChange === "changed");
|
|
3108
|
+
rows.push({
|
|
3109
|
+
label: "Trace drift cases",
|
|
3110
|
+
value: changedCases.length > 0 ? changedCases.map((entry) => entry.label ?? entry.caseId).slice(0, 4).join(", ") : "none"
|
|
3111
|
+
});
|
|
3112
|
+
rows.push({
|
|
3113
|
+
label: "Lead media cues",
|
|
3114
|
+
value: Array.from(new Set(history.caseTraceSnapshots.map((entry) => formatLeadMediaCueSummary({
|
|
3115
|
+
leadChannelAttributionCue: entry.leadChannelAttributionCue,
|
|
3116
|
+
leadChannelCue: entry.leadChannelCue,
|
|
3117
|
+
leadContinuityCue: entry.leadContinuityCue,
|
|
3118
|
+
leadSpeakerAttributionCue: entry.leadSpeakerAttributionCue,
|
|
3119
|
+
leadSpeakerCue: entry.leadSpeakerCue
|
|
3120
|
+
})).filter((value) => value !== "none"))).slice(0, 3).join(" \xB7 ") || "none"
|
|
3121
|
+
});
|
|
3122
|
+
}
|
|
3123
|
+
return rows;
|
|
3124
|
+
};
|
|
3125
|
+
var buildRAGEvaluationCaseTracePresentations = (history) => {
|
|
3126
|
+
if (!history?.caseTraceSnapshots.length) {
|
|
3127
|
+
return [];
|
|
3128
|
+
}
|
|
3129
|
+
return history.caseTraceSnapshots.map((entry) => {
|
|
3130
|
+
const label = entry.label ?? entry.caseId;
|
|
3131
|
+
const currentMode = entry.traceMode ?? "no-trace";
|
|
3132
|
+
const previousMode = entry.previousTraceMode ?? "n/a";
|
|
3133
|
+
const currentBalance = entry.sourceBalanceStrategy ?? "cap";
|
|
3134
|
+
const previousBalance = entry.previousSourceBalanceStrategy ?? "n/a";
|
|
3135
|
+
const currentVariants = entry.variantQueries.length > 0 ? entry.variantQueries.join(", ") : "none";
|
|
3136
|
+
const previousVariants = entry.previousVariantQueries.length > 0 ? entry.previousVariantQueries.join(", ") : "none";
|
|
3137
|
+
const currentStages = Object.keys(entry.stageCounts).length > 0 ? Object.entries(entry.stageCounts).map(([stage, count]) => `${stage} ${count}`).join(", ") : "none";
|
|
3138
|
+
const previousStages = Object.keys(entry.previousStageCounts).length > 0 ? Object.entries(entry.previousStageCounts).map(([stage, count]) => `${stage} ${count}`).join(", ") : "none";
|
|
3139
|
+
const currentLeadContext = entry.topContextLabel ?? "n/a";
|
|
3140
|
+
const previousLeadContext = entry.previousTopContextLabel ?? "n/a";
|
|
3141
|
+
const currentLeadLocation = entry.topLocatorLabel ?? "n/a";
|
|
3142
|
+
const previousLeadLocation = entry.previousTopLocatorLabel ?? "n/a";
|
|
3143
|
+
const currentBoundary = entry.sourceAwareChunkReasonLabel ?? "n/a";
|
|
3144
|
+
const previousBoundary = entry.previousSourceAwareChunkReasonLabel ?? "n/a";
|
|
3145
|
+
const currentScope = entry.sourceAwareUnitScopeLabel ?? "n/a";
|
|
3146
|
+
const previousScope = entry.previousSourceAwareUnitScopeLabel ?? "n/a";
|
|
3147
|
+
const currentMediaCues = formatLeadMediaCueSummary({
|
|
3148
|
+
leadChannelAttributionCue: entry.leadChannelAttributionCue,
|
|
3149
|
+
leadChannelCue: entry.leadChannelCue,
|
|
3150
|
+
leadContinuityCue: entry.leadContinuityCue,
|
|
3151
|
+
leadSpeakerAttributionCue: entry.leadSpeakerAttributionCue,
|
|
3152
|
+
leadSpeakerCue: entry.leadSpeakerCue
|
|
3153
|
+
});
|
|
3154
|
+
const previousMediaCues = formatLeadMediaCueSummary({
|
|
3155
|
+
leadChannelAttributionCue: entry.previousLeadChannelAttributionCue,
|
|
3156
|
+
leadChannelCue: entry.previousLeadChannelCue,
|
|
3157
|
+
leadContinuityCue: entry.previousLeadContinuityCue,
|
|
3158
|
+
leadSpeakerAttributionCue: entry.previousLeadSpeakerAttributionCue,
|
|
3159
|
+
leadSpeakerCue: entry.previousLeadSpeakerCue
|
|
3160
|
+
});
|
|
3161
|
+
const leadSummary = entry.topLocatorLabel ?? entry.topContextLabel ?? entry.sourceAwareUnitScopeLabel ?? "n/a";
|
|
3162
|
+
const previousLeadSummary = entry.previousTopLocatorLabel ?? entry.previousTopContextLabel ?? entry.previousSourceAwareUnitScopeLabel ?? "n/a";
|
|
3163
|
+
return {
|
|
3164
|
+
caseId: entry.caseId,
|
|
3165
|
+
label,
|
|
3166
|
+
summary: `${entry.traceChange} \xB7 ${previousMode}\u2192${currentMode} \xB7 ${previousBalance}\u2192${currentBalance} \xB7 lead ${previousLeadSummary}\u2192${leadSummary} \xB7 final ${entry.previousFinalCount ?? 0}\u2192${entry.finalCount}`,
|
|
3167
|
+
traceChange: entry.traceChange,
|
|
3168
|
+
rows: [
|
|
3169
|
+
{ label: "Query", value: entry.query },
|
|
3170
|
+
{ label: "Trace change", value: entry.traceChange },
|
|
3171
|
+
{ label: "Mode", value: `${previousMode}\u2192${currentMode}` },
|
|
3172
|
+
{
|
|
3173
|
+
label: "Balance",
|
|
3174
|
+
value: `${previousBalance}\u2192${currentBalance}`
|
|
3175
|
+
},
|
|
3176
|
+
{
|
|
3177
|
+
label: "Transformed query",
|
|
3178
|
+
value: `${entry.previousTransformedQuery?.trim() || "n/a"}\u2192${entry.transformedQuery?.trim() || "n/a"}`
|
|
3179
|
+
},
|
|
3180
|
+
{
|
|
3181
|
+
label: "Final",
|
|
3182
|
+
value: `${entry.previousFinalCount ?? 0}\u2192${entry.finalCount}`
|
|
3183
|
+
},
|
|
3184
|
+
{
|
|
3185
|
+
label: "Vector",
|
|
3186
|
+
value: `${entry.previousVectorCount ?? 0}\u2192${entry.vectorCount}`
|
|
3187
|
+
},
|
|
3188
|
+
{
|
|
3189
|
+
label: "Lexical",
|
|
3190
|
+
value: `${entry.previousLexicalCount ?? 0}\u2192${entry.lexicalCount}`
|
|
3191
|
+
},
|
|
3192
|
+
{
|
|
3193
|
+
label: "Candidate topK",
|
|
3194
|
+
value: `${entry.previousCandidateTopK ?? 0}\u2192${entry.candidateTopK}`
|
|
3195
|
+
},
|
|
3196
|
+
{
|
|
3197
|
+
label: "Lexical topK",
|
|
3198
|
+
value: `${entry.previousLexicalTopK ?? 0}\u2192${entry.lexicalTopK}`
|
|
3199
|
+
},
|
|
3200
|
+
{
|
|
3201
|
+
label: "Variants",
|
|
3202
|
+
value: `${previousVariants}\u2192${currentVariants}`
|
|
3203
|
+
},
|
|
3204
|
+
{
|
|
3205
|
+
label: "Lead context",
|
|
3206
|
+
value: `${previousLeadContext}\u2192${currentLeadContext}`
|
|
3207
|
+
},
|
|
3208
|
+
{
|
|
3209
|
+
label: "Lead location",
|
|
3210
|
+
value: `${previousLeadLocation}\u2192${currentLeadLocation}`
|
|
3211
|
+
},
|
|
3212
|
+
{
|
|
3213
|
+
label: "Chunk boundary",
|
|
3214
|
+
value: `${previousBoundary}\u2192${currentBoundary}`
|
|
3215
|
+
},
|
|
3216
|
+
{
|
|
3217
|
+
label: "Source-aware scope",
|
|
3218
|
+
value: `${previousScope}\u2192${currentScope}`
|
|
3219
|
+
},
|
|
3220
|
+
{
|
|
3221
|
+
label: "Lead media cues",
|
|
3222
|
+
value: `${previousMediaCues}\u2192${currentMediaCues}`
|
|
3223
|
+
},
|
|
3224
|
+
{
|
|
3225
|
+
label: "Stages",
|
|
3226
|
+
value: `${previousStages}\u2192${currentStages}`
|
|
3227
|
+
},
|
|
3228
|
+
{ label: "Status", value: entry.status }
|
|
3229
|
+
]
|
|
3230
|
+
};
|
|
3231
|
+
});
|
|
3232
|
+
};
|
|
3233
|
+
var buildRAGEvaluationHistoryPresentation = (history) => ({
|
|
3234
|
+
caseTraces: buildRAGEvaluationCaseTracePresentations(history),
|
|
3235
|
+
rows: buildRAGEvaluationHistoryRows(history),
|
|
3236
|
+
summary: history?.latestRun ? history.latestRun.label : "No persisted benchmark runs yet."
|
|
3237
|
+
});
|
|
3238
|
+
var buildRAGEvaluationSuiteSnapshotRows = (history) => {
|
|
3239
|
+
if (!history?.latestSnapshot) {
|
|
3240
|
+
return [
|
|
3241
|
+
{ label: "Suite snapshots", value: "No saved suite snapshots yet." }
|
|
3242
|
+
];
|
|
3243
|
+
}
|
|
3244
|
+
const rows = [
|
|
3245
|
+
{
|
|
3246
|
+
label: "Snapshots recorded",
|
|
3247
|
+
value: String(history.snapshots.length)
|
|
3248
|
+
},
|
|
3249
|
+
{
|
|
3250
|
+
label: "Latest snapshot",
|
|
3251
|
+
value: `v${history.latestSnapshot.version} \xB7 ${history.latestSnapshot.caseCount} cases`
|
|
3252
|
+
}
|
|
3253
|
+
];
|
|
3254
|
+
if (history.previousSnapshot) {
|
|
3255
|
+
rows.push({
|
|
3256
|
+
label: "Previous snapshot",
|
|
3257
|
+
value: `v${history.previousSnapshot.version} \xB7 ${history.previousSnapshot.caseCount} cases`
|
|
3258
|
+
});
|
|
3259
|
+
}
|
|
3260
|
+
if (!history.diff) {
|
|
3261
|
+
rows.push({
|
|
3262
|
+
label: "Snapshot diff",
|
|
3263
|
+
value: "Save another suite snapshot to compare dataset changes."
|
|
3264
|
+
});
|
|
3265
|
+
return rows;
|
|
3266
|
+
}
|
|
3267
|
+
rows.push({
|
|
3268
|
+
label: "Case count change",
|
|
3269
|
+
value: formatSignedDelta(history.diff.caseCountDelta)
|
|
3270
|
+
}, {
|
|
3271
|
+
label: "Added cases",
|
|
3272
|
+
value: history.diff.addedCaseIds.length > 0 ? history.diff.addedCaseIds.join(", ") : "none"
|
|
3273
|
+
}, {
|
|
3274
|
+
label: "Removed cases",
|
|
3275
|
+
value: history.diff.removedCaseIds.length > 0 ? history.diff.removedCaseIds.join(", ") : "none"
|
|
3276
|
+
}, {
|
|
3277
|
+
label: "Changed cases",
|
|
3278
|
+
value: history.diff.changedCaseIds.length > 0 ? history.diff.changedCaseIds.join(", ") : "none"
|
|
3279
|
+
}, {
|
|
3280
|
+
label: "Order changed",
|
|
3281
|
+
value: history.diff.orderChanged ? "changed" : "stable"
|
|
3282
|
+
});
|
|
3283
|
+
return rows;
|
|
3284
|
+
};
|
|
3285
|
+
var buildRAGEvaluationSuiteSnapshotPresentations = (history) => (history?.snapshots ?? []).map((snapshot) => ({
|
|
3286
|
+
id: snapshot.id,
|
|
3287
|
+
label: snapshot.label ?? snapshot.suiteId,
|
|
3288
|
+
rows: [
|
|
3289
|
+
{ label: "Version", value: `v${snapshot.version}` },
|
|
3290
|
+
{ label: "Created", value: formatDateLabel(snapshot.createdAt) },
|
|
3291
|
+
{ label: "Cases", value: String(snapshot.caseCount) },
|
|
3292
|
+
{
|
|
3293
|
+
label: "Case ids",
|
|
3294
|
+
value: snapshot.suite.input.cases.map((entry) => entry.id).join(", ") || "none"
|
|
3295
|
+
}
|
|
3296
|
+
],
|
|
3297
|
+
summary: `v${snapshot.version} \xB7 ${snapshot.caseCount} cases`,
|
|
3298
|
+
version: snapshot.version
|
|
3299
|
+
}));
|
|
3300
|
+
var buildRAGEvaluationSuiteSnapshotHistoryPresentation = (history) => ({
|
|
3301
|
+
rows: buildRAGEvaluationSuiteSnapshotRows(history),
|
|
3302
|
+
snapshots: buildRAGEvaluationSuiteSnapshotPresentations(history),
|
|
3303
|
+
summary: history?.latestSnapshot ? `v${history.latestSnapshot.version}` : "No saved suite snapshots yet."
|
|
3304
|
+
});
|
|
3305
|
+
var buildRAGAnswerGroundingCaseSnapshotPresentations = (history) => {
|
|
3306
|
+
if (!history?.caseSnapshots.length) {
|
|
3307
|
+
return [];
|
|
3308
|
+
}
|
|
3309
|
+
return history.caseSnapshots.map((entry) => {
|
|
3310
|
+
const label = entry.label ?? entry.caseId;
|
|
3311
|
+
return {
|
|
3312
|
+
answerChange: entry.answerChange,
|
|
3313
|
+
caseId: entry.caseId,
|
|
3314
|
+
label,
|
|
3315
|
+
rows: [
|
|
3316
|
+
{
|
|
3317
|
+
label: "Query",
|
|
3318
|
+
value: entry.query?.trim().length ? entry.query : "n/a"
|
|
3319
|
+
},
|
|
3320
|
+
{ label: "Answer change", value: entry.answerChange },
|
|
3321
|
+
{ label: "Coverage", value: entry.coverage },
|
|
3322
|
+
{
|
|
3323
|
+
label: "Resolved citations",
|
|
3324
|
+
value: `${entry.resolvedCitationCount}/${entry.citationCount}`
|
|
3325
|
+
},
|
|
3326
|
+
{
|
|
3327
|
+
label: "Resolved citation rate",
|
|
3328
|
+
value: entry.resolvedCitationRate.toFixed(3)
|
|
3329
|
+
},
|
|
3330
|
+
{ label: "Citation F1", value: entry.citationF1.toFixed(3) },
|
|
3331
|
+
{
|
|
3332
|
+
label: "Reference count",
|
|
3333
|
+
value: String(entry.referenceCount)
|
|
3334
|
+
},
|
|
3335
|
+
{
|
|
3336
|
+
label: "Cited IDs",
|
|
3337
|
+
value: entry.citedIds.length > 0 ? entry.citedIds.join(", ") : "none"
|
|
3338
|
+
},
|
|
3339
|
+
{
|
|
3340
|
+
label: "Matched IDs",
|
|
3341
|
+
value: entry.matchedIds.length > 0 ? entry.matchedIds.join(", ") : "none"
|
|
3342
|
+
},
|
|
3343
|
+
{
|
|
3344
|
+
label: "Missing IDs",
|
|
3345
|
+
value: entry.missingIds.length > 0 ? entry.missingIds.join(", ") : "none"
|
|
3346
|
+
},
|
|
3347
|
+
{
|
|
3348
|
+
label: "Extra IDs",
|
|
3349
|
+
value: entry.extraIds.length > 0 ? entry.extraIds.join(", ") : "none"
|
|
3350
|
+
},
|
|
3351
|
+
{
|
|
3352
|
+
label: "Unresolved refs",
|
|
3353
|
+
value: entry.ungroundedReferenceNumbers.length > 0 ? entry.ungroundedReferenceNumbers.join(", ") : "none"
|
|
3354
|
+
},
|
|
3355
|
+
{
|
|
3356
|
+
label: "Answer",
|
|
3357
|
+
value: entry.answer.trim().length > 0 ? entry.answer : "n/a"
|
|
3358
|
+
},
|
|
3359
|
+
{
|
|
3360
|
+
label: "Previous answer",
|
|
3361
|
+
value: entry.previousAnswer && entry.previousAnswer.trim().length > 0 ? entry.previousAnswer : "n/a"
|
|
3362
|
+
}
|
|
3363
|
+
],
|
|
3364
|
+
summary: `${entry.answerChange} \xB7 ${entry.coverage} \xB7 resolved ${entry.resolvedCitationCount}/${entry.citationCount} \xB7 refs ${entry.referenceCount}`
|
|
3365
|
+
};
|
|
3366
|
+
});
|
|
3367
|
+
};
|
|
3368
|
+
var buildRAGAnswerGroundingHistoryRows = (history) => {
|
|
3369
|
+
if (!history?.latestRun) {
|
|
3370
|
+
return [{ label: "History", value: "No persisted provider runs yet." }];
|
|
3371
|
+
}
|
|
3372
|
+
const rows = [
|
|
3373
|
+
{ label: "Runs recorded", value: String(history.runs.length) },
|
|
3374
|
+
{
|
|
3375
|
+
label: "Latest",
|
|
3376
|
+
value: `${history.latestRun.label} \xB7 ${formatGroundingHistorySummaryValue(history.latestRun.response)}`
|
|
3377
|
+
}
|
|
3378
|
+
];
|
|
3379
|
+
if (history.previousRun) {
|
|
3380
|
+
rows.push({
|
|
3381
|
+
label: "Previous",
|
|
3382
|
+
value: `${history.previousRun.label} \xB7 ${formatGroundingHistorySummaryValue(history.previousRun.response)}`
|
|
3383
|
+
});
|
|
3384
|
+
}
|
|
3385
|
+
if (history.leaderboard[0]) {
|
|
3386
|
+
rows.push({
|
|
3387
|
+
label: "Best recorded",
|
|
3388
|
+
value: `#${history.leaderboard[0].rank} \xB7 ${history.leaderboard[0].label} \xB7 passing ${formatEvaluationPassingRate(history.leaderboard[0].passingRate)} \xB7 citation f1 ${history.leaderboard[0].averageCitationF1.toFixed(3)} \xB7 resolved ${formatEvaluationPassingRate(history.leaderboard[0].averageResolvedCitationRate)}`
|
|
3389
|
+
});
|
|
3390
|
+
}
|
|
3391
|
+
if (history.caseSnapshots.length > 0) {
|
|
3392
|
+
const changedAnswers = history.caseSnapshots.filter((entry) => entry.answerChange === "changed").length;
|
|
3393
|
+
rows.push({
|
|
3394
|
+
label: "Answer drift",
|
|
3395
|
+
value: `${changedAnswers}/${history.caseSnapshots.length} changed`
|
|
3396
|
+
});
|
|
3397
|
+
}
|
|
3398
|
+
if (!history.diff) {
|
|
3399
|
+
rows.push({
|
|
3400
|
+
label: "History diff",
|
|
3401
|
+
value: "Run the provider comparison again to diff grounding regressions over time."
|
|
3402
|
+
});
|
|
3403
|
+
return rows;
|
|
3404
|
+
}
|
|
3405
|
+
rows.push({
|
|
3406
|
+
label: "Passing delta",
|
|
3407
|
+
value: formatSignedDelta(history.diff.summaryDelta.passingRate, 1, "%")
|
|
3408
|
+
}, {
|
|
3409
|
+
label: "Citation F1 delta",
|
|
3410
|
+
value: formatSignedDelta(history.diff.summaryDelta.averageCitationF1, 3)
|
|
3411
|
+
}, {
|
|
3412
|
+
label: "Resolved citation delta",
|
|
3413
|
+
value: formatSignedDelta(history.diff.summaryDelta.averageResolvedCitationRate * 100, 1, "%")
|
|
3414
|
+
}, {
|
|
3415
|
+
label: "Improved",
|
|
3416
|
+
value: formatGroundingHistoryCaseLabels(history.diff.improvedCases)
|
|
3417
|
+
}, {
|
|
3418
|
+
label: "Regressed",
|
|
3419
|
+
value: formatGroundingHistoryCaseLabels(history.diff.regressedCases)
|
|
3420
|
+
});
|
|
3421
|
+
if (history.previousRun) {
|
|
3422
|
+
const currentEntityView = buildRAGAnswerGroundingEntityQualityView(history.latestRun.response);
|
|
3423
|
+
const previousEntityView = buildRAGAnswerGroundingEntityQualityView(history.previousRun.response);
|
|
3424
|
+
rows.push({
|
|
3425
|
+
label: "Source regression hotspots",
|
|
3426
|
+
value: formatEntityRegressionHotspots(currentEntityView.bySource, previousEntityView.bySource, (entry) => entry.averageCitationF1, "citation f1")
|
|
3427
|
+
}, {
|
|
3428
|
+
label: "Document regression hotspots",
|
|
3429
|
+
value: formatEntityRegressionHotspots(currentEntityView.byDocument, previousEntityView.byDocument, (entry) => entry.averageCitationF1, "citation f1")
|
|
3430
|
+
});
|
|
3431
|
+
}
|
|
3432
|
+
return rows;
|
|
3433
|
+
};
|
|
3434
|
+
var buildRAGAnswerGroundingHistoryPresentation = (history) => ({
|
|
3435
|
+
caseSnapshots: buildRAGAnswerGroundingCaseSnapshotPresentations(history),
|
|
3436
|
+
rows: buildRAGAnswerGroundingHistoryRows(history),
|
|
3437
|
+
summary: history?.latestRun ? history.latestRun.label : "No persisted provider runs yet."
|
|
3438
|
+
});
|
|
3439
|
+
var formatFailureCounts = (failureCounts) => {
|
|
3440
|
+
const entries = Object.entries(failureCounts).sort((left, right) => {
|
|
3441
|
+
if (right[1] !== left[1]) {
|
|
3442
|
+
return right[1] - left[1];
|
|
3443
|
+
}
|
|
3444
|
+
return left[0].localeCompare(right[0]);
|
|
3445
|
+
});
|
|
3446
|
+
return entries.length > 0 ? entries.map(([key, count]) => `${key} ${count}`).join(" \xB7 ") : "none";
|
|
3447
|
+
};
|
|
3448
|
+
var formatFailureCountDelta = (current, previous) => {
|
|
3449
|
+
const deltas = Object.keys({ ...current, ...previous }).map((key) => ({
|
|
3450
|
+
delta: (current[key] ?? 0) - (previous[key] ?? 0),
|
|
3451
|
+
key
|
|
3452
|
+
})).filter((entry) => entry.delta > 0).sort((left, right) => {
|
|
3453
|
+
if (right.delta !== left.delta) {
|
|
3454
|
+
return right.delta - left.delta;
|
|
3455
|
+
}
|
|
3456
|
+
return left.key.localeCompare(right.key);
|
|
3457
|
+
});
|
|
3458
|
+
return deltas.length > 0 ? deltas.slice(0, 3).map((entry) => `${entry.key} +${entry.delta}`).join(" \xB7 ") : "stable";
|
|
3459
|
+
};
|
|
3460
|
+
var formatEntityRegressionHotspots = (currentEntries, previousEntries, metricSelector, metricLabel) => {
|
|
3461
|
+
const previousMap = new Map(previousEntries.map((entry) => [entry.key, entry]));
|
|
3462
|
+
const regressions = currentEntries.map((entry) => {
|
|
3463
|
+
const previous = previousMap.get(entry.key);
|
|
3464
|
+
const passDelta = entry.passingRate - (previous?.passingRate ?? 0);
|
|
3465
|
+
const metricDelta = metricSelector(entry) - (previous ? metricSelector(previous) : 0);
|
|
3466
|
+
const failureDelta = formatFailureCountDelta(entry.failureCounts, previous?.failureCounts ?? {});
|
|
3467
|
+
return {
|
|
3468
|
+
entry,
|
|
3469
|
+
failureDelta,
|
|
3470
|
+
metricDelta,
|
|
3471
|
+
passDelta
|
|
3472
|
+
};
|
|
3473
|
+
}).filter((entry) => entry.passDelta < 0 || entry.metricDelta < 0 || entry.failureDelta !== "stable").sort((left, right) => {
|
|
3474
|
+
if (left.passDelta !== right.passDelta) {
|
|
3475
|
+
return left.passDelta - right.passDelta;
|
|
3476
|
+
}
|
|
3477
|
+
if (left.metricDelta !== right.metricDelta) {
|
|
3478
|
+
return left.metricDelta - right.metricDelta;
|
|
3479
|
+
}
|
|
3480
|
+
return left.entry.label.localeCompare(right.entry.label);
|
|
3481
|
+
});
|
|
3482
|
+
return regressions.length > 0 ? regressions.slice(0, 3).map(({ entry, failureDelta, metricDelta, passDelta }) => `${entry.label} pass ${formatSignedDelta(passDelta, 1, "%")} \xB7 ${metricLabel} ${formatSignedDelta(metricDelta, 3)} \xB7 ${failureDelta}`).join(" | ") : "none";
|
|
3483
|
+
};
|
|
3484
|
+
var buildEvaluationEntityPresentation = (entry) => ({
|
|
3485
|
+
key: entry.key,
|
|
3486
|
+
label: entry.label,
|
|
3487
|
+
rows: [
|
|
3488
|
+
{ label: "Entity type", value: entry.entityType },
|
|
3489
|
+
{ label: "Cases", value: String(entry.totalCases) },
|
|
3490
|
+
{
|
|
3491
|
+
label: "Status mix",
|
|
3492
|
+
value: `${entry.passedCases} pass \xB7 ${entry.partialCases} partial \xB7 ${entry.failedCases} fail`
|
|
3493
|
+
},
|
|
3494
|
+
{
|
|
3495
|
+
label: "Passing rate",
|
|
3496
|
+
value: formatEvaluationPassingRate(entry.passingRate)
|
|
3497
|
+
},
|
|
3498
|
+
{ label: "Average F1", value: entry.averageF1.toFixed(3) },
|
|
3499
|
+
{
|
|
3500
|
+
label: "Failure classes",
|
|
3501
|
+
value: formatFailureCounts(entry.failureCounts)
|
|
3502
|
+
},
|
|
3503
|
+
{
|
|
3504
|
+
label: "Cases",
|
|
3505
|
+
value: entry.caseIds.length > 0 ? entry.caseIds.join(", ") : "none"
|
|
3506
|
+
}
|
|
3507
|
+
],
|
|
3508
|
+
summary: `${entry.entityType} \xB7 passing ${formatEvaluationPassingRate(entry.passingRate)} \xB7 f1 ${entry.averageF1.toFixed(3)} \xB7 failures ${formatFailureCounts(entry.failureCounts)}`
|
|
3509
|
+
});
|
|
3510
|
+
var buildGroundingEntityPresentation = (entry) => ({
|
|
3511
|
+
key: entry.key,
|
|
3512
|
+
label: entry.label,
|
|
3513
|
+
rows: [
|
|
3514
|
+
{ label: "Entity type", value: entry.entityType },
|
|
3515
|
+
{ label: "Cases", value: String(entry.totalCases) },
|
|
3516
|
+
{
|
|
3517
|
+
label: "Status mix",
|
|
3518
|
+
value: `${entry.passedCases} pass \xB7 ${entry.partialCases} partial \xB7 ${entry.failedCases} fail`
|
|
3519
|
+
},
|
|
3520
|
+
{
|
|
3521
|
+
label: "Passing rate",
|
|
3522
|
+
value: formatEvaluationPassingRate(entry.passingRate)
|
|
3523
|
+
},
|
|
3524
|
+
{
|
|
3525
|
+
label: "Average citation F1",
|
|
3526
|
+
value: entry.averageCitationF1.toFixed(3)
|
|
3527
|
+
},
|
|
3528
|
+
{
|
|
3529
|
+
label: "Average resolved citation rate",
|
|
3530
|
+
value: formatEvaluationPassingRate(entry.averageResolvedCitationRate * 100)
|
|
3531
|
+
},
|
|
3532
|
+
{
|
|
3533
|
+
label: "Failure classes",
|
|
3534
|
+
value: formatFailureCounts(entry.failureCounts)
|
|
3535
|
+
},
|
|
3536
|
+
{
|
|
3537
|
+
label: "Cases",
|
|
3538
|
+
value: entry.caseIds.length > 0 ? entry.caseIds.join(", ") : "none"
|
|
3539
|
+
}
|
|
3540
|
+
],
|
|
3541
|
+
summary: `${entry.entityType} \xB7 passing ${formatEvaluationPassingRate(entry.passingRate)} \xB7 citation f1 ${entry.averageCitationF1.toFixed(3)} \xB7 failures ${formatFailureCounts(entry.failureCounts)}`
|
|
3542
|
+
});
|
|
3543
|
+
var buildRAGEvaluationEntityQualityPresentation = (view, entityType) => {
|
|
3544
|
+
const entities = entityType === "source" ? view.bySource.map(buildEvaluationEntityPresentation) : view.byDocument.map(buildEvaluationEntityPresentation);
|
|
3545
|
+
return {
|
|
3546
|
+
entities,
|
|
3547
|
+
rows: [
|
|
3548
|
+
{ label: "Entity type", value: entityType },
|
|
3549
|
+
{ label: "Entities tracked", value: String(entities.length) },
|
|
3550
|
+
{
|
|
3551
|
+
label: "Best coverage",
|
|
3552
|
+
value: entities[0]?.summary ?? "No entity quality data yet."
|
|
3553
|
+
}
|
|
3554
|
+
],
|
|
3555
|
+
summary: entities[0]?.label ?? `No ${entityType === "source" ? "source" : "document"} quality data yet.`
|
|
3556
|
+
};
|
|
3557
|
+
};
|
|
3558
|
+
var buildRAGAnswerGroundingEntityQualityPresentation = (view, entityType) => {
|
|
3559
|
+
const entities = entityType === "source" ? view.bySource.map(buildGroundingEntityPresentation) : view.byDocument.map(buildGroundingEntityPresentation);
|
|
3560
|
+
return {
|
|
3561
|
+
entities,
|
|
3562
|
+
rows: [
|
|
3563
|
+
{ label: "Entity type", value: entityType },
|
|
3564
|
+
{ label: "Entities tracked", value: String(entities.length) },
|
|
3565
|
+
{
|
|
3566
|
+
label: "Best coverage",
|
|
3567
|
+
value: entities[0]?.summary ?? "No entity quality data yet."
|
|
3568
|
+
}
|
|
3569
|
+
],
|
|
3570
|
+
summary: entities[0]?.label ?? `No ${entityType === "source" ? "source" : "document"} quality data yet.`
|
|
3571
|
+
};
|
|
3572
|
+
};
|
|
3573
|
+
|
|
656
3574
|
// src/ai/rag/quality.ts
|
|
657
3575
|
var DEFAULT_TOP_K = 6;
|
|
658
3576
|
var DEFAULT_HISTORY_LIMIT = 20;
|
|
659
3577
|
var DEFAULT_SEARCH_TRACE_TABLE_NAME = "rag_search_traces";
|
|
3578
|
+
var getTraceLabelString = (value) => typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
|
|
3579
|
+
var getTraceLabelNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
3580
|
+
var formatTraceSpreadsheetTableLabel = (tableIndex, tableCount) => {
|
|
3581
|
+
if (typeof tableIndex !== "number" || !Number.isFinite(tableIndex) || tableIndex < 1) {
|
|
3582
|
+
return;
|
|
3583
|
+
}
|
|
3584
|
+
if (typeof tableCount === "number" && Number.isFinite(tableCount) && tableCount >= tableIndex) {
|
|
3585
|
+
return `Table ${tableIndex} of ${tableCount}`;
|
|
3586
|
+
}
|
|
3587
|
+
return `Table ${tableIndex}`;
|
|
3588
|
+
};
|
|
3589
|
+
var formatTraceSourceAwareChunkReason = (value) => {
|
|
3590
|
+
const reason = getTraceLabelString(value);
|
|
3591
|
+
if (reason === "section_boundary") {
|
|
3592
|
+
return "Chunk boundary section";
|
|
3593
|
+
}
|
|
3594
|
+
if (reason === "size_limit") {
|
|
3595
|
+
return "Chunk boundary size limit";
|
|
3596
|
+
}
|
|
3597
|
+
if (reason === "source_native_unit") {
|
|
3598
|
+
return "Chunk boundary source-native unit";
|
|
3599
|
+
}
|
|
3600
|
+
return;
|
|
3601
|
+
};
|
|
3602
|
+
var buildTraceSourceAwareUnitScopeLabel = (metadata) => {
|
|
3603
|
+
if (!metadata) {
|
|
3604
|
+
return;
|
|
3605
|
+
}
|
|
3606
|
+
const sectionKind = getTraceLabelString(metadata.sectionKind);
|
|
3607
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getTraceLabelString(value)).filter((value) => typeof value === "string") : [];
|
|
3608
|
+
const sectionTitle = getTraceLabelString(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
3609
|
+
const pdfTextKind = getTraceLabelString(metadata.pdfTextKind);
|
|
3610
|
+
const officeBlockKind = getTraceLabelString(metadata.officeBlockKind);
|
|
3611
|
+
const sheetName = getTraceLabelString(metadata.sheetName);
|
|
3612
|
+
const spreadsheetTableLabel = formatTraceSpreadsheetTableLabel(getTraceLabelNumber(metadata.spreadsheetTableIndex), getTraceLabelNumber(metadata.spreadsheetTableCount));
|
|
3613
|
+
const slideTitle = getTraceLabelString(metadata.slideTitle);
|
|
3614
|
+
const slideNumber = getTraceLabelNumber(metadata.slideNumber) ?? (typeof metadata.slideIndex === "number" ? metadata.slideIndex + 1 : undefined);
|
|
3615
|
+
if (sectionPath.length > 0 && (sectionKind === "markdown_heading" || sectionKind === "html_heading" || sectionKind === "office_heading" || sectionKind === undefined)) {
|
|
3616
|
+
return `Source-aware section ${sectionPath.join(" > ")}`;
|
|
3617
|
+
}
|
|
3618
|
+
if (sectionKind === "pdf_block") {
|
|
3619
|
+
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
3620
|
+
return `Source-aware PDF table block ${sectionTitle}`;
|
|
3621
|
+
}
|
|
3622
|
+
if (sectionTitle) {
|
|
3623
|
+
return `Source-aware PDF block ${sectionTitle}`;
|
|
3624
|
+
}
|
|
3625
|
+
return "Source-aware PDF block";
|
|
3626
|
+
}
|
|
3627
|
+
if (sectionKind === "office_block") {
|
|
3628
|
+
if (officeBlockKind && sectionTitle) {
|
|
3629
|
+
return `Source-aware office ${officeBlockKind} block ${sectionTitle}`;
|
|
3630
|
+
}
|
|
3631
|
+
if (sectionTitle) {
|
|
3632
|
+
return `Source-aware office block ${sectionTitle}`;
|
|
3633
|
+
}
|
|
3634
|
+
return "Source-aware office block";
|
|
3635
|
+
}
|
|
3636
|
+
if (sectionKind === "spreadsheet_rows" || sectionKind === undefined && (sheetName || spreadsheetTableLabel || getTraceLabelNumber(metadata.spreadsheetRowStart) !== undefined || getTraceLabelNumber(metadata.spreadsheetRowEnd) !== undefined)) {
|
|
3637
|
+
if (sheetName && spreadsheetTableLabel) {
|
|
3638
|
+
return `Source-aware spreadsheet ${sheetName} ${spreadsheetTableLabel}`;
|
|
3639
|
+
}
|
|
3640
|
+
if (sheetName) {
|
|
3641
|
+
return `Source-aware spreadsheet ${sheetName}`;
|
|
3642
|
+
}
|
|
3643
|
+
return "Source-aware spreadsheet";
|
|
3644
|
+
}
|
|
3645
|
+
if (sectionKind === "presentation_slide") {
|
|
3646
|
+
if (slideNumber && slideTitle) {
|
|
3647
|
+
return `Source-aware slide ${slideNumber} ${slideTitle}`;
|
|
3648
|
+
}
|
|
3649
|
+
if (slideTitle) {
|
|
3650
|
+
return `Source-aware slide ${slideTitle}`;
|
|
3651
|
+
}
|
|
3652
|
+
if (slideNumber) {
|
|
3653
|
+
return `Source-aware slide ${slideNumber}`;
|
|
3654
|
+
}
|
|
3655
|
+
return "Source-aware slide";
|
|
3656
|
+
}
|
|
3657
|
+
return;
|
|
3658
|
+
};
|
|
3659
|
+
var buildEvaluationLeadSnapshot = (input) => {
|
|
3660
|
+
const labels = buildRAGSourceLabels({
|
|
3661
|
+
metadata: input?.metadata,
|
|
3662
|
+
source: input?.source,
|
|
3663
|
+
title: input?.title
|
|
3664
|
+
});
|
|
3665
|
+
return {
|
|
3666
|
+
sourceAwareChunkReasonLabel: formatTraceSourceAwareChunkReason(input?.metadata?.sourceAwareChunkReason),
|
|
3667
|
+
sourceAwareUnitScopeLabel: buildTraceSourceAwareUnitScopeLabel(input?.metadata),
|
|
3668
|
+
topContextLabel: labels?.contextLabel,
|
|
3669
|
+
topLocatorLabel: labels?.locatorLabel
|
|
3670
|
+
};
|
|
3671
|
+
};
|
|
3672
|
+
var buildEvaluationLeadMediaCueSnapshot = (trace) => {
|
|
3673
|
+
const rerankStep = trace?.steps.find((step) => step.stage === "rerank");
|
|
3674
|
+
const metadata = rerankStep?.metadata ?? {};
|
|
3675
|
+
return {
|
|
3676
|
+
leadChannelAttributionCue: typeof metadata.leadChannelAttributionCue === "string" ? metadata.leadChannelAttributionCue : undefined,
|
|
3677
|
+
leadChannelCue: typeof metadata.leadChannelCue === "string" ? metadata.leadChannelCue : undefined,
|
|
3678
|
+
leadContinuityCue: typeof metadata.leadContinuityCue === "string" ? metadata.leadContinuityCue : undefined,
|
|
3679
|
+
leadSpeakerAttributionCue: typeof metadata.leadSpeakerAttributionCue === "string" ? metadata.leadSpeakerAttributionCue : undefined,
|
|
3680
|
+
leadSpeakerCue: typeof metadata.leadSpeakerCue === "string" ? metadata.leadSpeakerCue : undefined
|
|
3681
|
+
};
|
|
3682
|
+
};
|
|
3683
|
+
var buildEvaluationSQLiteQueryPlanSnapshot = (trace) => {
|
|
3684
|
+
const vectorStep = trace?.steps.find((step) => step.stage === "vector_search");
|
|
3685
|
+
const metadata = vectorStep?.metadata ?? {};
|
|
3686
|
+
return {
|
|
3687
|
+
sqliteQueryBackfillCount: typeof metadata.sqliteQueryBackfillCount === "number" ? metadata.sqliteQueryBackfillCount : undefined,
|
|
3688
|
+
sqliteQueryFilteredCandidates: typeof metadata.sqliteQueryFilteredCandidates === "number" ? metadata.sqliteQueryFilteredCandidates : undefined,
|
|
3689
|
+
sqliteQueryFinalSearchK: typeof metadata.sqliteQueryFinalSearchK === "number" ? metadata.sqliteQueryFinalSearchK : undefined,
|
|
3690
|
+
sqliteQueryInitialSearchK: typeof metadata.sqliteQueryInitialSearchK === "number" ? metadata.sqliteQueryInitialSearchK : undefined,
|
|
3691
|
+
sqliteQueryMode: metadata.sqliteQueryMode === "json_fallback" || metadata.sqliteQueryMode === "native_vec0" ? metadata.sqliteQueryMode : undefined,
|
|
3692
|
+
sqliteQueryPushdownApplied: typeof metadata.sqliteQueryPushdownApplied === "boolean" ? metadata.sqliteQueryPushdownApplied : undefined,
|
|
3693
|
+
sqliteQueryPushdownClauseCount: typeof metadata.sqliteQueryPushdownClauseCount === "number" ? metadata.sqliteQueryPushdownClauseCount : undefined
|
|
3694
|
+
};
|
|
3695
|
+
};
|
|
660
3696
|
var DEFAULT_RETRIEVAL_COMPARISON_HISTORY_TABLE_NAME = "rag_retrieval_comparison_history";
|
|
661
3697
|
var DEFAULT_RETRIEVAL_RELEASE_DECISION_TABLE_NAME = "rag_retrieval_release_decisions";
|
|
662
3698
|
var DEFAULT_RETRIEVAL_BASELINE_TABLE_NAME = "rag_retrieval_baselines";
|
|
@@ -804,6 +3840,151 @@ var parseJSONArray = (value, fallback) => {
|
|
|
804
3840
|
}
|
|
805
3841
|
};
|
|
806
3842
|
var normalizeExpectedIds = (input) => Array.from(new Set(input.map((item) => item.trim()).filter(Boolean)));
|
|
3843
|
+
var hasSourceExtension = (value, extensions) => {
|
|
3844
|
+
if (!value) {
|
|
3845
|
+
return false;
|
|
3846
|
+
}
|
|
3847
|
+
const normalized = value.trim().toLowerCase();
|
|
3848
|
+
return extensions.some((extension) => normalized.endsWith(extension));
|
|
3849
|
+
};
|
|
3850
|
+
var hasMetadataNumber = (metadata, key) => typeof metadata?.[key] === "number";
|
|
3851
|
+
var hasMetadataString = (metadata, key) => typeof metadata?.[key] === "string" && metadata[key] !== "";
|
|
3852
|
+
var hasMetadataArray = (metadata, key) => Array.isArray(metadata?.[key]) && metadata[key].length > 0;
|
|
3853
|
+
var detectSectionEvidence = (ids, sources = []) => sources.some((source) => hasMetadataArray(source.metadata, "sectionPath") || hasMetadataString(source.metadata, "sectionTitle") || hasMetadataString(source.metadata, "sectionKind"));
|
|
3854
|
+
var detectSectionGraphEvidence = (sources = []) => sources.some((source) => hasMetadataString(source.metadata, "sectionChunkId") || hasMetadataNumber(source.metadata, "sectionChunkIndex") || hasMetadataNumber(source.metadata, "sectionChunkCount") && Number(source.metadata?.sectionChunkCount) > 1);
|
|
3855
|
+
var detectSectionHierarchyEvidence = (sources = []) => sources.some((source) => {
|
|
3856
|
+
const path = Array.isArray(source.metadata?.sectionPath) ? source.metadata.sectionPath.filter((entry) => typeof entry === "string" && entry.trim().length > 0) : [];
|
|
3857
|
+
return path.length > 1;
|
|
3858
|
+
});
|
|
3859
|
+
var detectSpreadsheetEvidence = (ids, sources = []) => ids.some((id) => hasSourceExtension(id, [".xlsx", ".xls", ".csv", ".ods", ".tsv"])) || sources.some((source) => hasSourceExtension(source.source, [
|
|
3860
|
+
".xlsx",
|
|
3861
|
+
".xls",
|
|
3862
|
+
".csv",
|
|
3863
|
+
".ods",
|
|
3864
|
+
".tsv"
|
|
3865
|
+
]) || hasMetadataString(source.metadata, "sheetName") || hasMetadataNumber(source.metadata, "spreadsheetRowStart") || hasMetadataNumber(source.metadata, "spreadsheetTableIndex"));
|
|
3866
|
+
var detectMediaEvidence = (ids, sources = []) => ids.some((id) => hasSourceExtension(id, [
|
|
3867
|
+
".mp3",
|
|
3868
|
+
".wav",
|
|
3869
|
+
".m4a",
|
|
3870
|
+
".aac",
|
|
3871
|
+
".ogg",
|
|
3872
|
+
".flac",
|
|
3873
|
+
".mp4",
|
|
3874
|
+
".mov",
|
|
3875
|
+
".mkv",
|
|
3876
|
+
".webm"
|
|
3877
|
+
])) || sources.some((source) => hasSourceExtension(source.source, [
|
|
3878
|
+
".mp3",
|
|
3879
|
+
".wav",
|
|
3880
|
+
".m4a",
|
|
3881
|
+
".aac",
|
|
3882
|
+
".ogg",
|
|
3883
|
+
".flac",
|
|
3884
|
+
".mp4",
|
|
3885
|
+
".mov",
|
|
3886
|
+
".mkv",
|
|
3887
|
+
".webm"
|
|
3888
|
+
]) || hasMetadataString(source.metadata, "mediaKind") || hasMetadataString(source.metadata, "speaker") || hasMetadataNumber(source.metadata, "startMs"));
|
|
3889
|
+
var detectOCREvidence = (ids, sources = []) => ids.some((id) => hasSourceExtension(id, [".png", ".jpg", ".jpeg", ".webp"])) || sources.some((source) => hasSourceExtension(source.source, [
|
|
3890
|
+
".png",
|
|
3891
|
+
".jpg",
|
|
3892
|
+
".jpeg",
|
|
3893
|
+
".webp"
|
|
3894
|
+
]) || hasMetadataNumber(source.metadata, "ocrConfidence") || hasMetadataNumber(source.metadata, "ocrRegionConfidence") || hasMetadataString(source.metadata, "ocrEngine") || source.metadata?.pdfTextMode === "ocr");
|
|
3895
|
+
var getRetrievalTrace = (trace) => trace && ("trace" in trace) ? trace.trace : trace;
|
|
3896
|
+
var classifyRAGEvaluationFailure = (input) => {
|
|
3897
|
+
const classes = [];
|
|
3898
|
+
if (input.expectedCount === 0) {
|
|
3899
|
+
classes.push("no_expected_targets");
|
|
3900
|
+
}
|
|
3901
|
+
if (input.retrievedCount === 0) {
|
|
3902
|
+
classes.push("no_results");
|
|
3903
|
+
}
|
|
3904
|
+
if (input.expectedCount > 0 && input.matchedCount === 0) {
|
|
3905
|
+
classes.push("no_match");
|
|
3906
|
+
}
|
|
3907
|
+
if (input.expectedCount > 0 && input.matchedCount > 0 && input.matchedCount < input.expectedCount) {
|
|
3908
|
+
classes.push("partial_recall");
|
|
3909
|
+
}
|
|
3910
|
+
if (input.retrievedIds.length > input.matchedCount) {
|
|
3911
|
+
classes.push("extra_noise");
|
|
3912
|
+
}
|
|
3913
|
+
const sourceSignals = input.retrievedSources ?? [];
|
|
3914
|
+
const relevantIds = [...input.missingIds, ...input.retrievedIds];
|
|
3915
|
+
if ((input.missingIds.length > 0 || input.matchedCount === 0) && detectSectionEvidence(relevantIds, sourceSignals)) {
|
|
3916
|
+
classes.push("section_evidence_miss");
|
|
3917
|
+
if (detectSectionGraphEvidence(sourceSignals)) {
|
|
3918
|
+
classes.push("section_graph_miss");
|
|
3919
|
+
}
|
|
3920
|
+
if (detectSectionHierarchyEvidence(sourceSignals)) {
|
|
3921
|
+
classes.push("section_hierarchy_miss");
|
|
3922
|
+
}
|
|
3923
|
+
}
|
|
3924
|
+
if ((input.missingIds.length > 0 || input.matchedCount === 0) && detectSpreadsheetEvidence(relevantIds, sourceSignals)) {
|
|
3925
|
+
classes.push("spreadsheet_evidence_miss");
|
|
3926
|
+
}
|
|
3927
|
+
if ((input.missingIds.length > 0 || input.matchedCount === 0) && detectMediaEvidence(relevantIds, sourceSignals)) {
|
|
3928
|
+
classes.push("media_evidence_miss");
|
|
3929
|
+
}
|
|
3930
|
+
if ((input.missingIds.length > 0 || input.matchedCount === 0) && detectOCREvidence(relevantIds, sourceSignals)) {
|
|
3931
|
+
classes.push("ocr_evidence_miss");
|
|
3932
|
+
}
|
|
3933
|
+
const retrievalTrace = getRetrievalTrace(input.trace);
|
|
3934
|
+
if ((input.retrievedCount === 0 || input.matchedCount === 0) && (retrievalTrace?.routingLabel || retrievalTrace?.routingReason)) {
|
|
3935
|
+
classes.push("routing_miss");
|
|
3936
|
+
}
|
|
3937
|
+
return classes;
|
|
3938
|
+
};
|
|
3939
|
+
var classifyRAGGroundingFailure = (input) => {
|
|
3940
|
+
const classes = [];
|
|
3941
|
+
if (input.expectedCount === 0) {
|
|
3942
|
+
classes.push("no_expected_targets");
|
|
3943
|
+
}
|
|
3944
|
+
if (input.citationCount === 0) {
|
|
3945
|
+
classes.push("no_citations");
|
|
3946
|
+
}
|
|
3947
|
+
if (input.unresolvedCitationCount > 0) {
|
|
3948
|
+
classes.push("unresolved_citations");
|
|
3949
|
+
}
|
|
3950
|
+
if (input.missingIds.length > 0 || input.matchedCount < input.expectedCount) {
|
|
3951
|
+
classes.push("missing_expected_sources");
|
|
3952
|
+
}
|
|
3953
|
+
if (input.extraIds.length > 0) {
|
|
3954
|
+
classes.push("extra_citations");
|
|
3955
|
+
}
|
|
3956
|
+
const sourceSignals = input.availableSources ?? [];
|
|
3957
|
+
const relevantIds = [...input.missingIds, ...input.extraIds];
|
|
3958
|
+
if ((input.missingIds.length > 0 || input.citationCount === 0) && detectSectionEvidence(relevantIds, sourceSignals)) {
|
|
3959
|
+
classes.push("section_source_miss");
|
|
3960
|
+
if (detectSectionGraphEvidence(sourceSignals)) {
|
|
3961
|
+
classes.push("section_graph_source_miss");
|
|
3962
|
+
}
|
|
3963
|
+
if (detectSectionHierarchyEvidence(sourceSignals)) {
|
|
3964
|
+
classes.push("section_hierarchy_source_miss");
|
|
3965
|
+
}
|
|
3966
|
+
}
|
|
3967
|
+
if ((input.missingIds.length > 0 || input.citationCount === 0) && detectSpreadsheetEvidence(relevantIds, sourceSignals)) {
|
|
3968
|
+
classes.push("spreadsheet_source_miss");
|
|
3969
|
+
}
|
|
3970
|
+
if ((input.missingIds.length > 0 || input.citationCount === 0) && detectMediaEvidence(relevantIds, sourceSignals)) {
|
|
3971
|
+
classes.push("media_source_miss");
|
|
3972
|
+
}
|
|
3973
|
+
if ((input.missingIds.length > 0 || input.citationCount === 0) && detectOCREvidence(relevantIds, sourceSignals)) {
|
|
3974
|
+
classes.push("ocr_source_miss");
|
|
3975
|
+
}
|
|
3976
|
+
return classes;
|
|
3977
|
+
};
|
|
3978
|
+
var normalizeSyntheticText = (value) => value.replace(/^#+\s*/gm, "").replace(/`+/g, "").replace(/\[(.*?)\]\((.*?)\)/g, "$1").replace(/\s+/g, " ").trim();
|
|
3979
|
+
var buildSyntheticEvaluationQuery = (document) => {
|
|
3980
|
+
const title = normalizeSyntheticText(document.title);
|
|
3981
|
+
const sourceTail = normalizeSyntheticText(document.source.split("/").at(-1) ?? document.source);
|
|
3982
|
+
const text = normalizeSyntheticText(document.text ?? "");
|
|
3983
|
+
const firstSentence = text.split(/(?<=[.!?])\s+/).map((entry) => entry.trim()).find((entry) => entry.length >= 24) ?? text.split(/\n+/).map((entry) => entry.trim()).find((entry) => entry.length >= 24) ?? title ?? sourceTail;
|
|
3984
|
+
const prefixedSentence = title.length > 0 && firstSentence.toLowerCase().startsWith(title.toLowerCase()) ? firstSentence : [title, firstSentence].filter(Boolean).join(" ");
|
|
3985
|
+
const queryTerms = normalizeSyntheticText(prefixedSentence).split(/\s+/).slice(0, 12).join(" ");
|
|
3986
|
+
return queryTerms.length > 0 ? queryTerms : sourceTail;
|
|
3987
|
+
};
|
|
807
3988
|
var resolveEvaluationMode = (caseInput) => {
|
|
808
3989
|
if (normalizeStringArray(caseInput.expectedChunkIds).length > 0) {
|
|
809
3990
|
return "chunkId";
|
|
@@ -1279,6 +4460,7 @@ var evaluateRAGCollectionCases = async ({
|
|
|
1279
4460
|
caseResult,
|
|
1280
4461
|
filter: typeof input.cases?.[caseIndex]?.filter === "object" ? input.cases[caseIndex]?.filter : input.filter,
|
|
1281
4462
|
retrieval: input.cases?.[caseIndex]?.retrieval ?? input.retrieval,
|
|
4463
|
+
topResult: undefined,
|
|
1282
4464
|
trace: undefined
|
|
1283
4465
|
}));
|
|
1284
4466
|
}
|
|
@@ -1315,11 +4497,14 @@ var evaluateRAGCollectionCases = async ({
|
|
|
1315
4497
|
expectedIds,
|
|
1316
4498
|
mode,
|
|
1317
4499
|
query,
|
|
1318
|
-
retrievedIds
|
|
4500
|
+
retrievedIds,
|
|
4501
|
+
retrievedSources: sources,
|
|
4502
|
+
trace: searchOutcome.trace
|
|
1319
4503
|
}),
|
|
1320
4504
|
trace: searchOutcome.trace,
|
|
1321
4505
|
filter: searchInput.filter,
|
|
1322
|
-
retrieval: searchInput.retrieval
|
|
4506
|
+
retrieval: searchInput.retrieval,
|
|
4507
|
+
topResult: searchOutcome.results[0]
|
|
1323
4508
|
};
|
|
1324
4509
|
}));
|
|
1325
4510
|
};
|
|
@@ -1507,6 +4692,44 @@ var normalizeHistoryRuns = (runs) => [...runs].sort(toHistorySortOrder);
|
|
|
1507
4692
|
var normalizeRetrievalComparisonRuns = (runs) => [...runs].sort((left, right) => right.finishedAt - left.finishedAt);
|
|
1508
4693
|
var toTraceSummaryRunSortOrder = (left, right) => right.finishedAt - left.finishedAt;
|
|
1509
4694
|
var normalizeTraceSummaryRuns = (runs) => [...runs].sort(toTraceSummaryRunSortOrder);
|
|
4695
|
+
var applyRAGEvaluationHistoryPrunePolicy = ({
|
|
4696
|
+
input,
|
|
4697
|
+
runs,
|
|
4698
|
+
sort
|
|
4699
|
+
}) => {
|
|
4700
|
+
const sorted = sort(runs);
|
|
4701
|
+
const targeted = sorted.filter((run) => !input?.suiteId || run.suiteId === input.suiteId);
|
|
4702
|
+
const untouched = sorted.filter((run) => input?.suiteId && run.suiteId !== input.suiteId);
|
|
4703
|
+
const now = input?.now ?? Date.now();
|
|
4704
|
+
let kept = [...targeted];
|
|
4705
|
+
if (typeof input?.maxAgeMs === "number" && Number.isFinite(input.maxAgeMs)) {
|
|
4706
|
+
const cutoff = now - input.maxAgeMs;
|
|
4707
|
+
kept = kept.filter((run) => {
|
|
4708
|
+
const timestamp = run.finishedAt ?? run.createdAt ?? 0;
|
|
4709
|
+
return timestamp >= cutoff;
|
|
4710
|
+
});
|
|
4711
|
+
}
|
|
4712
|
+
if (typeof input?.maxRunsPerSuite === "number" && Number.isFinite(input.maxRunsPerSuite) && input.maxRunsPerSuite >= 0) {
|
|
4713
|
+
const remainingBySuite = new Map;
|
|
4714
|
+
kept = kept.filter((run) => {
|
|
4715
|
+
const current = remainingBySuite.get(run.suiteId) ?? 0;
|
|
4716
|
+
if (current >= input.maxRunsPerSuite) {
|
|
4717
|
+
return false;
|
|
4718
|
+
}
|
|
4719
|
+
remainingBySuite.set(run.suiteId, current + 1);
|
|
4720
|
+
return true;
|
|
4721
|
+
});
|
|
4722
|
+
}
|
|
4723
|
+
const keptIds = new Set(kept.map((run) => run.id));
|
|
4724
|
+
const removed = targeted.filter((run) => !keptIds.has(run.id));
|
|
4725
|
+
const next = sort([...untouched, ...kept]);
|
|
4726
|
+
return {
|
|
4727
|
+
keptCount: next.length,
|
|
4728
|
+
next,
|
|
4729
|
+
removed,
|
|
4730
|
+
removedCount: removed.length
|
|
4731
|
+
};
|
|
4732
|
+
};
|
|
1510
4733
|
var applyRAGSearchTracePrunePolicy = ({
|
|
1511
4734
|
input,
|
|
1512
4735
|
traces
|
|
@@ -1600,7 +4823,9 @@ var buildCaseDiff = (currentCase, previousCase) => ({
|
|
|
1600
4823
|
previousMatchedIds: previousCase?.matchedIds ?? [],
|
|
1601
4824
|
previousMissingIds: previousCase?.missingIds ?? [],
|
|
1602
4825
|
previousStatus: previousCase?.status,
|
|
1603
|
-
|
|
4826
|
+
previousFailureClasses: previousCase?.failureClasses ?? [],
|
|
4827
|
+
query: currentCase.query,
|
|
4828
|
+
currentFailureClasses: currentCase.failureClasses ?? []
|
|
1604
4829
|
});
|
|
1605
4830
|
var buildGroundingCaseDiff = (currentCase, previousCase) => ({
|
|
1606
4831
|
answerChanged: typeof previousCase?.answer === "string" ? previousCase.answer !== currentCase.answer : true,
|
|
@@ -1623,6 +4848,7 @@ var buildGroundingCaseDiff = (currentCase, previousCase) => ({
|
|
|
1623
4848
|
previousCitedIds: previousCase?.citedIds ?? [],
|
|
1624
4849
|
previousCoverage: previousCase?.coverage,
|
|
1625
4850
|
previousExtraIds: previousCase?.extraIds ?? [],
|
|
4851
|
+
previousFailureClasses: previousCase?.failureClasses ?? [],
|
|
1626
4852
|
previousMatchedIds: previousCase?.matchedIds ?? [],
|
|
1627
4853
|
previousMissingIds: previousCase?.missingIds ?? [],
|
|
1628
4854
|
previousReferenceCount: previousCase?.referenceCount,
|
|
@@ -1630,7 +4856,8 @@ var buildGroundingCaseDiff = (currentCase, previousCase) => ({
|
|
|
1630
4856
|
previousStatus: previousCase?.status,
|
|
1631
4857
|
previousUngroundedReferenceNumbers: previousCase?.groundedAnswer.ungroundedReferenceNumbers ?? [],
|
|
1632
4858
|
previousUnresolvedCitationCount: previousCase?.unresolvedCitationCount,
|
|
1633
|
-
query: currentCase.query
|
|
4859
|
+
query: currentCase.query,
|
|
4860
|
+
currentFailureClasses: currentCase.failureClasses ?? []
|
|
1634
4861
|
});
|
|
1635
4862
|
var buildGroundingCaseSnapshots = ({
|
|
1636
4863
|
current,
|
|
@@ -1651,6 +4878,7 @@ var buildGroundingCaseSnapshots = ({
|
|
|
1651
4878
|
citedIds: entry.citedIds,
|
|
1652
4879
|
coverage: entry.coverage,
|
|
1653
4880
|
extraIds: entry.extraIds,
|
|
4881
|
+
failureClasses: entry.failureClasses,
|
|
1654
4882
|
label: entry.label,
|
|
1655
4883
|
matchedIds: entry.matchedIds,
|
|
1656
4884
|
missingIds: entry.missingIds,
|
|
@@ -1682,15 +4910,40 @@ var buildEvaluationCaseTraceSnapshot = ({
|
|
|
1682
4910
|
filter,
|
|
1683
4911
|
retrieval,
|
|
1684
4912
|
currentTrace,
|
|
1685
|
-
previousTrace
|
|
4913
|
+
previousTrace,
|
|
4914
|
+
currentSnapshot,
|
|
4915
|
+
topResult
|
|
1686
4916
|
}) => {
|
|
1687
4917
|
const stageCounts = currentTrace ? buildTraceStageCounts([currentTrace]) : {};
|
|
1688
4918
|
const previousStageCounts = previousTrace?.stageCounts ?? {};
|
|
4919
|
+
const currentLeadSnapshot = topResult || currentSnapshot ? buildEvaluationLeadSnapshot({
|
|
4920
|
+
metadata: topResult?.metadata,
|
|
4921
|
+
source: topResult?.source,
|
|
4922
|
+
title: topResult?.title
|
|
4923
|
+
}) : {};
|
|
4924
|
+
const topContextLabel = currentLeadSnapshot.topContextLabel ?? currentSnapshot?.topContextLabel;
|
|
4925
|
+
const topLocatorLabel = currentLeadSnapshot.topLocatorLabel ?? currentSnapshot?.topLocatorLabel;
|
|
4926
|
+
const sourceAwareChunkReasonLabel = currentLeadSnapshot.sourceAwareChunkReasonLabel ?? currentSnapshot?.sourceAwareChunkReasonLabel;
|
|
4927
|
+
const sourceAwareUnitScopeLabel = currentLeadSnapshot.sourceAwareUnitScopeLabel ?? currentSnapshot?.sourceAwareUnitScopeLabel;
|
|
4928
|
+
const currentLeadMediaCueSnapshot = currentTrace ? buildEvaluationLeadMediaCueSnapshot(currentTrace) : {};
|
|
4929
|
+
const currentSQLiteQueryPlanSnapshot = currentTrace ? buildEvaluationSQLiteQueryPlanSnapshot(currentTrace) : {};
|
|
4930
|
+
const leadSpeakerCue = currentLeadMediaCueSnapshot.leadSpeakerCue ?? currentSnapshot?.leadSpeakerCue;
|
|
4931
|
+
const leadSpeakerAttributionCue = currentLeadMediaCueSnapshot.leadSpeakerAttributionCue ?? currentSnapshot?.leadSpeakerAttributionCue;
|
|
4932
|
+
const leadChannelCue = currentLeadMediaCueSnapshot.leadChannelCue ?? currentSnapshot?.leadChannelCue;
|
|
4933
|
+
const leadChannelAttributionCue = currentLeadMediaCueSnapshot.leadChannelAttributionCue ?? currentSnapshot?.leadChannelAttributionCue;
|
|
4934
|
+
const leadContinuityCue = currentLeadMediaCueSnapshot.leadContinuityCue ?? currentSnapshot?.leadContinuityCue;
|
|
4935
|
+
const sqliteQueryMode = currentSQLiteQueryPlanSnapshot.sqliteQueryMode ?? currentSnapshot?.sqliteQueryMode;
|
|
4936
|
+
const sqliteQueryPushdownApplied = currentSQLiteQueryPlanSnapshot.sqliteQueryPushdownApplied ?? currentSnapshot?.sqliteQueryPushdownApplied;
|
|
4937
|
+
const sqliteQueryPushdownClauseCount = currentSQLiteQueryPlanSnapshot.sqliteQueryPushdownClauseCount ?? currentSnapshot?.sqliteQueryPushdownClauseCount;
|
|
4938
|
+
const sqliteQueryFilteredCandidates = currentSQLiteQueryPlanSnapshot.sqliteQueryFilteredCandidates ?? currentSnapshot?.sqliteQueryFilteredCandidates;
|
|
4939
|
+
const sqliteQueryInitialSearchK = currentSQLiteQueryPlanSnapshot.sqliteQueryInitialSearchK ?? currentSnapshot?.sqliteQueryInitialSearchK;
|
|
4940
|
+
const sqliteQueryFinalSearchK = currentSQLiteQueryPlanSnapshot.sqliteQueryFinalSearchK ?? currentSnapshot?.sqliteQueryFinalSearchK;
|
|
4941
|
+
const sqliteQueryBackfillCount = currentSQLiteQueryPlanSnapshot.sqliteQueryBackfillCount ?? currentSnapshot?.sqliteQueryBackfillCount;
|
|
1689
4942
|
const currentFilterSignature = JSON.stringify(filter ?? undefined);
|
|
1690
4943
|
const previousFilterSignature = JSON.stringify(previousTrace?.inputFilter ?? undefined);
|
|
1691
4944
|
const currentRetrievalSignature = JSON.stringify(retrieval ?? undefined);
|
|
1692
4945
|
const previousRetrievalSignature = JSON.stringify(previousTrace?.inputRetrieval ?? undefined);
|
|
1693
|
-
const traceChange = !previousTrace ? currentTrace ? "new" : "unchanged" : previousTrace.traceMode !== currentTrace?.mode || previousFilterSignature !== currentFilterSignature || previousRetrievalSignature !== currentRetrievalSignature || previousTrace.sourceBalanceStrategy !== currentTrace?.sourceBalanceStrategy || previousTrace.transformedQuery !== (currentTrace?.transformedQuery || undefined) || previousTrace.variantQueries.join("|") !== (currentTrace?.variantQueries ?? []).join("|") || previousTrace.finalCount !== (currentTrace?.resultCounts.final ?? 0) || previousTrace.vectorCount !== (currentTrace?.resultCounts.vector ?? 0) || previousTrace.lexicalCount !== (currentTrace?.resultCounts.lexical ?? 0) || previousTrace.candidateTopK !== (currentTrace?.candidateTopK ?? 0) || previousTrace.lexicalTopK !== (currentTrace?.lexicalTopK ?? 0) || !areStageCountsEqual(previousStageCounts, stageCounts) ? "changed" : "unchanged";
|
|
4946
|
+
const traceChange = !previousTrace ? currentTrace ? "new" : "unchanged" : previousTrace.traceMode !== currentTrace?.mode || previousFilterSignature !== currentFilterSignature || previousRetrievalSignature !== currentRetrievalSignature || previousTrace.sourceBalanceStrategy !== currentTrace?.sourceBalanceStrategy || previousTrace.transformedQuery !== (currentTrace?.transformedQuery || undefined) || previousTrace.variantQueries.join("|") !== (currentTrace?.variantQueries ?? []).join("|") || previousTrace.finalCount !== (currentTrace?.resultCounts.final ?? 0) || previousTrace.vectorCount !== (currentTrace?.resultCounts.vector ?? 0) || previousTrace.lexicalCount !== (currentTrace?.resultCounts.lexical ?? 0) || previousTrace.candidateTopK !== (currentTrace?.candidateTopK ?? 0) || previousTrace.lexicalTopK !== (currentTrace?.lexicalTopK ?? 0) || previousTrace.topContextLabel !== topContextLabel || previousTrace.topLocatorLabel !== topLocatorLabel || previousTrace.sourceAwareChunkReasonLabel !== sourceAwareChunkReasonLabel || previousTrace.sourceAwareUnitScopeLabel !== sourceAwareUnitScopeLabel || previousTrace.leadSpeakerCue !== leadSpeakerCue || previousTrace.leadSpeakerAttributionCue !== leadSpeakerAttributionCue || previousTrace.leadChannelCue !== leadChannelCue || previousTrace.leadChannelAttributionCue !== leadChannelAttributionCue || previousTrace.leadContinuityCue !== leadContinuityCue || previousTrace.sqliteQueryMode !== sqliteQueryMode || previousTrace.sqliteQueryPushdownApplied !== sqliteQueryPushdownApplied || previousTrace.sqliteQueryPushdownClauseCount !== sqliteQueryPushdownClauseCount || previousTrace.sqliteQueryFilteredCandidates !== sqliteQueryFilteredCandidates || previousTrace.sqliteQueryInitialSearchK !== sqliteQueryInitialSearchK || previousTrace.sqliteQueryFinalSearchK !== sqliteQueryFinalSearchK || previousTrace.sqliteQueryBackfillCount !== sqliteQueryBackfillCount || !areStageCountsEqual(previousStageCounts, stageCounts) ? "changed" : "unchanged";
|
|
1694
4947
|
return {
|
|
1695
4948
|
candidateTopK: currentTrace?.candidateTopK ?? 0,
|
|
1696
4949
|
caseId: caseResult.caseId,
|
|
@@ -1707,16 +4960,48 @@ var buildEvaluationCaseTraceSnapshot = ({
|
|
|
1707
4960
|
previousLexicalTopK: previousTrace?.lexicalTopK,
|
|
1708
4961
|
previousInputFilter: previousTrace?.inputFilter,
|
|
1709
4962
|
previousInputRetrieval: previousTrace?.inputRetrieval,
|
|
4963
|
+
previousLeadChannelAttributionCue: previousTrace?.leadChannelAttributionCue,
|
|
4964
|
+
previousLeadChannelCue: previousTrace?.leadChannelCue,
|
|
4965
|
+
previousLeadContinuityCue: previousTrace?.leadContinuityCue,
|
|
4966
|
+
previousLeadSpeakerAttributionCue: previousTrace?.leadSpeakerAttributionCue,
|
|
4967
|
+
previousLeadSpeakerCue: previousTrace?.leadSpeakerCue,
|
|
4968
|
+
previousSqliteQueryBackfillCount: previousTrace?.sqliteQueryBackfillCount,
|
|
4969
|
+
previousSqliteQueryFilteredCandidates: previousTrace?.sqliteQueryFilteredCandidates,
|
|
4970
|
+
previousSqliteQueryFinalSearchK: previousTrace?.sqliteQueryFinalSearchK,
|
|
4971
|
+
previousSqliteQueryInitialSearchK: previousTrace?.sqliteQueryInitialSearchK,
|
|
4972
|
+
previousSqliteQueryMode: previousTrace?.sqliteQueryMode,
|
|
4973
|
+
previousSqliteQueryPushdownApplied: previousTrace?.sqliteQueryPushdownApplied,
|
|
4974
|
+
previousSqliteQueryPushdownClauseCount: previousTrace?.sqliteQueryPushdownClauseCount,
|
|
1710
4975
|
previousSourceBalanceStrategy: previousTrace?.sourceBalanceStrategy,
|
|
4976
|
+
previousSourceAwareChunkReasonLabel: previousTrace?.sourceAwareChunkReasonLabel,
|
|
1711
4977
|
previousStageCounts,
|
|
4978
|
+
previousTopContextLabel: previousTrace?.topContextLabel,
|
|
4979
|
+
previousTopLocatorLabel: previousTrace?.topLocatorLabel,
|
|
1712
4980
|
previousTraceMode: previousTrace?.traceMode,
|
|
1713
4981
|
previousTransformedQuery: previousTrace?.transformedQuery,
|
|
4982
|
+
previousSourceAwareUnitScopeLabel: previousTrace?.sourceAwareUnitScopeLabel,
|
|
1714
4983
|
previousVariantQueries: previousTrace?.variantQueries ?? [],
|
|
1715
4984
|
previousVectorCount: previousTrace?.vectorCount,
|
|
1716
4985
|
query: caseResult.query,
|
|
4986
|
+
leadChannelAttributionCue,
|
|
4987
|
+
leadChannelCue,
|
|
4988
|
+
leadContinuityCue,
|
|
4989
|
+
leadSpeakerAttributionCue,
|
|
4990
|
+
leadSpeakerCue,
|
|
4991
|
+
sqliteQueryBackfillCount,
|
|
4992
|
+
sqliteQueryFilteredCandidates,
|
|
4993
|
+
sqliteQueryFinalSearchK,
|
|
4994
|
+
sqliteQueryInitialSearchK,
|
|
4995
|
+
sqliteQueryMode,
|
|
4996
|
+
sqliteQueryPushdownApplied,
|
|
4997
|
+
sqliteQueryPushdownClauseCount,
|
|
4998
|
+
sourceAwareChunkReasonLabel,
|
|
4999
|
+
sourceAwareUnitScopeLabel,
|
|
1717
5000
|
stageCounts,
|
|
1718
5001
|
status: caseResult.status,
|
|
1719
5002
|
sourceBalanceStrategy: currentTrace?.sourceBalanceStrategy,
|
|
5003
|
+
topContextLabel,
|
|
5004
|
+
topLocatorLabel,
|
|
1720
5005
|
traceChange,
|
|
1721
5006
|
traceMode: currentTrace?.mode,
|
|
1722
5007
|
transformedQuery: currentTrace?.transformedQuery || undefined,
|
|
@@ -1724,11 +5009,12 @@ var buildEvaluationCaseTraceSnapshot = ({
|
|
|
1724
5009
|
vectorCount: currentTrace?.resultCounts.vector ?? 0
|
|
1725
5010
|
};
|
|
1726
5011
|
};
|
|
1727
|
-
var buildEvaluationCaseTraceSnapshotsFromEvaluated = (evaluated) => evaluated.map(({ caseResult, filter, retrieval, trace }) => buildEvaluationCaseTraceSnapshot({
|
|
5012
|
+
var buildEvaluationCaseTraceSnapshotsFromEvaluated = (evaluated) => evaluated.map(({ caseResult, filter, retrieval, trace, topResult }) => buildEvaluationCaseTraceSnapshot({
|
|
1728
5013
|
caseResult,
|
|
1729
5014
|
filter,
|
|
1730
5015
|
retrieval,
|
|
1731
|
-
currentTrace: trace
|
|
5016
|
+
currentTrace: trace,
|
|
5017
|
+
topResult
|
|
1732
5018
|
}));
|
|
1733
5019
|
var buildEvaluationCaseTraceSnapshots = ({
|
|
1734
5020
|
current,
|
|
@@ -1744,6 +5030,7 @@ var buildEvaluationCaseTraceSnapshots = ({
|
|
|
1744
5030
|
]));
|
|
1745
5031
|
return current.response.cases.map((caseResult) => buildEvaluationCaseTraceSnapshot({
|
|
1746
5032
|
caseResult,
|
|
5033
|
+
currentSnapshot: currentTraces.get(caseResult.caseId),
|
|
1747
5034
|
currentTrace: (() => {
|
|
1748
5035
|
const currentSnapshot = currentTraces.get(caseResult.caseId);
|
|
1749
5036
|
if (!currentSnapshot) {
|
|
@@ -1774,6 +5061,7 @@ var buildEvaluationCaseTraceSnapshots = ({
|
|
|
1774
5061
|
}));
|
|
1775
5062
|
};
|
|
1776
5063
|
var getStatusRank = (status) => status === "pass" ? 2 : status === "partial" ? 1 : 0;
|
|
5064
|
+
var buildCaseTraceLeadLabel = (trace) => trace?.topLocatorLabel ?? trace?.topContextLabel ?? trace?.sourceAwareUnitScopeLabel ?? trace?.sourceAwareChunkReasonLabel;
|
|
1777
5065
|
var buildRAGEvaluationRunDiff = ({
|
|
1778
5066
|
current,
|
|
1779
5067
|
previous
|
|
@@ -1783,12 +5071,31 @@ var buildRAGEvaluationRunDiff = ({
|
|
|
1783
5071
|
const regressedCases = diffs.filter((entry) => getStatusRank(entry.currentStatus) < getStatusRank(entry.previousStatus ?? "fail"));
|
|
1784
5072
|
const improvedCases = diffs.filter((entry) => getStatusRank(entry.currentStatus) > getStatusRank(entry.previousStatus ?? "fail"));
|
|
1785
5073
|
const unchangedCases = diffs.filter((entry) => getStatusRank(entry.currentStatus) === getStatusRank(entry.previousStatus ?? "fail"));
|
|
5074
|
+
const previousCaseTraces = new Map((previous?.caseTraceSnapshots ?? []).map((entry) => [
|
|
5075
|
+
entry.caseId,
|
|
5076
|
+
entry
|
|
5077
|
+
]));
|
|
5078
|
+
const traceLeadChanges = (current.caseTraceSnapshots ?? []).map((entry) => {
|
|
5079
|
+
const previousEntry = previousCaseTraces.get(entry.caseId);
|
|
5080
|
+
const currentLead = buildCaseTraceLeadLabel(entry);
|
|
5081
|
+
const previousLead = buildCaseTraceLeadLabel(previousEntry);
|
|
5082
|
+
if (!currentLead || currentLead === previousLead) {
|
|
5083
|
+
return;
|
|
5084
|
+
}
|
|
5085
|
+
return {
|
|
5086
|
+
caseId: entry.caseId,
|
|
5087
|
+
currentLead,
|
|
5088
|
+
label: entry.label,
|
|
5089
|
+
previousLead
|
|
5090
|
+
};
|
|
5091
|
+
}).filter((entry) => typeof entry !== "undefined");
|
|
1786
5092
|
return {
|
|
1787
5093
|
currentRunId: current.id,
|
|
1788
5094
|
improvedCases,
|
|
1789
5095
|
previousRunId: previous?.id,
|
|
1790
5096
|
regressedCases,
|
|
1791
5097
|
suiteId: current.suiteId,
|
|
5098
|
+
traceLeadChanges: traceLeadChanges.length > 0 ? traceLeadChanges : undefined,
|
|
1792
5099
|
summaryDelta: {
|
|
1793
5100
|
averageF1: current.response.summary.averageF1 - (previous?.response.summary.averageF1 ?? 0),
|
|
1794
5101
|
averageLatencyMs: current.response.summary.averageLatencyMs - (previous?.response.summary.averageLatencyMs ?? 0),
|
|
@@ -1881,6 +5188,101 @@ var createRAGFileEvaluationHistoryStore = (path) => ({
|
|
|
1881
5188
|
await mkdir(dirname(path), { recursive: true });
|
|
1882
5189
|
await writeFile(path, JSON.stringify(next, null, "\t") + `
|
|
1883
5190
|
`, "utf8");
|
|
5191
|
+
},
|
|
5192
|
+
pruneRuns: async (input) => {
|
|
5193
|
+
let existing = [];
|
|
5194
|
+
try {
|
|
5195
|
+
const content = await readFile(path, "utf8");
|
|
5196
|
+
const value = JSON.parse(content);
|
|
5197
|
+
existing = Array.isArray(value) ? value : [];
|
|
5198
|
+
} catch (error) {
|
|
5199
|
+
if (error.code !== "ENOENT") {
|
|
5200
|
+
throw error;
|
|
5201
|
+
}
|
|
5202
|
+
}
|
|
5203
|
+
const pruned = applyRAGEvaluationHistoryPrunePolicy({
|
|
5204
|
+
input,
|
|
5205
|
+
runs: existing,
|
|
5206
|
+
sort: normalizeHistoryRuns
|
|
5207
|
+
});
|
|
5208
|
+
await mkdir(dirname(path), { recursive: true });
|
|
5209
|
+
await writeFile(path, JSON.stringify(pruned.next, null, "\t") + `
|
|
5210
|
+
`, "utf8");
|
|
5211
|
+
return {
|
|
5212
|
+
keptCount: pruned.keptCount,
|
|
5213
|
+
removedCount: pruned.removedCount
|
|
5214
|
+
};
|
|
5215
|
+
}
|
|
5216
|
+
});
|
|
5217
|
+
var normalizeEvaluationSuiteSnapshots = (snapshots) => [...snapshots].sort((left, right) => {
|
|
5218
|
+
if (right.createdAt !== left.createdAt) {
|
|
5219
|
+
return right.createdAt - left.createdAt;
|
|
5220
|
+
}
|
|
5221
|
+
if (right.version !== left.version) {
|
|
5222
|
+
return right.version - left.version;
|
|
5223
|
+
}
|
|
5224
|
+
return right.id.localeCompare(left.id);
|
|
5225
|
+
});
|
|
5226
|
+
var createRAGFileEvaluationSuiteSnapshotHistoryStore = (path) => ({
|
|
5227
|
+
listSnapshots: async ({ limit, suiteId } = {}) => {
|
|
5228
|
+
let parsed = [];
|
|
5229
|
+
try {
|
|
5230
|
+
const content = await readFile(path, "utf8");
|
|
5231
|
+
const value = JSON.parse(content);
|
|
5232
|
+
parsed = Array.isArray(value) ? value : [];
|
|
5233
|
+
} catch (error) {
|
|
5234
|
+
if (error.code !== "ENOENT") {
|
|
5235
|
+
throw error;
|
|
5236
|
+
}
|
|
5237
|
+
}
|
|
5238
|
+
const filtered = parsed.filter((entry) => !suiteId || entry.suiteId === suiteId);
|
|
5239
|
+
const sorted = normalizeEvaluationSuiteSnapshots(filtered);
|
|
5240
|
+
return typeof limit === "number" ? sorted.slice(0, limit) : sorted;
|
|
5241
|
+
},
|
|
5242
|
+
saveSnapshot: async (snapshot) => {
|
|
5243
|
+
const existing = await (async () => {
|
|
5244
|
+
try {
|
|
5245
|
+
const content = await readFile(path, "utf8");
|
|
5246
|
+
const value = JSON.parse(content);
|
|
5247
|
+
return Array.isArray(value) ? value : [];
|
|
5248
|
+
} catch (error) {
|
|
5249
|
+
if (error.code !== "ENOENT") {
|
|
5250
|
+
throw error;
|
|
5251
|
+
}
|
|
5252
|
+
return [];
|
|
5253
|
+
}
|
|
5254
|
+
})();
|
|
5255
|
+
const next = normalizeEvaluationSuiteSnapshots([
|
|
5256
|
+
snapshot,
|
|
5257
|
+
...existing.filter((entry) => entry.id !== snapshot.id)
|
|
5258
|
+
]);
|
|
5259
|
+
await mkdir(dirname(path), { recursive: true });
|
|
5260
|
+
await writeFile(path, JSON.stringify(next, null, "\t") + `
|
|
5261
|
+
`, "utf8");
|
|
5262
|
+
},
|
|
5263
|
+
pruneSnapshots: async (input) => {
|
|
5264
|
+
let existing = [];
|
|
5265
|
+
try {
|
|
5266
|
+
const content = await readFile(path, "utf8");
|
|
5267
|
+
const value = JSON.parse(content);
|
|
5268
|
+
existing = Array.isArray(value) ? value : [];
|
|
5269
|
+
} catch (error) {
|
|
5270
|
+
if (error.code !== "ENOENT") {
|
|
5271
|
+
throw error;
|
|
5272
|
+
}
|
|
5273
|
+
}
|
|
5274
|
+
const pruned = applyRAGEvaluationHistoryPrunePolicy({
|
|
5275
|
+
input,
|
|
5276
|
+
runs: existing,
|
|
5277
|
+
sort: normalizeEvaluationSuiteSnapshots
|
|
5278
|
+
});
|
|
5279
|
+
await mkdir(dirname(path), { recursive: true });
|
|
5280
|
+
await writeFile(path, JSON.stringify(pruned.next, null, "\t") + `
|
|
5281
|
+
`, "utf8");
|
|
5282
|
+
return {
|
|
5283
|
+
keptCount: pruned.keptCount,
|
|
5284
|
+
removedCount: pruned.removedCount
|
|
5285
|
+
};
|
|
1884
5286
|
}
|
|
1885
5287
|
});
|
|
1886
5288
|
var createRAGFileRetrievalComparisonHistoryStore = (path) => ({
|
|
@@ -2504,32 +5906,209 @@ var createRAGFileSearchTracePruneHistoryStore = (path) => ({
|
|
|
2504
5906
|
throw error;
|
|
2505
5907
|
}
|
|
2506
5908
|
}
|
|
2507
|
-
const filtered = parsed.filter((entry) => !input?.trigger || entry.trigger === input.trigger);
|
|
2508
|
-
const sorted = normalizeSearchTracePruneRuns(filtered);
|
|
2509
|
-
return typeof input?.limit === "number" ? sorted.slice(0, input.limit) : sorted;
|
|
2510
|
-
},
|
|
2511
|
-
saveRun: async (run) => {
|
|
2512
|
-
const existing = await (async () => {
|
|
2513
|
-
try {
|
|
2514
|
-
const content = await readFile(path, "utf8");
|
|
2515
|
-
const value = JSON.parse(content);
|
|
2516
|
-
return Array.isArray(value) ? value : [];
|
|
2517
|
-
} catch (error) {
|
|
2518
|
-
if (error.code !== "ENOENT") {
|
|
2519
|
-
throw error;
|
|
2520
|
-
}
|
|
2521
|
-
return [];
|
|
5909
|
+
const filtered = parsed.filter((entry) => !input?.trigger || entry.trigger === input.trigger);
|
|
5910
|
+
const sorted = normalizeSearchTracePruneRuns(filtered);
|
|
5911
|
+
return typeof input?.limit === "number" ? sorted.slice(0, input.limit) : sorted;
|
|
5912
|
+
},
|
|
5913
|
+
saveRun: async (run) => {
|
|
5914
|
+
const existing = await (async () => {
|
|
5915
|
+
try {
|
|
5916
|
+
const content = await readFile(path, "utf8");
|
|
5917
|
+
const value = JSON.parse(content);
|
|
5918
|
+
return Array.isArray(value) ? value : [];
|
|
5919
|
+
} catch (error) {
|
|
5920
|
+
if (error.code !== "ENOENT") {
|
|
5921
|
+
throw error;
|
|
5922
|
+
}
|
|
5923
|
+
return [];
|
|
5924
|
+
}
|
|
5925
|
+
})();
|
|
5926
|
+
const next = normalizeSearchTracePruneRuns([
|
|
5927
|
+
run,
|
|
5928
|
+
...existing.filter((entry) => entry.id !== run.id)
|
|
5929
|
+
]);
|
|
5930
|
+
await mkdir(dirname(path), { recursive: true });
|
|
5931
|
+
await writeFile(path, JSON.stringify(next, null, "\t") + `
|
|
5932
|
+
`, "utf8");
|
|
5933
|
+
}
|
|
5934
|
+
});
|
|
5935
|
+
var createRAGSQLiteEvaluationHistoryStore = (options) => {
|
|
5936
|
+
const tableName = options.tableName ?? "rag_evaluation_history";
|
|
5937
|
+
assertSupportedIdentifier(tableName);
|
|
5938
|
+
const db = options.db ?? new (loadBunSQLiteModule()).Database(options.path ?? ":memory:");
|
|
5939
|
+
db.exec(`
|
|
5940
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
5941
|
+
id TEXT PRIMARY KEY,
|
|
5942
|
+
suite_id TEXT NOT NULL,
|
|
5943
|
+
finished_at INTEGER NOT NULL,
|
|
5944
|
+
record_json TEXT NOT NULL
|
|
5945
|
+
)
|
|
5946
|
+
`);
|
|
5947
|
+
db.exec(`CREATE INDEX IF NOT EXISTS ${tableName}_suite_finished_at_idx ON ${tableName} (suite_id, finished_at DESC)`);
|
|
5948
|
+
const insert = db.prepare(`
|
|
5949
|
+
INSERT INTO ${tableName} (
|
|
5950
|
+
id,
|
|
5951
|
+
suite_id,
|
|
5952
|
+
finished_at,
|
|
5953
|
+
record_json
|
|
5954
|
+
) VALUES (?, ?, ?, ?)
|
|
5955
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
5956
|
+
suite_id = excluded.suite_id,
|
|
5957
|
+
finished_at = excluded.finished_at,
|
|
5958
|
+
record_json = excluded.record_json
|
|
5959
|
+
`);
|
|
5960
|
+
return {
|
|
5961
|
+
listRuns(input = {}) {
|
|
5962
|
+
const where = [];
|
|
5963
|
+
const params = [];
|
|
5964
|
+
if (input.suiteId) {
|
|
5965
|
+
where.push("suite_id = ?");
|
|
5966
|
+
params.push(input.suiteId);
|
|
5967
|
+
}
|
|
5968
|
+
const sql = `SELECT record_json FROM ${tableName}` + `${where.length > 0 ? ` WHERE ${where.join(" AND ")}` : ""} ` + "ORDER BY finished_at DESC";
|
|
5969
|
+
const rows = db.prepare(sql).all(...params);
|
|
5970
|
+
const runs = normalizeHistoryRuns(rows.map((row) => JSON.parse(row.record_json)));
|
|
5971
|
+
return typeof input.limit === "number" ? runs.slice(0, input.limit) : runs;
|
|
5972
|
+
},
|
|
5973
|
+
saveRun(run) {
|
|
5974
|
+
insert.run(run.id, run.suiteId, run.finishedAt, JSON.stringify(run));
|
|
5975
|
+
},
|
|
5976
|
+
pruneRuns(input) {
|
|
5977
|
+
const allRows = db.prepare(`SELECT record_json FROM ${tableName}`).all();
|
|
5978
|
+
const pruned = applyRAGEvaluationHistoryPrunePolicy({
|
|
5979
|
+
input,
|
|
5980
|
+
runs: allRows.map((row) => JSON.parse(row.record_json)),
|
|
5981
|
+
sort: normalizeHistoryRuns
|
|
5982
|
+
});
|
|
5983
|
+
for (const run of pruned.removed) {
|
|
5984
|
+
db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(run.id);
|
|
5985
|
+
}
|
|
5986
|
+
return {
|
|
5987
|
+
keptCount: pruned.keptCount,
|
|
5988
|
+
removedCount: pruned.removedCount
|
|
5989
|
+
};
|
|
5990
|
+
}
|
|
5991
|
+
};
|
|
5992
|
+
};
|
|
5993
|
+
var createRAGSQLiteEvaluationSuiteSnapshotHistoryStore = (options) => {
|
|
5994
|
+
const tableName = options.tableName ?? "rag_evaluation_suite_snapshot_history";
|
|
5995
|
+
assertSupportedIdentifier(tableName);
|
|
5996
|
+
const db = options.db ?? new (loadBunSQLiteModule()).Database(options.path ?? ":memory:");
|
|
5997
|
+
db.exec(`
|
|
5998
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
5999
|
+
id TEXT PRIMARY KEY,
|
|
6000
|
+
suite_id TEXT NOT NULL,
|
|
6001
|
+
version INTEGER NOT NULL,
|
|
6002
|
+
created_at INTEGER NOT NULL,
|
|
6003
|
+
record_json TEXT NOT NULL
|
|
6004
|
+
)
|
|
6005
|
+
`);
|
|
6006
|
+
db.exec(`CREATE INDEX IF NOT EXISTS ${tableName}_suite_created_at_idx ON ${tableName} (suite_id, created_at DESC, version DESC)`);
|
|
6007
|
+
const insert = db.prepare(`
|
|
6008
|
+
INSERT INTO ${tableName} (
|
|
6009
|
+
id,
|
|
6010
|
+
suite_id,
|
|
6011
|
+
version,
|
|
6012
|
+
created_at,
|
|
6013
|
+
record_json
|
|
6014
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
6015
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
6016
|
+
suite_id = excluded.suite_id,
|
|
6017
|
+
version = excluded.version,
|
|
6018
|
+
created_at = excluded.created_at,
|
|
6019
|
+
record_json = excluded.record_json
|
|
6020
|
+
`);
|
|
6021
|
+
return {
|
|
6022
|
+
listSnapshots(input = {}) {
|
|
6023
|
+
const where = [];
|
|
6024
|
+
const params = [];
|
|
6025
|
+
if (input.suiteId) {
|
|
6026
|
+
where.push("suite_id = ?");
|
|
6027
|
+
params.push(input.suiteId);
|
|
6028
|
+
}
|
|
6029
|
+
const sql = `SELECT record_json FROM ${tableName}` + `${where.length > 0 ? ` WHERE ${where.join(" AND ")}` : ""} ` + "ORDER BY created_at DESC, version DESC";
|
|
6030
|
+
const rows = db.prepare(sql).all(...params);
|
|
6031
|
+
const snapshots = normalizeEvaluationSuiteSnapshots(rows.map((row) => JSON.parse(row.record_json)));
|
|
6032
|
+
return typeof input.limit === "number" ? snapshots.slice(0, input.limit) : snapshots;
|
|
6033
|
+
},
|
|
6034
|
+
saveSnapshot(snapshot) {
|
|
6035
|
+
insert.run(snapshot.id, snapshot.suiteId, snapshot.version, snapshot.createdAt, JSON.stringify(snapshot));
|
|
6036
|
+
},
|
|
6037
|
+
pruneSnapshots(input) {
|
|
6038
|
+
const allRows = db.prepare(`SELECT record_json FROM ${tableName}`).all();
|
|
6039
|
+
const pruned = applyRAGEvaluationHistoryPrunePolicy({
|
|
6040
|
+
input,
|
|
6041
|
+
runs: allRows.map((row) => JSON.parse(row.record_json)),
|
|
6042
|
+
sort: normalizeEvaluationSuiteSnapshots
|
|
6043
|
+
});
|
|
6044
|
+
for (const snapshot of pruned.removed) {
|
|
6045
|
+
db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(snapshot.id);
|
|
6046
|
+
}
|
|
6047
|
+
return {
|
|
6048
|
+
keptCount: pruned.keptCount,
|
|
6049
|
+
removedCount: pruned.removedCount
|
|
6050
|
+
};
|
|
6051
|
+
}
|
|
6052
|
+
};
|
|
6053
|
+
};
|
|
6054
|
+
var createRAGSQLiteAnswerGroundingEvaluationHistoryStore = (options) => {
|
|
6055
|
+
const tableName = options.tableName ?? "rag_answer_grounding_history";
|
|
6056
|
+
assertSupportedIdentifier(tableName);
|
|
6057
|
+
const db = options.db ?? new (loadBunSQLiteModule()).Database(options.path ?? ":memory:");
|
|
6058
|
+
db.exec(`
|
|
6059
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
6060
|
+
id TEXT PRIMARY KEY,
|
|
6061
|
+
suite_id TEXT NOT NULL,
|
|
6062
|
+
finished_at INTEGER NOT NULL,
|
|
6063
|
+
record_json TEXT NOT NULL
|
|
6064
|
+
)
|
|
6065
|
+
`);
|
|
6066
|
+
db.exec(`CREATE INDEX IF NOT EXISTS ${tableName}_suite_finished_at_idx ON ${tableName} (suite_id, finished_at DESC)`);
|
|
6067
|
+
const insert = db.prepare(`
|
|
6068
|
+
INSERT INTO ${tableName} (
|
|
6069
|
+
id,
|
|
6070
|
+
suite_id,
|
|
6071
|
+
finished_at,
|
|
6072
|
+
record_json
|
|
6073
|
+
) VALUES (?, ?, ?, ?)
|
|
6074
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
6075
|
+
suite_id = excluded.suite_id,
|
|
6076
|
+
finished_at = excluded.finished_at,
|
|
6077
|
+
record_json = excluded.record_json
|
|
6078
|
+
`);
|
|
6079
|
+
return {
|
|
6080
|
+
listRuns(input = {}) {
|
|
6081
|
+
const where = [];
|
|
6082
|
+
const params = [];
|
|
6083
|
+
if (input.suiteId) {
|
|
6084
|
+
where.push("suite_id = ?");
|
|
6085
|
+
params.push(input.suiteId);
|
|
2522
6086
|
}
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
});
|
|
6087
|
+
const sql = `SELECT record_json FROM ${tableName}` + `${where.length > 0 ? ` WHERE ${where.join(" AND ")}` : ""} ` + "ORDER BY finished_at DESC";
|
|
6088
|
+
const rows = db.prepare(sql).all(...params);
|
|
6089
|
+
const runs = normalizeGroundingHistoryRuns(rows.map((row) => JSON.parse(row.record_json)));
|
|
6090
|
+
return typeof input.limit === "number" ? runs.slice(0, input.limit) : runs;
|
|
6091
|
+
},
|
|
6092
|
+
saveRun(run) {
|
|
6093
|
+
insert.run(run.id, run.suiteId, run.finishedAt, JSON.stringify(run));
|
|
6094
|
+
},
|
|
6095
|
+
pruneRuns(input) {
|
|
6096
|
+
const allRows = db.prepare(`SELECT record_json FROM ${tableName}`).all();
|
|
6097
|
+
const pruned = applyRAGEvaluationHistoryPrunePolicy({
|
|
6098
|
+
input,
|
|
6099
|
+
runs: allRows.map((row) => JSON.parse(row.record_json)),
|
|
6100
|
+
sort: normalizeGroundingHistoryRuns
|
|
6101
|
+
});
|
|
6102
|
+
for (const run of pruned.removed) {
|
|
6103
|
+
db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(run.id);
|
|
6104
|
+
}
|
|
6105
|
+
return {
|
|
6106
|
+
keptCount: pruned.keptCount,
|
|
6107
|
+
removedCount: pruned.removedCount
|
|
6108
|
+
};
|
|
6109
|
+
}
|
|
6110
|
+
};
|
|
6111
|
+
};
|
|
2533
6112
|
var isSQLiteSearchTraceRow = (value) => isObjectRecord(value) && typeof value.id === "string" && typeof value.query === "string" && typeof value.label === "string" && (typeof value.group_key === "string" || value.group_key === null) && (typeof value.tags_json === "string" || value.tags_json === null) && typeof value.started_at === "number" && typeof value.finished_at === "number" && typeof value.elapsed_ms === "number" && typeof value.trace_json === "string" && typeof value.summary_json === "string" && typeof value.results_json === "string" && (typeof value.metadata_json === "string" || value.metadata_json === null);
|
|
2534
6113
|
var isSQLiteSearchTracePruneHistoryRow = (value) => isObjectRecord(value) && typeof value.id === "string" && typeof value.trigger === "string" && typeof value.started_at === "number" && typeof value.finished_at === "number" && typeof value.elapsed_ms === "number" && typeof value.run_json === "string";
|
|
2535
6114
|
var isSQLiteRetrievalComparisonHistoryRow = (value) => isObjectRecord(value) && typeof value.id === "string" && typeof value.suite_id === "string" && typeof value.suite_label === "string" && typeof value.label === "string" && (typeof value.corpus_group_key === "string" || value.corpus_group_key === null) && (typeof value.group_key === "string" || value.group_key === null) && (typeof value.tags_json === "string" || value.tags_json === null) && typeof value.started_at === "number" && typeof value.finished_at === "number" && typeof value.elapsed_ms === "number" && typeof value.comparison_json === "string" && (typeof value.decision_summary_json === "string" || value.decision_summary_json === null);
|
|
@@ -2560,6 +6139,57 @@ var ensureSQLiteColumns = (db, tableName, columns) => {
|
|
|
2560
6139
|
db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${column.name} ${column.definition}`);
|
|
2561
6140
|
}
|
|
2562
6141
|
};
|
|
6142
|
+
var getMissingSQLiteColumns = (db, tableName, columns) => {
|
|
6143
|
+
const pragma = db.prepare(`PRAGMA table_info(${tableName})`).all();
|
|
6144
|
+
const existing = new Set(Array.isArray(pragma) ? pragma.filter((row) => isObjectRecord(row) && ("name" in row)).map((row) => typeof row.name === "string" ? row.name : undefined).filter((name) => typeof name === "string") : []);
|
|
6145
|
+
return columns.filter((column) => !existing.has(column.name)).map((column) => ({
|
|
6146
|
+
columnName: column.name,
|
|
6147
|
+
definition: column.definition,
|
|
6148
|
+
tableName
|
|
6149
|
+
}));
|
|
6150
|
+
};
|
|
6151
|
+
var getDefaultSQLiteStoreMigrationDescriptors = () => [
|
|
6152
|
+
{
|
|
6153
|
+
tableName: DEFAULT_RETRIEVAL_COMPARISON_HISTORY_TABLE_NAME,
|
|
6154
|
+
columns: [{ definition: "TEXT", name: "corpus_group_key" }]
|
|
6155
|
+
},
|
|
6156
|
+
{
|
|
6157
|
+
tableName: DEFAULT_RETRIEVAL_RELEASE_DECISION_TABLE_NAME,
|
|
6158
|
+
columns: [{ definition: "TEXT", name: "corpus_group_key" }]
|
|
6159
|
+
},
|
|
6160
|
+
{
|
|
6161
|
+
tableName: DEFAULT_RETRIEVAL_BASELINE_TABLE_NAME,
|
|
6162
|
+
columns: [
|
|
6163
|
+
{ definition: "TEXT", name: "group_key" },
|
|
6164
|
+
{ definition: "TEXT", name: "corpus_group_key" }
|
|
6165
|
+
]
|
|
6166
|
+
}
|
|
6167
|
+
];
|
|
6168
|
+
var summarizeSQLiteStoreMigrationIssues = (issues) => issues.length > 0 ? `${issues.length} SQLite schema migration issue${issues.length === 1 ? "" : "s"} detected` : undefined;
|
|
6169
|
+
var inspectRAGSQLiteStoreMigrations = (options = {}) => {
|
|
6170
|
+
const db = options.db ?? new (loadBunSQLiteModule()).Database(options.path ?? ":memory:");
|
|
6171
|
+
const descriptors = options.descriptors ?? getDefaultSQLiteStoreMigrationDescriptors();
|
|
6172
|
+
const issues = descriptors.flatMap((descriptor) => getMissingSQLiteColumns(db, descriptor.tableName, descriptor.columns));
|
|
6173
|
+
return {
|
|
6174
|
+
issues,
|
|
6175
|
+
summary: summarizeSQLiteStoreMigrationIssues(issues)
|
|
6176
|
+
};
|
|
6177
|
+
};
|
|
6178
|
+
var applyRAGSQLiteStoreMigrations = (options = {}) => {
|
|
6179
|
+
const db = options.db ?? new (loadBunSQLiteModule()).Database(options.path ?? ":memory:");
|
|
6180
|
+
const descriptors = options.descriptors ?? getDefaultSQLiteStoreMigrationDescriptors();
|
|
6181
|
+
const inspection = inspectRAGSQLiteStoreMigrations({
|
|
6182
|
+
db,
|
|
6183
|
+
descriptors
|
|
6184
|
+
});
|
|
6185
|
+
for (const descriptor of descriptors) {
|
|
6186
|
+
ensureSQLiteColumns(db, descriptor.tableName, descriptor.columns);
|
|
6187
|
+
}
|
|
6188
|
+
return {
|
|
6189
|
+
...inspection,
|
|
6190
|
+
applied: inspection.issues
|
|
6191
|
+
};
|
|
6192
|
+
};
|
|
2563
6193
|
var mapSQLiteSearchTraceRow = (row) => ({
|
|
2564
6194
|
elapsedMs: row.elapsed_ms,
|
|
2565
6195
|
finishedAt: row.finished_at,
|
|
@@ -3930,6 +7560,31 @@ var createRAGFileAnswerGroundingEvaluationHistoryStore = (path) => ({
|
|
|
3930
7560
|
await writeFile(path, JSON.stringify({
|
|
3931
7561
|
runs: nextRuns
|
|
3932
7562
|
}, null, 2));
|
|
7563
|
+
},
|
|
7564
|
+
async pruneRuns(input) {
|
|
7565
|
+
let runs = [];
|
|
7566
|
+
try {
|
|
7567
|
+
const raw = await readFile(path, "utf8");
|
|
7568
|
+
const data = JSON.parse(raw);
|
|
7569
|
+
runs = Array.isArray(data.runs) ? data.runs : [];
|
|
7570
|
+
} catch (error) {
|
|
7571
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") {
|
|
7572
|
+
throw error;
|
|
7573
|
+
}
|
|
7574
|
+
}
|
|
7575
|
+
const pruned = applyRAGEvaluationHistoryPrunePolicy({
|
|
7576
|
+
input,
|
|
7577
|
+
runs,
|
|
7578
|
+
sort: normalizeGroundingHistoryRuns
|
|
7579
|
+
});
|
|
7580
|
+
await mkdir(dirname(path), { recursive: true });
|
|
7581
|
+
await writeFile(path, JSON.stringify({
|
|
7582
|
+
runs: pruned.next
|
|
7583
|
+
}, null, 2));
|
|
7584
|
+
return {
|
|
7585
|
+
keptCount: pruned.keptCount,
|
|
7586
|
+
removedCount: pruned.removedCount
|
|
7587
|
+
};
|
|
3933
7588
|
}
|
|
3934
7589
|
});
|
|
3935
7590
|
var createRAGFileAnswerGroundingCaseDifficultyHistoryStore = (path) => ({
|
|
@@ -3996,6 +7651,29 @@ var loadRAGEvaluationHistory = async ({
|
|
|
3996
7651
|
suiteLabel: suite.label ?? suite.id
|
|
3997
7652
|
};
|
|
3998
7653
|
};
|
|
7654
|
+
var loadRAGEvaluationSuiteSnapshotHistory = async ({
|
|
7655
|
+
store,
|
|
7656
|
+
suite,
|
|
7657
|
+
limit = DEFAULT_HISTORY_LIMIT
|
|
7658
|
+
}) => {
|
|
7659
|
+
const snapshots = normalizeEvaluationSuiteSnapshots(await Promise.resolve(store.listSnapshots({
|
|
7660
|
+
limit,
|
|
7661
|
+
suiteId: suite.id
|
|
7662
|
+
})));
|
|
7663
|
+
const latestSnapshot = snapshots[0];
|
|
7664
|
+
const previousSnapshot = snapshots[1];
|
|
7665
|
+
return {
|
|
7666
|
+
diff: latestSnapshot && previousSnapshot ? buildRAGEvaluationSuiteSnapshotDiff({
|
|
7667
|
+
current: latestSnapshot,
|
|
7668
|
+
previous: previousSnapshot
|
|
7669
|
+
}) : undefined,
|
|
7670
|
+
latestSnapshot,
|
|
7671
|
+
previousSnapshot,
|
|
7672
|
+
snapshots,
|
|
7673
|
+
suiteId: suite.id,
|
|
7674
|
+
suiteLabel: suite.label ?? suite.id
|
|
7675
|
+
};
|
|
7676
|
+
};
|
|
3999
7677
|
var loadRAGSearchTraceHistory = async ({
|
|
4000
7678
|
store,
|
|
4001
7679
|
query,
|
|
@@ -4626,6 +8304,118 @@ var buildRAGEvaluationResponse = (cases) => {
|
|
|
4626
8304
|
totalCases
|
|
4627
8305
|
};
|
|
4628
8306
|
};
|
|
8307
|
+
var incrementFailureCounts = (target, failureClasses) => {
|
|
8308
|
+
for (const failureClass of failureClasses ?? []) {
|
|
8309
|
+
target[failureClass] = (target[failureClass] ?? 0) + 1;
|
|
8310
|
+
}
|
|
8311
|
+
};
|
|
8312
|
+
var sortEntityQualitySummaries = (entries) => entries.sort((left, right) => {
|
|
8313
|
+
if (right.passingRate !== left.passingRate) {
|
|
8314
|
+
return right.passingRate - left.passingRate;
|
|
8315
|
+
}
|
|
8316
|
+
if (right.totalCases !== left.totalCases) {
|
|
8317
|
+
return right.totalCases - left.totalCases;
|
|
8318
|
+
}
|
|
8319
|
+
return left.label.localeCompare(right.label);
|
|
8320
|
+
});
|
|
8321
|
+
var buildRAGEvaluationEntityQualityView = (response) => {
|
|
8322
|
+
const bySource = new Map;
|
|
8323
|
+
const byDocument = new Map;
|
|
8324
|
+
for (const entry of response.cases) {
|
|
8325
|
+
const targets = entry.mode === "source" ? entry.expectedIds : entry.mode === "documentId" ? entry.expectedIds : [];
|
|
8326
|
+
const targetMap = entry.mode === "source" ? bySource : entry.mode === "documentId" ? byDocument : undefined;
|
|
8327
|
+
if (!targetMap) {
|
|
8328
|
+
continue;
|
|
8329
|
+
}
|
|
8330
|
+
for (const target of targets) {
|
|
8331
|
+
const current = targetMap.get(target) ?? {
|
|
8332
|
+
averageF1: 0,
|
|
8333
|
+
caseIds: [],
|
|
8334
|
+
entityType: entry.mode === "source" ? "source" : "document",
|
|
8335
|
+
failedCases: 0,
|
|
8336
|
+
failureCounts: {},
|
|
8337
|
+
key: target,
|
|
8338
|
+
label: target,
|
|
8339
|
+
passedCases: 0,
|
|
8340
|
+
passingRate: 0,
|
|
8341
|
+
partialCases: 0,
|
|
8342
|
+
totalCases: 0
|
|
8343
|
+
};
|
|
8344
|
+
current.totalCases += 1;
|
|
8345
|
+
current.averageF1 += entry.f1;
|
|
8346
|
+
current.caseIds.push(entry.caseId);
|
|
8347
|
+
if (entry.status === "pass") {
|
|
8348
|
+
current.passedCases += 1;
|
|
8349
|
+
} else if (entry.status === "partial") {
|
|
8350
|
+
current.partialCases += 1;
|
|
8351
|
+
} else {
|
|
8352
|
+
current.failedCases += 1;
|
|
8353
|
+
}
|
|
8354
|
+
incrementFailureCounts(current.failureCounts, entry.failureClasses);
|
|
8355
|
+
targetMap.set(target, current);
|
|
8356
|
+
}
|
|
8357
|
+
}
|
|
8358
|
+
for (const map of [bySource, byDocument]) {
|
|
8359
|
+
for (const entry of map.values()) {
|
|
8360
|
+
entry.averageF1 = entry.totalCases > 0 ? entry.averageF1 / entry.totalCases : 0;
|
|
8361
|
+
entry.passingRate = entry.totalCases > 0 ? entry.passedCases / entry.totalCases * 100 : 0;
|
|
8362
|
+
}
|
|
8363
|
+
}
|
|
8364
|
+
return {
|
|
8365
|
+
byDocument: sortEntityQualitySummaries([...byDocument.values()]),
|
|
8366
|
+
bySource: sortEntityQualitySummaries([...bySource.values()])
|
|
8367
|
+
};
|
|
8368
|
+
};
|
|
8369
|
+
var buildRAGAnswerGroundingEntityQualityView = (response) => {
|
|
8370
|
+
const bySource = new Map;
|
|
8371
|
+
const byDocument = new Map;
|
|
8372
|
+
for (const entry of response.cases) {
|
|
8373
|
+
const targetMap = entry.mode === "source" ? bySource : entry.mode === "documentId" ? byDocument : undefined;
|
|
8374
|
+
if (!targetMap) {
|
|
8375
|
+
continue;
|
|
8376
|
+
}
|
|
8377
|
+
for (const target of entry.expectedIds) {
|
|
8378
|
+
const current = targetMap.get(target) ?? {
|
|
8379
|
+
averageCitationF1: 0,
|
|
8380
|
+
averageResolvedCitationRate: 0,
|
|
8381
|
+
caseIds: [],
|
|
8382
|
+
entityType: entry.mode === "source" ? "source" : "document",
|
|
8383
|
+
failedCases: 0,
|
|
8384
|
+
failureCounts: {},
|
|
8385
|
+
key: target,
|
|
8386
|
+
label: target,
|
|
8387
|
+
passedCases: 0,
|
|
8388
|
+
passingRate: 0,
|
|
8389
|
+
partialCases: 0,
|
|
8390
|
+
totalCases: 0
|
|
8391
|
+
};
|
|
8392
|
+
current.totalCases += 1;
|
|
8393
|
+
current.averageCitationF1 += entry.citationF1;
|
|
8394
|
+
current.averageResolvedCitationRate += entry.resolvedCitationRate;
|
|
8395
|
+
current.caseIds.push(entry.caseId);
|
|
8396
|
+
if (entry.status === "pass") {
|
|
8397
|
+
current.passedCases += 1;
|
|
8398
|
+
} else if (entry.status === "partial") {
|
|
8399
|
+
current.partialCases += 1;
|
|
8400
|
+
} else {
|
|
8401
|
+
current.failedCases += 1;
|
|
8402
|
+
}
|
|
8403
|
+
incrementFailureCounts(current.failureCounts, entry.failureClasses);
|
|
8404
|
+
targetMap.set(target, current);
|
|
8405
|
+
}
|
|
8406
|
+
}
|
|
8407
|
+
for (const map of [bySource, byDocument]) {
|
|
8408
|
+
for (const entry of map.values()) {
|
|
8409
|
+
entry.averageCitationF1 = entry.totalCases > 0 ? entry.averageCitationF1 / entry.totalCases : 0;
|
|
8410
|
+
entry.averageResolvedCitationRate = entry.totalCases > 0 ? entry.averageResolvedCitationRate / entry.totalCases : 0;
|
|
8411
|
+
entry.passingRate = entry.totalCases > 0 ? entry.passedCases / entry.totalCases * 100 : 0;
|
|
8412
|
+
}
|
|
8413
|
+
}
|
|
8414
|
+
return {
|
|
8415
|
+
byDocument: sortEntityQualitySummaries([...byDocument.values()]),
|
|
8416
|
+
bySource: sortEntityQualitySummaries([...bySource.values()])
|
|
8417
|
+
};
|
|
8418
|
+
};
|
|
4629
8419
|
var evaluateRAGAnswerGroundingCase = ({
|
|
4630
8420
|
caseIndex,
|
|
4631
8421
|
caseInput
|
|
@@ -4667,6 +8457,15 @@ var evaluateRAGAnswerGroundingCase = ({
|
|
|
4667
8457
|
label: caseInput.label,
|
|
4668
8458
|
matchedCount,
|
|
4669
8459
|
matchedIds,
|
|
8460
|
+
failureClasses: classifyRAGGroundingFailure({
|
|
8461
|
+
availableSources: caseInput.sources,
|
|
8462
|
+
citationCount,
|
|
8463
|
+
expectedCount,
|
|
8464
|
+
extraIds,
|
|
8465
|
+
matchedCount,
|
|
8466
|
+
missingIds,
|
|
8467
|
+
unresolvedCitationCount
|
|
8468
|
+
}),
|
|
4670
8469
|
metadata: caseInput.metadata,
|
|
4671
8470
|
missingIds,
|
|
4672
8471
|
mode,
|
|
@@ -4982,7 +8781,327 @@ var buildRAGRetrievalTraceHistoryTrend = ({
|
|
|
4982
8781
|
latestToPrevious
|
|
4983
8782
|
};
|
|
4984
8783
|
};
|
|
4985
|
-
var createRAGEvaluationSuite = (suite) =>
|
|
8784
|
+
var createRAGEvaluationSuite = (suite) => {
|
|
8785
|
+
const cases = suite.input.cases.map((entry) => ({
|
|
8786
|
+
...entry,
|
|
8787
|
+
...entry.goldenSet === true ? { goldenSet: true } : {},
|
|
8788
|
+
...entry.expectedChunkIds ? { expectedChunkIds: [...entry.expectedChunkIds] } : {},
|
|
8789
|
+
...entry.expectedSources ? { expectedSources: [...entry.expectedSources] } : {},
|
|
8790
|
+
...entry.expectedDocumentIds ? { expectedDocumentIds: [...entry.expectedDocumentIds] } : {},
|
|
8791
|
+
...entry.hardNegativeChunkIds ? {
|
|
8792
|
+
hardNegativeChunkIds: normalizeExpectedIds(entry.hardNegativeChunkIds)
|
|
8793
|
+
} : {},
|
|
8794
|
+
...entry.hardNegativeSources ? {
|
|
8795
|
+
hardNegativeSources: normalizeExpectedIds(entry.hardNegativeSources)
|
|
8796
|
+
} : {},
|
|
8797
|
+
...entry.hardNegativeDocumentIds ? {
|
|
8798
|
+
hardNegativeDocumentIds: normalizeExpectedIds(entry.hardNegativeDocumentIds)
|
|
8799
|
+
} : {},
|
|
8800
|
+
...entry.filter ? { filter: { ...entry.filter } } : {},
|
|
8801
|
+
...entry.metadata ? { metadata: { ...entry.metadata } } : {},
|
|
8802
|
+
...entry.retrieval ? { retrieval: entry.retrieval } : {}
|
|
8803
|
+
}));
|
|
8804
|
+
const duplicateCaseIds = Array.from(new Set(cases.map((entry) => entry.id).filter((id, index, ids) => typeof id === "string" && ids.indexOf(id) !== index)));
|
|
8805
|
+
if (duplicateCaseIds.length > 0) {
|
|
8806
|
+
throw new Error(`RAG evaluation suite contains duplicate case ids: ${duplicateCaseIds.join(", ")}`);
|
|
8807
|
+
}
|
|
8808
|
+
return {
|
|
8809
|
+
...suite,
|
|
8810
|
+
input: {
|
|
8811
|
+
...suite.input,
|
|
8812
|
+
cases,
|
|
8813
|
+
...suite.input.filter ? { filter: { ...suite.input.filter } } : {},
|
|
8814
|
+
...suite.input.retrieval ? { retrieval: suite.input.retrieval } : {}
|
|
8815
|
+
},
|
|
8816
|
+
...suite.metadata ? { metadata: { ...suite.metadata } } : {}
|
|
8817
|
+
};
|
|
8818
|
+
};
|
|
8819
|
+
var addRAGEvaluationSuiteCase = ({
|
|
8820
|
+
suite,
|
|
8821
|
+
caseInput,
|
|
8822
|
+
index
|
|
8823
|
+
}) => {
|
|
8824
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8825
|
+
if (normalizedSuite.input.cases.some((entry) => entry.id === caseInput.id)) {
|
|
8826
|
+
throw new Error(`RAG evaluation suite already contains case id ${caseInput.id}`);
|
|
8827
|
+
}
|
|
8828
|
+
const nextCases = [...normalizedSuite.input.cases];
|
|
8829
|
+
const insertAt = typeof index === "number" && Number.isFinite(index) ? Math.max(0, Math.min(nextCases.length, Math.trunc(index))) : nextCases.length;
|
|
8830
|
+
nextCases.splice(insertAt, 0, caseInput);
|
|
8831
|
+
return createRAGEvaluationSuite({
|
|
8832
|
+
...normalizedSuite,
|
|
8833
|
+
input: {
|
|
8834
|
+
...normalizedSuite.input,
|
|
8835
|
+
cases: nextCases
|
|
8836
|
+
}
|
|
8837
|
+
});
|
|
8838
|
+
};
|
|
8839
|
+
var updateRAGEvaluationSuiteCase = ({
|
|
8840
|
+
suite,
|
|
8841
|
+
caseId,
|
|
8842
|
+
caseInput
|
|
8843
|
+
}) => {
|
|
8844
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8845
|
+
const targetIndex = normalizedSuite.input.cases.findIndex((entry) => entry.id === caseId);
|
|
8846
|
+
if (targetIndex < 0) {
|
|
8847
|
+
throw new Error(`RAG evaluation suite does not contain case id ${caseId}`);
|
|
8848
|
+
}
|
|
8849
|
+
if (caseInput.id !== caseId && normalizedSuite.input.cases.some((entry) => entry.id === caseInput.id)) {
|
|
8850
|
+
throw new Error(`RAG evaluation suite already contains case id ${caseInput.id}`);
|
|
8851
|
+
}
|
|
8852
|
+
const nextCases = [...normalizedSuite.input.cases];
|
|
8853
|
+
nextCases[targetIndex] = caseInput;
|
|
8854
|
+
return createRAGEvaluationSuite({
|
|
8855
|
+
...normalizedSuite,
|
|
8856
|
+
input: {
|
|
8857
|
+
...normalizedSuite.input,
|
|
8858
|
+
cases: nextCases
|
|
8859
|
+
}
|
|
8860
|
+
});
|
|
8861
|
+
};
|
|
8862
|
+
var removeRAGEvaluationSuiteCase = ({
|
|
8863
|
+
suite,
|
|
8864
|
+
caseId
|
|
8865
|
+
}) => {
|
|
8866
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8867
|
+
const nextCases = normalizedSuite.input.cases.filter((entry) => entry.id !== caseId);
|
|
8868
|
+
if (nextCases.length === normalizedSuite.input.cases.length) {
|
|
8869
|
+
throw new Error(`RAG evaluation suite does not contain case id ${caseId}`);
|
|
8870
|
+
}
|
|
8871
|
+
return {
|
|
8872
|
+
...normalizedSuite,
|
|
8873
|
+
input: {
|
|
8874
|
+
...normalizedSuite.input,
|
|
8875
|
+
cases: nextCases
|
|
8876
|
+
}
|
|
8877
|
+
};
|
|
8878
|
+
};
|
|
8879
|
+
var reorderRAGEvaluationSuiteCases = ({
|
|
8880
|
+
suite,
|
|
8881
|
+
caseIds
|
|
8882
|
+
}) => {
|
|
8883
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8884
|
+
const caseMap = new Map(normalizedSuite.input.cases.map((entry) => [entry.id, entry]));
|
|
8885
|
+
if (caseIds.length !== normalizedSuite.input.cases.length) {
|
|
8886
|
+
throw new Error("RAG evaluation suite reorder requires exactly one id per case");
|
|
8887
|
+
}
|
|
8888
|
+
const unknownCaseIds = caseIds.filter((id) => !caseMap.has(id));
|
|
8889
|
+
if (unknownCaseIds.length > 0) {
|
|
8890
|
+
throw new Error(`RAG evaluation suite reorder contains unknown case ids: ${unknownCaseIds.join(", ")}`);
|
|
8891
|
+
}
|
|
8892
|
+
if (new Set(caseIds).size !== caseIds.length) {
|
|
8893
|
+
throw new Error("RAG evaluation suite reorder contains duplicate case ids");
|
|
8894
|
+
}
|
|
8895
|
+
return {
|
|
8896
|
+
...normalizedSuite,
|
|
8897
|
+
input: {
|
|
8898
|
+
...normalizedSuite.input,
|
|
8899
|
+
cases: caseIds.map((id) => caseMap.get(id))
|
|
8900
|
+
}
|
|
8901
|
+
};
|
|
8902
|
+
};
|
|
8903
|
+
var setRAGEvaluationSuiteCaseGoldenSet = ({
|
|
8904
|
+
suite,
|
|
8905
|
+
caseId,
|
|
8906
|
+
goldenSet
|
|
8907
|
+
}) => {
|
|
8908
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8909
|
+
const target = normalizedSuite.input.cases.find((entry) => entry.id === caseId);
|
|
8910
|
+
if (!target) {
|
|
8911
|
+
throw new Error(`RAG evaluation suite does not contain case id ${caseId}`);
|
|
8912
|
+
}
|
|
8913
|
+
return updateRAGEvaluationSuiteCase({
|
|
8914
|
+
caseId,
|
|
8915
|
+
caseInput: {
|
|
8916
|
+
...target,
|
|
8917
|
+
...goldenSet ? { goldenSet: true } : { goldenSet: undefined }
|
|
8918
|
+
},
|
|
8919
|
+
suite: normalizedSuite
|
|
8920
|
+
});
|
|
8921
|
+
};
|
|
8922
|
+
var resolveHardNegativeCaseField = (kind) => kind === "chunkId" ? "hardNegativeChunkIds" : kind === "source" ? "hardNegativeSources" : "hardNegativeDocumentIds";
|
|
8923
|
+
var addRAGEvaluationSuiteCaseHardNegative = ({
|
|
8924
|
+
suite,
|
|
8925
|
+
caseId,
|
|
8926
|
+
kind,
|
|
8927
|
+
value
|
|
8928
|
+
}) => {
|
|
8929
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8930
|
+
const target = normalizedSuite.input.cases.find((entry) => entry.id === caseId);
|
|
8931
|
+
if (!target) {
|
|
8932
|
+
throw new Error(`RAG evaluation suite does not contain case id ${caseId}`);
|
|
8933
|
+
}
|
|
8934
|
+
const field = resolveHardNegativeCaseField(kind);
|
|
8935
|
+
const nextValues = normalizeExpectedIds([
|
|
8936
|
+
...normalizeStringArray(target[field]),
|
|
8937
|
+
value
|
|
8938
|
+
]);
|
|
8939
|
+
return updateRAGEvaluationSuiteCase({
|
|
8940
|
+
caseId,
|
|
8941
|
+
caseInput: {
|
|
8942
|
+
...target,
|
|
8943
|
+
[field]: nextValues
|
|
8944
|
+
},
|
|
8945
|
+
suite: normalizedSuite
|
|
8946
|
+
});
|
|
8947
|
+
};
|
|
8948
|
+
var removeRAGEvaluationSuiteCaseHardNegative = ({
|
|
8949
|
+
suite,
|
|
8950
|
+
caseId,
|
|
8951
|
+
kind,
|
|
8952
|
+
value
|
|
8953
|
+
}) => {
|
|
8954
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8955
|
+
const target = normalizedSuite.input.cases.find((entry) => entry.id === caseId);
|
|
8956
|
+
if (!target) {
|
|
8957
|
+
throw new Error(`RAG evaluation suite does not contain case id ${caseId}`);
|
|
8958
|
+
}
|
|
8959
|
+
const field = resolveHardNegativeCaseField(kind);
|
|
8960
|
+
const nextValues = normalizeStringArray(target[field]).filter((entry) => entry !== value.trim());
|
|
8961
|
+
return updateRAGEvaluationSuiteCase({
|
|
8962
|
+
caseId,
|
|
8963
|
+
caseInput: {
|
|
8964
|
+
...target,
|
|
8965
|
+
[field]: nextValues.length > 0 ? nextValues : undefined
|
|
8966
|
+
},
|
|
8967
|
+
suite: normalizedSuite
|
|
8968
|
+
});
|
|
8969
|
+
};
|
|
8970
|
+
var summarizeRAGEvaluationSuiteDataset = ({
|
|
8971
|
+
suite
|
|
8972
|
+
}) => {
|
|
8973
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
8974
|
+
const cases = normalizedSuite.input.cases;
|
|
8975
|
+
return {
|
|
8976
|
+
caseCount: cases.length,
|
|
8977
|
+
goldenSetCount: cases.filter((entry) => entry.goldenSet === true).length,
|
|
8978
|
+
hardNegativeCaseCount: cases.filter((entry) => normalizeStringArray(entry.hardNegativeChunkIds).length > 0 || normalizeStringArray(entry.hardNegativeSources).length > 0 || normalizeStringArray(entry.hardNegativeDocumentIds).length > 0).length,
|
|
8979
|
+
hardNegativeChunkIdCount: cases.reduce((sum, entry) => sum + normalizeStringArray(entry.hardNegativeChunkIds).length, 0),
|
|
8980
|
+
hardNegativeDocumentIdCount: cases.reduce((sum, entry) => sum + normalizeStringArray(entry.hardNegativeDocumentIds).length, 0),
|
|
8981
|
+
hardNegativeSourceCount: cases.reduce((sum, entry) => sum + normalizeStringArray(entry.hardNegativeSources).length, 0),
|
|
8982
|
+
suiteId: normalizedSuite.id
|
|
8983
|
+
};
|
|
8984
|
+
};
|
|
8985
|
+
var generateRAGEvaluationSuiteFromDocuments = ({
|
|
8986
|
+
suiteId,
|
|
8987
|
+
documents,
|
|
8988
|
+
label,
|
|
8989
|
+
description,
|
|
8990
|
+
maxCases = 20,
|
|
8991
|
+
topK = 5,
|
|
8992
|
+
scoreThreshold,
|
|
8993
|
+
filter,
|
|
8994
|
+
retrieval,
|
|
8995
|
+
includeGoldenSet = true,
|
|
8996
|
+
hardNegativePerCase = 1,
|
|
8997
|
+
metadata
|
|
8998
|
+
}) => {
|
|
8999
|
+
const normalizedDocuments = documents.filter((document) => typeof document.id === "string" && document.id.trim().length > 0 && typeof document.source === "string" && document.source.trim().length > 0 && typeof document.title === "string" && document.title.trim().length > 0).sort((left, right) => `${left.source}\x00${left.title}\x00${left.id}`.localeCompare(`${right.source}\x00${right.title}\x00${right.id}`)).slice(0, Math.max(0, Math.trunc(maxCases)));
|
|
9000
|
+
const cases = normalizedDocuments.map((document, index) => {
|
|
9001
|
+
const negativeCandidates = normalizedDocuments.slice(index + 1).concat(normalizedDocuments.slice(0, index)).filter((candidate) => candidate.id !== document.id);
|
|
9002
|
+
const hardNegativeDocumentIds = negativeCandidates.slice(0, Math.max(0, Math.trunc(hardNegativePerCase))).map((candidate) => candidate.id);
|
|
9003
|
+
const hardNegativeSources = negativeCandidates.slice(0, Math.max(0, Math.trunc(hardNegativePerCase))).map((candidate) => candidate.source);
|
|
9004
|
+
return {
|
|
9005
|
+
corpusKey: document.corpusKey,
|
|
9006
|
+
expectedDocumentIds: [document.id],
|
|
9007
|
+
expectedSources: [document.source],
|
|
9008
|
+
...includeGoldenSet ? { goldenSet: true } : {},
|
|
9009
|
+
...hardNegativeDocumentIds.length > 0 ? { hardNegativeDocumentIds } : {},
|
|
9010
|
+
...hardNegativeSources.length > 0 ? { hardNegativeSources } : {},
|
|
9011
|
+
id: `synthetic-${document.id}`,
|
|
9012
|
+
label: document.title,
|
|
9013
|
+
query: buildSyntheticEvaluationQuery(document)
|
|
9014
|
+
};
|
|
9015
|
+
});
|
|
9016
|
+
return createRAGEvaluationSuite({
|
|
9017
|
+
description,
|
|
9018
|
+
id: suiteId,
|
|
9019
|
+
input: {
|
|
9020
|
+
cases,
|
|
9021
|
+
...filter ? { filter: { ...filter } } : {},
|
|
9022
|
+
...typeof scoreThreshold === "number" ? { scoreThreshold } : {},
|
|
9023
|
+
...typeof topK === "number" ? { topK } : {},
|
|
9024
|
+
...retrieval ? { retrieval } : {}
|
|
9025
|
+
},
|
|
9026
|
+
label,
|
|
9027
|
+
metadata
|
|
9028
|
+
});
|
|
9029
|
+
};
|
|
9030
|
+
var createRAGEvaluationSuiteSnapshot = ({
|
|
9031
|
+
suite,
|
|
9032
|
+
id,
|
|
9033
|
+
version = 1,
|
|
9034
|
+
createdAt = Date.now(),
|
|
9035
|
+
metadata
|
|
9036
|
+
}) => {
|
|
9037
|
+
const normalizedSuite = createRAGEvaluationSuite(suite);
|
|
9038
|
+
return {
|
|
9039
|
+
caseCount: normalizedSuite.input.cases.length,
|
|
9040
|
+
createdAt,
|
|
9041
|
+
id: id ?? `${normalizedSuite.id}:snapshot:${String(version)}`,
|
|
9042
|
+
label: normalizedSuite.label,
|
|
9043
|
+
description: normalizedSuite.description,
|
|
9044
|
+
metadata: metadata ? { ...metadata } : undefined,
|
|
9045
|
+
suite: normalizedSuite,
|
|
9046
|
+
suiteId: normalizedSuite.id,
|
|
9047
|
+
version
|
|
9048
|
+
};
|
|
9049
|
+
};
|
|
9050
|
+
var stableStringifyEvaluationCase = (entry) => JSON.stringify({
|
|
9051
|
+
...entry,
|
|
9052
|
+
expectedChunkIds: entry.expectedChunkIds ?? [],
|
|
9053
|
+
expectedDocumentIds: entry.expectedDocumentIds ?? [],
|
|
9054
|
+
expectedSources: entry.expectedSources ?? [],
|
|
9055
|
+
goldenSet: entry.goldenSet === true,
|
|
9056
|
+
hardNegativeChunkIds: entry.hardNegativeChunkIds ?? [],
|
|
9057
|
+
hardNegativeDocumentIds: entry.hardNegativeDocumentIds ?? [],
|
|
9058
|
+
hardNegativeSources: entry.hardNegativeSources ?? [],
|
|
9059
|
+
filter: entry.filter ?? {},
|
|
9060
|
+
metadata: entry.metadata ?? {},
|
|
9061
|
+
retrieval: entry.retrieval ?? null
|
|
9062
|
+
});
|
|
9063
|
+
var buildRAGEvaluationSuiteSnapshotDiff = ({
|
|
9064
|
+
current,
|
|
9065
|
+
previous
|
|
9066
|
+
}) => {
|
|
9067
|
+
const currentCases = current.suite.input.cases;
|
|
9068
|
+
const previousCases = previous?.suite.input.cases ?? [];
|
|
9069
|
+
const currentMap = new Map(currentCases.map((entry) => [entry.id, entry]));
|
|
9070
|
+
const previousMap = new Map(previousCases.map((entry) => [entry.id, entry]));
|
|
9071
|
+
const currentIds = currentCases.map((entry) => entry.id);
|
|
9072
|
+
const previousIds = previousCases.map((entry) => entry.id);
|
|
9073
|
+
const addedCaseIds = currentIds.filter((id) => !previousMap.has(id));
|
|
9074
|
+
const removedCaseIds = previousIds.filter((id) => !currentMap.has(id));
|
|
9075
|
+
const changedCaseIds = currentIds.filter((id) => {
|
|
9076
|
+
const currentCase = currentMap.get(id);
|
|
9077
|
+
const previousCase = previousMap.get(id);
|
|
9078
|
+
if (!currentCase || !previousCase) {
|
|
9079
|
+
return false;
|
|
9080
|
+
}
|
|
9081
|
+
return stableStringifyEvaluationCase(currentCase) !== stableStringifyEvaluationCase(previousCase);
|
|
9082
|
+
});
|
|
9083
|
+
const unchangedCaseIds = currentIds.filter((id) => {
|
|
9084
|
+
const currentCase = currentMap.get(id);
|
|
9085
|
+
const previousCase = previousMap.get(id);
|
|
9086
|
+
if (!currentCase || !previousCase) {
|
|
9087
|
+
return false;
|
|
9088
|
+
}
|
|
9089
|
+
return stableStringifyEvaluationCase(currentCase) === stableStringifyEvaluationCase(previousCase);
|
|
9090
|
+
});
|
|
9091
|
+
const sharedIds = currentIds.filter((id) => previousMap.has(id));
|
|
9092
|
+
const orderChanged = sharedIds.length > 0 && JSON.stringify(sharedIds) !== JSON.stringify(previousIds.filter((id) => currentMap.has(id)));
|
|
9093
|
+
return {
|
|
9094
|
+
addedCaseIds,
|
|
9095
|
+
caseCountDelta: current.caseCount - (previous?.caseCount ?? 0),
|
|
9096
|
+
changedCaseIds,
|
|
9097
|
+
currentSnapshotId: current.id,
|
|
9098
|
+
orderChanged,
|
|
9099
|
+
previousSnapshotId: previous?.id,
|
|
9100
|
+
removedCaseIds,
|
|
9101
|
+
suiteId: current.suiteId,
|
|
9102
|
+
unchangedCaseIds
|
|
9103
|
+
};
|
|
9104
|
+
};
|
|
4986
9105
|
var evaluateRAGCollection = async ({
|
|
4987
9106
|
collection,
|
|
4988
9107
|
input,
|
|
@@ -5007,6 +9126,13 @@ var executeDryRunRAGEvaluation = (input, defaultTopK = DEFAULT_TOP_K) => input.c
|
|
|
5007
9126
|
elapsedMs: 0,
|
|
5008
9127
|
expectedCount: expectedIds.length,
|
|
5009
9128
|
expectedIds,
|
|
9129
|
+
failureClasses: classifyRAGEvaluationFailure({
|
|
9130
|
+
expectedCount: expectedIds.length,
|
|
9131
|
+
matchedCount: 0,
|
|
9132
|
+
missingIds: expectedIds,
|
|
9133
|
+
retrievedCount: 0,
|
|
9134
|
+
retrievedIds: []
|
|
9135
|
+
}),
|
|
5010
9136
|
f1: 0,
|
|
5011
9137
|
label: caseInput.label,
|
|
5012
9138
|
matchedCount: 0,
|
|
@@ -5055,7 +9181,9 @@ var summarizeRAGEvaluationCase = ({
|
|
|
5055
9181
|
mode,
|
|
5056
9182
|
retrievedIds,
|
|
5057
9183
|
expectedIds,
|
|
5058
|
-
elapsedMs
|
|
9184
|
+
elapsedMs,
|
|
9185
|
+
retrievedSources,
|
|
9186
|
+
trace
|
|
5059
9187
|
}) => {
|
|
5060
9188
|
const expectedSet = new Set(expectedIds);
|
|
5061
9189
|
const retrievedSet = new Set(retrievedIds);
|
|
@@ -5074,6 +9202,15 @@ var summarizeRAGEvaluationCase = ({
|
|
|
5074
9202
|
elapsedMs,
|
|
5075
9203
|
expectedCount,
|
|
5076
9204
|
expectedIds,
|
|
9205
|
+
failureClasses: classifyRAGEvaluationFailure({
|
|
9206
|
+
expectedCount,
|
|
9207
|
+
matchedCount,
|
|
9208
|
+
missingIds,
|
|
9209
|
+
retrievedCount,
|
|
9210
|
+
retrievedIds,
|
|
9211
|
+
retrievedSources,
|
|
9212
|
+
trace
|
|
9213
|
+
}),
|
|
5077
9214
|
f1,
|
|
5078
9215
|
label: caseInput.label,
|
|
5079
9216
|
matchedCount,
|
|
@@ -5095,12 +9232,18 @@ var summarizeRAGRerankerComparison = (entries) => {
|
|
|
5095
9232
|
};
|
|
5096
9233
|
var summarizeRAGRetrievalComparison = (entries) => summarizeEvaluationResponseComparison(entries, "retrievalId");
|
|
5097
9234
|
export {
|
|
9235
|
+
updateRAGEvaluationSuiteCase,
|
|
5098
9236
|
summarizeRAGSearchTraceStore,
|
|
5099
9237
|
summarizeRAGRetrievalTraces,
|
|
5100
9238
|
summarizeRAGRetrievalComparison,
|
|
5101
9239
|
summarizeRAGRerankerComparison,
|
|
9240
|
+
summarizeRAGEvaluationSuiteDataset,
|
|
5102
9241
|
summarizeRAGEvaluationCase,
|
|
9242
|
+
setRAGEvaluationSuiteCaseGoldenSet,
|
|
5103
9243
|
runRAGEvaluationSuite,
|
|
9244
|
+
reorderRAGEvaluationSuiteCases,
|
|
9245
|
+
removeRAGEvaluationSuiteCaseHardNegative,
|
|
9246
|
+
removeRAGEvaluationSuiteCase,
|
|
5104
9247
|
pruneRAGSearchTraceStore,
|
|
5105
9248
|
previewRAGSearchTraceStorePrune,
|
|
5106
9249
|
persistRAGSearchTraceRecord,
|
|
@@ -5137,9 +9280,12 @@ export {
|
|
|
5137
9280
|
loadRAGRetrievalComparisonHistory,
|
|
5138
9281
|
loadRAGRetrievalBaselines,
|
|
5139
9282
|
loadRAGRetrievalBaselineGatePolicyHistory,
|
|
9283
|
+
loadRAGEvaluationSuiteSnapshotHistory,
|
|
5140
9284
|
loadRAGEvaluationHistory,
|
|
5141
9285
|
loadRAGAnswerGroundingEvaluationHistory,
|
|
5142
9286
|
loadRAGAnswerGroundingCaseDifficultyHistory,
|
|
9287
|
+
inspectRAGSQLiteStoreMigrations,
|
|
9288
|
+
generateRAGEvaluationSuiteFromDocuments,
|
|
5143
9289
|
executeDryRunRAGEvaluation,
|
|
5144
9290
|
evaluateRAGCollection,
|
|
5145
9291
|
evaluateRAGAnswerGroundingCase,
|
|
@@ -5160,6 +9306,9 @@ export {
|
|
|
5160
9306
|
createRAGSQLiteRetrievalBaselineStore,
|
|
5161
9307
|
createRAGSQLiteRetrievalBaselineGatePolicyHistoryStore,
|
|
5162
9308
|
createRAGSQLiteGovernanceStores,
|
|
9309
|
+
createRAGSQLiteEvaluationSuiteSnapshotHistoryStore,
|
|
9310
|
+
createRAGSQLiteEvaluationHistoryStore,
|
|
9311
|
+
createRAGSQLiteAnswerGroundingEvaluationHistoryStore,
|
|
5163
9312
|
createRAGFileSearchTraceStore,
|
|
5164
9313
|
createRAGFileSearchTracePruneHistoryStore,
|
|
5165
9314
|
createRAGFileRetrievalReleaseLanePolicyHistoryStore,
|
|
@@ -5175,9 +9324,11 @@ export {
|
|
|
5175
9324
|
createRAGFileRetrievalComparisonHistoryStore,
|
|
5176
9325
|
createRAGFileRetrievalBaselineStore,
|
|
5177
9326
|
createRAGFileRetrievalBaselineGatePolicyHistoryStore,
|
|
9327
|
+
createRAGFileEvaluationSuiteSnapshotHistoryStore,
|
|
5178
9328
|
createRAGFileEvaluationHistoryStore,
|
|
5179
9329
|
createRAGFileAnswerGroundingEvaluationHistoryStore,
|
|
5180
9330
|
createRAGFileAnswerGroundingCaseDifficultyHistoryStore,
|
|
9331
|
+
createRAGEvaluationSuiteSnapshot,
|
|
5181
9332
|
createRAGEvaluationSuite,
|
|
5182
9333
|
compareRAGRetrievalTraceSummaries,
|
|
5183
9334
|
compareRAGRetrievalStrategies,
|
|
@@ -5187,15 +9338,21 @@ export {
|
|
|
5187
9338
|
buildRAGRetrievalTraceHistoryTrend,
|
|
5188
9339
|
buildRAGRetrievalReleaseVerdict,
|
|
5189
9340
|
buildRAGRetrievalComparisonDecisionSummary,
|
|
9341
|
+
buildRAGEvaluationSuiteSnapshotDiff,
|
|
5190
9342
|
buildRAGEvaluationRunDiff,
|
|
5191
9343
|
buildRAGEvaluationResponse,
|
|
5192
9344
|
buildRAGEvaluationLeaderboard,
|
|
9345
|
+
buildRAGEvaluationEntityQualityView,
|
|
5193
9346
|
buildRAGAnswerGroundingEvaluationRunDiff,
|
|
5194
9347
|
buildRAGAnswerGroundingEvaluationResponse,
|
|
5195
9348
|
buildRAGAnswerGroundingEvaluationLeaderboard,
|
|
9349
|
+
buildRAGAnswerGroundingEntityQualityView,
|
|
5196
9350
|
buildRAGAnswerGroundingCaseDifficultyRunDiff,
|
|
5197
|
-
buildRAGAnswerGroundingCaseDifficultyLeaderboard
|
|
9351
|
+
buildRAGAnswerGroundingCaseDifficultyLeaderboard,
|
|
9352
|
+
applyRAGSQLiteStoreMigrations,
|
|
9353
|
+
addRAGEvaluationSuiteCaseHardNegative,
|
|
9354
|
+
addRAGEvaluationSuiteCase
|
|
5198
9355
|
};
|
|
5199
9356
|
|
|
5200
|
-
//# debugId=
|
|
9357
|
+
//# debugId=5E80012397BE4C1064756E2164756E21
|
|
5201
9358
|
//# sourceMappingURL=quality.js.map
|