@absolutejs/absolute 0.19.0-beta.643 → 0.19.0-beta.645
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/client/index.js +976 -15
- package/dist/ai/client/index.js.map +6 -6
- package/dist/ai/client/ui.js +807 -15
- package/dist/ai/client/ui.js.map +5 -5
- package/dist/ai/index.js +2737 -141
- package/dist/ai/index.js.map +13 -13
- package/dist/ai/rag/quality.js +813 -15
- package/dist/ai/rag/quality.js.map +5 -5
- package/dist/ai/rag/ui.js +807 -15
- package/dist/ai/rag/ui.js.map +5 -5
- package/dist/ai-client/angular/ai/index.js +436 -8
- package/dist/ai-client/react/ai/index.js +436 -8
- package/dist/ai-client/vue/ai/index.js +436 -8
- package/dist/angular/ai/index.js +976 -15
- package/dist/angular/ai/index.js.map +6 -6
- package/dist/index.js +6 -6
- package/dist/index.js.map +2 -2
- package/dist/react/ai/index.js +976 -15
- package/dist/react/ai/index.js.map +6 -6
- package/dist/src/ai/client/ragClient.d.ts +74 -1
- package/dist/src/ai/index.d.ts +2 -1
- package/dist/src/ai/rag/adapters/queryPlanning.d.ts +8 -0
- package/dist/src/ai/rag/chat.d.ts +135 -7
- package/dist/src/ai/rag/index.d.ts +1 -1
- package/dist/src/ai/rag/presentation.d.ts +5 -1
- package/dist/src/ai/rag/quality.d.ts +34 -1
- package/dist/src/vue/ai/useRAG.d.ts +84 -0
- package/dist/src/vue/ai/useRAGEvaluate.d.ts +74 -0
- package/dist/src/vue/ai/useRAGSearch.d.ts +10 -0
- package/dist/svelte/ai/index.js +976 -15
- package/dist/svelte/ai/index.js.map +6 -6
- package/dist/types/ai.d.ts +115 -13
- package/dist/types/index.d.ts +1 -0
- package/dist/types/session.d.ts +16 -0
- package/dist/vue/ai/index.js +976 -15
- package/dist/vue/ai/index.js.map +6 -6
- package/package.json +8 -7
package/dist/ai/rag/ui.js
CHANGED
|
@@ -200,6 +200,7 @@ var buildContextLabel = (metadata) => {
|
|
|
200
200
|
return;
|
|
201
201
|
}
|
|
202
202
|
const emailKind = getContextString(metadata.emailKind);
|
|
203
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
203
204
|
if (emailKind === "attachment") {
|
|
204
205
|
return "Attachment evidence";
|
|
205
206
|
}
|
|
@@ -237,6 +238,16 @@ var buildContextLabel = (metadata) => {
|
|
|
237
238
|
}
|
|
238
239
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
239
240
|
const sectionTitle = getContextString(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
241
|
+
const officeSectionLabel = sectionPath.length > 0 ? sectionPath.join(" > ") : sectionTitle;
|
|
242
|
+
if (officeBlockKind === "table" && officeSectionLabel) {
|
|
243
|
+
return `Office table block ${officeSectionLabel}`;
|
|
244
|
+
}
|
|
245
|
+
if (officeBlockKind === "list" && officeSectionLabel) {
|
|
246
|
+
return `Office list block ${officeSectionLabel}`;
|
|
247
|
+
}
|
|
248
|
+
if (officeBlockKind === "paragraph" && officeSectionLabel) {
|
|
249
|
+
return `Office paragraph block ${officeSectionLabel}`;
|
|
250
|
+
}
|
|
240
251
|
if (sectionTitle) {
|
|
241
252
|
return `Section ${sectionTitle}`;
|
|
242
253
|
}
|
|
@@ -258,6 +269,46 @@ var formatMediaDurationLabel = (value) => {
|
|
|
258
269
|
}
|
|
259
270
|
return formatMediaTimestamp(value);
|
|
260
271
|
};
|
|
272
|
+
var formatOfficeListLevelsLabel = (value) => {
|
|
273
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const levels = value.map((entry) => getContextNumber(entry)).filter((entry) => typeof entry === "number").sort((left, right) => left - right);
|
|
277
|
+
if (levels.length === 0) {
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const minLevel = levels[0];
|
|
281
|
+
const maxLevel = levels[levels.length - 1];
|
|
282
|
+
return minLevel === maxLevel ? `Office list level ${minLevel}` : `Office list levels ${minLevel}-${maxLevel}`;
|
|
283
|
+
};
|
|
284
|
+
var getOfficeTableCitationScope = (metadata) => {
|
|
285
|
+
if (!metadata) {
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
289
|
+
if (officeBlockKind !== "table" && officeBlockKind !== "list") {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
293
|
+
const sectionTitle = getContextString(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
294
|
+
const officeContextText = officeBlockKind === "table" ? getContextString(metadata.officeTableContextText) : getContextString(metadata.officeListContextText);
|
|
295
|
+
if (!sectionTitle) {
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
return {
|
|
299
|
+
blockKind: officeBlockKind,
|
|
300
|
+
pathDepth: sectionPath.length,
|
|
301
|
+
sectionTitle,
|
|
302
|
+
hasContext: typeof officeContextText === "string"
|
|
303
|
+
};
|
|
304
|
+
};
|
|
305
|
+
var getOfficeTableCitationPreference = (metadata) => {
|
|
306
|
+
const scope = getOfficeTableCitationScope(metadata);
|
|
307
|
+
if (!scope) {
|
|
308
|
+
return 0;
|
|
309
|
+
}
|
|
310
|
+
return scope.pathDepth * 10 + (scope.hasContext ? 1 : 0) + (scope.blockKind === "list" && typeof metadata?.officeListGroupItemCount === "number" && metadata.officeListGroupItemCount > 1 ? 1 : 0);
|
|
311
|
+
};
|
|
261
312
|
var buildLocatorLabel = (metadata, source, title) => {
|
|
262
313
|
if (!metadata) {
|
|
263
314
|
return;
|
|
@@ -283,6 +334,10 @@ var buildLocatorLabel = (metadata, source, title) => {
|
|
|
283
334
|
return `Archive entry ${archiveEntry}`;
|
|
284
335
|
}
|
|
285
336
|
const emailKind = getContextString(metadata.emailKind);
|
|
337
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
338
|
+
const officeBlockNumber = getContextNumber(metadata.officeBlockNumber);
|
|
339
|
+
const officeTableBodyRowStart = getContextNumber(metadata.officeTableBodyRowStart);
|
|
340
|
+
const officeTableBodyRowEnd = getContextNumber(metadata.officeTableBodyRowEnd);
|
|
286
341
|
if (emailKind === "attachment") {
|
|
287
342
|
const attachmentName = getContextString(metadata.attachmentName) ?? getAttachmentName(source, title);
|
|
288
343
|
return attachmentName ? `Attachment ${attachmentName}` : "Attachment";
|
|
@@ -295,6 +350,18 @@ var buildLocatorLabel = (metadata, source, title) => {
|
|
|
295
350
|
if (mediaStart) {
|
|
296
351
|
return `Timestamp ${mediaStart}`;
|
|
297
352
|
}
|
|
353
|
+
if (officeBlockNumber && officeBlockKind === "table") {
|
|
354
|
+
if (typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number") {
|
|
355
|
+
return officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table block ${officeBlockNumber} \xB7 Row ${officeTableBodyRowStart}` : `Office table block ${officeBlockNumber} \xB7 Rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}`;
|
|
356
|
+
}
|
|
357
|
+
return `Office table block ${officeBlockNumber}`;
|
|
358
|
+
}
|
|
359
|
+
if (officeBlockNumber && officeBlockKind === "list") {
|
|
360
|
+
return `Office list block ${officeBlockNumber}`;
|
|
361
|
+
}
|
|
362
|
+
if (officeBlockNumber && officeBlockKind === "paragraph") {
|
|
363
|
+
return `Office paragraph block ${officeBlockNumber}`;
|
|
364
|
+
}
|
|
298
365
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
299
366
|
if (sectionPath.length > 0) {
|
|
300
367
|
return `Section ${sectionPath.join(" > ")}`;
|
|
@@ -328,10 +395,31 @@ var buildProvenanceLabel = (metadata) => {
|
|
|
328
395
|
const mediaDurationLabel = formatMediaDurationLabel(metadata.mediaDurationMs);
|
|
329
396
|
const transcriptSource = getContextString(metadata.transcriptSource);
|
|
330
397
|
const pdfTextMode = getContextString(metadata.pdfTextMode);
|
|
398
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
399
|
+
const officeListContextText = getContextString(metadata.officeListContextText);
|
|
400
|
+
const officeListGroupItemCount = getContextNumber(metadata.officeListGroupItemCount);
|
|
401
|
+
const officeListLevelsLabel = formatOfficeListLevelsLabel(metadata.officeListLevels);
|
|
402
|
+
const officeTableHeaders = Array.isArray(metadata.officeTableHeaders) ? metadata.officeTableHeaders.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
403
|
+
const officeTableColumnCount = getContextNumber(metadata.officeTableColumnCount);
|
|
404
|
+
const officeTableBodyRowCount = getContextNumber(metadata.officeTableBodyRowCount);
|
|
405
|
+
const officeTableBodyRowStart = getContextNumber(metadata.officeTableBodyRowStart);
|
|
406
|
+
const officeTableBodyRowEnd = getContextNumber(metadata.officeTableBodyRowEnd);
|
|
407
|
+
const officeTableContextText = getContextString(metadata.officeTableContextText);
|
|
408
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
331
409
|
const ocrEngine = getContextString(metadata.ocrEngine);
|
|
332
410
|
const ocrConfidence = getContextNumber(metadata.ocrRegionConfidence) ?? getContextNumber(metadata.ocrConfidence);
|
|
333
411
|
const labels = [
|
|
334
412
|
pdfTextMode ? `PDF ${pdfTextMode}` : "",
|
|
413
|
+
officeBlockKind ? `Office ${officeBlockKind}` : "",
|
|
414
|
+
typeof officeListGroupItemCount === "number" ? `Office list ${officeListGroupItemCount} items` : "",
|
|
415
|
+
officeListLevelsLabel ?? "",
|
|
416
|
+
sectionPath.length > 0 && officeBlockKind ? `Source-aware office ${officeBlockKind} block ${sectionPath.join(" > ")}` : "",
|
|
417
|
+
officeListContextText ? `Office list context ${officeListContextText}` : "",
|
|
418
|
+
officeTableHeaders.length > 0 ? `Office table ${officeTableHeaders.join(", ")}` : "",
|
|
419
|
+
typeof officeTableColumnCount === "number" ? `Office table ${officeTableColumnCount} cols` : "",
|
|
420
|
+
typeof officeTableBodyRowCount === "number" ? `Office table ${officeTableBodyRowCount} body rows` : "",
|
|
421
|
+
typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number" ? officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table row ${officeTableBodyRowStart}` : `Office table rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}` : "",
|
|
422
|
+
officeTableContextText ? `Office table context ${officeTableContextText}` : "",
|
|
335
423
|
ocrEngine ? `OCR ${ocrEngine}` : "",
|
|
336
424
|
typeof ocrConfidence === "number" ? `Confidence ${ocrConfidence.toFixed(2)}` : "",
|
|
337
425
|
mediaKind ? `Media ${mediaKind}` : "",
|
|
@@ -495,6 +583,15 @@ var buildRAGCitations = (sources) => {
|
|
|
495
583
|
});
|
|
496
584
|
}
|
|
497
585
|
return [...unique.values()].sort((left, right) => {
|
|
586
|
+
const leftOfficeScope = getOfficeTableCitationScope(left.metadata);
|
|
587
|
+
const rightOfficeScope = getOfficeTableCitationScope(right.metadata);
|
|
588
|
+
if (left.source === right.source && leftOfficeScope && rightOfficeScope && leftOfficeScope.blockKind === rightOfficeScope.blockKind && leftOfficeScope.sectionTitle === rightOfficeScope.sectionTitle) {
|
|
589
|
+
const leftOfficePreference = getOfficeTableCitationPreference(left.metadata);
|
|
590
|
+
const rightOfficePreference = getOfficeTableCitationPreference(right.metadata);
|
|
591
|
+
if (rightOfficePreference !== leftOfficePreference) {
|
|
592
|
+
return rightOfficePreference - leftOfficePreference;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
498
595
|
if (right.score !== left.score) {
|
|
499
596
|
return right.score - left.score;
|
|
500
597
|
}
|
|
@@ -908,6 +1005,7 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
908
1005
|
const sectionKind = getContextString2(metadata.sectionKind);
|
|
909
1006
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
910
1007
|
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
1008
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
911
1009
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
912
1010
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
913
1011
|
const sheetName = getContextString2(metadata.sheetName);
|
|
@@ -918,6 +1016,12 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
918
1016
|
return `Source-aware section ${sectionPath.join(" > ")}`;
|
|
919
1017
|
}
|
|
920
1018
|
if (sectionKind === "pdf_block") {
|
|
1019
|
+
if (pdfSemanticRole === "figure_caption" && sectionTitle) {
|
|
1020
|
+
return `Source-aware PDF figure caption ${sectionTitle}`;
|
|
1021
|
+
}
|
|
1022
|
+
if (pdfSemanticRole === "figure_body" && sectionTitle) {
|
|
1023
|
+
return `Source-aware PDF figure body ${sectionTitle}`;
|
|
1024
|
+
}
|
|
921
1025
|
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
922
1026
|
return `Source-aware PDF table block ${sectionTitle}`;
|
|
923
1027
|
}
|
|
@@ -927,11 +1031,12 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
927
1031
|
return "Source-aware PDF block";
|
|
928
1032
|
}
|
|
929
1033
|
if (sectionKind === "office_block") {
|
|
930
|
-
|
|
931
|
-
|
|
1034
|
+
const officeSectionLabel = sectionPath.length > 0 ? sectionPath.join(" > ") : sectionTitle;
|
|
1035
|
+
if (officeBlockKind && officeSectionLabel) {
|
|
1036
|
+
return `Source-aware office ${officeBlockKind} block ${officeSectionLabel}`;
|
|
932
1037
|
}
|
|
933
|
-
if (
|
|
934
|
-
return `Source-aware office block ${
|
|
1038
|
+
if (officeSectionLabel) {
|
|
1039
|
+
return `Source-aware office block ${officeSectionLabel}`;
|
|
935
1040
|
}
|
|
936
1041
|
return "Source-aware office block";
|
|
937
1042
|
}
|
|
@@ -1319,6 +1424,18 @@ var formatSpreadsheetTableLabel = (tableIndex, tableCount) => {
|
|
|
1319
1424
|
}
|
|
1320
1425
|
return `Table ${tableIndex}`;
|
|
1321
1426
|
};
|
|
1427
|
+
var formatOfficeListLevelsLabel2 = (value) => {
|
|
1428
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
1429
|
+
return;
|
|
1430
|
+
}
|
|
1431
|
+
const levels = value.map((entry) => getContextNumber2(entry)).filter((entry) => typeof entry === "number").sort((left, right) => left - right);
|
|
1432
|
+
if (levels.length === 0) {
|
|
1433
|
+
return;
|
|
1434
|
+
}
|
|
1435
|
+
const minLevel = levels[0];
|
|
1436
|
+
const maxLevel = levels[levels.length - 1];
|
|
1437
|
+
return minLevel === maxLevel ? `Office list level ${minLevel}` : `Office list levels ${minLevel}-${maxLevel}`;
|
|
1438
|
+
};
|
|
1322
1439
|
var formatMediaDurationLabel2 = (value) => {
|
|
1323
1440
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1324
1441
|
return;
|
|
@@ -1330,9 +1447,18 @@ var buildContextLabel2 = (metadata) => {
|
|
|
1330
1447
|
return;
|
|
1331
1448
|
}
|
|
1332
1449
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1450
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1451
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1452
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1333
1453
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1334
1454
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1335
1455
|
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
1456
|
+
if (pdfSemanticRole === "figure_caption" && sectionTitle) {
|
|
1457
|
+
return `PDF figure caption ${sectionTitle}`;
|
|
1458
|
+
}
|
|
1459
|
+
if (pdfSemanticRole === "figure_body" && sectionTitle) {
|
|
1460
|
+
return `PDF figure body ${sectionTitle}`;
|
|
1461
|
+
}
|
|
1336
1462
|
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
1337
1463
|
return `PDF table block ${sectionTitle}`;
|
|
1338
1464
|
}
|
|
@@ -1340,13 +1466,13 @@ var buildContextLabel2 = (metadata) => {
|
|
|
1340
1466
|
return `PDF text block ${sectionTitle}`;
|
|
1341
1467
|
}
|
|
1342
1468
|
if (officeBlockKind === "table" && sectionTitle) {
|
|
1343
|
-
return `Office table block ${sectionTitle}`;
|
|
1469
|
+
return `Office table block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1344
1470
|
}
|
|
1345
1471
|
if (officeBlockKind === "list" && sectionTitle) {
|
|
1346
|
-
return `Office list block ${sectionTitle}`;
|
|
1472
|
+
return `Office list block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1347
1473
|
}
|
|
1348
1474
|
if (officeBlockKind === "paragraph" && sectionTitle) {
|
|
1349
|
-
return `Office paragraph block ${sectionTitle}`;
|
|
1475
|
+
return `Office paragraph block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1350
1476
|
}
|
|
1351
1477
|
const emailKind = getContextString2(metadata.emailKind);
|
|
1352
1478
|
if (emailKind === "attachment") {
|
|
@@ -1444,9 +1570,14 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1444
1570
|
return;
|
|
1445
1571
|
}
|
|
1446
1572
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1573
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1447
1574
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1448
1575
|
const pdfBlockNumber = getContextNumber2(metadata.pdfBlockNumber);
|
|
1576
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1577
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1449
1578
|
const officeBlockNumber = getContextNumber2(metadata.officeBlockNumber);
|
|
1579
|
+
const officeTableBodyRowStart = getContextNumber2(metadata.officeTableBodyRowStart);
|
|
1580
|
+
const officeTableBodyRowEnd = getContextNumber2(metadata.officeTableBodyRowEnd);
|
|
1450
1581
|
const spreadsheetRowStart = getContextNumber2(metadata.spreadsheetRowStart);
|
|
1451
1582
|
const spreadsheetRowEnd = getContextNumber2(metadata.spreadsheetRowEnd);
|
|
1452
1583
|
const slideTitle = getContextString2(metadata.slideTitle);
|
|
@@ -1457,7 +1588,16 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1457
1588
|
if (page && region) {
|
|
1458
1589
|
return `Page ${page} \xB7 Region ${region}`;
|
|
1459
1590
|
}
|
|
1591
|
+
if (page && pdfBlockNumber && pdfSemanticRole === "figure_caption") {
|
|
1592
|
+
return `Page ${page} \xB7 Figure Caption ${pdfBlockNumber}`;
|
|
1593
|
+
}
|
|
1594
|
+
if (page && pdfBlockNumber && pdfSemanticRole === "figure_body") {
|
|
1595
|
+
return `Page ${page} \xB7 Figure Body ${pdfBlockNumber}`;
|
|
1596
|
+
}
|
|
1460
1597
|
if (page && pdfBlockNumber && pdfTextKind === "table_like") {
|
|
1598
|
+
if (typeof pdfTableBodyRowStart === "number" && typeof pdfTableBodyRowEnd === "number") {
|
|
1599
|
+
return pdfTableBodyRowStart === pdfTableBodyRowEnd ? `Page ${page} \xB7 Table Block ${pdfBlockNumber} \xB7 Row ${pdfTableBodyRowStart}` : `Page ${page} \xB7 Table Block ${pdfBlockNumber} \xB7 Rows ${pdfTableBodyRowStart}-${pdfTableBodyRowEnd}`;
|
|
1600
|
+
}
|
|
1461
1601
|
return `Page ${page} \xB7 Table Block ${pdfBlockNumber}`;
|
|
1462
1602
|
}
|
|
1463
1603
|
if (page && pdfBlockNumber) {
|
|
@@ -1520,6 +1660,9 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1520
1660
|
return `Timestamp ${mediaStart}`;
|
|
1521
1661
|
}
|
|
1522
1662
|
if (officeBlockNumber && officeBlockKind === "table") {
|
|
1663
|
+
if (typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number") {
|
|
1664
|
+
return officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table block ${officeBlockNumber} \xB7 Row ${officeTableBodyRowStart}` : `Office table block ${officeBlockNumber} \xB7 Rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}`;
|
|
1665
|
+
}
|
|
1523
1666
|
return `Office table block ${officeBlockNumber}`;
|
|
1524
1667
|
}
|
|
1525
1668
|
if (officeBlockNumber && officeBlockKind === "list") {
|
|
@@ -1556,11 +1699,27 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1556
1699
|
const mediaSegmentWindowDurationLabel = formatMediaDurationLabel2(metadata.mediaSegmentGroupDurationMs);
|
|
1557
1700
|
const mediaSegmentGapLabel = formatMediaDurationLabel2(metadata.mediaSegmentGapFromPreviousMs);
|
|
1558
1701
|
const spreadsheetHeaders = getSpreadsheetHeaders(metadata);
|
|
1702
|
+
const pdfTableHeaders = Array.isArray(metadata.pdfTableHeaders) ? metadata.pdfTableHeaders.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1703
|
+
const pdfTableColumnCount = getContextNumber2(metadata.pdfTableColumnCount);
|
|
1704
|
+
const pdfTableBodyRowCount = getContextNumber2(metadata.pdfTableBodyRowCount);
|
|
1559
1705
|
const spreadsheetColumnRange = formatSpreadsheetColumnRange(getContextString2(metadata.spreadsheetColumnStart), getContextString2(metadata.spreadsheetColumnEnd));
|
|
1560
1706
|
const slideNotesText = getContextString2(metadata.slideNotesText);
|
|
1561
1707
|
const pdfTextMode = getContextString2(metadata.pdfTextMode);
|
|
1708
|
+
const pdfEvidenceMode = getContextString2(metadata.pdfEvidenceMode);
|
|
1709
|
+
const pdfEvidenceOrigin = getContextString2(metadata.pdfEvidenceOrigin);
|
|
1710
|
+
const pdfEvidenceSupplement = getContextString2(metadata.pdfEvidenceSupplement);
|
|
1562
1711
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1712
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1563
1713
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1714
|
+
const officeListContextText = getContextString2(metadata.officeListContextText);
|
|
1715
|
+
const officeListGroupItemCount = getContextNumber2(metadata.officeListGroupItemCount);
|
|
1716
|
+
const officeListLevelsLabel = formatOfficeListLevelsLabel2(metadata.officeListLevels);
|
|
1717
|
+
const officeTableHeaders = Array.isArray(metadata.officeTableHeaders) ? metadata.officeTableHeaders.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1718
|
+
const officeTableColumnCount = getContextNumber2(metadata.officeTableColumnCount);
|
|
1719
|
+
const officeTableBodyRowCount = getContextNumber2(metadata.officeTableBodyRowCount);
|
|
1720
|
+
const officeTableBodyRowStart = getContextNumber2(metadata.officeTableBodyRowStart);
|
|
1721
|
+
const officeTableBodyRowEnd = getContextNumber2(metadata.officeTableBodyRowEnd);
|
|
1722
|
+
const officeTableContextText = getContextString2(metadata.officeTableContextText);
|
|
1564
1723
|
const ocrEngine = getContextString2(metadata.ocrEngine);
|
|
1565
1724
|
const extractorRegistryMatch = getContextString2(metadata.extractorRegistryMatch);
|
|
1566
1725
|
const chunkingProfile = getContextString2(metadata.chunkingProfile);
|
|
@@ -1576,10 +1735,19 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1576
1735
|
const ocrMinConfidence = getContextNumber2(metadata.ocrPageMinConfidence) ?? getContextNumber2(metadata.ocrMinConfidence);
|
|
1577
1736
|
const ocrMaxConfidence = getContextNumber2(metadata.ocrPageMaxConfidence) ?? getContextNumber2(metadata.ocrMaxConfidence);
|
|
1578
1737
|
const ocrRegionCount = getContextNumber2(metadata.ocrRegionCount);
|
|
1738
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1739
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1579
1740
|
const labels = [
|
|
1580
1741
|
pdfTextMode ? `PDF ${pdfTextMode}` : "",
|
|
1581
|
-
|
|
1742
|
+
pdfEvidenceMode ? `PDF evidence ${pdfEvidenceMode}` : "",
|
|
1743
|
+
pdfEvidenceOrigin ? `PDF origin ${pdfEvidenceOrigin}` : "",
|
|
1744
|
+
pdfEvidenceSupplement ? `PDF supplement ${pdfEvidenceSupplement}` : "",
|
|
1745
|
+
pdfSemanticRole === "figure_caption" ? "PDF figure caption" : "",
|
|
1746
|
+
pdfSemanticRole === "figure_body" ? "PDF figure body" : "",
|
|
1747
|
+
pdfSemanticRole === "figure_caption" ? "" : pdfSemanticRole === "figure_body" ? "" : pdfTextKind === "table_like" ? "PDF table block" : pdfTextKind === "paragraph" ? "PDF text block" : "",
|
|
1582
1748
|
officeBlockKind ? `Office ${officeBlockKind}` : "",
|
|
1749
|
+
typeof officeListGroupItemCount === "number" ? `Office list ${officeListGroupItemCount} items` : "",
|
|
1750
|
+
officeListLevelsLabel ?? "",
|
|
1583
1751
|
ocrEngine ? `OCR ${ocrEngine}` : "",
|
|
1584
1752
|
extractorRegistryMatch ? `Extractor ${extractorRegistryMatch}` : "",
|
|
1585
1753
|
chunkingProfile ? `Chunking ${chunkingProfile}` : "",
|
|
@@ -1589,6 +1757,16 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1589
1757
|
typeof ocrAverageConfidence === "number" && ocrAverageConfidence !== ocrConfidence ? `Average ${ocrAverageConfidence.toFixed(2)}` : "",
|
|
1590
1758
|
typeof ocrMinConfidence === "number" && typeof ocrMaxConfidence === "number" && ocrMinConfidence !== ocrMaxConfidence ? `Range ${ocrMinConfidence.toFixed(2)}-${ocrMaxConfidence.toFixed(2)}` : "",
|
|
1591
1759
|
typeof ocrRegionCount === "number" ? `${ocrRegionCount} regions` : "",
|
|
1760
|
+
pdfTableHeaders.length > 0 ? `PDF table ${pdfTableHeaders.join(", ")}` : "",
|
|
1761
|
+
typeof pdfTableColumnCount === "number" ? `PDF table ${pdfTableColumnCount} cols` : "",
|
|
1762
|
+
typeof pdfTableBodyRowCount === "number" ? `PDF table ${pdfTableBodyRowCount} body rows` : "",
|
|
1763
|
+
typeof pdfTableBodyRowStart === "number" && typeof pdfTableBodyRowEnd === "number" ? pdfTableBodyRowStart === pdfTableBodyRowEnd ? `PDF table row ${pdfTableBodyRowStart}` : `PDF table rows ${pdfTableBodyRowStart}-${pdfTableBodyRowEnd}` : "",
|
|
1764
|
+
officeListContextText ? `Office list context ${officeListContextText}` : "",
|
|
1765
|
+
officeTableHeaders.length > 0 ? `Office table ${officeTableHeaders.join(", ")}` : "",
|
|
1766
|
+
typeof officeTableColumnCount === "number" ? `Office table ${officeTableColumnCount} cols` : "",
|
|
1767
|
+
typeof officeTableBodyRowCount === "number" ? `Office table ${officeTableBodyRowCount} body rows` : "",
|
|
1768
|
+
typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number" ? officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table row ${officeTableBodyRowStart}` : `Office table rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}` : "",
|
|
1769
|
+
officeTableContextText ? `Office table context ${officeTableContextText}` : "",
|
|
1592
1770
|
spreadsheetHeaders.length > 0 ? `Spreadsheet ${spreadsheetHeaders.join(", ")}` : "",
|
|
1593
1771
|
spreadsheetColumnRange ? `Spreadsheet ${spreadsheetColumnRange}` : "",
|
|
1594
1772
|
spreadsheetTableLabel ? `Spreadsheet ${spreadsheetTableLabel}` : "",
|
|
@@ -2020,12 +2198,92 @@ var getStructuredSectionScoreWeight = (metadata) => {
|
|
|
2020
2198
|
return 1;
|
|
2021
2199
|
};
|
|
2022
2200
|
var getStructuredSourceLeadScore = (source) => source.score * getStructuredSectionScoreWeight(source.metadata);
|
|
2201
|
+
var getPDFLeadEvidencePreference = (metadata) => {
|
|
2202
|
+
if (!metadata) {
|
|
2203
|
+
return 0;
|
|
2204
|
+
}
|
|
2205
|
+
const pdfEvidenceMode = getContextString2(metadata.pdfEvidenceMode);
|
|
2206
|
+
const pdfEvidenceOrigin = getContextString2(metadata.pdfEvidenceOrigin);
|
|
2207
|
+
const pdfEvidenceSupplement = getContextString2(metadata.pdfEvidenceSupplement);
|
|
2208
|
+
if (pdfEvidenceMode === "hybrid" && pdfEvidenceOrigin === "native" && pdfEvidenceSupplement === "ocr") {
|
|
2209
|
+
return 3;
|
|
2210
|
+
}
|
|
2211
|
+
if (pdfEvidenceMode === "native" && pdfEvidenceOrigin === "native") {
|
|
2212
|
+
return 2;
|
|
2213
|
+
}
|
|
2214
|
+
if (pdfEvidenceMode === "ocr" && pdfEvidenceOrigin === "ocr") {
|
|
2215
|
+
return 1;
|
|
2216
|
+
}
|
|
2217
|
+
return 0;
|
|
2218
|
+
};
|
|
2219
|
+
var getPDFLeadScope = (metadata) => {
|
|
2220
|
+
if (!metadata) {
|
|
2221
|
+
return;
|
|
2222
|
+
}
|
|
2223
|
+
const pageNumber = getContextNumber2(metadata.pageNumber) ?? getContextNumber2(metadata.page) ?? (typeof metadata.pageIndex === "number" ? metadata.pageIndex + 1 : undefined);
|
|
2224
|
+
const sectionTitle = getContextString2(metadata.sectionTitle);
|
|
2225
|
+
const sourceNativeKind = getContextString2(metadata.sourceNativeKind);
|
|
2226
|
+
if (typeof pageNumber !== "number" && !sectionTitle && !sourceNativeKind) {
|
|
2227
|
+
return;
|
|
2228
|
+
}
|
|
2229
|
+
return {
|
|
2230
|
+
pageNumber,
|
|
2231
|
+
sectionTitle,
|
|
2232
|
+
sourceNativeKind
|
|
2233
|
+
};
|
|
2234
|
+
};
|
|
2235
|
+
var getOfficeLeadScope = (metadata) => {
|
|
2236
|
+
if (!metadata) {
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
2240
|
+
if (officeBlockKind !== "table" && officeBlockKind !== "list") {
|
|
2241
|
+
return;
|
|
2242
|
+
}
|
|
2243
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
2244
|
+
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
2245
|
+
const officeContextText = officeBlockKind === "table" ? getContextString2(metadata.officeTableContextText) : getContextString2(metadata.officeListContextText);
|
|
2246
|
+
if (!sectionTitle) {
|
|
2247
|
+
return;
|
|
2248
|
+
}
|
|
2249
|
+
return {
|
|
2250
|
+
blockKind: officeBlockKind,
|
|
2251
|
+
pathDepth: sectionPath.length,
|
|
2252
|
+
sectionTitle,
|
|
2253
|
+
hasContext: typeof officeContextText === "string"
|
|
2254
|
+
};
|
|
2255
|
+
};
|
|
2256
|
+
var getOfficeLeadEvidencePreference = (metadata) => {
|
|
2257
|
+
const scope = getOfficeLeadScope(metadata);
|
|
2258
|
+
if (!scope) {
|
|
2259
|
+
return 0;
|
|
2260
|
+
}
|
|
2261
|
+
return scope.pathDepth * 10 + (scope.hasContext ? 1 : 0) + (scope.blockKind === "list" && typeof metadata?.officeListGroupItemCount === "number" && metadata.officeListGroupItemCount > 1 ? 1 : 0);
|
|
2262
|
+
};
|
|
2023
2263
|
var getPreferredSourceLeadChunk = (chunks) => chunks.slice().sort((left, right) => {
|
|
2264
|
+
const leftOfficeScope = getOfficeLeadScope(left.metadata);
|
|
2265
|
+
const rightOfficeScope = getOfficeLeadScope(right.metadata);
|
|
2266
|
+
if (left.source === right.source && leftOfficeScope && rightOfficeScope && leftOfficeScope.blockKind === rightOfficeScope.blockKind && leftOfficeScope.sectionTitle === rightOfficeScope.sectionTitle) {
|
|
2267
|
+
const leftOfficePreference = getOfficeLeadEvidencePreference(left.metadata);
|
|
2268
|
+
const rightOfficePreference = getOfficeLeadEvidencePreference(right.metadata);
|
|
2269
|
+
if (rightOfficePreference !== leftOfficePreference) {
|
|
2270
|
+
return rightOfficePreference - leftOfficePreference;
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2024
2273
|
const leftWeightedScore = getStructuredSourceLeadScore(left);
|
|
2025
2274
|
const rightWeightedScore = getStructuredSourceLeadScore(right);
|
|
2026
2275
|
if (rightWeightedScore !== leftWeightedScore) {
|
|
2027
2276
|
return rightWeightedScore - leftWeightedScore;
|
|
2028
2277
|
}
|
|
2278
|
+
const leftScope = getPDFLeadScope(left.metadata);
|
|
2279
|
+
const rightScope = getPDFLeadScope(right.metadata);
|
|
2280
|
+
if (left.source === right.source && leftScope && rightScope && (leftScope.sectionTitle && rightScope.sectionTitle && leftScope.sectionTitle === rightScope.sectionTitle || typeof leftScope.pageNumber === "number" && typeof rightScope.pageNumber === "number" && leftScope.pageNumber === rightScope.pageNumber)) {
|
|
2281
|
+
const leftEvidencePreference = getPDFLeadEvidencePreference(left.metadata);
|
|
2282
|
+
const rightEvidencePreference = getPDFLeadEvidencePreference(right.metadata);
|
|
2283
|
+
if (rightEvidencePreference !== leftEvidencePreference) {
|
|
2284
|
+
return rightEvidencePreference - leftEvidencePreference;
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2029
2287
|
if (right.score !== left.score) {
|
|
2030
2288
|
return right.score - left.score;
|
|
2031
2289
|
}
|
|
@@ -2279,6 +2537,7 @@ var buildRAGSectionRetrievalDiagnostics = (sources, trace) => {
|
|
|
2279
2537
|
queryTransformProvider: trace?.queryTransformProvider,
|
|
2280
2538
|
queryTransformReason: trace?.queryTransformReason,
|
|
2281
2539
|
reasons,
|
|
2540
|
+
evidenceReconcileApplied: trace?.steps.some((step) => step.stage === "evidence_reconcile"),
|
|
2282
2541
|
rerankApplied: trace?.steps.some((step) => step.stage === "rerank" && step.metadata?.applied === true),
|
|
2283
2542
|
scoreShare,
|
|
2284
2543
|
scoreThresholdApplied: trace?.steps.some((step) => step.stage === "score_filter"),
|
|
@@ -2957,6 +3216,24 @@ var buildComparisonOverviewPresentation = (input) => {
|
|
|
2957
3216
|
value: input.resolveLabel(input.summary.bestByMultivectorVectorHitCases)
|
|
2958
3217
|
});
|
|
2959
3218
|
}
|
|
3219
|
+
if (input.summary.bestByEvidenceReconcileCases) {
|
|
3220
|
+
rows.push({
|
|
3221
|
+
label: "Best evidence reconcile",
|
|
3222
|
+
value: input.resolveLabel(input.summary.bestByEvidenceReconcileCases)
|
|
3223
|
+
});
|
|
3224
|
+
}
|
|
3225
|
+
if (input.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases) {
|
|
3226
|
+
rows.push({
|
|
3227
|
+
label: "Lowest runtime budget exhaustion",
|
|
3228
|
+
value: input.resolveLabel(input.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases)
|
|
3229
|
+
});
|
|
3230
|
+
}
|
|
3231
|
+
if (input.summary.bestByLowestRuntimeUnderfilledTopKCases) {
|
|
3232
|
+
rows.push({
|
|
3233
|
+
label: "Lowest runtime underfilled TopK",
|
|
3234
|
+
value: input.resolveLabel(input.summary.bestByLowestRuntimeUnderfilledTopKCases)
|
|
3235
|
+
});
|
|
3236
|
+
}
|
|
2960
3237
|
return {
|
|
2961
3238
|
rows,
|
|
2962
3239
|
winnerLabel,
|
|
@@ -3009,6 +3286,12 @@ var buildRAGComparisonTraceSummaryRows = (entry) => {
|
|
|
3009
3286
|
}, {
|
|
3010
3287
|
label: "Multivector",
|
|
3011
3288
|
value: `${formatTraceRatio(trace.multiVectorCases, trace.totalCases)} \xB7 collapse ${formatTraceRatio(trace.multiVectorCollapsedCases, trace.totalCases)} \xB7 lexical ${formatTraceRatio(trace.multiVectorLexicalHitCases, trace.totalCases)} \xB7 vector ${formatTraceRatio(trace.multiVectorVectorHitCases, trace.totalCases)}`
|
|
3289
|
+
}, {
|
|
3290
|
+
label: "Runtime",
|
|
3291
|
+
value: `budget ${formatTraceRatio(trace.runtimeCandidateBudgetExhaustedCases, trace.totalCases)} \xB7 underfilled ${formatTraceRatio(trace.runtimeUnderfilledTopKCases, trace.totalCases)}`
|
|
3292
|
+
}, {
|
|
3293
|
+
label: "Evidence reconcile",
|
|
3294
|
+
value: `all ${formatTraceRatio(trace.stageCounts.evidence_reconcile ?? 0, trace.totalCases)} \xB7 office ${formatTraceRatio(trace.officeEvidenceReconcileCases, trace.totalCases)} \xB7 pdf ${formatTraceRatio(trace.pdfEvidenceReconcileCases, trace.totalCases)}`
|
|
3012
3295
|
}, {
|
|
3013
3296
|
label: "TopK",
|
|
3014
3297
|
value: `${trace.averageCandidateTopK.toFixed(1)} / ${trace.averageLexicalTopK.toFixed(1)}`
|
|
@@ -3113,6 +3396,12 @@ var buildRAGComparisonTraceDiffRows = (entry, leader) => {
|
|
|
3113
3396
|
}, {
|
|
3114
3397
|
label: "Round robin delta",
|
|
3115
3398
|
value: formatTraceCountDelta(trace.roundRobinCases - leaderTrace.roundRobinCases)
|
|
3399
|
+
}, {
|
|
3400
|
+
label: "Runtime budget delta",
|
|
3401
|
+
value: formatTraceCountDelta(trace.runtimeCandidateBudgetExhaustedCases - leaderTrace.runtimeCandidateBudgetExhaustedCases)
|
|
3402
|
+
}, {
|
|
3403
|
+
label: "Runtime underfilled delta",
|
|
3404
|
+
value: formatTraceCountDelta(trace.runtimeUnderfilledTopKCases - leaderTrace.runtimeUnderfilledTopKCases)
|
|
3116
3405
|
});
|
|
3117
3406
|
if (stageDelta) {
|
|
3118
3407
|
rows.push({ label: "Stage delta", value: stageDelta });
|
|
@@ -3360,6 +3649,25 @@ var buildRAGEvaluationHistoryRows = (history) => {
|
|
|
3360
3649
|
label: "Trace variant delta",
|
|
3361
3650
|
value: formatTraceCountDelta(history.diff.traceSummaryDelta.variantCases)
|
|
3362
3651
|
});
|
|
3652
|
+
const evidenceReconcileDelta = history.diff.traceSummaryDelta.stageCounts?.evidence_reconcile;
|
|
3653
|
+
if (typeof evidenceReconcileDelta === "number") {
|
|
3654
|
+
rows.push({
|
|
3655
|
+
label: "Trace evidence reconcile delta",
|
|
3656
|
+
value: formatTraceCountDelta(evidenceReconcileDelta)
|
|
3657
|
+
});
|
|
3658
|
+
}
|
|
3659
|
+
if (typeof history.diff.traceSummaryDelta.officeEvidenceReconcileCasesDelta === "number") {
|
|
3660
|
+
rows.push({
|
|
3661
|
+
label: "Trace office evidence reconcile delta",
|
|
3662
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.officeEvidenceReconcileCasesDelta)
|
|
3663
|
+
});
|
|
3664
|
+
}
|
|
3665
|
+
if (typeof history.diff.traceSummaryDelta.pdfEvidenceReconcileCasesDelta === "number") {
|
|
3666
|
+
rows.push({
|
|
3667
|
+
label: "Trace PDF evidence reconcile delta",
|
|
3668
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.pdfEvidenceReconcileCasesDelta)
|
|
3669
|
+
});
|
|
3670
|
+
}
|
|
3363
3671
|
const stageDelta = Object.entries(history.diff.traceSummaryDelta.stageCounts ?? {}).map(([stage, count]) => `${stage} ${formatTraceCountDelta(count)}`).join(", ");
|
|
3364
3672
|
if (stageDelta) {
|
|
3365
3673
|
rows.push({ label: "Trace stage delta", value: stageDelta });
|
|
@@ -3564,6 +3872,92 @@ var buildRAGEvaluationSuiteSnapshotHistoryPresentation = (history) => ({
|
|
|
3564
3872
|
snapshots: buildRAGEvaluationSuiteSnapshotPresentations(history),
|
|
3565
3873
|
summary: history?.latestSnapshot ? `v${history.latestSnapshot.version}` : "No saved suite snapshots yet."
|
|
3566
3874
|
});
|
|
3875
|
+
var isRuntimeGateReason = (reason) => /runtime|candidate-budget|underfilled/i.test(reason);
|
|
3876
|
+
var getFixtureVariantsFromRunTags = (tags) => (tags ?? []).filter((tag) => tag.startsWith("fixture:")).map((tag) => tag.slice("fixture:".length)).filter((tag, index, all) => tag.length > 0 && all.indexOf(tag) === index);
|
|
3877
|
+
var buildRAGRetrievalReleaseHistoryRunPresentation = (run) => {
|
|
3878
|
+
const runtimeGateReasons = (run.decisionSummary?.gate?.reasons ?? run.releaseVerdict?.gate?.reasons ?? []).filter(isRuntimeGateReason);
|
|
3879
|
+
const rows = [
|
|
3880
|
+
{ label: "Finished", value: formatDateLabel(run.finishedAt) },
|
|
3881
|
+
{
|
|
3882
|
+
label: "Passing-rate winner",
|
|
3883
|
+
value: run.comparison.summary.bestByPassingRate ?? "n/a"
|
|
3884
|
+
},
|
|
3885
|
+
{
|
|
3886
|
+
label: "Average F1 winner",
|
|
3887
|
+
value: run.comparison.summary.bestByAverageF1 ?? "n/a"
|
|
3888
|
+
}
|
|
3889
|
+
];
|
|
3890
|
+
const fixtureVariants = getFixtureVariantsFromRunTags(run.tags);
|
|
3891
|
+
if (fixtureVariants.length > 0) {
|
|
3892
|
+
rows.push({
|
|
3893
|
+
label: "Fixture variant",
|
|
3894
|
+
value: fixtureVariants.join(", ")
|
|
3895
|
+
});
|
|
3896
|
+
}
|
|
3897
|
+
if (run.comparison.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases) {
|
|
3898
|
+
rows.push({
|
|
3899
|
+
label: "Lowest runtime budget exhaustion",
|
|
3900
|
+
value: run.comparison.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases
|
|
3901
|
+
});
|
|
3902
|
+
}
|
|
3903
|
+
if (run.comparison.summary.bestByLowestRuntimeUnderfilledTopKCases) {
|
|
3904
|
+
rows.push({
|
|
3905
|
+
label: "Lowest runtime underfilled TopK",
|
|
3906
|
+
value: run.comparison.summary.bestByLowestRuntimeUnderfilledTopKCases
|
|
3907
|
+
});
|
|
3908
|
+
}
|
|
3909
|
+
rows.push({
|
|
3910
|
+
label: "Gate status",
|
|
3911
|
+
value: run.decisionSummary?.gate?.status ?? run.releaseVerdict?.gate?.status ?? "n/a"
|
|
3912
|
+
}, {
|
|
3913
|
+
label: "Runtime gate failures",
|
|
3914
|
+
value: runtimeGateReasons.length > 0 ? runtimeGateReasons.join("; ") : "none"
|
|
3915
|
+
});
|
|
3916
|
+
return {
|
|
3917
|
+
label: run.label,
|
|
3918
|
+
rows,
|
|
3919
|
+
runId: run.id,
|
|
3920
|
+
summary: runtimeGateReasons.length > 0 ? `${run.label} \xB7 runtime gate blocked` : `${run.label} \xB7 ${run.comparison.summary.bestByPassingRate ?? "n/a"} leads passing rate`
|
|
3921
|
+
};
|
|
3922
|
+
};
|
|
3923
|
+
var buildRAGRetrievalReleaseGroupHistoryPresentation = (input) => {
|
|
3924
|
+
const recentRuns = (input.runs ?? []).map(buildRAGRetrievalReleaseHistoryRunPresentation);
|
|
3925
|
+
const fixtureVariants = (input.runs ?? []).flatMap((run) => getFixtureVariantsFromRunTags(run.tags)).filter((tag, index, all) => all.indexOf(tag) === index);
|
|
3926
|
+
const runtimeBlockedRuns = recentRuns.filter((entry) => entry.rows.some((row) => row.label === "Runtime gate failures" && row.value !== "none")).length;
|
|
3927
|
+
const rows = [
|
|
3928
|
+
{
|
|
3929
|
+
label: "Latest decision",
|
|
3930
|
+
value: input.timeline?.latestDecisionKind ?? "none"
|
|
3931
|
+
},
|
|
3932
|
+
{
|
|
3933
|
+
label: "Latest decision at",
|
|
3934
|
+
value: formatDateLabel(input.timeline?.latestDecisionAt)
|
|
3935
|
+
},
|
|
3936
|
+
{
|
|
3937
|
+
label: "Last promoted",
|
|
3938
|
+
value: formatDateLabel(input.timeline?.lastPromotedAt)
|
|
3939
|
+
},
|
|
3940
|
+
{
|
|
3941
|
+
label: "Last reverted",
|
|
3942
|
+
value: formatDateLabel(input.timeline?.lastRevertedAt)
|
|
3943
|
+
},
|
|
3944
|
+
{
|
|
3945
|
+
label: "Recent runtime-blocked runs",
|
|
3946
|
+
value: String(runtimeBlockedRuns)
|
|
3947
|
+
}
|
|
3948
|
+
];
|
|
3949
|
+
if (fixtureVariants.length > 0) {
|
|
3950
|
+
rows.push({
|
|
3951
|
+
label: "Fixture variants",
|
|
3952
|
+
value: fixtureVariants.join(", ")
|
|
3953
|
+
});
|
|
3954
|
+
}
|
|
3955
|
+
return {
|
|
3956
|
+
recentRuns,
|
|
3957
|
+
rows,
|
|
3958
|
+
summary: input.timeline?.latestDecisionKind ? `${input.timeline.latestDecisionKind} \xB7 ${recentRuns.length} recent runs` : recentRuns.length > 0 ? `${recentRuns.length} recent runs` : "No release history yet."
|
|
3959
|
+
};
|
|
3960
|
+
};
|
|
3567
3961
|
var buildRAGAnswerGroundingCaseSnapshotPresentations = (history) => {
|
|
3568
3962
|
if (!history?.caseSnapshots.length) {
|
|
3569
3963
|
return [];
|
|
@@ -4074,6 +4468,15 @@ var evaluateRetrievalComparisonGate = ({
|
|
|
4074
4468
|
if (typeof policy.minMultiVectorVectorHitCasesDelta === "number" && (delta.multiVectorVectorHitCasesDelta ?? 0) < policy.minMultiVectorVectorHitCasesDelta) {
|
|
4075
4469
|
reasons.push(`multivector vector-hit delta ${delta.multiVectorVectorHitCasesDelta ?? 0} is below ${policy.minMultiVectorVectorHitCasesDelta}`);
|
|
4076
4470
|
}
|
|
4471
|
+
if (typeof policy.minEvidenceReconcileCasesDelta === "number" && (delta.evidenceReconcileCasesDelta ?? 0) < policy.minEvidenceReconcileCasesDelta) {
|
|
4472
|
+
reasons.push(`evidence reconcile delta ${delta.evidenceReconcileCasesDelta ?? 0} is below ${policy.minEvidenceReconcileCasesDelta}`);
|
|
4473
|
+
}
|
|
4474
|
+
if (typeof policy.maxRuntimeCandidateBudgetExhaustedCasesDelta === "number" && (delta.runtimeCandidateBudgetExhaustedCasesDelta ?? 0) > policy.maxRuntimeCandidateBudgetExhaustedCasesDelta) {
|
|
4475
|
+
reasons.push(`runtime candidate-budget-exhausted delta ${delta.runtimeCandidateBudgetExhaustedCasesDelta ?? 0} exceeds ${policy.maxRuntimeCandidateBudgetExhaustedCasesDelta}`);
|
|
4476
|
+
}
|
|
4477
|
+
if (typeof policy.maxRuntimeUnderfilledTopKCasesDelta === "number" && (delta.runtimeUnderfilledTopKCasesDelta ?? 0) > policy.maxRuntimeUnderfilledTopKCasesDelta) {
|
|
4478
|
+
reasons.push(`runtime underfilled-topk delta ${delta.runtimeUnderfilledTopKCasesDelta ?? 0} exceeds ${policy.maxRuntimeUnderfilledTopKCasesDelta}`);
|
|
4479
|
+
}
|
|
4077
4480
|
if (reasons.length === 0) {
|
|
4078
4481
|
return {
|
|
4079
4482
|
policy,
|
|
@@ -4121,13 +4524,14 @@ var buildRAGRetrievalReleaseVerdict = ({
|
|
|
4121
4524
|
};
|
|
4122
4525
|
}
|
|
4123
4526
|
if (delta) {
|
|
4527
|
+
const requiresReview = delta.passingRateDelta < 0 || delta.averageF1Delta < 0 || (delta.evidenceReconcileCasesDelta ?? 0) < 0;
|
|
4124
4528
|
return {
|
|
4125
4529
|
baselineGroupKey: groupKey,
|
|
4126
4530
|
baselineRetrievalId,
|
|
4127
4531
|
candidateRetrievalId,
|
|
4128
4532
|
delta,
|
|
4129
|
-
status:
|
|
4130
|
-
summary:
|
|
4533
|
+
status: requiresReview ? "needs_review" : "pass",
|
|
4534
|
+
summary: requiresReview ? "Candidate should be reviewed before promotion." : "Candidate improved or matched the baseline."
|
|
4131
4535
|
};
|
|
4132
4536
|
}
|
|
4133
4537
|
return {
|
|
@@ -4604,6 +5008,20 @@ var buildTraceSummaryAggregate = ({
|
|
|
4604
5008
|
direction: "flat",
|
|
4605
5009
|
metric: "multiVectorCollapsedCases",
|
|
4606
5010
|
previous: 0
|
|
5011
|
+
},
|
|
5012
|
+
{
|
|
5013
|
+
current: 0,
|
|
5014
|
+
delta: 0,
|
|
5015
|
+
direction: "flat",
|
|
5016
|
+
metric: "runtimeCandidateBudgetExhaustedCases",
|
|
5017
|
+
previous: 0
|
|
5018
|
+
},
|
|
5019
|
+
{
|
|
5020
|
+
current: 0,
|
|
5021
|
+
delta: 0,
|
|
5022
|
+
direction: "flat",
|
|
5023
|
+
metric: "runtimeUnderfilledTopKCases",
|
|
5024
|
+
previous: 0
|
|
4607
5025
|
}
|
|
4608
5026
|
];
|
|
4609
5027
|
return {
|
|
@@ -4726,6 +5144,20 @@ var buildTraceSummaryAggregate = ({
|
|
|
4726
5144
|
direction: buildTraceSummaryDirection(latest.multiVectorCollapsedCases - previous.multiVectorCollapsedCases),
|
|
4727
5145
|
metric: "multiVectorCollapsedCases",
|
|
4728
5146
|
previous: previous.multiVectorCollapsedCases
|
|
5147
|
+
},
|
|
5148
|
+
{
|
|
5149
|
+
current: latest.runtimeCandidateBudgetExhaustedCases,
|
|
5150
|
+
delta: latest.runtimeCandidateBudgetExhaustedCases - previous.runtimeCandidateBudgetExhaustedCases,
|
|
5151
|
+
direction: buildTraceSummaryDirection(latest.runtimeCandidateBudgetExhaustedCases - previous.runtimeCandidateBudgetExhaustedCases),
|
|
5152
|
+
metric: "runtimeCandidateBudgetExhaustedCases",
|
|
5153
|
+
previous: previous.runtimeCandidateBudgetExhaustedCases
|
|
5154
|
+
},
|
|
5155
|
+
{
|
|
5156
|
+
current: latest.runtimeUnderfilledTopKCases,
|
|
5157
|
+
delta: latest.runtimeUnderfilledTopKCases - previous.runtimeUnderfilledTopKCases,
|
|
5158
|
+
direction: buildTraceSummaryDirection(latest.runtimeUnderfilledTopKCases - previous.runtimeUnderfilledTopKCases),
|
|
5159
|
+
metric: "runtimeUnderfilledTopKCases",
|
|
5160
|
+
previous: previous.runtimeUnderfilledTopKCases
|
|
4729
5161
|
}
|
|
4730
5162
|
];
|
|
4731
5163
|
const absoluteSorted = [...aggregate].sort((left, right) => Math.abs(right.delta) - Math.abs(left.delta) || left.metric.localeCompare(right.metric));
|
|
@@ -4780,12 +5212,17 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4780
5212
|
let multiVectorVectorHitCases = 0;
|
|
4781
5213
|
let multiVectorLexicalHitCases = 0;
|
|
4782
5214
|
let multiVectorCollapsedCases = 0;
|
|
5215
|
+
let officeEvidenceReconcileCases = 0;
|
|
5216
|
+
let pdfEvidenceReconcileCases = 0;
|
|
5217
|
+
let runtimeCandidateBudgetExhaustedCases = 0;
|
|
5218
|
+
let runtimeUnderfilledTopKCases = 0;
|
|
4783
5219
|
let finalCountSum = 0;
|
|
4784
5220
|
let vectorCountSum = 0;
|
|
4785
5221
|
let lexicalCountSum = 0;
|
|
4786
5222
|
let candidateTopKSum = 0;
|
|
4787
5223
|
let lexicalTopKSum = 0;
|
|
4788
5224
|
for (const trace of traces) {
|
|
5225
|
+
const vectorSearchMetadata = trace.steps.find((step) => step.stage === "vector_search")?.metadata;
|
|
4789
5226
|
modeSet.add(trace.mode);
|
|
4790
5227
|
sourceBalanceStrategySet.add(trace.sourceBalanceStrategy ?? "cap");
|
|
4791
5228
|
if (trace.runVector) {
|
|
@@ -4818,6 +5255,25 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4818
5255
|
if ((trace.multiVector?.collapsedParents ?? 0) > 0) {
|
|
4819
5256
|
multiVectorCollapsedCases += 1;
|
|
4820
5257
|
}
|
|
5258
|
+
const evidenceReconcileMetadata = trace.steps.find((step) => step.stage === "evidence_reconcile")?.metadata;
|
|
5259
|
+
if (typeof evidenceReconcileMetadata?.officeAffectedScopes === "number" && evidenceReconcileMetadata.officeAffectedScopes > 0) {
|
|
5260
|
+
officeEvidenceReconcileCases += 1;
|
|
5261
|
+
}
|
|
5262
|
+
if (typeof evidenceReconcileMetadata?.pdfAffectedScopes === "number" && evidenceReconcileMetadata.pdfAffectedScopes > 0) {
|
|
5263
|
+
pdfEvidenceReconcileCases += 1;
|
|
5264
|
+
}
|
|
5265
|
+
if (vectorSearchMetadata?.sqliteQueryCandidateBudgetExhausted) {
|
|
5266
|
+
runtimeCandidateBudgetExhaustedCases += 1;
|
|
5267
|
+
}
|
|
5268
|
+
if (vectorSearchMetadata?.postgresQueryCandidateBudgetExhausted) {
|
|
5269
|
+
runtimeCandidateBudgetExhaustedCases += 1;
|
|
5270
|
+
}
|
|
5271
|
+
if (vectorSearchMetadata?.sqliteQueryUnderfilledTopK) {
|
|
5272
|
+
runtimeUnderfilledTopKCases += 1;
|
|
5273
|
+
}
|
|
5274
|
+
if (vectorSearchMetadata?.postgresQueryUnderfilledTopK) {
|
|
5275
|
+
runtimeUnderfilledTopKCases += 1;
|
|
5276
|
+
}
|
|
4821
5277
|
finalCountSum += trace.resultCounts.final;
|
|
4822
5278
|
vectorCountSum += trace.resultCounts.vector;
|
|
4823
5279
|
lexicalCountSum += trace.resultCounts.lexical;
|
|
@@ -4843,6 +5299,10 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4843
5299
|
multiVectorVectorHitCases,
|
|
4844
5300
|
multiVectorLexicalHitCases,
|
|
4845
5301
|
multiVectorCollapsedCases,
|
|
5302
|
+
officeEvidenceReconcileCases,
|
|
5303
|
+
pdfEvidenceReconcileCases,
|
|
5304
|
+
runtimeCandidateBudgetExhaustedCases,
|
|
5305
|
+
runtimeUnderfilledTopKCases,
|
|
4846
5306
|
vectorCases
|
|
4847
5307
|
};
|
|
4848
5308
|
};
|
|
@@ -5645,6 +6105,8 @@ var buildRAGEvaluationRunDiff = ({
|
|
|
5645
6105
|
averageLexicalTopK: (current.traceSummary?.averageLexicalTopK ?? 0) - (previous?.traceSummary?.averageLexicalTopK ?? 0),
|
|
5646
6106
|
averageVectorCount: (current.traceSummary?.averageVectorCount ?? 0) - (previous?.traceSummary?.averageVectorCount ?? 0),
|
|
5647
6107
|
balancedCases: (current.traceSummary?.balancedCases ?? 0) - (previous?.traceSummary?.balancedCases ?? 0),
|
|
6108
|
+
officeEvidenceReconcileCasesDelta: (current.traceSummary?.officeEvidenceReconcileCases ?? 0) - (previous?.traceSummary?.officeEvidenceReconcileCases ?? 0),
|
|
6109
|
+
pdfEvidenceReconcileCasesDelta: (current.traceSummary?.pdfEvidenceReconcileCases ?? 0) - (previous?.traceSummary?.pdfEvidenceReconcileCases ?? 0),
|
|
5648
6110
|
lexicalCases: (current.traceSummary?.lexicalCases ?? 0) - (previous?.traceSummary?.lexicalCases ?? 0),
|
|
5649
6111
|
modesChanged: (current.traceSummary?.modes ?? []).join("|") !== (previous?.traceSummary?.modes ?? []).join("|"),
|
|
5650
6112
|
roundRobinCases: (current.traceSummary?.roundRobinCases ?? 0) - (previous?.traceSummary?.roundRobinCases ?? 0),
|
|
@@ -8428,7 +8890,10 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8428
8890
|
passingRateDelta: candidateEntry.response.passingRate - baselineEntry.response.passingRate,
|
|
8429
8891
|
multiVectorCollapsedCasesDelta: (candidateEntry.traceSummary?.multiVectorCollapsedCases ?? 0) - (baselineEntry.traceSummary?.multiVectorCollapsedCases ?? 0),
|
|
8430
8892
|
multiVectorLexicalHitCasesDelta: (candidateEntry.traceSummary?.multiVectorLexicalHitCases ?? 0) - (baselineEntry.traceSummary?.multiVectorLexicalHitCases ?? 0),
|
|
8431
|
-
multiVectorVectorHitCasesDelta: (candidateEntry.traceSummary?.multiVectorVectorHitCases ?? 0) - (baselineEntry.traceSummary?.multiVectorVectorHitCases ?? 0)
|
|
8893
|
+
multiVectorVectorHitCasesDelta: (candidateEntry.traceSummary?.multiVectorVectorHitCases ?? 0) - (baselineEntry.traceSummary?.multiVectorVectorHitCases ?? 0),
|
|
8894
|
+
evidenceReconcileCasesDelta: (candidateEntry.traceSummary?.stageCounts?.evidence_reconcile ?? 0) - (baselineEntry.traceSummary?.stageCounts?.evidence_reconcile ?? 0),
|
|
8895
|
+
runtimeCandidateBudgetExhaustedCasesDelta: (candidateEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases ?? 0) - (baselineEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases ?? 0),
|
|
8896
|
+
runtimeUnderfilledTopKCasesDelta: (candidateEntry.traceSummary?.runtimeUnderfilledTopKCases ?? 0) - (baselineEntry.traceSummary?.runtimeUnderfilledTopKCases ?? 0)
|
|
8432
8897
|
} : undefined;
|
|
8433
8898
|
return {
|
|
8434
8899
|
baseline: baselineEntry ? {
|
|
@@ -8438,6 +8903,9 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8438
8903
|
multiVectorCollapsedCases: baselineEntry.traceSummary?.multiVectorCollapsedCases,
|
|
8439
8904
|
multiVectorLexicalHitCases: baselineEntry.traceSummary?.multiVectorLexicalHitCases,
|
|
8440
8905
|
multiVectorVectorHitCases: baselineEntry.traceSummary?.multiVectorVectorHitCases,
|
|
8906
|
+
evidenceReconcileCases: baselineEntry.traceSummary?.stageCounts?.evidence_reconcile,
|
|
8907
|
+
runtimeCandidateBudgetExhaustedCases: baselineEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases,
|
|
8908
|
+
runtimeUnderfilledTopKCases: baselineEntry.traceSummary?.runtimeUnderfilledTopKCases,
|
|
8441
8909
|
passingRate: baselineEntry.response.passingRate,
|
|
8442
8910
|
retrievalId: baselineEntry.retrievalId
|
|
8443
8911
|
} : undefined,
|
|
@@ -8449,6 +8917,9 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8449
8917
|
multiVectorCollapsedCases: candidateEntry.traceSummary?.multiVectorCollapsedCases,
|
|
8450
8918
|
multiVectorLexicalHitCases: candidateEntry.traceSummary?.multiVectorLexicalHitCases,
|
|
8451
8919
|
multiVectorVectorHitCases: candidateEntry.traceSummary?.multiVectorVectorHitCases,
|
|
8920
|
+
evidenceReconcileCases: candidateEntry.traceSummary?.stageCounts?.evidence_reconcile,
|
|
8921
|
+
runtimeCandidateBudgetExhaustedCases: candidateEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases,
|
|
8922
|
+
runtimeUnderfilledTopKCases: candidateEntry.traceSummary?.runtimeUnderfilledTopKCases,
|
|
8452
8923
|
passingRate: candidateEntry.response.passingRate,
|
|
8453
8924
|
retrievalId: candidateEntry.retrievalId
|
|
8454
8925
|
} : undefined,
|
|
@@ -8460,7 +8931,10 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8460
8931
|
winnerByPassingRate: comparison.summary.bestByPassingRate,
|
|
8461
8932
|
winnerByMultivectorCollapsedCases: comparison.summary.bestByMultivectorCollapsedCases,
|
|
8462
8933
|
winnerByMultivectorLexicalHitCases: comparison.summary.bestByMultivectorLexicalHitCases,
|
|
8463
|
-
winnerByMultivectorVectorHitCases: comparison.summary.bestByMultivectorVectorHitCases
|
|
8934
|
+
winnerByMultivectorVectorHitCases: comparison.summary.bestByMultivectorVectorHitCases,
|
|
8935
|
+
winnerByEvidenceReconcileCases: comparison.summary.bestByEvidenceReconcileCases,
|
|
8936
|
+
winnerByLowestRuntimeCandidateBudgetExhaustedCases: comparison.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases,
|
|
8937
|
+
winnerByLowestRuntimeUnderfilledTopKCases: comparison.summary.bestByLowestRuntimeUnderfilledTopKCases
|
|
8464
8938
|
};
|
|
8465
8939
|
};
|
|
8466
8940
|
var loadRAGSearchTracePruneHistory = async ({
|
|
@@ -9148,6 +9622,45 @@ var selectComparisonEntryByTraceMetric = (entries, idKey, metric) => {
|
|
|
9148
9622
|
}
|
|
9149
9623
|
return typeof winner[idKey] === "string" ? winner[idKey] : undefined;
|
|
9150
9624
|
};
|
|
9625
|
+
var selectComparisonEntryByLowestTraceMetric = (entries, idKey, metric) => {
|
|
9626
|
+
const ranked = [...entries].sort((left, right) => {
|
|
9627
|
+
const leftMetric = left.traceSummary?.[metric] ?? 0;
|
|
9628
|
+
const rightMetric = right.traceSummary?.[metric] ?? 0;
|
|
9629
|
+
if (leftMetric !== rightMetric) {
|
|
9630
|
+
return leftMetric - rightMetric;
|
|
9631
|
+
}
|
|
9632
|
+
if (right.response.passingRate !== left.response.passingRate) {
|
|
9633
|
+
return right.response.passingRate - left.response.passingRate;
|
|
9634
|
+
}
|
|
9635
|
+
if (right.response.summary.averageF1 !== left.response.summary.averageF1) {
|
|
9636
|
+
return right.response.summary.averageF1 - left.response.summary.averageF1;
|
|
9637
|
+
}
|
|
9638
|
+
return left.response.summary.averageLatencyMs - right.response.summary.averageLatencyMs;
|
|
9639
|
+
});
|
|
9640
|
+
const winner = ranked[0];
|
|
9641
|
+
return typeof winner?.[idKey] === "string" ? winner[idKey] : undefined;
|
|
9642
|
+
};
|
|
9643
|
+
var selectComparisonEntryByTraceStageCount = (entries, idKey, stage) => {
|
|
9644
|
+
const ranked = [...entries].sort((left, right) => {
|
|
9645
|
+
const leftMetric = left.traceSummary?.stageCounts?.[stage] ?? 0;
|
|
9646
|
+
const rightMetric = right.traceSummary?.stageCounts?.[stage] ?? 0;
|
|
9647
|
+
if (rightMetric !== leftMetric) {
|
|
9648
|
+
return rightMetric - leftMetric;
|
|
9649
|
+
}
|
|
9650
|
+
if (right.response.passingRate !== left.response.passingRate) {
|
|
9651
|
+
return right.response.passingRate - left.response.passingRate;
|
|
9652
|
+
}
|
|
9653
|
+
if (right.response.summary.averageF1 !== left.response.summary.averageF1) {
|
|
9654
|
+
return right.response.summary.averageF1 - left.response.summary.averageF1;
|
|
9655
|
+
}
|
|
9656
|
+
return left.response.summary.averageLatencyMs - right.response.summary.averageLatencyMs;
|
|
9657
|
+
});
|
|
9658
|
+
const winner = ranked[0];
|
|
9659
|
+
if (!winner || (winner.traceSummary?.stageCounts?.[stage] ?? 0) === 0) {
|
|
9660
|
+
return;
|
|
9661
|
+
}
|
|
9662
|
+
return typeof winner?.[idKey] === "string" ? winner[idKey] : undefined;
|
|
9663
|
+
};
|
|
9151
9664
|
var resolveRetrievalMode = (candidate) => {
|
|
9152
9665
|
if (!candidate.retrieval) {
|
|
9153
9666
|
return "vector";
|
|
@@ -9241,7 +9754,11 @@ var compareRAGRetrievalTraceSummaries = (current, previous) => ({
|
|
|
9241
9754
|
multiVectorCasesDelta: current.multiVectorCases - previous.multiVectorCases,
|
|
9242
9755
|
multiVectorVectorHitCasesDelta: current.multiVectorVectorHitCases - previous.multiVectorVectorHitCases,
|
|
9243
9756
|
multiVectorLexicalHitCasesDelta: current.multiVectorLexicalHitCases - previous.multiVectorLexicalHitCases,
|
|
9244
|
-
multiVectorCollapsedCasesDelta: current.multiVectorCollapsedCases - previous.multiVectorCollapsedCases
|
|
9757
|
+
multiVectorCollapsedCasesDelta: current.multiVectorCollapsedCases - previous.multiVectorCollapsedCases,
|
|
9758
|
+
officeEvidenceReconcileCasesDelta: current.officeEvidenceReconcileCases - previous.officeEvidenceReconcileCases,
|
|
9759
|
+
pdfEvidenceReconcileCasesDelta: current.pdfEvidenceReconcileCases - previous.pdfEvidenceReconcileCases,
|
|
9760
|
+
runtimeCandidateBudgetExhaustedCasesDelta: current.runtimeCandidateBudgetExhaustedCases - previous.runtimeCandidateBudgetExhaustedCases,
|
|
9761
|
+
runtimeUnderfilledTopKCasesDelta: current.runtimeUnderfilledTopKCases - previous.runtimeUnderfilledTopKCases
|
|
9245
9762
|
});
|
|
9246
9763
|
var buildSearchTraceResultSnapshots = (results) => results.map((result) => ({
|
|
9247
9764
|
chunkId: result.chunkId,
|
|
@@ -9603,6 +10120,278 @@ var generateRAGEvaluationSuiteFromDocuments = ({
|
|
|
9603
10120
|
metadata
|
|
9604
10121
|
});
|
|
9605
10122
|
};
|
|
10123
|
+
var DEFAULT_NATIVE_PLANNER_BENCHMARK_SUITE_ID = "rag-native-planner-larger-corpus";
|
|
10124
|
+
var DEFAULT_NATIVE_PLANNER_BENCHMARK_LABEL = "Adaptive Native Planner Benchmark";
|
|
10125
|
+
var DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_SUITE_ID = "rag-native-backend-larger-corpus";
|
|
10126
|
+
var DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_LABEL = "Native Backend Comparison Benchmark";
|
|
10127
|
+
var DEFAULT_NATIVE_PLANNER_BENCHMARK_QUERY = "Which launch checklist phrase is exact wording?";
|
|
10128
|
+
var DEFAULT_NATIVE_BACKEND_HYBRID_QUERY = "aurora promotion checklist wording";
|
|
10129
|
+
var DEFAULT_NATIVE_BACKEND_FILTERED_QUERY = "focus lane launch checklist wording";
|
|
10130
|
+
var DEFAULT_NATIVE_BACKEND_REORDERED_QUERY = "exact aurora focus lane checklist wording";
|
|
10131
|
+
var DEFAULT_NATIVE_BACKEND_GUIDE_QUERY = "which focus lane guide contains exact aurora promotion wording";
|
|
10132
|
+
var DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER = {
|
|
10133
|
+
lane: "focus"
|
|
10134
|
+
};
|
|
10135
|
+
var DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS = [
|
|
10136
|
+
"focus-distractor-0",
|
|
10137
|
+
"focus-distractor-1",
|
|
10138
|
+
"focus-distractor-2"
|
|
10139
|
+
];
|
|
10140
|
+
var createRAGNativeBackendBenchmarkMockEmbedding = async (text) => {
|
|
10141
|
+
const normalized = text.toLowerCase();
|
|
10142
|
+
if (normalized.includes("launch checklist exact wording for aurora promotion") || normalized.includes("launch checklist exact wording")) {
|
|
10143
|
+
return [0.995, 0.005];
|
|
10144
|
+
}
|
|
10145
|
+
if (normalized.includes("aurora") || normalized.includes("checklist") || normalized.includes("focus lane") || normalized.includes("exact wording") || normalized.includes("guide")) {
|
|
10146
|
+
return [1, 0];
|
|
10147
|
+
}
|
|
10148
|
+
return [0, 1];
|
|
10149
|
+
};
|
|
10150
|
+
var createRAGNativeBackendBenchmarkCorpus = (input) => {
|
|
10151
|
+
const noiseCount = input?.noiseCount ?? 5001;
|
|
10152
|
+
const backend = input?.backend ?? "generic";
|
|
10153
|
+
const genericChunks = [
|
|
10154
|
+
...Array.from({ length: noiseCount }, (_, index) => ({
|
|
10155
|
+
chunkId: `noise:${index}`,
|
|
10156
|
+
corpusKey: "noise",
|
|
10157
|
+
embedding: [0, 1],
|
|
10158
|
+
metadata: {
|
|
10159
|
+
corpusKey: "noise",
|
|
10160
|
+
documentId: `noise-${index}`,
|
|
10161
|
+
lane: "noise"
|
|
10162
|
+
},
|
|
10163
|
+
source: `noise/${index}.md`,
|
|
10164
|
+
text: `Background operations note ${index}.`
|
|
10165
|
+
})),
|
|
10166
|
+
...Array.from({ length: 3 }, (_, index) => ({
|
|
10167
|
+
chunkId: `focus:distractor:${index}`,
|
|
10168
|
+
corpusKey: "focus",
|
|
10169
|
+
embedding: [1, 0],
|
|
10170
|
+
metadata: {
|
|
10171
|
+
corpusKey: "focus",
|
|
10172
|
+
documentId: `focus-distractor-${index}`,
|
|
10173
|
+
lane: "focus"
|
|
10174
|
+
},
|
|
10175
|
+
source: `focus/distractor-${index}.md`,
|
|
10176
|
+
text: index === 0 ? "aurora promotion checklist overview" : index === 1 ? "launch checklist wording draft" : "focus lane promotion runbook notes"
|
|
10177
|
+
})),
|
|
10178
|
+
{
|
|
10179
|
+
chunkId: "focus:target",
|
|
10180
|
+
corpusKey: "focus",
|
|
10181
|
+
embedding: [0.995, 0.005],
|
|
10182
|
+
metadata: {
|
|
10183
|
+
corpusKey: "focus",
|
|
10184
|
+
documentId: "focus-target",
|
|
10185
|
+
lane: "focus"
|
|
10186
|
+
},
|
|
10187
|
+
source: "guide/planner-depth.md",
|
|
10188
|
+
text: "launch checklist exact wording for aurora promotion in the focus lane"
|
|
10189
|
+
}
|
|
10190
|
+
];
|
|
10191
|
+
const backendSpecificChunks = backend === "sqlite-native" ? [
|
|
10192
|
+
{
|
|
10193
|
+
chunkId: "focus:sqlite:phrase-matrix",
|
|
10194
|
+
corpusKey: "focus",
|
|
10195
|
+
embedding: [1, 0],
|
|
10196
|
+
metadata: {
|
|
10197
|
+
backendFixture: "sqlite-native",
|
|
10198
|
+
corpusKey: "focus",
|
|
10199
|
+
documentId: "focus-sqlite-phrase-matrix",
|
|
10200
|
+
lane: "focus"
|
|
10201
|
+
},
|
|
10202
|
+
source: "guide/sqlite-phrase-matrix.md",
|
|
10203
|
+
text: "exact aurora focus lane checklist wording matrix for sqlite validation"
|
|
10204
|
+
},
|
|
10205
|
+
{
|
|
10206
|
+
chunkId: "focus:sqlite:guide-table",
|
|
10207
|
+
corpusKey: "focus",
|
|
10208
|
+
embedding: [1, 0],
|
|
10209
|
+
metadata: {
|
|
10210
|
+
backendFixture: "sqlite-native",
|
|
10211
|
+
corpusKey: "focus",
|
|
10212
|
+
documentId: "focus-sqlite-guide-table",
|
|
10213
|
+
lane: "focus"
|
|
10214
|
+
},
|
|
10215
|
+
source: "guide/sqlite-guide-table.md",
|
|
10216
|
+
text: "which focus lane guide contains aurora promotion wording draft table for sqlite operators"
|
|
10217
|
+
}
|
|
10218
|
+
] : backend === "postgres" ? [
|
|
10219
|
+
{
|
|
10220
|
+
chunkId: "focus:postgres:appendix",
|
|
10221
|
+
corpusKey: "focus",
|
|
10222
|
+
embedding: [1, 0],
|
|
10223
|
+
metadata: {
|
|
10224
|
+
backendFixture: "postgres",
|
|
10225
|
+
corpusKey: "focus",
|
|
10226
|
+
documentId: "focus-postgres-appendix",
|
|
10227
|
+
lane: "focus"
|
|
10228
|
+
},
|
|
10229
|
+
source: "guide/postgres-appendix.md",
|
|
10230
|
+
text: "which focus lane guide contains exact aurora promotion wording appendix for postgres release review"
|
|
10231
|
+
},
|
|
10232
|
+
{
|
|
10233
|
+
chunkId: "focus:postgres:alternatives",
|
|
10234
|
+
corpusKey: "focus",
|
|
10235
|
+
embedding: [1, 0],
|
|
10236
|
+
metadata: {
|
|
10237
|
+
backendFixture: "postgres",
|
|
10238
|
+
corpusKey: "focus",
|
|
10239
|
+
documentId: "focus-postgres-alternatives",
|
|
10240
|
+
lane: "focus"
|
|
10241
|
+
},
|
|
10242
|
+
source: "guide/postgres-alternatives.md",
|
|
10243
|
+
text: "aurora promotion checklist wording alternatives and exact focus lane phrasing for postgres audits"
|
|
10244
|
+
}
|
|
10245
|
+
] : [];
|
|
10246
|
+
return [...genericChunks, ...backendSpecificChunks];
|
|
10247
|
+
};
|
|
10248
|
+
var createRAGAdaptiveNativePlannerBenchmarkSuite = (input) => createRAGEvaluationSuite({
|
|
10249
|
+
description: input?.description ?? "Stress-tests larger-corpus native planner selection, candidate-budget pressure, and transformed-query recovery on filtered retrieval.",
|
|
10250
|
+
id: input?.id ?? DEFAULT_NATIVE_PLANNER_BENCHMARK_SUITE_ID,
|
|
10251
|
+
input: {
|
|
10252
|
+
cases: [
|
|
10253
|
+
{
|
|
10254
|
+
expectedDocumentIds: ["focus-target"],
|
|
10255
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10256
|
+
hardNegativeDocumentIds: [
|
|
10257
|
+
"focus-distractor-0",
|
|
10258
|
+
"focus-distractor-1",
|
|
10259
|
+
"focus-distractor-2"
|
|
10260
|
+
],
|
|
10261
|
+
id: "planner-pressure-exact-phrase",
|
|
10262
|
+
label: "Exact phrase survives larger-corpus native pressure",
|
|
10263
|
+
query: DEFAULT_NATIVE_PLANNER_BENCHMARK_QUERY,
|
|
10264
|
+
topK: input?.topK ?? 1
|
|
10265
|
+
}
|
|
10266
|
+
],
|
|
10267
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10268
|
+
retrieval: "vector",
|
|
10269
|
+
topK: input?.topK ?? 1
|
|
10270
|
+
},
|
|
10271
|
+
label: input?.label ?? DEFAULT_NATIVE_PLANNER_BENCHMARK_LABEL,
|
|
10272
|
+
metadata: {
|
|
10273
|
+
benchmarkKind: "adaptive_native_planner",
|
|
10274
|
+
benchmarkScope: "larger_corpus",
|
|
10275
|
+
expectedSignals: [
|
|
10276
|
+
"selected native planner profile",
|
|
10277
|
+
"candidate-budget exhaustion",
|
|
10278
|
+
"underfilled topk"
|
|
10279
|
+
],
|
|
10280
|
+
recommendedGroupKey: "runtime-native-planner",
|
|
10281
|
+
recommendedTags: ["runtime", "native", "planner"],
|
|
10282
|
+
...input?.metadata
|
|
10283
|
+
}
|
|
10284
|
+
});
|
|
10285
|
+
var createRAGAdaptiveNativePlannerBenchmarkSnapshot = (input) => {
|
|
10286
|
+
const suite = input?.suite ?? createRAGAdaptiveNativePlannerBenchmarkSuite();
|
|
10287
|
+
return createRAGEvaluationSuiteSnapshot({
|
|
10288
|
+
createdAt: input?.createdAt,
|
|
10289
|
+
id: input?.id,
|
|
10290
|
+
metadata: {
|
|
10291
|
+
artifactKind: "adaptive_native_planner_benchmark",
|
|
10292
|
+
persistForReleaseHistory: true,
|
|
10293
|
+
...input?.metadata
|
|
10294
|
+
},
|
|
10295
|
+
suite,
|
|
10296
|
+
version: input?.version
|
|
10297
|
+
});
|
|
10298
|
+
};
|
|
10299
|
+
var createRAGNativeBackendComparisonBenchmarkSuite = (input) => createRAGEvaluationSuite({
|
|
10300
|
+
description: input?.description ?? "Captures larger-corpus native backend parity with filtered vector pressure and harder hybrid retrieval cases so sqlite-native and postgres runs can be compared over time.",
|
|
10301
|
+
id: input?.id ?? DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_SUITE_ID,
|
|
10302
|
+
input: {
|
|
10303
|
+
cases: [
|
|
10304
|
+
{
|
|
10305
|
+
expectedDocumentIds: ["focus-target"],
|
|
10306
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10307
|
+
hardNegativeDocumentIds: [
|
|
10308
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10309
|
+
],
|
|
10310
|
+
id: "planner-pressure-exact-phrase",
|
|
10311
|
+
label: "Exact phrase survives larger-corpus native pressure",
|
|
10312
|
+
query: DEFAULT_NATIVE_PLANNER_BENCHMARK_QUERY,
|
|
10313
|
+
topK: input?.topK ?? 1
|
|
10314
|
+
},
|
|
10315
|
+
{
|
|
10316
|
+
expectedDocumentIds: ["focus-target"],
|
|
10317
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10318
|
+
hardNegativeDocumentIds: [
|
|
10319
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10320
|
+
],
|
|
10321
|
+
id: "planner-pressure-hybrid-phrase",
|
|
10322
|
+
label: "Hybrid retrieval survives filtered lexical pressure",
|
|
10323
|
+
query: DEFAULT_NATIVE_BACKEND_HYBRID_QUERY,
|
|
10324
|
+
topK: input?.topK ?? 1
|
|
10325
|
+
},
|
|
10326
|
+
{
|
|
10327
|
+
expectedDocumentIds: ["focus-target"],
|
|
10328
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10329
|
+
hardNegativeDocumentIds: [
|
|
10330
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10331
|
+
],
|
|
10332
|
+
id: "planner-pressure-filtered-lane-query",
|
|
10333
|
+
label: "Filtered lane query survives broader corpus noise",
|
|
10334
|
+
query: DEFAULT_NATIVE_BACKEND_FILTERED_QUERY,
|
|
10335
|
+
topK: input?.topK ?? 1
|
|
10336
|
+
},
|
|
10337
|
+
{
|
|
10338
|
+
expectedDocumentIds: ["focus-target"],
|
|
10339
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10340
|
+
hardNegativeDocumentIds: [
|
|
10341
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10342
|
+
],
|
|
10343
|
+
id: "planner-pressure-reordered-phrase",
|
|
10344
|
+
label: "Reordered phrase survives transform pressure",
|
|
10345
|
+
query: DEFAULT_NATIVE_BACKEND_REORDERED_QUERY,
|
|
10346
|
+
topK: input?.topK ?? 1
|
|
10347
|
+
},
|
|
10348
|
+
{
|
|
10349
|
+
expectedDocumentIds: ["focus-target"],
|
|
10350
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10351
|
+
hardNegativeDocumentIds: [
|
|
10352
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10353
|
+
],
|
|
10354
|
+
id: "planner-pressure-guide-query",
|
|
10355
|
+
label: "Guide attribution survives filtered corpus pressure",
|
|
10356
|
+
query: DEFAULT_NATIVE_BACKEND_GUIDE_QUERY,
|
|
10357
|
+
topK: input?.topK ?? 1
|
|
10358
|
+
}
|
|
10359
|
+
],
|
|
10360
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10361
|
+
retrieval: "vector",
|
|
10362
|
+
topK: input?.topK ?? 1
|
|
10363
|
+
},
|
|
10364
|
+
label: input?.label ?? DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_LABEL,
|
|
10365
|
+
metadata: {
|
|
10366
|
+
benchmarkKind: "native_backend_comparison",
|
|
10367
|
+
benchmarkScope: "larger_corpus",
|
|
10368
|
+
expectedSignals: [
|
|
10369
|
+
"backend-tagged runtime artifacts",
|
|
10370
|
+
"selected native planner profile",
|
|
10371
|
+
"hybrid filtered retrieval",
|
|
10372
|
+
"candidate-budget exhaustion",
|
|
10373
|
+
"underfilled topk",
|
|
10374
|
+
"query transform pressure"
|
|
10375
|
+
],
|
|
10376
|
+
recommendedGroupKey: "runtime-native-backend-parity",
|
|
10377
|
+
recommendedTags: ["runtime", "backend", "native"],
|
|
10378
|
+
...input?.metadata
|
|
10379
|
+
}
|
|
10380
|
+
});
|
|
10381
|
+
var createRAGNativeBackendComparisonBenchmarkSnapshot = (input) => {
|
|
10382
|
+
const suite = input?.suite ?? createRAGNativeBackendComparisonBenchmarkSuite();
|
|
10383
|
+
return createRAGEvaluationSuiteSnapshot({
|
|
10384
|
+
createdAt: input?.createdAt,
|
|
10385
|
+
id: input?.id,
|
|
10386
|
+
metadata: {
|
|
10387
|
+
artifactKind: "native_backend_comparison_benchmark",
|
|
10388
|
+
persistForReleaseHistory: true,
|
|
10389
|
+
...input?.metadata
|
|
10390
|
+
},
|
|
10391
|
+
suite,
|
|
10392
|
+
version: input?.version
|
|
10393
|
+
});
|
|
10394
|
+
};
|
|
9606
10395
|
var createRAGEvaluationSuiteSnapshot = ({
|
|
9607
10396
|
suite,
|
|
9608
10397
|
id,
|
|
@@ -9810,7 +10599,10 @@ var summarizeRAGRetrievalComparison = (entries) => ({
|
|
|
9810
10599
|
...summarizeEvaluationResponseComparison(entries, "retrievalId"),
|
|
9811
10600
|
bestByMultivectorCollapsedCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorCollapsedCases"),
|
|
9812
10601
|
bestByMultivectorLexicalHitCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorLexicalHitCases"),
|
|
9813
|
-
bestByMultivectorVectorHitCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorVectorHitCases")
|
|
10602
|
+
bestByMultivectorVectorHitCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorVectorHitCases"),
|
|
10603
|
+
bestByEvidenceReconcileCases: selectComparisonEntryByTraceStageCount(entries, "retrievalId", "evidence_reconcile"),
|
|
10604
|
+
bestByLowestRuntimeCandidateBudgetExhaustedCases: selectComparisonEntryByLowestTraceMetric(entries, "retrievalId", "runtimeCandidateBudgetExhaustedCases"),
|
|
10605
|
+
bestByLowestRuntimeUnderfilledTopKCases: selectComparisonEntryByLowestTraceMetric(entries, "retrievalId", "runtimeUnderfilledTopKCases")
|
|
9814
10606
|
});
|
|
9815
10607
|
export {
|
|
9816
10608
|
resolveRAGStreamStage,
|
|
@@ -9869,5 +10661,5 @@ export {
|
|
|
9869
10661
|
buildRAGAdminActionPresentation
|
|
9870
10662
|
};
|
|
9871
10663
|
|
|
9872
|
-
//# debugId=
|
|
10664
|
+
//# debugId=ED996CFAFF41E50864756E2164756E21
|
|
9873
10665
|
//# sourceMappingURL=ui.js.map
|