@absolutejs/absolute 0.19.0-beta.644 → 0.19.0-beta.645
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/client/index.js +694 -11
- package/dist/ai/client/index.js.map +6 -6
- package/dist/ai/client/ui.js +573 -11
- package/dist/ai/client/ui.js.map +5 -5
- package/dist/ai/index.js +2150 -136
- package/dist/ai/index.js.map +10 -10
- package/dist/ai/rag/quality.js +577 -11
- package/dist/ai/rag/quality.js.map +5 -5
- package/dist/ai/rag/ui.js +573 -11
- package/dist/ai/rag/ui.js.map +5 -5
- package/dist/ai-client/angular/ai/index.js +388 -8
- package/dist/ai-client/react/ai/index.js +388 -8
- package/dist/ai-client/vue/ai/index.js +388 -8
- package/dist/angular/ai/index.js +694 -11
- package/dist/angular/ai/index.js.map +6 -6
- package/dist/index.js +6 -6
- package/dist/index.js.map +2 -2
- package/dist/react/ai/index.js +694 -11
- package/dist/react/ai/index.js.map +6 -6
- package/dist/src/ai/client/ragClient.d.ts +58 -0
- package/dist/src/ai/index.d.ts +2 -1
- package/dist/src/ai/rag/chat.d.ts +90 -4
- package/dist/src/ai/rag/index.d.ts +1 -1
- package/dist/src/ai/rag/quality.d.ts +20 -1
- package/dist/src/vue/ai/useRAG.d.ts +80 -0
- package/dist/src/vue/ai/useRAGEvaluate.d.ts +70 -0
- package/dist/src/vue/ai/useRAGSearch.d.ts +10 -0
- package/dist/svelte/ai/index.js +694 -11
- package/dist/svelte/ai/index.js.map +6 -6
- package/dist/types/ai.d.ts +56 -13
- package/dist/types/index.d.ts +1 -0
- package/dist/types/session.d.ts +16 -0
- package/dist/vue/ai/index.js +694 -11
- package/dist/vue/ai/index.js.map +6 -6
- package/package.json +2 -1
package/dist/ai/rag/quality.js
CHANGED
|
@@ -200,6 +200,7 @@ var buildContextLabel = (metadata) => {
|
|
|
200
200
|
return;
|
|
201
201
|
}
|
|
202
202
|
const emailKind = getContextString(metadata.emailKind);
|
|
203
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
203
204
|
if (emailKind === "attachment") {
|
|
204
205
|
return "Attachment evidence";
|
|
205
206
|
}
|
|
@@ -237,6 +238,16 @@ var buildContextLabel = (metadata) => {
|
|
|
237
238
|
}
|
|
238
239
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
239
240
|
const sectionTitle = getContextString(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
241
|
+
const officeSectionLabel = sectionPath.length > 0 ? sectionPath.join(" > ") : sectionTitle;
|
|
242
|
+
if (officeBlockKind === "table" && officeSectionLabel) {
|
|
243
|
+
return `Office table block ${officeSectionLabel}`;
|
|
244
|
+
}
|
|
245
|
+
if (officeBlockKind === "list" && officeSectionLabel) {
|
|
246
|
+
return `Office list block ${officeSectionLabel}`;
|
|
247
|
+
}
|
|
248
|
+
if (officeBlockKind === "paragraph" && officeSectionLabel) {
|
|
249
|
+
return `Office paragraph block ${officeSectionLabel}`;
|
|
250
|
+
}
|
|
240
251
|
if (sectionTitle) {
|
|
241
252
|
return `Section ${sectionTitle}`;
|
|
242
253
|
}
|
|
@@ -258,6 +269,46 @@ var formatMediaDurationLabel = (value) => {
|
|
|
258
269
|
}
|
|
259
270
|
return formatMediaTimestamp(value);
|
|
260
271
|
};
|
|
272
|
+
var formatOfficeListLevelsLabel = (value) => {
|
|
273
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const levels = value.map((entry) => getContextNumber(entry)).filter((entry) => typeof entry === "number").sort((left, right) => left - right);
|
|
277
|
+
if (levels.length === 0) {
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const minLevel = levels[0];
|
|
281
|
+
const maxLevel = levels[levels.length - 1];
|
|
282
|
+
return minLevel === maxLevel ? `Office list level ${minLevel}` : `Office list levels ${minLevel}-${maxLevel}`;
|
|
283
|
+
};
|
|
284
|
+
var getOfficeTableCitationScope = (metadata) => {
|
|
285
|
+
if (!metadata) {
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
289
|
+
if (officeBlockKind !== "table" && officeBlockKind !== "list") {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
293
|
+
const sectionTitle = getContextString(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
294
|
+
const officeContextText = officeBlockKind === "table" ? getContextString(metadata.officeTableContextText) : getContextString(metadata.officeListContextText);
|
|
295
|
+
if (!sectionTitle) {
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
return {
|
|
299
|
+
blockKind: officeBlockKind,
|
|
300
|
+
pathDepth: sectionPath.length,
|
|
301
|
+
sectionTitle,
|
|
302
|
+
hasContext: typeof officeContextText === "string"
|
|
303
|
+
};
|
|
304
|
+
};
|
|
305
|
+
var getOfficeTableCitationPreference = (metadata) => {
|
|
306
|
+
const scope = getOfficeTableCitationScope(metadata);
|
|
307
|
+
if (!scope) {
|
|
308
|
+
return 0;
|
|
309
|
+
}
|
|
310
|
+
return scope.pathDepth * 10 + (scope.hasContext ? 1 : 0) + (scope.blockKind === "list" && typeof metadata?.officeListGroupItemCount === "number" && metadata.officeListGroupItemCount > 1 ? 1 : 0);
|
|
311
|
+
};
|
|
261
312
|
var buildLocatorLabel = (metadata, source, title) => {
|
|
262
313
|
if (!metadata) {
|
|
263
314
|
return;
|
|
@@ -283,6 +334,10 @@ var buildLocatorLabel = (metadata, source, title) => {
|
|
|
283
334
|
return `Archive entry ${archiveEntry}`;
|
|
284
335
|
}
|
|
285
336
|
const emailKind = getContextString(metadata.emailKind);
|
|
337
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
338
|
+
const officeBlockNumber = getContextNumber(metadata.officeBlockNumber);
|
|
339
|
+
const officeTableBodyRowStart = getContextNumber(metadata.officeTableBodyRowStart);
|
|
340
|
+
const officeTableBodyRowEnd = getContextNumber(metadata.officeTableBodyRowEnd);
|
|
286
341
|
if (emailKind === "attachment") {
|
|
287
342
|
const attachmentName = getContextString(metadata.attachmentName) ?? getAttachmentName(source, title);
|
|
288
343
|
return attachmentName ? `Attachment ${attachmentName}` : "Attachment";
|
|
@@ -295,6 +350,18 @@ var buildLocatorLabel = (metadata, source, title) => {
|
|
|
295
350
|
if (mediaStart) {
|
|
296
351
|
return `Timestamp ${mediaStart}`;
|
|
297
352
|
}
|
|
353
|
+
if (officeBlockNumber && officeBlockKind === "table") {
|
|
354
|
+
if (typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number") {
|
|
355
|
+
return officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table block ${officeBlockNumber} \xB7 Row ${officeTableBodyRowStart}` : `Office table block ${officeBlockNumber} \xB7 Rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}`;
|
|
356
|
+
}
|
|
357
|
+
return `Office table block ${officeBlockNumber}`;
|
|
358
|
+
}
|
|
359
|
+
if (officeBlockNumber && officeBlockKind === "list") {
|
|
360
|
+
return `Office list block ${officeBlockNumber}`;
|
|
361
|
+
}
|
|
362
|
+
if (officeBlockNumber && officeBlockKind === "paragraph") {
|
|
363
|
+
return `Office paragraph block ${officeBlockNumber}`;
|
|
364
|
+
}
|
|
298
365
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
299
366
|
if (sectionPath.length > 0) {
|
|
300
367
|
return `Section ${sectionPath.join(" > ")}`;
|
|
@@ -328,10 +395,31 @@ var buildProvenanceLabel = (metadata) => {
|
|
|
328
395
|
const mediaDurationLabel = formatMediaDurationLabel(metadata.mediaDurationMs);
|
|
329
396
|
const transcriptSource = getContextString(metadata.transcriptSource);
|
|
330
397
|
const pdfTextMode = getContextString(metadata.pdfTextMode);
|
|
398
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
399
|
+
const officeListContextText = getContextString(metadata.officeListContextText);
|
|
400
|
+
const officeListGroupItemCount = getContextNumber(metadata.officeListGroupItemCount);
|
|
401
|
+
const officeListLevelsLabel = formatOfficeListLevelsLabel(metadata.officeListLevels);
|
|
402
|
+
const officeTableHeaders = Array.isArray(metadata.officeTableHeaders) ? metadata.officeTableHeaders.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
403
|
+
const officeTableColumnCount = getContextNumber(metadata.officeTableColumnCount);
|
|
404
|
+
const officeTableBodyRowCount = getContextNumber(metadata.officeTableBodyRowCount);
|
|
405
|
+
const officeTableBodyRowStart = getContextNumber(metadata.officeTableBodyRowStart);
|
|
406
|
+
const officeTableBodyRowEnd = getContextNumber(metadata.officeTableBodyRowEnd);
|
|
407
|
+
const officeTableContextText = getContextString(metadata.officeTableContextText);
|
|
408
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
331
409
|
const ocrEngine = getContextString(metadata.ocrEngine);
|
|
332
410
|
const ocrConfidence = getContextNumber(metadata.ocrRegionConfidence) ?? getContextNumber(metadata.ocrConfidence);
|
|
333
411
|
const labels = [
|
|
334
412
|
pdfTextMode ? `PDF ${pdfTextMode}` : "",
|
|
413
|
+
officeBlockKind ? `Office ${officeBlockKind}` : "",
|
|
414
|
+
typeof officeListGroupItemCount === "number" ? `Office list ${officeListGroupItemCount} items` : "",
|
|
415
|
+
officeListLevelsLabel ?? "",
|
|
416
|
+
sectionPath.length > 0 && officeBlockKind ? `Source-aware office ${officeBlockKind} block ${sectionPath.join(" > ")}` : "",
|
|
417
|
+
officeListContextText ? `Office list context ${officeListContextText}` : "",
|
|
418
|
+
officeTableHeaders.length > 0 ? `Office table ${officeTableHeaders.join(", ")}` : "",
|
|
419
|
+
typeof officeTableColumnCount === "number" ? `Office table ${officeTableColumnCount} cols` : "",
|
|
420
|
+
typeof officeTableBodyRowCount === "number" ? `Office table ${officeTableBodyRowCount} body rows` : "",
|
|
421
|
+
typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number" ? officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table row ${officeTableBodyRowStart}` : `Office table rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}` : "",
|
|
422
|
+
officeTableContextText ? `Office table context ${officeTableContextText}` : "",
|
|
335
423
|
ocrEngine ? `OCR ${ocrEngine}` : "",
|
|
336
424
|
typeof ocrConfidence === "number" ? `Confidence ${ocrConfidence.toFixed(2)}` : "",
|
|
337
425
|
mediaKind ? `Media ${mediaKind}` : "",
|
|
@@ -495,6 +583,15 @@ var buildRAGCitations = (sources) => {
|
|
|
495
583
|
});
|
|
496
584
|
}
|
|
497
585
|
return [...unique.values()].sort((left, right) => {
|
|
586
|
+
const leftOfficeScope = getOfficeTableCitationScope(left.metadata);
|
|
587
|
+
const rightOfficeScope = getOfficeTableCitationScope(right.metadata);
|
|
588
|
+
if (left.source === right.source && leftOfficeScope && rightOfficeScope && leftOfficeScope.blockKind === rightOfficeScope.blockKind && leftOfficeScope.sectionTitle === rightOfficeScope.sectionTitle) {
|
|
589
|
+
const leftOfficePreference = getOfficeTableCitationPreference(left.metadata);
|
|
590
|
+
const rightOfficePreference = getOfficeTableCitationPreference(right.metadata);
|
|
591
|
+
if (rightOfficePreference !== leftOfficePreference) {
|
|
592
|
+
return rightOfficePreference - leftOfficePreference;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
498
595
|
if (right.score !== left.score) {
|
|
499
596
|
return right.score - left.score;
|
|
500
597
|
}
|
|
@@ -908,6 +1005,7 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
908
1005
|
const sectionKind = getContextString2(metadata.sectionKind);
|
|
909
1006
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
910
1007
|
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
1008
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
911
1009
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
912
1010
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
913
1011
|
const sheetName = getContextString2(metadata.sheetName);
|
|
@@ -918,6 +1016,12 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
918
1016
|
return `Source-aware section ${sectionPath.join(" > ")}`;
|
|
919
1017
|
}
|
|
920
1018
|
if (sectionKind === "pdf_block") {
|
|
1019
|
+
if (pdfSemanticRole === "figure_caption" && sectionTitle) {
|
|
1020
|
+
return `Source-aware PDF figure caption ${sectionTitle}`;
|
|
1021
|
+
}
|
|
1022
|
+
if (pdfSemanticRole === "figure_body" && sectionTitle) {
|
|
1023
|
+
return `Source-aware PDF figure body ${sectionTitle}`;
|
|
1024
|
+
}
|
|
921
1025
|
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
922
1026
|
return `Source-aware PDF table block ${sectionTitle}`;
|
|
923
1027
|
}
|
|
@@ -927,11 +1031,12 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
927
1031
|
return "Source-aware PDF block";
|
|
928
1032
|
}
|
|
929
1033
|
if (sectionKind === "office_block") {
|
|
930
|
-
|
|
931
|
-
|
|
1034
|
+
const officeSectionLabel = sectionPath.length > 0 ? sectionPath.join(" > ") : sectionTitle;
|
|
1035
|
+
if (officeBlockKind && officeSectionLabel) {
|
|
1036
|
+
return `Source-aware office ${officeBlockKind} block ${officeSectionLabel}`;
|
|
932
1037
|
}
|
|
933
|
-
if (
|
|
934
|
-
return `Source-aware office block ${
|
|
1038
|
+
if (officeSectionLabel) {
|
|
1039
|
+
return `Source-aware office block ${officeSectionLabel}`;
|
|
935
1040
|
}
|
|
936
1041
|
return "Source-aware office block";
|
|
937
1042
|
}
|
|
@@ -1319,6 +1424,18 @@ var formatSpreadsheetTableLabel = (tableIndex, tableCount) => {
|
|
|
1319
1424
|
}
|
|
1320
1425
|
return `Table ${tableIndex}`;
|
|
1321
1426
|
};
|
|
1427
|
+
var formatOfficeListLevelsLabel2 = (value) => {
|
|
1428
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
1429
|
+
return;
|
|
1430
|
+
}
|
|
1431
|
+
const levels = value.map((entry) => getContextNumber2(entry)).filter((entry) => typeof entry === "number").sort((left, right) => left - right);
|
|
1432
|
+
if (levels.length === 0) {
|
|
1433
|
+
return;
|
|
1434
|
+
}
|
|
1435
|
+
const minLevel = levels[0];
|
|
1436
|
+
const maxLevel = levels[levels.length - 1];
|
|
1437
|
+
return minLevel === maxLevel ? `Office list level ${minLevel}` : `Office list levels ${minLevel}-${maxLevel}`;
|
|
1438
|
+
};
|
|
1322
1439
|
var formatMediaDurationLabel2 = (value) => {
|
|
1323
1440
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1324
1441
|
return;
|
|
@@ -1330,9 +1447,18 @@ var buildContextLabel2 = (metadata) => {
|
|
|
1330
1447
|
return;
|
|
1331
1448
|
}
|
|
1332
1449
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1450
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1451
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1452
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1333
1453
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1334
1454
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1335
1455
|
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
1456
|
+
if (pdfSemanticRole === "figure_caption" && sectionTitle) {
|
|
1457
|
+
return `PDF figure caption ${sectionTitle}`;
|
|
1458
|
+
}
|
|
1459
|
+
if (pdfSemanticRole === "figure_body" && sectionTitle) {
|
|
1460
|
+
return `PDF figure body ${sectionTitle}`;
|
|
1461
|
+
}
|
|
1336
1462
|
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
1337
1463
|
return `PDF table block ${sectionTitle}`;
|
|
1338
1464
|
}
|
|
@@ -1340,13 +1466,13 @@ var buildContextLabel2 = (metadata) => {
|
|
|
1340
1466
|
return `PDF text block ${sectionTitle}`;
|
|
1341
1467
|
}
|
|
1342
1468
|
if (officeBlockKind === "table" && sectionTitle) {
|
|
1343
|
-
return `Office table block ${sectionTitle}`;
|
|
1469
|
+
return `Office table block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1344
1470
|
}
|
|
1345
1471
|
if (officeBlockKind === "list" && sectionTitle) {
|
|
1346
|
-
return `Office list block ${sectionTitle}`;
|
|
1472
|
+
return `Office list block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1347
1473
|
}
|
|
1348
1474
|
if (officeBlockKind === "paragraph" && sectionTitle) {
|
|
1349
|
-
return `Office paragraph block ${sectionTitle}`;
|
|
1475
|
+
return `Office paragraph block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1350
1476
|
}
|
|
1351
1477
|
const emailKind = getContextString2(metadata.emailKind);
|
|
1352
1478
|
if (emailKind === "attachment") {
|
|
@@ -1444,9 +1570,14 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1444
1570
|
return;
|
|
1445
1571
|
}
|
|
1446
1572
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1573
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1447
1574
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1448
1575
|
const pdfBlockNumber = getContextNumber2(metadata.pdfBlockNumber);
|
|
1576
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1577
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1449
1578
|
const officeBlockNumber = getContextNumber2(metadata.officeBlockNumber);
|
|
1579
|
+
const officeTableBodyRowStart = getContextNumber2(metadata.officeTableBodyRowStart);
|
|
1580
|
+
const officeTableBodyRowEnd = getContextNumber2(metadata.officeTableBodyRowEnd);
|
|
1450
1581
|
const spreadsheetRowStart = getContextNumber2(metadata.spreadsheetRowStart);
|
|
1451
1582
|
const spreadsheetRowEnd = getContextNumber2(metadata.spreadsheetRowEnd);
|
|
1452
1583
|
const slideTitle = getContextString2(metadata.slideTitle);
|
|
@@ -1457,7 +1588,16 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1457
1588
|
if (page && region) {
|
|
1458
1589
|
return `Page ${page} \xB7 Region ${region}`;
|
|
1459
1590
|
}
|
|
1591
|
+
if (page && pdfBlockNumber && pdfSemanticRole === "figure_caption") {
|
|
1592
|
+
return `Page ${page} \xB7 Figure Caption ${pdfBlockNumber}`;
|
|
1593
|
+
}
|
|
1594
|
+
if (page && pdfBlockNumber && pdfSemanticRole === "figure_body") {
|
|
1595
|
+
return `Page ${page} \xB7 Figure Body ${pdfBlockNumber}`;
|
|
1596
|
+
}
|
|
1460
1597
|
if (page && pdfBlockNumber && pdfTextKind === "table_like") {
|
|
1598
|
+
if (typeof pdfTableBodyRowStart === "number" && typeof pdfTableBodyRowEnd === "number") {
|
|
1599
|
+
return pdfTableBodyRowStart === pdfTableBodyRowEnd ? `Page ${page} \xB7 Table Block ${pdfBlockNumber} \xB7 Row ${pdfTableBodyRowStart}` : `Page ${page} \xB7 Table Block ${pdfBlockNumber} \xB7 Rows ${pdfTableBodyRowStart}-${pdfTableBodyRowEnd}`;
|
|
1600
|
+
}
|
|
1461
1601
|
return `Page ${page} \xB7 Table Block ${pdfBlockNumber}`;
|
|
1462
1602
|
}
|
|
1463
1603
|
if (page && pdfBlockNumber) {
|
|
@@ -1520,6 +1660,9 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1520
1660
|
return `Timestamp ${mediaStart}`;
|
|
1521
1661
|
}
|
|
1522
1662
|
if (officeBlockNumber && officeBlockKind === "table") {
|
|
1663
|
+
if (typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number") {
|
|
1664
|
+
return officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table block ${officeBlockNumber} \xB7 Row ${officeTableBodyRowStart}` : `Office table block ${officeBlockNumber} \xB7 Rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}`;
|
|
1665
|
+
}
|
|
1523
1666
|
return `Office table block ${officeBlockNumber}`;
|
|
1524
1667
|
}
|
|
1525
1668
|
if (officeBlockNumber && officeBlockKind === "list") {
|
|
@@ -1556,11 +1699,27 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1556
1699
|
const mediaSegmentWindowDurationLabel = formatMediaDurationLabel2(metadata.mediaSegmentGroupDurationMs);
|
|
1557
1700
|
const mediaSegmentGapLabel = formatMediaDurationLabel2(metadata.mediaSegmentGapFromPreviousMs);
|
|
1558
1701
|
const spreadsheetHeaders = getSpreadsheetHeaders(metadata);
|
|
1702
|
+
const pdfTableHeaders = Array.isArray(metadata.pdfTableHeaders) ? metadata.pdfTableHeaders.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1703
|
+
const pdfTableColumnCount = getContextNumber2(metadata.pdfTableColumnCount);
|
|
1704
|
+
const pdfTableBodyRowCount = getContextNumber2(metadata.pdfTableBodyRowCount);
|
|
1559
1705
|
const spreadsheetColumnRange = formatSpreadsheetColumnRange(getContextString2(metadata.spreadsheetColumnStart), getContextString2(metadata.spreadsheetColumnEnd));
|
|
1560
1706
|
const slideNotesText = getContextString2(metadata.slideNotesText);
|
|
1561
1707
|
const pdfTextMode = getContextString2(metadata.pdfTextMode);
|
|
1708
|
+
const pdfEvidenceMode = getContextString2(metadata.pdfEvidenceMode);
|
|
1709
|
+
const pdfEvidenceOrigin = getContextString2(metadata.pdfEvidenceOrigin);
|
|
1710
|
+
const pdfEvidenceSupplement = getContextString2(metadata.pdfEvidenceSupplement);
|
|
1562
1711
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1712
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1563
1713
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1714
|
+
const officeListContextText = getContextString2(metadata.officeListContextText);
|
|
1715
|
+
const officeListGroupItemCount = getContextNumber2(metadata.officeListGroupItemCount);
|
|
1716
|
+
const officeListLevelsLabel = formatOfficeListLevelsLabel2(metadata.officeListLevels);
|
|
1717
|
+
const officeTableHeaders = Array.isArray(metadata.officeTableHeaders) ? metadata.officeTableHeaders.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1718
|
+
const officeTableColumnCount = getContextNumber2(metadata.officeTableColumnCount);
|
|
1719
|
+
const officeTableBodyRowCount = getContextNumber2(metadata.officeTableBodyRowCount);
|
|
1720
|
+
const officeTableBodyRowStart = getContextNumber2(metadata.officeTableBodyRowStart);
|
|
1721
|
+
const officeTableBodyRowEnd = getContextNumber2(metadata.officeTableBodyRowEnd);
|
|
1722
|
+
const officeTableContextText = getContextString2(metadata.officeTableContextText);
|
|
1564
1723
|
const ocrEngine = getContextString2(metadata.ocrEngine);
|
|
1565
1724
|
const extractorRegistryMatch = getContextString2(metadata.extractorRegistryMatch);
|
|
1566
1725
|
const chunkingProfile = getContextString2(metadata.chunkingProfile);
|
|
@@ -1576,10 +1735,19 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1576
1735
|
const ocrMinConfidence = getContextNumber2(metadata.ocrPageMinConfidence) ?? getContextNumber2(metadata.ocrMinConfidence);
|
|
1577
1736
|
const ocrMaxConfidence = getContextNumber2(metadata.ocrPageMaxConfidence) ?? getContextNumber2(metadata.ocrMaxConfidence);
|
|
1578
1737
|
const ocrRegionCount = getContextNumber2(metadata.ocrRegionCount);
|
|
1738
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1739
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1579
1740
|
const labels = [
|
|
1580
1741
|
pdfTextMode ? `PDF ${pdfTextMode}` : "",
|
|
1581
|
-
|
|
1742
|
+
pdfEvidenceMode ? `PDF evidence ${pdfEvidenceMode}` : "",
|
|
1743
|
+
pdfEvidenceOrigin ? `PDF origin ${pdfEvidenceOrigin}` : "",
|
|
1744
|
+
pdfEvidenceSupplement ? `PDF supplement ${pdfEvidenceSupplement}` : "",
|
|
1745
|
+
pdfSemanticRole === "figure_caption" ? "PDF figure caption" : "",
|
|
1746
|
+
pdfSemanticRole === "figure_body" ? "PDF figure body" : "",
|
|
1747
|
+
pdfSemanticRole === "figure_caption" ? "" : pdfSemanticRole === "figure_body" ? "" : pdfTextKind === "table_like" ? "PDF table block" : pdfTextKind === "paragraph" ? "PDF text block" : "",
|
|
1582
1748
|
officeBlockKind ? `Office ${officeBlockKind}` : "",
|
|
1749
|
+
typeof officeListGroupItemCount === "number" ? `Office list ${officeListGroupItemCount} items` : "",
|
|
1750
|
+
officeListLevelsLabel ?? "",
|
|
1583
1751
|
ocrEngine ? `OCR ${ocrEngine}` : "",
|
|
1584
1752
|
extractorRegistryMatch ? `Extractor ${extractorRegistryMatch}` : "",
|
|
1585
1753
|
chunkingProfile ? `Chunking ${chunkingProfile}` : "",
|
|
@@ -1589,6 +1757,16 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1589
1757
|
typeof ocrAverageConfidence === "number" && ocrAverageConfidence !== ocrConfidence ? `Average ${ocrAverageConfidence.toFixed(2)}` : "",
|
|
1590
1758
|
typeof ocrMinConfidence === "number" && typeof ocrMaxConfidence === "number" && ocrMinConfidence !== ocrMaxConfidence ? `Range ${ocrMinConfidence.toFixed(2)}-${ocrMaxConfidence.toFixed(2)}` : "",
|
|
1591
1759
|
typeof ocrRegionCount === "number" ? `${ocrRegionCount} regions` : "",
|
|
1760
|
+
pdfTableHeaders.length > 0 ? `PDF table ${pdfTableHeaders.join(", ")}` : "",
|
|
1761
|
+
typeof pdfTableColumnCount === "number" ? `PDF table ${pdfTableColumnCount} cols` : "",
|
|
1762
|
+
typeof pdfTableBodyRowCount === "number" ? `PDF table ${pdfTableBodyRowCount} body rows` : "",
|
|
1763
|
+
typeof pdfTableBodyRowStart === "number" && typeof pdfTableBodyRowEnd === "number" ? pdfTableBodyRowStart === pdfTableBodyRowEnd ? `PDF table row ${pdfTableBodyRowStart}` : `PDF table rows ${pdfTableBodyRowStart}-${pdfTableBodyRowEnd}` : "",
|
|
1764
|
+
officeListContextText ? `Office list context ${officeListContextText}` : "",
|
|
1765
|
+
officeTableHeaders.length > 0 ? `Office table ${officeTableHeaders.join(", ")}` : "",
|
|
1766
|
+
typeof officeTableColumnCount === "number" ? `Office table ${officeTableColumnCount} cols` : "",
|
|
1767
|
+
typeof officeTableBodyRowCount === "number" ? `Office table ${officeTableBodyRowCount} body rows` : "",
|
|
1768
|
+
typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number" ? officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table row ${officeTableBodyRowStart}` : `Office table rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}` : "",
|
|
1769
|
+
officeTableContextText ? `Office table context ${officeTableContextText}` : "",
|
|
1592
1770
|
spreadsheetHeaders.length > 0 ? `Spreadsheet ${spreadsheetHeaders.join(", ")}` : "",
|
|
1593
1771
|
spreadsheetColumnRange ? `Spreadsheet ${spreadsheetColumnRange}` : "",
|
|
1594
1772
|
spreadsheetTableLabel ? `Spreadsheet ${spreadsheetTableLabel}` : "",
|
|
@@ -2020,12 +2198,92 @@ var getStructuredSectionScoreWeight = (metadata) => {
|
|
|
2020
2198
|
return 1;
|
|
2021
2199
|
};
|
|
2022
2200
|
var getStructuredSourceLeadScore = (source) => source.score * getStructuredSectionScoreWeight(source.metadata);
|
|
2201
|
+
var getPDFLeadEvidencePreference = (metadata) => {
|
|
2202
|
+
if (!metadata) {
|
|
2203
|
+
return 0;
|
|
2204
|
+
}
|
|
2205
|
+
const pdfEvidenceMode = getContextString2(metadata.pdfEvidenceMode);
|
|
2206
|
+
const pdfEvidenceOrigin = getContextString2(metadata.pdfEvidenceOrigin);
|
|
2207
|
+
const pdfEvidenceSupplement = getContextString2(metadata.pdfEvidenceSupplement);
|
|
2208
|
+
if (pdfEvidenceMode === "hybrid" && pdfEvidenceOrigin === "native" && pdfEvidenceSupplement === "ocr") {
|
|
2209
|
+
return 3;
|
|
2210
|
+
}
|
|
2211
|
+
if (pdfEvidenceMode === "native" && pdfEvidenceOrigin === "native") {
|
|
2212
|
+
return 2;
|
|
2213
|
+
}
|
|
2214
|
+
if (pdfEvidenceMode === "ocr" && pdfEvidenceOrigin === "ocr") {
|
|
2215
|
+
return 1;
|
|
2216
|
+
}
|
|
2217
|
+
return 0;
|
|
2218
|
+
};
|
|
2219
|
+
var getPDFLeadScope = (metadata) => {
|
|
2220
|
+
if (!metadata) {
|
|
2221
|
+
return;
|
|
2222
|
+
}
|
|
2223
|
+
const pageNumber = getContextNumber2(metadata.pageNumber) ?? getContextNumber2(metadata.page) ?? (typeof metadata.pageIndex === "number" ? metadata.pageIndex + 1 : undefined);
|
|
2224
|
+
const sectionTitle = getContextString2(metadata.sectionTitle);
|
|
2225
|
+
const sourceNativeKind = getContextString2(metadata.sourceNativeKind);
|
|
2226
|
+
if (typeof pageNumber !== "number" && !sectionTitle && !sourceNativeKind) {
|
|
2227
|
+
return;
|
|
2228
|
+
}
|
|
2229
|
+
return {
|
|
2230
|
+
pageNumber,
|
|
2231
|
+
sectionTitle,
|
|
2232
|
+
sourceNativeKind
|
|
2233
|
+
};
|
|
2234
|
+
};
|
|
2235
|
+
var getOfficeLeadScope = (metadata) => {
|
|
2236
|
+
if (!metadata) {
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
2240
|
+
if (officeBlockKind !== "table" && officeBlockKind !== "list") {
|
|
2241
|
+
return;
|
|
2242
|
+
}
|
|
2243
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
2244
|
+
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
2245
|
+
const officeContextText = officeBlockKind === "table" ? getContextString2(metadata.officeTableContextText) : getContextString2(metadata.officeListContextText);
|
|
2246
|
+
if (!sectionTitle) {
|
|
2247
|
+
return;
|
|
2248
|
+
}
|
|
2249
|
+
return {
|
|
2250
|
+
blockKind: officeBlockKind,
|
|
2251
|
+
pathDepth: sectionPath.length,
|
|
2252
|
+
sectionTitle,
|
|
2253
|
+
hasContext: typeof officeContextText === "string"
|
|
2254
|
+
};
|
|
2255
|
+
};
|
|
2256
|
+
var getOfficeLeadEvidencePreference = (metadata) => {
|
|
2257
|
+
const scope = getOfficeLeadScope(metadata);
|
|
2258
|
+
if (!scope) {
|
|
2259
|
+
return 0;
|
|
2260
|
+
}
|
|
2261
|
+
return scope.pathDepth * 10 + (scope.hasContext ? 1 : 0) + (scope.blockKind === "list" && typeof metadata?.officeListGroupItemCount === "number" && metadata.officeListGroupItemCount > 1 ? 1 : 0);
|
|
2262
|
+
};
|
|
2023
2263
|
var getPreferredSourceLeadChunk = (chunks) => chunks.slice().sort((left, right) => {
|
|
2264
|
+
const leftOfficeScope = getOfficeLeadScope(left.metadata);
|
|
2265
|
+
const rightOfficeScope = getOfficeLeadScope(right.metadata);
|
|
2266
|
+
if (left.source === right.source && leftOfficeScope && rightOfficeScope && leftOfficeScope.blockKind === rightOfficeScope.blockKind && leftOfficeScope.sectionTitle === rightOfficeScope.sectionTitle) {
|
|
2267
|
+
const leftOfficePreference = getOfficeLeadEvidencePreference(left.metadata);
|
|
2268
|
+
const rightOfficePreference = getOfficeLeadEvidencePreference(right.metadata);
|
|
2269
|
+
if (rightOfficePreference !== leftOfficePreference) {
|
|
2270
|
+
return rightOfficePreference - leftOfficePreference;
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2024
2273
|
const leftWeightedScore = getStructuredSourceLeadScore(left);
|
|
2025
2274
|
const rightWeightedScore = getStructuredSourceLeadScore(right);
|
|
2026
2275
|
if (rightWeightedScore !== leftWeightedScore) {
|
|
2027
2276
|
return rightWeightedScore - leftWeightedScore;
|
|
2028
2277
|
}
|
|
2278
|
+
const leftScope = getPDFLeadScope(left.metadata);
|
|
2279
|
+
const rightScope = getPDFLeadScope(right.metadata);
|
|
2280
|
+
if (left.source === right.source && leftScope && rightScope && (leftScope.sectionTitle && rightScope.sectionTitle && leftScope.sectionTitle === rightScope.sectionTitle || typeof leftScope.pageNumber === "number" && typeof rightScope.pageNumber === "number" && leftScope.pageNumber === rightScope.pageNumber)) {
|
|
2281
|
+
const leftEvidencePreference = getPDFLeadEvidencePreference(left.metadata);
|
|
2282
|
+
const rightEvidencePreference = getPDFLeadEvidencePreference(right.metadata);
|
|
2283
|
+
if (rightEvidencePreference !== leftEvidencePreference) {
|
|
2284
|
+
return rightEvidencePreference - leftEvidencePreference;
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2029
2287
|
if (right.score !== left.score) {
|
|
2030
2288
|
return right.score - left.score;
|
|
2031
2289
|
}
|
|
@@ -2279,6 +2537,7 @@ var buildRAGSectionRetrievalDiagnostics = (sources, trace) => {
|
|
|
2279
2537
|
queryTransformProvider: trace?.queryTransformProvider,
|
|
2280
2538
|
queryTransformReason: trace?.queryTransformReason,
|
|
2281
2539
|
reasons,
|
|
2540
|
+
evidenceReconcileApplied: trace?.steps.some((step) => step.stage === "evidence_reconcile"),
|
|
2282
2541
|
rerankApplied: trace?.steps.some((step) => step.stage === "rerank" && step.metadata?.applied === true),
|
|
2283
2542
|
scoreShare,
|
|
2284
2543
|
scoreThresholdApplied: trace?.steps.some((step) => step.stage === "score_filter"),
|
|
@@ -2957,6 +3216,12 @@ var buildComparisonOverviewPresentation = (input) => {
|
|
|
2957
3216
|
value: input.resolveLabel(input.summary.bestByMultivectorVectorHitCases)
|
|
2958
3217
|
});
|
|
2959
3218
|
}
|
|
3219
|
+
if (input.summary.bestByEvidenceReconcileCases) {
|
|
3220
|
+
rows.push({
|
|
3221
|
+
label: "Best evidence reconcile",
|
|
3222
|
+
value: input.resolveLabel(input.summary.bestByEvidenceReconcileCases)
|
|
3223
|
+
});
|
|
3224
|
+
}
|
|
2960
3225
|
if (input.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases) {
|
|
2961
3226
|
rows.push({
|
|
2962
3227
|
label: "Lowest runtime budget exhaustion",
|
|
@@ -3024,6 +3289,9 @@ var buildRAGComparisonTraceSummaryRows = (entry) => {
|
|
|
3024
3289
|
}, {
|
|
3025
3290
|
label: "Runtime",
|
|
3026
3291
|
value: `budget ${formatTraceRatio(trace.runtimeCandidateBudgetExhaustedCases, trace.totalCases)} \xB7 underfilled ${formatTraceRatio(trace.runtimeUnderfilledTopKCases, trace.totalCases)}`
|
|
3292
|
+
}, {
|
|
3293
|
+
label: "Evidence reconcile",
|
|
3294
|
+
value: `all ${formatTraceRatio(trace.stageCounts.evidence_reconcile ?? 0, trace.totalCases)} \xB7 office ${formatTraceRatio(trace.officeEvidenceReconcileCases, trace.totalCases)} \xB7 pdf ${formatTraceRatio(trace.pdfEvidenceReconcileCases, trace.totalCases)}`
|
|
3027
3295
|
}, {
|
|
3028
3296
|
label: "TopK",
|
|
3029
3297
|
value: `${trace.averageCandidateTopK.toFixed(1)} / ${trace.averageLexicalTopK.toFixed(1)}`
|
|
@@ -3381,6 +3649,25 @@ var buildRAGEvaluationHistoryRows = (history) => {
|
|
|
3381
3649
|
label: "Trace variant delta",
|
|
3382
3650
|
value: formatTraceCountDelta(history.diff.traceSummaryDelta.variantCases)
|
|
3383
3651
|
});
|
|
3652
|
+
const evidenceReconcileDelta = history.diff.traceSummaryDelta.stageCounts?.evidence_reconcile;
|
|
3653
|
+
if (typeof evidenceReconcileDelta === "number") {
|
|
3654
|
+
rows.push({
|
|
3655
|
+
label: "Trace evidence reconcile delta",
|
|
3656
|
+
value: formatTraceCountDelta(evidenceReconcileDelta)
|
|
3657
|
+
});
|
|
3658
|
+
}
|
|
3659
|
+
if (typeof history.diff.traceSummaryDelta.officeEvidenceReconcileCasesDelta === "number") {
|
|
3660
|
+
rows.push({
|
|
3661
|
+
label: "Trace office evidence reconcile delta",
|
|
3662
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.officeEvidenceReconcileCasesDelta)
|
|
3663
|
+
});
|
|
3664
|
+
}
|
|
3665
|
+
if (typeof history.diff.traceSummaryDelta.pdfEvidenceReconcileCasesDelta === "number") {
|
|
3666
|
+
rows.push({
|
|
3667
|
+
label: "Trace PDF evidence reconcile delta",
|
|
3668
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.pdfEvidenceReconcileCasesDelta)
|
|
3669
|
+
});
|
|
3670
|
+
}
|
|
3384
3671
|
const stageDelta = Object.entries(history.diff.traceSummaryDelta.stageCounts ?? {}).map(([stage, count]) => `${stage} ${formatTraceCountDelta(count)}`).join(", ");
|
|
3385
3672
|
if (stageDelta) {
|
|
3386
3673
|
rows.push({ label: "Trace stage delta", value: stageDelta });
|
|
@@ -3586,6 +3873,7 @@ var buildRAGEvaluationSuiteSnapshotHistoryPresentation = (history) => ({
|
|
|
3586
3873
|
summary: history?.latestSnapshot ? `v${history.latestSnapshot.version}` : "No saved suite snapshots yet."
|
|
3587
3874
|
});
|
|
3588
3875
|
var isRuntimeGateReason = (reason) => /runtime|candidate-budget|underfilled/i.test(reason);
|
|
3876
|
+
var getFixtureVariantsFromRunTags = (tags) => (tags ?? []).filter((tag) => tag.startsWith("fixture:")).map((tag) => tag.slice("fixture:".length)).filter((tag, index, all) => tag.length > 0 && all.indexOf(tag) === index);
|
|
3589
3877
|
var buildRAGRetrievalReleaseHistoryRunPresentation = (run) => {
|
|
3590
3878
|
const runtimeGateReasons = (run.decisionSummary?.gate?.reasons ?? run.releaseVerdict?.gate?.reasons ?? []).filter(isRuntimeGateReason);
|
|
3591
3879
|
const rows = [
|
|
@@ -3599,6 +3887,13 @@ var buildRAGRetrievalReleaseHistoryRunPresentation = (run) => {
|
|
|
3599
3887
|
value: run.comparison.summary.bestByAverageF1 ?? "n/a"
|
|
3600
3888
|
}
|
|
3601
3889
|
];
|
|
3890
|
+
const fixtureVariants = getFixtureVariantsFromRunTags(run.tags);
|
|
3891
|
+
if (fixtureVariants.length > 0) {
|
|
3892
|
+
rows.push({
|
|
3893
|
+
label: "Fixture variant",
|
|
3894
|
+
value: fixtureVariants.join(", ")
|
|
3895
|
+
});
|
|
3896
|
+
}
|
|
3602
3897
|
if (run.comparison.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases) {
|
|
3603
3898
|
rows.push({
|
|
3604
3899
|
label: "Lowest runtime budget exhaustion",
|
|
@@ -3627,6 +3922,7 @@ var buildRAGRetrievalReleaseHistoryRunPresentation = (run) => {
|
|
|
3627
3922
|
};
|
|
3628
3923
|
var buildRAGRetrievalReleaseGroupHistoryPresentation = (input) => {
|
|
3629
3924
|
const recentRuns = (input.runs ?? []).map(buildRAGRetrievalReleaseHistoryRunPresentation);
|
|
3925
|
+
const fixtureVariants = (input.runs ?? []).flatMap((run) => getFixtureVariantsFromRunTags(run.tags)).filter((tag, index, all) => all.indexOf(tag) === index);
|
|
3630
3926
|
const runtimeBlockedRuns = recentRuns.filter((entry) => entry.rows.some((row) => row.label === "Runtime gate failures" && row.value !== "none")).length;
|
|
3631
3927
|
const rows = [
|
|
3632
3928
|
{
|
|
@@ -3650,6 +3946,12 @@ var buildRAGRetrievalReleaseGroupHistoryPresentation = (input) => {
|
|
|
3650
3946
|
value: String(runtimeBlockedRuns)
|
|
3651
3947
|
}
|
|
3652
3948
|
];
|
|
3949
|
+
if (fixtureVariants.length > 0) {
|
|
3950
|
+
rows.push({
|
|
3951
|
+
label: "Fixture variants",
|
|
3952
|
+
value: fixtureVariants.join(", ")
|
|
3953
|
+
});
|
|
3954
|
+
}
|
|
3653
3955
|
return {
|
|
3654
3956
|
recentRuns,
|
|
3655
3957
|
rows,
|
|
@@ -4166,6 +4468,9 @@ var evaluateRetrievalComparisonGate = ({
|
|
|
4166
4468
|
if (typeof policy.minMultiVectorVectorHitCasesDelta === "number" && (delta.multiVectorVectorHitCasesDelta ?? 0) < policy.minMultiVectorVectorHitCasesDelta) {
|
|
4167
4469
|
reasons.push(`multivector vector-hit delta ${delta.multiVectorVectorHitCasesDelta ?? 0} is below ${policy.minMultiVectorVectorHitCasesDelta}`);
|
|
4168
4470
|
}
|
|
4471
|
+
if (typeof policy.minEvidenceReconcileCasesDelta === "number" && (delta.evidenceReconcileCasesDelta ?? 0) < policy.minEvidenceReconcileCasesDelta) {
|
|
4472
|
+
reasons.push(`evidence reconcile delta ${delta.evidenceReconcileCasesDelta ?? 0} is below ${policy.minEvidenceReconcileCasesDelta}`);
|
|
4473
|
+
}
|
|
4169
4474
|
if (typeof policy.maxRuntimeCandidateBudgetExhaustedCasesDelta === "number" && (delta.runtimeCandidateBudgetExhaustedCasesDelta ?? 0) > policy.maxRuntimeCandidateBudgetExhaustedCasesDelta) {
|
|
4170
4475
|
reasons.push(`runtime candidate-budget-exhausted delta ${delta.runtimeCandidateBudgetExhaustedCasesDelta ?? 0} exceeds ${policy.maxRuntimeCandidateBudgetExhaustedCasesDelta}`);
|
|
4171
4476
|
}
|
|
@@ -4219,13 +4524,14 @@ var buildRAGRetrievalReleaseVerdict = ({
|
|
|
4219
4524
|
};
|
|
4220
4525
|
}
|
|
4221
4526
|
if (delta) {
|
|
4527
|
+
const requiresReview = delta.passingRateDelta < 0 || delta.averageF1Delta < 0 || (delta.evidenceReconcileCasesDelta ?? 0) < 0;
|
|
4222
4528
|
return {
|
|
4223
4529
|
baselineGroupKey: groupKey,
|
|
4224
4530
|
baselineRetrievalId,
|
|
4225
4531
|
candidateRetrievalId,
|
|
4226
4532
|
delta,
|
|
4227
|
-
status:
|
|
4228
|
-
summary:
|
|
4533
|
+
status: requiresReview ? "needs_review" : "pass",
|
|
4534
|
+
summary: requiresReview ? "Candidate should be reviewed before promotion." : "Candidate improved or matched the baseline."
|
|
4229
4535
|
};
|
|
4230
4536
|
}
|
|
4231
4537
|
return {
|
|
@@ -4906,6 +5212,8 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4906
5212
|
let multiVectorVectorHitCases = 0;
|
|
4907
5213
|
let multiVectorLexicalHitCases = 0;
|
|
4908
5214
|
let multiVectorCollapsedCases = 0;
|
|
5215
|
+
let officeEvidenceReconcileCases = 0;
|
|
5216
|
+
let pdfEvidenceReconcileCases = 0;
|
|
4909
5217
|
let runtimeCandidateBudgetExhaustedCases = 0;
|
|
4910
5218
|
let runtimeUnderfilledTopKCases = 0;
|
|
4911
5219
|
let finalCountSum = 0;
|
|
@@ -4947,6 +5255,13 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4947
5255
|
if ((trace.multiVector?.collapsedParents ?? 0) > 0) {
|
|
4948
5256
|
multiVectorCollapsedCases += 1;
|
|
4949
5257
|
}
|
|
5258
|
+
const evidenceReconcileMetadata = trace.steps.find((step) => step.stage === "evidence_reconcile")?.metadata;
|
|
5259
|
+
if (typeof evidenceReconcileMetadata?.officeAffectedScopes === "number" && evidenceReconcileMetadata.officeAffectedScopes > 0) {
|
|
5260
|
+
officeEvidenceReconcileCases += 1;
|
|
5261
|
+
}
|
|
5262
|
+
if (typeof evidenceReconcileMetadata?.pdfAffectedScopes === "number" && evidenceReconcileMetadata.pdfAffectedScopes > 0) {
|
|
5263
|
+
pdfEvidenceReconcileCases += 1;
|
|
5264
|
+
}
|
|
4950
5265
|
if (vectorSearchMetadata?.sqliteQueryCandidateBudgetExhausted) {
|
|
4951
5266
|
runtimeCandidateBudgetExhaustedCases += 1;
|
|
4952
5267
|
}
|
|
@@ -4984,6 +5299,8 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4984
5299
|
multiVectorVectorHitCases,
|
|
4985
5300
|
multiVectorLexicalHitCases,
|
|
4986
5301
|
multiVectorCollapsedCases,
|
|
5302
|
+
officeEvidenceReconcileCases,
|
|
5303
|
+
pdfEvidenceReconcileCases,
|
|
4987
5304
|
runtimeCandidateBudgetExhaustedCases,
|
|
4988
5305
|
runtimeUnderfilledTopKCases,
|
|
4989
5306
|
vectorCases
|
|
@@ -5788,6 +6105,8 @@ var buildRAGEvaluationRunDiff = ({
|
|
|
5788
6105
|
averageLexicalTopK: (current.traceSummary?.averageLexicalTopK ?? 0) - (previous?.traceSummary?.averageLexicalTopK ?? 0),
|
|
5789
6106
|
averageVectorCount: (current.traceSummary?.averageVectorCount ?? 0) - (previous?.traceSummary?.averageVectorCount ?? 0),
|
|
5790
6107
|
balancedCases: (current.traceSummary?.balancedCases ?? 0) - (previous?.traceSummary?.balancedCases ?? 0),
|
|
6108
|
+
officeEvidenceReconcileCasesDelta: (current.traceSummary?.officeEvidenceReconcileCases ?? 0) - (previous?.traceSummary?.officeEvidenceReconcileCases ?? 0),
|
|
6109
|
+
pdfEvidenceReconcileCasesDelta: (current.traceSummary?.pdfEvidenceReconcileCases ?? 0) - (previous?.traceSummary?.pdfEvidenceReconcileCases ?? 0),
|
|
5791
6110
|
lexicalCases: (current.traceSummary?.lexicalCases ?? 0) - (previous?.traceSummary?.lexicalCases ?? 0),
|
|
5792
6111
|
modesChanged: (current.traceSummary?.modes ?? []).join("|") !== (previous?.traceSummary?.modes ?? []).join("|"),
|
|
5793
6112
|
roundRobinCases: (current.traceSummary?.roundRobinCases ?? 0) - (previous?.traceSummary?.roundRobinCases ?? 0),
|
|
@@ -8572,6 +8891,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8572
8891
|
multiVectorCollapsedCasesDelta: (candidateEntry.traceSummary?.multiVectorCollapsedCases ?? 0) - (baselineEntry.traceSummary?.multiVectorCollapsedCases ?? 0),
|
|
8573
8892
|
multiVectorLexicalHitCasesDelta: (candidateEntry.traceSummary?.multiVectorLexicalHitCases ?? 0) - (baselineEntry.traceSummary?.multiVectorLexicalHitCases ?? 0),
|
|
8574
8893
|
multiVectorVectorHitCasesDelta: (candidateEntry.traceSummary?.multiVectorVectorHitCases ?? 0) - (baselineEntry.traceSummary?.multiVectorVectorHitCases ?? 0),
|
|
8894
|
+
evidenceReconcileCasesDelta: (candidateEntry.traceSummary?.stageCounts?.evidence_reconcile ?? 0) - (baselineEntry.traceSummary?.stageCounts?.evidence_reconcile ?? 0),
|
|
8575
8895
|
runtimeCandidateBudgetExhaustedCasesDelta: (candidateEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases ?? 0) - (baselineEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases ?? 0),
|
|
8576
8896
|
runtimeUnderfilledTopKCasesDelta: (candidateEntry.traceSummary?.runtimeUnderfilledTopKCases ?? 0) - (baselineEntry.traceSummary?.runtimeUnderfilledTopKCases ?? 0)
|
|
8577
8897
|
} : undefined;
|
|
@@ -8583,6 +8903,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8583
8903
|
multiVectorCollapsedCases: baselineEntry.traceSummary?.multiVectorCollapsedCases,
|
|
8584
8904
|
multiVectorLexicalHitCases: baselineEntry.traceSummary?.multiVectorLexicalHitCases,
|
|
8585
8905
|
multiVectorVectorHitCases: baselineEntry.traceSummary?.multiVectorVectorHitCases,
|
|
8906
|
+
evidenceReconcileCases: baselineEntry.traceSummary?.stageCounts?.evidence_reconcile,
|
|
8586
8907
|
runtimeCandidateBudgetExhaustedCases: baselineEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases,
|
|
8587
8908
|
runtimeUnderfilledTopKCases: baselineEntry.traceSummary?.runtimeUnderfilledTopKCases,
|
|
8588
8909
|
passingRate: baselineEntry.response.passingRate,
|
|
@@ -8596,6 +8917,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8596
8917
|
multiVectorCollapsedCases: candidateEntry.traceSummary?.multiVectorCollapsedCases,
|
|
8597
8918
|
multiVectorLexicalHitCases: candidateEntry.traceSummary?.multiVectorLexicalHitCases,
|
|
8598
8919
|
multiVectorVectorHitCases: candidateEntry.traceSummary?.multiVectorVectorHitCases,
|
|
8920
|
+
evidenceReconcileCases: candidateEntry.traceSummary?.stageCounts?.evidence_reconcile,
|
|
8599
8921
|
runtimeCandidateBudgetExhaustedCases: candidateEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases,
|
|
8600
8922
|
runtimeUnderfilledTopKCases: candidateEntry.traceSummary?.runtimeUnderfilledTopKCases,
|
|
8601
8923
|
passingRate: candidateEntry.response.passingRate,
|
|
@@ -8610,6 +8932,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8610
8932
|
winnerByMultivectorCollapsedCases: comparison.summary.bestByMultivectorCollapsedCases,
|
|
8611
8933
|
winnerByMultivectorLexicalHitCases: comparison.summary.bestByMultivectorLexicalHitCases,
|
|
8612
8934
|
winnerByMultivectorVectorHitCases: comparison.summary.bestByMultivectorVectorHitCases,
|
|
8935
|
+
winnerByEvidenceReconcileCases: comparison.summary.bestByEvidenceReconcileCases,
|
|
8613
8936
|
winnerByLowestRuntimeCandidateBudgetExhaustedCases: comparison.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases,
|
|
8614
8937
|
winnerByLowestRuntimeUnderfilledTopKCases: comparison.summary.bestByLowestRuntimeUnderfilledTopKCases
|
|
8615
8938
|
};
|
|
@@ -9317,6 +9640,27 @@ var selectComparisonEntryByLowestTraceMetric = (entries, idKey, metric) => {
|
|
|
9317
9640
|
const winner = ranked[0];
|
|
9318
9641
|
return typeof winner?.[idKey] === "string" ? winner[idKey] : undefined;
|
|
9319
9642
|
};
|
|
9643
|
+
var selectComparisonEntryByTraceStageCount = (entries, idKey, stage) => {
|
|
9644
|
+
const ranked = [...entries].sort((left, right) => {
|
|
9645
|
+
const leftMetric = left.traceSummary?.stageCounts?.[stage] ?? 0;
|
|
9646
|
+
const rightMetric = right.traceSummary?.stageCounts?.[stage] ?? 0;
|
|
9647
|
+
if (rightMetric !== leftMetric) {
|
|
9648
|
+
return rightMetric - leftMetric;
|
|
9649
|
+
}
|
|
9650
|
+
if (right.response.passingRate !== left.response.passingRate) {
|
|
9651
|
+
return right.response.passingRate - left.response.passingRate;
|
|
9652
|
+
}
|
|
9653
|
+
if (right.response.summary.averageF1 !== left.response.summary.averageF1) {
|
|
9654
|
+
return right.response.summary.averageF1 - left.response.summary.averageF1;
|
|
9655
|
+
}
|
|
9656
|
+
return left.response.summary.averageLatencyMs - right.response.summary.averageLatencyMs;
|
|
9657
|
+
});
|
|
9658
|
+
const winner = ranked[0];
|
|
9659
|
+
if (!winner || (winner.traceSummary?.stageCounts?.[stage] ?? 0) === 0) {
|
|
9660
|
+
return;
|
|
9661
|
+
}
|
|
9662
|
+
return typeof winner?.[idKey] === "string" ? winner[idKey] : undefined;
|
|
9663
|
+
};
|
|
9320
9664
|
var resolveRetrievalMode = (candidate) => {
|
|
9321
9665
|
if (!candidate.retrieval) {
|
|
9322
9666
|
return "vector";
|
|
@@ -9411,6 +9755,8 @@ var compareRAGRetrievalTraceSummaries = (current, previous) => ({
|
|
|
9411
9755
|
multiVectorVectorHitCasesDelta: current.multiVectorVectorHitCases - previous.multiVectorVectorHitCases,
|
|
9412
9756
|
multiVectorLexicalHitCasesDelta: current.multiVectorLexicalHitCases - previous.multiVectorLexicalHitCases,
|
|
9413
9757
|
multiVectorCollapsedCasesDelta: current.multiVectorCollapsedCases - previous.multiVectorCollapsedCases,
|
|
9758
|
+
officeEvidenceReconcileCasesDelta: current.officeEvidenceReconcileCases - previous.officeEvidenceReconcileCases,
|
|
9759
|
+
pdfEvidenceReconcileCasesDelta: current.pdfEvidenceReconcileCases - previous.pdfEvidenceReconcileCases,
|
|
9414
9760
|
runtimeCandidateBudgetExhaustedCasesDelta: current.runtimeCandidateBudgetExhaustedCases - previous.runtimeCandidateBudgetExhaustedCases,
|
|
9415
9761
|
runtimeUnderfilledTopKCasesDelta: current.runtimeUnderfilledTopKCases - previous.runtimeUnderfilledTopKCases
|
|
9416
9762
|
});
|
|
@@ -9776,10 +10122,129 @@ var generateRAGEvaluationSuiteFromDocuments = ({
|
|
|
9776
10122
|
};
|
|
9777
10123
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_SUITE_ID = "rag-native-planner-larger-corpus";
|
|
9778
10124
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_LABEL = "Adaptive Native Planner Benchmark";
|
|
10125
|
+
var DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_SUITE_ID = "rag-native-backend-larger-corpus";
|
|
10126
|
+
var DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_LABEL = "Native Backend Comparison Benchmark";
|
|
9779
10127
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_QUERY = "Which launch checklist phrase is exact wording?";
|
|
10128
|
+
var DEFAULT_NATIVE_BACKEND_HYBRID_QUERY = "aurora promotion checklist wording";
|
|
10129
|
+
var DEFAULT_NATIVE_BACKEND_FILTERED_QUERY = "focus lane launch checklist wording";
|
|
10130
|
+
var DEFAULT_NATIVE_BACKEND_REORDERED_QUERY = "exact aurora focus lane checklist wording";
|
|
10131
|
+
var DEFAULT_NATIVE_BACKEND_GUIDE_QUERY = "which focus lane guide contains exact aurora promotion wording";
|
|
9780
10132
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER = {
|
|
9781
10133
|
lane: "focus"
|
|
9782
10134
|
};
|
|
10135
|
+
var DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS = [
|
|
10136
|
+
"focus-distractor-0",
|
|
10137
|
+
"focus-distractor-1",
|
|
10138
|
+
"focus-distractor-2"
|
|
10139
|
+
];
|
|
10140
|
+
var createRAGNativeBackendBenchmarkMockEmbedding = async (text) => {
|
|
10141
|
+
const normalized = text.toLowerCase();
|
|
10142
|
+
if (normalized.includes("launch checklist exact wording for aurora promotion") || normalized.includes("launch checklist exact wording")) {
|
|
10143
|
+
return [0.995, 0.005];
|
|
10144
|
+
}
|
|
10145
|
+
if (normalized.includes("aurora") || normalized.includes("checklist") || normalized.includes("focus lane") || normalized.includes("exact wording") || normalized.includes("guide")) {
|
|
10146
|
+
return [1, 0];
|
|
10147
|
+
}
|
|
10148
|
+
return [0, 1];
|
|
10149
|
+
};
|
|
10150
|
+
var createRAGNativeBackendBenchmarkCorpus = (input) => {
|
|
10151
|
+
const noiseCount = input?.noiseCount ?? 5001;
|
|
10152
|
+
const backend = input?.backend ?? "generic";
|
|
10153
|
+
const genericChunks = [
|
|
10154
|
+
...Array.from({ length: noiseCount }, (_, index) => ({
|
|
10155
|
+
chunkId: `noise:${index}`,
|
|
10156
|
+
corpusKey: "noise",
|
|
10157
|
+
embedding: [0, 1],
|
|
10158
|
+
metadata: {
|
|
10159
|
+
corpusKey: "noise",
|
|
10160
|
+
documentId: `noise-${index}`,
|
|
10161
|
+
lane: "noise"
|
|
10162
|
+
},
|
|
10163
|
+
source: `noise/${index}.md`,
|
|
10164
|
+
text: `Background operations note ${index}.`
|
|
10165
|
+
})),
|
|
10166
|
+
...Array.from({ length: 3 }, (_, index) => ({
|
|
10167
|
+
chunkId: `focus:distractor:${index}`,
|
|
10168
|
+
corpusKey: "focus",
|
|
10169
|
+
embedding: [1, 0],
|
|
10170
|
+
metadata: {
|
|
10171
|
+
corpusKey: "focus",
|
|
10172
|
+
documentId: `focus-distractor-${index}`,
|
|
10173
|
+
lane: "focus"
|
|
10174
|
+
},
|
|
10175
|
+
source: `focus/distractor-${index}.md`,
|
|
10176
|
+
text: index === 0 ? "aurora promotion checklist overview" : index === 1 ? "launch checklist wording draft" : "focus lane promotion runbook notes"
|
|
10177
|
+
})),
|
|
10178
|
+
{
|
|
10179
|
+
chunkId: "focus:target",
|
|
10180
|
+
corpusKey: "focus",
|
|
10181
|
+
embedding: [0.995, 0.005],
|
|
10182
|
+
metadata: {
|
|
10183
|
+
corpusKey: "focus",
|
|
10184
|
+
documentId: "focus-target",
|
|
10185
|
+
lane: "focus"
|
|
10186
|
+
},
|
|
10187
|
+
source: "guide/planner-depth.md",
|
|
10188
|
+
text: "launch checklist exact wording for aurora promotion in the focus lane"
|
|
10189
|
+
}
|
|
10190
|
+
];
|
|
10191
|
+
const backendSpecificChunks = backend === "sqlite-native" ? [
|
|
10192
|
+
{
|
|
10193
|
+
chunkId: "focus:sqlite:phrase-matrix",
|
|
10194
|
+
corpusKey: "focus",
|
|
10195
|
+
embedding: [1, 0],
|
|
10196
|
+
metadata: {
|
|
10197
|
+
backendFixture: "sqlite-native",
|
|
10198
|
+
corpusKey: "focus",
|
|
10199
|
+
documentId: "focus-sqlite-phrase-matrix",
|
|
10200
|
+
lane: "focus"
|
|
10201
|
+
},
|
|
10202
|
+
source: "guide/sqlite-phrase-matrix.md",
|
|
10203
|
+
text: "exact aurora focus lane checklist wording matrix for sqlite validation"
|
|
10204
|
+
},
|
|
10205
|
+
{
|
|
10206
|
+
chunkId: "focus:sqlite:guide-table",
|
|
10207
|
+
corpusKey: "focus",
|
|
10208
|
+
embedding: [1, 0],
|
|
10209
|
+
metadata: {
|
|
10210
|
+
backendFixture: "sqlite-native",
|
|
10211
|
+
corpusKey: "focus",
|
|
10212
|
+
documentId: "focus-sqlite-guide-table",
|
|
10213
|
+
lane: "focus"
|
|
10214
|
+
},
|
|
10215
|
+
source: "guide/sqlite-guide-table.md",
|
|
10216
|
+
text: "which focus lane guide contains aurora promotion wording draft table for sqlite operators"
|
|
10217
|
+
}
|
|
10218
|
+
] : backend === "postgres" ? [
|
|
10219
|
+
{
|
|
10220
|
+
chunkId: "focus:postgres:appendix",
|
|
10221
|
+
corpusKey: "focus",
|
|
10222
|
+
embedding: [1, 0],
|
|
10223
|
+
metadata: {
|
|
10224
|
+
backendFixture: "postgres",
|
|
10225
|
+
corpusKey: "focus",
|
|
10226
|
+
documentId: "focus-postgres-appendix",
|
|
10227
|
+
lane: "focus"
|
|
10228
|
+
},
|
|
10229
|
+
source: "guide/postgres-appendix.md",
|
|
10230
|
+
text: "which focus lane guide contains exact aurora promotion wording appendix for postgres release review"
|
|
10231
|
+
},
|
|
10232
|
+
{
|
|
10233
|
+
chunkId: "focus:postgres:alternatives",
|
|
10234
|
+
corpusKey: "focus",
|
|
10235
|
+
embedding: [1, 0],
|
|
10236
|
+
metadata: {
|
|
10237
|
+
backendFixture: "postgres",
|
|
10238
|
+
corpusKey: "focus",
|
|
10239
|
+
documentId: "focus-postgres-alternatives",
|
|
10240
|
+
lane: "focus"
|
|
10241
|
+
},
|
|
10242
|
+
source: "guide/postgres-alternatives.md",
|
|
10243
|
+
text: "aurora promotion checklist wording alternatives and exact focus lane phrasing for postgres audits"
|
|
10244
|
+
}
|
|
10245
|
+
] : [];
|
|
10246
|
+
return [...genericChunks, ...backendSpecificChunks];
|
|
10247
|
+
};
|
|
9783
10248
|
var createRAGAdaptiveNativePlannerBenchmarkSuite = (input) => createRAGEvaluationSuite({
|
|
9784
10249
|
description: input?.description ?? "Stress-tests larger-corpus native planner selection, candidate-budget pressure, and transformed-query recovery on filtered retrieval.",
|
|
9785
10250
|
id: input?.id ?? DEFAULT_NATIVE_PLANNER_BENCHMARK_SUITE_ID,
|
|
@@ -9831,6 +10296,102 @@ var createRAGAdaptiveNativePlannerBenchmarkSnapshot = (input) => {
|
|
|
9831
10296
|
version: input?.version
|
|
9832
10297
|
});
|
|
9833
10298
|
};
|
|
10299
|
+
var createRAGNativeBackendComparisonBenchmarkSuite = (input) => createRAGEvaluationSuite({
|
|
10300
|
+
description: input?.description ?? "Captures larger-corpus native backend parity with filtered vector pressure and harder hybrid retrieval cases so sqlite-native and postgres runs can be compared over time.",
|
|
10301
|
+
id: input?.id ?? DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_SUITE_ID,
|
|
10302
|
+
input: {
|
|
10303
|
+
cases: [
|
|
10304
|
+
{
|
|
10305
|
+
expectedDocumentIds: ["focus-target"],
|
|
10306
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10307
|
+
hardNegativeDocumentIds: [
|
|
10308
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10309
|
+
],
|
|
10310
|
+
id: "planner-pressure-exact-phrase",
|
|
10311
|
+
label: "Exact phrase survives larger-corpus native pressure",
|
|
10312
|
+
query: DEFAULT_NATIVE_PLANNER_BENCHMARK_QUERY,
|
|
10313
|
+
topK: input?.topK ?? 1
|
|
10314
|
+
},
|
|
10315
|
+
{
|
|
10316
|
+
expectedDocumentIds: ["focus-target"],
|
|
10317
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10318
|
+
hardNegativeDocumentIds: [
|
|
10319
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10320
|
+
],
|
|
10321
|
+
id: "planner-pressure-hybrid-phrase",
|
|
10322
|
+
label: "Hybrid retrieval survives filtered lexical pressure",
|
|
10323
|
+
query: DEFAULT_NATIVE_BACKEND_HYBRID_QUERY,
|
|
10324
|
+
topK: input?.topK ?? 1
|
|
10325
|
+
},
|
|
10326
|
+
{
|
|
10327
|
+
expectedDocumentIds: ["focus-target"],
|
|
10328
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10329
|
+
hardNegativeDocumentIds: [
|
|
10330
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10331
|
+
],
|
|
10332
|
+
id: "planner-pressure-filtered-lane-query",
|
|
10333
|
+
label: "Filtered lane query survives broader corpus noise",
|
|
10334
|
+
query: DEFAULT_NATIVE_BACKEND_FILTERED_QUERY,
|
|
10335
|
+
topK: input?.topK ?? 1
|
|
10336
|
+
},
|
|
10337
|
+
{
|
|
10338
|
+
expectedDocumentIds: ["focus-target"],
|
|
10339
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10340
|
+
hardNegativeDocumentIds: [
|
|
10341
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10342
|
+
],
|
|
10343
|
+
id: "planner-pressure-reordered-phrase",
|
|
10344
|
+
label: "Reordered phrase survives transform pressure",
|
|
10345
|
+
query: DEFAULT_NATIVE_BACKEND_REORDERED_QUERY,
|
|
10346
|
+
topK: input?.topK ?? 1
|
|
10347
|
+
},
|
|
10348
|
+
{
|
|
10349
|
+
expectedDocumentIds: ["focus-target"],
|
|
10350
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10351
|
+
hardNegativeDocumentIds: [
|
|
10352
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10353
|
+
],
|
|
10354
|
+
id: "planner-pressure-guide-query",
|
|
10355
|
+
label: "Guide attribution survives filtered corpus pressure",
|
|
10356
|
+
query: DEFAULT_NATIVE_BACKEND_GUIDE_QUERY,
|
|
10357
|
+
topK: input?.topK ?? 1
|
|
10358
|
+
}
|
|
10359
|
+
],
|
|
10360
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10361
|
+
retrieval: "vector",
|
|
10362
|
+
topK: input?.topK ?? 1
|
|
10363
|
+
},
|
|
10364
|
+
label: input?.label ?? DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_LABEL,
|
|
10365
|
+
metadata: {
|
|
10366
|
+
benchmarkKind: "native_backend_comparison",
|
|
10367
|
+
benchmarkScope: "larger_corpus",
|
|
10368
|
+
expectedSignals: [
|
|
10369
|
+
"backend-tagged runtime artifacts",
|
|
10370
|
+
"selected native planner profile",
|
|
10371
|
+
"hybrid filtered retrieval",
|
|
10372
|
+
"candidate-budget exhaustion",
|
|
10373
|
+
"underfilled topk",
|
|
10374
|
+
"query transform pressure"
|
|
10375
|
+
],
|
|
10376
|
+
recommendedGroupKey: "runtime-native-backend-parity",
|
|
10377
|
+
recommendedTags: ["runtime", "backend", "native"],
|
|
10378
|
+
...input?.metadata
|
|
10379
|
+
}
|
|
10380
|
+
});
|
|
10381
|
+
var createRAGNativeBackendComparisonBenchmarkSnapshot = (input) => {
|
|
10382
|
+
const suite = input?.suite ?? createRAGNativeBackendComparisonBenchmarkSuite();
|
|
10383
|
+
return createRAGEvaluationSuiteSnapshot({
|
|
10384
|
+
createdAt: input?.createdAt,
|
|
10385
|
+
id: input?.id,
|
|
10386
|
+
metadata: {
|
|
10387
|
+
artifactKind: "native_backend_comparison_benchmark",
|
|
10388
|
+
persistForReleaseHistory: true,
|
|
10389
|
+
...input?.metadata
|
|
10390
|
+
},
|
|
10391
|
+
suite,
|
|
10392
|
+
version: input?.version
|
|
10393
|
+
});
|
|
10394
|
+
};
|
|
9834
10395
|
var createRAGEvaluationSuiteSnapshot = ({
|
|
9835
10396
|
suite,
|
|
9836
10397
|
id,
|
|
@@ -10039,6 +10600,7 @@ var summarizeRAGRetrievalComparison = (entries) => ({
|
|
|
10039
10600
|
bestByMultivectorCollapsedCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorCollapsedCases"),
|
|
10040
10601
|
bestByMultivectorLexicalHitCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorLexicalHitCases"),
|
|
10041
10602
|
bestByMultivectorVectorHitCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorVectorHitCases"),
|
|
10603
|
+
bestByEvidenceReconcileCases: selectComparisonEntryByTraceStageCount(entries, "retrievalId", "evidence_reconcile"),
|
|
10042
10604
|
bestByLowestRuntimeCandidateBudgetExhaustedCases: selectComparisonEntryByLowestTraceMetric(entries, "retrievalId", "runtimeCandidateBudgetExhaustedCases"),
|
|
10043
10605
|
bestByLowestRuntimeUnderfilledTopKCases: selectComparisonEntryByLowestTraceMetric(entries, "retrievalId", "runtimeUnderfilledTopKCases")
|
|
10044
10606
|
});
|
|
@@ -10120,6 +10682,10 @@ export {
|
|
|
10120
10682
|
createRAGSQLiteEvaluationSuiteSnapshotHistoryStore,
|
|
10121
10683
|
createRAGSQLiteEvaluationHistoryStore,
|
|
10122
10684
|
createRAGSQLiteAnswerGroundingEvaluationHistoryStore,
|
|
10685
|
+
createRAGNativeBackendComparisonBenchmarkSuite,
|
|
10686
|
+
createRAGNativeBackendComparisonBenchmarkSnapshot,
|
|
10687
|
+
createRAGNativeBackendBenchmarkMockEmbedding,
|
|
10688
|
+
createRAGNativeBackendBenchmarkCorpus,
|
|
10123
10689
|
createRAGFileSearchTraceStore,
|
|
10124
10690
|
createRAGFileSearchTracePruneHistoryStore,
|
|
10125
10691
|
createRAGFileRetrievalReleaseLanePolicyHistoryStore,
|
|
@@ -10167,5 +10733,5 @@ export {
|
|
|
10167
10733
|
addRAGEvaluationSuiteCase
|
|
10168
10734
|
};
|
|
10169
10735
|
|
|
10170
|
-
//# debugId=
|
|
10736
|
+
//# debugId=5CB8FFAF88D0745664756E2164756E21
|
|
10171
10737
|
//# sourceMappingURL=quality.js.map
|