@absolutejs/absolute 0.19.0-beta.644 → 0.19.0-beta.646
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/client/index.js +694 -11
- package/dist/ai/client/index.js.map +6 -6
- package/dist/ai/client/ui.js +573 -11
- package/dist/ai/client/ui.js.map +5 -5
- package/dist/ai/index.js +2150 -136
- package/dist/ai/index.js.map +10 -10
- package/dist/ai/rag/quality.js +577 -11
- package/dist/ai/rag/quality.js.map +5 -5
- package/dist/ai/rag/ui.js +573 -11
- package/dist/ai/rag/ui.js.map +5 -5
- package/dist/ai-client/angular/ai/index.js +388 -8
- package/dist/ai-client/react/ai/index.js +388 -8
- package/dist/ai-client/vue/ai/index.js +388 -8
- package/dist/angular/ai/index.js +694 -11
- package/dist/angular/ai/index.js.map +6 -6
- package/dist/index.js +6 -6
- package/dist/index.js.map +2 -2
- package/dist/react/ai/index.js +694 -11
- package/dist/react/ai/index.js.map +6 -6
- package/dist/src/ai/client/ragClient.d.ts +58 -0
- package/dist/src/ai/index.d.ts +2 -1
- package/dist/src/ai/rag/chat.d.ts +90 -4
- package/dist/src/ai/rag/index.d.ts +1 -1
- package/dist/src/ai/rag/quality.d.ts +20 -1
- package/dist/src/vue/ai/useRAG.d.ts +80 -0
- package/dist/src/vue/ai/useRAGEvaluate.d.ts +70 -0
- package/dist/src/vue/ai/useRAGSearch.d.ts +10 -0
- package/dist/svelte/ai/index.js +694 -11
- package/dist/svelte/ai/index.js.map +6 -6
- package/dist/types/ai.d.ts +56 -13
- package/dist/types/index.d.ts +1 -0
- package/dist/types/session.d.ts +16 -0
- package/dist/vue/ai/index.js +694 -11
- package/dist/vue/ai/index.js.map +6 -6
- package/package.json +8 -7
package/dist/ai/client/index.js
CHANGED
|
@@ -208,6 +208,7 @@ var buildContextLabel = (metadata) => {
|
|
|
208
208
|
return;
|
|
209
209
|
}
|
|
210
210
|
const emailKind = getContextString(metadata.emailKind);
|
|
211
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
211
212
|
if (emailKind === "attachment") {
|
|
212
213
|
return "Attachment evidence";
|
|
213
214
|
}
|
|
@@ -245,6 +246,16 @@ var buildContextLabel = (metadata) => {
|
|
|
245
246
|
}
|
|
246
247
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
247
248
|
const sectionTitle = getContextString(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
249
|
+
const officeSectionLabel = sectionPath.length > 0 ? sectionPath.join(" > ") : sectionTitle;
|
|
250
|
+
if (officeBlockKind === "table" && officeSectionLabel) {
|
|
251
|
+
return `Office table block ${officeSectionLabel}`;
|
|
252
|
+
}
|
|
253
|
+
if (officeBlockKind === "list" && officeSectionLabel) {
|
|
254
|
+
return `Office list block ${officeSectionLabel}`;
|
|
255
|
+
}
|
|
256
|
+
if (officeBlockKind === "paragraph" && officeSectionLabel) {
|
|
257
|
+
return `Office paragraph block ${officeSectionLabel}`;
|
|
258
|
+
}
|
|
248
259
|
if (sectionTitle) {
|
|
249
260
|
return `Section ${sectionTitle}`;
|
|
250
261
|
}
|
|
@@ -266,6 +277,46 @@ var formatMediaDurationLabel = (value) => {
|
|
|
266
277
|
}
|
|
267
278
|
return formatMediaTimestamp(value);
|
|
268
279
|
};
|
|
280
|
+
var formatOfficeListLevelsLabel = (value) => {
|
|
281
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
const levels = value.map((entry) => getContextNumber(entry)).filter((entry) => typeof entry === "number").sort((left, right) => left - right);
|
|
285
|
+
if (levels.length === 0) {
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
const minLevel = levels[0];
|
|
289
|
+
const maxLevel = levels[levels.length - 1];
|
|
290
|
+
return minLevel === maxLevel ? `Office list level ${minLevel}` : `Office list levels ${minLevel}-${maxLevel}`;
|
|
291
|
+
};
|
|
292
|
+
var getOfficeTableCitationScope = (metadata) => {
|
|
293
|
+
if (!metadata) {
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
297
|
+
if (officeBlockKind !== "table" && officeBlockKind !== "list") {
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
301
|
+
const sectionTitle = getContextString(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
302
|
+
const officeContextText = officeBlockKind === "table" ? getContextString(metadata.officeTableContextText) : getContextString(metadata.officeListContextText);
|
|
303
|
+
if (!sectionTitle) {
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
return {
|
|
307
|
+
blockKind: officeBlockKind,
|
|
308
|
+
pathDepth: sectionPath.length,
|
|
309
|
+
sectionTitle,
|
|
310
|
+
hasContext: typeof officeContextText === "string"
|
|
311
|
+
};
|
|
312
|
+
};
|
|
313
|
+
var getOfficeTableCitationPreference = (metadata) => {
|
|
314
|
+
const scope = getOfficeTableCitationScope(metadata);
|
|
315
|
+
if (!scope) {
|
|
316
|
+
return 0;
|
|
317
|
+
}
|
|
318
|
+
return scope.pathDepth * 10 + (scope.hasContext ? 1 : 0) + (scope.blockKind === "list" && typeof metadata?.officeListGroupItemCount === "number" && metadata.officeListGroupItemCount > 1 ? 1 : 0);
|
|
319
|
+
};
|
|
269
320
|
var buildLocatorLabel = (metadata, source, title) => {
|
|
270
321
|
if (!metadata) {
|
|
271
322
|
return;
|
|
@@ -291,6 +342,10 @@ var buildLocatorLabel = (metadata, source, title) => {
|
|
|
291
342
|
return `Archive entry ${archiveEntry}`;
|
|
292
343
|
}
|
|
293
344
|
const emailKind = getContextString(metadata.emailKind);
|
|
345
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
346
|
+
const officeBlockNumber = getContextNumber(metadata.officeBlockNumber);
|
|
347
|
+
const officeTableBodyRowStart = getContextNumber(metadata.officeTableBodyRowStart);
|
|
348
|
+
const officeTableBodyRowEnd = getContextNumber(metadata.officeTableBodyRowEnd);
|
|
294
349
|
if (emailKind === "attachment") {
|
|
295
350
|
const attachmentName = getContextString(metadata.attachmentName) ?? getAttachmentName(source, title);
|
|
296
351
|
return attachmentName ? `Attachment ${attachmentName}` : "Attachment";
|
|
@@ -303,6 +358,18 @@ var buildLocatorLabel = (metadata, source, title) => {
|
|
|
303
358
|
if (mediaStart) {
|
|
304
359
|
return `Timestamp ${mediaStart}`;
|
|
305
360
|
}
|
|
361
|
+
if (officeBlockNumber && officeBlockKind === "table") {
|
|
362
|
+
if (typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number") {
|
|
363
|
+
return officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table block ${officeBlockNumber} \xB7 Row ${officeTableBodyRowStart}` : `Office table block ${officeBlockNumber} \xB7 Rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}`;
|
|
364
|
+
}
|
|
365
|
+
return `Office table block ${officeBlockNumber}`;
|
|
366
|
+
}
|
|
367
|
+
if (officeBlockNumber && officeBlockKind === "list") {
|
|
368
|
+
return `Office list block ${officeBlockNumber}`;
|
|
369
|
+
}
|
|
370
|
+
if (officeBlockNumber && officeBlockKind === "paragraph") {
|
|
371
|
+
return `Office paragraph block ${officeBlockNumber}`;
|
|
372
|
+
}
|
|
306
373
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
307
374
|
if (sectionPath.length > 0) {
|
|
308
375
|
return `Section ${sectionPath.join(" > ")}`;
|
|
@@ -336,10 +403,31 @@ var buildProvenanceLabel = (metadata) => {
|
|
|
336
403
|
const mediaDurationLabel = formatMediaDurationLabel(metadata.mediaDurationMs);
|
|
337
404
|
const transcriptSource = getContextString(metadata.transcriptSource);
|
|
338
405
|
const pdfTextMode = getContextString(metadata.pdfTextMode);
|
|
406
|
+
const officeBlockKind = getContextString(metadata.officeBlockKind);
|
|
407
|
+
const officeListContextText = getContextString(metadata.officeListContextText);
|
|
408
|
+
const officeListGroupItemCount = getContextNumber(metadata.officeListGroupItemCount);
|
|
409
|
+
const officeListLevelsLabel = formatOfficeListLevelsLabel(metadata.officeListLevels);
|
|
410
|
+
const officeTableHeaders = Array.isArray(metadata.officeTableHeaders) ? metadata.officeTableHeaders.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
411
|
+
const officeTableColumnCount = getContextNumber(metadata.officeTableColumnCount);
|
|
412
|
+
const officeTableBodyRowCount = getContextNumber(metadata.officeTableBodyRowCount);
|
|
413
|
+
const officeTableBodyRowStart = getContextNumber(metadata.officeTableBodyRowStart);
|
|
414
|
+
const officeTableBodyRowEnd = getContextNumber(metadata.officeTableBodyRowEnd);
|
|
415
|
+
const officeTableContextText = getContextString(metadata.officeTableContextText);
|
|
416
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString(value)).filter((value) => typeof value === "string") : [];
|
|
339
417
|
const ocrEngine = getContextString(metadata.ocrEngine);
|
|
340
418
|
const ocrConfidence = getContextNumber(metadata.ocrRegionConfidence) ?? getContextNumber(metadata.ocrConfidence);
|
|
341
419
|
const labels = [
|
|
342
420
|
pdfTextMode ? `PDF ${pdfTextMode}` : "",
|
|
421
|
+
officeBlockKind ? `Office ${officeBlockKind}` : "",
|
|
422
|
+
typeof officeListGroupItemCount === "number" ? `Office list ${officeListGroupItemCount} items` : "",
|
|
423
|
+
officeListLevelsLabel ?? "",
|
|
424
|
+
sectionPath.length > 0 && officeBlockKind ? `Source-aware office ${officeBlockKind} block ${sectionPath.join(" > ")}` : "",
|
|
425
|
+
officeListContextText ? `Office list context ${officeListContextText}` : "",
|
|
426
|
+
officeTableHeaders.length > 0 ? `Office table ${officeTableHeaders.join(", ")}` : "",
|
|
427
|
+
typeof officeTableColumnCount === "number" ? `Office table ${officeTableColumnCount} cols` : "",
|
|
428
|
+
typeof officeTableBodyRowCount === "number" ? `Office table ${officeTableBodyRowCount} body rows` : "",
|
|
429
|
+
typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number" ? officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table row ${officeTableBodyRowStart}` : `Office table rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}` : "",
|
|
430
|
+
officeTableContextText ? `Office table context ${officeTableContextText}` : "",
|
|
343
431
|
ocrEngine ? `OCR ${ocrEngine}` : "",
|
|
344
432
|
typeof ocrConfidence === "number" ? `Confidence ${ocrConfidence.toFixed(2)}` : "",
|
|
345
433
|
mediaKind ? `Media ${mediaKind}` : "",
|
|
@@ -503,6 +591,15 @@ var buildRAGCitations = (sources) => {
|
|
|
503
591
|
});
|
|
504
592
|
}
|
|
505
593
|
return [...unique.values()].sort((left, right) => {
|
|
594
|
+
const leftOfficeScope = getOfficeTableCitationScope(left.metadata);
|
|
595
|
+
const rightOfficeScope = getOfficeTableCitationScope(right.metadata);
|
|
596
|
+
if (left.source === right.source && leftOfficeScope && rightOfficeScope && leftOfficeScope.blockKind === rightOfficeScope.blockKind && leftOfficeScope.sectionTitle === rightOfficeScope.sectionTitle) {
|
|
597
|
+
const leftOfficePreference = getOfficeTableCitationPreference(left.metadata);
|
|
598
|
+
const rightOfficePreference = getOfficeTableCitationPreference(right.metadata);
|
|
599
|
+
if (rightOfficePreference !== leftOfficePreference) {
|
|
600
|
+
return rightOfficePreference - leftOfficePreference;
|
|
601
|
+
}
|
|
602
|
+
}
|
|
506
603
|
if (right.score !== left.score) {
|
|
507
604
|
return right.score - left.score;
|
|
508
605
|
}
|
|
@@ -916,6 +1013,7 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
916
1013
|
const sectionKind = getContextString2(metadata.sectionKind);
|
|
917
1014
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
918
1015
|
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
1016
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
919
1017
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
920
1018
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
921
1019
|
const sheetName = getContextString2(metadata.sheetName);
|
|
@@ -926,6 +1024,12 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
926
1024
|
return `Source-aware section ${sectionPath.join(" > ")}`;
|
|
927
1025
|
}
|
|
928
1026
|
if (sectionKind === "pdf_block") {
|
|
1027
|
+
if (pdfSemanticRole === "figure_caption" && sectionTitle) {
|
|
1028
|
+
return `Source-aware PDF figure caption ${sectionTitle}`;
|
|
1029
|
+
}
|
|
1030
|
+
if (pdfSemanticRole === "figure_body" && sectionTitle) {
|
|
1031
|
+
return `Source-aware PDF figure body ${sectionTitle}`;
|
|
1032
|
+
}
|
|
929
1033
|
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
930
1034
|
return `Source-aware PDF table block ${sectionTitle}`;
|
|
931
1035
|
}
|
|
@@ -935,11 +1039,12 @@ var buildSourceAwareUnitScopeLabel = (metadata) => {
|
|
|
935
1039
|
return "Source-aware PDF block";
|
|
936
1040
|
}
|
|
937
1041
|
if (sectionKind === "office_block") {
|
|
938
|
-
|
|
939
|
-
|
|
1042
|
+
const officeSectionLabel = sectionPath.length > 0 ? sectionPath.join(" > ") : sectionTitle;
|
|
1043
|
+
if (officeBlockKind && officeSectionLabel) {
|
|
1044
|
+
return `Source-aware office ${officeBlockKind} block ${officeSectionLabel}`;
|
|
940
1045
|
}
|
|
941
|
-
if (
|
|
942
|
-
return `Source-aware office block ${
|
|
1046
|
+
if (officeSectionLabel) {
|
|
1047
|
+
return `Source-aware office block ${officeSectionLabel}`;
|
|
943
1048
|
}
|
|
944
1049
|
return "Source-aware office block";
|
|
945
1050
|
}
|
|
@@ -1327,6 +1432,18 @@ var formatSpreadsheetTableLabel = (tableIndex, tableCount) => {
|
|
|
1327
1432
|
}
|
|
1328
1433
|
return `Table ${tableIndex}`;
|
|
1329
1434
|
};
|
|
1435
|
+
var formatOfficeListLevelsLabel2 = (value) => {
|
|
1436
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
const levels = value.map((entry) => getContextNumber2(entry)).filter((entry) => typeof entry === "number").sort((left, right) => left - right);
|
|
1440
|
+
if (levels.length === 0) {
|
|
1441
|
+
return;
|
|
1442
|
+
}
|
|
1443
|
+
const minLevel = levels[0];
|
|
1444
|
+
const maxLevel = levels[levels.length - 1];
|
|
1445
|
+
return minLevel === maxLevel ? `Office list level ${minLevel}` : `Office list levels ${minLevel}-${maxLevel}`;
|
|
1446
|
+
};
|
|
1330
1447
|
var formatMediaDurationLabel2 = (value) => {
|
|
1331
1448
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1332
1449
|
return;
|
|
@@ -1338,9 +1455,18 @@ var buildContextLabel2 = (metadata) => {
|
|
|
1338
1455
|
return;
|
|
1339
1456
|
}
|
|
1340
1457
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1458
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1459
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1460
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1341
1461
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1342
1462
|
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1343
1463
|
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
1464
|
+
if (pdfSemanticRole === "figure_caption" && sectionTitle) {
|
|
1465
|
+
return `PDF figure caption ${sectionTitle}`;
|
|
1466
|
+
}
|
|
1467
|
+
if (pdfSemanticRole === "figure_body" && sectionTitle) {
|
|
1468
|
+
return `PDF figure body ${sectionTitle}`;
|
|
1469
|
+
}
|
|
1344
1470
|
if (pdfTextKind === "table_like" && sectionTitle) {
|
|
1345
1471
|
return `PDF table block ${sectionTitle}`;
|
|
1346
1472
|
}
|
|
@@ -1348,13 +1474,13 @@ var buildContextLabel2 = (metadata) => {
|
|
|
1348
1474
|
return `PDF text block ${sectionTitle}`;
|
|
1349
1475
|
}
|
|
1350
1476
|
if (officeBlockKind === "table" && sectionTitle) {
|
|
1351
|
-
return `Office table block ${sectionTitle}`;
|
|
1477
|
+
return `Office table block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1352
1478
|
}
|
|
1353
1479
|
if (officeBlockKind === "list" && sectionTitle) {
|
|
1354
|
-
return `Office list block ${sectionTitle}`;
|
|
1480
|
+
return `Office list block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1355
1481
|
}
|
|
1356
1482
|
if (officeBlockKind === "paragraph" && sectionTitle) {
|
|
1357
|
-
return `Office paragraph block ${sectionTitle}`;
|
|
1483
|
+
return `Office paragraph block ${sectionPath.join(" > ") || sectionTitle}`;
|
|
1358
1484
|
}
|
|
1359
1485
|
const emailKind = getContextString2(metadata.emailKind);
|
|
1360
1486
|
if (emailKind === "attachment") {
|
|
@@ -1452,9 +1578,14 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1452
1578
|
return;
|
|
1453
1579
|
}
|
|
1454
1580
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1581
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1455
1582
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1456
1583
|
const pdfBlockNumber = getContextNumber2(metadata.pdfBlockNumber);
|
|
1584
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1585
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1457
1586
|
const officeBlockNumber = getContextNumber2(metadata.officeBlockNumber);
|
|
1587
|
+
const officeTableBodyRowStart = getContextNumber2(metadata.officeTableBodyRowStart);
|
|
1588
|
+
const officeTableBodyRowEnd = getContextNumber2(metadata.officeTableBodyRowEnd);
|
|
1458
1589
|
const spreadsheetRowStart = getContextNumber2(metadata.spreadsheetRowStart);
|
|
1459
1590
|
const spreadsheetRowEnd = getContextNumber2(metadata.spreadsheetRowEnd);
|
|
1460
1591
|
const slideTitle = getContextString2(metadata.slideTitle);
|
|
@@ -1465,7 +1596,16 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1465
1596
|
if (page && region) {
|
|
1466
1597
|
return `Page ${page} \xB7 Region ${region}`;
|
|
1467
1598
|
}
|
|
1599
|
+
if (page && pdfBlockNumber && pdfSemanticRole === "figure_caption") {
|
|
1600
|
+
return `Page ${page} \xB7 Figure Caption ${pdfBlockNumber}`;
|
|
1601
|
+
}
|
|
1602
|
+
if (page && pdfBlockNumber && pdfSemanticRole === "figure_body") {
|
|
1603
|
+
return `Page ${page} \xB7 Figure Body ${pdfBlockNumber}`;
|
|
1604
|
+
}
|
|
1468
1605
|
if (page && pdfBlockNumber && pdfTextKind === "table_like") {
|
|
1606
|
+
if (typeof pdfTableBodyRowStart === "number" && typeof pdfTableBodyRowEnd === "number") {
|
|
1607
|
+
return pdfTableBodyRowStart === pdfTableBodyRowEnd ? `Page ${page} \xB7 Table Block ${pdfBlockNumber} \xB7 Row ${pdfTableBodyRowStart}` : `Page ${page} \xB7 Table Block ${pdfBlockNumber} \xB7 Rows ${pdfTableBodyRowStart}-${pdfTableBodyRowEnd}`;
|
|
1608
|
+
}
|
|
1469
1609
|
return `Page ${page} \xB7 Table Block ${pdfBlockNumber}`;
|
|
1470
1610
|
}
|
|
1471
1611
|
if (page && pdfBlockNumber) {
|
|
@@ -1528,6 +1668,9 @@ var buildLocatorLabel2 = (metadata, source, title) => {
|
|
|
1528
1668
|
return `Timestamp ${mediaStart}`;
|
|
1529
1669
|
}
|
|
1530
1670
|
if (officeBlockNumber && officeBlockKind === "table") {
|
|
1671
|
+
if (typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number") {
|
|
1672
|
+
return officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table block ${officeBlockNumber} \xB7 Row ${officeTableBodyRowStart}` : `Office table block ${officeBlockNumber} \xB7 Rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}`;
|
|
1673
|
+
}
|
|
1531
1674
|
return `Office table block ${officeBlockNumber}`;
|
|
1532
1675
|
}
|
|
1533
1676
|
if (officeBlockNumber && officeBlockKind === "list") {
|
|
@@ -1564,11 +1707,27 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1564
1707
|
const mediaSegmentWindowDurationLabel = formatMediaDurationLabel2(metadata.mediaSegmentGroupDurationMs);
|
|
1565
1708
|
const mediaSegmentGapLabel = formatMediaDurationLabel2(metadata.mediaSegmentGapFromPreviousMs);
|
|
1566
1709
|
const spreadsheetHeaders = getSpreadsheetHeaders(metadata);
|
|
1710
|
+
const pdfTableHeaders = Array.isArray(metadata.pdfTableHeaders) ? metadata.pdfTableHeaders.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1711
|
+
const pdfTableColumnCount = getContextNumber2(metadata.pdfTableColumnCount);
|
|
1712
|
+
const pdfTableBodyRowCount = getContextNumber2(metadata.pdfTableBodyRowCount);
|
|
1567
1713
|
const spreadsheetColumnRange = formatSpreadsheetColumnRange(getContextString2(metadata.spreadsheetColumnStart), getContextString2(metadata.spreadsheetColumnEnd));
|
|
1568
1714
|
const slideNotesText = getContextString2(metadata.slideNotesText);
|
|
1569
1715
|
const pdfTextMode = getContextString2(metadata.pdfTextMode);
|
|
1716
|
+
const pdfEvidenceMode = getContextString2(metadata.pdfEvidenceMode);
|
|
1717
|
+
const pdfEvidenceOrigin = getContextString2(metadata.pdfEvidenceOrigin);
|
|
1718
|
+
const pdfEvidenceSupplement = getContextString2(metadata.pdfEvidenceSupplement);
|
|
1570
1719
|
const pdfTextKind = getContextString2(metadata.pdfTextKind);
|
|
1720
|
+
const pdfSemanticRole = getContextString2(metadata.pdfSemanticRole);
|
|
1571
1721
|
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
1722
|
+
const officeListContextText = getContextString2(metadata.officeListContextText);
|
|
1723
|
+
const officeListGroupItemCount = getContextNumber2(metadata.officeListGroupItemCount);
|
|
1724
|
+
const officeListLevelsLabel = formatOfficeListLevelsLabel2(metadata.officeListLevels);
|
|
1725
|
+
const officeTableHeaders = Array.isArray(metadata.officeTableHeaders) ? metadata.officeTableHeaders.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
1726
|
+
const officeTableColumnCount = getContextNumber2(metadata.officeTableColumnCount);
|
|
1727
|
+
const officeTableBodyRowCount = getContextNumber2(metadata.officeTableBodyRowCount);
|
|
1728
|
+
const officeTableBodyRowStart = getContextNumber2(metadata.officeTableBodyRowStart);
|
|
1729
|
+
const officeTableBodyRowEnd = getContextNumber2(metadata.officeTableBodyRowEnd);
|
|
1730
|
+
const officeTableContextText = getContextString2(metadata.officeTableContextText);
|
|
1572
1731
|
const ocrEngine = getContextString2(metadata.ocrEngine);
|
|
1573
1732
|
const extractorRegistryMatch = getContextString2(metadata.extractorRegistryMatch);
|
|
1574
1733
|
const chunkingProfile = getContextString2(metadata.chunkingProfile);
|
|
@@ -1584,10 +1743,19 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1584
1743
|
const ocrMinConfidence = getContextNumber2(metadata.ocrPageMinConfidence) ?? getContextNumber2(metadata.ocrMinConfidence);
|
|
1585
1744
|
const ocrMaxConfidence = getContextNumber2(metadata.ocrPageMaxConfidence) ?? getContextNumber2(metadata.ocrMaxConfidence);
|
|
1586
1745
|
const ocrRegionCount = getContextNumber2(metadata.ocrRegionCount);
|
|
1746
|
+
const pdfTableBodyRowStart = getContextNumber2(metadata.pdfTableBodyRowStart);
|
|
1747
|
+
const pdfTableBodyRowEnd = getContextNumber2(metadata.pdfTableBodyRowEnd);
|
|
1587
1748
|
const labels = [
|
|
1588
1749
|
pdfTextMode ? `PDF ${pdfTextMode}` : "",
|
|
1589
|
-
|
|
1750
|
+
pdfEvidenceMode ? `PDF evidence ${pdfEvidenceMode}` : "",
|
|
1751
|
+
pdfEvidenceOrigin ? `PDF origin ${pdfEvidenceOrigin}` : "",
|
|
1752
|
+
pdfEvidenceSupplement ? `PDF supplement ${pdfEvidenceSupplement}` : "",
|
|
1753
|
+
pdfSemanticRole === "figure_caption" ? "PDF figure caption" : "",
|
|
1754
|
+
pdfSemanticRole === "figure_body" ? "PDF figure body" : "",
|
|
1755
|
+
pdfSemanticRole === "figure_caption" ? "" : pdfSemanticRole === "figure_body" ? "" : pdfTextKind === "table_like" ? "PDF table block" : pdfTextKind === "paragraph" ? "PDF text block" : "",
|
|
1590
1756
|
officeBlockKind ? `Office ${officeBlockKind}` : "",
|
|
1757
|
+
typeof officeListGroupItemCount === "number" ? `Office list ${officeListGroupItemCount} items` : "",
|
|
1758
|
+
officeListLevelsLabel ?? "",
|
|
1591
1759
|
ocrEngine ? `OCR ${ocrEngine}` : "",
|
|
1592
1760
|
extractorRegistryMatch ? `Extractor ${extractorRegistryMatch}` : "",
|
|
1593
1761
|
chunkingProfile ? `Chunking ${chunkingProfile}` : "",
|
|
@@ -1597,6 +1765,16 @@ var buildProvenanceLabel2 = (metadata) => {
|
|
|
1597
1765
|
typeof ocrAverageConfidence === "number" && ocrAverageConfidence !== ocrConfidence ? `Average ${ocrAverageConfidence.toFixed(2)}` : "",
|
|
1598
1766
|
typeof ocrMinConfidence === "number" && typeof ocrMaxConfidence === "number" && ocrMinConfidence !== ocrMaxConfidence ? `Range ${ocrMinConfidence.toFixed(2)}-${ocrMaxConfidence.toFixed(2)}` : "",
|
|
1599
1767
|
typeof ocrRegionCount === "number" ? `${ocrRegionCount} regions` : "",
|
|
1768
|
+
pdfTableHeaders.length > 0 ? `PDF table ${pdfTableHeaders.join(", ")}` : "",
|
|
1769
|
+
typeof pdfTableColumnCount === "number" ? `PDF table ${pdfTableColumnCount} cols` : "",
|
|
1770
|
+
typeof pdfTableBodyRowCount === "number" ? `PDF table ${pdfTableBodyRowCount} body rows` : "",
|
|
1771
|
+
typeof pdfTableBodyRowStart === "number" && typeof pdfTableBodyRowEnd === "number" ? pdfTableBodyRowStart === pdfTableBodyRowEnd ? `PDF table row ${pdfTableBodyRowStart}` : `PDF table rows ${pdfTableBodyRowStart}-${pdfTableBodyRowEnd}` : "",
|
|
1772
|
+
officeListContextText ? `Office list context ${officeListContextText}` : "",
|
|
1773
|
+
officeTableHeaders.length > 0 ? `Office table ${officeTableHeaders.join(", ")}` : "",
|
|
1774
|
+
typeof officeTableColumnCount === "number" ? `Office table ${officeTableColumnCount} cols` : "",
|
|
1775
|
+
typeof officeTableBodyRowCount === "number" ? `Office table ${officeTableBodyRowCount} body rows` : "",
|
|
1776
|
+
typeof officeTableBodyRowStart === "number" && typeof officeTableBodyRowEnd === "number" ? officeTableBodyRowStart === officeTableBodyRowEnd ? `Office table row ${officeTableBodyRowStart}` : `Office table rows ${officeTableBodyRowStart}-${officeTableBodyRowEnd}` : "",
|
|
1777
|
+
officeTableContextText ? `Office table context ${officeTableContextText}` : "",
|
|
1600
1778
|
spreadsheetHeaders.length > 0 ? `Spreadsheet ${spreadsheetHeaders.join(", ")}` : "",
|
|
1601
1779
|
spreadsheetColumnRange ? `Spreadsheet ${spreadsheetColumnRange}` : "",
|
|
1602
1780
|
spreadsheetTableLabel ? `Spreadsheet ${spreadsheetTableLabel}` : "",
|
|
@@ -2028,12 +2206,92 @@ var getStructuredSectionScoreWeight = (metadata) => {
|
|
|
2028
2206
|
return 1;
|
|
2029
2207
|
};
|
|
2030
2208
|
var getStructuredSourceLeadScore = (source) => source.score * getStructuredSectionScoreWeight(source.metadata);
|
|
2209
|
+
var getPDFLeadEvidencePreference = (metadata) => {
|
|
2210
|
+
if (!metadata) {
|
|
2211
|
+
return 0;
|
|
2212
|
+
}
|
|
2213
|
+
const pdfEvidenceMode = getContextString2(metadata.pdfEvidenceMode);
|
|
2214
|
+
const pdfEvidenceOrigin = getContextString2(metadata.pdfEvidenceOrigin);
|
|
2215
|
+
const pdfEvidenceSupplement = getContextString2(metadata.pdfEvidenceSupplement);
|
|
2216
|
+
if (pdfEvidenceMode === "hybrid" && pdfEvidenceOrigin === "native" && pdfEvidenceSupplement === "ocr") {
|
|
2217
|
+
return 3;
|
|
2218
|
+
}
|
|
2219
|
+
if (pdfEvidenceMode === "native" && pdfEvidenceOrigin === "native") {
|
|
2220
|
+
return 2;
|
|
2221
|
+
}
|
|
2222
|
+
if (pdfEvidenceMode === "ocr" && pdfEvidenceOrigin === "ocr") {
|
|
2223
|
+
return 1;
|
|
2224
|
+
}
|
|
2225
|
+
return 0;
|
|
2226
|
+
};
|
|
2227
|
+
var getPDFLeadScope = (metadata) => {
|
|
2228
|
+
if (!metadata) {
|
|
2229
|
+
return;
|
|
2230
|
+
}
|
|
2231
|
+
const pageNumber = getContextNumber2(metadata.pageNumber) ?? getContextNumber2(metadata.page) ?? (typeof metadata.pageIndex === "number" ? metadata.pageIndex + 1 : undefined);
|
|
2232
|
+
const sectionTitle = getContextString2(metadata.sectionTitle);
|
|
2233
|
+
const sourceNativeKind = getContextString2(metadata.sourceNativeKind);
|
|
2234
|
+
if (typeof pageNumber !== "number" && !sectionTitle && !sourceNativeKind) {
|
|
2235
|
+
return;
|
|
2236
|
+
}
|
|
2237
|
+
return {
|
|
2238
|
+
pageNumber,
|
|
2239
|
+
sectionTitle,
|
|
2240
|
+
sourceNativeKind
|
|
2241
|
+
};
|
|
2242
|
+
};
|
|
2243
|
+
var getOfficeLeadScope = (metadata) => {
|
|
2244
|
+
if (!metadata) {
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
2247
|
+
const officeBlockKind = getContextString2(metadata.officeBlockKind);
|
|
2248
|
+
if (officeBlockKind !== "table" && officeBlockKind !== "list") {
|
|
2249
|
+
return;
|
|
2250
|
+
}
|
|
2251
|
+
const sectionPath = Array.isArray(metadata.sectionPath) ? metadata.sectionPath.map((value) => getContextString2(value)).filter((value) => typeof value === "string") : [];
|
|
2252
|
+
const sectionTitle = getContextString2(metadata.sectionTitle) ?? sectionPath.at(-1);
|
|
2253
|
+
const officeContextText = officeBlockKind === "table" ? getContextString2(metadata.officeTableContextText) : getContextString2(metadata.officeListContextText);
|
|
2254
|
+
if (!sectionTitle) {
|
|
2255
|
+
return;
|
|
2256
|
+
}
|
|
2257
|
+
return {
|
|
2258
|
+
blockKind: officeBlockKind,
|
|
2259
|
+
pathDepth: sectionPath.length,
|
|
2260
|
+
sectionTitle,
|
|
2261
|
+
hasContext: typeof officeContextText === "string"
|
|
2262
|
+
};
|
|
2263
|
+
};
|
|
2264
|
+
var getOfficeLeadEvidencePreference = (metadata) => {
|
|
2265
|
+
const scope = getOfficeLeadScope(metadata);
|
|
2266
|
+
if (!scope) {
|
|
2267
|
+
return 0;
|
|
2268
|
+
}
|
|
2269
|
+
return scope.pathDepth * 10 + (scope.hasContext ? 1 : 0) + (scope.blockKind === "list" && typeof metadata?.officeListGroupItemCount === "number" && metadata.officeListGroupItemCount > 1 ? 1 : 0);
|
|
2270
|
+
};
|
|
2031
2271
|
var getPreferredSourceLeadChunk = (chunks) => chunks.slice().sort((left, right) => {
|
|
2272
|
+
const leftOfficeScope = getOfficeLeadScope(left.metadata);
|
|
2273
|
+
const rightOfficeScope = getOfficeLeadScope(right.metadata);
|
|
2274
|
+
if (left.source === right.source && leftOfficeScope && rightOfficeScope && leftOfficeScope.blockKind === rightOfficeScope.blockKind && leftOfficeScope.sectionTitle === rightOfficeScope.sectionTitle) {
|
|
2275
|
+
const leftOfficePreference = getOfficeLeadEvidencePreference(left.metadata);
|
|
2276
|
+
const rightOfficePreference = getOfficeLeadEvidencePreference(right.metadata);
|
|
2277
|
+
if (rightOfficePreference !== leftOfficePreference) {
|
|
2278
|
+
return rightOfficePreference - leftOfficePreference;
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2032
2281
|
const leftWeightedScore = getStructuredSourceLeadScore(left);
|
|
2033
2282
|
const rightWeightedScore = getStructuredSourceLeadScore(right);
|
|
2034
2283
|
if (rightWeightedScore !== leftWeightedScore) {
|
|
2035
2284
|
return rightWeightedScore - leftWeightedScore;
|
|
2036
2285
|
}
|
|
2286
|
+
const leftScope = getPDFLeadScope(left.metadata);
|
|
2287
|
+
const rightScope = getPDFLeadScope(right.metadata);
|
|
2288
|
+
if (left.source === right.source && leftScope && rightScope && (leftScope.sectionTitle && rightScope.sectionTitle && leftScope.sectionTitle === rightScope.sectionTitle || typeof leftScope.pageNumber === "number" && typeof rightScope.pageNumber === "number" && leftScope.pageNumber === rightScope.pageNumber)) {
|
|
2289
|
+
const leftEvidencePreference = getPDFLeadEvidencePreference(left.metadata);
|
|
2290
|
+
const rightEvidencePreference = getPDFLeadEvidencePreference(right.metadata);
|
|
2291
|
+
if (rightEvidencePreference !== leftEvidencePreference) {
|
|
2292
|
+
return rightEvidencePreference - leftEvidencePreference;
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2037
2295
|
if (right.score !== left.score) {
|
|
2038
2296
|
return right.score - left.score;
|
|
2039
2297
|
}
|
|
@@ -2287,6 +2545,7 @@ var buildRAGSectionRetrievalDiagnostics = (sources, trace) => {
|
|
|
2287
2545
|
queryTransformProvider: trace?.queryTransformProvider,
|
|
2288
2546
|
queryTransformReason: trace?.queryTransformReason,
|
|
2289
2547
|
reasons,
|
|
2548
|
+
evidenceReconcileApplied: trace?.steps.some((step) => step.stage === "evidence_reconcile"),
|
|
2290
2549
|
rerankApplied: trace?.steps.some((step) => step.stage === "rerank" && step.metadata?.applied === true),
|
|
2291
2550
|
scoreShare,
|
|
2292
2551
|
scoreThresholdApplied: trace?.steps.some((step) => step.stage === "score_filter"),
|
|
@@ -2965,6 +3224,12 @@ var buildComparisonOverviewPresentation = (input) => {
|
|
|
2965
3224
|
value: input.resolveLabel(input.summary.bestByMultivectorVectorHitCases)
|
|
2966
3225
|
});
|
|
2967
3226
|
}
|
|
3227
|
+
if (input.summary.bestByEvidenceReconcileCases) {
|
|
3228
|
+
rows.push({
|
|
3229
|
+
label: "Best evidence reconcile",
|
|
3230
|
+
value: input.resolveLabel(input.summary.bestByEvidenceReconcileCases)
|
|
3231
|
+
});
|
|
3232
|
+
}
|
|
2968
3233
|
if (input.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases) {
|
|
2969
3234
|
rows.push({
|
|
2970
3235
|
label: "Lowest runtime budget exhaustion",
|
|
@@ -3032,6 +3297,9 @@ var buildRAGComparisonTraceSummaryRows = (entry) => {
|
|
|
3032
3297
|
}, {
|
|
3033
3298
|
label: "Runtime",
|
|
3034
3299
|
value: `budget ${formatTraceRatio(trace.runtimeCandidateBudgetExhaustedCases, trace.totalCases)} \xB7 underfilled ${formatTraceRatio(trace.runtimeUnderfilledTopKCases, trace.totalCases)}`
|
|
3300
|
+
}, {
|
|
3301
|
+
label: "Evidence reconcile",
|
|
3302
|
+
value: `all ${formatTraceRatio(trace.stageCounts.evidence_reconcile ?? 0, trace.totalCases)} \xB7 office ${formatTraceRatio(trace.officeEvidenceReconcileCases, trace.totalCases)} \xB7 pdf ${formatTraceRatio(trace.pdfEvidenceReconcileCases, trace.totalCases)}`
|
|
3035
3303
|
}, {
|
|
3036
3304
|
label: "TopK",
|
|
3037
3305
|
value: `${trace.averageCandidateTopK.toFixed(1)} / ${trace.averageLexicalTopK.toFixed(1)}`
|
|
@@ -3389,6 +3657,25 @@ var buildRAGEvaluationHistoryRows = (history) => {
|
|
|
3389
3657
|
label: "Trace variant delta",
|
|
3390
3658
|
value: formatTraceCountDelta(history.diff.traceSummaryDelta.variantCases)
|
|
3391
3659
|
});
|
|
3660
|
+
const evidenceReconcileDelta = history.diff.traceSummaryDelta.stageCounts?.evidence_reconcile;
|
|
3661
|
+
if (typeof evidenceReconcileDelta === "number") {
|
|
3662
|
+
rows.push({
|
|
3663
|
+
label: "Trace evidence reconcile delta",
|
|
3664
|
+
value: formatTraceCountDelta(evidenceReconcileDelta)
|
|
3665
|
+
});
|
|
3666
|
+
}
|
|
3667
|
+
if (typeof history.diff.traceSummaryDelta.officeEvidenceReconcileCasesDelta === "number") {
|
|
3668
|
+
rows.push({
|
|
3669
|
+
label: "Trace office evidence reconcile delta",
|
|
3670
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.officeEvidenceReconcileCasesDelta)
|
|
3671
|
+
});
|
|
3672
|
+
}
|
|
3673
|
+
if (typeof history.diff.traceSummaryDelta.pdfEvidenceReconcileCasesDelta === "number") {
|
|
3674
|
+
rows.push({
|
|
3675
|
+
label: "Trace PDF evidence reconcile delta",
|
|
3676
|
+
value: formatTraceCountDelta(history.diff.traceSummaryDelta.pdfEvidenceReconcileCasesDelta)
|
|
3677
|
+
});
|
|
3678
|
+
}
|
|
3392
3679
|
const stageDelta = Object.entries(history.diff.traceSummaryDelta.stageCounts ?? {}).map(([stage, count]) => `${stage} ${formatTraceCountDelta(count)}`).join(", ");
|
|
3393
3680
|
if (stageDelta) {
|
|
3394
3681
|
rows.push({ label: "Trace stage delta", value: stageDelta });
|
|
@@ -3594,6 +3881,7 @@ var buildRAGEvaluationSuiteSnapshotHistoryPresentation = (history) => ({
|
|
|
3594
3881
|
summary: history?.latestSnapshot ? `v${history.latestSnapshot.version}` : "No saved suite snapshots yet."
|
|
3595
3882
|
});
|
|
3596
3883
|
var isRuntimeGateReason = (reason) => /runtime|candidate-budget|underfilled/i.test(reason);
|
|
3884
|
+
var getFixtureVariantsFromRunTags = (tags) => (tags ?? []).filter((tag) => tag.startsWith("fixture:")).map((tag) => tag.slice("fixture:".length)).filter((tag, index, all) => tag.length > 0 && all.indexOf(tag) === index);
|
|
3597
3885
|
var buildRAGRetrievalReleaseHistoryRunPresentation = (run) => {
|
|
3598
3886
|
const runtimeGateReasons = (run.decisionSummary?.gate?.reasons ?? run.releaseVerdict?.gate?.reasons ?? []).filter(isRuntimeGateReason);
|
|
3599
3887
|
const rows = [
|
|
@@ -3607,6 +3895,13 @@ var buildRAGRetrievalReleaseHistoryRunPresentation = (run) => {
|
|
|
3607
3895
|
value: run.comparison.summary.bestByAverageF1 ?? "n/a"
|
|
3608
3896
|
}
|
|
3609
3897
|
];
|
|
3898
|
+
const fixtureVariants = getFixtureVariantsFromRunTags(run.tags);
|
|
3899
|
+
if (fixtureVariants.length > 0) {
|
|
3900
|
+
rows.push({
|
|
3901
|
+
label: "Fixture variant",
|
|
3902
|
+
value: fixtureVariants.join(", ")
|
|
3903
|
+
});
|
|
3904
|
+
}
|
|
3610
3905
|
if (run.comparison.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases) {
|
|
3611
3906
|
rows.push({
|
|
3612
3907
|
label: "Lowest runtime budget exhaustion",
|
|
@@ -3635,6 +3930,7 @@ var buildRAGRetrievalReleaseHistoryRunPresentation = (run) => {
|
|
|
3635
3930
|
};
|
|
3636
3931
|
var buildRAGRetrievalReleaseGroupHistoryPresentation = (input) => {
|
|
3637
3932
|
const recentRuns = (input.runs ?? []).map(buildRAGRetrievalReleaseHistoryRunPresentation);
|
|
3933
|
+
const fixtureVariants = (input.runs ?? []).flatMap((run) => getFixtureVariantsFromRunTags(run.tags)).filter((tag, index, all) => all.indexOf(tag) === index);
|
|
3638
3934
|
const runtimeBlockedRuns = recentRuns.filter((entry) => entry.rows.some((row) => row.label === "Runtime gate failures" && row.value !== "none")).length;
|
|
3639
3935
|
const rows = [
|
|
3640
3936
|
{
|
|
@@ -3658,6 +3954,12 @@ var buildRAGRetrievalReleaseGroupHistoryPresentation = (input) => {
|
|
|
3658
3954
|
value: String(runtimeBlockedRuns)
|
|
3659
3955
|
}
|
|
3660
3956
|
];
|
|
3957
|
+
if (fixtureVariants.length > 0) {
|
|
3958
|
+
rows.push({
|
|
3959
|
+
label: "Fixture variants",
|
|
3960
|
+
value: fixtureVariants.join(", ")
|
|
3961
|
+
});
|
|
3962
|
+
}
|
|
3661
3963
|
return {
|
|
3662
3964
|
recentRuns,
|
|
3663
3965
|
rows,
|
|
@@ -4174,6 +4476,9 @@ var evaluateRetrievalComparisonGate = ({
|
|
|
4174
4476
|
if (typeof policy.minMultiVectorVectorHitCasesDelta === "number" && (delta.multiVectorVectorHitCasesDelta ?? 0) < policy.minMultiVectorVectorHitCasesDelta) {
|
|
4175
4477
|
reasons.push(`multivector vector-hit delta ${delta.multiVectorVectorHitCasesDelta ?? 0} is below ${policy.minMultiVectorVectorHitCasesDelta}`);
|
|
4176
4478
|
}
|
|
4479
|
+
if (typeof policy.minEvidenceReconcileCasesDelta === "number" && (delta.evidenceReconcileCasesDelta ?? 0) < policy.minEvidenceReconcileCasesDelta) {
|
|
4480
|
+
reasons.push(`evidence reconcile delta ${delta.evidenceReconcileCasesDelta ?? 0} is below ${policy.minEvidenceReconcileCasesDelta}`);
|
|
4481
|
+
}
|
|
4177
4482
|
if (typeof policy.maxRuntimeCandidateBudgetExhaustedCasesDelta === "number" && (delta.runtimeCandidateBudgetExhaustedCasesDelta ?? 0) > policy.maxRuntimeCandidateBudgetExhaustedCasesDelta) {
|
|
4178
4483
|
reasons.push(`runtime candidate-budget-exhausted delta ${delta.runtimeCandidateBudgetExhaustedCasesDelta ?? 0} exceeds ${policy.maxRuntimeCandidateBudgetExhaustedCasesDelta}`);
|
|
4179
4484
|
}
|
|
@@ -4227,13 +4532,14 @@ var buildRAGRetrievalReleaseVerdict = ({
|
|
|
4227
4532
|
};
|
|
4228
4533
|
}
|
|
4229
4534
|
if (delta) {
|
|
4535
|
+
const requiresReview = delta.passingRateDelta < 0 || delta.averageF1Delta < 0 || (delta.evidenceReconcileCasesDelta ?? 0) < 0;
|
|
4230
4536
|
return {
|
|
4231
4537
|
baselineGroupKey: groupKey,
|
|
4232
4538
|
baselineRetrievalId,
|
|
4233
4539
|
candidateRetrievalId,
|
|
4234
4540
|
delta,
|
|
4235
|
-
status:
|
|
4236
|
-
summary:
|
|
4541
|
+
status: requiresReview ? "needs_review" : "pass",
|
|
4542
|
+
summary: requiresReview ? "Candidate should be reviewed before promotion." : "Candidate improved or matched the baseline."
|
|
4237
4543
|
};
|
|
4238
4544
|
}
|
|
4239
4545
|
return {
|
|
@@ -4914,6 +5220,8 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4914
5220
|
let multiVectorVectorHitCases = 0;
|
|
4915
5221
|
let multiVectorLexicalHitCases = 0;
|
|
4916
5222
|
let multiVectorCollapsedCases = 0;
|
|
5223
|
+
let officeEvidenceReconcileCases = 0;
|
|
5224
|
+
let pdfEvidenceReconcileCases = 0;
|
|
4917
5225
|
let runtimeCandidateBudgetExhaustedCases = 0;
|
|
4918
5226
|
let runtimeUnderfilledTopKCases = 0;
|
|
4919
5227
|
let finalCountSum = 0;
|
|
@@ -4955,6 +5263,13 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4955
5263
|
if ((trace.multiVector?.collapsedParents ?? 0) > 0) {
|
|
4956
5264
|
multiVectorCollapsedCases += 1;
|
|
4957
5265
|
}
|
|
5266
|
+
const evidenceReconcileMetadata = trace.steps.find((step) => step.stage === "evidence_reconcile")?.metadata;
|
|
5267
|
+
if (typeof evidenceReconcileMetadata?.officeAffectedScopes === "number" && evidenceReconcileMetadata.officeAffectedScopes > 0) {
|
|
5268
|
+
officeEvidenceReconcileCases += 1;
|
|
5269
|
+
}
|
|
5270
|
+
if (typeof evidenceReconcileMetadata?.pdfAffectedScopes === "number" && evidenceReconcileMetadata.pdfAffectedScopes > 0) {
|
|
5271
|
+
pdfEvidenceReconcileCases += 1;
|
|
5272
|
+
}
|
|
4958
5273
|
if (vectorSearchMetadata?.sqliteQueryCandidateBudgetExhausted) {
|
|
4959
5274
|
runtimeCandidateBudgetExhaustedCases += 1;
|
|
4960
5275
|
}
|
|
@@ -4992,6 +5307,8 @@ var summarizeRetrievalTraces = (traces) => {
|
|
|
4992
5307
|
multiVectorVectorHitCases,
|
|
4993
5308
|
multiVectorLexicalHitCases,
|
|
4994
5309
|
multiVectorCollapsedCases,
|
|
5310
|
+
officeEvidenceReconcileCases,
|
|
5311
|
+
pdfEvidenceReconcileCases,
|
|
4995
5312
|
runtimeCandidateBudgetExhaustedCases,
|
|
4996
5313
|
runtimeUnderfilledTopKCases,
|
|
4997
5314
|
vectorCases
|
|
@@ -5796,6 +6113,8 @@ var buildRAGEvaluationRunDiff = ({
|
|
|
5796
6113
|
averageLexicalTopK: (current.traceSummary?.averageLexicalTopK ?? 0) - (previous?.traceSummary?.averageLexicalTopK ?? 0),
|
|
5797
6114
|
averageVectorCount: (current.traceSummary?.averageVectorCount ?? 0) - (previous?.traceSummary?.averageVectorCount ?? 0),
|
|
5798
6115
|
balancedCases: (current.traceSummary?.balancedCases ?? 0) - (previous?.traceSummary?.balancedCases ?? 0),
|
|
6116
|
+
officeEvidenceReconcileCasesDelta: (current.traceSummary?.officeEvidenceReconcileCases ?? 0) - (previous?.traceSummary?.officeEvidenceReconcileCases ?? 0),
|
|
6117
|
+
pdfEvidenceReconcileCasesDelta: (current.traceSummary?.pdfEvidenceReconcileCases ?? 0) - (previous?.traceSummary?.pdfEvidenceReconcileCases ?? 0),
|
|
5799
6118
|
lexicalCases: (current.traceSummary?.lexicalCases ?? 0) - (previous?.traceSummary?.lexicalCases ?? 0),
|
|
5800
6119
|
modesChanged: (current.traceSummary?.modes ?? []).join("|") !== (previous?.traceSummary?.modes ?? []).join("|"),
|
|
5801
6120
|
roundRobinCases: (current.traceSummary?.roundRobinCases ?? 0) - (previous?.traceSummary?.roundRobinCases ?? 0),
|
|
@@ -8580,6 +8899,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8580
8899
|
multiVectorCollapsedCasesDelta: (candidateEntry.traceSummary?.multiVectorCollapsedCases ?? 0) - (baselineEntry.traceSummary?.multiVectorCollapsedCases ?? 0),
|
|
8581
8900
|
multiVectorLexicalHitCasesDelta: (candidateEntry.traceSummary?.multiVectorLexicalHitCases ?? 0) - (baselineEntry.traceSummary?.multiVectorLexicalHitCases ?? 0),
|
|
8582
8901
|
multiVectorVectorHitCasesDelta: (candidateEntry.traceSummary?.multiVectorVectorHitCases ?? 0) - (baselineEntry.traceSummary?.multiVectorVectorHitCases ?? 0),
|
|
8902
|
+
evidenceReconcileCasesDelta: (candidateEntry.traceSummary?.stageCounts?.evidence_reconcile ?? 0) - (baselineEntry.traceSummary?.stageCounts?.evidence_reconcile ?? 0),
|
|
8583
8903
|
runtimeCandidateBudgetExhaustedCasesDelta: (candidateEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases ?? 0) - (baselineEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases ?? 0),
|
|
8584
8904
|
runtimeUnderfilledTopKCasesDelta: (candidateEntry.traceSummary?.runtimeUnderfilledTopKCases ?? 0) - (baselineEntry.traceSummary?.runtimeUnderfilledTopKCases ?? 0)
|
|
8585
8905
|
} : undefined;
|
|
@@ -8591,6 +8911,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8591
8911
|
multiVectorCollapsedCases: baselineEntry.traceSummary?.multiVectorCollapsedCases,
|
|
8592
8912
|
multiVectorLexicalHitCases: baselineEntry.traceSummary?.multiVectorLexicalHitCases,
|
|
8593
8913
|
multiVectorVectorHitCases: baselineEntry.traceSummary?.multiVectorVectorHitCases,
|
|
8914
|
+
evidenceReconcileCases: baselineEntry.traceSummary?.stageCounts?.evidence_reconcile,
|
|
8594
8915
|
runtimeCandidateBudgetExhaustedCases: baselineEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases,
|
|
8595
8916
|
runtimeUnderfilledTopKCases: baselineEntry.traceSummary?.runtimeUnderfilledTopKCases,
|
|
8596
8917
|
passingRate: baselineEntry.response.passingRate,
|
|
@@ -8604,6 +8925,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8604
8925
|
multiVectorCollapsedCases: candidateEntry.traceSummary?.multiVectorCollapsedCases,
|
|
8605
8926
|
multiVectorLexicalHitCases: candidateEntry.traceSummary?.multiVectorLexicalHitCases,
|
|
8606
8927
|
multiVectorVectorHitCases: candidateEntry.traceSummary?.multiVectorVectorHitCases,
|
|
8928
|
+
evidenceReconcileCases: candidateEntry.traceSummary?.stageCounts?.evidence_reconcile,
|
|
8607
8929
|
runtimeCandidateBudgetExhaustedCases: candidateEntry.traceSummary?.runtimeCandidateBudgetExhaustedCases,
|
|
8608
8930
|
runtimeUnderfilledTopKCases: candidateEntry.traceSummary?.runtimeUnderfilledTopKCases,
|
|
8609
8931
|
passingRate: candidateEntry.response.passingRate,
|
|
@@ -8618,6 +8940,7 @@ var buildRAGRetrievalComparisonDecisionSummary = ({
|
|
|
8618
8940
|
winnerByMultivectorCollapsedCases: comparison.summary.bestByMultivectorCollapsedCases,
|
|
8619
8941
|
winnerByMultivectorLexicalHitCases: comparison.summary.bestByMultivectorLexicalHitCases,
|
|
8620
8942
|
winnerByMultivectorVectorHitCases: comparison.summary.bestByMultivectorVectorHitCases,
|
|
8943
|
+
winnerByEvidenceReconcileCases: comparison.summary.bestByEvidenceReconcileCases,
|
|
8621
8944
|
winnerByLowestRuntimeCandidateBudgetExhaustedCases: comparison.summary.bestByLowestRuntimeCandidateBudgetExhaustedCases,
|
|
8622
8945
|
winnerByLowestRuntimeUnderfilledTopKCases: comparison.summary.bestByLowestRuntimeUnderfilledTopKCases
|
|
8623
8946
|
};
|
|
@@ -9325,6 +9648,27 @@ var selectComparisonEntryByLowestTraceMetric = (entries, idKey, metric) => {
|
|
|
9325
9648
|
const winner = ranked[0];
|
|
9326
9649
|
return typeof winner?.[idKey] === "string" ? winner[idKey] : undefined;
|
|
9327
9650
|
};
|
|
9651
|
+
var selectComparisonEntryByTraceStageCount = (entries, idKey, stage) => {
|
|
9652
|
+
const ranked = [...entries].sort((left, right) => {
|
|
9653
|
+
const leftMetric = left.traceSummary?.stageCounts?.[stage] ?? 0;
|
|
9654
|
+
const rightMetric = right.traceSummary?.stageCounts?.[stage] ?? 0;
|
|
9655
|
+
if (rightMetric !== leftMetric) {
|
|
9656
|
+
return rightMetric - leftMetric;
|
|
9657
|
+
}
|
|
9658
|
+
if (right.response.passingRate !== left.response.passingRate) {
|
|
9659
|
+
return right.response.passingRate - left.response.passingRate;
|
|
9660
|
+
}
|
|
9661
|
+
if (right.response.summary.averageF1 !== left.response.summary.averageF1) {
|
|
9662
|
+
return right.response.summary.averageF1 - left.response.summary.averageF1;
|
|
9663
|
+
}
|
|
9664
|
+
return left.response.summary.averageLatencyMs - right.response.summary.averageLatencyMs;
|
|
9665
|
+
});
|
|
9666
|
+
const winner = ranked[0];
|
|
9667
|
+
if (!winner || (winner.traceSummary?.stageCounts?.[stage] ?? 0) === 0) {
|
|
9668
|
+
return;
|
|
9669
|
+
}
|
|
9670
|
+
return typeof winner?.[idKey] === "string" ? winner[idKey] : undefined;
|
|
9671
|
+
};
|
|
9328
9672
|
var resolveRetrievalMode = (candidate) => {
|
|
9329
9673
|
if (!candidate.retrieval) {
|
|
9330
9674
|
return "vector";
|
|
@@ -9419,6 +9763,8 @@ var compareRAGRetrievalTraceSummaries = (current, previous) => ({
|
|
|
9419
9763
|
multiVectorVectorHitCasesDelta: current.multiVectorVectorHitCases - previous.multiVectorVectorHitCases,
|
|
9420
9764
|
multiVectorLexicalHitCasesDelta: current.multiVectorLexicalHitCases - previous.multiVectorLexicalHitCases,
|
|
9421
9765
|
multiVectorCollapsedCasesDelta: current.multiVectorCollapsedCases - previous.multiVectorCollapsedCases,
|
|
9766
|
+
officeEvidenceReconcileCasesDelta: current.officeEvidenceReconcileCases - previous.officeEvidenceReconcileCases,
|
|
9767
|
+
pdfEvidenceReconcileCasesDelta: current.pdfEvidenceReconcileCases - previous.pdfEvidenceReconcileCases,
|
|
9422
9768
|
runtimeCandidateBudgetExhaustedCasesDelta: current.runtimeCandidateBudgetExhaustedCases - previous.runtimeCandidateBudgetExhaustedCases,
|
|
9423
9769
|
runtimeUnderfilledTopKCasesDelta: current.runtimeUnderfilledTopKCases - previous.runtimeUnderfilledTopKCases
|
|
9424
9770
|
});
|
|
@@ -9784,10 +10130,129 @@ var generateRAGEvaluationSuiteFromDocuments = ({
|
|
|
9784
10130
|
};
|
|
9785
10131
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_SUITE_ID = "rag-native-planner-larger-corpus";
|
|
9786
10132
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_LABEL = "Adaptive Native Planner Benchmark";
|
|
10133
|
+
var DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_SUITE_ID = "rag-native-backend-larger-corpus";
|
|
10134
|
+
var DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_LABEL = "Native Backend Comparison Benchmark";
|
|
9787
10135
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_QUERY = "Which launch checklist phrase is exact wording?";
|
|
10136
|
+
var DEFAULT_NATIVE_BACKEND_HYBRID_QUERY = "aurora promotion checklist wording";
|
|
10137
|
+
var DEFAULT_NATIVE_BACKEND_FILTERED_QUERY = "focus lane launch checklist wording";
|
|
10138
|
+
var DEFAULT_NATIVE_BACKEND_REORDERED_QUERY = "exact aurora focus lane checklist wording";
|
|
10139
|
+
var DEFAULT_NATIVE_BACKEND_GUIDE_QUERY = "which focus lane guide contains exact aurora promotion wording";
|
|
9788
10140
|
var DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER = {
|
|
9789
10141
|
lane: "focus"
|
|
9790
10142
|
};
|
|
10143
|
+
var DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS = [
|
|
10144
|
+
"focus-distractor-0",
|
|
10145
|
+
"focus-distractor-1",
|
|
10146
|
+
"focus-distractor-2"
|
|
10147
|
+
];
|
|
10148
|
+
var createRAGNativeBackendBenchmarkMockEmbedding = async (text) => {
|
|
10149
|
+
const normalized = text.toLowerCase();
|
|
10150
|
+
if (normalized.includes("launch checklist exact wording for aurora promotion") || normalized.includes("launch checklist exact wording")) {
|
|
10151
|
+
return [0.995, 0.005];
|
|
10152
|
+
}
|
|
10153
|
+
if (normalized.includes("aurora") || normalized.includes("checklist") || normalized.includes("focus lane") || normalized.includes("exact wording") || normalized.includes("guide")) {
|
|
10154
|
+
return [1, 0];
|
|
10155
|
+
}
|
|
10156
|
+
return [0, 1];
|
|
10157
|
+
};
|
|
10158
|
+
var createRAGNativeBackendBenchmarkCorpus = (input) => {
|
|
10159
|
+
const noiseCount = input?.noiseCount ?? 5001;
|
|
10160
|
+
const backend = input?.backend ?? "generic";
|
|
10161
|
+
const genericChunks = [
|
|
10162
|
+
...Array.from({ length: noiseCount }, (_, index) => ({
|
|
10163
|
+
chunkId: `noise:${index}`,
|
|
10164
|
+
corpusKey: "noise",
|
|
10165
|
+
embedding: [0, 1],
|
|
10166
|
+
metadata: {
|
|
10167
|
+
corpusKey: "noise",
|
|
10168
|
+
documentId: `noise-${index}`,
|
|
10169
|
+
lane: "noise"
|
|
10170
|
+
},
|
|
10171
|
+
source: `noise/${index}.md`,
|
|
10172
|
+
text: `Background operations note ${index}.`
|
|
10173
|
+
})),
|
|
10174
|
+
...Array.from({ length: 3 }, (_, index) => ({
|
|
10175
|
+
chunkId: `focus:distractor:${index}`,
|
|
10176
|
+
corpusKey: "focus",
|
|
10177
|
+
embedding: [1, 0],
|
|
10178
|
+
metadata: {
|
|
10179
|
+
corpusKey: "focus",
|
|
10180
|
+
documentId: `focus-distractor-${index}`,
|
|
10181
|
+
lane: "focus"
|
|
10182
|
+
},
|
|
10183
|
+
source: `focus/distractor-${index}.md`,
|
|
10184
|
+
text: index === 0 ? "aurora promotion checklist overview" : index === 1 ? "launch checklist wording draft" : "focus lane promotion runbook notes"
|
|
10185
|
+
})),
|
|
10186
|
+
{
|
|
10187
|
+
chunkId: "focus:target",
|
|
10188
|
+
corpusKey: "focus",
|
|
10189
|
+
embedding: [0.995, 0.005],
|
|
10190
|
+
metadata: {
|
|
10191
|
+
corpusKey: "focus",
|
|
10192
|
+
documentId: "focus-target",
|
|
10193
|
+
lane: "focus"
|
|
10194
|
+
},
|
|
10195
|
+
source: "guide/planner-depth.md",
|
|
10196
|
+
text: "launch checklist exact wording for aurora promotion in the focus lane"
|
|
10197
|
+
}
|
|
10198
|
+
];
|
|
10199
|
+
const backendSpecificChunks = backend === "sqlite-native" ? [
|
|
10200
|
+
{
|
|
10201
|
+
chunkId: "focus:sqlite:phrase-matrix",
|
|
10202
|
+
corpusKey: "focus",
|
|
10203
|
+
embedding: [1, 0],
|
|
10204
|
+
metadata: {
|
|
10205
|
+
backendFixture: "sqlite-native",
|
|
10206
|
+
corpusKey: "focus",
|
|
10207
|
+
documentId: "focus-sqlite-phrase-matrix",
|
|
10208
|
+
lane: "focus"
|
|
10209
|
+
},
|
|
10210
|
+
source: "guide/sqlite-phrase-matrix.md",
|
|
10211
|
+
text: "exact aurora focus lane checklist wording matrix for sqlite validation"
|
|
10212
|
+
},
|
|
10213
|
+
{
|
|
10214
|
+
chunkId: "focus:sqlite:guide-table",
|
|
10215
|
+
corpusKey: "focus",
|
|
10216
|
+
embedding: [1, 0],
|
|
10217
|
+
metadata: {
|
|
10218
|
+
backendFixture: "sqlite-native",
|
|
10219
|
+
corpusKey: "focus",
|
|
10220
|
+
documentId: "focus-sqlite-guide-table",
|
|
10221
|
+
lane: "focus"
|
|
10222
|
+
},
|
|
10223
|
+
source: "guide/sqlite-guide-table.md",
|
|
10224
|
+
text: "which focus lane guide contains aurora promotion wording draft table for sqlite operators"
|
|
10225
|
+
}
|
|
10226
|
+
] : backend === "postgres" ? [
|
|
10227
|
+
{
|
|
10228
|
+
chunkId: "focus:postgres:appendix",
|
|
10229
|
+
corpusKey: "focus",
|
|
10230
|
+
embedding: [1, 0],
|
|
10231
|
+
metadata: {
|
|
10232
|
+
backendFixture: "postgres",
|
|
10233
|
+
corpusKey: "focus",
|
|
10234
|
+
documentId: "focus-postgres-appendix",
|
|
10235
|
+
lane: "focus"
|
|
10236
|
+
},
|
|
10237
|
+
source: "guide/postgres-appendix.md",
|
|
10238
|
+
text: "which focus lane guide contains exact aurora promotion wording appendix for postgres release review"
|
|
10239
|
+
},
|
|
10240
|
+
{
|
|
10241
|
+
chunkId: "focus:postgres:alternatives",
|
|
10242
|
+
corpusKey: "focus",
|
|
10243
|
+
embedding: [1, 0],
|
|
10244
|
+
metadata: {
|
|
10245
|
+
backendFixture: "postgres",
|
|
10246
|
+
corpusKey: "focus",
|
|
10247
|
+
documentId: "focus-postgres-alternatives",
|
|
10248
|
+
lane: "focus"
|
|
10249
|
+
},
|
|
10250
|
+
source: "guide/postgres-alternatives.md",
|
|
10251
|
+
text: "aurora promotion checklist wording alternatives and exact focus lane phrasing for postgres audits"
|
|
10252
|
+
}
|
|
10253
|
+
] : [];
|
|
10254
|
+
return [...genericChunks, ...backendSpecificChunks];
|
|
10255
|
+
};
|
|
9791
10256
|
var createRAGAdaptiveNativePlannerBenchmarkSuite = (input) => createRAGEvaluationSuite({
|
|
9792
10257
|
description: input?.description ?? "Stress-tests larger-corpus native planner selection, candidate-budget pressure, and transformed-query recovery on filtered retrieval.",
|
|
9793
10258
|
id: input?.id ?? DEFAULT_NATIVE_PLANNER_BENCHMARK_SUITE_ID,
|
|
@@ -9839,6 +10304,102 @@ var createRAGAdaptiveNativePlannerBenchmarkSnapshot = (input) => {
|
|
|
9839
10304
|
version: input?.version
|
|
9840
10305
|
});
|
|
9841
10306
|
};
|
|
10307
|
+
var createRAGNativeBackendComparisonBenchmarkSuite = (input) => createRAGEvaluationSuite({
|
|
10308
|
+
description: input?.description ?? "Captures larger-corpus native backend parity with filtered vector pressure and harder hybrid retrieval cases so sqlite-native and postgres runs can be compared over time.",
|
|
10309
|
+
id: input?.id ?? DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_SUITE_ID,
|
|
10310
|
+
input: {
|
|
10311
|
+
cases: [
|
|
10312
|
+
{
|
|
10313
|
+
expectedDocumentIds: ["focus-target"],
|
|
10314
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10315
|
+
hardNegativeDocumentIds: [
|
|
10316
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10317
|
+
],
|
|
10318
|
+
id: "planner-pressure-exact-phrase",
|
|
10319
|
+
label: "Exact phrase survives larger-corpus native pressure",
|
|
10320
|
+
query: DEFAULT_NATIVE_PLANNER_BENCHMARK_QUERY,
|
|
10321
|
+
topK: input?.topK ?? 1
|
|
10322
|
+
},
|
|
10323
|
+
{
|
|
10324
|
+
expectedDocumentIds: ["focus-target"],
|
|
10325
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10326
|
+
hardNegativeDocumentIds: [
|
|
10327
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10328
|
+
],
|
|
10329
|
+
id: "planner-pressure-hybrid-phrase",
|
|
10330
|
+
label: "Hybrid retrieval survives filtered lexical pressure",
|
|
10331
|
+
query: DEFAULT_NATIVE_BACKEND_HYBRID_QUERY,
|
|
10332
|
+
topK: input?.topK ?? 1
|
|
10333
|
+
},
|
|
10334
|
+
{
|
|
10335
|
+
expectedDocumentIds: ["focus-target"],
|
|
10336
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10337
|
+
hardNegativeDocumentIds: [
|
|
10338
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10339
|
+
],
|
|
10340
|
+
id: "planner-pressure-filtered-lane-query",
|
|
10341
|
+
label: "Filtered lane query survives broader corpus noise",
|
|
10342
|
+
query: DEFAULT_NATIVE_BACKEND_FILTERED_QUERY,
|
|
10343
|
+
topK: input?.topK ?? 1
|
|
10344
|
+
},
|
|
10345
|
+
{
|
|
10346
|
+
expectedDocumentIds: ["focus-target"],
|
|
10347
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10348
|
+
hardNegativeDocumentIds: [
|
|
10349
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10350
|
+
],
|
|
10351
|
+
id: "planner-pressure-reordered-phrase",
|
|
10352
|
+
label: "Reordered phrase survives transform pressure",
|
|
10353
|
+
query: DEFAULT_NATIVE_BACKEND_REORDERED_QUERY,
|
|
10354
|
+
topK: input?.topK ?? 1
|
|
10355
|
+
},
|
|
10356
|
+
{
|
|
10357
|
+
expectedDocumentIds: ["focus-target"],
|
|
10358
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10359
|
+
hardNegativeDocumentIds: [
|
|
10360
|
+
...DEFAULT_NATIVE_PLANNER_HARD_NEGATIVE_DOCUMENT_IDS
|
|
10361
|
+
],
|
|
10362
|
+
id: "planner-pressure-guide-query",
|
|
10363
|
+
label: "Guide attribution survives filtered corpus pressure",
|
|
10364
|
+
query: DEFAULT_NATIVE_BACKEND_GUIDE_QUERY,
|
|
10365
|
+
topK: input?.topK ?? 1
|
|
10366
|
+
}
|
|
10367
|
+
],
|
|
10368
|
+
filter: { ...DEFAULT_NATIVE_PLANNER_BENCHMARK_FILTER },
|
|
10369
|
+
retrieval: "vector",
|
|
10370
|
+
topK: input?.topK ?? 1
|
|
10371
|
+
},
|
|
10372
|
+
label: input?.label ?? DEFAULT_NATIVE_BACKEND_COMPARISON_BENCHMARK_LABEL,
|
|
10373
|
+
metadata: {
|
|
10374
|
+
benchmarkKind: "native_backend_comparison",
|
|
10375
|
+
benchmarkScope: "larger_corpus",
|
|
10376
|
+
expectedSignals: [
|
|
10377
|
+
"backend-tagged runtime artifacts",
|
|
10378
|
+
"selected native planner profile",
|
|
10379
|
+
"hybrid filtered retrieval",
|
|
10380
|
+
"candidate-budget exhaustion",
|
|
10381
|
+
"underfilled topk",
|
|
10382
|
+
"query transform pressure"
|
|
10383
|
+
],
|
|
10384
|
+
recommendedGroupKey: "runtime-native-backend-parity",
|
|
10385
|
+
recommendedTags: ["runtime", "backend", "native"],
|
|
10386
|
+
...input?.metadata
|
|
10387
|
+
}
|
|
10388
|
+
});
|
|
10389
|
+
var createRAGNativeBackendComparisonBenchmarkSnapshot = (input) => {
|
|
10390
|
+
const suite = input?.suite ?? createRAGNativeBackendComparisonBenchmarkSuite();
|
|
10391
|
+
return createRAGEvaluationSuiteSnapshot({
|
|
10392
|
+
createdAt: input?.createdAt,
|
|
10393
|
+
id: input?.id,
|
|
10394
|
+
metadata: {
|
|
10395
|
+
artifactKind: "native_backend_comparison_benchmark",
|
|
10396
|
+
persistForReleaseHistory: true,
|
|
10397
|
+
...input?.metadata
|
|
10398
|
+
},
|
|
10399
|
+
suite,
|
|
10400
|
+
version: input?.version
|
|
10401
|
+
});
|
|
10402
|
+
};
|
|
9842
10403
|
var createRAGEvaluationSuiteSnapshot = ({
|
|
9843
10404
|
suite,
|
|
9844
10405
|
id,
|
|
@@ -10047,6 +10608,7 @@ var summarizeRAGRetrievalComparison = (entries) => ({
|
|
|
10047
10608
|
bestByMultivectorCollapsedCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorCollapsedCases"),
|
|
10048
10609
|
bestByMultivectorLexicalHitCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorLexicalHitCases"),
|
|
10049
10610
|
bestByMultivectorVectorHitCases: selectComparisonEntryByTraceMetric(entries, "retrievalId", "multiVectorVectorHitCases"),
|
|
10611
|
+
bestByEvidenceReconcileCases: selectComparisonEntryByTraceStageCount(entries, "retrievalId", "evidence_reconcile"),
|
|
10050
10612
|
bestByLowestRuntimeCandidateBudgetExhaustedCases: selectComparisonEntryByLowestTraceMetric(entries, "retrievalId", "runtimeCandidateBudgetExhaustedCases"),
|
|
10051
10613
|
bestByLowestRuntimeUnderfilledTopKCases: selectComparisonEntryByLowestTraceMetric(entries, "retrievalId", "runtimeUnderfilledTopKCases")
|
|
10052
10614
|
});
|
|
@@ -11094,12 +11656,21 @@ var createRAGClient = (options) => {
|
|
|
11094
11656
|
if (typeof input?.limit === "number") {
|
|
11095
11657
|
searchParams.set("limit", String(input.limit));
|
|
11096
11658
|
}
|
|
11659
|
+
if (typeof input?.runLimit === "number") {
|
|
11660
|
+
searchParams.set("runLimit", String(input.runLimit));
|
|
11661
|
+
}
|
|
11097
11662
|
if (input?.label) {
|
|
11098
11663
|
searchParams.set("label", input.label);
|
|
11099
11664
|
}
|
|
11100
11665
|
if (input?.description) {
|
|
11101
11666
|
searchParams.set("description", input.description);
|
|
11102
11667
|
}
|
|
11668
|
+
if (input?.groupKey) {
|
|
11669
|
+
searchParams.set("benchmarkGroupKey", input.groupKey);
|
|
11670
|
+
}
|
|
11671
|
+
if (input?.corpusGroupKey) {
|
|
11672
|
+
searchParams.set("benchmarkCorpusGroupKey", input.corpusGroupKey);
|
|
11673
|
+
}
|
|
11103
11674
|
const suffix = searchParams.size ? `?${searchParams}` : "";
|
|
11104
11675
|
const response = await fetchImpl(`${basePath}/compare/retrieval/benchmarks/adaptive-native-planner${suffix}`);
|
|
11105
11676
|
if (!response.ok) {
|
|
@@ -11111,6 +11682,35 @@ var createRAGClient = (options) => {
|
|
|
11111
11682
|
}
|
|
11112
11683
|
return payload;
|
|
11113
11684
|
},
|
|
11685
|
+
async runAdaptiveNativePlannerBenchmark(input) {
|
|
11686
|
+
const response = await fetchImpl(`${basePath}/compare/retrieval/benchmarks/adaptive-native-planner/run`, {
|
|
11687
|
+
body: JSON.stringify({
|
|
11688
|
+
baselineRetrievalId: input?.baselineRetrievalId,
|
|
11689
|
+
candidateRetrievalId: input?.candidateRetrievalId,
|
|
11690
|
+
corpusGroupKey: input?.corpusGroupKey,
|
|
11691
|
+
description: input?.description,
|
|
11692
|
+
groupKey: input?.groupKey,
|
|
11693
|
+
label: input?.label,
|
|
11694
|
+
limit: input?.limit,
|
|
11695
|
+
metadata: input?.metadata,
|
|
11696
|
+
persistRun: input?.persistRun,
|
|
11697
|
+
retrievals: input?.retrievals,
|
|
11698
|
+
runLimit: input?.runLimit,
|
|
11699
|
+
tags: input?.tags,
|
|
11700
|
+
topK: input?.topK
|
|
11701
|
+
}),
|
|
11702
|
+
headers: jsonHeaders,
|
|
11703
|
+
method: "POST"
|
|
11704
|
+
});
|
|
11705
|
+
if (!response.ok) {
|
|
11706
|
+
throw new Error(await toErrorMessage(response));
|
|
11707
|
+
}
|
|
11708
|
+
const payload = await parseJson(response);
|
|
11709
|
+
if (!payload.ok) {
|
|
11710
|
+
throw new Error(payload.error ?? "Adaptive native planner benchmark run failed");
|
|
11711
|
+
}
|
|
11712
|
+
return payload;
|
|
11713
|
+
},
|
|
11114
11714
|
async saveAdaptiveNativePlannerBenchmarkSnapshot(input) {
|
|
11115
11715
|
const response = await fetchImpl(`${basePath}/compare/retrieval/benchmarks/adaptive-native-planner/snapshots`, {
|
|
11116
11716
|
body: JSON.stringify({
|
|
@@ -11134,6 +11734,89 @@ var createRAGClient = (options) => {
|
|
|
11134
11734
|
}
|
|
11135
11735
|
return payload;
|
|
11136
11736
|
},
|
|
11737
|
+
async nativeBackendComparisonBenchmark(input) {
|
|
11738
|
+
const searchParams = new URLSearchParams;
|
|
11739
|
+
if (typeof input?.limit === "number") {
|
|
11740
|
+
searchParams.set("limit", String(input.limit));
|
|
11741
|
+
}
|
|
11742
|
+
if (typeof input?.runLimit === "number") {
|
|
11743
|
+
searchParams.set("runLimit", String(input.runLimit));
|
|
11744
|
+
}
|
|
11745
|
+
if (input?.label) {
|
|
11746
|
+
searchParams.set("label", input.label);
|
|
11747
|
+
}
|
|
11748
|
+
if (input?.description) {
|
|
11749
|
+
searchParams.set("description", input.description);
|
|
11750
|
+
}
|
|
11751
|
+
if (input?.groupKey) {
|
|
11752
|
+
searchParams.set("benchmarkGroupKey", input.groupKey);
|
|
11753
|
+
}
|
|
11754
|
+
if (input?.corpusGroupKey) {
|
|
11755
|
+
searchParams.set("benchmarkCorpusGroupKey", input.corpusGroupKey);
|
|
11756
|
+
}
|
|
11757
|
+
const suffix = searchParams.size ? `?${searchParams}` : "";
|
|
11758
|
+
const response = await fetchImpl(`${basePath}/compare/retrieval/benchmarks/native-backend-comparison${suffix}`);
|
|
11759
|
+
if (!response.ok) {
|
|
11760
|
+
throw new Error(await toErrorMessage(response));
|
|
11761
|
+
}
|
|
11762
|
+
const payload = await parseJson(response);
|
|
11763
|
+
if (!payload.ok) {
|
|
11764
|
+
throw new Error(payload.error ?? "Native backend comparison benchmark history failed");
|
|
11765
|
+
}
|
|
11766
|
+
return payload;
|
|
11767
|
+
},
|
|
11768
|
+
async runNativeBackendComparisonBenchmark(input) {
|
|
11769
|
+
const response = await fetchImpl(`${basePath}/compare/retrieval/benchmarks/native-backend-comparison/run`, {
|
|
11770
|
+
body: JSON.stringify({
|
|
11771
|
+
baselineRetrievalId: input?.baselineRetrievalId,
|
|
11772
|
+
candidateRetrievalId: input?.candidateRetrievalId,
|
|
11773
|
+
corpusGroupKey: input?.corpusGroupKey,
|
|
11774
|
+
description: input?.description,
|
|
11775
|
+
groupKey: input?.groupKey,
|
|
11776
|
+
label: input?.label,
|
|
11777
|
+
limit: input?.limit,
|
|
11778
|
+
metadata: input?.metadata,
|
|
11779
|
+
persistRun: input?.persistRun,
|
|
11780
|
+
retrievals: input?.retrievals,
|
|
11781
|
+
runLimit: input?.runLimit,
|
|
11782
|
+
tags: input?.tags,
|
|
11783
|
+
topK: input?.topK
|
|
11784
|
+
}),
|
|
11785
|
+
headers: jsonHeaders,
|
|
11786
|
+
method: "POST"
|
|
11787
|
+
});
|
|
11788
|
+
if (!response.ok) {
|
|
11789
|
+
throw new Error(await toErrorMessage(response));
|
|
11790
|
+
}
|
|
11791
|
+
const payload = await parseJson(response);
|
|
11792
|
+
if (!payload.ok) {
|
|
11793
|
+
throw new Error(payload.error ?? "Native backend comparison benchmark run failed");
|
|
11794
|
+
}
|
|
11795
|
+
return payload;
|
|
11796
|
+
},
|
|
11797
|
+
async saveNativeBackendComparisonBenchmarkSnapshot(input) {
|
|
11798
|
+
const response = await fetchImpl(`${basePath}/compare/retrieval/benchmarks/native-backend-comparison/snapshots`, {
|
|
11799
|
+
body: JSON.stringify({
|
|
11800
|
+
createdAt: input?.createdAt,
|
|
11801
|
+
description: input?.description,
|
|
11802
|
+
label: input?.label,
|
|
11803
|
+
limit: input?.limit,
|
|
11804
|
+
metadata: input?.metadata,
|
|
11805
|
+
snapshotMetadata: input?.snapshotMetadata,
|
|
11806
|
+
version: input?.version
|
|
11807
|
+
}),
|
|
11808
|
+
headers: jsonHeaders,
|
|
11809
|
+
method: "POST"
|
|
11810
|
+
});
|
|
11811
|
+
if (!response.ok) {
|
|
11812
|
+
throw new Error(await toErrorMessage(response));
|
|
11813
|
+
}
|
|
11814
|
+
const payload = await parseJson(response);
|
|
11815
|
+
if (!payload.ok) {
|
|
11816
|
+
throw new Error(payload.error ?? "Native backend comparison benchmark snapshot failed");
|
|
11817
|
+
}
|
|
11818
|
+
return payload;
|
|
11819
|
+
},
|
|
11137
11820
|
async retrievalLaneHandoffs(input) {
|
|
11138
11821
|
const searchParams = new URLSearchParams;
|
|
11139
11822
|
if (input?.groupKey) {
|
|
@@ -11928,5 +12611,5 @@ export {
|
|
|
11928
12611
|
buildRAGEvaluationLeaderboard
|
|
11929
12612
|
};
|
|
11930
12613
|
|
|
11931
|
-
//# debugId=
|
|
12614
|
+
//# debugId=72A6DEC169D4AAA764756E2164756E21
|
|
11932
12615
|
//# sourceMappingURL=index.js.map
|