@tangle-network/agent-knowledge 22.0.1 → 22.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +9 -1
- package/CHANGELOG.md +11 -0
- package/README.md +3 -3
- package/dist/cli.js +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +316 -379
- package/dist/index.js.map +1 -1
- package/dist/{inspect-DBxyQSdt.js → inspect-rnlXAb4O.js} +25 -14
- package/dist/{inspect-DBxyQSdt.js.map → inspect-rnlXAb4O.js.map} +1 -1
- package/docs/architecture.md +17 -0
- package/docs/compatibility.md +5 -4
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as loadSourceRegistry, $n as mediaTypeFor, $t as KNOWLEDGE_EVENT_TYPES, A as validateKnowledgePage, An as finishKnowledgeFileTransaction, At as isScaffoldPath, B as CHECKABLE_RUNG_THRESHOLD, Bn as listRegularFilesWithinRoot, Bt as KnowledgeEventSchema, C as immutablePageDigest, Cn as runInKnowledgeMutationScope, Ct as resolveKnowledgeCitation, D as normalizeVisibilityArtifact, Dn as applyKnowledgeFileTransaction, Dt as SCAFFOLD_PAGE_BASENAMES, E as nonEmpty$1, En as withKnowledgeRead, Et as resolveRunScopedCitations, F as detectNearDuplicatePages, Fn as normalizePagesDirectory, Ft as WIKILINK_REGEX, G as gradeClaims, Gn as renameDurable, Gt as KnowledgePageSchema, H as UncheckableClaimError, Hn as readRegularFileWithinRoot, Ht as KnowledgeGraphNodeSchema, I as normalizePageText, In as canonicalPathsEqual, It as extractWikilinks, J as verifyGradeableEvidence, Jn as withSafeDirectory, Jt as ResearchClaimLedgerSchema, K as gradeFor, Kn as syncDirectory, Kt as KnowledgeRelationSchema, L as prepareKnowledgeValidation, Ln as canonicalRelativeWithinRoot, Lt as normalizeLinkTarget, M as verifyKnowledgeVisibilitySnapshot, Mn as prepareKnowledgeFileTransaction, Mt as layoutFor, N as visibilityIdentity, Nn as rollbackKnowledgeFileTransaction, Nt as loadKnowledgePages, O as normalizeVisibilityRef, On as assertKnowledgeMutationPath, Ot as initKnowledgeBase, P as visibilityIndexOf, Pn as DEFAULT_PAGES_DIRECTORY, Pt as writeJson, Q as addSourceText, Qn as writeJsonDurableWithinRoot, Qt as SourceRecordSchema, R as validateKnowledgeIndex, Rn as isKernelAnchoredPath, Rt as DeepQuestionSchema, S as encodeKnowledgeVisibilitySnapshot, Sn as recoverPendingKnowledgeMutation, St as parseKnowledgeCitationReference, T as knowledgeVisibilityArtifactRef, Tn as withKnowledgeMutationLabel, Tt as resolveRunScopedCitation, U as assertGradeableEvidence, Un as relativeWithinRoot, Ut as KnowledgeIndexSchema, V as DEADLINE_EXIT_CODE, Vn as readRegularFileNoFollow, Vt as KnowledgeGraphEdgeSchema, W as claimCheckKey, Wn as removeDurable, Wt as KnowledgePageInvalidationSchema, X as writeKnowledgeIndex, Xn as writeFileDurableWithinRoot, Xt as ResearchSourceVersionSchema, Y as buildKnowledgeIndex, Yn as writeFileDurable, Yt as ResearchClaimRecordSchema, Z as addSourcePath, Zn as writeJsonDurable, Zt as SourceAnchorSchema, _ as KNOWLEDGE_USE_RECEIPT_SCHEMA_VERSION, _n as parseFrontmatter, _t as assertKnowledgeCitationsResolved, a as applyKnowledgeWriteBlocksFile, an as claimEvidenceId, at as KB_CLAIM_LEDGER_DIR, b as decodeKnowledgeVisibilitySnapshot, bn as isKnowledgeMutationHeld, bt as auditKnowledgeCitations, c as KnowledgeDuplicateIntakeError, cn as deepQuestionId, ct as KB_STORE_DIR, d as createFileRunLineageAuthority, dn as materializeRegisteredClaimEvidence, dt as buildKnowledgeGraph, en as ClaimLedgerGoalConflictError, er as textSourceAdapter, et as snapshotSourceTextInput, f as createRunScopedStores, fn as mergeClaimLedgers, ft as knowledgePageRelations, g as KNOWLEDGE_RECEIPT_DIGEST_ALGORITHM, gn as formatFrontmatter, gt as assertKnowledgeCitationAudit, h as originatedPages, hn as researchSourceVersionKey, ht as assertCurrentRunCitationsResolved, i as applyKnowledgeWriteBlocks, in as assertTrackedClaimIntegrity, it as FileSystemKbStore, j as validateOrigin, jn as knowledgeFileTransactionPlanHash, jt as knowledgePageFromMarkdown, k as snapshotImmutableVisibility, kn as commitKnowledgeFileMutations, kt as isKnowledgePagePath, l as assertKnowledgeWriteIntake, ln as emptyClaimLedger, lt as MemoryKbStore, m as loadKnowledgeToolChain, mn as normalizeClaimText, mt as KnowledgeCitationResolutionError, n as inspectKnowledgeIndex, nn as assertResearchClaimEvidenceIntegrity, nt as writeSourceRegistry, o as isSafeKnowledgePath, on as claimId, ot as KB_EVENTS_PATH, p as isKnowledgeToolChain, pn as mergeTrackedClaims, pt as KnowledgeCitationAuditError, q as verdictFor, qn as withSafeDescendant, qt as ResearchClaimEvidenceSchema, r as stringMetadata, rn as assertResearchClaimLedgerIntegrity, rt as ClaimLedgerMigrationRequiredError, s as parseKnowledgeWriteBlocks, sn as claimSourceHost, st as KB_INDEX_PATH, t as explainKnowledgeTarget, tn as assertDeepQuestionIntegrity, tt as sourceRegistryPath, u as RUN_LINEAGE_BASENAME, un as linkClaimContradictions, ut as assertClaimLedgerId, v as assertSchemaVersion, vn as acquireDurableFileLock, vt as assertRunScopedCitationsResolved, w as knowledgePageDigest, wn as withKnowledgeMutation, wt as resolveKnowledgeCitations, x as digest, xn as measureKnowledgeMutation, xt as formatKnowledgeCitationReference, y as createKnowledgeVisibilitySnapshot, yn as inspectPendingKnowledgeMutation, yt as auditCurrentRunCitations, z as lintKnowledgeIndex, zn as isMissingFile, zt as KnowledgeBaseCandidateSchema } from "./inspect-rnlXAb4O.js";
|
|
2
2
|
import { i as textSourceId, n as slugify, r as stableId, t as sha256 } from "./ids-Bevz_pXV.js";
|
|
3
3
|
import { a as buildKnowledgeLexicalIndex, c as tokenizeText, d as lexicalPageTerms, i as searchKnowledgePages, l as DEFAULT_FIELD_BOOSTS, n as reciprocalRankFusion, o as scoreBm25, r as searchKnowledge, s as tokenizeQuery, t as KNOWLEDGE_SEARCH_RETRIEVER_ID, u as assembleLexicalIndex } from "./search-aZdP2685.js";
|
|
4
4
|
import { n as memoryWriteResultToSourceRecord, t as memoryHitToSourceRecord } from "./source-record-CdWvzvO6.js";
|
|
@@ -520,10 +520,177 @@ function exceedsCostCeiling$1(totalCostUsd, costCeiling) {
|
|
|
520
520
|
return totalCostUsd - costCeiling > tolerance;
|
|
521
521
|
}
|
|
522
522
|
//#endregion
|
|
523
|
+
//#region src/optimization.ts
|
|
524
|
+
/**
|
|
525
|
+
* Runs one complete agent-eval method over a serialized knowledge candidate.
|
|
526
|
+
* The method receives train and selection data; agent-eval owns final scoring.
|
|
527
|
+
*/
|
|
528
|
+
async function runSerializedKnowledgeOptimization(options) {
|
|
529
|
+
assertCompleteOptimizationMethod(options.method);
|
|
530
|
+
assertImmutableRef(options.executionRef, "knowledge optimization executionRef");
|
|
531
|
+
const codec = options.codec ?? jsonCandidateCodec();
|
|
532
|
+
const baseline = normalizeCandidate(options.baseline, codec, "baseline");
|
|
533
|
+
const dispatchRef = [
|
|
534
|
+
"knowledge-optimization",
|
|
535
|
+
options.executionRef,
|
|
536
|
+
baseline.surfaceHash,
|
|
537
|
+
surfaceHash(options.method.name)
|
|
538
|
+
].join(":");
|
|
539
|
+
assertPartitionContent(options.trainScenarios, options.selectionScenarios, options.finalScenarios, options.scenarioFingerprint ?? scenarioContentFingerprint);
|
|
540
|
+
const method = canonicalCandidateMethod(options.method, codec);
|
|
541
|
+
const { baseline: _baseline, method: _method, trainScenarios, selectionScenarios, finalScenarios, dispatchCandidate, executionRef: _executionRef, codec: _codec, scenarioFingerprint: _scenarioFingerprint, ...comparisonOptions } = options;
|
|
542
|
+
const optimizationRunOptions = {
|
|
543
|
+
...comparisonOptions.reps !== void 0 ? { reps: comparisonOptions.reps } : {},
|
|
544
|
+
...comparisonOptions.resumable !== void 0 ? { resumable: comparisonOptions.resumable } : {},
|
|
545
|
+
...comparisonOptions.labeledStore !== void 0 ? { labeledStore: comparisonOptions.labeledStore } : {},
|
|
546
|
+
...comparisonOptions.captureSource !== void 0 ? { captureSource: comparisonOptions.captureSource } : {},
|
|
547
|
+
...comparisonOptions.captureSourceVersionHash !== void 0 ? { captureSourceVersionHash: comparisonOptions.captureSourceVersionHash } : {},
|
|
548
|
+
...comparisonOptions.maxConcurrency !== void 0 ? { maxConcurrency: comparisonOptions.maxConcurrency } : {},
|
|
549
|
+
...comparisonOptions.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: comparisonOptions.dispatchTimeoutMs } : {},
|
|
550
|
+
...comparisonOptions.repo !== void 0 ? { repo: comparisonOptions.repo } : {},
|
|
551
|
+
...comparisonOptions.tracing !== void 0 ? { tracing: comparisonOptions.tracing } : {},
|
|
552
|
+
...comparisonOptions.expectUsage !== void 0 ? { expectUsage: comparisonOptions.expectUsage } : {},
|
|
553
|
+
...comparisonOptions.now !== void 0 ? { now: comparisonOptions.now } : {},
|
|
554
|
+
...comparisonOptions.buildTraceWriter !== void 0 ? { buildTraceWriter: comparisonOptions.buildTraceWriter } : {},
|
|
555
|
+
...comparisonOptions.storage !== void 0 ? { storage: comparisonOptions.storage } : {},
|
|
556
|
+
...comparisonOptions.cellPlacement !== void 0 ? { cellPlacement: comparisonOptions.cellPlacement } : {},
|
|
557
|
+
...comparisonOptions.optimizationRunOptions ?? {},
|
|
558
|
+
dispatchRef
|
|
559
|
+
};
|
|
560
|
+
const comparison = await compareOptimizationMethods({
|
|
561
|
+
...comparisonOptions,
|
|
562
|
+
dispatchRef,
|
|
563
|
+
optimizationRunOptions,
|
|
564
|
+
methods: [method],
|
|
565
|
+
baselineSurface: baseline.surface,
|
|
566
|
+
trainScenarios: trainScenarios.map((scenario) => structuredClone(scenario)),
|
|
567
|
+
selectionScenarios: selectionScenarios.map((scenario) => structuredClone(scenario)),
|
|
568
|
+
testScenarios: finalScenarios.map((scenario) => structuredClone(scenario)),
|
|
569
|
+
dispatchWithSurface: async (surface, scenario, context) => {
|
|
570
|
+
const candidate = normalizeSurface(surface, codec, "candidate");
|
|
571
|
+
return dispatchCandidate({
|
|
572
|
+
candidate: structuredClone(candidate.value),
|
|
573
|
+
candidateSurface: candidate.surface,
|
|
574
|
+
candidateSurfaceHash: candidate.surfaceHash,
|
|
575
|
+
scenario,
|
|
576
|
+
context
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
});
|
|
580
|
+
const winner = normalizeSurface(comparison.best.winnerSurface, codec, "winner");
|
|
581
|
+
return {
|
|
582
|
+
methodName: options.method.name,
|
|
583
|
+
baseline,
|
|
584
|
+
winner,
|
|
585
|
+
comparison
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
function assertCompleteOptimizationMethod(method) {
|
|
589
|
+
const candidate = method;
|
|
590
|
+
if (!candidate || typeof candidate.name !== "string" || candidate.name.trim().length === 0 || typeof candidate.optimize !== "function") throw new Error("knowledge optimization requires a complete OptimizationMethod with a non-empty name and optimize()");
|
|
591
|
+
}
|
|
592
|
+
function jsonCandidateCodec() {
|
|
593
|
+
return {
|
|
594
|
+
serialize: (candidate) => canonicalJson(candidate),
|
|
595
|
+
parse(surface) {
|
|
596
|
+
let parsed;
|
|
597
|
+
try {
|
|
598
|
+
parsed = JSON.parse(surface);
|
|
599
|
+
} catch (error) {
|
|
600
|
+
throw new Error("serialized knowledge candidate is not valid JSON", { cause: error });
|
|
601
|
+
}
|
|
602
|
+
return parsed;
|
|
603
|
+
}
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
function jsonObjectCandidateCodec() {
|
|
607
|
+
const codec = jsonCandidateCodec();
|
|
608
|
+
return {
|
|
609
|
+
serialize: codec.serialize,
|
|
610
|
+
parse(surface) {
|
|
611
|
+
const candidate = codec.parse(surface);
|
|
612
|
+
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) throw new Error("serialized knowledge candidate must be a JSON object");
|
|
613
|
+
return candidate;
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
function scenarioContentFingerprint(scenario) {
|
|
618
|
+
const { id: _id, tags: _tags, split: _split, splitTag: _splitTag, ...content } = scenario;
|
|
619
|
+
return surfaceHash(canonicalJson(content));
|
|
620
|
+
}
|
|
621
|
+
function canonicalCandidateMethod(method, codec) {
|
|
622
|
+
return {
|
|
623
|
+
name: method.name,
|
|
624
|
+
async optimize(input) {
|
|
625
|
+
const result = await method.optimize(input);
|
|
626
|
+
const winner = normalizeSurface(result.winnerSurface, codec, `${method.name} winner`);
|
|
627
|
+
return {
|
|
628
|
+
...result,
|
|
629
|
+
winnerSurface: winner.surface
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
};
|
|
633
|
+
}
|
|
634
|
+
function normalizeCandidate(candidate, codec, label) {
|
|
635
|
+
let surface;
|
|
636
|
+
try {
|
|
637
|
+
surface = codec.serialize(candidate);
|
|
638
|
+
} catch (error) {
|
|
639
|
+
throw new Error(`${label} serializer failed`, { cause: error });
|
|
640
|
+
}
|
|
641
|
+
return normalizeSurface(surface, codec, label);
|
|
642
|
+
}
|
|
643
|
+
function normalizeSurface(surface, codec, label) {
|
|
644
|
+
if (typeof surface !== "string" || surface.trim().length === 0) throw new Error(`${label} must be a non-empty serialized string`);
|
|
645
|
+
let value;
|
|
646
|
+
try {
|
|
647
|
+
value = codec.parse(surface);
|
|
648
|
+
} catch (error) {
|
|
649
|
+
const detail = error instanceof Error ? `: ${error.message}` : "";
|
|
650
|
+
throw new Error(`${label} parser failed${detail}`, { cause: error });
|
|
651
|
+
}
|
|
652
|
+
let canonicalSurface;
|
|
653
|
+
try {
|
|
654
|
+
canonicalSurface = codec.serialize(value);
|
|
655
|
+
} catch (error) {
|
|
656
|
+
throw new Error(`${label} serializer failed after parsing`, { cause: error });
|
|
657
|
+
}
|
|
658
|
+
if (typeof canonicalSurface !== "string" || canonicalSurface.trim().length === 0) throw new Error(`${label} serializer must return a non-empty string`);
|
|
659
|
+
if (codec.serialize(codec.parse(canonicalSurface)) !== canonicalSurface) throw new Error(`${label} codec must round-trip to one canonical surface`);
|
|
660
|
+
return {
|
|
661
|
+
value: structuredClone(value),
|
|
662
|
+
surface: canonicalSurface,
|
|
663
|
+
surfaceHash: surfaceHash(canonicalSurface)
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
function assertPartitionContent(train, selection, final, fingerprint) {
|
|
667
|
+
if (!fingerprint) return;
|
|
668
|
+
const owner = /* @__PURE__ */ new Map();
|
|
669
|
+
for (const [name, scenarios] of [
|
|
670
|
+
["train", train],
|
|
671
|
+
["selection", selection],
|
|
672
|
+
["final", final]
|
|
673
|
+
]) for (const scenario of scenarios) {
|
|
674
|
+
const identity = fingerprint(scenario);
|
|
675
|
+
if (typeof identity !== "string" || identity.trim().length === 0) throw new Error(`scenarioFingerprint returned no identity for '${scenario.id}'`);
|
|
676
|
+
const prior = owner.get(identity);
|
|
677
|
+
if (prior) {
|
|
678
|
+
const scope = prior.partition === name ? `${name} partition duplicates` : `${prior.partition}/${name} partitions duplicate`;
|
|
679
|
+
throw new Error(`serialized knowledge optimization ${scope} scenario content at '${prior.scenarioId}'/'${scenario.id}'`);
|
|
680
|
+
}
|
|
681
|
+
owner.set(identity, {
|
|
682
|
+
partition: name,
|
|
683
|
+
scenarioId: scenario.id
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
//#endregion
|
|
523
688
|
//#region src/rag-eval/knowledge-base.ts
|
|
524
689
|
function scoreKnowledgeBaseIndex(index, options = {}) {
|
|
525
|
-
|
|
526
|
-
|
|
690
|
+
return scoreValidatedKnowledgeBaseIndex(index, options, validateKnowledgeIndex(index, options));
|
|
691
|
+
}
|
|
692
|
+
/** Internal scoring over the validation of this exact index and strictness policy. */
|
|
693
|
+
function scoreValidatedKnowledgeBaseIndex(index, options, validation) {
|
|
527
694
|
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
528
695
|
const staleSourceCount = index.sources.filter((source) => {
|
|
529
696
|
return source.validUntil ? Date.parse(source.validUntil) < now.getTime() : false;
|
|
@@ -543,7 +710,7 @@ function scoreKnowledgeBaseIndex(index, options = {}) {
|
|
|
543
710
|
stale_source_rate: index.sources.length === 0 ? 0 : staleSourceCount / index.sources.length,
|
|
544
711
|
duplicate_source_hash_rate: index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
|
|
545
712
|
lint_error_count: validation.findings.filter((finding) => finding.severity === "error").length,
|
|
546
|
-
lint_warning_count:
|
|
713
|
+
lint_warning_count: validation.findings.filter((finding) => finding.severity === "warning").length,
|
|
547
714
|
near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
|
|
548
715
|
near_duplicate_page_count: nearDuplicates.duplicatePageCount,
|
|
549
716
|
near_duplicate_pair_count: nearDuplicates.duplicatePairCount
|
|
@@ -1233,171 +1400,6 @@ function below(metric, threshold) {
|
|
|
1233
1400
|
return threshold !== void 0 && metric < threshold;
|
|
1234
1401
|
}
|
|
1235
1402
|
//#endregion
|
|
1236
|
-
//#region src/optimization.ts
|
|
1237
|
-
/**
|
|
1238
|
-
* Runs one complete agent-eval method over a serialized knowledge candidate.
|
|
1239
|
-
* The method receives train and selection data; agent-eval owns final scoring.
|
|
1240
|
-
*/
|
|
1241
|
-
async function runSerializedKnowledgeOptimization(options) {
|
|
1242
|
-
assertCompleteOptimizationMethod(options.method);
|
|
1243
|
-
assertImmutableRef(options.executionRef, "knowledge optimization executionRef");
|
|
1244
|
-
const codec = options.codec ?? jsonCandidateCodec();
|
|
1245
|
-
const baseline = normalizeCandidate(options.baseline, codec, "baseline");
|
|
1246
|
-
const dispatchRef = [
|
|
1247
|
-
"knowledge-optimization",
|
|
1248
|
-
options.executionRef,
|
|
1249
|
-
baseline.surfaceHash,
|
|
1250
|
-
surfaceHash(options.method.name)
|
|
1251
|
-
].join(":");
|
|
1252
|
-
assertPartitionContent(options.trainScenarios, options.selectionScenarios, options.finalScenarios, options.scenarioFingerprint ?? scenarioContentFingerprint);
|
|
1253
|
-
const method = canonicalCandidateMethod(options.method, codec);
|
|
1254
|
-
const { baseline: _baseline, method: _method, trainScenarios, selectionScenarios, finalScenarios, dispatchCandidate, executionRef: _executionRef, codec: _codec, scenarioFingerprint: _scenarioFingerprint, ...comparisonOptions } = options;
|
|
1255
|
-
const optimizationRunOptions = {
|
|
1256
|
-
...comparisonOptions.reps !== void 0 ? { reps: comparisonOptions.reps } : {},
|
|
1257
|
-
...comparisonOptions.resumable !== void 0 ? { resumable: comparisonOptions.resumable } : {},
|
|
1258
|
-
...comparisonOptions.labeledStore !== void 0 ? { labeledStore: comparisonOptions.labeledStore } : {},
|
|
1259
|
-
...comparisonOptions.captureSource !== void 0 ? { captureSource: comparisonOptions.captureSource } : {},
|
|
1260
|
-
...comparisonOptions.captureSourceVersionHash !== void 0 ? { captureSourceVersionHash: comparisonOptions.captureSourceVersionHash } : {},
|
|
1261
|
-
...comparisonOptions.maxConcurrency !== void 0 ? { maxConcurrency: comparisonOptions.maxConcurrency } : {},
|
|
1262
|
-
...comparisonOptions.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: comparisonOptions.dispatchTimeoutMs } : {},
|
|
1263
|
-
...comparisonOptions.repo !== void 0 ? { repo: comparisonOptions.repo } : {},
|
|
1264
|
-
...comparisonOptions.tracing !== void 0 ? { tracing: comparisonOptions.tracing } : {},
|
|
1265
|
-
...comparisonOptions.expectUsage !== void 0 ? { expectUsage: comparisonOptions.expectUsage } : {},
|
|
1266
|
-
...comparisonOptions.now !== void 0 ? { now: comparisonOptions.now } : {},
|
|
1267
|
-
...comparisonOptions.buildTraceWriter !== void 0 ? { buildTraceWriter: comparisonOptions.buildTraceWriter } : {},
|
|
1268
|
-
...comparisonOptions.storage !== void 0 ? { storage: comparisonOptions.storage } : {},
|
|
1269
|
-
...comparisonOptions.cellPlacement !== void 0 ? { cellPlacement: comparisonOptions.cellPlacement } : {},
|
|
1270
|
-
...comparisonOptions.optimizationRunOptions ?? {},
|
|
1271
|
-
dispatchRef
|
|
1272
|
-
};
|
|
1273
|
-
const comparison = await compareOptimizationMethods({
|
|
1274
|
-
...comparisonOptions,
|
|
1275
|
-
dispatchRef,
|
|
1276
|
-
optimizationRunOptions,
|
|
1277
|
-
methods: [method],
|
|
1278
|
-
baselineSurface: baseline.surface,
|
|
1279
|
-
trainScenarios: trainScenarios.map((scenario) => structuredClone(scenario)),
|
|
1280
|
-
selectionScenarios: selectionScenarios.map((scenario) => structuredClone(scenario)),
|
|
1281
|
-
testScenarios: finalScenarios.map((scenario) => structuredClone(scenario)),
|
|
1282
|
-
dispatchWithSurface: async (surface, scenario, context) => {
|
|
1283
|
-
const candidate = normalizeSurface(surface, codec, "candidate");
|
|
1284
|
-
return dispatchCandidate({
|
|
1285
|
-
candidate: structuredClone(candidate.value),
|
|
1286
|
-
candidateSurface: candidate.surface,
|
|
1287
|
-
candidateSurfaceHash: candidate.surfaceHash,
|
|
1288
|
-
scenario,
|
|
1289
|
-
context
|
|
1290
|
-
});
|
|
1291
|
-
}
|
|
1292
|
-
});
|
|
1293
|
-
const winner = normalizeSurface(comparison.best.winnerSurface, codec, "winner");
|
|
1294
|
-
return {
|
|
1295
|
-
methodName: options.method.name,
|
|
1296
|
-
baseline,
|
|
1297
|
-
winner,
|
|
1298
|
-
comparison
|
|
1299
|
-
};
|
|
1300
|
-
}
|
|
1301
|
-
function assertCompleteOptimizationMethod(method) {
|
|
1302
|
-
const candidate = method;
|
|
1303
|
-
if (!candidate || typeof candidate.name !== "string" || candidate.name.trim().length === 0 || typeof candidate.optimize !== "function") throw new Error("knowledge optimization requires a complete OptimizationMethod with a non-empty name and optimize()");
|
|
1304
|
-
}
|
|
1305
|
-
function jsonCandidateCodec() {
|
|
1306
|
-
return {
|
|
1307
|
-
serialize: (candidate) => canonicalJson(candidate),
|
|
1308
|
-
parse(surface) {
|
|
1309
|
-
let parsed;
|
|
1310
|
-
try {
|
|
1311
|
-
parsed = JSON.parse(surface);
|
|
1312
|
-
} catch (error) {
|
|
1313
|
-
throw new Error("serialized knowledge candidate is not valid JSON", { cause: error });
|
|
1314
|
-
}
|
|
1315
|
-
return parsed;
|
|
1316
|
-
}
|
|
1317
|
-
};
|
|
1318
|
-
}
|
|
1319
|
-
function jsonObjectCandidateCodec() {
|
|
1320
|
-
const codec = jsonCandidateCodec();
|
|
1321
|
-
return {
|
|
1322
|
-
serialize: codec.serialize,
|
|
1323
|
-
parse(surface) {
|
|
1324
|
-
const candidate = codec.parse(surface);
|
|
1325
|
-
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) throw new Error("serialized knowledge candidate must be a JSON object");
|
|
1326
|
-
return candidate;
|
|
1327
|
-
}
|
|
1328
|
-
};
|
|
1329
|
-
}
|
|
1330
|
-
function scenarioContentFingerprint(scenario) {
|
|
1331
|
-
const { id: _id, tags: _tags, split: _split, splitTag: _splitTag, ...content } = scenario;
|
|
1332
|
-
return surfaceHash(canonicalJson(content));
|
|
1333
|
-
}
|
|
1334
|
-
function canonicalCandidateMethod(method, codec) {
|
|
1335
|
-
return {
|
|
1336
|
-
name: method.name,
|
|
1337
|
-
async optimize(input) {
|
|
1338
|
-
const result = await method.optimize(input);
|
|
1339
|
-
const winner = normalizeSurface(result.winnerSurface, codec, `${method.name} winner`);
|
|
1340
|
-
return {
|
|
1341
|
-
...result,
|
|
1342
|
-
winnerSurface: winner.surface
|
|
1343
|
-
};
|
|
1344
|
-
}
|
|
1345
|
-
};
|
|
1346
|
-
}
|
|
1347
|
-
function normalizeCandidate(candidate, codec, label) {
|
|
1348
|
-
let surface;
|
|
1349
|
-
try {
|
|
1350
|
-
surface = codec.serialize(candidate);
|
|
1351
|
-
} catch (error) {
|
|
1352
|
-
throw new Error(`${label} serializer failed`, { cause: error });
|
|
1353
|
-
}
|
|
1354
|
-
return normalizeSurface(surface, codec, label);
|
|
1355
|
-
}
|
|
1356
|
-
function normalizeSurface(surface, codec, label) {
|
|
1357
|
-
if (typeof surface !== "string" || surface.trim().length === 0) throw new Error(`${label} must be a non-empty serialized string`);
|
|
1358
|
-
let value;
|
|
1359
|
-
try {
|
|
1360
|
-
value = codec.parse(surface);
|
|
1361
|
-
} catch (error) {
|
|
1362
|
-
const detail = error instanceof Error ? `: ${error.message}` : "";
|
|
1363
|
-
throw new Error(`${label} parser failed${detail}`, { cause: error });
|
|
1364
|
-
}
|
|
1365
|
-
let canonicalSurface;
|
|
1366
|
-
try {
|
|
1367
|
-
canonicalSurface = codec.serialize(value);
|
|
1368
|
-
} catch (error) {
|
|
1369
|
-
throw new Error(`${label} serializer failed after parsing`, { cause: error });
|
|
1370
|
-
}
|
|
1371
|
-
if (typeof canonicalSurface !== "string" || canonicalSurface.trim().length === 0) throw new Error(`${label} serializer must return a non-empty string`);
|
|
1372
|
-
if (codec.serialize(codec.parse(canonicalSurface)) !== canonicalSurface) throw new Error(`${label} codec must round-trip to one canonical surface`);
|
|
1373
|
-
return {
|
|
1374
|
-
value: structuredClone(value),
|
|
1375
|
-
surface: canonicalSurface,
|
|
1376
|
-
surfaceHash: surfaceHash(canonicalSurface)
|
|
1377
|
-
};
|
|
1378
|
-
}
|
|
1379
|
-
function assertPartitionContent(train, selection, final, fingerprint) {
|
|
1380
|
-
if (!fingerprint) return;
|
|
1381
|
-
const owner = /* @__PURE__ */ new Map();
|
|
1382
|
-
for (const [name, scenarios] of [
|
|
1383
|
-
["train", train],
|
|
1384
|
-
["selection", selection],
|
|
1385
|
-
["final", final]
|
|
1386
|
-
]) for (const scenario of scenarios) {
|
|
1387
|
-
const identity = fingerprint(scenario);
|
|
1388
|
-
if (typeof identity !== "string" || identity.trim().length === 0) throw new Error(`scenarioFingerprint returned no identity for '${scenario.id}'`);
|
|
1389
|
-
const prior = owner.get(identity);
|
|
1390
|
-
if (prior) {
|
|
1391
|
-
const scope = prior.partition === name ? `${name} partition duplicates` : `${prior.partition}/${name} partitions duplicate`;
|
|
1392
|
-
throw new Error(`serialized knowledge optimization ${scope} scenario content at '${prior.scenarioId}'/'${scenario.id}'`);
|
|
1393
|
-
}
|
|
1394
|
-
owner.set(identity, {
|
|
1395
|
-
partition: name,
|
|
1396
|
-
scenarioId: scenario.id
|
|
1397
|
-
});
|
|
1398
|
-
}
|
|
1399
|
-
}
|
|
1400
|
-
//#endregion
|
|
1401
1403
|
//#region src/rag-optimization.ts
|
|
1402
1404
|
/** Optimizes retrieval and answer behavior together as one serialized RAG configuration. */
|
|
1403
1405
|
async function runRagOptimization(options) {
|
|
@@ -1501,15 +1503,20 @@ function defineReadinessSpec(input) {
|
|
|
1501
1503
|
function buildEvalKnowledgeBundle(options) {
|
|
1502
1504
|
const searchLimit = options.searchLimit ?? 5;
|
|
1503
1505
|
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
1506
|
+
const lexicalIndex = options.specs.some((spec) => spec.query.trim() !== "") ? buildKnowledgeLexicalIndex(options.index.pages) : void 0;
|
|
1504
1507
|
const searchResultsByRequirement = {};
|
|
1505
1508
|
const requirements = options.specs.map((spec) => {
|
|
1506
|
-
const results = searchKnowledge(options.index, spec.query,
|
|
1509
|
+
const results = searchKnowledge(options.index, spec.query, {
|
|
1510
|
+
limit: searchLimit,
|
|
1511
|
+
lexicalIndex
|
|
1512
|
+
});
|
|
1507
1513
|
searchResultsByRequirement[spec.id] = results;
|
|
1508
1514
|
return requirementFromSearch(options.index, spec, results, now);
|
|
1509
1515
|
});
|
|
1510
1516
|
const report = scoreKnowledgeReadiness({
|
|
1511
1517
|
taskId: options.taskId,
|
|
1512
1518
|
requirements,
|
|
1519
|
+
now,
|
|
1513
1520
|
userAnswers: options.userAnswers,
|
|
1514
1521
|
evidenceIds: requirements.flatMap((requirement) => requirement.evidenceIds),
|
|
1515
1522
|
claimIds: [],
|
|
@@ -1536,6 +1543,10 @@ function requirementFromSearch(index, spec, results, now) {
|
|
|
1536
1543
|
const hitCoverage = spec.minHits ? Math.min(1, hitCount / spec.minHits) : hitCount > 0 ? 1 : 0;
|
|
1537
1544
|
const freshness = sourceFreshness(sources, now);
|
|
1538
1545
|
const currentConfidence = round(Math.min(bestScore, sourceCoverage, hitCoverage, freshness.score));
|
|
1546
|
+
const freshnessFields = {
|
|
1547
|
+
...freshness.validUntil === void 0 ? {} : { validUntil: freshness.validUntil },
|
|
1548
|
+
...freshness.lastVerifiedAt === void 0 ? {} : { lastVerifiedAt: freshness.lastVerifiedAt }
|
|
1549
|
+
};
|
|
1539
1550
|
return {
|
|
1540
1551
|
id: spec.id,
|
|
1541
1552
|
description: spec.description,
|
|
@@ -1547,6 +1558,7 @@ function requirementFromSearch(index, spec, results, now) {
|
|
|
1547
1558
|
sensitivity: spec.sensitivity,
|
|
1548
1559
|
confidenceNeeded: spec.confidenceNeeded,
|
|
1549
1560
|
currentConfidence,
|
|
1561
|
+
...freshnessFields,
|
|
1550
1562
|
evidenceIds: unique([...sourceIds.map((sourceId) => `source:${sourceId}`), ...results.map((result) => `page:${result.page.id}`)]),
|
|
1551
1563
|
fallbackPolicy: spec.fallbackPolicy ?? (spec.importance === "blocking" ? "block" : "continue_with_caveat"),
|
|
1552
1564
|
metadata: {
|
|
@@ -1557,8 +1569,7 @@ function requirementFromSearch(index, spec, results, now) {
|
|
|
1557
1569
|
bestNormalizedScore: bestScore,
|
|
1558
1570
|
expiredSourceIds: freshness.expiredSourceIds,
|
|
1559
1571
|
freshnessScore: freshness.score,
|
|
1560
|
-
...
|
|
1561
|
-
...freshness.lastVerifiedAt === void 0 ? {} : { lastVerifiedAt: freshness.lastVerifiedAt }
|
|
1572
|
+
...freshnessFields
|
|
1562
1573
|
}
|
|
1563
1574
|
};
|
|
1564
1575
|
}
|
|
@@ -1610,9 +1621,24 @@ function readinessFor(options, index) {
|
|
|
1610
1621
|
...options.readiness ?? {},
|
|
1611
1622
|
taskId: options.readinessTaskId ?? options.goal,
|
|
1612
1623
|
index,
|
|
1613
|
-
specs: options.readinessSpecs
|
|
1624
|
+
specs: [...options.readinessSpecs]
|
|
1614
1625
|
});
|
|
1615
1626
|
}
|
|
1627
|
+
/** Assess one index without repeating lint/schema work for its two policy gates. */
|
|
1628
|
+
function assessKnowledgeIndex(options, index) {
|
|
1629
|
+
const prepared = prepareKnowledgeValidation(index);
|
|
1630
|
+
const validation = prepared.validate({ strict: options.strict });
|
|
1631
|
+
const readiness = readinessFor(options, index);
|
|
1632
|
+
const qualityOptions = {
|
|
1633
|
+
strict: options.strict,
|
|
1634
|
+
...options.kbQuality
|
|
1635
|
+
};
|
|
1636
|
+
return {
|
|
1637
|
+
validation,
|
|
1638
|
+
readiness,
|
|
1639
|
+
kbQuality: scoreValidatedKnowledgeBaseIndex(index, qualityOptions, Boolean(qualityOptions.strict) === Boolean(options.strict) ? validation : prepared.validate(qualityOptions))
|
|
1640
|
+
};
|
|
1641
|
+
}
|
|
1616
1642
|
//#endregion
|
|
1617
1643
|
//#region src/research-loop.ts
|
|
1618
1644
|
/**
|
|
@@ -1631,18 +1657,12 @@ function createKnowledgeControlLoopAdapter(options) {
|
|
|
1631
1657
|
await initKnowledgeBase(options.root);
|
|
1632
1658
|
initialized = true;
|
|
1633
1659
|
}
|
|
1634
|
-
const
|
|
1635
|
-
const validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
1636
|
-
const lintFindings = lintKnowledgeIndex(index);
|
|
1637
|
-
const readiness = readinessFor(options, index);
|
|
1660
|
+
const observation = await observeKnowledgeResearch(options);
|
|
1638
1661
|
return {
|
|
1639
1662
|
root: options.root,
|
|
1640
1663
|
goal: options.goal,
|
|
1641
1664
|
iteration: history.length + 1,
|
|
1642
|
-
|
|
1643
|
-
lintFindings,
|
|
1644
|
-
validation,
|
|
1645
|
-
readiness,
|
|
1665
|
+
...observation,
|
|
1646
1666
|
previousSteps: [...appliedSteps],
|
|
1647
1667
|
signal: abortSignal
|
|
1648
1668
|
};
|
|
@@ -1674,7 +1694,7 @@ function createKnowledgeControlLoopAdapter(options) {
|
|
|
1674
1694
|
},
|
|
1675
1695
|
stopPolicies: { stateFingerprint: ({ signal: _signal, ...observable }) => canonicalJson(observable) },
|
|
1676
1696
|
async act(action, ctx) {
|
|
1677
|
-
const step = await applyKnowledgeResearchDecision(options, action, ctx.state.iteration);
|
|
1697
|
+
const { step } = await applyKnowledgeResearchDecision(options, action, ctx.state.iteration);
|
|
1678
1698
|
appliedSteps.push(step);
|
|
1679
1699
|
return step;
|
|
1680
1700
|
}
|
|
@@ -1684,10 +1704,7 @@ async function runKnowledgeResearchLoop(options) {
|
|
|
1684
1704
|
const maxIterations = Math.max(1, options.maxIterations ?? 3);
|
|
1685
1705
|
await initKnowledgeBase(options.root);
|
|
1686
1706
|
const steps = [];
|
|
1687
|
-
let
|
|
1688
|
-
let validation = validateKnowledgeIndex(index, { strict: options.strict });
|
|
1689
|
-
let lintFindings = lintKnowledgeIndex(index);
|
|
1690
|
-
let readiness = readinessFor(options, index);
|
|
1707
|
+
let observation = await observeKnowledgeResearch(options);
|
|
1691
1708
|
let done = false;
|
|
1692
1709
|
for (let iteration = 1; iteration <= maxIterations; iteration++) {
|
|
1693
1710
|
if (options.signal?.aborted) throw new Error("Knowledge research loop aborted");
|
|
@@ -1695,19 +1712,14 @@ async function runKnowledgeResearchLoop(options) {
|
|
|
1695
1712
|
root: options.root,
|
|
1696
1713
|
goal: options.goal,
|
|
1697
1714
|
iteration,
|
|
1698
|
-
|
|
1699
|
-
lintFindings,
|
|
1700
|
-
validation,
|
|
1701
|
-
readiness,
|
|
1715
|
+
...observation,
|
|
1702
1716
|
previousSteps: steps,
|
|
1703
1717
|
signal: options.signal
|
|
1704
1718
|
});
|
|
1705
1719
|
done = Boolean(decision.done);
|
|
1706
|
-
const
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
lintFindings = step.lintFindings;
|
|
1710
|
-
readiness = step.readiness;
|
|
1720
|
+
const applied = await applyKnowledgeResearchDecision(options, decision, iteration);
|
|
1721
|
+
const { step } = applied;
|
|
1722
|
+
observation = applied.observation;
|
|
1711
1723
|
steps.push(step);
|
|
1712
1724
|
await options.onStep?.(step);
|
|
1713
1725
|
if (done) break;
|
|
@@ -1718,10 +1730,7 @@ async function runKnowledgeResearchLoop(options) {
|
|
|
1718
1730
|
goal: options.goal,
|
|
1719
1731
|
iterations: steps.length,
|
|
1720
1732
|
done,
|
|
1721
|
-
|
|
1722
|
-
lintFindings,
|
|
1723
|
-
validation,
|
|
1724
|
-
readiness,
|
|
1733
|
+
...observation,
|
|
1725
1734
|
steps
|
|
1726
1735
|
};
|
|
1727
1736
|
}
|
|
@@ -1730,10 +1739,8 @@ async function applyKnowledgeResearchDecision(options, decision, iteration = 1)
|
|
|
1730
1739
|
for (const sourcePath of decision.sourcePaths ?? []) addedSources.push(...await addSourcePath(options.root, sourcePath, options.sourceOptions));
|
|
1731
1740
|
for (const sourceText of decision.sourceTexts ?? []) addedSources.push(await addSourceText(options.root, sourceText, options.sourceOptions));
|
|
1732
1741
|
const applied = decision.proposalText ? await applyKnowledgeWriteBlocks(options.root, decision.proposalText, { pagesDirectory: options.pagesDirectory }) : void 0;
|
|
1733
|
-
const
|
|
1734
|
-
const validation
|
|
1735
|
-
const lintFindings = lintKnowledgeIndex(index);
|
|
1736
|
-
const readiness = readinessFor(options, index);
|
|
1742
|
+
const observation = await observeKnowledgeResearch(options);
|
|
1743
|
+
const { validation, lintFindings, readiness } = observation;
|
|
1737
1744
|
const done = Boolean(decision.done);
|
|
1738
1745
|
const event = createKnowledgeEvent({
|
|
1739
1746
|
type: "research.iteration",
|
|
@@ -1750,16 +1757,29 @@ async function applyKnowledgeResearchDecision(options, decision, iteration = 1)
|
|
|
1750
1757
|
}
|
|
1751
1758
|
});
|
|
1752
1759
|
return {
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1760
|
+
step: {
|
|
1761
|
+
iteration,
|
|
1762
|
+
...decision.notes === void 0 ? {} : { notes: decision.notes },
|
|
1763
|
+
addedSources,
|
|
1764
|
+
...applied === void 0 ? {} : { applied },
|
|
1765
|
+
lintFindings,
|
|
1766
|
+
validation,
|
|
1767
|
+
...readiness === void 0 ? {} : { readiness },
|
|
1768
|
+
event,
|
|
1769
|
+
done,
|
|
1770
|
+
...decision.metadata === void 0 ? {} : { metadata: decision.metadata }
|
|
1771
|
+
},
|
|
1772
|
+
observation
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
async function observeKnowledgeResearch(options) {
|
|
1776
|
+
const index = await buildKnowledgeIndex(options.root, options);
|
|
1777
|
+
const prepared = prepareKnowledgeValidation(index);
|
|
1778
|
+
return {
|
|
1779
|
+
index,
|
|
1780
|
+
lintFindings: structuredClone(prepared.lintFindings),
|
|
1781
|
+
validation: prepared.validate({ strict: options.strict }),
|
|
1782
|
+
readiness: readinessFor(options, index)
|
|
1763
1783
|
};
|
|
1764
1784
|
}
|
|
1765
1785
|
//#endregion
|
|
@@ -2038,114 +2058,90 @@ async function runRagKnowledgeImprovementPhases(options, previous) {
|
|
|
2038
2058
|
let knowledgeUpdate = previous?.knowledgeUpdate;
|
|
2039
2059
|
let answerQuality = previous?.answerQuality;
|
|
2040
2060
|
let promotion = previous?.promotion;
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
goal: options.goal,
|
|
2059
|
-
phases,
|
|
2060
|
-
optimization: selectRagOptimization(optimization),
|
|
2061
|
-
signal: options.signal,
|
|
2062
|
-
retrieval: selectRetrievalOptimization(retrieval),
|
|
2063
|
-
findings
|
|
2064
|
-
});
|
|
2065
|
-
}, summarizeAcquisitionDecision);
|
|
2066
|
-
else skipPhase(phases, now, "knowledge-acquisition", "no acquisition hook provided");
|
|
2067
|
-
}
|
|
2068
|
-
if (phaseEnabled(options, "knowledge-update")) {
|
|
2069
|
-
if (options.updateKnowledge) knowledgeUpdate = await runPhase(phases, now, "knowledge-update", async () => {
|
|
2070
|
-
assertNotAborted(options.signal);
|
|
2071
|
-
return options.updateKnowledge({
|
|
2072
|
-
goal: options.goal,
|
|
2073
|
-
phases,
|
|
2074
|
-
optimization: selectRagOptimization(optimization),
|
|
2075
|
-
signal: options.signal,
|
|
2076
|
-
retrieval: selectRetrievalOptimization(retrieval),
|
|
2077
|
-
findings,
|
|
2078
|
-
acquisition
|
|
2061
|
+
const context = () => ({
|
|
2062
|
+
goal: options.goal,
|
|
2063
|
+
phases,
|
|
2064
|
+
optimization: selectOptimization(optimization),
|
|
2065
|
+
signal: options.signal,
|
|
2066
|
+
retrieval: selectOptimization(retrieval)
|
|
2067
|
+
});
|
|
2068
|
+
async function runPhase(phase, previousValue, action, summarize, missing) {
|
|
2069
|
+
if (!phaseEnabled(options, phase)) return previousValue;
|
|
2070
|
+
const startedAt = now().toISOString();
|
|
2071
|
+
if (!action) {
|
|
2072
|
+
phases.push({
|
|
2073
|
+
phase,
|
|
2074
|
+
status: "skipped",
|
|
2075
|
+
summary: missing,
|
|
2076
|
+
startedAt,
|
|
2077
|
+
finishedAt: startedAt
|
|
2079
2078
|
});
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
return runKnowledgeResearchUpdate(options, acquisition);
|
|
2084
|
-
}, (result) => result.summary);
|
|
2085
|
-
else skipPhase(phases, now, "knowledge-update", "no update hook or research loop provided");
|
|
2086
|
-
}
|
|
2087
|
-
if (phaseEnabled(options, "rag-optimization") && (options.optimization || options.enabledPhases?.includes("rag-optimization") || options.requiredPhases?.includes("rag-optimization"))) {
|
|
2088
|
-
if (options.optimization) optimization = await runPhase(phases, now, "rag-optimization", async () => {
|
|
2089
|
-
assertNotAborted(options.signal);
|
|
2090
|
-
return runRagOptimization(options.optimization);
|
|
2091
|
-
}, summarizeRagOptimization);
|
|
2092
|
-
else skipPhase(phases, now, "rag-optimization", "no full RAG optimization options provided");
|
|
2093
|
-
}
|
|
2094
|
-
if (phaseEnabled(options, "retrieval-tuning")) {
|
|
2095
|
-
if (options.retrieval) retrieval = await runPhase(phases, now, "retrieval-tuning", async () => {
|
|
2096
|
-
assertNotAborted(options.signal);
|
|
2097
|
-
return runRetrievalImprovementLoop(options.retrieval);
|
|
2098
|
-
}, summarizeRetrievalResult);
|
|
2099
|
-
else skipPhase(phases, now, "retrieval-tuning", "no retrieval options provided");
|
|
2100
|
-
}
|
|
2101
|
-
if (phaseEnabled(options, "answer-quality")) {
|
|
2102
|
-
if (options.evaluateAnswers) {
|
|
2103
|
-
answerQuality = await runPhase(phases, now, "answer-quality", async () => {
|
|
2104
|
-
assertNotAborted(options.signal);
|
|
2105
|
-
const result = await options.evaluateAnswers({
|
|
2106
|
-
goal: options.goal,
|
|
2107
|
-
phases,
|
|
2108
|
-
optimization: selectRagOptimization(optimization),
|
|
2109
|
-
signal: options.signal,
|
|
2110
|
-
retrieval: selectRetrievalOptimization(retrieval),
|
|
2111
|
-
findings,
|
|
2112
|
-
acquisition,
|
|
2113
|
-
knowledgeUpdate
|
|
2114
|
-
});
|
|
2115
|
-
assertRagAnswerEvidence(result);
|
|
2116
|
-
return result;
|
|
2117
|
-
}, summarizeAnswerQuality);
|
|
2118
|
-
findings = [...findings, ...answerQuality.findings ?? []];
|
|
2119
|
-
} else skipPhase(phases, now, "answer-quality", "no answer-quality hook provided");
|
|
2120
|
-
}
|
|
2121
|
-
if (phaseEnabled(options, "promotion")) {
|
|
2122
|
-
if (options.decidePromotion) promotion = await runPhase(phases, now, "promotion", async () => {
|
|
2079
|
+
return previousValue;
|
|
2080
|
+
}
|
|
2081
|
+
try {
|
|
2123
2082
|
assertNotAborted(options.signal);
|
|
2124
|
-
const
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2083
|
+
const result = await action();
|
|
2084
|
+
phases.push({
|
|
2085
|
+
phase,
|
|
2086
|
+
status: "completed",
|
|
2087
|
+
summary: summarize(result),
|
|
2088
|
+
startedAt,
|
|
2089
|
+
finishedAt: now().toISOString()
|
|
2131
2090
|
});
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
retrievalComparison: retrieval?.comparison,
|
|
2141
|
-
findings,
|
|
2142
|
-
acquisition,
|
|
2143
|
-
knowledgeUpdate,
|
|
2144
|
-
answerQuality
|
|
2091
|
+
return result;
|
|
2092
|
+
} catch (error) {
|
|
2093
|
+
phases.push({
|
|
2094
|
+
phase,
|
|
2095
|
+
status: "failed",
|
|
2096
|
+
summary: error.message,
|
|
2097
|
+
startedAt,
|
|
2098
|
+
finishedAt: now().toISOString()
|
|
2145
2099
|
});
|
|
2146
|
-
|
|
2147
|
-
|
|
2100
|
+
throw error;
|
|
2101
|
+
}
|
|
2148
2102
|
}
|
|
2103
|
+
findings = [...await runPhase("gap-diagnosis", findings, options.diagnose && (() => options.diagnose(context())), (diagnosed) => `${diagnosed.length} finding(s)`, "no diagnosis hook provided")];
|
|
2104
|
+
acquisition = await runPhase("knowledge-acquisition", acquisition, options.acquireKnowledge && (() => options.acquireKnowledge({
|
|
2105
|
+
...context(),
|
|
2106
|
+
findings
|
|
2107
|
+
})), summarizeAcquisitionDecision, "no acquisition hook provided");
|
|
2108
|
+
knowledgeUpdate = await runPhase("knowledge-update", knowledgeUpdate, options.updateKnowledge ? () => options.updateKnowledge({
|
|
2109
|
+
...context(),
|
|
2110
|
+
findings,
|
|
2111
|
+
acquisition
|
|
2112
|
+
}) : options.knowledgeResearch ? () => runKnowledgeResearchUpdate(options, acquisition) : void 0, (result) => result.summary, "no update hook or research loop provided");
|
|
2113
|
+
if (options.optimization || options.enabledPhases?.includes("rag-optimization") || options.requiredPhases?.includes("rag-optimization")) optimization = await runPhase("rag-optimization", optimization, options.optimization && (() => runRagOptimization(options.optimization)), summarizeOptimization, "no full RAG optimization options provided");
|
|
2114
|
+
retrieval = await runPhase("retrieval-tuning", retrieval, options.retrieval && (() => runRetrievalImprovementLoop(options.retrieval)), summarizeOptimization, "no retrieval options provided");
|
|
2115
|
+
const runsAnswerQuality = phaseEnabled(options, "answer-quality") && Boolean(options.evaluateAnswers);
|
|
2116
|
+
answerQuality = await runPhase("answer-quality", answerQuality, options.evaluateAnswers && (async () => {
|
|
2117
|
+
const result = await options.evaluateAnswers({
|
|
2118
|
+
...context(),
|
|
2119
|
+
findings,
|
|
2120
|
+
acquisition,
|
|
2121
|
+
knowledgeUpdate
|
|
2122
|
+
});
|
|
2123
|
+
assertRagAnswerEvidence(result);
|
|
2124
|
+
return result;
|
|
2125
|
+
}), summarizeAnswerQuality, "no answer-quality hook provided");
|
|
2126
|
+
if (runsAnswerQuality) findings = [...findings, ...answerQuality.findings ?? []];
|
|
2127
|
+
promotion = await runPhase("promotion", promotion, options.decidePromotion && (() => {
|
|
2128
|
+
return rejectUnsafePromotionEvidence({
|
|
2129
|
+
optimization: optimization?.comparison,
|
|
2130
|
+
optimizationCostCeiling: options.optimization?.costCeiling,
|
|
2131
|
+
retrieval: retrieval?.comparison,
|
|
2132
|
+
retrievalCostCeiling: options.retrieval?.costCeiling,
|
|
2133
|
+
answerQuality,
|
|
2134
|
+
answerQualityCostCeiling: options.answerQualityCostCeiling
|
|
2135
|
+
}) ?? options.decidePromotion({
|
|
2136
|
+
...context(),
|
|
2137
|
+
optimizationComparison: optimization?.comparison,
|
|
2138
|
+
retrievalComparison: retrieval?.comparison,
|
|
2139
|
+
findings,
|
|
2140
|
+
acquisition,
|
|
2141
|
+
knowledgeUpdate,
|
|
2142
|
+
answerQuality
|
|
2143
|
+
});
|
|
2144
|
+
}), (result) => `${result.promoted ? "promoted" : "held"}: ${result.reason}`, "no promotion decision hook provided");
|
|
2149
2145
|
return {
|
|
2150
2146
|
goal: options.goal,
|
|
2151
2147
|
phases,
|
|
@@ -2191,7 +2187,7 @@ function exceedsCostCeiling(totalCostUsd, costCeiling) {
|
|
|
2191
2187
|
const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling)) * 8;
|
|
2192
2188
|
return totalCostUsd - costCeiling > tolerance;
|
|
2193
2189
|
}
|
|
2194
|
-
function
|
|
2190
|
+
function summarizeOptimization(result) {
|
|
2195
2191
|
return `${result.methodName}; winner=${result.winner.surfaceHash}`;
|
|
2196
2192
|
}
|
|
2197
2193
|
async function runKnowledgeResearchUpdate(options, acquisition) {
|
|
@@ -2222,53 +2218,7 @@ function acquisitionBackedResearchStep(acquisition) {
|
|
|
2222
2218
|
done: acquisition.done ?? true
|
|
2223
2219
|
});
|
|
2224
2220
|
}
|
|
2225
|
-
|
|
2226
|
-
const startedAt = now().toISOString();
|
|
2227
|
-
try {
|
|
2228
|
-
const result = await action();
|
|
2229
|
-
phases.push({
|
|
2230
|
-
phase,
|
|
2231
|
-
status: "completed",
|
|
2232
|
-
summary: summarize(result),
|
|
2233
|
-
startedAt,
|
|
2234
|
-
finishedAt: now().toISOString()
|
|
2235
|
-
});
|
|
2236
|
-
return result;
|
|
2237
|
-
} catch (error) {
|
|
2238
|
-
phases.push({
|
|
2239
|
-
phase,
|
|
2240
|
-
status: "failed",
|
|
2241
|
-
summary: error.message,
|
|
2242
|
-
startedAt,
|
|
2243
|
-
finishedAt: now().toISOString()
|
|
2244
|
-
});
|
|
2245
|
-
throw error;
|
|
2246
|
-
}
|
|
2247
|
-
}
|
|
2248
|
-
function skipPhase(phases, now, phase, summary) {
|
|
2249
|
-
const timestamp = now().toISOString();
|
|
2250
|
-
phases.push({
|
|
2251
|
-
phase,
|
|
2252
|
-
status: "skipped",
|
|
2253
|
-
summary,
|
|
2254
|
-
startedAt: timestamp,
|
|
2255
|
-
finishedAt: timestamp
|
|
2256
|
-
});
|
|
2257
|
-
}
|
|
2258
|
-
function summarizeRetrievalResult(result) {
|
|
2259
|
-
return `${result.methodName}; winner=${result.winner.surfaceHash}`;
|
|
2260
|
-
}
|
|
2261
|
-
function selectRagOptimization(result) {
|
|
2262
|
-
if (!result) return void 0;
|
|
2263
|
-
return {
|
|
2264
|
-
methodName: result.methodName,
|
|
2265
|
-
baseline: structuredClone(result.baseline),
|
|
2266
|
-
winner: structuredClone(result.winner),
|
|
2267
|
-
baselineConfig: structuredClone(result.baselineConfig),
|
|
2268
|
-
winnerConfig: structuredClone(result.winnerConfig)
|
|
2269
|
-
};
|
|
2270
|
-
}
|
|
2271
|
-
function selectRetrievalOptimization(result) {
|
|
2221
|
+
function selectOptimization(result) {
|
|
2272
2222
|
if (!result) return void 0;
|
|
2273
2223
|
return {
|
|
2274
2224
|
methodName: result.methodName,
|
|
@@ -3109,12 +3059,7 @@ function selectedStageRequiredPhases(options, stagePhases) {
|
|
|
3109
3059
|
async function evaluateCandidate(runDir, state, candidate, snapshot, lifecycle, options, now, useConfiguredEvaluator) {
|
|
3110
3060
|
return withBaselineSnapshot(runDir, state.baseHash, async (baselineRoot) => {
|
|
3111
3061
|
const [baselineIndex, candidateIndex] = await Promise.all([buildKnowledgeIndex(baselineRoot, options.stateScope), buildKnowledgeIndex(snapshot.root, options.stateScope)]);
|
|
3112
|
-
const validation
|
|
3113
|
-
const readiness = readinessFor(options, candidateIndex);
|
|
3114
|
-
const kbQuality = scoreKnowledgeBaseIndex(candidateIndex, {
|
|
3115
|
-
strict: options.strict,
|
|
3116
|
-
...options.kbQuality
|
|
3117
|
-
});
|
|
3062
|
+
const { validation, readiness, kbQuality } = assessKnowledgeIndex(options, candidateIndex);
|
|
3118
3063
|
const candidateHash = snapshot.hash;
|
|
3119
3064
|
const evaluator = useConfiguredEvaluator ? options.evaluate : options.evaluateDevelopment;
|
|
3120
3065
|
const evaluation = applyLifecycleFailures(normalizeMetric(await ((evaluator ? evaluator({
|
|
@@ -5409,15 +5354,7 @@ function createQmdKnowledgeTools(options) {
|
|
|
5409
5354
|
//#region src/readiness-check.ts
|
|
5410
5355
|
async function evaluateKnowledgeBaseReadiness(options) {
|
|
5411
5356
|
const index = await buildKnowledgeIndex(options.root);
|
|
5412
|
-
const validation
|
|
5413
|
-
const readiness = readinessFor({
|
|
5414
|
-
...options,
|
|
5415
|
-
readinessSpecs: options.readinessSpecs ? [...options.readinessSpecs] : void 0
|
|
5416
|
-
}, index);
|
|
5417
|
-
const kbQuality = scoreKnowledgeBaseIndex(index, {
|
|
5418
|
-
strict: options.strict,
|
|
5419
|
-
...options.kbQuality
|
|
5420
|
-
});
|
|
5357
|
+
const { validation, readiness, kbQuality } = assessKnowledgeIndex(options, index);
|
|
5421
5358
|
const blockingMissing = readiness?.report.blockingMissingRequirements.length ?? 0;
|
|
5422
5359
|
const blockingTotal = options.readinessSpecs?.filter((spec) => spec.importance === "blocking").length ?? 0;
|
|
5423
5360
|
const blockingReadiness = blockingTotal === 0 ? 1 : Math.max(0, blockingTotal - blockingMissing) / blockingTotal;
|