@tangle-network/agent-eval 0.83.0 → 0.84.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,95 @@
1
+ // src/runtime-trajectory.ts
2
+ var DEFAULT_SPLIT_TAG = "search";
3
+ function projectRuntimeTrajectoryEvidence(options) {
4
+ const diagnostics = [];
5
+ const runsById = /* @__PURE__ */ new Map();
6
+ const events = [];
7
+ let recordWithRuntimeEventsCount = 0;
8
+ let defaultedSplitCount = 0;
9
+ for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {
10
+ const record = options.records[recordIndex];
11
+ const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf);
12
+ const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG;
13
+ if (record.splitTag === void 0) defaultedSplitCount += 1;
14
+ const rawEvents = record.runtimeEvents;
15
+ if (!Array.isArray(rawEvents)) {
16
+ diagnostics.push(
17
+ `${key}: runtimeEvents is not an array; no runtime run join can be extracted`
18
+ );
19
+ continue;
20
+ }
21
+ if (rawEvents.length === 0) {
22
+ diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`);
23
+ continue;
24
+ }
25
+ recordWithRuntimeEventsCount += 1;
26
+ for (let index = 0; index < rawEvents.length; index += 1) {
27
+ const event = parseRuntimeTrajectoryHookEvent(rawEvents[index]);
28
+ if (!event) {
29
+ diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`);
30
+ continue;
31
+ }
32
+ events.push(event);
33
+ const scenarioId = event.scenarioId ?? stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ?? stringOrUndefined(record.scenarioId);
34
+ const prior = runsById.get(event.runId);
35
+ if (!prior) {
36
+ runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag });
37
+ continue;
38
+ }
39
+ if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {
40
+ diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`);
41
+ }
42
+ }
43
+ }
44
+ const runs = [...runsById.values()];
45
+ return {
46
+ runs,
47
+ events,
48
+ summary: {
49
+ recordCount: options.records.length,
50
+ recordWithRuntimeEventsCount,
51
+ runtimeRunCount: runs.length,
52
+ lifecycleEventCount: events.length,
53
+ defaultedSplitCount
54
+ },
55
+ diagnostics
56
+ };
57
+ }
58
+ function parseRuntimeTrajectoryHookEvent(input) {
59
+ if (!isRecord(input)) return null;
60
+ if (typeof input.id !== "string" || input.id.length === 0) return null;
61
+ if (typeof input.runId !== "string" || input.runId.length === 0) return null;
62
+ if (typeof input.target !== "string" || input.target.length === 0) return null;
63
+ if (typeof input.phase !== "string" || input.phase.length === 0) return null;
64
+ if (typeof input.timestamp !== "number" || !Number.isFinite(input.timestamp)) return null;
65
+ return {
66
+ id: input.id,
67
+ runId: input.runId,
68
+ scenarioId: stringOrUndefined(input.scenarioId),
69
+ target: input.target,
70
+ phase: input.phase,
71
+ timestamp: input.timestamp,
72
+ stepIndex: finiteNumberOrUndefined(input.stepIndex),
73
+ parentId: stringOrUndefined(input.parentId),
74
+ payload: input.payload,
75
+ metadata: isRecord(input.metadata) ? { ...input.metadata } : void 0
76
+ };
77
+ }
78
+ function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
79
+ return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
80
+ }
81
+ function isRecord(value) {
82
+ return typeof value === "object" && value !== null && !Array.isArray(value);
83
+ }
84
+ function stringOrUndefined(value) {
85
+ return typeof value === "string" && value.length > 0 ? value : void 0;
86
+ }
87
+ function finiteNumberOrUndefined(value) {
88
+ return typeof value === "number" && Number.isFinite(value) ? value : void 0;
89
+ }
90
+
91
+ export {
92
+ projectRuntimeTrajectoryEvidence,
93
+ parseRuntimeTrajectoryHookEvent
94
+ };
95
+ //# sourceMappingURL=chunk-T4SQEITX.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/runtime-trajectory.ts"],"sourcesContent":["import type { RunSplitTag } from './run-record'\n\nexport interface RuntimeTrajectoryHookEvent {\n id: string\n runId: string\n scenarioId?: string\n target: string\n phase: string\n timestamp: number\n stepIndex?: number\n parentId?: string\n payload?: unknown\n metadata?: Record<string, unknown>\n}\n\nexport interface RuntimeTrajectoryRecord {\n id?: string\n scenarioId?: string\n splitTag?: RunSplitTag\n runtimeEvents?: unknown\n [key: string]: unknown\n}\n\nexport interface RuntimeTrajectoryRunRecord {\n runId: string\n scenarioId?: string\n splitTag: RunSplitTag\n}\n\nexport interface RuntimeTrajectoryEvidenceSummary {\n recordCount: number\n recordWithRuntimeEventsCount: number\n runtimeRunCount: number\n lifecycleEventCount: number\n defaultedSplitCount: number\n}\n\nexport interface RuntimeTrajectoryEvidenceProjection {\n runs: RuntimeTrajectoryRunRecord[]\n events: RuntimeTrajectoryHookEvent[]\n summary: RuntimeTrajectoryEvidenceSummary\n diagnostics: string[]\n}\n\nexport interface ProjectRuntimeTrajectoryEvidenceOptions<\n TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord,\n> {\n records: TRecord[]\n defaultSplitTag?: RunSplitTag\n recordIdOf?: (record: TRecord, index: number) => string | undefined\n scenarioIdOf?: (record: TRecord, index: number) => string | undefined\n}\n\nconst DEFAULT_SPLIT_TAG: RunSplitTag = 'search'\n\nexport function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(\n options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>,\n): RuntimeTrajectoryEvidenceProjection {\n const diagnostics: string[] = []\n const runsById = new Map<string, RuntimeTrajectoryRunRecord>()\n const events: RuntimeTrajectoryHookEvent[] = []\n let recordWithRuntimeEventsCount = 0\n let defaultedSplitCount = 0\n\n for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {\n const record = options.records[recordIndex]!\n const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf)\n const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG\n if (record.splitTag === undefined) defaultedSplitCount += 1\n\n const rawEvents = record.runtimeEvents\n if (!Array.isArray(rawEvents)) {\n diagnostics.push(\n `${key}: runtimeEvents is not an array; no runtime run join can be extracted`,\n )\n continue\n }\n if (rawEvents.length === 0) {\n diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`)\n continue\n }\n recordWithRuntimeEventsCount += 1\n\n for (let index = 0; index < rawEvents.length; index += 1) {\n const event = parseRuntimeTrajectoryHookEvent(rawEvents[index])\n if (!event) {\n diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`)\n continue\n }\n events.push(event)\n\n const scenarioId =\n event.scenarioId ??\n stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ??\n stringOrUndefined(record.scenarioId)\n const prior = runsById.get(event.runId)\n if (!prior) {\n runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag })\n continue\n }\n if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {\n diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`)\n }\n }\n }\n\n const runs = [...runsById.values()]\n return {\n runs,\n events,\n summary: {\n recordCount: options.records.length,\n recordWithRuntimeEventsCount,\n runtimeRunCount: runs.length,\n lifecycleEventCount: events.length,\n defaultedSplitCount,\n },\n diagnostics,\n }\n}\n\nexport function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null {\n if (!isRecord(input)) return null\n if (typeof input.id !== 'string' || input.id.length === 0) return null\n if (typeof input.runId !== 'string' || input.runId.length === 0) return null\n if (typeof input.target !== 'string' || input.target.length === 0) return null\n if (typeof input.phase !== 'string' || input.phase.length === 0) return null\n if (typeof input.timestamp !== 'number' || !Number.isFinite(input.timestamp)) return null\n\n return {\n id: input.id,\n runId: input.runId,\n scenarioId: stringOrUndefined(input.scenarioId),\n target: input.target,\n phase: input.phase,\n timestamp: input.timestamp,\n stepIndex: finiteNumberOrUndefined(input.stepIndex),\n parentId: stringOrUndefined(input.parentId),\n payload: input.payload,\n metadata: isRecord(input.metadata) ? { ...input.metadata } : undefined,\n }\n}\n\nfunction runtimeTrajectoryRecordKey<TRecord extends RuntimeTrajectoryRecord>(\n record: TRecord,\n index: number,\n recordIdOf?: (record: TRecord, index: number) => string | undefined,\n): string {\n return (\n stringOrUndefined(recordIdOf?.(record, index)) ??\n stringOrUndefined(record.id) ??\n `record[${index}]`\n )\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n\nfunction stringOrUndefined(value: unknown): string | undefined {\n return typeof value === 'string' && value.length > 0 ? value : undefined\n}\n\nfunction finiteNumberOrUndefined(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isFinite(value) ? value : undefined\n}\n"],"mappings":";AAqDA,IAAM,oBAAiC;AAEhC,SAAS,iCACd,SACqC;AACrC,QAAM,cAAwB,CAAC;AAC/B,QAAM,WAAW,oBAAI,IAAwC;AAC7D,QAAM,SAAuC,CAAC;AAC9C,MAAI,+BAA+B;AACnC,MAAI,sBAAsB;AAE1B,WAAS,cAAc,GAAG,cAAc,QAAQ,QAAQ,QAAQ,eAAe,GAAG;AAChF,UAAM,SAAS,QAAQ,QAAQ,WAAW;AAC1C,UAAM,MAAM,2BAA2B,QAAQ,aAAa,QAAQ,UAAU;AAC9E,UAAM,WAAW,OAAO,YAAY,QAAQ,mBAAmB;AAC/D,QAAI,OAAO,aAAa,OAAW,wBAAuB;AAE1D,UAAM,YAAY,OAAO;AACzB,QAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;AAC7B,kBAAY;AAAA,QACV,GAAG,GAAG;AAAA,MACR;AACA;AAAA,IACF;AACA,QAAI,UAAU,WAAW,GAAG;AAC1B,kBAAY,KAAK,GAAG,GAAG,0DAA0D;AACjF;AAAA,IACF;AACA,oCAAgC;AAEhC,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,GAAG;AACxD,YAAM,QAAQ,gCAAgC,UAAU,KAAK,CAAC;AAC9D,UAAI,CAAC,OAAO;AACV,oBAAY,KAAK,GAAG,GAAG,mBAAmB,KAAK,6BAA6B;AAC5E;AAAA,MACF;AACA,aAAO,KAAK,KAAK;AAEjB,YAAM,aACJ,MAAM,cACN,kBAAkB,QAAQ,eAAe,QAAQ,WAAW,CAAC,KAC7D,kBAAkB,OAAO,UAAU;AACrC,YAAM,QAAQ,SAAS,IAAI,MAAM,KAAK;AACtC,UAAI,CAAC,OAAO;AACV,iBAAS,IAAI,MAAM,OAAO,EAAE,OAAO,MAAM,OAAO,YAAY,SAAS,CAAC;AACtE;AAAA,MACF;AACA,UAAI,MAAM,eAAe,cAAc,MAAM,aAAa,UAAU;AAClE,oBAAY,KAAK,GAAG,GAAG,WAAW,MAAM,KAAK,0CAA0C;AAAA,MACzF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;AAClC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS;AAAA,MACP,aAAa,QAAQ,QAAQ;AAAA,MAC7B;AAAA,MACA,iBAAiB,KAAK;AAAA,MACtB,qBAAqB,OAAO;AAAA,MAC5B;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,gCAAgC,OAAmD;AACjG,MAAI,CAAC,SAAS,KAAK,EAAG,QAAO;AAC7B,MAAI,OAAO,MAAM,OAAO,YAAY,MAAM,GAAG,WAAW,EAAG,QAAO;AAClE,MAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,EAAG,QAAO;AACxE,MAAI,OAAO,MAAM,WAAW,YAAY,MAAM,OAAO,WAAW,EAAG,QAAO;AAC1E,MAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,EAAG,QAAO;AACxE,MAAI,OAAO,MAAM,cAAc,YAAY,CAAC,OAAO,SAAS,MAAM,SAAS,EAAG,QAAO;AAErF,SAAO;AAAA,IACL,IAAI,MAAM;AAAA,IACV,OAAO,MAAM;AAAA,IACb,YAAY,kBAAkB,MAAM,UAAU;AAAA,IAC9C,QAAQ,MAAM;AAAA,IACd,OAAO,MAAM;AAAA,IACb,WAAW,MAAM;AAAA,IACjB,WAAW,wBAAwB,MAAM,SAAS;AAAA,IAClD,UAAU,kBAAkB,MAAM,QAAQ;AAAA,IAC1C,SAAS,MAAM;AAAA,IACf,UAAU,SAAS,MAAM,QAAQ,IAAI,EAAE,GAAG,MAAM,SAAS,IAAI;AAAA,EAC/D;AACF;AAEA,SAAS,2BACP,QACA,OACA,YACQ;AACR,SACE,kBAAkB,aAAa,QAAQ,KAAK,CAAC,KAC7C,kBAAkB,OAAO,EAAE,KAC3B,UAAU,KAAK;AAEnB;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,kBAAkB,OAAoC;AAC7D,SAAO,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ;AACjE;AAEA,SAAS,wBAAwB,OAAoC;AACnE,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;","names":[]}
package/dist/index.d.ts CHANGED
@@ -42,6 +42,7 @@ export { A as Artifact, E as EventKind, i as FAILURE_CLASSES, F as FailureClass,
42
42
  import { T as TraceStore, R as RunFilter } from './store-CKUAgsJz.js';
43
43
  export { E as EventFilter, F as FileSystemTraceStore, a as FileSystemTraceStoreOptions, I as InMemoryTraceStore, S as SpanFilter } from './store-CKUAgsJz.js';
44
44
  export { D as DEFAULT_FAILURE_RULES, b as FailureClassification, c as FailureContext, d as FailureRule, e as classifyFailure } from './failure-cluster-CL7IVgkJ.js';
45
+ export { P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection, b as RuntimeTrajectoryEvidenceSummary, c as RuntimeTrajectoryHookEvent, R as RuntimeTrajectoryRecord, d as RuntimeTrajectoryRunRecord, p as parseRuntimeTrajectoryHookEvent, e as projectRuntimeTrajectoryEvidence } from './runtime-trajectory-BLRiaifm.js';
45
46
  import { a as BaselineReport } from './baseline-DE36-Np7.js';
46
47
  export { B as BaselineOptions, M as MetricSamples, b as MetricVerdict, T as ToolStats, d as ToolUseMetrics, e as ToolUseOptions, f as compareToBaseline, c as computeToolUseMetrics, i as iqr, w as welchsTTest } from './baseline-DE36-Np7.js';
47
48
  import { T as Trajectory, a as TrajectoryStep } from './trajectory-GEdXJCL5.js';
@@ -3738,6 +3739,124 @@ declare class Mutex {
3738
3739
  get pending(): number;
3739
3740
  }
3740
3741
 
3742
+ /**
3743
+ * DescriptionLengthGate — a Minimum-Description-Length promotion gate, the
3744
+ * Builder/Breaker acceptance rule from Wang & Buehler, "Self-Revising Discovery
3745
+ * Systems for Science" (arXiv:2606.01444, MIT LAMM), eq. 5:
3746
+ *
3747
+ * L(M, D) = L_model(M) + L_data(D | M)
3748
+ * accept M' over M iff L(M', D∪E) < L(M, D∪E)
3749
+ *
3750
+ * Both candidate and baseline are scored on the SAME enlarged evidence set
3751
+ * (every accumulated task — NOT just the held-out split), and the candidate is
3752
+ * accepted only if it lowers the TOTAL bit cost. This is the gate's whole
3753
+ * point and what distinguishes it from a monotone held-out delta:
3754
+ *
3755
+ * - L_model(M) — the candidate's own description length: the compressed size
3756
+ * of its model text (a prompt, skill, profile, or symbolic model). A bigger
3757
+ * model pays more bits.
3758
+ * - L_data(D|M) — the residual: bits of "surprise" that the model did not
3759
+ * simply succeed, −Σ_i log2(s_i) over the model's per-task score s_i.
3760
+ * Perfect scores cost 0 bits; failure costs a lot (capped, not infinite).
3761
+ *
3762
+ * A candidate that merely memorizes new counterexamples grows L_model faster
3763
+ * than it shrinks L_data and LOSES — a principled, complexity-penalized
3764
+ * alternative to HeldOutGate's held-out paired delta. Use this gate
3765
+ * when the model text whose size you want to penalize is available; use
3766
+ * HeldOutGate when promotion should turn on held-out generalization with an
3767
+ * overfit-gap check instead.
3768
+ *
3769
+ * Scale / calibration: a gzip'd prose model is hundreds–thousands of bits;
3770
+ * a single task contributes at most −log2(scoreFloor) data bits (≈10). So with
3771
+ * little evidence the model term dominates and the gate is conservative about
3772
+ * model GROWTH — it promotes a larger model only once accumulated evidence
3773
+ * genuinely pays for the added bits (exactly the paper's regime, where D∪E
3774
+ * grows). `lambda` is the lever: λ<1 discounts model bits (more permissive),
3775
+ * λ>1 is stricter. A shrinking-or-equal model that does no worse always wins.
3776
+ *
3777
+ * Stateless: construct once with the description-length budget, call
3778
+ * `evaluate` per (candidate, baseline) pair.
3779
+ */
3780
+ type DescriptionLengthRejectionCode = 'few_tasks' | 'no_total_gain' | 'model_bloat';
3781
+ interface DescriptionLengthConfig {
3782
+ /** Stable label of the baseline. Required — paper-grade evaluation never
3783
+ * compares two unlabelled candidates. */
3784
+ baselineKey: string;
3785
+ /** Weight on model bits relative to data bits (the description-length
3786
+ * budget λ). 1 = bits are bits. >1 = more complexity-averse. Default 1. */
3787
+ lambda?: number;
3788
+ /** The candidate must beat the baseline by at least this many bits to
3789
+ * promote — a robustness margin against measurement noise. Default 0
3790
+ * (strict `<`, as the paper). */
3791
+ marginBits?: number;
3792
+ /** Per-task score floor for the residual code: −log2(max(s, floor)). Caps a
3793
+ * total-failure task's surprise instead of letting it diverge. Default
3794
+ * 2^-10 (a failed task costs 10 bits, not ∞). */
3795
+ scoreFloor?: number;
3796
+ /** Minimum number of shared (candidate, baseline) tasks before the gate will
3797
+ * consider promoting. Default 3. */
3798
+ minTasks?: number;
3799
+ }
3800
+ interface DescriptionLengthEvidence {
3801
+ /** Shared tasks scored on both sides (the enlarged evidence D∪E). */
3802
+ tasks: number;
3803
+ /** Compressed-model bits — L_model. */
3804
+ modelBits: {
3805
+ candidate: number;
3806
+ baseline: number;
3807
+ };
3808
+ /** Residual surprise bits — L_data(D|M). */
3809
+ dataBits: {
3810
+ candidate: number;
3811
+ baseline: number;
3812
+ };
3813
+ /** λ·L_model + L_data — the quantity the gate minimizes. */
3814
+ totalBits: {
3815
+ candidate: number;
3816
+ baseline: number;
3817
+ };
3818
+ /** candidate − baseline total. Negative = candidate compresses better. */
3819
+ deltaBits: number;
3820
+ /** Per-component deltas, for audit: did the win come from a smaller model,
3821
+ * better outcomes, or both? */
3822
+ modelBitsDelta: number;
3823
+ dataBitsDelta: number;
3824
+ }
3825
+ interface DescriptionLengthDecision {
3826
+ promote: boolean;
3827
+ candidateId: string;
3828
+ baselineId: string;
3829
+ evidence: DescriptionLengthEvidence;
3830
+ reason: string;
3831
+ rejectionCode: DescriptionLengthRejectionCode | null;
3832
+ }
3833
+ interface DescriptionLengthCandidate {
3834
+ /** The model text whose size is L_model (a prompt, skill, profile, or
3835
+ * symbolic model; concatenated if several files). */
3836
+ content: string;
3837
+ /** Runs whose per-task scores form L_data. */
3838
+ runs: RunRecord[];
3839
+ }
3840
+ /** Compressed-model bits — the model's description length L_model. gzip is a
3841
+ * deterministic, dependency-free stand-in for Kolmogorov complexity; it
3842
+ * rewards genuine compactness and penalizes boilerplate padding. */
3843
+ declare function modelDescriptionBits(content: string): number;
3844
+ /** Residual surprise L_data(D|M) = −Σ_i log2(max(s_i, floor)) over the given
3845
+ * per-task scores. Lower = the model more reliably succeeds. */
3846
+ declare function dataDescriptionBits(scoreByTask: Map<string, number>, keys: Iterable<string>, scoreFloor: number): number;
3847
+ declare class DescriptionLengthGate {
3848
+ private readonly baselineKey;
3849
+ private readonly lambda;
3850
+ private readonly marginBits;
3851
+ private readonly scoreFloor;
3852
+ private readonly minTasks;
3853
+ constructor(config: DescriptionLengthConfig);
3854
+ /** Decide whether `candidate` should replace `baseline`. Both are scored on
3855
+ * the shared task set (the enlarged evidence); the candidate promotes only
3856
+ * if λ·L_model + L_data is strictly lower by at least `marginBits`. */
3857
+ evaluate(candidate: DescriptionLengthCandidate, baseline: DescriptionLengthCandidate): DescriptionLengthDecision;
3858
+ }
3859
+
3741
3860
  /**
3742
3861
  * Walk a personas directory and return every file matching the convention
3743
3862
  * `NN-slug.{yaml,yml,json,md}`. Sorted by filename so the numeric prefix
@@ -4610,4 +4729,4 @@ declare namespace index {
4610
4729
  export { type index_AgentProfile as AgentProfile, type index_AgentProfileSection as AgentProfileSection, index_BASELINE_ROLES as BASELINE_ROLES, type index_BaselineRoleKey as BaselineRoleKey, type index_ProfileSkill as ProfileSkill, index_applyDomainPatch as applyDomainPatch, index_baselineProfile as baselineProfile, index_baselineProfileFromRole as baselineProfileFromRole, index_engineerRole as engineerRole, index_generalistRole as generalistRole, index_prodProfile as prodProfile, index_profileToSurface as profileToSurface, index_renderProfile as renderProfile, index_researcherRole as researcherRole, index_sectionHash as sectionHash };
4611
4730
  }
4612
4731
 
4613
- export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, AgentProfile$1 as AgentProfile, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type ErrorCountPattern, type EvolutionRound, type ExecutorConfig, type Expectation, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, HoldoutAuditor, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeFamily, type JudgeFleetOptions, JudgeFn, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, ProductClient, ProductClientConfig, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, RunScore, RunScoreWeights, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type SynthesisReason, type SynthesisTarget, TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, analyzeSeries, appendScorecard, assertCrossFamily, assertSingleBackend, attributeCounterfactuals, bisect, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, ghCliClient, precision as goldenPrecision, hashContent, hashJson, htmlContainsElement, httpGithubClient, inMemoryReferenceReplayStore, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, judgeFamily, keyPreserved, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, matchGoldens, mergeLayerResults, multiToolchainLayer, notBlocked, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, printDriverSummary, index as profile, promptBisect, proposeAutomatedPullRequest, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, replayScorerOverCorpus, replayTraceThroughJudge, resetLockedAppendersForTesting, resolveModelPricing, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, securityJudge, sentenceReorderMutator, signManifest, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
4732
+ export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, AgentProfile$1 as AgentProfile, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type ErrorCountPattern, type EvolutionRound, type ExecutorConfig, type Expectation, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, HoldoutAuditor, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeFamily, type JudgeFleetOptions, JudgeFn, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, ProductClient, ProductClientConfig, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, RunScore, RunScoreWeights, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type SynthesisReason, type SynthesisTarget, TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, analyzeSeries, appendScorecard, assertCrossFamily, assertSingleBackend, attributeCounterfactuals, bisect, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, ghCliClient, precision as goldenPrecision, hashContent, hashJson, htmlContainsElement, httpGithubClient, inMemoryReferenceReplayStore, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, judgeFamily, keyPreserved, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, matchGoldens, mergeLayerResults, modelDescriptionBits, multiToolchainLayer, notBlocked, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, printDriverSummary, index as profile, promptBisect, proposeAutomatedPullRequest, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, replayScorerOverCorpus, replayTraceThroughJudge, resetLockedAppendersForTesting, resolveModelPricing, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, securityJudge, sentenceReorderMutator, signManifest, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
package/dist/index.js CHANGED
@@ -6,6 +6,10 @@ import {
6
6
  parseCorrectnessResponse,
7
7
  verifyCompletion
8
8
  } from "./chunk-YGYXHNAQ.js";
9
+ import {
10
+ parseRuntimeTrajectoryHookEvent,
11
+ projectRuntimeTrajectoryEvidence
12
+ } from "./chunk-T4SQEITX.js";
9
13
  import {
10
14
  HoldoutAuditor,
11
15
  canaryLeakView,
@@ -2911,14 +2915,14 @@ async function runHarnessExperiment(config) {
2911
2915
  const score = config.score ?? ((trace) => critic.scoreTrace(trace));
2912
2916
  const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
2913
2917
  const trace = await config.adapter.run(request);
2914
- const runScore2 = await score(trace, request);
2918
+ const runScore3 = await score(trace, request);
2915
2919
  const result = {
2916
2920
  variant: request.variant,
2917
2921
  scenario: request.scenario,
2918
2922
  trialIndex: request.trialIndex,
2919
2923
  trace,
2920
- score: runScore2,
2921
- aggregate: aggregateRunScore(runScore2, config.weights)
2924
+ score: runScore3,
2925
+ aggregate: aggregateRunScore(runScore3, config.weights)
2922
2926
  };
2923
2927
  await config.onResult?.(result);
2924
2928
  return result;
@@ -7089,6 +7093,121 @@ function createDefaultReviewer(options) {
7089
7093
  };
7090
7094
  }
7091
7095
 
7096
+ // src/description-length-gate.ts
7097
+ import { gzipSync } from "zlib";
7098
+ function runScore2(run) {
7099
+ const o = run.outcome;
7100
+ const s = o.holdoutScore ?? o.searchScore ?? o.raw?.score;
7101
+ return typeof s === "number" && Number.isFinite(s) ? s : void 0;
7102
+ }
7103
+ function taskKey(run) {
7104
+ return run.scenarioId ?? run.experimentId;
7105
+ }
7106
+ function perTaskMeanScore(runs) {
7107
+ const acc = /* @__PURE__ */ new Map();
7108
+ for (const run of runs) {
7109
+ const s = runScore2(run);
7110
+ if (s === void 0) continue;
7111
+ const key = taskKey(run);
7112
+ const cur = acc.get(key) ?? { sum: 0, n: 0 };
7113
+ cur.sum += s;
7114
+ cur.n += 1;
7115
+ acc.set(key, cur);
7116
+ }
7117
+ return new Map([...acc].map(([k, v]) => [k, v.sum / v.n]));
7118
+ }
7119
+ function modelDescriptionBits(content) {
7120
+ return gzipSync(Buffer.from(content, "utf8")).byteLength * 8;
7121
+ }
7122
+ function dataDescriptionBits(scoreByTask, keys, scoreFloor) {
7123
+ let bits = 0;
7124
+ for (const key of keys) {
7125
+ const s = scoreByTask.get(key);
7126
+ if (s === void 0) continue;
7127
+ bits += -Math.log2(Math.max(s, scoreFloor));
7128
+ }
7129
+ return bits;
7130
+ }
7131
+ var DescriptionLengthGate = class {
7132
+ baselineKey;
7133
+ lambda;
7134
+ marginBits;
7135
+ scoreFloor;
7136
+ minTasks;
7137
+ constructor(config) {
7138
+ if (!config.baselineKey) throw new Error("DescriptionLengthGate: baselineKey is required");
7139
+ this.baselineKey = config.baselineKey;
7140
+ this.lambda = config.lambda ?? 1;
7141
+ this.marginBits = config.marginBits ?? 0;
7142
+ this.scoreFloor = config.scoreFloor ?? 2 ** -10;
7143
+ this.minTasks = config.minTasks ?? 3;
7144
+ if (!(this.lambda >= 0)) throw new Error("DescriptionLengthGate: lambda must be \u2265 0");
7145
+ if (!(this.scoreFloor > 0 && this.scoreFloor < 1))
7146
+ throw new Error("DescriptionLengthGate: scoreFloor must be in (0,1)");
7147
+ }
7148
+ /** Decide whether `candidate` should replace `baseline`. Both are scored on
7149
+ * the shared task set (the enlarged evidence); the candidate promotes only
7150
+ * if λ·L_model + L_data is strictly lower by at least `marginBits`. */
7151
+ evaluate(candidate, baseline) {
7152
+ const candidateId = inferCandidateId(candidate.runs, this.baselineKey);
7153
+ const candScores = perTaskMeanScore(candidate.runs);
7154
+ const baseScores = perTaskMeanScore(baseline.runs);
7155
+ const shared = [...candScores.keys()].filter((k) => baseScores.has(k));
7156
+ const modelBits = {
7157
+ candidate: this.lambda * modelDescriptionBits(candidate.content),
7158
+ baseline: this.lambda * modelDescriptionBits(baseline.content)
7159
+ };
7160
+ const dataBits = {
7161
+ candidate: dataDescriptionBits(candScores, shared, this.scoreFloor),
7162
+ baseline: dataDescriptionBits(baseScores, shared, this.scoreFloor)
7163
+ };
7164
+ const totalBits = {
7165
+ candidate: modelBits.candidate + dataBits.candidate,
7166
+ baseline: modelBits.baseline + dataBits.baseline
7167
+ };
7168
+ const evidence = {
7169
+ tasks: shared.length,
7170
+ modelBits,
7171
+ dataBits,
7172
+ totalBits,
7173
+ deltaBits: totalBits.candidate - totalBits.baseline,
7174
+ modelBitsDelta: modelBits.candidate - modelBits.baseline,
7175
+ dataBitsDelta: dataBits.candidate - dataBits.baseline
7176
+ };
7177
+ const base = { candidateId, baselineId: this.baselineKey, evidence };
7178
+ const fmt2 = (n) => n.toFixed(1);
7179
+ if (shared.length < this.minTasks) {
7180
+ return {
7181
+ ...base,
7182
+ promote: false,
7183
+ reason: `few_tasks: ${shared.length} shared task(s) < min ${this.minTasks}`,
7184
+ rejectionCode: "few_tasks"
7185
+ };
7186
+ }
7187
+ if (!(evidence.deltaBits < -this.marginBits)) {
7188
+ const code = evidence.dataBitsDelta < 0 ? "model_bloat" : "no_total_gain";
7189
+ const why = code === "model_bloat" ? `model grew ${fmt2(evidence.modelBitsDelta)} bits, outpacing a ${fmt2(-evidence.dataBitsDelta)}-bit data gain` : `outcomes did not improve (data \u0394=${fmt2(evidence.dataBitsDelta)} bits)`;
7190
+ return {
7191
+ ...base,
7192
+ promote: false,
7193
+ reason: `${code}: total \u0394=${fmt2(evidence.deltaBits)} bits does not clear the ${fmt2(this.marginBits)}-bit margin \u2014 ${why}`,
7194
+ rejectionCode: code
7195
+ };
7196
+ }
7197
+ return {
7198
+ ...base,
7199
+ promote: true,
7200
+ reason: `promote: total \u0394=${fmt2(evidence.deltaBits)} bits (model \u0394=${fmt2(evidence.modelBitsDelta)}, data \u0394=${fmt2(evidence.dataBitsDelta)}) over ${shared.length} tasks`,
7201
+ rejectionCode: null
7202
+ };
7203
+ }
7204
+ };
7205
+ function inferCandidateId(runs, baselineKey) {
7206
+ for (const run of runs)
7207
+ if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
7208
+ return runs[0]?.candidateId ?? "(unknown candidate)";
7209
+ }
7210
+
7092
7211
  // src/discover-personas.ts
7093
7212
  import { promises as fs } from "fs";
7094
7213
  import { basename, extname, join as join3 } from "path";
@@ -7240,7 +7359,7 @@ var HeldOutGate = class {
7240
7359
  * the candidate run with the matching baseline run. Pairs without
7241
7360
  * a holdout score on both sides are dropped. */
7242
7361
  evaluate(candidate, baseline) {
7243
- const candidateId = inferCandidateId(candidate, this.baselineKey);
7362
+ const candidateId = inferCandidateId2(candidate, this.baselineKey);
7244
7363
  const baselineId = this.baselineKey;
7245
7364
  const baselineHoldoutByKey = indexHoldoutByKey(baseline);
7246
7365
  const beforeHoldout = [];
@@ -7343,7 +7462,7 @@ var HeldOutGate = class {
7343
7462
  };
7344
7463
  }
7345
7464
  };
7346
- function inferCandidateId(candidate, baselineKey) {
7465
+ function inferCandidateId2(candidate, baselineKey) {
7347
7466
  for (const run of candidate) {
7348
7467
  if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
7349
7468
  }
@@ -8314,6 +8433,7 @@ export {
8314
8433
  DEFAULT_TRACE_ANALYST_BUDGETS,
8315
8434
  DEFAULT_TRACE_ANALYST_KINDS,
8316
8435
  Dataset,
8436
+ DescriptionLengthGate,
8317
8437
  DockerSandboxDriver,
8318
8438
  DualAgentBench,
8319
8439
  ERROR_COUNT_PATTERNS,
@@ -8485,6 +8605,7 @@ export {
8485
8605
  createTraceAnalystKind,
8486
8606
  crossTraceDiff,
8487
8607
  crowdingDistance,
8608
+ dataDescriptionBits,
8488
8609
  decideNextUserTurn,
8489
8610
  decideReferenceReplayPromotion,
8490
8611
  decideReferenceReplayRunPromotion,
@@ -8591,6 +8712,7 @@ export {
8591
8712
  matchGoldens,
8592
8713
  mergeLayerResults,
8593
8714
  mergeSteeringBundle,
8715
+ modelDescriptionBits,
8594
8716
  multiToolchainLayer,
8595
8717
  nistAiRmfReport,
8596
8718
  normalizeScores,
@@ -8613,6 +8735,7 @@ export {
8613
8735
  parseGoldJsonl,
8614
8736
  parseReflectionResponse,
8615
8737
  parseRunRecordSafe,
8738
+ parseRuntimeTrajectoryHookEvent,
8616
8739
  partialCredit,
8617
8740
  passOrthogonality,
8618
8741
  pixelDeltaRatio,
@@ -8623,6 +8746,7 @@ export {
8623
8746
  probeLlm,
8624
8747
  profile_exports as profile,
8625
8748
  projectOtlpFlatLine,
8749
+ projectRuntimeTrajectoryEvidence,
8626
8750
  promptBisect,
8627
8751
  proposeAutomatedPullRequest,
8628
8752
  proposeSynthesisTargets,