@tangle-network/agent-eval 0.83.0 → 0.84.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/belief-state/index.d.ts +81 -2
- package/dist/belief-state/index.js +160 -4
- package/dist/belief-state/index.js.map +1 -1
- package/dist/chunk-T4SQEITX.js +95 -0
- package/dist/chunk-T4SQEITX.js.map +1 -0
- package/dist/index.d.ts +120 -1
- package/dist/index.js +129 -5
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/runtime-trajectory-BLRiaifm.d.ts +49 -0
- package/docs/research/belief-state-agent-eval-roadmap.md +3 -0
- package/package.json +1 -1
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// src/runtime-trajectory.ts
|
|
2
|
+
var DEFAULT_SPLIT_TAG = "search";
|
|
3
|
+
function projectRuntimeTrajectoryEvidence(options) {
|
|
4
|
+
const diagnostics = [];
|
|
5
|
+
const runsById = /* @__PURE__ */ new Map();
|
|
6
|
+
const events = [];
|
|
7
|
+
let recordWithRuntimeEventsCount = 0;
|
|
8
|
+
let defaultedSplitCount = 0;
|
|
9
|
+
for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {
|
|
10
|
+
const record = options.records[recordIndex];
|
|
11
|
+
const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf);
|
|
12
|
+
const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG;
|
|
13
|
+
if (record.splitTag === void 0) defaultedSplitCount += 1;
|
|
14
|
+
const rawEvents = record.runtimeEvents;
|
|
15
|
+
if (!Array.isArray(rawEvents)) {
|
|
16
|
+
diagnostics.push(
|
|
17
|
+
`${key}: runtimeEvents is not an array; no runtime run join can be extracted`
|
|
18
|
+
);
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
if (rawEvents.length === 0) {
|
|
22
|
+
diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`);
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
recordWithRuntimeEventsCount += 1;
|
|
26
|
+
for (let index = 0; index < rawEvents.length; index += 1) {
|
|
27
|
+
const event = parseRuntimeTrajectoryHookEvent(rawEvents[index]);
|
|
28
|
+
if (!event) {
|
|
29
|
+
diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`);
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
events.push(event);
|
|
33
|
+
const scenarioId = event.scenarioId ?? stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ?? stringOrUndefined(record.scenarioId);
|
|
34
|
+
const prior = runsById.get(event.runId);
|
|
35
|
+
if (!prior) {
|
|
36
|
+
runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag });
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {
|
|
40
|
+
diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const runs = [...runsById.values()];
|
|
45
|
+
return {
|
|
46
|
+
runs,
|
|
47
|
+
events,
|
|
48
|
+
summary: {
|
|
49
|
+
recordCount: options.records.length,
|
|
50
|
+
recordWithRuntimeEventsCount,
|
|
51
|
+
runtimeRunCount: runs.length,
|
|
52
|
+
lifecycleEventCount: events.length,
|
|
53
|
+
defaultedSplitCount
|
|
54
|
+
},
|
|
55
|
+
diagnostics
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function parseRuntimeTrajectoryHookEvent(input) {
|
|
59
|
+
if (!isRecord(input)) return null;
|
|
60
|
+
if (typeof input.id !== "string" || input.id.length === 0) return null;
|
|
61
|
+
if (typeof input.runId !== "string" || input.runId.length === 0) return null;
|
|
62
|
+
if (typeof input.target !== "string" || input.target.length === 0) return null;
|
|
63
|
+
if (typeof input.phase !== "string" || input.phase.length === 0) return null;
|
|
64
|
+
if (typeof input.timestamp !== "number" || !Number.isFinite(input.timestamp)) return null;
|
|
65
|
+
return {
|
|
66
|
+
id: input.id,
|
|
67
|
+
runId: input.runId,
|
|
68
|
+
scenarioId: stringOrUndefined(input.scenarioId),
|
|
69
|
+
target: input.target,
|
|
70
|
+
phase: input.phase,
|
|
71
|
+
timestamp: input.timestamp,
|
|
72
|
+
stepIndex: finiteNumberOrUndefined(input.stepIndex),
|
|
73
|
+
parentId: stringOrUndefined(input.parentId),
|
|
74
|
+
payload: input.payload,
|
|
75
|
+
metadata: isRecord(input.metadata) ? { ...input.metadata } : void 0
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
|
|
79
|
+
return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
|
|
80
|
+
}
|
|
81
|
+
function isRecord(value) {
|
|
82
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
83
|
+
}
|
|
84
|
+
function stringOrUndefined(value) {
|
|
85
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
86
|
+
}
|
|
87
|
+
function finiteNumberOrUndefined(value) {
|
|
88
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export {
|
|
92
|
+
projectRuntimeTrajectoryEvidence,
|
|
93
|
+
parseRuntimeTrajectoryHookEvent
|
|
94
|
+
};
|
|
95
|
+
//# sourceMappingURL=chunk-T4SQEITX.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/runtime-trajectory.ts"],"sourcesContent":["import type { RunSplitTag } from './run-record'\n\nexport interface RuntimeTrajectoryHookEvent {\n id: string\n runId: string\n scenarioId?: string\n target: string\n phase: string\n timestamp: number\n stepIndex?: number\n parentId?: string\n payload?: unknown\n metadata?: Record<string, unknown>\n}\n\nexport interface RuntimeTrajectoryRecord {\n id?: string\n scenarioId?: string\n splitTag?: RunSplitTag\n runtimeEvents?: unknown\n [key: string]: unknown\n}\n\nexport interface RuntimeTrajectoryRunRecord {\n runId: string\n scenarioId?: string\n splitTag: RunSplitTag\n}\n\nexport interface RuntimeTrajectoryEvidenceSummary {\n recordCount: number\n recordWithRuntimeEventsCount: number\n runtimeRunCount: number\n lifecycleEventCount: number\n defaultedSplitCount: number\n}\n\nexport interface RuntimeTrajectoryEvidenceProjection {\n runs: RuntimeTrajectoryRunRecord[]\n events: RuntimeTrajectoryHookEvent[]\n summary: RuntimeTrajectoryEvidenceSummary\n diagnostics: string[]\n}\n\nexport interface ProjectRuntimeTrajectoryEvidenceOptions<\n TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord,\n> {\n records: TRecord[]\n defaultSplitTag?: RunSplitTag\n recordIdOf?: (record: TRecord, index: number) => string | undefined\n scenarioIdOf?: (record: TRecord, index: number) => string | undefined\n}\n\nconst DEFAULT_SPLIT_TAG: RunSplitTag = 'search'\n\nexport function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(\n options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>,\n): RuntimeTrajectoryEvidenceProjection {\n const diagnostics: string[] = []\n const runsById = new Map<string, RuntimeTrajectoryRunRecord>()\n const events: RuntimeTrajectoryHookEvent[] = []\n let recordWithRuntimeEventsCount = 0\n let defaultedSplitCount = 0\n\n for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {\n const record = options.records[recordIndex]!\n const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf)\n const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG\n if (record.splitTag === undefined) defaultedSplitCount += 1\n\n const rawEvents = record.runtimeEvents\n if (!Array.isArray(rawEvents)) {\n diagnostics.push(\n `${key}: runtimeEvents is not an array; no runtime run join can be extracted`,\n )\n continue\n }\n if (rawEvents.length === 0) {\n diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`)\n continue\n }\n recordWithRuntimeEventsCount += 1\n\n for (let index = 0; index < rawEvents.length; index += 1) {\n const event = parseRuntimeTrajectoryHookEvent(rawEvents[index])\n if (!event) {\n diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`)\n continue\n }\n events.push(event)\n\n const scenarioId =\n event.scenarioId ??\n stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ??\n stringOrUndefined(record.scenarioId)\n const prior = runsById.get(event.runId)\n if (!prior) {\n runsById.set(event.runId, { runId: event.runId, scenarioId, splitTag })\n continue\n }\n if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) {\n diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`)\n }\n }\n }\n\n const runs = [...runsById.values()]\n return {\n runs,\n events,\n summary: {\n recordCount: options.records.length,\n recordWithRuntimeEventsCount,\n runtimeRunCount: runs.length,\n lifecycleEventCount: events.length,\n defaultedSplitCount,\n },\n diagnostics,\n }\n}\n\nexport function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null {\n if (!isRecord(input)) return null\n if (typeof input.id !== 'string' || input.id.length === 0) return null\n if (typeof input.runId !== 'string' || input.runId.length === 0) return null\n if (typeof input.target !== 'string' || input.target.length === 0) return null\n if (typeof input.phase !== 'string' || input.phase.length === 0) return null\n if (typeof input.timestamp !== 'number' || !Number.isFinite(input.timestamp)) return null\n\n return {\n id: input.id,\n runId: input.runId,\n scenarioId: stringOrUndefined(input.scenarioId),\n target: input.target,\n phase: input.phase,\n timestamp: input.timestamp,\n stepIndex: finiteNumberOrUndefined(input.stepIndex),\n parentId: stringOrUndefined(input.parentId),\n payload: input.payload,\n metadata: isRecord(input.metadata) ? { ...input.metadata } : undefined,\n }\n}\n\nfunction runtimeTrajectoryRecordKey<TRecord extends RuntimeTrajectoryRecord>(\n record: TRecord,\n index: number,\n recordIdOf?: (record: TRecord, index: number) => string | undefined,\n): string {\n return (\n stringOrUndefined(recordIdOf?.(record, index)) ??\n stringOrUndefined(record.id) ??\n `record[${index}]`\n )\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n\nfunction stringOrUndefined(value: unknown): string | undefined {\n return typeof value === 'string' && value.length > 0 ? value : undefined\n}\n\nfunction finiteNumberOrUndefined(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isFinite(value) ? value : undefined\n}\n"],"mappings":";AAqDA,IAAM,oBAAiC;AAEhC,SAAS,iCACd,SACqC;AACrC,QAAM,cAAwB,CAAC;AAC/B,QAAM,WAAW,oBAAI,IAAwC;AAC7D,QAAM,SAAuC,CAAC;AAC9C,MAAI,+BAA+B;AACnC,MAAI,sBAAsB;AAE1B,WAAS,cAAc,GAAG,cAAc,QAAQ,QAAQ,QAAQ,eAAe,GAAG;AAChF,UAAM,SAAS,QAAQ,QAAQ,WAAW;AAC1C,UAAM,MAAM,2BAA2B,QAAQ,aAAa,QAAQ,UAAU;AAC9E,UAAM,WAAW,OAAO,YAAY,QAAQ,mBAAmB;AAC/D,QAAI,OAAO,aAAa,OAAW,wBAAuB;AAE1D,UAAM,YAAY,OAAO;AACzB,QAAI,CAAC,MAAM,QAAQ,SAAS,GAAG;AAC7B,kBAAY;AAAA,QACV,GAAG,GAAG;AAAA,MACR;AACA;AAAA,IACF;AACA,QAAI,UAAU,WAAW,GAAG;AAC1B,kBAAY,KAAK,GAAG,GAAG,0DAA0D;AACjF;AAAA,IACF;AACA,oCAAgC;AAEhC,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,GAAG;AACxD,YAAM,QAAQ,gCAAgC,UAAU,KAAK,CAAC;AAC9D,UAAI,CAAC,OAAO;AACV,oBAAY,KAAK,GAAG,GAAG,mBAAmB,KAAK,6BAA6B;AAC5E;AAAA,MACF;AACA,aAAO,KAAK,KAAK;AAEjB,YAAM,aACJ,MAAM,cACN,kBAAkB,QAAQ,eAAe,QAAQ,WAAW,CAAC,KAC7D,kBAAkB,OAAO,UAAU;AACrC,YAAM,QAAQ,SAAS,IAAI,MAAM,KAAK;AACtC,UAAI,CAAC,OAAO;AACV,iBAAS,IAAI,MAAM,OAAO,EAAE,OAAO,MAAM,OAAO,YAAY,SAAS,CAAC;AACtE;AAAA,MACF;AACA,UAAI,MAAM,eAAe,cAAc,MAAM,aAAa,UAAU;AAClE,oBAAY,KAAK,GAAG,GAAG,WAAW,MAAM,KAAK,0CAA0C;AAAA,MACzF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;AAClC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS;AAAA,MACP,aAAa,QAAQ,QAAQ;AAAA,MAC7B;AAAA,MACA,iBAAiB,KAAK;AAAA,MACtB,qBAAqB,OAAO;AAAA,MAC5B;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,gCAAgC,OAAmD;AACjG,MAAI,CAAC,SAAS,KAAK,EAAG,QAAO;AAC7B,MAAI,OAAO,MAAM,OAAO,YAAY,MAAM,GAAG,WAAW,EAAG,QAAO;AAClE,MAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,EAAG,QAAO;AACxE,MAAI,OAAO,MAAM,WAAW,YAAY,MAAM,OAAO,WAAW,EAAG,QAAO;AAC1E,MAAI,OAAO,MAAM,UAAU,YAAY,MAAM,MAAM,WAAW,EAAG,QAAO;AACxE,MAAI,OAAO,MAAM,cAAc,YAAY,CAAC,OAAO,SAAS,MAAM,SAAS,EAAG,QAAO;AAErF,SAAO;AAAA,IACL,IAAI,MAAM;AAAA,IACV,OAAO,MAAM;AAAA,IACb,YAAY,kBAAkB,MAAM,UAAU;AAAA,IAC9C,QAAQ,MAAM;AAAA,IACd,OAAO,MAAM;AAAA,IACb,WAAW,MAAM;AAAA,IACjB,WAAW,wBAAwB,MAAM,SAAS;AAAA,IAClD,UAAU,kBAAkB,MAAM,QAAQ;AAAA,IAC1C,SAAS,MAAM;AAAA,IACf,UAAU,SAAS,MAAM,QAAQ,IAAI,EAAE,GAAG,MAAM,SAAS,IAAI;AAAA,EAC/D;AACF;AAEA,SAAS,2BACP,QACA,OACA,YACQ;AACR,SACE,kBAAkB,aAAa,QAAQ,KAAK,CAAC,KAC7C,kBAAkB,OAAO,EAAE,KAC3B,UAAU,KAAK;AAEnB;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,kBAAkB,OAAoC;AAC7D,SAAO,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ;AACjE;AAEA,SAAS,wBAAwB,OAAoC;AACnE,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;","names":[]}
|
package/dist/index.d.ts
CHANGED
|
@@ -42,6 +42,7 @@ export { A as Artifact, E as EventKind, i as FAILURE_CLASSES, F as FailureClass,
|
|
|
42
42
|
import { T as TraceStore, R as RunFilter } from './store-CKUAgsJz.js';
|
|
43
43
|
export { E as EventFilter, F as FileSystemTraceStore, a as FileSystemTraceStoreOptions, I as InMemoryTraceStore, S as SpanFilter } from './store-CKUAgsJz.js';
|
|
44
44
|
export { D as DEFAULT_FAILURE_RULES, b as FailureClassification, c as FailureContext, d as FailureRule, e as classifyFailure } from './failure-cluster-CL7IVgkJ.js';
|
|
45
|
+
export { P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection, b as RuntimeTrajectoryEvidenceSummary, c as RuntimeTrajectoryHookEvent, R as RuntimeTrajectoryRecord, d as RuntimeTrajectoryRunRecord, p as parseRuntimeTrajectoryHookEvent, e as projectRuntimeTrajectoryEvidence } from './runtime-trajectory-BLRiaifm.js';
|
|
45
46
|
import { a as BaselineReport } from './baseline-DE36-Np7.js';
|
|
46
47
|
export { B as BaselineOptions, M as MetricSamples, b as MetricVerdict, T as ToolStats, d as ToolUseMetrics, e as ToolUseOptions, f as compareToBaseline, c as computeToolUseMetrics, i as iqr, w as welchsTTest } from './baseline-DE36-Np7.js';
|
|
47
48
|
import { T as Trajectory, a as TrajectoryStep } from './trajectory-GEdXJCL5.js';
|
|
@@ -3738,6 +3739,124 @@ declare class Mutex {
|
|
|
3738
3739
|
get pending(): number;
|
|
3739
3740
|
}
|
|
3740
3741
|
|
|
3742
|
+
/**
|
|
3743
|
+
* DescriptionLengthGate — a Minimum-Description-Length promotion gate, the
|
|
3744
|
+
* Builder/Breaker acceptance rule from Wang & Buehler, "Self-Revising Discovery
|
|
3745
|
+
* Systems for Science" (arXiv:2606.01444, MIT LAMM), eq. 5:
|
|
3746
|
+
*
|
|
3747
|
+
* L(M, D) = L_model(M) + L_data(D | M)
|
|
3748
|
+
* accept M' over M iff L(M', D∪E) < L(M, D∪E)
|
|
3749
|
+
*
|
|
3750
|
+
* Both candidate and baseline are scored on the SAME enlarged evidence set
|
|
3751
|
+
* (every accumulated task — NOT just the held-out split), and the candidate is
|
|
3752
|
+
* accepted only if it lowers the TOTAL bit cost. This is the gate's whole
|
|
3753
|
+
* point and what distinguishes it from a monotone held-out delta:
|
|
3754
|
+
*
|
|
3755
|
+
* - L_model(M) — the candidate's own description length: the compressed size
|
|
3756
|
+
* of its model text (a prompt, skill, profile, or symbolic model). A bigger
|
|
3757
|
+
* model pays more bits.
|
|
3758
|
+
* - L_data(D|M) — the residual: bits of "surprise" that the model did not
|
|
3759
|
+
* simply succeed, −Σ_i log2(s_i) over the model's per-task score s_i.
|
|
3760
|
+
* Perfect scores cost 0 bits; failure costs a lot (capped, not infinite).
|
|
3761
|
+
*
|
|
3762
|
+
* A candidate that merely memorizes new counterexamples grows L_model faster
|
|
3763
|
+
* than it shrinks L_data and LOSES — a principled, complexity-penalized
|
|
3764
|
+
* alternative to HeldOutGate's held-out paired delta. Use this gate
|
|
3765
|
+
* when the model text whose size you want to penalize is available; use
|
|
3766
|
+
* HeldOutGate when promotion should turn on held-out generalization with an
|
|
3767
|
+
* overfit-gap check instead.
|
|
3768
|
+
*
|
|
3769
|
+
* Scale / calibration: a gzip'd prose model is hundreds–thousands of bits;
|
|
3770
|
+
* a single task contributes at most −log2(scoreFloor) data bits (≈10). So with
|
|
3771
|
+
* little evidence the model term dominates and the gate is conservative about
|
|
3772
|
+
* model GROWTH — it promotes a larger model only once accumulated evidence
|
|
3773
|
+
* genuinely pays for the added bits (exactly the paper's regime, where D∪E
|
|
3774
|
+
* grows). `lambda` is the lever: λ<1 discounts model bits (more permissive),
|
|
3775
|
+
* λ>1 is stricter. A shrinking-or-equal model that does no worse always wins.
|
|
3776
|
+
*
|
|
3777
|
+
* Stateless: construct once with the description-length budget, call
|
|
3778
|
+
* `evaluate` per (candidate, baseline) pair.
|
|
3779
|
+
*/
|
|
3780
|
+
type DescriptionLengthRejectionCode = 'few_tasks' | 'no_total_gain' | 'model_bloat';
|
|
3781
|
+
interface DescriptionLengthConfig {
|
|
3782
|
+
/** Stable label of the baseline. Required — paper-grade evaluation never
|
|
3783
|
+
* compares two unlabelled candidates. */
|
|
3784
|
+
baselineKey: string;
|
|
3785
|
+
/** Weight on model bits relative to data bits (the description-length
|
|
3786
|
+
* budget λ). 1 = bits are bits. >1 = more complexity-averse. Default 1. */
|
|
3787
|
+
lambda?: number;
|
|
3788
|
+
/** The candidate must beat the baseline by at least this many bits to
|
|
3789
|
+
* promote — a robustness margin against measurement noise. Default 0
|
|
3790
|
+
* (strict `<`, as the paper). */
|
|
3791
|
+
marginBits?: number;
|
|
3792
|
+
/** Per-task score floor for the residual code: −log2(max(s, floor)). Caps a
|
|
3793
|
+
* total-failure task's surprise instead of letting it diverge. Default
|
|
3794
|
+
* 2^-10 (a failed task costs 10 bits, not ∞). */
|
|
3795
|
+
scoreFloor?: number;
|
|
3796
|
+
/** Minimum number of shared (candidate, baseline) tasks before the gate will
|
|
3797
|
+
* consider promoting. Default 3. */
|
|
3798
|
+
minTasks?: number;
|
|
3799
|
+
}
|
|
3800
|
+
interface DescriptionLengthEvidence {
|
|
3801
|
+
/** Shared tasks scored on both sides (the enlarged evidence D∪E). */
|
|
3802
|
+
tasks: number;
|
|
3803
|
+
/** Compressed-model bits — L_model. */
|
|
3804
|
+
modelBits: {
|
|
3805
|
+
candidate: number;
|
|
3806
|
+
baseline: number;
|
|
3807
|
+
};
|
|
3808
|
+
/** Residual surprise bits — L_data(D|M). */
|
|
3809
|
+
dataBits: {
|
|
3810
|
+
candidate: number;
|
|
3811
|
+
baseline: number;
|
|
3812
|
+
};
|
|
3813
|
+
/** λ·L_model + L_data — the quantity the gate minimizes. */
|
|
3814
|
+
totalBits: {
|
|
3815
|
+
candidate: number;
|
|
3816
|
+
baseline: number;
|
|
3817
|
+
};
|
|
3818
|
+
/** candidate − baseline total. Negative = candidate compresses better. */
|
|
3819
|
+
deltaBits: number;
|
|
3820
|
+
/** Per-component deltas, for audit: did the win come from a smaller model,
|
|
3821
|
+
* better outcomes, or both? */
|
|
3822
|
+
modelBitsDelta: number;
|
|
3823
|
+
dataBitsDelta: number;
|
|
3824
|
+
}
|
|
3825
|
+
interface DescriptionLengthDecision {
|
|
3826
|
+
promote: boolean;
|
|
3827
|
+
candidateId: string;
|
|
3828
|
+
baselineId: string;
|
|
3829
|
+
evidence: DescriptionLengthEvidence;
|
|
3830
|
+
reason: string;
|
|
3831
|
+
rejectionCode: DescriptionLengthRejectionCode | null;
|
|
3832
|
+
}
|
|
3833
|
+
interface DescriptionLengthCandidate {
|
|
3834
|
+
/** The model text whose size is L_model (a prompt, skill, profile, or
|
|
3835
|
+
* symbolic model; concatenated if several files). */
|
|
3836
|
+
content: string;
|
|
3837
|
+
/** Runs whose per-task scores form L_data. */
|
|
3838
|
+
runs: RunRecord[];
|
|
3839
|
+
}
|
|
3840
|
+
/** Compressed-model bits — the model's description length L_model. gzip is a
|
|
3841
|
+
* deterministic, dependency-free stand-in for Kolmogorov complexity; it
|
|
3842
|
+
* rewards genuine compactness and penalizes boilerplate padding. */
|
|
3843
|
+
declare function modelDescriptionBits(content: string): number;
|
|
3844
|
+
/** Residual surprise L_data(D|M) = −Σ_i log2(max(s_i, floor)) over the given
|
|
3845
|
+
* per-task scores. Lower = the model more reliably succeeds. */
|
|
3846
|
+
declare function dataDescriptionBits(scoreByTask: Map<string, number>, keys: Iterable<string>, scoreFloor: number): number;
|
|
3847
|
+
declare class DescriptionLengthGate {
|
|
3848
|
+
private readonly baselineKey;
|
|
3849
|
+
private readonly lambda;
|
|
3850
|
+
private readonly marginBits;
|
|
3851
|
+
private readonly scoreFloor;
|
|
3852
|
+
private readonly minTasks;
|
|
3853
|
+
constructor(config: DescriptionLengthConfig);
|
|
3854
|
+
/** Decide whether `candidate` should replace `baseline`. Both are scored on
|
|
3855
|
+
* the shared task set (the enlarged evidence); the candidate promotes only
|
|
3856
|
+
* if λ·L_model + L_data is strictly lower by at least `marginBits`. */
|
|
3857
|
+
evaluate(candidate: DescriptionLengthCandidate, baseline: DescriptionLengthCandidate): DescriptionLengthDecision;
|
|
3858
|
+
}
|
|
3859
|
+
|
|
3741
3860
|
/**
|
|
3742
3861
|
* Walk a personas directory and return every file matching the convention
|
|
3743
3862
|
* `NN-slug.{yaml,yml,json,md}`. Sorted by filename so the numeric prefix
|
|
@@ -4610,4 +4729,4 @@ declare namespace index {
|
|
|
4610
4729
|
export { type index_AgentProfile as AgentProfile, type index_AgentProfileSection as AgentProfileSection, index_BASELINE_ROLES as BASELINE_ROLES, type index_BaselineRoleKey as BaselineRoleKey, type index_ProfileSkill as ProfileSkill, index_applyDomainPatch as applyDomainPatch, index_baselineProfile as baselineProfile, index_baselineProfileFromRole as baselineProfileFromRole, index_engineerRole as engineerRole, index_generalistRole as generalistRole, index_prodProfile as prodProfile, index_profileToSurface as profileToSurface, index_renderProfile as renderProfile, index_researcherRole as researcherRole, index_sectionHash as sectionHash };
|
|
4611
4730
|
}
|
|
4612
4731
|
|
|
4613
|
-
export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, AgentProfile$1 as AgentProfile, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type ErrorCountPattern, type EvolutionRound, type ExecutorConfig, type Expectation, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, HoldoutAuditor, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeFamily, type JudgeFleetOptions, JudgeFn, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, ProductClient, ProductClientConfig, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, RunScore, RunScoreWeights, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type SynthesisReason, type SynthesisTarget, TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, analyzeSeries, appendScorecard, assertCrossFamily, assertSingleBackend, attributeCounterfactuals, bisect, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, ghCliClient, precision as goldenPrecision, hashContent, hashJson, htmlContainsElement, httpGithubClient, inMemoryReferenceReplayStore, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, judgeFamily, keyPreserved, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, matchGoldens, mergeLayerResults, multiToolchainLayer, notBlocked, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, printDriverSummary, index as profile, promptBisect, proposeAutomatedPullRequest, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, replayScorerOverCorpus, replayTraceThroughJudge, resetLockedAppendersForTesting, resolveModelPricing, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, securityJudge, sentenceReorderMutator, signManifest, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
|
|
4732
|
+
export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, AgentProfile$1 as AgentProfile, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type ErrorCountPattern, type EvolutionRound, type ExecutorConfig, type Expectation, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, HoldoutAuditor, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeFamily, type JudgeFleetOptions, JudgeFn, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, ProductClient, ProductClientConfig, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, RunScore, RunScoreWeights, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type SynthesisReason, type SynthesisTarget, TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, analyzeSeries, appendScorecard, assertCrossFamily, assertSingleBackend, attributeCounterfactuals, bisect, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, ghCliClient, precision as goldenPrecision, hashContent, hashJson, htmlContainsElement, httpGithubClient, inMemoryReferenceReplayStore, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, judgeFamily, keyPreserved, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, matchGoldens, mergeLayerResults, modelDescriptionBits, multiToolchainLayer, notBlocked, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, printDriverSummary, index as profile, promptBisect, proposeAutomatedPullRequest, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, replayScorerOverCorpus, replayTraceThroughJudge, resetLockedAppendersForTesting, resolveModelPricing, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, securityJudge, sentenceReorderMutator, signManifest, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,10 @@ import {
|
|
|
6
6
|
parseCorrectnessResponse,
|
|
7
7
|
verifyCompletion
|
|
8
8
|
} from "./chunk-YGYXHNAQ.js";
|
|
9
|
+
import {
|
|
10
|
+
parseRuntimeTrajectoryHookEvent,
|
|
11
|
+
projectRuntimeTrajectoryEvidence
|
|
12
|
+
} from "./chunk-T4SQEITX.js";
|
|
9
13
|
import {
|
|
10
14
|
HoldoutAuditor,
|
|
11
15
|
canaryLeakView,
|
|
@@ -2911,14 +2915,14 @@ async function runHarnessExperiment(config) {
|
|
|
2911
2915
|
const score = config.score ?? ((trace) => critic.scoreTrace(trace));
|
|
2912
2916
|
const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
|
|
2913
2917
|
const trace = await config.adapter.run(request);
|
|
2914
|
-
const
|
|
2918
|
+
const runScore3 = await score(trace, request);
|
|
2915
2919
|
const result = {
|
|
2916
2920
|
variant: request.variant,
|
|
2917
2921
|
scenario: request.scenario,
|
|
2918
2922
|
trialIndex: request.trialIndex,
|
|
2919
2923
|
trace,
|
|
2920
|
-
score:
|
|
2921
|
-
aggregate: aggregateRunScore(
|
|
2924
|
+
score: runScore3,
|
|
2925
|
+
aggregate: aggregateRunScore(runScore3, config.weights)
|
|
2922
2926
|
};
|
|
2923
2927
|
await config.onResult?.(result);
|
|
2924
2928
|
return result;
|
|
@@ -7089,6 +7093,121 @@ function createDefaultReviewer(options) {
|
|
|
7089
7093
|
};
|
|
7090
7094
|
}
|
|
7091
7095
|
|
|
7096
|
+
// src/description-length-gate.ts
|
|
7097
|
+
import { gzipSync } from "zlib";
|
|
7098
|
+
function runScore2(run) {
|
|
7099
|
+
const o = run.outcome;
|
|
7100
|
+
const s = o.holdoutScore ?? o.searchScore ?? o.raw?.score;
|
|
7101
|
+
return typeof s === "number" && Number.isFinite(s) ? s : void 0;
|
|
7102
|
+
}
|
|
7103
|
+
function taskKey(run) {
|
|
7104
|
+
return run.scenarioId ?? run.experimentId;
|
|
7105
|
+
}
|
|
7106
|
+
function perTaskMeanScore(runs) {
|
|
7107
|
+
const acc = /* @__PURE__ */ new Map();
|
|
7108
|
+
for (const run of runs) {
|
|
7109
|
+
const s = runScore2(run);
|
|
7110
|
+
if (s === void 0) continue;
|
|
7111
|
+
const key = taskKey(run);
|
|
7112
|
+
const cur = acc.get(key) ?? { sum: 0, n: 0 };
|
|
7113
|
+
cur.sum += s;
|
|
7114
|
+
cur.n += 1;
|
|
7115
|
+
acc.set(key, cur);
|
|
7116
|
+
}
|
|
7117
|
+
return new Map([...acc].map(([k, v]) => [k, v.sum / v.n]));
|
|
7118
|
+
}
|
|
7119
|
+
function modelDescriptionBits(content) {
|
|
7120
|
+
return gzipSync(Buffer.from(content, "utf8")).byteLength * 8;
|
|
7121
|
+
}
|
|
7122
|
+
function dataDescriptionBits(scoreByTask, keys, scoreFloor) {
|
|
7123
|
+
let bits = 0;
|
|
7124
|
+
for (const key of keys) {
|
|
7125
|
+
const s = scoreByTask.get(key);
|
|
7126
|
+
if (s === void 0) continue;
|
|
7127
|
+
bits += -Math.log2(Math.max(s, scoreFloor));
|
|
7128
|
+
}
|
|
7129
|
+
return bits;
|
|
7130
|
+
}
|
|
7131
|
+
var DescriptionLengthGate = class {
|
|
7132
|
+
baselineKey;
|
|
7133
|
+
lambda;
|
|
7134
|
+
marginBits;
|
|
7135
|
+
scoreFloor;
|
|
7136
|
+
minTasks;
|
|
7137
|
+
constructor(config) {
|
|
7138
|
+
if (!config.baselineKey) throw new Error("DescriptionLengthGate: baselineKey is required");
|
|
7139
|
+
this.baselineKey = config.baselineKey;
|
|
7140
|
+
this.lambda = config.lambda ?? 1;
|
|
7141
|
+
this.marginBits = config.marginBits ?? 0;
|
|
7142
|
+
this.scoreFloor = config.scoreFloor ?? 2 ** -10;
|
|
7143
|
+
this.minTasks = config.minTasks ?? 3;
|
|
7144
|
+
if (!(this.lambda >= 0)) throw new Error("DescriptionLengthGate: lambda must be \u2265 0");
|
|
7145
|
+
if (!(this.scoreFloor > 0 && this.scoreFloor < 1))
|
|
7146
|
+
throw new Error("DescriptionLengthGate: scoreFloor must be in (0,1)");
|
|
7147
|
+
}
|
|
7148
|
+
/** Decide whether `candidate` should replace `baseline`. Both are scored on
|
|
7149
|
+
* the shared task set (the enlarged evidence); the candidate promotes only
|
|
7150
|
+
* if λ·L_model + L_data is strictly lower by at least `marginBits`. */
|
|
7151
|
+
evaluate(candidate, baseline) {
|
|
7152
|
+
const candidateId = inferCandidateId(candidate.runs, this.baselineKey);
|
|
7153
|
+
const candScores = perTaskMeanScore(candidate.runs);
|
|
7154
|
+
const baseScores = perTaskMeanScore(baseline.runs);
|
|
7155
|
+
const shared = [...candScores.keys()].filter((k) => baseScores.has(k));
|
|
7156
|
+
const modelBits = {
|
|
7157
|
+
candidate: this.lambda * modelDescriptionBits(candidate.content),
|
|
7158
|
+
baseline: this.lambda * modelDescriptionBits(baseline.content)
|
|
7159
|
+
};
|
|
7160
|
+
const dataBits = {
|
|
7161
|
+
candidate: dataDescriptionBits(candScores, shared, this.scoreFloor),
|
|
7162
|
+
baseline: dataDescriptionBits(baseScores, shared, this.scoreFloor)
|
|
7163
|
+
};
|
|
7164
|
+
const totalBits = {
|
|
7165
|
+
candidate: modelBits.candidate + dataBits.candidate,
|
|
7166
|
+
baseline: modelBits.baseline + dataBits.baseline
|
|
7167
|
+
};
|
|
7168
|
+
const evidence = {
|
|
7169
|
+
tasks: shared.length,
|
|
7170
|
+
modelBits,
|
|
7171
|
+
dataBits,
|
|
7172
|
+
totalBits,
|
|
7173
|
+
deltaBits: totalBits.candidate - totalBits.baseline,
|
|
7174
|
+
modelBitsDelta: modelBits.candidate - modelBits.baseline,
|
|
7175
|
+
dataBitsDelta: dataBits.candidate - dataBits.baseline
|
|
7176
|
+
};
|
|
7177
|
+
const base = { candidateId, baselineId: this.baselineKey, evidence };
|
|
7178
|
+
const fmt2 = (n) => n.toFixed(1);
|
|
7179
|
+
if (shared.length < this.minTasks) {
|
|
7180
|
+
return {
|
|
7181
|
+
...base,
|
|
7182
|
+
promote: false,
|
|
7183
|
+
reason: `few_tasks: ${shared.length} shared task(s) < min ${this.minTasks}`,
|
|
7184
|
+
rejectionCode: "few_tasks"
|
|
7185
|
+
};
|
|
7186
|
+
}
|
|
7187
|
+
if (!(evidence.deltaBits < -this.marginBits)) {
|
|
7188
|
+
const code = evidence.dataBitsDelta < 0 ? "model_bloat" : "no_total_gain";
|
|
7189
|
+
const why = code === "model_bloat" ? `model grew ${fmt2(evidence.modelBitsDelta)} bits, outpacing a ${fmt2(-evidence.dataBitsDelta)}-bit data gain` : `outcomes did not improve (data \u0394=${fmt2(evidence.dataBitsDelta)} bits)`;
|
|
7190
|
+
return {
|
|
7191
|
+
...base,
|
|
7192
|
+
promote: false,
|
|
7193
|
+
reason: `${code}: total \u0394=${fmt2(evidence.deltaBits)} bits does not clear the ${fmt2(this.marginBits)}-bit margin \u2014 ${why}`,
|
|
7194
|
+
rejectionCode: code
|
|
7195
|
+
};
|
|
7196
|
+
}
|
|
7197
|
+
return {
|
|
7198
|
+
...base,
|
|
7199
|
+
promote: true,
|
|
7200
|
+
reason: `promote: total \u0394=${fmt2(evidence.deltaBits)} bits (model \u0394=${fmt2(evidence.modelBitsDelta)}, data \u0394=${fmt2(evidence.dataBitsDelta)}) over ${shared.length} tasks`,
|
|
7201
|
+
rejectionCode: null
|
|
7202
|
+
};
|
|
7203
|
+
}
|
|
7204
|
+
};
|
|
7205
|
+
function inferCandidateId(runs, baselineKey) {
|
|
7206
|
+
for (const run of runs)
|
|
7207
|
+
if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
|
|
7208
|
+
return runs[0]?.candidateId ?? "(unknown candidate)";
|
|
7209
|
+
}
|
|
7210
|
+
|
|
7092
7211
|
// src/discover-personas.ts
|
|
7093
7212
|
import { promises as fs } from "fs";
|
|
7094
7213
|
import { basename, extname, join as join3 } from "path";
|
|
@@ -7240,7 +7359,7 @@ var HeldOutGate = class {
|
|
|
7240
7359
|
* the candidate run with the matching baseline run. Pairs without
|
|
7241
7360
|
* a holdout score on both sides are dropped. */
|
|
7242
7361
|
evaluate(candidate, baseline) {
|
|
7243
|
-
const candidateId =
|
|
7362
|
+
const candidateId = inferCandidateId2(candidate, this.baselineKey);
|
|
7244
7363
|
const baselineId = this.baselineKey;
|
|
7245
7364
|
const baselineHoldoutByKey = indexHoldoutByKey(baseline);
|
|
7246
7365
|
const beforeHoldout = [];
|
|
@@ -7343,7 +7462,7 @@ var HeldOutGate = class {
|
|
|
7343
7462
|
};
|
|
7344
7463
|
}
|
|
7345
7464
|
};
|
|
7346
|
-
function
|
|
7465
|
+
function inferCandidateId2(candidate, baselineKey) {
|
|
7347
7466
|
for (const run of candidate) {
|
|
7348
7467
|
if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
|
|
7349
7468
|
}
|
|
@@ -8314,6 +8433,7 @@ export {
|
|
|
8314
8433
|
DEFAULT_TRACE_ANALYST_BUDGETS,
|
|
8315
8434
|
DEFAULT_TRACE_ANALYST_KINDS,
|
|
8316
8435
|
Dataset,
|
|
8436
|
+
DescriptionLengthGate,
|
|
8317
8437
|
DockerSandboxDriver,
|
|
8318
8438
|
DualAgentBench,
|
|
8319
8439
|
ERROR_COUNT_PATTERNS,
|
|
@@ -8485,6 +8605,7 @@ export {
|
|
|
8485
8605
|
createTraceAnalystKind,
|
|
8486
8606
|
crossTraceDiff,
|
|
8487
8607
|
crowdingDistance,
|
|
8608
|
+
dataDescriptionBits,
|
|
8488
8609
|
decideNextUserTurn,
|
|
8489
8610
|
decideReferenceReplayPromotion,
|
|
8490
8611
|
decideReferenceReplayRunPromotion,
|
|
@@ -8591,6 +8712,7 @@ export {
|
|
|
8591
8712
|
matchGoldens,
|
|
8592
8713
|
mergeLayerResults,
|
|
8593
8714
|
mergeSteeringBundle,
|
|
8715
|
+
modelDescriptionBits,
|
|
8594
8716
|
multiToolchainLayer,
|
|
8595
8717
|
nistAiRmfReport,
|
|
8596
8718
|
normalizeScores,
|
|
@@ -8613,6 +8735,7 @@ export {
|
|
|
8613
8735
|
parseGoldJsonl,
|
|
8614
8736
|
parseReflectionResponse,
|
|
8615
8737
|
parseRunRecordSafe,
|
|
8738
|
+
parseRuntimeTrajectoryHookEvent,
|
|
8616
8739
|
partialCredit,
|
|
8617
8740
|
passOrthogonality,
|
|
8618
8741
|
pixelDeltaRatio,
|
|
@@ -8623,6 +8746,7 @@ export {
|
|
|
8623
8746
|
probeLlm,
|
|
8624
8747
|
profile_exports as profile,
|
|
8625
8748
|
projectOtlpFlatLine,
|
|
8749
|
+
projectRuntimeTrajectoryEvidence,
|
|
8626
8750
|
promptBisect,
|
|
8627
8751
|
proposeAutomatedPullRequest,
|
|
8628
8752
|
proposeSynthesisTargets,
|