@tangle-network/agent-eval 0.128.2 → 0.129.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +265 -0
- package/README.md +18 -0
- package/dist/analyst/index.d.ts +107 -165
- package/dist/analyst/index.js +5 -9
- package/dist/analyst/index.js.map +1 -1
- package/dist/belief-state/index.d.ts +2 -19
- package/dist/belief-state/index.js +30 -31
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +5 -8
- package/dist/benchmarks/index.js +12 -11
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +30 -39
- package/dist/campaign/index.js +11 -10
- package/dist/{chunk-NKAGIDE2.js → chunk-2QU3YOPR.js} +15 -274
- package/dist/chunk-2QU3YOPR.js.map +1 -0
- package/dist/{chunk-EJGRPCO3.js → chunk-3OCR4R5I.js} +245 -134
- package/dist/chunk-3OCR4R5I.js.map +1 -0
- package/dist/{chunk-2JX3CFMB.js → chunk-56TAVBOK.js} +5 -2
- package/dist/chunk-56TAVBOK.js.map +1 -0
- package/dist/{chunk-DPUHNQLN.js → chunk-7FO3TNPI.js} +2 -2
- package/dist/{chunk-DJKY2TSY.js → chunk-BSO5JDQH.js} +27 -120
- package/dist/chunk-BSO5JDQH.js.map +1 -0
- package/dist/{chunk-EZJEIH2R.js → chunk-C6LXANRU.js} +11 -20
- package/dist/chunk-C6LXANRU.js.map +1 -0
- package/dist/{chunk-ZUUWPZCV.js → chunk-DODXQREJ.js} +4 -4
- package/dist/{chunk-2MKQIFS4.js → chunk-E7QXT7SX.js} +2 -2
- package/dist/{chunk-NYLOYM6N.js → chunk-EG66UGL4.js} +37 -28
- package/dist/chunk-EG66UGL4.js.map +1 -0
- package/dist/{chunk-P5W7RQKK.js → chunk-FXTVJPYD.js} +2 -2
- package/dist/{chunk-VZSRQ272.js → chunk-G7MGMCZD.js} +6 -2
- package/dist/chunk-G7MGMCZD.js.map +1 -0
- package/dist/{chunk-IHQDPH7D.js → chunk-H23X7XKK.js} +85 -75
- package/dist/chunk-H23X7XKK.js.map +1 -0
- package/dist/{chunk-VBQ3CRKH.js → chunk-HPWUNB47.js} +4 -6
- package/dist/chunk-HPWUNB47.js.map +1 -0
- package/dist/{chunk-XPRT64IE.js → chunk-IYCLP2N2.js} +3 -3
- package/dist/chunk-IYCLP2N2.js.map +1 -0
- package/dist/{chunk-XDWDC2MP.js → chunk-JQSF5DQT.js} +11 -5
- package/dist/chunk-JQSF5DQT.js.map +1 -0
- package/dist/{chunk-NACAGYSY.js → chunk-M4YBQKIJ.js} +11 -11
- package/dist/chunk-M4YBQKIJ.js.map +1 -0
- package/dist/{chunk-YJBNWCAA.js → chunk-NY44NC4A.js} +3 -3
- package/dist/chunk-OIUOT4QD.js +44 -0
- package/dist/chunk-OIUOT4QD.js.map +1 -0
- package/dist/chunk-OWN5NPMC.js +152 -0
- package/dist/chunk-OWN5NPMC.js.map +1 -0
- package/dist/chunk-PC5DOSM7.js +579 -0
- package/dist/chunk-PC5DOSM7.js.map +1 -0
- package/dist/{chunk-UB2LOJ6Q.js → chunk-QB6BDBP2.js} +23 -20
- package/dist/chunk-QB6BDBP2.js.map +1 -0
- package/dist/chunk-RXHCETDZ.js +536 -0
- package/dist/chunk-RXHCETDZ.js.map +1 -0
- package/dist/{chunk-PBE2LOSS.js → chunk-SFLLL76A.js} +7 -7
- package/dist/chunk-SFLLL76A.js.map +1 -0
- package/dist/{chunk-VGRCHJON.js → chunk-T6RLYGAD.js} +3 -8
- package/dist/chunk-T6RLYGAD.js.map +1 -0
- package/dist/{chunk-VLOATJQ2.js → chunk-TJVT4QFF.js} +21 -18
- package/dist/chunk-TJVT4QFF.js.map +1 -0
- package/dist/{chunk-S5YLIBFX.js → chunk-TQ7LNKZ3.js} +2 -2
- package/dist/{chunk-EOSZT7PL.js → chunk-U4L7JRPZ.js} +2 -297
- package/dist/chunk-U4L7JRPZ.js.map +1 -0
- package/dist/chunk-U4PHLT2N.js +419 -0
- package/dist/chunk-U4PHLT2N.js.map +1 -0
- package/dist/{chunk-WS3NZZQQ.js → chunk-VCZ5FQYW.js} +3 -4
- package/dist/chunk-VCZ5FQYW.js.map +1 -0
- package/dist/{chunk-BYT7ELPS.js → chunk-WVATSFCP.js} +2 -2
- package/dist/{chunk-TSN7JT6D.js → chunk-X4YIBDER.js} +21 -5
- package/dist/{chunk-TSN7JT6D.js.map → chunk-X4YIBDER.js.map} +1 -1
- package/dist/{chunk-TBL77AUT.js → chunk-YQN4ICPP.js} +5 -5
- package/dist/{chunk-MHELPNRP.js → chunk-ZHTZ4EYI.js} +1 -1
- package/dist/chunk-ZHTZ4EYI.js.map +1 -0
- package/dist/cli.js +6 -5
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.d.ts +47 -87
- package/dist/contract/index.js +14 -13
- package/dist/contract/index.js.map +1 -1
- package/dist/control.js +3 -2
- package/dist/fuzz.js +3 -2
- package/dist/fuzz.js.map +1 -1
- package/dist/index.d.ts +659 -203
- package/dist/index.js +145 -117
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +3 -4
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +5 -5
- package/dist/reporting.d.ts +14 -0
- package/dist/reporting.js +7 -6
- package/dist/rl.d.ts +652 -82
- package/dist/rl.js +415 -171
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +1071 -32
- package/dist/rollout/index.js +68 -10
- package/dist/run-campaign-OJJ7CZF4.js +18 -0
- package/dist/supervisor-run/index.d.ts +114 -4
- package/dist/supervisor-run/index.js +4 -3
- package/dist/traces.d.ts +1 -1
- package/dist/traces.js +6 -5
- package/dist/wire/index.d.ts +10 -11
- package/dist/wire/index.js +3 -3
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +4 -4
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- /package/dist/{chunk-DPUHNQLN.js.map → chunk-7FO3TNPI.js.map} +0 -0
- /package/dist/{chunk-ZUUWPZCV.js.map → chunk-DODXQREJ.js.map} +0 -0
- /package/dist/{chunk-2MKQIFS4.js.map → chunk-E7QXT7SX.js.map} +0 -0
- /package/dist/{chunk-P5W7RQKK.js.map → chunk-FXTVJPYD.js.map} +0 -0
- /package/dist/{chunk-YJBNWCAA.js.map → chunk-NY44NC4A.js.map} +0 -0
- /package/dist/{chunk-S5YLIBFX.js.map → chunk-TQ7LNKZ3.js.map} +0 -0
- /package/dist/{chunk-BYT7ELPS.js.map → chunk-WVATSFCP.js.map} +0 -0
- /package/dist/{chunk-TBL77AUT.js.map → chunk-YQN4ICPP.js.map} +0 -0
- /package/dist/{run-campaign-ISHFZ7FJ.js.map → run-campaign-OJJ7CZF4.js.map} +0 -0
|
@@ -0,0 +1,579 @@
|
|
|
1
|
+
// src/rollout/gate-checks.ts
|
|
2
|
+
var GATE_CHECK_IDS = [
|
|
3
|
+
"reward-relationship",
|
|
4
|
+
"gated-evidence",
|
|
5
|
+
"undeclared-step-payload",
|
|
6
|
+
"unscreened-reward"
|
|
7
|
+
];
|
|
8
|
+
var field = (subject, name) => subject.outcome?.[name];
|
|
9
|
+
function readReward(value) {
|
|
10
|
+
if (value === null || value === void 0) return { kind: "absent" };
|
|
11
|
+
if (typeof value === "number") {
|
|
12
|
+
if (Number.isNaN(value)) return { kind: "unreadable", shown: "NaN" };
|
|
13
|
+
return value > 0 ? { kind: "positive", shown: String(value) } : { kind: "cleared" };
|
|
14
|
+
}
|
|
15
|
+
return {
|
|
16
|
+
kind: "unreadable",
|
|
17
|
+
shown: `${JSON.stringify(value) ?? String(value)} (${typeof value})`
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
var isPlainRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
21
|
+
function payloadIsPopulated(value) {
|
|
22
|
+
if (value === void 0 || value === null) return false;
|
|
23
|
+
if (isPlainRecord(value)) return Object.keys(value).length > 0;
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
var DECLARED_STEP_KEYS = {
|
|
27
|
+
kind: true,
|
|
28
|
+
name: true,
|
|
29
|
+
input: true,
|
|
30
|
+
output: true,
|
|
31
|
+
status: true,
|
|
32
|
+
durationMs: true,
|
|
33
|
+
llm_call_count: true,
|
|
34
|
+
prompt_token_ids: true,
|
|
35
|
+
completion_token_ids: true,
|
|
36
|
+
logprobs: true
|
|
37
|
+
};
|
|
38
|
+
function undeclaredStepPayload(steps) {
|
|
39
|
+
if (steps === void 0 || steps === null) return void 0;
|
|
40
|
+
if (!Array.isArray(steps)) return steps;
|
|
41
|
+
const extras = [];
|
|
42
|
+
let found = false;
|
|
43
|
+
for (const step of steps) {
|
|
44
|
+
const extra = {};
|
|
45
|
+
if (isPlainRecord(step)) {
|
|
46
|
+
for (const [key, value] of Object.entries(step)) {
|
|
47
|
+
if (key in DECLARED_STEP_KEYS) continue;
|
|
48
|
+
extra[key] = value;
|
|
49
|
+
found = true;
|
|
50
|
+
}
|
|
51
|
+
} else if (step !== void 0 && step !== null) {
|
|
52
|
+
extras.push({ step });
|
|
53
|
+
found = true;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
extras.push(extra);
|
|
57
|
+
}
|
|
58
|
+
return found ? extras : void 0;
|
|
59
|
+
}
|
|
60
|
+
function declaredSteps(steps) {
|
|
61
|
+
if (!Array.isArray(steps)) return void 0;
|
|
62
|
+
const kept = [];
|
|
63
|
+
for (const step of steps) {
|
|
64
|
+
if (!isPlainRecord(step)) continue;
|
|
65
|
+
const projected = {};
|
|
66
|
+
for (const [key, value] of Object.entries(step)) {
|
|
67
|
+
if (key in DECLARED_STEP_KEYS) projected[key] = value;
|
|
68
|
+
}
|
|
69
|
+
kept.push(projected);
|
|
70
|
+
}
|
|
71
|
+
return kept;
|
|
72
|
+
}
|
|
73
|
+
var rewardRelationship = {
|
|
74
|
+
id: "reward-relationship",
|
|
75
|
+
refuses: "a positive \u2014 or unreadable \u2014 reward on a line the authenticity screen flagged as gamed",
|
|
76
|
+
tripwires: [
|
|
77
|
+
{ outcome: { reward: 0.95, realness_gated: true } },
|
|
78
|
+
// The second population, stated separately because the check held for the
|
|
79
|
+
// first while passing this one for a full round.
|
|
80
|
+
{ outcome: { reward: "0.95", realness_gated: true } }
|
|
81
|
+
],
|
|
82
|
+
errors: (subject) => {
|
|
83
|
+
if (field(subject, "realness_gated") !== true) return [];
|
|
84
|
+
const reading = readReward(field(subject, "reward"));
|
|
85
|
+
if (reading.kind === "absent" || reading.kind === "cleared") return [];
|
|
86
|
+
if (reading.kind === "unreadable") {
|
|
87
|
+
return [
|
|
88
|
+
`outcome.reward: ${reading.shown} with outcome.realness_gated: true \u2014 the wire format declares this field \`number | null\`, so the gate cannot read this value as a number and cannot establish that the reward was cleared. On a run flagged as gamed that is not evidence of a zero, it is the absence of it, and a consumer that coerces the value (\`"0.95"\`, \`"2"\`) reads a positive reward off a faked success. Emit a number or null \u2014 \`trainingReward\` / \`trainingScore\` from \`rollout/reward.ts\` produce one.`
|
|
89
|
+
];
|
|
90
|
+
}
|
|
91
|
+
return [
|
|
92
|
+
`outcome.reward: ${reading.shown} with outcome.realness_gated: true \u2014 a run flagged as gamed may not carry a positive reward. The anti-Goodhart gate forces the reward to 0 (a real verdict: the gate decided) before the line is written, so a fine-tune cannot learn from a faked success. Derive the reward with \`trainingReward\` / \`trainingScore\` from \`rollout/reward.ts\`, or drop the line.`
|
|
93
|
+
];
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
function gatedEvidenceOf(subject) {
|
|
97
|
+
const evidence = {};
|
|
98
|
+
const metrics = field(subject, "metrics");
|
|
99
|
+
if (payloadIsPopulated(metrics)) evidence.metrics = metrics;
|
|
100
|
+
const verdict = field(subject, "verdict");
|
|
101
|
+
if (verdict !== null && verdict !== void 0) evidence.verdict = verdict;
|
|
102
|
+
const steps = undeclaredStepPayload(subject.steps);
|
|
103
|
+
if (steps !== void 0) evidence.steps = steps;
|
|
104
|
+
return evidence.metrics === void 0 && evidence.verdict === void 0 && steps === void 0 ? void 0 : evidence;
|
|
105
|
+
}
|
|
106
|
+
var gatedEvidence = {
|
|
107
|
+
id: "gated-evidence",
|
|
108
|
+
refuses: "the numbers a fabricated reward was computed from, riding along at reward 0",
|
|
109
|
+
tripwires: [
|
|
110
|
+
{ outcome: { reward: 0, realness_gated: true, metrics: { "layer.tests": 1 } } },
|
|
111
|
+
// A PRIMITIVE `metrics`: `Object.keys(0.95)` is `[]`, so the emptiness test
|
|
112
|
+
// this check used to run reported it clean and the verifiers format shipped
|
|
113
|
+
// it as its per-rubric score dict.
|
|
114
|
+
{ outcome: { reward: 0, realness_gated: true, metrics: 0.95 } }
|
|
115
|
+
],
|
|
116
|
+
errors: (subject) => {
|
|
117
|
+
if (field(subject, "realness_gated") !== true) return [];
|
|
118
|
+
const evidence = gatedEvidenceOf(subject);
|
|
119
|
+
if (evidence === void 0) return [];
|
|
120
|
+
if (evidence.metrics === void 0 && evidence.verdict === void 0) return [];
|
|
121
|
+
const path = evidence.metrics !== void 0 ? "outcome.metrics" : "outcome.verdict";
|
|
122
|
+
return [
|
|
123
|
+
`${path} is populated with outcome.realness_gated: true \u2014 a run flagged as gamed may not ship the numbers its fabricated reward was computed from, even at reward 0. The per-layer scores in \`metrics\` ARE the reward signal in the verifiers format, and \`verdict\` is the record that claimed the success. Mint through \`assertMinted\`, which relocates both to \`provenance.gated_evidence\` (see \`gateGamedOutcome\`), or drop the line.`
|
|
124
|
+
];
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
var undeclaredStepPayloadCheck = {
|
|
128
|
+
id: "undeclared-step-payload",
|
|
129
|
+
refuses: "per-step reward signal hung on a gated line\u2019s steps, outside every field the gate read",
|
|
130
|
+
tripwires: [
|
|
131
|
+
{
|
|
132
|
+
outcome: { reward: 0, realness_gated: true },
|
|
133
|
+
steps: [{ kind: "tool", name: "edit", reward: 0.9 }]
|
|
134
|
+
}
|
|
135
|
+
],
|
|
136
|
+
errors: (subject) => {
|
|
137
|
+
if (field(subject, "realness_gated") !== true) return [];
|
|
138
|
+
const extra = undeclaredStepPayload(subject.steps);
|
|
139
|
+
if (extra === void 0) return [];
|
|
140
|
+
return [
|
|
141
|
+
`steps[] carries fields \`${ROLLOUT_SCHEMA_NAME}\` does not declare (${JSON.stringify(extra).slice(0, 120)}) with outcome.realness_gated: true \u2014 a per-step reward is training signal exactly like the scalar, and the exporters copy \`steps\` through verbatim, so zeroing \`outcome.reward\` while leaving it in place ships the gamed run\u2019s step-level credit assignment at full value. Mint through \`assertMinted\`, which projects the steps down to the declared keys and relocates the rest to \`provenance.gated_evidence.steps\` (see \`gateGamedOutcome\`).`
|
|
142
|
+
];
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
var ROLLOUT_SCHEMA_NAME = "tangle.rollout.v1";
|
|
146
|
+
var unscreenedReward = {
|
|
147
|
+
id: "unscreened-reward",
|
|
148
|
+
refuses: "a positive reward whose producer declares that no authenticity screen ever ran",
|
|
149
|
+
tripwires: [
|
|
150
|
+
{ outcome: { reward: 1, realness_gated: false, realness_screened: false } },
|
|
151
|
+
{
|
|
152
|
+
outcome: {
|
|
153
|
+
reward: "2",
|
|
154
|
+
realness_gated: false,
|
|
155
|
+
realness_screened: false
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
],
|
|
159
|
+
errors: (subject) => {
|
|
160
|
+
if (field(subject, "realness_screened") !== false) return [];
|
|
161
|
+
const reading = readReward(field(subject, "reward"));
|
|
162
|
+
if (reading.kind === "absent" || reading.kind === "cleared") return [];
|
|
163
|
+
return [
|
|
164
|
+
`outcome.reward: ${reading.shown} with outcome.realness_screened: false \u2014 this producer declares that NO authenticity screen ran on this reward, so nothing has established the success is real, and an unscreened positive reward is exactly the signal the anti-Goodhart gate exists to qualify. Screen the run and write the verdict (\`rolloutRewardFields\` from a \`RunRecord\` carrying \`outcome.realness\`), or emit \`reward: null\` \u2014 an unqualified verdict is a labeled gap, not a measured success.`
|
|
165
|
+
];
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
var GATE_CHECKS = {
|
|
169
|
+
"reward-relationship": rewardRelationship,
|
|
170
|
+
"gated-evidence": gatedEvidence,
|
|
171
|
+
"undeclared-step-payload": undeclaredStepPayloadCheck,
|
|
172
|
+
"unscreened-reward": unscreenedReward
|
|
173
|
+
};
|
|
174
|
+
var enforced = { kind: "enforce" };
|
|
175
|
+
var repairedBy = (by) => ({ kind: "repair", by });
|
|
176
|
+
var omittedBecause = (because) => ({
|
|
177
|
+
kind: "omit",
|
|
178
|
+
because
|
|
179
|
+
});
|
|
180
|
+
var GATE_POLICIES = {
|
|
181
|
+
/**
|
|
182
|
+
* The schema validator. Rejects the reward relationship and NOTHING ELSE, on
|
|
183
|
+
* purpose: it runs on every line read off disk, and the other two conditions
|
|
184
|
+
* describe artifacts that already exist.
|
|
185
|
+
*/
|
|
186
|
+
validateRolloutLine: {
|
|
187
|
+
// Stays a REJECTION rather than a transformation. A caller emitting
|
|
188
|
+
// `{reward: 0.95, realness_gated: true}` is a producer defect and has to
|
|
189
|
+
// fail loudly; laundering it into `reward: 0` here would hide the producer,
|
|
190
|
+
// which is the actual bug.
|
|
191
|
+
"reward-relationship": enforced,
|
|
192
|
+
"gated-evidence": omittedBecause(
|
|
193
|
+
"a ledger written before the relocation existed carries `metrics` on its gated lines; rejecting those would make every already-published artifact unreadable. The condition is REPAIRED at `assertMinted` (`gateGamedOutcome`) instead, which is the only door into the training path, so the artifact stays readable and the leak still closes."
|
|
194
|
+
),
|
|
195
|
+
"undeclared-step-payload": omittedBecause(
|
|
196
|
+
"identical reasoning to `gated-evidence`, one field over: a foreign or pre-unification ledger may carry producer-invented keys on its steps, and refusing to READ those files buys nothing the relocation at `assertMinted` does not already buy. Promotion into training is where it closes."
|
|
197
|
+
),
|
|
198
|
+
"unscreened-reward": omittedBecause(
|
|
199
|
+
"supervision-journal rows legitimately carry an unscreened positive reward (there is no `RunRecord.outcome.realness` behind them) and must stay writable, readable and reportable. Only PROMOTION into training is closed, at `assertMinted`."
|
|
200
|
+
)
|
|
201
|
+
},
|
|
202
|
+
/**
|
|
203
|
+
* The mint funnel — the single door every `MintedRolloutLine` passes. Validates
|
|
204
|
+
* first (so the reward relationship has already been rejected), then refuses
|
|
205
|
+
* what cannot be repaired, then repairs what can.
|
|
206
|
+
*/
|
|
207
|
+
assertMinted: {
|
|
208
|
+
// Re-run after `assertRolloutLine`, which already rejected it. Cheap, and it
|
|
209
|
+
// makes this entry point independently complete rather than complete only
|
|
210
|
+
// because of what it happens to call first.
|
|
211
|
+
"reward-relationship": enforced,
|
|
212
|
+
"gated-evidence": repairedBy("gateGamedOutcome"),
|
|
213
|
+
"undeclared-step-payload": repairedBy("gateGamedOutcome"),
|
|
214
|
+
"unscreened-reward": enforced
|
|
215
|
+
},
|
|
216
|
+
/**
|
|
217
|
+
* The runtime backstop, and the one entry point with no license to omit
|
|
218
|
+
* anything: it exists for callers the type system never saw (plain JavaScript
|
|
219
|
+
* handing an object literal to a published exporter), so a check it skips is a
|
|
220
|
+
* check that does not run at all for them. This is where the fourth leak was.
|
|
221
|
+
*/
|
|
222
|
+
assertRewardGate: {
|
|
223
|
+
"reward-relationship": enforced,
|
|
224
|
+
"gated-evidence": enforced,
|
|
225
|
+
"undeclared-step-payload": enforced,
|
|
226
|
+
"unscreened-reward": enforced
|
|
227
|
+
},
|
|
228
|
+
/**
|
|
229
|
+
* The release certifier. Same checks, measured over the rows a release is
|
|
230
|
+
* ABOUT TO WRITE rather than over one line's outcome — see `REPORT_MEASURES`
|
|
231
|
+
* in `release/gate-report.ts`, which is the second total map this policy
|
|
232
|
+
* drives.
|
|
233
|
+
*/
|
|
234
|
+
assertGateReport: {
|
|
235
|
+
"reward-relationship": enforced,
|
|
236
|
+
"gated-evidence": enforced,
|
|
237
|
+
"undeclared-step-payload": enforced,
|
|
238
|
+
"unscreened-reward": enforced
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
function gateErrors(subject, policy) {
|
|
242
|
+
const errors = [];
|
|
243
|
+
for (const id of GATE_CHECK_IDS) {
|
|
244
|
+
if (policy[id].kind !== "enforce") continue;
|
|
245
|
+
errors.push(...GATE_CHECKS[id].errors(subject));
|
|
246
|
+
}
|
|
247
|
+
return errors;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// src/rollout/schema.ts
|
|
251
|
+
var ROLLOUT_SCHEMA = "tangle.rollout.v1";
|
|
252
|
+
var ROLLOUT_ROLES = [
|
|
253
|
+
"agent",
|
|
254
|
+
"supervisor",
|
|
255
|
+
"worker",
|
|
256
|
+
"proposer",
|
|
257
|
+
"judge",
|
|
258
|
+
"analyst"
|
|
259
|
+
];
|
|
260
|
+
var ROLLOUT_SPLITS = ["search", "dev", "holdout", "canary"];
|
|
261
|
+
var TRAINABLE_SPLITS = ["search"];
|
|
262
|
+
function isTrainableSplit(split) {
|
|
263
|
+
return TRAINABLE_SPLITS.includes(split);
|
|
264
|
+
}
|
|
265
|
+
var ROLLOUT_CAPTURES = ["mint", "settle-time", "backfill"];
|
|
266
|
+
var CHAT_ROLES = ["system", "user", "assistant", "tool"];
|
|
267
|
+
var isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
268
|
+
var isNumberOrNull = (v) => v === null || typeof v === "number";
|
|
269
|
+
var isNumberArray = (v) => Array.isArray(v) && v.every((n) => typeof n === "number" && Number.isFinite(n));
|
|
270
|
+
var isIntegerArray = (v) => Array.isArray(v) && v.every((n) => Number.isInteger(n));
|
|
271
|
+
var isStringOrNull = (v) => v === null || typeof v === "string";
|
|
272
|
+
var isIntegerOrNull = (v) => v === null || Number.isInteger(v);
|
|
273
|
+
function validateChatMessage(value, path, errors) {
|
|
274
|
+
if (!isRecord(value)) {
|
|
275
|
+
errors.push(`${path}: not an object`);
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
if (!CHAT_ROLES.includes(value.role))
|
|
279
|
+
errors.push(`${path}.role: invalid role ${String(value.role)}`);
|
|
280
|
+
if (!isStringOrNull(value.content)) errors.push(`${path}.content: must be string|null`);
|
|
281
|
+
if (value.reasoning_content !== void 0 && typeof value.reasoning_content !== "string") {
|
|
282
|
+
errors.push(`${path}.reasoning_content: must be string when present`);
|
|
283
|
+
}
|
|
284
|
+
if (value.tool_call_id !== void 0 && typeof value.tool_call_id !== "string") {
|
|
285
|
+
errors.push(`${path}.tool_call_id: must be string when present`);
|
|
286
|
+
}
|
|
287
|
+
if (value.role === "tool" && typeof value.tool_call_id !== "string") {
|
|
288
|
+
errors.push(`${path}.tool_call_id: required on role:"tool"`);
|
|
289
|
+
}
|
|
290
|
+
if (value.is_copied_context !== void 0 && typeof value.is_copied_context !== "boolean") {
|
|
291
|
+
errors.push(`${path}.is_copied_context: must be boolean when present`);
|
|
292
|
+
}
|
|
293
|
+
if (value.tool_calls !== void 0) {
|
|
294
|
+
if (!Array.isArray(value.tool_calls)) {
|
|
295
|
+
errors.push(`${path}.tool_calls: must be an array when present`);
|
|
296
|
+
} else {
|
|
297
|
+
value.tool_calls.forEach((call, i) => {
|
|
298
|
+
if (!isRecord(call) || typeof call.id !== "string" || call.type !== "function") {
|
|
299
|
+
errors.push(`${path}.tool_calls[${i}]: must be {id, type:"function", function}`);
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
const fn = call.function;
|
|
303
|
+
if (!isRecord(fn) || typeof fn.name !== "string" || typeof fn.arguments !== "string") {
|
|
304
|
+
errors.push(
|
|
305
|
+
`${path}.tool_calls[${i}].function: must be {name: string, arguments: string}`
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
function validateSection(value, path, fields, errors) {
|
|
313
|
+
if (!isRecord(value)) {
|
|
314
|
+
errors.push(`${path}: not an object`);
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
for (const [name, check, expect] of fields) {
|
|
318
|
+
if (!check(value[name])) errors.push(`${path}.${name}: expected ${expect}`);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
function gateGamedOutcome(line) {
|
|
322
|
+
if (line.outcome.realness_gated !== true) return line;
|
|
323
|
+
const moved = gatedEvidenceOf(line);
|
|
324
|
+
if (moved === void 0) return line;
|
|
325
|
+
const kept = line.provenance.gated_evidence;
|
|
326
|
+
const evidence = {};
|
|
327
|
+
const metrics = { ...kept?.metrics, ...moved.metrics };
|
|
328
|
+
if (Object.keys(metrics).length > 0) evidence.metrics = metrics;
|
|
329
|
+
if (moved.verdict !== void 0) evidence.verdict = moved.verdict;
|
|
330
|
+
else if (kept?.verdict !== void 0) evidence.verdict = kept.verdict;
|
|
331
|
+
if (moved.steps !== void 0) evidence.steps = moved.steps;
|
|
332
|
+
else if (kept?.steps !== void 0) evidence.steps = kept.steps;
|
|
333
|
+
const steps = moved.steps === void 0 ? line.steps : declaredSteps(line.steps);
|
|
334
|
+
return {
|
|
335
|
+
...line,
|
|
336
|
+
...steps === void 0 ? {} : { steps },
|
|
337
|
+
outcome: { ...line.outcome, metrics: {}, verdict: null },
|
|
338
|
+
provenance: { ...line.provenance, gated_evidence: evidence }
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
function assertRewardGate(line, context) {
|
|
342
|
+
const errors = gateErrors(line, GATE_POLICIES.assertRewardGate);
|
|
343
|
+
if (errors.length > 0) {
|
|
344
|
+
throw new Error(`${context}: rollout ${line.rollout_id} \u2014 ${errors[0]}`);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
function validateRolloutLine(value) {
|
|
348
|
+
const errors = [];
|
|
349
|
+
if (!isRecord(value)) return ["line: not an object"];
|
|
350
|
+
if (value.schema !== ROLLOUT_SCHEMA) errors.push(`schema: expected "${ROLLOUT_SCHEMA}"`);
|
|
351
|
+
if (typeof value.rollout_id !== "string" || value.rollout_id.length === 0)
|
|
352
|
+
errors.push("rollout_id: expected non-empty string");
|
|
353
|
+
if (!isStringOrNull(value.parent_rollout_id))
|
|
354
|
+
errors.push("parent_rollout_id: expected string|null");
|
|
355
|
+
if (typeof value.run_id !== "string" || value.run_id.length === 0)
|
|
356
|
+
errors.push("run_id: expected non-empty string");
|
|
357
|
+
if (!isStringOrNull(value.experiment_id)) errors.push("experiment_id: expected string|null");
|
|
358
|
+
if (!isStringOrNull(value.candidate_id)) errors.push("candidate_id: expected string|null");
|
|
359
|
+
if (!isIntegerOrNull(value.generation)) errors.push("generation: expected integer|null");
|
|
360
|
+
if (!isIntegerOrNull(value.candidate_index)) errors.push("candidate_index: expected integer|null");
|
|
361
|
+
if (!ROLLOUT_ROLES.includes(value.role))
|
|
362
|
+
errors.push(`role: invalid role ${String(value.role)}`);
|
|
363
|
+
validateSection(
|
|
364
|
+
value.task,
|
|
365
|
+
"task",
|
|
366
|
+
[
|
|
367
|
+
["suite", (v) => typeof v === "string" && v.length > 0, "non-empty string"],
|
|
368
|
+
["instance_id", (v) => typeof v === "string" && v.length > 0, "non-empty string"],
|
|
369
|
+
[
|
|
370
|
+
"split",
|
|
371
|
+
(v) => ROLLOUT_SPLITS.includes(v),
|
|
372
|
+
`one of ${ROLLOUT_SPLITS.join("|")}`
|
|
373
|
+
],
|
|
374
|
+
["seed", isNumberOrNull, "number|null"],
|
|
375
|
+
["rep", (v) => Number.isInteger(v), "integer"]
|
|
376
|
+
],
|
|
377
|
+
errors
|
|
378
|
+
);
|
|
379
|
+
validateSection(
|
|
380
|
+
value.policy,
|
|
381
|
+
"policy",
|
|
382
|
+
[
|
|
383
|
+
["harness", isStringOrNull, "string|null"],
|
|
384
|
+
["harness_version", isStringOrNull, "string|null"],
|
|
385
|
+
["model", isStringOrNull, "string|null"],
|
|
386
|
+
["provider", isStringOrNull, "string|null"],
|
|
387
|
+
["profile_commit", isStringOrNull, "string|null"],
|
|
388
|
+
["sampling", (v) => v === null || isRecord(v), "object|null"]
|
|
389
|
+
],
|
|
390
|
+
errors
|
|
391
|
+
);
|
|
392
|
+
if (isRecord(value.policy)) {
|
|
393
|
+
for (const key of ["prompt_hash", "config_hash", "agent_profile_cell_id"]) {
|
|
394
|
+
if (value.policy[key] !== void 0 && !isStringOrNull(value.policy[key])) {
|
|
395
|
+
errors.push(`policy.${key}: expected string|null when present`);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
if (!Array.isArray(value.messages)) {
|
|
400
|
+
errors.push("messages: expected array");
|
|
401
|
+
} else {
|
|
402
|
+
for (const [i, m] of value.messages.entries()) validateChatMessage(m, `messages[${i}]`, errors);
|
|
403
|
+
}
|
|
404
|
+
if (!Array.isArray(value.tool_defs)) {
|
|
405
|
+
errors.push("tool_defs: expected array");
|
|
406
|
+
} else {
|
|
407
|
+
value.tool_defs.forEach((d, i) => {
|
|
408
|
+
if (!isRecord(d) || d.type !== "function" || !isRecord(d.function) || typeof d.function.name !== "string") {
|
|
409
|
+
errors.push(`tool_defs[${i}]: must be {type:"function", function:{name}}`);
|
|
410
|
+
}
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
if (value.steps !== void 0) {
|
|
414
|
+
if (!Array.isArray(value.steps)) {
|
|
415
|
+
errors.push("steps: expected array when present");
|
|
416
|
+
} else {
|
|
417
|
+
value.steps.forEach((s, i) => {
|
|
418
|
+
if (!isRecord(s) || typeof s.kind !== "string" || typeof s.name !== "string") {
|
|
419
|
+
errors.push(`steps[${i}]: must be {kind: string, name: string, \u2026}`);
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
if (s.llm_call_count !== void 0 && !Number.isInteger(s.llm_call_count)) {
|
|
423
|
+
errors.push(`steps[${i}].llm_call_count: expected integer when present`);
|
|
424
|
+
}
|
|
425
|
+
for (const key of ["prompt_token_ids", "completion_token_ids"]) {
|
|
426
|
+
if (s[key] !== void 0 && !isIntegerArray(s[key])) {
|
|
427
|
+
errors.push(`steps[${i}].${key}: expected integer[] when present`);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
if (s.logprobs !== void 0 && !isNumberArray(s.logprobs)) {
|
|
431
|
+
errors.push(`steps[${i}].logprobs: expected number[] when present`);
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
validateSection(
|
|
437
|
+
value.outcome,
|
|
438
|
+
"outcome",
|
|
439
|
+
[
|
|
440
|
+
["reward", isNumberOrNull, "number|null"],
|
|
441
|
+
["reward_source", isStringOrNull, "string|null"],
|
|
442
|
+
["metrics", isRecord, "object"],
|
|
443
|
+
["is_completed", (v) => typeof v === "boolean", "boolean"],
|
|
444
|
+
["is_truncated", (v) => typeof v === "boolean", "boolean"],
|
|
445
|
+
["error", isStringOrNull, "string|null"]
|
|
446
|
+
],
|
|
447
|
+
errors
|
|
448
|
+
);
|
|
449
|
+
if (isRecord(value.outcome)) {
|
|
450
|
+
if (!("verdict" in value.outcome)) errors.push("outcome.verdict: field required (may be null)");
|
|
451
|
+
if (typeof value.outcome.realness_gated !== "boolean") {
|
|
452
|
+
errors.push("outcome.realness_gated: expected boolean");
|
|
453
|
+
}
|
|
454
|
+
if (value.outcome.realness_screened !== void 0 && typeof value.outcome.realness_screened !== "boolean") {
|
|
455
|
+
errors.push("outcome.realness_screened: expected boolean when present");
|
|
456
|
+
}
|
|
457
|
+
errors.push(
|
|
458
|
+
...gateErrors(
|
|
459
|
+
{ outcome: value.outcome, steps: value.steps },
|
|
460
|
+
GATE_POLICIES.validateRolloutLine
|
|
461
|
+
)
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
validateSection(
|
|
465
|
+
value.cost,
|
|
466
|
+
"cost",
|
|
467
|
+
[
|
|
468
|
+
["usd", isNumberOrNull, "number|null"],
|
|
469
|
+
["tokens_in", isNumberOrNull, "number|null"],
|
|
470
|
+
["tokens_out", isNumberOrNull, "number|null"],
|
|
471
|
+
["tokens_reasoning", isNumberOrNull, "number|null"],
|
|
472
|
+
["cache_read", isNumberOrNull, "number|null"],
|
|
473
|
+
["cache_write", isNumberOrNull, "number|null"],
|
|
474
|
+
["wall_s", isNumberOrNull, "number|null"]
|
|
475
|
+
],
|
|
476
|
+
errors
|
|
477
|
+
);
|
|
478
|
+
if (isRecord(value.cost) && value.cost.llm_call_count !== void 0 && !(value.cost.llm_call_count === null || Number.isInteger(value.cost.llm_call_count))) {
|
|
479
|
+
errors.push("cost.llm_call_count: expected integer|null when present");
|
|
480
|
+
}
|
|
481
|
+
validateSection(
|
|
482
|
+
value.artifacts,
|
|
483
|
+
"artifacts",
|
|
484
|
+
[
|
|
485
|
+
["patch_path", isStringOrNull, "string|null"],
|
|
486
|
+
["run_dir", isStringOrNull, "string|null"],
|
|
487
|
+
["transcript_ref", isStringOrNull, "string|null"]
|
|
488
|
+
],
|
|
489
|
+
errors
|
|
490
|
+
);
|
|
491
|
+
validateSection(
|
|
492
|
+
value.provenance,
|
|
493
|
+
"provenance",
|
|
494
|
+
[
|
|
495
|
+
[
|
|
496
|
+
"captured_at",
|
|
497
|
+
(v) => typeof v === "string" && !Number.isNaN(Date.parse(v)),
|
|
498
|
+
"ISO-8601 timestamp"
|
|
499
|
+
],
|
|
500
|
+
[
|
|
501
|
+
"capture",
|
|
502
|
+
(v) => ROLLOUT_CAPTURES.includes(v),
|
|
503
|
+
`one of ${ROLLOUT_CAPTURES.join("|")}`
|
|
504
|
+
]
|
|
505
|
+
],
|
|
506
|
+
errors
|
|
507
|
+
);
|
|
508
|
+
if (isRecord(value.provenance)) {
|
|
509
|
+
if (value.provenance.gap !== void 0 && typeof value.provenance.gap !== "string") {
|
|
510
|
+
errors.push("provenance.gap: must be string when present");
|
|
511
|
+
}
|
|
512
|
+
const evidence = value.provenance.gated_evidence;
|
|
513
|
+
if (evidence !== void 0) {
|
|
514
|
+
if (!isRecord(evidence)) {
|
|
515
|
+
errors.push("provenance.gated_evidence: must be an object when present");
|
|
516
|
+
} else if (evidence.metrics !== void 0 && !isRecord(evidence.metrics)) {
|
|
517
|
+
errors.push("provenance.gated_evidence.metrics: must be an object when present");
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
if (Array.isArray(value.messages) && isRecord(value.provenance)) {
|
|
522
|
+
if (value.messages.length === 0 && typeof value.provenance.gap !== "string") {
|
|
523
|
+
errors.push("provenance.gap: required when messages is empty");
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
return errors;
|
|
527
|
+
}
|
|
528
|
+
function assertRolloutLine(value, context = "rollout line") {
|
|
529
|
+
const errors = validateRolloutLine(value);
|
|
530
|
+
if (errors.length > 0) {
|
|
531
|
+
throw new Error(`invalid ${context}:
|
|
532
|
+
${errors.join("\n ")}`);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
function isRolloutLine(value) {
|
|
536
|
+
return validateRolloutLine(value).length === 0;
|
|
537
|
+
}
|
|
538
|
+
function assertMinted(value, context = "rollout line") {
|
|
539
|
+
assertRolloutLine(value, context);
|
|
540
|
+
const refused = gateErrors(value, GATE_POLICIES.assertMinted);
|
|
541
|
+
if (refused.length > 0) {
|
|
542
|
+
throw new Error(`invalid ${context}: rollout ${value.rollout_id} \u2014 ${refused[0]}`);
|
|
543
|
+
}
|
|
544
|
+
const gated = gateGamedOutcome(value);
|
|
545
|
+
if (gated.outcome.realness_gated === void 0) {
|
|
546
|
+
return {
|
|
547
|
+
...gated,
|
|
548
|
+
outcome: { ...gated.outcome, realness_gated: false }
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
return gated;
|
|
552
|
+
}
|
|
553
|
+
function assertMintedLines(values, context = "rollout line") {
|
|
554
|
+
return values.map((value, i) => assertMinted(value, `${context} [${i}]`));
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
export {
|
|
558
|
+
GATE_CHECK_IDS,
|
|
559
|
+
undeclaredStepPayload,
|
|
560
|
+
gatedEvidenceOf,
|
|
561
|
+
GATE_CHECKS,
|
|
562
|
+
GATE_POLICIES,
|
|
563
|
+
gateErrors,
|
|
564
|
+
ROLLOUT_SCHEMA,
|
|
565
|
+
ROLLOUT_ROLES,
|
|
566
|
+
ROLLOUT_SPLITS,
|
|
567
|
+
TRAINABLE_SPLITS,
|
|
568
|
+
isTrainableSplit,
|
|
569
|
+
ROLLOUT_CAPTURES,
|
|
570
|
+
CHAT_ROLES,
|
|
571
|
+
gateGamedOutcome,
|
|
572
|
+
assertRewardGate,
|
|
573
|
+
validateRolloutLine,
|
|
574
|
+
assertRolloutLine,
|
|
575
|
+
isRolloutLine,
|
|
576
|
+
assertMinted,
|
|
577
|
+
assertMintedLines
|
|
578
|
+
};
|
|
579
|
+
//# sourceMappingURL=chunk-PC5DOSM7.js.map
|