@dzhechkov/harness-cli 0.3.253 → 0.3.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +11 -7
- package/README.md +103 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +351 -26
- package/dist/cli.js.map +1 -1
- package/package.json +2 -2
- package/sbom.json +16 -6
- package/src/cli.ts +354 -22
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dzhechkov/harness-cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.254",
|
|
4
4
|
"description": "The dz CLI — install AI skills for Claude Code, Codex, OpenCode, Hermes, OpenClaude, GitHub Copilot. 35 commands, 13 presets, 6 platform targets.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"sbom.json"
|
|
42
42
|
],
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@dzhechkov/harness-core": "^0.3.
|
|
44
|
+
"@dzhechkov/harness-core": "^0.3.144",
|
|
45
45
|
"@dzhechkov/harness-presets": "^0.5.0",
|
|
46
46
|
"@dzhechkov/scout": "^0.8.0",
|
|
47
47
|
"@dzhechkov/skills-devops": "^0.3.0",
|
package/sbom.json
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"hashes": [
|
|
26
26
|
{
|
|
27
27
|
"alg": "SHA-256",
|
|
28
|
-
"content": "
|
|
28
|
+
"content": "fc57c105e5031d2d6751ef08b64769dc463cd79c5cf7637b48bc8cb7190e5b3a"
|
|
29
29
|
}
|
|
30
30
|
]
|
|
31
31
|
},
|
|
@@ -105,7 +105,7 @@
|
|
|
105
105
|
"hashes": [
|
|
106
106
|
{
|
|
107
107
|
"alg": "SHA-256",
|
|
108
|
-
"content": "
|
|
108
|
+
"content": "4268f56e38f72625c2c490add4b17ee12d7ff7db991a2090f426afbb026b98a9"
|
|
109
109
|
}
|
|
110
110
|
]
|
|
111
111
|
},
|
|
@@ -115,7 +115,7 @@
|
|
|
115
115
|
"hashes": [
|
|
116
116
|
{
|
|
117
117
|
"alg": "SHA-256",
|
|
118
|
-
"content": "
|
|
118
|
+
"content": "fe24568930cc226e0bad8b4ccba559e2f837bb4cb068c2cc37750b5a10c27a70"
|
|
119
119
|
}
|
|
120
120
|
]
|
|
121
121
|
},
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
"hashes": [
|
|
126
126
|
{
|
|
127
127
|
"alg": "SHA-256",
|
|
128
|
-
"content": "
|
|
128
|
+
"content": "4a0b554fab78cd50fc46cff653411b2003b7dd9a4df1b16af6e9f5a0e24379ff"
|
|
129
129
|
}
|
|
130
130
|
]
|
|
131
131
|
},
|
|
@@ -185,7 +185,7 @@
|
|
|
185
185
|
"hashes": [
|
|
186
186
|
{
|
|
187
187
|
"alg": "SHA-256",
|
|
188
|
-
"content": "
|
|
188
|
+
"content": "c6469d0f7539ce6c0a8176d0b795952a667df2b975c9f93c66e7cd6eba843383"
|
|
189
189
|
}
|
|
190
190
|
]
|
|
191
191
|
},
|
|
@@ -205,7 +205,7 @@
|
|
|
205
205
|
"hashes": [
|
|
206
206
|
{
|
|
207
207
|
"alg": "SHA-256",
|
|
208
|
-
"content": "
|
|
208
|
+
"content": "13d3946992186e019310c1c8666fd724ce74880b8fda19850eeb8b6e0399e3d7"
|
|
209
209
|
}
|
|
210
210
|
]
|
|
211
211
|
},
|
|
@@ -229,6 +229,16 @@
|
|
|
229
229
|
}
|
|
230
230
|
]
|
|
231
231
|
},
|
|
232
|
+
{
|
|
233
|
+
"type": "file",
|
|
234
|
+
"name": "test/epoch-replay-cli.test.ts",
|
|
235
|
+
"hashes": [
|
|
236
|
+
{
|
|
237
|
+
"alg": "SHA-256",
|
|
238
|
+
"content": "a5a48585b565288ca1ea96030b02b2ccb37a621ef7b3cfe54cd7aaa7164d6904"
|
|
239
|
+
}
|
|
240
|
+
]
|
|
241
|
+
},
|
|
232
242
|
{
|
|
233
243
|
"type": "file",
|
|
234
244
|
"name": "tsconfig.json",
|
package/src/cli.ts
CHANGED
|
@@ -201,6 +201,24 @@ import {
|
|
|
201
201
|
renderContentProbe,
|
|
202
202
|
findNonRegistrableSkillDirs,
|
|
203
203
|
assembleCompoundingReport,
|
|
204
|
+
// Cold-vs-warm EPOCH RUNNER (feature epoch-replay) — orchestrates + scores, never calls a model.
|
|
205
|
+
replayableInstances,
|
|
206
|
+
buildWorkOrder,
|
|
207
|
+
buildJudgePrompts,
|
|
208
|
+
unblindJudgments,
|
|
209
|
+
verifyWorkOrder,
|
|
210
|
+
isValidMargin,
|
|
211
|
+
DIGEST_HONEST_SCOPE,
|
|
212
|
+
scoreEpochReplay,
|
|
213
|
+
generateMockOutcomes,
|
|
214
|
+
renderEpochReplayResult,
|
|
215
|
+
renderWorkOrderSummary,
|
|
216
|
+
renderJudgePromptsSummary,
|
|
217
|
+
WORK_ORDER_KIND,
|
|
218
|
+
DEFAULT_MOCK_N,
|
|
219
|
+
DEFAULT_MOCK_SEED,
|
|
220
|
+
type WorkOrder,
|
|
221
|
+
type EpochOutcome,
|
|
204
222
|
scoreRun,
|
|
205
223
|
renderScorecard,
|
|
206
224
|
renderCompoundingReport,
|
|
@@ -265,6 +283,10 @@ Usage:
|
|
|
265
283
|
dz delivery-check --slug <slug> [--context-only] [--findings <f.json>] [--strict] [--author <model>] [--json] (portable Step-10 Delivery Gate: prints the 4-plane review brief + artifact probes; --findings classifies a fed-back review into a fail-closed ready|blocked hand-off and writes features/<slug>/10_delivery_review.md; --strict exits 1 on blocked)
|
|
266
284
|
dz skills-verify [--dir <project>] [--expect a,b] [--static] [--strict] [--json] (does .claude/skills/ actually REGISTER? --static = instant layout scan for CI; default reads the authoritative system/init listing from a real session. exit 0 pass / 1 fail / 2 inconclusive — never a false pass)
|
|
267
285
|
dz compounding [--project <dir>] [--json] (honest learning-loop payoff report: pool write-only ratio, guard repeat-violation trajectory, cold-vs-warm replay readiness, instrumentation health — a gate without enough data says INSUFFICIENT_DATA, never a fake verdict)
|
|
286
|
+
dz epoch-replay --mock [--n <N>] [--effect <-1..1>] [--tie-rate <0..1>] [--seed <N>] [--slice <name>] [--json] ($0 synthetic run — exercises the verdict math, NOT evidence)
|
|
287
|
+
dz epoch-replay --emit [--project <dir>] [--limit <N>] [--seed <N>] [--out <file>] (cold-vs-warm work order: instances + PRE-REGISTERED blind A/B assignment; the runner never calls a model)
|
|
288
|
+
dz epoch-replay --judge <filled-work-order.json> [--out <file>] (blind judge prompts from the filled plans)
|
|
289
|
+
dz epoch-replay --score <judgments.json> --work-order <file> [--slice <name>] [--json] (un-blind against the pre-registered assignment → SUPPORTED only when the two 95% Wilson CIs are DISJOINT, else FALSIFIED / INCONCLUSIVE)
|
|
268
290
|
dz score --slug <feature> [--project <dir>] [--json] (process scorecard for ONE feature-adr run, from its artifacts: ADR confirmation, discrimination, cross-model QE grade, live verification, README-first, learning loop, amendments — descriptive-only, a low score exits 0)
|
|
269
291
|
dz backlog add "<idea>" [--effort 1-5] [--proposal <text>] [--dry-run] [--project <dir>] [--json] (capture an idea: semantic dedup against existing ideas via the Brain vector engine (DUPLICATE>=0.92 merges, RELATED links, NEW creates) + GoalMap alignment; --dry-run classifies without writing)
|
|
270
292
|
dz backlog list [--status <s>] [--goal <id>] [--project <dir>] [--json] (list captured ideas, filterable by status/goal)
|
|
@@ -5773,6 +5795,46 @@ function cmdScore(options: Map<string, string>, flags: Set<string>, cwd: string,
|
|
|
5773
5795
|
return 0;
|
|
5774
5796
|
}
|
|
5775
5797
|
|
|
5798
|
+
/**
|
|
5799
|
+
* Read the apply-leg usage log into the shape `compounding.ts` / `epoch-replay.ts` expect.
|
|
5800
|
+
*
|
|
5801
|
+
* ONE reader, because there were two and they disagreed: the `dz compounding` fact-gatherer used
|
|
5802
|
+
* to drop `eventId` and `queryTruncated`, so the readiness gate counted 48 "replayable pairs" over
|
|
5803
|
+
* a log whose honest count is 25 (MEASURED on this repo, 2026-07-29). Both defences documented in
|
|
5804
|
+
* `assembleCompoundingReport` — "one prompt = one pair" (Codex #1) and "a truncated query is a
|
|
5805
|
+
* prefix, not the prompt" (Codex #3) — were correct in the pure function and DEAD at its only
|
|
5806
|
+
* caller, because the caller never passed the fields they read.
|
|
5807
|
+
*/
|
|
5808
|
+
function readRecallUsageEvents(
|
|
5809
|
+
root: string,
|
|
5810
|
+
): { dzId: string; ts: string; query?: string; runId?: string; eventId?: string; queryTruncated?: boolean }[] {
|
|
5811
|
+
const usage: { dzId: string; ts: string; query?: string; runId?: string; eventId?: string; queryTruncated?: boolean }[] = [];
|
|
5812
|
+
try {
|
|
5813
|
+
const text = readFileSync(join(root, '.dz', 'recall-usage.jsonl'), 'utf-8');
|
|
5814
|
+
for (const line of text.split('\n')) {
|
|
5815
|
+
if (line.trim() === '') continue;
|
|
5816
|
+
try {
|
|
5817
|
+
const o = JSON.parse(line) as Record<string, unknown>;
|
|
5818
|
+
if (typeof o.dzId === 'string' && typeof o.ts === 'string' && o.kind !== 'aggregate') {
|
|
5819
|
+
usage.push({
|
|
5820
|
+
dzId: o.dzId,
|
|
5821
|
+
ts: o.ts,
|
|
5822
|
+
...(typeof o.query === 'string' ? { query: o.query } : {}),
|
|
5823
|
+
...(typeof o.runId === 'string' ? { runId: o.runId } : {}),
|
|
5824
|
+
...(typeof o.eventId === 'string' ? { eventId: o.eventId } : {}),
|
|
5825
|
+
...(o.queryTruncated === true ? { queryTruncated: true } : {}),
|
|
5826
|
+
});
|
|
5827
|
+
}
|
|
5828
|
+
} catch {
|
|
5829
|
+
/* one bad line must not kill the read */
|
|
5830
|
+
}
|
|
5831
|
+
}
|
|
5832
|
+
} catch {
|
|
5833
|
+
/* no log yet — callers report the absence */
|
|
5834
|
+
}
|
|
5835
|
+
return usage;
|
|
5836
|
+
}
|
|
5837
|
+
|
|
5776
5838
|
/**
|
|
5777
5839
|
* `dz compounding` — does the learning loop actually PAY? (feature compounding, scout C2.)
|
|
5778
5840
|
* Gathers the facts (store rows, apply-leg usage log, guard audit) and hands them to the PURE
|
|
@@ -5812,28 +5874,7 @@ function cmdCompounding(options: Map<string, string>, flags: Set<string>, cwd: s
|
|
|
5812
5874
|
}));
|
|
5813
5875
|
|
|
5814
5876
|
// apply-leg usage events (read records only; aggregate rows carry no query by construction)
|
|
5815
|
-
const usage
|
|
5816
|
-
try {
|
|
5817
|
-
const text = readFileSync(join(root, '.dz', 'recall-usage.jsonl'), 'utf-8');
|
|
5818
|
-
for (const line of text.split('\n')) {
|
|
5819
|
-
if (line.trim() === '') continue;
|
|
5820
|
-
try {
|
|
5821
|
-
const o = JSON.parse(line) as Record<string, unknown>;
|
|
5822
|
-
if (typeof o.dzId === 'string' && typeof o.ts === 'string' && o.kind !== 'aggregate') {
|
|
5823
|
-
usage.push({
|
|
5824
|
-
dzId: o.dzId,
|
|
5825
|
-
ts: o.ts,
|
|
5826
|
-
...(typeof o.query === 'string' ? { query: o.query } : {}),
|
|
5827
|
-
...(typeof o.runId === 'string' ? { runId: o.runId } : {}),
|
|
5828
|
-
});
|
|
5829
|
-
}
|
|
5830
|
-
} catch {
|
|
5831
|
-
/* one bad line must not kill the report */
|
|
5832
|
-
}
|
|
5833
|
-
}
|
|
5834
|
-
} catch {
|
|
5835
|
-
/* no log yet — the report says so */
|
|
5836
|
-
}
|
|
5877
|
+
const usage = readRecallUsageEvents(root);
|
|
5837
5878
|
|
|
5838
5879
|
// guard audit events
|
|
5839
5880
|
const guard: { ts: string; verdict: string; rules: string[] }[] = [];
|
|
@@ -5864,6 +5905,295 @@ function cmdCompounding(options: Map<string, string>, flags: Set<string>, cwd: s
|
|
|
5864
5905
|
return 0;
|
|
5865
5906
|
}
|
|
5866
5907
|
|
|
5908
|
+
// ── `dz epoch-replay` (feature epoch-replay) ────────────────────────────────────────────────────
|
|
5909
|
+
|
|
5910
|
+
const EPOCH_REPLAY_DIR = join('.dz', 'epoch-replay');
|
|
5911
|
+
|
|
5912
|
+
const EPOCH_REPLAY_USAGE = [
|
|
5913
|
+
'dz epoch-replay --mock [--n <N>] [--effect <-1..1>] [--tie-rate <0..1>] [--seed <N>] [--slice <name>] [--margin <0..1>] [--json]',
|
|
5914
|
+
'dz epoch-replay --emit [--project <dir>] [--limit <N>] [--seed <N>] [--margin <0..0.5>] [--out <file>] [--json]',
|
|
5915
|
+
'dz epoch-replay --judge <filled-work-order.json> [--out <file>] [--json]',
|
|
5916
|
+
'dz epoch-replay --score <judgments.json> --work-order <file> [--slice <name>] [--json] (margin comes from the work order)',
|
|
5917
|
+
].join('\n ');
|
|
5918
|
+
|
|
5919
|
+
/** Read a JSON file into an object, or return a parse/IO error string. */
|
|
5920
|
+
function readJsonFile(path: string): { value: unknown } | { error: string } {
|
|
5921
|
+
try {
|
|
5922
|
+
return { value: JSON.parse(readFileSync(path, 'utf-8')) as unknown };
|
|
5923
|
+
} catch (e) {
|
|
5924
|
+
return { error: `cannot read ${path}: ${e instanceof Error ? e.message : String(e)}` };
|
|
5925
|
+
}
|
|
5926
|
+
}
|
|
5927
|
+
|
|
5928
|
+
/**
|
|
5929
|
+
* Integrity-check a parsed work order. Checking `kind` + `Array.isArray(items)` was VACUOUS: a
|
|
5930
|
+
* hand-written file with those two fields and a fabricated `warmIsA` bought whatever verdict its
|
|
5931
|
+
* author wanted (Codex QE HIGH-2). `verifyWorkOrder` recomputes the digest AND re-derives every
|
|
5932
|
+
* assignment from the stated seed.
|
|
5933
|
+
*/
|
|
5934
|
+
function asVerifiedWorkOrder(value: unknown): { order: WorkOrder } | { problems: readonly string[] } {
|
|
5935
|
+
const v = verifyWorkOrder(value);
|
|
5936
|
+
if (!v.ok) return { problems: v.problems };
|
|
5937
|
+
return { order: value as WorkOrder };
|
|
5938
|
+
}
|
|
5939
|
+
|
|
5940
|
+
function writeJsonOut(path: string, value: unknown): void {
|
|
5941
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
5942
|
+
writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`, 'utf-8');
|
|
5943
|
+
}
|
|
5944
|
+
|
|
5945
|
+
/** Numeric option parsing that never silently accepts garbage. */
|
|
5946
|
+
function numOpt(options: Map<string, string>, key: string): { value: number } | { error: string } | null {
|
|
5947
|
+
const raw = options.get(key);
|
|
5948
|
+
if (raw === undefined) return null;
|
|
5949
|
+
const n = Number(raw);
|
|
5950
|
+
if (!Number.isFinite(n)) return { error: `--${key} expects a finite number, got ${JSON.stringify(raw)}` };
|
|
5951
|
+
return { value: n };
|
|
5952
|
+
}
|
|
5953
|
+
|
|
5954
|
+
/**
|
|
5955
|
+
* `dz epoch-replay` — the executable cold-vs-warm epoch runner (feature epoch-replay, scout #4).
|
|
5956
|
+
*
|
|
5957
|
+
* `dz compounding` says whether a replay CAN be run; this says what it FOUND. The runner never
|
|
5958
|
+
* calls a model: real mode emits a work order, renders blind judge prompts, and scores filled
|
|
5959
|
+
* judgments; `--mock` exercises the same verdict math on seeded synthetic outcomes at $0.
|
|
5960
|
+
*/
|
|
5961
|
+
function cmdEpochReplay(options: Map<string, string>, flags: Set<string>, cwd: string, write: Write): number {
|
|
5962
|
+
const json = flags.has('json');
|
|
5963
|
+
const fail = (msg: string): number => {
|
|
5964
|
+
write(json ? JSON.stringify({ error: msg, exitCode: 1 }) : `dz epoch-replay: ${msg}\n usage:\n ${EPOCH_REPLAY_USAGE}`);
|
|
5965
|
+
return 1;
|
|
5966
|
+
};
|
|
5967
|
+
if (flags.has('help')) {
|
|
5968
|
+
write(`dz epoch-replay — cold (epoch 0) vs warm (epoch 1), Wilson-CI three-valued verdict\n ${EPOCH_REPLAY_USAGE}`);
|
|
5969
|
+
write('');
|
|
5970
|
+
write(' ONE binomial over DECISIVE pairs (ties carry no direction and are excluded from the test).');
|
|
5971
|
+
write(' SUPPORTED only when the paired lift interval lies ENTIRELY above zero.');
|
|
5972
|
+
write(' FALSIFIED only on HARM (entirely below zero), or on a passed NON-SUPERIORITY test (the');
|
|
5973
|
+
write(' lift upper bound below the PRE-REGISTERED margin, default 0.05). Otherwise INCONCLUSIVE —');
|
|
5974
|
+
write(' a first-class honest outcome; a tie is UNDER-POWERED, never "refuted".');
|
|
5975
|
+
write(' The margin is pre-registered at --emit and stored in the work order; --score reads it there.');
|
|
5976
|
+
write(' This runner ORCHESTRATES and SCORES; it never calls a model. The judge-facing file holds');
|
|
5977
|
+
write(' {id, prompt} and nothing else; --score refuses any work order whose digest or seed-derived');
|
|
5978
|
+
write(' assignment does not check out, and refuses duplicate judgement ids.');
|
|
5979
|
+
return 0;
|
|
5980
|
+
}
|
|
5981
|
+
|
|
5982
|
+
const allowedFlags = new Set(['mock', 'emit', 'json', 'help']);
|
|
5983
|
+
for (const flag of flags) {
|
|
5984
|
+
if (!allowedFlags.has(flag)) return fail(`unknown option --${flag}`);
|
|
5985
|
+
}
|
|
5986
|
+
const allowedOptions = new Set(['judge', 'score', 'work-order', 'project', 'out', 'n', 'effect', 'tie-rate', 'seed', 'slice', 'limit', 'margin']);
|
|
5987
|
+
for (const key of options.keys()) {
|
|
5988
|
+
if (key.startsWith('_positional_')) return fail(`unexpected argument "${options.get(key)}"`);
|
|
5989
|
+
if (!allowedOptions.has(key)) return fail(`unknown option --${key}`);
|
|
5990
|
+
}
|
|
5991
|
+
|
|
5992
|
+
// Exactly ONE mode. A command whose default mode is a report can silently swallow a typo'd mode
|
|
5993
|
+
// flag and print something that reads like a result — so there is NO default mode here.
|
|
5994
|
+
const modes = [
|
|
5995
|
+
flags.has('mock') ? 'mock' : null,
|
|
5996
|
+
flags.has('emit') ? 'emit' : null,
|
|
5997
|
+
options.has('judge') ? 'judge' : null,
|
|
5998
|
+
options.has('score') ? 'score' : null,
|
|
5999
|
+
].filter((m): m is string => m !== null);
|
|
6000
|
+
if (modes.length === 0) return fail('pick exactly one mode: --mock | --emit | --judge <file> | --score <file>');
|
|
6001
|
+
if (modes.length > 1) return fail(`modes are exclusive, got: ${modes.join(', ')}`);
|
|
6002
|
+
const mode = modes[0]!;
|
|
6003
|
+
|
|
6004
|
+
const num = (key: string): number | undefined | { error: string } => {
|
|
6005
|
+
const r = numOpt(options, key);
|
|
6006
|
+
if (r === null) return undefined;
|
|
6007
|
+
if ('error' in r) return r;
|
|
6008
|
+
return r.value;
|
|
6009
|
+
};
|
|
6010
|
+
|
|
6011
|
+
// ── --mock: seeded synthetic outcomes, $0, exercises the real verdict math ──
|
|
6012
|
+
if (mode === 'mock') {
|
|
6013
|
+
for (const key of ['judge', 'score', 'work-order', 'project', 'out', 'limit']) {
|
|
6014
|
+
if (options.has(key)) return fail(`--${key} is not valid with --mock`);
|
|
6015
|
+
}
|
|
6016
|
+
const parsed: Record<string, number | undefined> = {};
|
|
6017
|
+
for (const key of ['n', 'effect', 'tie-rate', 'seed'] as const) {
|
|
6018
|
+
const v = num(key);
|
|
6019
|
+
if (typeof v === 'object' && v !== null) return fail(v.error);
|
|
6020
|
+
parsed[key] = v;
|
|
6021
|
+
}
|
|
6022
|
+
const outcomes = generateMockOutcomes({
|
|
6023
|
+
n: parsed.n ?? DEFAULT_MOCK_N,
|
|
6024
|
+
effect: parsed.effect ?? 0,
|
|
6025
|
+
tieRate: parsed['tie-rate'] ?? 0,
|
|
6026
|
+
seed: parsed.seed ?? DEFAULT_MOCK_SEED,
|
|
6027
|
+
});
|
|
6028
|
+
const marginOpt = numOpt(options, 'margin');
|
|
6029
|
+
if (marginOpt !== null && 'error' in marginOpt) return fail(marginOpt.error);
|
|
6030
|
+
const result = scoreEpochReplay(outcomes, {
|
|
6031
|
+
slice: options.get('slice') ?? 'all',
|
|
6032
|
+
...(marginOpt !== null ? { margin: marginOpt.value } : {}),
|
|
6033
|
+
});
|
|
6034
|
+
if (result.refusal !== null) return fail(result.refusal);
|
|
6035
|
+
if (json) {
|
|
6036
|
+
write(JSON.stringify({ mode: 'mock', synthetic: true, ...result, exitCode: 0 }, null, 2));
|
|
6037
|
+
} else {
|
|
6038
|
+
write(renderEpochReplayResult(result));
|
|
6039
|
+
write('');
|
|
6040
|
+
write(
|
|
6041
|
+
` SYNTHETIC (--mock): outcomes generated with seed ${parsed.seed ?? DEFAULT_MOCK_SEED} at a TRUE effect of ${parsed.effect ?? 0}. ` +
|
|
6042
|
+
'This exercises the protocol, it is NOT evidence about the learning loop.',
|
|
6043
|
+
);
|
|
6044
|
+
}
|
|
6045
|
+
return 0;
|
|
6046
|
+
}
|
|
6047
|
+
|
|
6048
|
+
// ── --emit: the generation work order (real mode, stage 1) ──
|
|
6049
|
+
if (mode === 'emit') {
|
|
6050
|
+
for (const key of ['judge', 'score', 'work-order', 'n', 'effect', 'tie-rate', 'slice']) {
|
|
6051
|
+
if (options.has(key)) return fail(`--${key} is not valid with --emit`);
|
|
6052
|
+
}
|
|
6053
|
+
const root = resolve(cwd, options.get('project') ?? '.');
|
|
6054
|
+
const seed = num('seed');
|
|
6055
|
+
if (typeof seed === 'object' && seed !== null) return fail(seed.error);
|
|
6056
|
+
const limit = num('limit');
|
|
6057
|
+
if (typeof limit === 'object' && limit !== null) return fail(limit.error);
|
|
6058
|
+
// HIGH-B: the non-superiority margin is PRE-REGISTERED here, digest-covered, and read back by
|
|
6059
|
+
// --score. Out of range is refused, never clamped — `--margin 99` must not buy FALSIFIED.
|
|
6060
|
+
const emitMargin = num('margin');
|
|
6061
|
+
if (typeof emitMargin === 'object' && emitMargin !== null) return fail(emitMargin.error);
|
|
6062
|
+
if (typeof emitMargin === 'number' && !isValidMargin(emitMargin)) {
|
|
6063
|
+
return fail(`--margin ${emitMargin} must be in (0, 0.5] — refused, not clamped: an oversized margin buys FALSIFIED`);
|
|
6064
|
+
}
|
|
6065
|
+
|
|
6066
|
+
const lessonText = new Map<string, string>();
|
|
6067
|
+
for (const r of loadStoreRecords(root)) {
|
|
6068
|
+
if (typeof r.text === 'string' && r.text.trim() !== '') lessonText.set(r.id, r.text);
|
|
6069
|
+
}
|
|
6070
|
+
// The SAME reader `dz compounding` uses — readiness and the runner must see one corpus.
|
|
6071
|
+
const instances = replayableInstances(readRecallUsageEvents(root), lessonText);
|
|
6072
|
+
const order = buildWorkOrder(instances, {
|
|
6073
|
+
...(typeof seed === 'number' ? { seed } : {}),
|
|
6074
|
+
...(typeof limit === 'number' ? { limit } : {}),
|
|
6075
|
+
...(typeof emitMargin === 'number' ? { margin: emitMargin } : {}),
|
|
6076
|
+
});
|
|
6077
|
+
const outPath = resolve(cwd, options.get('out') ?? join(root, EPOCH_REPLAY_DIR, 'work-order.json'));
|
|
6078
|
+
try {
|
|
6079
|
+
writeJsonOut(outPath, order);
|
|
6080
|
+
} catch (e) {
|
|
6081
|
+
return fail(`cannot write ${outPath}: ${e instanceof Error ? e.message : String(e)}`);
|
|
6082
|
+
}
|
|
6083
|
+
if (json) {
|
|
6084
|
+
write(JSON.stringify({ mode: 'emit', out: outPath, instances: order.items.length, seed: order.seed, margin: order.margin, digest: order.digest, corpusFingerprint: order.corpusFingerprint, emittedAt: order.emittedAt, exitCode: 0 }, null, 2));
|
|
6085
|
+
}
|
|
6086
|
+
else write(renderWorkOrderSummary(order, outPath));
|
|
6087
|
+
return 0;
|
|
6088
|
+
}
|
|
6089
|
+
|
|
6090
|
+
// ── --judge: blind judge prompts from a FILLED work order (real mode, stage 2) ──
|
|
6091
|
+
if (mode === 'judge') {
|
|
6092
|
+
for (const key of ['score', 'work-order', 'n', 'effect', 'tie-rate', 'slice', 'limit', 'seed', 'margin']) {
|
|
6093
|
+
if (options.has(key)) return fail(`--${key} is not valid with --judge`);
|
|
6094
|
+
}
|
|
6095
|
+
const inPath = resolve(cwd, options.get('judge')!);
|
|
6096
|
+
const read = readJsonFile(inPath);
|
|
6097
|
+
if ('error' in read) return fail(read.error);
|
|
6098
|
+
const verified = asVerifiedWorkOrder(read.value);
|
|
6099
|
+
if ('problems' in verified) {
|
|
6100
|
+
return fail(`${inPath} is not a verifiable ${WORK_ORDER_KIND}: ${verified.problems.join('; ')} (emit one with \`dz epoch-replay --emit\`)`);
|
|
6101
|
+
}
|
|
6102
|
+
const result = buildJudgePrompts(verified.order);
|
|
6103
|
+
const outPath = resolve(cwd, options.get('out') ?? join(dirname(inPath), 'judge-prompts.json'));
|
|
6104
|
+
try {
|
|
6105
|
+
// The JUDGE-FACING artifact. Its whole content is {id, prompt} per item — no `warmIsA`, no
|
|
6106
|
+
// arm names, no path back to the work order, and NOT the `skipped` list (whose reasons name
|
|
6107
|
+
// arms). Anything else here hands the judge the answer key (Codex QE CRITICAL-1).
|
|
6108
|
+
writeJsonOut(outPath, {
|
|
6109
|
+
kind: 'dz-epoch-replay-judge-prompts',
|
|
6110
|
+
version: 1,
|
|
6111
|
+
prompts: result.prompts.map((p) => ({ id: p.id, prompt: p.prompt })),
|
|
6112
|
+
});
|
|
6113
|
+
} catch (e) {
|
|
6114
|
+
return fail(`cannot write ${outPath}: ${e instanceof Error ? e.message : String(e)}`);
|
|
6115
|
+
}
|
|
6116
|
+
// `skipped` is OPERATOR-facing only — stdout / --json, never the file.
|
|
6117
|
+
if (json) write(JSON.stringify({ mode: 'judge', out: outPath, prompts: result.prompts.length, skipped: result.skipped, exitCode: 0 }, null, 2));
|
|
6118
|
+
else write(renderJudgePromptsSummary(result, outPath));
|
|
6119
|
+
return 0;
|
|
6120
|
+
}
|
|
6121
|
+
|
|
6122
|
+
// ── --score: un-blind + verdict (real mode, stage 3) ──
|
|
6123
|
+
for (const key of ['n', 'effect', 'tie-rate', 'limit', 'seed', 'out', 'project']) {
|
|
6124
|
+
if (options.has(key)) return fail(`--${key} is not valid with --score`);
|
|
6125
|
+
}
|
|
6126
|
+
// HIGH-B: a margin chosen once the counts are visible is not a pre-registration — and `--margin 99`
|
|
6127
|
+
// at scoring time would simply buy FALSIFIED. Real mode reads it from the work order, full stop.
|
|
6128
|
+
if (options.has('margin')) {
|
|
6129
|
+
return fail('--margin is not valid with --score: the non-superiority margin is PRE-REGISTERED at --emit and stored in the work order (re-emit to change it)');
|
|
6130
|
+
}
|
|
6131
|
+
const orderPath = options.get('work-order');
|
|
6132
|
+
if (orderPath === undefined) {
|
|
6133
|
+
return fail('--score requires --work-order <file>: un-blinding must use the PRE-REGISTERED assignment, not a label in the judgments file');
|
|
6134
|
+
}
|
|
6135
|
+
const orderRead = readJsonFile(resolve(cwd, orderPath));
|
|
6136
|
+
if ('error' in orderRead) return fail(orderRead.error);
|
|
6137
|
+
const verifiedOrder = asVerifiedWorkOrder(orderRead.value);
|
|
6138
|
+
if ('problems' in verifiedOrder) {
|
|
6139
|
+
return fail(
|
|
6140
|
+
`${resolve(cwd, orderPath)} is not a verifiable ${WORK_ORDER_KIND} — refusing to un-blind against it: ${verifiedOrder.problems.join('; ')}`,
|
|
6141
|
+
);
|
|
6142
|
+
}
|
|
6143
|
+
const order = verifiedOrder.order;
|
|
6144
|
+
const judgePath = resolve(cwd, options.get('score')!);
|
|
6145
|
+
const judgeRead = readJsonFile(judgePath);
|
|
6146
|
+
if ('error' in judgeRead) return fail(judgeRead.error);
|
|
6147
|
+
const rawRows = Array.isArray(judgeRead.value)
|
|
6148
|
+
? judgeRead.value
|
|
6149
|
+
: typeof judgeRead.value === 'object' && judgeRead.value !== null && Array.isArray((judgeRead.value as { judgments?: unknown }).judgments)
|
|
6150
|
+
? (judgeRead.value as { judgments: unknown[] }).judgments
|
|
6151
|
+
: null;
|
|
6152
|
+
if (rawRows === null) return fail(`${judgePath} must be an array of {id, winner} rows (or {"judgments": [...]})`);
|
|
6153
|
+
const unblind = unblindJudgments(
|
|
6154
|
+
order,
|
|
6155
|
+
rawRows.map((r) => {
|
|
6156
|
+
const o = (typeof r === 'object' && r !== null ? r : {}) as Record<string, unknown>;
|
|
6157
|
+
return { id: typeof o.id === 'string' ? o.id : '', winner: typeof o.winner === 'string' ? o.winner : '' };
|
|
6158
|
+
}),
|
|
6159
|
+
);
|
|
6160
|
+
// A duplicated judgement id is corrupt input, not a skippable row — refuse loudly.
|
|
6161
|
+
if (!unblind.ok) return fail(unblind.error ?? 'judgments refused');
|
|
6162
|
+
const { outcomes, skipped } = unblind;
|
|
6163
|
+
|
|
6164
|
+
const result = scoreEpochReplay(outcomes as EpochOutcome[], {
|
|
6165
|
+
slice: options.get('slice') ?? 'all',
|
|
6166
|
+
margin: order.margin, // PRE-REGISTERED in the work order, verified by the digest
|
|
6167
|
+
});
|
|
6168
|
+
if (result.refusal !== null) return fail(result.refusal);
|
|
6169
|
+
if (json) {
|
|
6170
|
+
write(JSON.stringify({
|
|
6171
|
+
mode: 'score',
|
|
6172
|
+
workOrder: resolve(cwd, orderPath),
|
|
6173
|
+
judgments: judgePath,
|
|
6174
|
+
scored: outcomes.length,
|
|
6175
|
+
skipped,
|
|
6176
|
+
provenance: { seed: order.seed, margin: order.margin, emittedAt: order.emittedAt, digest: order.digest, corpusFingerprint: order.corpusFingerprint, digestScope: DIGEST_HONEST_SCOPE },
|
|
6177
|
+
...result,
|
|
6178
|
+
exitCode: 0,
|
|
6179
|
+
}, null, 2));
|
|
6180
|
+
} else {
|
|
6181
|
+
write(renderEpochReplayResult(result));
|
|
6182
|
+
if (skipped.length > 0) {
|
|
6183
|
+
write('');
|
|
6184
|
+
write(` ${skipped.length} judgment(s) SKIPPED (never guessed):`);
|
|
6185
|
+
for (const s of skipped) write(` · ${s.id}: ${s.reason}`);
|
|
6186
|
+
}
|
|
6187
|
+
// Provenance, so a reviewer can ask for the original emitted file and compare.
|
|
6188
|
+
write('');
|
|
6189
|
+
write(` WORK ORDER: seed ${order.seed} · margin ${order.margin} (pre-registered) · emitted ${order.emittedAt}`);
|
|
6190
|
+
write(` digest ${order.digest}`);
|
|
6191
|
+
write(` corpus ${order.corpusFingerprint}`);
|
|
6192
|
+
write(` ${DIGEST_HONEST_SCOPE}`);
|
|
6193
|
+
}
|
|
6194
|
+
return 0;
|
|
6195
|
+
}
|
|
6196
|
+
|
|
5867
6197
|
/**
|
|
5868
6198
|
* Env vars an INHERITED Claude session leaks into a child. Left in place, the probe can silently
|
|
5869
6199
|
* read the parent's project instead of the target — the exact confound that made a hand-rolled
|
|
@@ -6974,6 +7304,8 @@ export async function runCli(argv: string[], io: CliIo = {}): Promise<number> {
|
|
|
6974
7304
|
return cmdSkillsVerify(options, flags, cwd, write);
|
|
6975
7305
|
case 'compounding':
|
|
6976
7306
|
return cmdCompounding(options, flags, cwd, write);
|
|
7307
|
+
case 'epoch-replay':
|
|
7308
|
+
return cmdEpochReplay(options, flags, cwd, write);
|
|
6977
7309
|
case 'score':
|
|
6978
7310
|
return cmdScore(options, flags, cwd, write);
|
|
6979
7311
|
case 'backlog':
|