@mikenguyen69/harness 0.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +43 -0
- package/dist/build-identity.json +1 -0
- package/dist/cancellation.js +138 -0
- package/dist/cli.js +47 -0
- package/dist/dispatch.js +70 -0
- package/dist/help.js +47 -0
- package/dist/internal/agent/adapter/conformance.js +53 -0
- package/dist/internal/agent/adapter/hook.js +113 -0
- package/dist/internal/agent/adapter/install.js +30 -0
- package/dist/internal/agent/adapter/scope.js +55 -0
- package/dist/internal/agent/adapter/session.js +63 -0
- package/dist/internal/agent/adapter/settings.js +51 -0
- package/dist/internal/agent/cli.js +79 -0
- package/dist/internal/agent/evals/index.js +111 -0
- package/dist/internal/agent/index.js +15 -0
- package/dist/internal/agent/ledger-read.js +27 -0
- package/dist/internal/agent/routing/cli.js +104 -0
- package/dist/internal/agent/routing/index.js +123 -0
- package/dist/internal/agent/telemetry/index.js +97 -0
- package/dist/internal/orchestration/activity/index.js +111 -0
- package/dist/internal/orchestration/analyze/index.js +237 -0
- package/dist/internal/orchestration/cli.js +475 -0
- package/dist/internal/orchestration/config/index.js +66 -0
- package/dist/internal/orchestration/doctor/index.js +202 -0
- package/dist/internal/orchestration/forge/index.js +121 -0
- package/dist/internal/orchestration/git/index.js +181 -0
- package/dist/internal/orchestration/harness/cli.js +378 -0
- package/dist/internal/orchestration/harness/index.js +186 -0
- package/dist/internal/orchestration/harness/shim.js +182 -0
- package/dist/internal/orchestration/index.js +29 -0
- package/dist/internal/orchestration/loop/index.js +1107 -0
- package/dist/internal/orchestration/prompts/index.js +199 -0
- package/dist/internal/orchestration/report/index.js +125 -0
- package/dist/internal/orchestration/routes/index.js +85 -0
- package/dist/internal/orchestration/run/index.js +221 -0
- package/dist/internal/orchestration/runner/index.js +90 -0
- package/dist/internal/orchestration/runners/claude/index.js +246 -0
- package/dist/internal/orchestration/runners/claude/sdk.js +198 -0
- package/dist/internal/orchestration/runners/claude/wiring.js +98 -0
- package/dist/internal/orchestration/runners/codex/exec.js +114 -0
- package/dist/internal/orchestration/runners/codex/index.js +72 -0
- package/dist/internal/orchestration/runners/conformance/index.js +169 -0
- package/dist/internal/orchestration/runners/cursor/exec.js +531 -0
- package/dist/internal/orchestration/runners/cursor/index.js +184 -0
- package/dist/internal/orchestration/runners/stub/index.js +168 -0
- package/dist/internal/orchestration/runners/telemetry.js +36 -0
- package/dist/internal/orchestration/runstate/index.js +213 -0
- package/dist/internal/orchestration/runstate/publish.js +115 -0
- package/dist/internal/orchestration/worktree/index.js +89 -0
- package/dist/internal/system/adapters/index.js +19 -0
- package/dist/internal/system/agentops/index.js +16 -0
- package/dist/internal/system/agentops/scope.js +67 -0
- package/dist/internal/system/assets.js +45 -0
- package/dist/internal/system/board/cli.js +56 -0
- package/dist/internal/system/board/index.js +276 -0
- package/dist/internal/system/board/live.js +277 -0
- package/dist/internal/system/calibration/cli.js +45 -0
- package/dist/internal/system/calibration/index.js +141 -0
- package/dist/internal/system/cli.js +174 -0
- package/dist/internal/system/corpus/replay.js +79 -0
- package/dist/internal/system/digest/cli.js +23 -0
- package/dist/internal/system/digest/index.js +145 -0
- package/dist/internal/system/doors/classify.js +94 -0
- package/dist/internal/system/doors/cli.js +184 -0
- package/dist/internal/system/doors/index.js +249 -0
- package/dist/internal/system/doors/metrics.js +47 -0
- package/dist/internal/system/escapes/cli.js +45 -0
- package/dist/internal/system/escapes/index.js +81 -0
- package/dist/internal/system/explore/cli.js +77 -0
- package/dist/internal/system/explore/index.js +91 -0
- package/dist/internal/system/gates/cli.js +43 -0
- package/dist/internal/system/gates/detectors.js +55 -0
- package/dist/internal/system/gates/pre.js +51 -0
- package/dist/internal/system/hub/cli.js +51 -0
- package/dist/internal/system/hub/index.js +146 -0
- package/dist/internal/system/init/index.js +83 -0
- package/dist/internal/system/ledger/cli.js +54 -0
- package/dist/internal/system/ledger/index.js +176 -0
- package/dist/internal/system/manifest/index.js +207 -0
- package/dist/internal/system/provenance/index.js +13 -0
- package/dist/internal/system/report/index.js +15 -0
- package/dist/internal/system/runner/index.js +74 -0
- package/dist/internal/system/runstatus/cli.js +86 -0
- package/dist/internal/system/runstatus/index.js +168 -0
- package/dist/internal/system/sequencer/briefing.js +75 -0
- package/dist/internal/system/sequencer/cli.js +170 -0
- package/dist/internal/system/sequencer/index.js +156 -0
- package/dist/internal/system/spec/cli.js +110 -0
- package/dist/internal/system/spec/index.js +212 -0
- package/dist/internal/system/spec/materialize.js +111 -0
- package/dist/internal/system/spec/openspec-parse.js +83 -0
- package/dist/internal/system/spec/project.js +208 -0
- package/dist/internal/system/substrate/index.js +14 -0
- package/dist/internal/system/system/index.js +159 -0
- package/dist/internal/system/units/index.js +87 -0
- package/dist/internal/system/verify/cli.js +103 -0
- package/dist/internal/system/verify/declined-reasons.js +61 -0
- package/dist/internal/system/verify/index.js +188 -0
- package/dist/internal/system/verify/signing.js +79 -0
- package/dist/paths.js +50 -0
- package/dist/routes.js +93 -0
- package/dist/version.js +30 -0
- package/dist/win32-process-group.js +360 -0
- package/package.json +41 -0
- package/profiles/python/bindings.toml +59 -0
- package/profiles/terraform/bindings.toml +66 -0
- package/profiles/typescript/bindings.toml +64 -0
- package/profiles/typescript/smoke/README.md +8 -0
- package/schema/baseline.schema.json +26 -0
- package/schema/decision-classes.yaml +45 -0
- package/schema/envelope.schema.json +186 -0
- package/schema/ledger-event.schema.json +235 -0
- package/schema/manifest.schema.json +119 -0
- package/schema/routes.schema.json +55 -0
- package/schema/run-status.schema.json +162 -0
- package/schema/session-event.schema.json +74 -0
- package/schema/spec.schema.json +114 -0
- package/schema/system.schema.json +142 -0
- package/templates/ci/verify.yml +23 -0
- package/templates/target-kit/README.md +78 -0
- package/templates/target-kit/component.toml +20 -0
- package/templates/target-kit/orchestration.json +9 -0
- package/templates/target-kit/profiles/target/bindings.toml +6 -0
- package/templates/target-kit/routes.toml +5 -0
- package/templates/target-kit/specs/first-unit.yaml +11 -0
- package/templates/target-kit/system.toml +7 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* calibration — the weekly job. Reads the fleet's envelopes over a trailing
|
|
3
|
+
* window and proposes gate severity changes as data, each citing its evidence
|
|
4
|
+
* query. Proposals are appended as `calibration.proposed`; a human applies (or
|
|
5
|
+
* rejects) them — severity never moves silently.
|
|
6
|
+
*
|
|
7
|
+
* Below an event-volume floor the cadence auto-derates to monthly and no
|
|
8
|
+
* proposals are made ("insufficient data").
|
|
9
|
+
*/
|
|
10
|
+
import { append, read as ledgerRead, } from "../ledger/index.js";
|
|
11
|
+
const RUNS_TO_PROMOTE = 20;
|
|
12
|
+
const RUNS_FOR_JUDGEMENT = 10;
|
|
13
|
+
const FAIL_RATE_DEMOTE = 0.5;
|
|
14
|
+
const RETRY_RATE_DEMOTE = 0.2;
|
|
15
|
+
const MONTHLY_FLOOR = 30; // fewer than this many envelopes in the window → monthly cadence
|
|
16
|
+
const RAMP = ["advisory", "changed-lines", "blocking"];
|
|
17
|
+
function nextUp(s) {
|
|
18
|
+
const i = RAMP.indexOf(s);
|
|
19
|
+
return i >= 0 && i < RAMP.length - 1 ? RAMP[i + 1] : null;
|
|
20
|
+
}
|
|
21
|
+
function nextDown(s) {
|
|
22
|
+
const i = RAMP.indexOf(s);
|
|
23
|
+
return i > 0 ? RAMP[i - 1] : null;
|
|
24
|
+
}
|
|
25
|
+
export function analyzeGates(events, now, windowDays) {
|
|
26
|
+
const cutoff = now - windowDays * 24 * 60 * 60 * 1000;
|
|
27
|
+
const stats = new Map();
|
|
28
|
+
for (const e of events) {
|
|
29
|
+
if (e.type !== "envelope" || Date.parse(e.ts) < cutoff)
|
|
30
|
+
continue;
|
|
31
|
+
const b = e.body;
|
|
32
|
+
for (const g of b.gates ?? []) {
|
|
33
|
+
const s = stats.get(g.name) ?? {
|
|
34
|
+
runs: 0,
|
|
35
|
+
fails: 0,
|
|
36
|
+
retries: 0,
|
|
37
|
+
declined: 0,
|
|
38
|
+
severity: g.severity,
|
|
39
|
+
};
|
|
40
|
+
s.severity = g.severity;
|
|
41
|
+
if (g.outcome === "declined")
|
|
42
|
+
s.declined++;
|
|
43
|
+
else {
|
|
44
|
+
s.runs++;
|
|
45
|
+
if (g.outcome === "fail")
|
|
46
|
+
s.fails++;
|
|
47
|
+
if ((g.retries ?? 0) > 0)
|
|
48
|
+
s.retries++;
|
|
49
|
+
}
|
|
50
|
+
stats.set(g.name, s);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return stats;
|
|
54
|
+
}
|
|
55
|
+
export function proposeCalibration(opts) {
|
|
56
|
+
const now = opts.now ?? Date.now();
|
|
57
|
+
const windowDays = opts.windowDays ?? 30;
|
|
58
|
+
const events = ledgerRead(opts.ledgerDir);
|
|
59
|
+
const envelopesInWindow = events.filter((e) => e.type === "envelope" &&
|
|
60
|
+
Date.parse(e.ts) >= now - windowDays * 24 * 60 * 60 * 1000).length;
|
|
61
|
+
if (envelopesInWindow < MONTHLY_FLOOR) {
|
|
62
|
+
return {
|
|
63
|
+
cadence: "monthly",
|
|
64
|
+
windowDays,
|
|
65
|
+
envelopesInWindow,
|
|
66
|
+
insufficient: true,
|
|
67
|
+
proposals: [],
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
const stats = analyzeGates(events, now, windowDays);
|
|
71
|
+
const debt = opts.baseline?.debt ?? {};
|
|
72
|
+
const proposals = [];
|
|
73
|
+
for (const [name, s] of stats) {
|
|
74
|
+
// delete: never actually ran, no frozen debt for it
|
|
75
|
+
if (s.runs === 0 && s.declined > 0 && (debt[name] ?? 0) === 0) {
|
|
76
|
+
proposals.push({
|
|
77
|
+
action: "delete",
|
|
78
|
+
target: name,
|
|
79
|
+
evidence: `declined ${s.declined}× / ran 0× in ${windowDays}d, baseline debt 0`,
|
|
80
|
+
});
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const failRate = s.runs ? s.fails / s.runs : 0;
|
|
84
|
+
const retryRate = s.runs ? s.retries / s.runs : 0;
|
|
85
|
+
// demote: failing or flaky
|
|
86
|
+
if (s.runs >= RUNS_FOR_JUDGEMENT &&
|
|
87
|
+
(failRate > FAIL_RATE_DEMOTE || retryRate > RETRY_RATE_DEMOTE)) {
|
|
88
|
+
const to = nextDown(s.severity);
|
|
89
|
+
if (to) {
|
|
90
|
+
proposals.push({
|
|
91
|
+
action: "demote",
|
|
92
|
+
target: name,
|
|
93
|
+
from: s.severity,
|
|
94
|
+
to,
|
|
95
|
+
evidence: `${s.fails}/${s.runs} fail, ${s.retries}/${s.runs} pass-on-retry in ${windowDays}d`,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
// promote: clean over a long run
|
|
101
|
+
if (s.runs >= RUNS_TO_PROMOTE && s.fails === 0 && s.retries === 0) {
|
|
102
|
+
const to = nextUp(s.severity);
|
|
103
|
+
if (to) {
|
|
104
|
+
proposals.push({
|
|
105
|
+
action: "promote",
|
|
106
|
+
target: name,
|
|
107
|
+
from: s.severity,
|
|
108
|
+
to,
|
|
109
|
+
evidence: `0 failures in ${s.runs} runs over ${windowDays}d`,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
proposals.sort((a, b) => a.target.localeCompare(b.target));
|
|
115
|
+
return {
|
|
116
|
+
cadence: "weekly",
|
|
117
|
+
windowDays,
|
|
118
|
+
envelopesInWindow,
|
|
119
|
+
insufficient: false,
|
|
120
|
+
proposals,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/** Append the proposals to the ledger (calibration.proposed, one per proposal). */
|
|
124
|
+
export function emitProposals(report, ledgerDir, system = "fleet") {
|
|
125
|
+
for (const p of report.proposals) {
|
|
126
|
+
append(ledgerDir, {
|
|
127
|
+
system,
|
|
128
|
+
actor: { kind: "ci", id: "calibration-job" },
|
|
129
|
+
type: "calibration.proposed",
|
|
130
|
+
body: {
|
|
131
|
+
action: p.action,
|
|
132
|
+
target: p.target,
|
|
133
|
+
...(p.from ? { from: p.from } : {}),
|
|
134
|
+
...(p.to ? { to: p.to } : {}),
|
|
135
|
+
evidence: p.evidence,
|
|
136
|
+
},
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
return report.proposals.length;
|
|
140
|
+
}
|
|
141
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* harness — the single entrypoint, identical for agent, human, and CI.
|
|
4
|
+
*
|
|
5
|
+
* The moment any caller gets its own weaker path, that path becomes the
|
|
6
|
+
* default and the standard is decorative. So: no flags that weaken gates
|
|
7
|
+
* per-caller.
|
|
8
|
+
*
|
|
9
|
+
* Structural rules the runner enforces (not configurable):
|
|
10
|
+
* - a vocabulary gate with no binding and no declination is an error
|
|
11
|
+
* - all-skip / all-declined is never a pass
|
|
12
|
+
* - retries are recorded; a pass-on-retry is not a clean pass
|
|
13
|
+
* - out-of-scope file touches halt (fail closed), never warn
|
|
14
|
+
*/
|
|
15
|
+
import { doorCommand, mergeCommand } from "./doors/cli.js";
|
|
16
|
+
import { escapeCommand } from "./escapes/cli.js";
|
|
17
|
+
import { gateCommand } from "./gates/cli.js";
|
|
18
|
+
import { exploreCommand } from "./explore/cli.js";
|
|
19
|
+
import { boardCommand, boardServeCommand } from "./board/cli.js";
|
|
20
|
+
import { calibrateCommand } from "./calibration/cli.js";
|
|
21
|
+
import { digestCommand } from "./digest/cli.js";
|
|
22
|
+
import { hubCommand } from "./hub/cli.js";
|
|
23
|
+
import { runstatusCommand } from "./runstatus/cli.js";
|
|
24
|
+
import { ledgerCommand } from "./ledger/cli.js";
|
|
25
|
+
import { initCommand } from "./init/index.js";
|
|
26
|
+
import { claimCommand, heartbeatCommand, nextCommand, } from "./sequencer/cli.js";
|
|
27
|
+
import { specCommand } from "./spec/cli.js";
|
|
28
|
+
import { verifyCommand } from "./verify/cli.js";
|
|
29
|
+
const USAGE = `harness — deterministic scaffolding around nondeterministic agents
|
|
30
|
+
|
|
31
|
+
Usage:
|
|
32
|
+
harness init scaffold component.toml, hooks, CI workflow, generated adapters
|
|
33
|
+
--audit: read-only brownfield inventory (stack, tooling, substrate,
|
|
34
|
+
debt counts, flaky tests) — writes nothing, prints the plan
|
|
35
|
+
--mode ratchet: brownfield adoption — freeze the debt inventory as
|
|
36
|
+
.harness/baseline.json; bulk gates run changed-lines against
|
|
37
|
+
a floor that may only fall; flakes quarantined by name
|
|
38
|
+
harness verify run all bound gates (cheap first, fail fast) and emit the report
|
|
39
|
+
harness gate --pre --file <path> ring-0: run bound cheap-cost gates
|
|
40
|
+
(a single named gate, e.g. \`gate spec\`, is not yet implemented)
|
|
41
|
+
harness spec check lint specs/*.yaml + dep DAG + consumes/produces
|
|
42
|
+
[--specs <dir>] [--system <system.toml>]
|
|
43
|
+
materialize validated specs into idempotent unit.planned facts
|
|
44
|
+
--specs <dir> --system <system.toml> [--ledger <dir>] [--json]
|
|
45
|
+
project emit harness unit YAML from OpenSpec + harness.projection.yaml
|
|
46
|
+
--change <dir> [--openspec <dir>] [--out <dir>] [--json]
|
|
47
|
+
harness next list units ready to claim (deps merged, unclaimed or stale)
|
|
48
|
+
harness claim <unit-id> --session <id> atomically take a ready unit
|
|
49
|
+
harness heartbeat <unit-id> --session <id> keep a claim alive
|
|
50
|
+
harness door list [--sweep] | show <id> | approve <id> --reason … | reject …
|
|
51
|
+
harness merge <unit-id> <--check | --landed <sha>> [facts…] authorize/record
|
|
52
|
+
harness ledger append/read/verify the hash-chained record plane
|
|
53
|
+
append (stdin JSON) | read | verify-chain [--dir <path>]
|
|
54
|
+
harness runstatus publish [--file <path>] --spool <dir> [--machine <id>]
|
|
55
|
+
push a run-status document to the hub spool (actuator → board)
|
|
56
|
+
harness board build [--ledger <dir>] [--spool <dir>] [--out board.html]
|
|
57
|
+
serve [--root <dir>]... [--port 4317] [--refresh 2000]
|
|
58
|
+
run the local live dashboard across initialized repositories
|
|
59
|
+
harness attest write/verify provenance attestations for a merge
|
|
60
|
+
harness report aggregate envelopes (calibration input)
|
|
61
|
+
`;
|
|
62
|
+
const COMMANDS = [
|
|
63
|
+
"init",
|
|
64
|
+
"verify",
|
|
65
|
+
"gate",
|
|
66
|
+
"spec",
|
|
67
|
+
"next",
|
|
68
|
+
"claim",
|
|
69
|
+
"heartbeat",
|
|
70
|
+
"door",
|
|
71
|
+
"merge",
|
|
72
|
+
"escape",
|
|
73
|
+
"explore",
|
|
74
|
+
"hub",
|
|
75
|
+
"runstatus",
|
|
76
|
+
"board",
|
|
77
|
+
"calibrate",
|
|
78
|
+
"digest",
|
|
79
|
+
"ledger",
|
|
80
|
+
"attest",
|
|
81
|
+
"report",
|
|
82
|
+
];
|
|
83
|
+
function isCommand(arg) {
|
|
84
|
+
return COMMANDS.includes(arg);
|
|
85
|
+
}
|
|
86
|
+
const cmd = process.argv[2];
|
|
87
|
+
if (!isCommand(cmd)) {
|
|
88
|
+
process.stderr.write(USAGE);
|
|
89
|
+
process.exit(2);
|
|
90
|
+
}
|
|
91
|
+
if (cmd === "ledger") {
|
|
92
|
+
process.exit(ledgerCommand(process.argv.slice(3)));
|
|
93
|
+
}
|
|
94
|
+
if (cmd === "init") {
|
|
95
|
+
process.exit(initCommand(process.argv.slice(3)));
|
|
96
|
+
}
|
|
97
|
+
if (cmd === "verify") {
|
|
98
|
+
process.exit(verifyCommand(process.argv.slice(3)));
|
|
99
|
+
}
|
|
100
|
+
if (cmd === "spec") {
|
|
101
|
+
process.exit(specCommand(process.argv.slice(3)));
|
|
102
|
+
}
|
|
103
|
+
if (cmd === "next") {
|
|
104
|
+
process.exit(nextCommand(process.argv.slice(3)));
|
|
105
|
+
}
|
|
106
|
+
if (cmd === "door") {
|
|
107
|
+
process.exit(doorCommand(process.argv.slice(3)));
|
|
108
|
+
}
|
|
109
|
+
if (cmd === "merge") {
|
|
110
|
+
process.exit(mergeCommand(process.argv.slice(3)));
|
|
111
|
+
}
|
|
112
|
+
if (cmd === "escape") {
|
|
113
|
+
process.exit(escapeCommand(process.argv.slice(3)));
|
|
114
|
+
}
|
|
115
|
+
if (cmd === "explore") {
|
|
116
|
+
process.exit(exploreCommand(process.argv.slice(3)));
|
|
117
|
+
}
|
|
118
|
+
if (cmd === "hub") {
|
|
119
|
+
process.exit(hubCommand(process.argv.slice(3)));
|
|
120
|
+
}
|
|
121
|
+
if (cmd === "runstatus") {
|
|
122
|
+
process.exit(runstatusCommand(process.argv.slice(3)));
|
|
123
|
+
}
|
|
124
|
+
if (cmd === "board") {
|
|
125
|
+
if (process.argv[3] === "serve") {
|
|
126
|
+
try {
|
|
127
|
+
const running = await boardServeCommand(process.argv.slice(4));
|
|
128
|
+
process.stdout.write(`Harness Operations live at ${running.url}\nPress Ctrl+C to stop.\n`);
|
|
129
|
+
// Safety valve for automated contexts: a verifier/agent that spawns
|
|
130
|
+
// `board serve` and forgets it would otherwise run forever and hold its
|
|
131
|
+
// parent's stdio pipes open. HARNESS_BOARD_SERVE_MAX_MS makes it
|
|
132
|
+
// self-terminate. Unset = unlimited, for interactive use.
|
|
133
|
+
const maxMs = Number(process.env.HARNESS_BOARD_SERVE_MAX_MS ?? 0);
|
|
134
|
+
await new Promise((accept) => {
|
|
135
|
+
const stop = () => void running.close().finally(accept);
|
|
136
|
+
process.once("SIGINT", stop);
|
|
137
|
+
process.once("SIGTERM", stop);
|
|
138
|
+
if (Number.isFinite(maxMs) && maxMs > 0) {
|
|
139
|
+
setTimeout(() => {
|
|
140
|
+
process.stderr.write(`harness board serve: self-stopping after ${maxMs}ms\n`);
|
|
141
|
+
stop();
|
|
142
|
+
}, maxMs);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
process.exit(0);
|
|
146
|
+
}
|
|
147
|
+
catch (error) {
|
|
148
|
+
process.stderr.write(`harness board serve: ${error.message}\n`);
|
|
149
|
+
process.exit(2);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
process.exit(boardCommand(process.argv.slice(3)));
|
|
153
|
+
}
|
|
154
|
+
if (cmd === "calibrate") {
|
|
155
|
+
process.exit(calibrateCommand(process.argv.slice(3)));
|
|
156
|
+
}
|
|
157
|
+
if (cmd === "digest") {
|
|
158
|
+
process.exit(digestCommand(process.argv.slice(3)));
|
|
159
|
+
}
|
|
160
|
+
if (cmd === "claim") {
|
|
161
|
+
process.exit(claimCommand(process.argv.slice(3)));
|
|
162
|
+
}
|
|
163
|
+
if (cmd === "heartbeat") {
|
|
164
|
+
process.exit(heartbeatCommand(process.argv.slice(3)));
|
|
165
|
+
}
|
|
166
|
+
if (cmd === "gate") {
|
|
167
|
+
const code = gateCommand(process.argv.slice(3));
|
|
168
|
+
if (code !== undefined)
|
|
169
|
+
process.exit(code);
|
|
170
|
+
// fall through: a bare `harness gate <name>` (not --pre) is still a skeleton
|
|
171
|
+
}
|
|
172
|
+
process.stderr.write(`harness ${cmd}: not implemented (skeleton)\n`);
|
|
173
|
+
process.exit(1);
|
|
174
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Corpus replay — compiled so subprocess validation does not need a TypeScript
|
|
3
|
+
* loader. Every active fixture must still be blocked; a reopened fixture fails
|
|
4
|
+
* the build even when its detector would block it again.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync, } from "node:fs";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import { join, resolve } from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { parse as parseYaml } from "yaml";
|
|
11
|
+
import { checkSpecs } from "../spec/index.js";
|
|
12
|
+
import { scopeCross, suppressionDelta, testWeakening, } from "../gates/detectors.js";
|
|
13
|
+
function detectorBlocks(f) {
|
|
14
|
+
const input = f.replay.input;
|
|
15
|
+
switch (f.replay.kind) {
|
|
16
|
+
case "spec": {
|
|
17
|
+
const dir = mkdtempSync(join(tmpdir(), "corpus-spec-"));
|
|
18
|
+
try {
|
|
19
|
+
writeFileSync(join(dir, "s.yaml"), JSON.stringify(input.spec));
|
|
20
|
+
return checkSpecs(dir).some((finding) => finding.level === "error");
|
|
21
|
+
}
|
|
22
|
+
finally {
|
|
23
|
+
rmSync(dir, { recursive: true, force: true });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
case "suppression":
|
|
27
|
+
return suppressionDelta(String(input.before ?? ""), String(input.after ?? "")).length > 0;
|
|
28
|
+
case "scope":
|
|
29
|
+
return scopeCross(input.changed ?? [], input.scope ?? []).length > 0;
|
|
30
|
+
case "test-weakening":
|
|
31
|
+
return testWeakening({
|
|
32
|
+
changed: input.changed ?? [],
|
|
33
|
+
assertionsRemoved: input.assertionsRemoved ?? {},
|
|
34
|
+
}).length > 0;
|
|
35
|
+
default:
|
|
36
|
+
throw new Error(`unknown replay kind: ${f.replay.kind}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
export function replayCorpus(fixturesDir) {
|
|
40
|
+
let failures = 0;
|
|
41
|
+
const lines = [];
|
|
42
|
+
const ids = readdirSync(fixturesDir).sort();
|
|
43
|
+
for (const id of ids) {
|
|
44
|
+
const fixture = parseYaml(readFileSync(join(fixturesDir, id, "fixture.yaml"), "utf8"));
|
|
45
|
+
if (fixture.status === "reopened") {
|
|
46
|
+
lines.push(`✗ ${fixture.id} — REOPENED (a recurred escape is unfinished work)`);
|
|
47
|
+
failures++;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const blocked = detectorBlocks(fixture);
|
|
51
|
+
if (blocked === fixture.replay.expect_block) {
|
|
52
|
+
lines.push(`✓ ${fixture.id} — ${fixture.must_block.gate} blocks as expected`);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
lines.push(`✗ ${fixture.id} — REGRESSION: expected block=${fixture.replay.expect_block}, got ${blocked}`);
|
|
56
|
+
failures++;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
lines.push(`\n${ids.length - failures}/${ids.length} fixtures pass`);
|
|
60
|
+
return { total: ids.length, failures, lines };
|
|
61
|
+
}
|
|
62
|
+
export function corpusReplayCommand(args) {
|
|
63
|
+
const i = args.indexOf("--dir");
|
|
64
|
+
const fixturesDir = i >= 0 && args[i + 1]
|
|
65
|
+
? args[i + 1]
|
|
66
|
+
: fileURLToPath(new URL("../../corpus/fixtures/", import.meta.url));
|
|
67
|
+
const result = replayCorpus(fixturesDir);
|
|
68
|
+
for (const line of result.lines) {
|
|
69
|
+
const stream = line.startsWith("✗") ? process.stderr : process.stdout;
|
|
70
|
+
stream.write(line + "\n");
|
|
71
|
+
}
|
|
72
|
+
return result.failures === 0 ? 0 : 1;
|
|
73
|
+
}
|
|
74
|
+
const invokedDirectly = process.argv[1] &&
|
|
75
|
+
fileURLToPath(import.meta.url).toLowerCase() ===
|
|
76
|
+
resolve(process.argv[1]).toLowerCase();
|
|
77
|
+
if (invokedDirectly)
|
|
78
|
+
process.exit(corpusReplayCommand(process.argv.slice(2)));
|
|
79
|
+
//# sourceMappingURL=replay.js.map
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `harness digest` — render the weekly operator digest from the ledger.
|
|
3
|
+
*
|
|
4
|
+
* harness digest [--ledger <dir>] [--window 7] [--out digest.html] [--json]
|
|
5
|
+
*/
|
|
6
|
+
import { writeFileSync } from "node:fs";
|
|
7
|
+
import { buildDigestFromLedger, renderDigestHtml } from "./index.js";
|
|
8
|
+
function flag(args, name) {
|
|
9
|
+
const i = args.indexOf(`--${name}`);
|
|
10
|
+
return i >= 0 && args[i + 1] ? args[i + 1] : undefined;
|
|
11
|
+
}
|
|
12
|
+
export function digestCommand(args) {
|
|
13
|
+
const d = buildDigestFromLedger(flag(args, "ledger") ?? ".harness/ledger", Date.now(), Number(flag(args, "window") ?? 7));
|
|
14
|
+
if (args.includes("--json")) {
|
|
15
|
+
process.stdout.write(JSON.stringify(d, null, 2) + "\n");
|
|
16
|
+
return 0;
|
|
17
|
+
}
|
|
18
|
+
const out = flag(args, "out") ?? "digest.html";
|
|
19
|
+
writeFileSync(out, renderDigestHtml(d));
|
|
20
|
+
process.stdout.write(`wrote ${out} — ${d.newFixtures.length} new fixture(s), ${d.approvedThenEscaped.length} approved-then-escaped, ${d.routeDrift.length} drift flag(s)\n`);
|
|
21
|
+
return 0;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* digest — the weekly operator digest (solution design §9.2). Records must
|
|
3
|
+
* teach, not just render: this composes a page from
|
|
4
|
+
* - new fixtures + their failure-class taxonomy
|
|
5
|
+
* - your door decisions vs outcomes (anything you approved that later escaped?)
|
|
6
|
+
* - route drift (eval catch rate falling across a model rev)
|
|
7
|
+
* - one spec-wording suggestion traced from an escape back to the unit it hit
|
|
8
|
+
*/
|
|
9
|
+
import { read as ledgerRead } from "../ledger/index.js";
|
|
10
|
+
function inWindow(e, now, windowDays) {
|
|
11
|
+
return Date.parse(e.ts) >= now - windowDays * 24 * 60 * 60 * 1000;
|
|
12
|
+
}
|
|
13
|
+
export function buildDigest(events, now = Date.now(), windowDays = 7) {
|
|
14
|
+
const win = events.filter((e) => inWindow(e, now, windowDays));
|
|
15
|
+
// 1. new fixtures + taxonomy
|
|
16
|
+
const newFixtures = [];
|
|
17
|
+
const failureTaxonomy = {};
|
|
18
|
+
for (const e of win) {
|
|
19
|
+
if (e.type !== "fixture.added")
|
|
20
|
+
continue;
|
|
21
|
+
const b = e.body;
|
|
22
|
+
const klass = b.klass ?? b.escape_class ?? "unclassified";
|
|
23
|
+
newFixtures.push({ id: b.id ?? "?", klass });
|
|
24
|
+
failureTaxonomy[klass] = (failureTaxonomy[klass] ?? 0) + 1;
|
|
25
|
+
}
|
|
26
|
+
// 2. approved-then-escaped: a door approved for a unit, then an escape naming it
|
|
27
|
+
const approvedUnits = new Map(); // unit → door id
|
|
28
|
+
for (const e of events) {
|
|
29
|
+
if (e.type !== "door.decided")
|
|
30
|
+
continue;
|
|
31
|
+
const b = e.body;
|
|
32
|
+
if (b.verdict === "approved" && b.unit_id)
|
|
33
|
+
approvedUnits.set(b.unit_id, b.door_id ?? "?");
|
|
34
|
+
}
|
|
35
|
+
const approvedThenEscaped = [];
|
|
36
|
+
for (const e of win) {
|
|
37
|
+
if (e.type !== "escape.routed" && e.type !== "escape.detected")
|
|
38
|
+
continue;
|
|
39
|
+
const b = e.body;
|
|
40
|
+
for (const [unit, door] of approvedUnits) {
|
|
41
|
+
const bareUnit = unit.includes("#")
|
|
42
|
+
? unit.slice(unit.indexOf("#") + 1)
|
|
43
|
+
: unit;
|
|
44
|
+
if ((b.new_unit ?? "").includes(bareUnit) || (b.unit_id ?? "") === unit) {
|
|
45
|
+
approvedThenEscaped.push({ unit, door, escape: b.scenario ?? "?" });
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
// 3. route drift — consecutive eval.run for one route with a falling catch rate
|
|
50
|
+
const byRoute = new Map();
|
|
51
|
+
for (const e of events) {
|
|
52
|
+
if (e.type !== "eval.run")
|
|
53
|
+
continue;
|
|
54
|
+
const b = e.body;
|
|
55
|
+
if (!b.route || typeof b.catch_rate !== "number")
|
|
56
|
+
continue;
|
|
57
|
+
if (!byRoute.has(b.route))
|
|
58
|
+
byRoute.set(b.route, []);
|
|
59
|
+
byRoute.get(b.route).push(b.catch_rate);
|
|
60
|
+
}
|
|
61
|
+
const routeDrift = [];
|
|
62
|
+
for (const [route, rates] of byRoute) {
|
|
63
|
+
for (let i = 1; i < rates.length; i++) {
|
|
64
|
+
const delta = Number((rates[i] - rates[i - 1]).toFixed(3));
|
|
65
|
+
if (delta <= -0.09)
|
|
66
|
+
routeDrift.push({ route, from: rates[i - 1], to: rates[i], delta });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
// 4. one spec-wording suggestion traced from an escape
|
|
70
|
+
let specSuggestion = null;
|
|
71
|
+
for (const e of win) {
|
|
72
|
+
if (e.type !== "escape.routed")
|
|
73
|
+
continue;
|
|
74
|
+
const b = e.body;
|
|
75
|
+
const detected = win.find((x) => x.type === "escape.detected" &&
|
|
76
|
+
x.body.scenario === b.scenario);
|
|
77
|
+
const ctx = detected?.body?.context;
|
|
78
|
+
specSuggestion =
|
|
79
|
+
`Escape "${b.scenario}" hit ${b.owner ?? "a component"} → its next spec should state the behaviour that was missing` +
|
|
80
|
+
(ctx ? `: "${ctx}"` : ". Add a falsifiable acceptance criterion for it.");
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
// 5. process cost per unit — from run.analyzed (P3)
|
|
84
|
+
const processCost = [];
|
|
85
|
+
for (const e of win) {
|
|
86
|
+
if (e.type !== "run.analyzed")
|
|
87
|
+
continue;
|
|
88
|
+
const b = e.body;
|
|
89
|
+
for (const u of b.units ?? []) {
|
|
90
|
+
processCost.push({
|
|
91
|
+
unit: u.unit_id ?? "",
|
|
92
|
+
turnsTotal: u.turns_total ?? 0,
|
|
93
|
+
wallMs: u.wall_ms ?? 0,
|
|
94
|
+
costUsd: typeof u.cost_usd === "number" ? u.cost_usd : null,
|
|
95
|
+
runId: b.run_id ?? "",
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return {
|
|
100
|
+
windowDays,
|
|
101
|
+
generatedAt: new Date(now).toISOString(),
|
|
102
|
+
newFixtures,
|
|
103
|
+
failureTaxonomy,
|
|
104
|
+
approvedThenEscaped,
|
|
105
|
+
routeDrift,
|
|
106
|
+
specSuggestion,
|
|
107
|
+
processCost,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
export function buildDigestFromLedger(ledgerDir, now, windowDays) {
|
|
111
|
+
return buildDigest(ledgerRead(ledgerDir), now, windowDays);
|
|
112
|
+
}
|
|
113
|
+
export function renderDigestHtml(d) {
|
|
114
|
+
const json = JSON.stringify(d, null, 2).replace(/</g, "\\u003c");
|
|
115
|
+
const section = (title, body) => `<section><h2>${title}</h2>${body}</section>`;
|
|
116
|
+
const list = (xs) => xs.length
|
|
117
|
+
? `<ul>${xs.map((x) => `<li>${x}</li>`).join("")}</ul>`
|
|
118
|
+
: `<p class="empty">nothing this week</p>`;
|
|
119
|
+
return `<!doctype html>
|
|
120
|
+
<html lang="en"><head><meta charset="utf-8"><title>Operator digest</title>
|
|
121
|
+
<style>
|
|
122
|
+
:root{color-scheme:light dark}
|
|
123
|
+
body{font:15px/1.6 ui-sans-serif,system-ui,sans-serif;max-width:760px;margin:40px auto;padding:0 20px;background:#0e1216;color:#e5e9ee}
|
|
124
|
+
h1{font-size:1.4rem} h2{font-size:.95rem;letter-spacing:.04em;text-transform:uppercase;color:#9aa4b1;margin:26px 0 8px}
|
|
125
|
+
.empty{color:#6b7583;font-style:italic} code{font-family:ui-monospace,monospace;color:#5bb8d4}
|
|
126
|
+
.suggest{border-left:3px solid #5bb8d4;background:#16333d;padding:10px 14px;border-radius:0 8px 8px 0}
|
|
127
|
+
</style></head>
|
|
128
|
+
<body>
|
|
129
|
+
<h1>Operator digest — last ${d.windowDays} days</h1>
|
|
130
|
+
<p style="color:#9aa4b1">generated ${d.generatedAt}</p>
|
|
131
|
+
${section("New fixtures + failure taxonomy", list(d.newFixtures.map((f) => `<code>${f.id}</code> — ${f.klass}`)) +
|
|
132
|
+
(Object.keys(d.failureTaxonomy).length
|
|
133
|
+
? `<p>${Object.entries(d.failureTaxonomy)
|
|
134
|
+
.map(([k, n]) => `${k}×${n}`)
|
|
135
|
+
.join(" · ")}</p>`
|
|
136
|
+
: ""))}
|
|
137
|
+
${section("Door decisions vs outcomes", list(d.approvedThenEscaped.map((a) => `approved <code>${a.door}</code> for <code>${a.unit}</code> → later escaped via <code>${a.escape}</code>`)))}
|
|
138
|
+
${section("Route drift", list(d.routeDrift.map((r) => `<code>${r.route}</code>: ${r.from} → ${r.to} (${r.delta})`)))}
|
|
139
|
+
${section("Spec-wording suggestion", d.specSuggestion ? `<div class="suggest">${d.specSuggestion}</div>` : `<p class="empty">no escape to trace this week</p>`)}
|
|
140
|
+
${section("Process cost per unit", list(d.processCost.map((p) => `<code>${p.unit}</code> — ${p.turnsTotal} turns, ${p.wallMs}ms${p.costUsd != null ? `, $${p.costUsd.toFixed(2)}` : ""} (run <code>${p.runId}</code>)`)))}
|
|
141
|
+
<script type="application/json" id="data">${json}</script>
|
|
142
|
+
</body></html>
|
|
143
|
+
`;
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* doors/classify — a merge is routed to a door CLASS by data only. Nothing here
|
|
3
|
+
* is hardcoded policy: the always-human classes come from
|
|
4
|
+
* schema/decision-classes.yaml and the routing rules from system.toml's
|
|
5
|
+
* `[[door_class]]` (Contract C, solution design §6.3).
|
|
6
|
+
*
|
|
7
|
+
* Output: the classes that fired, and whether an operator decision is required
|
|
8
|
+
* (any `human_required` class, or a `one-way` / `propose-only` routing).
|
|
9
|
+
*/
|
|
10
|
+
import { readFileSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { parse as parseToml } from "smol-toml";
|
|
13
|
+
import { parse as parseYaml } from "yaml";
|
|
14
|
+
import { packageSchemaDir } from "../assets.js";
|
|
15
|
+
function decisionClassesPath() {
|
|
16
|
+
return join(packageSchemaDir(), "decision-classes.yaml");
|
|
17
|
+
}
|
|
18
|
+
let _dc;
|
|
19
|
+
function decisionClasses() {
|
|
20
|
+
_dc ??= parseYaml(readFileSync(decisionClassesPath(), "utf8"));
|
|
21
|
+
return _dc;
|
|
22
|
+
}
|
|
23
|
+
/** system.toml `[[door_class]]` entries, if a path is given. */
|
|
24
|
+
function systemDoorClasses(systemTomlPath) {
|
|
25
|
+
if (!systemTomlPath)
|
|
26
|
+
return [];
|
|
27
|
+
const raw = parseToml(readFileSync(systemTomlPath, "utf8"));
|
|
28
|
+
return raw.door_class ?? [];
|
|
29
|
+
}
|
|
30
|
+
const HUMAN_FACT = {
|
|
31
|
+
"one-way-door": (f) => f.unitDoor === undefined || f.unitDoor === "one-way",
|
|
32
|
+
"new-dependency": (f) => f.addsDependency === true,
|
|
33
|
+
"schema-or-api-change": (f) => f.touchesSchemaOrApi === true,
|
|
34
|
+
"security-touching": (f) => f.touchesSecurity === true,
|
|
35
|
+
};
|
|
36
|
+
const DOOR_FACT = {
|
|
37
|
+
"gate:external": (f) => f.gateExternal === true,
|
|
38
|
+
"artifact-producer-change": (f) => f.artifactProducerChange === true,
|
|
39
|
+
"new-dependency": (f) => f.addsDependency === true,
|
|
40
|
+
"schema-or-api-change": (f) => f.touchesSchemaOrApi === true,
|
|
41
|
+
"security-touching": (f) => f.touchesSecurity === true,
|
|
42
|
+
};
|
|
43
|
+
function doorMatches(match, f) {
|
|
44
|
+
if (match.startsWith("component:"))
|
|
45
|
+
return f.component === match.slice("component:".length);
|
|
46
|
+
if (match.startsWith("mode:"))
|
|
47
|
+
return f.mode === match.slice("mode:".length);
|
|
48
|
+
return DOOR_FACT[match]?.(f) ?? false;
|
|
49
|
+
}
|
|
50
|
+
export function classifyMerge(facts, systemTomlPath) {
|
|
51
|
+
const dc = decisionClasses();
|
|
52
|
+
const fired = [];
|
|
53
|
+
for (const c of dc.human_required ?? []) {
|
|
54
|
+
if (HUMAN_FACT[c.id]?.(facts))
|
|
55
|
+
fired.push({ match: c.id, class: "one-way", source: "human_required" });
|
|
56
|
+
}
|
|
57
|
+
for (const c of dc.system_scope ?? []) {
|
|
58
|
+
// system_scope conditions are described in prose; only infra-apply is fact-checkable here
|
|
59
|
+
if (c.id === "infra-apply" && facts.gateExternal) {
|
|
60
|
+
fired.push({
|
|
61
|
+
match: c.id,
|
|
62
|
+
class: c.door ?? "one-way",
|
|
63
|
+
source: "system_scope",
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
for (const dcls of systemDoorClasses(systemTomlPath)) {
|
|
68
|
+
if (doorMatches(dcls.match, facts)) {
|
|
69
|
+
fired.push({
|
|
70
|
+
match: dcls.match,
|
|
71
|
+
class: dcls.class,
|
|
72
|
+
source: "door_class",
|
|
73
|
+
...(dcls.until ? { until: dcls.until } : {}),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
if (facts.mode === "propose-only") {
|
|
78
|
+
fired.push({
|
|
79
|
+
match: `mode:propose-only`,
|
|
80
|
+
class: "propose-only",
|
|
81
|
+
source: "component_mode",
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
const requiresHuman = fired.some((f) => f.class === "one-way" || f.class === "propose-only");
|
|
85
|
+
return {
|
|
86
|
+
fired,
|
|
87
|
+
requiresHuman,
|
|
88
|
+
antiRubberStamp: {
|
|
89
|
+
approvalRateThreshold: dc.anti_rubber_stamp?.approval_rate_threshold ?? 0.95,
|
|
90
|
+
minDeliberationSeconds: dc.anti_rubber_stamp?.min_deliberation_seconds ?? 30,
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=classify.js.map
|