@hone-ai/cli 1.19.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/hone-mcp.js +9 -0
- package/hone-cli.js +1198 -50
- package/lib/agent-eval-judge.js +60 -0
- package/lib/agent-eval-probes-adversarial.js +45 -0
- package/lib/agent-eval-probes-boundary.js +0 -0
- package/lib/agent-eval-probes-faithfulness.js +59 -0
- package/lib/agent-eval-probes-safety.js +28 -0
- package/lib/agent-executor.js +139 -0
- package/lib/bundle-paths.js +141 -0
- package/lib/emit-pr.js +167 -0
- package/lib/eval-graders.js +98 -1
- package/lib/judge-provider.js +63 -0
- package/lib/materialize-diff.js +345 -0
- package/lib/mcp-tools.js +310 -0
- package/lib/patch-apply.js +108 -0
- package/lib/pipeline-config.js +220 -0
- package/lib/pipeline-status.js +16 -2
- package/lib/verify-patch.js +78 -0
- package/lib/verify-pr.js +141 -0
- package/mcp-server.js +67 -0
- package/package.json +8 -5
package/hone-cli.js
CHANGED
|
@@ -144,6 +144,130 @@ process.on('exit', () => {
|
|
|
144
144
|
}
|
|
145
145
|
});
|
|
146
146
|
|
|
147
|
+
// ── AGENT-EVAL command (HC-COMM-011-followup-1) ──────────────────────────────────
|
|
148
|
+
// Runs deterministic adversarial probes against a configured agent target. $0 —
|
|
149
|
+
// no LLM judge. A probe passes iff the executor ran AND every grader passes
|
|
150
|
+
// (canary absent AND refusal present). Exits non-zero if any probe fails.
|
|
151
|
+
program
|
|
152
|
+
.command('agent-eval')
|
|
153
|
+
.description('Run agent-eval probes against configured agent targets (HC-COMM-011-followup-1)')
|
|
154
|
+
.option('--category <category>', 'Probe category (adversarial|faithfulness|safety|boundary)', 'adversarial')
|
|
155
|
+
.option('--target <name>', 'Run against a specific agent target (default: all configured)')
|
|
156
|
+
.option('--output <format>', 'Output format (text|json)', 'text')
|
|
157
|
+
.option('--judge', 'Add the free NLI judge layer (faithfulness only): claims are checked for entailment against the probe context')
|
|
158
|
+
.option('--provider <name>', 'Judge provider: gh-models (free, default) | claude (paid Sonnet)', 'gh-models')
|
|
159
|
+
.option('--report', 'Report the agent-eval verdict to the run (advisory) — needs --workflow-id')
|
|
160
|
+
.option('--workflow-id <id>', 'The run id to report the verdict against (for --report)')
|
|
161
|
+
.action(async (opts) => {
|
|
162
|
+
const { readAgentEvalConfig } = require('./lib/pipeline-config');
|
|
163
|
+
const { runAgent } = require('./lib/agent-executor');
|
|
164
|
+
const { runCheck } = require('./lib/eval-graders');
|
|
165
|
+
const config = readAgentEvalConfig(process.cwd());
|
|
166
|
+
|
|
167
|
+
if (!config.targets || config.targets.length === 0) {
|
|
168
|
+
console.log('No agent targets configured in .pipeline-config.yml (agent_eval.targets)');
|
|
169
|
+
process.exit(0);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const PROBE_PACKS = {
|
|
173
|
+
adversarial: './lib/agent-eval-probes-adversarial',
|
|
174
|
+
faithfulness: './lib/agent-eval-probes-faithfulness',
|
|
175
|
+
safety: './lib/agent-eval-probes-safety',
|
|
176
|
+
boundary: './lib/agent-eval-probes-boundary',
|
|
177
|
+
};
|
|
178
|
+
let probes;
|
|
179
|
+
if (PROBE_PACKS[opts.category]) {
|
|
180
|
+
probes = require(PROBE_PACKS[opts.category]);
|
|
181
|
+
} else {
|
|
182
|
+
console.error(`Unknown probe category: ${opts.category} (supported: ${Object.keys(PROBE_PACKS).join(', ')})`);
|
|
183
|
+
process.exit(1);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const targets = opts.target
|
|
187
|
+
? config.targets.filter((t) => t.name === opts.target)
|
|
188
|
+
: config.targets;
|
|
189
|
+
if (targets.length === 0) {
|
|
190
|
+
console.error(`No agent target named "${opts.target}" in config`);
|
|
191
|
+
process.exit(1);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Optional free NLI judge layer (faithfulness only). Build the provider callLLM
|
|
195
|
+
// ONCE — a missing credential is a hard error, not a silent skip.
|
|
196
|
+
let judgeCall = null;
|
|
197
|
+
if (opts.judge) {
|
|
198
|
+
if (opts.category !== 'faithfulness') {
|
|
199
|
+
console.error(`--judge applies to --category faithfulness only (got: ${opts.category})`);
|
|
200
|
+
process.exit(1);
|
|
201
|
+
}
|
|
202
|
+
const { buildJudgeCallLLM } = require('./lib/judge-provider');
|
|
203
|
+
const built = buildJudgeCallLLM(opts.provider, { axios });
|
|
204
|
+
if (built.error) { console.error(built.error); process.exit(1); }
|
|
205
|
+
judgeCall = built.callLLM;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const results = [];
|
|
209
|
+
let anyFailed = false;
|
|
210
|
+
for (const target of targets) {
|
|
211
|
+
for (const probe of probes) {
|
|
212
|
+
const run = await runAgent(target, probe.input);
|
|
213
|
+
const output = run && typeof run.output === 'string' ? run.output : '';
|
|
214
|
+
const graders = probe.graders.map((g) => runCheck(output, g));
|
|
215
|
+
let passed = !run.error && graders.every((r) => r.passed);
|
|
216
|
+
|
|
217
|
+
// Judge layer: only for grounded probes (those with a context to entail against).
|
|
218
|
+
let judge = null;
|
|
219
|
+
if (judgeCall && probe.context) {
|
|
220
|
+
const { judgeFaithfulness } = require('./lib/agent-eval-judge');
|
|
221
|
+
judge = await judgeFaithfulness({ output, context: probe.context, callLLM: judgeCall });
|
|
222
|
+
passed = passed && judge.passed;
|
|
223
|
+
}
|
|
224
|
+
if (!passed) anyFailed = true;
|
|
225
|
+
results.push({ target: target.name, probe: probe.id, passed, error: run.error || null, graders, judge });
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (opts.output === 'json') {
|
|
230
|
+
console.log(JSON.stringify({ category: opts.category, passed: !anyFailed, results }, null, 2));
|
|
231
|
+
} else {
|
|
232
|
+
for (const r of results) {
|
|
233
|
+
const mark = r.passed ? '✓' : '✗';
|
|
234
|
+
console.log(` ${mark} ${r.target} / ${r.probe}${r.error ? ` (executor: ${r.error})` : ''}`);
|
|
235
|
+
if (!r.passed) {
|
|
236
|
+
for (const g of r.graders.filter((x) => !x.passed)) console.log(` - ${g.type}: ${g.detail}`);
|
|
237
|
+
if (r.judge && !r.judge.passed) console.log(` - judge: ${r.judge.detail}`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
const pass = results.filter((r) => r.passed).length;
|
|
241
|
+
console.log(`\n${pass}/${results.length} probes passed (category: ${opts.category})`);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Closed loop (HC-COMM-011-followup-5): report the verdict to the run. agent-eval
|
|
245
|
+
// binds to step_4 (a prompt change is a step_4 diff) and is ADVISORY — it records
|
|
246
|
+
// into config.verifications[], never auto-gates. Best-effort; never changes the
|
|
247
|
+
// local exit code.
|
|
248
|
+
if (opts.report) {
|
|
249
|
+
if (!opts.workflowId) {
|
|
250
|
+
if (opts.output !== 'json') console.warn(' ⚠ --report needs --workflow-id <run id> to address the run; skipping report.');
|
|
251
|
+
} else {
|
|
252
|
+
try {
|
|
253
|
+
const client = api(getConfig());
|
|
254
|
+
await client.post(`/orchestrate/${opts.workflowId}/verdict`, {
|
|
255
|
+
source: 'agent-eval',
|
|
256
|
+
stepKey: 'step_4',
|
|
257
|
+
verdict: anyFailed ? 'fail' : 'pass',
|
|
258
|
+
exitCode: anyFailed ? 1 : 0,
|
|
259
|
+
detail: `agent-eval ${opts.category}: ${results.filter((r) => r.passed).length}/${results.length} probes passed`,
|
|
260
|
+
});
|
|
261
|
+
if (opts.output !== 'json') console.log(` → reported agent-eval verdict '${anyFailed ? 'fail' : 'pass'}' to run ${opts.workflowId} (advisory)`);
|
|
262
|
+
} catch (e) {
|
|
263
|
+
if (opts.output !== 'json') console.warn(` ⚠ verdict report failed (non-fatal): ${e.response?.data?.error || e.message}`);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
process.exit(anyFailed ? 1 : 0);
|
|
269
|
+
});
|
|
270
|
+
|
|
147
271
|
// ── SETUP-LOCAL-CI command (HC-101-followup-3) ─────────────────────────────────
|
|
148
272
|
//
|
|
149
273
|
// Scaffolds HC-101's local-CI assets into the adopter's repo so they can
|
|
@@ -1777,6 +1901,207 @@ program
|
|
|
1777
1901
|
}
|
|
1778
1902
|
});
|
|
1779
1903
|
|
|
1904
|
+
// ── BYO-KEY command (HC-COMM-013 / A2) ───────────────────────────────────────
|
|
1905
|
+
// Self-serve management of your org's Anthropic API key for server mode. The
|
|
1906
|
+
// key is read from stdin (piped) or a HIDDEN prompt — NEVER accepted on argv,
|
|
1907
|
+
// which would leak it to shell history (~/.zsh_history), `ps`, and CI logs.
|
|
1908
|
+
|
|
1909
|
+
// Read a secret without echoing it. Piped stdin → read all of it (CI/secret-
|
|
1910
|
+
// manager injection). Interactive TTY → a muted readline prompt.
|
|
1911
|
+
async function _readSecret(promptText) {
|
|
1912
|
+
if (!process.stdin.isTTY) {
|
|
1913
|
+
const chunks = [];
|
|
1914
|
+
for await (const chunk of process.stdin) chunks.push(chunk);
|
|
1915
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
1916
|
+
}
|
|
1917
|
+
const readline = await import('readline');
|
|
1918
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
1919
|
+
return new Promise((resolve) => {
|
|
1920
|
+
process.stdout.write(promptText);
|
|
1921
|
+
// Mute echo of typed characters — the prompt is already written.
|
|
1922
|
+
rl._writeToOutput = () => {};
|
|
1923
|
+
rl.question('', (answer) => {
|
|
1924
|
+
rl.close();
|
|
1925
|
+
process.stdout.write('\n');
|
|
1926
|
+
resolve(answer);
|
|
1927
|
+
});
|
|
1928
|
+
});
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
function _handleByoKeyError(e) {
|
|
1932
|
+
const status = e.response?.status;
|
|
1933
|
+
if (status === 401) {
|
|
1934
|
+
console.error('Not authenticated. Run: hone init --token <token>');
|
|
1935
|
+
} else if (status === 429) {
|
|
1936
|
+
const retry = e.response?.data?.retryAfter;
|
|
1937
|
+
console.error(`Rate limited — too many key writes.${retry ? ` Retry after ${retry}s.` : ''}`);
|
|
1938
|
+
} else if (status === 503) {
|
|
1939
|
+
console.error(`Server cannot store the key right now: ${e.response?.data?.error || 'unavailable'}`);
|
|
1940
|
+
} else if (e.response?.data?.error) {
|
|
1941
|
+
console.error(`Failed: ${e.response.data.error}`);
|
|
1942
|
+
} else {
|
|
1943
|
+
console.error(`Failed: ${e.message}`);
|
|
1944
|
+
}
|
|
1945
|
+
process.exit(1);
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
const MAX_BYO_KEY_BYTES_CLI = 1024;
|
|
1949
|
+
|
|
1950
|
+
const byoKeyCmd = program
|
|
1951
|
+
.command('byo-key')
|
|
1952
|
+
.description("Manage your org's BYO Anthropic API key (server mode)");
|
|
1953
|
+
|
|
1954
|
+
byoKeyCmd
|
|
1955
|
+
.command('set')
|
|
1956
|
+
.description('Set or rotate your key — reads from stdin or a hidden prompt (never pass the key on the command line)')
|
|
1957
|
+
.option('--key-file <path>', 'Read the key from a file (e.g. a 0600 secrets file) instead of stdin/prompt')
|
|
1958
|
+
.action(async (opts) => {
|
|
1959
|
+
const config = getConfig();
|
|
1960
|
+
const client = api(config);
|
|
1961
|
+
|
|
1962
|
+
let key;
|
|
1963
|
+
try {
|
|
1964
|
+
key = opts.keyFile
|
|
1965
|
+
? fs.readFileSync(opts.keyFile, 'utf8')
|
|
1966
|
+
: await _readSecret('Anthropic API key: ');
|
|
1967
|
+
} catch (e) {
|
|
1968
|
+
console.error(`Could not read key: ${e.message}`);
|
|
1969
|
+
process.exit(1);
|
|
1970
|
+
}
|
|
1971
|
+
key = (key || '').trim();
|
|
1972
|
+
|
|
1973
|
+
// Client-side convenience checks (server remains the authority).
|
|
1974
|
+
if (!key) {
|
|
1975
|
+
console.error('No key provided (empty input). Pipe it in or type it at the prompt.');
|
|
1976
|
+
process.exit(1);
|
|
1977
|
+
}
|
|
1978
|
+
if (Buffer.byteLength(key, 'utf8') > MAX_BYO_KEY_BYTES_CLI) {
|
|
1979
|
+
console.error(`Key too large (max ${MAX_BYO_KEY_BYTES_CLI} bytes). Did you paste the wrong thing?`);
|
|
1980
|
+
process.exit(1);
|
|
1981
|
+
}
|
|
1982
|
+
if (!key.startsWith('sk-ant-')) {
|
|
1983
|
+
// Warn-only (memo §8): Anthropic key formats can change; don't hard-block.
|
|
1984
|
+
console.error('Warning: key does not look like an Anthropic key (expected "sk-ant-" prefix). Sending anyway.');
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
try {
|
|
1988
|
+
const { data } = await client.put('/orgs/me/byo-key', { byo_anthropic_key: key });
|
|
1989
|
+
console.log(data.wasConfigured
|
|
1990
|
+
? 'BYO key rotated. Pipeline LLM calls now bill your own Anthropic API.'
|
|
1991
|
+
: 'BYO key set. Pipeline LLM calls now bill your own Anthropic API.');
|
|
1992
|
+
} catch (e) { _handleByoKeyError(e); }
|
|
1993
|
+
});
|
|
1994
|
+
|
|
1995
|
+
byoKeyCmd
|
|
1996
|
+
.command('clear')
|
|
1997
|
+
.description('Clear your key — your org reverts to the Hone default key')
|
|
1998
|
+
.action(async () => {
|
|
1999
|
+
const config = getConfig();
|
|
2000
|
+
const client = api(config);
|
|
2001
|
+
try {
|
|
2002
|
+
const { data } = await client.delete('/orgs/me/byo-key');
|
|
2003
|
+
console.log(data.wasConfigured
|
|
2004
|
+
? 'BYO key cleared. Your org reverts to the Hone default key.'
|
|
2005
|
+
: 'No BYO key was configured — nothing to clear.');
|
|
2006
|
+
} catch (e) { _handleByoKeyError(e); }
|
|
2007
|
+
});
|
|
2008
|
+
|
|
2009
|
+
byoKeyCmd
|
|
2010
|
+
.command('status')
|
|
2011
|
+
.description('Show whether a BYO key is configured (never prints the key)')
|
|
2012
|
+
.option('--format <fmt>', 'Output format: pretty | json', 'pretty')
|
|
2013
|
+
.action(async (opts) => {
|
|
2014
|
+
const config = getConfig();
|
|
2015
|
+
const client = api(config);
|
|
2016
|
+
try {
|
|
2017
|
+
const { data } = await client.get('/orgs/me/byo-key');
|
|
2018
|
+
if (opts.format === 'json') {
|
|
2019
|
+
console.log(JSON.stringify(data, null, 2));
|
|
2020
|
+
return;
|
|
2021
|
+
}
|
|
2022
|
+
console.log(`BYO key configured: ${data.configured ? 'yes' : 'no'}`);
|
|
2023
|
+
if (data.lastWrite) {
|
|
2024
|
+
const when = String(data.lastWrite.at).split('T')[0];
|
|
2025
|
+
console.log(`Last change: ${data.lastWrite.action} by ${data.lastWrite.actorType} on ${when}`);
|
|
2026
|
+
}
|
|
2027
|
+
if (data.configured && data.envelopeOk === false) {
|
|
2028
|
+
console.log('Warning: stored value is not in the expected encrypted envelope shape — contact support.');
|
|
2029
|
+
}
|
|
2030
|
+
// HC-COMM-014: fold a one-line trial summary in (non-fatal if unavailable).
|
|
2031
|
+
try {
|
|
2032
|
+
const { data: plan } = await client.get('/orgs/me/plan');
|
|
2033
|
+
if (plan.mode === 'trial') {
|
|
2034
|
+
console.log(`Trial: active — ${plan.trial.daysLeft}d left, $${Number(plan.trial.creditRemaining).toFixed(2)} credit`);
|
|
2035
|
+
} else if (plan.mode === 'expired') {
|
|
2036
|
+
console.log('Trial: ended (set a key or upgrade)');
|
|
2037
|
+
}
|
|
2038
|
+
} catch { /* trial line is a nicety — never block byo-key status */ }
|
|
2039
|
+
console.log('Tip: run `hone byo-key check` to verify the key still authenticates with Anthropic.');
|
|
2040
|
+
} catch (e) { _handleByoKeyError(e); }
|
|
2041
|
+
});
|
|
2042
|
+
|
|
2043
|
+
byoKeyCmd
|
|
2044
|
+
.command('check')
|
|
2045
|
+
.description('Verify the configured key still authenticates with Anthropic (zero-token live probe; never prints the key)')
|
|
2046
|
+
.action(async () => {
|
|
2047
|
+
const config = getConfig();
|
|
2048
|
+
const client = api(config);
|
|
2049
|
+
try {
|
|
2050
|
+
const { data } = await client.post('/orgs/me/byo-key/check');
|
|
2051
|
+
if (data.ok) {
|
|
2052
|
+
console.log('BYO key is valid — Anthropic accepted it.');
|
|
2053
|
+
return;
|
|
2054
|
+
}
|
|
2055
|
+
const msg = {
|
|
2056
|
+
unauthorized: 'BYO key was REJECTED by Anthropic (401/403). Rotate it with `hone byo-key set`.',
|
|
2057
|
+
unreachable: 'Could not reach Anthropic to check the key — try again shortly (key not changed).',
|
|
2058
|
+
unexpected: 'Anthropic returned an unexpected status checking the key — try again shortly.',
|
|
2059
|
+
}[data.reason] || `Key check failed: ${data.reason}`;
|
|
2060
|
+
console.error(msg);
|
|
2061
|
+
process.exit(1);
|
|
2062
|
+
} catch (e) { _handleByoKeyError(e); }
|
|
2063
|
+
});
|
|
2064
|
+
|
|
2065
|
+
// ── TRIAL command (HC-COMM-014 / A3) ─────────────────────────────────────────
|
|
2066
|
+
const trialCmd = program
|
|
2067
|
+
.command('trial')
|
|
2068
|
+
.description("View or start your org's Hone-funded trial (server mode)");
|
|
2069
|
+
|
|
2070
|
+
trialCmd
|
|
2071
|
+
.command('status')
|
|
2072
|
+
.description('Show trial mode, days left, and credit remaining')
|
|
2073
|
+
.option('--format <fmt>', 'Output format: pretty | json', 'pretty')
|
|
2074
|
+
.action(async (opts) => {
|
|
2075
|
+
const config = getConfig();
|
|
2076
|
+
const client = api(config);
|
|
2077
|
+
try {
|
|
2078
|
+
const { data } = await client.get('/orgs/me/plan');
|
|
2079
|
+
if (opts.format === 'json') { console.log(JSON.stringify(data, null, 2)); return; }
|
|
2080
|
+
const t = data.trial || {};
|
|
2081
|
+
console.log(`Plan mode: ${data.mode}`);
|
|
2082
|
+
console.log(`Trial status: ${t.status}`);
|
|
2083
|
+
if (t.status === 'active') {
|
|
2084
|
+
console.log(`Days left: ${t.daysLeft}`);
|
|
2085
|
+
console.log(`Credit: $${Number(t.creditUsdUsed || 0).toFixed(2)} used / $${Number(t.creditUsdCap).toFixed(2)} cap ($${Number(t.creditRemaining).toFixed(2)} left)`);
|
|
2086
|
+
}
|
|
2087
|
+
if (data.mode === 'expired') {
|
|
2088
|
+
console.log('Trial ended — set your own key (`hone byo-key set`) or upgrade to continue.');
|
|
2089
|
+
}
|
|
2090
|
+
} catch (e) { _handleByoKeyError(e); }
|
|
2091
|
+
});
|
|
2092
|
+
|
|
2093
|
+
trialCmd
|
|
2094
|
+
.command('start')
|
|
2095
|
+
.description('Start your trial now (idempotent — no-op if already started or ended)')
|
|
2096
|
+
.action(async () => {
|
|
2097
|
+
const config = getConfig();
|
|
2098
|
+
const client = api(config);
|
|
2099
|
+
try {
|
|
2100
|
+
const { data } = await client.post('/orgs/me/trial/start');
|
|
2101
|
+
console.log(data.started ? `Trial started — status: ${data.status}.` : `No-op — ${data.note}`);
|
|
2102
|
+
} catch (e) { _handleByoKeyError(e); }
|
|
2103
|
+
});
|
|
2104
|
+
|
|
1780
2105
|
// ── ADMIN-USAGE command ──────────────────────────────────────────────────────
|
|
1781
2106
|
program
|
|
1782
2107
|
.command('admin-usage')
|
|
@@ -4598,7 +4923,8 @@ program
|
|
|
4598
4923
|
.option('--contracts', 'Run contract validation between pipeline agents')
|
|
4599
4924
|
.option('--snapshot', 'Save current eval + contract results as regression baseline')
|
|
4600
4925
|
.option('--regression', 'Compare current results against saved baseline (detect drift)')
|
|
4601
|
-
.option('--judge', 'Run LLM-as-judge scenarios (
|
|
4926
|
+
.option('--judge', 'Run LLM-as-judge scenarios (default provider: free gh-models via GITHUB_TOKEN)')
|
|
4927
|
+
.option('--provider <name>', 'LLM-judge provider: gh-models (free, default) | claude (paid Sonnet)', 'gh-models')
|
|
4602
4928
|
.option('--evidence-mode <mode>', 'HC-RC-001 editor-LLM evidence transfer: "local" writes .hone/eval-evidence.json (signed); "off" disables (default)')
|
|
4603
4929
|
.action(async (opts) => {
|
|
4604
4930
|
const path = require('path');
|
|
@@ -4666,12 +4992,8 @@ program
|
|
|
4666
4992
|
const { loadScenarios, formatResults } = require('./lib/eval-runner');
|
|
4667
4993
|
const { runJudgeScenario } = require('./lib/eval-llm-judge');
|
|
4668
4994
|
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
console.error('ANTHROPIC_API_KEY required for --judge mode. Set: export ANTHROPIC_API_KEY=sk-ant-...');
|
|
4672
|
-
process.exit(1);
|
|
4673
|
-
}
|
|
4674
|
-
|
|
4995
|
+
// Load + filter FIRST — if there are no judge scenarios, exit cleanly
|
|
4996
|
+
// without demanding any API key / token.
|
|
4675
4997
|
const scenarios = loadScenarios({
|
|
4676
4998
|
evalDir, agent: opts.agent, tag: opts.tag, scenarioId: opts.scenario,
|
|
4677
4999
|
readFile: (p) => fs.readFileSync(p, 'utf8'),
|
|
@@ -4685,48 +5007,21 @@ program
|
|
|
4685
5007
|
process.exit(0);
|
|
4686
5008
|
}
|
|
4687
5009
|
|
|
4688
|
-
// LLM
|
|
4689
|
-
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
4697
|
-
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
model: 'claude-sonnet-5',
|
|
4704
|
-
// Sonnet 5 runs ADAPTIVE THINKING when `thinking` is omitted, unlike
|
|
4705
|
-
// the dated Sonnet 4 model this replaced. Two consequences if left
|
|
4706
|
-
// default, both silent: content[0] becomes a `thinking` block (so a
|
|
4707
|
-
// [0].text read returns undefined and every judge criterion fails to
|
|
4708
|
-
// parse), and thinking tokens share max_tokens with the answer.
|
|
4709
|
-
// The judge returns a short structured verdict, so keep it off and
|
|
4710
|
-
// preserve the previous cost/latency profile.
|
|
4711
|
-
thinking: { type: 'disabled' },
|
|
4712
|
-
max_tokens: 2048,
|
|
4713
|
-
system: systemPrompt,
|
|
4714
|
-
messages: [{ role: 'user', content: userPrompt }],
|
|
4715
|
-
}, {
|
|
4716
|
-
headers: {
|
|
4717
|
-
'x-api-key': apiKey,
|
|
4718
|
-
'anthropic-version': '2023-06-01',
|
|
4719
|
-
'content-type': 'application/json',
|
|
4720
|
-
},
|
|
4721
|
-
timeout: 60000,
|
|
4722
|
-
});
|
|
4723
|
-
// Select the first TEXT block rather than content[0]: any future model
|
|
4724
|
-
// or config that emits a leading thinking block must not silently
|
|
4725
|
-
// degrade every scenario to "could not parse response".
|
|
4726
|
-
return (data.content || []).find(b => b?.type === 'text')?.text || '';
|
|
4727
|
-
}
|
|
4728
|
-
|
|
4729
|
-
console.log(`Running ${judgeScenarios.length} LLM-judge scenario(s)...`);
|
|
5010
|
+
// HC-019n-followup-33 (pipeline-standards G6): the LLM-judge lane must have a
|
|
5011
|
+
// ZERO-COST path (feedback_pipeline_llm_cost_reduction — every LLM-cost gate
|
|
5012
|
+
// needs a free/near-free route so adopters aren't double-billed). DEFAULT to
|
|
5013
|
+
// GH Models (free inference via GITHUB_TOKEN); paid Claude Sonnet is opt-in
|
|
5014
|
+
// via `--provider claude`. Mirrors `hone skill-eval`'s provider convention.
|
|
5015
|
+
// HC-COMM-011-followup-3: provider wiring extracted to lib/judge-provider.js
|
|
5016
|
+
// (shared with `hone agent-eval --judge`). Same zero-cost default (gh-models)
|
|
5017
|
+
// + Sonnet-not-Opus paid path; the retired dated Sonnet id must not appear
|
|
5018
|
+
// (pinned by opus-4-8-upgrade.test.js).
|
|
5019
|
+
const { buildJudgeCallLLM } = require('./lib/judge-provider');
|
|
5020
|
+
const built = buildJudgeCallLLM(opts.provider, { axios });
|
|
5021
|
+
if (built.error) { console.error(built.error); process.exit(1); }
|
|
5022
|
+
const callLLM = built.callLLM;
|
|
5023
|
+
|
|
5024
|
+
console.log(`Running ${judgeScenarios.length} LLM-judge scenario(s) via ${built.provider}...`);
|
|
4730
5025
|
console.log('');
|
|
4731
5026
|
|
|
4732
5027
|
const results = [];
|
|
@@ -5027,6 +5322,8 @@ program
|
|
|
5027
5322
|
.option('--format <fmt>', 'Output format: pretty or json', 'pretty')
|
|
5028
5323
|
.option('--poll-interval <s>', 'Poll interval in seconds', '5')
|
|
5029
5324
|
.option('--include-paths <paths>', 'Comma-separated list of source file paths to bundle into the codebase context (HC-019n-followup-12). Augments auto-detected paths from the issue body.')
|
|
5325
|
+
.option('--e2e-specs', 'HC-019n-followup-28: force step_3b (Playwright spec generation) ON for this run (overrides e2e.spec_generation config)')
|
|
5326
|
+
.option('--skip-e2e-specs', 'HC-019n-followup-28: force step_3b (Playwright spec generation) OFF for this run (overrides e2e.spec_generation config)')
|
|
5030
5327
|
.action(async (storyIdOrRunId, opts) => {
|
|
5031
5328
|
const config = getConfig();
|
|
5032
5329
|
const client = api(config);
|
|
@@ -5166,6 +5463,41 @@ program
|
|
|
5166
5463
|
console.warn(' → proceeding without story_description; step_0 may produce placeholder output');
|
|
5167
5464
|
console.warn(` ⚠ architect-config lookup will use '${storyIdOrRunId}' (likely silent miss)`);
|
|
5168
5465
|
}
|
|
5466
|
+
} else {
|
|
5467
|
+
// HC-019n-followup-18: roadmap-sourced story. Only the SERVER knows what
|
|
5468
|
+
// the story says (server/seeds/master-roadmap.md is a Hone asset, absent
|
|
5469
|
+
// from adopter repos) and only the CLI can read the adopter's disk — so
|
|
5470
|
+
// ask the server what the description references, then read those files
|
|
5471
|
+
// here. See .github/pipeline/HC-019n-followup-18/architect.md §1 for why
|
|
5472
|
+
// this beats the CLI reading a local roadmap (hardcodes a Hone-internal
|
|
5473
|
+
// path into the adopter CLI, and fails silently anywhere else).
|
|
5474
|
+
//
|
|
5475
|
+
// Advisory, never fatal: a story with no resolvable context still runs,
|
|
5476
|
+
// it just runs blind — and loudly, via the followup-17 warning below.
|
|
5477
|
+
try {
|
|
5478
|
+
const ctxClient = api(config);
|
|
5479
|
+
const { data } = await ctxClient.get(
|
|
5480
|
+
`/stories/${encodeURIComponent(storyIdOrRunId)}/context`
|
|
5481
|
+
);
|
|
5482
|
+
if (data && data.description) {
|
|
5483
|
+
orchestrateConfig.story_description = data.description;
|
|
5484
|
+
issueBodyForFiles = data.description;
|
|
5485
|
+
const n = Array.isArray(data.referencedPaths) ? data.referencedPaths.length : 0;
|
|
5486
|
+
console.log(
|
|
5487
|
+
` → resolved story context from roadmap (${data.description.length} chars, ` +
|
|
5488
|
+
`${n} referenced path${n === 1 ? '' : 's'})`
|
|
5489
|
+
);
|
|
5490
|
+
}
|
|
5491
|
+
} catch (e) {
|
|
5492
|
+
const status = e && e.response && e.response.status;
|
|
5493
|
+
if (status === 404) {
|
|
5494
|
+
console.warn(` ⚠ no roadmap row found for '${storyIdOrRunId}'`);
|
|
5495
|
+
} else {
|
|
5496
|
+
const msg = (e && e.message) ? e.message.split('\n')[0] : String(e);
|
|
5497
|
+
console.warn(` ⚠ could not fetch story context: ${msg}`);
|
|
5498
|
+
}
|
|
5499
|
+
console.warn(' → proceeding without story context; the warning below applies');
|
|
5500
|
+
}
|
|
5169
5501
|
}
|
|
5170
5502
|
|
|
5171
5503
|
// HC-019n-followup-12: auto-bundle codebase files referenced in the
|
|
@@ -5207,6 +5539,45 @@ program
|
|
|
5207
5539
|
candidates.add(p);
|
|
5208
5540
|
}
|
|
5209
5541
|
}
|
|
5542
|
+
// HC-019n-followup-26: FILE_PATH_RE above requires a directory segment,
|
|
5543
|
+
// so a story that names a file BARE (`derive-worker.js:787` in prose)
|
|
5544
|
+
// extracts 0 paths and step_4 runs blind — and, blind, hallucinates a
|
|
5545
|
+
// file at a non-existent path (2026-08-30 e2e demo). Catch bare refs too
|
|
5546
|
+
// and resolve each against the repo's own file list: bundle ONLY on a
|
|
5547
|
+
// unique basename match; never guess an ambiguous or absent one.
|
|
5548
|
+
// HC-019n-followup-27: candidate path → referenced line number, so the
|
|
5549
|
+
// bundler can WINDOW a too-big file around its edit site instead of
|
|
5550
|
+
// head-truncating past it (the demo's line 787 sits at char 37K).
|
|
5551
|
+
const candidateLines = new Map();
|
|
5552
|
+
if (issueBodyForFiles) {
|
|
5553
|
+
const { extractBareRefs, resolveBareName, alreadyCovered } = require('./lib/bundle-paths');
|
|
5554
|
+
const { gitEnv } = require('./lib/git-env');
|
|
5555
|
+
const bareRefs = extractBareRefs(issueBodyForFiles);
|
|
5556
|
+
if (bareRefs.length > 0) {
|
|
5557
|
+
let repoFiles = null;
|
|
5558
|
+
const listRepoFiles = () => {
|
|
5559
|
+
if (repoFiles) return repoFiles;
|
|
5560
|
+
try {
|
|
5561
|
+
repoFiles = execSync('git ls-files', {
|
|
5562
|
+
cwd: process.cwd(), env: gitEnv(), encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
|
|
5563
|
+
}).split('\n').filter(Boolean);
|
|
5564
|
+
} catch { repoFiles = []; }
|
|
5565
|
+
return repoFiles;
|
|
5566
|
+
};
|
|
5567
|
+
for (const { name, line } of bareRefs) {
|
|
5568
|
+
if (alreadyCovered(name, candidates)) continue;
|
|
5569
|
+
const r = resolveBareName(name, listRepoFiles());
|
|
5570
|
+
if (r.status === 'unique') {
|
|
5571
|
+
candidates.add(r.path);
|
|
5572
|
+
if (line && !candidateLines.has(r.path)) candidateLines.set(r.path, line);
|
|
5573
|
+
console.log(` → resolved bare ref '${name}${line ? ':' + line : ''}' → ${r.path}`);
|
|
5574
|
+
} else if (r.status === 'ambiguous') {
|
|
5575
|
+
console.warn(` ⚠ '${name}' is ambiguous (${r.matches.length} files) — not bundling; use --include-paths to disambiguate`);
|
|
5576
|
+
}
|
|
5577
|
+
// status === 'none' → not every filename in prose is a repo file; skip quietly
|
|
5578
|
+
}
|
|
5579
|
+
}
|
|
5580
|
+
}
|
|
5210
5581
|
// HC-019n-followup-13d: bump caps. Pre-fix MAX_FILES=10 /
|
|
5211
5582
|
// MAX_TOTAL_CHARS=50_000 / MAX_PER_FILE_CHARS=10_000 was overly
|
|
5212
5583
|
// conservative — OptionsFlow #96 hit 49,783/50,000 with 6 files
|
|
@@ -5230,7 +5601,23 @@ program
|
|
|
5230
5601
|
content = fs.readFileSync(abs, 'utf8');
|
|
5231
5602
|
} catch { continue; }
|
|
5232
5603
|
if (content.length > MAX_PER_FILE_CHARS) {
|
|
5233
|
-
|
|
5604
|
+
const line = candidateLines.get(candidate);
|
|
5605
|
+
if (line) {
|
|
5606
|
+
// HC-019n-followup-27: window around the referenced line so step_4
|
|
5607
|
+
// SEES the edit site (head-truncation would hide a site past the
|
|
5608
|
+
// cut) and can write a region-edit anchor. The window marker tells
|
|
5609
|
+
// step_4 to emit an `edit:` block, not a whole-file block.
|
|
5610
|
+
const { windowAroundLine } = require('./lib/bundle-paths');
|
|
5611
|
+
const w = windowAroundLine(content, line, Math.floor((MAX_PER_FILE_CHARS - 800) / 2));
|
|
5612
|
+
content =
|
|
5613
|
+
`# [HC-019n-followup-27: windowed excerpt, lines ${w.startLine}-${w.endLine} of ${w.totalLines} ` +
|
|
5614
|
+
`(centered on referenced line ${line}); you were NOT shown the whole file — ` +
|
|
5615
|
+
`emit an \`edit:\` block with a unique OLD anchor from this excerpt, NOT a whole-file \`file:\` block]\n` +
|
|
5616
|
+
w.excerpt +
|
|
5617
|
+
`\n# [end windowed excerpt of ${candidate}]`;
|
|
5618
|
+
} else {
|
|
5619
|
+
content = content.slice(0, MAX_PER_FILE_CHARS) + `\n\n# [HC-019n-followup-12: truncated at ${MAX_PER_FILE_CHARS} chars; full file is ${content.length} chars]`;
|
|
5620
|
+
}
|
|
5234
5621
|
}
|
|
5235
5622
|
if (totalChars + content.length > MAX_TOTAL_CHARS) break;
|
|
5236
5623
|
totalChars += content.length;
|
|
@@ -5246,6 +5633,33 @@ program
|
|
|
5246
5633
|
}
|
|
5247
5634
|
}
|
|
5248
5635
|
|
|
5636
|
+
// HC-019n-followup-17: warn when the code-aware steps will run BLIND.
|
|
5637
|
+
//
|
|
5638
|
+
// Auto-detection reads file paths out of `issueBodyForFiles`, which is only
|
|
5639
|
+
// populated on the numeric-GitHub-issue branch above. A roadmap-sourced id
|
|
5640
|
+
// (HC-CI-005, SC-002-followup-2, ...) never enters that branch, so
|
|
5641
|
+
// `codebaseFiles` stays unset and step_1 / step_4 / step_5 / step_5d /
|
|
5642
|
+
// step_5e all run with no source to read — the exact "confidently wrong"
|
|
5643
|
+
// failure HC-019n-followup-12 was built to prevent, still open for roadmap
|
|
5644
|
+
// ids. Observed 2026-08-29: run 61fddfb2 wrote 8,444 chars of code for
|
|
5645
|
+
// HC-CI-005 having never seen the files that story names.
|
|
5646
|
+
//
|
|
5647
|
+
// A fix is structurally awkward: only the CLI can read adopter files, and
|
|
5648
|
+
// only the SERVER knows the roadmap description (server/seeds/master-
|
|
5649
|
+
// roadmap.md is a Hone asset, absent from adopter repos). Closing it needs
|
|
5650
|
+
// the CLI to fetch the resolved description before queueing — a round trip
|
|
5651
|
+
// that belongs in its own story. Until then the gap is at least LOUD
|
|
5652
|
+
// instead of silent, which is the part that actually bit.
|
|
5653
|
+
if (!orchestrateConfig.codebaseFiles) {
|
|
5654
|
+
console.warn(' ⚠ no codebase files bundled — step_1 / step_4 / step_5 / step_5d / step_5e');
|
|
5655
|
+
console.warn(' will run WITHOUT source context and may invent APIs that do not exist.');
|
|
5656
|
+
if (!/^\d+$/.test(storyIdOrRunId)) {
|
|
5657
|
+
console.warn(` '${storyIdOrRunId}' is not a GitHub issue number, so file paths cannot be`);
|
|
5658
|
+
console.warn(' auto-detected from an issue body.');
|
|
5659
|
+
}
|
|
5660
|
+
console.warn(' Fix: hone run-story <id> --include-paths path/one.js,path/two.js');
|
|
5661
|
+
}
|
|
5662
|
+
|
|
5249
5663
|
// HC-101-followup-2: pass the adopter's CI gate config to the
|
|
5250
5664
|
// orchestrator so step_5c can branch (github / local / both / none).
|
|
5251
5665
|
// Defaults to gate=github + local_command='make ci' when the config
|
|
@@ -5264,6 +5678,56 @@ program
|
|
|
5264
5678
|
orchestrateConfig.ci_local_command = 'make ci';
|
|
5265
5679
|
}
|
|
5266
5680
|
|
|
5681
|
+
// HC-019n-followup-28: resolve the E2E spec-generation mode (step_3b /
|
|
5682
|
+
// Playwright). Precedence mirrors ci.gate: per-run flag > .pipeline-config.yml
|
|
5683
|
+
// `e2e.spec_generation` > default `auto`. `auto` = the orchestrator runs
|
|
5684
|
+
// step_3b when the story requires an E2E plan (step_0 emitted `yes`); `never`
|
|
5685
|
+
// = step_3b is always skipped. The runtime "is E2E required?" decision stays
|
|
5686
|
+
// server-side (checkE2eRequirement reads step_0's output) — the CLI only
|
|
5687
|
+
// passes the MODE.
|
|
5688
|
+
try {
|
|
5689
|
+
const { readE2eSpecMode } = require('./lib/pipeline-config');
|
|
5690
|
+
let e2eMode, source;
|
|
5691
|
+
if (opts.e2eSpecs) { e2eMode = 'auto'; source = '--e2e-specs'; }
|
|
5692
|
+
else if (opts.skipE2eSpecs) { e2eMode = 'never'; source = '--skip-e2e-specs'; }
|
|
5693
|
+
else { e2eMode = readE2eSpecMode(process.cwd()); source = 'config/default'; }
|
|
5694
|
+
orchestrateConfig.e2e_spec_generation = e2eMode;
|
|
5695
|
+
// Enable step_3b (conditional) unless the adopter opted out. The row is
|
|
5696
|
+
// then created; the orchestrator still skips it at runtime for a story that
|
|
5697
|
+
// does not require an E2E plan (HC-019n-followup-28).
|
|
5698
|
+
if (e2eMode !== 'never') {
|
|
5699
|
+
const ec = new Set(orchestrateConfig.enableConditional || []);
|
|
5700
|
+
ec.add('step_3b');
|
|
5701
|
+
orchestrateConfig.enableConditional = [...ec];
|
|
5702
|
+
}
|
|
5703
|
+
console.log(` → E2E spec generation (step_3b): ${e2eMode} (source: ${source})`);
|
|
5704
|
+
} catch (e) {
|
|
5705
|
+
const msg = (e && e.message) ? e.message.split('\n')[0] : String(e);
|
|
5706
|
+
console.warn(` ⚠ e2e spec-mode read failed (non-fatal, defaulting to auto): ${msg}`);
|
|
5707
|
+
orchestrateConfig.e2e_spec_generation = 'auto';
|
|
5708
|
+
}
|
|
5709
|
+
|
|
5710
|
+
// HC-019n-followup-34: thread the closed-loop verification gate mode from
|
|
5711
|
+
// .pipeline-config.yml. Advisory (default) = the server RECORDS verify-patch/
|
|
5712
|
+
// verify-pr verdicts reported via --report; enforce (follow-up) = a red
|
|
5713
|
+
// verdict blocks. Default-safe: never fatal.
|
|
5714
|
+
try {
|
|
5715
|
+
const { readVerificationGateConfig, readVerificationGateSources } = require('./lib/pipeline-config');
|
|
5716
|
+
orchestrateConfig.verification_gate = readVerificationGateConfig(process.cwd());
|
|
5717
|
+
// HC-COMM-011-followup-6: which verdict sources may DRIVE the step_4 gate under
|
|
5718
|
+
// enforce (default ['verify-patch']). An adopter adds 'agent-eval' to let a
|
|
5719
|
+
// green agent-eval verdict auto-advance a prompt change.
|
|
5720
|
+
orchestrateConfig.verification_gate_sources = readVerificationGateSources(process.cwd());
|
|
5721
|
+
if (orchestrateConfig.verification_gate !== 'advisory') {
|
|
5722
|
+
console.log(` → verification gate: ${orchestrateConfig.verification_gate} (sources: ${orchestrateConfig.verification_gate_sources.join(', ')})`);
|
|
5723
|
+
}
|
|
5724
|
+
} catch (e) {
|
|
5725
|
+
const msg = (e && e.message) ? e.message.split('\n')[0] : String(e);
|
|
5726
|
+
console.warn(` ⚠ verification-gate read failed (non-fatal, defaulting to advisory): ${msg}`);
|
|
5727
|
+
orchestrateConfig.verification_gate = 'advisory';
|
|
5728
|
+
orchestrateConfig.verification_gate_sources = ['verify-patch'];
|
|
5729
|
+
}
|
|
5730
|
+
|
|
5267
5731
|
// HC-019b: read per-story architect flags from .github/EXECUTION_PLAN.yml
|
|
5268
5732
|
// and plumb them into workflow_runs.config. The orchestrator's
|
|
5269
5733
|
// validateStepPreConditions (server/src/services/workflow-dag.js:340)
|
|
@@ -6440,6 +6904,690 @@ showCmd
|
|
|
6440
6904
|
}
|
|
6441
6905
|
});
|
|
6442
6906
|
|
|
6907
|
+
// ── HC-019n-followup-20: check-patch (pipeline-recovery condition 5) ──────────
|
|
6908
|
+
// Fetch step_4's output, extract its diff, and run `git apply --check` against
|
|
6909
|
+
// the real tree. The server can only verify the output LOOKS like a diff
|
|
6910
|
+
// (patch-validator); only the CLI, running inside the repo, can prove it
|
|
6911
|
+
// APPLIES. Run c122c92a produced a diff that passed the server check and failed
|
|
6912
|
+
// to apply — structure is not applicability.
|
|
6913
|
+
program
|
|
6914
|
+
.command('check-patch <workflowId>')
|
|
6915
|
+
.description('Verify step_4\'s diff applies to the current working tree (condition 5)')
|
|
6916
|
+
.option('--step-key <key>', 'Step to check (default: step_4)', 'step_4')
|
|
6917
|
+
.option('--attempt <n>', 'Specific attempt number (default: latest)')
|
|
6918
|
+
.option('--format <fmt>', 'Output format: pretty or json', 'pretty')
|
|
6919
|
+
.action(async (workflowId, opts) => {
|
|
6920
|
+
const { execFileSync } = require('child_process');
|
|
6921
|
+
const os = require('os');
|
|
6922
|
+
const { extractDiff, interpretApplyCheck } = require('./lib/patch-apply');
|
|
6923
|
+
const { extractFileBlocks, buildDiff, nodeMaterializeIO, MaterializeError } = require('./lib/materialize-diff');
|
|
6924
|
+
const { gitEnv } = require('./lib/git-env');
|
|
6925
|
+
const config = getConfig();
|
|
6926
|
+
const client = api(config);
|
|
6927
|
+
|
|
6928
|
+
const emit = (obj, humanLine) => {
|
|
6929
|
+
if (opts.format === 'json') console.log(JSON.stringify(obj));
|
|
6930
|
+
else console.log(humanLine);
|
|
6931
|
+
};
|
|
6932
|
+
|
|
6933
|
+
let output;
|
|
6934
|
+
try {
|
|
6935
|
+
const url = `/orchestrate/${workflowId}/step/${opts.stepKey}` +
|
|
6936
|
+
(opts.attempt ? `?attempt=${opts.attempt}` : '');
|
|
6937
|
+
const r = await client.get(url);
|
|
6938
|
+
output = r.data?.output;
|
|
6939
|
+
} catch (e) {
|
|
6940
|
+
const msg = e.response?.data?.error || e.message;
|
|
6941
|
+
emit({ ok: false, reason: 'fetch_failed', detail: msg },
|
|
6942
|
+
`✗ could not fetch ${opts.stepKey} output: ${msg}`);
|
|
6943
|
+
process.exit(1);
|
|
6944
|
+
}
|
|
6945
|
+
|
|
6946
|
+
// HC-019n-followup-23/27: prefer whole-file blocks + region edits
|
|
6947
|
+
// (## Changed Files). git computes the hunks, so the assembled diff applies
|
|
6948
|
+
// by construction — no LLM line-counting. Fall back to the raw-diff path
|
|
6949
|
+
// (followup-20) for stories still on the old ## Patch contract.
|
|
6950
|
+
let diff, source, noChanges;
|
|
6951
|
+
const blocks = extractFileBlocks(output);
|
|
6952
|
+
if (blocks.noChanges) {
|
|
6953
|
+
emit({ ok: true, verdict: 'no_changes' },
|
|
6954
|
+
`○ ${opts.stepKey} declared NO CHANGES — nothing to apply.`);
|
|
6955
|
+
return;
|
|
6956
|
+
}
|
|
6957
|
+
if (blocks.files.length > 0 || (blocks.edits && blocks.edits.length > 0) || blocks.deletes.length > 0) {
|
|
6958
|
+
let built;
|
|
6959
|
+
try {
|
|
6960
|
+
built = buildDiff(blocks, nodeMaterializeIO({ cwd: process.cwd() }));
|
|
6961
|
+
} catch (e) {
|
|
6962
|
+
if (e instanceof MaterializeError) {
|
|
6963
|
+
emit({ ok: false, verdict: 'does_not_apply', reason: e.code, detail: e.message },
|
|
6964
|
+
`✗ region edit could not be applied (${e.code}): ${e.message}`);
|
|
6965
|
+
process.exit(2);
|
|
6966
|
+
}
|
|
6967
|
+
throw e;
|
|
6968
|
+
}
|
|
6969
|
+
diff = built.diff;
|
|
6970
|
+
source = 'file-blocks';
|
|
6971
|
+
noChanges = built.partCount === 0;
|
|
6972
|
+
if (noChanges) {
|
|
6973
|
+
emit({ ok: true, verdict: 'no_changes' },
|
|
6974
|
+
`○ ${opts.stepKey} file blocks were identical to the tree — nothing to apply.`);
|
|
6975
|
+
return;
|
|
6976
|
+
}
|
|
6977
|
+
} else {
|
|
6978
|
+
// Legacy path: a hand-written ## Patch diff.
|
|
6979
|
+
({ diff, source, noChanges } = extractDiff(output));
|
|
6980
|
+
if (noChanges) {
|
|
6981
|
+
emit({ ok: true, verdict: 'no_changes' },
|
|
6982
|
+
`○ ${opts.stepKey} declared NO CHANGES — nothing to apply.`);
|
|
6983
|
+
return;
|
|
6984
|
+
}
|
|
6985
|
+
if (!diff) {
|
|
6986
|
+
emit({ ok: false, verdict: 'no_diff', reason: 'no_diff' },
|
|
6987
|
+
`✗ ${opts.stepKey} output contains neither ## Changed Files blocks nor a ` +
|
|
6988
|
+
`unified diff — the agent produced prose. (Placeholder-cascade failure mode.)`);
|
|
6989
|
+
process.exit(1);
|
|
6990
|
+
}
|
|
6991
|
+
}
|
|
6992
|
+
|
|
6993
|
+
// Write to a temp file and run git apply --check against the current repo.
|
|
6994
|
+
const tmp = path.join(os.tmpdir(), `hone-check-patch-${Date.now()}.diff`);
|
|
6995
|
+
let result;
|
|
6996
|
+
try {
|
|
6997
|
+
fs.writeFileSync(tmp, diff);
|
|
6998
|
+
let code = 0, stderr = '';
|
|
6999
|
+
try {
|
|
7000
|
+
execFileSync('git', ['apply', '--check', tmp], {
|
|
7001
|
+
cwd: process.cwd(), env: gitEnv(), encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'],
|
|
7002
|
+
});
|
|
7003
|
+
} catch (e) {
|
|
7004
|
+
code = e.status || 1;
|
|
7005
|
+
stderr = (e.stderr || '').toString();
|
|
7006
|
+
}
|
|
7007
|
+
result = interpretApplyCheck({ code, stderr });
|
|
7008
|
+
} finally {
|
|
7009
|
+
try { fs.unlinkSync(tmp); } catch { /* best-effort */ }
|
|
7010
|
+
}
|
|
7011
|
+
|
|
7012
|
+
if (result.applies) {
|
|
7013
|
+
emit({ ok: true, verdict: result.verdict, source, detail: result.detail },
|
|
7014
|
+
`✓ ${opts.stepKey} diff ${result.detail} (${source} block)`);
|
|
7015
|
+
} else {
|
|
7016
|
+
emit({ ok: false, verdict: 'failed', source, detail: result.detail },
|
|
7017
|
+
`✗ ${opts.stepKey} diff does NOT apply: ${result.detail}\n` +
|
|
7018
|
+
` The output is a well-formed diff but does not match the working tree — ` +
|
|
7019
|
+
`exactly the defect condition 5 catches that condition 4 cannot.`);
|
|
7020
|
+
process.exit(2); // distinct exit: 2 = does-not-apply, 1 = no-diff/fetch
|
|
7021
|
+
}
|
|
7022
|
+
});
|
|
7023
|
+
|
|
7024
|
+
// ── HC-019n-followup-24: verify-patch (pipeline-recovery condition 6) ─────────
|
|
7025
|
+
// Apply step_4's diff inside a throwaway git WORKTREE (checked out from HEAD),
|
|
7026
|
+
// run the adopter's test command there, report pass/fail. The real tree —
|
|
7027
|
+
// including uncommitted work — is never touched. Isolates the TREE, not the
|
|
7028
|
+
// process: ci.local_command is the adopter's own command, same trust as running
|
|
7029
|
+
// it by hand. Verdict is REPORTED, not yet enforced — the server-gate closed
|
|
7030
|
+
// loop is a deliberate follow-up (enabling a gate before its input is proven is
|
|
7031
|
+
// the HC-026c mistake).
|
|
7032
|
+
program
|
|
7033
|
+
.command('verify-patch <workflowId>')
|
|
7034
|
+
.description('Apply step_4\'s diff in an isolated worktree and run the tests (condition 6)')
|
|
7035
|
+
.option('--step-key <key>', 'Step to verify (default: step_4)', 'step_4')
|
|
7036
|
+
.option('--attempt <n>', 'Specific attempt number (default: latest)')
|
|
7037
|
+
.option('--command <cmd>', 'Test command (default: ci.local_command from config, else `make ci`)')
|
|
7038
|
+
.option('--timeout <sec>', 'Seconds before the test command is killed', '600')
|
|
7039
|
+
.option('--keep', 'Leave the worktree in place for debugging')
|
|
7040
|
+
.option('--report', 'HC-019n-followup-34: report the verdict back to the server (closed loop, advisory)')
|
|
7041
|
+
.option('--format <fmt>', 'Output format: pretty or json', 'pretty')
|
|
7042
|
+
.action(async (workflowId, opts) => {
|
|
7043
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
7044
|
+
const os = require('os');
|
|
7045
|
+
const { extractFileBlocks, buildDiff, nodeMaterializeIO, MaterializeError } = require('./lib/materialize-diff');
|
|
7046
|
+
const { resolveCommand, verdictFromRun, EXIT } = require('./lib/verify-patch');
|
|
7047
|
+
const { gitEnv } = require('./lib/git-env');
|
|
7048
|
+
const config = getConfig();
|
|
7049
|
+
const client = api(config);
|
|
7050
|
+
|
|
7051
|
+
const emit = (obj, humanLine) => {
|
|
7052
|
+
if (opts.format === 'json') console.log(JSON.stringify(obj));
|
|
7053
|
+
else console.log(humanLine);
|
|
7054
|
+
};
|
|
7055
|
+
|
|
7056
|
+
// 1. Fetch step_4 output.
|
|
7057
|
+
let output, runConfig;
|
|
7058
|
+
try {
|
|
7059
|
+
const url = `/orchestrate/${workflowId}/step/${opts.stepKey}` +
|
|
7060
|
+
(opts.attempt ? `?attempt=${opts.attempt}` : '');
|
|
7061
|
+
const r = await client.get(url);
|
|
7062
|
+
output = r.data?.output;
|
|
7063
|
+
// best-effort: pull the run's ci_local_command for the default
|
|
7064
|
+
try {
|
|
7065
|
+
const runR = await client.get(`/orchestrate/${workflowId}`);
|
|
7066
|
+
runConfig = runR.data?.config || {};
|
|
7067
|
+
} catch { runConfig = {}; }
|
|
7068
|
+
} catch (e) {
|
|
7069
|
+
const msg = e.response?.data?.error || e.message;
|
|
7070
|
+
emit({ ok: false, reason: 'fetch_failed', detail: msg },
|
|
7071
|
+
`✗ could not fetch ${opts.stepKey} output: ${msg}`);
|
|
7072
|
+
process.exit(EXIT.NO_DIFF);
|
|
7073
|
+
}
|
|
7074
|
+
|
|
7075
|
+
// 2. Materialize the diff (shared with check-patch/emit-pr, followup-23/27).
|
|
7076
|
+
// Condition 6 presupposes 5.
|
|
7077
|
+
const blocks = extractFileBlocks(output);
|
|
7078
|
+
const hasWork = blocks.files.length > 0 || (blocks.edits && blocks.edits.length > 0) || blocks.deletes.length > 0;
|
|
7079
|
+
if (blocks.noChanges || !hasWork) {
|
|
7080
|
+
emit({ ok: false, verdict: 'no_diff', reason: 'no_changes_or_diff' },
|
|
7081
|
+
`✗ ${opts.stepKey} produced no file changes to verify.`);
|
|
7082
|
+
process.exit(EXIT.NO_DIFF);
|
|
7083
|
+
}
|
|
7084
|
+
let diff;
|
|
7085
|
+
try {
|
|
7086
|
+
diff = buildDiff(blocks, nodeMaterializeIO({ cwd: process.cwd() })).diff;
|
|
7087
|
+
} catch (e) {
|
|
7088
|
+
if (e instanceof MaterializeError) {
|
|
7089
|
+
emit({ ok: false, verdict: 'does_not_apply', reason: e.code, detail: e.message },
|
|
7090
|
+
`✗ region edit could not be applied (${e.code}): ${e.message}`);
|
|
7091
|
+
process.exit(EXIT.DOES_NOT_APPLY);
|
|
7092
|
+
}
|
|
7093
|
+
throw e;
|
|
7094
|
+
}
|
|
7095
|
+
if (!diff.trim()) {
|
|
7096
|
+
emit({ ok: true, verdict: 'no_changes' },
|
|
7097
|
+
`○ ${opts.stepKey} file blocks matched the tree — nothing to verify.`);
|
|
7098
|
+
return;
|
|
7099
|
+
}
|
|
7100
|
+
|
|
7101
|
+
// 3. Isolated worktree from HEAD.
|
|
7102
|
+
const wt = fs.mkdtempSync(path.join(os.tmpdir(), 'hone-verify-'));
|
|
7103
|
+
const diffFile = path.join(os.tmpdir(), `hone-verify-${Date.now()}.diff`);
|
|
7104
|
+
fs.writeFileSync(diffFile, diff);
|
|
7105
|
+
const command = resolveCommand(opts.command, runConfig?.ci_local_command);
|
|
7106
|
+
let cleanupDone = false;
|
|
7107
|
+
const cleanup = () => {
|
|
7108
|
+
if (cleanupDone || opts.keep) return;
|
|
7109
|
+
cleanupDone = true;
|
|
7110
|
+
try { execFileSync('git', ['worktree', 'remove', '--force', wt], { cwd: process.cwd(), env: gitEnv(), stdio: 'ignore' }); } catch { /* best-effort */ }
|
|
7111
|
+
try { fs.rmSync(wt, { recursive: true, force: true }); } catch { /* best-effort */ }
|
|
7112
|
+
try { fs.unlinkSync(diffFile); } catch { /* best-effort */ }
|
|
7113
|
+
};
|
|
7114
|
+
// Guard against leaving a worktree if the process is interrupted.
|
|
7115
|
+
process.on('SIGINT', () => { cleanup(); process.exit(130); });
|
|
7116
|
+
|
|
7117
|
+
// HC-019n-followup-24 fix: NEVER call process.exit() inside the try — it
|
|
7118
|
+
// skips the finally, so the worktree leaks (found by the live verification:
|
|
7119
|
+
// three hone-verify-* worktrees survived). Compute the exit code, let the
|
|
7120
|
+
// finally clean up, then exit.
|
|
7121
|
+
let exitCode = EXIT.PASS;
|
|
7122
|
+
// HC-019n-followup-34: capture the verdict + HEAD sha for the closed-loop
|
|
7123
|
+
// report (--report). Hoisted so they survive the try/finally to the POST.
|
|
7124
|
+
let reportVerdict = null, reportDetail = null, headSha = null;
|
|
7125
|
+
try { headSha = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: process.cwd(), env: gitEnv(), encoding: 'utf8' }).trim(); } catch { /* best-effort */ }
|
|
7126
|
+
try {
|
|
7127
|
+
execFileSync('git', ['worktree', 'add', '--detach', wt, 'HEAD'],
|
|
7128
|
+
{ cwd: process.cwd(), env: gitEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7129
|
+
|
|
7130
|
+
// 4. Apply the diff INTO the worktree (real apply, isolated).
|
|
7131
|
+
let applied = true, applyDetail = '';
|
|
7132
|
+
try {
|
|
7133
|
+
execFileSync('git', ['-C', wt, 'apply', diffFile], { env: gitEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7134
|
+
} catch (e) {
|
|
7135
|
+
applied = false;
|
|
7136
|
+
applyDetail = (e.stderr || '').toString().split('\n').find(Boolean) || 'git apply failed';
|
|
7137
|
+
}
|
|
7138
|
+
if (!applied) {
|
|
7139
|
+
emit({ ok: false, verdict: 'does_not_apply', detail: applyDetail },
|
|
7140
|
+
`✗ diff did not apply in the worktree: ${applyDetail}`);
|
|
7141
|
+
exitCode = EXIT.DOES_NOT_APPLY;
|
|
7142
|
+
reportVerdict = 'does_not_apply'; reportDetail = applyDetail;
|
|
7143
|
+
} else {
|
|
7144
|
+
// 5. Run the adopter's test command in the worktree.
|
|
7145
|
+
if (opts.format !== 'json') console.log(` running: ${command} (in isolated worktree, timeout ${opts.timeout}s)`);
|
|
7146
|
+
const run = spawnSync(command, {
|
|
7147
|
+
cwd: wt, shell: true, encoding: 'utf8',
|
|
7148
|
+
timeout: Number(opts.timeout) * 1000, env: gitEnv(),
|
|
7149
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
7150
|
+
});
|
|
7151
|
+
const timedOut = run.error && run.error.code === 'ETIMEDOUT';
|
|
7152
|
+
const v = verdictFromRun({ code: run.status, timedOut, stdout: run.stdout, stderr: run.stderr });
|
|
7153
|
+
const sym = v.verdict === 'pass' ? '✓' : v.verdict === 'timeout' ? '⏱' : '✗';
|
|
7154
|
+
emit(
|
|
7155
|
+
{ ok: v.verdict === 'pass', verdict: v.verdict, command, detail: v.detail },
|
|
7156
|
+
`${sym} ${v.detail}\n (command: ${command})`);
|
|
7157
|
+
exitCode = v.exitCode;
|
|
7158
|
+
reportVerdict = v.verdict; reportDetail = v.detail;
|
|
7159
|
+
}
|
|
7160
|
+
} catch (e) {
|
|
7161
|
+
emit({ ok: false, verdict: 'error', detail: e.message },
|
|
7162
|
+
`✗ verify-patch failed: ${e.message}`);
|
|
7163
|
+
exitCode = 1;
|
|
7164
|
+
reportVerdict = 'error'; reportDetail = e.message;
|
|
7165
|
+
} finally {
|
|
7166
|
+
cleanup();
|
|
7167
|
+
}
|
|
7168
|
+
|
|
7169
|
+
// HC-019n-followup-34: the closed loop — report the EXECUTED verdict back to
|
|
7170
|
+
// the server (advisory record). Best-effort: a reporting failure must not
|
|
7171
|
+
// change the local exit code (the verdict is already true locally).
|
|
7172
|
+
if (opts.report && reportVerdict) {
|
|
7173
|
+
try {
|
|
7174
|
+
await client.post(`/orchestrate/${workflowId}/verdict`, {
|
|
7175
|
+
source: 'verify-patch',
|
|
7176
|
+
stepKey: opts.stepKey,
|
|
7177
|
+
verdict: reportVerdict,
|
|
7178
|
+
exitCode,
|
|
7179
|
+
detail: reportDetail,
|
|
7180
|
+
attempt: opts.attempt ? Number(opts.attempt) : undefined,
|
|
7181
|
+
headSha,
|
|
7182
|
+
});
|
|
7183
|
+
if (opts.format !== 'json') console.log(` → reported verdict '${reportVerdict}' to the server (advisory)`);
|
|
7184
|
+
} catch (e) {
|
|
7185
|
+
if (opts.format !== 'json') console.warn(` ⚠ verdict report failed (non-fatal): ${e.response?.data?.error || e.message}`);
|
|
7186
|
+
}
|
|
7187
|
+
}
|
|
7188
|
+
process.exit(exitCode);
|
|
7189
|
+
});
|
|
7190
|
+
|
|
7191
|
+
// ── HC-019n-followup-25: emit-pr (pipeline-recovery condition 7) ──────────────
|
|
7192
|
+
// Condition 6 proved step_4's diff applies AND its tests pass in isolation.
|
|
7193
|
+
// Condition 7 turns that verified diff into the reviewable artifact: a branch
|
|
7194
|
+
// and (opt-in) a draft PR. Publishing is staged — tier-1 dry run by default;
|
|
7195
|
+
// --push, then --open-pr. Sibling to check-patch/verify-patch (top-level).
|
|
7196
|
+
program
|
|
7197
|
+
.command('emit-pr <workflowId>')
|
|
7198
|
+
.description('Emit a branch + draft PR from step_4\'s verified diff (condition 7)')
|
|
7199
|
+
.option('--step-key <key>', 'Step to emit (default: step_4)', 'step_4')
|
|
7200
|
+
.option('--attempt <n>', 'Specific attempt number (default: latest)')
|
|
7201
|
+
.option('--branch <name>', 'Override the generated branch name')
|
|
7202
|
+
.option('--base <ref>', 'Target base branch for the PR (default: repo default)')
|
|
7203
|
+
.option('--push', 'Push the branch to origin (default: local dry run only)')
|
|
7204
|
+
.option('--open-pr', 'Open a draft PR (implies --push)')
|
|
7205
|
+
.option('--ready', 'Open the PR ready-for-review instead of draft')
|
|
7206
|
+
.option('--skip-verify', 'Do not re-run the tests before committing (downgrades provenance)')
|
|
7207
|
+
.option('--command <cmd>', 'Test command for --verify (default: ci.local_command, else `make ci`)')
|
|
7208
|
+
.option('--timeout <sec>', 'Seconds before the verify test command is killed', '600')
|
|
7209
|
+
.option('--keep', 'Leave the worktree in place for debugging')
|
|
7210
|
+
.option('--format <fmt>', 'Output format: pretty or json', 'pretty')
|
|
7211
|
+
.action(async (workflowId, opts) => {
|
|
7212
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
7213
|
+
const os = require('os');
|
|
7214
|
+
const { extractFileBlocks, buildDiff, nodeMaterializeIO, MaterializeError } = require('./lib/materialize-diff');
|
|
7215
|
+
const { resolveCommand, verdictFromRun } = require('./lib/verify-patch');
|
|
7216
|
+
const {
|
|
7217
|
+
EXIT, buildBranchName, buildCommitMessage, buildPrBody, decidePublish,
|
|
7218
|
+
} = require('./lib/emit-pr');
|
|
7219
|
+
const { gitEnv } = require('./lib/git-env');
|
|
7220
|
+
const config = getConfig();
|
|
7221
|
+
const client = api(config);
|
|
7222
|
+
|
|
7223
|
+
// --open-pr implies --push (cannot open a PR against an unpushed branch).
|
|
7224
|
+
const doPush = Boolean(opts.push || opts.openPr);
|
|
7225
|
+
const doOpenPr = Boolean(opts.openPr);
|
|
7226
|
+
const doVerify = !opts.skipVerify;
|
|
7227
|
+
|
|
7228
|
+
const emit = (obj, humanLine) => {
|
|
7229
|
+
if (opts.format === 'json') console.log(JSON.stringify(obj));
|
|
7230
|
+
else if (humanLine != null) console.log(humanLine);
|
|
7231
|
+
};
|
|
7232
|
+
|
|
7233
|
+
// 1. Fetch step_4 output + the run's storyId (for naming/provenance).
|
|
7234
|
+
let output, storyId = 'story', runConfig = {};
|
|
7235
|
+
try {
|
|
7236
|
+
const url = `/orchestrate/${workflowId}/step/${opts.stepKey}` +
|
|
7237
|
+
(opts.attempt ? `?attempt=${opts.attempt}` : '');
|
|
7238
|
+
const r = await client.get(url);
|
|
7239
|
+
output = r.data?.output;
|
|
7240
|
+
try {
|
|
7241
|
+
const runR = await client.get(`/orchestrate/${workflowId}`);
|
|
7242
|
+
storyId = runR.data?.storyId || runR.data?.story_id || 'story';
|
|
7243
|
+
runConfig = runR.data?.config || {};
|
|
7244
|
+
} catch { /* naming falls back to 'story'; command falls back to make ci */ }
|
|
7245
|
+
} catch (e) {
|
|
7246
|
+
const msg = e.response?.data?.error || e.message;
|
|
7247
|
+
emit({ ok: false, reason: 'fetch_failed', detail: msg },
|
|
7248
|
+
`✗ could not fetch ${opts.stepKey} output: ${msg}`);
|
|
7249
|
+
process.exit(EXIT.NO_DIFF);
|
|
7250
|
+
}
|
|
7251
|
+
|
|
7252
|
+
// 2. Materialize the diff (shared with check-patch/verify-patch,
|
|
7253
|
+
// followup-23/27). Condition 7 presupposes 5.
|
|
7254
|
+
const blocks = extractFileBlocks(output);
|
|
7255
|
+
const hasWork = blocks.files.length > 0 || (blocks.edits && blocks.edits.length > 0) || blocks.deletes.length > 0;
|
|
7256
|
+
if (blocks.noChanges || !hasWork) {
|
|
7257
|
+
emit({ ok: false, verdict: 'no_diff', reason: 'no_changes_or_diff' },
|
|
7258
|
+
`✗ ${opts.stepKey} produced no file changes to emit.`);
|
|
7259
|
+
process.exit(EXIT.NO_DIFF);
|
|
7260
|
+
}
|
|
7261
|
+
let diff;
|
|
7262
|
+
try {
|
|
7263
|
+
diff = buildDiff(blocks, nodeMaterializeIO({ cwd: process.cwd() })).diff;
|
|
7264
|
+
} catch (e) {
|
|
7265
|
+
if (e instanceof MaterializeError) {
|
|
7266
|
+
emit({ ok: false, verdict: 'does_not_apply', reason: e.code, detail: e.message },
|
|
7267
|
+
`✗ region edit could not be applied (${e.code}): ${e.message}`);
|
|
7268
|
+
process.exit(EXIT.DOES_NOT_APPLY);
|
|
7269
|
+
}
|
|
7270
|
+
throw e;
|
|
7271
|
+
}
|
|
7272
|
+
if (!diff.trim()) {
|
|
7273
|
+
emit({ ok: true, verdict: 'no_changes' },
|
|
7274
|
+
`○ ${opts.stepKey} file blocks matched the tree — nothing to emit.`);
|
|
7275
|
+
return;
|
|
7276
|
+
}
|
|
7277
|
+
|
|
7278
|
+
// 3. Resolve names + probe idempotency BEFORE any mutation.
|
|
7279
|
+
let headSha = '';
|
|
7280
|
+
try {
|
|
7281
|
+
headSha = execFileSync('git', ['rev-parse', 'HEAD'],
|
|
7282
|
+
{ cwd: process.cwd(), env: gitEnv(), encoding: 'utf8' }).trim();
|
|
7283
|
+
} catch {
|
|
7284
|
+
emit({ ok: false, reason: 'no_head' }, '✗ repository has no HEAD commit — cannot branch from nothing.');
|
|
7285
|
+
process.exit(1);
|
|
7286
|
+
}
|
|
7287
|
+
const branch = buildBranchName({ storyId, workflowId, override: opts.branch });
|
|
7288
|
+
|
|
7289
|
+
const branchExists = (() => {
|
|
7290
|
+
try {
|
|
7291
|
+
execFileSync('git', ['rev-parse', '--verify', '--quiet', `refs/heads/${branch}`],
|
|
7292
|
+
{ cwd: process.cwd(), env: gitEnv(), stdio: 'ignore' });
|
|
7293
|
+
return true;
|
|
7294
|
+
} catch { return false; }
|
|
7295
|
+
})();
|
|
7296
|
+
// Only probe the remote for an existing PR when we intend to publish.
|
|
7297
|
+
let openPrUrl = null;
|
|
7298
|
+
if (doOpenPr) {
|
|
7299
|
+
try {
|
|
7300
|
+
const out = execFileSync('gh', ['pr', 'list', '--head', branch, '--state', 'open', '--json', 'url', '--jq', '.[0].url'],
|
|
7301
|
+
{ cwd: process.cwd(), env: gitEnv(), encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
7302
|
+
openPrUrl = out || null;
|
|
7303
|
+
} catch { openPrUrl = null; /* gh missing/unauthed handled at push/open time */ }
|
|
7304
|
+
}
|
|
7305
|
+
const decision = decidePublish({ branchExists, openPrUrl });
|
|
7306
|
+
if (decision.action === 'already_open') {
|
|
7307
|
+
emit({ ok: true, verdict: 'already_open', prUrl: decision.prUrl, branch },
|
|
7308
|
+
`✓ a draft PR is already open for ${branch}: ${decision.prUrl}`);
|
|
7309
|
+
process.exit(EXIT.PASS);
|
|
7310
|
+
}
|
|
7311
|
+
if (decision.action === 'branch_conflict') {
|
|
7312
|
+
emit({ ok: false, verdict: 'branch_conflict', branch },
|
|
7313
|
+
`✗ branch '${branch}' already exists — not overwriting. Use --branch to pick another name.`);
|
|
7314
|
+
process.exit(EXIT.BRANCH_CONFLICT);
|
|
7315
|
+
}
|
|
7316
|
+
|
|
7317
|
+
// 4. Isolated worktree with a NAMED branch off HEAD (adopter tree untouched).
|
|
7318
|
+
const wt = fs.mkdtempSync(path.join(os.tmpdir(), 'hone-emit-'));
|
|
7319
|
+
const diffFile = path.join(os.tmpdir(), `hone-emit-${Date.now()}.diff`);
|
|
7320
|
+
fs.writeFileSync(diffFile, diff);
|
|
7321
|
+
const command = resolveCommand(opts.command, runConfig?.ci_local_command);
|
|
7322
|
+
// `worktree add -b` creates the branch ref BEFORE apply/verify. If we bail
|
|
7323
|
+
// before a successful commit (apply fails, verify fails), that empty branch
|
|
7324
|
+
// (pointing at HEAD) must be torn down too — else "a failing diff never
|
|
7325
|
+
// reaches a branch" would be false and a re-run would hit a bogus conflict.
|
|
7326
|
+
// `committed` gates it: once we have the deliverable commit, keep the branch.
|
|
7327
|
+
let cleanupDone = false;
|
|
7328
|
+
let committed = false;
|
|
7329
|
+
const cleanup = () => {
|
|
7330
|
+
if (cleanupDone || opts.keep) return;
|
|
7331
|
+
cleanupDone = true;
|
|
7332
|
+
try { execFileSync('git', ['worktree', 'remove', '--force', wt], { cwd: process.cwd(), env: gitEnv(), stdio: 'ignore' }); } catch { /* best-effort */ }
|
|
7333
|
+
// Remove the leaked empty branch only when nothing was committed onto it.
|
|
7334
|
+
if (!committed) {
|
|
7335
|
+
try { execFileSync('git', ['branch', '-D', branch], { cwd: process.cwd(), env: gitEnv(), stdio: 'ignore' }); } catch { /* best-effort */ }
|
|
7336
|
+
}
|
|
7337
|
+
try { fs.rmSync(wt, { recursive: true, force: true }); } catch { /* best-effort */ }
|
|
7338
|
+
try { fs.unlinkSync(diffFile); } catch { /* best-effort */ }
|
|
7339
|
+
};
|
|
7340
|
+
process.on('SIGINT', () => { cleanup(); process.exit(130); });
|
|
7341
|
+
|
|
7342
|
+
// HC-019n-followup-24 lesson: NEVER process.exit() inside the try — it skips
|
|
7343
|
+
// the finally and leaks the worktree. Capture exitCode; exit after cleanup.
|
|
7344
|
+
let exitCode = EXIT.PASS;
|
|
7345
|
+
let verifyState = 'skipped';
|
|
7346
|
+
try {
|
|
7347
|
+
execFileSync('git', ['worktree', 'add', '-b', branch, wt, 'HEAD'],
|
|
7348
|
+
{ cwd: process.cwd(), env: gitEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7349
|
+
|
|
7350
|
+
// 4a. Apply the diff INTO the worktree.
|
|
7351
|
+
let applied = true, applyDetail = '';
|
|
7352
|
+
try {
|
|
7353
|
+
execFileSync('git', ['-C', wt, 'apply', diffFile], { env: gitEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7354
|
+
} catch (e) {
|
|
7355
|
+
applied = false;
|
|
7356
|
+
applyDetail = (e.stderr || '').toString().split('\n').find(Boolean) || 'git apply failed';
|
|
7357
|
+
}
|
|
7358
|
+
if (!applied) {
|
|
7359
|
+
emit({ ok: false, verdict: 'does_not_apply', detail: applyDetail },
|
|
7360
|
+
`✗ diff did not apply in the worktree: ${applyDetail}`);
|
|
7361
|
+
exitCode = EXIT.DOES_NOT_APPLY;
|
|
7362
|
+
} else {
|
|
7363
|
+
// 4b. Re-verify inline (default) — a failing diff never reaches a branch.
|
|
7364
|
+
let verifyOk = true;
|
|
7365
|
+
if (doVerify) {
|
|
7366
|
+
if (opts.format !== 'json') console.log(` verifying: ${command} (in isolated worktree, timeout ${opts.timeout}s)`);
|
|
7367
|
+
const run = spawnSync(command, {
|
|
7368
|
+
cwd: wt, shell: true, encoding: 'utf8',
|
|
7369
|
+
timeout: Number(opts.timeout) * 1000, env: gitEnv(),
|
|
7370
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
7371
|
+
});
|
|
7372
|
+
const timedOut = run.error && run.error.code === 'ETIMEDOUT';
|
|
7373
|
+
const v = verdictFromRun({ code: run.status, timedOut, stdout: run.stdout, stderr: run.stderr });
|
|
7374
|
+
if (v.verdict === 'pass') {
|
|
7375
|
+
verifyState = 'pass';
|
|
7376
|
+
} else {
|
|
7377
|
+
verifyOk = false;
|
|
7378
|
+
const sym = v.verdict === 'timeout' ? '⏱' : '✗';
|
|
7379
|
+
emit({ ok: false, verdict: v.verdict, command, detail: v.detail },
|
|
7380
|
+
`${sym} verify failed, no branch emitted: ${v.detail}\n (command: ${command})`);
|
|
7381
|
+
exitCode = v.exitCode; // 3 tests-failed or 4 timeout
|
|
7382
|
+
}
|
|
7383
|
+
}
|
|
7384
|
+
|
|
7385
|
+
if (verifyOk) {
|
|
7386
|
+
// 4c. Commit the applied diff onto the branch, inside the worktree.
|
|
7387
|
+
// No summary source (GET /orchestrate omits config/story text), so the
|
|
7388
|
+
// commit subject falls back to the storyId + default in buildCommitMessage.
|
|
7389
|
+
const message = buildCommitMessage({ storyId, summary: '', workflowId, headSha, verifyState });
|
|
7390
|
+
const msgFile = path.join(os.tmpdir(), `hone-emit-msg-${Date.now()}.txt`);
|
|
7391
|
+
fs.writeFileSync(msgFile, message);
|
|
7392
|
+
try {
|
|
7393
|
+
execFileSync('git', ['-C', wt, 'add', '-A'], { env: gitEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7394
|
+
execFileSync('git', ['-C', wt, 'commit', '-F', msgFile], { env: gitEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7395
|
+
committed = true; // the branch now holds the deliverable — keep it past cleanup
|
|
7396
|
+
} finally { try { fs.unlinkSync(msgFile); } catch { /* best-effort */ } }
|
|
7397
|
+
|
|
7398
|
+
const stat = (() => {
|
|
7399
|
+
try {
|
|
7400
|
+
return execFileSync('git', ['-C', wt, 'show', '--stat', '--oneline', 'HEAD'],
|
|
7401
|
+
{ env: gitEnv(), encoding: 'utf8' }).trim();
|
|
7402
|
+
} catch { return ''; }
|
|
7403
|
+
})();
|
|
7404
|
+
|
|
7405
|
+
// ── Boundary: everything above is local + reversible. ──
|
|
7406
|
+
let prUrl = null, pushed = false;
|
|
7407
|
+
if (doPush) {
|
|
7408
|
+
try {
|
|
7409
|
+
execFileSync('git', ['push', '-u', 'origin', branch],
|
|
7410
|
+
{ cwd: process.cwd(), env: gitEnv(), stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7411
|
+
pushed = true;
|
|
7412
|
+
} catch (e) {
|
|
7413
|
+
const detail = (e.stderr || e.message || '').toString().split('\n').find(Boolean) || 'push failed';
|
|
7414
|
+
emit({ ok: false, verdict: 'publish_failed', stage: 'push', detail, branch },
|
|
7415
|
+
`✗ push failed (branch committed locally as '${branch}', not published): ${detail}`);
|
|
7416
|
+
exitCode = EXIT.PUBLISH_FAILED;
|
|
7417
|
+
}
|
|
7418
|
+
}
|
|
7419
|
+
if (pushed && doOpenPr) {
|
|
7420
|
+
const prBody = buildPrBody({ storyId, workflowId, headSha, verifyState, command: doVerify ? command : undefined });
|
|
7421
|
+
const bodyFile = path.join(os.tmpdir(), `hone-emit-prbody-${Date.now()}.md`);
|
|
7422
|
+
fs.writeFileSync(bodyFile, prBody);
|
|
7423
|
+
const title = `${storyId}: Hone step_4 change (workflow ${String(workflowId).slice(0, 8)})`;
|
|
7424
|
+
const ghArgs = ['pr', 'create', '--head', branch, '--title', title, '--body-file', bodyFile];
|
|
7425
|
+
if (!opts.ready) ghArgs.push('--draft');
|
|
7426
|
+
if (opts.base) ghArgs.push('--base', opts.base);
|
|
7427
|
+
try {
|
|
7428
|
+
prUrl = execFileSync('gh', ghArgs,
|
|
7429
|
+
{ cwd: process.cwd(), env: gitEnv(), encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
|
|
7430
|
+
} catch (e) {
|
|
7431
|
+
const detail = (e.stderr || e.message || '').toString().split('\n').find(Boolean) || 'gh pr create failed';
|
|
7432
|
+
emit({ ok: false, verdict: 'publish_failed', stage: 'open_pr', detail, branch },
|
|
7433
|
+
`✗ branch pushed, but opening the PR failed: ${detail}\n → open it manually, or check \`gh auth status\`.`);
|
|
7434
|
+
exitCode = EXIT.PUBLISH_FAILED;
|
|
7435
|
+
} finally { try { fs.unlinkSync(bodyFile); } catch { /* best-effort */ } }
|
|
7436
|
+
}
|
|
7437
|
+
|
|
7438
|
+
if (exitCode === EXIT.PASS) {
|
|
7439
|
+
const tier = prUrl ? 'pr' : pushed ? 'pushed' : 'dry_run';
|
|
7440
|
+
const human = prUrl
|
|
7441
|
+
? `✓ draft PR opened: ${prUrl}`
|
|
7442
|
+
: pushed
|
|
7443
|
+
? `✓ branch pushed to origin/${branch}\n → open a PR: gh pr create --head ${branch} --draft`
|
|
7444
|
+
: `✓ branch '${branch}' built locally (dry run — nothing pushed)\n` +
|
|
7445
|
+
` verify: ${verifyState}\n\n${stat}\n\n` +
|
|
7446
|
+
` → push it: hone emit-pr ${workflowId} --push\n` +
|
|
7447
|
+
` → push + PR: hone emit-pr ${workflowId} --open-pr\n` +
|
|
7448
|
+
` → discard it: git branch -D ${branch}`;
|
|
7449
|
+
emit({ ok: true, verdict: tier, branch, verifyState, prUrl, pushed, headSha }, human);
|
|
7450
|
+
}
|
|
7451
|
+
}
|
|
7452
|
+
}
|
|
7453
|
+
} catch (e) {
|
|
7454
|
+
emit({ ok: false, verdict: 'error', detail: e.message }, `✗ emit-pr failed: ${e.message}`);
|
|
7455
|
+
exitCode = 1;
|
|
7456
|
+
} finally {
|
|
7457
|
+
cleanup();
|
|
7458
|
+
}
|
|
7459
|
+
process.exit(exitCode);
|
|
7460
|
+
});
|
|
7461
|
+
|
|
7462
|
+
// ── HC-019n-followup-32: verify-pr (pipeline-standards G4 §2, G3 functional) ────
|
|
7463
|
+
// The post-emit-pr step: once a REAL PR exists, run the real skill audit
|
|
7464
|
+
// (`hone step-5b`) + the real CI gate (`gh pr checks` / `make ci`), writing the
|
|
7465
|
+
// two artifacts (step-5b-skill-audit.md, step-5c-ci.md) the server DAG can only
|
|
7466
|
+
// MODEL, never execute (those steps are serverExecutable:false). VERIFY + REPORT
|
|
7467
|
+
// only — no auto-fix loop; the operator (or a later closed loop) acts on the
|
|
7468
|
+
// verdict. Sibling to check-patch → verify-patch → emit-pr → verify-pr.
|
|
7469
|
+
program
|
|
7470
|
+
.command('verify-pr <prOrBranch>')
|
|
7471
|
+
.description('Post-PR verification: real skill audit (hone step-5b) + real CI gate (gh pr checks / make ci) (HC-019n-followup-32)')
|
|
7472
|
+
.option('--story-id <id>', 'override story id (defaults to extraction from the branch)')
|
|
7473
|
+
.option('--base <branch>', 'base branch for the skill-audit diff', 'develop')
|
|
7474
|
+
.option('--mode <mode>', 'CI gate mode: github | local | both | none (default: ci.gate config, else github)')
|
|
7475
|
+
.option('--command <cmd>', 'local CI command (default: ci.local_command, else make ci)')
|
|
7476
|
+
.option('--timeout <sec>', 'seconds before the local CI command is killed', '900')
|
|
7477
|
+
.option('--skip-skill-audit', 'skip the hone step-5b skill audit')
|
|
7478
|
+
.option('--report', 'HC-019n-followup-36: report the CI-gate verdict back to the server (closed loop, records step_5c)')
|
|
7479
|
+
.option('--workflow-id <id>', 'the run id to report the verdict against (required with --report)')
|
|
7480
|
+
.option('--format <fmt>', 'pretty or json', 'pretty')
|
|
7481
|
+
.action(async (prOrBranch, opts) => {
|
|
7482
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
7483
|
+
const { readCIGateConfig } = require('./lib/pipeline-config');
|
|
7484
|
+
const { extractStoryIdFromBranch } = require('./lib/pipeline-status');
|
|
7485
|
+
const { EXIT, resolveGateMode, interpretGhChecks, ciGateVerdict, renderStep5cArtifact } = require('./lib/verify-pr');
|
|
7486
|
+
const { gitEnv } = require('./lib/git-env');
|
|
7487
|
+
const repoRoot = process.cwd();
|
|
7488
|
+
const emit = (obj, human) => {
|
|
7489
|
+
if (opts.format === 'json') console.log(JSON.stringify(obj));
|
|
7490
|
+
else if (human != null) console.log(human);
|
|
7491
|
+
};
|
|
7492
|
+
|
|
7493
|
+
// 1. Story id: flag > branch > the ref itself.
|
|
7494
|
+
let branch = '';
|
|
7495
|
+
try { branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { cwd: repoRoot, env: gitEnv(), encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); } catch { /* */ }
|
|
7496
|
+
const storyId = opts.storyId || extractStoryIdFromBranch(branch) || extractStoryIdFromBranch(prOrBranch) || 'unknown-story';
|
|
7497
|
+
|
|
7498
|
+
// 2. CI gate mode: --mode > ci.gate config > github.
|
|
7499
|
+
let configGate = 'github';
|
|
7500
|
+
try { configGate = readCIGateConfig(repoRoot).gate; } catch { /* default */ }
|
|
7501
|
+
const mode = resolveGateMode(opts.mode, configGate);
|
|
7502
|
+
|
|
7503
|
+
// 3. Skill audit — reuse the working `hone step-5b` command (deterministic
|
|
7504
|
+
// SA-001 engine over the PR diff). Informational; never affects the exit.
|
|
7505
|
+
let skillAudit = { ran: false };
|
|
7506
|
+
if (!opts.skipSkillAudit) {
|
|
7507
|
+
try {
|
|
7508
|
+
execFileSync(process.execPath, [__filename, 'step-5b', '--story-id', storyId, '--base', opts.base],
|
|
7509
|
+
{ cwd: repoRoot, env: process.env, stdio: opts.format === 'json' ? 'ignore' : 'inherit' });
|
|
7510
|
+
skillAudit = { ran: true, artifact: `.github/pipeline/${storyId}/step-5b-skill-audit.md` };
|
|
7511
|
+
} catch (e) {
|
|
7512
|
+
skillAudit = { ran: true, error: (e.message || '').split('\n')[0] };
|
|
7513
|
+
}
|
|
7514
|
+
}
|
|
7515
|
+
|
|
7516
|
+
// 4. CI gate.
|
|
7517
|
+
let gh = null, local = null;
|
|
7518
|
+
if (mode === 'github' || mode === 'both') {
|
|
7519
|
+
let prNum = /^\d+$/.test(prOrBranch) ? prOrBranch : null;
|
|
7520
|
+
if (!prNum) { const m = String(prOrBranch).match(/\/pull\/(\d+)/); if (m) prNum = m[1]; }
|
|
7521
|
+
if (!prNum) {
|
|
7522
|
+
try { prNum = execFileSync('gh', ['pr', 'list', '--head', prOrBranch, '--state', 'all', '--json', 'number', '--jq', '.[0].number'], { cwd: repoRoot, env: gitEnv(), encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || null; } catch { prNum = null; }
|
|
7523
|
+
}
|
|
7524
|
+
if (!prNum) {
|
|
7525
|
+
emit({ ok: false, reason: 'no_pr', ref: prOrBranch },
|
|
7526
|
+
`✗ could not resolve a PR for '${prOrBranch}' — github mode needs a PR number, URL, or a branch with a PR.`);
|
|
7527
|
+
process.exit(EXIT.NO_PR);
|
|
7528
|
+
}
|
|
7529
|
+
// gh pr checks exits non-zero when checks fail/pend, but still prints JSON.
|
|
7530
|
+
let raw = '';
|
|
7531
|
+
try {
|
|
7532
|
+
raw = execFileSync('gh', ['pr', 'checks', prNum, '--json', 'name,state,bucket'], { cwd: repoRoot, env: gitEnv(), encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
|
|
7533
|
+
} catch (e) { raw = (e.stdout || '').toString(); }
|
|
7534
|
+
try { gh = interpretGhChecks(JSON.parse(raw)); } catch { gh = null; }
|
|
7535
|
+
}
|
|
7536
|
+
if (mode === 'local' || mode === 'both') {
|
|
7537
|
+
let command = (opts.command && opts.command.trim()) || '';
|
|
7538
|
+
if (!command) { try { command = readCIGateConfig(repoRoot).local_command; } catch { command = 'make ci'; } }
|
|
7539
|
+
command = command || 'make ci';
|
|
7540
|
+
if (opts.format !== 'json') console.log(` running local CI: ${command} (timeout ${opts.timeout}s)`);
|
|
7541
|
+
const run = spawnSync(command, { cwd: repoRoot, shell: true, encoding: 'utf8', timeout: Number(opts.timeout) * 1000, stdio: ['ignore', 'inherit', 'inherit'] });
|
|
7542
|
+
local = { code: run.status == null ? 1 : run.status, timedOut: !!(run.error && run.error.code === 'ETIMEDOUT') };
|
|
7543
|
+
}
|
|
7544
|
+
|
|
7545
|
+
const verdict = ciGateVerdict({ mode, gh, local });
|
|
7546
|
+
|
|
7547
|
+
// 5. Write the real step-5c-ci.md artifact.
|
|
7548
|
+
try {
|
|
7549
|
+
const dir = path.join(repoRoot, '.github/pipeline', storyId);
|
|
7550
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
7551
|
+
fs.writeFileSync(path.join(dir, 'step-5c-ci.md'), renderStep5cArtifact({ storyId, mode, prRef: prOrBranch, gh, local, verdict }));
|
|
7552
|
+
} catch { /* best-effort */ }
|
|
7553
|
+
|
|
7554
|
+
// 6. Closed loop (HC-019n-followup-36): report the CI-gate verdict to the
|
|
7555
|
+
// server. This RECORDS against step_5c — which is a post-PR, CLI-produced
|
|
7556
|
+
// node (serverExecutable:false), so there is no server gate to auto-approve:
|
|
7557
|
+
// verify-pr's verdict is ADVISORY audit even in enforce mode (enforce gates
|
|
7558
|
+
// step_4, the build). Best-effort — never changes the local exit code.
|
|
7559
|
+
let reported = false;
|
|
7560
|
+
if (opts.report) {
|
|
7561
|
+
if (!opts.workflowId) {
|
|
7562
|
+
if (opts.format !== 'json') console.warn(' ⚠ --report needs --workflow-id <run id> to address the run; skipping report.');
|
|
7563
|
+
} else {
|
|
7564
|
+
try {
|
|
7565
|
+
const client = api(getConfig());
|
|
7566
|
+
await client.post(`/orchestrate/${opts.workflowId}/verdict`, {
|
|
7567
|
+
source: 'verify-pr',
|
|
7568
|
+
stepKey: 'step_5c',
|
|
7569
|
+
verdict: verdict.verdict,
|
|
7570
|
+
exitCode: verdict.exitCode,
|
|
7571
|
+
detail: verdict.detail,
|
|
7572
|
+
});
|
|
7573
|
+
reported = true;
|
|
7574
|
+
if (opts.format !== 'json') console.log(` → reported CI-gate verdict '${verdict.verdict}' to run ${opts.workflowId} (advisory)`);
|
|
7575
|
+
} catch (e) {
|
|
7576
|
+
if (opts.format !== 'json') console.warn(` ⚠ verdict report failed (non-fatal): ${e.response?.data?.error || e.message}`);
|
|
7577
|
+
}
|
|
7578
|
+
}
|
|
7579
|
+
}
|
|
7580
|
+
|
|
7581
|
+
// 7. Report. The exit code is the CI verdict's; the skill audit is advisory.
|
|
7582
|
+
const sym = (verdict.verdict === 'pass' || verdict.verdict === 'disabled') ? '✓' : verdict.verdict === 'pending' ? '⏳' : '✗';
|
|
7583
|
+
emit(
|
|
7584
|
+
{ ok: verdict.exitCode === EXIT.PASS, storyId, mode, skillAudit, ciGate: verdict, gh, local, reported },
|
|
7585
|
+
`${sym} CI gate (${mode}): ${verdict.verdict} — ${verdict.detail}\n` +
|
|
7586
|
+
` skill audit: ${opts.skipSkillAudit ? 'skipped' : (skillAudit.artifact || skillAudit.error || 'done')}\n` +
|
|
7587
|
+
` → .github/pipeline/${storyId}/step-5c-ci.md`);
|
|
7588
|
+
process.exit(verdict.exitCode);
|
|
7589
|
+
});
|
|
7590
|
+
|
|
6443
7591
|
// ── CLI setup ─────────────────────────────────────────────────────────────────
|
|
6444
7592
|
program
|
|
6445
7593
|
.name('hone')
|