@hone-ai/cli 1.8.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hone-cli.js +680 -73
- package/lib/compare-reviews.js +279 -0
- package/lib/parse-review-json.js +87 -0
- package/lib/release-review-config.js +98 -0
- package/package.json +1 -1
package/hone-cli.js
CHANGED
|
@@ -25,6 +25,8 @@ const crypto = require('crypto');
|
|
|
25
25
|
const { execSync } = require('child_process');
|
|
26
26
|
|
|
27
27
|
const pkg = require('./package.json');
|
|
28
|
+
const { parseReviewJSON } = require('./lib/parse-review-json');
|
|
29
|
+
const { resolveBaseRef, getMaxDiffChars } = require('./lib/release-review-config');
|
|
28
30
|
const program = new Command();
|
|
29
31
|
|
|
30
32
|
// ── Config resolution ─────────────────────────────────────────────────────────
|
|
@@ -362,10 +364,16 @@ program
|
|
|
362
364
|
console.log('Running setup-ai-pipeline.sh v3.1...');
|
|
363
365
|
console.log('');
|
|
364
366
|
|
|
367
|
+
// Auto-detect non-TTY (CI, piped, Claude Code) and add --non-interactive
|
|
368
|
+
const isNonInteractive = opts.nonInteractive || !process.stdin.isTTY;
|
|
369
|
+
if (isNonInteractive && !opts.nonInteractive) {
|
|
370
|
+
console.log(' (non-TTY detected — running in non-interactive mode)');
|
|
371
|
+
}
|
|
372
|
+
|
|
365
373
|
const flags = [
|
|
366
374
|
`--source "${path.join(tmpDir, 'enterprise-github')}"`,
|
|
367
|
-
opts.dryRun
|
|
368
|
-
|
|
375
|
+
opts.dryRun ? '--dry-run' : '',
|
|
376
|
+
isNonInteractive ? '--non-interactive' : '',
|
|
369
377
|
].filter(Boolean).join(' ');
|
|
370
378
|
|
|
371
379
|
try {
|
|
@@ -4144,13 +4152,14 @@ program
|
|
|
4144
4152
|
.option('--contracts', 'Run contract validation between pipeline agents')
|
|
4145
4153
|
.option('--snapshot', 'Save current eval + contract results as regression baseline')
|
|
4146
4154
|
.option('--regression', 'Compare current results against saved baseline (detect drift)')
|
|
4155
|
+
.option('--judge', 'Run LLM-as-judge scenarios (requires ANTHROPIC_API_KEY, costs tokens)')
|
|
4147
4156
|
.action(async (opts) => {
|
|
4148
4157
|
const path = require('path');
|
|
4149
4158
|
const fs = require('fs');
|
|
4150
4159
|
const yaml = require('js-yaml');
|
|
4151
4160
|
|
|
4152
|
-
// Load agent prompts from seed-agent-prompts.js
|
|
4153
|
-
const seedPath = path.resolve(__dirname, '..', 'scripts', 'seed-agent-prompts.js');
|
|
4161
|
+
// Load agent prompts from seed-agent-prompts.js (HC-019n-hotfix: moved into /server/)
|
|
4162
|
+
const seedPath = path.resolve(__dirname, '..', 'server', 'scripts', 'seed-agent-prompts.js');
|
|
4154
4163
|
const { AGENT_PROMPTS } = require(seedPath);
|
|
4155
4164
|
const evalDir = opts.evalsDir || path.resolve(__dirname, '..', 'evals');
|
|
4156
4165
|
|
|
@@ -4205,7 +4214,84 @@ program
|
|
|
4205
4214
|
process.exit(results.failed > 0 ? 1 : 0);
|
|
4206
4215
|
}
|
|
4207
4216
|
|
|
4208
|
-
//
|
|
4217
|
+
// LLM-judge mode (HC-019i / #268)
|
|
4218
|
+
if (opts.judge) {
|
|
4219
|
+
const { loadScenarios, formatResults } = require('./lib/eval-runner');
|
|
4220
|
+
const { runJudgeScenario } = require('./lib/eval-llm-judge');
|
|
4221
|
+
|
|
4222
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
4223
|
+
if (!apiKey) {
|
|
4224
|
+
console.error('ANTHROPIC_API_KEY required for --judge mode. Set: export ANTHROPIC_API_KEY=sk-ant-...');
|
|
4225
|
+
process.exit(1);
|
|
4226
|
+
}
|
|
4227
|
+
|
|
4228
|
+
const scenarios = loadScenarios({
|
|
4229
|
+
evalDir, agent: opts.agent, tag: opts.tag, scenarioId: opts.scenario,
|
|
4230
|
+
readFile: (p) => fs.readFileSync(p, 'utf8'),
|
|
4231
|
+
listDir: (p) => fs.readdirSync(p), isDir: (p) => fs.statSync(p).isDirectory(),
|
|
4232
|
+
parseYaml: (text) => yaml.load(text),
|
|
4233
|
+
});
|
|
4234
|
+
|
|
4235
|
+
const judgeScenarios = scenarios.filter(s => s.grading?.mode === 'llm-judge');
|
|
4236
|
+
if (judgeScenarios.length === 0) {
|
|
4237
|
+
console.log('No llm-judge scenarios found. Add grading.mode: llm-judge to eval YAML files.');
|
|
4238
|
+
process.exit(0);
|
|
4239
|
+
}
|
|
4240
|
+
|
|
4241
|
+
// LLM call function using Anthropic API
|
|
4242
|
+
async function callLLM(systemPrompt, userPrompt) {
|
|
4243
|
+
const { data } = await axios.post('https://api.anthropic.com/v1/messages', {
|
|
4244
|
+
model: 'claude-sonnet-4-20250514',
|
|
4245
|
+
max_tokens: 2048,
|
|
4246
|
+
system: systemPrompt,
|
|
4247
|
+
messages: [{ role: 'user', content: userPrompt }],
|
|
4248
|
+
}, {
|
|
4249
|
+
headers: {
|
|
4250
|
+
'x-api-key': apiKey,
|
|
4251
|
+
'anthropic-version': '2023-06-01',
|
|
4252
|
+
'content-type': 'application/json',
|
|
4253
|
+
},
|
|
4254
|
+
timeout: 60000,
|
|
4255
|
+
});
|
|
4256
|
+
return data.content?.[0]?.text || '';
|
|
4257
|
+
}
|
|
4258
|
+
|
|
4259
|
+
console.log(`Running ${judgeScenarios.length} LLM-judge scenario(s)...`);
|
|
4260
|
+
console.log('');
|
|
4261
|
+
|
|
4262
|
+
const results = [];
|
|
4263
|
+
for (const scenario of judgeScenarios) {
|
|
4264
|
+
const agentName = scenario.evalAgent || scenario.agent;
|
|
4265
|
+
const promptText = AGENT_PROMPTS[agentName];
|
|
4266
|
+
if (!promptText) {
|
|
4267
|
+
results.push({ id: scenario.id, agent: agentName, result: 'error',
|
|
4268
|
+
checks: 0, checks_passed: 0, failures: [{ type: 'missing_prompt', passed: false, detail: `agent "${agentName}" not found` }] });
|
|
4269
|
+
continue;
|
|
4270
|
+
}
|
|
4271
|
+
try {
|
|
4272
|
+
const result = await runJudgeScenario({ scenario, agentPrompt: promptText, callLLM });
|
|
4273
|
+
results.push(result);
|
|
4274
|
+
} catch (e) {
|
|
4275
|
+
results.push({ id: scenario.id, agent: agentName, result: 'error',
|
|
4276
|
+
checks: 0, checks_passed: 0, failures: [{ type: 'llm_error', passed: false, detail: e.message }] });
|
|
4277
|
+
}
|
|
4278
|
+
|
|
4279
|
+
if (opts.failFast && results[results.length - 1].result !== 'pass') break;
|
|
4280
|
+
}
|
|
4281
|
+
|
|
4282
|
+
const summary = {
|
|
4283
|
+
total: results.length,
|
|
4284
|
+
passed: results.filter(r => r.result === 'pass').length,
|
|
4285
|
+
failed: results.filter(r => r.result === 'fail').length,
|
|
4286
|
+
errors: results.filter(r => r.result === 'error').length,
|
|
4287
|
+
scenarios: results,
|
|
4288
|
+
};
|
|
4289
|
+
|
|
4290
|
+
console.log(formatResults(summary, opts.format));
|
|
4291
|
+
process.exit(summary.failed + summary.errors > 0 ? 1 : 0);
|
|
4292
|
+
}
|
|
4293
|
+
|
|
4294
|
+
// Scenario evaluation mode (deterministic)
|
|
4209
4295
|
const { loadScenarios, runAllScenarios, formatResults } = require('./lib/eval-runner');
|
|
4210
4296
|
|
|
4211
4297
|
if (!fs.existsSync(evalDir)) {
|
|
@@ -4429,7 +4515,379 @@ process.on('SIGINT', () => {
|
|
|
4429
4515
|
process.exit(0);
|
|
4430
4516
|
});
|
|
4431
4517
|
|
|
4518
|
+
// ── HC-053: Batch story queue (Epic 6 — Autonomous Mode) ─────────────────────
|
|
4519
|
+
//
|
|
4520
|
+
// Fire-and-forget submission of N stories. Each story runs as an autonomous
|
|
4521
|
+
// workflow in 'batch' mode (auto-approves early gates per workflow-dag).
|
|
4522
|
+
// One failure does NOT halt siblings.
|
|
4523
|
+
//
|
|
4524
|
+
// hone queue-stories --file stories.txt
|
|
4525
|
+
// hone queue-stories --status <batchId>
|
|
4526
|
+
// hone queue-stories --list
|
|
4527
|
+
program
|
|
4528
|
+
.command('queue-stories')
|
|
4529
|
+
.description('Queue a batch of stories for autonomous (overnight) execution')
|
|
4530
|
+
.option('--file <path>', 'Text file with one story ID per line (cap: 50 lines, 16KB)')
|
|
4531
|
+
.option('--repo <name>', 'Repository name applied to every story (default: directory name)')
|
|
4532
|
+
.option('--branch <name>', 'Git branch applied to every story (default: current branch)')
|
|
4533
|
+
.option('--status <batchId>', 'Show status of a previously submitted batch')
|
|
4534
|
+
.option('--report <batchId>', 'Fetch the morning report for a batch (markdown by default)')
|
|
4535
|
+
.option('--save <path>', 'When used with --report, save markdown to a file instead of stdout')
|
|
4536
|
+
.option('--list', 'List recent batches for this org')
|
|
4537
|
+
.option('--limit <n>', 'Number of batches to list (default 20, max 100)', '20')
|
|
4538
|
+
.option('--format <fmt>', 'Output format: pretty or json (with --report, also: md)', 'pretty')
|
|
4539
|
+
// HC-054: Night Shift mode. Conditional auto-approve at step_4/step_5
|
|
4540
|
+
// when the agent output passes isCleanOutput's positive allowlist +
|
|
4541
|
+
// inverse blocklist. Extended LLM timeout (480s), orchestrator watchdog
|
|
4542
|
+
// (900s/step), default token budget of 400K × story_count when none
|
|
4543
|
+
// explicitly set. Cap of 25 stories per overnight batch.
|
|
4544
|
+
.option('--overnight', 'Enable Night Shift mode (conditional auto-approve, longer timeouts, max 25 stories)', false)
|
|
4545
|
+
.action(async (opts) => {
|
|
4546
|
+
const config = getConfig();
|
|
4547
|
+
const client = api(config);
|
|
4548
|
+
|
|
4549
|
+
// Enforce mutually-exclusive modes — combining e.g. --list with --report
|
|
4550
|
+
// would silently drop one. Explicit error is friendlier than 'silently
|
|
4551
|
+
// did nothing' or 'silently did the wrong thing'.
|
|
4552
|
+
const modes = [];
|
|
4553
|
+
if (opts.list) modes.push('--list');
|
|
4554
|
+
if (opts.status) modes.push('--status');
|
|
4555
|
+
if (opts.report) modes.push('--report');
|
|
4556
|
+
if (opts.file) modes.push('--file');
|
|
4557
|
+
if (modes.length > 1) {
|
|
4558
|
+
console.error(`These flags are mutually exclusive: ${modes.join(', ')}`);
|
|
4559
|
+
console.error('Specify only one of --file, --status, --report, --list.');
|
|
4560
|
+
process.exit(1);
|
|
4561
|
+
}
|
|
4562
|
+
|
|
4563
|
+
// ── --list ────────────────────────────────────────────────
|
|
4564
|
+
if (opts.list) {
|
|
4565
|
+
try {
|
|
4566
|
+
const { data } = await client.get(`/orchestrate/batch`, { params: { limit: opts.limit } });
|
|
4567
|
+
if (opts.format === 'json') { console.log(JSON.stringify(data, null, 2)); return; }
|
|
4568
|
+
if (!data.batches || data.batches.length === 0) {
|
|
4569
|
+
console.log('No batches found.');
|
|
4570
|
+
return;
|
|
4571
|
+
}
|
|
4572
|
+
console.log('');
|
|
4573
|
+
console.log('Recent batches:');
|
|
4574
|
+
for (const b of data.batches) {
|
|
4575
|
+
console.log(` ${b.batchId} status=${b.status} stories=${b.storyCount} created=${b.createdAt}`);
|
|
4576
|
+
}
|
|
4577
|
+
console.log('');
|
|
4578
|
+
} catch (e) {
|
|
4579
|
+
console.error(`Failed: ${e.response?.data?.error || e.message}`);
|
|
4580
|
+
process.exit(1);
|
|
4581
|
+
}
|
|
4582
|
+
return;
|
|
4583
|
+
}
|
|
4584
|
+
|
|
4585
|
+
// ── --report <batchId> (HC-055) ──────────────────────────
|
|
4586
|
+
if (opts.report) {
|
|
4587
|
+
try {
|
|
4588
|
+
const format = opts.format === 'json' ? 'json' : 'md';
|
|
4589
|
+
const { data, headers } = await client.get(
|
|
4590
|
+
`/orchestrate/batch/${opts.report}/report`,
|
|
4591
|
+
{ params: { format }, responseType: 'text', transformResponse: [(d) => d] }
|
|
4592
|
+
);
|
|
4593
|
+
if (format === 'json') {
|
|
4594
|
+
// JSON path — parse and pretty-print, or save raw.
|
|
4595
|
+
if (opts.save) {
|
|
4596
|
+
fs.writeFileSync(opts.save, data, 'utf8');
|
|
4597
|
+
console.log(`Saved JSON report to ${opts.save}`);
|
|
4598
|
+
} else {
|
|
4599
|
+
try { console.log(JSON.stringify(JSON.parse(data), null, 2)); }
|
|
4600
|
+
catch { console.log(data); }
|
|
4601
|
+
}
|
|
4602
|
+
} else {
|
|
4603
|
+
// Markdown path.
|
|
4604
|
+
if (opts.save) {
|
|
4605
|
+
fs.writeFileSync(opts.save, data, 'utf8');
|
|
4606
|
+
const cached = headers['x-report-cached'] === 'true' ? ' (cached)' : '';
|
|
4607
|
+
console.log(`Saved markdown report${cached} to ${opts.save}`);
|
|
4608
|
+
} else {
|
|
4609
|
+
// Guard: axios with responseType:'text' is expected to return a
|
|
4610
|
+
// string, but a server regression or unusual proxy could send
|
|
4611
|
+
// an empty/undefined body. Don't crash trying to print it.
|
|
4612
|
+
const text = typeof data === 'string' ? data : (data == null ? '' : String(data));
|
|
4613
|
+
process.stdout.write(text);
|
|
4614
|
+
if (!text.endsWith('\n')) process.stdout.write('\n');
|
|
4615
|
+
}
|
|
4616
|
+
}
|
|
4617
|
+
} catch (e) {
|
|
4618
|
+
if (e.response?.status === 404) {
|
|
4619
|
+
console.error('Batch not found.');
|
|
4620
|
+
} else {
|
|
4621
|
+
// responseType:'text' keeps the body as a raw string even for JSON
|
|
4622
|
+
// error responses, so e.response.data is the unparsed `{"error":...}`
|
|
4623
|
+
// and `.error` would be undefined. Parse string bodies before reading.
|
|
4624
|
+
let serverError;
|
|
4625
|
+
const rawData = e.response?.data;
|
|
4626
|
+
if (typeof rawData === 'string') {
|
|
4627
|
+
try { serverError = JSON.parse(rawData)?.error; } catch { /* not JSON */ }
|
|
4628
|
+
} else {
|
|
4629
|
+
serverError = rawData?.error;
|
|
4630
|
+
}
|
|
4631
|
+
console.error(`Failed: ${serverError || e.message}`);
|
|
4632
|
+
}
|
|
4633
|
+
process.exit(1);
|
|
4634
|
+
}
|
|
4635
|
+
return;
|
|
4636
|
+
}
|
|
4637
|
+
|
|
4638
|
+
// ── --status <batchId> ────────────────────────────────────
|
|
4639
|
+
if (opts.status) {
|
|
4640
|
+
try {
|
|
4641
|
+
const { data } = await client.get(`/orchestrate/batch/${opts.status}`);
|
|
4642
|
+
if (opts.format === 'json') { console.log(JSON.stringify(data, null, 2)); return; }
|
|
4643
|
+
console.log('');
|
|
4644
|
+
console.log(`Batch: ${data.batchId}`);
|
|
4645
|
+
console.log(`Status: ${data.status}`);
|
|
4646
|
+
console.log(`Stories: ${data.storyCount}`);
|
|
4647
|
+
console.log(`Tokens: ${(data.totalTokens || 0).toLocaleString()}`);
|
|
4648
|
+
const c = data.counts || {};
|
|
4649
|
+
console.log(`Counts: completed=${c.completed||0} running=${c.running||0} paused=${c.paused||0} queued=${c.queued||0} failed=${c.failed||0} killed=${c.killed||0} stalled=${c.stalled||0}`);
|
|
4650
|
+
console.log('');
|
|
4651
|
+
console.log('Runs:');
|
|
4652
|
+
for (const r of (data.runs || [])) {
|
|
4653
|
+
const icon = r.status === 'completed' ? '✓'
|
|
4654
|
+
: r.status === 'running' ? '⏳'
|
|
4655
|
+
: r.status === 'paused' ? '⏸'
|
|
4656
|
+
: r.status === 'failed' ? '✗'
|
|
4657
|
+
: r.status === 'killed' ? '☠'
|
|
4658
|
+
: r.status === 'stalled' ? '⚠'
|
|
4659
|
+
: '⬜';
|
|
4660
|
+
const step = r.currentStep ? ` @${r.currentStep}` : '';
|
|
4661
|
+
const err = r.errorMessage ? ` err=${r.errorMessage}` : '';
|
|
4662
|
+
console.log(` ${icon} ${r.storyId} (${r.repoName}) ${r.status}${step}${err}`);
|
|
4663
|
+
}
|
|
4664
|
+
console.log('');
|
|
4665
|
+
} catch (e) {
|
|
4666
|
+
if (e.response?.status === 404) console.error('Batch not found.');
|
|
4667
|
+
else console.error(`Failed: ${e.response?.data?.error || e.message}`);
|
|
4668
|
+
process.exit(1);
|
|
4669
|
+
}
|
|
4670
|
+
return;
|
|
4671
|
+
}
|
|
4672
|
+
|
|
4673
|
+
// ── Submit a new batch from --file ─────────────────────────
|
|
4674
|
+
if (!opts.file) {
|
|
4675
|
+
console.error('Specify --file <path>, --status <batchId>, --report <batchId>, or --list.');
|
|
4676
|
+
console.error('Example: hone queue-stories --file stories.txt');
|
|
4677
|
+
process.exit(1);
|
|
4678
|
+
}
|
|
4679
|
+
|
|
4680
|
+
let storyIds;
|
|
4681
|
+
try {
|
|
4682
|
+
const stat = fs.statSync(opts.file);
|
|
4683
|
+
if (stat.size > 16 * 1024) {
|
|
4684
|
+
console.error(`File too large: ${stat.size} bytes (max 16KB).`);
|
|
4685
|
+
process.exit(1);
|
|
4686
|
+
}
|
|
4687
|
+
// Strip UTF-8 BOM (U+FEFF) that Windows editors prepend; trim()
|
|
4688
|
+
// does not remove it, so without this the first storyId becomes
|
|
4689
|
+
// an invisible-prefixed string that fails downstream lookups.
|
|
4690
|
+
const raw = fs.readFileSync(opts.file, 'utf8').replace(/^\uFEFF/, '');
|
|
4691
|
+
// HC-059: each non-comment line is `STORY-ID` OR
|
|
4692
|
+
// `STORY-ID depends:DEP1,DEP2`. Whitespace tolerant. Empty lines and
|
|
4693
|
+
// lines starting with # are ignored.
|
|
4694
|
+
storyIds = raw.split(/\r?\n/)
|
|
4695
|
+
.map(l => l.trim())
|
|
4696
|
+
.filter(l => l && !l.startsWith('#'));
|
|
4697
|
+
} catch (e) {
|
|
4698
|
+
console.error(`Cannot read --file: ${e.message}`);
|
|
4699
|
+
process.exit(1);
|
|
4700
|
+
}
|
|
4701
|
+
if (storyIds.length === 0) {
|
|
4702
|
+
console.error('No story IDs found in file.');
|
|
4703
|
+
process.exit(1);
|
|
4704
|
+
}
|
|
4705
|
+
if (storyIds.length > 50) {
|
|
4706
|
+
console.error(`Too many stories: ${storyIds.length} (max 50 per batch).`);
|
|
4707
|
+
process.exit(1);
|
|
4708
|
+
}
|
|
4709
|
+
|
|
4710
|
+
const repoName = opts.repo || path.basename(process.cwd());
|
|
4711
|
+
let branch = opts.branch;
|
|
4712
|
+
if (!branch) {
|
|
4713
|
+
try { branch = execSync('git symbolic-ref --short HEAD', { encoding: 'utf8' }).trim(); }
|
|
4714
|
+
catch { branch = null; }
|
|
4715
|
+
}
|
|
4716
|
+
|
|
4717
|
+
// HC-059: parse `STORY-A depends:STORY-B,STORY-C` per-line syntax. The
|
|
4718
|
+
// `depends:` token is case-sensitive and must come AFTER the storyId.
|
|
4719
|
+
// Multiple deps separated by commas, whitespace tolerant. Lines without
|
|
4720
|
+
// `depends:` yield no `dependsOn` (server validates absence vs empty).
|
|
4721
|
+
const stories = storyIds.map(line => {
|
|
4722
|
+
const depsMatch = line.match(/^(\S+)\s+depends:(\S+)\s*$/);
|
|
4723
|
+
if (depsMatch) {
|
|
4724
|
+
const [, id, depsCsv] = depsMatch;
|
|
4725
|
+
const dependsOn = depsCsv.split(',').map(s => s.trim()).filter(Boolean);
|
|
4726
|
+
return { storyId: id, repoName, branch, dependsOn };
|
|
4727
|
+
}
|
|
4728
|
+
// Reject ambiguous lines (storyId followed by garbage) \u2014 better than
|
|
4729
|
+
// silently treating `STORY-A something` as just `STORY-A`.
|
|
4730
|
+
if (/\s/.test(line)) {
|
|
4731
|
+
console.error(`Malformed line in --file: "${line}"`);
|
|
4732
|
+
console.error(` Expected: "STORY-ID" OR "STORY-ID depends:STORY-B,STORY-C"`);
|
|
4733
|
+
process.exit(1);
|
|
4734
|
+
}
|
|
4735
|
+
return { storyId: line, repoName, branch };
|
|
4736
|
+
});
|
|
4737
|
+
|
|
4738
|
+
// HC-054: Night Shift opt-in. config.overnight=true plumbs end-to-end
|
|
4739
|
+
// (server validates the 25-story cap + applies default token budget +
|
|
4740
|
+
// denormalizes flag into each child's workflow_runs.config).
|
|
4741
|
+
const body = { stories };
|
|
4742
|
+
if (opts.overnight) {
|
|
4743
|
+
body.config = { overnight: true };
|
|
4744
|
+
}
|
|
4745
|
+
|
|
4746
|
+
try {
|
|
4747
|
+
const { data } = await client.post('/orchestrate/batch', body);
|
|
4748
|
+
|
|
4749
|
+
if (opts.format === 'json') { console.log(JSON.stringify(data, null, 2)); return; }
|
|
4750
|
+
|
|
4751
|
+
console.log('');
|
|
4752
|
+
console.log(`Batch queued: ${data.batchId}`);
|
|
4753
|
+
console.log(`Stories: ${data.storyCount} Enqueued: ${data.enqueued}` +
|
|
4754
|
+
(data.enqueueFailed ? ` Enqueue-failed: ${data.enqueueFailed}` : ''));
|
|
4755
|
+
console.log('');
|
|
4756
|
+
console.log('Stories will run autonomously in batch mode (early gates auto-approve).');
|
|
4757
|
+
console.log('Steps that require human approval (step_4, step_5) will pause those runs.');
|
|
4758
|
+
console.log('');
|
|
4759
|
+
console.log(`Check progress: hone queue-stories --status ${data.batchId}`);
|
|
4760
|
+
console.log(`Per-story: hone run-story <runId> --status`);
|
|
4761
|
+
console.log('');
|
|
4762
|
+
} catch (e) {
|
|
4763
|
+
if (e.response?.status === 429) {
|
|
4764
|
+
console.error(`Org concurrency limit reached: ${e.response.data.error}`);
|
|
4765
|
+
} else if (e.response?.status === 400) {
|
|
4766
|
+
console.error(`Rejected: ${e.response.data.error}`);
|
|
4767
|
+
} else {
|
|
4768
|
+
console.error(`Failed: ${e.response?.data?.error || e.message}`);
|
|
4769
|
+
}
|
|
4770
|
+
process.exit(1);
|
|
4771
|
+
}
|
|
4772
|
+
});
|
|
4773
|
+
|
|
4774
|
+
// ── HC-056: Schedule install (GitHub Actions overnight template) ────────────
|
|
4775
|
+
//
|
|
4776
|
+
// Installs a parameterized .github/workflows/<name>.yml that runs `hone
|
|
4777
|
+
// queue-stories` on a cron schedule. workflow_dispatch trigger gives
|
|
4778
|
+
// on-demand execution from the GitHub Actions UI or `gh workflow run`.
|
|
4779
|
+
//
|
|
4780
|
+
// hone schedule install
|
|
4781
|
+
// hone schedule install --name overnight --cron "0 18 * * 1-5" --file stories.txt
|
|
4782
|
+
//
|
|
4783
|
+
// Future server-side cron is filed as HC-075. For now the workflow lives
|
|
4784
|
+
// in the adopter's repo, which gives them version control on the schedule
|
|
4785
|
+
// + free history in the GitHub Actions UI.
|
|
4786
|
+
program
|
|
4787
|
+
.command('schedule')
|
|
4788
|
+
.description('Manage overnight batch schedules (GitHub Actions templates)')
|
|
4789
|
+
.argument('<action>', 'Action to perform: install')
|
|
4790
|
+
.option('--name <name>', 'Schedule name (used as workflow filename)', 'overnight')
|
|
4791
|
+
.option('--cron <cron>', 'Cron expression (UTC). Default: weekdays 6pm', '0 18 * * 1-5')
|
|
4792
|
+
.option('--file <path>', 'Default stories file path (relative to repo root)', 'stories.txt')
|
|
4793
|
+
.option('--out <dir>', 'Output directory for the workflow file', '.github/workflows')
|
|
4794
|
+
.option('--force', 'Overwrite existing workflow file', false)
|
|
4795
|
+
.action(async (action, opts) => {
|
|
4796
|
+
if (action !== 'install') {
|
|
4797
|
+
console.error(`Unknown schedule action: ${action}. Supported: install`);
|
|
4798
|
+
console.error('(Server-side scheduling is filed as HC-075.)');
|
|
4799
|
+
process.exit(1);
|
|
4800
|
+
}
|
|
4801
|
+
|
|
4802
|
+
// Basic cron validation — five whitespace-separated fields.
|
|
4803
|
+
// Catches the most common mistakes (six fields, single token, etc.)
|
|
4804
|
+
// without trying to validate semantics.
|
|
4805
|
+
const fields = String(opts.cron).trim().split(/\s+/);
|
|
4806
|
+
if (fields.length !== 5) {
|
|
4807
|
+
console.error(`Invalid cron expression: "${opts.cron}"`);
|
|
4808
|
+
console.error('Expected 5 fields: minute hour day-of-month month day-of-week.');
|
|
4809
|
+
console.error('GitHub Actions does NOT support aliases like @daily / @hourly — use the 5-field form.');
|
|
4810
|
+
console.error('Test with https://crontab.guru');
|
|
4811
|
+
process.exit(1);
|
|
4812
|
+
}
|
|
4813
|
+
|
|
4814
|
+
// Name → workflow filename. Reject path traversal / slashes.
|
|
4815
|
+
if (!/^[a-z0-9][a-z0-9_-]{0,40}$/i.test(opts.name)) {
|
|
4816
|
+
console.error(`Invalid --name "${opts.name}". Use letters, digits, hyphens, underscores (max 41 chars).`);
|
|
4817
|
+
process.exit(1);
|
|
4818
|
+
}
|
|
4819
|
+
|
|
4820
|
+
const config = getConfig();
|
|
4821
|
+
const client = api(config);
|
|
4822
|
+
|
|
4823
|
+
// 1. Fetch the template from the server.
|
|
4824
|
+
let template;
|
|
4825
|
+
try {
|
|
4826
|
+
const { data } = await client.get('/scripts/overnight-schedule-template', {
|
|
4827
|
+
responseType: 'text', transformResponse: [(d) => d],
|
|
4828
|
+
});
|
|
4829
|
+
template = data;
|
|
4830
|
+
} catch (e) {
|
|
4831
|
+
// responseType:'text' keeps JSON error bodies as raw strings — parse
|
|
4832
|
+
// before reading .error so the user sees the server's actual message
|
|
4833
|
+
// (e.g. "overnight-schedule.yml not bundled") instead of axios's
|
|
4834
|
+
// generic "Request failed with status code 503".
|
|
4835
|
+
let serverError;
|
|
4836
|
+
const rawData = e.response?.data;
|
|
4837
|
+
if (typeof rawData === 'string') {
|
|
4838
|
+
try { serverError = JSON.parse(rawData)?.error; } catch { /* not JSON */ }
|
|
4839
|
+
} else {
|
|
4840
|
+
serverError = rawData?.error;
|
|
4841
|
+
}
|
|
4842
|
+
console.error(`Failed to fetch template: ${e.response?.status || ''} ${serverError || e.message}`);
|
|
4843
|
+
process.exit(1);
|
|
4844
|
+
}
|
|
4845
|
+
|
|
4846
|
+
// 2. Substitute placeholders. Use replace-all so any future template
|
|
4847
|
+
// additions referencing the same placeholder are handled.
|
|
4848
|
+
const populated = template
|
|
4849
|
+
.replace(/\{\{NAME\}\}/g, opts.name)
|
|
4850
|
+
.replace(/\{\{CRON\}\}/g, opts.cron)
|
|
4851
|
+
.replace(/\{\{STORIES_FILE\}\}/g, opts.file);
|
|
4852
|
+
|
|
4853
|
+
// 3. Decide output path. Default writes to `.github/workflows/<name>.yml`.
|
|
4854
|
+
const outDir = path.resolve(process.cwd(), opts.out);
|
|
4855
|
+
const outFile = path.join(outDir, `${opts.name}.yml`);
|
|
4856
|
+
|
|
4857
|
+
if (fs.existsSync(outFile) && !opts.force) {
|
|
4858
|
+
console.error(`File already exists: ${outFile}`);
|
|
4859
|
+
console.error('Use --force to overwrite, or pick a different --name.');
|
|
4860
|
+
process.exit(1);
|
|
4861
|
+
}
|
|
4862
|
+
|
|
4863
|
+
// 4. Ensure output dir exists.
|
|
4864
|
+
fs.mkdirSync(outDir, { recursive: true });
|
|
4865
|
+
|
|
4866
|
+
// 5. Write the file.
|
|
4867
|
+
fs.writeFileSync(outFile, populated, 'utf8');
|
|
4868
|
+
|
|
4869
|
+
console.log('');
|
|
4870
|
+
console.log(`✓ Installed schedule: ${path.relative(process.cwd(), outFile)}`);
|
|
4871
|
+
console.log('');
|
|
4872
|
+
console.log(' Schedule: ' + opts.cron + ' (UTC)');
|
|
4873
|
+
console.log(' Stories: ' + opts.file);
|
|
4874
|
+
console.log('');
|
|
4875
|
+
console.log('Next steps:');
|
|
4876
|
+
console.log(' 1. Ensure repo secret HONE_TOKEN is set');
|
|
4877
|
+
console.log(' (Settings → Secrets and variables → Actions → New repository secret)');
|
|
4878
|
+
console.log(' 2. Commit and push the workflow file:');
|
|
4879
|
+
console.log(` git add ${path.relative(process.cwd(), outFile)} && git commit -m "feat: nightly Hone batch" && git push`);
|
|
4880
|
+
console.log(` 3. Test on-demand: gh workflow run ${opts.name}.yml --field dry_run=true`);
|
|
4881
|
+
console.log(` Or: GitHub Actions UI → "${opts.name}" → Run workflow`);
|
|
4882
|
+
console.log('');
|
|
4883
|
+
console.log('Test the cron expression at https://crontab.guru');
|
|
4884
|
+
console.log('');
|
|
4885
|
+
});
|
|
4886
|
+
|
|
4432
4887
|
// ── Release Review (pre-deployment holistic review) ──────────────────────────
|
|
4888
|
+
// parseReviewJSON is extracted to cli/lib/parse-review-json.js for unit
|
|
4889
|
+
// coverage. See that file's docstring for the three-shape extraction strategy.
|
|
4890
|
+
|
|
4433
4891
|
program
|
|
4434
4892
|
.command('release-review')
|
|
4435
4893
|
.description('Holistic code review of all changed files before deployment (runs Opus)')
|
|
@@ -4437,15 +4895,49 @@ program
|
|
|
4437
4895
|
.option('--format <fmt>', 'Output format: pretty or json', 'pretty')
|
|
4438
4896
|
.option('--dry-run', 'Show what would be reviewed without calling the LLM', false)
|
|
4439
4897
|
.option('--max-files <n>', 'Max source files to include in review', '40')
|
|
4898
|
+
.option('--provider <name>', 'LLM provider: opus | gh-models (HC-080a-spike)', 'opus')
|
|
4440
4899
|
.action(async (opts) => {
|
|
4441
4900
|
const { execSync } = require('child_process');
|
|
4442
4901
|
const fs = require('fs');
|
|
4443
4902
|
const repoRoot = process.cwd();
|
|
4444
4903
|
|
|
4904
|
+
// When emitting JSON to stdout, route banner/status output to stderr so
|
|
4905
|
+
// `tee file.json` in CI captures pure JSON. Otherwise the artifact ends
|
|
4906
|
+
// up with `Hone AI — Production Review\n=====\n...` mixed in front of
|
|
4907
|
+
// the JSON envelope, breaking JSON.parse. (HC-080a-spike PR #313 data
|
|
4908
|
+
// point 1 found this — both lanes' artifacts were unparseable.)
|
|
4909
|
+
//
|
|
4910
|
+
// Normalize the format value defensively — `--format JSON` or
|
|
4911
|
+
// `--format=json ` (trailing space) would otherwise silently fall into
|
|
4912
|
+
// pretty-mode and re-pollute the artifact.
|
|
4913
|
+
const isJsonOut = String(opts.format || '').trim().toLowerCase() === 'json';
|
|
4914
|
+
const banner = (line) => (isJsonOut ? console.error(line) : console.log(line));
|
|
4915
|
+
|
|
4916
|
+
// emitStatusEnvelope is for non-success exit paths in JSON mode (no
|
|
4917
|
+
// changes, auth failure, rate limit, empty response). Ensures the
|
|
4918
|
+
// artifact is ALWAYS valid JSON, never zero-byte. The status field
|
|
4919
|
+
// tells compare-reviews.js to treat this row as a known non-finding
|
|
4920
|
+
// outcome rather than an unparseable file.
|
|
4921
|
+
const emitStatusEnvelope = (status, extra = {}) => {
|
|
4922
|
+
if (!isJsonOut) return;
|
|
4923
|
+
console.log(JSON.stringify({
|
|
4924
|
+
status,
|
|
4925
|
+
base: opts.base, // raw user input — preserved for audit
|
|
4926
|
+
provider: opts.provider,
|
|
4927
|
+
...extra,
|
|
4928
|
+
}, null, 2));
|
|
4929
|
+
};
|
|
4930
|
+
|
|
4931
|
+
// Normalize the base ref to avoid the double-`origin/` prefix bug. The
|
|
4932
|
+
// CI step passes `--base origin/main` and the previous code prefixed
|
|
4933
|
+
// again → `origin/origin/main` → git rejected → silent fallback to
|
|
4934
|
+
// `HEAD~10`. Both lanes reviewed the wrong diff in PR #313 first run.
|
|
4935
|
+
const baseRef = resolveBaseRef(opts.base);
|
|
4936
|
+
|
|
4445
4937
|
// 1. Get changed files
|
|
4446
4938
|
let changedFiles;
|
|
4447
4939
|
try {
|
|
4448
|
-
const raw = execSync(`git diff --name-only
|
|
4940
|
+
const raw = execSync(`git diff --name-only ${baseRef}...HEAD`, { encoding: 'utf8', cwd: repoRoot });
|
|
4449
4941
|
changedFiles = raw.trim().split('\n').filter(Boolean);
|
|
4450
4942
|
} catch {
|
|
4451
4943
|
try {
|
|
@@ -4458,7 +4950,8 @@ program
|
|
|
4458
4950
|
}
|
|
4459
4951
|
|
|
4460
4952
|
if (changedFiles.length === 0) {
|
|
4461
|
-
|
|
4953
|
+
banner('No changed files found. Nothing to review.');
|
|
4954
|
+
emitStatusEnvelope('no_changes', { resolvedBase: baseRef });
|
|
4462
4955
|
process.exit(0);
|
|
4463
4956
|
}
|
|
4464
4957
|
|
|
@@ -4472,32 +4965,49 @@ program
|
|
|
4472
4965
|
const maxFiles = parseInt(opts.maxFiles, 10) || 40;
|
|
4473
4966
|
const filesToReview = sourceFiles.slice(0, maxFiles);
|
|
4474
4967
|
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4968
|
+
banner('');
|
|
4969
|
+
banner('Hone AI — Production Review');
|
|
4970
|
+
banner('================================');
|
|
4971
|
+
banner(`Base: ${baseRef}`);
|
|
4972
|
+
banner(`Changed files: ${changedFiles.length} total, ${sourceFiles.length} source, ${filesToReview.length} to review`);
|
|
4973
|
+
banner('');
|
|
4481
4974
|
|
|
4482
4975
|
if (opts.dryRun) {
|
|
4483
|
-
|
|
4484
|
-
for (const f of filesToReview)
|
|
4485
|
-
if (sourceFiles.length > maxFiles)
|
|
4976
|
+
banner('Source files that would be reviewed:');
|
|
4977
|
+
for (const f of filesToReview) banner(` ${f}`);
|
|
4978
|
+
if (sourceFiles.length > maxFiles) banner(` ... and ${sourceFiles.length - maxFiles} more (increase --max-files)`);
|
|
4486
4979
|
process.exit(0);
|
|
4487
4980
|
}
|
|
4488
4981
|
|
|
4489
|
-
// 3.
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
console.error('ANTHROPIC_API_KEY not set. Required for production review (Opus model).');
|
|
4493
|
-
console.error('Set it: export ANTHROPIC_API_KEY=sk-ant-...');
|
|
4982
|
+
// 3. Provider validation + credential check
|
|
4983
|
+
if (opts.provider !== 'opus' && opts.provider !== 'gh-models') {
|
|
4984
|
+
console.error(`Invalid --provider: ${opts.provider}. Use 'opus' or 'gh-models'.`);
|
|
4494
4985
|
process.exit(1);
|
|
4495
4986
|
}
|
|
4496
4987
|
|
|
4988
|
+
let apiKey, providerLabel;
|
|
4989
|
+
if (opts.provider === 'gh-models') {
|
|
4990
|
+
apiKey = process.env.GITHUB_TOKEN;
|
|
4991
|
+
providerLabel = 'GitHub Models (openai/gpt-4.1)';
|
|
4992
|
+
if (!apiKey) {
|
|
4993
|
+
console.error('GITHUB_TOKEN not set. Required for --provider gh-models.');
|
|
4994
|
+
console.error('In CI: GITHUB_TOKEN is auto-injected. Locally: export GITHUB_TOKEN=<your PAT>.');
|
|
4995
|
+
process.exit(1);
|
|
4996
|
+
}
|
|
4997
|
+
} else {
|
|
4998
|
+
apiKey = process.env.ANTHROPIC_API_KEY;
|
|
4999
|
+
providerLabel = 'Anthropic Opus (claude-opus-4-20250514)';
|
|
5000
|
+
if (!apiKey) {
|
|
5001
|
+
console.error('ANTHROPIC_API_KEY not set. Required for --provider opus.');
|
|
5002
|
+
console.error('Set it: export ANTHROPIC_API_KEY=sk-ant-...');
|
|
5003
|
+
process.exit(1);
|
|
5004
|
+
}
|
|
5005
|
+
}
|
|
5006
|
+
|
|
4497
5007
|
// 4. Build the diff content (truncated per-file to stay within context)
|
|
4498
5008
|
let diffContent;
|
|
4499
5009
|
try {
|
|
4500
|
-
diffContent = execSync(`git diff
|
|
5010
|
+
diffContent = execSync(`git diff ${baseRef}...HEAD -- ${filesToReview.map(f => `'${f}'`).join(' ')}`, {
|
|
4501
5011
|
encoding: 'utf8', cwd: repoRoot, maxBuffer: 10 * 1024 * 1024,
|
|
4502
5012
|
});
|
|
4503
5013
|
} catch {
|
|
@@ -4509,10 +5019,13 @@ program
|
|
|
4509
5019
|
}
|
|
4510
5020
|
}
|
|
4511
5021
|
|
|
4512
|
-
//
|
|
4513
|
-
|
|
5022
|
+
// Provider-specific truncation. GH Models GPT-4.1 has an 8K-token
|
|
5023
|
+
// request-body cap via models.github.ai/inference — 100K chars overflows.
|
|
5024
|
+
// See cli/lib/release-review-config.js for the budget math.
|
|
5025
|
+
const MAX_DIFF_CHARS = getMaxDiffChars(opts.provider);
|
|
4514
5026
|
if (diffContent.length > MAX_DIFF_CHARS) {
|
|
4515
|
-
diffContent = diffContent.slice(0, MAX_DIFF_CHARS) +
|
|
5027
|
+
diffContent = diffContent.slice(0, MAX_DIFF_CHARS) +
|
|
5028
|
+
`\n\n[... diff truncated at ${MAX_DIFF_CHARS} chars for --provider ${opts.provider} ...]`;
|
|
4516
5029
|
}
|
|
4517
5030
|
|
|
4518
5031
|
// 5. Build the prompt
|
|
@@ -4569,72 +5082,166 @@ program
|
|
|
4569
5082
|
'Review ALL files holistically. Return findings as JSON.',
|
|
4570
5083
|
].join('\n');
|
|
4571
5084
|
|
|
4572
|
-
|
|
4573
|
-
|
|
5085
|
+
banner(`Calling ${providerLabel}...`);
|
|
5086
|
+
banner('');
|
|
4574
5087
|
|
|
4575
|
-
// 6. Call
|
|
5088
|
+
// 6. Call LLM (provider-branched, HC-080a-spike)
|
|
5089
|
+
// max_tokens is held SYMMETRIC across providers so the HC-080a-spike
|
|
5090
|
+
// comparison measures model capability, not output budget. 4096 is the
|
|
5091
|
+
// safe ceiling for openai/gpt-4.1 via GH Models; Opus supports more but
|
|
5092
|
+
// running it with 4096 keeps the comparison apples-to-apples.
|
|
5093
|
+
const MAX_OUTPUT_TOKENS = 4096;
|
|
5094
|
+
const startedAt = Date.now();
|
|
4576
5095
|
try {
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
|
|
4588
|
-
|
|
4589
|
-
|
|
5096
|
+
let responseText, inputTokens, outputTokens, modelLabel;
|
|
5097
|
+
|
|
5098
|
+
if (opts.provider === 'gh-models') {
|
|
5099
|
+
// GitHub Models — free LLM inference using GITHUB_TOKEN. Mirrors the
|
|
5100
|
+
// path used by server/scripts/ai-reviewer.js for per-PR review.
|
|
5101
|
+
const { data } = await axios.post(
|
|
5102
|
+
'https://models.github.ai/inference/chat/completions',
|
|
5103
|
+
{
|
|
5104
|
+
model: 'openai/gpt-4.1',
|
|
5105
|
+
messages: [
|
|
5106
|
+
{ role: 'system', content: systemPrompt },
|
|
5107
|
+
{ role: 'user', content: userPrompt },
|
|
5108
|
+
],
|
|
5109
|
+
max_tokens: MAX_OUTPUT_TOKENS,
|
|
5110
|
+
},
|
|
5111
|
+
{
|
|
5112
|
+
headers: {
|
|
5113
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
5114
|
+
'Content-Type': 'application/json',
|
|
5115
|
+
},
|
|
5116
|
+
timeout: 120000,
|
|
5117
|
+
}
|
|
5118
|
+
);
|
|
5119
|
+
responseText = data.choices?.[0]?.message?.content || '';
|
|
5120
|
+
inputTokens = data.usage?.prompt_tokens || 0;
|
|
5121
|
+
outputTokens = data.usage?.completion_tokens || 0;
|
|
5122
|
+
modelLabel = 'openai/gpt-4.1';
|
|
5123
|
+
} else {
|
|
5124
|
+
const { data } = await axios.post('https://api.anthropic.com/v1/messages', {
|
|
5125
|
+
model: 'claude-opus-4-20250514',
|
|
5126
|
+
max_tokens: MAX_OUTPUT_TOKENS,
|
|
5127
|
+
system: systemPrompt,
|
|
5128
|
+
messages: [{ role: 'user', content: userPrompt }],
|
|
5129
|
+
}, {
|
|
5130
|
+
headers: {
|
|
5131
|
+
'x-api-key': apiKey,
|
|
5132
|
+
'anthropic-version': '2023-06-01',
|
|
5133
|
+
'content-type': 'application/json',
|
|
5134
|
+
},
|
|
5135
|
+
timeout: 120000,
|
|
5136
|
+
});
|
|
5137
|
+
responseText = data.content?.[0]?.text || '';
|
|
5138
|
+
inputTokens = data.usage?.input_tokens || 0;
|
|
5139
|
+
outputTokens = data.usage?.output_tokens || 0;
|
|
5140
|
+
modelLabel = 'claude-opus-4-20250514';
|
|
5141
|
+
}
|
|
4590
5142
|
|
|
4591
|
-
const
|
|
5143
|
+
const elapsedMs = Date.now() - startedAt;
|
|
5144
|
+
|
|
5145
|
+
// Empty-response detection: some providers return HTTP 200 with empty
|
|
5146
|
+
// choices/content (content-filter trip, soft quota limit, model alias
|
|
5147
|
+
// typo). Without this guard the CLI would silently exit 0 and the CI
|
|
5148
|
+
// artifact would look like a clean pass — a false-negative on the
|
|
5149
|
+
// entire review.
|
|
5150
|
+
if (!responseText || responseText.trim().length === 0) {
|
|
5151
|
+
console.error(`Empty response from --provider ${opts.provider}. ` +
|
|
5152
|
+
`Possible causes: content filter, quota exhausted, or model name rejected.`);
|
|
5153
|
+
emitStatusEnvelope('empty_response', {
|
|
5154
|
+
resolvedBase: baseRef,
|
|
5155
|
+
model: modelLabel,
|
|
5156
|
+
inputTokens,
|
|
5157
|
+
outputTokens,
|
|
5158
|
+
elapsedMs: Date.now() - startedAt,
|
|
5159
|
+
});
|
|
5160
|
+
process.exit(1);
|
|
5161
|
+
}
|
|
4592
5162
|
|
|
4593
|
-
//
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
5163
|
+
// Parse the response once, robustly. Try the full string first (the
|
|
5164
|
+
// happy path when the LLM emits clean JSON), then fall back to greedy
|
|
5165
|
+
// brace extraction. A failed parse leaves parsed = null and downstream
|
|
5166
|
+
// code uses raw.
|
|
5167
|
+
const parsed = parseReviewJSON(responseText);
|
|
5168
|
+
|
|
5169
|
+
// 7. Output envelope + content. Envelope fields are spread LAST so an
|
|
5170
|
+
// LLM cannot rewrite audit fields (provider, model, inputTokens, etc.)
|
|
5171
|
+
// via prompt injection in the diff content.
|
|
5172
|
+
//
|
|
5173
|
+
// base (raw user input) is preserved alongside resolvedBase (the actual
|
|
5174
|
+
// git revision that was diffed) so audit trails are complete and
|
|
5175
|
+
// comparison tooling can distinguish "the operator passed `main`" from
|
|
5176
|
+
// "the operator passed `origin/main`."
|
|
5177
|
+
if (isJsonOut) {
|
|
5178
|
+
const envelope = {
|
|
5179
|
+
status: 'reviewed',
|
|
5180
|
+
base: opts.base,
|
|
5181
|
+
resolvedBase: baseRef,
|
|
5182
|
+
provider: opts.provider,
|
|
5183
|
+
totalFiles: changedFiles.length,
|
|
5184
|
+
sourceFiles: sourceFiles.length,
|
|
5185
|
+
reviewedFiles: filesToReview.length,
|
|
5186
|
+
model: modelLabel,
|
|
5187
|
+
inputTokens,
|
|
5188
|
+
outputTokens,
|
|
5189
|
+
elapsedMs,
|
|
5190
|
+
};
|
|
5191
|
+
if (parsed) {
|
|
5192
|
+
console.log(JSON.stringify({ ...parsed, ...envelope }, null, 2));
|
|
4613
5193
|
} else {
|
|
4614
|
-
console.log(JSON.stringify({ raw: responseText }, null, 2));
|
|
5194
|
+
console.log(JSON.stringify({ ...envelope, raw: responseText }, null, 2));
|
|
4615
5195
|
}
|
|
4616
5196
|
} else {
|
|
4617
5197
|
console.log(responseText);
|
|
4618
5198
|
}
|
|
4619
5199
|
|
|
4620
|
-
// 8. Exit code
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
5200
|
+
// 8. Exit code — defense in depth:
|
|
5201
|
+
// (a) structured check against the parsed JSON, then
|
|
5202
|
+
// (b) loose substring check on the raw response (catches LLMs that
|
|
5203
|
+
// emit markdown-formatted CRITICAL findings without strict JSON).
|
|
5204
|
+
let hasCritical = false, doNotDeploy = false;
|
|
5205
|
+
if (parsed) {
|
|
5206
|
+
hasCritical = (parsed.summary?.critical || 0) > 0 ||
|
|
5207
|
+
(Array.isArray(parsed.findings) &&
|
|
5208
|
+
parsed.findings.some(f => String(f.severity).toUpperCase() === 'CRITICAL'));
|
|
5209
|
+
doNotDeploy = parsed.recommendation === 'DO_NOT_DEPLOY';
|
|
5210
|
+
}
|
|
5211
|
+
if (!hasCritical) {
|
|
5212
|
+
// Loose match — case-insensitive, no quote requirement. False-positive
|
|
5213
|
+
// tolerance is acceptable here because the cost of a false-positive is
|
|
5214
|
+
// "CI shows yellow, human looks at the report"; the cost of a false-
|
|
5215
|
+
// negative is "real CRITICAL bug ships to prod undetected".
|
|
5216
|
+
hasCritical = /\bCRITICAL\b/.test(responseText);
|
|
5217
|
+
}
|
|
5218
|
+
if (!doNotDeploy) {
|
|
5219
|
+
doNotDeploy = /\bDO_NOT_DEPLOY\b/.test(responseText);
|
|
5220
|
+
}
|
|
5221
|
+
if (hasCritical || doNotDeploy) {
|
|
5222
|
+
banner('');
|
|
5223
|
+
banner('CRITICAL issues found. Fix before deploying.');
|
|
4626
5224
|
process.exit(1);
|
|
4627
5225
|
}
|
|
4628
5226
|
} catch (e) {
|
|
4629
5227
|
const status = e.response?.status;
|
|
4630
5228
|
const msg = e.response?.data?.error?.message || e.message;
|
|
4631
|
-
|
|
4632
|
-
|
|
5229
|
+
let kind = 'http_error';
|
|
5230
|
+
if (status === 401 || status === 403) {
|
|
5231
|
+
console.error(`Invalid auth for --provider ${opts.provider}. Check credential and try again.`);
|
|
5232
|
+
kind = 'auth_error';
|
|
4633
5233
|
} else if (status === 429) {
|
|
4634
|
-
console.error(
|
|
5234
|
+
console.error(`Rate limited by ${opts.provider}. Try again shortly.`);
|
|
5235
|
+
kind = 'rate_limited';
|
|
4635
5236
|
} else {
|
|
4636
|
-
console.error(`Production review failed: ${msg}`);
|
|
5237
|
+
console.error(`Production review failed (${opts.provider}): ${msg}`);
|
|
4637
5238
|
}
|
|
5239
|
+
emitStatusEnvelope(kind, {
|
|
5240
|
+
resolvedBase: baseRef,
|
|
5241
|
+
httpStatus: status || null,
|
|
5242
|
+
errorMessage: msg,
|
|
5243
|
+
elapsedMs: Date.now() - startedAt,
|
|
5244
|
+
});
|
|
4638
5245
|
process.exit(1);
|
|
4639
5246
|
}
|
|
4640
5247
|
});
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* compare-reviews.js — HC-080a-spike side-by-side comparison helper.
|
|
4
|
+
*
|
|
5
|
+
* Reads two release-review JSON artifacts (Opus + GH Models GPT-4.1) and
|
|
6
|
+
* prints a markdown comparison table that can be pasted into the decision
|
|
7
|
+
* record in docs/architecture/release-review-llm-provider-decision.md.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* node cli/lib/compare-reviews.js <opus-findings.json> <gh-models-findings.json>
|
|
11
|
+
*
|
|
12
|
+
* Both files should be JSON envelopes produced by `hone release-review --format json`
|
|
13
|
+
* (see cli/hone-cli.js release-review handler — both providers emit the same
|
|
14
|
+
* envelope shape).
|
|
15
|
+
*
|
|
16
|
+
* The script is intentionally tolerant of partial/raw outputs: if a provider
|
|
17
|
+
* returned a non-JSON response, the comparison still shows what's there and
|
|
18
|
+
* flags the format problem in the rubric.
|
|
19
|
+
*
|
|
20
|
+
* Exit codes:
|
|
21
|
+
* 0 — both files parsed, comparison printed
|
|
22
|
+
* 1 — missing file, invalid JSON, or other unrecoverable error
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
const fs = require('fs');
|
|
26
|
+
const path = require('path');
|
|
27
|
+
|
|
28
|
+
// ── helpers ────────────────────────────────────────────────────────────────
|
|
29
|
+
|
|
30
|
+
// Normalize file paths for cross-provider equality. Strips leading "./",
|
|
31
|
+
// converts backslashes, and removes duplicate separators. Coarse: does NOT
|
|
32
|
+
// resolve symlinks, just compares string-equivalent paths.
|
|
33
|
+
function normalizePath(p) {
|
|
34
|
+
if (typeof p !== 'string') return '';
|
|
35
|
+
let n = p.replace(/\\/g, '/').replace(/^\.\//, '').trim();
|
|
36
|
+
// collapse repeated slashes but preserve leading slash
|
|
37
|
+
n = n.replace(/\/+/g, '/');
|
|
38
|
+
return n;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Escape a string for safe interpolation into a markdown table cell or bullet.
|
|
42
|
+
// Replaces pipes (`|` — breaks table column count), backticks (corrupt code
|
|
43
|
+
// spans), and newlines (break bullet lines). Backticks are removed (rather
|
|
44
|
+
// than backslash-escaped) because backslash-escapes inside code spans are NOT
|
|
45
|
+
// honored by CommonMark — replacing with a similar-looking char preserves
|
|
46
|
+
// rendering. This is defense against LLM-emitted content that may contain
|
|
47
|
+
// any of these chars in `issue` / `file` strings.
|
|
48
|
+
function escapeMd(s) {
|
|
49
|
+
if (s == null) return '';
|
|
50
|
+
return String(s)
|
|
51
|
+
.replace(/\|/g, '\\|')
|
|
52
|
+
.replace(/`/g, '‘') // ` → left single quote (visually close)
|
|
53
|
+
.replace(/\r?\n/g, ' '); // newlines → spaces inside a cell
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function loadEnvelope(filePath) {
|
|
57
|
+
if (!fs.existsSync(filePath)) {
|
|
58
|
+
return { error: `file not found: ${filePath}` };
|
|
59
|
+
}
|
|
60
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
61
|
+
try {
|
|
62
|
+
return JSON.parse(raw);
|
|
63
|
+
} catch (e) {
|
|
64
|
+
return { error: `JSON parse failed: ${e.message}`, raw };
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function bucketSeverity(findings) {
|
|
69
|
+
const buckets = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, LOW: 0, OTHER: 0 };
|
|
70
|
+
if (!Array.isArray(findings)) return buckets;
|
|
71
|
+
for (const f of findings) {
|
|
72
|
+
const sev = String(f.severity || '').toUpperCase();
|
|
73
|
+
if (sev in buckets) buckets[sev]++;
|
|
74
|
+
else buckets.OTHER++;
|
|
75
|
+
}
|
|
76
|
+
return buckets;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function describe(env) {
|
|
80
|
+
if (env.error) {
|
|
81
|
+
return {
|
|
82
|
+
ok: false,
|
|
83
|
+
error: env.error,
|
|
84
|
+
status: 'unparseable',
|
|
85
|
+
provider: 'unknown',
|
|
86
|
+
model: 'unknown',
|
|
87
|
+
counts: bucketSeverity([]),
|
|
88
|
+
totalFindings: 0,
|
|
89
|
+
cost: { inputTokens: 0, outputTokens: 0 },
|
|
90
|
+
elapsedMs: null,
|
|
91
|
+
recommendation: 'n/a',
|
|
92
|
+
formatOk: false,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
const findings = Array.isArray(env.findings) ? env.findings : [];
|
|
96
|
+
const counts = bucketSeverity(findings);
|
|
97
|
+
// `status` field (added by HC-080a-spike fix pass) tells operators whether
|
|
98
|
+
// this row is a real review, a known error envelope, or an unknown shape.
|
|
99
|
+
// Status values produced by the CLI: 'reviewed' | 'no_changes' |
|
|
100
|
+
// 'empty_response' | 'auth_error' | 'rate_limited' | 'http_error'.
|
|
101
|
+
// Default to 'reviewed' if absent so older artifacts still display.
|
|
102
|
+
const status = env.status || (findings.length > 0 ? 'reviewed' : 'unknown');
|
|
103
|
+
return {
|
|
104
|
+
ok: true,
|
|
105
|
+
status,
|
|
106
|
+
provider: env.provider || 'unknown',
|
|
107
|
+
model: env.model || 'unknown',
|
|
108
|
+
counts,
|
|
109
|
+
totalFindings: findings.length,
|
|
110
|
+
cost: {
|
|
111
|
+
inputTokens: env.inputTokens || 0,
|
|
112
|
+
outputTokens: env.outputTokens || 0,
|
|
113
|
+
},
|
|
114
|
+
elapsedMs: typeof env.elapsedMs === 'number' ? env.elapsedMs : null,
|
|
115
|
+
recommendation: env.recommendation || (env.raw ? 'raw-output' : 'n/a'),
|
|
116
|
+
formatOk: !env.raw,
|
|
117
|
+
summary: env.summary || null,
|
|
118
|
+
findings,
|
|
119
|
+
httpStatus: env.httpStatus ?? null,
|
|
120
|
+
errorMessage: env.errorMessage || null,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ── Findings overlap detection ────────────────────────────────────────────
|
|
125
|
+
//
|
|
126
|
+
// Two findings are considered the "same" bug if they reference the same
|
|
127
|
+
// normalized file path AND both have explicit line numbers within ±3 of each
|
|
128
|
+
// other. Findings without explicit line numbers are NOT auto-merged — two
|
|
129
|
+
// unrelated file-level findings on the same file would otherwise collapse
|
|
130
|
+
// into a false overlap.
|
|
131
|
+
//
|
|
132
|
+
// This is coarse — a precise match would require semantic similarity scoring,
|
|
133
|
+
// which is overkill for a 5-release spike. The output is meant to anchor
|
|
134
|
+
// manual review, not auto-decide.
|
|
135
|
+
function findOverlap(aFindings, bFindings) {
|
|
136
|
+
const overlap = [];
|
|
137
|
+
const aOnly = [];
|
|
138
|
+
const bUsed = new Set();
|
|
139
|
+
|
|
140
|
+
for (const a of aFindings) {
|
|
141
|
+
let matched = false;
|
|
142
|
+
const aFile = normalizePath(a.file);
|
|
143
|
+
const aLine = Number.isFinite(a.line) ? a.line : null;
|
|
144
|
+
for (let i = 0; i < bFindings.length; i++) {
|
|
145
|
+
if (bUsed.has(i)) continue;
|
|
146
|
+
const b = bFindings[i];
|
|
147
|
+
const bFile = normalizePath(b.file);
|
|
148
|
+
const bLine = Number.isFinite(b.line) ? b.line : null;
|
|
149
|
+
if (aFile !== bFile || aFile === '') continue;
|
|
150
|
+
// Require both lines to be explicit AND nearby. Two undefined lines
|
|
151
|
+
// do NOT count as a match — they're treated as distinct file-level
|
|
152
|
+
// findings until a human merges them manually in the rubric.
|
|
153
|
+
if (aLine === null || bLine === null) continue;
|
|
154
|
+
if (Math.abs(aLine - bLine) > 3) continue;
|
|
155
|
+
overlap.push({ a, b });
|
|
156
|
+
bUsed.add(i);
|
|
157
|
+
matched = true;
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
if (!matched) aOnly.push(a);
|
|
161
|
+
}
|
|
162
|
+
const bOnly = bFindings.filter((_, i) => !bUsed.has(i));
|
|
163
|
+
return { overlap, aOnly, bOnly };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function renderMarkdown(opus, gh) {
|
|
167
|
+
const lines = [];
|
|
168
|
+
lines.push('# Release-Review Provider Comparison (HC-080a-spike)');
|
|
169
|
+
lines.push('');
|
|
170
|
+
lines.push(`_Generated: ${new Date().toISOString()}_`);
|
|
171
|
+
lines.push('');
|
|
172
|
+
|
|
173
|
+
lines.push('## Summary');
|
|
174
|
+
lines.push('');
|
|
175
|
+
lines.push('| Dimension | Opus (Anthropic) | GPT-4.1 (GitHub Models) |');
|
|
176
|
+
lines.push('|---|---|---|');
|
|
177
|
+
lines.push(`| Provider | ${opus.provider} | ${gh.provider} |`);
|
|
178
|
+
lines.push(`| Status | ${opus.status} | ${gh.status} |`);
|
|
179
|
+
lines.push(`| Model | ${opus.model} | ${gh.model} |`);
|
|
180
|
+
lines.push(`| Format OK | ${opus.formatOk ? 'yes' : 'NO (raw)'} | ${gh.formatOk ? 'yes' : 'NO (raw)'} |`);
|
|
181
|
+
lines.push(`| Latency (ms) | ${opus.elapsedMs ?? 'n/a'} | ${gh.elapsedMs ?? 'n/a'} |`);
|
|
182
|
+
lines.push(`| Input tokens | ${opus.cost.inputTokens} | ${gh.cost.inputTokens} |`);
|
|
183
|
+
lines.push(`| Output tokens | ${opus.cost.outputTokens} | ${gh.cost.outputTokens} |`);
|
|
184
|
+
lines.push(`| Total findings | ${opus.totalFindings} | ${gh.totalFindings} |`);
|
|
185
|
+
lines.push(`| Findings: CRITICAL | ${opus.counts.CRITICAL} | ${gh.counts.CRITICAL} |`);
|
|
186
|
+
lines.push(`| Findings: HIGH | ${opus.counts.HIGH} | ${gh.counts.HIGH} |`);
|
|
187
|
+
lines.push(`| Findings: MEDIUM | ${opus.counts.MEDIUM} | ${gh.counts.MEDIUM} |`);
|
|
188
|
+
lines.push(`| Findings: LOW | ${opus.counts.LOW} | ${gh.counts.LOW} |`);
|
|
189
|
+
lines.push(`| Recommendation | ${opus.recommendation} | ${gh.recommendation} |`);
|
|
190
|
+
lines.push('');
|
|
191
|
+
|
|
192
|
+
if (!opus.ok || !gh.ok) {
|
|
193
|
+
lines.push('## Errors');
|
|
194
|
+
lines.push('');
|
|
195
|
+
if (!opus.ok) lines.push(`- Opus envelope: ${opus.error}`);
|
|
196
|
+
if (!gh.ok) lines.push(`- GH Models envelope: ${gh.error}`);
|
|
197
|
+
lines.push('');
|
|
198
|
+
return lines.join('\n');
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const { overlap, aOnly: opusOnly, bOnly: ghOnly } = findOverlap(opus.findings, gh.findings);
|
|
202
|
+
|
|
203
|
+
lines.push('## Overlap analysis (coarse: same file + line ±3)');
|
|
204
|
+
lines.push('');
|
|
205
|
+
lines.push(`- Shared findings (both providers caught): ${overlap.length}`);
|
|
206
|
+
lines.push(`- Opus-only findings: ${opusOnly.length}`);
|
|
207
|
+
lines.push(`- GH Models-only findings: ${ghOnly.length}`);
|
|
208
|
+
lines.push('');
|
|
209
|
+
|
|
210
|
+
if (overlap.length) {
|
|
211
|
+
lines.push('### Shared findings (sample, up to 5)');
|
|
212
|
+
lines.push('');
|
|
213
|
+
for (const { a, b } of overlap.slice(0, 5)) {
|
|
214
|
+
lines.push(`- **${escapeMd(a.file)}:${a.line}** — Opus says: _${escapeMd(a.issue)}_ — GPT says: _${escapeMd(b.issue)}_`);
|
|
215
|
+
}
|
|
216
|
+
lines.push('');
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (opusOnly.length) {
|
|
220
|
+
lines.push('### Opus-only findings (POTENTIAL QUALITY GAP if real bugs)');
|
|
221
|
+
lines.push('');
|
|
222
|
+
for (const f of opusOnly.slice(0, 10)) {
|
|
223
|
+
lines.push(`- [${escapeMd(f.severity)}] **${escapeMd(f.file)}:${f.line ?? '?'}** — ${escapeMd(f.issue)}`);
|
|
224
|
+
}
|
|
225
|
+
if (opusOnly.length > 10) lines.push(`- ... and ${opusOnly.length - 10} more`);
|
|
226
|
+
lines.push('');
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (ghOnly.length) {
|
|
230
|
+
lines.push('### GH Models-only findings');
|
|
231
|
+
lines.push('');
|
|
232
|
+
for (const f of ghOnly.slice(0, 10)) {
|
|
233
|
+
lines.push(`- [${escapeMd(f.severity)}] **${escapeMd(f.file)}:${f.line ?? '?'}** — ${escapeMd(f.issue)}`);
|
|
234
|
+
}
|
|
235
|
+
if (ghOnly.length > 10) lines.push(`- ... and ${ghOnly.length - 10} more`);
|
|
236
|
+
lines.push('');
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
lines.push('## Decision rubric — fill after manual review');
|
|
240
|
+
lines.push('');
|
|
241
|
+
lines.push('- [ ] Did GH Models catch ≥80% of Opus HIGH/MEDIUM findings on this release?');
|
|
242
|
+
lines.push('- [ ] Were any Opus-only findings actually real bugs (not noise)?');
|
|
243
|
+
lines.push('- [ ] Were any GH Models-only findings actually real bugs that Opus missed?');
|
|
244
|
+
lines.push('- [ ] Was the output format clean for both providers?');
|
|
245
|
+
lines.push('');
|
|
246
|
+
lines.push('Append this comparison row to the rubric in');
|
|
247
|
+
lines.push('`docs/architecture/release-review-llm-provider-decision.md`.');
|
|
248
|
+
|
|
249
|
+
return lines.join('\n');
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function main(argv) {
|
|
253
|
+
if (argv.length < 2) {
|
|
254
|
+
console.error('Usage: node cli/lib/compare-reviews.js <opus-findings.json> <gh-models-findings.json>');
|
|
255
|
+
console.error('');
|
|
256
|
+
console.error('Download both artifacts from a release CI run, then run this script.');
|
|
257
|
+
process.exit(1);
|
|
258
|
+
}
|
|
259
|
+
const opusFile = path.resolve(argv[0]);
|
|
260
|
+
const ghFile = path.resolve(argv[1]);
|
|
261
|
+
const opus = describe(loadEnvelope(opusFile));
|
|
262
|
+
const gh = describe(loadEnvelope(ghFile));
|
|
263
|
+
console.log(renderMarkdown(opus, gh));
|
|
264
|
+
process.exit(0);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (require.main === module) {
|
|
268
|
+
main(process.argv.slice(2));
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
module.exports = {
|
|
272
|
+
loadEnvelope,
|
|
273
|
+
describe,
|
|
274
|
+
findOverlap,
|
|
275
|
+
bucketSeverity,
|
|
276
|
+
renderMarkdown,
|
|
277
|
+
normalizePath,
|
|
278
|
+
escapeMd,
|
|
279
|
+
};
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* parse-review-json.js — HC-080a-spike extraction helper.
|
|
4
|
+
*
|
|
5
|
+
* Robust JSON extraction from LLM output. Used by `hone release-review` to
|
|
6
|
+
* parse the structured `{ findings, summary, recommendation }` response from
|
|
7
|
+
* either Anthropic Opus or GitHub Models GPT-4.1.
|
|
8
|
+
*
|
|
9
|
+
* Why this exists: LLMs don't reliably emit pure JSON. The function tries
|
|
10
|
+
* three shapes in order:
|
|
11
|
+
*
|
|
12
|
+
* 1. Pure JSON — the whole response IS valid JSON.
|
|
13
|
+
* 2. Code-fenced — ```json { ... } ``` block somewhere in the response.
|
|
14
|
+
* 3. Forward-scan balanced — find every balanced `{...}` span in the
|
|
15
|
+
* response and JSON.parse the LONGEST one (LLMs that emit a schema
|
|
16
|
+
* example before their real answer would otherwise corrupt a naive
|
|
17
|
+
* greedy regex; the real answer is almost always the largest block).
|
|
18
|
+
*
|
|
19
|
+
* The function returns the parsed object, or null on failure. Callers should
|
|
20
|
+
* use the loose substring fallback (e.g., `/\bCRITICAL\b/`) on the raw text
|
|
21
|
+
* when this returns null — that's the safety net against silent CRITICAL
|
|
22
|
+
* passes on non-JSON output.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {string} text Raw LLM response.
|
|
27
|
+
* @returns {object|null} Parsed JSON, or null if no parseable JSON found.
|
|
28
|
+
*/
|
|
29
|
+
function parseReviewJSON(text) {
|
|
30
|
+
if (!text || typeof text !== 'string') return null;
|
|
31
|
+
|
|
32
|
+
// Shape 1: try the whole string. Cheap; happens when the LLM follows the
|
|
33
|
+
// "Output as JSON" instruction strictly.
|
|
34
|
+
try { return JSON.parse(text); } catch { /* fall through */ }
|
|
35
|
+
|
|
36
|
+
// Shape 2: ```json ... ``` fenced block. Common when the LLM emits prose
|
|
37
|
+
// commentary before the structured answer.
|
|
38
|
+
const fenced = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/i);
|
|
39
|
+
if (fenced) {
|
|
40
|
+
try { return JSON.parse(fenced[1]); } catch { /* fall through */ }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Shape 3: forward-scan for balanced `{...}` spans, then try parsing each
|
|
44
|
+
// from longest to shortest. This handles LLMs that emit a schema example
|
|
45
|
+
// then their real answer (we'd otherwise capture both as one bad blob if
|
|
46
|
+
// we used a greedy `/\{[\s\S]*\}/` regex).
|
|
47
|
+
//
|
|
48
|
+
// The scanner tracks whether we're inside a JSON string literal so braces
|
|
49
|
+
// inside strings don't disrupt depth counting. Escape handling: a `\`
|
|
50
|
+
// inside a string marks the next character as escaped, so `\"` doesn't
|
|
51
|
+
// terminate the string.
|
|
52
|
+
const spans = [];
|
|
53
|
+
let depth = 0, start = -1, inString = false, escaped = false;
|
|
54
|
+
for (let i = 0; i < text.length; i++) {
|
|
55
|
+
const ch = text[i];
|
|
56
|
+
if (escaped) { escaped = false; continue; }
|
|
57
|
+
if (inString) {
|
|
58
|
+
if (ch === '\\') escaped = true;
|
|
59
|
+
else if (ch === '"') inString = false;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
if (ch === '"') { inString = true; continue; }
|
|
63
|
+
if (ch === '{') {
|
|
64
|
+
if (depth === 0) start = i;
|
|
65
|
+
depth++;
|
|
66
|
+
} else if (ch === '}') {
|
|
67
|
+
depth--;
|
|
68
|
+
if (depth === 0 && start >= 0) {
|
|
69
|
+
spans.push([start, i + 1]);
|
|
70
|
+
start = -1;
|
|
71
|
+
} else if (depth < 0) {
|
|
72
|
+
// Recover from malformed input — extra `}` without matching `{`.
|
|
73
|
+
depth = 0;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Try longest spans first — the LLM's real answer is usually the largest
|
|
79
|
+
// block in the response.
|
|
80
|
+
spans.sort((a, b) => (b[1] - b[0]) - (a[1] - a[0]));
|
|
81
|
+
for (const [a, b] of spans) {
|
|
82
|
+
try { return JSON.parse(text.slice(a, b)); } catch { /* try next span */ }
|
|
83
|
+
}
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
module.exports = { parseReviewJSON };
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* release-review-config.js — HC-080a-spike helpers extracted for unit coverage.
|
|
4
|
+
*
|
|
5
|
+
* Two helpers exist here because they were sources of bugs in the first
|
|
6
|
+
* spike CI run (PR #313 data point 1):
|
|
7
|
+
*
|
|
8
|
+
* 1. resolveBaseRef() — the CI step passes `--base origin/main` and the
|
|
9
|
+
* CLI was prefixing `origin/` again, producing `origin/origin/main`
|
|
10
|
+
* which `git diff` rejects. Either calling convention (`main` or
|
|
11
|
+
* `origin/main`) should now work.
|
|
12
|
+
*
|
|
13
|
+
* 2. getMaxDiffChars() — GH Models GPT-4.1 enforces an 8000-token cap
|
|
14
|
+
* on the REQUEST BODY (input). The previous 100k-char truncation
|
|
15
|
+
* worked fine for Opus (200k context) but failed every GH Models call
|
|
16
|
+
* with "Request body too large." Provider-specific truncation fixes it.
|
|
17
|
+
* See the function's docstring for budget math + the input-only vs
|
|
18
|
+
* total-body interpretation.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Resolve a base ref to a stable `origin/<branch>` form regardless of
|
|
23
|
+
* whether the caller already prefixed it.
|
|
24
|
+
*
|
|
25
|
+
* Examples:
|
|
26
|
+
* resolveBaseRef('main') // → 'origin/main'
|
|
27
|
+
* resolveBaseRef('origin/main') // → 'origin/main' (no double prefix)
|
|
28
|
+
* resolveBaseRef('refs/remotes/origin/main') // → 'origin/main'
|
|
29
|
+
*
|
|
30
|
+
* @param {string} base User-supplied base branch.
|
|
31
|
+
* @returns {string} Normalized git revision suitable for `git diff <ref>...HEAD`.
|
|
32
|
+
*/
|
|
33
|
+
function resolveBaseRef(base) {
|
|
34
|
+
if (!base || typeof base !== 'string') return 'origin/main';
|
|
35
|
+
const trimmed = base.trim();
|
|
36
|
+
if (trimmed.startsWith('refs/remotes/origin/')) {
|
|
37
|
+
return 'origin/' + trimmed.slice('refs/remotes/origin/'.length);
|
|
38
|
+
}
|
|
39
|
+
if (trimmed.startsWith('origin/')) return trimmed;
|
|
40
|
+
return 'origin/' + trimmed;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Provider-specific max diff size (in chars). Models with smaller input
|
|
45
|
+
* windows need more aggressive truncation.
|
|
46
|
+
*
|
|
47
|
+
* ## Budget math
|
|
48
|
+
*
|
|
49
|
+
* GH Models `openai/gpt-4.1` via `models.github.ai/inference` enforces an
|
|
50
|
+
* 8000-token cap on the REQUEST BODY (everything in the JSON payload sent
|
|
51
|
+
* to /chat/completions: system message + user message + model name + the
|
|
52
|
+
* `max_tokens` integer). The OUTPUT cap (`max_tokens: 4096`) is counted
|
|
53
|
+
* separately by the model and does NOT consume the 8000-token request
|
|
54
|
+
* budget — that 4096 governs how many tokens the response is allowed to
|
|
55
|
+
* have, not how many tokens our request can include.
|
|
56
|
+
*
|
|
57
|
+
* Decomposition of the 8000-token input budget (typical):
|
|
58
|
+
* system prompt ~600 tokens (~2400 chars)
|
|
59
|
+
* user prompt template ~200 tokens (~800 chars)
|
|
60
|
+
* file list (40 entries) ~400 tokens (~1600 chars)
|
|
61
|
+
* safety headroom ~800 tokens (~3200 chars)
|
|
62
|
+
* -------------------------------
|
|
63
|
+
* diff budget ~6000 tokens ≈ 24000 chars
|
|
64
|
+
*
|
|
65
|
+
* We use 20000 chars (≈5000 tokens) as a conservative cap to leave room
|
|
66
|
+
* for prompt drift if we evolve the system/user templates. If GH Models
|
|
67
|
+
* documentation later confirms the cap is TOTAL (input + output) instead
|
|
68
|
+
* of input-only, drop this number AND `max_tokens` together.
|
|
69
|
+
*
|
|
70
|
+
* Anthropic Opus `claude-opus-4-20250514`: 200K context window. The 100K
|
|
71
|
+
* char cap is generous and unchanged from the original implementation.
|
|
72
|
+
*
|
|
73
|
+
* ## Default behavior — FAIL CLOSED
|
|
74
|
+
*
|
|
75
|
+
* Unknown providers receive the SMALLEST known cap (20000), not the
|
|
76
|
+
* largest. Reasoning: if a future small-context provider (haiku, mini,
|
|
77
|
+
* flash) is added and a developer forgets to add a switch case here,
|
|
78
|
+
* the 20000 default truncates safely. Falling back to 100000 would
|
|
79
|
+
* re-introduce exactly the bug class this helper exists to fix.
|
|
80
|
+
*
|
|
81
|
+
* @param {string} provider 'opus' or 'gh-models'.
|
|
82
|
+
* @returns {number} Max diff chars to send to this provider.
|
|
83
|
+
*/
|
|
84
|
+
function getMaxDiffChars(provider) {
|
|
85
|
+
switch (provider) {
|
|
86
|
+
case 'opus':
|
|
87
|
+
return 100000;
|
|
88
|
+
case 'gh-models':
|
|
89
|
+
return 20000;
|
|
90
|
+
default:
|
|
91
|
+
return 20000;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
module.exports = {
|
|
96
|
+
resolveBaseRef,
|
|
97
|
+
getMaxDiffChars,
|
|
98
|
+
};
|