aiki-cli 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -5
- package/README.md +73 -15
- package/dist/bench/idea-v3-bench.js +104 -5
- package/dist/bench/idea-v3-rating.js +159 -4
- package/dist/cli/bench.js +5 -5
- package/dist/cli/index.js +12 -2
- package/dist/cli/resume.js +20 -0
- package/dist/cli/run.js +76 -5
- package/dist/cli/serve.js +48 -0
- package/dist/config/config.js +7 -2
- package/dist/council/view.js +64 -15
- package/dist/orchestration/auto-profile.js +97 -0
- package/dist/orchestration/claim-groups.js +56 -0
- package/dist/orchestration/context.js +69 -6
- package/dist/orchestration/decision-dossier.js +450 -89
- package/dist/orchestration/decision-graph.js +33 -2
- package/dist/orchestration/engine.js +8 -4
- package/dist/orchestration/evidence-origin.js +17 -0
- package/dist/orchestration/jsonStage.js +47 -5
- package/dist/orchestration/legacy-idea-adapter.js +3 -0
- package/dist/orchestration/modes.js +18 -10
- package/dist/orchestration/preflight.js +36 -3
- package/dist/orchestration/quick-analysis.js +29 -7
- package/dist/orchestration/sanitize-paths.js +10 -0
- package/dist/orchestration/stages/s10-render.js +196 -84
- package/dist/orchestration/stages/s4-analyze.js +10 -2
- package/dist/orchestration/stages/s4b-challenge.js +97 -0
- package/dist/orchestration/stages/s5-drift.js +13 -3
- package/dist/orchestration/stages/s6-positions.js +18 -0
- package/dist/orchestration/stages/s7-decision-graph.js +3 -0
- package/dist/orchestration/stages/s8-verify.js +66 -12
- package/dist/orchestration/stages/s8b-rebuttal.js +11 -4
- package/dist/orchestration/stages/s9-judge.js +52 -14
- package/dist/orchestration/stages/s9b-plan.js +227 -48
- package/dist/orchestration/url-sources.js +21 -0
- package/dist/providers/adapter-core.js +1 -1
- package/dist/providers/claude.js +18 -0
- package/dist/schemas/index.js +112 -3
- package/dist/serve/flight-deck.js +830 -0
- package/dist/serve/followup.js +50 -0
- package/dist/serve/frames.js +168 -0
- package/dist/serve/gates.js +72 -0
- package/dist/serve/projections.js +283 -0
- package/dist/serve/server.js +219 -0
- package/dist/serve/threads.js +145 -0
- package/dist/serve-ui/Five.png +0 -0
- package/dist/serve-ui/app.js +820 -0
- package/dist/serve-ui/index.html +171 -0
- package/dist/serve-ui/workspace.css +662 -0
- package/dist/skills/idea-refinement/analyst.md +5 -5
- package/dist/skills/idea-refinement/chair.md +18 -0
- package/dist/skills/idea-refinement/economics-delivery.md +11 -0
- package/dist/skills/idea-refinement/market-adoption.md +12 -0
- package/dist/skills/idea-refinement/planner.md +25 -19
- package/dist/skills/idea-refinement/rebuttal.md +15 -0
- package/dist/skills/idea-refinement/verifier.md +17 -0
- package/dist/storage/runs.js +2 -1
- package/dist/workflows/idea-refinement.js +130 -24
- package/package.json +2 -2
|
@@ -118,7 +118,7 @@ function blindId(seed) {
|
|
|
118
118
|
return createHash('sha256').update(seed).digest('hex').slice(0, 12).toUpperCase();
|
|
119
119
|
}
|
|
120
120
|
/** Remove the explicit identifiers forbidden by BENCHMARK-IDEA-V3.md §4 before human rating. §4 bars
|
|
121
|
-
* provider/model names, run ids, arm labels, AND costs — so the
|
|
121
|
+
* provider/model names, run ids, arm labels, AND costs — so the dossier's "Run details" cost
|
|
122
122
|
* block (mode, call counts, categories, per-provider calls, model time, degradation flags) and the
|
|
123
123
|
* inline `> ⚠ DEGRADED: <flag tokens>` callouts are redacted too; the DEGRADED marker and any prose
|
|
124
124
|
* note stay, since those are quality self-assessments raters legitimately read. */
|
|
@@ -132,7 +132,7 @@ export function blindIdeaV3Report(report, runId, caseDir) {
|
|
|
132
132
|
.replace(/^(- Report ID:).*$/gm, '$1 [redacted]')
|
|
133
133
|
.replace(/^(- Generated:).*$/gm, '$1 [redacted]')
|
|
134
134
|
.replace(/^(- Models and roles:).*$/gm, '$1 [redacted]')
|
|
135
|
-
.replace(/^(- (?:Mode|Provider calls|Categories|By provider|Recorded model time|Degradation flags):).*$/gm, '$1 [redacted]')
|
|
135
|
+
.replace(/^(- (?:Mode|Provider calls|Categories|By provider|Recorded model time|Tokens|Degradation flags):).*$/gm, '$1 [redacted]')
|
|
136
136
|
.replace(/^(> ⚠ DEGRADED): [a-z0-9_]+(?:, [a-z0-9_]+)*\.?$/gm, '$1 [redacted]');
|
|
137
137
|
}
|
|
138
138
|
async function renderCasePacket(item) {
|
|
@@ -384,6 +384,82 @@ function median(values) {
|
|
|
384
384
|
const middle = Math.floor(sorted.length / 2);
|
|
385
385
|
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
386
386
|
}
|
|
387
|
+
function nearestRank(values, percentile) {
|
|
388
|
+
if (!values.length)
|
|
389
|
+
return 0;
|
|
390
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
391
|
+
return sorted[Math.ceil(percentile * sorted.length) - 1];
|
|
392
|
+
}
|
|
393
|
+
/** Phase F additive metric: adjudicated matched expert claims per 1,000 recorded input+output tokens. */
|
|
394
|
+
export function ideaV3TokenEfficiency(scored, campaign) {
|
|
395
|
+
scored = IdeaV3ScoredCampaign.parse(scored);
|
|
396
|
+
campaign = IdeaV3Campaign.parse(campaign);
|
|
397
|
+
return campaign.arms.map((arm) => {
|
|
398
|
+
const observations = campaign.observations.filter((item) => item.arm === arm);
|
|
399
|
+
const complete = observations.length > 0 && observations.every((item) => item.usage !== undefined);
|
|
400
|
+
const tokens = complete
|
|
401
|
+
? observations.reduce((sum, item) => sum + item.usage.inputTokens + item.usage.outputTokens, 0)
|
|
402
|
+
: null;
|
|
403
|
+
const matched = scored.summary.find((item) => item.arm === arm)?.score.matched ?? 0;
|
|
404
|
+
return {
|
|
405
|
+
arm,
|
|
406
|
+
matched,
|
|
407
|
+
tokens,
|
|
408
|
+
matched_per_1k_tokens: tokens ? matched * 1_000 / tokens : null,
|
|
409
|
+
estimated_calls: observations.reduce((sum, item) => sum + (item.usage?.estimatedCalls ?? 0), 0),
|
|
410
|
+
};
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
/** Phase F build-only product targets. Original frozen holdout ship gates remain separate. */
|
|
414
|
+
export function evaluateIdeaV3AdaptiveGates(scored, campaign) {
|
|
415
|
+
scored = IdeaV3ScoredCampaign.parse(scored);
|
|
416
|
+
campaign = IdeaV3Campaign.parse(campaign);
|
|
417
|
+
if (scored.set !== 'build' || campaign.set !== 'build') {
|
|
418
|
+
throw new Error('adaptive product targets require the Phase F build campaign');
|
|
419
|
+
}
|
|
420
|
+
const required = ['A', 'B', 'B2', 'D2', 'R'];
|
|
421
|
+
const caseIds = new Set(campaign.observations.filter((item) => item.arm === 'A').map((item) => item.case_id));
|
|
422
|
+
const expectedPairs = [...caseIds].flatMap((caseId) => required.map((arm) => `${caseId}:${arm}`));
|
|
423
|
+
const observedPairs = campaign.observations.filter((item) => required.includes(item.arm));
|
|
424
|
+
const scoredPairs = new Set(scored.reports.map((item) => `${item.case_id}:${item.arm}`));
|
|
425
|
+
const matrixComplete = caseIds.size > 0
|
|
426
|
+
&& observedPairs.length === expectedPairs.length
|
|
427
|
+
&& expectedPairs.every((pair) => observedPairs.some((item) => `${item.case_id}:${item.arm}` === pair) && scoredPairs.has(pair))
|
|
428
|
+
&& required.every((arm) => scored.summary.some((item) => item.arm === arm));
|
|
429
|
+
const score = (arm) => scored.summary.find((item) => item.arm === arm)?.score.f1;
|
|
430
|
+
const a = score('A'), b = score('B'), d2 = score('D2'), r = score('R');
|
|
431
|
+
const aObservations = campaign.observations.filter((item) => item.arm === 'A');
|
|
432
|
+
const rObservations = campaign.observations.filter((item) => item.arm === 'R');
|
|
433
|
+
const usageComplete = aObservations.length > 0 && rObservations.length > 0
|
|
434
|
+
&& [...aObservations, ...rObservations].every((item) => item.usage !== undefined);
|
|
435
|
+
const perCaseTokens = (items) => items.map((item) => item.usage.inputTokens + item.usage.outputTokens);
|
|
436
|
+
const aMedianTokens = usageComplete ? median(perCaseTokens(aObservations)) : null;
|
|
437
|
+
const rMedianTokens = usageComplete ? median(perCaseTokens(rObservations)) : null;
|
|
438
|
+
const tokenReduction = aMedianTokens !== null && rMedianTokens
|
|
439
|
+
? 1 - aMedianTokens / rMedianTokens
|
|
440
|
+
: null;
|
|
441
|
+
const aMedianCalls = median(aObservations.map((item) => item.calls));
|
|
442
|
+
const aP95Calls = nearestRank(aObservations.map((item) => item.calls), 0.95);
|
|
443
|
+
const gates = {
|
|
444
|
+
matrix_complete: matrixComplete,
|
|
445
|
+
usage_complete: usageComplete,
|
|
446
|
+
primary_vs_b: a !== undefined && b !== undefined && a > b,
|
|
447
|
+
quality_floor_d2: a !== undefined && d2 !== undefined && a >= d2 - 0.05,
|
|
448
|
+
quality_floor_r: a !== undefined && r !== undefined && a >= r - 0.05,
|
|
449
|
+
token_savings: tokenReduction !== null && tokenReduction >= 0.4,
|
|
450
|
+
calls_median: aObservations.length > 0 && aMedianCalls <= 3,
|
|
451
|
+
calls_p95: aObservations.length > 0 && aP95Calls <= 6,
|
|
452
|
+
};
|
|
453
|
+
return {
|
|
454
|
+
...gates,
|
|
455
|
+
pass: Object.values(gates).every(Boolean),
|
|
456
|
+
a_median_calls: aMedianCalls,
|
|
457
|
+
a_p95_calls: aP95Calls,
|
|
458
|
+
a_median_tokens: aMedianTokens,
|
|
459
|
+
r_median_tokens: rMedianTokens,
|
|
460
|
+
token_reduction: tokenReduction,
|
|
461
|
+
};
|
|
462
|
+
}
|
|
387
463
|
/** Pure frozen ship-gate calculation. Secondary wins cannot rescue either primary F1 loss. */
|
|
388
464
|
export function evaluateIdeaV3Gates(scored, campaign) {
|
|
389
465
|
scored = IdeaV3ScoredCampaign.parse(scored);
|
|
@@ -441,6 +517,75 @@ export function evaluateIdeaV3Gates(scored, campaign) {
|
|
|
441
517
|
holdout_cases: cases.size,
|
|
442
518
|
};
|
|
443
519
|
}
|
|
520
|
+
/** Publish Phase F build validation without turning it into a frozen holdout claim. */
|
|
521
|
+
export async function writeIdeaV3AdaptiveResults(opts) {
|
|
522
|
+
const root = opts.root ?? process.cwd();
|
|
523
|
+
const scoredPath = resolve(opts.scoredPath);
|
|
524
|
+
const scored = IdeaV3ScoredCampaign.parse(JSON.parse(await readFile(scoredPath, 'utf8')));
|
|
525
|
+
const campaignPath = resolveFrom(dirname(scoredPath), scored.campaign);
|
|
526
|
+
const campaign = IdeaV3Campaign.parse(JSON.parse(await readFile(campaignPath, 'utf8')));
|
|
527
|
+
const gates = evaluateIdeaV3AdaptiveGates(scored, campaign);
|
|
528
|
+
const efficiency = ideaV3TokenEfficiency(scored, campaign);
|
|
529
|
+
const summary = new Map(scored.summary.map((item) => [item.arm, item.score]));
|
|
530
|
+
const rows = [
|
|
531
|
+
['Complete scored A/B/B2/D2/R matrix', gates.matrix_complete, gates.matrix_complete ? 'complete' : 'missing or duplicate pairs'],
|
|
532
|
+
['A/R token usage complete', gates.usage_complete, gates.usage_complete ? 'complete' : 'usage missing'],
|
|
533
|
+
['A strictly beats B on F1', gates.primary_vs_b, `${summary.get('A')?.f1.toFixed(3)} vs ${summary.get('B')?.f1.toFixed(3)}`],
|
|
534
|
+
['A within 0.05 F1 of D2', gates.quality_floor_d2, `${summary.get('A')?.f1.toFixed(3)} vs ${summary.get('D2')?.f1.toFixed(3)}`],
|
|
535
|
+
['A within 0.05 F1 of R', gates.quality_floor_r, `${summary.get('A')?.f1.toFixed(3)} vs ${summary.get('R')?.f1.toFixed(3)}`],
|
|
536
|
+
['A median tokens ≥40% below R', gates.token_savings, gates.token_reduction === null ? 'usage missing' : pct(gates.token_reduction)],
|
|
537
|
+
['A median calls ≤3', gates.calls_median, gates.a_median_calls.toFixed(1)],
|
|
538
|
+
['A p95 calls ≤6', gates.calls_p95, gates.a_p95_calls.toFixed(0)],
|
|
539
|
+
];
|
|
540
|
+
const lines = [
|
|
541
|
+
'# RESULTS-IDEA-V3-ADAPTIVE — Phase F build validation',
|
|
542
|
+
'',
|
|
543
|
+
`**Adaptive build targets: ${gates.pass ? 'PASS' : 'FAIL'}**`,
|
|
544
|
+
'',
|
|
545
|
+
'Build-set product validation; not a frozen holdout claim.',
|
|
546
|
+
'',
|
|
547
|
+
'## Product targets',
|
|
548
|
+
'',
|
|
549
|
+
'| Target | Result | Evidence |',
|
|
550
|
+
'|---|---|---|',
|
|
551
|
+
...rows.map(([name, pass, evidence]) => `| ${name} | ${pass ? 'PASS' : 'FAIL'} | ${evidence} |`),
|
|
552
|
+
'',
|
|
553
|
+
'## Primary metric',
|
|
554
|
+
'',
|
|
555
|
+
'| Arm | Matched | Recall | Precision | F1 |',
|
|
556
|
+
'|---|---:|---:|---:|---:|',
|
|
557
|
+
...campaign.arms.map((arm) => {
|
|
558
|
+
const item = summary.get(arm);
|
|
559
|
+
return `| ${arm} | ${item.matched} | ${pct(item.recall)} | ${pct(item.precision)} | ${item.f1.toFixed(3)} |`;
|
|
560
|
+
}),
|
|
561
|
+
'',
|
|
562
|
+
'## Verified insights per 1,000 tokens',
|
|
563
|
+
'',
|
|
564
|
+
'| Arm | Matched | Tokens | Matched / 1k tokens | Estimated calls |',
|
|
565
|
+
'|---|---:|---:|---:|---:|',
|
|
566
|
+
...efficiency.map((item) => `| ${item.arm} | ${item.matched} | ${item.tokens ?? 'missing'} | ${item.matched_per_1k_tokens?.toFixed(3) ?? 'missing'} | ${item.estimated_calls} |`),
|
|
567
|
+
'',
|
|
568
|
+
'## Every case and arm',
|
|
569
|
+
'',
|
|
570
|
+
'| Case | Arm | Status | Calls | Tokens | Repairs | Wall time | Flags / failure |',
|
|
571
|
+
'|---|---|---|---:|---:|---:|---:|---|',
|
|
572
|
+
...[...campaign.observations]
|
|
573
|
+
.sort((left, right) => left.case_id.localeCompare(right.case_id) || left.arm.localeCompare(right.arm))
|
|
574
|
+
.map((item) => {
|
|
575
|
+
const tokens = item.usage ? item.usage.inputTokens + item.usage.outputTokens : 'missing';
|
|
576
|
+
const detail = item.status === 'error' ? item.error : item.flags.join(', ') || 'none';
|
|
577
|
+
return `| ${item.case_id} | ${item.arm} | ${item.status} | ${item.calls} | ${tokens} | ${item.repair_calls} | ${(item.latency_ms / 1000).toFixed(1)}s | ${mdCell(detail)} |`;
|
|
578
|
+
}),
|
|
579
|
+
'',
|
|
580
|
+
`Token values include input plus output. ${efficiency.some((item) => item.estimated_calls > 0) ? 'At least one call uses Phase A labeled estimation.' : 'All recorded calls use provider-reported totals.'}`,
|
|
581
|
+
'',
|
|
582
|
+
];
|
|
583
|
+
const path = opts.outPath ?? join(root, 'RESULTS-IDEA-V3-ADAPTIVE.md');
|
|
584
|
+
const tmp = `${path}.tmp`;
|
|
585
|
+
await writeFile(tmp, lines.join('\n'), 'utf8');
|
|
586
|
+
await rename(tmp, path);
|
|
587
|
+
return { path, gates };
|
|
588
|
+
}
|
|
444
589
|
function mdCell(value) {
|
|
445
590
|
return value.replaceAll('|', '\\|').replaceAll('\n', ' ');
|
|
446
591
|
}
|
|
@@ -511,6 +656,16 @@ export async function writeIdeaV3Results(opts) {
|
|
|
511
656
|
await rename(tmp, path);
|
|
512
657
|
return { path, gates };
|
|
513
658
|
}
|
|
659
|
+
/** CLI publication dispatch: build amendments and frozen holdout results stay visibly separate. */
|
|
660
|
+
export async function publishIdeaV3Results(opts) {
|
|
661
|
+
const scored = IdeaV3ScoredCampaign.parse(JSON.parse(await readFile(resolve(opts.scoredPath), 'utf8')));
|
|
662
|
+
if (scored.set === 'build') {
|
|
663
|
+
const result = await writeIdeaV3AdaptiveResults(opts);
|
|
664
|
+
return { path: result.path, passed: result.gates.pass, label: 'adaptive build targets' };
|
|
665
|
+
}
|
|
666
|
+
const result = await writeIdeaV3Results(opts);
|
|
667
|
+
return { path: result.path, passed: result.gates.ship, label: 'frozen ship gate' };
|
|
668
|
+
}
|
|
514
669
|
export const IdeaV3ProtocolDraft = z.object({
|
|
515
670
|
build_scores: z.string().min(1),
|
|
516
671
|
baseline_provider: z.enum(['claude', 'codex', 'agy']),
|
|
@@ -523,7 +678,7 @@ export const IdeaV3ProtocolDraft = z.object({
|
|
|
523
678
|
}).strict(),
|
|
524
679
|
lane_assignment: z.enum(['agy-market', 'codex-market']),
|
|
525
680
|
}).strict();
|
|
526
|
-
/** Freeze only
|
|
681
|
+
/** Freeze only the original complete, already-scored B/C/D2/R build campaign; the file is one-shot. */
|
|
527
682
|
export async function writeFrozenIdeaV3Protocol(opts) {
|
|
528
683
|
const root = opts.root ?? process.cwd();
|
|
529
684
|
const draftPath = resolve(opts.draftPath);
|
|
@@ -539,7 +694,7 @@ export async function writeFrozenIdeaV3Protocol(opts) {
|
|
|
539
694
|
if (campaign.baseline_provider !== draft.baseline_provider) {
|
|
540
695
|
throw new Error(`draft baseline ${draft.baseline_provider} does not match campaign ${campaign.baseline_provider}`);
|
|
541
696
|
}
|
|
542
|
-
const arms = [
|
|
697
|
+
const arms = ['B', 'C', 'D2', 'R'];
|
|
543
698
|
if (arms.some((arm) => !campaign.arms.includes(arm) || !scored.summary.some((item) => item.arm === arm))) {
|
|
544
699
|
throw new Error('protocol freeze requires scored B, C, D2, and R build arms');
|
|
545
700
|
}
|
package/dist/cli/bench.js
CHANGED
|
@@ -6,7 +6,7 @@ import { planBench, renderTable, runBench } from '../bench/harness.js';
|
|
|
6
6
|
import { setupProviders } from '../orchestration/context.js';
|
|
7
7
|
import { chooseLaneDefault, importLaneAdjudications, planIdeaLaneBench, runIdeaLaneBench } from '../bench/idea-lane-rotation.js';
|
|
8
8
|
import { IDEA_V3_ARM_IDS, planIdeaV3Bench, runIdeaV3Bench } from '../bench/idea-v3-bench.js';
|
|
9
|
-
import { exportIdeaV3BlindBundle, importIdeaV3Ratings,
|
|
9
|
+
import { exportIdeaV3BlindBundle, importIdeaV3Ratings, publishIdeaV3Results, writeFrozenIdeaV3Protocol } from '../bench/idea-v3-rating.js';
|
|
10
10
|
import { DISPLAY_NAME } from '../providers/types.js';
|
|
11
11
|
const VALID_ARMS = ['A', 'B', 'C', 'D', 'E', 'L'];
|
|
12
12
|
/** One-block pre-run summary: what will run + the ≈Opus cost, so the user commits knowingly (§19). */
|
|
@@ -34,9 +34,9 @@ export async function benchCommand(workflow, opts = {}) {
|
|
|
34
34
|
}
|
|
35
35
|
if (opts.publishResults) {
|
|
36
36
|
try {
|
|
37
|
-
const result = await
|
|
37
|
+
const result = await publishIdeaV3Results({ scoredPath: opts.publishResults });
|
|
38
38
|
process.stdout.write(`\nresults: ${result.path}\n`);
|
|
39
|
-
process.stdout.write(
|
|
39
|
+
process.stdout.write(`${result.label}: ${result.passed ? 'PASS' : 'FAIL'}\n\n`);
|
|
40
40
|
return 0;
|
|
41
41
|
}
|
|
42
42
|
catch (error) {
|
|
@@ -82,7 +82,7 @@ export async function benchCommand(workflow, opts = {}) {
|
|
|
82
82
|
: set === 'build' ? ['B', 'C', 'D2', 'R'] : ['B', 'C', 'R'];
|
|
83
83
|
const invalid = requested.filter((arm) => !IDEA_V3_ARM_IDS.includes(arm));
|
|
84
84
|
if (invalid.length) {
|
|
85
|
-
process.stderr.write(`invalid idea-v3 arm(s): ${invalid.join(', ')}. Valid: B,C,D2,R\n`);
|
|
85
|
+
process.stderr.write(`invalid idea-v3 arm(s): ${invalid.join(', ')}. Valid: A,B,B2,C,D2,R\n`);
|
|
86
86
|
return 1;
|
|
87
87
|
}
|
|
88
88
|
const arms = requested;
|
|
@@ -100,7 +100,7 @@ export async function benchCommand(workflow, opts = {}) {
|
|
|
100
100
|
return 1;
|
|
101
101
|
}
|
|
102
102
|
process.stdout.write(`\nidea-v3 protocol comparison — ${set} — ${plan.cases.length} case(s) × arms ${arms.join(',')}\n`);
|
|
103
|
-
process.stdout.write(`B/C baseline provider: ${DISPLAY_NAME[provider]}\n`);
|
|
103
|
+
process.stdout.write(`B/B2/C baseline provider: ${DISPLAY_NAME[provider]}\n`);
|
|
104
104
|
if (plan.resumedFrom)
|
|
105
105
|
process.stdout.write(`resume: continuing ${plan.resumedFrom} — ${plan.skipCompleted} recorded pair(s) kept\n`);
|
|
106
106
|
process.stdout.write(`to run: ${plan.toRun.length} case×arm pair(s) → ≤${plan.estimatedProviderCalls} nominal provider call(s)\n`);
|
package/dist/cli/index.js
CHANGED
|
@@ -12,6 +12,7 @@ import { sessionsCommand } from './sessions.js';
|
|
|
12
12
|
import { config } from './config.js';
|
|
13
13
|
import { modelsCommand } from './models.js';
|
|
14
14
|
import { benchCommand } from './bench.js';
|
|
15
|
+
import { serveCommand } from './serve.js';
|
|
15
16
|
import { VERSION } from './version.js';
|
|
16
17
|
import { ConfigError, loadLayeredConfig } from '../config/config.js';
|
|
17
18
|
import { resolveRunsRoot } from '../storage/paths.js';
|
|
@@ -51,7 +52,8 @@ program
|
|
|
51
52
|
.option('--head <ref>', 'code-review: head git ref to diff to (default HEAD)')
|
|
52
53
|
.option('--diff <file>', 'code-review: review a patch file instead of computing a git diff')
|
|
53
54
|
.option('--evidence <path>', 'idea-refinement: local source file/directory (stores paths + hashes, not copies)')
|
|
54
|
-
.option('--mode <mode>', 'idea-refinement: quick | council |
|
|
55
|
+
.option('--mode <mode>', 'idea-refinement: quick | council | auto (research is an alias for council)')
|
|
56
|
+
.option('--allow-blocked-sources', 'idea-refinement: proceed even when a supplied URL cannot be read (default: stop and ask)')
|
|
55
57
|
.option('--cheap', 'code-review: Gemini+Codex review, Claude judges only disputes (~⅓ the Opus; experimental)')
|
|
56
58
|
.option('--yes', 'skip the run-cost confirmation prompt')
|
|
57
59
|
.action(async (workflow, input, opts) => {
|
|
@@ -93,7 +95,7 @@ program
|
|
|
93
95
|
.command('bench')
|
|
94
96
|
.description('Run code-review, idea lane, or frozen idea-v3 benchmark arms; writes bench/results/*.json.')
|
|
95
97
|
.argument('<workflow>', 'code-review | idea-refinement | idea-v3')
|
|
96
|
-
.option('--arms <list>', 'comma-separated arms to run (code review defaults A,B,C,D; idea-v3 defaults B,C,D2,R
|
|
98
|
+
.option('--arms <list>', 'comma-separated arms to run (code review defaults A,B,C,D; idea-v3 defaults B,C,D2,R; Phase F adds explicit A,B2)')
|
|
97
99
|
.option('--set <name>', 'task set: build | holdout', 'build')
|
|
98
100
|
.option('--resume', 'continue the latest results file: keep already-scored case×arm pairs, retry the rest')
|
|
99
101
|
.option('--yes', 'actually run; without it, print the pre-run Opus-call estimate and exit')
|
|
@@ -123,6 +125,14 @@ program
|
|
|
123
125
|
campaign: opts.campaign,
|
|
124
126
|
}));
|
|
125
127
|
});
|
|
128
|
+
program
|
|
129
|
+
.command('serve')
|
|
130
|
+
.description('Open the chat workspace in the browser (localhost only; no paid calls until you convene a run).')
|
|
131
|
+
.option('--port <n>', 'port to bind (default: first free in 4173–4183)', (v) => parseInt(v, 10))
|
|
132
|
+
.option('--no-open', 'do not open the browser automatically')
|
|
133
|
+
.action(async (opts) => {
|
|
134
|
+
process.exit(await serveCommand({ port: opts.port, open: opts.open }));
|
|
135
|
+
});
|
|
126
136
|
program
|
|
127
137
|
.command('models')
|
|
128
138
|
.description('Show configurable models per provider (lists via the CLI where supported) + your current pins.')
|
package/dist/cli/resume.js
CHANGED
|
@@ -9,6 +9,7 @@ import { resolveRunsRoot } from '../storage/paths.js';
|
|
|
9
9
|
import { buildReplayCache } from '../storage/replay.js';
|
|
10
10
|
import { findSession } from '../storage/sessions.js';
|
|
11
11
|
import { readJsonArtifact, resolveRunId, runDir } from '../storage/runs-read.js';
|
|
12
|
+
import { UrlSourceSet } from '../schemas/index.js';
|
|
12
13
|
import { EvidencePack } from '../orchestration/evidence-pack.js';
|
|
13
14
|
export async function resumeCommand(runArg, opts = {}) {
|
|
14
15
|
if (!runArg) {
|
|
@@ -72,6 +73,22 @@ export async function resumeCommand(runArg, opts = {}) {
|
|
|
72
73
|
}
|
|
73
74
|
evidencePack = parsed.data;
|
|
74
75
|
}
|
|
76
|
+
// v6 T10: the URL snapshot + the user's proceed-past-blocked decision are run inputs, like
|
|
77
|
+
// inputs/idea.md and evidence-pack.json. Reusing the ORIGINAL snapshot keeps every prompt that
|
|
78
|
+
// embeds it byte-identical (replay cache hits, no refetch); and a non-empty replay cache proves
|
|
79
|
+
// the old run cleared the S0 gate, so a blocked source in the snapshot means consent was given
|
|
80
|
+
// (interactive `y` or --allow-blocked-sources) — restore that decision instead of re-asking.
|
|
81
|
+
// (Real failure: resume c46e of 626e died SOURCE_UNREADABLE re-gating a consented run.)
|
|
82
|
+
let urlSources;
|
|
83
|
+
const savedUrlSources = await readJsonArtifact(oldDir, '00a-url-sources.json');
|
|
84
|
+
if (savedUrlSources) {
|
|
85
|
+
const parsedSources = UrlSourceSet.safeParse(savedUrlSources);
|
|
86
|
+
if (!parsedSources.success) {
|
|
87
|
+
process.stderr.write(`cannot validate 00a-url-sources.json for ${oldId} — nothing to resume.\n`);
|
|
88
|
+
return 1;
|
|
89
|
+
}
|
|
90
|
+
urlSources = parsedSources.data;
|
|
91
|
+
}
|
|
75
92
|
const replay = await buildReplayCache(oldDir);
|
|
76
93
|
if (replay.size === 0) {
|
|
77
94
|
process.stderr.write(`no completed calls found for ${oldId} — start a fresh run instead.\n`);
|
|
@@ -100,6 +117,9 @@ export async function resumeCommand(runArg, opts = {}) {
|
|
|
100
117
|
resumedFrom: oldId,
|
|
101
118
|
providerModels: cfg.models,
|
|
102
119
|
evidencePack,
|
|
120
|
+
urlSources,
|
|
121
|
+
allowBlockedSources: urlSources?.sources.some((s) => s.status !== 'FETCHED') ?? false,
|
|
122
|
+
autoDecision: previousMeta.auto_decision,
|
|
103
123
|
});
|
|
104
124
|
if (outcome.ok) {
|
|
105
125
|
process.stdout.write(`\n ✔ resumed run ${outcome.runId} complete — ${outcome.callCount} new provider call(s)\n artifacts: ${outcome.dir}\n\n`);
|
package/dist/cli/run.js
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
import { readFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { createInterface } from 'node:readline';
|
|
6
|
+
import { blockedSourceStop, extractPublicUrls, snapshotUrlSources } from '../orchestration/url-sources.js';
|
|
7
|
+
import { buildTaskProfile, resolveAutoMode } from '../orchestration/auto-profile.js';
|
|
8
|
+
import { requestedOutputsFor } from '../orchestration/preflight.js';
|
|
6
9
|
import { renderTerminalSummary } from '../orchestration/stages/s10-render.js';
|
|
7
10
|
import { run as runEngine } from '../orchestration/engine.js';
|
|
8
11
|
import { ConfigError, loadLayeredConfig } from '../config/config.js';
|
|
@@ -16,12 +19,16 @@ const WORKFLOWS = ['idea-refinement', 'code-review'];
|
|
|
16
19
|
export function estimateRun(workflow, opts = {}) {
|
|
17
20
|
if (workflow === 'code-review')
|
|
18
21
|
return { calls: 5, opus: opts.cheap ? 1 : 2 };
|
|
22
|
+
if (opts.auto)
|
|
23
|
+
return { calls: 4, opus: 1, minCalls: opts.fastPath ? 1 : 3, reserved: 0 };
|
|
24
|
+
if (opts.fastPath)
|
|
25
|
+
return { calls: 1, opus: 1, minCalls: 1, reserved: 0 };
|
|
19
26
|
const mode = opts.mode ?? 'council';
|
|
20
27
|
const plan = IDEA_MODE_PLANS[mode];
|
|
21
28
|
return {
|
|
22
29
|
calls: plan.maxCalls,
|
|
23
30
|
opus: mode === 'quick' ? 1 : 2,
|
|
24
|
-
minCalls:
|
|
31
|
+
minCalls: plan.maxCalls - plan.reservedCalls,
|
|
25
32
|
reserved: plan.reservedCalls,
|
|
26
33
|
};
|
|
27
34
|
}
|
|
@@ -35,6 +42,19 @@ function confirm(question) {
|
|
|
35
42
|
});
|
|
36
43
|
});
|
|
37
44
|
}
|
|
45
|
+
/** One question, one trimmed answer. Resolve BEFORE close — closing fires the EOF handler, and
|
|
46
|
+
* the live "typed Y, got cancelled" bug was that handler resolving '' first. EOF alone (Ctrl+D)
|
|
47
|
+
* still resolves '' = the safe deny default. Exported (with injectable streams) for the test. */
|
|
48
|
+
export function ask(question, io = { input: process.stdin, output: process.stdout }) {
|
|
49
|
+
return new Promise((resolve) => {
|
|
50
|
+
const rl = createInterface({ input: io.input, output: io.output });
|
|
51
|
+
rl.on('close', () => resolve(''));
|
|
52
|
+
rl.question(question, (a) => {
|
|
53
|
+
resolve(a.trim());
|
|
54
|
+
rl.close();
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
}
|
|
38
58
|
/** Resolve an idea-refinement input: an existing file path → its contents, else the arg as inline text. */
|
|
39
59
|
async function resolveInput(arg) {
|
|
40
60
|
if (!arg)
|
|
@@ -90,11 +110,15 @@ export async function runCommand(workflow, input, opts = {}) {
|
|
|
90
110
|
return 1;
|
|
91
111
|
}
|
|
92
112
|
let mode;
|
|
113
|
+
let autoRequested = false; // v7 Phase B: `--mode auto` → resolved to quick|council once inputs are known
|
|
93
114
|
if (workflow === 'idea-refinement') {
|
|
94
|
-
if (opts.mode
|
|
115
|
+
if (opts.mode === 'auto') {
|
|
116
|
+
autoRequested = true;
|
|
117
|
+
}
|
|
118
|
+
else if (opts.mode !== undefined) {
|
|
95
119
|
const parsed = IdeaModeSchema.safeParse(opts.mode);
|
|
96
120
|
if (!parsed.success) {
|
|
97
|
-
process.stderr.write(`unknown idea mode "${opts.mode}". Available: quick, council, research\n`);
|
|
121
|
+
process.stderr.write(`unknown idea mode "${opts.mode}". Available: quick, council, research, auto\n`);
|
|
98
122
|
return 1;
|
|
99
123
|
}
|
|
100
124
|
mode = parsed.data;
|
|
@@ -120,7 +144,8 @@ export async function runCommand(workflow, input, opts = {}) {
|
|
|
120
144
|
return 1;
|
|
121
145
|
}
|
|
122
146
|
text = resolved;
|
|
123
|
-
|
|
147
|
+
if (!autoRequested)
|
|
148
|
+
mode ??= inferIdeaMode(text);
|
|
124
149
|
}
|
|
125
150
|
if (opts.evidence) {
|
|
126
151
|
if (workflow !== 'idea-refinement') {
|
|
@@ -135,6 +160,19 @@ export async function runCommand(workflow, input, opts = {}) {
|
|
|
135
160
|
return 1;
|
|
136
161
|
}
|
|
137
162
|
}
|
|
163
|
+
// v7 Phase B: resolve `--mode auto` here, before any RunCtx/engine work, so meta.mode stays quick|council
|
|
164
|
+
// and the estimate below reflects the resolved mode. urlCount is fetch-free (extract only, no snapshot).
|
|
165
|
+
let autoDecision;
|
|
166
|
+
if (autoRequested) {
|
|
167
|
+
const resolved = resolveAutoMode(buildTaskProfile(text, {
|
|
168
|
+
urlCount: extractPublicUrls(text).length,
|
|
169
|
+
hasEvidencePack: !!evidencePack,
|
|
170
|
+
requestedOutputs: requestedOutputsFor(text),
|
|
171
|
+
}));
|
|
172
|
+
mode = resolved.mode;
|
|
173
|
+
autoDecision = { resolved: resolved.mode, reasons: resolved.reasons, ...(resolved.fastPath ? { fast_path: true } : {}) };
|
|
174
|
+
process.stdout.write(` auto → ${resolved.mode}: ${resolved.reasons.join(', ')}.\n`);
|
|
175
|
+
}
|
|
138
176
|
// Precedence (§10/T9): --budget flag > config.budget > built-in default. roles/deadline/models are
|
|
139
177
|
// config-only (layered: global ~/.aiki base, project .aiki override).
|
|
140
178
|
let cfg;
|
|
@@ -158,7 +196,7 @@ export async function runCommand(workflow, input, opts = {}) {
|
|
|
158
196
|
process.stderr.write('--cheap only applies to code-review; ignoring for this workflow.\n');
|
|
159
197
|
}
|
|
160
198
|
// Run-cost preview (V5): show the estimate; confirm interactively unless --yes or non-interactive.
|
|
161
|
-
const est = estimateRun(workflow, { cheap: opts.cheap, mode });
|
|
199
|
+
const est = estimateRun(workflow, { cheap: opts.cheap, mode, auto: !!autoDecision, fastPath: autoDecision?.fast_path });
|
|
162
200
|
const callEstimate = est.minCalls !== undefined && est.minCalls !== est.calls ? `${est.minCalls}–${est.calls}` : `${est.calls}`;
|
|
163
201
|
const resolvedBudget = opts.budget ?? cfg.budget ?? defaultBudgetFor(workflow, mode);
|
|
164
202
|
const reservation = est.reserved ? `; ${est.reserved} call(s) reserved for chair + planner` : '';
|
|
@@ -173,6 +211,36 @@ export async function runCommand(workflow, input, opts = {}) {
|
|
|
173
211
|
else {
|
|
174
212
|
process.stdout.write(`${note}\n`);
|
|
175
213
|
}
|
|
214
|
+
// v6 T10b: check supplied URLs BEFORE any provider spend. A blocked source in a TTY gets a
|
|
215
|
+
// simple permission prompt — Y allow, anything else deny. Headless keeps the fail-fast gate.
|
|
216
|
+
let allowBlockedSources = opts.allowBlockedSources ?? false;
|
|
217
|
+
let urlSources;
|
|
218
|
+
if (workflow === 'idea-refinement') {
|
|
219
|
+
urlSources = await snapshotUrlSources(text);
|
|
220
|
+
const stopMessage = blockedSourceStop(urlSources, mode ?? 'council', allowBlockedSources);
|
|
221
|
+
if (stopMessage && process.stdin.isTTY && process.stdout.isTTY) {
|
|
222
|
+
const unreadable = urlSources.sources.filter((item) => item.status !== 'FETCHED');
|
|
223
|
+
const noun = unreadable.length > 1 ? 'these pages' : 'this page';
|
|
224
|
+
for (const source of unreadable) {
|
|
225
|
+
process.stdout.write(`\n ⚠ Couldn't read a page you attached:\n ${source.url}\n reason: ${source.error ?? source.status}\n`);
|
|
226
|
+
}
|
|
227
|
+
process.stdout.write(`\n Do you want to run without ${noun}?\n`);
|
|
228
|
+
process.stdout.write(` y = yes, run anyway (facts from ${noun} will be marked unverified)\n`);
|
|
229
|
+
process.stdout.write(' n = no, cancel — paste the page text into your prompt and rerun\n\n');
|
|
230
|
+
const answer = await ask(' Choice [y/N]: ');
|
|
231
|
+
if (/^y/i.test(answer)) {
|
|
232
|
+
allowBlockedSources = true;
|
|
233
|
+
}
|
|
234
|
+
else {
|
|
235
|
+
process.stdout.write('\n Cancelled. Paste the page text into your prompt and rerun, or rerun with --allow-blocked-sources.\n');
|
|
236
|
+
return 0;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
else if (stopMessage) {
|
|
240
|
+
process.stderr.write(` ✖ ${stopMessage}\n`);
|
|
241
|
+
return 1;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
176
244
|
const outcome = await runEngine(workflow, text, {
|
|
177
245
|
mode,
|
|
178
246
|
budget: resolvedBudget,
|
|
@@ -182,6 +250,9 @@ export async function runCommand(workflow, input, opts = {}) {
|
|
|
182
250
|
runsRoot: await resolveRunsRoot(), // hybrid: repo .aiki when in a repo, else ~/.aiki
|
|
183
251
|
providerModels: cfg.models, // V8: per-provider model → CLI --model
|
|
184
252
|
evidencePack,
|
|
253
|
+
allowBlockedSources,
|
|
254
|
+
urlSources,
|
|
255
|
+
autoDecision,
|
|
185
256
|
});
|
|
186
257
|
if (outcome.ok) {
|
|
187
258
|
process.stdout.write(`\n ✔ run ${outcome.runId} complete — ${outcome.callCount} provider call(s)\n artifacts: ${outcome.dir}\n\n`);
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// `aiki serve` — open the chat workspace on localhost. Resolves the built serve-ui assets, builds a
|
|
2
|
+
// FlightDeck over the hybrid runs root, starts the server, and (unless --no-open) opens the browser.
|
|
3
|
+
// The process stays up until Ctrl+C; there are no paid calls until the user convenes a run in the UI.
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import { dirname, join } from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
import { FlightDeck } from '../serve/flight-deck.js';
|
|
8
|
+
import { startServer } from '../serve/server.js';
|
|
9
|
+
import { openInBrowser } from '../council/open.js';
|
|
10
|
+
import { resolveRunsRoot } from '../storage/paths.js';
|
|
11
|
+
import { VERSION } from './version.js';
|
|
12
|
+
/** Locate the serve-ui assets: dist/serve-ui next to this module (built), else repo-root serve-ui (dev). */
|
|
13
|
+
function resolveStaticDir() {
|
|
14
|
+
const here = dirname(fileURLToPath(import.meta.url)); // dist/cli
|
|
15
|
+
const built = join(here, '..', 'serve-ui'); // dist/serve-ui
|
|
16
|
+
if (existsSync(join(built, 'index.html')))
|
|
17
|
+
return built;
|
|
18
|
+
return join(here, '..', '..', 'serve-ui'); // repo-root serve-ui (running from src)
|
|
19
|
+
}
|
|
20
|
+
export async function serveCommand(opts = {}) {
|
|
21
|
+
const runsRoot = await resolveRunsRoot();
|
|
22
|
+
const log = (line) => {
|
|
23
|
+
const time = new Date().toLocaleTimeString('en-US', { hour12: false });
|
|
24
|
+
process.stdout.write(` \x1b[2m${time}\x1b[0m ${line}\n`);
|
|
25
|
+
};
|
|
26
|
+
const flightDeck = new FlightDeck({ runsRoot, version: VERSION, log });
|
|
27
|
+
const staticDir = resolveStaticDir();
|
|
28
|
+
let server;
|
|
29
|
+
try {
|
|
30
|
+
server = await startServer({ flightDeck, staticDir, port: opts.port });
|
|
31
|
+
}
|
|
32
|
+
catch (e) {
|
|
33
|
+
process.stderr.write(`aiki serve: ${e instanceof Error ? e.message : String(e)}\n`);
|
|
34
|
+
return 1;
|
|
35
|
+
}
|
|
36
|
+
const url = `http://127.0.0.1:${server.port}`;
|
|
37
|
+
process.stdout.write(`\n aiki workspace → ${url}\n (Ctrl+C to stop)\n\n`);
|
|
38
|
+
if (opts.open !== false)
|
|
39
|
+
openInBrowser(url);
|
|
40
|
+
await new Promise((resolve) => {
|
|
41
|
+
const shutdown = () => {
|
|
42
|
+
server.close().finally(() => resolve());
|
|
43
|
+
};
|
|
44
|
+
process.on('SIGINT', shutdown);
|
|
45
|
+
process.on('SIGTERM', shutdown);
|
|
46
|
+
});
|
|
47
|
+
return 0;
|
|
48
|
+
}
|
package/dist/config/config.js
CHANGED
|
@@ -22,6 +22,7 @@ const ConfigRoles = z
|
|
|
22
22
|
judge: ProviderIdSchema.optional(),
|
|
23
23
|
verifier: ProviderIdSchema.optional(),
|
|
24
24
|
s4: z.array(ProviderIdSchema).min(1).optional(),
|
|
25
|
+
responder: ProviderIdSchema.optional(),
|
|
25
26
|
})
|
|
26
27
|
.strict();
|
|
27
28
|
/** Per-provider model override (V8). Each CLI runs its own model families, so model choice is per
|
|
@@ -96,7 +97,11 @@ export function mergeConfig(base, over) {
|
|
|
96
97
|
}
|
|
97
98
|
/** Layered config: global `~/.aiki/config.json` (base) overlaid by the project `.aiki/config.json`. What
|
|
98
99
|
* the CLI actually uses (V8) — a user can set defaults once in the global file and override per project. */
|
|
99
|
-
export async function loadLayeredConfig() {
|
|
100
|
-
const
|
|
100
|
+
export async function loadLayeredConfig(projectRoot = '.aiki') {
|
|
101
|
+
const globalRoot = homeAikiRoot();
|
|
102
|
+
const global = await loadConfig(globalRoot);
|
|
103
|
+
if (projectRoot === globalRoot)
|
|
104
|
+
return global;
|
|
105
|
+
const project = await loadConfig(projectRoot);
|
|
101
106
|
return mergeConfig(global, project);
|
|
102
107
|
}
|