aiki-cli 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -7
- package/README.md +109 -30
- package/dist/bench/arms.js +10 -9
- package/dist/bench/harness.js +13 -10
- package/dist/bench/idea-lane-rotation.js +237 -0
- package/dist/bench/idea-v3-bench.js +506 -0
- package/dist/bench/idea-v3-rating.js +582 -0
- package/dist/bench/results.js +10 -3
- package/dist/bench/scoring/decision-insights.js +112 -0
- package/dist/bench/scoring/seeded-bugs.js +4 -0
- package/dist/cli/bench.js +180 -3
- package/dist/cli/doctor.js +56 -24
- package/dist/cli/index.js +32 -7
- package/dist/cli/resolve.js +7 -6
- package/dist/cli/resume.js +18 -0
- package/dist/cli/run.js +63 -8
- package/dist/cli/version.js +8 -0
- package/dist/council/view.js +378 -109
- package/dist/orchestration/calculations.js +97 -0
- package/dist/orchestration/context.js +37 -9
- package/dist/orchestration/decision-dossier.js +262 -0
- package/dist/orchestration/decision-graph.js +256 -0
- package/dist/orchestration/engine.js +5 -2
- package/dist/orchestration/evidence-pack.js +46 -0
- package/dist/orchestration/idea-lanes.js +20 -0
- package/dist/orchestration/jsonStage.js +72 -0
- package/dist/orchestration/legacy-idea-adapter.js +102 -0
- package/dist/orchestration/modes.js +33 -0
- package/dist/orchestration/preflight.js +183 -0
- package/dist/orchestration/quick-analysis.js +81 -0
- package/dist/orchestration/stages/cr-ladder.js +80 -0
- package/dist/orchestration/stages/s10-render.js +562 -150
- package/dist/orchestration/stages/s4-analyze.js +12 -7
- package/dist/orchestration/stages/s5-drift.js +9 -9
- package/dist/orchestration/stages/s6-positions.js +10 -0
- package/dist/orchestration/stages/s7-decision-graph.js +76 -0
- package/dist/orchestration/stages/s8-verify.js +153 -46
- package/dist/orchestration/stages/s8b-rebuttal.js +208 -0
- package/dist/orchestration/stages/s9-judge.js +329 -108
- package/dist/orchestration/stages/s9b-plan.js +85 -75
- package/dist/providers/codex.js +2 -1
- package/dist/providers/spawn.js +5 -0
- package/dist/schemas/index.js +572 -13
- package/dist/skills/idea-refinement/analyst.md +18 -14
- package/dist/skills/idea-refinement/economics-delivery.md +7 -0
- package/dist/skills/idea-refinement/market-adoption.md +7 -0
- package/dist/storage/runs.js +11 -4
- package/dist/tui/app.js +37 -13
- package/dist/tui/format.js +4 -5
- package/dist/tui/smart-entry.js +17 -1
- package/dist/tui/timeline.js +2 -4
- package/dist/workflows/code-review.js +4 -2
- package/dist/workflows/idea-refinement.js +110 -46
- package/package.json +12 -4
- package/dist/orchestration/stages/s0-grill.js +0 -79
- package/dist/orchestration/stages/s1-intent.js +0 -25
- package/dist/orchestration/stages/s2-misread.js +0 -76
- package/dist/orchestration/stages/s3-prompts.js +0 -55
- package/dist/orchestration/stages/s6-claims.js +0 -56
- package/dist/orchestration/stages/s7-disagreement.js +0 -134
|
@@ -9,20 +9,24 @@
|
|
|
9
9
|
// Discriminator injection (§13): the model returns JSON with NO `workflow` field. We validate the
|
|
10
10
|
// call against `IdeaRoleOutputModel` (that exact shape), then inject `workflow` and persist via
|
|
11
11
|
// `writeRoleOutput`, which re-validates the full `RoleOutput`.
|
|
12
|
-
import { IdeaRoleOutputModel } from '../../schemas/index.js';
|
|
12
|
+
import { IdeaRoleOutputModel, salvageIdeaRoleOutputModel } from '../../schemas/index.js';
|
|
13
13
|
import { isFatal, StageError } from '../context.js';
|
|
14
14
|
import { jsonCall } from '../jsonStage.js';
|
|
15
|
+
import { IDEA_LANES } from '../idea-lanes.js';
|
|
15
16
|
/** Run one analyst seat → validated `RoleOutput`, persisted to 04-role-outputs/<label>.json.
|
|
16
17
|
* `label` doubles as the artifact filename, so a resample uses a distinct label. */
|
|
17
|
-
async function runSeat(ctx, seat, label, prompt) {
|
|
18
|
-
const model = await jsonCall(ctx, ctx.handle(seat), `S4-${label}`, prompt, IdeaRoleOutputModel);
|
|
18
|
+
async function runSeat(ctx, seat, label, lane, prompt) {
|
|
19
|
+
const model = await jsonCall(ctx, ctx.handle(seat), `S4-${label}`, prompt, IdeaRoleOutputModel, { salvage: salvageIdeaRoleOutputModel });
|
|
19
20
|
const output = { workflow: 'idea-refinement', ...model };
|
|
20
21
|
await ctx.writer.writeRoleOutput(label, output);
|
|
21
|
-
return { provider: seat, output };
|
|
22
|
+
return { provider: seat, sample: label, lane, output };
|
|
22
23
|
}
|
|
23
|
-
export async function s4Analyze(ctx,
|
|
24
|
+
export async function s4Analyze(ctx, prompts) {
|
|
24
25
|
const seats = ctx.roles.s4;
|
|
25
|
-
const settled = await Promise.allSettled(seats.map((seat) =>
|
|
26
|
+
const settled = await Promise.allSettled(seats.map((seat, index) => {
|
|
27
|
+
const lane = IDEA_LANES[index % IDEA_LANES.length];
|
|
28
|
+
return runSeat(ctx, seat, seat, lane, prompts[lane]);
|
|
29
|
+
}));
|
|
26
30
|
const survivors = [];
|
|
27
31
|
const dropped = [];
|
|
28
32
|
for (let i = 0; i < settled.length; i++) {
|
|
@@ -43,7 +47,8 @@ export async function s4Analyze(ctx, analystPrompt) {
|
|
|
43
47
|
if (survivors.length === 1) {
|
|
44
48
|
const only = survivors[0];
|
|
45
49
|
ctx.addFlag('low_diversity');
|
|
46
|
-
const
|
|
50
|
+
const lane = only.lane ?? IDEA_LANES[0];
|
|
51
|
+
const resample = await runSeat(ctx, only.provider, `${only.provider}-2`, lane, prompts[lane]);
|
|
47
52
|
return [only, resample];
|
|
48
53
|
}
|
|
49
54
|
throw new StageError('S4', 'QUORUM', `no analyst seats survived (dropped: ${dropped.map((d) => `${d.provider}:${d.error}`).join('; ')})`);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// S5 — drift detection (§9, §12.1). Deterministic gate: does each S4 output still address the
|
|
2
2
|
// contract's task? Two code checks (no model call — the §601/§532 T6 "deterministic core"; the §9
|
|
3
|
-
// "verifier spot-check" is deferred): (1) the output produced ≥1
|
|
3
|
+
// "verifier spot-check" is deferred): (1) the output produced ≥1 position (an analyst that
|
|
4
4
|
// produced none did not engage the task), and (2) its `task_echo` is similar enough to the contract
|
|
5
5
|
// task (overlap coefficient ≥ DRIFT_MIN_SIMILARITY). Drifted outputs are excluded from everything
|
|
6
6
|
// downstream and logged; if exclusion drops the survivor count below quorum (2), the run aborts.
|
|
@@ -11,19 +11,19 @@ import { overlapCoefficient, tokenize } from '../cluster.js';
|
|
|
11
11
|
* paragraph. Lenient by design — this catches an analyst that wandered off-task, not paraphrase
|
|
12
12
|
* distance. Tunable (same footnote class as the S2 clustering threshold). */
|
|
13
13
|
export const DRIFT_MIN_SIMILARITY = 0.3;
|
|
14
|
-
export async function s5Drift(ctx, contract, seats) {
|
|
14
|
+
export async function s5Drift(ctx, contract, seats, minSeats = 2) {
|
|
15
15
|
const taskTokens = tokenize(contract.task);
|
|
16
16
|
const entries = [];
|
|
17
17
|
const kept = [];
|
|
18
18
|
const excluded = [];
|
|
19
19
|
for (const seat of seats) {
|
|
20
20
|
const sim = overlapCoefficient(tokenize(seat.output.task_echo), taskTokens);
|
|
21
|
-
const
|
|
22
|
-
const on_task =
|
|
23
|
-
const evidence = !
|
|
24
|
-
? 'no
|
|
21
|
+
const hasPositions = seat.output.positions.length > 0;
|
|
22
|
+
const on_task = hasPositions && sim >= DRIFT_MIN_SIMILARITY;
|
|
23
|
+
const evidence = !hasPositions
|
|
24
|
+
? 'no positions produced'
|
|
25
25
|
: on_task
|
|
26
|
-
? `task_echo overlap ${sim.toFixed(2)} ≥ ${DRIFT_MIN_SIMILARITY}; ${seat.output.
|
|
26
|
+
? `task_echo overlap ${sim.toFixed(2)} ≥ ${DRIFT_MIN_SIMILARITY}; ${seat.output.positions.length} position(s)`
|
|
27
27
|
: `task_echo overlap ${sim.toFixed(2)} < ${DRIFT_MIN_SIMILARITY} (off-task)`;
|
|
28
28
|
entries.push({ provider: seat.provider, on_task, similarity: Number(sim.toFixed(3)), evidence });
|
|
29
29
|
if (on_task)
|
|
@@ -33,8 +33,8 @@ export async function s5Drift(ctx, contract, seats) {
|
|
|
33
33
|
}
|
|
34
34
|
const report = { entries, excluded };
|
|
35
35
|
await ctx.writer.writeJson('drift-report', report);
|
|
36
|
-
if (kept.length <
|
|
37
|
-
throw new StageError('S5', 'QUORUM', `drift exclusion left ${kept.length} on-task output(s); need
|
|
36
|
+
if (kept.length < minSeats) {
|
|
37
|
+
throw new StageError('S5', 'QUORUM', `drift exclusion left ${kept.length} on-task output(s); need ≥${minSeats}`);
|
|
38
38
|
}
|
|
39
39
|
return { report, kept };
|
|
40
40
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// S6 — preserve validated analyst submissions for graph compilation. No lexical merging: positions
|
|
2
|
+
// keep their original text and are grouped only by stable references in S7.
|
|
3
|
+
export function collectPositions(seats) {
|
|
4
|
+
return seats.map((seat) => ({ provider: seat.provider, source_id: seat.sample ?? seat.provider, submission: seat.output }));
|
|
5
|
+
}
|
|
6
|
+
export async function s6Positions(ctx, seats) {
|
|
7
|
+
const positions = collectPositions(seats);
|
|
8
|
+
await ctx.writer.writeJson('positions', positions);
|
|
9
|
+
return positions;
|
|
10
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// S7 — deterministic conservative grouping plus pure decision-graph compilation. R6 removes the
|
|
2
|
+
// model-authored grouping call so graph audit spends no provider call on an obvious run.
|
|
3
|
+
import { compileDecisionGraph, coverageHoleQueue, positionId } from '../decision-graph.js';
|
|
4
|
+
import { IdeaRoleOutputModel } from '../../schemas/index.js';
|
|
5
|
+
import { isFatal } from '../context.js';
|
|
6
|
+
import { jsonCall } from '../jsonStage.js';
|
|
7
|
+
import { overlapCoefficient, tokenize } from '../cluster.js';
|
|
8
|
+
const GROUP_SIMILARITY = 0.8;
|
|
9
|
+
export function buildCoverageFillPrompt(task, holes) {
|
|
10
|
+
return `TARGETED COVERAGE FILL. Answer only the missing dimensions below; do not repeat the full analysis.
|
|
11
|
+
TASK: ${task}
|
|
12
|
+
MISSING DIMENSIONS:
|
|
13
|
+
${holes.map((hole) => `- ${hole.dimension_id}: ${hole.label}`).join('\n')}
|
|
14
|
+
|
|
15
|
+
Output ONLY the idea analyst JSON schema. Emit positions/evidence only when needed. Include one explicit
|
|
16
|
+
COVERED entry anchored to a position, or a reasoned NOT_APPLICABLE entry, for every listed dimension.
|
|
17
|
+
Required top-level keys: task_echo, strongest_version, positions, evidence, calculations, coverage, decision_questions.
|
|
18
|
+
Use calculations: [] unless a new position makes a derived numeric claim.
|
|
19
|
+
Use the same position and evidence-card shapes as the initial analysis; JSON only.`;
|
|
20
|
+
}
|
|
21
|
+
async function fillCoverage(ctx, submissions, rubric, task) {
|
|
22
|
+
const holes = coverageHoleQueue(submissions, rubric);
|
|
23
|
+
if (holes.length === 0 || ctx.optionalCallsRemaining() === 0)
|
|
24
|
+
return submissions;
|
|
25
|
+
const provider = ctx.roles.s4[0] ?? ctx.roles.analyst;
|
|
26
|
+
try {
|
|
27
|
+
const model = await jsonCall(ctx, ctx.handle(provider), 'S7-coverage-fill', buildCoverageFillPrompt(task, holes), IdeaRoleOutputModel, { repair: false });
|
|
28
|
+
const output = { workflow: 'idea-refinement', ...model };
|
|
29
|
+
await ctx.writer.writeJson('coverage-fill', output);
|
|
30
|
+
return [...submissions, { provider, source_id: `${provider}-coverage-fill`, submission: output }];
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
if (isFatal(error))
|
|
34
|
+
throw error;
|
|
35
|
+
return submissions;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/** Conservative exact/high-overlap grouping. False negatives stay separate; false merges are worse. */
|
|
39
|
+
export function deterministicClaimGroups(submissions) {
|
|
40
|
+
const positions = submissions.flatMap(({ provider, source_id = provider, submission }) => submission.positions.map((position) => ({
|
|
41
|
+
id: positionId(provider, position.local_id, source_id),
|
|
42
|
+
provider,
|
|
43
|
+
dimension: position.dimension_id,
|
|
44
|
+
proposition: position.proposition.trim().toLowerCase().replace(/[^a-z0-9]+/g, ' '),
|
|
45
|
+
tokens: tokenize(position.proposition),
|
|
46
|
+
})));
|
|
47
|
+
const used = new Set();
|
|
48
|
+
const groups = [];
|
|
49
|
+
for (const first of positions) {
|
|
50
|
+
if (used.has(first.id))
|
|
51
|
+
continue;
|
|
52
|
+
const group = [first];
|
|
53
|
+
for (const candidate of positions) {
|
|
54
|
+
if (used.has(candidate.id) || candidate.id === first.id)
|
|
55
|
+
continue;
|
|
56
|
+
if (candidate.provider === first.provider || candidate.dimension !== first.dimension)
|
|
57
|
+
continue;
|
|
58
|
+
const same = candidate.proposition === first.proposition
|
|
59
|
+
|| overlapCoefficient(candidate.tokens, first.tokens) >= GROUP_SIMILARITY;
|
|
60
|
+
if (same && !group.some((item) => item.provider === candidate.provider))
|
|
61
|
+
group.push(candidate);
|
|
62
|
+
}
|
|
63
|
+
if (group.length >= 2) {
|
|
64
|
+
group.forEach((item) => used.add(item.id));
|
|
65
|
+
groups.push(group.map((item) => item.id));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return groups;
|
|
69
|
+
}
|
|
70
|
+
export async function s7DecisionGraph(ctx, submissions, rubric, task = '') {
|
|
71
|
+
const completed = await fillCoverage(ctx, submissions, rubric, task);
|
|
72
|
+
const groups = deterministicClaimGroups(completed);
|
|
73
|
+
const graph = compileDecisionGraph(completed, rubric, groups);
|
|
74
|
+
await ctx.writer.writeJson('decision-graph', graph);
|
|
75
|
+
return graph;
|
|
76
|
+
}
|
|
@@ -1,56 +1,163 @@
|
|
|
1
|
-
// S8 —
|
|
2
|
-
//
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
// graceful: every item is marked UNCERTAIN ("unverified") and passed to the judge, never an abort (§259).
|
|
7
|
-
import { VerificationSet } from '../../schemas/index.js';
|
|
8
|
-
import { isFatal } from '../context.js';
|
|
1
|
+
// S8 — independently verify graph-selected claims from stored evidence/calculation references.
|
|
2
|
+
// Provider identity stays hidden; model output is translated back to graph evidence IDs before S9.
|
|
3
|
+
import { selectEscalations } from '../decision-graph.js';
|
|
4
|
+
import { ClaimVerificationSet } from '../../schemas/index.js';
|
|
5
|
+
import { isFatal, StageError } from '../context.js';
|
|
9
6
|
import { jsonCall } from '../jsonStage.js';
|
|
10
|
-
const S8_PROMPT = `ROLE:
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
"
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
7
|
+
const S8_PROMPT = `ROLE: Independent verifier. Check each anonymous decision claim below against the
|
|
8
|
+
provided evidence cards and deterministic calculation checks. Output ONLY JSON:
|
|
9
|
+
{"verifications": [{"claim_id": "<G-id>", "status": "VERIFIED|PARTIAL|CONTRADICTED|UNVERIFIABLE",
|
|
10
|
+
"reasoning": "<concise reason>", "evidence_ids": ["<E-id>"],
|
|
11
|
+
"calculation_check": "PASS|FAIL|NOT_APPLICABLE", "missing_evidence": ["<gap>"]}]}
|
|
12
|
+
VERIFIED = the proposition is supported. CONTRADICTED = it is refuted. PARTIAL = evidence is mixed or
|
|
13
|
+
incomplete. UNVERIFIABLE = the supplied sources cannot decide. Cite only supplied evidence IDs. Model
|
|
14
|
+
knowledge cannot verify a current, numeric, legal, medical, financial, market, or regulatory fact.
|
|
15
|
+
Judge each item independently; no verdict quota. JSON only.
|
|
16
|
+
ITEMS: {{ITEMS_JSON}}`;
|
|
17
|
+
function evidenceIdsForClaim(graph, claimId) {
|
|
18
|
+
const positionById = new Map(graph.positions.map((position) => [position.id, position]));
|
|
19
|
+
const claim = graph.claims.find((item) => item.id === claimId);
|
|
20
|
+
const positionEvidence = claim?.position_ids.flatMap((id) => {
|
|
21
|
+
const position = positionById.get(id);
|
|
22
|
+
return position?.evidence_ids.map((evidenceId) => `${position.source_id}/${evidenceId}`) ?? [];
|
|
23
|
+
}) ?? [];
|
|
24
|
+
const calculationEvidence = graph.calculations.filter((calculation) => calculation.claim_id === claimId)
|
|
25
|
+
.flatMap((calculation) => calculation.inputs.flatMap((input) => input.evidence_ids));
|
|
26
|
+
return new Set([...positionEvidence, ...calculationEvidence]);
|
|
27
|
+
}
|
|
28
|
+
function verifierInput(graph) {
|
|
29
|
+
const positionById = new Map(graph.positions.map((position) => [position.id, position]));
|
|
30
|
+
const evidenceRefs = new Map(graph.evidence.map((evidence, index) => [`E${index + 1}`, evidence.id]));
|
|
31
|
+
const aliasByEvidence = new Map([...evidenceRefs].map(([alias, id]) => [id, alias]));
|
|
32
|
+
const evidenceById = new Map(graph.evidence.map((evidence) => [evidence.id, evidence]));
|
|
33
|
+
const items = selectEscalations(graph, { max: 8 }).map((escalation) => {
|
|
34
|
+
const claim = graph.claims.find((item) => item.id === escalation.claim_id);
|
|
35
|
+
const linkedEvidenceIds = evidenceIdsForClaim(graph, claim.id);
|
|
36
|
+
return {
|
|
37
|
+
id: claim.id,
|
|
38
|
+
kind: escalation.kind,
|
|
39
|
+
proposition: claim.proposition,
|
|
40
|
+
positions: claim.position_ids.map((id) => {
|
|
41
|
+
const position = positionById.get(id);
|
|
42
|
+
return { stance: position.stance, reasoning: position.reasoning };
|
|
43
|
+
}),
|
|
44
|
+
evidence: [...linkedEvidenceIds].flatMap((id) => {
|
|
45
|
+
const evidence = evidenceById.get(id);
|
|
46
|
+
if (!evidence)
|
|
47
|
+
return [];
|
|
48
|
+
const { provider: _provider, source_id: _sourceId, ...card } = evidence;
|
|
49
|
+
return [{ ...card, id: aliasByEvidence.get(id) }];
|
|
50
|
+
}),
|
|
51
|
+
calculations: graph.calculations.filter((calculation) => calculation.claim_id === claim.id).map((calculation, index) => ({
|
|
52
|
+
id: `C${index + 1}`,
|
|
53
|
+
inputs: calculation.inputs.map((input) => ({
|
|
54
|
+
...input,
|
|
55
|
+
evidence_ids: input.evidence_ids.map((id) => aliasByEvidence.get(id)).filter((id) => id !== undefined),
|
|
56
|
+
})),
|
|
57
|
+
steps: calculation.steps,
|
|
58
|
+
result_step: calculation.result_step,
|
|
59
|
+
deterministic_check: (() => {
|
|
60
|
+
const check = graph.calculation_checks.find((item) => item.calculation_id === calculation.id);
|
|
61
|
+
return check ? { status: check.status, issues: check.issues } : undefined;
|
|
62
|
+
})(),
|
|
63
|
+
})),
|
|
64
|
+
evidence_hole: graph.holes.evidence.find((hole) => hole.claim_id === claim.id)?.reason,
|
|
65
|
+
};
|
|
66
|
+
});
|
|
67
|
+
return { prompt: S8_PROMPT.replace('{{ITEMS_JSON}}', JSON.stringify(items, null, 2)), evidenceRefs };
|
|
68
|
+
}
|
|
69
|
+
export function buildVerifierPrompt(graph) {
|
|
70
|
+
return verifierInput(graph).prompt;
|
|
71
|
+
}
|
|
72
|
+
/** Validate every S8 claim/evidence reference before the chair can see it. */
|
|
73
|
+
export function claimVerificationRefIssues(graph, verifications, expectedIds = selectEscalations(graph, { max: 8 }).map((item) => item.claim_id)) {
|
|
74
|
+
const expected = new Set(expectedIds);
|
|
75
|
+
const evidence = new Map(graph.evidence.map((item) => [item.id, item]));
|
|
76
|
+
const seen = new Set();
|
|
77
|
+
const issues = [];
|
|
78
|
+
for (const verification of verifications.verifications) {
|
|
79
|
+
if (!expected.has(verification.claim_id))
|
|
80
|
+
issues.push(`unknown claim id: ${verification.claim_id}`);
|
|
81
|
+
if (seen.has(verification.claim_id))
|
|
82
|
+
issues.push(`duplicate claim verification: ${verification.claim_id}`);
|
|
83
|
+
seen.add(verification.claim_id);
|
|
84
|
+
const cards = verification.evidence_ids.map((id) => evidence.get(id));
|
|
85
|
+
for (const [index, card] of cards.entries())
|
|
86
|
+
if (!card)
|
|
87
|
+
issues.push(`unknown evidence id for ${verification.claim_id}: ${verification.evidence_ids[index]}`);
|
|
88
|
+
const linkedEvidence = evidenceIdsForClaim(graph, verification.claim_id);
|
|
89
|
+
for (const id of verification.evidence_ids)
|
|
90
|
+
if (evidence.has(id) && !linkedEvidence.has(id))
|
|
91
|
+
issues.push(`unrelated evidence id for ${verification.claim_id}: ${id}`);
|
|
92
|
+
if (verification.status !== 'UNVERIFIABLE' && verification.evidence_ids.length === 0) {
|
|
93
|
+
issues.push(`${verification.claim_id}: ${verification.status} requires evidence ids`);
|
|
94
|
+
}
|
|
95
|
+
const evidenceHole = graph.holes.evidence.find((hole) => hole.claim_id === verification.claim_id);
|
|
96
|
+
const onlyModelKnowledge = cards.length > 0 && cards.every((card) => card?.source_kind === 'MODEL_KNOWLEDGE');
|
|
97
|
+
if (verification.status === 'VERIFIED' && evidenceHole && onlyModelKnowledge) {
|
|
98
|
+
issues.push(`${verification.claim_id}: model knowledge cannot close its evidence hole`);
|
|
99
|
+
}
|
|
100
|
+
const calculationChecks = graph.calculation_checks.filter((check) => check.claim_id === verification.claim_id);
|
|
101
|
+
if (calculationChecks.some((check) => check.status === 'FAIL')) {
|
|
102
|
+
if (verification.calculation_check !== 'FAIL')
|
|
103
|
+
issues.push(`${verification.claim_id}: failed deterministic calculation must be reported`);
|
|
104
|
+
if (verification.status === 'VERIFIED')
|
|
105
|
+
issues.push(`${verification.claim_id}: failed deterministic calculation cannot be verified`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
for (const id of expected)
|
|
109
|
+
if (!seen.has(id))
|
|
110
|
+
issues.push(`missing claim verification: ${id}`);
|
|
111
|
+
return issues;
|
|
112
|
+
}
|
|
113
|
+
export async function s8Verify(ctx, graph) {
|
|
114
|
+
const escalations = selectEscalations(graph, { max: 8 });
|
|
115
|
+
if (escalations.length === 0) {
|
|
22
116
|
const empty = { verifications: [] };
|
|
23
117
|
await ctx.writer.writeJson('verifications', empty);
|
|
24
118
|
return empty;
|
|
25
119
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
120
|
+
const unavailable = () => ({
|
|
121
|
+
verifications: escalations.map((escalation) => {
|
|
122
|
+
const checks = graph.calculation_checks.filter((check) => check.claim_id === escalation.claim_id);
|
|
123
|
+
return {
|
|
124
|
+
claim_id: escalation.claim_id,
|
|
125
|
+
status: 'UNVERIFIABLE',
|
|
126
|
+
reasoning: 'Independent verification was unavailable or skipped to preserve required calls.',
|
|
127
|
+
evidence_ids: [],
|
|
128
|
+
calculation_check: checks.length === 0 ? 'NOT_APPLICABLE'
|
|
129
|
+
: checks.some((check) => check.status === 'FAIL') ? 'FAIL' : 'PASS',
|
|
130
|
+
missing_evidence: ['independent verification'],
|
|
131
|
+
};
|
|
132
|
+
}),
|
|
133
|
+
});
|
|
134
|
+
if (ctx.optionalCallsRemaining() === 0) {
|
|
135
|
+
ctx.addFlag('verification_skipped');
|
|
136
|
+
const skipped = unavailable();
|
|
137
|
+
await ctx.writer.writeJson('verifications', skipped);
|
|
138
|
+
return skipped;
|
|
40
139
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
const
|
|
46
|
-
verifications:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
evidence: '(verifier unavailable — unverified)',
|
|
50
|
-
note: '',
|
|
140
|
+
try {
|
|
141
|
+
const input = verifierInput(graph);
|
|
142
|
+
// Optional work gets no model repair: one logical optional call must remain one call.
|
|
143
|
+
const result = await jsonCall(ctx, ctx.handle(ctx.roles.verifier), 'S8', input.prompt, ClaimVerificationSet, { repair: false });
|
|
144
|
+
const translated = {
|
|
145
|
+
verifications: result.verifications.map((verification) => ({
|
|
146
|
+
...verification,
|
|
147
|
+
evidence_ids: verification.evidence_ids.map((id) => input.evidenceRefs.get(id) ?? id),
|
|
51
148
|
})),
|
|
52
149
|
};
|
|
53
|
-
|
|
54
|
-
|
|
150
|
+
const issues = claimVerificationRefIssues(graph, translated, escalations.map((item) => item.claim_id));
|
|
151
|
+
if (issues.length)
|
|
152
|
+
throw new StageError('S8', 'BAD_OUTPUT', issues.join('; '));
|
|
153
|
+
await ctx.writer.writeJson('verifications', translated);
|
|
154
|
+
return translated;
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
if (isFatal(error))
|
|
158
|
+
throw error;
|
|
159
|
+
const failed = unavailable();
|
|
160
|
+
await ctx.writer.writeJson('verifications', failed);
|
|
161
|
+
return failed;
|
|
55
162
|
}
|
|
56
163
|
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// R5 — one optional, bounded rebuttal round over verdict-flipping graph nodes. Original graph
|
|
2
|
+
// objects are never rewritten; validated responses are appended as a separate 08b event artifact.
|
|
3
|
+
import { RebuttalEventSet as RebuttalEventSetSchema, RebuttalResponseSet } from '../../schemas/index.js';
|
|
4
|
+
import { selectRebuttalEscalations, } from '../decision-graph.js';
|
|
5
|
+
import { isFatal, StageError } from '../context.js';
|
|
6
|
+
import { jsonCall } from '../jsonStage.js';
|
|
7
|
+
import { IDEA_MODE_PLANS } from '../modes.js';
|
|
8
|
+
/** Internal protocol caps only. R6 owns exposing modes and changing the surrounding topology. */
|
|
9
|
+
export const REBUTTAL_LIMITS = {
|
|
10
|
+
quick: { maxNodes: 0, optionalCalls: 0 },
|
|
11
|
+
council: { maxNodes: 3, optionalCalls: 2 },
|
|
12
|
+
research: { maxNodes: 3, optionalCalls: 2 },
|
|
13
|
+
};
|
|
14
|
+
const REBUTTAL_PROMPT = `ROLE: Scout rebuttal. This is the council's only rebuttal round. Respond only
|
|
15
|
+
to the anonymous decision nodes assigned below. You may not rewrite any original claim or evidence.
|
|
16
|
+
|
|
17
|
+
Output ONLY JSON:
|
|
18
|
+
{"events":[{"claim_id":"<G-id>","response":"CONCEDE|COUNTER|NARROW|UNRESOLVED",
|
|
19
|
+
"reasoning":"<precise reason>","evidence_ids":["<E-id>"],
|
|
20
|
+
"narrowed_proposition":"<required only for NARROW>"}]}
|
|
21
|
+
|
|
22
|
+
CONCEDE = your position changes. COUNTER = retain it using a cited supplied card or a precise logical
|
|
23
|
+
rebuttal. NARROW = state the smaller proposition actually supported. UNRESOLVED = evidence cannot decide.
|
|
24
|
+
Return exactly one event per assigned claim. Cite only evidence IDs shown for that claim. Do not invent
|
|
25
|
+
sources, URLs, claim IDs, or evidence IDs. JSON only.
|
|
26
|
+
NODES: {{NODES_JSON}}`;
|
|
27
|
+
function claimAuthors(graph, claimId) {
|
|
28
|
+
const positionById = new Map(graph.positions.map((position) => [position.id, position]));
|
|
29
|
+
const claim = graph.claims.find((item) => item.id === claimId);
|
|
30
|
+
return [...new Set(claim?.position_ids
|
|
31
|
+
.map((id) => positionById.get(id)?.provider)
|
|
32
|
+
.filter((provider) => provider !== undefined) ?? [])];
|
|
33
|
+
}
|
|
34
|
+
function linkedEvidenceIds(graph, claimId) {
|
|
35
|
+
const evidenceIds = new Set(graph.evidence.map((item) => item.id));
|
|
36
|
+
const linked = graph.edges
|
|
37
|
+
.filter((edge) => edge.to === claimId && evidenceIds.has(edge.from))
|
|
38
|
+
.map((edge) => edge.from);
|
|
39
|
+
for (const calculation of graph.calculations.filter((item) => item.claim_id === claimId)) {
|
|
40
|
+
for (const input of calculation.inputs)
|
|
41
|
+
linked.push(...input.evidence_ids);
|
|
42
|
+
}
|
|
43
|
+
return [...new Set(linked)];
|
|
44
|
+
}
|
|
45
|
+
/** One grouped call per relevant scout; judge-authored nodes never enter the rebuttal/chair path. */
|
|
46
|
+
export function planRebuttalCalls(graph, escalations, scouts, judge) {
|
|
47
|
+
const assigned = new Map();
|
|
48
|
+
const add = (provider, claimId) => {
|
|
49
|
+
const ids = assigned.get(provider) ?? [];
|
|
50
|
+
if (!ids.includes(claimId))
|
|
51
|
+
ids.push(claimId);
|
|
52
|
+
assigned.set(provider, ids);
|
|
53
|
+
};
|
|
54
|
+
for (const escalation of escalations) {
|
|
55
|
+
const authors = claimAuthors(graph, escalation.claim_id);
|
|
56
|
+
if (authors.includes(judge))
|
|
57
|
+
continue;
|
|
58
|
+
if (escalation.kind === 'DISAGREEMENT' || escalation.kind === 'EVIDENCE_CONFLICT') {
|
|
59
|
+
for (const scout of scouts)
|
|
60
|
+
if (authors.includes(scout) && scout !== judge)
|
|
61
|
+
add(scout, escalation.claim_id);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const challenger = scouts.find((scout) => scout !== judge && !authors.includes(scout));
|
|
65
|
+
if (challenger)
|
|
66
|
+
add(challenger, escalation.claim_id);
|
|
67
|
+
}
|
|
68
|
+
return scouts.flatMap((provider) => {
|
|
69
|
+
const claim_ids = assigned.get(provider);
|
|
70
|
+
return claim_ids?.length ? [{ provider, claim_ids }] : [];
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
function rebuttalInput(graph, verifications, escalationById, assignment) {
|
|
74
|
+
const claimIds = new Set(assignment.claim_ids);
|
|
75
|
+
const positionRefs = new Map();
|
|
76
|
+
const evidenceRefs = new Map();
|
|
77
|
+
const aliasPosition = (id) => {
|
|
78
|
+
const found = [...positionRefs].find(([, value]) => value === id)?.[0];
|
|
79
|
+
if (found)
|
|
80
|
+
return found;
|
|
81
|
+
const alias = `P${positionRefs.size + 1}`;
|
|
82
|
+
positionRefs.set(alias, id);
|
|
83
|
+
return alias;
|
|
84
|
+
};
|
|
85
|
+
const aliasEvidence = (id) => {
|
|
86
|
+
const found = [...evidenceRefs].find(([, value]) => value === id)?.[0];
|
|
87
|
+
if (found)
|
|
88
|
+
return found;
|
|
89
|
+
const alias = `E${evidenceRefs.size + 1}`;
|
|
90
|
+
evidenceRefs.set(alias, id);
|
|
91
|
+
return alias;
|
|
92
|
+
};
|
|
93
|
+
const positionById = new Map(graph.positions.map((position) => [position.id, position]));
|
|
94
|
+
const evidenceById = new Map(graph.evidence.map((evidence) => [evidence.id, evidence]));
|
|
95
|
+
const verificationById = new Map(verifications.verifications.map((item) => [item.claim_id, item]));
|
|
96
|
+
const allowedEvidence = new Map();
|
|
97
|
+
const nodes = graph.claims.filter((claim) => claimIds.has(claim.id)).map((claim) => {
|
|
98
|
+
const evidenceIds = linkedEvidenceIds(graph, claim.id);
|
|
99
|
+
const aliases = new Set(evidenceIds.map(aliasEvidence));
|
|
100
|
+
allowedEvidence.set(claim.id, aliases);
|
|
101
|
+
const positions = claim.position_ids.map((id) => positionById.get(id)).map((position) => ({
|
|
102
|
+
id: aliasPosition(position.id),
|
|
103
|
+
proposition: position.proposition,
|
|
104
|
+
stance: position.stance,
|
|
105
|
+
reasoning: position.reasoning,
|
|
106
|
+
}));
|
|
107
|
+
const verification = verificationById.get(claim.id);
|
|
108
|
+
return {
|
|
109
|
+
claim_id: claim.id,
|
|
110
|
+
escalation_kind: escalationById.get(claim.id)?.kind,
|
|
111
|
+
proposition: claim.proposition,
|
|
112
|
+
consequence_if_false: claim.if_false,
|
|
113
|
+
your_positions: claim.position_ids
|
|
114
|
+
.filter((id) => positionById.get(id)?.provider === assignment.provider)
|
|
115
|
+
.map((id) => positions.find((position) => position.id === aliasPosition(id))),
|
|
116
|
+
opposing_positions: claim.position_ids
|
|
117
|
+
.filter((id) => positionById.get(id)?.provider !== assignment.provider)
|
|
118
|
+
.map((id) => positions.find((position) => position.id === aliasPosition(id))),
|
|
119
|
+
evidence: evidenceIds.flatMap((id) => {
|
|
120
|
+
const evidence = evidenceById.get(id);
|
|
121
|
+
if (!evidence)
|
|
122
|
+
return [];
|
|
123
|
+
const { provider: _provider, source_id: _sourceId, ...card } = evidence;
|
|
124
|
+
return [{ ...card, id: aliasEvidence(id) }];
|
|
125
|
+
}),
|
|
126
|
+
verification: verification ? {
|
|
127
|
+
...verification,
|
|
128
|
+
evidence_ids: verification.evidence_ids.map(aliasEvidence),
|
|
129
|
+
} : undefined,
|
|
130
|
+
};
|
|
131
|
+
});
|
|
132
|
+
return {
|
|
133
|
+
prompt: REBUTTAL_PROMPT.replace('{{NODES_JSON}}', JSON.stringify(nodes, null, 2)),
|
|
134
|
+
evidenceRefs,
|
|
135
|
+
allowedEvidence,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
function translateEvents(graph, assignment, input, output, startIndex) {
|
|
139
|
+
const expected = new Set(assignment.claim_ids);
|
|
140
|
+
const seen = new Set();
|
|
141
|
+
const issues = [];
|
|
142
|
+
for (const event of output.events) {
|
|
143
|
+
if (!expected.has(event.claim_id))
|
|
144
|
+
issues.push(`unknown claim id: ${event.claim_id}`);
|
|
145
|
+
if (seen.has(event.claim_id))
|
|
146
|
+
issues.push(`duplicate claim response: ${event.claim_id}`);
|
|
147
|
+
seen.add(event.claim_id);
|
|
148
|
+
const allowed = input.allowedEvidence.get(event.claim_id) ?? new Set();
|
|
149
|
+
for (const id of event.evidence_ids)
|
|
150
|
+
if (!allowed.has(id))
|
|
151
|
+
issues.push(`invalid evidence id for ${event.claim_id}: ${id}`);
|
|
152
|
+
}
|
|
153
|
+
for (const id of expected)
|
|
154
|
+
if (!seen.has(id))
|
|
155
|
+
issues.push(`missing claim response: ${id}`);
|
|
156
|
+
if (issues.length)
|
|
157
|
+
throw new StageError('S8b', 'BAD_OUTPUT', issues.join('; '));
|
|
158
|
+
const positionById = new Map(graph.positions.map((position) => [position.id, position]));
|
|
159
|
+
const claimById = new Map(graph.claims.map((claim) => [claim.id, claim]));
|
|
160
|
+
return output.events.map((event, index) => ({
|
|
161
|
+
...event,
|
|
162
|
+
id: `R${startIndex + index + 1}`,
|
|
163
|
+
round: 1,
|
|
164
|
+
responder: assignment.provider,
|
|
165
|
+
target_position_ids: claimById.get(event.claim_id)?.position_ids
|
|
166
|
+
.filter((id) => positionById.get(id)?.provider === assignment.provider) ?? [],
|
|
167
|
+
evidence_ids: event.evidence_ids.map((id) => input.evidenceRefs.get(id)),
|
|
168
|
+
}));
|
|
169
|
+
}
|
|
170
|
+
export async function s8bRebuttal(ctx, graph, verifications, mode = ctx.mode) {
|
|
171
|
+
const limits = REBUTTAL_LIMITS[mode];
|
|
172
|
+
const escalations = selectRebuttalEscalations(graph, verifications, { maxNodes: limits.maxNodes });
|
|
173
|
+
const selected_claim_ids = escalations.map((item) => item.claim_id);
|
|
174
|
+
const finish = async (events, stop_reason) => {
|
|
175
|
+
const result = RebuttalEventSetSchema.parse({ round: 1, selected_claim_ids, events, stop_reason });
|
|
176
|
+
await ctx.writer.writeJson('rebuttals', result);
|
|
177
|
+
return result;
|
|
178
|
+
};
|
|
179
|
+
if (escalations.length === 0)
|
|
180
|
+
return finish([], 'NO_ESCALATIONS');
|
|
181
|
+
const planned = planRebuttalCalls(graph, escalations, ctx.roles.s4, ctx.roles.judge);
|
|
182
|
+
if (planned.length === 0)
|
|
183
|
+
return finish([], 'NO_ELIGIBLE_SCOUT');
|
|
184
|
+
const optionalCalls = Math.min(limits.optionalCalls, ctx.optionalCallsRemaining());
|
|
185
|
+
if (optionalCalls === 0) {
|
|
186
|
+
const budgetRoom = ctx.budget.limit - ctx.budget.used - IDEA_MODE_PLANS[mode].reservedCalls;
|
|
187
|
+
return finish([], budgetRoom <= 0 ? 'BUDGET_RESERVED' : 'CALL_CAP_REACHED');
|
|
188
|
+
}
|
|
189
|
+
const assignments = planned.slice(0, optionalCalls);
|
|
190
|
+
const escalationById = new Map(escalations.map((item) => [item.claim_id, item]));
|
|
191
|
+
const events = [];
|
|
192
|
+
for (const assignment of assignments) {
|
|
193
|
+
const input = rebuttalInput(graph, verifications, escalationById, assignment);
|
|
194
|
+
try {
|
|
195
|
+
const output = await jsonCall(ctx, ctx.handle(assignment.provider), `S8b-${assignment.provider}`, input.prompt, RebuttalResponseSet, { repair: false });
|
|
196
|
+
events.push(...translateEvents(graph, assignment, input, output, events.length));
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
if (isFatal(error))
|
|
200
|
+
throw error;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (planned.length > optionalCalls)
|
|
204
|
+
return finish(events, 'CALL_CAP_REACHED');
|
|
205
|
+
if (events.length > 0 && events.every((event) => event.evidence_ids.length === 0))
|
|
206
|
+
return finish(events, 'NO_NEW_EVIDENCE');
|
|
207
|
+
return finish(events, 'ROUND_COMPLETE');
|
|
208
|
+
}
|