@kienntpixon/dieu-phoi 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/bin/dieu-phoi.cjs +20 -7
- package/hook.cjs +625 -15
- package/lib/aggregate.cjs +181 -16
- package/lib/args.cjs +1 -1
- package/lib/choose.cjs +101 -27
- package/lib/commands.cjs +243 -22
- package/lib/knowledge.cjs +74 -18
- package/lib/setup.cjs +16 -8
- package/package.json +1 -1
package/lib/aggregate.cjs
CHANGED
|
@@ -56,13 +56,14 @@ function round6(v) {
|
|
|
56
56
|
|
|
57
57
|
/**
|
|
58
58
|
* Normalize a verdict to the two-layer model. Legacy `rework` (no `by`) = user reject/quality;
|
|
59
|
-
* missing `by` otherwise = agent layer.
|
|
59
|
+
* missing `by` otherwise = agent layer. Core 0.7.0: `by=auto` (fast-mode self-acceptance) is its own layer —
|
|
60
|
+
* never a user verdict, never an agent verdict (CONTRACT "Mode theo yêu cầu").
|
|
60
61
|
*/
|
|
61
62
|
function normalizeVerdict(ev) {
|
|
62
63
|
if (ev.outcome === 'rework' && !ev.by) {
|
|
63
64
|
return { by: 'user', outcome: 'reject', cause: 'quality', round: Number(ev.round) || null };
|
|
64
65
|
}
|
|
65
|
-
const by = ev.by === 'user' ? 'user' : 'agent';
|
|
66
|
+
const by = ev.by === 'user' ? 'user' : (ev.by === 'auto' ? 'auto' : 'agent');
|
|
66
67
|
let cause = ev.cause || null;
|
|
67
68
|
if (by === 'user' && ev.outcome === 'reject' && cause !== 'scope') cause = 'quality';
|
|
68
69
|
let outcome = ev.outcome;
|
|
@@ -111,12 +112,18 @@ function steerIdOf(ev) {
|
|
|
111
112
|
return typeof ev.steer_id === 'string' && ev.steer_id ? ev.steer_id : `${ev.agent_id}|${ev.ts}`;
|
|
112
113
|
}
|
|
113
114
|
|
|
115
|
+
/** core 0.7.0: an exec's own selective-test verdict (`verify=self`) is not evidence for the agent layer. */
|
|
116
|
+
function isSelf(v) {
|
|
117
|
+
return !!v && v.verify === 'self';
|
|
118
|
+
}
|
|
119
|
+
|
|
114
120
|
/**
|
|
115
|
-
* Join events. Returns agents (one per real agent), userVerdicts (task layer), overrides, steers.
|
|
121
|
+
* Join events. Returns agents (one per real agent), userVerdicts (task layer), autoVerdicts (by=auto), overrides, steers.
|
|
116
122
|
*/
|
|
117
123
|
function buildAgents(events) {
|
|
118
124
|
const map = new Map();
|
|
119
125
|
const userVerdicts = [];
|
|
126
|
+
const autoVerdicts = [];
|
|
120
127
|
const overrides = [];
|
|
121
128
|
const steers = [];
|
|
122
129
|
const get = (id) => {
|
|
@@ -132,6 +139,7 @@ function buildAgents(events) {
|
|
|
132
139
|
if (ev.kind === 'verdict') {
|
|
133
140
|
const v = Object.assign(e, normalizeVerdict(ev));
|
|
134
141
|
if (v.by === 'user') { userVerdicts.push(v); return; }
|
|
142
|
+
if (v.by === 'auto') { autoVerdicts.push(v); return; }
|
|
135
143
|
}
|
|
136
144
|
if (ev.kind === 'steer') steers.push(e);
|
|
137
145
|
if (typeof ev.agent_id !== 'string' || !ev.agent_id) return;
|
|
@@ -149,6 +157,8 @@ function buildAgents(events) {
|
|
|
149
157
|
const vAll = a.verdicts.slice().sort((x, y) => tsOf(x) - tsOf(y) || x._idx - y._idx);
|
|
150
158
|
// `aborted` (quota pause) is neither pass nor fail: it never sets the outcome nor counts as a result.
|
|
151
159
|
const v = vAll.filter((x) => x.outcome !== ABORTED);
|
|
160
|
+
const checked = v.filter((x) => !isSelf(x));
|
|
161
|
+
const selfs = v.filter(isSelf);
|
|
152
162
|
const verdict = v.length ? v[v.length - 1] : null;
|
|
153
163
|
const firstVerdictTs = v.length ? tsOf(v[0]) : Infinity;
|
|
154
164
|
const pick = (k) => (d[k] !== undefined && d[k] !== null ? d[k] : (c0[k] !== undefined && c0[k] !== null ? c0[k] : null));
|
|
@@ -163,6 +173,8 @@ function buildAgents(events) {
|
|
|
163
173
|
task_id: pick('task_id') || (verdict && verdict.task_id) || (a.steers[0] && a.steers[0].task_id) || null,
|
|
164
174
|
try: Number(pick('try')) || 1,
|
|
165
175
|
role: pick('role') || 'exec',
|
|
176
|
+
// core 0.7.1 (C5): the one batched verifier of a fast request (tag `|batch`, dispatch `batch:true`)
|
|
177
|
+
batch: pick('batch') === true,
|
|
166
178
|
score: d.score !== undefined ? d.score : null,
|
|
167
179
|
urgency: d.urgency || null,
|
|
168
180
|
model: core.normalizeModel(cl.model) || core.normalizeModel(d.model) || null,
|
|
@@ -173,7 +185,10 @@ function buildAgents(events) {
|
|
|
173
185
|
duration_s: a.completes.length ? sumOrNull(a.completes.map((x) => x.duration_s)) : null,
|
|
174
186
|
tool_uses: a.completes.length ? sumOrNull(a.completes.map((x) => x.tool_uses)) : null,
|
|
175
187
|
outcome: verdict ? verdict.outcome : null,
|
|
176
|
-
|
|
188
|
+
// agent layer = first verdict that is not the exec's own selective tests (verify=self, core 0.7.0)
|
|
189
|
+
first_outcome: checked.length ? checked[0].outcome : null,
|
|
190
|
+
self_first: selfs.length ? selfs[0].outcome : null,
|
|
191
|
+
self_verdicts: selfs.length,
|
|
177
192
|
// every non-aborted agent verdict in time order: a retry by SendMessage to the same agent is a new result
|
|
178
193
|
verdict_seq: v.map((x) => ({ outcome: x.outcome, ts: tsOf(x), idx: x._idx })),
|
|
179
194
|
aborted: vAll.length > 0 && vAll[vAll.length - 1].outcome === ABORTED,
|
|
@@ -187,24 +202,28 @@ function buildAgents(events) {
|
|
|
187
202
|
last_complete_ts: a.completes.length ? Math.max(...a.completes.map(tsOf)) : 0,
|
|
188
203
|
});
|
|
189
204
|
}
|
|
190
|
-
return { agents, userVerdicts, overrides, steers };
|
|
205
|
+
return { agents, userVerdicts, autoVerdicts, overrides, steers };
|
|
191
206
|
}
|
|
192
207
|
|
|
193
208
|
/** Group agents into tasks (sample unit = task_id; agents without task_id are their own task). */
|
|
194
|
-
function buildTasks(agents, userVerdicts) {
|
|
209
|
+
function buildTasks(agents, userVerdicts, autoVerdicts) {
|
|
195
210
|
const map = new Map();
|
|
196
211
|
const byAgent = new Map();
|
|
197
212
|
for (const a of agents) {
|
|
198
213
|
const key = a.task_id ? `task:${a.task_id}` : `agent:${a.agent_id}`;
|
|
199
|
-
if (!map.has(key)) map.set(key, { key, task_id: a.task_id, agents: [], user: [] });
|
|
214
|
+
if (!map.has(key)) map.set(key, { key, task_id: a.task_id, agents: [], user: [], auto: [] });
|
|
200
215
|
map.get(key).agents.push(a);
|
|
201
216
|
byAgent.set(a.agent_id, key);
|
|
202
217
|
}
|
|
218
|
+
for (const v of autoVerdicts || []) {
|
|
219
|
+
const t = v.task_id ? map.get(`task:${v.task_id}`) : null;
|
|
220
|
+
if (t) t.auto.push(v);
|
|
221
|
+
}
|
|
203
222
|
for (const v of userVerdicts || []) {
|
|
204
223
|
let key = v.task_id ? `task:${v.task_id}` : null;
|
|
205
224
|
if (!key && v.agent_id) key = byAgent.get(v.agent_id) || null; // legacy rework without task_id
|
|
206
225
|
if (!key) continue;
|
|
207
|
-
if (!map.has(key)) map.set(key, { key, task_id: v.task_id, agents: [], user: [] });
|
|
226
|
+
if (!map.has(key)) map.set(key, { key, task_id: v.task_id, agents: [], user: [], auto: [] });
|
|
208
227
|
map.get(key).user.push(v);
|
|
209
228
|
}
|
|
210
229
|
const tasks = [];
|
|
@@ -261,9 +280,15 @@ function buildTasks(agents, userVerdicts) {
|
|
|
261
280
|
if (userOutcome === 'accept') { success = true; source = 'user'; } else if (userOutcome === 'reject_quality') { success = false; source = 'user'; } else if (!userOutcome && agentFirst !== null) {
|
|
262
281
|
success = agentFirst; source = 'agent';
|
|
263
282
|
}
|
|
283
|
+
// core 0.7.0: by=auto never decides success; a user reject after the first auto accept overturns it.
|
|
284
|
+
const autoTs = t.auto.filter((v) => v.outcome === 'accept').map(tsOf);
|
|
285
|
+
const autoAccept = autoTs.length > 0;
|
|
286
|
+
const autoOverturned = autoAccept && user.some((v) => v.outcome === 'reject' && tsOf(v) > Math.min(...autoTs));
|
|
264
287
|
const lastUser = user[user.length - 1] || null;
|
|
265
288
|
const accepted = lastUser ? lastUser.outcome === 'accept' : attempts.some((a) => a.outcome === 'pass');
|
|
266
|
-
|
|
289
|
+
// core 0.7.1 (C5): the batched verifier checks every fast step of a request; it is billed to its own cell
|
|
290
|
+
// (batchVerifyCells), never to the task_type x tier of the fast step whose task_id it carries.
|
|
291
|
+
const costed = t.agents.filter((a) => (a.completed || a.outcome) && !isBatchVerify(a));
|
|
267
292
|
tasks.push({
|
|
268
293
|
key: t.key,
|
|
269
294
|
task_id: t.task_id,
|
|
@@ -283,6 +308,9 @@ function buildTasks(agents, userVerdicts) {
|
|
|
283
308
|
excluded: !!(first && first.steered) || userOutcome === 'reject_scope',
|
|
284
309
|
accepted,
|
|
285
310
|
agent_first_pass: agentFirst,
|
|
311
|
+
self_first: firstRes ? firstRes.self_first : null,
|
|
312
|
+
auto_accept: autoAccept,
|
|
313
|
+
auto_overturned: autoOverturned,
|
|
286
314
|
// agents with a verdict but no complete event (hook missed) make cost/time unknown (null)
|
|
287
315
|
total_cost: sumOrNull(costed.map((a) => a.cost_usd)),
|
|
288
316
|
total_time_s: sumOrNull(costed.map((a) => a.duration_s)),
|
|
@@ -291,6 +319,43 @@ function buildTasks(agents, userVerdicts) {
|
|
|
291
319
|
return tasks;
|
|
292
320
|
}
|
|
293
321
|
|
|
322
|
+
/** core 0.7.1 (C5): a role=verify agent dispatched with `batch:true`. */
|
|
323
|
+
function isBatchVerify(a) {
|
|
324
|
+
return !!a && a.role === 'verify' && a.batch === true;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* core 0.7.1 (C5), CLI only: batched verifiers grouped by their own model x effort. Agents with a complete or a
|
|
329
|
+
* verdict; cost/time per agent = sum over its complete segments (null when unknown, left out of the percentiles).
|
|
330
|
+
*/
|
|
331
|
+
function batchVerifyCells(agents) {
|
|
332
|
+
const groups = new Map();
|
|
333
|
+
for (const a of agents || []) {
|
|
334
|
+
if (!isBatchVerify(a) || !(a.completed || a.outcome)) continue;
|
|
335
|
+
const key = `${a.model || '?'}|${a.effort || '?'}`;
|
|
336
|
+
if (!groups.has(key)) groups.set(key, { model: a.model, effort: a.effort, agents: [] });
|
|
337
|
+
groups.get(key).agents.push(a);
|
|
338
|
+
}
|
|
339
|
+
const out = [];
|
|
340
|
+
for (const g of groups.values()) {
|
|
341
|
+
const costs = g.agents.map((a) => a.cost_usd);
|
|
342
|
+
const times = g.agents.map((a) => (a.duration_s === null ? null : a.duration_s / 60));
|
|
343
|
+
out.push({
|
|
344
|
+
model: g.model,
|
|
345
|
+
effort: g.effort,
|
|
346
|
+
n: g.agents.length,
|
|
347
|
+
tasks: [...new Set(g.agents.map((a) => a.task_id).filter(Boolean))].length,
|
|
348
|
+
cost_total_usd: round6(costs.reduce((s, c) => s + (typeof c === 'number' && isFinite(c) ? c : 0), 0)),
|
|
349
|
+
cost_mean_usd: round6(mean(costs)),
|
|
350
|
+
cost_p50_usd: round6(percentile(costs, 0.5)),
|
|
351
|
+
time_mean_min: round6(mean(times)),
|
|
352
|
+
time_p50_min: round6(percentile(times, 0.5)),
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
out.sort((a, b) => String(a.model).localeCompare(String(b.model)) || String(a.effort).localeCompare(String(b.effort)));
|
|
356
|
+
return out;
|
|
357
|
+
}
|
|
358
|
+
|
|
294
359
|
function cellKey(type, model, effort) {
|
|
295
360
|
return `${type}|${model || '?'}|${effort || '?'}`;
|
|
296
361
|
}
|
|
@@ -309,11 +374,17 @@ function buildCells(tasks, opts) {
|
|
|
309
374
|
if (!f) continue;
|
|
310
375
|
const key = cellKey(t.task_type, f.model, f.effort);
|
|
311
376
|
if (!cells.has(key)) {
|
|
312
|
-
cells.set(key, {
|
|
377
|
+
cells.set(key, {
|
|
378
|
+
task_type: t.task_type, model: f.model, effort: f.effort, dispatched: 0, excluded: 0, tasks: [], agentTasks: [], selfTasks: [], verdicted: 0, auto: 0, overturned: 0,
|
|
379
|
+
});
|
|
313
380
|
}
|
|
314
381
|
const c = cells.get(key);
|
|
315
382
|
c.dispatched++;
|
|
383
|
+
if (t.auto_accept) c.auto++;
|
|
384
|
+
if (t.auto_overturned) c.overturned++;
|
|
316
385
|
if (t.excluded) { c.excluded++; continue; }
|
|
386
|
+
if (t.self_first) c.selfTasks.push(t);
|
|
387
|
+
if (t.self_first || t.agent_first_pass !== null) c.verdicted++;
|
|
317
388
|
if (t.success !== null) c.tasks.push(t);
|
|
318
389
|
if (t.agent_first_pass !== null) c.agentTasks.push(t);
|
|
319
390
|
}
|
|
@@ -331,6 +402,9 @@ function buildCells(tasks, opts) {
|
|
|
331
402
|
// score uses the time median, not the mean: waits for the user (gate 3, quota pauses) inflate the mean
|
|
332
403
|
// 3-13x (retro 2026-10-18: art-gen sonnet mean 483 vs p50 37 min) and would flip priors to slower tiers.
|
|
333
404
|
const timeP50 = percentile(timesMin, 0.5);
|
|
405
|
+
const notes = [];
|
|
406
|
+
if (n < MIN_N) notes.push('chưa đủ dữ liệu');
|
|
407
|
+
if (c.verdicted && c.selfTasks.length / c.verdicted > 0.5) notes.push('ít bằng chứng');
|
|
334
408
|
out.push({
|
|
335
409
|
task_type: c.task_type,
|
|
336
410
|
model: c.model,
|
|
@@ -358,8 +432,12 @@ function buildCells(tasks, opts) {
|
|
|
358
432
|
attempt_p50_min: percentile(attemptTimes, 0.5),
|
|
359
433
|
attempt_p75_min: percentile(attemptTimes, 0.75),
|
|
360
434
|
score: costMean === null && timeP50 === null ? null : round6((costMean || 0) + lambda * (timeP50 || 0)),
|
|
435
|
+
n_self: c.selfTasks.length,
|
|
436
|
+
self_pass: c.selfTasks.filter((t) => t.self_first === 'pass').length,
|
|
437
|
+
auto_accept: c.auto,
|
|
438
|
+
auto_overturned: c.overturned,
|
|
361
439
|
enough: n >= MIN_N,
|
|
362
|
-
note:
|
|
440
|
+
note: notes.join('; '),
|
|
363
441
|
});
|
|
364
442
|
}
|
|
365
443
|
out.sort((a, b) => a.task_type.localeCompare(b.task_type)
|
|
@@ -485,16 +563,103 @@ function orchestrationSummary(events) {
|
|
|
485
563
|
}
|
|
486
564
|
|
|
487
565
|
function analyze(events, opts) {
|
|
488
|
-
const { agents, userVerdicts, overrides, steers } = buildAgents(withoutDiscarded(events));
|
|
489
|
-
const tasks = buildTasks(agents, userVerdicts);
|
|
566
|
+
const { agents, userVerdicts, autoVerdicts, overrides, steers } = buildAgents(withoutDiscarded(events));
|
|
567
|
+
const tasks = buildTasks(agents, userVerdicts, autoVerdicts);
|
|
490
568
|
const cells = buildCells(tasks, opts);
|
|
491
569
|
return {
|
|
492
|
-
agents, tasks, cells, overrides, userVerdicts, steerEvents: steers, steers: buildSteers(tasks, steers, agents),
|
|
570
|
+
agents, tasks, cells, overrides, userVerdicts, autoVerdicts, steerEvents: steers, steers: buildSteers(tasks, steers, agents),
|
|
493
571
|
orchestration: orchestrationSummary(events),
|
|
572
|
+
batch_verify: batchVerifyCells(agents),
|
|
494
573
|
};
|
|
495
574
|
}
|
|
496
575
|
|
|
497
576
|
module.exports = {
|
|
498
|
-
MIN_N, FAIL_OUTCOMES, HEAVY_STEERS, ABORTED, QUOTA_REASON_RE, HANDOFF_REASON_RE, discardedTaskIds, withoutDiscarded, CANONICAL_CELL_FIELDS, normalizeVerdict, isInternalComplete, buildAgents, buildTasks,
|
|
499
|
-
buildCells, buildSteers, canonicalCells, orchestrationSummary, analyze, cellKey, mean, percentile, tsOf, steerIdOf,
|
|
577
|
+
MIN_N, FAIL_OUTCOMES, HEAVY_STEERS, ABORTED, QUOTA_REASON_RE, HANDOFF_REASON_RE, discardedTaskIds, withoutDiscarded, CANONICAL_CELL_FIELDS, normalizeVerdict, isSelf, isInternalComplete, buildAgents, buildTasks,
|
|
578
|
+
buildCells, buildSteers, canonicalCells, isBatchVerify, batchVerifyCells, orchestrationSummary, analyze, cellKey, mean, percentile, tsOf, steerIdOf,
|
|
500
579
|
};
|
|
580
|
+
|
|
581
|
+
// ------------------------------------------------------------------ core 0.7.0 (s2): stats split by mode
|
|
582
|
+
//
|
|
583
|
+
// Kept apart from the canonical cells (not part of the CLI <-> sink parity yet). A task's mode = `mode` of its first
|
|
584
|
+
// exec dispatch; tasks dispatched before modes existed (no `mode`) count as normal.
|
|
585
|
+
|
|
586
|
+
const MODE_KEYS = ['fast', 'normal', 'strict'];
|
|
587
|
+
const NO_CATCH_VERIFY = new Set(['self', 'none']);
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Per mode: tasks, verified_tasks (a verify agent ran), catch_rate (share of exec agent verdicts checked by something
|
|
591
|
+
* other than self/none that came back fail), self_verdicts, batch_verify_fails, user_verdict_tasks, rejects (quality),
|
|
592
|
+
* reject_rate, verify_min (minutes of verify agents), test_s (sum of complete.test_s over the task's agents),
|
|
593
|
+
* test_s_per_task, full_suite_runs. `events` = the same events given to analyze(); `tasks` = analyze().tasks.
|
|
594
|
+
*/
|
|
595
|
+
function modeSummary(events, tasks) {
|
|
596
|
+
const evs = withoutDiscarded(events).filter(Boolean);
|
|
597
|
+
const dispatchMode = new Map();
|
|
598
|
+
const tests = new Map();
|
|
599
|
+
const verdicts = new Map();
|
|
600
|
+
for (const ev of evs) {
|
|
601
|
+
if (ev.kind === 'dispatch' && typeof ev.agent_id === 'string' && !dispatchMode.has(ev.agent_id)) dispatchMode.set(ev.agent_id, ev.mode);
|
|
602
|
+
if (ev.kind === 'complete' && typeof ev.agent_id === 'string') {
|
|
603
|
+
const t = tests.get(ev.agent_id) || { s: 0, timed: 0, full: 0 };
|
|
604
|
+
const s = finiteNum(ev.test_s);
|
|
605
|
+
if (s !== null) { t.s += s; t.timed++; }
|
|
606
|
+
t.full += finiteNum(ev.full_suite_runs) || 0;
|
|
607
|
+
tests.set(ev.agent_id, t);
|
|
608
|
+
}
|
|
609
|
+
if (ev.kind === 'verdict' && ev.by !== 'user' && typeof ev.agent_id === 'string') {
|
|
610
|
+
if (!verdicts.has(ev.agent_id)) verdicts.set(ev.agent_id, []);
|
|
611
|
+
verdicts.get(ev.agent_id).push(ev);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
const out = {};
|
|
615
|
+
for (const m of MODE_KEYS) {
|
|
616
|
+
out[m] = {
|
|
617
|
+
tasks: 0, verified_tasks: 0, checked_verdicts: 0, caught: 0, catch_rate: null, self_verdicts: 0, batch_verify_fails: 0,
|
|
618
|
+
user_verdict_tasks: 0, rejects: 0, reject_rate: null, verify_min: 0, test_s: 0, test_timed_tasks: 0, test_s_per_task: null, full_suite_runs: 0,
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
for (const t of tasks || []) {
|
|
622
|
+
if (!t.first) continue;
|
|
623
|
+
const raw = dispatchMode.get(t.first.agent_id);
|
|
624
|
+
const r = out[MODE_KEYS.includes(raw) ? raw : 'normal'];
|
|
625
|
+
r.tasks++;
|
|
626
|
+
if (t.verifies.length) r.verified_tasks++;
|
|
627
|
+
for (const v of t.verifies) r.verify_min += (finiteNum(v.duration_s) || 0) / 60;
|
|
628
|
+
let ts = 0;
|
|
629
|
+
let timed = false;
|
|
630
|
+
for (const a of t.attempts.concat(t.verifies)) {
|
|
631
|
+
const x = tests.get(a.agent_id);
|
|
632
|
+
if (!x) continue;
|
|
633
|
+
if (x.timed) { ts += x.s; timed = true; }
|
|
634
|
+
r.full_suite_runs += x.full;
|
|
635
|
+
}
|
|
636
|
+
if (timed) { r.test_s += ts; r.test_timed_tasks++; }
|
|
637
|
+
for (const a of t.attempts) {
|
|
638
|
+
if (a.role !== 'exec') continue;
|
|
639
|
+
for (const v of verdicts.get(a.agent_id) || []) {
|
|
640
|
+
if (v.outcome === 'fail' && /^batch-verify(?:-|$)/.test(String(v.reason_code || ''))) r.batch_verify_fails++;
|
|
641
|
+
if (v.verify === 'self') r.self_verdicts++;
|
|
642
|
+
if (v.outcome !== 'pass' && v.outcome !== 'fail') continue;
|
|
643
|
+
if (NO_CATCH_VERIFY.has(v.verify || 'none')) continue;
|
|
644
|
+
r.checked_verdicts++;
|
|
645
|
+
if (v.outcome === 'fail') r.caught++;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
if (t.user_verdicts.length) {
|
|
649
|
+
r.user_verdict_tasks++;
|
|
650
|
+
if (t.user_verdicts.some((v) => v.outcome === 'reject' && v.cause !== 'scope')) r.rejects++;
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
for (const m of MODE_KEYS) {
|
|
654
|
+
const r = out[m];
|
|
655
|
+
r.catch_rate = r.checked_verdicts ? round6(r.caught / r.checked_verdicts) : null;
|
|
656
|
+
r.reject_rate = r.user_verdict_tasks ? round6(r.rejects / r.user_verdict_tasks) : null;
|
|
657
|
+
r.verify_min = round6(r.verify_min);
|
|
658
|
+
r.test_s = round6(r.test_s);
|
|
659
|
+
r.test_s_per_task = r.test_timed_tasks ? round6(r.test_s / r.test_timed_tasks) : null;
|
|
660
|
+
}
|
|
661
|
+
return out;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
module.exports.MODE_KEYS = MODE_KEYS;
|
|
665
|
+
module.exports.modeSummary = modeSummary;
|
package/lib/args.cjs
CHANGED
|
@@ -4,7 +4,7 @@ const fs = require('fs');
|
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const core = require('../hook.cjs');
|
|
6
6
|
|
|
7
|
-
const BOOLEAN_FLAGS = new Set(['json', 'help', 'h', 'purge', 'version', 'v', 'dry-run', 'no-sync', 'approve', 'yes', 'force', 'follow', 'remove', 'dirty', 'no-cache', 'untagged', 'no-knowledge', 'net', 'with-codebase-memory', 'cbm', 'allow-custom-source']);
|
|
7
|
+
const BOOLEAN_FLAGS = new Set(['json', 'help', 'h', 'purge', 'version', 'v', 'dry-run', 'no-sync', 'approve', 'yes', 'force', 'follow', 'remove', 'dirty', 'no-cache', 'untagged', 'no-knowledge', 'net', 'with-codebase-memory', 'cbm', 'allow-custom-source', 'auto']);
|
|
8
8
|
|
|
9
9
|
/** Minimal argv parser: positionals, --k v, --k=v, boolean flags. */
|
|
10
10
|
function parseArgs(argv) {
|
package/lib/choose.cjs
CHANGED
|
@@ -132,9 +132,19 @@ function loadProfile(skillDir) {
|
|
|
132
132
|
const v = Number(r.orch_ctx_soft_cap);
|
|
133
133
|
if (isFinite(v) && v > 0) softCap = Math.min(DEFAULT_ORCH_CTX_SOFT_CAP, Math.floor(v));
|
|
134
134
|
}
|
|
135
|
+
// core 0.7.1 (retro 2026-10-26, C4): per-type tier floor from the user's preference (e.g. art-gen -> opus/medium).
|
|
136
|
+
// Only tightens: choose takes max(floor, safety floors, escalation); an entry off the ladder is ignored.
|
|
137
|
+
const tierFloor = {};
|
|
138
|
+
if (r.tier_floor && typeof r.tier_floor === 'object' && !Array.isArray(r.tier_floor)) {
|
|
139
|
+
for (const [k, v] of Object.entries(r.tier_floor)) {
|
|
140
|
+
if (k.startsWith('_')) continue;
|
|
141
|
+
const idx = parsePrior(v);
|
|
142
|
+
if (idx !== null) tierFloor[String(k).toLowerCase()] = idx;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
135
145
|
return {
|
|
136
146
|
source, dept: r.dept || null, priors, lambda, explore_rate: explore, time_cap_min: timeCap, extra_task_types: extra,
|
|
137
|
-
orch_ctx_soft_cap: softCap,
|
|
147
|
+
orch_ctx_soft_cap: softCap, tier_floor: tierFloor,
|
|
138
148
|
};
|
|
139
149
|
}
|
|
140
150
|
|
|
@@ -302,20 +312,8 @@ function estimateFor(idx, cell, pricing) {
|
|
|
302
312
|
|
|
303
313
|
// ------------------------------------------------------------------ task type guard
|
|
304
314
|
|
|
305
|
-
// Ad-hoc labels
|
|
306
|
-
|
|
307
|
-
const TYPE_ALIASES = {
|
|
308
|
-
'bug-fix': 'debug', bugfix: 'debug', fix: 'debug', bug: 'debug',
|
|
309
|
-
config: 'mechanical-edit', shell: 'mechanical-edit', deploy: 'mechanical-edit', ops: 'mechanical-edit', edit: 'mechanical-edit',
|
|
310
|
-
docs: 'content-write', doc: 'content-write', documentation: 'content-write', writing: 'content-write', copy: 'content-write',
|
|
311
|
-
design: 'plan-design', plan: 'plan-design', planning: 'plan-design', architecture: 'plan-design',
|
|
312
|
-
research: 'research-synth', synth: 'research-synth',
|
|
313
|
-
test: 'test-write', tests: 'test-write',
|
|
314
|
-
analysis: 'data-analysis', data: 'data-analysis',
|
|
315
|
-
impl: 'feature-impl', feature: 'feature-impl', implement: 'feature-impl',
|
|
316
|
-
'code-review': 'review',
|
|
317
|
-
'art-fix': 'art-gen', art: 'art-gen',
|
|
318
|
-
};
|
|
315
|
+
// Ad-hoc labels → the closest fixed type: the table lives in hook.cjs (core 0.7.1) so dispatch tags map the same way.
|
|
316
|
+
const TYPE_ALIASES = core.TYPE_ALIASES;
|
|
319
317
|
|
|
320
318
|
/**
|
|
321
319
|
* { type, known, mapped_from, warning }: a fixed/extra type passes; an ad-hoc label with a known alias target
|
|
@@ -334,10 +332,56 @@ function resolveTaskType(raw, profile) {
|
|
|
334
332
|
return { type: t, known: false, mapped_from: null, warning: `loại "${t}" không có trong danh sách cố định (${list}) — chọn loại gần nhất hoặc hỏi người dùng` };
|
|
335
333
|
}
|
|
336
334
|
|
|
335
|
+
// ------------------------------------------------------------------ verify plan (core 0.7.0, mode per request)
|
|
336
|
+
|
|
337
|
+
const VERIFY_PLANS = ['independent-blocking', 'independent-nonblocking', 'self+batch', 'spot'];
|
|
338
|
+
/** fast: highest score still allowed to drop its own blocking verifier (s<=6 self+batch, s7-8 independent-nonblocking). */
|
|
339
|
+
const FAST_SELF_MAX = 6;
|
|
340
|
+
const FAST_SOFT_MAX = 8;
|
|
341
|
+
/** strict: share of steps that also get a random Opus reviewer spot check. */
|
|
342
|
+
const STRICT_SPOT_RATE = 1 / 3;
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* How one exec step is checked, from the request mode and the step's rubric inputs.
|
|
346
|
+
* The safety floor is evaluated FIRST and wins in every mode: reversible=n, publish/deploy, verifiable=n, score>=9
|
|
347
|
+
* always get an independent blocking verifier. Only after that does the mode loosen (fast) or tighten (strict).
|
|
348
|
+
* opts: { mode, score, verifiable, reversible, publish, dependents, seed }
|
|
349
|
+
* Returns { verify_plan, verify_reason, verify_floor: string[], test_scope: 'selective'|'full' }.
|
|
350
|
+
*/
|
|
351
|
+
function verifyPlanFor(opts) {
|
|
352
|
+
const mode = opts.mode || core.DEFAULT_MODE;
|
|
353
|
+
if (!core.MODES.includes(mode)) throw new Error(`--mode phải là ${core.MODES.join('|')}`);
|
|
354
|
+
const score = Number(opts.score);
|
|
355
|
+
const floor = [];
|
|
356
|
+
if (opts.reversible === false) floor.push('khó hoàn tác');
|
|
357
|
+
if (opts.publish === true) floor.push('publish/deploy');
|
|
358
|
+
if (opts.verifiable === false) floor.push('không kiểm chứng bằng máy');
|
|
359
|
+
if (score >= 9) floor.push(`score ${score}≥9`);
|
|
360
|
+
const out = (plan, reason, scope) => ({ verify_plan: plan, verify_reason: reason, verify_floor: floor, test_scope: scope });
|
|
361
|
+
if (floor.length) return out('independent-blocking', `sàn an toàn (${floor.join(', ')}) → verifier độc lập chặn bước, mọi mode`, 'full');
|
|
362
|
+
if (mode === 'fast') {
|
|
363
|
+
if (score <= FAST_SELF_MAX) {
|
|
364
|
+
return out('self+batch', `fast, s${score}≤${FAST_SELF_MAX}: exec tự kiểm (test chọn lọc theo file đổi, --verify self); verifier gộp chạy cả suite một lần trước cổng 3`, 'selective');
|
|
365
|
+
}
|
|
366
|
+
if (score <= FAST_SOFT_MAX) {
|
|
367
|
+
return opts.dependents === true
|
|
368
|
+
? out('independent-blocking', `fast, s${score}: bước sau phụ thuộc (depends_on) vào bước này → verifier độc lập vẫn chặn`, 'full')
|
|
369
|
+
: out('independent-nonblocking', `fast, s${score}: verifier độc lập nhưng không chặn bước sau (không bước nào depends_on)`, 'full');
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
if (mode === 'strict') {
|
|
373
|
+
const u = makeRng(opts.seed === undefined || opts.seed === null || opts.seed === true ? undefined : `spot:${opts.seed}`)();
|
|
374
|
+
if (u < STRICT_SPOT_RATE) return out('spot', 'strict: verifier độc lập chặn + reviewer Opus kiểm ngẫu nhiên (1/3), luôn chạy cả suite', 'full');
|
|
375
|
+
return out('independent-blocking', 'strict: verifier độc lập chặn, luôn chạy cả suite', 'full');
|
|
376
|
+
}
|
|
377
|
+
return out('independent-blocking', `${mode}: verifier độc lập chặn bước`, 'full');
|
|
378
|
+
}
|
|
379
|
+
|
|
337
380
|
// ------------------------------------------------------------------ choose
|
|
338
381
|
|
|
339
382
|
/**
|
|
340
|
-
* opts: { type, score, urgency, reversible, verifiable, role, seed, events, profile, taskId, override, pricing
|
|
383
|
+
* opts: { type, score, urgency, reversible, verifiable, role, seed, events, profile, taskId, override, pricing,
|
|
384
|
+
* mode, publish, dependents } (mode/publish/dependents only drive verify_plan, never the tier)
|
|
341
385
|
* Returns { action, model, model_id, effort, reason, explored, tier, budget_s, est_cost_usd,
|
|
342
386
|
* est_time_min, est_source, profile_source, suggested_tier?, overridden? , fails?, tried? }.
|
|
343
387
|
* Throws when an override goes below a safety floor or off the ladder.
|
|
@@ -368,6 +412,13 @@ function choose(opts) {
|
|
|
368
412
|
}
|
|
369
413
|
const reversible = opts.reversible !== false;
|
|
370
414
|
const verifiable = opts.verifiable !== false;
|
|
415
|
+
if (opts.mode !== undefined && opts.mode !== null && !core.MODES.includes(opts.mode)) {
|
|
416
|
+
throw new Error(`--mode phải là ${core.MODES.join('|')} (nhận "${opts.mode}")`);
|
|
417
|
+
}
|
|
418
|
+
const mode = opts.mode || core.DEFAULT_MODE;
|
|
419
|
+
const vp = role === 'exec'
|
|
420
|
+
? verifyPlanFor({ mode, score, verifiable, reversible, publish: opts.publish === true, dependents: opts.dependents === true, seed: opts.seed })
|
|
421
|
+
: null;
|
|
371
422
|
const profile = opts.profile || loadProfile(null);
|
|
372
423
|
const lambda = profile.lambda[urgency];
|
|
373
424
|
const rng = makeRng(opts.seed);
|
|
@@ -398,6 +449,11 @@ function choose(opts) {
|
|
|
398
449
|
if (!verifiable) { safetyFloor = Math.max(safetyFloor, OPUS_MEDIUM); floorWhy.push('không kiểm chứng bằng máy → tối thiểu opus/medium'); }
|
|
399
450
|
if (!reversible && role === 'exec') { safetyFloor = Math.max(safetyFloor, OPUS_MEDIUM); floorWhy.push('khó hoàn tác → tối thiểu opus/medium'); }
|
|
400
451
|
|
|
452
|
+
// Preference floor from the profile (tier_floor): never lowers a safety floor (max), never above the ceiling.
|
|
453
|
+
const prefFloor = known && profile.tier_floor && Number.isInteger(profile.tier_floor[type])
|
|
454
|
+
? Math.min(TOP, profile.tier_floor[type]) : 0;
|
|
455
|
+
const prefWhy = prefFloor > 0 ? `tier_floor của profile: ${type} tối thiểu ${tierName(prefFloor)}` : null;
|
|
456
|
+
|
|
401
457
|
const noExploreWhy = [];
|
|
402
458
|
if (score >= 9) noExploreWhy.push(`score ${score}≥9`);
|
|
403
459
|
if (!verifiable) noExploreWhy.push('không kiểm chứng bằng máy');
|
|
@@ -436,6 +492,12 @@ function choose(opts) {
|
|
|
436
492
|
tier: tierName(idx),
|
|
437
493
|
role,
|
|
438
494
|
task_type: type,
|
|
495
|
+
score,
|
|
496
|
+
mode,
|
|
497
|
+
verify_plan: vp ? vp.verify_plan : null,
|
|
498
|
+
verify_reason: vp ? vp.verify_reason : null,
|
|
499
|
+
verify_floor: vp ? vp.verify_floor : null,
|
|
500
|
+
test_scope: vp ? vp.test_scope : null,
|
|
439
501
|
budget_s: budgetFor(cell),
|
|
440
502
|
}, estimateFor(idx, cell, opts.pricing), { profile_source: src },
|
|
441
503
|
rt.mapped_from ? { mapped_from: rt.mapped_from } : {}, rt.warning ? { type_warning: rt.warning } : {}, extra || {});
|
|
@@ -443,7 +505,7 @@ function choose(opts) {
|
|
|
443
505
|
|
|
444
506
|
// Unknown task type (not fixed, not in the profile, no alias): its stats would never pool — ask first.
|
|
445
507
|
if (!known) {
|
|
446
|
-
return finish(Math.max(prior, safetyFloor), rt.warning, false, { action: 'ask_user', unknown_type: true });
|
|
508
|
+
return finish(Math.max(prior, safetyFloor), rt.warning, false, { action: 'ask_user', unknown_type: true, reason_code: 'unknown-type' });
|
|
447
509
|
}
|
|
448
510
|
|
|
449
511
|
if (overrideIdx !== null && overrideIdx < safetyFloor) {
|
|
@@ -451,34 +513,37 @@ function choose(opts) {
|
|
|
451
513
|
}
|
|
452
514
|
// Escalation beats override/approval: the user must decide first.
|
|
453
515
|
if (esc.ask) {
|
|
454
|
-
const userIdx = overrideIdx !== null ? overrideIdx : (approvedIdx !== null ? approvedIdx :
|
|
516
|
+
const userIdx = overrideIdx !== null ? overrideIdx : (approvedIdx !== null ? approvedIdx : prefFloor);
|
|
455
517
|
const idx = Math.max(userIdx, esc.floor, safetyFloor);
|
|
456
|
-
return finish(idx, esc.note, false, { action: 'ask_user', fails: esc.fails, tried: esc.tried });
|
|
518
|
+
return finish(idx, esc.note, false, { action: 'ask_user', fails: esc.fails, tried: esc.tried, reason_code: 'ask' });
|
|
457
519
|
}
|
|
458
520
|
|
|
459
521
|
// Tier approved by the user in the plan table: no Thompson/exploration; only floors can raise it.
|
|
460
522
|
if (approvedIdx !== null) {
|
|
461
523
|
const forced = Math.max(safetyFloor, esc.floor);
|
|
462
524
|
if (approvedIdx >= forced) {
|
|
463
|
-
|
|
464
|
-
|
|
525
|
+
let n = esc.note ? `; ${esc.note}` : '';
|
|
526
|
+
if (approvedIdx < prefFloor) n += `; dưới ${prefWhy} (người dùng duyệt nên giữ)`;
|
|
527
|
+
return finish(approvedIdx, `người dùng đã duyệt ${tierName(approvedIdx)} trong bảng phân công${n}`, false, { deviation: false, reason_code: 'plan' });
|
|
465
528
|
}
|
|
466
529
|
const why = [];
|
|
467
530
|
if (safetyFloor > approvedIdx) why.push(`sàn an toàn ${tierName(safetyFloor)} (${floorWhy.join(', ')})`);
|
|
468
531
|
if (esc.floor > approvedIdx) why.push(esc.note || `leo thang → ${tierName(esc.floor)}`);
|
|
469
|
-
return finish(forced, `lệch khỏi tier đã duyệt ${tierName(approvedIdx)} → ${tierName(forced)}: ${why.join('; ')}`, false, { deviation: true, approved_tier: tierName(approvedIdx) });
|
|
532
|
+
return finish(forced, `lệch khỏi tier đã duyệt ${tierName(approvedIdx)} → ${tierName(forced)}: ${why.join('; ')}`, false, { deviation: true, approved_tier: tierName(approvedIdx), reason_code: 'plan' });
|
|
470
533
|
}
|
|
471
534
|
if (esc.note && !esc.ask) notes.push(esc.note);
|
|
472
535
|
|
|
473
|
-
const floor = Math.max(safetyFloor, esc.ask ? 0 : esc.floor);
|
|
536
|
+
const floor = Math.max(safetyFloor, esc.ask ? 0 : esc.floor, prefFloor);
|
|
474
537
|
const base = Math.min(TOP, Math.max(prior, floor));
|
|
475
538
|
if (base > prior) {
|
|
476
539
|
const why = floorWhy.slice();
|
|
477
540
|
if (esc.floor > prior && esc.floor >= safetyFloor && !esc.ask) why.push(`leo thang → ${tierName(esc.floor)}`);
|
|
541
|
+
if (prefFloor > prior) why.push(prefWhy);
|
|
478
542
|
notes.unshift(`sàn ${tierName(base)} cao hơn ${priorWhy} ${tierName(prior)}: ${why.join(', ') || 'luật an toàn'}`);
|
|
479
543
|
} else if (floorWhy.length) {
|
|
480
544
|
notes.push(`sàn an toàn: ${floorWhy.join(', ')}`);
|
|
481
545
|
}
|
|
546
|
+
if (prefFloor > 0 && base <= prior) notes.push(prefWhy);
|
|
482
547
|
|
|
483
548
|
const minIdx = noExplore ? base : floor;
|
|
484
549
|
const candidates = [];
|
|
@@ -494,10 +559,15 @@ function choose(opts) {
|
|
|
494
559
|
|
|
495
560
|
let chosen = base;
|
|
496
561
|
let why;
|
|
562
|
+
// core 0.7.1 (C3): why this tier, as a code — thompson | floor | escalation | prior | user-prior | explore | plan | override | ask | unknown-type
|
|
563
|
+
let code;
|
|
564
|
+
let tsScore = null;
|
|
497
565
|
const baseCell = cellAt(base);
|
|
498
566
|
const scored = pool.filter((x) => x.cell && x.cell.enough);
|
|
499
567
|
if (!baseCell || !baseCell.enough || !scored.length) {
|
|
500
568
|
why = `${priorWhy} ${tierName(base)} (${baseCell ? `n=${baseCell.n}` : 'n=0'}, chưa đủ dữ liệu)`;
|
|
569
|
+
if (base > prior) code = esc.floor === base && esc.floor > Math.max(safetyFloor, prefFloor) ? 'escalation' : 'floor';
|
|
570
|
+
else code = priorWhy === 'prior' ? 'prior' : 'user-prior';
|
|
501
571
|
} else {
|
|
502
572
|
let best = null;
|
|
503
573
|
for (const x of scored) {
|
|
@@ -510,6 +580,8 @@ function choose(opts) {
|
|
|
510
580
|
if (!best || s < best.s) best = { i: x.i, s, p };
|
|
511
581
|
}
|
|
512
582
|
chosen = best.i;
|
|
583
|
+
code = 'thompson';
|
|
584
|
+
tsScore = Math.round(best.s * 1000) / 1000;
|
|
513
585
|
why = `Thompson: ${tierName(chosen)} điểm ${best.s.toFixed(3)} (p≈${best.p.toFixed(2)}, ${scored.length} ô đủ dữ liệu)`;
|
|
514
586
|
}
|
|
515
587
|
|
|
@@ -518,22 +590,24 @@ function choose(opts) {
|
|
|
518
590
|
if (!noExplore && chosen - 1 >= floor && u < profile.explore_rate) {
|
|
519
591
|
chosen -= 1;
|
|
520
592
|
explored = true;
|
|
593
|
+
code = 'explore';
|
|
521
594
|
why = `thăm dò xuống ${tierName(chosen)} (xác suất ${profile.explore_rate}); ${why}`;
|
|
522
595
|
}
|
|
523
596
|
|
|
524
597
|
if (overrideIdx !== null) {
|
|
525
598
|
const sugg = tierName(chosen);
|
|
526
|
-
|
|
527
|
-
|
|
599
|
+
const below = overrideIdx < prefFloor ? `; dưới ${prefWhy} (người dùng chọn nên giữ)` : '';
|
|
600
|
+
return finish(overrideIdx, `người dùng chọn ${tierName(overrideIdx)} (đề xuất là ${sugg}: ${why})${below}`, false, {
|
|
601
|
+
overridden: overrideIdx !== chosen, suggested_tier: sugg, reason_code: 'override', suggested_reason_code: code, thompson_score: tsScore,
|
|
528
602
|
});
|
|
529
603
|
}
|
|
530
604
|
if (noExplore) notes.push(`không thăm dò: ${noExploreWhy.join(', ')}`);
|
|
531
|
-
return finish(chosen, [why].concat(notes).join('; '), explored);
|
|
605
|
+
return finish(chosen, [why].concat(notes).join('; '), explored, { reason_code: code, thompson_score: tsScore });
|
|
532
606
|
}
|
|
533
607
|
|
|
534
608
|
module.exports = {
|
|
535
609
|
DEFAULT_ORCH_CTX_SOFT_CAP,
|
|
536
610
|
LADDER, DEFAULT_PRIORS, DEFAULT_LAMBDA, DEFAULT_TIME_CAP_MIN, ROUGH, tierIndex, tierName, parsePrior,
|
|
537
611
|
parseTier, loadProfile, makeRng, betaSample, escalation, overridePrior, roughEstimate, choose, shortModel,
|
|
538
|
-
TYPE_ALIASES, resolveTaskType,
|
|
612
|
+
TYPE_ALIASES, resolveTaskType, verifyPlanFor, VERIFY_PLANS,
|
|
539
613
|
};
|