@event4u/agent-config 14.2.0 → 14.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +286 -218
- package/agents/templates/.ai-council.yml.example +35 -0
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +83 -12
- package/dist/agent-src/rules/roadmap-progress-sync.md +48 -4
- package/dist/agent-src/scripts/resume_probe.ts +154 -9
- package/dist/agent-src/scripts/roadmap_gates.ts +210 -6
- package/dist/agent-src/skills/roadmap-management/SKILL.md +9 -4
- package/dist/agent-src/templates/agent-settings.md +70 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/scripts/telemetry/remote.ts +571 -0
- package/dist/agent-src/templates/scripts/telemetry/settings.ts +222 -44
- package/dist/cli/python/workspace_drive.js +8 -3
- package/dist/cli/python/workspace_drive.js.map +1 -1
- package/dist/cli/python/workspace_hosts.js +16 -12
- package/dist/cli/python/workspace_hosts.js.map +1 -1
- package/dist/cli/registry.js +2 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/{chunk-F3YUF7SA.js → chunk-3UCJWMMP.js} +1 -1
- package/dist/cli-delegate/chunk-F7DQWYR5.js +69 -0
- package/dist/cli-delegate/{chunk-7PZ4FSZB.js → chunk-FBKL6JWT.js} +61 -2
- package/dist/cli-delegate/{chunk-MNWRDMR5.js → chunk-PKOBQADP.js} +1 -1
- package/dist/cli-delegate/cmd_conformance.js +26 -11
- package/dist/cli-delegate/cmd_doctor.js +2 -2
- package/dist/cli-delegate/cmd_handoff.js +3 -3
- package/dist/cli-delegate/cmd_route_audit.js +2 -2
- package/dist/cli-delegate/cmd_settings_set.js +17 -3
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +4 -4
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +3684 -1719
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +1070 -408
- package/dist/scripts/_lib/host_launch.js +66 -0
- package/dist/scripts/_lib/host_launch.js.map +1 -0
- package/dist/server/io/yamlIO.js +175 -3
- package/dist/server/io/yamlIO.js.map +1 -1
- package/dist/server/routes/wizard.js +20 -1
- package/dist/server/routes/wizard.js.map +1 -1
- package/dist/shared/settingsConsent.js +25 -0
- package/dist/shared/settingsConsent.js.map +1 -1
- package/docs/contracts/settings-classes.md +11 -0
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +2 -1
- package/package.json +1 -1
- package/src/agent-src/scripts/resume_probe.ts +154 -9
- package/src/agent-src/scripts/roadmap_gates.ts +210 -6
- package/src/agent-src/templates/scripts/telemetry/remote.ts +571 -0
- package/src/agent-src/templates/scripts/telemetry/settings.ts +222 -44
- package/src/config/estate-count-budget.json +120 -0
- package/src/config/evaluator-budgets.json +4 -4
- package/src/config/gate-coverage.yml +32 -4
- package/src/config/gate-violation-baselines.json +63 -63
- package/src/config/gitignore-block.txt +5 -0
- package/src/config/hook-latency-budget.json +40 -1
- package/src/config/hook-token-budget.json +16 -0
- package/src/config/rule-stub-ceilings.json +11 -3
- package/src/scripts/_dispatch.bash +42 -1
- package/src/scripts/_lib/council_fallback_posture.ts +81 -0
- package/src/scripts/_lib/council_fallback_wiring.ts +126 -0
- package/src/scripts/_lib/council_settings_block.ts +152 -0
- package/src/scripts/_lib/headless_invocation.ts +285 -0
- package/src/scripts/_lib/host_launch.ts +72 -0
- package/src/scripts/_lib/map_to_object.ts +16 -0
- package/src/scripts/_lib/roadmap_checkboxes.ts +88 -0
- package/src/scripts/_lib/run_checkpoint.ts +409 -0
- package/src/scripts/_lib/session_register.ts +32 -1
- package/src/scripts/_lib/stdin.ts +35 -1
- package/src/scripts/_lib/turn_end_refusals.ts +12 -4
- package/src/scripts/_lib/unattended_guard.ts +371 -0
- package/src/scripts/ai_council/config.ts +27 -1
- package/src/scripts/ai_council/events_log.ts +12 -1
- package/src/scripts/ai_council/fallback_config.ts +114 -0
- package/src/scripts/ai_council/mid_flight_fallback.ts +295 -0
- package/src/scripts/ai_council/orchestrator.ts +251 -292
- package/src/scripts/ai_council/py_parity.ts +139 -0
- package/src/scripts/ai_council/response_render.ts +99 -0
- package/src/scripts/ai_council/spend_gate.ts +123 -0
- package/src/scripts/ai_council/transport_resolver.ts +81 -13
- package/src/scripts/bench_hook_injection.ts +100 -6
- package/src/scripts/bench_hook_latency.ts +442 -16
- package/src/scripts/check_estate_count.ts +924 -0
- package/src/scripts/compile_hook_manifest.ts +67 -0
- package/src/scripts/council_cli.ts +100 -125
- package/src/scripts/decision_memo.ts +325 -0
- package/src/scripts/hook_manifest.json +1 -0
- package/src/scripts/hook_manifest.yaml +172 -25
- package/src/scripts/hooks/concern_registry.ts +6 -0
- package/src/scripts/hooks/dispatch_hook.ts +183 -10
- package/src/scripts/hooks/dispatch_issues.ts +26 -0
- package/src/scripts/hooks/end_review_nudge_hook.ts +97 -24
- package/src/scripts/hooks/hook_stdin.ts +111 -5
- package/src/scripts/hooks/injection_budget.ts +595 -0
- package/src/scripts/hooks/payload_stub.ts +446 -0
- package/src/scripts/hooks/run_continuation_hook.ts +627 -0
- package/src/scripts/hooks/session_eol_hook.ts +52 -0
- package/src/scripts/hooks/telemetry_usage_hook.ts +339 -0
- package/src/scripts/hooks/tool_result_bytes_hook.ts +27 -0
- package/src/scripts/hooks/turn_end_gate_hook.ts +45 -7
- package/src/scripts/hooks_doctor.ts +16 -2
- package/src/scripts/interruption_report.ts +330 -13
- package/src/scripts/lint_hook_manifest.ts +167 -0
- package/src/scripts/roadmap_progress_hook.ts +254 -46
- package/src/scripts/run_supervise.ts +623 -0
- package/src/scripts/session_register_hook.ts +108 -8
- package/src/scripts/sessions_cli.ts +18 -6
- package/src/scripts/telemetry_disclosure_hook.ts +244 -0
- package/src/server/io/yamlIO.ts +167 -3
- package/src/shared/settingsConsent.ts +49 -2
- package/dist/cli-delegate/chunk-MMJL4TIX.js +0 -18
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* py_parity — the small Python-stdlib parity helpers the council modules share.
|
|
3
|
+
*
|
|
4
|
+
* Relocated out of `orchestrator.ts` UNCHANGED. That file sits ~870 lines over
|
|
5
|
+
* the source ceiling and this repository answers growth with extraction rather
|
|
6
|
+
* than a raised baseline (`gate-violation-baselines.json`
|
|
7
|
+
* § check_source_size_budget: "extract, then measure"). A reviewer diffs a
|
|
8
|
+
* move: no body here is edited, only exported.
|
|
9
|
+
*
|
|
10
|
+
* They exist at all because this module family is a faithful port of a retired
|
|
11
|
+
* Python implementation whose formatting and slicing semantics are asserted
|
|
12
|
+
* byte-for-byte by golden tests — JS `Math.trunc`, `String.slice` and
|
|
13
|
+
* `str.length` disagree with their Python counterparts on truncation sign,
|
|
14
|
+
* code points and surrogate pairs respectively.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
// ── small stdlib parity helpers ───────────────────────────────────────
|
|
18
|
+
|
|
19
|
+
/** Mirror Python `int(x)` truncation-toward-zero on a float total. */
|
|
20
|
+
export function _pyInt(x: number): number {
|
|
21
|
+
return Math.trunc(x);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Mirror Python `repr(int)` for the error message in estimate_debate_cost. */
|
|
25
|
+
export function _pyReprInt(x: number): string {
|
|
26
|
+
return String(x);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Mirror `type(exc).__name__ + ": " + str(exc)`. */
|
|
30
|
+
export function _excTag(exc: unknown): string {
|
|
31
|
+
if (exc instanceof Error) {
|
|
32
|
+
return `${exc.name}: ${exc.message}`;
|
|
33
|
+
}
|
|
34
|
+
return `Error: ${String(exc)}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** dict.setdefault(key, value) — only set when key absent. */
|
|
38
|
+
export function _setdefault(
|
|
39
|
+
obj: Record<string, unknown>,
|
|
40
|
+
key: string,
|
|
41
|
+
value: unknown,
|
|
42
|
+
): void {
|
|
43
|
+
if (!(key in obj)) {
|
|
44
|
+
obj[key] = value;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** dict.get(key, default). */
|
|
49
|
+
export function _metaGet(
|
|
50
|
+
obj: Record<string, unknown>,
|
|
51
|
+
key: string,
|
|
52
|
+
fallback: unknown,
|
|
53
|
+
): unknown {
|
|
54
|
+
return key in obj ? obj[key] : fallback;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Python `str.lstrip()` (no-arg) — strip leading whitespace. */
|
|
58
|
+
export function _pyLStrip(s: string): string {
|
|
59
|
+
return s.replace(/^\s+/, '');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Python `str.rstrip()` (no-arg) — strip trailing whitespace. */
|
|
63
|
+
export function _pyRStrip(s: string): string {
|
|
64
|
+
return s.replace(/\s+$/, '');
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Python `len(str)` — code-point count, not UTF-16 unit count. */
|
|
68
|
+
export function _pyLen(s: string): number {
|
|
69
|
+
let n = 0;
|
|
70
|
+
for (const _ of s) {
|
|
71
|
+
n += 1;
|
|
72
|
+
}
|
|
73
|
+
return n;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Python `s[start:end]` — code-point slicing. */
|
|
77
|
+
export function _pySlice(s: string, start: number, end: number): string {
|
|
78
|
+
return [...s].slice(start, end).join('');
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// ── Python-format / stdlib parity helpers ────────────────────────────────
|
|
82
|
+
//
|
|
83
|
+
// The orchestrator formats USD / scores / strengths via Python f-string
|
|
84
|
+
// specs (`:.4f`, `:.2f`, `:.1f`) which round half-to-even on the decimal
|
|
85
|
+
// representation. JS `toFixed` rounds half away from zero, so the spec
|
|
86
|
+
// formatting is reimplemented to stay byte-exact with the retired Python implementation.
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Format `x` to `ndigits` decimals using round-half-to-even, matching
|
|
90
|
+
* CPython's `format(x, ".<ndigits>f")`.
|
|
91
|
+
*/
|
|
92
|
+
export function _pyFixed(x: number, ndigits: number): string {
|
|
93
|
+
if (!Number.isFinite(x)) {
|
|
94
|
+
return String(x);
|
|
95
|
+
}
|
|
96
|
+
const neg = x < 0 || Object.is(x, -0);
|
|
97
|
+
const abs = Math.abs(x);
|
|
98
|
+
const factor = Math.pow(10, ndigits);
|
|
99
|
+
const scaled = abs * factor;
|
|
100
|
+
const floor = Math.floor(scaled);
|
|
101
|
+
const frac = scaled - floor;
|
|
102
|
+
const tol = Math.max(Math.abs(scaled), 1) * 2 ** -40;
|
|
103
|
+
let rounded: number;
|
|
104
|
+
if (Math.abs(frac - 0.5) <= tol) {
|
|
105
|
+
rounded = floor % 2 === 0 ? floor : floor + 1;
|
|
106
|
+
} else {
|
|
107
|
+
rounded = Math.round(scaled);
|
|
108
|
+
}
|
|
109
|
+
let intStr = String(rounded);
|
|
110
|
+
let result: string;
|
|
111
|
+
if (ndigits === 0) {
|
|
112
|
+
result = intStr;
|
|
113
|
+
} else {
|
|
114
|
+
if (intStr.length <= ndigits) {
|
|
115
|
+
intStr = '0'.repeat(ndigits - intStr.length + 1) + intStr;
|
|
116
|
+
}
|
|
117
|
+
const whole = intStr.slice(0, intStr.length - ndigits);
|
|
118
|
+
const dec = intStr.slice(intStr.length - ndigits);
|
|
119
|
+
result = `${whole}.${dec}`;
|
|
120
|
+
}
|
|
121
|
+
return neg ? `-${result}` : result;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Mirror Python `str.strip()` (no-arg). Sibling-twin convention uses trim(). */
|
|
125
|
+
export function _pyStrip(s: string): string {
|
|
126
|
+
return s.trim();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Mirror Python `str.split()` (no separator) — split on runs of whitespace,
|
|
131
|
+
* dropping leading / trailing whitespace (no empty tokens).
|
|
132
|
+
*/
|
|
133
|
+
export function _pySplitWhitespace(s: string): string[] {
|
|
134
|
+
const trimmed = s.trim();
|
|
135
|
+
if (trimmed === '') {
|
|
136
|
+
return [];
|
|
137
|
+
}
|
|
138
|
+
return trimmed.split(/\s+/);
|
|
139
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* response_render — the per-member meta line, and the one-call helper.
|
|
3
|
+
*
|
|
4
|
+
* Both are plumbing the orchestrator uses rather than orchestration, and both
|
|
5
|
+
* were extracted when the mid-flight fallback grew them: `orchestrator.ts` is
|
|
6
|
+
* ~1,000 lines over the source ceiling, and the documented response to growing
|
|
7
|
+
* an over-ceiling file is extraction, never a raised baseline
|
|
8
|
+
* (`gate-violation-baselines.json` § check_source_size_budget).
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { CouncilResponse, type ExternalAIClient } from './clients.js';
|
|
12
|
+
import { _metaGet } from './py_parity.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The shape `callMember` reads off a question. Structural rather than an
|
|
16
|
+
* import of `CouncilQuestion`: that class lives in `orchestrator.ts`, and
|
|
17
|
+
* importing it back would make this extraction a cycle.
|
|
18
|
+
*/
|
|
19
|
+
interface QuestionShape {
|
|
20
|
+
readonly user_prompt: string;
|
|
21
|
+
readonly max_tokens: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Round half-to-even to `n` decimals, matching CPython's `format(x, ".nf")`. */
|
|
25
|
+
type FixedFn = (x: number, n: number) => string;
|
|
26
|
+
|
|
27
|
+
export interface MetaRenderDeps {
|
|
28
|
+
readonly pyFixed: FixedFn;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function renderResponseMeta(r: CouncilResponse, deps: MetaRenderDeps): string {
|
|
32
|
+
const meta_dict: Record<string, unknown> = r.metadata ?? {};
|
|
33
|
+
const billable = Boolean(_metaGet(meta_dict, 'billable', true));
|
|
34
|
+
const estimated = Boolean(_metaGet(meta_dict, 'tokens_estimated', false));
|
|
35
|
+
const parts: string[] = [];
|
|
36
|
+
if (!billable) {
|
|
37
|
+
const label = (_metaGet(meta_dict, 'subscription_label', null) ||
|
|
38
|
+
'flat-rate') as string;
|
|
39
|
+
parts.push(`cost: subscription (${label})`);
|
|
40
|
+
} else {
|
|
41
|
+
const cost_usd = _metaGet(meta_dict, 'cost_usd', undefined);
|
|
42
|
+
if (typeof cost_usd === 'number' && !(typeof cost_usd === 'boolean')) {
|
|
43
|
+
parts.push(`cost: $${deps.pyFixed(cost_usd, 4)}`);
|
|
44
|
+
}
|
|
45
|
+
const prefix = estimated ? '~' : '';
|
|
46
|
+
parts.push(
|
|
47
|
+
`tokens: ${prefix}${r.input_tokens} in / ${prefix}${r.output_tokens} out`,
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
parts.push(`${r.latency_ms} ms`);
|
|
51
|
+
// A seat that answered over api because its cli transport died says so
|
|
52
|
+
// in the ARTEFACT, not only in metadata: otherwise the only visible
|
|
53
|
+
// difference from "was api all along" is a cost line nobody planned.
|
|
54
|
+
const fallback_from = _metaGet(meta_dict, 'fallback_from', null);
|
|
55
|
+
if (typeof fallback_from === 'string' && fallback_from !== '') {
|
|
56
|
+
const reason = _metaGet(meta_dict, 'fallback_reason', null);
|
|
57
|
+
const sticky = _metaGet(meta_dict, 'fallback_sticky', false) === true;
|
|
58
|
+
const why = typeof reason === 'string' && reason !== '' ? `: ${reason}` : '';
|
|
59
|
+
parts.push(
|
|
60
|
+
sticky
|
|
61
|
+
? `transport: api (${fallback_from} lost earlier this pass${why})`
|
|
62
|
+
: `transport: api (fell back from ${fallback_from}${why})`,
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
return `*${parts.join(' · ')}*`;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
/** Error-tagged member call — the split/ask branch is one shape, not three. */
|
|
70
|
+
/**
|
|
71
|
+
* One member call, error-tagged rather than throwing — the split/ask branch
|
|
72
|
+
* appeared three times before this and is one shape, not three.
|
|
73
|
+
*/
|
|
74
|
+
export function callMember(
|
|
75
|
+
client: ExternalAIClient,
|
|
76
|
+
question: QuestionShape,
|
|
77
|
+
opts: { split?: { stable: string; suffix: string } | null },
|
|
78
|
+
systemFor: (m: ExternalAIClient) => string,
|
|
79
|
+
excTag: (e: unknown) => string,
|
|
80
|
+
): CouncilResponse {
|
|
81
|
+
try {
|
|
82
|
+
return opts.split
|
|
83
|
+
? client.ask_split(
|
|
84
|
+
systemFor(client),
|
|
85
|
+
opts.split.stable,
|
|
86
|
+
opts.split.suffix,
|
|
87
|
+
question.max_tokens,
|
|
88
|
+
)
|
|
89
|
+
: client.ask(systemFor(client), question.user_prompt, question.max_tokens);
|
|
90
|
+
} catch (exc) {
|
|
91
|
+
return new CouncilResponse({
|
|
92
|
+
provider: client.name,
|
|
93
|
+
model: client.model,
|
|
94
|
+
text: '',
|
|
95
|
+
error: excTag(exc),
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* spend_gate — the ceilings, the breach decision, and the event a breach raises.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `orchestrator.ts` unchanged, and the ordering is the point
|
|
5
|
+
* (`gate-violation-baselines.json` § check_source_size_budget: "extract, then
|
|
6
|
+
* measure"): the orchestrator sits far over the 1,500-line source ceiling, so
|
|
7
|
+
* a change that grows it has to hand a cohesive unit out rather than pin a
|
|
8
|
+
* higher number — the gate's own text calls raising the baseline a defect.
|
|
9
|
+
*
|
|
10
|
+
* The unit is cohesive on its own terms, not merely convenient: `CostBudget`
|
|
11
|
+
* states the ceilings, `_breach` decides whether one is crossed, `OverrunEvent`
|
|
12
|
+
* is what a crossing hands to the operator, and `_total_usd` is the arithmetic
|
|
13
|
+
* all three share. Nothing here calls a provider or knows what a round is.
|
|
14
|
+
*
|
|
15
|
+
* Re-exported from `orchestrator.ts`, so no caller changes.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { CostEstimate } from './pricing.js';
|
|
19
|
+
import type { ExternalAIClient } from './clients.js';
|
|
20
|
+
import { would_exceed as _would_exceed_daily } from './budget_guard.js';
|
|
21
|
+
|
|
22
|
+
export class CostBudget {
|
|
23
|
+
max_input_tokens: number;
|
|
24
|
+
max_output_tokens: number;
|
|
25
|
+
max_calls: number;
|
|
26
|
+
max_total_usd: number; // 0 = USD ceiling disabled (token caps still apply)
|
|
27
|
+
daily_limit_usd: number; // 0 = rolling 24h cap disabled (D3)
|
|
28
|
+
|
|
29
|
+
constructor(
|
|
30
|
+
args: {
|
|
31
|
+
max_input_tokens?: number;
|
|
32
|
+
max_output_tokens?: number;
|
|
33
|
+
max_calls?: number;
|
|
34
|
+
max_total_usd?: number;
|
|
35
|
+
daily_limit_usd?: number;
|
|
36
|
+
} = {},
|
|
37
|
+
) {
|
|
38
|
+
this.max_input_tokens = args.max_input_tokens ?? 50_000;
|
|
39
|
+
this.max_output_tokens = args.max_output_tokens ?? 20_000;
|
|
40
|
+
this.max_calls = args.max_calls ?? 10;
|
|
41
|
+
this.max_total_usd = args.max_total_usd ?? 0.0;
|
|
42
|
+
this.daily_limit_usd = args.daily_limit_usd ?? 0.0;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
/** Passed to `on_overrun` when projected spend exceeds the budget. */
|
|
48
|
+
export class OverrunEvent {
|
|
49
|
+
member_index: number;
|
|
50
|
+
member: ExternalAIClient;
|
|
51
|
+
next_estimate: CostEstimate; // this member's projected cost
|
|
52
|
+
spent_input_tokens: number; // already-billed totals BEFORE this member
|
|
53
|
+
spent_output_tokens: number;
|
|
54
|
+
spent_usd: number;
|
|
55
|
+
projected_total_usd: number; // spent_usd + next_estimate.total_usd
|
|
56
|
+
daily_spent_usd: number; // rolling 24h spend BEFORE this member (D3)
|
|
57
|
+
daily_limit_usd: number; // the configured daily cap (0 = disabled)
|
|
58
|
+
breach_kind: string; // "session" | "daily" | "tokens"
|
|
59
|
+
|
|
60
|
+
constructor(args: {
|
|
61
|
+
member_index: number;
|
|
62
|
+
member: ExternalAIClient;
|
|
63
|
+
next_estimate: CostEstimate;
|
|
64
|
+
spent_input_tokens: number;
|
|
65
|
+
spent_output_tokens: number;
|
|
66
|
+
spent_usd: number;
|
|
67
|
+
projected_total_usd: number;
|
|
68
|
+
daily_spent_usd?: number;
|
|
69
|
+
daily_limit_usd?: number;
|
|
70
|
+
breach_kind?: string;
|
|
71
|
+
}) {
|
|
72
|
+
this.member_index = args.member_index;
|
|
73
|
+
this.member = args.member;
|
|
74
|
+
this.next_estimate = args.next_estimate;
|
|
75
|
+
this.spent_input_tokens = args.spent_input_tokens;
|
|
76
|
+
this.spent_output_tokens = args.spent_output_tokens;
|
|
77
|
+
this.spent_usd = args.spent_usd;
|
|
78
|
+
this.projected_total_usd = args.projected_total_usd;
|
|
79
|
+
this.daily_spent_usd = args.daily_spent_usd ?? 0.0;
|
|
80
|
+
this.daily_limit_usd = args.daily_limit_usd ?? 0.0;
|
|
81
|
+
this.breach_kind = args.breach_kind ?? 'session';
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
/** Mirror the Python `CostEstimate.total_usd` property. */
|
|
87
|
+
export function _total_usd(e: CostEstimate): number {
|
|
88
|
+
return e.input_usd + e.output_usd;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export type BreachKind = 'tokens' | 'daily' | 'session' | null;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The projected-spend verdict. One implementation because the round head and
|
|
95
|
+
* the fallback's metered retry must gate on the SAME rules — two copies of
|
|
96
|
+
* four comparisons is how they drift.
|
|
97
|
+
*/
|
|
98
|
+
export function _breach(
|
|
99
|
+
est: CostEstimate | null,
|
|
100
|
+
spent: Spent,
|
|
101
|
+
budget: CostBudget,
|
|
102
|
+
): BreachKind {
|
|
103
|
+
const usd = est ? _total_usd(est) : 0.0;
|
|
104
|
+
if (
|
|
105
|
+
spent.input + (est ? est.input_tokens : 0) > budget.max_input_tokens ||
|
|
106
|
+
spent.output + (est ? est.output_tokens : 0) > budget.max_output_tokens
|
|
107
|
+
) {
|
|
108
|
+
return 'tokens';
|
|
109
|
+
}
|
|
110
|
+
if (budget.daily_limit_usd > 0 && _would_exceed_daily(budget.daily_limit_usd, usd)) {
|
|
111
|
+
return 'daily';
|
|
112
|
+
}
|
|
113
|
+
if (budget.max_total_usd > 0 && spent.usd + usd > budget.max_total_usd) return 'session';
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
export interface Spent {
|
|
119
|
+
input: number;
|
|
120
|
+
output: number;
|
|
121
|
+
usd: number;
|
|
122
|
+
}
|
|
123
|
+
|
|
@@ -381,18 +381,35 @@ const FALLBACK_ELIGIBLE: ReadonlySet<CliFailureClass> = new Set([
|
|
|
381
381
|
// of its shapes: the pre-spend gate refuses before any subprocess exists,
|
|
382
382
|
// and the call-time shape is a provider 400 — rejected at the request
|
|
383
383
|
// boundary, no generation performed.
|
|
384
|
-
//
|
|
385
|
-
// HONEST SCOPE: this set is not consumed by any shipped path today.
|
|
386
|
-
// `isFallbackEligible` and `MidFlightFallback` have zero call sites outside
|
|
387
|
-
// this module and its tests, so membership here is a stated property of the
|
|
388
|
-
// class, NOT a behaviour a caller performs. Said plainly because the
|
|
389
|
-
// opposite reading is the attractive one — the constraint really is the
|
|
390
|
-
// subscription transport's rather than the model's, so the api rung really
|
|
391
|
-
// would serve the same id, and it would be easy to write that as if it
|
|
392
|
-
// happened. It does not happen yet.
|
|
393
384
|
'model_unservable',
|
|
394
385
|
]);
|
|
395
386
|
|
|
387
|
+
/**
|
|
388
|
+
* Caller-supplied policy that widens the base eligibility set.
|
|
389
|
+
*
|
|
390
|
+
* `apiOnQuota` covers BOTH quota shapes, and both satisfy the same
|
|
391
|
+
* no-double-charge property the base set is built on:
|
|
392
|
+
*
|
|
393
|
+
* - the LOCAL `cli_call_budget` refusal happens before any subprocess exists
|
|
394
|
+
* ("nothing sent, nothing booked" — `clients.ts` returns the response
|
|
395
|
+
* without spawning);
|
|
396
|
+
* - the provider-side plan-quota rejection is refused at the request
|
|
397
|
+
* boundary, no generation performed.
|
|
398
|
+
*
|
|
399
|
+
* They stay behind an opt-in for a different reason than double-spend: the
|
|
400
|
+
* fallback flips the billing class. A vendor CLI under a subscription login
|
|
401
|
+
* is unmetered; its api twin is metered USD. Silently converting exhausted
|
|
402
|
+
* plan quota into API spend is a decision the operator must state, not one
|
|
403
|
+
* `auto` may infer — the USD spend itself remains guarded by the ordinary
|
|
404
|
+
* `cost_budget` gates the orchestrator runs on the retry.
|
|
405
|
+
*/
|
|
406
|
+
export interface FallbackPolicy {
|
|
407
|
+
readonly apiOnQuota: boolean;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** The base policy: exactly the three-plus-one no-double-charge classes. */
|
|
411
|
+
export const DEFAULT_FALLBACK_POLICY: FallbackPolicy = { apiOnQuota: false };
|
|
412
|
+
|
|
396
413
|
/** Normalise a raw client error / skip reason into a `CliFailureClass`. */
|
|
397
414
|
export function classifyCliFailure(raw: string): CliFailureClass {
|
|
398
415
|
const s = raw.trim().toLowerCase();
|
|
@@ -409,7 +426,19 @@ export function classifyCliFailure(raw: string): CliFailureClass {
|
|
|
409
426
|
// Also matched on the provider's own wording, so a model refused at call
|
|
410
427
|
// time (one the deny-list has never seen) classifies the same as one the
|
|
411
428
|
// pre-spend gate caught.
|
|
412
|
-
if (
|
|
429
|
+
if (
|
|
430
|
+
s.startsWith('model_unservable') ||
|
|
431
|
+
// The token the PRODUCERS actually write. R2 round 6, finding 4:
|
|
432
|
+
// `clients.ts` emits `model_unsupported_on_transport` at both sites
|
|
433
|
+
// (the pre-spend gate at :1857 and the turn-failed path at :2002) and
|
|
434
|
+
// puts the vendor sentence in `metadata.detail`, never in `error` — so
|
|
435
|
+
// the class matched nothing a caller could produce and the contract's
|
|
436
|
+
// fourth eligible class was dead. The token is matched here rather
|
|
437
|
+
// than renamed at the producers because `code:` values reach the
|
|
438
|
+
// events log and a rename would break every reader of the history.
|
|
439
|
+
s.startsWith('model_unsupported_on_transport') ||
|
|
440
|
+
s.includes('is not supported when using codex')
|
|
441
|
+
) {
|
|
413
442
|
return 'model_unservable';
|
|
414
443
|
}
|
|
415
444
|
if (s === 'timeout') return 'timeout';
|
|
@@ -420,7 +449,22 @@ export function classifyCliFailure(raw: string): CliFailureClass {
|
|
|
420
449
|
|
|
421
450
|
/** True when `failure` is eligible to fall through to the `api` rung. */
|
|
422
451
|
export function isFallbackEligible(failure: CliFailureClass): boolean {
|
|
423
|
-
return
|
|
452
|
+
return isFallbackEligibleUnder(failure, DEFAULT_FALLBACK_POLICY);
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* Policy-aware variant: the base set, plus `quota_exhausted` when the
|
|
457
|
+
* caller's policy opted in (`ai_council.fallback.api_on_quota`).
|
|
458
|
+
* `timeout` / `server_error` are ineligible under EVERY policy — a
|
|
459
|
+
* half-completed call must never be paid for twice, and no opt-in key
|
|
460
|
+
* exists (or may be added) to override that.
|
|
461
|
+
*/
|
|
462
|
+
export function isFallbackEligibleUnder(
|
|
463
|
+
failure: CliFailureClass,
|
|
464
|
+
policy: FallbackPolicy,
|
|
465
|
+
): boolean {
|
|
466
|
+
if (FALLBACK_ELIGIBLE.has(failure)) return true;
|
|
467
|
+
return policy.apiOnQuota && failure === 'quota_exhausted';
|
|
424
468
|
}
|
|
425
469
|
|
|
426
470
|
/**
|
|
@@ -467,13 +511,37 @@ export class MidFlightFallback {
|
|
|
467
511
|
private readonly used = new Set<string>();
|
|
468
512
|
|
|
469
513
|
/** `'api'` to retry on the api rung, `'stop'` to surface the failure. */
|
|
470
|
-
attempt(
|
|
471
|
-
|
|
514
|
+
attempt(
|
|
515
|
+
provider: string,
|
|
516
|
+
failure: CliFailureClass,
|
|
517
|
+
policy: FallbackPolicy = DEFAULT_FALLBACK_POLICY,
|
|
518
|
+
): 'api' | 'stop' {
|
|
519
|
+
if (!isFallbackEligibleUnder(failure, policy)) return 'stop';
|
|
472
520
|
if (this.used.has(provider)) return 'stop';
|
|
473
521
|
this.used.add(provider);
|
|
474
522
|
return 'api';
|
|
475
523
|
}
|
|
476
524
|
|
|
525
|
+
/**
|
|
526
|
+
* Give a provider its one fallback back, because the twin never ran.
|
|
527
|
+
*
|
|
528
|
+
* R2 round 2, finding 8. `attempt` claims the slot at DECISION time, which
|
|
529
|
+
* is correct — the claim is what stops a dead binary being retried once per
|
|
530
|
+
* round. But a claim that is never CONSUMED must not stay spent: when the
|
|
531
|
+
* projected-spend gate refuses the escalation, no api call happens, and
|
|
532
|
+
* leaving the provider marked means the seat can never re-decide. Every
|
|
533
|
+
* later round then re-spawns the dead cli binary and gets `'stop'`, which
|
|
534
|
+
* is the exact per-round-ledger behaviour the invocation-scoped ledger was
|
|
535
|
+
* chosen to avoid.
|
|
536
|
+
*
|
|
537
|
+
* Called ONLY on a path where the twin was built and then not called.
|
|
538
|
+
* A provider that actually retried keeps its claim spent — that is the cap
|
|
539
|
+
* working.
|
|
540
|
+
*/
|
|
541
|
+
release(provider: string): void {
|
|
542
|
+
this.used.delete(provider);
|
|
543
|
+
}
|
|
544
|
+
|
|
477
545
|
/** Providers that have already spent their one fallback. */
|
|
478
546
|
spent(): readonly string[] {
|
|
479
547
|
return [...this.used].sort();
|
|
@@ -62,10 +62,74 @@ export interface Measurement {
|
|
|
62
62
|
breach: boolean;
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
interface PerTurnAggregate {
|
|
66
|
+
tool_calls: number;
|
|
67
|
+
slots_counted: string[];
|
|
68
|
+
ceiling_bytes: number;
|
|
69
|
+
gate_on_ceiling: boolean;
|
|
70
|
+
}
|
|
71
|
+
|
|
65
72
|
interface Budget {
|
|
66
73
|
default_cap_bytes: number;
|
|
67
74
|
per_concern_caps_bytes: Record<string, number | string>;
|
|
68
75
|
per_slot_sum_caps_bytes: Record<string, number | string>;
|
|
76
|
+
per_turn_aggregate_bytes?: PerTurnAggregate;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The per-turn aggregate reading, or the reason there isn't one. */
|
|
80
|
+
export interface TurnAggregate {
|
|
81
|
+
/** Bytes a representative turn injects, or `null` when it is not derivable. */
|
|
82
|
+
bytes: number | null;
|
|
83
|
+
ceiling: number;
|
|
84
|
+
gated: boolean;
|
|
85
|
+
breach: boolean;
|
|
86
|
+
/** Present only when `bytes` is null — names the slot that was missing. */
|
|
87
|
+
unresolved?: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Derive the per-turn aggregate from the same per-slot sums the rows above
|
|
92
|
+
* report, per `per_turn_aggregate_bytes.definition`.
|
|
93
|
+
*
|
|
94
|
+
* Returns `null` bytes rather than a number when a counted slot produced no
|
|
95
|
+
* reading. This is the latency twin's rule, adopted deliberately: a composite
|
|
96
|
+
* over a subset reads LOW, and low is the direction that makes a ceiling look
|
|
97
|
+
* met. A missing slot is an unresolved measurement, never a small one.
|
|
98
|
+
*/
|
|
99
|
+
export function perTurnAggregate(
|
|
100
|
+
slotSums: Record<string, { bytes: number; unresolved?: boolean }>,
|
|
101
|
+
cfg: PerTurnAggregate | undefined,
|
|
102
|
+
): TurnAggregate | null {
|
|
103
|
+
if (cfg === undefined) return null;
|
|
104
|
+
const perCallSlots = new Set(['pre_tool_use', 'post_tool_use']);
|
|
105
|
+
let bytes = 0;
|
|
106
|
+
for (const slot of cfg.slots_counted) {
|
|
107
|
+
const reading = slotSums[slot];
|
|
108
|
+
if (reading === undefined || reading.unresolved === true) {
|
|
109
|
+
return {
|
|
110
|
+
bytes: null,
|
|
111
|
+
ceiling: cfg.ceiling_bytes,
|
|
112
|
+
gated: cfg.gate_on_ceiling,
|
|
113
|
+
// AN UNRESOLVED COMPOSITE BREACHES WHEN THE ROW IS ARMED. It used
|
|
114
|
+
// to return `false` here, which meant the protection this
|
|
115
|
+
// function's docstring calls load-bearing was a printed word and
|
|
116
|
+
// not a gate: once `gate_on_ceiling` is armed, a composite over a
|
|
117
|
+
// subset of its slots reads low, and low is the direction that
|
|
118
|
+
// makes a ceiling look met — so the run would have exited green
|
|
119
|
+
// over a measurement that does not exist. Found by the R2
|
|
120
|
+
// completion review, 2026-08-19.
|
|
121
|
+
breach: cfg.gate_on_ceiling,
|
|
122
|
+
unresolved: slot,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
bytes += perCallSlots.has(slot) ? reading.bytes * cfg.tool_calls : reading.bytes;
|
|
126
|
+
}
|
|
127
|
+
return {
|
|
128
|
+
bytes,
|
|
129
|
+
ceiling: cfg.ceiling_bytes,
|
|
130
|
+
gated: cfg.gate_on_ceiling,
|
|
131
|
+
breach: cfg.gate_on_ceiling && bytes > cfg.ceiling_bytes,
|
|
132
|
+
};
|
|
69
133
|
}
|
|
70
134
|
|
|
71
135
|
function capFor(budget: Budget, concern: string): number {
|
|
@@ -129,12 +193,12 @@ export function runConcernOnce(
|
|
|
129
193
|
}
|
|
130
194
|
}
|
|
131
195
|
|
|
132
|
-
export function bench(opts: { platform?: string; budgetPath?: string } = {}): { measurements: Measurement[]; slotSums: Record<string, { bytes: number; cap: number; breach: boolean }>; errors: string[] } {
|
|
196
|
+
export function bench(opts: { platform?: string; budgetPath?: string } = {}): { measurements: Measurement[]; slotSums: Record<string, { bytes: number; cap: number; breach: boolean }>; errors: string[]; turnAggregate: TurnAggregate | null } {
|
|
133
197
|
const platform = opts.platform ?? 'claude';
|
|
134
198
|
const manifest = _load_yaml(MANIFEST);
|
|
135
199
|
const budget = JSON.parse(fs.readFileSync(opts.budgetPath ?? BUDGET, 'utf8')) as Budget;
|
|
136
200
|
const measurements: Measurement[] = [];
|
|
137
|
-
const slotSums: Record<string, { bytes: number; cap: number; breach: boolean }> = {};
|
|
201
|
+
const slotSums: Record<string, { bytes: number; cap: number; breach: boolean; unresolved?: boolean }> = {};
|
|
138
202
|
const errors: string[] = [];
|
|
139
203
|
|
|
140
204
|
for (const event of EVENT_VOCABULARY) {
|
|
@@ -143,10 +207,19 @@ export function bench(opts: { platform?: string; budgetPath?: string } = {}): {
|
|
|
143
207
|
const envelope = JSON.parse(fs.readFileSync(fixtureFile, 'utf8')) as JsonObject;
|
|
144
208
|
const concerns = _resolve_concerns(manifest, platform, event);
|
|
145
209
|
let sum = 0;
|
|
210
|
+
// A concern the registry cannot resolve contributes NOTHING to `sum`, so
|
|
211
|
+
// the slot total is short by an unknown amount. It used to be reported as
|
|
212
|
+
// a resolved low number, which is the one direction that makes a ceiling
|
|
213
|
+
// look met — `perTurnAggregate`'s own docstring refuses exactly that for a
|
|
214
|
+
// missing fixture, and a missing concern is the same defect one level
|
|
215
|
+
// down. Marked unresolved so the composite refuses to derive from it.
|
|
216
|
+
// (R2 review, 2026-08-19.)
|
|
217
|
+
let slotUnresolved = false;
|
|
146
218
|
for (const concern of concerns) {
|
|
147
219
|
const bytes = runConcernOnce(String(concern['script']), Array.isArray(concern['args']) ? concern['args'].map(String) : [], envelope);
|
|
148
220
|
if (bytes < 0) {
|
|
149
221
|
errors.push(`${event}/${String(concern['name'])}: not in CONCERN_REGISTRY`);
|
|
222
|
+
slotUnresolved = true;
|
|
150
223
|
continue;
|
|
151
224
|
}
|
|
152
225
|
const cap = capFor(budget, String(concern['name']));
|
|
@@ -156,9 +229,9 @@ export function bench(opts: { platform?: string; budgetPath?: string } = {}): {
|
|
|
156
229
|
}
|
|
157
230
|
const slotCapRaw = budget.per_slot_sum_caps_bytes[event];
|
|
158
231
|
const slotCap = typeof slotCapRaw === 'number' ? slotCapRaw : Number.MAX_SAFE_INTEGER;
|
|
159
|
-
slotSums[event] = { bytes: sum, cap: slotCap, breach: sum > slotCap };
|
|
232
|
+
slotSums[event] = { bytes: sum, cap: slotCap, breach: sum > slotCap, ...(slotUnresolved ? { unresolved: true } : {}) };
|
|
160
233
|
}
|
|
161
|
-
return { measurements, slotSums, errors };
|
|
234
|
+
return { measurements, slotSums, errors, turnAggregate: perTurnAggregate(slotSums, budget.per_turn_aggregate_bytes) };
|
|
162
235
|
}
|
|
163
236
|
|
|
164
237
|
export function main(argv: readonly string[]): number {
|
|
@@ -211,10 +284,31 @@ export function main(argv: readonly string[]): number {
|
|
|
211
284
|
for (const [slot, s] of Object.entries(result.slotSums)) {
|
|
212
285
|
if (s.bytes > 0) process.stdout.write(` slot-sum ${slot.padEnd(20)} ${String(s.bytes).padStart(6)} B (cap ${s.cap === Number.MAX_SAFE_INTEGER ? '—' : s.cap})${s.breach ? ' ❌ BREACH' : ''}\n`);
|
|
213
286
|
}
|
|
287
|
+
const agg = result.turnAggregate;
|
|
288
|
+
if (agg !== null) {
|
|
289
|
+
const reading =
|
|
290
|
+
agg.bytes === null
|
|
291
|
+
? `UNRESOLVED (slot '${String(agg.unresolved)}' produced no reading)`
|
|
292
|
+
: `${String(agg.bytes)} B`;
|
|
293
|
+
const arm = agg.gated ? 'gated' : 'reported, not gated';
|
|
294
|
+
process.stdout.write(
|
|
295
|
+
` per-turn aggregate ${reading} (ceiling ${String(agg.ceiling)} B, ${arm})${agg.breach ? ' ❌ BREACH' : ''}\n`,
|
|
296
|
+
);
|
|
297
|
+
}
|
|
214
298
|
for (const e of result.errors) process.stdout.write(` warn: ${e}\n`);
|
|
215
299
|
}
|
|
216
|
-
|
|
217
|
-
|
|
300
|
+
const aggBreach = result.turnAggregate?.breach === true;
|
|
301
|
+
if (breaches.length > 0 || slotBreaches.length > 0 || aggBreach) {
|
|
302
|
+
// The aggregate is named in the failure line. It used to be absent, so an
|
|
303
|
+
// aggregate-only red read "0 concern breach(es), 0 slot-sum breach(es)"
|
|
304
|
+
// and exited 1 — a CI failure that named no cause. (R2 review, 2026-08-19.)
|
|
305
|
+
const agg = result.turnAggregate;
|
|
306
|
+
const aggPart = !aggBreach
|
|
307
|
+
? ''
|
|
308
|
+
: agg?.bytes === null
|
|
309
|
+
? `, per-turn aggregate UNRESOLVED (slot '${String(agg.unresolved)}' produced no reading) while the ceiling is armed`
|
|
310
|
+
: `, per-turn aggregate ${String(agg?.bytes)} B over its ${String(agg?.ceiling)} B ceiling`;
|
|
311
|
+
process.stderr.write(`bench_hook_injection: ${breaches.length} concern breach(es), ${slotBreaches.length} slot-sum breach(es)${aggPart} — the budget row is the decision surface (src/config/hook-token-budget.json)\n`);
|
|
218
312
|
return 1;
|
|
219
313
|
}
|
|
220
314
|
return 0;
|