@hegemonart/get-design-done 1.27.0 → 1.27.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +69 -0
- package/SKILL.md +1 -0
- package/agents/design-reflector.md +52 -0
- package/hooks/budget-enforcer.ts +249 -5
- package/package.json +2 -2
- package/reference/bandit-integration.md +163 -0
- package/reference/peer-protocols.md +1 -1
- package/reference/registry.json +7 -0
- package/scripts/install.cjs +100 -1
- package/scripts/lib/bandit-arbitrage.cjs +423 -0
- package/scripts/lib/bandit-router/integration.cjs +309 -0
- package/scripts/lib/peer-cli/spawn-cmd.cjs +2 -2
- package/scripts/lib/session-runner/index.ts +381 -28
- package/skills/bandit-status/SKILL.md +129 -0
- package/skills/peers/SKILL.md +27 -8
|
@@ -80,12 +80,130 @@ const rateGuard = _nodeRequire(
|
|
|
80
80
|
ingestHeaders: (provider: string, headers: unknown) => Promise<unknown>;
|
|
81
81
|
};
|
|
82
82
|
|
|
83
|
+
// ── Plan 27.5-03 — Bandit posterior feedback loop ────────────────────────────
|
|
84
|
+
//
|
|
85
|
+
// `integration.cjs` is the Phase 27.5-01 production-integration shim for the
|
|
86
|
+
// Phase 23.5 bandit posterior. It exposes `recordOutcome({agent, bin, delegate,
|
|
87
|
+
// tier, status, costUsd, adaptiveMode, baseDir?, posteriorPath?})` which writes
|
|
88
|
+
// the (status + cost) reward back to the posterior arm for the (agent × bin ×
|
|
89
|
+
// tier × delegate) joint. Per CONTEXT D-04, the call fires AFTER every
|
|
90
|
+
// `emit('session.completed', …)` site so the posterior reflects the measured
|
|
91
|
+
// signal — correctness + cost.
|
|
92
|
+
//
|
|
93
|
+
// The shim is no-throw (best-effort write). The session-runner wraps each
|
|
94
|
+
// `recordOutcome` call in its own try/catch as a defensive belt-and-braces
|
|
95
|
+
// guard against future shim changes.
|
|
96
|
+
const banditIntegration = _nodeRequire(
|
|
97
|
+
_resolve(_REPO_ROOT, 'scripts/lib/bandit-router/integration.cjs'),
|
|
98
|
+
) as {
|
|
99
|
+
recordOutcome: (input: {
|
|
100
|
+
agent: string;
|
|
101
|
+
bin: string;
|
|
102
|
+
delegate?: string;
|
|
103
|
+
tier: string;
|
|
104
|
+
status: string;
|
|
105
|
+
costUsd?: number;
|
|
106
|
+
adaptiveMode?: 'static' | 'hedge' | 'full';
|
|
107
|
+
baseDir?: string;
|
|
108
|
+
posteriorPath?: string;
|
|
109
|
+
}) => void;
|
|
110
|
+
DELEGATE_NONE: string;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
// ── Plan 27.5-03 — adaptive-mode read once per run ──────────────────────────
|
|
114
|
+
//
|
|
115
|
+
// `adaptive-mode.cjs.getMode({quiet: true})` reads `.design/budget.json` and
|
|
116
|
+
// returns `'static' | 'hedge' | 'full'`. We cache the resolved mode locally
|
|
117
|
+
// on each `run()` invocation so the recordOutcome calls at the 4 terminal
|
|
118
|
+
// emit sites all see the same value (consistent gating per session).
|
|
119
|
+
const adaptiveModeLib = _nodeRequire(
|
|
120
|
+
_resolve(_REPO_ROOT, 'scripts/lib/adaptive-mode.cjs'),
|
|
121
|
+
) as {
|
|
122
|
+
getMode: (opts?: { baseDir?: string; budgetPath?: string; quiet?: boolean }) => 'static' | 'hedge' | 'full';
|
|
123
|
+
};
|
|
124
|
+
|
|
83
125
|
/** Rate-guard provider key for the Anthropic Agent SDK. */
|
|
84
126
|
const RATE_GUARD_PROVIDER = 'anthropic';
|
|
85
127
|
|
|
86
128
|
/** Default retries (first attempt + 1 retry). */
|
|
87
129
|
const DEFAULT_MAX_RETRIES = 2;
|
|
88
130
|
|
|
131
|
+
/**
|
|
132
|
+
* Default bin marker for bandit posterior writes from session-runner.
|
|
133
|
+
*
|
|
134
|
+
* Per CONTEXT D-12, session-runner uses a deterministic placeholder bin
|
|
135
|
+
* (`'medium'`) for now; real complexity-class-based bin selection is
|
|
136
|
+
* deferred to a later plan. This matches the 27.5-02 budget-enforcer
|
|
137
|
+
* convention so the (agent × bin) posterior slices stay consistent
|
|
138
|
+
* across both write paths.
|
|
139
|
+
*/
|
|
140
|
+
const SESSION_RUNNER_DEFAULT_BIN = 'medium';
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Infer a tier ('opus' | 'sonnet' | 'haiku') from a model identifier.
|
|
144
|
+
*
|
|
145
|
+
* Used at the 4 terminal-emit sites where the final tier isn't already
|
|
146
|
+
* carried on `opts` — we fall back to inspecting `usage.model` (folded
|
|
147
|
+
* during the run loop from SDK chunks). Unknown / empty model names
|
|
148
|
+
* default to 'sonnet' (matches the DEFAULT_MODEL_RATE choice and is
|
|
149
|
+
* the safest middle tier for posterior arms).
|
|
150
|
+
*/
|
|
151
|
+
function tierFromModel(modelName: string | null | undefined): 'opus' | 'sonnet' | 'haiku' {
|
|
152
|
+
if (typeof modelName !== 'string' || modelName.length === 0) return 'sonnet';
|
|
153
|
+
const lower = modelName.toLowerCase();
|
|
154
|
+
if (lower.includes('opus')) return 'opus';
|
|
155
|
+
if (lower.includes('haiku')) return 'haiku';
|
|
156
|
+
return 'sonnet';
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Best-effort bandit posterior write following `emit('session.completed', …)`.
|
|
161
|
+
*
|
|
162
|
+
* Per CONTEXT D-04: posterior updates happen AT the terminal emit site so the
|
|
163
|
+
* recorded reward reflects the same (status + cost) the rest of the system
|
|
164
|
+
* just observed. The shim (`integration.cjs`) is no-throw and short-circuits
|
|
165
|
+
* silently in static/hedge mode; the outer try/catch here is a defensive
|
|
166
|
+
* belt-and-braces guard for any future shim change.
|
|
167
|
+
*
|
|
168
|
+
* Failures NEVER bubble out — the session-runner contract is that `run()`
|
|
169
|
+
* never throws, and that contract MUST hold even when telemetry is broken.
|
|
170
|
+
*/
|
|
171
|
+
function _recordBanditOutcome(input: {
|
|
172
|
+
agent: string;
|
|
173
|
+
bin: string;
|
|
174
|
+
delegate: string;
|
|
175
|
+
tier: string;
|
|
176
|
+
status: string;
|
|
177
|
+
costUsd: number;
|
|
178
|
+
adaptiveMode: 'static' | 'hedge' | 'full';
|
|
179
|
+
}): void {
|
|
180
|
+
try {
|
|
181
|
+
banditIntegration.recordOutcome({
|
|
182
|
+
agent: input.agent,
|
|
183
|
+
bin: input.bin,
|
|
184
|
+
delegate: input.delegate,
|
|
185
|
+
tier: input.tier,
|
|
186
|
+
status: input.status,
|
|
187
|
+
costUsd: input.costUsd,
|
|
188
|
+
adaptiveMode: input.adaptiveMode,
|
|
189
|
+
});
|
|
190
|
+
} catch (err) {
|
|
191
|
+
// Defensive: shim is no-throw, but a future change could regress.
|
|
192
|
+
// Telemetry failure must never break a session — swallow.
|
|
193
|
+
if (process.env['GDD_BANDIT_DEBUG'] === '1') {
|
|
194
|
+
try {
|
|
195
|
+
process.stderr.write(
|
|
196
|
+
'[session-runner] _recordBanditOutcome swallowed: ' +
|
|
197
|
+
(err instanceof Error ? err.message : String(err)) +
|
|
198
|
+
'\n',
|
|
199
|
+
);
|
|
200
|
+
} catch {
|
|
201
|
+
/* swallow */
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
89
207
|
// ── Plan 27-06 — Peer-CLI delegation primitives ─────────────────────────────
|
|
90
208
|
//
|
|
91
209
|
// Lazy registry loader: the registry is a .cjs module under scripts/lib/peer-cli
|
|
@@ -200,15 +318,22 @@ async function tryDelegate(args: {
|
|
|
200
318
|
return reg !== null ? reg.dispatch : null;
|
|
201
319
|
})();
|
|
202
320
|
if (dispatcher === null) {
|
|
203
|
-
// No registry available at all — fall through to local.
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
_logPeerCallFailed({ peer: parsed.peer, role, errorClass: 'registry_missing' });
|
|
321
|
+
// No registry available at all — fall through to local.
|
|
322
|
+
_logPeerCallFailed({
|
|
323
|
+
peer: parsed.peer, role, errorClass: 'registry_missing',
|
|
324
|
+
sessionId: args.sessionId, stage: opts.stage,
|
|
325
|
+
});
|
|
209
326
|
return null;
|
|
210
327
|
}
|
|
211
328
|
|
|
329
|
+
// v1.27.1 — emit peer_call_started before dispatcher invocation so the
|
|
330
|
+
// events.jsonl trail captures the attempt even if the dispatcher hangs.
|
|
331
|
+
_logPeerCallStarted({
|
|
332
|
+
peer: parsed.peer, role,
|
|
333
|
+
sessionId: args.sessionId, stage: opts.stage,
|
|
334
|
+
});
|
|
335
|
+
const dispatchStartedAt = Date.now();
|
|
336
|
+
|
|
212
337
|
let dispatchResult: { result: unknown; peer: string; protocol: 'acp' | 'asp' } | null = null;
|
|
213
338
|
try {
|
|
214
339
|
dispatchResult = await dispatcher(role, tier, sanitizedPrompt, { cwd: process.cwd() });
|
|
@@ -218,6 +343,8 @@ async function tryDelegate(args: {
|
|
|
218
343
|
role,
|
|
219
344
|
errorClass: 'dispatch_threw',
|
|
220
345
|
message: err instanceof Error ? err.message : String(err),
|
|
346
|
+
sessionId: args.sessionId,
|
|
347
|
+
stage: opts.stage,
|
|
221
348
|
});
|
|
222
349
|
return null; // transparent fallback
|
|
223
350
|
}
|
|
@@ -225,10 +352,28 @@ async function tryDelegate(args: {
|
|
|
225
352
|
if (dispatchResult === null) {
|
|
226
353
|
// Registry returned null — peer absent, capability mismatch, or
|
|
227
354
|
// adapter-side error. Per D-07 we fall back silently.
|
|
228
|
-
_logPeerCallFailed({
|
|
355
|
+
_logPeerCallFailed({
|
|
356
|
+
peer: parsed.peer, role, errorClass: 'registry_returned_null',
|
|
357
|
+
sessionId: args.sessionId, stage: opts.stage,
|
|
358
|
+
});
|
|
229
359
|
return null;
|
|
230
360
|
}
|
|
231
361
|
|
|
362
|
+
// v1.27.1 — peer round-trip succeeded. Emit peer_call_complete with the
|
|
363
|
+
// measured latency. Token counts + cost are 0 / null because adapters
|
|
364
|
+
// don't surface usage in v1.27 (Plan 27-04 spec deferred it); reflector
|
|
365
|
+
// tolerates null cost (Plan 26-06 cost-arbitrage analysis).
|
|
366
|
+
_logPeerCallComplete({
|
|
367
|
+
peer: dispatchResult.peer,
|
|
368
|
+
role,
|
|
369
|
+
latencyMs: Date.now() - dispatchStartedAt,
|
|
370
|
+
tokensIn: 0,
|
|
371
|
+
tokensOut: 0,
|
|
372
|
+
costUsd: null,
|
|
373
|
+
sessionId: args.sessionId,
|
|
374
|
+
stage: opts.stage,
|
|
375
|
+
});
|
|
376
|
+
|
|
232
377
|
// Peer succeeded. Build a SessionResult that mirrors the local path's
|
|
233
378
|
// shape so downstream consumers (stage-handlers, transcript readers,
|
|
234
379
|
// tests) treat both paths uniformly. We do NOT write a transcript file
|
|
@@ -271,34 +416,122 @@ function _coerceFinalText(result: unknown): string | undefined {
|
|
|
271
416
|
}
|
|
272
417
|
|
|
273
418
|
/**
|
|
274
|
-
*
|
|
275
|
-
*
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
*
|
|
419
|
+
* v1.27.1 — wires Plan 27-08's `peer_call_failed` event for real.
|
|
420
|
+
* Phase 22 `appendEvent` accepts the new event type (registered in
|
|
421
|
+
* KNOWN_EVENT_TYPES via Plan 27-08), so the reflector and downstream
|
|
422
|
+
* telemetry consumers see delegation drops as a measurement signal.
|
|
423
|
+
*
|
|
424
|
+
* Errors from `appendEvent` (e.g., events.jsonl unwritable) are
|
|
425
|
+
* swallowed — peer-call telemetry is observability, not critical
|
|
426
|
+
* path. STATE.md remains the durable record of session outcomes.
|
|
427
|
+
*
|
|
428
|
+
* Operators can additionally set `GDD_PEER_DEBUG=1` to emit a
|
|
429
|
+
* one-line stderr breadcrumb mirroring the event for live tailing.
|
|
281
430
|
*/
|
|
282
431
|
function _logPeerCallFailed(args: {
|
|
283
432
|
peer: string;
|
|
284
433
|
role: string;
|
|
285
434
|
errorClass: string;
|
|
286
435
|
message?: string;
|
|
436
|
+
sessionId?: string;
|
|
437
|
+
stage?: SessionRunnerOptions['stage'];
|
|
287
438
|
}): void {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
439
|
+
try {
|
|
440
|
+
appendEvent({
|
|
441
|
+
type: 'peer_call_failed',
|
|
442
|
+
timestamp: new Date().toISOString(),
|
|
443
|
+
sessionId: args.sessionId ?? 'unknown',
|
|
444
|
+
...(args.stage !== undefined && args.stage !== 'init' && args.stage !== 'custom' ? { stage: args.stage } : {}),
|
|
445
|
+
payload: {
|
|
446
|
+
runtime_role: 'peer',
|
|
447
|
+
peer_id: args.peer,
|
|
448
|
+
role: args.role,
|
|
449
|
+
error_class: args.errorClass,
|
|
450
|
+
...(args.message !== undefined ? { message: args.message } : {}),
|
|
451
|
+
},
|
|
452
|
+
});
|
|
453
|
+
} catch {
|
|
454
|
+
// Telemetry is best-effort — never let an event-stream failure
|
|
455
|
+
// break the actual session flow.
|
|
456
|
+
}
|
|
457
|
+
if (process.env['GDD_PEER_DEBUG'] === '1') {
|
|
458
|
+
const payload = JSON.stringify({
|
|
459
|
+
type: 'peer_call_failed',
|
|
460
|
+
peer_id: args.peer,
|
|
461
|
+
role: args.role,
|
|
462
|
+
error_class: args.errorClass,
|
|
463
|
+
...(args.message !== undefined ? { message: args.message } : {}),
|
|
464
|
+
ts: new Date().toISOString(),
|
|
465
|
+
});
|
|
466
|
+
// eslint-disable-next-line no-console
|
|
467
|
+
console.error(`[peer-cli] ${payload}`);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* v1.27.1 — emit `peer_call_started` event. Fired once at the beginning
|
|
473
|
+
* of a delegation attempt, before the dispatcher is invoked. Pairs with
|
|
474
|
+
* `peer_call_complete` (success path) or `peer_call_failed` (any failure
|
|
475
|
+
* path, transparent to caller per D-07).
|
|
476
|
+
*/
|
|
477
|
+
function _logPeerCallStarted(args: {
|
|
478
|
+
peer: string;
|
|
479
|
+
role: string;
|
|
480
|
+
sessionId?: string;
|
|
481
|
+
stage?: SessionRunnerOptions['stage'];
|
|
482
|
+
}): void {
|
|
483
|
+
try {
|
|
484
|
+
appendEvent({
|
|
485
|
+
type: 'peer_call_started',
|
|
486
|
+
timestamp: new Date().toISOString(),
|
|
487
|
+
sessionId: args.sessionId ?? 'unknown',
|
|
488
|
+
...(args.stage !== undefined && args.stage !== 'init' && args.stage !== 'custom' ? { stage: args.stage } : {}),
|
|
489
|
+
payload: {
|
|
490
|
+
runtime_role: 'peer',
|
|
491
|
+
peer_id: args.peer,
|
|
492
|
+
role: args.role,
|
|
493
|
+
},
|
|
494
|
+
});
|
|
495
|
+
} catch {
|
|
496
|
+
// best-effort
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/**
|
|
501
|
+
* v1.27.1 — emit `peer_call_complete` event. Fired after a successful
|
|
502
|
+
* dispatcher round-trip. Cost is null when the adapter doesn't return
|
|
503
|
+
* usage data (some peers don't surface token counts); the reflector
|
|
504
|
+
* tolerates null cost for arbitrage analysis (Plan 26-06).
|
|
505
|
+
*/
|
|
506
|
+
function _logPeerCallComplete(args: {
|
|
507
|
+
peer: string;
|
|
508
|
+
role: string;
|
|
509
|
+
latencyMs: number;
|
|
510
|
+
tokensIn: number;
|
|
511
|
+
tokensOut: number;
|
|
512
|
+
costUsd: number | null;
|
|
513
|
+
sessionId?: string;
|
|
514
|
+
stage?: SessionRunnerOptions['stage'];
|
|
515
|
+
}): void {
|
|
516
|
+
try {
|
|
517
|
+
appendEvent({
|
|
518
|
+
type: 'peer_call_complete',
|
|
519
|
+
timestamp: new Date().toISOString(),
|
|
520
|
+
sessionId: args.sessionId ?? 'unknown',
|
|
521
|
+
...(args.stage !== undefined && args.stage !== 'init' && args.stage !== 'custom' ? { stage: args.stage } : {}),
|
|
522
|
+
payload: {
|
|
523
|
+
runtime_role: 'peer',
|
|
524
|
+
peer_id: args.peer,
|
|
525
|
+
role: args.role,
|
|
526
|
+
latency_ms: args.latencyMs,
|
|
527
|
+
tokens_in: args.tokensIn,
|
|
528
|
+
tokens_out: args.tokensOut,
|
|
529
|
+
cost_usd: args.costUsd,
|
|
530
|
+
},
|
|
531
|
+
});
|
|
532
|
+
} catch {
|
|
533
|
+
// best-effort
|
|
534
|
+
}
|
|
302
535
|
}
|
|
303
536
|
|
|
304
537
|
/** Baseline retry backoff parameters (matches jittered-backoff defaults for
|
|
@@ -565,6 +798,22 @@ export async function run(opts: SessionRunnerOptions): Promise<SessionResult> {
|
|
|
565
798
|
const toolCalls: SessionResult['tool_calls'] = [];
|
|
566
799
|
const usage = { input: 0, output: 0, model: null as string | null };
|
|
567
800
|
let turns = 0;
|
|
801
|
+
|
|
802
|
+
// -- 3a. Resolve adaptive-mode once for the entire session (Plan 27.5-03). --
|
|
803
|
+
// Cached locally so all four `recordOutcome()` call sites below see the
|
|
804
|
+
// same gating decision (consistent posterior-write semantics across
|
|
805
|
+
// rate-limit, peer, turnCap=0, and terminal-completion paths).
|
|
806
|
+
//
|
|
807
|
+
// Wrapped in try/catch because adaptive-mode.getMode reads
|
|
808
|
+
// `.design/budget.json`; a broken fs.readFile / JSON.parse must not
|
|
809
|
+
// crash the session before it even starts. Fallback = 'static' which
|
|
810
|
+
// short-circuits the recordOutcome shim (no-op).
|
|
811
|
+
let adaptiveMode: 'static' | 'hedge' | 'full' = 'static';
|
|
812
|
+
try {
|
|
813
|
+
adaptiveMode = adaptiveModeLib.getMode({ quiet: true });
|
|
814
|
+
} catch {
|
|
815
|
+
// swallow — fallback to 'static' means no posterior writes
|
|
816
|
+
}
|
|
568
817
|
let finalText: string | undefined;
|
|
569
818
|
|
|
570
819
|
// -- 4. Emit session.started. -------------------------------------------
|
|
@@ -604,6 +853,20 @@ export async function run(opts: SessionRunnerOptions): Promise<SessionResult> {
|
|
|
604
853
|
transcript_path: transcriptPath,
|
|
605
854
|
sanitizer: { applied: [...result.sanitizer.applied], removedSections: [...result.sanitizer.removedSections] },
|
|
606
855
|
});
|
|
856
|
+
// Plan 27.5-03 — feedback loop. Posterior records the
|
|
857
|
+
// measured outcome (status + cost) for the (agent × bin × tier × delegate)
|
|
858
|
+
// slice. The rate-limit preflight failure path has no peer dispatch and no
|
|
859
|
+
// usage data (zero cost), so delegate=DELEGATE_NONE and tier falls back to
|
|
860
|
+
// 'sonnet' via tierFromModel(null). Shim no-ops in static/hedge mode.
|
|
861
|
+
_recordBanditOutcome({
|
|
862
|
+
agent: opts.stage,
|
|
863
|
+
bin: SESSION_RUNNER_DEFAULT_BIN,
|
|
864
|
+
delegate: banditIntegration.DELEGATE_NONE,
|
|
865
|
+
tier: tierFromModel(usage.model),
|
|
866
|
+
status: result.status,
|
|
867
|
+
costUsd: result.usage.usd_cost,
|
|
868
|
+
adaptiveMode,
|
|
869
|
+
});
|
|
607
870
|
transcript.close();
|
|
608
871
|
return result;
|
|
609
872
|
}
|
|
@@ -623,6 +886,69 @@ export async function run(opts: SessionRunnerOptions): Promise<SessionResult> {
|
|
|
623
886
|
}
|
|
624
887
|
}
|
|
625
888
|
|
|
889
|
+
// -- 6.5. Peer-CLI delegation try (Plan 27-06 wiring, v1.27.1). ---------
|
|
890
|
+
// If the agent's frontmatter declares `delegate_to: <peer>-<role>` AND the
|
|
891
|
+
// peer is allowlisted AND the registry can route, run the prompt on the
|
|
892
|
+
// peer-CLI and return early. On peer-absent / peer-error / null result,
|
|
893
|
+
// fall through transparently to the local SDK loop (D-07).
|
|
894
|
+
//
|
|
895
|
+
// tryDelegate is a no-op when opts.delegateTo is undefined / 'none', when
|
|
896
|
+
// the registry can't load, when the peer isn't allowlisted, when the
|
|
897
|
+
// dispatcher returns null, or when the dispatcher throws. In all those
|
|
898
|
+
// cases tryDelegate returns null and we proceed to the local SDK path.
|
|
899
|
+
const peerResult = await tryDelegate({
|
|
900
|
+
opts,
|
|
901
|
+
sanitizedPrompt,
|
|
902
|
+
transcriptPath,
|
|
903
|
+
sessionId,
|
|
904
|
+
sanitizer: sanResult,
|
|
905
|
+
});
|
|
906
|
+
if (peerResult !== null) {
|
|
907
|
+
emit('session.completed', opts.stage, sessionId, {
|
|
908
|
+
stage: opts.stage,
|
|
909
|
+
sessionId,
|
|
910
|
+
status: peerResult.status,
|
|
911
|
+
turns: peerResult.turns,
|
|
912
|
+
usage: peerResult.usage,
|
|
913
|
+
transcript_path: transcriptPath,
|
|
914
|
+
sanitizer: { applied: [...peerResult.sanitizer.applied], removedSections: [...peerResult.sanitizer.removedSections] },
|
|
915
|
+
});
|
|
916
|
+
// Plan 27.5-03 — feedback loop, peer path. The
|
|
917
|
+
// delegate dimension is the peer name parsed from opts.delegateTo (e.g.
|
|
918
|
+
// 'gemini-research' → 'gemini'). Per CONTEXT D-04 we use the peer name
|
|
919
|
+
// for the delegate slice of the posterior so peer-success arms get the
|
|
920
|
+
// reward signal separately from local arms. Tier is 'sonnet' by default
|
|
921
|
+
// since the peer adapter doesn't surface a model identifier in v1.27.
|
|
922
|
+
// Re-parse opts.delegateTo here — tryDelegate already validated it but
|
|
923
|
+
// didn't expose the peer name on the returned SessionResult.
|
|
924
|
+
const _peerParsed = parseDelegateTo(opts.delegateTo);
|
|
925
|
+
const _delegate = _peerParsed !== null
|
|
926
|
+
? _peerParsed.peer
|
|
927
|
+
: banditIntegration.DELEGATE_NONE;
|
|
928
|
+
// Tier resolution priority for the peer path:
|
|
929
|
+
// 1. opts.delegateTier when it's a bare tier name (opus/sonnet/haiku)
|
|
930
|
+
// 2. tierFromModel(opts.delegateTier) when it's a model id string
|
|
931
|
+
// 3. tierFromModel(usage.model) fallback
|
|
932
|
+
// tierFromModel() is safe for any string and returns 'sonnet' on miss,
|
|
933
|
+
// so the second branch covers both bare-tier and model-id inputs.
|
|
934
|
+
const _peerTier: 'opus' | 'sonnet' | 'haiku' =
|
|
935
|
+
typeof opts.delegateTier === 'string' && opts.delegateTier.length > 0
|
|
936
|
+
? tierFromModel(opts.delegateTier)
|
|
937
|
+
: tierFromModel(usage.model);
|
|
938
|
+
_recordBanditOutcome({
|
|
939
|
+
agent: opts.stage,
|
|
940
|
+
bin: SESSION_RUNNER_DEFAULT_BIN,
|
|
941
|
+
delegate: _delegate,
|
|
942
|
+
tier: _peerTier,
|
|
943
|
+
status: peerResult.status,
|
|
944
|
+
costUsd: peerResult.usage.usd_cost,
|
|
945
|
+
adaptiveMode,
|
|
946
|
+
});
|
|
947
|
+
transcript.close();
|
|
948
|
+
if (opts.signal !== undefined) opts.signal.removeEventListener('abort', onExternalAbort);
|
|
949
|
+
return peerResult;
|
|
950
|
+
}
|
|
951
|
+
|
|
626
952
|
// -- 7. Retry-once loop. ------------------------------------------------
|
|
627
953
|
const maxAttempts = opts.maxRetries !== undefined && opts.maxRetries > 0
|
|
628
954
|
? opts.maxRetries
|
|
@@ -649,6 +975,18 @@ export async function run(opts: SessionRunnerOptions): Promise<SessionResult> {
|
|
|
649
975
|
transcript_path: transcriptPath,
|
|
650
976
|
sanitizer: { applied: [...sanResult.applied], removedSections: [...sanResult.removedSections] },
|
|
651
977
|
});
|
|
978
|
+
// Plan 27.5-03 — feedback loop, turnCap=0 path. No
|
|
979
|
+
// SDK call was ever made, so no peer involvement and no model id was
|
|
980
|
+
// ever observed. Reward will be 0 (status !== 'completed') with cost 0.
|
|
981
|
+
_recordBanditOutcome({
|
|
982
|
+
agent: opts.stage,
|
|
983
|
+
bin: SESSION_RUNNER_DEFAULT_BIN,
|
|
984
|
+
delegate: banditIntegration.DELEGATE_NONE,
|
|
985
|
+
tier: tierFromModel(usage.model),
|
|
986
|
+
status,
|
|
987
|
+
costUsd: result.usage.usd_cost,
|
|
988
|
+
adaptiveMode,
|
|
989
|
+
});
|
|
652
990
|
transcript.close();
|
|
653
991
|
if (opts.signal !== undefined) opts.signal.removeEventListener('abort', onExternalAbort);
|
|
654
992
|
return result;
|
|
@@ -743,6 +1081,21 @@ export async function run(opts: SessionRunnerOptions): Promise<SessionResult> {
|
|
|
743
1081
|
transcript_path: transcriptPath,
|
|
744
1082
|
sanitizer: { applied: [...sanResult.applied], removedSections: [...sanResult.removedSections] },
|
|
745
1083
|
});
|
|
1084
|
+
// Plan 27.5-03 — feedback loop, terminal main-loop path.
|
|
1085
|
+
// This is the dominant write site: covers natural completion, budget cap,
|
|
1086
|
+
// turn cap (after first turn), abort, and error (post-retry-exhaustion).
|
|
1087
|
+
// Tier is inferred from the model actually observed during the run
|
|
1088
|
+
// (usage.model). Delegate=DELEGATE_NONE because tryDelegate either returned
|
|
1089
|
+
// null (we wouldn't be here otherwise) or wasn't invoked at all.
|
|
1090
|
+
_recordBanditOutcome({
|
|
1091
|
+
agent: opts.stage,
|
|
1092
|
+
bin: SESSION_RUNNER_DEFAULT_BIN,
|
|
1093
|
+
delegate: banditIntegration.DELEGATE_NONE,
|
|
1094
|
+
tier: tierFromModel(usage.model),
|
|
1095
|
+
status: result.status,
|
|
1096
|
+
costUsd: result.usage.usd_cost,
|
|
1097
|
+
adaptiveMode,
|
|
1098
|
+
});
|
|
746
1099
|
|
|
747
1100
|
return result;
|
|
748
1101
|
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: gdd-bandit-status
|
|
3
|
+
description: "Surface read-only per-(agent, bin, delegate) bandit posterior snapshot — alpha/beta/mean/stddev/count/last-used per arm. Phase 27.5 (v1.27.5) diagnostic. Use when investigating 'why did the bandit pick tier X for agent Y?' or when verifying posterior convergence after enabling adaptive_mode: full."
|
|
4
|
+
argument-hint: ""
|
|
5
|
+
tools: Read, Bash
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# gdd-bandit-status
|
|
9
|
+
|
|
10
|
+
## Role
|
|
11
|
+
|
|
12
|
+
You are a deterministic, read-only diagnostic skill. You do not spawn agents and do not modify the bandit posterior. You read `.design/telemetry/posterior.json` (the path declared by `scripts/lib/bandit-router.cjs`'s `DEFAULT_POSTERIOR_PATH` constant), aggregate per-`(agent, bin, delegate, tier)` arm state, and emit a single Markdown table summarizing the posterior. The user runs this when they want to inspect bandit decisions without touching the posterior.
|
|
13
|
+
|
|
14
|
+
Strictly read-only per Phase 27.5 D-11. To reset the posterior, use `/gdd:bandit-reset` from Phase 23.5.
|
|
15
|
+
|
|
16
|
+
## Invocation Contract
|
|
17
|
+
|
|
18
|
+
- **Input**: none. The skill takes no arguments.
|
|
19
|
+
- **Output**: a Markdown bandit-status table to stdout. No JSON wrapper. The table is the entire output.
|
|
20
|
+
|
|
21
|
+
## Procedure
|
|
22
|
+
|
|
23
|
+
### 1. Locate the posterior file
|
|
24
|
+
|
|
25
|
+
Read `.design/telemetry/posterior.json`. If the file does not exist:
|
|
26
|
+
|
|
27
|
+
- Emit the empty-state message:
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
## Bandit Posterior Snapshot
|
|
31
|
+
|
|
32
|
+
No posterior data yet — run a few pipeline cycles with `adaptive_mode: full` first.
|
|
33
|
+
|
|
34
|
+
No posterior data found at `.design/telemetry/posterior.json`.
|
|
35
|
+
|
|
36
|
+
Possible reasons:
|
|
37
|
+
- `adaptive_mode` is `static` or `hedge` (bandit is silent — see `.design/budget.json` `adaptive_mode` setting).
|
|
38
|
+
- No spawns have fired since Phase 27.5 wiring landed.
|
|
39
|
+
- Posterior was cleared via `/gdd:bandit-reset`.
|
|
40
|
+
|
|
41
|
+
See `reference/bandit-integration.md` for setup guidance.
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
- Skip to Section 4 (Record).
|
|
45
|
+
|
|
46
|
+
### 2. Parse the posterior
|
|
47
|
+
|
|
48
|
+
Parse the file as JSON. If parsing fails (truncated/corrupted file), emit:
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
## Bandit Posterior Snapshot
|
|
52
|
+
|
|
53
|
+
Posterior file at `.design/telemetry/posterior.json` exists but is unparseable (truncated or corrupted).
|
|
54
|
+
|
|
55
|
+
Run `/gdd:bandit-reset` to start fresh, or restore from a backup.
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The posterior schema is:
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
{
|
|
62
|
+
"schema_version": "1.0.0",
|
|
63
|
+
"generated_at": "<ISO timestamp>",
|
|
64
|
+
"arms": [
|
|
65
|
+
{ "agent": "...", "bin": "...", "tier": "...", "delegate": "...", "alpha": N, "beta": N, "last_used": "...", "count": N }
|
|
66
|
+
]
|
|
67
|
+
}
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The `delegate` field is optional — when absent, the arm is the Phase 23.5 legacy slice (equivalent to `delegate: 'none'`). The status output renders `delegate: '-'` for legacy arms to distinguish them visually from explicit `'none'` arms.
|
|
71
|
+
|
|
72
|
+
### 3. Render the table
|
|
73
|
+
|
|
74
|
+
Compute per arm:
|
|
75
|
+
|
|
76
|
+
- `mean = alpha / (alpha + beta)` (rounded to 3 decimals)
|
|
77
|
+
- `stddev = sqrt(alpha * beta / ((alpha + beta)^2 * (alpha + beta + 1)))` (rounded to 3 decimals)
|
|
78
|
+
|
|
79
|
+
Sort arms by `(agent ascending, bin ascending, delegate ascending where '-' sorts first, tier ascending where opus < sonnet < haiku is the canonical tier ordering, last_used descending tiebreaker)`. Group rows by agent for readability.
|
|
80
|
+
|
|
81
|
+
Emit:
|
|
82
|
+
|
|
83
|
+
```
|
|
84
|
+
## Bandit Posterior Snapshot
|
|
85
|
+
|
|
86
|
+
Per-(agent, bin, delegate, tier) posterior state. Read-only — to reset the posterior, use `/gdd:bandit-reset` (Phase 23.5).
|
|
87
|
+
|
|
88
|
+
Posterior file: `.design/telemetry/posterior.json` (last updated: <generated_at>)
|
|
89
|
+
Total arms: <count>
|
|
90
|
+
|
|
91
|
+
| Agent | Bin | Delegate | Tier | Alpha | Beta | Mean | Stddev | Count | Last Used |
|
|
92
|
+
|-----------------|--------|----------|--------|-------|-------|-------|--------|-------|----------------------|
|
|
93
|
+
| <agent> | <bin> | <deleg> | <tier> | <a> | <b> | <m> | <s> | <c> | <last_used or '-'> |
|
|
94
|
+
|
|
95
|
+
> Mean = alpha / (alpha + beta). Stddev = sqrt(alpha*beta / ((alpha+beta)^2 * (alpha+beta+1))).
|
|
96
|
+
> Delegate '-' = Phase 23.5 legacy slice (equivalent to 'none').
|
|
97
|
+
> See `reference/bandit-integration.md` for interpretation.
|
|
98
|
+
> Read-only — use `/gdd:bandit-reset` to clear posterior state.
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Format numbers to fixed precision: alpha/beta to 2 decimals, mean/stddev to 3 decimals, count as integer, last_used truncated to the minute precision (`YYYY-MM-DDTHH:MM`).
|
|
102
|
+
|
|
103
|
+
When `last_used` is null (arm exists but never selected — possible if the arm was created by `ensureArm` without a subsequent `pull`), render `-` in the Last Used column.
|
|
104
|
+
|
|
105
|
+
After the table, surface a brief best-arm summary per `(agent, bin)` slice — for each unique `(agent, bin)` pair, identify the arm with the highest `mean` (tie-broken by `count` descending) and display it as the "best-arm" recommendation. This helps the operator answer "why did the bandit pick tier X?" at a glance.
|
|
106
|
+
|
|
107
|
+
### 4. Record
|
|
108
|
+
|
|
109
|
+
After execution, append one JSONL line to `.design/skill-records.jsonl`:
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
{"skill": "gdd-bandit-status", "ts": "<ISO timestamp>", "arms_seen": <count>, "posterior_present": <bool>}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The skill writes ONLY to `.design/skill-records.jsonl` for telemetry purposes. It never touches `.design/telemetry/posterior.json`.
|
|
116
|
+
|
|
117
|
+
## Cross-references
|
|
118
|
+
|
|
119
|
+
- `scripts/lib/bandit-router.cjs` (Phase 23.5) — posterior shape, `DEFAULT_POSTERIOR_PATH` constant, `loadPosterior()` helper.
|
|
120
|
+
- `scripts/lib/bandit-router/integration.cjs` (Phase 27.5-01) — production-integration shim.
|
|
121
|
+
- `hooks/budget-enforcer.ts` (Phase 27.5-02) — bandit consultation site.
|
|
122
|
+
- `scripts/lib/session-runner/index.ts` (Phase 27.5-03) — outcome recording site.
|
|
123
|
+
- `scripts/lib/bandit-arbitrage.cjs` (Phase 27.5-04) — automated stale-frontmatter analysis.
|
|
124
|
+
- `reference/bandit-integration.md` (Phase 27.5-06) — operator guide.
|
|
125
|
+
- `/gdd:bandit-reset` (Phase 23.5) — the ONLY surface that mutates the posterior.
|
|
126
|
+
|
|
127
|
+
## Record
|
|
128
|
+
|
|
129
|
+
See Section 4 above.
|