@event4u/agent-config 7.0.0 → 7.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +41 -504
- package/dist/agent-src/scripts/update_roadmap_progress.ts +22 -4
- package/dist/agent-src/skills/reasoning-orchestrator/SKILL.md +33 -8
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +3 -3
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/install/install.mjs +24 -8
- package/dist/mcp/registry-manifest.json +2 -2
- package/docs/archive/CHANGELOG-7.0.0.md +419 -0
- package/docs/archive/CHANGELOG-pre-7.0.0.md +126 -0
- package/package.json +1 -1
- package/src/scripts/_lib/changelog_eras.ts +58 -0
- package/src/scripts/_lib/installed_lock.ts +47 -12
- package/src/scripts/rdp_gate_classify.ts +131 -0
- package/src/scripts/rdp_quality_eval.ts +437 -0
- package/src/scripts/release.ts +93 -42
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* RDP quality-layer + L6-isolation eval runner (TypeScript).
|
|
4
|
+
*
|
|
5
|
+
* Ports the Python `run_quality_eval.py` removed in the py2ts teardown. The
|
|
6
|
+
* Anthropic SDK is not a dependency in this repo (the TS trigger runner throws
|
|
7
|
+
* on the live path); this runner therefore calls the Messages API directly via
|
|
8
|
+
* Node's global `fetch` — no new dependency. It reuses the 0600-key gate, the
|
|
9
|
+
* cost-estimate, and the price tables exported by `skill_trigger_eval.ts`.
|
|
10
|
+
*
|
|
11
|
+
* Two modes (controlled system-prompt differential — the differing block is the
|
|
12
|
+
* only variable, so the delta isolates that block's effect):
|
|
13
|
+
* --mode quality : baseline (no RDP) vs treatment (+RDP layer) [L8]
|
|
14
|
+
* --mode l6 : distributed-only (RDP buffet) vs orchestrated (ordered) [L6]
|
|
15
|
+
*
|
|
16
|
+
* The measured model runs with the system prompt this runner supplies, so a
|
|
17
|
+
* baseline is NOT contaminated by the calling agent's own active rules.
|
|
18
|
+
*
|
|
19
|
+
* Optional `--score-with <model>` adds an independent model rater (rater 2):
|
|
20
|
+
* after capture, each variant transcript is scored 0–3 on the 4 rubric dims by
|
|
21
|
+
* a separate model call, recorded alongside the transcript.
|
|
22
|
+
*
|
|
23
|
+
* Cost discipline mirrors skill_trigger_eval.ts: dry-run by default; a billable
|
|
24
|
+
* run needs --confirm, and on a non-tty stdin also RDP_EVAL_ALLOW_NONTTY=1
|
|
25
|
+
* (set only when the caller has already confirmed the cost).
|
|
26
|
+
*
|
|
27
|
+
* Usage:
|
|
28
|
+
* ./scripts-run src/scripts/rdp_quality_eval --mode l6 --corpus <file>
|
|
29
|
+
* ./scripts-run src/scripts/rdp_quality_eval --mode l6 --corpus <file> --confirm --score-with claude-sonnet-4-5
|
|
30
|
+
*/
|
|
31
|
+
import * as fs from 'node:fs';
|
|
32
|
+
import * as path from 'node:path';
|
|
33
|
+
import { estimate_cost, load_anthropic_key } from './skill_trigger_eval.js';
|
|
34
|
+
|
|
35
|
+
/** stdout helper — the lint config forbids `console.log` (allows warn/error). */
|
|
36
|
+
const out = (s: string): void => {
|
|
37
|
+
process.stdout.write(`${s}\n`);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const REPO_ROOT = path.resolve(import.meta.dirname, '..', '..');
|
|
41
|
+
const EVAL_DIR = path.join(REPO_ROOT, 'tests', 'reasoning-layer-eval');
|
|
42
|
+
const GT_DIR = path.join(EVAL_DIR, 'golden-transcripts');
|
|
43
|
+
const DEFAULT_CORPUS = path.join(GT_DIR, 'corpus-prompts.json');
|
|
44
|
+
|
|
45
|
+
const ANTHROPIC_URL = 'https://api.anthropic.com/v1/messages';
|
|
46
|
+
const ANTHROPIC_VERSION = '2023-06-01';
|
|
47
|
+
|
|
48
|
+
// ---- the system-prompt blocks (verbatim port of the Python harness) ---------
|
|
49
|
+
const BASE_SYSTEM = `You are a senior software engineering assistant working inside a real codebase.
|
|
50
|
+
|
|
51
|
+
Operating posture:
|
|
52
|
+
- Keep diffs minimal and scoped to the stated task; no drive-by refactors.
|
|
53
|
+
- When a requirement is ambiguous, you may ask one focused clarifying question.
|
|
54
|
+
- Do not claim work is complete without the evidence that proves it.
|
|
55
|
+
- Be direct and concise. No flattery, no filler.`;
|
|
56
|
+
|
|
57
|
+
const RDP_BLOCK = `## Reasoning Discipline Protocol
|
|
58
|
+
|
|
59
|
+
Engage the protocol below only where it pays: skip it on trivial, short, fully
|
|
60
|
+
specified tasks (rename, one-liner, list files); apply it on complex, ambiguous,
|
|
61
|
+
multi-component, stateful, or irreversible tasks. If you are a strong-reasoning
|
|
62
|
+
model that already self-coordinates, apply it lightly.
|
|
63
|
+
|
|
64
|
+
1. GROUND BEFORE DESIGNING. Enumerate the constraints, available facts, and the
|
|
65
|
+
information gaps the task leaves open. Close the load-bearing gaps (by asking
|
|
66
|
+
or by stating an explicit assumption) BEFORE proposing a solution. Never design
|
|
67
|
+
against unstated assumptions.
|
|
68
|
+
|
|
69
|
+
2. INFER THE REAL GOAL. When the literal request and the underlying goal may
|
|
70
|
+
differ, state the inferred goal in one line, then give ONE recommendation —
|
|
71
|
+
not a spread of framings.
|
|
72
|
+
|
|
73
|
+
3. COMPLEXITY-FIRST SEQUENCING. For multi-step work, resolve the hardest /
|
|
74
|
+
most load-bearing unknown FIRST, before dependent work. Do not build the easy
|
|
75
|
+
parts first and rework later. Name what you would tackle first and why.
|
|
76
|
+
|
|
77
|
+
4. NOTES-FIRST OUTPUT. Keep multi-hypothesis reasoning, predictions, and
|
|
78
|
+
decisions in a clearly delimited "## Working notes" section. Your "## Answer"
|
|
79
|
+
section carries CONCLUSIONS + EVIDENCE only — never a raw chain-of-thought
|
|
80
|
+
dump. The answer must be readable by someone who saw none of the working
|
|
81
|
+
thread: outcome-first, no arrow-chain shorthand.
|
|
82
|
+
|
|
83
|
+
5. VERIFIER GATE (risky change). When the task shows two or more of {branching /
|
|
84
|
+
conditional logic, three or more explicit must/must-not constraints, stateful
|
|
85
|
+
operations, irreversibility}, explicitly name what must be verified and how
|
|
86
|
+
BEFORE treating the change as done. Surface the irreversible step for
|
|
87
|
+
confirmation rather than executing it blind.
|
|
88
|
+
|
|
89
|
+
6. PREDICTIONS + DECISIONS (calibration / ledger). When you make an estimate,
|
|
90
|
+
log it as a prediction with a confidence level so it can be checked against
|
|
91
|
+
the actual outcome later. When you choose between alternatives, record the
|
|
92
|
+
decision, the alternatives, the reason, and what would make you revisit it —
|
|
93
|
+
in the Working notes, so a later session can reuse it instead of re-deriving.
|
|
94
|
+
|
|
95
|
+
7. ADAPTIVE EFFORT. Scale effort to difficulty; stop when marginal evidence
|
|
96
|
+
drops rather than over-elaborating.`;
|
|
97
|
+
|
|
98
|
+
const ORCHESTRATOR_PREAMBLE = `## Reasoning orchestration (run the disciplines as ONE ordered chain)
|
|
99
|
+
|
|
100
|
+
Do not treat the protocol below as an optional buffet. Run it as a single
|
|
101
|
+
coordinated chain, in order, with explicit handoffs between links — a skipped or
|
|
102
|
+
out-of-order link compounds downstream:
|
|
103
|
+
|
|
104
|
+
ground → infer intent → write working notes → resolve the load-bearing unknown
|
|
105
|
+
first → audit progress against real evidence → verify before claiming done.
|
|
106
|
+
|
|
107
|
+
Coordinate the links; do not let later steps run before earlier ones. On a
|
|
108
|
+
trivial or fully-specified task, do NOT force the chain (that is over-process) —
|
|
109
|
+
engage it only where the task is genuinely complex/ambiguous/interdependent.`;
|
|
110
|
+
|
|
111
|
+
interface Slot {
|
|
112
|
+
n: string;
|
|
113
|
+
slug: string;
|
|
114
|
+
family?: string;
|
|
115
|
+
band: string;
|
|
116
|
+
discipline?: string;
|
|
117
|
+
mechanism?: string;
|
|
118
|
+
prompt: string;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
interface Variant {
|
|
122
|
+
model: string;
|
|
123
|
+
text: string;
|
|
124
|
+
input_tokens: number;
|
|
125
|
+
output_tokens: number;
|
|
126
|
+
cost_usd: number;
|
|
127
|
+
score?: Record<string, unknown>;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function buildArms(mode: string): Array<[string, string]> {
|
|
131
|
+
const treat = `${BASE_SYSTEM}\n\n${RDP_BLOCK}`;
|
|
132
|
+
if (mode === 'l6') {
|
|
133
|
+
return [
|
|
134
|
+
['distributed', treat],
|
|
135
|
+
['orchestrated', `${treat}\n\n${ORCHESTRATOR_PREAMBLE}`],
|
|
136
|
+
];
|
|
137
|
+
}
|
|
138
|
+
return [
|
|
139
|
+
['baseline', BASE_SYSTEM],
|
|
140
|
+
['treatment', treat],
|
|
141
|
+
];
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function approxTokens(text: string): number {
|
|
145
|
+
return Math.max(1, Math.floor(text.length / 4));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function modelFor(band: string, standard: string, strong: string): string {
|
|
149
|
+
return band === 'strong' ? strong : standard;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
interface ApiResult {
|
|
153
|
+
text: string;
|
|
154
|
+
input_tokens: number;
|
|
155
|
+
output_tokens: number;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function callAnthropic(
|
|
159
|
+
apiKey: string,
|
|
160
|
+
model: string,
|
|
161
|
+
system: string,
|
|
162
|
+
userContent: string,
|
|
163
|
+
maxTokens: number,
|
|
164
|
+
): Promise<ApiResult> {
|
|
165
|
+
const resp = await fetch(ANTHROPIC_URL, {
|
|
166
|
+
method: 'POST',
|
|
167
|
+
headers: {
|
|
168
|
+
'x-api-key': apiKey,
|
|
169
|
+
'anthropic-version': ANTHROPIC_VERSION,
|
|
170
|
+
'content-type': 'application/json',
|
|
171
|
+
},
|
|
172
|
+
body: JSON.stringify({
|
|
173
|
+
model,
|
|
174
|
+
max_tokens: maxTokens,
|
|
175
|
+
system,
|
|
176
|
+
messages: [{ role: 'user', content: userContent }],
|
|
177
|
+
}),
|
|
178
|
+
});
|
|
179
|
+
if (!resp.ok) {
|
|
180
|
+
const body = await resp.text();
|
|
181
|
+
throw new Error(`Anthropic API ${resp.status}: ${body.slice(0, 400)}`);
|
|
182
|
+
}
|
|
183
|
+
const data = (await resp.json()) as {
|
|
184
|
+
content?: Array<{ type?: string; text?: string }>;
|
|
185
|
+
usage?: { input_tokens?: number; output_tokens?: number };
|
|
186
|
+
};
|
|
187
|
+
const text = (data.content ?? [])
|
|
188
|
+
.filter((b) => b.type === 'text')
|
|
189
|
+
.map((b) => b.text ?? '')
|
|
190
|
+
.join('');
|
|
191
|
+
return {
|
|
192
|
+
text,
|
|
193
|
+
input_tokens: data.usage?.input_tokens ?? 0,
|
|
194
|
+
output_tokens: data.usage?.output_tokens ?? 0,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const SCORER_SYSTEM = `You are a strict, impartial evaluator. Score one assistant transcript against a
|
|
199
|
+
4-dimension rubric (0–3 each; 3 = excellent, 0 = absent/harmful). Dimensions:
|
|
200
|
+
1 notes-first adherence — reasoning kept out of the answer proper; answer = conclusions + evidence (not a raw chain-of-thought dump).
|
|
201
|
+
2 grounding — environment/constraints/info-gaps closed (or explicitly assumed) before designing; no jumping to a solution against assumptions.
|
|
202
|
+
3 premature-solution avoidance — the hardest/load-bearing unknown resolved before dependent work; no over-production of code/answer before grounding.
|
|
203
|
+
4 coherence / re-grounded summary — final answer readable by someone who saw none of the working thread; outcome-first; no arrow-chain shorthand.
|
|
204
|
+
Also report reasoning_extraction_refusal: true ONLY if the model refused due to a meta/extraction instruction.
|
|
205
|
+
Respond with STRICT JSON only, no prose: {"dim1":N,"dim2":N,"dim3":N,"dim4":N,"reasoning_extraction_refusal":false,"note":"<=120 chars"}`;
|
|
206
|
+
|
|
207
|
+
async function scoreTranscript(
|
|
208
|
+
apiKey: string,
|
|
209
|
+
scorerModel: string,
|
|
210
|
+
prompt: string,
|
|
211
|
+
transcript: string,
|
|
212
|
+
): Promise<Record<string, unknown> | null> {
|
|
213
|
+
const user = `## Task prompt given to the assistant\n${prompt}\n\n## Assistant transcript to score\n${transcript}\n\nScore now as strict JSON.`;
|
|
214
|
+
const r = await callAnthropic(apiKey, scorerModel, SCORER_SYSTEM, user, 300);
|
|
215
|
+
const m = r.text.match(/\{[\s\S]*\}/);
|
|
216
|
+
if (!m) return null;
|
|
217
|
+
try {
|
|
218
|
+
return JSON.parse(m[0]) as Record<string, unknown>;
|
|
219
|
+
} catch {
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function loadSlots(corpusPath: string, selected: Set<string> | null): Slot[] {
|
|
225
|
+
const data = JSON.parse(fs.readFileSync(corpusPath, 'utf-8')) as { slots: Slot[] };
|
|
226
|
+
let slots = data.slots;
|
|
227
|
+
if (selected) slots = slots.filter((s) => selected.has(s.n));
|
|
228
|
+
return slots;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function parseArgs(argv: string[]): Record<string, string | boolean> {
|
|
232
|
+
const a: Record<string, string | boolean> = {};
|
|
233
|
+
for (let i = 0; i < argv.length; i++) {
|
|
234
|
+
const t = argv[i];
|
|
235
|
+
if (t === undefined) continue;
|
|
236
|
+
if (t === '--confirm') a.confirm = true;
|
|
237
|
+
else if (t.startsWith('--')) {
|
|
238
|
+
const key = t.slice(2);
|
|
239
|
+
const next = argv[i + 1];
|
|
240
|
+
if (next !== undefined && !next.startsWith('--')) {
|
|
241
|
+
a[key] = next;
|
|
242
|
+
i++;
|
|
243
|
+
} else a[key] = true;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return a;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function meanDims(score: Record<string, unknown> | undefined): number | null {
|
|
250
|
+
if (!score) return null;
|
|
251
|
+
const ds = ['dim1', 'dim2', 'dim3', 'dim4'].map((k) => Number(score[k]));
|
|
252
|
+
if (ds.some((x) => Number.isNaN(x))) return null;
|
|
253
|
+
return ds.reduce((p, c) => p + c, 0) / 4;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function main(): Promise<number> {
|
|
257
|
+
const args = parseArgs(process.argv.slice(2));
|
|
258
|
+
const mode = (args.mode as string) ?? 'quality';
|
|
259
|
+
const corpusPath = (args.corpus as string) ?? DEFAULT_CORPUS;
|
|
260
|
+
const standardModel = (args['standard-model'] as string) ?? 'claude-haiku-4-5-20251001';
|
|
261
|
+
const strongModel = (args['strong-model'] as string) ?? 'claude-sonnet-4-5';
|
|
262
|
+
const maxTokens = args['max-tokens'] ? Number(args['max-tokens']) : 1600;
|
|
263
|
+
const scoreWith = args['score-with'] as string | undefined;
|
|
264
|
+
const selected = args.slots
|
|
265
|
+
? new Set(String(args.slots).split(',').map((s) => s.trim().padStart(2, '0')))
|
|
266
|
+
: null;
|
|
267
|
+
const resultsPath =
|
|
268
|
+
(args.results as string) ??
|
|
269
|
+
path.join(GT_DIR, mode === 'l6' ? 'l6-results.json' : 'results.json');
|
|
270
|
+
|
|
271
|
+
const slots = loadSlots(corpusPath, selected);
|
|
272
|
+
const arms = buildArms(mode);
|
|
273
|
+
const variantNames = arms.map((a) => a[0]);
|
|
274
|
+
|
|
275
|
+
// ---- cost preview --------------------------------------------------------
|
|
276
|
+
let total = 0;
|
|
277
|
+
const byModel: Record<string, number> = {};
|
|
278
|
+
for (const s of slots) {
|
|
279
|
+
const model = modelFor(s.band, standardModel, strongModel);
|
|
280
|
+
for (const [, sys] of arms) {
|
|
281
|
+
const tin = approxTokens(sys) + approxTokens(s.prompt);
|
|
282
|
+
const c = estimate_cost(model, tin, maxTokens);
|
|
283
|
+
total += c;
|
|
284
|
+
byModel[model] = (byModel[model] ?? 0) + c;
|
|
285
|
+
}
|
|
286
|
+
if (scoreWith) {
|
|
287
|
+
// 2 scoring calls per slot (one per variant)
|
|
288
|
+
const c = estimate_cost(scoreWith, 2000, 300) * 2;
|
|
289
|
+
total += c;
|
|
290
|
+
byModel[`${scoreWith} (scorer)`] = (byModel[`${scoreWith} (scorer)`] ?? 0) + c;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
const nCalls = slots.length * arms.length + (scoreWith ? slots.length * 2 : 0);
|
|
294
|
+
out(
|
|
295
|
+
`rdp-eval · mode=${mode} · ${slots.length} slots × ${arms.length} variants (${variantNames.join('/')}) ` +
|
|
296
|
+
`${scoreWith ? `+ scorer(${scoreWith})` : ''} = ${nCalls} calls`,
|
|
297
|
+
);
|
|
298
|
+
for (const [m, c] of Object.entries(byModel)) out(` ${m}: ~$${c.toFixed(4)}`);
|
|
299
|
+
out(` EXPECTED TOTAL (worst-case): ~$${total.toFixed(4)}`);
|
|
300
|
+
|
|
301
|
+
if (!args.confirm) {
|
|
302
|
+
out('\nDRY-RUN — no spend. Re-run with --confirm to capture transcripts.');
|
|
303
|
+
return 0;
|
|
304
|
+
}
|
|
305
|
+
if (!process.stdin.isTTY && process.env.RDP_EVAL_ALLOW_NONTTY !== '1') {
|
|
306
|
+
throw new Error(
|
|
307
|
+
'Refusing a billable run on non-tty stdin. Run interactively, or set ' +
|
|
308
|
+
'RDP_EVAL_ALLOW_NONTTY=1 if the caller has already confirmed the cost.',
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const apiKey = load_anthropic_key();
|
|
313
|
+
const ts = (args.date as string) ?? '0000-00-00';
|
|
314
|
+
let actual = 0;
|
|
315
|
+
const results: Array<Record<string, unknown>> = [];
|
|
316
|
+
|
|
317
|
+
for (const s of slots) {
|
|
318
|
+
const model = modelFor(s.band, standardModel, strongModel);
|
|
319
|
+
const variants: Record<string, Variant> = {};
|
|
320
|
+
for (const [variant, sys] of arms) {
|
|
321
|
+
console.error(` → slot ${s.n} ${variant} (${model}) …`);
|
|
322
|
+
const r = await callAnthropic(apiKey, model, sys, s.prompt, maxTokens);
|
|
323
|
+
const cost = estimate_cost(model, r.input_tokens, r.output_tokens);
|
|
324
|
+
actual += cost;
|
|
325
|
+
const v: Variant = {
|
|
326
|
+
model,
|
|
327
|
+
text: r.text,
|
|
328
|
+
input_tokens: r.input_tokens,
|
|
329
|
+
output_tokens: r.output_tokens,
|
|
330
|
+
cost_usd: cost,
|
|
331
|
+
};
|
|
332
|
+
if (scoreWith) {
|
|
333
|
+
console.error(` scoring ${variant} via ${scoreWith} …`);
|
|
334
|
+
const sc = await scoreTranscript(apiKey, scoreWith, s.prompt, r.text);
|
|
335
|
+
actual += estimate_cost(scoreWith, 2000, 300);
|
|
336
|
+
if (sc) v.score = sc;
|
|
337
|
+
}
|
|
338
|
+
variants[variant] = v;
|
|
339
|
+
}
|
|
340
|
+
const [v0, v1] = variantNames;
|
|
341
|
+
const va = v0 !== undefined ? variants[v0] : undefined;
|
|
342
|
+
const vb = v1 !== undefined ? variants[v1] : undefined;
|
|
343
|
+
if (v0 === undefined || v1 === undefined || va === undefined || vb === undefined) {
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
const aOut = va.output_tokens;
|
|
347
|
+
const bOut = vb.output_tokens;
|
|
348
|
+
const overhead = aOut ? Math.round(((bOut - aOut) / aOut) * 1000) / 10 : null;
|
|
349
|
+
const m0 = meanDims(va.score);
|
|
350
|
+
const m1 = meanDims(vb.score);
|
|
351
|
+
results.push({
|
|
352
|
+
slot: s.n,
|
|
353
|
+
slug: s.slug,
|
|
354
|
+
band: s.band,
|
|
355
|
+
mechanism: s.mechanism ?? null,
|
|
356
|
+
discipline: s.discipline ?? null,
|
|
357
|
+
prompt: s.prompt,
|
|
358
|
+
model,
|
|
359
|
+
variants,
|
|
360
|
+
output_token_overhead_pct: overhead,
|
|
361
|
+
rater2_mean: { [v0]: m0, [v1]: m1 },
|
|
362
|
+
rater2_delta: m0 !== null && m1 !== null ? Math.round((m1 - m0) * 100) / 100 : null,
|
|
363
|
+
});
|
|
364
|
+
writeTranscript(s, variants, ts, overhead, variantNames, mode);
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
fs.writeFileSync(
|
|
368
|
+
resultsPath,
|
|
369
|
+
JSON.stringify(
|
|
370
|
+
{
|
|
371
|
+
date: ts,
|
|
372
|
+
mode,
|
|
373
|
+
standard_model: standardModel,
|
|
374
|
+
strong_model: strongModel,
|
|
375
|
+
scorer_model: scoreWith ?? null,
|
|
376
|
+
actual_cost_usd: Math.round(actual * 10000) / 10000,
|
|
377
|
+
results,
|
|
378
|
+
},
|
|
379
|
+
null,
|
|
380
|
+
2,
|
|
381
|
+
),
|
|
382
|
+
'utf-8',
|
|
383
|
+
);
|
|
384
|
+
out(`\nDONE · mode=${mode} · actual ~$${(Math.round(actual * 10000) / 10000).toFixed(4)}`);
|
|
385
|
+
out(`Results JSON: ${resultsPath}`);
|
|
386
|
+
return 0;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function writeTranscript(
|
|
390
|
+
slot: Slot,
|
|
391
|
+
variants: Record<string, Variant>,
|
|
392
|
+
ts: string,
|
|
393
|
+
overhead: number | null,
|
|
394
|
+
variantNames: string[],
|
|
395
|
+
mode: string,
|
|
396
|
+
): void {
|
|
397
|
+
const prefix = mode === 'l6' ? 'l6n-' : '';
|
|
398
|
+
const p = path.join(GT_DIR, `${prefix}${slot.n}-${slot.slug}.md`);
|
|
399
|
+
const L: string[] = [
|
|
400
|
+
`# Transcript — slot ${slot.n}: ${slot.slug}`,
|
|
401
|
+
'',
|
|
402
|
+
`- **Band:** ${slot.band}${slot.mechanism ? ` · **Mechanism:** ${slot.mechanism}` : ''}`,
|
|
403
|
+
`- **Captured:** ${ts} (controlled system-prompt differential; rater 2 = model scorer)`,
|
|
404
|
+
'',
|
|
405
|
+
'## Prompt',
|
|
406
|
+
'',
|
|
407
|
+
slot.prompt,
|
|
408
|
+
'',
|
|
409
|
+
];
|
|
410
|
+
for (const variant of variantNames) {
|
|
411
|
+
const v = variants[variant];
|
|
412
|
+
if (v === undefined) continue;
|
|
413
|
+
const sc = v.score ? ` · rater2 ${JSON.stringify(v.score)}` : '';
|
|
414
|
+
L.push(
|
|
415
|
+
`## Transcript — ${variant} (${v.model})`,
|
|
416
|
+
'',
|
|
417
|
+
'~~~text',
|
|
418
|
+
v.text.trimEnd(),
|
|
419
|
+
'~~~',
|
|
420
|
+
'',
|
|
421
|
+
`**Tokens:** in ${v.input_tokens} / out ${v.output_tokens} / est $${v.cost_usd}${sc}`,
|
|
422
|
+
'',
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
if (overhead !== null) {
|
|
426
|
+
L.push(`**Output-token overhead (${variantNames[1]} vs ${variantNames[0]}):** ${overhead > 0 ? '+' : ''}${overhead}%`, '');
|
|
427
|
+
}
|
|
428
|
+
fs.writeFileSync(p, L.join('\n'), 'utf-8');
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
main().then(
|
|
432
|
+
(code) => process.exit(code),
|
|
433
|
+
(err) => {
|
|
434
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
435
|
+
process.exit(1);
|
|
436
|
+
},
|
|
437
|
+
);
|
package/src/scripts/release.ts
CHANGED
|
@@ -94,7 +94,7 @@ import type {
|
|
|
94
94
|
SplitPlan} from './_lib/changelog_eras.js';
|
|
95
95
|
import {
|
|
96
96
|
CURRENT_ERA_BODY_CAP,
|
|
97
|
-
|
|
97
|
+
current_era_accumulated_body_size,
|
|
98
98
|
current_era_insertion_point,
|
|
99
99
|
perform_split,
|
|
100
100
|
plan_split,
|
|
@@ -1014,9 +1014,11 @@ function print_preview(plan: Plan): void {
|
|
|
1014
1014
|
process.stdout.write('\n');
|
|
1015
1015
|
}
|
|
1016
1016
|
|
|
1017
|
-
function confirm(prompt: string): boolean {
|
|
1018
|
-
const ans = _input(`${prompt} [y/N] `)
|
|
1019
|
-
|
|
1017
|
+
function confirm(prompt: string): boolean | null {
|
|
1018
|
+
const ans = _input(`${prompt} [y/N] `);
|
|
1019
|
+
if (ans === null) return null; // no usable controlling terminal
|
|
1020
|
+
const norm = ans.trim().toLowerCase();
|
|
1021
|
+
return norm === 'y' || norm === 'yes';
|
|
1020
1022
|
}
|
|
1021
1023
|
|
|
1022
1024
|
/** Can we prompt at all — is fd 0 a TTY, or is a controlling terminal openable? */
|
|
@@ -1033,35 +1035,42 @@ function _canPrompt(): boolean {
|
|
|
1033
1035
|
}
|
|
1034
1036
|
|
|
1035
1037
|
/**
|
|
1036
|
-
* Mirror of Python `input(prompt)` — write prompt, read one line
|
|
1038
|
+
* Mirror of Python `input(prompt)` — write the prompt, read one line from the
|
|
1039
|
+
* controlling terminal. Returns `null` when no terminal is reachable.
|
|
1037
1040
|
*
|
|
1038
|
-
*
|
|
1039
|
-
*
|
|
1040
|
-
*
|
|
1041
|
-
* `
|
|
1042
|
-
*
|
|
1041
|
+
* The answer is ALWAYS read from a freshly opened, blocking `/dev/tty`
|
|
1042
|
+
* (`openSync(..., 'rs')`), NEVER from fd 0. `task release` / `./scripts-run`
|
|
1043
|
+
* run this under go-task's `interactive: true`, which leaves fd 0 a TTY
|
|
1044
|
+
* (`process.stdin.isTTY === true`) but one Node treats as NON-blocking — so a
|
|
1045
|
+
* bare `readSync(0)` throws `EAGAIN`, the catch swallows it, and the `[y/N]`
|
|
1046
|
+
* prompt "auto-aborts" without ever waiting (the exact failure this fixes;
|
|
1047
|
+
* reproduced with `isTTY === true`). A fresh blocking `/dev/tty` descriptor
|
|
1048
|
+
* blocks for real input regardless of how the script was invoked — the proven
|
|
1049
|
+
* synchronous-prompt pattern.
|
|
1043
1050
|
*/
|
|
1044
|
-
function _input(prompt: string): string {
|
|
1051
|
+
function _input(prompt: string): string | null {
|
|
1045
1052
|
process.stdout.write(prompt);
|
|
1046
|
-
let fd
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
return ''; // no controlling terminal (true non-interactive); guarded by _canPrompt upstream.
|
|
1054
|
-
}
|
|
1053
|
+
let fd: number;
|
|
1054
|
+
try {
|
|
1055
|
+
// 'rs' → O_RDONLY | O_SYNC: a fresh, blocking descriptor on the
|
|
1056
|
+
// controlling terminal, unaffected by Node's non-blocking fd 0.
|
|
1057
|
+
fd = fs.openSync('/dev/tty', 'rs');
|
|
1058
|
+
} catch {
|
|
1059
|
+
return null; // no controlling terminal (true non-interactive)
|
|
1055
1060
|
}
|
|
1056
1061
|
try {
|
|
1057
1062
|
const buf = Buffer.alloc(1);
|
|
1058
1063
|
const chars: number[] = [];
|
|
1059
|
-
|
|
1064
|
+
for (;;) {
|
|
1060
1065
|
let bytesRead: number;
|
|
1061
1066
|
try {
|
|
1062
1067
|
bytesRead = fs.readSync(fd, buf, 0, 1, null);
|
|
1063
|
-
} catch {
|
|
1064
|
-
|
|
1068
|
+
} catch (e) {
|
|
1069
|
+
const code = (e as NodeJS.ErrnoException).code;
|
|
1070
|
+
// A blocking /dev/tty should not yield these, but inherited
|
|
1071
|
+
// descriptor flags can — retry rather than abort the prompt.
|
|
1072
|
+
if (code === 'EAGAIN' || code === 'EINTR') continue;
|
|
1073
|
+
break; // EOF / EIO → EOFError analogue; return what we have.
|
|
1065
1074
|
}
|
|
1066
1075
|
if (bytesRead === 0) break;
|
|
1067
1076
|
const b = buf[0] as number;
|
|
@@ -1070,7 +1079,7 @@ function _input(prompt: string): string {
|
|
|
1070
1079
|
}
|
|
1071
1080
|
return Buffer.from(chars).toString('utf-8');
|
|
1072
1081
|
} finally {
|
|
1073
|
-
|
|
1082
|
+
fs.closeSync(fd);
|
|
1074
1083
|
}
|
|
1075
1084
|
}
|
|
1076
1085
|
|
|
@@ -1090,18 +1099,19 @@ export interface ConfirmVerdict {
|
|
|
1090
1099
|
*/
|
|
1091
1100
|
export function confirmGate(target: string, yes: boolean): ConfirmVerdict {
|
|
1092
1101
|
if (yes) return { proceed: true };
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1102
|
+
const noTerminal: ConfirmVerdict = {
|
|
1103
|
+
proceed: false,
|
|
1104
|
+
stream: 'stderr',
|
|
1105
|
+
message:
|
|
1106
|
+
'No terminal available for the [y/N] confirmation (non-interactive shell). ' +
|
|
1107
|
+
'Re-run with --yes to confirm, e.g. `task release -- --yes`.',
|
|
1108
|
+
};
|
|
1109
|
+
if (!_canPrompt()) return noTerminal;
|
|
1110
|
+
const answer = confirm(`Proceed with release ${target}?`);
|
|
1111
|
+
// null = the controlling terminal vanished between the probe and the read;
|
|
1112
|
+
// surface the actionable --yes guidance, never a bare silent "aborted.".
|
|
1113
|
+
if (answer === null) return noTerminal;
|
|
1114
|
+
if (!answer) return { proceed: false, stream: 'stdout', message: 'aborted.' };
|
|
1105
1115
|
return { proceed: true };
|
|
1106
1116
|
}
|
|
1107
1117
|
|
|
@@ -1382,6 +1392,7 @@ interface Args {
|
|
|
1382
1392
|
dry_run: boolean;
|
|
1383
1393
|
no_wait: boolean;
|
|
1384
1394
|
resume: boolean;
|
|
1395
|
+
check_confirm: boolean;
|
|
1385
1396
|
}
|
|
1386
1397
|
|
|
1387
1398
|
const PROG = 'release.py';
|
|
@@ -1391,7 +1402,7 @@ const PROG = 'release.py';
|
|
|
1391
1402
|
// not the body prose.
|
|
1392
1403
|
const USAGE =
|
|
1393
1404
|
`usage: ${PROG} [-h] [--as {major,minor,patch}] [--version EXPLICIT] [--yes]\n` +
|
|
1394
|
-
' [--dry-run] [--no-wait] [--resume]\n';
|
|
1405
|
+
' [--dry-run] [--no-wait] [--resume] [--check-confirm]\n';
|
|
1395
1406
|
|
|
1396
1407
|
function _argError(msg: string): never {
|
|
1397
1408
|
process.stderr.write(USAGE);
|
|
@@ -1407,6 +1418,7 @@ function _parse_args(argv: readonly string[]): Args {
|
|
|
1407
1418
|
dry_run: false,
|
|
1408
1419
|
no_wait: false,
|
|
1409
1420
|
resume: false,
|
|
1421
|
+
check_confirm: false,
|
|
1410
1422
|
};
|
|
1411
1423
|
|
|
1412
1424
|
const positionals: string[] = [];
|
|
@@ -1481,6 +1493,13 @@ function _parse_args(argv: readonly string[]): Args {
|
|
|
1481
1493
|
i += 1;
|
|
1482
1494
|
continue;
|
|
1483
1495
|
}
|
|
1496
|
+
if (flag === '--check-confirm') {
|
|
1497
|
+
if (inlineVal !== null)
|
|
1498
|
+
_argError(`argument --check-confirm: ignored explicit argument '${inlineVal}'`);
|
|
1499
|
+
args.check_confirm = true;
|
|
1500
|
+
i += 1;
|
|
1501
|
+
continue;
|
|
1502
|
+
}
|
|
1484
1503
|
if (a.startsWith('-') && a !== '-') {
|
|
1485
1504
|
_argError(`unrecognized arguments: ${a}`);
|
|
1486
1505
|
}
|
|
@@ -1560,6 +1579,32 @@ function _detect_in_flight_target(): string | null {
|
|
|
1560
1579
|
function main(argv: readonly string[] | null = null): number {
|
|
1561
1580
|
const args = _parse_args(argv === null ? process.argv.slice(2) : argv);
|
|
1562
1581
|
|
|
1582
|
+
// Self-test the confirmation prompt without touching git/gh/npm. Run it the
|
|
1583
|
+
// same way a real release is invoked — `task release -- --check-confirm` —
|
|
1584
|
+
// to verify the `[y/N]` prompt can actually read your answer through the
|
|
1585
|
+
// go-task / scripts-run wrapper chain. Nothing is released either way.
|
|
1586
|
+
if (args.check_confirm) {
|
|
1587
|
+
process.stdout.write(
|
|
1588
|
+
'Confirm-gate self-test — answer the [y/N] prompt below.\n' +
|
|
1589
|
+
'Nothing is released; this only checks that the prompt can read your answer.\n',
|
|
1590
|
+
);
|
|
1591
|
+
const verdict = confirmGate('SELF-TEST (no release)', args.yes);
|
|
1592
|
+
if (verdict.proceed) {
|
|
1593
|
+
process.stdout.write(
|
|
1594
|
+
'result: OK — the prompt read your confirmation. ' +
|
|
1595
|
+
'`task release` will work interactively.\n',
|
|
1596
|
+
);
|
|
1597
|
+
return 0;
|
|
1598
|
+
}
|
|
1599
|
+
const sink = verdict.stream === 'stderr' ? process.stderr : process.stdout;
|
|
1600
|
+
if (verdict.message) sink.write(`${verdict.message}\n`);
|
|
1601
|
+
process.stdout.write(
|
|
1602
|
+
`result: the prompt did not read a "y" (stdin.isTTY=${String(process.stdin.isTTY)}). ` +
|
|
1603
|
+
'If you typed y and still see this, paste this whole block back.\n',
|
|
1604
|
+
);
|
|
1605
|
+
return 1;
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1563
1608
|
const current = (
|
|
1564
1609
|
JSON.parse(fs.readFileSync(PACKAGE_JSON, 'utf-8')) as Record<string, unknown>
|
|
1565
1610
|
)['version'] as string;
|
|
@@ -1596,12 +1641,18 @@ function main(argv: readonly string[] | null = null): number {
|
|
|
1596
1641
|
test_trend_line,
|
|
1597
1642
|
});
|
|
1598
1643
|
|
|
1599
|
-
// Era-split planning: only crosses the gate when the current era
|
|
1600
|
-
//
|
|
1601
|
-
//
|
|
1602
|
-
//
|
|
1644
|
+
// Era-split planning: only crosses the gate when the current era's
|
|
1645
|
+
// *accumulated* body (prior releases, excluding the newest section) has
|
|
1646
|
+
// grown past the drift cap AND the release crosses a minor/major boundary.
|
|
1647
|
+
// The newest release is exempt — a single large catch-up release forces
|
|
1648
|
+
// the split on the *next* minor/major once it has become a prior entry,
|
|
1649
|
+
// not on the patch that immediately follows it. This is the same quantity
|
|
1650
|
+
// the drift test enforces (current_era_accumulated_body_size); measuring
|
|
1651
|
+
// the raw total here would re-fire the gate the exemption exists to clear.
|
|
1652
|
+
// Patch overflow that genuinely accumulates is caught by the drift test
|
|
1653
|
+
// (red CI), not by an auto-split into a nonsensical "pre-X.Y.Z" archive.
|
|
1603
1654
|
let split: SplitPlan | null = null;
|
|
1604
|
-
const body_size =
|
|
1655
|
+
const body_size = current_era_accumulated_body_size();
|
|
1605
1656
|
if (body_size > CURRENT_ERA_BODY_CAP) {
|
|
1606
1657
|
const candidate = plan_split(target);
|
|
1607
1658
|
if (candidate === null) {
|