@tekyzinc/gsd-t 3.29.10 → 4.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +176 -0
- package/bin/gsd-t-parallel.cjs +64 -13
- package/bin/gsd-t-test-data-adapters/file-json-array.cjs +27 -3
- package/bin/gsd-t-test-data-adapters/localstorage-key-prefix.cjs +6 -1
- package/bin/gsd-t-test-data-adapters/sqlite-table-where.cjs +16 -1
- package/bin/gsd-t-test-data-ledger.cjs +1 -0
- package/bin/gsd-t.js +48 -376
- package/bin/journey-coverage.cjs +6 -3
- package/bin/parallel-cli.cjs +13 -1
- package/commands/gsd-t-debug.md +47 -533
- package/commands/gsd-t-design-decompose.md +24 -497
- package/commands/gsd-t-doc-ripple.md +23 -139
- package/commands/gsd-t-execute.md +41 -958
- package/commands/gsd-t-help.md +0 -2
- package/commands/gsd-t-impact.md +26 -295
- package/commands/gsd-t-integrate.md +44 -369
- package/commands/gsd-t-milestone.md +25 -124
- package/commands/gsd-t-partition.md +28 -539
- package/commands/gsd-t-plan.md +26 -472
- package/commands/gsd-t-prd.md +25 -311
- package/commands/gsd-t-quick.md +1 -1
- package/commands/gsd-t-resume.md +0 -33
- package/commands/gsd-t-verify.md +52 -569
- package/commands/gsd-t-wave.md +41 -430
- package/package.json +1 -1
- package/scripts/gsd-t-calibration-hook.js +3 -1
- package/scripts/hooks/pre-commit-journey-coverage +0 -2
- package/scripts/hooks/pre-commit-playwright-gate +0 -2
- package/templates/CLAUDE-global.md +43 -173
- package/templates/CLAUDE-project.md +118 -104
- package/templates/prompts/design-verify-subagent.md +3 -0
- package/templates/prompts/qa-subagent.md +3 -0
- package/templates/prompts/red-team-subagent.md +5 -2
- package/templates/workflows/_lib.js +176 -0
- package/templates/workflows/gsd-t-debug.workflow.js +86 -0
- package/templates/workflows/gsd-t-execute.workflow.js +206 -0
- package/templates/workflows/gsd-t-integrate.workflow.js +71 -0
- package/templates/workflows/gsd-t-phase.workflow.js +94 -0
- package/templates/workflows/gsd-t-quick.workflow.js +72 -0
- package/templates/workflows/gsd-t-verify.workflow.js +348 -0
- package/templates/workflows/gsd-t-wave.workflow.js +43 -0
- package/bin/check-headless-sessions.js +0 -140
- package/bin/context-budget-audit.cjs +0 -447
- package/bin/context-meter-config.cjs +0 -101
- package/bin/context-meter-config.test.cjs +0 -101
- package/bin/event-stream.cjs +0 -205
- package/bin/gsd-t-benchmark-orchestrator.js +0 -437
- package/bin/gsd-t-capture-lint.cjs +0 -440
- package/bin/gsd-t-economics.cjs +0 -315
- package/bin/gsd-t-in-session-usage.cjs +0 -213
- package/bin/gsd-t-orchestrator-config.cjs +0 -161
- package/bin/gsd-t-orchestrator-queue.cjs +0 -180
- package/bin/gsd-t-orchestrator-recover.cjs +0 -231
- package/bin/gsd-t-orchestrator-worker.cjs +0 -219
- package/bin/gsd-t-orchestrator.js +0 -535
- package/bin/gsd-t-parallel-probe.cjs +0 -132
- package/bin/gsd-t-ratelimit-probe-worker.cjs +0 -237
- package/bin/gsd-t-ratelimit-probe.cjs +0 -648
- package/bin/gsd-t-report-tokens.cjs +0 -549
- package/bin/gsd-t-stream-feed-client.cjs +0 -151
- package/bin/gsd-t-token-backfill.cjs +0 -366
- package/bin/gsd-t-token-capture.cjs +0 -321
- package/bin/gsd-t-token-dashboard.cjs +0 -353
- package/bin/gsd-t-token-regenerate-log.cjs +0 -129
- package/bin/gsd-t-tool-attribution.cjs +0 -377
- package/bin/gsd-t-tool-cost.cjs +0 -195
- package/bin/gsd-t-transcript-tee.cjs +0 -246
- package/bin/gsd-t-unattended-heartbeat.cjs +0 -188
- package/bin/gsd-t-unattended-platform.cjs +0 -551
- package/bin/gsd-t-unattended-platform.js +0 -381
- package/bin/gsd-t-unattended-safety.cjs +0 -773
- package/bin/gsd-t-unattended-safety.js +0 -788
- package/bin/gsd-t-unattended.cjs +0 -2109
- package/bin/gsd-t-unattended.js +0 -1367
- package/bin/gsd-t-worker-dispatch.cjs +0 -211
- package/bin/handoff-lock.cjs +0 -249
- package/bin/handoff-lock.js +0 -249
- package/bin/headless-auto-spawn.cjs +0 -711
- package/bin/headless-auto-spawn.js +0 -373
- package/bin/headless-exit-codes.cjs +0 -67
- package/bin/live-activity-report.cjs +0 -615
- package/bin/log-tail.cjs +0 -81
- package/bin/m44-proof-measure.cjs +0 -285
- package/bin/m46-iter-proof.cjs +0 -149
- package/bin/m46-worker-proof.cjs +0 -201
- package/bin/m55-substrate-proof.cjs +0 -134
- package/bin/metrics-rollup.js +0 -200
- package/bin/model-windows.cjs +0 -99
- package/bin/model-windows.test.cjs +0 -75
- package/bin/parallelism-report.cjs +0 -535
- package/bin/runway-estimator.cjs +0 -242
- package/bin/spawn-plan-derive.cjs +0 -163
- package/bin/spawn-plan-status-updater.cjs +0 -292
- package/bin/spawn-plan-writer.cjs +0 -204
- package/bin/supervisor-pid-fingerprint.cjs +0 -126
- package/bin/token-budget.cjs +0 -265
- package/bin/unattended-watch-format.cjs +0 -178
- package/bin/watch-progress.js +0 -155
- package/commands/gsd-t-unattended-stop.md +0 -85
- package/commands/gsd-t-unattended-watch.md +0 -465
- package/commands/gsd-t-unattended.md +0 -476
- package/commands/gsd-t-visualize.md +0 -116
- package/scripts/gsd-t-context-meter.e2e.test.js +0 -364
- package/scripts/gsd-t-context-meter.js +0 -341
- package/scripts/gsd-t-context-meter.test.js +0 -471
- package/scripts/gsd-t-dashboard-server.js +0 -1203
- package/scripts/gsd-t-post-commit-spawn-plan.sh +0 -86
- package/scripts/gsd-t-transcript.html +0 -1821
- package/scripts/hooks/gsd-t-conversation-capture.js +0 -439
- package/scripts/hooks/gsd-t-in-session-usage-hook.js +0 -84
- package/scripts/spawn-plan-fmt-tokens.cjs +0 -80
- package/templates/hooks/post-commit-spawn-plan.sh +0 -85
package/bin/event-stream.cjs
DELETED
|
@@ -1,205 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* bin/event-stream.cjs
|
|
3
|
-
*
|
|
4
|
-
* Structured event stream for the unattended supervisor watch-tick (M38 ES).
|
|
5
|
-
* Workers append events to `.gsd-t/events/YYYY-MM-DD.jsonl`; watch tick reads
|
|
6
|
-
* new events since the persisted cursor.
|
|
7
|
-
*
|
|
8
|
-
* Contract: `.gsd-t/contracts/unattended-event-stream-contract.md` v1.0.0.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
"use strict";
|
|
12
|
-
|
|
13
|
-
const fs = require("fs");
|
|
14
|
-
const path = require("path");
|
|
15
|
-
|
|
16
|
-
const EVENTS_DIR_REL = path.join(".gsd-t", "events");
|
|
17
|
-
const CURSOR_PATH_REL = path.join(".gsd-t", ".unattended", "event-cursor");
|
|
18
|
-
|
|
19
|
-
function eventsDir(projectDir) {
|
|
20
|
-
return path.join(projectDir, EVENTS_DIR_REL);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function cursorPath(projectDir) {
|
|
24
|
-
return path.join(projectDir, CURSOR_PATH_REL);
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
function todayFileDate(now) {
|
|
28
|
-
const d = now || new Date();
|
|
29
|
-
const y = d.getUTCFullYear();
|
|
30
|
-
const m = String(d.getUTCMonth() + 1).padStart(2, "0");
|
|
31
|
-
const day = String(d.getUTCDate()).padStart(2, "0");
|
|
32
|
-
return `${y}-${m}-${day}`;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function eventFileFor(projectDir, fileDate) {
|
|
36
|
-
return path.join(eventsDir(projectDir), `${fileDate}.jsonl`);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Append one event to today's jsonl file. Atomic via append-only write with
|
|
41
|
-
* a newline-terminated record. Never throws — failures are logged to stderr.
|
|
42
|
-
*
|
|
43
|
-
* @param {string} projectDir
|
|
44
|
-
* @param {object} eventObj — `type`, `iter`, `source`, plus type-specific fields
|
|
45
|
-
* @returns {boolean} true on success, false on failure
|
|
46
|
-
*/
|
|
47
|
-
function appendEvent(projectDir, eventObj) {
|
|
48
|
-
try {
|
|
49
|
-
if (!eventObj || typeof eventObj !== "object") return false;
|
|
50
|
-
const ev = Object.assign({}, eventObj);
|
|
51
|
-
if (typeof ev.ts !== "string" || !ev.ts) {
|
|
52
|
-
ev.ts = new Date().toISOString();
|
|
53
|
-
}
|
|
54
|
-
const dir = eventsDir(projectDir);
|
|
55
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
56
|
-
const fileDate = todayFileDate(new Date(ev.ts));
|
|
57
|
-
const file = eventFileFor(projectDir, fileDate);
|
|
58
|
-
const line = JSON.stringify(ev) + "\n";
|
|
59
|
-
fs.appendFileSync(file, line, "utf8");
|
|
60
|
-
return true;
|
|
61
|
-
} catch (err) {
|
|
62
|
-
try {
|
|
63
|
-
process.stderr.write(`[event-stream] appendEvent failed: ${err.message}\n`);
|
|
64
|
-
} catch (_) { /* ignore */ }
|
|
65
|
-
return false;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function readCursor(projectDir) {
|
|
70
|
-
try {
|
|
71
|
-
const raw = fs.readFileSync(cursorPath(projectDir), "utf8");
|
|
72
|
-
const parsed = JSON.parse(raw);
|
|
73
|
-
if (parsed && typeof parsed.fileDate === "string" && Number.isFinite(parsed.offset)) {
|
|
74
|
-
return parsed;
|
|
75
|
-
}
|
|
76
|
-
} catch (_) { /* missing or malformed → treated as uninitialized */ }
|
|
77
|
-
return null;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
function writeCursor(projectDir, cursor) {
|
|
81
|
-
try {
|
|
82
|
-
const p = cursorPath(projectDir);
|
|
83
|
-
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
84
|
-
const tmp = `${p}.tmp`;
|
|
85
|
-
fs.writeFileSync(tmp, JSON.stringify(cursor), "utf8");
|
|
86
|
-
fs.renameSync(tmp, p);
|
|
87
|
-
return true;
|
|
88
|
-
} catch (err) {
|
|
89
|
-
try {
|
|
90
|
-
process.stderr.write(`[event-stream] writeCursor failed: ${err.message}\n`);
|
|
91
|
-
} catch (_) { /* ignore */ }
|
|
92
|
-
return false;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function fileSize(p) {
|
|
97
|
-
try {
|
|
98
|
-
return fs.statSync(p).size;
|
|
99
|
-
} catch (_) {
|
|
100
|
-
return 0;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
function parseJsonlSlice(buf) {
|
|
105
|
-
const text = buf.toString("utf8");
|
|
106
|
-
const lines = text.split("\n");
|
|
107
|
-
const hasPartialTail = text.length > 0 && !text.endsWith("\n");
|
|
108
|
-
const consumable = hasPartialTail ? lines.slice(0, -1) : lines;
|
|
109
|
-
const events = [];
|
|
110
|
-
for (const line of consumable) {
|
|
111
|
-
if (!line) continue;
|
|
112
|
-
try {
|
|
113
|
-
events.push(JSON.parse(line));
|
|
114
|
-
} catch (_) {
|
|
115
|
-
try {
|
|
116
|
-
process.stderr.write(`[event-stream] skipping malformed line\n`);
|
|
117
|
-
} catch (__) { /* ignore */ }
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
const consumedBytes = hasPartialTail
|
|
121
|
-
? Buffer.byteLength(consumable.join("\n") + (consumable.length ? "\n" : ""), "utf8")
|
|
122
|
-
: buf.length;
|
|
123
|
-
return { events, consumedBytes };
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
function readSlice(filePath, startOffset) {
|
|
127
|
-
const size = fileSize(filePath);
|
|
128
|
-
if (size <= startOffset) return { events: [], consumedBytes: 0, fileSize: size };
|
|
129
|
-
const fd = fs.openSync(filePath, "r");
|
|
130
|
-
try {
|
|
131
|
-
const length = size - startOffset;
|
|
132
|
-
const buf = Buffer.alloc(length);
|
|
133
|
-
fs.readSync(fd, buf, 0, length, startOffset);
|
|
134
|
-
const { events, consumedBytes } = parseJsonlSlice(buf);
|
|
135
|
-
return { events, consumedBytes, fileSize: size };
|
|
136
|
-
} finally {
|
|
137
|
-
try { fs.closeSync(fd); } catch (_) { /* ignore */ }
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/**
|
|
142
|
-
* Read all events from cursor to EOF. Handles day-boundary per contract §3:
|
|
143
|
-
* if cursor points to a past file date, read remaining events from that file,
|
|
144
|
-
* then read today's file from byte 0; advance cursor to today's EOF.
|
|
145
|
-
*
|
|
146
|
-
* If cursor is missing, initialize to today's file at current EOF (no backlog
|
|
147
|
-
* surfaced on first read — matches contract §3 initial-cursor behavior).
|
|
148
|
-
*
|
|
149
|
-
* @param {string} projectDir
|
|
150
|
-
* @returns {{events: object[], newCursor: {fileDate: string, offset: number}}}
|
|
151
|
-
*/
|
|
152
|
-
function readSinceCursor(projectDir) {
|
|
153
|
-
const today = todayFileDate(new Date());
|
|
154
|
-
let cursor = readCursor(projectDir);
|
|
155
|
-
if (!cursor) {
|
|
156
|
-
const todayFile = eventFileFor(projectDir, today);
|
|
157
|
-
const size = fileSize(todayFile);
|
|
158
|
-
const newCursor = { fileDate: today, offset: size };
|
|
159
|
-
writeCursor(projectDir, newCursor);
|
|
160
|
-
return { events: [], newCursor };
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
const result = [];
|
|
164
|
-
let { fileDate, offset } = cursor;
|
|
165
|
-
|
|
166
|
-
if (fileDate !== today) {
|
|
167
|
-
const prev = eventFileFor(projectDir, fileDate);
|
|
168
|
-
if (fs.existsSync(prev)) {
|
|
169
|
-
const slice = readSlice(prev, offset);
|
|
170
|
-
result.push(...slice.events);
|
|
171
|
-
}
|
|
172
|
-
fileDate = today;
|
|
173
|
-
offset = 0;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
const todayFile = eventFileFor(projectDir, today);
|
|
177
|
-
if (fs.existsSync(todayFile)) {
|
|
178
|
-
const slice = readSlice(todayFile, offset);
|
|
179
|
-
result.push(...slice.events);
|
|
180
|
-
offset = offset + slice.consumedBytes;
|
|
181
|
-
} else {
|
|
182
|
-
return { events: result, newCursor: { fileDate, offset } };
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
return { events: result, newCursor: { fileDate, offset } };
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
/**
|
|
189
|
-
* Persist the cursor returned by readSinceCursor.
|
|
190
|
-
* @param {string} projectDir
|
|
191
|
-
* @param {{fileDate: string, offset: number}} newCursor
|
|
192
|
-
*/
|
|
193
|
-
function advanceCursor(projectDir, newCursor) {
|
|
194
|
-
if (!newCursor || typeof newCursor.fileDate !== "string" || !Number.isFinite(newCursor.offset)) {
|
|
195
|
-
return false;
|
|
196
|
-
}
|
|
197
|
-
return writeCursor(projectDir, newCursor);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
module.exports = {
|
|
201
|
-
appendEvent,
|
|
202
|
-
readSinceCursor,
|
|
203
|
-
advanceCursor,
|
|
204
|
-
_internal: { todayFileDate, eventsDir, cursorPath, readCursor, writeCursor },
|
|
205
|
-
};
|
|
@@ -1,437 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
'use strict';
|
|
3
|
-
|
|
4
|
-
// D0-T2 — M40 speed-benchmark gate driver.
|
|
5
|
-
// Compares wall-clock time for two ways of executing the same fixed workload:
|
|
6
|
-
// (a) orchestrator path — one `claude -p` spawn per task, wave-join loop
|
|
7
|
-
// driven by bin/gsd-t-orchestrator.js
|
|
8
|
-
// (b) in-session path — a single `claude -p` session that runs the same
|
|
9
|
-
// tasks sequentially (the honest "one Claude Code
|
|
10
|
-
// window" comparison)
|
|
11
|
-
// Contract: .gsd-t/contracts/wave-join-contract.md v1.0.0
|
|
12
|
-
|
|
13
|
-
const fs = require('fs');
|
|
14
|
-
const os = require('os');
|
|
15
|
-
const path = require('path');
|
|
16
|
-
const { spawnSync } = require('child_process');
|
|
17
|
-
|
|
18
|
-
const PKG_ROOT = path.resolve(__dirname, '..');
|
|
19
|
-
const DEFAULT_FIXTURE = path.join(PKG_ROOT, 'test', 'fixtures', 'm40-benchmark-workload');
|
|
20
|
-
const PASS_TOLERANCE = 1.05;
|
|
21
|
-
|
|
22
|
-
function parseCliArgs(argv) {
|
|
23
|
-
const args = {
|
|
24
|
-
runs: 3,
|
|
25
|
-
reportPath: null,
|
|
26
|
-
resultsPath: null,
|
|
27
|
-
fixtureDir: DEFAULT_FIXTURE,
|
|
28
|
-
mockClaude: null,
|
|
29
|
-
projectDir: process.cwd(),
|
|
30
|
-
keepTmp: false,
|
|
31
|
-
help: false,
|
|
32
|
-
};
|
|
33
|
-
for (let i = 0; i < argv.length; i++) {
|
|
34
|
-
const a = argv[i];
|
|
35
|
-
if (a === '-h' || a === '--help') { args.help = true; }
|
|
36
|
-
else if (a === '--runs') { args.runs = parseInt(argv[++i], 10); }
|
|
37
|
-
else if (a === '--report-path') { args.reportPath = argv[++i]; }
|
|
38
|
-
else if (a === '--results-path') { args.resultsPath = argv[++i]; }
|
|
39
|
-
else if (a === '--fixture-dir') { args.fixtureDir = path.resolve(argv[++i]); }
|
|
40
|
-
else if (a === '--mock-claude') { args.mockClaude = path.resolve(argv[++i]); }
|
|
41
|
-
else if (a === '--project-dir') { args.projectDir = path.resolve(argv[++i]); }
|
|
42
|
-
else if (a === '--keep-tmp') { args.keepTmp = true; }
|
|
43
|
-
}
|
|
44
|
-
if (!Number.isInteger(args.runs) || args.runs < 1) {
|
|
45
|
-
throw new Error('--runs must be a positive integer');
|
|
46
|
-
}
|
|
47
|
-
return args;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function printHelp() {
|
|
51
|
-
process.stdout.write([
|
|
52
|
-
'Usage: gsd-t benchmark-orchestrator [options]',
|
|
53
|
-
'',
|
|
54
|
-
'Runs the M40 speed-benchmark kill-switch gate: compares orchestrator',
|
|
55
|
-
'path vs in-session path on a fixed workload and emits a verdict.',
|
|
56
|
-
'',
|
|
57
|
-
'Options:',
|
|
58
|
-
' --runs <n> Number of runs per side (default 3).',
|
|
59
|
-
' --report-path <path> Human-readable report (default docs/m40-benchmark-report.md).',
|
|
60
|
-
' --results-path <path> Machine-readable JSON (default .gsd-t/benchmark-results.json).',
|
|
61
|
-
' --fixture-dir <path> Override benchmark workload fixture directory.',
|
|
62
|
-
' --mock-claude <path> Use this binary as `claude` (for smoke tests).',
|
|
63
|
-
' --project-dir <path> Project directory to write outputs into (default cwd).',
|
|
64
|
-
' --keep-tmp Preserve per-run tmp directories for diagnosis.',
|
|
65
|
-
' -h, --help Show this help.',
|
|
66
|
-
'',
|
|
67
|
-
'Verdict is PASS when median(orchestrator_ms) <= median(in-session_ms) * 1.05.',
|
|
68
|
-
'',
|
|
69
|
-
].join('\n'));
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function median(nums) {
|
|
73
|
-
if (!nums.length) return 0;
|
|
74
|
-
const sorted = [...nums].sort((a, b) => a - b);
|
|
75
|
-
const mid = Math.floor(sorted.length / 2);
|
|
76
|
-
return sorted.length % 2 ? sorted[mid] : Math.round((sorted[mid - 1] + sorted[mid]) / 2);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function copyDirSync(src, dst) {
|
|
80
|
-
fs.mkdirSync(dst, { recursive: true });
|
|
81
|
-
for (const entry of fs.readdirSync(src, { withFileTypes: true })) {
|
|
82
|
-
const s = path.join(src, entry.name);
|
|
83
|
-
const d = path.join(dst, entry.name);
|
|
84
|
-
if (entry.isDirectory()) copyDirSync(s, d);
|
|
85
|
-
else if (entry.isSymbolicLink()) fs.symlinkSync(fs.readlinkSync(s), d);
|
|
86
|
-
else fs.copyFileSync(s, d);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
function gitInitRepo(dir) {
|
|
91
|
-
const opts = { cwd: dir, stdio: 'ignore' };
|
|
92
|
-
spawnSync('git', ['init', '-q', '-b', 'main'], opts);
|
|
93
|
-
spawnSync('git', ['config', 'user.email', 'bench@gsd-t.local'], opts);
|
|
94
|
-
spawnSync('git', ['config', 'user.name', 'benchmark'], opts);
|
|
95
|
-
spawnSync('git', ['config', 'commit.gpgsign', 'false'], opts);
|
|
96
|
-
spawnSync('git', ['add', '-A'], opts);
|
|
97
|
-
spawnSync('git', ['commit', '-q', '-m', 'benchmark fixture baseline'], opts);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
function prepareRun(fixtureDir, label, runIdx) {
|
|
101
|
-
const tmpBase = fs.mkdtempSync(path.join(os.tmpdir(), `m40-bench-${label}-${runIdx}-`));
|
|
102
|
-
const dest = path.join(tmpBase, 'workload');
|
|
103
|
-
copyDirSync(fixtureDir, dest);
|
|
104
|
-
gitInitRepo(dest);
|
|
105
|
-
return { tmpBase, dest };
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
function cleanupRun(tmpBase) {
|
|
109
|
-
try { fs.rmSync(tmpBase, { recursive: true, force: true }); } catch (_) { /* best effort */ }
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
function buildChildEnv(mockClaude) {
|
|
113
|
-
const env = { ...process.env };
|
|
114
|
-
if (mockClaude) env.GSD_T_CLAUDE_BIN = mockClaude;
|
|
115
|
-
return env;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
function captureOutput(res) {
|
|
119
|
-
const parts = [];
|
|
120
|
-
if (res && res.stdout) parts.push('--- stdout ---\n' + String(res.stdout));
|
|
121
|
-
if (res && res.stderr) parts.push('--- stderr ---\n' + String(res.stderr));
|
|
122
|
-
return parts.join('\n').slice(-4096);
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function runOrchestratorSide({ fixtureDir, runIdx, mockClaude, logger, keepTmp = false, spawnImpl = spawnSync }) {
|
|
126
|
-
const { tmpBase, dest } = prepareRun(fixtureDir, 'orch', runIdx);
|
|
127
|
-
const bin = path.join(__dirname, 'gsd-t-orchestrator.js');
|
|
128
|
-
const t0 = Date.now();
|
|
129
|
-
let exitCode = null;
|
|
130
|
-
let output = '';
|
|
131
|
-
try {
|
|
132
|
-
const res = spawnImpl(process.execPath, [
|
|
133
|
-
bin,
|
|
134
|
-
'--milestone', 'M40-bench',
|
|
135
|
-
'--project-dir', dest,
|
|
136
|
-
'--max-parallel', '8',
|
|
137
|
-
'--worker-timeout', '180000',
|
|
138
|
-
], {
|
|
139
|
-
env: buildChildEnv(mockClaude),
|
|
140
|
-
encoding: 'utf8',
|
|
141
|
-
timeout: 900000,
|
|
142
|
-
});
|
|
143
|
-
exitCode = res.status;
|
|
144
|
-
output = captureOutput(res);
|
|
145
|
-
} catch (e) {
|
|
146
|
-
output = 'spawn_error: ' + (e && e.message);
|
|
147
|
-
}
|
|
148
|
-
const durationMs = Date.now() - t0;
|
|
149
|
-
const commitAudit = auditTaskCommits(dest);
|
|
150
|
-
if (logger) logger.log(`[bench orch #${runIdx}] exit=${exitCode} duration=${durationMs}ms commits=${commitAudit.taskCommitCount}/${commitAudit.expectedTaskCount}`);
|
|
151
|
-
if (keepTmp && logger) logger.log(` kept: ${tmpBase}`);
|
|
152
|
-
if (!keepTmp) cleanupRun(tmpBase);
|
|
153
|
-
return {
|
|
154
|
-
durationMs,
|
|
155
|
-
exitCode,
|
|
156
|
-
stderr: output,
|
|
157
|
-
tmpDir: keepTmp ? tmpBase : null,
|
|
158
|
-
commitAudit
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
function runInsessionSide({ fixtureDir, runIdx, mockClaude, logger, keepTmp = false, spawnImpl = spawnSync }) {
|
|
163
|
-
const { tmpBase, dest } = prepareRun(fixtureDir, 'insession', runIdx);
|
|
164
|
-
const claudeBin = mockClaude || process.env.GSD_T_CLAUDE_BIN || 'claude';
|
|
165
|
-
const prompt = [
|
|
166
|
-
'# In-session equivalent — M40 benchmark control',
|
|
167
|
-
'',
|
|
168
|
-
'You are simulating a single Claude Code window running /gsd-t-execute',
|
|
169
|
-
'against this fixture sequentially (no external orchestrator).',
|
|
170
|
-
'',
|
|
171
|
-
`Project dir: ${dest}`,
|
|
172
|
-
'',
|
|
173
|
-
'## Required discipline (enforces parity with orchestrator path)',
|
|
174
|
-
'',
|
|
175
|
-
'For the benchmark to be a fair comparison, you MUST apply the same',
|
|
176
|
-
'per-task discipline the external orchestrator enforces on its workers.',
|
|
177
|
-
'Shortcuts that collapse multiple tasks into one batch INVALIDATE the',
|
|
178
|
-
'benchmark.',
|
|
179
|
-
'',
|
|
180
|
-
'1. Read `.gsd-t/contracts/completion-signal-contract.md` once at the start.',
|
|
181
|
-
'2. Read every `.gsd-t/domains/*/tasks.md` to enumerate the tasks.',
|
|
182
|
-
'3. Process tasks strictly in wave order: ALL wave 0 before ANY wave 1,',
|
|
183
|
-
' ALL wave 1 before ANY wave 2, etc. Within a wave you may interleave',
|
|
184
|
-
' tasks but each task MUST complete its own commit before the next',
|
|
185
|
-
' task begins writing files.',
|
|
186
|
-
'4. For EACH task, in order:',
|
|
187
|
-
' a. Read the task body from its domain tasks.md',
|
|
188
|
-
' b. Create/modify ONLY the files listed in that task\'s `**Files**:` field',
|
|
189
|
-
' c. Run `npm test` and verify it passes',
|
|
190
|
-
' d. `git add` ONLY that task\'s owned files',
|
|
191
|
-
' e. `git commit` with a message starting with the canonical task id',
|
|
192
|
-
' (e.g. `bench-d1-t1: …`) on branch `main`',
|
|
193
|
-
' f. Add a Decision Log entry to `.gsd-t/progress.md` naming the task id',
|
|
194
|
-
' g. Commit the progress.md update as part of (e) OR as a follow-up',
|
|
195
|
-
' commit whose message also starts with the same task id',
|
|
196
|
-
'',
|
|
197
|
-
'## Hard rules',
|
|
198
|
-
'- ONE commit per task minimum. Twenty tasks → at least twenty commits',
|
|
199
|
-
' beyond the baseline commit. DO NOT bulk-commit multiple tasks.',
|
|
200
|
-
'- DO NOT write files for task N+1 before task N is committed.',
|
|
201
|
-
'- DO NOT spawn subagents — you are the single-session baseline.',
|
|
202
|
-
'- DO NOT push to any git remote (no `git push`).',
|
|
203
|
-
'- If you finish early, STOP. Do not run extra work.',
|
|
204
|
-
'',
|
|
205
|
-
'This is the baseline the external orchestrator is being compared to.',
|
|
206
|
-
'The comparison is only meaningful if both sides obey the same rules.',
|
|
207
|
-
'',
|
|
208
|
-
].join('\n');
|
|
209
|
-
const t0 = Date.now();
|
|
210
|
-
let exitCode = null;
|
|
211
|
-
let output = '';
|
|
212
|
-
try {
|
|
213
|
-
const res = spawnImpl(claudeBin, [
|
|
214
|
-
'-p',
|
|
215
|
-
'--dangerously-skip-permissions',
|
|
216
|
-
'--output-format', 'stream-json',
|
|
217
|
-
'--verbose',
|
|
218
|
-
'--model', 'sonnet',
|
|
219
|
-
], {
|
|
220
|
-
cwd: dest,
|
|
221
|
-
env: buildChildEnv(mockClaude),
|
|
222
|
-
input: prompt,
|
|
223
|
-
encoding: 'utf8',
|
|
224
|
-
timeout: 900000,
|
|
225
|
-
});
|
|
226
|
-
exitCode = res.status;
|
|
227
|
-
output = captureOutput(res);
|
|
228
|
-
} catch (e) {
|
|
229
|
-
output = 'spawn_error: ' + (e && e.message);
|
|
230
|
-
}
|
|
231
|
-
const durationMs = Date.now() - t0;
|
|
232
|
-
const commitAudit = auditTaskCommits(dest);
|
|
233
|
-
if (logger) logger.log(`[bench insession #${runIdx}] exit=${exitCode} duration=${durationMs}ms commits=${commitAudit.taskCommitCount}/${commitAudit.expectedTaskCount}`);
|
|
234
|
-
if (keepTmp && logger) logger.log(` kept: ${tmpBase}`);
|
|
235
|
-
if (!keepTmp) cleanupRun(tmpBase);
|
|
236
|
-
return {
|
|
237
|
-
durationMs,
|
|
238
|
-
exitCode,
|
|
239
|
-
stderr: output,
|
|
240
|
-
tmpDir: keepTmp ? tmpBase : null,
|
|
241
|
-
commitAudit
|
|
242
|
-
};
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
function auditTaskCommits(workloadDir) {
|
|
246
|
-
const { spawnSync: ss } = require('child_process');
|
|
247
|
-
const res = ss('git', ['-C', workloadDir, 'log', '--pretty=%s'], { encoding: 'utf8' });
|
|
248
|
-
const subjects = String(res.stdout || '').split('\n').filter(Boolean);
|
|
249
|
-
const taskIdRe = /^bench-d\d+-t\d+[:\s]/;
|
|
250
|
-
const taskCommits = subjects.filter((s) => taskIdRe.test(s));
|
|
251
|
-
const uniqueTaskIds = new Set(taskCommits.map((s) => s.split(/[:\s]/)[0]));
|
|
252
|
-
const expectedTaskCount = 20;
|
|
253
|
-
return {
|
|
254
|
-
totalCommits: subjects.length,
|
|
255
|
-
taskCommitCount: taskCommits.length,
|
|
256
|
-
uniqueTaskIds: uniqueTaskIds.size,
|
|
257
|
-
expectedTaskCount,
|
|
258
|
-
discipline: uniqueTaskIds.size >= expectedTaskCount ? 'compliant' : 'noncompliant'
|
|
259
|
-
};
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
function collectEnv() {
|
|
263
|
-
return {
|
|
264
|
-
node: process.version,
|
|
265
|
-
platform: `${process.platform}-${os.release()}`,
|
|
266
|
-
arch: process.arch,
|
|
267
|
-
cpuCount: os.cpus().length,
|
|
268
|
-
totalMemMb: Math.round(os.totalmem() / (1024 * 1024)),
|
|
269
|
-
freeMemMb: Math.round(os.freemem() / (1024 * 1024)),
|
|
270
|
-
};
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
function renderReportMd(results) {
|
|
274
|
-
const lines = [];
|
|
275
|
-
lines.push('# M40 Speed Benchmark — Gate Verdict');
|
|
276
|
-
lines.push('');
|
|
277
|
-
lines.push(`- **Generated**: ${results.generatedAt}`);
|
|
278
|
-
lines.push(`- **Runs per side**: ${results.runs}`);
|
|
279
|
-
lines.push(`- **Fixture**: ${results.fixtureDir}`);
|
|
280
|
-
lines.push(`- **Verdict**: **${results.verdict}** — ${results.verdictDetail}`);
|
|
281
|
-
lines.push('');
|
|
282
|
-
lines.push('## Environment');
|
|
283
|
-
lines.push('');
|
|
284
|
-
lines.push(`- Node: ${results.env.node}`);
|
|
285
|
-
lines.push(`- Platform: ${results.env.platform} (${results.env.arch})`);
|
|
286
|
-
lines.push(`- CPUs: ${results.env.cpuCount}`);
|
|
287
|
-
lines.push(`- RAM: ${results.env.freeMemMb} MB free / ${results.env.totalMemMb} MB total`);
|
|
288
|
-
lines.push('');
|
|
289
|
-
lines.push('## Per-run timings (ms) and commit-discipline audit');
|
|
290
|
-
lines.push('');
|
|
291
|
-
lines.push('| # | Orchestrator (ms / exit / commits) | In-session (ms / exit / commits) |');
|
|
292
|
-
lines.push('|---|------------------------------------|----------------------------------|');
|
|
293
|
-
for (let i = 0; i < results.runs; i++) {
|
|
294
|
-
const o = results.orchestrator[i] || {};
|
|
295
|
-
const s = results.insession[i] || {};
|
|
296
|
-
const oc = o.commitAudit || {};
|
|
297
|
-
const sc = s.commitAudit || {};
|
|
298
|
-
const oCommits = `${oc.uniqueTaskIds ?? '—'}/${oc.expectedTaskCount ?? '—'}`;
|
|
299
|
-
const sCommits = `${sc.uniqueTaskIds ?? '—'}/${sc.expectedTaskCount ?? '—'}`;
|
|
300
|
-
lines.push(`| ${i + 1} | ${o.durationMs ?? '—'} / ${o.exitCode ?? '—'} / ${oCommits} | ${s.durationMs ?? '—'} / ${s.exitCode ?? '—'} / ${sCommits} |`);
|
|
301
|
-
}
|
|
302
|
-
lines.push('');
|
|
303
|
-
lines.push(`- **Median orchestrator**: ${results.summary.medianOrchMs} ms`);
|
|
304
|
-
lines.push(`- **Median in-session**: ${results.summary.medianInsessionMs} ms`);
|
|
305
|
-
lines.push(`- **Threshold** (insession × ${PASS_TOLERANCE}): ${results.summary.thresholdMs} ms`);
|
|
306
|
-
lines.push('');
|
|
307
|
-
lines.push('## Methodology');
|
|
308
|
-
lines.push('');
|
|
309
|
-
lines.push('- Same fixture (`test/fixtures/m40-benchmark-workload/`) copied to a fresh');
|
|
310
|
-
lines.push(' tmp dir per run; git initialized; no cross-run state.');
|
|
311
|
-
lines.push('- Orchestrator path: `bin/gsd-t-orchestrator.js` drives waves via the');
|
|
312
|
-
lines.push(' D1 spawn loop + D2 brief builder.');
|
|
313
|
-
lines.push('- In-session path: a single `claude -p` session handed the tasks');
|
|
314
|
-
lines.push(' sequentially — no subagents.');
|
|
315
|
-
lines.push('- `Date.now()` wall-clock, millisecond precision. Both sides include');
|
|
316
|
-
lines.push(' their full lifecycle (startup + work + teardown).');
|
|
317
|
-
lines.push('- PASS when `median(orchestrator_ms) ≤ median(in-session_ms) × 1.05`.');
|
|
318
|
-
lines.push('');
|
|
319
|
-
return lines.join('\n');
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
function computeVerdict(orchTimings, insessionTimings) {
|
|
323
|
-
const orchOk = orchTimings.length && orchTimings.every((r) => r.exitCode === 0);
|
|
324
|
-
const insOk = insessionTimings.length && insessionTimings.every((r) => r.exitCode === 0);
|
|
325
|
-
const orchCompliant = orchTimings.every((r) => !r.commitAudit || r.commitAudit.discipline === 'compliant');
|
|
326
|
-
const insCompliant = insessionTimings.every((r) => !r.commitAudit || r.commitAudit.discipline === 'compliant');
|
|
327
|
-
const medianOrchMs = median(orchTimings.map((r) => r.durationMs));
|
|
328
|
-
const medianInsessionMs = median(insessionTimings.map((r) => r.durationMs));
|
|
329
|
-
const thresholdMs = Math.round(medianInsessionMs * PASS_TOLERANCE);
|
|
330
|
-
let verdict = 'FAIL';
|
|
331
|
-
let verdictDetail;
|
|
332
|
-
if (!orchOk || !insOk) {
|
|
333
|
-
verdictDetail = `one or more runs failed (orchestrator_ok=${orchOk}, insession_ok=${insOk}) — cannot trust comparison — M40 HALT RECOMMENDED`;
|
|
334
|
-
} else if (!orchCompliant || !insCompliant) {
|
|
335
|
-
verdict = 'INVALID';
|
|
336
|
-
verdictDetail = `commit-discipline audit failed (orchestrator_compliant=${orchCompliant}, insession_compliant=${insCompliant}) — one side did not produce one-commit-per-task; timing comparison is meaningless until both sides obey the same rules`;
|
|
337
|
-
} else if (medianOrchMs <= thresholdMs) {
|
|
338
|
-
verdict = 'PASS';
|
|
339
|
-
verdictDetail = `orchestrator ${medianOrchMs}ms ≤ in-session ${medianInsessionMs}ms × ${PASS_TOLERANCE} (${thresholdMs}ms) — Waves 2+3+4 unlocked`;
|
|
340
|
-
} else {
|
|
341
|
-
verdictDetail = `orchestrator ${medianOrchMs}ms > in-session ${medianInsessionMs}ms × ${PASS_TOLERANCE} (${thresholdMs}ms) — M40 HALT RECOMMENDED`;
|
|
342
|
-
}
|
|
343
|
-
return { verdict, verdictDetail, medianOrchMs, medianInsessionMs, thresholdMs, orchCompliant, insCompliant };
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
async function runBenchmark(opts) {
|
|
347
|
-
const {
|
|
348
|
-
runs,
|
|
349
|
-
fixtureDir,
|
|
350
|
-
mockClaude,
|
|
351
|
-
projectDir,
|
|
352
|
-
reportPath,
|
|
353
|
-
resultsPath,
|
|
354
|
-
keepTmp = false,
|
|
355
|
-
logger = console,
|
|
356
|
-
orchestratorImpl = runOrchestratorSide,
|
|
357
|
-
insessionImpl = runInsessionSide,
|
|
358
|
-
} = opts;
|
|
359
|
-
|
|
360
|
-
if (!fs.existsSync(fixtureDir)) {
|
|
361
|
-
throw new Error(`Fixture dir not found: ${fixtureDir}`);
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
const orchestrator = [];
|
|
365
|
-
const insession = [];
|
|
366
|
-
for (let i = 1; i <= runs; i++) {
|
|
367
|
-
orchestrator.push(orchestratorImpl({ fixtureDir, runIdx: i, mockClaude, logger, keepTmp }));
|
|
368
|
-
insession.push(insessionImpl({ fixtureDir, runIdx: i, mockClaude, logger, keepTmp }));
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
const summary = computeVerdict(orchestrator, insession);
|
|
372
|
-
const results = {
|
|
373
|
-
schemaVersion: 1,
|
|
374
|
-
generatedAt: new Date().toISOString(),
|
|
375
|
-
runs,
|
|
376
|
-
fixtureDir,
|
|
377
|
-
env: collectEnv(),
|
|
378
|
-
orchestrator,
|
|
379
|
-
insession,
|
|
380
|
-
summary,
|
|
381
|
-
verdict: summary.verdict,
|
|
382
|
-
verdictDetail: summary.verdictDetail,
|
|
383
|
-
};
|
|
384
|
-
|
|
385
|
-
const rp = resultsPath || path.join(projectDir, '.gsd-t', 'benchmark-results.json');
|
|
386
|
-
const rrp = reportPath || path.join(projectDir, 'docs', 'm40-benchmark-report.md');
|
|
387
|
-
fs.mkdirSync(path.dirname(rp), { recursive: true });
|
|
388
|
-
fs.mkdirSync(path.dirname(rrp), { recursive: true });
|
|
389
|
-
fs.writeFileSync(rp, JSON.stringify(results, null, 2));
|
|
390
|
-
fs.writeFileSync(rrp, renderReportMd(results));
|
|
391
|
-
|
|
392
|
-
const verdictLine = results.verdict === 'PASS'
|
|
393
|
-
? `BENCHMARK: PASS — orchestrator ${summary.medianOrchMs}ms vs in-session ${summary.medianInsessionMs}ms — Waves 2+3+4 unlocked`
|
|
394
|
-
: `BENCHMARK: FAIL — orchestrator ${summary.medianOrchMs}ms vs in-session ${summary.medianInsessionMs}ms — M40 HALT RECOMMENDED`;
|
|
395
|
-
logger.log('');
|
|
396
|
-
logger.log(verdictLine);
|
|
397
|
-
logger.log(` Results: ${rp}`);
|
|
398
|
-
logger.log(` Report: ${rrp}`);
|
|
399
|
-
|
|
400
|
-
return { results, resultsPath: rp, reportPath: rrp };
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
async function main() {
|
|
404
|
-
let args;
|
|
405
|
-
try { args = parseCliArgs(process.argv.slice(2)); }
|
|
406
|
-
catch (e) { process.stderr.write(`Error: ${e.message}\n\n`); printHelp(); process.exit(2); }
|
|
407
|
-
if (args.help) { printHelp(); process.exit(0); }
|
|
408
|
-
|
|
409
|
-
try {
|
|
410
|
-
const { results } = await runBenchmark(args);
|
|
411
|
-
process.exit(results.verdict === 'PASS' ? 0 : 1);
|
|
412
|
-
} catch (e) {
|
|
413
|
-
process.stderr.write(`benchmark-orchestrator failed: ${e && e.message}\n`);
|
|
414
|
-
if (e && e.stack) process.stderr.write(e.stack + '\n');
|
|
415
|
-
process.exit(2);
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
if (require.main === module) {
|
|
420
|
-
main();
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
module.exports = {
|
|
424
|
-
parseCliArgs,
|
|
425
|
-
median,
|
|
426
|
-
copyDirSync,
|
|
427
|
-
gitInitRepo,
|
|
428
|
-
prepareRun,
|
|
429
|
-
cleanupRun,
|
|
430
|
-
collectEnv,
|
|
431
|
-
renderReportMd,
|
|
432
|
-
computeVerdict,
|
|
433
|
-
runOrchestratorSide,
|
|
434
|
-
runInsessionSide,
|
|
435
|
-
runBenchmark,
|
|
436
|
-
PASS_TOLERANCE,
|
|
437
|
-
};
|