@tekyzinc/gsd-t 3.13.16 → 3.16.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +1 -0
- package/bin/gsd-t-benchmark-orchestrator.js +437 -0
- package/bin/gsd-t-capture-lint.cjs +276 -0
- package/bin/gsd-t-completion-check.cjs +106 -0
- package/bin/gsd-t-orchestrator-config.cjs +64 -0
- package/bin/gsd-t-orchestrator-queue.cjs +180 -0
- package/bin/gsd-t-orchestrator-recover.cjs +231 -0
- package/bin/gsd-t-orchestrator-worker.cjs +219 -0
- package/bin/gsd-t-orchestrator.js +534 -0
- package/bin/gsd-t-stream-feed-client.cjs +151 -0
- package/bin/gsd-t-task-brief-compactor.cjs +89 -0
- package/bin/gsd-t-task-brief-template.cjs +96 -0
- package/bin/gsd-t-task-brief.js +249 -0
- package/bin/gsd-t-token-backfill.cjs +366 -0
- package/bin/gsd-t-token-capture.cjs +306 -0
- package/bin/gsd-t-token-dashboard.cjs +318 -0
- package/bin/gsd-t-token-regenerate-log.cjs +129 -0
- package/bin/gsd-t-transcript-tee.cjs +246 -0
- package/bin/gsd-t-unattended-heartbeat.cjs +188 -0
- package/bin/gsd-t-unattended-platform.cjs +191 -27
- package/bin/gsd-t-unattended-safety.cjs +8 -1
- package/bin/gsd-t-unattended.cjs +192 -31
- package/bin/gsd-t.js +329 -2
- package/bin/supervisor-pid-fingerprint.cjs +126 -0
- package/commands/gsd-t-debug.md +63 -51
- package/commands/gsd-t-design-decompose.md +2 -7
- package/commands/gsd-t-doc-ripple.md +20 -11
- package/commands/gsd-t-execute.md +82 -50
- package/commands/gsd-t-integrate.md +43 -16
- package/commands/gsd-t-plan.md +20 -7
- package/commands/gsd-t-prd.md +19 -12
- package/commands/gsd-t-quick.md +64 -29
- package/commands/gsd-t-resume.md +51 -4
- package/commands/gsd-t-unattended.md +19 -20
- package/commands/gsd-t-verify.md +48 -32
- package/commands/gsd-t-visualize.md +19 -17
- package/commands/gsd-t-wave.md +29 -27
- package/docs/architecture.md +16 -0
- package/docs/m40-benchmark-report.md +35 -0
- package/docs/requirements.md +20 -0
- package/package.json +1 -1
- package/scripts/gsd-t-dashboard-server.js +291 -4
- package/scripts/gsd-t-dashboard.html +31 -1
- package/scripts/gsd-t-design-review-server.js +3 -1
- package/scripts/gsd-t-stream-feed-server.js +428 -0
- package/scripts/gsd-t-stream-feed.html +1168 -0
- package/scripts/gsd-t-token-aggregator.js +373 -0
- package/scripts/gsd-t-transcript.html +422 -0
- package/scripts/hooks/gsd-t-in-session-probe.js +62 -0
- package/scripts/hooks/pre-commit-capture-lint +26 -0
- package/templates/CLAUDE-global.md +69 -0
- package/scripts/gsd-t-agent-dashboard-server.js +0 -424
- package/scripts/gsd-t-agent-dashboard.html +0 -1043
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* GSD-T Token Dashboard (M41 D4)
|
|
4
|
+
*
|
|
5
|
+
* Cumulative historical view of `.gsd-t/metrics/token-usage.jsonl`.
|
|
6
|
+
* Feeds the `gsd-t tokens` CLI and the token-block tail of `gsd-t status`.
|
|
7
|
+
*
|
|
8
|
+
* Zero external deps. `.cjs` for ESM/CJS compat.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const fs = require('fs');
|
|
12
|
+
const path = require('path');
|
|
13
|
+
const readline = require('readline');
|
|
14
|
+
|
|
15
|
+
const DEFAULT_JSONL_PATH = (projectDir) => path.join(projectDir, '.gsd-t', 'metrics', 'token-usage.jsonl');
|
|
16
|
+
|
|
17
|
+
function _safeParse(line) {
|
|
18
|
+
try { return JSON.parse(line); } catch (_) { return null; }
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function _day(startedAt) {
|
|
22
|
+
if (!startedAt) return 'unknown';
|
|
23
|
+
// YYYY-MM-DD slice works for both ISO and our 'YYYY-MM-DD HH:MM' format.
|
|
24
|
+
return String(startedAt).slice(0, 10);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function _cost(r) {
|
|
28
|
+
return (typeof r.costUSD === 'number' && r.costUSD >= 0) ? r.costUSD : 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Aggregate token-usage records.
|
|
33
|
+
*
|
|
34
|
+
* @param {object} opts
|
|
35
|
+
* @param {string} opts.projectDir
|
|
36
|
+
* @param {string} [opts.since] YYYY-MM-DD (inclusive)
|
|
37
|
+
* @param {string} [opts.milestone] e.g. 'M41'
|
|
38
|
+
* @returns {Promise<object>}
|
|
39
|
+
*/
|
|
40
|
+
async function aggregate(opts) {
|
|
41
|
+
const projectDir = opts.projectDir || '.';
|
|
42
|
+
const jsonlPath = opts.jsonlPath || DEFAULT_JSONL_PATH(projectDir);
|
|
43
|
+
const sinceDay = opts.since || null;
|
|
44
|
+
const milestone = opts.milestone || null;
|
|
45
|
+
|
|
46
|
+
const agg = {
|
|
47
|
+
totalRecords: 0,
|
|
48
|
+
totalCostUSD: 0,
|
|
49
|
+
totalInputTokens: 0,
|
|
50
|
+
totalOutputTokens: 0,
|
|
51
|
+
totalCacheReadTokens: 0,
|
|
52
|
+
totalCacheCreateTokens: 0,
|
|
53
|
+
byDay: {},
|
|
54
|
+
byCommand: {},
|
|
55
|
+
byModel: {},
|
|
56
|
+
topSpawns: [],
|
|
57
|
+
rolling7d: { days: 0, totalCostUSD: 0, dailyAvgUSD: 0, monthlyProjectionUSD: 0 },
|
|
58
|
+
currentMilestone: milestone,
|
|
59
|
+
source: jsonlPath,
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
if (!fs.existsSync(jsonlPath)) {
|
|
63
|
+
return agg;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const rs = fs.createReadStream(jsonlPath, { encoding: 'utf8' });
|
|
67
|
+
const rl = readline.createInterface({ input: rs, crlfDelay: Infinity });
|
|
68
|
+
|
|
69
|
+
const rawRecords = [];
|
|
70
|
+
|
|
71
|
+
for await (const line of rl) {
|
|
72
|
+
const trimmed = line && line.trim();
|
|
73
|
+
if (!trimmed) continue;
|
|
74
|
+
const r = _safeParse(trimmed);
|
|
75
|
+
if (!r || typeof r !== 'object') continue;
|
|
76
|
+
|
|
77
|
+
const day = _day(r.startedAt);
|
|
78
|
+
if (sinceDay && day < sinceDay) continue;
|
|
79
|
+
if (milestone && r.milestone && r.milestone !== milestone) continue;
|
|
80
|
+
if (milestone && !r.milestone) continue;
|
|
81
|
+
|
|
82
|
+
rawRecords.push(r);
|
|
83
|
+
|
|
84
|
+
agg.totalRecords += 1;
|
|
85
|
+
agg.totalCostUSD += _cost(r);
|
|
86
|
+
agg.totalInputTokens += Number(r.inputTokens || 0);
|
|
87
|
+
agg.totalOutputTokens += Number(r.outputTokens || 0);
|
|
88
|
+
agg.totalCacheReadTokens += Number(r.cacheReadInputTokens || 0);
|
|
89
|
+
agg.totalCacheCreateTokens += Number(r.cacheCreationInputTokens || 0);
|
|
90
|
+
|
|
91
|
+
// byDay
|
|
92
|
+
if (!agg.byDay[day]) agg.byDay[day] = { day, records: 0, costUSD: 0, inputTokens: 0, outputTokens: 0 };
|
|
93
|
+
const d = agg.byDay[day];
|
|
94
|
+
d.records += 1;
|
|
95
|
+
d.costUSD += _cost(r);
|
|
96
|
+
d.inputTokens += Number(r.inputTokens || 0);
|
|
97
|
+
d.outputTokens += Number(r.outputTokens || 0);
|
|
98
|
+
|
|
99
|
+
// byCommand
|
|
100
|
+
const cmd = r.command || 'unknown';
|
|
101
|
+
if (!agg.byCommand[cmd]) agg.byCommand[cmd] = { command: cmd, records: 0, costUSD: 0, inputTokens: 0, outputTokens: 0 };
|
|
102
|
+
const c = agg.byCommand[cmd];
|
|
103
|
+
c.records += 1;
|
|
104
|
+
c.costUSD += _cost(r);
|
|
105
|
+
c.inputTokens += Number(r.inputTokens || 0);
|
|
106
|
+
c.outputTokens += Number(r.outputTokens || 0);
|
|
107
|
+
|
|
108
|
+
// byModel
|
|
109
|
+
const m = r.model || 'unknown';
|
|
110
|
+
if (!agg.byModel[m]) agg.byModel[m] = { model: m, records: 0, costUSD: 0, inputTokens: 0, cacheReadTokens: 0, cacheHitRate: 0 };
|
|
111
|
+
const mm = agg.byModel[m];
|
|
112
|
+
mm.records += 1;
|
|
113
|
+
mm.costUSD += _cost(r);
|
|
114
|
+
mm.inputTokens += Number(r.inputTokens || 0);
|
|
115
|
+
mm.cacheReadTokens += Number(r.cacheReadInputTokens || 0);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Top 10 spawns by cost desc
|
|
119
|
+
agg.topSpawns = rawRecords
|
|
120
|
+
.slice()
|
|
121
|
+
.sort((a, b) => _cost(b) - _cost(a))
|
|
122
|
+
.slice(0, 10)
|
|
123
|
+
.map((r) => ({
|
|
124
|
+
startedAt: r.startedAt,
|
|
125
|
+
command: r.command,
|
|
126
|
+
step: r.step,
|
|
127
|
+
model: r.model,
|
|
128
|
+
costUSD: _cost(r),
|
|
129
|
+
inputTokens: r.inputTokens || 0,
|
|
130
|
+
outputTokens: r.outputTokens || 0,
|
|
131
|
+
}));
|
|
132
|
+
|
|
133
|
+
// Cache-hit rate per model
|
|
134
|
+
for (const key of Object.keys(agg.byModel)) {
|
|
135
|
+
const mm = agg.byModel[key];
|
|
136
|
+
const denom = mm.inputTokens + mm.cacheReadTokens;
|
|
137
|
+
mm.cacheHitRate = denom > 0 ? mm.cacheReadTokens / denom : 0;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Rolling 7-day window by calendar day
|
|
141
|
+
const now = Date.now();
|
|
142
|
+
const sevenDaysMs = 7 * 24 * 60 * 60 * 1000;
|
|
143
|
+
const cutoffMs = now - sevenDaysMs;
|
|
144
|
+
let rollingCost = 0;
|
|
145
|
+
const rollingDays = new Set();
|
|
146
|
+
for (const r of rawRecords) {
|
|
147
|
+
const ts = Date.parse(String(r.startedAt).replace(' ', 'T') + (r.startedAt && r.startedAt.length === 16 ? ':00Z' : ''));
|
|
148
|
+
if (Number.isFinite(ts) && ts >= cutoffMs) {
|
|
149
|
+
rollingCost += _cost(r);
|
|
150
|
+
rollingDays.add(_day(r.startedAt));
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
agg.rolling7d.days = rollingDays.size;
|
|
154
|
+
agg.rolling7d.totalCostUSD = rollingCost;
|
|
155
|
+
agg.rolling7d.dailyAvgUSD = rollingCost / 7;
|
|
156
|
+
agg.rolling7d.monthlyProjectionUSD = agg.rolling7d.dailyAvgUSD * 30;
|
|
157
|
+
|
|
158
|
+
return agg;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function _fmtMoney(n) {
|
|
162
|
+
if (n == null || !Number.isFinite(n)) return '-';
|
|
163
|
+
return `$${n.toFixed(2)}`;
|
|
164
|
+
}
|
|
165
|
+
function _fmtPct(n) {
|
|
166
|
+
if (!Number.isFinite(n)) return '-';
|
|
167
|
+
return `${(n * 100).toFixed(1)}%`;
|
|
168
|
+
}
|
|
169
|
+
function _fmtInt(n) {
|
|
170
|
+
return String(Number(n || 0).toLocaleString('en-US'));
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function renderTable(agg) {
|
|
174
|
+
const lines = [];
|
|
175
|
+
lines.push('═══ Token Dashboard ═══');
|
|
176
|
+
lines.push(`Records: ${_fmtInt(agg.totalRecords)} | Total cost: ${_fmtMoney(agg.totalCostUSD)} | Input: ${_fmtInt(agg.totalInputTokens)} | Output: ${_fmtInt(agg.totalOutputTokens)}`);
|
|
177
|
+
if (agg.currentMilestone) lines.push(`Milestone filter: ${agg.currentMilestone}`);
|
|
178
|
+
lines.push('');
|
|
179
|
+
|
|
180
|
+
lines.push('── By Day ──');
|
|
181
|
+
const days = Object.values(agg.byDay).sort((a, b) => a.day.localeCompare(b.day));
|
|
182
|
+
if (days.length === 0) lines.push(' (no data)');
|
|
183
|
+
for (const d of days) {
|
|
184
|
+
lines.push(` ${d.day} ${String(d.records).padStart(4)} spawns ${_fmtMoney(d.costUSD).padStart(8)} in=${_fmtInt(d.inputTokens)} out=${_fmtInt(d.outputTokens)}`);
|
|
185
|
+
}
|
|
186
|
+
lines.push('');
|
|
187
|
+
|
|
188
|
+
lines.push('── By Command ──');
|
|
189
|
+
const cmds = Object.values(agg.byCommand).sort((a, b) => b.costUSD - a.costUSD);
|
|
190
|
+
if (cmds.length === 0) lines.push(' (no data)');
|
|
191
|
+
for (const c of cmds) {
|
|
192
|
+
lines.push(` ${c.command.padEnd(30)} ${String(c.records).padStart(4)} ${_fmtMoney(c.costUSD).padStart(8)}`);
|
|
193
|
+
}
|
|
194
|
+
lines.push('');
|
|
195
|
+
|
|
196
|
+
lines.push('── By Model ──');
|
|
197
|
+
const models = Object.values(agg.byModel).sort((a, b) => b.costUSD - a.costUSD);
|
|
198
|
+
if (models.length === 0) lines.push(' (no data)');
|
|
199
|
+
for (const m of models) {
|
|
200
|
+
lines.push(` ${m.model.padEnd(20)} ${String(m.records).padStart(4)} ${_fmtMoney(m.costUSD).padStart(8)} cache-hit: ${_fmtPct(m.cacheHitRate)}`);
|
|
201
|
+
}
|
|
202
|
+
lines.push('');
|
|
203
|
+
|
|
204
|
+
lines.push('── Top 10 Spawns by Cost ──');
|
|
205
|
+
if (agg.topSpawns.length === 0) lines.push(' (no data)');
|
|
206
|
+
for (let i = 0; i < agg.topSpawns.length; i++) {
|
|
207
|
+
const s = agg.topSpawns[i];
|
|
208
|
+
lines.push(` ${String(i + 1).padStart(2)}. ${s.startedAt} ${String(s.command || '').padEnd(22)} ${String(s.step || '').padEnd(14)} ${_fmtMoney(s.costUSD)}`);
|
|
209
|
+
}
|
|
210
|
+
lines.push('');
|
|
211
|
+
|
|
212
|
+
lines.push('── Rolling 7-Day Projection ──');
|
|
213
|
+
lines.push(` 7-day cost: ${_fmtMoney(agg.rolling7d.totalCostUSD)} across ${agg.rolling7d.days} day(s)`);
|
|
214
|
+
lines.push(` Daily avg: ${_fmtMoney(agg.rolling7d.dailyAvgUSD)}`);
|
|
215
|
+
lines.push(` Monthly projection (× 30): ${_fmtMoney(agg.rolling7d.monthlyProjectionUSD)}`);
|
|
216
|
+
|
|
217
|
+
return lines.join('\n');
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function renderJson(agg) {
|
|
221
|
+
return JSON.stringify(agg, null, 2);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Renders exactly 2 content lines + 1 separator = 3 output lines total.
|
|
226
|
+
*/
|
|
227
|
+
function renderStatusBlock(agg) {
|
|
228
|
+
const sep = '───';
|
|
229
|
+
if (!agg || agg.totalRecords === 0) {
|
|
230
|
+
return sep + '\nTokens: no data yet (run a command to populate)\n(run `gsd-t tokens` for full dashboard)';
|
|
231
|
+
}
|
|
232
|
+
const line1 = `Tokens: ${_fmtInt(agg.totalRecords)} spawns, ${_fmtMoney(agg.totalCostUSD)} total${agg.currentMilestone ? ` (${agg.currentMilestone})` : ''}`;
|
|
233
|
+
const line2 = `Rolling 7d: ${_fmtMoney(agg.rolling7d.totalCostUSD)} (${_fmtMoney(agg.rolling7d.dailyAvgUSD)}/day → ${_fmtMoney(agg.rolling7d.monthlyProjectionUSD)}/mo proj.)`;
|
|
234
|
+
return `${sep}\n${line1}\n${line2}`;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Synchronous variant of aggregate() — for callers that need a blocking read
|
|
239
|
+
* (e.g. the `gsd-t status` tail where we can't await). Uses fs.readFileSync
|
|
240
|
+
* and shares the same record-processing loop. Safe for typical JSONL sizes
|
|
241
|
+
* (<100k lines); prefer aggregate() for large files or background work.
|
|
242
|
+
*/
|
|
243
|
+
function aggregateSync(opts) {
|
|
244
|
+
const projectDir = opts.projectDir || '.';
|
|
245
|
+
const jsonlPath = opts.jsonlPath || DEFAULT_JSONL_PATH(projectDir);
|
|
246
|
+
const sinceDay = opts.since || null;
|
|
247
|
+
const milestone = opts.milestone || null;
|
|
248
|
+
|
|
249
|
+
const agg = {
|
|
250
|
+
totalRecords: 0,
|
|
251
|
+
totalCostUSD: 0,
|
|
252
|
+
totalInputTokens: 0,
|
|
253
|
+
totalOutputTokens: 0,
|
|
254
|
+
totalCacheReadTokens: 0,
|
|
255
|
+
totalCacheCreateTokens: 0,
|
|
256
|
+
byDay: {},
|
|
257
|
+
byCommand: {},
|
|
258
|
+
byModel: {},
|
|
259
|
+
topSpawns: [],
|
|
260
|
+
rolling7d: { days: 0, totalCostUSD: 0, dailyAvgUSD: 0, monthlyProjectionUSD: 0 },
|
|
261
|
+
currentMilestone: milestone,
|
|
262
|
+
source: jsonlPath,
|
|
263
|
+
};
|
|
264
|
+
|
|
265
|
+
if (!fs.existsSync(jsonlPath)) return agg;
|
|
266
|
+
|
|
267
|
+
const raw = fs.readFileSync(jsonlPath, 'utf8');
|
|
268
|
+
const lines = raw.split('\n');
|
|
269
|
+
const rawRecords = [];
|
|
270
|
+
|
|
271
|
+
for (const line of lines) {
|
|
272
|
+
const trimmed = line && line.trim();
|
|
273
|
+
if (!trimmed) continue;
|
|
274
|
+
const r = _safeParse(trimmed);
|
|
275
|
+
if (!r || typeof r !== 'object') continue;
|
|
276
|
+
|
|
277
|
+
const day = _day(r.startedAt);
|
|
278
|
+
if (sinceDay && day < sinceDay) continue;
|
|
279
|
+
if (milestone && r.milestone && r.milestone !== milestone) continue;
|
|
280
|
+
if (milestone && !r.milestone) continue;
|
|
281
|
+
|
|
282
|
+
rawRecords.push(r);
|
|
283
|
+
agg.totalRecords += 1;
|
|
284
|
+
agg.totalCostUSD += _cost(r);
|
|
285
|
+
agg.totalInputTokens += Number(r.inputTokens || 0);
|
|
286
|
+
agg.totalOutputTokens += Number(r.outputTokens || 0);
|
|
287
|
+
agg.totalCacheReadTokens += Number(r.cacheReadInputTokens || 0);
|
|
288
|
+
agg.totalCacheCreateTokens += Number(r.cacheCreationInputTokens || 0);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
const now = Date.now();
|
|
292
|
+
const cutoffMs = now - 7 * 24 * 60 * 60 * 1000;
|
|
293
|
+
let rollingCost = 0;
|
|
294
|
+
const rollingDays = new Set();
|
|
295
|
+
for (const r of rawRecords) {
|
|
296
|
+
const ts = Date.parse(String(r.startedAt).replace(' ', 'T') + (r.startedAt && r.startedAt.length === 16 ? ':00Z' : ''));
|
|
297
|
+
if (Number.isFinite(ts) && ts >= cutoffMs) {
|
|
298
|
+
rollingCost += _cost(r);
|
|
299
|
+
rollingDays.add(_day(r.startedAt));
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
agg.rolling7d.days = rollingDays.size;
|
|
303
|
+
agg.rolling7d.totalCostUSD = rollingCost;
|
|
304
|
+
agg.rolling7d.dailyAvgUSD = rollingCost / 7;
|
|
305
|
+
agg.rolling7d.monthlyProjectionUSD = agg.rolling7d.dailyAvgUSD * 30;
|
|
306
|
+
|
|
307
|
+
return agg;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
module.exports = {
|
|
311
|
+
aggregate,
|
|
312
|
+
aggregateSync,
|
|
313
|
+
renderTable,
|
|
314
|
+
renderJson,
|
|
315
|
+
renderStatusBlock,
|
|
316
|
+
_safeParse,
|
|
317
|
+
_day,
|
|
318
|
+
};
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* GSD-T Token Log Regenerator (M43 D3)
|
|
4
|
+
*
|
|
5
|
+
* Reads `.gsd-t/metrics/token-usage.jsonl` end-to-end and writes
|
|
6
|
+
* `.gsd-t/token-log.md` deterministically. Per metrics-schema-contract v2
|
|
7
|
+
* §Derived Artifact, `token-log.md` is a regenerated view post-v2.
|
|
8
|
+
*
|
|
9
|
+
* Sort (v2 §5): startedAt asc → session_id asc → turn_id asc.
|
|
10
|
+
* Numeric turn_ids sort numerically; mixed/non-numeric falls back to lex.
|
|
11
|
+
*
|
|
12
|
+
* Idempotent and deterministic: running twice produces byte-identical output.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const fs = require('fs');
|
|
16
|
+
const path = require('path');
|
|
17
|
+
|
|
18
|
+
const capture = require('./gsd-t-token-capture.cjs');
|
|
19
|
+
const { NEW_HEADER, NEW_SEP, _formatTokensCell } = capture;
|
|
20
|
+
|
|
21
|
+
function _readJsonl(jsonlPath) {
|
|
22
|
+
if (!fs.existsSync(jsonlPath)) return [];
|
|
23
|
+
const text = fs.readFileSync(jsonlPath, 'utf8');
|
|
24
|
+
const rows = [];
|
|
25
|
+
for (const line of text.split('\n')) {
|
|
26
|
+
const s = line.trim();
|
|
27
|
+
if (!s) continue;
|
|
28
|
+
try { rows.push(JSON.parse(s)); }
|
|
29
|
+
catch (_) { /* skip malformed line */ }
|
|
30
|
+
}
|
|
31
|
+
return rows;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function _tokenCellFromRow(row) {
|
|
35
|
+
if (!row.hasUsage) return '—';
|
|
36
|
+
const u = {
|
|
37
|
+
input_tokens: row.inputTokens || 0,
|
|
38
|
+
output_tokens: row.outputTokens || 0,
|
|
39
|
+
cache_read_input_tokens: row.cacheReadInputTokens || 0,
|
|
40
|
+
cache_creation_input_tokens: row.cacheCreationInputTokens || 0,
|
|
41
|
+
total_cost_usd: (typeof row.costUSD === 'number') ? row.costUSD : undefined,
|
|
42
|
+
};
|
|
43
|
+
return _formatTokensCell(u);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function _cmpStart(a, b) {
|
|
47
|
+
const av = a.startedAt || '';
|
|
48
|
+
const bv = b.startedAt || '';
|
|
49
|
+
if (av < bv) return -1;
|
|
50
|
+
if (av > bv) return 1;
|
|
51
|
+
return 0;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function _cmpSession(a, b) {
|
|
55
|
+
const av = a.session_id == null ? '' : String(a.session_id);
|
|
56
|
+
const bv = b.session_id == null ? '' : String(b.session_id);
|
|
57
|
+
if (av < bv) return -1;
|
|
58
|
+
if (av > bv) return 1;
|
|
59
|
+
return 0;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function _cmpTurn(a, b) {
|
|
63
|
+
const av = a.turn_id == null ? '' : String(a.turn_id);
|
|
64
|
+
const bv = b.turn_id == null ? '' : String(b.turn_id);
|
|
65
|
+
const an = Number(av), bn = Number(bv);
|
|
66
|
+
if (av !== '' && bv !== '' && Number.isFinite(an) && Number.isFinite(bn)) {
|
|
67
|
+
return an - bn;
|
|
68
|
+
}
|
|
69
|
+
if (av < bv) return -1;
|
|
70
|
+
if (av > bv) return 1;
|
|
71
|
+
return 0;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function sortRows(rows) {
|
|
75
|
+
return rows.slice().sort((a, b) =>
|
|
76
|
+
_cmpStart(a, b) || _cmpSession(a, b) || _cmpTurn(a, b)
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function _durationCell(row) {
|
|
81
|
+
const ms = Number(row.durationMs || 0);
|
|
82
|
+
return `${Math.max(0, Math.round(ms / 1000))}s`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function _renderRow(row) {
|
|
86
|
+
const tokensCell = _tokenCellFromRow(row);
|
|
87
|
+
const notes = (row.notes == null || row.notes === '') ? '-' : String(row.notes).replace(/\|/g, '\\|');
|
|
88
|
+
const domain = (row.domain == null || row.domain === '') ? '-' : String(row.domain);
|
|
89
|
+
const task = (row.task == null || row.task === '') ? '-' : String(row.task);
|
|
90
|
+
const ctxPct = (row.ctxPct == null) ? 'N/A' : String(row.ctxPct);
|
|
91
|
+
return `| ${row.startedAt || ''} | ${row.endedAt || ''} | ${row.command || ''} | ${row.step || ''} | ${row.model || ''} | ${_durationCell(row)} | ${tokensCell} | ${notes} | ${domain} | ${task} | ${ctxPct} |`;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function renderMarkdown(rows) {
|
|
95
|
+
const sorted = sortRows(rows);
|
|
96
|
+
const lines = ['# GSD-T Token Log', '', NEW_HEADER, NEW_SEP];
|
|
97
|
+
for (const r of sorted) lines.push(_renderRow(r));
|
|
98
|
+
lines.push('');
|
|
99
|
+
return lines.join('\n');
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Regenerate token-log.md from token-usage.jsonl.
|
|
104
|
+
* @param {object} [opts]
|
|
105
|
+
* @param {string} [opts.projectDir='.']
|
|
106
|
+
* @param {string} [opts.jsonlPath] override input path
|
|
107
|
+
* @param {string} [opts.tokenLogPath] override output path
|
|
108
|
+
* @returns {{ wrote: string, rowCount: number }}
|
|
109
|
+
*/
|
|
110
|
+
function regenerateLog(opts = {}) {
|
|
111
|
+
const projectDir = opts.projectDir || '.';
|
|
112
|
+
const jsonlPath = opts.jsonlPath || path.join(projectDir, '.gsd-t', 'metrics', 'token-usage.jsonl');
|
|
113
|
+
const tokenLogPath = opts.tokenLogPath || path.join(projectDir, '.gsd-t', 'token-log.md');
|
|
114
|
+
const rows = _readJsonl(jsonlPath);
|
|
115
|
+
const markdown = renderMarkdown(rows);
|
|
116
|
+
const dir = path.dirname(tokenLogPath);
|
|
117
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
118
|
+
fs.writeFileSync(tokenLogPath, markdown);
|
|
119
|
+
return { wrote: tokenLogPath, rowCount: rows.length };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
module.exports = {
|
|
123
|
+
regenerateLog,
|
|
124
|
+
renderMarkdown,
|
|
125
|
+
sortRows,
|
|
126
|
+
_readJsonl,
|
|
127
|
+
_renderRow,
|
|
128
|
+
_tokenCellFromRow,
|
|
129
|
+
};
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* GSD-T Transcript Tee (M42 D1)
|
|
4
|
+
*
|
|
5
|
+
* Captures raw stream-json frames from every unattended spawn to
|
|
6
|
+
* `.gsd-t/transcripts/{spawn-id}.ndjson` + maintains a registry at
|
|
7
|
+
* `.gsd-t/transcripts/.index.json` used by the dashboard sidebar.
|
|
8
|
+
*
|
|
9
|
+
* Zero external deps. Append-only. Does not parse the frames — that's the
|
|
10
|
+
* renderer's job. One frame per line; lines that fail JSON parse are still
|
|
11
|
+
* tee'd so nothing is silently dropped.
|
|
12
|
+
*
|
|
13
|
+
* Contracts:
|
|
14
|
+
* - .gsd-t/contracts/stream-json-sink-contract.md v1.1.0 (frame shape)
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const crypto = require('crypto');
|
|
20
|
+
|
|
21
|
+
const TRANSCRIPTS_DIRNAME = path.join('.gsd-t', 'transcripts');
|
|
22
|
+
const INDEX_FILENAME = '.index.json';
|
|
23
|
+
|
|
24
|
+
function _transcriptsDir(projectDir) {
|
|
25
|
+
return path.join(projectDir || '.', TRANSCRIPTS_DIRNAME);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function _indexPath(projectDir) {
|
|
29
|
+
return path.join(_transcriptsDir(projectDir), INDEX_FILENAME);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function _ensureDir(p) {
|
|
33
|
+
if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true });
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Allocate a hierarchical spawn-id. Shape: `{parent-prefix}-{short}` where
|
|
38
|
+
* `short` is an 8-char hex from a random UUID. Root spawns get no prefix.
|
|
39
|
+
*
|
|
40
|
+
* @param {object} [opts]
|
|
41
|
+
* @param {string|null} [opts.parentId]
|
|
42
|
+
* @returns {string}
|
|
43
|
+
*/
|
|
44
|
+
function allocateSpawnId(opts) {
|
|
45
|
+
const parentId = opts && opts.parentId ? String(opts.parentId) : null;
|
|
46
|
+
const short = crypto.randomBytes(4).toString('hex');
|
|
47
|
+
return parentId ? `${parentId}.${short}` : `s-${short}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function _readIndex(projectDir) {
|
|
51
|
+
const p = _indexPath(projectDir);
|
|
52
|
+
if (!fs.existsSync(p)) return { spawns: [] };
|
|
53
|
+
try {
|
|
54
|
+
const raw = fs.readFileSync(p, 'utf8');
|
|
55
|
+
const parsed = JSON.parse(raw);
|
|
56
|
+
if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.spawns)) return { spawns: [] };
|
|
57
|
+
return parsed;
|
|
58
|
+
} catch (_) {
|
|
59
|
+
return { spawns: [] };
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function _writeIndex(projectDir, idx) {
|
|
64
|
+
const p = _indexPath(projectDir);
|
|
65
|
+
_ensureDir(path.dirname(p));
|
|
66
|
+
const tmp = p + '.tmp';
|
|
67
|
+
fs.writeFileSync(tmp, JSON.stringify(idx, null, 2));
|
|
68
|
+
fs.renameSync(tmp, p);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Register a new transcript + create the ndjson file.
|
|
73
|
+
*
|
|
74
|
+
* @param {object} opts
|
|
75
|
+
* @param {string} opts.spawnId
|
|
76
|
+
* @param {string} [opts.projectDir='.']
|
|
77
|
+
* @param {object} [opts.meta] { parentId?, command?, description?, workerPid?, model? }
|
|
78
|
+
* @returns {{spawnId, transcriptPath, startedAt}}
|
|
79
|
+
*/
|
|
80
|
+
function openTranscript(opts) {
|
|
81
|
+
if (!opts || !opts.spawnId) throw new Error('openTranscript: spawnId required');
|
|
82
|
+
const projectDir = opts.projectDir || '.';
|
|
83
|
+
const meta = opts.meta || {};
|
|
84
|
+
const dir = _transcriptsDir(projectDir);
|
|
85
|
+
_ensureDir(dir);
|
|
86
|
+
|
|
87
|
+
const transcriptPath = path.join(dir, `${opts.spawnId}.ndjson`);
|
|
88
|
+
if (!fs.existsSync(transcriptPath)) {
|
|
89
|
+
fs.writeFileSync(transcriptPath, '');
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const startedAt = new Date().toISOString();
|
|
93
|
+
const entry = {
|
|
94
|
+
spawnId: opts.spawnId,
|
|
95
|
+
parentId: meta.parentId || null,
|
|
96
|
+
command: meta.command || null,
|
|
97
|
+
description: meta.description || null,
|
|
98
|
+
model: meta.model || null,
|
|
99
|
+
workerPid: meta.workerPid || null,
|
|
100
|
+
startedAt,
|
|
101
|
+
endedAt: null,
|
|
102
|
+
status: 'running',
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const idx = _readIndex(projectDir);
|
|
106
|
+
const existing = idx.spawns.findIndex((s) => s.spawnId === opts.spawnId);
|
|
107
|
+
if (existing >= 0) {
|
|
108
|
+
idx.spawns[existing] = { ...idx.spawns[existing], ...entry };
|
|
109
|
+
} else {
|
|
110
|
+
idx.spawns.push(entry);
|
|
111
|
+
}
|
|
112
|
+
_writeIndex(projectDir, idx);
|
|
113
|
+
|
|
114
|
+
return { spawnId: opts.spawnId, transcriptPath, startedAt };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Append a single frame (already JSON-serializable) to the transcript.
|
|
119
|
+
* `frame` may be a parsed object OR a raw string line — strings are wrapped
|
|
120
|
+
* as `{type:"raw",line}` so the ndjson shape stays uniform.
|
|
121
|
+
*
|
|
122
|
+
* @param {object} opts
|
|
123
|
+
* @param {string} opts.spawnId
|
|
124
|
+
* @param {string} [opts.projectDir='.']
|
|
125
|
+
* @param {object|string} opts.frame
|
|
126
|
+
*/
|
|
127
|
+
function appendFrame(opts) {
|
|
128
|
+
if (!opts || !opts.spawnId) throw new Error('appendFrame: spawnId required');
|
|
129
|
+
if (opts.frame === undefined || opts.frame === null) return;
|
|
130
|
+
const projectDir = opts.projectDir || '.';
|
|
131
|
+
const p = path.join(_transcriptsDir(projectDir), `${opts.spawnId}.ndjson`);
|
|
132
|
+
_ensureDir(path.dirname(p));
|
|
133
|
+
|
|
134
|
+
let line;
|
|
135
|
+
if (typeof opts.frame === 'string') {
|
|
136
|
+
const trimmed = opts.frame.trim();
|
|
137
|
+
if (!trimmed) return;
|
|
138
|
+
try {
|
|
139
|
+
JSON.parse(trimmed);
|
|
140
|
+
line = trimmed;
|
|
141
|
+
} catch (_) {
|
|
142
|
+
line = JSON.stringify({ type: 'raw', line: opts.frame });
|
|
143
|
+
}
|
|
144
|
+
} else {
|
|
145
|
+
try {
|
|
146
|
+
line = JSON.stringify(opts.frame);
|
|
147
|
+
} catch (_) {
|
|
148
|
+
line = JSON.stringify({ type: 'raw', line: String(opts.frame) });
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
fs.appendFileSync(p, line + '\n');
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Mark a transcript as ended. Idempotent — subsequent calls update endedAt.
|
|
156
|
+
*
|
|
157
|
+
* @param {object} opts
|
|
158
|
+
* @param {string} opts.spawnId
|
|
159
|
+
* @param {string} [opts.projectDir='.']
|
|
160
|
+
* @param {'done'|'failed'|'stopped'|'ended'} [opts.status='ended']
|
|
161
|
+
*/
|
|
162
|
+
function closeTranscript(opts) {
|
|
163
|
+
if (!opts || !opts.spawnId) throw new Error('closeTranscript: spawnId required');
|
|
164
|
+
const projectDir = opts.projectDir || '.';
|
|
165
|
+
const idx = _readIndex(projectDir);
|
|
166
|
+
const i = idx.spawns.findIndex((s) => s.spawnId === opts.spawnId);
|
|
167
|
+
if (i < 0) return false;
|
|
168
|
+
idx.spawns[i].endedAt = new Date().toISOString();
|
|
169
|
+
idx.spawns[i].status = opts.status || 'ended';
|
|
170
|
+
_writeIndex(projectDir, idx);
|
|
171
|
+
return true;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* List registered spawns (most recent first).
|
|
176
|
+
*
|
|
177
|
+
* @param {string} [projectDir='.']
|
|
178
|
+
* @returns {Array<object>}
|
|
179
|
+
*/
|
|
180
|
+
function listTranscripts(projectDir) {
|
|
181
|
+
const idx = _readIndex(projectDir || '.');
|
|
182
|
+
return idx.spawns.slice().sort((a, b) => {
|
|
183
|
+
const ta = Date.parse(a.startedAt) || 0;
|
|
184
|
+
const tb = Date.parse(b.startedAt) || 0;
|
|
185
|
+
return tb - ta;
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function readTranscriptMeta(projectDir, spawnId) {
|
|
190
|
+
const idx = _readIndex(projectDir || '.');
|
|
191
|
+
return idx.spawns.find((s) => s.spawnId === spawnId) || null;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Tee a stream-json stdout stream. Returns an `onChunk(buffer)` callback that
|
|
196
|
+
* you feed chunks to (typed as Buffer or string). Frames are split at `\n`
|
|
197
|
+
* and each complete line is written to the transcript ndjson. Incomplete
|
|
198
|
+
* tails are buffered until the next chunk.
|
|
199
|
+
*
|
|
200
|
+
* Returns also a `flush()` to call on child exit — writes any stranded tail
|
|
201
|
+
* as a single `{type:"raw"}` line so nothing is dropped.
|
|
202
|
+
*
|
|
203
|
+
* @param {object} opts
|
|
204
|
+
* @param {string} opts.spawnId
|
|
205
|
+
* @param {string} [opts.projectDir='.']
|
|
206
|
+
* @returns {{onChunk: (chunk) => void, flush: () => void}}
|
|
207
|
+
*/
|
|
208
|
+
function makeStreamTee(opts) {
|
|
209
|
+
if (!opts || !opts.spawnId) throw new Error('makeStreamTee: spawnId required');
|
|
210
|
+
const projectDir = opts.projectDir || '.';
|
|
211
|
+
const spawnId = opts.spawnId;
|
|
212
|
+
let buf = '';
|
|
213
|
+
|
|
214
|
+
return {
|
|
215
|
+
onChunk(chunk) {
|
|
216
|
+
if (chunk == null) return;
|
|
217
|
+
buf += Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk);
|
|
218
|
+
let nl;
|
|
219
|
+
while ((nl = buf.indexOf('\n')) >= 0) {
|
|
220
|
+
const line = buf.slice(0, nl);
|
|
221
|
+
buf = buf.slice(nl + 1);
|
|
222
|
+
if (line.length > 0) appendFrame({ spawnId, projectDir, frame: line });
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
flush() {
|
|
226
|
+
if (buf.length > 0) {
|
|
227
|
+
appendFrame({ spawnId, projectDir, frame: buf });
|
|
228
|
+
buf = '';
|
|
229
|
+
}
|
|
230
|
+
},
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
module.exports = {
|
|
235
|
+
allocateSpawnId,
|
|
236
|
+
openTranscript,
|
|
237
|
+
appendFrame,
|
|
238
|
+
closeTranscript,
|
|
239
|
+
listTranscripts,
|
|
240
|
+
readTranscriptMeta,
|
|
241
|
+
makeStreamTee,
|
|
242
|
+
_readIndex,
|
|
243
|
+
_writeIndex,
|
|
244
|
+
TRANSCRIPTS_DIRNAME,
|
|
245
|
+
INDEX_FILENAME,
|
|
246
|
+
};
|