@tekyzinc/gsd-t 3.13.16 → 3.16.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +1 -0
- package/bin/gsd-t-benchmark-orchestrator.js +437 -0
- package/bin/gsd-t-capture-lint.cjs +276 -0
- package/bin/gsd-t-completion-check.cjs +106 -0
- package/bin/gsd-t-orchestrator-config.cjs +64 -0
- package/bin/gsd-t-orchestrator-queue.cjs +180 -0
- package/bin/gsd-t-orchestrator-recover.cjs +231 -0
- package/bin/gsd-t-orchestrator-worker.cjs +219 -0
- package/bin/gsd-t-orchestrator.js +534 -0
- package/bin/gsd-t-stream-feed-client.cjs +151 -0
- package/bin/gsd-t-task-brief-compactor.cjs +89 -0
- package/bin/gsd-t-task-brief-template.cjs +96 -0
- package/bin/gsd-t-task-brief.js +249 -0
- package/bin/gsd-t-token-backfill.cjs +366 -0
- package/bin/gsd-t-token-capture.cjs +306 -0
- package/bin/gsd-t-token-dashboard.cjs +318 -0
- package/bin/gsd-t-token-regenerate-log.cjs +129 -0
- package/bin/gsd-t-transcript-tee.cjs +246 -0
- package/bin/gsd-t-unattended-heartbeat.cjs +188 -0
- package/bin/gsd-t-unattended-platform.cjs +191 -27
- package/bin/gsd-t-unattended-safety.cjs +8 -1
- package/bin/gsd-t-unattended.cjs +192 -31
- package/bin/gsd-t.js +329 -2
- package/bin/supervisor-pid-fingerprint.cjs +126 -0
- package/commands/gsd-t-debug.md +63 -51
- package/commands/gsd-t-design-decompose.md +2 -7
- package/commands/gsd-t-doc-ripple.md +20 -11
- package/commands/gsd-t-execute.md +82 -50
- package/commands/gsd-t-integrate.md +43 -16
- package/commands/gsd-t-plan.md +20 -7
- package/commands/gsd-t-prd.md +19 -12
- package/commands/gsd-t-quick.md +64 -29
- package/commands/gsd-t-resume.md +51 -4
- package/commands/gsd-t-unattended.md +19 -20
- package/commands/gsd-t-verify.md +48 -32
- package/commands/gsd-t-visualize.md +19 -17
- package/commands/gsd-t-wave.md +29 -27
- package/docs/architecture.md +16 -0
- package/docs/m40-benchmark-report.md +35 -0
- package/docs/requirements.md +20 -0
- package/package.json +1 -1
- package/scripts/gsd-t-dashboard-server.js +291 -4
- package/scripts/gsd-t-dashboard.html +31 -1
- package/scripts/gsd-t-design-review-server.js +3 -1
- package/scripts/gsd-t-stream-feed-server.js +428 -0
- package/scripts/gsd-t-stream-feed.html +1168 -0
- package/scripts/gsd-t-token-aggregator.js +373 -0
- package/scripts/gsd-t-transcript.html +422 -0
- package/scripts/hooks/gsd-t-in-session-probe.js +62 -0
- package/scripts/hooks/pre-commit-capture-lint +26 -0
- package/templates/CLAUDE-global.md +69 -0
- package/scripts/gsd-t-agent-dashboard-server.js +0 -424
- package/scripts/gsd-t-agent-dashboard.html +0 -1043
|
@@ -0,0 +1,534 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
|
|
7
|
+
const { loadConfig } = require('./gsd-t-orchestrator-config.cjs');
|
|
8
|
+
const { readAllTasks, groupByWave, validateNoForwardDeps } = require('./gsd-t-orchestrator-queue.cjs');
|
|
9
|
+
const { runWorker } = require('./gsd-t-orchestrator-worker.cjs');
|
|
10
|
+
const { buildTaskBrief } = require('./gsd-t-task-brief.js');
|
|
11
|
+
const { createStreamFeedClient } = require('./gsd-t-stream-feed-client.cjs');
|
|
12
|
+
const { recoverRunState, writeRecoveredState, archiveState } = require('./gsd-t-orchestrator-recover.cjs');
|
|
13
|
+
|
|
14
|
+
const STATE_DIR = '.gsd-t/orchestrator';
|
|
15
|
+
const STATE_FILE = 'state.json';
|
|
16
|
+
|
|
17
|
+
function nowIso() { return new Date().toISOString(); }
|
|
18
|
+
|
|
19
|
+
function parseCliArgs(argv) {
|
|
20
|
+
const args = {
|
|
21
|
+
milestone: null,
|
|
22
|
+
maxParallel: null,
|
|
23
|
+
workerTimeoutMs: null,
|
|
24
|
+
projectDir: process.cwd(),
|
|
25
|
+
resume: false,
|
|
26
|
+
noArchive: false,
|
|
27
|
+
help: false,
|
|
28
|
+
streamFeed: true,
|
|
29
|
+
streamFeedPort: null,
|
|
30
|
+
streamFeedHost: null
|
|
31
|
+
};
|
|
32
|
+
for (let i = 0; i < argv.length; i++) {
|
|
33
|
+
const a = argv[i];
|
|
34
|
+
if (a === '--help' || a === '-h') { args.help = true; }
|
|
35
|
+
else if (a === '--milestone') { args.milestone = argv[++i]; }
|
|
36
|
+
else if (a === '--max-parallel') { args.maxParallel = argv[++i]; }
|
|
37
|
+
else if (a === '--worker-timeout') { args.workerTimeoutMs = argv[++i]; }
|
|
38
|
+
else if (a === '--project-dir') { args.projectDir = path.resolve(argv[++i]); }
|
|
39
|
+
else if (a === '--resume') { args.resume = true; }
|
|
40
|
+
else if (a === '--no-archive') { args.noArchive = true; }
|
|
41
|
+
else if (a === '--no-stream-feed') { args.streamFeed = false; }
|
|
42
|
+
else if (a === '--stream-feed-port') { args.streamFeedPort = Number(argv[++i]); }
|
|
43
|
+
else if (a === '--stream-feed-host') { args.streamFeedHost = argv[++i]; }
|
|
44
|
+
}
|
|
45
|
+
return args;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function printHelp() {
|
|
49
|
+
process.stdout.write([
|
|
50
|
+
'Usage: gsd-t orchestrate --milestone <id> [options]',
|
|
51
|
+
'',
|
|
52
|
+
'Options:',
|
|
53
|
+
' --milestone <id> Milestone id (e.g. M40). Required.',
|
|
54
|
+
' --max-parallel <n> Max concurrent workers (default 3, max 15).',
|
|
55
|
+
' --worker-timeout <ms> Per-worker timeout in ms (default 270000).',
|
|
56
|
+
' --project-dir <path> Project directory (default cwd).',
|
|
57
|
+
' --resume Resume from .gsd-t/orchestrator/state.json.',
|
|
58
|
+
' --no-archive When --resume + state is terminal, do NOT archive — fail instead.',
|
|
59
|
+
' --no-stream-feed Disable pushing frames to local stream-feed server.',
|
|
60
|
+
' --stream-feed-port <N> Override stream-feed port (default 7842 / env GSD_T_STREAM_FEED_PORT).',
|
|
61
|
+
' --stream-feed-host <H> Override stream-feed host (default 127.0.0.1).',
|
|
62
|
+
' -h, --help Show this help.',
|
|
63
|
+
''
|
|
64
|
+
].join('\n'));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function atomicWriteJson(fp, obj) {
|
|
68
|
+
const dir = path.dirname(fp);
|
|
69
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
70
|
+
const tmp = fp + '.tmp.' + process.pid;
|
|
71
|
+
fs.writeFileSync(tmp, JSON.stringify(obj, null, 2));
|
|
72
|
+
fs.renameSync(tmp, fp);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function appendEventLine(projectDir, event) {
|
|
76
|
+
const dayStr = nowIso().slice(0, 10);
|
|
77
|
+
const eventsDir = path.join(projectDir, '.gsd-t', 'events');
|
|
78
|
+
if (!fs.existsSync(eventsDir)) fs.mkdirSync(eventsDir, { recursive: true });
|
|
79
|
+
const fp = path.join(eventsDir, dayStr + '.jsonl');
|
|
80
|
+
fs.appendFileSync(fp, JSON.stringify(event) + '\n');
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function writeEvent(projectDir, eventType, extra) {
|
|
84
|
+
const event = {
|
|
85
|
+
ts: nowIso(),
|
|
86
|
+
command: 'orchestrate',
|
|
87
|
+
phase: null,
|
|
88
|
+
trace_id: null,
|
|
89
|
+
event_type: eventType,
|
|
90
|
+
agent_id: process.env.GSD_T_AGENT_ID || null,
|
|
91
|
+
parent_agent_id: process.env.GSD_T_PARENT_AGENT_ID || null,
|
|
92
|
+
reasoning: null,
|
|
93
|
+
outcome: null,
|
|
94
|
+
...(extra || {})
|
|
95
|
+
};
|
|
96
|
+
try { appendEventLine(projectDir, event); } catch (_) { /* best effort */ }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function makeState(projectDir) {
|
|
100
|
+
const fp = path.join(projectDir, STATE_DIR, STATE_FILE);
|
|
101
|
+
const defaults = {
|
|
102
|
+
startedAt: nowIso(),
|
|
103
|
+
status: 'running',
|
|
104
|
+
currentWave: null,
|
|
105
|
+
tasks: {}
|
|
106
|
+
};
|
|
107
|
+
return {
|
|
108
|
+
fp,
|
|
109
|
+
data: defaults,
|
|
110
|
+
save(patch) {
|
|
111
|
+
if (patch) Object.assign(this.data, patch);
|
|
112
|
+
atomicWriteJson(this.fp, this.data);
|
|
113
|
+
},
|
|
114
|
+
patchTask(id, fields) {
|
|
115
|
+
this.data.tasks[id] = { ...(this.data.tasks[id] || {}), ...fields };
|
|
116
|
+
atomicWriteJson(this.fp, this.data);
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function filterTasksByMilestone(tasks, milestone) {
|
|
122
|
+
// M40 tasks don't yet carry a milestone field in tasks.md — for now,
|
|
123
|
+
// the milestone context is global. This function is a placeholder for
|
|
124
|
+
// when tasks.md entries include an explicit milestone marker.
|
|
125
|
+
return tasks;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async function runWaveTasks({ tasks, config, state, logger, spawnImpl, runWorkerImpl = runWorker, onFrame, interrupt, liveChildrenRef, streamFeedFactory }) {
|
|
129
|
+
const queue = [...tasks];
|
|
130
|
+
const running = new Set();
|
|
131
|
+
const results = [];
|
|
132
|
+
const liveChildren = liveChildrenRef || new Map();
|
|
133
|
+
let haltRequested = false;
|
|
134
|
+
if (!interrupt) interrupt = { interrupted: false };
|
|
135
|
+
|
|
136
|
+
const emit = (frame) => {
|
|
137
|
+
if (typeof onFrame === 'function') { try { onFrame(frame); } catch (_) {} }
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
const launch = async (task) => {
|
|
141
|
+
const started = nowIso();
|
|
142
|
+
state.patchTask(task.id, { status: 'running', startedAt: started, retryCount: 0 });
|
|
143
|
+
writeEvent(config.projectDir, 'task_start', { task_id: task.id, wave: task.wave, domain: task.domain });
|
|
144
|
+
|
|
145
|
+
const canonicalTaskId = task.id.includes(':T')
|
|
146
|
+
? `${task.domain}-t${task.id.split(':T')[1]}`
|
|
147
|
+
: task.id;
|
|
148
|
+
const taskForWorker = { ...task, canonicalId: canonicalTaskId };
|
|
149
|
+
|
|
150
|
+
const attempt = async (retryCount) => {
|
|
151
|
+
let brief;
|
|
152
|
+
try {
|
|
153
|
+
brief = buildTaskBrief({
|
|
154
|
+
milestone: config.milestone,
|
|
155
|
+
domain: task.domain,
|
|
156
|
+
taskId: canonicalTaskId,
|
|
157
|
+
projectDir: config.projectDir,
|
|
158
|
+
expectedBranch: config.expectedBranch || 'main'
|
|
159
|
+
});
|
|
160
|
+
} catch (err) {
|
|
161
|
+
logger.log(`[orchestrator] brief build failed for ${task.id}: ${err.message}`);
|
|
162
|
+
return {
|
|
163
|
+
result: { ok: false, missing: ['brief_build_error'], details: { error: String(err) } },
|
|
164
|
+
exitCode: -1,
|
|
165
|
+
durationMs: 0,
|
|
166
|
+
timedOut: false
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
if (retryCount > 0) {
|
|
170
|
+
brief += `\n\n## Retry Note\nPrevious attempt failed. Try again, paying attention to the Done Signal checklist.\n`;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// D1-T7: per-worker stream-feed client. Opens when we get a pid; every
|
|
174
|
+
// onFrame call tees to both the user callback and the feed client so the
|
|
175
|
+
// D4 server + UI see every frame.
|
|
176
|
+
let feedClient = null;
|
|
177
|
+
|
|
178
|
+
const teeFrame = (frame) => {
|
|
179
|
+
emit(frame);
|
|
180
|
+
if (feedClient) {
|
|
181
|
+
try { feedClient.pushFrame(frame); } catch (_) { /* best-effort */ }
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
const outcome = await runWorkerImpl({
|
|
186
|
+
task: taskForWorker,
|
|
187
|
+
brief,
|
|
188
|
+
config,
|
|
189
|
+
onFrame: teeFrame,
|
|
190
|
+
onSpawn: ({ child, pid }) => {
|
|
191
|
+
if (pid != null) {
|
|
192
|
+
state.patchTask(task.id, { workerPid: pid });
|
|
193
|
+
liveChildren.set(task.id, child);
|
|
194
|
+
if (streamFeedFactory) {
|
|
195
|
+
try {
|
|
196
|
+
feedClient = streamFeedFactory({
|
|
197
|
+
workerPid: pid,
|
|
198
|
+
taskId: canonicalTaskId,
|
|
199
|
+
projectDir: config.projectDir
|
|
200
|
+
});
|
|
201
|
+
} catch (err) {
|
|
202
|
+
logger.log(`[orchestrator] stream-feed client open failed for ${task.id}: ${err.message}`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
},
|
|
207
|
+
env: process.env,
|
|
208
|
+
spawnImpl
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
if (feedClient) {
|
|
212
|
+
try { await feedClient.close(); } catch (_) { /* best-effort */ }
|
|
213
|
+
}
|
|
214
|
+
return outcome;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
let outcome = await attempt(0);
|
|
218
|
+
liveChildren.delete(task.id);
|
|
219
|
+
if (!outcome.result.ok && !interrupt.interrupted && config.retryOnFail) {
|
|
220
|
+
state.patchTask(task.id, { retryCount: 1 });
|
|
221
|
+
outcome = await attempt(1);
|
|
222
|
+
liveChildren.delete(task.id);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const endedAt = nowIso();
|
|
226
|
+
state.patchTask(task.id, {
|
|
227
|
+
status: outcome.result.ok ? 'done' : 'failed',
|
|
228
|
+
endedAt,
|
|
229
|
+
exitCode: outcome.exitCode,
|
|
230
|
+
durationMs: outcome.durationMs,
|
|
231
|
+
missing: outcome.result.missing || []
|
|
232
|
+
});
|
|
233
|
+
writeEvent(config.projectDir, outcome.result.ok ? 'task_done' : 'task_failed', {
|
|
234
|
+
task_id: task.id,
|
|
235
|
+
wave: task.wave,
|
|
236
|
+
domain: task.domain,
|
|
237
|
+
exit_code: outcome.exitCode,
|
|
238
|
+
duration_ms: outcome.durationMs
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
results.push({ task, outcome });
|
|
242
|
+
if (!outcome.result.ok && config.haltOnSecondFail) {
|
|
243
|
+
haltRequested = true;
|
|
244
|
+
}
|
|
245
|
+
return outcome;
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
const pump = async () => {
|
|
249
|
+
while (queue.length && running.size < config.maxParallel && !haltRequested && !interrupt.interrupted) {
|
|
250
|
+
const task = queue.shift();
|
|
251
|
+
const p = launch(task).finally(() => running.delete(p));
|
|
252
|
+
running.add(p);
|
|
253
|
+
}
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
await pump();
|
|
257
|
+
while (running.size) {
|
|
258
|
+
await Promise.race(running);
|
|
259
|
+
if (!haltRequested && !interrupt.interrupted) await pump();
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
return { results, halted: haltRequested, liveChildren };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function emitWaveBoundary(onFrame, state, wave, taskCount, extra) {
|
|
266
|
+
if (typeof onFrame !== 'function') return;
|
|
267
|
+
const frame = { type: 'wave-boundary', wave, state, taskCount, ts: nowIso() };
|
|
268
|
+
if (extra) Object.assign(frame, extra);
|
|
269
|
+
try { onFrame(frame); } catch (_) {}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
async function runOrchestrator(opts) {
|
|
273
|
+
const {
|
|
274
|
+
projectDir,
|
|
275
|
+
milestone,
|
|
276
|
+
maxParallel,
|
|
277
|
+
workerTimeoutMs,
|
|
278
|
+
logger = console,
|
|
279
|
+
spawnImpl,
|
|
280
|
+
runWorkerImpl,
|
|
281
|
+
onFrame,
|
|
282
|
+
installSignalHandlers = true,
|
|
283
|
+
streamFeed = null, // D1-T7: null|false = off, true = default (env-aware), object = client opts
|
|
284
|
+
streamFeedFactory: streamFeedFactoryOverride = null, // test hook
|
|
285
|
+
resume = false,
|
|
286
|
+
noArchive = false
|
|
287
|
+
} = opts;
|
|
288
|
+
|
|
289
|
+
const config = loadConfig({
|
|
290
|
+
projectDir,
|
|
291
|
+
cliFlags: {
|
|
292
|
+
...(maxParallel != null ? { maxParallel } : {}),
|
|
293
|
+
...(workerTimeoutMs != null ? { workerTimeoutMs } : {})
|
|
294
|
+
},
|
|
295
|
+
env: process.env
|
|
296
|
+
});
|
|
297
|
+
config.milestone = milestone || 'unknown';
|
|
298
|
+
|
|
299
|
+
const allTasks = readAllTasks(projectDir);
|
|
300
|
+
const scopedTasks = filterTasksByMilestone(allTasks, milestone);
|
|
301
|
+
if (!scopedTasks.length) {
|
|
302
|
+
logger.log(`[orchestrator] no tasks found under ${projectDir}/.gsd-t/domains/*/tasks.md`);
|
|
303
|
+
return { status: 'empty', waves: [] };
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
validateNoForwardDeps(scopedTasks);
|
|
307
|
+
const waves = groupByWave(scopedTasks);
|
|
308
|
+
|
|
309
|
+
const state = makeState(projectDir);
|
|
310
|
+
|
|
311
|
+
// D6-T2: --resume handling. Ran before we touch state.save().
|
|
312
|
+
// fresh → error (no run to resume)
|
|
313
|
+
// terminal → archive (unless --no-archive) and start fresh
|
|
314
|
+
// resume → seed state.tasks from reconciled recovery output, skip done/ambiguous
|
|
315
|
+
let resumedSkipIds = new Set(); // task ids to not re-launch (done + ambiguous)
|
|
316
|
+
let resumeStartWave = null;
|
|
317
|
+
if (resume) {
|
|
318
|
+
const recovery = recoverRunState({ projectDir });
|
|
319
|
+
for (const note of (recovery.notes || [])) logger.log(`[resume] ${note}`);
|
|
320
|
+
if (recovery.mode === 'fresh') {
|
|
321
|
+
const err = new Error('no run to resume — .gsd-t/orchestrator/state.json is missing');
|
|
322
|
+
err.code = 'NO_RESUME_STATE';
|
|
323
|
+
throw err;
|
|
324
|
+
}
|
|
325
|
+
if (recovery.mode === 'terminal') {
|
|
326
|
+
if (noArchive) {
|
|
327
|
+
const err = new Error('--resume requested but state is terminal and --no-archive was set');
|
|
328
|
+
err.code = 'TERMINAL_NO_ARCHIVE';
|
|
329
|
+
throw err;
|
|
330
|
+
}
|
|
331
|
+
const { archived, archivePath } = archiveState(projectDir);
|
|
332
|
+
if (archived) logger.log(`[resume] archived terminal state → ${archivePath}; starting fresh`);
|
|
333
|
+
} else if (recovery.mode === 'resume') {
|
|
334
|
+
// Persist reconciled task statuses and load them into the in-memory state.
|
|
335
|
+
writeRecoveredState(projectDir, { ...recovery.state, tasks: recovery.tasks });
|
|
336
|
+
state.data = { ...recovery.state, tasks: recovery.tasks };
|
|
337
|
+
for (const [tid, t] of Object.entries(recovery.tasks)) {
|
|
338
|
+
if (t.status === 'done' || t.status === 'ambiguous') resumedSkipIds.add(tid);
|
|
339
|
+
if (t.status === 'ambiguous') logger.log(`[resume] task ${tid} is AMBIGUOUS — commit without progress entry; skipped, needs operator triage`);
|
|
340
|
+
}
|
|
341
|
+
resumeStartWave = recovery.currentWave;
|
|
342
|
+
if (resumeStartWave != null) {
|
|
343
|
+
logger.log(`[resume] continuing from wave ${resumeStartWave} (${resumedSkipIds.size}/${Object.keys(recovery.tasks).length} tasks already done or ambiguous)`);
|
|
344
|
+
} else {
|
|
345
|
+
logger.log('[resume] all tasks reconciled as done/ambiguous — nothing left to run');
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
state.save({ milestone: config.milestone, totalTasks: scopedTasks.length, waves: [...waves.keys()] });
|
|
351
|
+
writeEvent(projectDir, resume ? 'orchestrator_resume' : 'orchestrator_start', { milestone: config.milestone, total_tasks: scopedTasks.length });
|
|
352
|
+
|
|
353
|
+
// D1-T7: stream-feed wiring. `streamFeed: true` means "open clients pointed at
|
|
354
|
+
// the default local server". `streamFeed: { port, host }` overrides. `false`/null
|
|
355
|
+
// disables. If a user-supplied onFrame is the consumer, they can still opt-in.
|
|
356
|
+
let streamFeedFactory = streamFeedFactoryOverride;
|
|
357
|
+
let orchestratorFeedClient = null;
|
|
358
|
+
const feedOpts = (streamFeed && typeof streamFeed === 'object') ? streamFeed : {};
|
|
359
|
+
const feedEnabled = streamFeed === true || (streamFeed && typeof streamFeed === 'object');
|
|
360
|
+
if (feedEnabled && !streamFeedFactory) {
|
|
361
|
+
streamFeedFactory = ({ workerPid, taskId, projectDir }) => createStreamFeedClient({
|
|
362
|
+
...feedOpts,
|
|
363
|
+
projectDir,
|
|
364
|
+
workerPid,
|
|
365
|
+
taskId
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
if (streamFeedFactory) {
|
|
369
|
+
try {
|
|
370
|
+
orchestratorFeedClient = streamFeedFactory({
|
|
371
|
+
workerPid: process.pid,
|
|
372
|
+
taskId: 'orchestrator',
|
|
373
|
+
projectDir
|
|
374
|
+
});
|
|
375
|
+
} catch (err) {
|
|
376
|
+
logger.log(`[orchestrator] stream-feed orchestrator client open failed: ${err.message}`);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const teeWaveFrame = (frame) => {
|
|
381
|
+
if (typeof onFrame === 'function') { try { onFrame(frame); } catch (_) {} }
|
|
382
|
+
if (orchestratorFeedClient) {
|
|
383
|
+
try { orchestratorFeedClient.pushFrame(frame); } catch (_) {}
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
|
|
387
|
+
const interrupt = { interrupted: false, currentWaveChildren: null };
|
|
388
|
+
|
|
389
|
+
let sigintHandler = null;
|
|
390
|
+
if (installSignalHandlers && typeof process.on === 'function') {
|
|
391
|
+
sigintHandler = () => {
|
|
392
|
+
if (interrupt.interrupted) return;
|
|
393
|
+
interrupt.interrupted = true;
|
|
394
|
+
logger.log('[orchestrator] SIGINT received — terminating workers');
|
|
395
|
+
writeEvent(projectDir, 'orchestrator_interrupt', {});
|
|
396
|
+
if (interrupt.currentWaveChildren) {
|
|
397
|
+
for (const child of interrupt.currentWaveChildren.values()) {
|
|
398
|
+
try { child.kill('SIGTERM'); } catch (_) {}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
};
|
|
402
|
+
process.on('SIGINT', sigintHandler);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
const cleanup = () => {
|
|
406
|
+
if (sigintHandler && typeof process.removeListener === 'function') {
|
|
407
|
+
try { process.removeListener('SIGINT', sigintHandler); } catch (_) {}
|
|
408
|
+
}
|
|
409
|
+
if (orchestratorFeedClient) {
|
|
410
|
+
try { orchestratorFeedClient.close(); } catch (_) {}
|
|
411
|
+
orchestratorFeedClient = null;
|
|
412
|
+
}
|
|
413
|
+
};
|
|
414
|
+
|
|
415
|
+
try {
|
|
416
|
+
const waveResults = [];
|
|
417
|
+
for (const [waveNum, waveTasks] of waves) {
|
|
418
|
+
if (interrupt.interrupted) break;
|
|
419
|
+
// D6-T2: on resume, skip waves entirely behind the resume point.
|
|
420
|
+
if (resumeStartWave != null && waveNum < resumeStartWave) {
|
|
421
|
+
writeEvent(projectDir, 'wave_skipped', { wave: waveNum, reason: 'resume_before' });
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
424
|
+
// Filter out tasks that recovery marked done/ambiguous.
|
|
425
|
+
const effectiveTasks = resumedSkipIds.size
|
|
426
|
+
? waveTasks.filter((t) => !resumedSkipIds.has(t.id))
|
|
427
|
+
: waveTasks;
|
|
428
|
+
if (effectiveTasks.length === 0) {
|
|
429
|
+
writeEvent(projectDir, 'wave_skipped', { wave: waveNum, reason: 'all_tasks_reconciled' });
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
state.save({ currentWave: waveNum, status: 'running' });
|
|
434
|
+
writeEvent(projectDir, 'wave_start', { wave: waveNum, task_count: effectiveTasks.length });
|
|
435
|
+
emitWaveBoundary(teeWaveFrame, 'start', waveNum, effectiveTasks.length);
|
|
436
|
+
|
|
437
|
+
const waveStartedMs = Date.now();
|
|
438
|
+
const waveLiveChildren = new Map();
|
|
439
|
+
interrupt.currentWaveChildren = waveLiveChildren;
|
|
440
|
+
const { results, halted } = await runWaveTasks({
|
|
441
|
+
tasks: effectiveTasks,
|
|
442
|
+
config,
|
|
443
|
+
state,
|
|
444
|
+
logger,
|
|
445
|
+
spawnImpl,
|
|
446
|
+
runWorkerImpl,
|
|
447
|
+
onFrame,
|
|
448
|
+
interrupt,
|
|
449
|
+
liveChildrenRef: waveLiveChildren,
|
|
450
|
+
streamFeedFactory
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
const failed = results.filter((r) => !r.outcome.result.ok);
|
|
454
|
+
const waveDurationMs = Date.now() - waveStartedMs;
|
|
455
|
+
waveResults.push({ wave: waveNum, total: waveTasks.length, done: results.length - failed.length, failed: failed.length });
|
|
456
|
+
writeEvent(projectDir, failed.length ? 'wave_failed' : 'wave_done', {
|
|
457
|
+
wave: waveNum,
|
|
458
|
+
failed_count: failed.length
|
|
459
|
+
});
|
|
460
|
+
emitWaveBoundary(teeWaveFrame, failed.length ? 'failed' : 'done', waveNum, waveTasks.length, { durationMs: waveDurationMs, failed: failed.length });
|
|
461
|
+
|
|
462
|
+
if (interrupt.interrupted) break;
|
|
463
|
+
|
|
464
|
+
if (halted || failed.length) {
|
|
465
|
+
state.save({ status: 'failed' });
|
|
466
|
+
logger.log(`[wave_halt] wave=${waveNum} failed_tasks=${failed.length}`);
|
|
467
|
+
writeEvent(projectDir, 'orchestrator_halt', { wave: waveNum, failed_count: failed.length });
|
|
468
|
+
return { status: 'failed', waves: waveResults, failedWave: waveNum };
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
if (interrupt.interrupted) {
|
|
473
|
+
state.save({ status: 'interrupted', endedAt: nowIso() });
|
|
474
|
+
writeEvent(projectDir, 'orchestrator_done_interrupted', {});
|
|
475
|
+
return { status: 'interrupted', waves: waveResults };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
state.save({ status: 'done', endedAt: nowIso() });
|
|
479
|
+
writeEvent(projectDir, 'orchestrator_done', { waves: waveResults.length });
|
|
480
|
+
return { status: 'done', waves: waveResults };
|
|
481
|
+
} finally {
|
|
482
|
+
cleanup();
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
async function main() {
|
|
487
|
+
const args = parseCliArgs(process.argv.slice(2));
|
|
488
|
+
if (args.help) { printHelp(); process.exit(0); }
|
|
489
|
+
if (!args.milestone) {
|
|
490
|
+
process.stderr.write('Error: --milestone is required\n\n');
|
|
491
|
+
printHelp();
|
|
492
|
+
process.exit(2);
|
|
493
|
+
}
|
|
494
|
+
let streamFeedOpt = args.streamFeed;
|
|
495
|
+
if (streamFeedOpt) {
|
|
496
|
+
if (args.streamFeedPort || args.streamFeedHost) {
|
|
497
|
+
streamFeedOpt = {};
|
|
498
|
+
if (args.streamFeedPort) streamFeedOpt.port = args.streamFeedPort;
|
|
499
|
+
if (args.streamFeedHost) streamFeedOpt.host = args.streamFeedHost;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
try {
|
|
503
|
+
const res = await runOrchestrator({
|
|
504
|
+
projectDir: args.projectDir,
|
|
505
|
+
milestone: args.milestone,
|
|
506
|
+
maxParallel: args.maxParallel,
|
|
507
|
+
workerTimeoutMs: args.workerTimeoutMs,
|
|
508
|
+
streamFeed: streamFeedOpt,
|
|
509
|
+
resume: args.resume,
|
|
510
|
+
noArchive: args.noArchive
|
|
511
|
+
});
|
|
512
|
+
if (res.status === 'interrupted') {
|
|
513
|
+
process.exit(130);
|
|
514
|
+
}
|
|
515
|
+
if (res.status === 'failed') {
|
|
516
|
+
process.exit(1);
|
|
517
|
+
}
|
|
518
|
+
process.exit(0);
|
|
519
|
+
} catch (err) {
|
|
520
|
+
process.stderr.write(`[orchestrator] ${err.message}\n`);
|
|
521
|
+
process.exit(1);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
if (require.main === module) {
|
|
526
|
+
main();
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
module.exports = {
|
|
530
|
+
runOrchestrator,
|
|
531
|
+
runWaveTasks,
|
|
532
|
+
parseCliArgs,
|
|
533
|
+
atomicWriteJson
|
|
534
|
+
};
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* GSD-T Stream Feed Client (M40 D4-T3)
|
|
4
|
+
*
|
|
5
|
+
* Orchestrator/worker-side push helper. Opens a single keep-alive chunked
|
|
6
|
+
* HTTP POST to /ingest?workerPid=&taskId=; each pushFrame writes one JSON line.
|
|
7
|
+
* On server unreachable: spools to .gsd-t/stream-feed/spool-{pid}.jsonl and
|
|
8
|
+
* stays in spool mode. close() flushes + ends the HTTP stream.
|
|
9
|
+
*
|
|
10
|
+
* Contract: .gsd-t/contracts/stream-json-sink-contract.md
|
|
11
|
+
* Zero external deps — node http + fs only.
|
|
12
|
+
*/
|
|
13
|
+
const http = require('http');
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const path = require('path');
|
|
16
|
+
|
|
17
|
+
const DEFAULT_PORT = 7842;
|
|
18
|
+
|
|
19
|
+
function createStreamFeedClient(opts = {}) {
|
|
20
|
+
const port = Number(opts.port || process.env.GSD_T_STREAM_FEED_PORT || DEFAULT_PORT);
|
|
21
|
+
const host = opts.host || '127.0.0.1';
|
|
22
|
+
const projectDir = opts.projectDir || process.cwd();
|
|
23
|
+
const workerPid = opts.workerPid || process.pid;
|
|
24
|
+
const taskId = opts.taskId || '';
|
|
25
|
+
const spoolDir = path.join(projectDir, '.gsd-t', 'stream-feed');
|
|
26
|
+
const spoolPath = path.join(spoolDir, `spool-${workerPid}.jsonl`);
|
|
27
|
+
const httpImpl = opts.httpImpl || http;
|
|
28
|
+
|
|
29
|
+
let req = null;
|
|
30
|
+
let spooling = false;
|
|
31
|
+
let closed = false;
|
|
32
|
+
let spoolStream = null;
|
|
33
|
+
const pendingLines = []; // lines written to req before confirmed delivery
|
|
34
|
+
const stats = { pushed: 0, spooled: 0, dropped: 0 };
|
|
35
|
+
|
|
36
|
+
function ensureSpoolDir() {
|
|
37
|
+
try { fs.mkdirSync(spoolDir, { recursive: true }); } catch { /* exists */ }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function switchToSpool(reason) {
|
|
41
|
+
if (spooling) return;
|
|
42
|
+
spooling = true;
|
|
43
|
+
ensureSpoolDir();
|
|
44
|
+
try {
|
|
45
|
+
spoolStream = fs.createWriteStream(spoolPath, { flags: 'a' });
|
|
46
|
+
} catch (e) {
|
|
47
|
+
spoolStream = null;
|
|
48
|
+
}
|
|
49
|
+
try { process.stderr.write(`[stream-feed-client] switching to spool mode (${reason}) → ${spoolPath}\n`); } catch { /* noop */ }
|
|
50
|
+
// Flush any pending lines (written to req but never ack'd) into spool.
|
|
51
|
+
while (pendingLines.length > 0) {
|
|
52
|
+
writeToSpool(pendingLines.shift());
|
|
53
|
+
if (stats.pushed > 0) stats.pushed -= 1;
|
|
54
|
+
}
|
|
55
|
+
if (req) {
|
|
56
|
+
try { req.destroy(); } catch { /* noop */ }
|
|
57
|
+
req = null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function openRequest() {
|
|
62
|
+
if (closed || spooling) return;
|
|
63
|
+
try {
|
|
64
|
+
req = httpImpl.request({
|
|
65
|
+
host, port,
|
|
66
|
+
method: 'POST',
|
|
67
|
+
path: `/ingest?workerPid=${encodeURIComponent(String(workerPid))}&taskId=${encodeURIComponent(String(taskId))}`,
|
|
68
|
+
headers: {
|
|
69
|
+
'Content-Type': 'application/x-ndjson',
|
|
70
|
+
'Transfer-Encoding': 'chunked',
|
|
71
|
+
'Connection': 'keep-alive',
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
req.on('error', (err) => {
|
|
75
|
+
switchToSpool(err.code || err.message || 'http-error');
|
|
76
|
+
});
|
|
77
|
+
req.on('socket', (sock) => {
|
|
78
|
+
// Clear pending once the socket is actually usable (connected).
|
|
79
|
+
sock.once('connect', () => { pendingLines.length = 0; });
|
|
80
|
+
});
|
|
81
|
+
req.on('response', (res) => {
|
|
82
|
+
pendingLines.length = 0; // response means server accepted delivery
|
|
83
|
+
res.resume();
|
|
84
|
+
});
|
|
85
|
+
} catch (e) {
|
|
86
|
+
switchToSpool('request-ctor-error');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function writeToSpool(line) {
|
|
91
|
+
ensureSpoolDir();
|
|
92
|
+
if (!spoolStream) {
|
|
93
|
+
try { spoolStream = fs.createWriteStream(spoolPath, { flags: 'a' }); }
|
|
94
|
+
catch { stats.dropped += 1; return; }
|
|
95
|
+
}
|
|
96
|
+
try { spoolStream.write(line + '\n'); stats.spooled += 1; }
|
|
97
|
+
catch { stats.dropped += 1; }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function pushFrame(frame) {
|
|
101
|
+
if (closed) return;
|
|
102
|
+
let line;
|
|
103
|
+
if (typeof frame === 'string') line = frame;
|
|
104
|
+
else {
|
|
105
|
+
try { line = JSON.stringify(frame); }
|
|
106
|
+
catch { stats.dropped += 1; return; }
|
|
107
|
+
}
|
|
108
|
+
if (spooling) { writeToSpool(line); return; }
|
|
109
|
+
if (!req) openRequest();
|
|
110
|
+
if (spooling) { writeToSpool(line); return; }
|
|
111
|
+
try {
|
|
112
|
+
pendingLines.push(line);
|
|
113
|
+
const ok = req.write(line + '\n');
|
|
114
|
+
if (!ok) {
|
|
115
|
+
// Drain if needed; we don't backpressure the caller.
|
|
116
|
+
}
|
|
117
|
+
stats.pushed += 1;
|
|
118
|
+
} catch (e) {
|
|
119
|
+
switchToSpool('write-error');
|
|
120
|
+
writeToSpool(line);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function close() {
|
|
125
|
+
if (closed) return Promise.resolve();
|
|
126
|
+
closed = true;
|
|
127
|
+
return new Promise((resolve) => {
|
|
128
|
+
const done = () => resolve();
|
|
129
|
+
if (req && !spooling) {
|
|
130
|
+
try { req.end(done); }
|
|
131
|
+
catch { done(); }
|
|
132
|
+
} else if (spoolStream) {
|
|
133
|
+
try { spoolStream.end(done); }
|
|
134
|
+
catch { done(); }
|
|
135
|
+
} else {
|
|
136
|
+
done();
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return {
|
|
142
|
+
pushFrame,
|
|
143
|
+
close,
|
|
144
|
+
get mode() { return closed ? 'closed' : (spooling ? 'spool' : 'http'); },
|
|
145
|
+
get stats() { return { ...stats }; },
|
|
146
|
+
get spoolPath() { return spoolPath; },
|
|
147
|
+
_internal: { switchToSpool },
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
module.exports = { createStreamFeedClient };
|