@gh-symphony/cli 0.0.21 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +100 -69
  2. package/dist/chunk-6I753NYO.js +18 -0
  3. package/dist/{workflow-BLJH2HC3.js → chunk-B4ZJMAZL.js} +27 -19
  4. package/dist/{chunk-SXGT7LOF.js → chunk-DLZAJXZL.js} +600 -12
  5. package/dist/chunk-GHVDABFO.js +235 -0
  6. package/dist/{chunk-QEONJ5DZ.js → chunk-GPRCOJDJ.js} +1314 -35
  7. package/dist/{chunk-A67CMOYE.js → chunk-VFHMHHZW.js} +1 -1
  8. package/dist/{chunk-JN3TQVFV.js → chunk-WM2B6BJ7.js} +16 -62
  9. package/dist/{chunk-ROGRTUFI.js → chunk-WOVNN5NW.js} +16 -6
  10. package/dist/{chunk-C67H3OUL.js → chunk-Z3NZOPLZ.js} +0 -81
  11. package/dist/{config-cmd-DNXNL26Z.js → config-cmd-2ADPUYWA.js} +1 -1
  12. package/dist/{doctor-4HBRICHP.js → doctor-EEPNFCGF.js} +464 -40
  13. package/dist/index.js +357 -244
  14. package/dist/repo-RX4OK7XH.js +6783 -0
  15. package/dist/{setup-B2SVLW2R.js → setup-XNHHRBGU.js} +57 -91
  16. package/dist/{upgrade-OJXPZRYE.js → upgrade-NS53EO2B.js} +2 -2
  17. package/dist/{version-TBDCTKDO.js → version-2RHFZ5CI.js} +1 -1
  18. package/dist/worker-entry.js +376 -15
  19. package/dist/workflow-26QNZZWH.js +22 -0
  20. package/package.json +5 -5
  21. package/dist/chunk-5NV3LSAJ.js +0 -11
  22. package/dist/chunk-C7G7RJ4G.js +0 -146
  23. package/dist/chunk-KY6WKH66.js +0 -1300
  24. package/dist/chunk-MYVJ6HK4.js +0 -3510
  25. package/dist/chunk-S6VIK4FF.js +0 -723
  26. package/dist/chunk-XN5ABWZ6.js +0 -486
  27. package/dist/chunk-Y6TYJMNT.js +0 -109
  28. package/dist/init-HZ3JEDGQ.js +0 -38
  29. package/dist/logs-6JKKYDGJ.js +0 -188
  30. package/dist/project-25NQ4J4Y.js +0 -24
  31. package/dist/recover-L3MJHHDA.js +0 -133
  32. package/dist/repo-TDCWQR6P.js +0 -379
  33. package/dist/run-XJQ6BF7U.js +0 -110
  34. package/dist/start-I2CC7BLW.js +0 -18
  35. package/dist/status-QSCFVGRQ.js +0 -11
  36. package/dist/stop-7MFCBQVW.js +0 -9
@@ -1,3510 +0,0 @@
1
- #!/usr/bin/env node
2
- import {
3
- resolveTrackerAdapter
4
- } from "./chunk-SXGT7LOF.js";
5
- import {
6
- DEFAULT_MAX_FAILURE_RETRIES,
7
- DEFAULT_WORKFLOW_LIFECYCLE,
8
- WorkflowConfigStore,
9
- buildHookEnv,
10
- buildProjectSnapshot,
11
- buildPromptVariables,
12
- createDefaultWorkflowResolution,
13
- createInvalidWorkflowResolution,
14
- deriveIssueWorkspaceKey,
15
- deriveIssueWorkspaceKeyFromIdentifier,
16
- deriveLegacyIssueWorkspaceKey,
17
- executeWorkspaceHook,
18
- isFileMissing,
19
- isMatchingIssueRun,
20
- isOrchestratorChannelEvent,
21
- isStateActive,
22
- isStateTerminal,
23
- mapIssueOrchestrationStateToStatus,
24
- matchesWorkflowState,
25
- parseRecentEvents,
26
- readEnvFile,
27
- readJsonFile,
28
- renderPrompt,
29
- resolveIssueWorkspaceDirectory,
30
- resolveWorkflowRuntimeCommand,
31
- resolveWorkflowRuntimeTimeouts,
32
- safeReadDir,
33
- scheduleRetryAt
34
- } from "./chunk-QEONJ5DZ.js";
35
-
36
- // ../orchestrator/src/service.ts
37
- import { mkdir as mkdir3, readFile as readFile3, rm as rm3, writeFile as writeFile3 } from "fs/promises";
38
- import { createWriteStream, mkdirSync } from "fs";
39
- import { spawn as spawn2 } from "child_process";
40
- import { join as join3 } from "path";
41
- import { StringDecoder } from "string_decoder";
42
- import { fileURLToPath } from "url";
43
-
44
- // ../orchestrator/src/git.ts
45
- import { spawn } from "child_process";
46
- import { randomUUID } from "crypto";
47
- import {
48
- access,
49
- mkdir,
50
- readFile,
51
- rename,
52
- rm,
53
- stat,
54
- writeFile
55
- } from "fs/promises";
56
- import { constants } from "fs";
57
- import { join } from "path";
58
- var workflowConfigStore = new WorkflowConfigStore();
59
- var LOCK_RETRY_MS = 100;
60
- var LOCK_STALE_MS = 30 * 60 * 1e3;
61
- var LOCK_TIMEOUT_MS = 2 * 60 * 1e3;
62
- async function cloneRepositoryForRun(input) {
63
- const result = await syncRepositoryForRun(input);
64
- return result.repositoryDirectory;
65
- }
66
- async function syncRepositoryForRun(input) {
67
- await mkdir(input.targetDirectory, { recursive: true });
68
- const repositoryDirectory = join(input.targetDirectory, "repository");
69
- const lockDirectory = join(input.targetDirectory, "repository.lock");
70
- return withRepositoryLock(lockDirectory, async () => {
71
- let hasGit = false;
72
- try {
73
- await access(join(repositoryDirectory, ".git"), constants.R_OK);
74
- hasGit = true;
75
- } catch {
76
- }
77
- if (hasGit) {
78
- try {
79
- const beforeHead = await readGitHead(repositoryDirectory);
80
- await runCommand("git", [
81
- "-C",
82
- repositoryDirectory,
83
- "pull",
84
- "--ff-only"
85
- ]);
86
- const afterHead = await readGitHead(repositoryDirectory);
87
- return {
88
- repositoryDirectory,
89
- changed: beforeHead !== afterHead
90
- };
91
- } catch {
92
- await rm(repositoryDirectory, { recursive: true, force: true });
93
- }
94
- } else {
95
- await rm(repositoryDirectory, { recursive: true, force: true });
96
- }
97
- const tempRepositoryDirectory = join(
98
- input.targetDirectory,
99
- `repository.tmp-${process.pid}-${Date.now()}`
100
- );
101
- await rm(tempRepositoryDirectory, { recursive: true, force: true });
102
- try {
103
- await runCommand("git", [
104
- "clone",
105
- "--depth",
106
- "1",
107
- input.repository.cloneUrl,
108
- tempRepositoryDirectory
109
- ]);
110
- await rename(tempRepositoryDirectory, repositoryDirectory);
111
- return {
112
- repositoryDirectory,
113
- changed: true
114
- };
115
- } finally {
116
- await rm(tempRepositoryDirectory, { recursive: true, force: true });
117
- }
118
- });
119
- }
120
- async function ensureIssueWorkspaceRepository(input) {
121
- return cloneRepositoryForRun({
122
- repository: input.repository,
123
- targetDirectory: input.issueWorkspacePath
124
- });
125
- }
126
- async function loadRepositoryWorkflow(repositoryDirectory, _repository) {
127
- const workflowPath = join(repositoryDirectory, "WORKFLOW.md");
128
- try {
129
- return await workflowConfigStore.load(workflowPath);
130
- } catch (error) {
131
- if (isMissingFileError(error)) {
132
- return createDefaultWorkflowResolution();
133
- }
134
- return createInvalidWorkflowResolution(
135
- workflowPath,
136
- error instanceof Error ? error.message : "workflow_parse_error"
137
- );
138
- }
139
- }
140
- function runCommand(command, args) {
141
- return new Promise((resolve4, reject) => {
142
- const child = spawn(command, args, {
143
- stdio: "pipe"
144
- });
145
- let stderr = "";
146
- child.stderr?.on("data", (chunk) => {
147
- stderr += String(chunk);
148
- });
149
- child.once("error", reject);
150
- child.once("exit", (code) => {
151
- if (code === 0) {
152
- resolve4();
153
- return;
154
- }
155
- reject(
156
- new Error(
157
- stderr.trim() || `${command} exited with code ${code ?? "unknown"}`
158
- )
159
- );
160
- });
161
- });
162
- }
163
- async function readGitHead(repositoryDirectory) {
164
- try {
165
- return await runCommandCapture("git", [
166
- "-C",
167
- repositoryDirectory,
168
- "rev-parse",
169
- "HEAD"
170
- ]);
171
- } catch {
172
- return null;
173
- }
174
- }
175
- function runCommandCapture(command, args) {
176
- return new Promise((resolve4, reject) => {
177
- const child = spawn(command, args, {
178
- stdio: ["ignore", "pipe", "pipe"]
179
- });
180
- let stdout = "";
181
- let stderr = "";
182
- child.stdout?.on("data", (chunk) => {
183
- stdout += String(chunk);
184
- });
185
- child.stderr?.on("data", (chunk) => {
186
- stderr += String(chunk);
187
- });
188
- child.once("error", reject);
189
- child.once("exit", (code) => {
190
- if (code === 0) {
191
- resolve4(stdout.trim());
192
- return;
193
- }
194
- reject(
195
- new Error(
196
- stderr.trim() || `${command} exited with code ${code ?? "unknown"}`
197
- )
198
- );
199
- });
200
- });
201
- }
202
- async function withRepositoryLock(lockDirectory, fn) {
203
- const ownerToken = await acquireRepositoryLock(lockDirectory);
204
- try {
205
- return await fn();
206
- } finally {
207
- await releaseRepositoryLock(lockDirectory, ownerToken);
208
- }
209
- }
210
- async function acquireRepositoryLock(lockDirectory) {
211
- const startedAt = Date.now();
212
- const ownerToken = `${process.pid}:${randomUUID()}`;
213
- for (; ; ) {
214
- try {
215
- await mkdir(lockDirectory);
216
- await writeFile(
217
- join(lockDirectory, "owner"),
218
- `${ownerToken}
219
- ${(/* @__PURE__ */ new Date()).toISOString()}
220
- `,
221
- "utf8"
222
- );
223
- return ownerToken;
224
- } catch (error) {
225
- if (!isAlreadyExistsError(error)) {
226
- throw error;
227
- }
228
- }
229
- const stale = await isStaleLock(lockDirectory);
230
- if (stale) {
231
- await rm(lockDirectory, { recursive: true, force: true });
232
- continue;
233
- }
234
- if (Date.now() - startedAt >= LOCK_TIMEOUT_MS) {
235
- throw new Error(
236
- `Timed out waiting for repository cache lock: ${lockDirectory}`
237
- );
238
- }
239
- await wait(LOCK_RETRY_MS);
240
- }
241
- }
242
- async function releaseRepositoryLock(lockDirectory, ownerToken) {
243
- try {
244
- const owner = await readLockOwner(lockDirectory);
245
- if (owner !== ownerToken) {
246
- return;
247
- }
248
- } catch (error) {
249
- if (isMissingFileError(error)) {
250
- return;
251
- }
252
- throw error;
253
- }
254
- await rm(lockDirectory, { recursive: true, force: true });
255
- }
256
- async function isStaleLock(lockDirectory) {
257
- try {
258
- const details = await stat(lockDirectory);
259
- return Date.now() - details.mtimeMs >= LOCK_STALE_MS;
260
- } catch (error) {
261
- if (isMissingFileError(error)) {
262
- return false;
263
- }
264
- throw error;
265
- }
266
- }
267
- function isAlreadyExistsError(error) {
268
- return Boolean(
269
- error && typeof error === "object" && "code" in error && error.code === "EEXIST"
270
- );
271
- }
272
- async function readLockOwner(lockDirectory) {
273
- await access(join(lockDirectory, "owner"), constants.R_OK);
274
- const owner = await readFile(join(lockDirectory, "owner"), "utf8");
275
- return owner.split("\n", 1)[0] || null;
276
- }
277
- function wait(ms) {
278
- return new Promise((resolve4) => {
279
- setTimeout(resolve4, ms);
280
- });
281
- }
282
- function isMissingFileError(error) {
283
- return Boolean(
284
- error && typeof error === "object" && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR")
285
- );
286
- }
287
-
288
- // ../orchestrator/src/fs-store.ts
289
- import {
290
- mkdir as mkdir2,
291
- open,
292
- rename as rename2,
293
- rm as rm2,
294
- stat as stat2,
295
- writeFile as writeFile2,
296
- appendFile
297
- } from "fs/promises";
298
- import { dirname, join as join2, relative, resolve } from "path";
299
- var OrchestratorFsStore = class {
300
- constructor(runtimeRoot, options = {}) {
301
- this.runtimeRoot = runtimeRoot;
302
- this.resolvedRuntimeRoot = resolve(runtimeRoot);
303
- this.resolvedEventsMirrorRoot = options.eventsMirrorRoot ? resolve(options.eventsMirrorRoot) : null;
304
- }
305
- resolvedRuntimeRoot;
306
- resolvedEventsMirrorRoot;
307
- projectsRoot() {
308
- return join2(this.runtimeRoot, "projects");
309
- }
310
- projectDir(projectId) {
311
- return join2(this.projectsRoot(), projectId);
312
- }
313
- projectRunsDir(projectId) {
314
- return join2(this.projectDir(projectId), "runs");
315
- }
316
- runDir(runId, projectId) {
317
- if (!projectId) {
318
- return join2(this.runtimeRoot, "projects", "__unknown__", "runs", runId);
319
- }
320
- return join2(this.projectRunsDir(projectId), runId);
321
- }
322
- async loadProjectConfig(projectId) {
323
- return readJsonFile(
324
- join2(this.projectDir(projectId), "project.json")
325
- );
326
- }
327
- async saveProjectConfig(config) {
328
- await writeJsonFile(
329
- join2(this.projectDir(config.projectId), "project.json"),
330
- config
331
- );
332
- }
333
- async loadProjectIssueOrchestrations(projectId) {
334
- const issuesPath = join2(this.projectDir(projectId), "issues.json");
335
- const issues = await readJsonFile(issuesPath);
336
- if (issues) {
337
- return issues.map((issue) => ({
338
- ...issue,
339
- completedOnce: issue.completedOnce ?? false,
340
- failureRetryCount: issue.failureRetryCount ?? 0
341
- }));
342
- }
343
- const legacyLeases = await readJsonFile(join2(this.projectDir(projectId), "leases.json")) ?? [];
344
- if (legacyLeases.length === 0) {
345
- return [];
346
- }
347
- const migratedIssues = legacyLeases.map(
348
- (lease) => ({
349
- issueId: lease.issueId,
350
- identifier: lease.issueIdentifier,
351
- workspaceKey: deriveIssueWorkspaceKeyFromIdentifier(
352
- lease.issueIdentifier
353
- ),
354
- completedOnce: false,
355
- failureRetryCount: 0,
356
- state: lease.status === "active" ? "claimed" : "released",
357
- currentRunId: lease.status === "active" ? lease.runId : null,
358
- retryEntry: null,
359
- updatedAt: lease.updatedAt
360
- })
361
- );
362
- await this.saveProjectIssueOrchestrations(projectId, migratedIssues);
363
- return migratedIssues;
364
- }
365
- async saveProjectIssueOrchestrations(projectId, issues) {
366
- await writeJsonFile(
367
- join2(this.projectDir(projectId), "issues.json"),
368
- issues
369
- );
370
- }
371
- async saveProjectStatus(status) {
372
- await writeJsonFile(
373
- join2(this.projectDir(status.projectId), "status.json"),
374
- status
375
- );
376
- }
377
- async loadProjectStatus(projectId) {
378
- return await readJsonFile(
379
- join2(this.projectDir(projectId), "status.json")
380
- ) ?? null;
381
- }
382
- async loadRun(runId, projectId) {
383
- const runDirectory = projectId !== void 0 ? this.runDir(runId, projectId) : await this.findRunDir(runId);
384
- if (!runDirectory) {
385
- return null;
386
- }
387
- return await readJsonFile(
388
- join2(runDirectory, "run.json")
389
- ) ?? null;
390
- }
391
- async loadAllRuns() {
392
- const projectIds = await safeReadDir(this.projectsRoot());
393
- const runDirectories = await Promise.all(
394
- projectIds.map(async (projectId) => {
395
- const entries = await safeReadDir(this.projectRunsDir(projectId));
396
- return entries.map((entry) => this.runDir(entry, projectId));
397
- })
398
- );
399
- const runs = await Promise.all(
400
- runDirectories.flat().map(
401
- (directory) => readJsonFile(join2(directory, "run.json"))
402
- )
403
- );
404
- return runs.filter((run) => Boolean(run));
405
- }
406
- async saveRun(run) {
407
- await writeJsonFile(
408
- join2(this.runDir(run.runId, run.projectId), "run.json"),
409
- run
410
- );
411
- }
412
- async appendRunEvent(runId, event) {
413
- const resolvedProjectId = "projectId" in event && typeof event.projectId === "string" ? event.projectId : void 0;
414
- const runDirectory = resolvedProjectId !== void 0 ? this.runDir(runId, resolvedProjectId) : await this.findRunDir(runId);
415
- if (!runDirectory) {
416
- throw new Error(
417
- `Unable to resolve run directory for event append: ${runId}`
418
- );
419
- }
420
- const path = join2(runDirectory, "events.ndjson");
421
- const resolvedPath = resolve(path);
422
- const serializedEvent = JSON.stringify(event) + "\n";
423
- await mkdir2(dirname(path), { recursive: true });
424
- await appendFile(path, serializedEvent, {
425
- encoding: "utf8",
426
- mode: 420
427
- });
428
- const mirrorPath = this.resolveMirroredEventsPath(resolvedPath);
429
- if (!mirrorPath) {
430
- return;
431
- }
432
- try {
433
- await mkdir2(dirname(mirrorPath), { recursive: true });
434
- await appendFile(mirrorPath, serializedEvent, {
435
- encoding: "utf8",
436
- mode: 420
437
- });
438
- } catch (error) {
439
- console.warn(
440
- `Failed to mirror orchestrator event log to ${mirrorPath}: ${error instanceof Error ? error.message : String(error)}`
441
- );
442
- }
443
- }
444
- async loadRecentRunEvents(runId, limit = 20, projectId) {
445
- const runDirectory = projectId !== void 0 ? this.runDir(runId, projectId) : await this.findRunDir(runId);
446
- if (!runDirectory) {
447
- return [];
448
- }
449
- const path = join2(runDirectory, "events.ndjson");
450
- try {
451
- if (limit <= 0) {
452
- return [];
453
- }
454
- const handle = await open(path, "r");
455
- try {
456
- const stats = await handle.stat();
457
- let position = stats.size;
458
- let tail = Buffer.alloc(0);
459
- while (position > 0) {
460
- const readSize = Math.min(position, 4096);
461
- position -= readSize;
462
- const chunk = Buffer.allocUnsafe(readSize);
463
- await handle.read(chunk, 0, readSize, position);
464
- tail = Buffer.concat([chunk, tail]);
465
- const events = parseRecentEvents(tail.toString("utf8"), limit, {
466
- allowPartialFirstLine: position > 0
467
- });
468
- if (events.length >= limit) {
469
- return events;
470
- }
471
- }
472
- return parseRecentEvents(tail.toString("utf8"), limit, {
473
- allowPartialFirstLine: false
474
- });
475
- } finally {
476
- await handle.close();
477
- }
478
- } catch (error) {
479
- if (isFileMissing(error)) {
480
- return [];
481
- }
482
- throw error;
483
- }
484
- }
485
- issueWorkspaceDir(projectId, workspaceKey) {
486
- return join2(this.projectDir(projectId), "issues", workspaceKey);
487
- }
488
- async loadIssueWorkspace(projectId, workspaceKey) {
489
- return await readJsonFile(
490
- join2(this.issueWorkspaceDir(projectId, workspaceKey), "workspace.json")
491
- ) ?? null;
492
- }
493
- async loadIssueWorkspaces(projectId) {
494
- const issuesDir = join2(this.projectDir(projectId), "issues");
495
- const entries = await safeReadDir(issuesDir);
496
- const records = await Promise.all(
497
- entries.map((entry) => this.loadIssueWorkspace(projectId, entry))
498
- );
499
- return records.filter(
500
- (record) => Boolean(record)
501
- );
502
- }
503
- async saveIssueWorkspace(record) {
504
- await writeJsonFile(
505
- join2(
506
- this.issueWorkspaceDir(record.projectId, record.workspaceKey),
507
- "workspace.json"
508
- ),
509
- record
510
- );
511
- }
512
- async removeIssueWorkspace(projectId, workspaceKey) {
513
- const dir = this.issueWorkspaceDir(projectId, workspaceKey);
514
- await rm2(dir, { recursive: true, force: true });
515
- }
516
- async findRunDir(runId) {
517
- const projectIds = await safeReadDir(this.projectsRoot());
518
- for (const projectId of projectIds) {
519
- const candidate = this.runDir(runId, projectId);
520
- const run = await readJsonFile(
521
- join2(candidate, "run.json")
522
- );
523
- if (run || await pathExists(join2(candidate, "events.ndjson"))) {
524
- return candidate;
525
- }
526
- }
527
- return null;
528
- }
529
- resolveMirroredEventsPath(primaryPath) {
530
- if (!this.resolvedEventsMirrorRoot) {
531
- return null;
532
- }
533
- const relativePath = relative(this.resolvedRuntimeRoot, primaryPath);
534
- if (relativePath.startsWith("..")) {
535
- return null;
536
- }
537
- const mirrorPath = join2(this.resolvedEventsMirrorRoot, relativePath);
538
- return mirrorPath === primaryPath ? null : mirrorPath;
539
- }
540
- };
541
- async function writeJsonFile(path, value) {
542
- await mkdir2(dirname(path), { recursive: true });
543
- const temporaryPath = `${path}.tmp`;
544
- await writeFile2(temporaryPath, JSON.stringify(value, null, 2) + "\n", "utf8");
545
- await rename2(temporaryPath, path);
546
- }
547
- async function pathExists(path) {
548
- try {
549
- await stat2(path);
550
- return true;
551
- } catch (error) {
552
- if (isFileMissing(error)) {
553
- return false;
554
- }
555
- throw error;
556
- }
557
- }
558
-
559
- // ../tracker-file/src/file-tracker-adapter.ts
560
- import { readFile as readFile2 } from "fs/promises";
561
- function requireTrackerSetting(project, key) {
562
- const value = project.tracker.settings?.[key];
563
- if (typeof value !== "string" || value.length === 0) {
564
- throw new Error(
565
- `Tracker adapter "file" requires the "${key}" setting.`
566
- );
567
- }
568
- return value;
569
- }
570
- function parseIssueNumber(identifier) {
571
- const match = identifier.match(/#(\d+)$/);
572
- return match ? Number.parseInt(match[1] ?? "0", 10) : 0;
573
- }
574
- function isValidIssueShape(entry) {
575
- if (!entry || typeof entry !== "object") return false;
576
- const e = entry;
577
- return typeof e.id === "string" && typeof e.identifier === "string" && typeof e.state === "string" && e.repository !== null && typeof e.repository === "object" && e.tracker !== null && typeof e.tracker === "object";
578
- }
579
- var fileTrackerAdapter = {
580
- async listIssues(project) {
581
- const issuesPath = requireTrackerSetting(project, "issuesPath");
582
- try {
583
- const raw = await readFile2(issuesPath, "utf-8");
584
- const parsed = JSON.parse(raw);
585
- if (!Array.isArray(parsed)) {
586
- throw new Error(
587
- `Expected an array of issues in ${issuesPath}, got ${typeof parsed}`
588
- );
589
- }
590
- const valid = [];
591
- for (let i = 0; i < parsed.length; i++) {
592
- if (isValidIssueShape(parsed[i])) {
593
- valid.push(parsed[i]);
594
- } else {
595
- process.stderr.write(
596
- `[tracker-file] Skipping invalid issue at index ${i} in ${issuesPath}
597
- `
598
- );
599
- }
600
- }
601
- return valid;
602
- } catch (err) {
603
- if (err instanceof Error && "code" in err && err.code === "ENOENT") {
604
- return [];
605
- }
606
- if (err instanceof SyntaxError) {
607
- return [];
608
- }
609
- throw err;
610
- }
611
- },
612
- async listIssuesByStates(project, states) {
613
- if (states.length === 0) {
614
- return [];
615
- }
616
- const issues = await this.listIssues(project);
617
- const normalizedStates = new Set(
618
- states.map((state) => state.trim().toLowerCase())
619
- );
620
- return issues.filter(
621
- (issue) => normalizedStates.has(issue.state.trim().toLowerCase())
622
- );
623
- },
624
- async fetchIssueStatesByIds(project, issueIds) {
625
- if (issueIds.length === 0) {
626
- return [];
627
- }
628
- const issues = await this.listIssues(project);
629
- const ids = new Set(issueIds);
630
- return issues.filter((issue) => ids.has(issue.id));
631
- },
632
- buildWorkerEnvironment(_project, _issue) {
633
- return {
634
- SYMPHONY_FILE_TRACKER: "true"
635
- };
636
- },
637
- reviveIssue(project, run) {
638
- return {
639
- id: run.issueId,
640
- identifier: run.issueIdentifier,
641
- number: parseIssueNumber(run.issueIdentifier),
642
- title: run.issueTitle ?? run.issueIdentifier,
643
- description: null,
644
- priority: null,
645
- state: run.issueState,
646
- branchName: null,
647
- url: null,
648
- labels: [],
649
- blockedBy: [],
650
- createdAt: null,
651
- updatedAt: null,
652
- repository: run.repository,
653
- tracker: {
654
- adapter: "file",
655
- bindingId: project.tracker.bindingId,
656
- itemId: run.issueId
657
- },
658
- metadata: {}
659
- };
660
- }
661
- };
662
-
663
- // ../orchestrator/src/tracker-adapters.ts
664
- var localAdapters = /* @__PURE__ */ new Map([
665
- ["file", fileTrackerAdapter]
666
- ]);
667
- function resolveTrackerAdapter2(tracker) {
668
- const local = localAdapters.get(tracker.adapter);
669
- if (local) return local;
670
- return resolveTrackerAdapter(tracker);
671
- }
672
-
673
- // ../orchestrator/src/service.ts
674
- var DEFAULT_POLL_INTERVAL_MS = 3e4;
675
- var DEFAULT_CONCURRENCY = 3;
676
- var DEFAULT_RETRY_BACKOFF_MS = 3e4;
677
- var CONTINUATION_RETRY_DELAY_MS = 1e3;
678
- var DEFAULT_WORKER_COMMAND = "node packages/worker/dist/index.js";
679
- var DEFAULT_MAX_NONPRODUCTIVE_TURNS = 3;
680
- var LOW_RATE_LIMIT_WARNING_THRESHOLD = 0.05;
681
- var MAX_FAILURE_RETRIES_EXCEEDED_REASON = "max_failure_retries_exceeded";
682
- var STUCK_WORKER_TIMEOUT_MS = 30 * 60 * 1e3;
683
- function isUsableWorkflowResolution(resolution) {
684
- return resolution.isValid || resolution.usedLastKnownGood;
685
- }
686
- function parseTimestampMs(value) {
687
- if (!value) {
688
- return null;
689
- }
690
- const parsed = new Date(value).getTime();
691
- return Number.isFinite(parsed) ? parsed : null;
692
- }
693
- function parseFiniteNumber(value) {
694
- if (typeof value === "number" && Number.isFinite(value)) {
695
- return value;
696
- }
697
- if (typeof value === "string" && value.trim()) {
698
- const parsed = Number(value);
699
- return Number.isFinite(parsed) ? parsed : null;
700
- }
701
- return null;
702
- }
703
- var OrchestratorService = class {
704
- constructor(store, projectConfig, dependencies = {}) {
705
- this.store = store;
706
- this.projectConfig = projectConfig;
707
- this.dependencies = dependencies;
708
- }
709
- projectPollIntervals = /* @__PURE__ */ new Map();
710
- activeWorkerPids = /* @__PURE__ */ new Set();
711
- workerStderrBuffers = /* @__PURE__ */ new Map();
712
- workerStderrDecoders = /* @__PURE__ */ new Map();
713
- lastKnownGoodWorkflows = /* @__PURE__ */ new Map();
714
- lastReportedWorkflowErrors = /* @__PURE__ */ new Map();
715
- workflowResolutionCache = null;
716
- running = true;
717
- shuttingDown = false;
718
- shutdownPromise = null;
719
- sleepTimer = null;
720
- sleepResolver = null;
721
- reconcilePromise = Promise.resolve();
722
- reconcileRequested = false;
723
- async run(options = {}) {
724
- this.running = true;
725
- await this.runSerialized(
726
- () => this.performStartupCleanup(this.createTrackerDependencies())
727
- );
728
- while (this.running) {
729
- try {
730
- const snapshot = await this.runOnceInternal(
731
- options.issueIdentifier,
732
- this.createTrackerDependencies()
733
- );
734
- await this.notifyTick(snapshot);
735
- } catch (error) {
736
- if (options.once) {
737
- throw error;
738
- }
739
- this.writeStderr(
740
- `[orchestrator] run loop failed for ${this.projectConfig.projectId}: ${this.formatErrorMessage(error)}`
741
- );
742
- }
743
- if (options.once || !this.running) {
744
- return;
745
- }
746
- await this.waitForNextPoll();
747
- }
748
- }
749
- async runOnce(options = {}) {
750
- return this.runOnceInternal(
751
- options.issueIdentifier,
752
- this.createTrackerDependencies()
753
- );
754
- }
755
- async status() {
756
- return this.store.loadProjectStatus(this.projectConfig.projectId);
757
- }
758
- async statusForIssue(issueIdentifier) {
759
- const issueRecords = await this.store.loadProjectIssueOrchestrations(
760
- this.projectConfig.projectId
761
- );
762
- const issueRecord = issueRecords.find(
763
- (record) => record.identifier === issueIdentifier
764
- );
765
- if (!issueRecord) {
766
- return null;
767
- }
768
- const currentRunCandidate = issueRecord.currentRunId ? await this.store.loadRun(
769
- issueRecord.currentRunId,
770
- this.projectConfig.projectId
771
- ) : null;
772
- const currentRun = isMatchingIssueRun(
773
- currentRunCandidate,
774
- this.projectConfig.projectId,
775
- issueRecord.issueId,
776
- issueIdentifier
777
- ) ? currentRunCandidate : await this.findLatestRunForIssue(issueRecord.issueId, issueIdentifier);
778
- const recentEvents = currentRun === null ? [] : await this.store.loadRecentRunEvents(
779
- currentRun.runId,
780
- 20,
781
- currentRun.projectId
782
- );
783
- const latestEventMessage = recentEvents[recentEvents.length - 1]?.message ?? null;
784
- const currentAttempt = currentRun?.attempt ?? issueRecord.retryEntry?.attempt ?? 0;
785
- return {
786
- issue_identifier: issueRecord.identifier,
787
- issue_id: issueRecord.issueId,
788
- status: currentRun?.status ?? mapIssueOrchestrationStateToStatus(issueRecord.state),
789
- workspace: {
790
- path: currentRun?.workingDirectory ?? null
791
- },
792
- attempts: {
793
- restart_count: Math.max(0, currentAttempt - 1),
794
- current_retry_attempt: currentAttempt
795
- },
796
- running: currentRun === null ? null : {
797
- session_id: currentRun.runtimeSession?.sessionId ?? null,
798
- turn_count: currentRun.turnCount ?? null,
799
- state: currentRun.issueState ?? null,
800
- started_at: currentRun.startedAt ?? null,
801
- last_event: currentRun.lastEvent ?? null,
802
- last_message: latestEventMessage,
803
- last_event_at: currentRun.lastEventAt ?? null,
804
- tokens: currentRun.tokenUsage ? {
805
- input_tokens: currentRun.tokenUsage.inputTokens,
806
- output_tokens: currentRun.tokenUsage.outputTokens,
807
- total_tokens: currentRun.tokenUsage.totalTokens
808
- } : null
809
- },
810
- retry: currentRun?.nextRetryAt ?? issueRecord.retryEntry?.dueAt ? {
811
- due_at: currentRun?.nextRetryAt ?? issueRecord.retryEntry?.dueAt ?? "",
812
- kind: currentRun?.retryKind ?? null,
813
- error: currentRun?.lastError ?? issueRecord.retryEntry?.error ?? null
814
- } : null,
815
- logs: {
816
- codex_session_logs: currentRun === null ? [] : [
817
- {
818
- label: "worker",
819
- path: join3(
820
- this.store.runDir(currentRun.runId, currentRun.projectId),
821
- "worker.log"
822
- ),
823
- url: null
824
- }
825
- ]
826
- },
827
- recent_events: recentEvents,
828
- last_error: currentRun?.lastError ?? issueRecord.retryEntry?.error ?? null,
829
- tracked: {
830
- issue_orchestration_state: issueRecord.state,
831
- current_run_id: issueRecord.currentRunId,
832
- workspace_key: issueRecord.workspaceKey,
833
- run_phase: currentRun?.runPhase ?? null,
834
- execution_phase: currentRun?.executionPhase ?? null
835
- }
836
- };
837
- }
838
- async recover() {
839
- return this.runOnce();
840
- }
841
- requestReconcile() {
842
- this.reconcileRequested = true;
843
- this.cancelPendingSleep();
844
- }
845
- async shutdown() {
846
- if (this.shutdownPromise) {
847
- return this.shutdownPromise;
848
- }
849
- this.shuttingDown = true;
850
- this.shutdownPromise = (async () => {
851
- this.running = false;
852
- this.cancelPendingSleep();
853
- const workerPids = [...this.activeWorkerPids];
854
- for (const pid of workerPids) {
855
- this.sendSignal(pid, "SIGTERM");
856
- }
857
- if (workerPids.length === 0) {
858
- return;
859
- }
860
- let waitedMs = 0;
861
- while (this.activeWorkerPids.size > 0 && waitedMs < 1e4) {
862
- this.pruneExitedWorkerPids();
863
- if (this.activeWorkerPids.size === 0) {
864
- return;
865
- }
866
- await (this.dependencies.waitImpl ?? wait2)(100);
867
- waitedMs += 100;
868
- }
869
- for (const pid of [...this.activeWorkerPids]) {
870
- if (!this.isProcessRunning(pid)) {
871
- this.retireWorkerPid(pid);
872
- continue;
873
- }
874
- this.sendSignal(pid, "SIGKILL");
875
- this.retireWorkerPid(pid);
876
- }
877
- })();
878
- return this.shutdownPromise;
879
- }
880
- getEffectivePollIntervalMs() {
881
- if (this.dependencies.pollIntervalMs) {
882
- return this.dependencies.pollIntervalMs;
883
- }
884
- const configuredIntervals = [...this.projectPollIntervals.values()].filter(
885
- (value) => Number.isFinite(value) && value > 0
886
- );
887
- return configuredIntervals.length ? Math.min(...configuredIntervals) : DEFAULT_POLL_INTERVAL_MS;
888
- }
889
- async reconcileProject(tenant, issueIdentifier, trackerDependencies = {}) {
890
- const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
891
- const now = this.now();
892
- let lastError = null;
893
- let dispatched = 0;
894
- let suppressed = 0;
895
- let recovered = 0;
896
- let pollIntervalMs = DEFAULT_POLL_INTERVAL_MS;
897
- let rateLimits = null;
898
- let trackerRateLimits = null;
899
- let issueRecords = await this.store.loadProjectIssueOrchestrations(
900
- tenant.projectId
901
- );
902
- const allRuns = (await this.store.loadAllRuns()).filter(
903
- (run) => run.projectId === tenant.projectId
904
- );
905
- const activeRuns = allRuns.filter((run) => isActiveRunStatus(run.status));
906
- for (const run of activeRuns) {
907
- const outcome = await this.reconcileRun(
908
- tenant,
909
- run,
910
- issueRecords,
911
- trackerDependencies
912
- );
913
- issueRecords = outcome.issueRecords;
914
- if (outcome.recovered) {
915
- recovered += 1;
916
- }
917
- }
918
- const reconciledRuns = (await this.store.loadAllRuns()).filter(
919
- (run) => run.projectId === tenant.projectId && isActiveRunStatus(run.status)
920
- );
921
- const projectRunsAfterReconcile = (await this.store.loadAllRuns()).filter(
922
- (run) => run.projectId === tenant.projectId
923
- );
924
- rateLimits = resolveProjectRateLimits(reconciledRuns, []);
925
- try {
926
- pollIntervalMs = await this.loadProjectPollInterval(tenant);
927
- const currentActiveRuns = (await this.store.loadAllRuns()).filter(
928
- (run) => run.projectId === tenant.projectId && isActiveRunStatus(run.status)
929
- );
930
- const {
931
- runs: syncedActiveRuns,
932
- issuesByIdentifier: syncedIssuesByIdentifier
933
- } = await this.syncActiveRunIssueStates(
934
- tenant,
935
- trackerAdapter,
936
- currentActiveRuns,
937
- now
938
- );
939
- const issues = await trackerAdapter.listIssues(
940
- tenant,
941
- trackerDependencies
942
- );
943
- const filteredIssues = issueIdentifier ? issues.filter(
944
- (issue) => issue.identifier === issueIdentifier
945
- ) : issues;
946
- const { candidates: actionableCandidates, lifecycle } = await this.resolveActionableCandidates(tenant, filteredIssues);
947
- const trackedIssuesByIdentifier = new Map(
948
- syncedIssuesByIdentifier
949
- );
950
- for (const issue of filteredIssues) {
951
- const existing = trackedIssuesByIdentifier.get(issue.identifier);
952
- trackedIssuesByIdentifier.set(issue.identifier, {
953
- ...existing ?? issue,
954
- ...issue,
955
- rateLimits: issue.rateLimits ?? existing?.rateLimits ?? null
956
- });
957
- }
958
- for (const [identifier, issue] of syncedIssuesByIdentifier) {
959
- const existing = trackedIssuesByIdentifier.get(identifier);
960
- if (!existing) {
961
- trackedIssuesByIdentifier.set(identifier, issue);
962
- continue;
963
- }
964
- trackedIssuesByIdentifier.set(identifier, {
965
- ...issue,
966
- ...existing,
967
- rateLimits: existing.rateLimits ?? issue.rateLimits ?? null
968
- });
969
- }
970
- rateLimits = resolveProjectRateLimits(
971
- syncedActiveRuns,
972
- trackedIssuesByIdentifier.values()
973
- );
974
- trackerRateLimits = resolveTrackerRateLimits(
975
- trackedIssuesByIdentifier.values()
976
- );
977
- const concurrency = await this.getProjectConcurrency(tenant);
978
- const currentlyActive = issueRecords.filter(
979
- (record) => isIssueOrchestrationClaimed(record.state)
980
- ).length;
981
- const availableSlots = Math.max(0, concurrency - currentlyActive);
982
- const latestRunsByIssueId = buildLatestRunMapByIssueId(
983
- projectRunsAfterReconcile
984
- );
985
- const unscheduledCandidates = actionableCandidates.filter((issue) => {
986
- if (hasConvergenceLockedRun(
987
- projectRunsAfterReconcile,
988
- issue.id,
989
- issue.state,
990
- issue.updatedAt
991
- )) {
992
- return false;
993
- }
994
- return !issueRecords.some(
995
- (record) => record.issueId === issue.id && isIssueOrchestrationClaimed(record.state)
996
- );
997
- });
998
- const sortedCandidates = sortCandidatesForDispatch(unscheduledCandidates);
999
- const activeByState = /* @__PURE__ */ new Map();
1000
- for (const run of syncedActiveRuns) {
1001
- const state = run.issueState;
1002
- const count = activeByState.get(state) ?? 0;
1003
- activeByState.set(state, count + 1);
1004
- }
1005
- const maxConcurrentByState = await this.loadProjectMaxConcurrentByState(tenant);
1006
- let slotsRemaining = availableSlots;
1007
- for (const issue of sortedCandidates) {
1008
- if (this.shuttingDown) {
1009
- break;
1010
- }
1011
- if (slotsRemaining <= 0) break;
1012
- if (await this.isFailureRetrySuppressedIssue(
1013
- tenant,
1014
- issue,
1015
- issueRecords,
1016
- latestRunsByIssueId.get(issue.id) ?? null
1017
- )) {
1018
- continue;
1019
- }
1020
- const stateLimit = maxConcurrentByState[issue.state];
1021
- if (stateLimit !== void 0) {
1022
- const activeInState = activeByState.get(issue.state) ?? 0;
1023
- if (activeInState >= stateLimit) {
1024
- continue;
1025
- }
1026
- }
1027
- const preferredWorkspaceKey = deriveIssueWorkspaceKey(
1028
- {
1029
- projectId: tenant.projectId,
1030
- adapter: issue.tracker.adapter,
1031
- issueSubjectId: issue.id
1032
- },
1033
- issue.identifier
1034
- );
1035
- issueRecords = upsertIssueOrchestration(issueRecords, {
1036
- issueId: issue.id,
1037
- identifier: issue.identifier,
1038
- workspaceKey: preferredWorkspaceKey,
1039
- state: "claimed",
1040
- failureRetryCount: 0,
1041
- currentRunId: null,
1042
- retryEntry: null,
1043
- updatedAt: now.toISOString()
1044
- });
1045
- let run;
1046
- try {
1047
- run = await this.startRun(tenant, issue);
1048
- } catch (error) {
1049
- issueRecords = releaseIssueOrchestration(issueRecords, issue.id, now);
1050
- throw error;
1051
- }
1052
- issueRecords = upsertIssueOrchestration(issueRecords, {
1053
- issueId: run.issueId,
1054
- identifier: run.issueIdentifier,
1055
- workspaceKey: run.issueWorkspaceKey ?? preferredWorkspaceKey,
1056
- state: "running",
1057
- currentRunId: run.runId,
1058
- retryEntry: null,
1059
- updatedAt: now.toISOString()
1060
- });
1061
- await this.store.saveRun(run);
1062
- await this.store.appendRunEvent(run.runId, {
1063
- at: now.toISOString(),
1064
- event: "run-dispatched",
1065
- projectId: tenant.projectId,
1066
- issueIdentifier: issue.identifier,
1067
- issueId: run.issueId,
1068
- issueState: issue.state
1069
- });
1070
- this.logVerbose(
1071
- `[dispatch] Issue ${issue.identifier} \u2192 run ${run.runId}`
1072
- );
1073
- dispatched += 1;
1074
- slotsRemaining -= 1;
1075
- activeByState.set(
1076
- issue.state,
1077
- (activeByState.get(issue.state) ?? 0) + 1
1078
- );
1079
- }
1080
- for (const issueRecord of issueRecords) {
1081
- if (!isIssueOrchestrationClaimed(issueRecord.state)) {
1082
- continue;
1083
- }
1084
- const issue = trackedIssuesByIdentifier.get(issueRecord.identifier);
1085
- if (!issue) {
1086
- continue;
1087
- }
1088
- const persistedRun = issueRecord.currentRunId ? await this.store.loadRun(issueRecord.currentRunId, tenant.projectId) : null;
1089
- const activeRun = syncedActiveRuns.find(
1090
- (run) => isMatchingIssueRun(
1091
- run,
1092
- tenant.projectId,
1093
- issueRecord.issueId,
1094
- issueRecord.identifier
1095
- )
1096
- ) ?? persistedRun;
1097
- const resolvedIssue = actionableCandidates.find(
1098
- (candidate) => candidate.identifier === issue.identifier
1099
- );
1100
- if (resolvedIssue) {
1101
- continue;
1102
- }
1103
- if (activeRun?.processId) {
1104
- this.sendSignal(activeRun.processId, "SIGTERM");
1105
- this.retireWorkerPid(activeRun.processId);
1106
- }
1107
- if (activeRun) {
1108
- const suppressedRun = {
1109
- ...activeRun,
1110
- status: "suppressed",
1111
- processId: null,
1112
- completedAt: now.toISOString(),
1113
- updatedAt: now.toISOString(),
1114
- runPhase: "canceled_by_reconciliation",
1115
- lastError: "Run suppressed because the tracker state is no longer actionable."
1116
- };
1117
- await this.store.saveRun(suppressedRun);
1118
- this.logVerbose(
1119
- `[run-completed] ${suppressedRun.runId} status=${suppressedRun.status}`
1120
- );
1121
- }
1122
- issueRecords = releaseIssueOrchestration(
1123
- issueRecords,
1124
- issueRecord.issueId,
1125
- now
1126
- );
1127
- suppressed += 1;
1128
- }
1129
- const terminalIssuesByIdentifier = /* @__PURE__ */ new Map();
1130
- for (const issue of trackedIssuesByIdentifier.values()) {
1131
- if (!isStateTerminal(issue.state, lifecycle)) {
1132
- continue;
1133
- }
1134
- terminalIssuesByIdentifier.set(issue.identifier, issue);
1135
- }
1136
- for (const issue of terminalIssuesByIdentifier.values()) {
1137
- await this.cleanupTerminalIssueWorkspace(tenant, issue, now);
1138
- }
1139
- } catch (error) {
1140
- lastError = error instanceof Error ? error.message : "Unknown orchestration error";
1141
- }
1142
- const effectivePollIntervalMs = resolveAdaptivePollIntervalMs(
1143
- pollIntervalMs,
1144
- trackerRateLimits
1145
- );
1146
- if (effectivePollIntervalMs > pollIntervalMs && isLowRateLimit(trackerRateLimits, LOW_RATE_LIMIT_WARNING_THRESHOLD)) {
1147
- this.writeStderr(
1148
- `[orchestrator] low GitHub rate limit for ${tenant.projectId}: interval=${effectivePollIntervalMs}ms rateLimits=${JSON.stringify(
1149
- trackerRateLimits
1150
- )}`
1151
- );
1152
- }
1153
- this.projectPollIntervals.set(tenant.projectId, effectivePollIntervalMs);
1154
- await this.store.saveProjectIssueOrchestrations(
1155
- tenant.projectId,
1156
- issueRecords
1157
- );
1158
- const allTenantRuns = (await this.store.loadAllRuns()).filter(
1159
- (run) => run.projectId === tenant.projectId
1160
- );
1161
- const latestRuns = allTenantRuns.filter(
1162
- (run) => isActiveRunStatus(run.status)
1163
- );
1164
- rateLimits = rateLimits ?? resolveProjectRateLimits(latestRuns, []);
1165
- const status = buildProjectSnapshot({
1166
- project: tenant,
1167
- activeRuns: latestRuns,
1168
- allRuns: allTenantRuns,
1169
- summary: { dispatched, suppressed, recovered },
1170
- lastTickAt: now.toISOString(),
1171
- lastError,
1172
- rateLimits
1173
- });
1174
- await this.store.saveProjectStatus(status);
1175
- return status;
1176
- }
1177
- async performStartupCleanup(trackerDependencies = {}) {
1178
- const tenant = this.projectConfig;
1179
- const now = this.now();
1180
- const workspaceRecords = await this.store.loadIssueWorkspaces(
1181
- tenant.projectId
1182
- );
1183
- if (workspaceRecords.length === 0) {
1184
- return;
1185
- }
1186
- const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
1187
- const workflowCache = /* @__PURE__ */ new Map();
1188
- let issues;
1189
- try {
1190
- issues = await trackerAdapter.listIssuesByStates(
1191
- tenant,
1192
- await this.resolveStartupCleanupTerminalStates(
1193
- tenant,
1194
- workspaceRecords,
1195
- workflowCache
1196
- ),
1197
- trackerDependencies
1198
- );
1199
- } catch (error) {
1200
- const message = error instanceof Error ? error.message : "Unknown tracker error";
1201
- console.warn(
1202
- `[orchestrator] Startup cleanup skipped for project ${tenant.projectId}: ${message}`
1203
- );
1204
- return;
1205
- }
1206
- const issuesById = new Map(issues.map((issue) => [issue.id, issue]));
1207
- for (const workspaceRecord of workspaceRecords) {
1208
- if (workspaceRecord.status === "removed") {
1209
- continue;
1210
- }
1211
- const issue = issuesById.get(workspaceRecord.issueSubjectId);
1212
- if (!issue) {
1213
- continue;
1214
- }
1215
- try {
1216
- const resolution = await this.loadStartupCleanupWorkflow(
1217
- tenant,
1218
- issue.repository,
1219
- workflowCache
1220
- );
1221
- if (!resolution.isValid) {
1222
- continue;
1223
- }
1224
- if (!isStateTerminal(issue.state, resolution.lifecycle)) {
1225
- continue;
1226
- }
1227
- await this.cleanupTerminalIssueWorkspace(
1228
- tenant,
1229
- issue,
1230
- now,
1231
- resolution
1232
- );
1233
- } catch (error) {
1234
- const message = error instanceof Error ? error.message : "Unknown startup cleanup error";
1235
- console.warn(
1236
- `[orchestrator] Startup cleanup skipped workspace for ${issue.identifier}: ${message}`
1237
- );
1238
- }
1239
- }
1240
- }
1241
- async notifyTick(snapshot) {
1242
- if (!this.dependencies.onTick) {
1243
- return;
1244
- }
1245
- try {
1246
- await this.dependencies.onTick(snapshot);
1247
- } catch (error) {
1248
- this.writeStderr(
1249
- `[orchestrator] onTick callback failed: ${this.formatErrorMessage(error)}`
1250
- );
1251
- }
1252
- }
1253
- formatErrorMessage(error) {
1254
- if (error instanceof Error) {
1255
- return error.stack ?? error.message;
1256
- }
1257
- return String(error);
1258
- }
1259
- async resolveStartupCleanupTerminalStates(tenant, workspaceRecords, workflowCache) {
1260
- const terminalStates = /* @__PURE__ */ new Map();
1261
- const repositories = this.resolveStartupCleanupRepositories(
1262
- tenant,
1263
- workspaceRecords
1264
- );
1265
- for (const repository of repositories) {
1266
- let resolution;
1267
- try {
1268
- resolution = await this.loadStartupCleanupWorkflow(
1269
- tenant,
1270
- repository,
1271
- workflowCache
1272
- );
1273
- } catch {
1274
- continue;
1275
- }
1276
- if (!isUsableWorkflowResolution(resolution)) {
1277
- continue;
1278
- }
1279
- for (const state of resolution.lifecycle.terminalStates) {
1280
- const normalizedState = state.trim().toLowerCase();
1281
- if (!terminalStates.has(normalizedState)) {
1282
- terminalStates.set(normalizedState, state);
1283
- }
1284
- }
1285
- }
1286
- if (terminalStates.size === 0) {
1287
- for (const state of DEFAULT_WORKFLOW_LIFECYCLE.terminalStates) {
1288
- terminalStates.set(state.trim().toLowerCase(), state);
1289
- }
1290
- }
1291
- return [...terminalStates.values()];
1292
- }
1293
- resolveStartupCleanupRepositories(tenant, workspaceRecords) {
1294
- const repositories = /* @__PURE__ */ new Map();
1295
- for (const repository of tenant.repositories) {
1296
- repositories.set(
1297
- this.startupCleanupRepositoryKey(repository.owner, repository.name),
1298
- repository
1299
- );
1300
- }
1301
- for (const workspaceRecord of workspaceRecords) {
1302
- const repository = this.parseWorkspaceRepositoryRef(workspaceRecord);
1303
- if (!repository) {
1304
- continue;
1305
- }
1306
- const key = this.startupCleanupRepositoryKey(
1307
- repository.owner,
1308
- repository.name
1309
- );
1310
- if (!repositories.has(key)) {
1311
- repositories.set(key, repository);
1312
- }
1313
- }
1314
- return [...repositories.values()];
1315
- }
1316
- parseWorkspaceRepositoryRef(workspaceRecord) {
1317
- const match = workspaceRecord.issueIdentifier.match(
1318
- /^([^/]+)\/([^#]+)#\d+$/
1319
- );
1320
- if (!match) {
1321
- return null;
1322
- }
1323
- const owner = match[1];
1324
- const name = match[2];
1325
- if (!owner || !name) {
1326
- return null;
1327
- }
1328
- return {
1329
- owner,
1330
- name,
1331
- cloneUrl: workspaceRecord.repositoryPath
1332
- };
1333
- }
1334
- startupCleanupRepositoryKey(owner, name) {
1335
- return `${owner}/${name}`;
1336
- }
1337
- async loadStartupCleanupWorkflow(tenant, repository, workflowCache) {
1338
- const cacheKey = this.workflowCacheKey(repository);
1339
- const cachedResolution = workflowCache.get(cacheKey);
1340
- if (cachedResolution) {
1341
- return cachedResolution;
1342
- }
1343
- const resolutionPromise = tenant.repositories.some(
1344
- (candidate) => candidate.owner === repository.owner && candidate.name === repository.name
1345
- ) ? this.loadProjectWorkflow(tenant, repository) : loadRepositoryWorkflow(repository.cloneUrl, repository);
1346
- workflowCache.set(cacheKey, resolutionPromise);
1347
- return resolutionPromise;
1348
- }
1349
- async runSerialized(operation) {
1350
- const previous = this.reconcilePromise;
1351
- let release;
1352
- this.reconcilePromise = new Promise((resolve4) => {
1353
- release = resolve4;
1354
- });
1355
- await previous;
1356
- try {
1357
- return await operation();
1358
- } finally {
1359
- release();
1360
- }
1361
- }
1362
- async runOnceInternal(issueIdentifier, trackerDependencies) {
1363
- return this.runSerialized(async () => {
1364
- const workflowResolutionCache = /* @__PURE__ */ new Map();
1365
- this.workflowResolutionCache = workflowResolutionCache;
1366
- try {
1367
- return await this.reconcileProject(
1368
- this.projectConfig,
1369
- issueIdentifier,
1370
- trackerDependencies
1371
- );
1372
- } finally {
1373
- if (this.workflowResolutionCache === workflowResolutionCache) {
1374
- this.workflowResolutionCache = null;
1375
- }
1376
- }
1377
- });
1378
- }
1379
- createTrackerDependencies() {
1380
- return {
1381
- fetchImpl: this.dependencies.fetchImpl,
1382
- projectItemsCache: createProjectItemsCache()
1383
- };
1384
- }
1385
- async findLatestRunForIssue(issueId, issueIdentifier) {
1386
- const matchingRuns = (await this.store.loadAllRuns()).filter((run) => run.projectId === this.projectConfig.projectId).filter(
1387
- (run) => run.issueId === issueId || run.issueIdentifier === issueIdentifier
1388
- ).sort(
1389
- (left, right) => new Date(right.updatedAt).getTime() - new Date(left.updatedAt).getTime()
1390
- );
1391
- return matchingRuns[0] ?? null;
1392
- }
1393
- async resolveActionableCandidates(tenant, issues) {
1394
- const candidates = [];
1395
- let lifecycle = null;
1396
- for (const issue of issues) {
1397
- const resolution = await this.loadProjectWorkflow(
1398
- tenant,
1399
- issue.repository
1400
- );
1401
- if (!isUsableWorkflowResolution(resolution)) {
1402
- continue;
1403
- }
1404
- if (!lifecycle) {
1405
- lifecycle = resolution.lifecycle;
1406
- }
1407
- if (!this.isIssueCandidateEligible(issue, resolution.lifecycle, issues)) {
1408
- continue;
1409
- }
1410
- candidates.push(issue);
1411
- }
1412
- if (!lifecycle && tenant.repositories.length > 0) {
1413
- const resolution = await this.loadProjectWorkflow(
1414
- tenant,
1415
- tenant.repositories[0]
1416
- );
1417
- if (isUsableWorkflowResolution(resolution)) {
1418
- lifecycle = resolution.lifecycle;
1419
- }
1420
- }
1421
- return {
1422
- candidates,
1423
- lifecycle: lifecycle ?? {
1424
- stateFieldName: "Status",
1425
- activeStates: ["Todo", "In Progress"],
1426
- terminalStates: ["Done"],
1427
- blockerCheckStates: ["Todo"]
1428
- }
1429
- };
1430
- }
1431
- isIssueCandidateEligible(issue, lifecycle, issues) {
1432
- if (!isStateActive(issue.state, lifecycle)) {
1433
- return false;
1434
- }
1435
- if (!matchesWorkflowState(issue.state, lifecycle.blockerCheckStates) || issue.blockedBy.length === 0) {
1436
- return true;
1437
- }
1438
- return !issue.blockedBy.some((blockerRef) => {
1439
- if (blockerRef.state && isStateTerminal(blockerRef.state, lifecycle)) {
1440
- return false;
1441
- }
1442
- if (blockerRef.identifier) {
1443
- const blockerIssue = issues.find(
1444
- (candidate) => candidate.identifier === blockerRef.identifier
1445
- );
1446
- if (blockerIssue?.state) {
1447
- return !isStateTerminal(blockerIssue.state, lifecycle);
1448
- }
1449
- }
1450
- return true;
1451
- });
1452
- }
1453
- async loadProjectWorkflow(tenant, repository) {
1454
- const cacheKey = this.workflowCacheKey(repository);
1455
- const pendingCache = this.workflowResolutionCache;
1456
- if (pendingCache) {
1457
- const cachedResolution = pendingCache.get(cacheKey);
1458
- if (cachedResolution) {
1459
- return cachedResolution;
1460
- }
1461
- const resolutionPromise = this.loadProjectWorkflowUncached(
1462
- tenant,
1463
- repository
1464
- );
1465
- pendingCache.set(cacheKey, resolutionPromise);
1466
- return resolutionPromise;
1467
- }
1468
- return this.loadProjectWorkflowUncached(tenant, repository);
1469
- }
1470
- async loadProjectWorkflowUncached(tenant, repository) {
1471
- const cacheRoot = join3(
1472
- this.store.projectDir(tenant.projectId),
1473
- "cache",
1474
- repository.owner,
1475
- repository.name
1476
- );
1477
- const { repositoryDirectory, changed } = await syncRepositoryForRun({
1478
- repository,
1479
- targetDirectory: cacheRoot
1480
- });
1481
- const resolution = await loadRepositoryWorkflow(
1482
- repositoryDirectory,
1483
- repository
1484
- );
1485
- return this.resolveWorkflowResolution(
1486
- repository,
1487
- cacheRoot,
1488
- resolution,
1489
- changed
1490
- );
1491
- }
1492
- async startRun(tenant, issue) {
1493
- if (this.shuttingDown || !this.running) {
1494
- throw new Error(
1495
- "Orchestrator is shutting down and cannot start new runs."
1496
- );
1497
- }
1498
- const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
1499
- const now = this.now();
1500
- const runId = createRunId(now, tenant.projectId, issue.identifier);
1501
- const runDir = this.store.runDir(runId, tenant.projectId);
1502
- const workspaceRuntimeDir = runDir;
1503
- const issueSubjectId = issue.id;
1504
- const identity = {
1505
- projectId: tenant.projectId,
1506
- adapter: issue.tracker.adapter,
1507
- issueSubjectId
1508
- };
1509
- const preferredWorkspaceKey = deriveIssueWorkspaceKey(
1510
- identity,
1511
- issue.identifier
1512
- );
1513
- const legacyWorkspaceKey = deriveLegacyIssueWorkspaceKey(identity);
1514
- const existingWorkspaceRecord = await this.store.loadIssueWorkspace(
1515
- tenant.projectId,
1516
- preferredWorkspaceKey
1517
- ) ?? (legacyWorkspaceKey === preferredWorkspaceKey ? null : await this.store.loadIssueWorkspace(
1518
- tenant.projectId,
1519
- legacyWorkspaceKey
1520
- ));
1521
- const workspaceKey = existingWorkspaceRecord?.workspaceKey ?? preferredWorkspaceKey;
1522
- const projectDir = this.store.projectDir(tenant.projectId);
1523
- const issueWorkspacePath = resolveIssueWorkspaceDirectory(
1524
- projectDir,
1525
- workspaceKey
1526
- );
1527
- const repositoryDirectory = await ensureIssueWorkspaceRepository({
1528
- repository: issue.repository,
1529
- issueWorkspacePath
1530
- });
1531
- if (!existingWorkspaceRecord) {
1532
- const workspaceRecord = {
1533
- workspaceKey,
1534
- projectId: tenant.projectId,
1535
- adapter: issue.tracker.adapter,
1536
- issueSubjectId,
1537
- issueIdentifier: issue.identifier,
1538
- workspacePath: issueWorkspacePath,
1539
- repositoryPath: repositoryDirectory,
1540
- status: "active",
1541
- createdAt: now.toISOString(),
1542
- updatedAt: now.toISOString(),
1543
- lastError: null
1544
- };
1545
- await this.store.saveIssueWorkspace(workspaceRecord);
1546
- const afterCreateResult = await this.runHook(
1547
- "after_create",
1548
- tenant,
1549
- repositoryDirectory,
1550
- issue.repository,
1551
- {
1552
- projectId: tenant.projectId,
1553
- workspaceKey,
1554
- issueSubjectId,
1555
- issueIdentifier: issue.identifier,
1556
- workspacePath: issueWorkspacePath,
1557
- repositoryPath: repositoryDirectory
1558
- }
1559
- );
1560
- if (afterCreateResult && afterCreateResult.outcome !== "success" && afterCreateResult.outcome !== "skipped") {
1561
- await this.store.appendRunEvent(runId, {
1562
- at: now.toISOString(),
1563
- event: "hook-failed",
1564
- projectId: tenant.projectId,
1565
- hook: "after_create",
1566
- error: afterCreateResult.error ?? null
1567
- });
1568
- }
1569
- }
1570
- const workflow = await this.loadProjectWorkflow(tenant, issue.repository);
1571
- if (!isUsableWorkflowResolution(workflow)) {
1572
- throw new Error(
1573
- workflow.validationError ?? "Invalid repository WORKFLOW.md"
1574
- );
1575
- }
1576
- const promptVariables = buildPromptVariables(issue, {
1577
- attempt: null
1578
- // first execution
1579
- });
1580
- const renderedPrompt = renderPrompt(
1581
- workflow.promptTemplate,
1582
- promptVariables
1583
- );
1584
- await this.runHook(
1585
- "before_run",
1586
- tenant,
1587
- repositoryDirectory,
1588
- issue.repository,
1589
- {
1590
- projectId: tenant.projectId,
1591
- workspaceKey,
1592
- issueSubjectId,
1593
- issueIdentifier: issue.identifier,
1594
- workspacePath: issueWorkspacePath,
1595
- repositoryPath: repositoryDirectory,
1596
- runId,
1597
- state: issue.state
1598
- }
1599
- );
1600
- const runtimeTimeouts = resolveWorkflowRuntimeTimeouts(workflow.workflow);
1601
- mkdirSync(runDir, { recursive: true });
1602
- const workerLogStream = (this.dependencies.createWriteStreamImpl ?? createWriteStream)(join3(runDir, "worker.log"), {
1603
- flags: "a"
1604
- });
1605
- let workerLogAvailable = true;
1606
- let workerExited = false;
1607
- let workerStderrFinalizing = false;
1608
- let workerLogBackpressured = false;
1609
- const resumeWorkerStderr = () => {
1610
- if (!workerLogBackpressured) {
1611
- return;
1612
- }
1613
- workerLogBackpressured = false;
1614
- child.stderr?.resume?.();
1615
- };
1616
- const markWorkerLogUnavailable = (error) => {
1617
- resumeWorkerStderr();
1618
- if (!workerLogAvailable) {
1619
- return;
1620
- }
1621
- workerLogAvailable = false;
1622
- const message = error instanceof Error ? error.message : String(error ?? "unknown");
1623
- this.writeStderr(
1624
- `[orchestrator] failed to write worker log for ${runId}: ${message}`
1625
- );
1626
- };
1627
- const child = (this.dependencies.spawnImpl ?? spawn2)(
1628
- "bash",
1629
- ["-lc", resolveWorkerCommand()],
1630
- {
1631
- cwd: process.cwd(),
1632
- env: this.buildProjectExecutionEnv(tenant.projectId, {
1633
- GITHUB_GRAPHQL_TOKEN: process.env.GITHUB_GRAPHQL_TOKEN ?? "",
1634
- CODEX_PROJECT_ID: tenant.projectId,
1635
- PROJECT_ID: tenant.projectId,
1636
- WORKING_DIRECTORY: repositoryDirectory,
1637
- WORKSPACE_RUNTIME_DIR: workspaceRuntimeDir,
1638
- SYMPHONY_RUN_ID: runId,
1639
- SYMPHONY_ISSUE_STATE: issue.state,
1640
- SYMPHONY_ISSUE_ID: issue.id,
1641
- SYMPHONY_ISSUE_IDENTIFIER: issue.identifier,
1642
- SYMPHONY_ISSUE_TITLE: issue.title,
1643
- SYMPHONY_ISSUE_SUBJECT_ID: issueSubjectId,
1644
- SYMPHONY_ISSUE_WORKSPACE_KEY: workspaceKey,
1645
- SYMPHONY_TRACKER_ADAPTER: issue.tracker.adapter,
1646
- SYMPHONY_TRACKER_BINDING_ID: issue.tracker.bindingId,
1647
- SYMPHONY_TRACKER_ITEM_ID: issue.tracker.itemId,
1648
- TARGET_REPOSITORY_CLONE_URL: issue.repository.cloneUrl,
1649
- TARGET_REPOSITORY_OWNER: issue.repository.owner,
1650
- TARGET_REPOSITORY_NAME: issue.repository.name,
1651
- TARGET_REPOSITORY_URL: issue.repository.url,
1652
- ...trackerAdapter.buildWorkerEnvironment(tenant, issue),
1653
- SYMPHONY_RENDERED_PROMPT: renderedPrompt,
1654
- SYMPHONY_WORKFLOW_PATH: workflow.workflowPath ?? "",
1655
- SYMPHONY_AGENT_COMMAND: resolveWorkflowRuntimeCommand(
1656
- workflow.workflow
1657
- ),
1658
- SYMPHONY_APPROVAL_POLICY: workflow.workflow.codex.approvalPolicy ?? "",
1659
- SYMPHONY_THREAD_SANDBOX: workflow.workflow.codex.threadSandbox ?? "",
1660
- SYMPHONY_TURN_SANDBOX_POLICY: workflow.workflow.codex.turnSandboxPolicy ?? "",
1661
- SYMPHONY_MAX_TURNS: String(workflow.workflow.agent.maxTurns),
1662
- SYMPHONY_MAX_NONPRODUCTIVE_TURNS: process.env.SYMPHONY_MAX_NONPRODUCTIVE_TURNS ?? String(DEFAULT_MAX_NONPRODUCTIVE_TURNS),
1663
- // Clear legacy resume/budget env so fresh worker sessions do not
1664
- // inherit stale process-level values.
1665
- SYMPHONY_GLOBAL_MAX_TURNS: "",
1666
- SYMPHONY_MAX_TOKENS: "",
1667
- SYMPHONY_SESSION_TIMEOUT_MS: "",
1668
- SYMPHONY_RESUME_THREAD_ID: "",
1669
- SYMPHONY_CUMULATIVE_TURN_COUNT: "0",
1670
- SYMPHONY_CUMULATIVE_INPUT_TOKENS: "0",
1671
- SYMPHONY_CUMULATIVE_OUTPUT_TOKENS: "0",
1672
- SYMPHONY_CUMULATIVE_TOTAL_TOKENS: "0",
1673
- SYMPHONY_LAST_TURN_SUMMARY: "",
1674
- SYMPHONY_SESSION_STARTED_AT: "",
1675
- SYMPHONY_READ_TIMEOUT_MS: String(
1676
- runtimeTimeouts.readTimeoutMs
1677
- ),
1678
- SYMPHONY_TURN_TIMEOUT_MS: String(
1679
- runtimeTimeouts.turnTimeoutMs
1680
- )
1681
- }),
1682
- detached: true,
1683
- stdio: ["ignore", "ignore", "pipe"]
1684
- }
1685
- );
1686
- const handleWorkerStderrChunk = (chunk) => {
1687
- const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk), "utf8");
1688
- if (workerLogAvailable) {
1689
- try {
1690
- if (!workerLogStream.write(buffer)) {
1691
- workerLogBackpressured = true;
1692
- child.stderr?.pause?.();
1693
- }
1694
- } catch (error) {
1695
- markWorkerLogUnavailable(error);
1696
- }
1697
- }
1698
- this.consumeWorkerStderrChunk(runId, buffer);
1699
- };
1700
- const drainWorkerStderr = () => {
1701
- const stderr = child.stderr;
1702
- if (!stderr || typeof stderr.read !== "function") {
1703
- return;
1704
- }
1705
- let chunk;
1706
- while ((chunk = stderr.read()) !== null) {
1707
- handleWorkerStderrChunk(chunk);
1708
- }
1709
- };
1710
- const completeWorkerStderrFinalization = (code, signal) => {
1711
- if (workerExited) {
1712
- return;
1713
- }
1714
- workerExited = true;
1715
- workerStderrFinalizing = false;
1716
- child.stderr?.removeListener("data", handleWorkerStderrChunk);
1717
- this.flushWorkerStderrBuffer(runId);
1718
- workerLogStream.end();
1719
- if (child.pid) {
1720
- this.retireWorkerPid(child.pid);
1721
- }
1722
- this.logVerbose(
1723
- `[worker-exited] ${runId} (code=${code ?? "null"}, signal=${signal ?? "null"})`
1724
- );
1725
- };
1726
- const finalizeWorkerStderr = (code, signal) => {
1727
- if (workerExited || workerStderrFinalizing) {
1728
- return;
1729
- }
1730
- workerStderrFinalizing = true;
1731
- const stderr = child.stderr;
1732
- const finish = () => {
1733
- stderr?.removeListener("end", finish);
1734
- stderr?.removeListener("close", finish);
1735
- drainWorkerStderr();
1736
- completeWorkerStderrFinalization(code, signal);
1737
- };
1738
- resumeWorkerStderr();
1739
- drainWorkerStderr();
1740
- if (!stderr) {
1741
- completeWorkerStderrFinalization(code, signal);
1742
- return;
1743
- }
1744
- if (stderr.readableEnded || stderr.readable === false) {
1745
- finish();
1746
- return;
1747
- }
1748
- stderr.once("end", finish);
1749
- stderr.once("close", finish);
1750
- };
1751
- workerLogStream.on("error", (error) => {
1752
- markWorkerLogUnavailable(error);
1753
- });
1754
- workerLogStream.on("drain", () => {
1755
- resumeWorkerStderr();
1756
- });
1757
- child.stderr?.on("data", handleWorkerStderrChunk);
1758
- if (child.pid) {
1759
- this.activeWorkerPids.add(child.pid);
1760
- this.logVerbose(`[worker-started] ${runId} (pid=${child.pid})`);
1761
- }
1762
- child.on?.("error", (error) => {
1763
- const message = error instanceof Error ? error.message : String(error ?? "unknown");
1764
- this.writeStderr(
1765
- `[orchestrator] worker process error for ${runId}: ${message}`
1766
- );
1767
- finalizeWorkerStderr(null, null);
1768
- });
1769
- child.on?.("close", (code, signal) => {
1770
- finalizeWorkerStderr(code, signal);
1771
- });
1772
- child.unref();
1773
- return {
1774
- runId,
1775
- projectId: tenant.projectId,
1776
- projectSlug: tenant.slug,
1777
- issueId: issue.id,
1778
- issueSubjectId,
1779
- issueIdentifier: issue.identifier,
1780
- issueTitle: issue.title,
1781
- issueState: issue.state,
1782
- repository: issue.repository,
1783
- status: "running",
1784
- attempt: 1,
1785
- processId: child.pid ?? null,
1786
- port: null,
1787
- workingDirectory: repositoryDirectory,
1788
- issueWorkspaceKey: workspaceKey,
1789
- workspaceRuntimeDir,
1790
- workflowPath: workflow.workflowPath,
1791
- retryKind: null,
1792
- threadId: null,
1793
- cumulativeTurnCount: 0,
1794
- lastTurnSummary: null,
1795
- createdAt: now.toISOString(),
1796
- updatedAt: now.toISOString(),
1797
- startedAt: now.toISOString(),
1798
- completedAt: null,
1799
- lastError: null,
1800
- nextRetryAt: null,
1801
- runPhase: "preparing_workspace",
1802
- rateLimits: issue.rateLimits ?? null
1803
- };
1804
- }
1805
- async syncActiveRunIssueStates(tenant, trackerAdapter, activeRuns, now) {
1806
- const activeIssueIds = [...new Set(activeRuns.map((run) => run.issueId))];
1807
- if (activeIssueIds.length === 0) {
1808
- return {
1809
- runs: activeRuns,
1810
- issuesByIdentifier: /* @__PURE__ */ new Map()
1811
- };
1812
- }
1813
- const issues = await trackerAdapter.fetchIssueStatesByIds(
1814
- tenant,
1815
- activeIssueIds,
1816
- {
1817
- fetchImpl: this.dependencies.fetchImpl
1818
- }
1819
- );
1820
- const issuesByIdentifier = new Map(
1821
- issues.map((issue) => [issue.identifier, issue])
1822
- );
1823
- const issueStateByIdentifier = new Map(
1824
- issues.map((issue) => [issue.identifier, issue.state])
1825
- );
1826
- const syncedRuns = [];
1827
- for (const run of activeRuns) {
1828
- const currentTrackerState = issueStateByIdentifier.get(
1829
- run.issueIdentifier
1830
- );
1831
- if (!currentTrackerState || currentTrackerState === run.issueState) {
1832
- syncedRuns.push(run);
1833
- continue;
1834
- }
1835
- const updatedRun = {
1836
- ...run,
1837
- issueState: currentTrackerState,
1838
- updatedAt: now.toISOString()
1839
- };
1840
- await this.store.saveRun(updatedRun);
1841
- syncedRuns.push(updatedRun);
1842
- }
1843
- return {
1844
- runs: syncedRuns,
1845
- issuesByIdentifier
1846
- };
1847
- }
1848
- async reconcileRun(tenant, run, issueRecords, trackerDependencies = {}) {
1849
- const now = this.now();
1850
- if (run.processId && this.isProcessRunning(run.processId)) {
1851
- const retryPolicy = await this.loadRetryPolicy(tenant, run.repository);
1852
- const configuredStallTimeoutMs = retryPolicy?.stallTimeoutMs ?? null;
1853
- const lastActivityAtMs = parseTimestampMs(
1854
- run.lastEventAt ?? run.startedAt
1855
- );
1856
- const startedAtMs = parseTimestampMs(run.startedAt);
1857
- const elapsedSinceLastActivityMs = lastActivityAtMs === null ? null : now.getTime() - lastActivityAtMs;
1858
- const runningSinceMs = startedAtMs === null ? null : now.getTime() - startedAtMs;
1859
- const isStalledByWorkflowTimeout = configuredStallTimeoutMs !== null && configuredStallTimeoutMs > 0 && elapsedSinceLastActivityMs !== null && elapsedSinceLastActivityMs > configuredStallTimeoutMs;
1860
- const isStalledByFallbackTimeout = runningSinceMs !== null && runningSinceMs > STUCK_WORKER_TIMEOUT_MS;
1861
- if (isStalledByWorkflowTimeout || isStalledByFallbackTimeout) {
1862
- const elapsedMs = isStalledByWorkflowTimeout ? elapsedSinceLastActivityMs : runningSinceMs;
1863
- const timeoutMs = isStalledByWorkflowTimeout ? configuredStallTimeoutMs : STUCK_WORKER_TIMEOUT_MS;
1864
- const elapsedSeconds = Math.round((elapsedMs ?? 0) / 1e3);
1865
- const timeoutSeconds = Math.round((timeoutMs ?? 0) / 1e3);
1866
- if (this.isVerboseLoggingEnabled()) {
1867
- this.writeStderr(
1868
- `[stall-detected] ${run.runId} (elapsed=${elapsedSeconds}s > ${timeoutSeconds}s)`
1869
- );
1870
- } else {
1871
- this.writeStderr(
1872
- `[orchestrator] stuck worker detected for ${run.runId} (elapsed ${elapsedSeconds}s > ${timeoutSeconds}s) \u2014 sending SIGTERM`
1873
- );
1874
- }
1875
- this.sendSignal(run.processId, "SIGTERM");
1876
- } else {
1877
- const runningRecord = {
1878
- ...run,
1879
- status: "running",
1880
- updatedAt: now.toISOString()
1881
- };
1882
- await this.store.saveRun(runningRecord);
1883
- issueRecords = upsertIssueOrchestration(issueRecords, {
1884
- issueId: run.issueId,
1885
- identifier: run.issueIdentifier,
1886
- workspaceKey: run.issueWorkspaceKey ?? deriveIssueWorkspaceKey(
1887
- {
1888
- projectId: tenant.projectId,
1889
- adapter: tenant.tracker.adapter,
1890
- issueSubjectId: run.issueSubjectId
1891
- },
1892
- run.issueIdentifier
1893
- ),
1894
- state: "running",
1895
- currentRunId: run.runId,
1896
- retryEntry: null,
1897
- updatedAt: now.toISOString()
1898
- });
1899
- return {
1900
- issueRecords,
1901
- recovered: false
1902
- };
1903
- }
1904
- }
1905
- if (run.processId) {
1906
- this.retireWorkerPid(run.processId);
1907
- }
1908
- const workerInfo = await this.fetchWorkerRunInfo(run);
1909
- const runWithTokens = {
1910
- ...run,
1911
- runtimeSession: buildRuntimeSession(
1912
- run.runtimeSession,
1913
- workerInfo.sessionId,
1914
- workerInfo.threadId,
1915
- run.status === "running" ? "failed" : run.runtimeSession?.status ?? null,
1916
- run.runtimeSession?.startedAt ?? run.startedAt ?? now.toISOString(),
1917
- now.toISOString(),
1918
- workerInfo.exitClassification
1919
- ),
1920
- threadId: workerInfo.threadId ?? run.threadId ?? null,
1921
- cumulativeTurnCount: resolveCumulativeTurnCount(
1922
- run,
1923
- workerInfo.turnCount ?? null
1924
- ),
1925
- tokenUsage: workerInfo.tokenUsage ?? run.tokenUsage,
1926
- lastEvent: workerInfo.lastEvent ?? run.lastEvent,
1927
- lastTurnSummary: resolveLastTurnSummary(
1928
- run.lastTurnSummary,
1929
- resolveLastTurnSummaryCandidate(
1930
- workerInfo.lastEvent,
1931
- workerInfo.lastError
1932
- )
1933
- ),
1934
- lastEventAt: workerInfo.lastEventAt ?? run.lastEventAt ?? void 0,
1935
- lastEventAtSource: workerInfo.lastEventAtSource ?? run.lastEventAtSource ?? void 0,
1936
- executionPhase: workerInfo.executionPhase ?? run.executionPhase ?? null,
1937
- runPhase: workerInfo.runPhase ?? run.runPhase ?? null,
1938
- rateLimits: workerInfo.rateLimits ?? run.rateLimits ?? null
1939
- };
1940
- const workerSessionId = workerInfo.sessionId;
1941
- if (workerInfo.lastError) {
1942
- await this.store.appendRunEvent(run.runId, {
1943
- at: now.toISOString(),
1944
- event: "worker-error",
1945
- projectId: run.projectId,
1946
- runId: run.runId,
1947
- issueIdentifier: run.issueIdentifier,
1948
- error: workerInfo.lastError,
1949
- attempt: run.attempt
1950
- });
1951
- }
1952
- if (run.status === "retrying" && run.nextRetryAt) {
1953
- if (new Date(run.nextRetryAt).getTime() > now.getTime()) {
1954
- return {
1955
- issueRecords,
1956
- recovered: false
1957
- };
1958
- }
1959
- if (await this.resolveRetryRestartAction(
1960
- tenant,
1961
- run,
1962
- trackerDependencies
1963
- ) === "release") {
1964
- return this.releaseRetryingRun(runWithTokens, issueRecords, now);
1965
- }
1966
- return this.restartRun(tenant, run, issueRecords, now, workerSessionId);
1967
- }
1968
- if (workerInfo.exitClassification === "convergence-detected") {
1969
- const completedRun = {
1970
- ...runWithTokens,
1971
- status: "failed",
1972
- processId: null,
1973
- updatedAt: now.toISOString(),
1974
- completedAt: now.toISOString(),
1975
- nextRetryAt: null,
1976
- retryKind: null,
1977
- lastError: runWithTokens.lastError,
1978
- runPhase: runWithTokens.runPhase ?? "failed"
1979
- };
1980
- await this.store.saveRun(completedRun);
1981
- this.logVerbose(
1982
- `[run-completed] ${completedRun.runId} status=${completedRun.status}`
1983
- );
1984
- return {
1985
- issueRecords: releaseIssueOrchestration(issueRecords, run.issueId, now),
1986
- recovered: false
1987
- };
1988
- }
1989
- if (run.issueWorkspaceKey) {
1990
- const issueWorkspacePath = resolveIssueWorkspaceDirectory(
1991
- this.store.projectDir(tenant.projectId),
1992
- run.issueWorkspaceKey
1993
- );
1994
- await this.runHook(
1995
- "after_run",
1996
- tenant,
1997
- run.workingDirectory,
1998
- run.repository,
1999
- {
2000
- projectId: run.projectId,
2001
- workspaceKey: run.issueWorkspaceKey,
2002
- issueSubjectId: run.issueSubjectId,
2003
- issueIdentifier: run.issueIdentifier,
2004
- workspacePath: issueWorkspacePath,
2005
- repositoryPath: run.workingDirectory,
2006
- runId: run.runId,
2007
- state: run.issueState
2008
- }
2009
- );
2010
- }
2011
- const retryKind = await this.classifyRetryKind(
2012
- tenant,
2013
- run,
2014
- trackerDependencies
2015
- );
2016
- const failureRetryCount = retryKind === "failure" ? (this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0) + 1 : this.resolveFailureRetryCount(issueRecords, run.issueId) ?? 0;
2017
- const maxFailureRetries = await this.loadMaxFailureRetries(
2018
- tenant,
2019
- run.repository
2020
- );
2021
- if (retryKind === "failure" && failureRetryCount >= maxFailureRetries) {
2022
- const lastError = [
2023
- `Run suppressed: ${MAX_FAILURE_RETRIES_EXCEEDED_REASON}.`,
2024
- `failureRetryCount=${failureRetryCount}.`,
2025
- `maxFailureRetries=${maxFailureRetries}.`
2026
- ].join(" ");
2027
- const suppressedRun = {
2028
- ...runWithTokens,
2029
- status: "suppressed",
2030
- processId: null,
2031
- updatedAt: now.toISOString(),
2032
- completedAt: now.toISOString(),
2033
- nextRetryAt: null,
2034
- retryKind: null,
2035
- runPhase: runWithTokens.runPhase ?? "failed",
2036
- lastError
2037
- };
2038
- await this.store.saveRun(suppressedRun);
2039
- await this.store.appendRunEvent(run.runId, {
2040
- at: now.toISOString(),
2041
- event: "run-suppressed",
2042
- projectId: run.projectId,
2043
- issueIdentifier: run.issueIdentifier,
2044
- issueId: run.issueId,
2045
- reason: MAX_FAILURE_RETRIES_EXCEEDED_REASON
2046
- });
2047
- this.logVerbose(
2048
- `[run-completed] ${suppressedRun.runId} status=${suppressedRun.status}`
2049
- );
2050
- return {
2051
- issueRecords: upsertIssueOrchestration(issueRecords, {
2052
- issueId: run.issueId,
2053
- identifier: run.issueIdentifier,
2054
- workspaceKey: run.issueWorkspaceKey ?? deriveIssueWorkspaceKey(
2055
- {
2056
- projectId: tenant.projectId,
2057
- adapter: tenant.tracker.adapter,
2058
- issueSubjectId: run.issueSubjectId
2059
- },
2060
- run.issueIdentifier
2061
- ),
2062
- state: "released",
2063
- failureRetryCount,
2064
- currentRunId: null,
2065
- retryEntry: null,
2066
- updatedAt: now.toISOString()
2067
- }),
2068
- recovered: false
2069
- };
2070
- }
2071
- let nextRetryAt;
2072
- if (retryKind === "continuation") {
2073
- nextRetryAt = new Date(
2074
- now.getTime() + CONTINUATION_RETRY_DELAY_MS
2075
- ).toISOString();
2076
- } else {
2077
- const retryOptions = await this.loadRetryPolicy(tenant, run.repository);
2078
- const backoffMs = this.dependencies.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS;
2079
- nextRetryAt = (retryOptions ? scheduleRetryAt(now, run.attempt + 1, retryOptions) : new Date(now.getTime() + backoffMs)).toISOString();
2080
- }
2081
- const retryRecord = {
2082
- ...runWithTokens,
2083
- status: "retrying",
2084
- attempt: runWithTokens.attempt + 1,
2085
- processId: null,
2086
- updatedAt: now.toISOString(),
2087
- nextRetryAt,
2088
- retryKind,
2089
- threadId: runWithTokens.threadId ?? runWithTokens.runtimeSession?.threadId ?? run.threadId ?? run.runtimeSession?.threadId ?? null,
2090
- cumulativeTurnCount: runWithTokens.cumulativeTurnCount ?? run.cumulativeTurnCount ?? 0,
2091
- lastTurnSummary: runWithTokens.lastTurnSummary ?? run.lastTurnSummary ?? null,
2092
- runPhase: runWithTokens.runPhase ?? "failed",
2093
- lastError: retryKind === "continuation" ? null : "Worker process exited unexpectedly."
2094
- };
2095
- await this.store.saveRun(retryRecord);
2096
- this.logVerbose(
2097
- `[retry-scheduled] ${retryRecord.runId} kind=${retryKind} attempt=${retryRecord.attempt} nextAt=${nextRetryAt}`
2098
- );
2099
- this.logVerbose(
2100
- `[run-completed] ${retryRecord.runId} status=${retryRecord.status}`
2101
- );
2102
- issueRecords = upsertIssueOrchestration(issueRecords, {
2103
- issueId: run.issueId,
2104
- identifier: run.issueIdentifier,
2105
- workspaceKey: run.issueWorkspaceKey ?? deriveIssueWorkspaceKey(
2106
- {
2107
- projectId: tenant.projectId,
2108
- adapter: tenant.tracker.adapter,
2109
- issueSubjectId: run.issueSubjectId
2110
- },
2111
- run.issueIdentifier
2112
- ),
2113
- state: "retry_queued",
2114
- completedOnce: retryKind === "continuation" ? true : void 0,
2115
- failureRetryCount,
2116
- currentRunId: run.runId,
2117
- retryEntry: {
2118
- attempt: retryRecord.attempt,
2119
- dueAt: nextRetryAt,
2120
- error: retryRecord.lastError
2121
- },
2122
- updatedAt: now.toISOString()
2123
- });
2124
- return {
2125
- issueRecords,
2126
- recovered: false
2127
- };
2128
- }
2129
- now() {
2130
- return this.dependencies.now?.() ?? /* @__PURE__ */ new Date();
2131
- }
2132
- isVerboseLoggingEnabled() {
2133
- return this.dependencies.logLevel === "verbose";
2134
- }
2135
- writeStderr(message) {
2136
- (this.dependencies.stderr ?? process.stderr).write(`${message}
2137
- `);
2138
- }
2139
- consumeWorkerStderrChunk(runId, chunk) {
2140
- let decoder = this.workerStderrDecoders.get(runId);
2141
- if (!decoder) {
2142
- decoder = new StringDecoder("utf8");
2143
- this.workerStderrDecoders.set(runId, decoder);
2144
- }
2145
- const nextBuffer = (this.workerStderrBuffers.get(runId) ?? "") + decoder.write(chunk);
2146
- const lines = nextBuffer.split("\n");
2147
- this.workerStderrBuffers.set(runId, lines.pop() ?? "");
2148
- for (const line of lines) {
2149
- this.consumeWorkerStderrLine(runId, line);
2150
- }
2151
- }
2152
- flushWorkerStderrBuffer(runId) {
2153
- const decoder = this.workerStderrDecoders.get(runId);
2154
- const remainder = (this.workerStderrBuffers.get(runId) ?? "") + (decoder?.end() ?? "");
2155
- this.workerStderrBuffers.delete(runId);
2156
- this.workerStderrDecoders.delete(runId);
2157
- if (remainder && remainder.trim()) {
2158
- this.consumeWorkerStderrLine(runId, remainder);
2159
- }
2160
- }
2161
- consumeWorkerStderrLine(runId, line) {
2162
- const trimmed = line.trim();
2163
- if (!trimmed || !trimmed.startsWith("{")) {
2164
- return;
2165
- }
2166
- try {
2167
- const parsed = JSON.parse(trimmed);
2168
- if (!isOrchestratorChannelEvent(parsed)) {
2169
- return;
2170
- }
2171
- void this.runSerialized(
2172
- () => this.applyWorkerChannelEvent(runId, parsed)
2173
- ).catch((error) => {
2174
- const message = error instanceof Error ? error.message : String(error ?? "unknown");
2175
- this.writeStderr(
2176
- `[orchestrator] failed to apply worker channel event for ${runId}: ${message}`
2177
- );
2178
- });
2179
- } catch {
2180
- }
2181
- }
2182
- async applyWorkerChannelEvent(runId, event) {
2183
- const run = await this.store.loadRun(runId, this.projectConfig.projectId);
2184
- if (!run || !canApplyWorkerChannelUpdate(run.status) || run.issueId !== event.issueId) {
2185
- return;
2186
- }
2187
- if (event.type === "heartbeat") {
2188
- const nowIso2 = this.now().toISOString();
2189
- const persistedLastEventAt = event.lastEventAt ?? run.lastEventAt ?? null;
2190
- await this.store.saveRun({
2191
- ...run,
2192
- updatedAt: nowIso2,
2193
- lastEvent: "heartbeat",
2194
- lastTurnSummary: resolveLastTurnSummary(
2195
- run.lastTurnSummary,
2196
- event.lastError
2197
- ),
2198
- lastEventAt: persistedLastEventAt,
2199
- lastEventAtSource: event.lastEventAt != null ? "event-channel" : run.lastEventAtSource ?? null,
2200
- tokenUsage: event.tokenUsage,
2201
- rateLimits: event.rateLimits,
2202
- runtimeSession: buildRuntimeSession(
2203
- run.runtimeSession,
2204
- resolveChannelSessionId(event.sessionInfo),
2205
- event.sessionInfo?.threadId ?? null,
2206
- "active",
2207
- run.startedAt ?? run.runtimeSession?.startedAt ?? nowIso2,
2208
- nowIso2,
2209
- event.sessionInfo?.exitClassification ?? null
2210
- ),
2211
- threadId: event.sessionInfo?.threadId ?? run.threadId ?? run.runtimeSession?.threadId ?? null,
2212
- turnCount: event.sessionInfo && event.sessionInfo.turnCount != null ? event.sessionInfo.turnCount : run.turnCount,
2213
- cumulativeTurnCount: resolveCumulativeTurnCount(
2214
- run,
2215
- event.sessionInfo?.turnCount ?? null
2216
- ),
2217
- executionPhase: event.executionPhase ?? run.executionPhase,
2218
- runPhase: event.runPhase ?? run.runPhase,
2219
- lastError: event.lastError
2220
- });
2221
- return;
2222
- }
2223
- if (event.type === "turn_started") {
2224
- await this.store.appendRunEvent(runId, {
2225
- at: event.startedAt,
2226
- event: "turn_started",
2227
- projectId: run.projectId,
2228
- issueIdentifier: run.issueIdentifier,
2229
- issueId: run.issueId,
2230
- sessionId: event.sessionId,
2231
- threadId: event.threadId,
2232
- turnId: event.turnId,
2233
- turnCount: event.turnCount
2234
- });
2235
- return;
2236
- }
2237
- if (event.type === "turn_completed") {
2238
- await this.store.appendRunEvent(runId, {
2239
- at: event.completedAt,
2240
- event: "turn_completed",
2241
- projectId: run.projectId,
2242
- issueIdentifier: run.issueIdentifier,
2243
- issueId: run.issueId,
2244
- sessionId: event.sessionId,
2245
- threadId: event.threadId,
2246
- turnId: event.turnId,
2247
- turnCount: event.turnCount,
2248
- startedAt: event.startedAt,
2249
- durationMs: event.durationMs,
2250
- tokenUsage: event.tokenUsage
2251
- });
2252
- return;
2253
- }
2254
- if (event.type === "turn_failed") {
2255
- await this.store.appendRunEvent(runId, {
2256
- at: event.failedAt,
2257
- event: "turn_failed",
2258
- projectId: run.projectId,
2259
- issueIdentifier: run.issueIdentifier,
2260
- issueId: run.issueId,
2261
- sessionId: event.sessionId,
2262
- threadId: event.threadId,
2263
- turnId: event.turnId,
2264
- turnCount: event.turnCount,
2265
- startedAt: event.startedAt,
2266
- durationMs: event.durationMs,
2267
- tokenUsage: event.tokenUsage,
2268
- error: event.error
2269
- });
2270
- return;
2271
- }
2272
- const nowIso = this.now().toISOString();
2273
- await this.store.saveRun({
2274
- ...run,
2275
- updatedAt: nowIso,
2276
- lastEvent: event.event ?? run.lastEvent ?? null,
2277
- lastTurnSummary: resolveLastTurnSummary(
2278
- run.lastTurnSummary,
2279
- resolveLastTurnSummaryCandidate(event.event, event.lastError)
2280
- ),
2281
- lastEventAt: event.lastEventAt,
2282
- lastEventAtSource: "event-channel",
2283
- tokenUsage: event.tokenUsage ?? run.tokenUsage,
2284
- rateLimits: event.rateLimits ?? run.rateLimits ?? null,
2285
- runtimeSession: buildRuntimeSession(
2286
- run.runtimeSession,
2287
- resolveChannelSessionId(event.sessionInfo),
2288
- event.sessionInfo?.threadId ?? run.runtimeSession?.threadId ?? null,
2289
- "active",
2290
- run.startedAt ?? run.runtimeSession?.startedAt ?? nowIso,
2291
- nowIso,
2292
- event.sessionInfo?.exitClassification ?? null
2293
- ),
2294
- threadId: event.sessionInfo?.threadId ?? run.threadId ?? run.runtimeSession?.threadId ?? null,
2295
- turnCount: event.sessionInfo && event.sessionInfo.turnCount != null ? event.sessionInfo.turnCount : run.turnCount,
2296
- cumulativeTurnCount: resolveCumulativeTurnCount(
2297
- run,
2298
- event.sessionInfo?.turnCount ?? null
2299
- ),
2300
- executionPhase: event.executionPhase ?? run.executionPhase ?? null,
2301
- runPhase: event.runPhase ?? run.runPhase ?? null,
2302
- lastError: event.lastError ?? run.lastError
2303
- });
2304
- }
2305
- logVerbose(message) {
2306
- if (!this.isVerboseLoggingEnabled()) {
2307
- return;
2308
- }
2309
- this.writeStderr(message);
2310
- }
2311
- async waitForNextPoll() {
2312
- if (this.consumePendingReconcileRequest()) {
2313
- return;
2314
- }
2315
- const customWait = this.dependencies.waitImpl;
2316
- const pollIntervalMs = this.getEffectivePollIntervalMs();
2317
- const waitPromise = this.createPendingSleepPromise();
2318
- try {
2319
- if (customWait) {
2320
- await Promise.race([customWait(pollIntervalMs), waitPromise]);
2321
- } else {
2322
- this.sleepTimer = setTimeout(() => {
2323
- this.sleepResolver?.();
2324
- }, pollIntervalMs);
2325
- await waitPromise;
2326
- }
2327
- } finally {
2328
- this.cancelPendingSleep();
2329
- }
2330
- this.consumePendingReconcileRequest();
2331
- }
2332
- cancelPendingSleep() {
2333
- if (this.sleepTimer) {
2334
- clearTimeout(this.sleepTimer);
2335
- this.sleepTimer = null;
2336
- }
2337
- this.sleepResolver?.();
2338
- this.sleepResolver = null;
2339
- }
2340
- createPendingSleepPromise() {
2341
- return new Promise((resolve4) => {
2342
- this.sleepResolver = () => {
2343
- this.sleepResolver = null;
2344
- this.sleepTimer = null;
2345
- resolve4();
2346
- };
2347
- });
2348
- }
2349
- consumePendingReconcileRequest() {
2350
- if (!this.reconcileRequested) {
2351
- return false;
2352
- }
2353
- this.reconcileRequested = false;
2354
- return true;
2355
- }
2356
- /**
2357
- * Classify whether a process exit should be treated as continuation retry
2358
- * or failure retry. Continuation applies when the issue is still actionable
2359
- * — the worker completed its session and the issue hasn't transitioned away.
2360
- * Failure applies when we cannot confirm the issue is still actionable.
2361
- */
2362
- async classifyRetryKind(tenant, run, trackerDependencies = {}) {
2363
- try {
2364
- const eligibleContext = await this.fetchTrackedIssueEligibilityContext(
2365
- tenant,
2366
- run.issueIdentifier,
2367
- trackerDependencies
2368
- );
2369
- if (!eligibleContext) {
2370
- return "failure";
2371
- }
2372
- const resolution = await this.loadProjectWorkflow(tenant, run.repository);
2373
- if (!isUsableWorkflowResolution(resolution)) {
2374
- return "failure";
2375
- }
2376
- return this.isIssueCandidateEligible(
2377
- eligibleContext.issue,
2378
- resolution.lifecycle,
2379
- eligibleContext.issues
2380
- ) ? "continuation" : "failure";
2381
- } catch {
2382
- return "failure";
2383
- }
2384
- }
2385
- async resolveRetryRestartAction(tenant, run, trackerDependencies = {}) {
2386
- try {
2387
- const eligibleContext = await this.fetchTrackedIssueEligibilityContext(
2388
- tenant,
2389
- run.issueIdentifier,
2390
- trackerDependencies
2391
- );
2392
- if (!eligibleContext) {
2393
- return "release";
2394
- }
2395
- const resolution = await this.loadProjectWorkflow(tenant, run.repository);
2396
- if (!isUsableWorkflowResolution(resolution)) {
2397
- return "restart";
2398
- }
2399
- return this.isIssueCandidateEligible(
2400
- eligibleContext.issue,
2401
- resolution.lifecycle,
2402
- eligibleContext.issues
2403
- ) ? "restart" : "release";
2404
- } catch {
2405
- return "restart";
2406
- }
2407
- }
2408
- async fetchTrackedIssueEligibilityContext(tenant, issueIdentifier, trackerDependencies = {}) {
2409
- const trackerAdapter = resolveTrackerAdapter2(tenant.tracker);
2410
- const issues = await trackerAdapter.listIssues(tenant, {
2411
- fetchImpl: this.dependencies.fetchImpl,
2412
- ...trackerDependencies
2413
- });
2414
- const issue = issues.find(
2415
- (candidate) => candidate.identifier === issueIdentifier
2416
- );
2417
- return issue ? { issue, issues } : null;
2418
- }
2419
- async fetchWorkerRunInfo(run) {
2420
- const latestRun = await this.store.loadRun(run.runId, run.projectId) ?? run;
2421
- const persistedTokenUsage = await this.readPersistedWorkerTokenUsage(latestRun);
2422
- return {
2423
- tokenUsage: persistedTokenUsage,
2424
- sessionId: latestRun.runtimeSession?.sessionId ?? null,
2425
- threadId: latestRun.threadId ?? latestRun.runtimeSession?.threadId ?? null,
2426
- turnCount: latestRun.turnCount ?? null,
2427
- exitClassification: latestRun.runtimeSession?.exitClassification ?? null,
2428
- lastError: latestRun.lastError ?? null,
2429
- lastEvent: latestRun.lastEvent ?? null,
2430
- lastEventAt: latestRun.lastEventAt ?? null,
2431
- lastEventAtSource: latestRun.lastEventAtSource ?? null,
2432
- executionPhase: latestRun.executionPhase ?? null,
2433
- runPhase: latestRun.runPhase ?? null,
2434
- rateLimits: latestRun.rateLimits ?? null
2435
- };
2436
- }
2437
- async readPersistedWorkerTokenUsage(run) {
2438
- const artifactPaths = [
2439
- join3(run.workspaceRuntimeDir, "token-usage.json"),
2440
- join3(
2441
- run.workspaceRuntimeDir,
2442
- ".orchestrator",
2443
- "runs",
2444
- run.runId,
2445
- "token-usage.json"
2446
- )
2447
- ];
2448
- for (const artifactPath of artifactPaths) {
2449
- try {
2450
- const raw = await readFile3(artifactPath, "utf8");
2451
- const tokenUsage = JSON.parse(
2452
- raw
2453
- );
2454
- if (hasTokenUsage(tokenUsage)) {
2455
- return tokenUsage;
2456
- }
2457
- } catch {
2458
- continue;
2459
- }
2460
- }
2461
- return null;
2462
- }
2463
- /**
2464
- * Execute a workspace lifecycle hook using the workflow configuration
2465
- * loaded from the repository. Returns the hook result or null if the
2466
- * workflow could not be loaded.
2467
- */
2468
- async runHook(kind, tenant, repositoryDirectory, repository, context, resolution) {
2469
- try {
2470
- const workflowResolution = resolution ?? await this.loadProjectWorkflow(tenant, repository);
2471
- if (!isUsableWorkflowResolution(workflowResolution)) {
2472
- return null;
2473
- }
2474
- const hookEnv = this.buildProjectExecutionEnv(
2475
- tenant.projectId,
2476
- buildHookEnv(context)
2477
- );
2478
- return executeWorkspaceHook({
2479
- kind,
2480
- hooks: workflowResolution.workflow.hooks,
2481
- repositoryPath: repositoryDirectory,
2482
- env: hookEnv,
2483
- timeoutMs: workflowResolution.workflow.hooks.timeoutMs
2484
- });
2485
- } catch {
2486
- return null;
2487
- }
2488
- }
2489
- readProjectEnv(projectId) {
2490
- const envPath = join3(this.store.projectDir(projectId), ".env");
2491
- try {
2492
- return readEnvFile(envPath);
2493
- } catch (error) {
2494
- const message = error instanceof Error ? error.message : "Unknown error occurred.";
2495
- (this.dependencies.stderr ?? process.stderr).write(
2496
- `[warn] Failed to load project env for ${projectId} from ${envPath}: ${message}
2497
- `
2498
- );
2499
- return {};
2500
- }
2501
- }
2502
- buildProjectExecutionEnv(projectId, env) {
2503
- const inheritedEnv = Object.fromEntries(
2504
- Object.entries(process.env).filter(
2505
- (entry) => typeof entry[1] === "string"
2506
- )
2507
- );
2508
- const explicitEnv = Object.fromEntries(
2509
- Object.entries(env).filter(
2510
- (entry) => typeof entry[1] === "string"
2511
- )
2512
- );
2513
- return {
2514
- ...this.readProjectEnv(projectId),
2515
- ...inheritedEnv,
2516
- ...explicitEnv
2517
- };
2518
- }
2519
- async restartRun(tenant, run, issueRecords, now, sessionId) {
2520
- const supersededRecord = {
2521
- ...run,
2522
- status: "failed",
2523
- completedAt: now.toISOString(),
2524
- updatedAt: now.toISOString(),
2525
- lastError: "Superseded by recovered run."
2526
- };
2527
- await this.store.saveRun(supersededRecord);
2528
- const issue = resolveTrackerAdapter2(tenant.tracker).reviveIssue(
2529
- tenant,
2530
- run
2531
- );
2532
- const restarted = await this.startRun(tenant, issue);
2533
- const recoveredRecord = {
2534
- ...restarted,
2535
- attempt: run.attempt,
2536
- retryKind: run.retryKind ?? "recovery",
2537
- createdAt: run.createdAt,
2538
- issueWorkspaceKey: run.issueWorkspaceKey,
2539
- threadId: null,
2540
- cumulativeTurnCount: resolvePersistedCumulativeTurnCount(run),
2541
- lastTurnSummary: run.lastTurnSummary ?? null,
2542
- turnCount: 0
2543
- };
2544
- await this.store.saveRun(recoveredRecord);
2545
- await this.store.appendRunEvent(run.runId, {
2546
- at: now.toISOString(),
2547
- event: "run-recovered",
2548
- projectId: run.projectId,
2549
- issueIdentifier: run.issueIdentifier,
2550
- issueId: run.issueId,
2551
- sessionId: sessionId ?? void 0
2552
- });
2553
- return {
2554
- issueRecords: upsertIssueOrchestration(issueRecords, {
2555
- issueId: recoveredRecord.issueId,
2556
- identifier: recoveredRecord.issueIdentifier,
2557
- workspaceKey: recoveredRecord.issueWorkspaceKey ?? deriveIssueWorkspaceKey(
2558
- {
2559
- projectId: tenant.projectId,
2560
- adapter: tenant.tracker.adapter,
2561
- issueSubjectId: recoveredRecord.issueSubjectId
2562
- },
2563
- recoveredRecord.issueIdentifier
2564
- ),
2565
- state: "running",
2566
- currentRunId: recoveredRecord.runId,
2567
- retryEntry: null,
2568
- updatedAt: now.toISOString()
2569
- }),
2570
- recovered: true
2571
- };
2572
- }
2573
- async releaseRetryingRun(run, issueRecords, now) {
2574
- const suppressedRun = {
2575
- ...run,
2576
- status: "suppressed",
2577
- processId: null,
2578
- completedAt: now.toISOString(),
2579
- updatedAt: now.toISOString(),
2580
- nextRetryAt: null,
2581
- runPhase: "canceled_by_reconciliation",
2582
- lastError: "Retry canceled because the tracker issue is no longer actionable."
2583
- };
2584
- await this.store.saveRun(suppressedRun);
2585
- this.logVerbose(
2586
- `[run-completed] ${suppressedRun.runId} status=${suppressedRun.status}`
2587
- );
2588
- return {
2589
- issueRecords: releaseIssueOrchestration(issueRecords, run.issueId, now),
2590
- recovered: false
2591
- };
2592
- }
2593
- async loadProjectPollInterval(tenant) {
2594
- const intervals = await Promise.all(
2595
- tenant.repositories.map(async (repository) => {
2596
- const resolution = await this.loadProjectWorkflow(tenant, repository);
2597
- return isUsableWorkflowResolution(resolution) ? resolution.workflow.polling.intervalMs : NaN;
2598
- })
2599
- );
2600
- const validIntervals = intervals.filter(
2601
- (value) => Number.isFinite(value) && value > 0
2602
- );
2603
- return validIntervals.length ? Math.min(...validIntervals) : DEFAULT_POLL_INTERVAL_MS;
2604
- }
2605
- async loadProjectMaxConcurrentByState(tenant) {
2606
- const result = {};
2607
- const resolutions = await Promise.all(
2608
- tenant.repositories.map(async (repository) => {
2609
- try {
2610
- return await this.loadProjectWorkflow(tenant, repository);
2611
- } catch {
2612
- return null;
2613
- }
2614
- })
2615
- );
2616
- for (const resolution of resolutions) {
2617
- if (!resolution) continue;
2618
- if (!isUsableWorkflowResolution(resolution)) continue;
2619
- const stateLimits = resolution.workflow.agent.maxConcurrentAgentsByState;
2620
- for (const [state, limit] of Object.entries(stateLimits)) {
2621
- const existing = result[state];
2622
- const numLimit = typeof limit === "number" ? limit : Number(limit);
2623
- result[state] = existing === void 0 ? numLimit : Math.min(existing, numLimit);
2624
- }
2625
- }
2626
- return result;
2627
- }
2628
- async loadRetryPolicy(tenant, repository) {
2629
- try {
2630
- const resolution = await this.loadProjectWorkflow(tenant, repository);
2631
- if (!isUsableWorkflowResolution(resolution)) {
2632
- return null;
2633
- }
2634
- return {
2635
- baseDelayMs: this.dependencies.retryBackoffMs ?? resolution.workflow.agent.retryBaseDelayMs,
2636
- maxDelayMs: this.dependencies.retryBackoffMs ?? resolution.workflow.agent.maxRetryBackoffMs,
2637
- stallTimeoutMs: resolveWorkflowRuntimeTimeouts(resolution.workflow).stallTimeoutMs
2638
- };
2639
- } catch {
2640
- if (!this.dependencies.retryBackoffMs) {
2641
- return null;
2642
- }
2643
- return {
2644
- baseDelayMs: this.dependencies.retryBackoffMs,
2645
- maxDelayMs: this.dependencies.retryBackoffMs,
2646
- stallTimeoutMs: null
2647
- };
2648
- }
2649
- }
2650
- async getProjectConcurrency(project) {
2651
- if (this.dependencies.concurrency !== void 0) {
2652
- return this.dependencies.concurrency;
2653
- }
2654
- const limits = await Promise.all(
2655
- project.repositories.map(async (repository) => {
2656
- try {
2657
- const resolution = await this.loadProjectWorkflow(
2658
- project,
2659
- repository
2660
- );
2661
- return isUsableWorkflowResolution(resolution) ? resolution.workflow.agent.maxConcurrentAgents : NaN;
2662
- } catch {
2663
- return NaN;
2664
- }
2665
- })
2666
- );
2667
- const validLimits = limits.filter(
2668
- (value) => Number.isFinite(value) && value >= 0
2669
- );
2670
- return validLimits.length ? Math.min(...validLimits) : DEFAULT_CONCURRENCY;
2671
- }
2672
- async resolveWorkflowResolution(repository, cacheRoot, resolution, changed) {
2673
- const cacheKey = this.workflowCacheKey(repository);
2674
- if (resolution.isValid) {
2675
- const effectiveResolution = {
2676
- ...resolution,
2677
- isValid: true,
2678
- usedLastKnownGood: false,
2679
- validationError: null
2680
- };
2681
- let workflowPath = effectiveResolution.workflowPath;
2682
- try {
2683
- workflowPath = await this.persistLastKnownGoodWorkflow(
2684
- cacheRoot,
2685
- effectiveResolution
2686
- ) ?? effectiveResolution.workflowPath;
2687
- } catch {
2688
- workflowPath = effectiveResolution.workflowPath;
2689
- }
2690
- this.lastKnownGoodWorkflows.set(cacheKey, {
2691
- ...effectiveResolution,
2692
- workflowPath
2693
- });
2694
- this.lastReportedWorkflowErrors.delete(cacheKey);
2695
- return effectiveResolution;
2696
- }
2697
- const cached = this.lastKnownGoodWorkflows.get(cacheKey);
2698
- const message = resolution.validationError ?? "Invalid repository WORKFLOW.md";
2699
- const previousMessage = this.lastReportedWorkflowErrors.get(cacheKey);
2700
- if (changed || previousMessage !== message) {
2701
- process.stderr.write(
2702
- `[orchestrator] failed to reload WORKFLOW.md for ${repository.owner}/${repository.name}: ${message}
2703
- `
2704
- );
2705
- this.lastReportedWorkflowErrors.set(cacheKey, message);
2706
- }
2707
- if (!cached) {
2708
- return resolution;
2709
- }
2710
- return {
2711
- ...cached,
2712
- workflowPath: cached.workflowPath,
2713
- isValid: false,
2714
- usedLastKnownGood: true,
2715
- validationError: message
2716
- };
2717
- }
2718
- async persistLastKnownGoodWorkflow(cacheRoot, resolution) {
2719
- if (!resolution.workflowPath) {
2720
- return null;
2721
- }
2722
- const snapshotPath = this.lastKnownGoodWorkflowPath(cacheRoot);
2723
- const markdown = await readFile3(resolution.workflowPath, "utf8");
2724
- await mkdir3(join3(cacheRoot, "last-known-good"), { recursive: true });
2725
- await writeFile3(snapshotPath, markdown, "utf8");
2726
- return snapshotPath;
2727
- }
2728
- lastKnownGoodWorkflowPath(cacheRoot) {
2729
- return join3(cacheRoot, "last-known-good", "WORKFLOW.md");
2730
- }
2731
- workflowCacheKey(repository) {
2732
- return `${repository.owner}/${repository.name}:${this.normalizeRepositoryCloneUrl(repository.cloneUrl)}`;
2733
- }
2734
- normalizeRepositoryCloneUrl(cloneUrl) {
2735
- if (cloneUrl.startsWith("file://")) {
2736
- try {
2737
- return fileURLToPath(cloneUrl);
2738
- } catch {
2739
- return cloneUrl;
2740
- }
2741
- }
2742
- return cloneUrl;
2743
- }
2744
- isProcessRunning(processId) {
2745
- if (this.dependencies.isProcessRunning) {
2746
- return this.dependencies.isProcessRunning(processId);
2747
- }
2748
- try {
2749
- process.kill(-processId, 0);
2750
- return true;
2751
- } catch {
2752
- return false;
2753
- }
2754
- }
2755
- sendSignal(processId, signal) {
2756
- try {
2757
- const kill = this.dependencies.killImpl;
2758
- if (kill) {
2759
- kill(processId, signal);
2760
- } else {
2761
- process.kill(-processId, signal);
2762
- }
2763
- } catch {
2764
- this.retireWorkerPid(processId);
2765
- }
2766
- }
2767
- pruneExitedWorkerPids() {
2768
- for (const pid of [...this.activeWorkerPids]) {
2769
- if (!this.isProcessRunning(pid)) {
2770
- this.retireWorkerPid(pid);
2771
- }
2772
- }
2773
- }
2774
- retireWorkerPid(processId) {
2775
- if (processId) {
2776
- this.activeWorkerPids.delete(processId);
2777
- }
2778
- }
2779
- /**
2780
- * Clean up the issue workspace for a terminal issue.
2781
- *
2782
- * Runs the `before_remove` hook if configured. Hook failures are logged and
2783
- * ignored so workspace cleanup still proceeds per spec 9.4. The workspace
2784
- * directory is removed and the record set to `removed`. Orchestration
2785
- * records (runs) are preserved.
2786
- */
2787
- async cleanupTerminalIssueWorkspace(tenant, issue, now, workflowResolution) {
2788
- const issueSubjectId = issue.id;
2789
- const identity = {
2790
- projectId: tenant.projectId,
2791
- adapter: issue.tracker.adapter,
2792
- issueSubjectId
2793
- };
2794
- const preferredWorkspaceKey = deriveIssueWorkspaceKey(
2795
- identity,
2796
- issue.identifier
2797
- );
2798
- const legacyWorkspaceKey = deriveLegacyIssueWorkspaceKey(identity);
2799
- const orchestrationRecord = (await this.store.loadProjectIssueOrchestrations(tenant.projectId)).find((record) => record.issueId === issue.id);
2800
- const workspaceRecord = (orchestrationRecord ? await this.store.loadIssueWorkspace(
2801
- tenant.projectId,
2802
- orchestrationRecord.workspaceKey
2803
- ) : null) ?? await this.store.loadIssueWorkspace(
2804
- tenant.projectId,
2805
- preferredWorkspaceKey
2806
- ) ?? (legacyWorkspaceKey === preferredWorkspaceKey ? null : await this.store.loadIssueWorkspace(
2807
- tenant.projectId,
2808
- legacyWorkspaceKey
2809
- ));
2810
- if (!workspaceRecord || workspaceRecord.status === "removed") {
2811
- return;
2812
- }
2813
- const pendingRecord = {
2814
- ...workspaceRecord,
2815
- status: "cleanup_pending",
2816
- updatedAt: now.toISOString()
2817
- };
2818
- await this.store.saveIssueWorkspace(pendingRecord);
2819
- const hookResult = await this.runHook(
2820
- "before_remove",
2821
- tenant,
2822
- workspaceRecord.repositoryPath,
2823
- issue.repository,
2824
- {
2825
- projectId: tenant.projectId,
2826
- workspaceKey: workspaceRecord.workspaceKey,
2827
- issueSubjectId,
2828
- issueIdentifier: issue.identifier,
2829
- workspacePath: workspaceRecord.workspacePath,
2830
- repositoryPath: workspaceRecord.repositoryPath
2831
- },
2832
- workflowResolution
2833
- );
2834
- if (hookResult && hookResult.outcome !== "success" && hookResult.outcome !== "skipped") {
2835
- const errorMessage = hookResult.error ?? `before_remove hook ${hookResult.outcome}`;
2836
- console.warn(
2837
- `[orchestrator] before_remove hook failed for ${issue.identifier}; continuing cleanup: ${errorMessage}`
2838
- );
2839
- }
2840
- try {
2841
- await rm3(workspaceRecord.workspacePath, { recursive: true, force: true });
2842
- } catch {
2843
- }
2844
- const removedRecord = {
2845
- ...workspaceRecord,
2846
- status: "removed",
2847
- updatedAt: now.toISOString(),
2848
- lastError: null
2849
- };
2850
- await this.store.saveIssueWorkspace(removedRecord);
2851
- }
2852
- resolveFailureRetryCount(issueRecords, issueId) {
2853
- return issueRecords.find((record) => record.issueId === issueId)?.failureRetryCount ?? null;
2854
- }
2855
- async isFailureRetrySuppressedIssue(tenant, issue, issueRecords, latestRun) {
2856
- const issueRecord = issueRecords.find(
2857
- (record) => record.issueId === issue.id || record.identifier === issue.identifier
2858
- ) ?? null;
2859
- if (!issueRecord || issueRecord.failureRetryCount <= 0) {
2860
- return false;
2861
- }
2862
- const maxFailureRetries = await this.loadMaxFailureRetries(
2863
- tenant,
2864
- issue.repository
2865
- );
2866
- if (issueRecord.failureRetryCount < maxFailureRetries) {
2867
- return false;
2868
- }
2869
- if (!latestRun || latestRun.status !== "suppressed" || latestRun.issueState !== issue.state || !latestRun.lastError?.includes(MAX_FAILURE_RETRIES_EXCEEDED_REASON)) {
2870
- return false;
2871
- }
2872
- const issueUpdatedAtMs = parseTimestampMs(issue.updatedAt);
2873
- const suppressedAtMs = parseTimestampMs(
2874
- latestRun.completedAt ?? latestRun.updatedAt
2875
- );
2876
- if (issueUpdatedAtMs === null || suppressedAtMs === null) {
2877
- return true;
2878
- }
2879
- return issueUpdatedAtMs <= suppressedAtMs;
2880
- }
2881
- async loadMaxFailureRetries(tenant, repository) {
2882
- try {
2883
- const resolution = await this.loadProjectWorkflow(tenant, repository);
2884
- return isUsableWorkflowResolution(resolution) ? resolution.workflow.agent.maxFailureRetries : DEFAULT_MAX_FAILURE_RETRIES;
2885
- } catch {
2886
- return DEFAULT_MAX_FAILURE_RETRIES;
2887
- }
2888
- }
2889
- };
2890
- function hasTokenUsage(tokenUsage) {
2891
- return Boolean(
2892
- tokenUsage && (tokenUsage.inputTokens > 0 || tokenUsage.outputTokens > 0 || tokenUsage.totalTokens > 0)
2893
- );
2894
- }
2895
- function isRecord(value) {
2896
- return !!value && typeof value === "object" && !Array.isArray(value);
2897
- }
2898
- function resolveProjectRateLimits(runs, issues) {
2899
- let latestRunRateLimits = null;
2900
- let latestRunTimestamp = -Infinity;
2901
- for (const run of runs) {
2902
- if (!isRecord(run.rateLimits)) {
2903
- continue;
2904
- }
2905
- const timestamp = parseTimestampMs(
2906
- run.lastEventAt ?? run.updatedAt ?? run.startedAt
2907
- );
2908
- const sortableTimestamp = timestamp ?? -Infinity;
2909
- if (sortableTimestamp >= latestRunTimestamp) {
2910
- latestRunTimestamp = sortableTimestamp;
2911
- latestRunRateLimits = run.rateLimits;
2912
- }
2913
- }
2914
- if (latestRunRateLimits) {
2915
- return latestRunRateLimits;
2916
- }
2917
- for (const issue of issues) {
2918
- if (isRecord(issue.rateLimits)) {
2919
- return issue.rateLimits;
2920
- }
2921
- }
2922
- return null;
2923
- }
2924
- function resolveTrackerRateLimits(issues) {
2925
- for (const issue of issues) {
2926
- if (isGitHubTrackerRateLimits(issue.rateLimits)) {
2927
- return issue.rateLimits;
2928
- }
2929
- }
2930
- return null;
2931
- }
2932
- function resolveAdaptivePollIntervalMs(basePollIntervalMs, rateLimits) {
2933
- if (!Number.isFinite(basePollIntervalMs) || basePollIntervalMs <= 0) {
2934
- return DEFAULT_POLL_INTERVAL_MS;
2935
- }
2936
- const ratio = extractRateLimitRatio(rateLimits);
2937
- if (ratio === null || ratio > 0.5) {
2938
- return basePollIntervalMs;
2939
- }
2940
- if (ratio >= 0.2) {
2941
- return basePollIntervalMs * 2;
2942
- }
2943
- if (ratio >= LOW_RATE_LIMIT_WARNING_THRESHOLD) {
2944
- return basePollIntervalMs * 4;
2945
- }
2946
- return basePollIntervalMs * 10;
2947
- }
2948
- function extractRateLimitRatio(rateLimits) {
2949
- if (!isRecord(rateLimits)) {
2950
- return null;
2951
- }
2952
- const limit = parseFiniteNumber(rateLimits.limit);
2953
- const remaining = parseFiniteNumber(rateLimits.remaining);
2954
- if (limit === null || remaining === null || limit <= 0 || remaining < 0) {
2955
- return null;
2956
- }
2957
- return remaining / limit;
2958
- }
2959
- function isGitHubTrackerRateLimits(rateLimits) {
2960
- if (!isRecord(rateLimits) || rateLimits.source !== "github") {
2961
- return false;
2962
- }
2963
- return rateLimits.resource === void 0 || rateLimits.resource === null || rateLimits.resource === "graphql";
2964
- }
2965
- function isLowRateLimit(rateLimits, threshold) {
2966
- const ratio = extractRateLimitRatio(rateLimits);
2967
- return ratio !== null && ratio < threshold;
2968
- }
2969
- function buildRuntimeSession(existing, sessionId, threadId, status, startedAt, updatedAt, exitClassification = void 0) {
2970
- if (existing === void 0 && sessionId === null && threadId === null && status === null && (exitClassification === void 0 || exitClassification === null)) {
2971
- return void 0;
2972
- }
2973
- return {
2974
- sessionId: sessionId ?? existing?.sessionId ?? null,
2975
- threadId: threadId ?? existing?.threadId ?? null,
2976
- status: status ?? existing?.status ?? null,
2977
- startedAt: existing?.startedAt ?? startedAt,
2978
- updatedAt,
2979
- exitClassification: exitClassification === void 0 ? existing?.exitClassification ?? null : exitClassification
2980
- };
2981
- }
2982
- function resolvePersistedCumulativeTurnCount(run) {
2983
- return run.cumulativeTurnCount ?? run.turnCount ?? 0;
2984
- }
2985
- function hasConvergenceLockedRun(runs, issueId, issueState, issueUpdatedAt) {
2986
- const latestRun = runs.filter((run) => run.issueId === issueId).sort(
2987
- (left, right) => new Date(right.updatedAt).getTime() - new Date(left.updatedAt).getTime()
2988
- )[0];
2989
- if (latestRun?.runtimeSession?.exitClassification !== "convergence-detected" || latestRun.issueState !== issueState) {
2990
- return false;
2991
- }
2992
- const convergedAtMs = parseTimestampMs(
2993
- latestRun.completedAt ?? latestRun.updatedAt
2994
- );
2995
- const issueUpdatedAtMs = parseTimestampMs(issueUpdatedAt);
2996
- if (convergedAtMs === null || issueUpdatedAtMs === null) {
2997
- return true;
2998
- }
2999
- return issueUpdatedAtMs <= convergedAtMs;
3000
- }
3001
- function resolveCumulativeTurnCount(run, turnCount) {
3002
- const carriedTotal = resolvePersistedCumulativeTurnCount(run);
3003
- if (turnCount === null) {
3004
- return carriedTotal;
3005
- }
3006
- const previousSessionTurnCount = run.turnCount ?? 0;
3007
- const baseTurnCount = Math.max(0, carriedTotal - previousSessionTurnCount);
3008
- return baseTurnCount + turnCount;
3009
- }
3010
- function isTerminalTurnEvent(event) {
3011
- return event === "turn/completed" || event === "turn/failed" || event === "turn/cancelled";
3012
- }
3013
- function resolveLastTurnSummaryCandidate(event, lastError) {
3014
- if (typeof lastError === "string" && lastError.trim()) {
3015
- return lastError.trim();
3016
- }
3017
- return typeof event === "string" && isTerminalTurnEvent(event) ? event : null;
3018
- }
3019
- function resolveLastTurnSummary(existing, candidate) {
3020
- if (typeof candidate === "string" && candidate.trim()) {
3021
- return candidate.trim();
3022
- }
3023
- return existing ?? null;
3024
- }
3025
- function canApplyWorkerChannelUpdate(status) {
3026
- return status === "running" || status === "retrying";
3027
- }
3028
- function resolveChannelSessionId(sessionInfo) {
3029
- if (!sessionInfo) {
3030
- return null;
3031
- }
3032
- return sessionInfo.sessionId ?? (sessionInfo.threadId && sessionInfo.turnId ? `${sessionInfo.threadId}-${sessionInfo.turnId}` : null);
3033
- }
3034
- function resolveWorkerCommand() {
3035
- if (process.env.SYMPHONY_WORKER_COMMAND) {
3036
- return process.env.SYMPHONY_WORKER_COMMAND;
3037
- }
3038
- try {
3039
- const workerUrl = import.meta.resolve("@gh-symphony/worker");
3040
- return `node ${fileURLToPath(workerUrl)}`;
3041
- } catch {
3042
- try {
3043
- const bundledWorker = join3(
3044
- fileURLToPath(new URL(".", import.meta.url)),
3045
- "worker-entry.js"
3046
- );
3047
- return `node ${bundledWorker}`;
3048
- } catch {
3049
- return DEFAULT_WORKER_COMMAND;
3050
- }
3051
- }
3052
- }
3053
- function createStore(runtimeRoot = ".runtime", options = {}) {
3054
- return new OrchestratorFsStore(runtimeRoot, options);
3055
- }
3056
- function sortCandidatesForDispatch(candidates) {
3057
- return [...candidates].sort((a, b) => {
3058
- if (a.priority !== b.priority) {
3059
- if (a.priority === null) return 1;
3060
- if (b.priority === null) return -1;
3061
- return a.priority - b.priority;
3062
- }
3063
- if (a.createdAt !== b.createdAt) {
3064
- if (a.createdAt === null) return 1;
3065
- if (b.createdAt === null) return -1;
3066
- return a.createdAt < b.createdAt ? -1 : 1;
3067
- }
3068
- return a.identifier.localeCompare(b.identifier);
3069
- });
3070
- }
3071
- function createProjectItemsCache() {
3072
- const entries = /* @__PURE__ */ new Map();
3073
- return {
3074
- getOrLoad(key, load) {
3075
- const cached = entries.get(key);
3076
- if (cached) {
3077
- return cached;
3078
- }
3079
- const pending = load().catch((error) => {
3080
- entries.delete(key);
3081
- throw error;
3082
- });
3083
- entries.set(key, pending);
3084
- return pending;
3085
- }
3086
- };
3087
- }
3088
- function wait2(ms) {
3089
- return new Promise((resolve4) => setTimeout(resolve4, ms));
3090
- }
3091
- function createRunId(now, projectId, issueIdentifier) {
3092
- return [
3093
- projectId,
3094
- issueIdentifier.replace(/[^a-zA-Z0-9]+/g, "-"),
3095
- now.getTime().toString(36)
3096
- ].join("-");
3097
- }
3098
- function buildLatestRunMapByIssueId(runs) {
3099
- const latestRuns = /* @__PURE__ */ new Map();
3100
- for (const run of runs) {
3101
- const existing = latestRuns.get(run.issueId);
3102
- if (!existing) {
3103
- latestRuns.set(run.issueId, run);
3104
- continue;
3105
- }
3106
- const runUpdatedAtMs = parseTimestampMs(run.updatedAt) ?? -Infinity;
3107
- const existingUpdatedAtMs = parseTimestampMs(existing.updatedAt) ?? -Infinity;
3108
- if (runUpdatedAtMs > existingUpdatedAtMs) {
3109
- latestRuns.set(run.issueId, run);
3110
- }
3111
- }
3112
- return latestRuns;
3113
- }
3114
- function isIssueOrchestrationClaimed(state) {
3115
- return state === "claimed" || state === "running" || state === "retry_queued";
3116
- }
3117
- function upsertIssueOrchestration(issueRecords, nextRecord) {
3118
- const existingRecord = issueRecords.find((record) => record.issueId === nextRecord.issueId) ?? null;
3119
- const remaining = issueRecords.filter(
3120
- (record) => record.issueId !== nextRecord.issueId
3121
- );
3122
- return [
3123
- ...remaining,
3124
- {
3125
- ...nextRecord,
3126
- completedOnce: nextRecord.completedOnce ?? existingRecord?.completedOnce ?? false,
3127
- failureRetryCount: nextRecord.failureRetryCount ?? existingRecord?.failureRetryCount ?? 0
3128
- }
3129
- ];
3130
- }
3131
- function releaseIssueOrchestration(issueRecords, issueId, now) {
3132
- return issueRecords.map(
3133
- (record) => record.issueId === issueId ? {
3134
- ...record,
3135
- state: "released",
3136
- currentRunId: null,
3137
- retryEntry: null,
3138
- updatedAt: now.toISOString()
3139
- } : record
3140
- );
3141
- }
3142
- function isActiveRunStatus(status) {
3143
- return status === "pending" || status === "starting" || status === "running" || status === "retrying";
3144
- }
3145
-
3146
- // ../orchestrator/src/lock.ts
3147
- import { randomUUID as randomUUID2 } from "crypto";
3148
- import { mkdir as mkdir4, open as open2, readFile as readFile4, rm as rm4 } from "fs/promises";
3149
- import { dirname as dirname2, isAbsolute, join as join4, relative as relative2, resolve as resolve2 } from "path";
3150
- import { setTimeout as delay } from "timers/promises";
3151
- var LOCK_READ_RETRY_DELAY_MS = 10;
3152
- var LOCK_READ_RETRY_LIMIT = 20;
3153
- async function acquireProjectLock(input) {
3154
- assertValidProjectId(input.projectId);
3155
- const pid = input.pid ?? process.pid;
3156
- const startedAt = (input.now ?? /* @__PURE__ */ new Date()).toISOString();
3157
- const ownerToken = `${pid}:${randomUUID2()}`;
3158
- const lockPath = resolveProjectLockPath(input.runtimeRoot, input.projectId);
3159
- const record = { ownerToken, pid, startedAt };
3160
- let invalidReadAttempts = 0;
3161
- for (; ; ) {
3162
- try {
3163
- await mkdir4(dirname2(lockPath), { recursive: true });
3164
- const handle = await open2(lockPath, "wx");
3165
- try {
3166
- await handle.writeFile(JSON.stringify(record, null, 2) + "\n", "utf8");
3167
- } finally {
3168
- await handle.close();
3169
- }
3170
- return { lockPath, ownerToken, pid, startedAt };
3171
- } catch (error) {
3172
- if (!isAlreadyExistsError2(error)) {
3173
- throw error;
3174
- }
3175
- }
3176
- const existing = await readProjectLock(lockPath);
3177
- if (existing.status === "missing") {
3178
- invalidReadAttempts = 0;
3179
- continue;
3180
- }
3181
- if (existing.status === "invalid") {
3182
- invalidReadAttempts += 1;
3183
- if (invalidReadAttempts >= LOCK_READ_RETRY_LIMIT) {
3184
- throw new Error(
3185
- `Project "${input.projectId}" lock file is unreadable at "${lockPath}".`
3186
- );
3187
- }
3188
- await delay(LOCK_READ_RETRY_DELAY_MS);
3189
- continue;
3190
- }
3191
- invalidReadAttempts = 0;
3192
- if ((input.isProcessRunning ?? isProcessRunning)(existing.record.pid)) {
3193
- throw new Error(
3194
- `Project "${input.projectId}" is already running (PID ${existing.record.pid}).`
3195
- );
3196
- }
3197
- await rm4(lockPath, { force: true });
3198
- }
3199
- }
3200
- async function releaseProjectLock(lock) {
3201
- if (!lock) {
3202
- return;
3203
- }
3204
- try {
3205
- const existing = await readProjectLock(lock.lockPath);
3206
- if (existing.status !== "valid" || existing.record.ownerToken !== lock.ownerToken) {
3207
- return;
3208
- }
3209
- } catch (error) {
3210
- if (isMissingFileError2(error)) {
3211
- return;
3212
- }
3213
- throw error;
3214
- }
3215
- await rm4(lock.lockPath, { force: true });
3216
- }
3217
- async function readProjectLock(lockPath) {
3218
- try {
3219
- const raw = await readFile4(lockPath, "utf8");
3220
- const record = parseProjectLock(raw);
3221
- if (!record) {
3222
- return { status: "invalid" };
3223
- }
3224
- return { status: "valid", record };
3225
- } catch (error) {
3226
- if (isMissingFileError2(error)) {
3227
- return { status: "missing" };
3228
- }
3229
- throw error;
3230
- }
3231
- }
3232
- function assertValidProjectId(projectId) {
3233
- if (projectId.length === 0 || projectId === "." || projectId === ".." || projectId.includes("/") || projectId.includes("\\")) {
3234
- throw new Error(
3235
- `Invalid project ID "${projectId}". Project IDs must not contain path separators or traversal segments.`
3236
- );
3237
- }
3238
- }
3239
- function resolveProjectLockPath(runtimeRoot, projectId) {
3240
- const store = new OrchestratorFsStore(runtimeRoot);
3241
- const projectsRoot = resolve2(runtimeRoot, "projects");
3242
- const projectDir = resolve2(store.projectDir(projectId));
3243
- const relativeProjectDir = relative2(projectsRoot, projectDir);
3244
- if (relativeProjectDir.length === 0 || relativeProjectDir.startsWith("..") || isAbsolute(relativeProjectDir)) {
3245
- throw new Error(
3246
- `Invalid project ID "${projectId}". Project lock path must stay within "${projectsRoot}".`
3247
- );
3248
- }
3249
- return join4(projectDir, ".lock");
3250
- }
3251
- function parseProjectLock(raw) {
3252
- try {
3253
- const parsed = JSON.parse(raw);
3254
- if (typeof parsed.ownerToken !== "string" || typeof parsed.pid !== "number" || !Number.isInteger(parsed.pid) || parsed.pid <= 0 || typeof parsed.startedAt !== "string") {
3255
- return null;
3256
- }
3257
- return {
3258
- ownerToken: parsed.ownerToken,
3259
- pid: parsed.pid,
3260
- startedAt: parsed.startedAt
3261
- };
3262
- } catch {
3263
- return null;
3264
- }
3265
- }
3266
- function isProcessRunning(pid) {
3267
- try {
3268
- process.kill(pid, 0);
3269
- return true;
3270
- } catch (error) {
3271
- return !(error && typeof error === "object" && "code" in error && error.code === "ESRCH");
3272
- }
3273
- }
3274
- function isAlreadyExistsError2(error) {
3275
- return Boolean(
3276
- error && typeof error === "object" && "code" in error && error.code === "EEXIST"
3277
- );
3278
- }
3279
- function isMissingFileError2(error) {
3280
- return Boolean(
3281
- error && typeof error === "object" && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR")
3282
- );
3283
- }
3284
-
3285
- // ../orchestrator/src/index.ts
3286
- import { pathToFileURL } from "url";
3287
- import { resolve as resolve3 } from "path";
3288
-
3289
- // ../orchestrator/src/runtime-factory.ts
3290
- import { join as join5 } from "path";
3291
-
3292
- // ../orchestrator/src/index.ts
3293
- function resolveOrchestratorLogLevel(value) {
3294
- if (!value || value === "normal") {
3295
- return "normal";
3296
- }
3297
- if (value === "verbose") {
3298
- return "verbose";
3299
- }
3300
- throw new Error(
3301
- `Unsupported log level: ${value}. Supported values: normal, verbose.`
3302
- );
3303
- }
3304
- async function runCli(argv, dependencies = {}) {
3305
- const [command = "run-once", ...args] = argv;
3306
- const parsed = parseArgs(args);
3307
- if (parsed.projectId) {
3308
- assertValidProjectId(parsed.projectId);
3309
- }
3310
- const runtimeRoot = resolve3(parsed.runtimeRoot ?? ".runtime");
3311
- const stderr = dependencies.stderr ?? process.stderr;
3312
- const eventsDir = resolveOptionalPath(
3313
- parsed.eventsDir ?? process.env.SYMPHONY_EVENTS_DIR
3314
- );
3315
- const logLevel = resolveOrchestratorLogLevel(
3316
- parsed.logLevel ?? process.env.SYMPHONY_LOG_LEVEL
3317
- );
3318
- const service = await dependencies.createService?.(runtimeRoot, parsed.projectId, {
3319
- eventsDir,
3320
- logLevel,
3321
- stderr
3322
- }) ?? await createServiceForRuntime(runtimeRoot, parsed.projectId, {
3323
- eventsDir,
3324
- logLevel,
3325
- stderr
3326
- });
3327
- const stdout = dependencies.stdout ?? process.stdout;
3328
- const exitProcess = dependencies.exitProcess ?? process.exit;
3329
- const signalTarget = dependencies.signalTarget ?? process;
3330
- switch (command) {
3331
- case "run": {
3332
- let lock = null;
3333
- let cleanupPromise = null;
3334
- let shuttingDownForSignal = false;
3335
- const cleanup = async () => {
3336
- if (cleanupPromise) {
3337
- return cleanupPromise;
3338
- }
3339
- cleanupPromise = (async () => {
3340
- let cleanupError;
3341
- const shutdownPromise = service.shutdown();
3342
- try {
3343
- await shutdownPromise;
3344
- } catch (error) {
3345
- cleanupError = error;
3346
- } finally {
3347
- try {
3348
- await (dependencies.releaseLock ?? releaseProjectLock)(lock);
3349
- lock = null;
3350
- } catch (lockError) {
3351
- cleanupError ??= lockError;
3352
- }
3353
- }
3354
- if (cleanupError) {
3355
- throw cleanupError;
3356
- }
3357
- })();
3358
- return cleanupPromise;
3359
- };
3360
- const handleSignal = (signal) => {
3361
- shuttingDownForSignal = true;
3362
- let exitCode = 0;
3363
- void cleanup().catch((error) => {
3364
- exitCode = 1;
3365
- stderr.write(
3366
- `Failed to shut down orchestrator after ${signal}: ${error instanceof Error ? error.message : String(error)}
3367
- `
3368
- );
3369
- }).finally(() => {
3370
- exitProcess(exitCode);
3371
- });
3372
- };
3373
- const sigintHandler = () => handleSignal("SIGINT");
3374
- const sigtermHandler = () => handleSignal("SIGTERM");
3375
- try {
3376
- if (parsed.projectId) {
3377
- lock = await (dependencies.acquireLock ?? acquireProjectLock)({
3378
- runtimeRoot,
3379
- projectId: parsed.projectId
3380
- });
3381
- }
3382
- signalTarget.once("SIGINT", sigintHandler);
3383
- signalTarget.once("SIGTERM", sigtermHandler);
3384
- await service.run({
3385
- issueIdentifier: parsed.issueIdentifier
3386
- });
3387
- await cleanup();
3388
- } finally {
3389
- signalTarget.off("SIGINT", sigintHandler);
3390
- signalTarget.off("SIGTERM", sigtermHandler);
3391
- if (!shuttingDownForSignal) {
3392
- await cleanup();
3393
- }
3394
- }
3395
- return;
3396
- }
3397
- case "run-once":
3398
- case "dispatch": {
3399
- const result = await service.runOnce({
3400
- issueIdentifier: parsed.issueIdentifier
3401
- });
3402
- stdout.write(JSON.stringify(result, null, 2) + "\n");
3403
- return;
3404
- }
3405
- case "run-issue": {
3406
- if (!parsed.projectId || !parsed.issueIdentifier) {
3407
- throw new Error("run-issue requires --project-id and --issue.");
3408
- }
3409
- const result = await service.runOnce({
3410
- issueIdentifier: parsed.issueIdentifier
3411
- });
3412
- stdout.write(JSON.stringify(result, null, 2) + "\n");
3413
- return;
3414
- }
3415
- case "recover": {
3416
- const result = await service.recover();
3417
- stdout.write(JSON.stringify(result, null, 2) + "\n");
3418
- return;
3419
- }
3420
- case "status": {
3421
- const result = await service.status();
3422
- stdout.write(JSON.stringify(result, null, 2) + "\n");
3423
- return;
3424
- }
3425
- default:
3426
- throw new Error(`Unsupported command: ${command}`);
3427
- }
3428
- }
3429
- async function createServiceForRuntime(runtimeRoot, projectId, options) {
3430
- if (!projectId) {
3431
- throw new Error("Orchestrator CLI requires --project-id.");
3432
- }
3433
- const store = createStore(runtimeRoot, {
3434
- eventsMirrorRoot: options?.eventsDir
3435
- });
3436
- const projectConfig = await store.loadProjectConfig(projectId);
3437
- if (!projectConfig) {
3438
- throw new Error(`Project config not found for "${projectId}".`);
3439
- }
3440
- return new OrchestratorService(store, projectConfig, options);
3441
- }
3442
- async function main() {
3443
- await runCli(process.argv.slice(2));
3444
- }
3445
- function parseArgs(args) {
3446
- const parsed = {};
3447
- for (let index = 0; index < args.length; index += 1) {
3448
- const argument = args[index];
3449
- const value = args[index + 1];
3450
- if (!argument?.startsWith("--")) {
3451
- continue;
3452
- }
3453
- switch (argument) {
3454
- case "--runtime-root":
3455
- parsed.runtimeRoot = value;
3456
- index += 1;
3457
- break;
3458
- case "--project":
3459
- case "--project-id":
3460
- parsed.projectId = value;
3461
- index += 1;
3462
- break;
3463
- case "--issue":
3464
- parsed.issueIdentifier = value;
3465
- index += 1;
3466
- break;
3467
- case "--events-dir":
3468
- if (!value || value.startsWith("-")) {
3469
- throw new Error(`Option '${argument}' argument missing`);
3470
- }
3471
- parsed.eventsDir = value;
3472
- index += 1;
3473
- break;
3474
- case "--log-level":
3475
- if (!value || value.startsWith("-")) {
3476
- throw new Error(`Option '${argument}' argument missing`);
3477
- }
3478
- parsed.logLevel = value;
3479
- index += 1;
3480
- break;
3481
- default:
3482
- throw new Error(`Unknown option: ${argument}`);
3483
- }
3484
- }
3485
- return parsed;
3486
- }
3487
- function resolveOptionalPath(value) {
3488
- if (!value || value.trim().length === 0) {
3489
- return void 0;
3490
- }
3491
- return resolve3(value.trim());
3492
- }
3493
- if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
3494
- main().catch((error) => {
3495
- process.stderr.write(
3496
- `${error instanceof Error ? error.message : "Unknown error"}
3497
- `
3498
- );
3499
- process.exitCode = 1;
3500
- });
3501
- }
3502
-
3503
- export {
3504
- OrchestratorService,
3505
- createStore,
3506
- acquireProjectLock,
3507
- releaseProjectLock,
3508
- resolveOrchestratorLogLevel,
3509
- runCli
3510
- };