@agent-relay/factory 0.0.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +240 -22
- package/bin/factory.mjs +7 -0
- package/dist/cli/fleet.d.ts +77 -0
- package/dist/cli/fleet.d.ts.map +1 -0
- package/dist/cli/fleet.js +500 -0
- package/dist/cli/fleet.js.map +1 -0
- package/dist/config/schema.d.ts +1313 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +309 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/constants/linear.d.ts +4 -0
- package/dist/constants/linear.d.ts.map +1 -0
- package/dist/constants/linear.js +15 -0
- package/dist/constants/linear.js.map +1 -0
- package/dist/constants/slack.d.ts +3 -0
- package/dist/constants/slack.d.ts.map +1 -0
- package/dist/constants/slack.js +3 -0
- package/dist/constants/slack.js.map +1 -0
- package/dist/dispatch/templates.d.ts +36 -0
- package/dist/dispatch/templates.d.ts.map +1 -0
- package/dist/dispatch/templates.js +110 -0
- package/dist/dispatch/templates.js.map +1 -0
- package/dist/fleet/create-fleet.d.ts +13 -0
- package/dist/fleet/create-fleet.d.ts.map +1 -0
- package/dist/fleet/create-fleet.js +14 -0
- package/dist/fleet/create-fleet.js.map +1 -0
- package/dist/fleet/internal-fleet-client.d.ts +85 -0
- package/dist/fleet/internal-fleet-client.d.ts.map +1 -0
- package/dist/fleet/internal-fleet-client.js +575 -0
- package/dist/fleet/internal-fleet-client.js.map +1 -0
- package/dist/fleet/relay-fleet-client.d.ts +28 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -0
- package/dist/fleet/relay-fleet-client.js +33 -0
- package/dist/fleet/relay-fleet-client.js.map +1 -0
- package/dist/github/index.d.ts +5 -0
- package/dist/github/index.d.ts.map +1 -0
- package/dist/github/index.js +3 -0
- package/dist/github/index.js.map +1 -0
- package/dist/github/merge-gate.d.ts +47 -0
- package/dist/github/merge-gate.d.ts.map +1 -0
- package/dist/github/merge-gate.js +147 -0
- package/dist/github/merge-gate.js.map +1 -0
- package/dist/github/probe-closer.d.ts +15 -0
- package/dist/github/probe-closer.d.ts.map +1 -0
- package/dist/github/probe-closer.js +67 -0
- package/dist/github/probe-closer.js.map +1 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -0
- package/dist/issue-key-match.d.ts +3 -0
- package/dist/issue-key-match.d.ts.map +1 -0
- package/dist/issue-key-match.js +22 -0
- package/dist/issue-key-match.js.map +1 -0
- package/dist/linear/state-resolver.d.ts +25 -0
- package/dist/linear/state-resolver.d.ts.map +1 -0
- package/dist/linear/state-resolver.js +221 -0
- package/dist/linear/state-resolver.js.map +1 -0
- package/dist/mount/local-mount-preflight.d.ts +8 -0
- package/dist/mount/local-mount-preflight.d.ts.map +1 -0
- package/dist/mount/local-mount-preflight.js +97 -0
- package/dist/mount/local-mount-preflight.js.map +1 -0
- package/dist/mount/relayfile-binary.d.ts +7 -0
- package/dist/mount/relayfile-binary.d.ts.map +1 -0
- package/dist/mount/relayfile-binary.js +143 -0
- package/dist/mount/relayfile-binary.js.map +1 -0
- package/dist/mount/relayfile-cloud-mount-client.d.ts +123 -0
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -0
- package/dist/mount/relayfile-cloud-mount-client.js +427 -0
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -0
- package/dist/orchestrator/batch-tracker.d.ts +42 -0
- package/dist/orchestrator/batch-tracker.d.ts.map +1 -0
- package/dist/orchestrator/batch-tracker.js +125 -0
- package/dist/orchestrator/batch-tracker.js.map +1 -0
- package/dist/orchestrator/factory.d.ts +55 -0
- package/dist/orchestrator/factory.d.ts.map +1 -0
- package/dist/orchestrator/factory.js +4018 -0
- package/dist/orchestrator/factory.js.map +1 -0
- package/dist/orchestrator/index.d.ts +5 -0
- package/dist/orchestrator/index.d.ts.map +1 -0
- package/dist/orchestrator/index.js +4 -0
- package/dist/orchestrator/index.js.map +1 -0
- package/dist/orchestrator/process-identity.d.ts +23 -0
- package/dist/orchestrator/process-identity.d.ts.map +1 -0
- package/dist/orchestrator/process-identity.js +122 -0
- package/dist/orchestrator/process-identity.js.map +1 -0
- package/dist/orchestrator/reaper.d.ts +60 -0
- package/dist/orchestrator/reaper.d.ts.map +1 -0
- package/dist/orchestrator/reaper.js +293 -0
- package/dist/orchestrator/reaper.js.map +1 -0
- package/dist/ports/fleet.d.ts +95 -0
- package/dist/ports/fleet.d.ts.map +1 -0
- package/dist/ports/fleet.js +2 -0
- package/dist/ports/fleet.js.map +1 -0
- package/dist/ports/index.d.ts +5 -0
- package/dist/ports/index.d.ts.map +1 -0
- package/dist/ports/index.js +2 -0
- package/dist/ports/index.js.map +1 -0
- package/dist/ports/mount.d.ts +56 -0
- package/dist/ports/mount.d.ts.map +1 -0
- package/dist/ports/mount.js +2 -0
- package/dist/ports/mount.js.map +1 -0
- package/dist/ports/state.d.ts +61 -0
- package/dist/ports/state.d.ts.map +1 -0
- package/dist/ports/state.js +2 -0
- package/dist/ports/state.js.map +1 -0
- package/dist/ports/system.d.ts +15 -0
- package/dist/ports/system.d.ts.map +1 -0
- package/dist/ports/system.js +2 -0
- package/dist/ports/system.js.map +1 -0
- package/dist/ports/writeback.d.ts +25 -0
- package/dist/ports/writeback.d.ts.map +1 -0
- package/dist/ports/writeback.js +2 -0
- package/dist/ports/writeback.js.map +1 -0
- package/dist/safety/factory-scope.d.ts +16 -0
- package/dist/safety/factory-scope.d.ts.map +1 -0
- package/dist/safety/factory-scope.js +107 -0
- package/dist/safety/factory-scope.js.map +1 -0
- package/dist/state/in-memory-state-store.d.ts +31 -0
- package/dist/state/in-memory-state-store.d.ts.map +1 -0
- package/dist/state/in-memory-state-store.js +115 -0
- package/dist/state/in-memory-state-store.js.map +1 -0
- package/dist/state/index.d.ts +3 -0
- package/dist/state/index.d.ts.map +1 -0
- package/dist/state/index.js +2 -0
- package/dist/state/index.js.map +1 -0
- package/dist/state/store.d.ts +29 -0
- package/dist/state/store.d.ts.map +1 -0
- package/dist/state/store.js +26 -0
- package/dist/state/store.js.map +1 -0
- package/dist/subscriptions/common.d.ts +7 -0
- package/dist/subscriptions/common.d.ts.map +1 -0
- package/dist/subscriptions/common.js +59 -0
- package/dist/subscriptions/common.js.map +1 -0
- package/dist/subscriptions/event-client.d.ts +40 -0
- package/dist/subscriptions/event-client.d.ts.map +1 -0
- package/dist/subscriptions/event-client.js +234 -0
- package/dist/subscriptions/event-client.js.map +1 -0
- package/dist/subscriptions/globs.d.ts +5 -0
- package/dist/subscriptions/globs.d.ts.map +1 -0
- package/dist/subscriptions/globs.js +50 -0
- package/dist/subscriptions/globs.js.map +1 -0
- package/dist/subscriptions/index.d.ts +8 -0
- package/dist/subscriptions/index.d.ts.map +1 -0
- package/dist/subscriptions/index.js +5 -0
- package/dist/subscriptions/index.js.map +1 -0
- package/dist/subscriptions/linear-filter.d.ts +26 -0
- package/dist/subscriptions/linear-filter.d.ts.map +1 -0
- package/dist/subscriptions/linear-filter.js +143 -0
- package/dist/subscriptions/linear-filter.js.map +1 -0
- package/dist/subscriptions/specs.d.ts +44 -0
- package/dist/subscriptions/specs.d.ts.map +1 -0
- package/dist/subscriptions/specs.js +195 -0
- package/dist/subscriptions/specs.js.map +1 -0
- package/dist/testing/fakes.d.ts +114 -0
- package/dist/testing/fakes.d.ts.map +1 -0
- package/dist/testing/fakes.js +219 -0
- package/dist/testing/fakes.js.map +1 -0
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +2 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/triage/heuristic.d.ts +38 -0
- package/dist/triage/heuristic.d.ts.map +1 -0
- package/dist/triage/heuristic.js +262 -0
- package/dist/triage/heuristic.js.map +1 -0
- package/dist/triage/index.d.ts +7 -0
- package/dist/triage/index.d.ts.map +1 -0
- package/dist/triage/index.js +5 -0
- package/dist/triage/index.js.map +1 -0
- package/dist/triage/llm.d.ts +11 -0
- package/dist/triage/llm.d.ts.map +1 -0
- package/dist/triage/llm.js +46 -0
- package/dist/triage/llm.js.map +1 -0
- package/dist/triage/schema.d.ts +240 -0
- package/dist/triage/schema.d.ts.map +1 -0
- package/dist/triage/schema.js +34 -0
- package/dist/triage/schema.js.map +1 -0
- package/dist/triage/tiered.d.ts +8 -0
- package/dist/triage/tiered.d.ts.map +1 -0
- package/dist/triage/tiered.js +40 -0
- package/dist/triage/tiered.js.map +1 -0
- package/dist/types.d.ts +211 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/webhook/handler.d.ts +22 -0
- package/dist/webhook/handler.d.ts.map +1 -0
- package/dist/webhook/handler.js +57 -0
- package/dist/webhook/handler.js.map +1 -0
- package/dist/webhook/index.d.ts +5 -0
- package/dist/webhook/index.d.ts.map +1 -0
- package/dist/webhook/index.js +3 -0
- package/dist/webhook/index.js.map +1 -0
- package/dist/webhook/registrar.d.ts +16 -0
- package/dist/webhook/registrar.d.ts.map +1 -0
- package/dist/webhook/registrar.js +50 -0
- package/dist/webhook/registrar.js.map +1 -0
- package/dist/writeback/github.d.ts +6 -0
- package/dist/writeback/github.d.ts.map +1 -0
- package/dist/writeback/github.js +48 -0
- package/dist/writeback/github.js.map +1 -0
- package/dist/writeback/index.d.ts +6 -0
- package/dist/writeback/index.d.ts.map +1 -0
- package/dist/writeback/index.js +4 -0
- package/dist/writeback/index.js.map +1 -0
- package/dist/writeback/linear.d.ts +41 -0
- package/dist/writeback/linear.d.ts.map +1 -0
- package/dist/writeback/linear.js +243 -0
- package/dist/writeback/linear.js.map +1 -0
- package/dist/writeback/shared.d.ts +7 -0
- package/dist/writeback/shared.d.ts.map +1 -0
- package/dist/writeback/shared.js +29 -0
- package/dist/writeback/shared.js.map +1 -0
- package/dist/writeback/slack.d.ts +16 -0
- package/dist/writeback/slack.d.ts.map +1 -0
- package/dist/writeback/slack.js +104 -0
- package/dist/writeback/slack.js.map +1 -0
- package/package.json +46 -9
- package/index.d.ts +0 -2
- package/index.js +0 -7
|
@@ -0,0 +1,4018 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
3
|
+
import { FactoryConfigSchema } from '../config/schema';
|
|
4
|
+
import { linearByStatePath } from '../constants/linear';
|
|
5
|
+
import { stateResolutionFromIds } from '../linear/state-resolver';
|
|
6
|
+
import { GithubMergeGate, closeProbePr } from '../github';
|
|
7
|
+
import { InMemoryStateStore } from '../state/in-memory-state-store';
|
|
8
|
+
import { containsExplicitIssueReference, containsIssueKey } from '../issue-key-match';
|
|
9
|
+
import { isInFactoryScope } from '../safety/factory-scope';
|
|
10
|
+
import { renderAgentTask } from '../dispatch/templates';
|
|
11
|
+
import { HeuristicTriage, TieredTriage, babysitterSpec } from '../triage';
|
|
12
|
+
import { MountGithubRead, MountLinearWriteback, MountSlackWriteback } from '../writeback';
|
|
13
|
+
import { asRecord, parseJsonContent, stableHash, wrappedPayload } from '../writeback/shared';
|
|
14
|
+
import { issueKey } from './batch-tracker';
|
|
15
|
+
import { findAgentProcessByName, readProcessIdentity } from './process-identity';
|
|
16
|
+
import { terminatePids } from './reaper';
|
|
17
|
+
const ISSUE_ROOT = '/linear/issues';
|
|
18
|
+
const GITHUB_ISSUE_ROOT = '/github/repos';
|
|
19
|
+
const READY_EVENTS_LIMIT = 100;
|
|
20
|
+
const LIVE_ISSUE_GLOB = `${ISSUE_ROOT}/**`;
|
|
21
|
+
// Subscribe broadly under /github/repos and let isGithubIssueFilePath() /
|
|
22
|
+
// githubPullPathParts() re-validate the exact shape in the callback.
|
|
23
|
+
// globMatchesPath() treats a non-terminal `**` as a single-segment wildcard, so
|
|
24
|
+
// a more specific glob like `.../issues/**/*.json` would miss the two-segment
|
|
25
|
+
// <owner>/<repo> prefix and the nested <number>__<slug>/meta.json (and
|
|
26
|
+
// directory) event shapes this factory accepts. A terminal `**` prefix-matches
|
|
27
|
+
// every descendant, so all supported issue AND pull-request path variants reach
|
|
28
|
+
// the handler — including the PR change events the babysitter reacts to.
|
|
29
|
+
export const LIVE_GITHUB_ISSUE_GLOB = `${GITHUB_ISSUE_ROOT}/**`;
|
|
30
|
+
const LIVE_DEDUPE_LIMIT = 5_000;
|
|
31
|
+
const LIVE_EVENT_DRAIN_BATCH_SIZE = 5;
|
|
32
|
+
const COMPLETION_SWEEP_INTERVAL_MS = 15_000;
|
|
33
|
+
const COMPLETION_SWEEP_BATCH_SIZE = 2;
|
|
34
|
+
const PROBE_PR_GH_BACKOFF_MS = 60_000;
|
|
35
|
+
const PROBE_PR_GH_CANDIDATE_LIMIT = 200;
|
|
36
|
+
const SLACK_REPLY_EVENTS_LIMIT = 100;
|
|
37
|
+
const SLACK_REPLY_POLL_INTERVAL_MS = 5_000;
|
|
38
|
+
const AGENT_QUESTION_DEDUPE_LIMIT = 500;
|
|
39
|
+
const AGENT_NEEDS_INPUT_MARKER = '[factory-needs-input]';
|
|
40
|
+
const LEGACY_AGENT_NEEDS_INPUT_MARKER = 'FACTORY_NEEDS_INPUT';
|
|
41
|
+
// The babysitter DMs `factory` with this marker once it believes the PR is
|
|
42
|
+
// green; the orchestrator confirms readiness with one authoritative gh read
|
|
43
|
+
// before transitioning the issue to Human Review.
|
|
44
|
+
const AGENT_PR_READY_MARKER = '[factory-pr-ready]';
|
|
45
|
+
const FACTORY_E2E_MARKER = '[factory-e2e]';
|
|
46
|
+
const INJECTION_CONFIRMATION_TIMEOUT_MS = 90_000;
|
|
47
|
+
const INJECTION_RETRY_DELAY_MS = 1_000;
|
|
48
|
+
const INJECTION_RETRY_ATTEMPT_TIMEOUT_MS = 15_000;
|
|
49
|
+
const INJECTION_MAX_ATTEMPTS = 6;
|
|
50
|
+
const STOP_TEARDOWN_TIMEOUT_MS = 2_500;
|
|
51
|
+
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
52
|
+
const MERGE_GATE_MAX_ATTEMPTS = 12;
|
|
53
|
+
const MERGE_GATE_POLL_DELAY_MS = 10_000;
|
|
54
|
+
const MAX_LABEL_IMPLEMENTERS = 4;
|
|
55
|
+
const DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS = 5 * 60_000;
|
|
56
|
+
const DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
57
|
+
const GITHUB_FACTORY_LABEL = 'factory';
|
|
58
|
+
const GITHUB_MIRROR_TITLE_PREFIX = '[factory]';
|
|
59
|
+
const GITHUB_MIRROR_SOURCE_PREFIX = 'Source: ';
|
|
60
|
+
export const DEFAULT_FACTORY_LOOP_HEARTBEAT_PATH = '/tmp/factory-run/factory-loop-heartbeat.json';
|
|
61
|
+
export const DEFAULT_FACTORY_LOOP_REGISTRY_PATH = '/tmp/factory-run/factory-loop-registry.json';
|
|
62
|
+
const realClock = {
|
|
63
|
+
now: () => Date.now(),
|
|
64
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
65
|
+
};
|
|
66
|
+
export function createFactory(config, ports) {
|
|
67
|
+
return new FactoryLoop(FactoryConfigSchema.parse(config), ports);
|
|
68
|
+
}
|
|
69
|
+
export class FactoryLoop {
|
|
70
|
+
#config;
|
|
71
|
+
#mount;
|
|
72
|
+
#states;
|
|
73
|
+
#fleet;
|
|
74
|
+
#triage;
|
|
75
|
+
#linear;
|
|
76
|
+
#slack;
|
|
77
|
+
#mergeGate;
|
|
78
|
+
#probeCloser;
|
|
79
|
+
#probePrResolver;
|
|
80
|
+
#customProbePrResolver;
|
|
81
|
+
#probePrGhRunner;
|
|
82
|
+
#logger;
|
|
83
|
+
#clock;
|
|
84
|
+
#processIdentityReader;
|
|
85
|
+
#processFinder;
|
|
86
|
+
#kill;
|
|
87
|
+
#readChildPids;
|
|
88
|
+
#terminationGraceMs;
|
|
89
|
+
#state;
|
|
90
|
+
#workspaceId;
|
|
91
|
+
#batchView;
|
|
92
|
+
#batchReady;
|
|
93
|
+
#listeners = new Map();
|
|
94
|
+
#counters = {};
|
|
95
|
+
#resumeInFlight = new Map();
|
|
96
|
+
#slackWatchers = new Map();
|
|
97
|
+
#slackWatcherStarts = new Map();
|
|
98
|
+
// Last invalid-label failure signature we posted per issue, so a stuck Ready
|
|
99
|
+
// issue (or the comment writeback's own change event) does not re-post the
|
|
100
|
+
// same notice every cycle. Cleared once the issue dispatches successfully.
|
|
101
|
+
#labelDispatchFailures = new Map();
|
|
102
|
+
#postMergeDoneAdvances = new Set();
|
|
103
|
+
#slackDegraded = false;
|
|
104
|
+
#slackDegradedReason;
|
|
105
|
+
#slackWritebackFailureDegraded = false;
|
|
106
|
+
#slackWritebackFailureBackoffUntilMs = 0;
|
|
107
|
+
#slackEventWatermarkCache;
|
|
108
|
+
#slackEventWatermarkRefresh;
|
|
109
|
+
#subscription;
|
|
110
|
+
#livePollTimer;
|
|
111
|
+
#livePollInFlight = false;
|
|
112
|
+
#liveEventCursor;
|
|
113
|
+
#liveEventHighWatermark;
|
|
114
|
+
#liveConnectStartedAtMs = 0;
|
|
115
|
+
#liveReplaySkewMarginMs = 0;
|
|
116
|
+
#liveHeartbeatTimer;
|
|
117
|
+
#liveHeartbeatActive = false;
|
|
118
|
+
#liveHeartbeatInFlight = false;
|
|
119
|
+
#liveHeartbeatRefresh;
|
|
120
|
+
#liveHeartbeatLastWriteMs = 0;
|
|
121
|
+
#stoppingHeartbeatRefreshActive = false;
|
|
122
|
+
#liveEventQueue = [];
|
|
123
|
+
#liveEventDrainScheduled = false;
|
|
124
|
+
#liveEventDrainActive = false;
|
|
125
|
+
// Holds back the live-event drain while the startup full pull runs, so events
|
|
126
|
+
// that arrive during the pull buffer and drain afterward (batch dedupe then
|
|
127
|
+
// suppresses any overlap with what the pull already dispatched).
|
|
128
|
+
#deferLiveEventDrain = false;
|
|
129
|
+
#completionSweepTimer;
|
|
130
|
+
#completionSweepActive = false;
|
|
131
|
+
#completionInFlight = new Set();
|
|
132
|
+
// Issue keys for which a babysitter has already been spawned, so repeated PR
|
|
133
|
+
// webhooks / agent-exit safety nets don't respawn it.
|
|
134
|
+
#babysitterSpawned = new Set();
|
|
135
|
+
// Issue key -> the open PR the babysitter is shepherding, including the
|
|
136
|
+
// webhook-fed mount path so readiness can re-read PR meta without a gh call.
|
|
137
|
+
#babysitterPr = new Map();
|
|
138
|
+
#probePrGhBackoffUntilMs = new Map();
|
|
139
|
+
#probePrResolvedCache = new Map();
|
|
140
|
+
// GitHub issue mirror-id -> resolved Linear mirror path, so repeat ingestion
|
|
141
|
+
// cycles read the mirror directly instead of re-scanning all Linear issues.
|
|
142
|
+
#githubMirrorPathCache = new Map();
|
|
143
|
+
#seenLiveEvents = new Set();
|
|
144
|
+
#offAgentExit;
|
|
145
|
+
#offDeliveryFailed;
|
|
146
|
+
#offAgentMessage;
|
|
147
|
+
// undefined = readiness not yet probed; resolved lazily so the standalone
|
|
148
|
+
// runOnce() path (which skips #start) still ingests when the mount is present.
|
|
149
|
+
#githubIngestionEnabled;
|
|
150
|
+
#starting;
|
|
151
|
+
#started = false;
|
|
152
|
+
#stopping = false;
|
|
153
|
+
constructor(config, ports) {
|
|
154
|
+
this.#config = config;
|
|
155
|
+
this.#mount = ports.mount;
|
|
156
|
+
// Resolved role<->state mapping. The CLI injects a name-resolved, per-team
|
|
157
|
+
// resolution via ports; fall back to one built from explicit stateIds.
|
|
158
|
+
this.#states = ports.stateResolution ?? stateResolutionFromIds(config.stateIds);
|
|
159
|
+
installFactoryDraftPredicate(this.#mount, config);
|
|
160
|
+
this.#fleet = ports.fleet;
|
|
161
|
+
this.#triage = ports.triage ?? new TieredTriage(new HeuristicTriage());
|
|
162
|
+
this.#linear = ports.linear ?? MountLinearWriteback(ports.mount, {
|
|
163
|
+
safety: config.safety,
|
|
164
|
+
});
|
|
165
|
+
this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
|
|
166
|
+
void (ports.github ?? MountGithubRead(ports.mount));
|
|
167
|
+
this.#mergeGate = ports.mergeGate ?? new GithubMergeGate();
|
|
168
|
+
this.#probeCloser = ports.probeCloser ?? closeProbePr;
|
|
169
|
+
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
170
|
+
this.#probePrGhRunner = ports.probePrGhRunner ?? failClosedGhRunner;
|
|
171
|
+
this.#probePrResolver = ports.probePrResolver ?? ((issue) => this.#resolveIssuePr(issue));
|
|
172
|
+
this.#logger = ports.logger ?? console;
|
|
173
|
+
this.#clock = ports.clock ?? realClock;
|
|
174
|
+
this.#processIdentityReader = ports.processIdentityReader ?? readProcessIdentity;
|
|
175
|
+
this.#processFinder = ports.processFinder ?? ((agentName, opts) => findAgentProcessByName(agentName, {
|
|
176
|
+
readProcessIdentity: this.#processIdentityReader,
|
|
177
|
+
protectedPids: opts?.protectedPids,
|
|
178
|
+
}));
|
|
179
|
+
this.#kill = ports.kill ?? process.kill;
|
|
180
|
+
this.#readChildPids = ports.readChildPids;
|
|
181
|
+
this.#terminationGraceMs = ports.terminationGraceMs;
|
|
182
|
+
this.#workspaceId = config.workspaceId ?? 'default';
|
|
183
|
+
this.#state = ports.stateStore ?? new InMemoryStateStore({
|
|
184
|
+
batchSize: config.batchSize,
|
|
185
|
+
agentQuestionDedupeLimit: AGENT_QUESTION_DEDUPE_LIMIT,
|
|
186
|
+
});
|
|
187
|
+
this.#batchReady = this.#state.getBatch(this.#workspaceId).then((batch) => {
|
|
188
|
+
this.#batchView = batch;
|
|
189
|
+
return batch;
|
|
190
|
+
});
|
|
191
|
+
this.#wireFleetEvents();
|
|
192
|
+
}
|
|
193
|
+
async #batch() {
|
|
194
|
+
if (this.#batchView) {
|
|
195
|
+
return this.#batchView;
|
|
196
|
+
}
|
|
197
|
+
const batch = await this.#batchReady;
|
|
198
|
+
this.#batchView = batch;
|
|
199
|
+
return batch;
|
|
200
|
+
}
|
|
201
|
+
async start(opts = {}) {
|
|
202
|
+
if (this.#started) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
if (this.#starting) {
|
|
206
|
+
return this.#starting;
|
|
207
|
+
}
|
|
208
|
+
this.#starting = this.#start(opts);
|
|
209
|
+
try {
|
|
210
|
+
await this.#starting;
|
|
211
|
+
}
|
|
212
|
+
finally {
|
|
213
|
+
this.#starting = undefined;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async #start(opts) {
|
|
217
|
+
this.#stopping = false;
|
|
218
|
+
const ready = await this.#mount.ensureSubRoot(ISSUE_ROOT, { timeoutMs: 90_000 });
|
|
219
|
+
if (ready !== 'ready') {
|
|
220
|
+
this.#error(new Error(`${ISSUE_ROOT} sub-root is not mounted`));
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
await this.#ensureGithubIngestionReady();
|
|
224
|
+
this.#wireFleetEvents();
|
|
225
|
+
if (opts.mode === 'live') {
|
|
226
|
+
this.#started = true;
|
|
227
|
+
try {
|
|
228
|
+
await this.#startLiveSubscription(opts.liveSubscription);
|
|
229
|
+
this.#scheduleCompletionSweep(0);
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
this.#started = false;
|
|
234
|
+
await this.#stopLiveHeartbeat('stopping');
|
|
235
|
+
throw error;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
await this.#backfillReadyIssues();
|
|
239
|
+
this.#subscription = this.#mount.subscribe([`${ISSUE_ROOT}/**/*.json`, LIVE_GITHUB_ISSUE_GLOB], (event) => {
|
|
240
|
+
if (isGithubPullFilePath(event.resource.path)) {
|
|
241
|
+
void this.#handlePrChange(event.resource.path);
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
if (isGithubIssueFilePath(event.resource.path)) {
|
|
245
|
+
void this.#handleGithubIssueChange(event.resource.path, { dryRun: this.#config.dryRun });
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
void this.#handleChange(event.resource.path);
|
|
249
|
+
});
|
|
250
|
+
this.#started = true;
|
|
251
|
+
this.#scheduleCompletionSweep(0);
|
|
252
|
+
}
|
|
253
|
+
async stop() {
|
|
254
|
+
this.#started = false;
|
|
255
|
+
this.#stopping = true;
|
|
256
|
+
if (this.#completionSweepTimer)
|
|
257
|
+
clearTimeout(this.#completionSweepTimer);
|
|
258
|
+
this.#completionSweepTimer = undefined;
|
|
259
|
+
this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping');
|
|
260
|
+
try {
|
|
261
|
+
await this.#releaseInFlightAgents('factory-stopped');
|
|
262
|
+
if (this.#livePollTimer)
|
|
263
|
+
clearTimeout(this.#livePollTimer);
|
|
264
|
+
this.#livePollTimer = undefined;
|
|
265
|
+
this.#livePollInFlight = false;
|
|
266
|
+
this.#liveEventQueue.length = 0;
|
|
267
|
+
this.#completionInFlight.clear();
|
|
268
|
+
this.#babysitterSpawned.clear();
|
|
269
|
+
this.#babysitterPr.clear();
|
|
270
|
+
const subscription = this.#subscription;
|
|
271
|
+
this.#subscription = undefined;
|
|
272
|
+
await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
|
|
273
|
+
await Promise.all([...this.#slackWatchers.values()].map((watcher) => watcher.stop()));
|
|
274
|
+
this.#slackWatchers.clear();
|
|
275
|
+
await this.#state.clearSlackThreads(this.#workspaceId);
|
|
276
|
+
this.#slackWatcherStarts.clear();
|
|
277
|
+
this.#offAgentExit?.();
|
|
278
|
+
this.#offDeliveryFailed?.();
|
|
279
|
+
this.#offAgentMessage?.();
|
|
280
|
+
this.#offAgentExit = undefined;
|
|
281
|
+
this.#offDeliveryFailed = undefined;
|
|
282
|
+
this.#offAgentMessage = undefined;
|
|
283
|
+
await this.#fleet.dispose();
|
|
284
|
+
}
|
|
285
|
+
finally {
|
|
286
|
+
this.#stoppingHeartbeatRefreshActive = false;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
async #boundedStopTeardown(label, teardown) {
|
|
290
|
+
let timer;
|
|
291
|
+
const action = Promise.resolve()
|
|
292
|
+
.then(teardown)
|
|
293
|
+
.then(() => ({ status: 'done' }), (error) => ({ status: 'error', error }));
|
|
294
|
+
const timeout = new Promise((resolve) => {
|
|
295
|
+
timer = setTimeout(() => resolve({ status: 'timeout' }), STOP_TEARDOWN_TIMEOUT_MS);
|
|
296
|
+
timer.unref?.();
|
|
297
|
+
});
|
|
298
|
+
const result = await Promise.race([action, timeout]);
|
|
299
|
+
if (timer) {
|
|
300
|
+
clearTimeout(timer);
|
|
301
|
+
}
|
|
302
|
+
if (result.status === 'timeout') {
|
|
303
|
+
this.#logger.warn?.(`[factory] ${label} timed out after ${STOP_TEARDOWN_TIMEOUT_MS}ms; continuing shutdown and allowing the server-side subscription to expire`, {
|
|
304
|
+
timeoutMs: STOP_TEARDOWN_TIMEOUT_MS,
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
else if (result.status === 'error') {
|
|
308
|
+
this.#logger.warn?.(`[factory] failed while stopping ${label}; continuing shutdown`, result.error);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
async dispose() {
|
|
312
|
+
await this.stop();
|
|
313
|
+
}
|
|
314
|
+
async #startLiveSubscription(overrides = {}) {
|
|
315
|
+
const options = this.#liveOptions(overrides);
|
|
316
|
+
await this.#startLiveHeartbeat();
|
|
317
|
+
this.#liveConnectStartedAtMs = this.#clock.now();
|
|
318
|
+
this.#liveReplaySkewMarginMs = options.replaySkewMarginMs;
|
|
319
|
+
const highWatermark = await this.#currentEventHighWatermark();
|
|
320
|
+
this.#liveEventHighWatermark = highWatermark.highWatermark;
|
|
321
|
+
this.#seenLiveEvents.clear();
|
|
322
|
+
this.#logger.info?.('[factory] live subscription starting', {
|
|
323
|
+
transport: options.transport,
|
|
324
|
+
highWatermark: this.#liveEventHighWatermark,
|
|
325
|
+
replaySkewMarginMs: this.#liveReplaySkewMarginMs,
|
|
326
|
+
highWatermarkRouteUnavailable: highWatermark.routeUnavailable,
|
|
327
|
+
});
|
|
328
|
+
// Register the live subscription BEFORE the startup full pull so an issue
|
|
329
|
+
// that becomes Ready *during* the pull is captured, not lost in the window
|
|
330
|
+
// between listTree and subscribe. Events buffer (deferred drain) until the
|
|
331
|
+
// pull finishes; batch dedupe then suppresses any overlap with what the
|
|
332
|
+
// pull already dispatched.
|
|
333
|
+
if (options.transport !== 'poll') {
|
|
334
|
+
// LIVE_GITHUB_ISSUE_GLOB is a terminal `${GITHUB_ISSUE_ROOT}/**`, so it
|
|
335
|
+
// already covers the PR change events the babysitter consumes; pull-event
|
|
336
|
+
// *processing* is gated on babysitter.enabled in #prepareLiveEventForDrain.
|
|
337
|
+
this.#subscription = this.#mount.subscribe([LIVE_ISSUE_GLOB, LIVE_GITHUB_ISSUE_GLOB], (event) => {
|
|
338
|
+
this.#enqueueLiveEvent(event);
|
|
339
|
+
}, { from: 'now', coalesce: 'none' });
|
|
340
|
+
}
|
|
341
|
+
if (highWatermark.routeUnavailable) {
|
|
342
|
+
this.#increment('liveHighWatermarkFullPullFallbacks');
|
|
343
|
+
this.#logger.info?.('[factory] live subscription high-watermark route unavailable; running startup full pull before draining buffered events');
|
|
344
|
+
this.#deferLiveEventDrain = true;
|
|
345
|
+
try {
|
|
346
|
+
await this.runOnce();
|
|
347
|
+
}
|
|
348
|
+
catch (error) {
|
|
349
|
+
// A startup pull failure must not abort the daemon: log it and fall back
|
|
350
|
+
// to the live event stream (plus any buffered events) instead of leaving
|
|
351
|
+
// the factory down.
|
|
352
|
+
this.#increment('liveHighWatermarkFullPullErrors');
|
|
353
|
+
this.#error(error);
|
|
354
|
+
}
|
|
355
|
+
finally {
|
|
356
|
+
this.#deferLiveEventDrain = false;
|
|
357
|
+
this.#scheduleLiveEventDrain();
|
|
358
|
+
}
|
|
359
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
360
|
+
}
|
|
361
|
+
if (options.transport === 'poll') {
|
|
362
|
+
this.#liveEventCursor = await this.#currentEventCursor(options.eventLimit);
|
|
363
|
+
this.#scheduleLivePoll(0, options);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
async #startLiveHeartbeat() {
|
|
367
|
+
this.#liveHeartbeatActive = true;
|
|
368
|
+
await this.#writeLiveHeartbeat('running');
|
|
369
|
+
this.#scheduleLiveHeartbeatRefresh();
|
|
370
|
+
}
|
|
371
|
+
async #stopLiveHeartbeat(status) {
|
|
372
|
+
if (!this.#liveHeartbeatActive && !this.#liveHeartbeatTimer) {
|
|
373
|
+
return false;
|
|
374
|
+
}
|
|
375
|
+
this.#liveHeartbeatActive = false;
|
|
376
|
+
if (this.#liveHeartbeatTimer) {
|
|
377
|
+
clearTimeout(this.#liveHeartbeatTimer);
|
|
378
|
+
this.#liveHeartbeatTimer = undefined;
|
|
379
|
+
}
|
|
380
|
+
await this.#liveHeartbeatRefresh;
|
|
381
|
+
await this.#writeLiveHeartbeat(status);
|
|
382
|
+
return status === 'stopping';
|
|
383
|
+
}
|
|
384
|
+
async #refreshStoppingHeartbeat() {
|
|
385
|
+
if (!this.#stoppingHeartbeatRefreshActive) {
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
try {
|
|
389
|
+
await this.#writeLiveHeartbeat('stopping');
|
|
390
|
+
}
|
|
391
|
+
catch (error) {
|
|
392
|
+
this.#logger.warn?.('[factory] failed to refresh stopping heartbeat', error);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
#scheduleLiveHeartbeatRefresh() {
|
|
396
|
+
if (!this.#liveHeartbeatActive || this.#liveHeartbeatTimer) {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
// This heartbeat proves daemon process liveness for the external crash reaper.
|
|
400
|
+
// MountClient subscriptions do not expose connected/keepalive state here, so
|
|
401
|
+
// subscription-wedge detection remains a separate watchdog concern.
|
|
402
|
+
this.#liveHeartbeatTimer = setTimeout(() => {
|
|
403
|
+
this.#liveHeartbeatTimer = undefined;
|
|
404
|
+
this.#liveHeartbeatRefresh = this.#refreshLiveHeartbeat()
|
|
405
|
+
.finally(() => {
|
|
406
|
+
this.#liveHeartbeatRefresh = undefined;
|
|
407
|
+
});
|
|
408
|
+
}, liveHeartbeatIntervalMs(this.#config.loop.heartbeatStaleMs));
|
|
409
|
+
this.#liveHeartbeatTimer.unref?.();
|
|
410
|
+
}
|
|
411
|
+
async #refreshLiveHeartbeat() {
|
|
412
|
+
if (!this.#liveHeartbeatActive || this.#liveHeartbeatInFlight) {
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
this.#liveHeartbeatInFlight = true;
|
|
416
|
+
try {
|
|
417
|
+
await this.#writeLiveHeartbeat('running');
|
|
418
|
+
}
|
|
419
|
+
catch (error) {
|
|
420
|
+
this.#logger.warn?.('[factory] failed to refresh live daemon heartbeat', error);
|
|
421
|
+
}
|
|
422
|
+
finally {
|
|
423
|
+
this.#liveHeartbeatInFlight = false;
|
|
424
|
+
this.#scheduleLiveHeartbeatRefresh();
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
async #writeLiveHeartbeat(status) {
|
|
428
|
+
await this.#writeLoopHeartbeat(this.#config.loop.heartbeatPath, this.#config.loop.registryPath, status, 0, 0);
|
|
429
|
+
this.#liveHeartbeatLastWriteMs = this.#clock.now();
|
|
430
|
+
}
|
|
431
|
+
#liveOptions(overrides) {
|
|
432
|
+
return {
|
|
433
|
+
transport: overrides.transport ?? this.#config.liveSubscription.transport,
|
|
434
|
+
pollIntervalMs: overrides.pollIntervalMs ?? this.#config.liveSubscription.pollIntervalMs,
|
|
435
|
+
eventLimit: overrides.eventLimit ?? this.#config.liveSubscription.eventLimit,
|
|
436
|
+
replaySkewMarginMs: overrides.replaySkewMarginMs ?? this.#config.liveSubscription.replaySkewMarginMs,
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
async #currentEventCursor(limit) {
|
|
440
|
+
let cursor;
|
|
441
|
+
for (;;) {
|
|
442
|
+
const page = await this.#mount.getEvents({ cursor, limit });
|
|
443
|
+
cursor = eventCursorAfterPage(cursor, page.events, page.nextCursor);
|
|
444
|
+
if (!page.nextCursor)
|
|
445
|
+
return cursor;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
async #currentEventHighWatermark() {
|
|
449
|
+
try {
|
|
450
|
+
return {
|
|
451
|
+
highWatermark: await this.#mount.getEventHighWatermark?.(),
|
|
452
|
+
routeUnavailable: false,
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
catch (error) {
|
|
456
|
+
this.#increment('liveHighWatermarkUnavailable');
|
|
457
|
+
this.#logger.warn?.('[factory] live subscription high-watermark unavailable', error);
|
|
458
|
+
return {
|
|
459
|
+
routeUnavailable: isHighWatermarkRouteUnavailable(error),
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
#scheduleLivePoll(delayMs, options) {
|
|
464
|
+
if (this.#livePollTimer || !this.#started)
|
|
465
|
+
return;
|
|
466
|
+
this.#livePollTimer = setTimeout(() => {
|
|
467
|
+
this.#livePollTimer = undefined;
|
|
468
|
+
void this.#pollLiveEvents(options).finally(() => {
|
|
469
|
+
if (this.#started)
|
|
470
|
+
this.#scheduleLivePoll(options.pollIntervalMs, options);
|
|
471
|
+
});
|
|
472
|
+
}, delayMs);
|
|
473
|
+
}
|
|
474
|
+
async #pollLiveEvents(options) {
|
|
475
|
+
if (this.#livePollInFlight)
|
|
476
|
+
return;
|
|
477
|
+
this.#livePollInFlight = true;
|
|
478
|
+
try {
|
|
479
|
+
let cursor = this.#liveEventCursor;
|
|
480
|
+
for (;;) {
|
|
481
|
+
const page = await this.#mount.getEvents({ cursor, limit: options.eventLimit });
|
|
482
|
+
await this.#handleLiveEventsWithYield(page.events);
|
|
483
|
+
const nextCursor = eventCursorAfterPage(cursor, page.events, page.nextCursor);
|
|
484
|
+
this.#liveEventCursor = nextCursor;
|
|
485
|
+
if (!page.nextCursor || page.nextCursor === cursor)
|
|
486
|
+
break;
|
|
487
|
+
cursor = page.nextCursor;
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
catch (error) {
|
|
491
|
+
this.#logger.warn?.('[factory] live subscription poll failed', error);
|
|
492
|
+
}
|
|
493
|
+
finally {
|
|
494
|
+
this.#livePollInFlight = false;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
#enqueueLiveEvent(event) {
|
|
498
|
+
if (!this.#started)
|
|
499
|
+
return;
|
|
500
|
+
this.#liveEventQueue.push(event);
|
|
501
|
+
this.#scheduleLiveEventDrain();
|
|
502
|
+
}
|
|
503
|
+
#scheduleLiveEventDrain() {
|
|
504
|
+
if (this.#liveEventDrainScheduled || this.#liveEventDrainActive || this.#deferLiveEventDrain || !this.#started) {
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
this.#liveEventDrainScheduled = true;
|
|
508
|
+
void liveEventYield()
|
|
509
|
+
.then(() => {
|
|
510
|
+
this.#liveEventDrainScheduled = false;
|
|
511
|
+
return this.#drainLiveEventQueue();
|
|
512
|
+
})
|
|
513
|
+
.catch((error) => {
|
|
514
|
+
this.#liveEventDrainScheduled = false;
|
|
515
|
+
this.#logger.warn?.('[factory] live issue event drain failed', error);
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
async #drainLiveEventQueue() {
|
|
519
|
+
if (this.#liveEventDrainActive) {
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
this.#liveEventDrainActive = true;
|
|
523
|
+
try {
|
|
524
|
+
await this.#handleLiveEventsWithYield(this.#liveEventQueue);
|
|
525
|
+
}
|
|
526
|
+
finally {
|
|
527
|
+
this.#liveEventDrainActive = false;
|
|
528
|
+
if (this.#liveEventQueue.length > 0 && this.#started) {
|
|
529
|
+
this.#scheduleLiveEventDrain();
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
async #handleLiveEventsWithYield(events) {
|
|
534
|
+
let handled = 0;
|
|
535
|
+
const seenIssueKeys = new Set();
|
|
536
|
+
while (events.length > 0 && this.#started) {
|
|
537
|
+
const batch = events.splice(0, LIVE_EVENT_DRAIN_BATCH_SIZE);
|
|
538
|
+
const paths = [];
|
|
539
|
+
for (const event of batch) {
|
|
540
|
+
const path = await this.#prepareLiveEventForDrain(event, seenIssueKeys);
|
|
541
|
+
if (path) {
|
|
542
|
+
paths.push(path);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
await Promise.all(paths.map((path) => this.#handlePreparedLiveChange(path)));
|
|
546
|
+
handled += batch.length;
|
|
547
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
548
|
+
if (events.length > 0) {
|
|
549
|
+
this.#increment('liveEventDrainYields');
|
|
550
|
+
await liveEventYield();
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
if (handled === 0) {
|
|
554
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
async #refreshLiveHeartbeatIfDue() {
|
|
558
|
+
if (!this.#liveHeartbeatActive)
|
|
559
|
+
return;
|
|
560
|
+
const intervalMs = liveHeartbeatIntervalMs(this.#config.loop.heartbeatStaleMs);
|
|
561
|
+
if (this.#clock.now() - this.#liveHeartbeatLastWriteMs < intervalMs)
|
|
562
|
+
return;
|
|
563
|
+
// A scheduled (real-timer) refresh may already be in flight carrying a
|
|
564
|
+
// timestamp captured before this drain batch advanced the clock. The
|
|
565
|
+
// #liveHeartbeatInFlight guard would otherwise make this due refresh a
|
|
566
|
+
// no-op, leaving the heartbeat stamped stale until the next interval. Wait
|
|
567
|
+
// for that write to settle, then re-check and force a current-timestamped
|
|
568
|
+
// write if we are still due so the heartbeat never falls behind the drain.
|
|
569
|
+
await this.#liveHeartbeatRefresh;
|
|
570
|
+
if (this.#clock.now() - this.#liveHeartbeatLastWriteMs < intervalMs)
|
|
571
|
+
return;
|
|
572
|
+
await this.#refreshLiveHeartbeat();
|
|
573
|
+
}
|
|
574
|
+
async #prepareLiveEventForDrain(event, seenIssueKeys) {
|
|
575
|
+
const path = event.resource.path;
|
|
576
|
+
const isPullPath = isGithubPullFilePath(path);
|
|
577
|
+
if (!isIssueFilePath(path) && !isGithubIssueFilePath(path) && !isPullPath) {
|
|
578
|
+
return undefined;
|
|
579
|
+
}
|
|
580
|
+
if (isBeforeLiveCutoff(event.occurredAt, this.#liveConnectStartedAtMs, this.#liveReplaySkewMarginMs)) {
|
|
581
|
+
this.#increment('liveReplayEventsSuppressed');
|
|
582
|
+
this.#increment('liveReplayEventsSuppressedByTime');
|
|
583
|
+
this.#logger.debug?.('[factory] suppressed stale live issue event', {
|
|
584
|
+
id: event.id,
|
|
585
|
+
path,
|
|
586
|
+
occurredAt: event.occurredAt,
|
|
587
|
+
connectStartedAt: new Date(this.#liveConnectStartedAtMs).toISOString(),
|
|
588
|
+
replaySkewMarginMs: this.#liveReplaySkewMarginMs,
|
|
589
|
+
});
|
|
590
|
+
return undefined;
|
|
591
|
+
}
|
|
592
|
+
if (isAtOrBeforeHighWatermark(event.id, this.#liveEventHighWatermark)) {
|
|
593
|
+
this.#increment('liveReplayEventsSuppressed');
|
|
594
|
+
this.#increment('liveReplayEventsSuppressedByWatermark');
|
|
595
|
+
this.#logger.debug?.('[factory] suppressed replayed live issue event', {
|
|
596
|
+
id: event.id,
|
|
597
|
+
highWatermark: this.#liveEventHighWatermark,
|
|
598
|
+
path,
|
|
599
|
+
});
|
|
600
|
+
return undefined;
|
|
601
|
+
}
|
|
602
|
+
const dedupeKey = liveEventDedupeKey(event);
|
|
603
|
+
if (dedupeKey) {
|
|
604
|
+
if (this.#seenLiveEvents.has(dedupeKey)) {
|
|
605
|
+
this.#increment('liveDuplicateEventsSuppressed');
|
|
606
|
+
this.#logger.debug?.('[factory] suppressed duplicate live issue event', {
|
|
607
|
+
id: event.id,
|
|
608
|
+
path,
|
|
609
|
+
});
|
|
610
|
+
return undefined;
|
|
611
|
+
}
|
|
612
|
+
rememberLiveEvent(this.#seenLiveEvents, dedupeKey);
|
|
613
|
+
}
|
|
614
|
+
else {
|
|
615
|
+
this.#increment('liveEventsMissingIdentity');
|
|
616
|
+
this.#logger.warn?.('[factory] live issue event missing stable identity', { path });
|
|
617
|
+
}
|
|
618
|
+
if (isGithubIssueFilePath(path)) {
|
|
619
|
+
const sourceKey = `github:${path}`;
|
|
620
|
+
if (seenIssueKeys.has(sourceKey)) {
|
|
621
|
+
this.#increment('liveDuplicateIssueEventsSuppressed');
|
|
622
|
+
this.#logger.debug?.('[factory] suppressed duplicate live GitHub issue event in current drain', {
|
|
623
|
+
id: event.id,
|
|
624
|
+
path,
|
|
625
|
+
});
|
|
626
|
+
return undefined;
|
|
627
|
+
}
|
|
628
|
+
seenIssueKeys.add(sourceKey);
|
|
629
|
+
this.#recordArrivalLatency(event);
|
|
630
|
+
return path;
|
|
631
|
+
}
|
|
632
|
+
if (isPullPath) {
|
|
633
|
+
// Dedupe PR change events by path within a drain; the babysitter routing
|
|
634
|
+
// re-derives the issue from the PR head ref downstream.
|
|
635
|
+
const sourceKey = `pull:${path}`;
|
|
636
|
+
if (seenIssueKeys.has(sourceKey)) {
|
|
637
|
+
this.#increment('liveDuplicatePrEventsSuppressed');
|
|
638
|
+
return undefined;
|
|
639
|
+
}
|
|
640
|
+
seenIssueKeys.add(sourceKey);
|
|
641
|
+
this.#recordArrivalLatency(event);
|
|
642
|
+
return path;
|
|
643
|
+
}
|
|
644
|
+
const issueKey = keyFromPath(path);
|
|
645
|
+
if (seenIssueKeys.has(issueKey)) {
|
|
646
|
+
this.#increment('liveDuplicateIssueEventsSuppressed');
|
|
647
|
+
this.#logger.debug?.('[factory] suppressed duplicate live issue event for issue in current drain', {
|
|
648
|
+
id: event.id,
|
|
649
|
+
path,
|
|
650
|
+
issue: issueKey,
|
|
651
|
+
});
|
|
652
|
+
return undefined;
|
|
653
|
+
}
|
|
654
|
+
const issueRef = { key: issueKey, uuid: uuidFromPath(path) ?? issueKey, path };
|
|
655
|
+
const batch = await this.#batch();
|
|
656
|
+
if (batch.isInFlight(issueRef) || batch.isQueued(issueRef)) {
|
|
657
|
+
this.#increment('liveDuplicateIssueEventsSuppressed');
|
|
658
|
+
this.#logger.debug?.('[factory] suppressed duplicate live issue event for tracked issue', {
|
|
659
|
+
id: event.id,
|
|
660
|
+
path,
|
|
661
|
+
issue: issueKey,
|
|
662
|
+
});
|
|
663
|
+
return undefined;
|
|
664
|
+
}
|
|
665
|
+
seenIssueKeys.add(issueKey);
|
|
666
|
+
this.#recordArrivalLatency(event);
|
|
667
|
+
return path;
|
|
668
|
+
}
|
|
669
|
+
async #handlePreparedLiveChange(path) {
|
|
670
|
+
if (isGithubPullFilePath(path)) {
|
|
671
|
+
await this.#handlePrChange(path);
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
674
|
+
if (isGithubIssueFilePath(path)) {
|
|
675
|
+
await this.#handleGithubIssueChange(path, { dryRun: this.#config.dryRun });
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
await this.#handleChange(path, { requireRealIssue: true });
|
|
679
|
+
}
|
|
680
|
+
#scheduleCompletionSweep(delayMs = COMPLETION_SWEEP_INTERVAL_MS) {
|
|
681
|
+
if (!this.#started || this.#completionSweepTimer || this.#completionSweepActive) {
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
this.#completionSweepTimer = setTimeout(() => {
|
|
685
|
+
this.#completionSweepTimer = undefined;
|
|
686
|
+
void this.#sweepPrStateCompletions('live-timer')
|
|
687
|
+
.catch((error) => {
|
|
688
|
+
this.#increment('completionSweepErrors');
|
|
689
|
+
this.#logger.warn?.('[factory] PR completion sweep failed', error);
|
|
690
|
+
})
|
|
691
|
+
.finally(() => {
|
|
692
|
+
if (this.#started)
|
|
693
|
+
this.#scheduleCompletionSweep();
|
|
694
|
+
});
|
|
695
|
+
}, delayMs);
|
|
696
|
+
this.#completionSweepTimer.unref?.();
|
|
697
|
+
}
|
|
698
|
+
async #sweepPrStateCompletions(reason) {
|
|
699
|
+
// When the babysitter owns PR-open, completion is driven by PR webhooks +
|
|
700
|
+
// the babysitter's readiness signal (see #handlePrChange / #handleAgentExit),
|
|
701
|
+
// not this polling sweep. Disabling it here is what makes the babysitter path
|
|
702
|
+
// webhook-driven rather than polled.
|
|
703
|
+
if (this.#config.babysitter.enabled) {
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
if (this.#completionSweepActive) {
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
this.#completionSweepActive = true;
|
|
710
|
+
try {
|
|
711
|
+
const batch = await this.#batch();
|
|
712
|
+
const records = batch.inFlight
|
|
713
|
+
.filter((record) => !record.dryRun && !this.#completionInFlight.has(issueKey(record.issue)));
|
|
714
|
+
if (records.length === 0) {
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
this.#increment('completionSweepRuns');
|
|
718
|
+
for (let index = 0; index < records.length; index += COMPLETION_SWEEP_BATCH_SIZE) {
|
|
719
|
+
const candidates = await Promise.all(records.slice(index, index + COMPLETION_SWEEP_BATCH_SIZE).map(async (record) => {
|
|
720
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
721
|
+
if (!issue || !isInFactoryScope(issue, this.#config.safety)) {
|
|
722
|
+
return undefined;
|
|
723
|
+
}
|
|
724
|
+
const pr = await this.#completionPrForIssue(issue);
|
|
725
|
+
if (!pr) {
|
|
726
|
+
this.#increment('completionSweepMissingPr');
|
|
727
|
+
return undefined;
|
|
728
|
+
}
|
|
729
|
+
if (pr.draft) {
|
|
730
|
+
this.#increment('completionSweepDraftPr');
|
|
731
|
+
this.#probePrGhBackoffUntilMs.set(issue.key, this.#clock.now() + PROBE_PR_GH_BACKOFF_MS);
|
|
732
|
+
return undefined;
|
|
733
|
+
}
|
|
734
|
+
return { record, pr };
|
|
735
|
+
}));
|
|
736
|
+
for (const candidate of candidates) {
|
|
737
|
+
if (!candidate || batch.getIssue(candidate.record.issue) !== candidate.record) {
|
|
738
|
+
continue;
|
|
739
|
+
}
|
|
740
|
+
this.#increment('completionSweepCompleted');
|
|
741
|
+
this.#logger.info?.('[factory] PR completion sweep completing issue', {
|
|
742
|
+
issue: candidate.record.issue.key,
|
|
743
|
+
repo: candidate.pr.repo,
|
|
744
|
+
prNumber: candidate.pr.prNumber,
|
|
745
|
+
reason,
|
|
746
|
+
});
|
|
747
|
+
// workaround for relay#1116: agents often exit as worker_exited after opening a PR,
|
|
748
|
+
// so PR state is the primary completion signal that frees the batch slot.
|
|
749
|
+
await this.#completeIssue(candidate.record);
|
|
750
|
+
}
|
|
751
|
+
await this.#refreshLiveHeartbeatIfDue();
|
|
752
|
+
if (index + COMPLETION_SWEEP_BATCH_SIZE < records.length) {
|
|
753
|
+
await liveEventYield();
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
finally {
|
|
758
|
+
this.#completionSweepActive = false;
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
async #completionPrForIssue(issue) {
|
|
762
|
+
if (this.#customProbePrResolver) {
|
|
763
|
+
return this.#probePrResolver(issue);
|
|
764
|
+
}
|
|
765
|
+
return this.#resolveIssuePr(issue, {
|
|
766
|
+
titleMarker: FACTORY_E2E_MARKER,
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
async #resolveIssuePr(issue, opts = {}) {
|
|
770
|
+
const key = issue.key;
|
|
771
|
+
const now = this.#clock.now();
|
|
772
|
+
const cached = this.#probePrResolvedCache.get(key);
|
|
773
|
+
if (cached && cached.expiresAtMs > now) {
|
|
774
|
+
return cached.pr;
|
|
775
|
+
}
|
|
776
|
+
const mountPr = await resolveIssuePrFromMount(this.#mount, this.#config, issue, opts);
|
|
777
|
+
if (mountPr) {
|
|
778
|
+
return mountPr;
|
|
779
|
+
}
|
|
780
|
+
const backoffUntil = this.#probePrGhBackoffUntilMs.get(key) ?? 0;
|
|
781
|
+
if (backoffUntil > now) {
|
|
782
|
+
this.#increment('probePrGhBackoffSkips');
|
|
783
|
+
return undefined;
|
|
784
|
+
}
|
|
785
|
+
const ghPr = await resolveIssuePrFromGh(this.#probePrGhRunner, this.#config, issue, opts, this.#logger);
|
|
786
|
+
this.#increment('probePrGhResolveAttempts');
|
|
787
|
+
if (ghPr) {
|
|
788
|
+
this.#probePrGhBackoffUntilMs.delete(key);
|
|
789
|
+
if (!ghPr.draft) {
|
|
790
|
+
this.#probePrResolvedCache.set(key, { pr: ghPr, expiresAtMs: now + PROBE_PR_GH_BACKOFF_MS });
|
|
791
|
+
}
|
|
792
|
+
this.#increment('probePrGhResolveHits');
|
|
793
|
+
return ghPr;
|
|
794
|
+
}
|
|
795
|
+
this.#probePrGhBackoffUntilMs.set(key, now + PROBE_PR_GH_BACKOFF_MS);
|
|
796
|
+
return undefined;
|
|
797
|
+
}
|
|
798
|
+
async runOnce(opts = {}) {
|
|
799
|
+
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
800
|
+
await this.#ingestGithubIssues({ dryRun });
|
|
801
|
+
const paths = await this.#readyIssuePaths();
|
|
802
|
+
const pulled = [];
|
|
803
|
+
const triaged = [];
|
|
804
|
+
const dispatched = [];
|
|
805
|
+
const skipped = [];
|
|
806
|
+
for (const path of paths) {
|
|
807
|
+
const issue = await this.#readIssue(path);
|
|
808
|
+
if (issue) {
|
|
809
|
+
await this.#recordCanonicalIssueState(issue);
|
|
810
|
+
}
|
|
811
|
+
if (!issue) {
|
|
812
|
+
continue;
|
|
813
|
+
}
|
|
814
|
+
pulled.push(issueRef(issue));
|
|
815
|
+
const dispatchBlock = await this.#dispatchBlockReason(issue);
|
|
816
|
+
if (dispatchBlock) {
|
|
817
|
+
skipped.push({ issue: issueRef(issue), reason: dispatchBlock });
|
|
818
|
+
continue;
|
|
819
|
+
}
|
|
820
|
+
const batch = await this.#batch();
|
|
821
|
+
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
822
|
+
skipped.push({ issue: issueRef(issue), reason: 'already tracked' });
|
|
823
|
+
continue;
|
|
824
|
+
}
|
|
825
|
+
if (!this.#states.isRole(issue.stateId, 'readyForAgent')) {
|
|
826
|
+
skipped.push({ issue: issueRef(issue), reason: 'live state is not ready-for-agent' });
|
|
827
|
+
continue;
|
|
828
|
+
}
|
|
829
|
+
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
830
|
+
skipped.push({ issue: issueRef(issue), reason: 'not factory-e2e scope' });
|
|
831
|
+
continue;
|
|
832
|
+
}
|
|
833
|
+
if (!isRealLinearIssue(issue)) {
|
|
834
|
+
skipped.push({ issue: issueRef(issue), reason: 'not reconciled real Linear issue' });
|
|
835
|
+
continue;
|
|
836
|
+
}
|
|
837
|
+
const decision = await this.triageIssue(issue);
|
|
838
|
+
triaged.push(decision);
|
|
839
|
+
const result = await this.dispatch(decision, { dryRun });
|
|
840
|
+
if (result.agents.length === 0 && !dryRun) {
|
|
841
|
+
skipped.push({ issue: decision.issue, reason: 'queued or escalated' });
|
|
842
|
+
}
|
|
843
|
+
else {
|
|
844
|
+
dispatched.push(result);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return { pulled, triaged, dispatched, skipped, dryRun, slackDegraded: this.#slackDegraded };
|
|
848
|
+
}
|
|
849
|
+
async runLoop(opts = {}) {
|
|
850
|
+
const maxIterations = Math.min(5, Math.max(1, Math.trunc(opts.maxIterations ?? this.#config.loop.maxIterations)));
|
|
851
|
+
const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
|
|
852
|
+
const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
|
|
853
|
+
const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
|
|
854
|
+
const reports = [];
|
|
855
|
+
let consecutiveFailures = 0;
|
|
856
|
+
let completed = false;
|
|
857
|
+
try {
|
|
858
|
+
for (let iteration = 0; iteration < maxIterations; iteration += 1) {
|
|
859
|
+
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'running', iteration, maxIterations);
|
|
860
|
+
try {
|
|
861
|
+
await this.#sweepPrStateCompletions('run-loop');
|
|
862
|
+
reports.push(await this.runOnce({ dryRun: opts.dryRun }));
|
|
863
|
+
consecutiveFailures = 0;
|
|
864
|
+
}
|
|
865
|
+
catch (error) {
|
|
866
|
+
consecutiveFailures += 1;
|
|
867
|
+
this.#increment('loopIterationFailures');
|
|
868
|
+
this.#error(error);
|
|
869
|
+
reports.push(failedIterationReport(error, opts.dryRun ?? this.#config.dryRun));
|
|
870
|
+
await this.#reapDispatchFailureHandoffsNow(heartbeatPath, registryPath);
|
|
871
|
+
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'running', iteration + 1, maxIterations);
|
|
872
|
+
if (consecutiveFailures >= maxConsecutiveFailures) {
|
|
873
|
+
this.#increment('loopCircuitBreaks');
|
|
874
|
+
this.#logger.error?.('[factory] stopping loop after consecutive iteration failures', {
|
|
875
|
+
consecutiveFailures,
|
|
876
|
+
maxConsecutiveFailures,
|
|
877
|
+
});
|
|
878
|
+
break;
|
|
879
|
+
}
|
|
880
|
+
continue;
|
|
881
|
+
}
|
|
882
|
+
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'running', iteration + 1, maxIterations);
|
|
883
|
+
}
|
|
884
|
+
this.#increment('loopIdle');
|
|
885
|
+
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'idle', reports.length, maxIterations);
|
|
886
|
+
completed = true;
|
|
887
|
+
return reports;
|
|
888
|
+
}
|
|
889
|
+
finally {
|
|
890
|
+
if (!completed) {
|
|
891
|
+
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'stopping', reports.length, maxIterations);
|
|
892
|
+
}
|
|
893
|
+
await this.stop();
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
async triageIssue(issue) {
|
|
897
|
+
return this.#triage.triage(issue, {
|
|
898
|
+
config: this.#config,
|
|
899
|
+
repoMap: repoMapFromConfig(this.#config),
|
|
900
|
+
});
|
|
901
|
+
}
|
|
902
|
+
async dispatch(decision, opts = {}) {
|
|
903
|
+
const dryRun = opts.dryRun ?? this.#config.dryRun;
|
|
904
|
+
const batch = await this.#batch();
|
|
905
|
+
const existingRecord = batch.getIssue(decision.issue);
|
|
906
|
+
if (existingRecord?.result) {
|
|
907
|
+
return existingRecord.result;
|
|
908
|
+
}
|
|
909
|
+
const blockReason = await this.#dispatchBlockReason(decision.issue);
|
|
910
|
+
if (blockReason) {
|
|
911
|
+
const error = new Error(`Refusing to dispatch ${decision.issue.key}: ${blockReason}`);
|
|
912
|
+
this.#error(error, decision.issue);
|
|
913
|
+
throw error;
|
|
914
|
+
}
|
|
915
|
+
const liveIssue = await this.#readIssue(decision.issue.path);
|
|
916
|
+
if (!liveIssue || !isInFactoryScope(liveIssue, this.#config.safety)) {
|
|
917
|
+
const error = new Error(`Refusing to dispatch ${decision.issue.key}: not factory-e2e scope`);
|
|
918
|
+
this.#error(error, decision.issue);
|
|
919
|
+
throw error;
|
|
920
|
+
}
|
|
921
|
+
if (!isRealLinearIssue(liveIssue)) {
|
|
922
|
+
const error = new Error(`Refusing to dispatch ${decision.issue.key}: not reconciled real Linear issue`);
|
|
923
|
+
this.#error(error, decision.issue);
|
|
924
|
+
throw error;
|
|
925
|
+
}
|
|
926
|
+
const escalationReason = triageEscalationReason(decision);
|
|
927
|
+
if (escalationReason) {
|
|
928
|
+
await this.#escalateTriageToSlack(decision, escalationReason, dryRun);
|
|
929
|
+
this.#error(new Error(`${escalationReason}; escalation required`), decision.issue);
|
|
930
|
+
return { issue: decision.issue, agents: [], dryRun };
|
|
931
|
+
}
|
|
932
|
+
// TODO(AR-274 follow-up): short-circuit LLM triage once label-derived
|
|
933
|
+
// routes are authoritative for dispatch identity.
|
|
934
|
+
const labelDispatch = labelDerivedDispatchDecision(liveIssue, decision, this.#config);
|
|
935
|
+
if (!labelDispatch.ok) {
|
|
936
|
+
const comment = labelDispatchFailureComment(decision.issue, labelDispatch);
|
|
937
|
+
this.#logger.warn?.('[factory] skipped dispatch due to invalid repo labels', {
|
|
938
|
+
issue: decision.issue.key,
|
|
939
|
+
labels: liveIssue.labels,
|
|
940
|
+
offendingLabels: labelDispatch.offendingLabels,
|
|
941
|
+
reason: labelDispatch.reason,
|
|
942
|
+
});
|
|
943
|
+
if (!dryRun) {
|
|
944
|
+
const signature = labelDispatchFailureSignature(labelDispatch);
|
|
945
|
+
if (this.#labelDispatchFailures.get(decision.issue.key) !== signature) {
|
|
946
|
+
try {
|
|
947
|
+
await this.#linear.postComment(liveIssue, comment);
|
|
948
|
+
// Record only after a successful post so a failed writeback retries
|
|
949
|
+
// next cycle rather than being suppressed as already-notified.
|
|
950
|
+
this.#labelDispatchFailures.set(decision.issue.key, signature);
|
|
951
|
+
}
|
|
952
|
+
catch (error) {
|
|
953
|
+
this.#logger.warn?.('[factory] label dispatch block comment writeback skipped', error);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
return { issue: decision.issue, agents: [], comments: [comment], dryRun };
|
|
958
|
+
}
|
|
959
|
+
const dispatchDecision = labelDispatch.decision;
|
|
960
|
+
// A valid label resolution clears any prior failure notice so a later
|
|
961
|
+
// regression posts a fresh, actionable comment instead of being deduped.
|
|
962
|
+
this.#labelDispatchFailures.delete(dispatchDecision.issue.key);
|
|
963
|
+
await this.#recordDispatchAttempt(dispatchDecision.issue);
|
|
964
|
+
const record = batch.start(dispatchDecision, dryRun);
|
|
965
|
+
if (!record) {
|
|
966
|
+
await this.#clearDispatchInFlight(dispatchDecision.issue);
|
|
967
|
+
this.#increment('queued');
|
|
968
|
+
this.#emit('issue-queued', { issue: dispatchDecision.issue });
|
|
969
|
+
return { issue: dispatchDecision.issue, agents: [], dryRun };
|
|
970
|
+
}
|
|
971
|
+
if (record.result) {
|
|
972
|
+
return record.result;
|
|
973
|
+
}
|
|
974
|
+
const spawnedForReaperHandoff = [];
|
|
975
|
+
try {
|
|
976
|
+
const agents = [];
|
|
977
|
+
for (const spec of [...dispatchDecision.implementers, dispatchDecision.reviewer]) {
|
|
978
|
+
const spawned = await this.#spawnAgent(record, spec, dryRun);
|
|
979
|
+
const tracked = record.agents.get(spawned.name);
|
|
980
|
+
if (tracked) {
|
|
981
|
+
spawnedForReaperHandoff.push({
|
|
982
|
+
issue: record.issue,
|
|
983
|
+
name: spawned.name,
|
|
984
|
+
tracked: cloneTrackedAgent(tracked),
|
|
985
|
+
persistedAtMs: this.#clock.now(),
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
agents.push({ name: spawned.name, role: spec.role });
|
|
989
|
+
}
|
|
990
|
+
await this.#writeInFlightRegistry();
|
|
991
|
+
const comment = dispatchComment(dispatchDecision, agents);
|
|
992
|
+
let implementingStateId;
|
|
993
|
+
if (!dryRun) {
|
|
994
|
+
const issue = await this.#readIssue(dispatchDecision.issue.path);
|
|
995
|
+
if (!issue || !this.#states.isRole(issue.stateId, 'readyForAgent')) {
|
|
996
|
+
throw new Error(`Live state changed before writeback for ${dispatchDecision.issue.key}`);
|
|
997
|
+
}
|
|
998
|
+
implementingStateId = this.#states.idFor(issue.team, 'agentImplementing');
|
|
999
|
+
try {
|
|
1000
|
+
await this.#linear.postComment(issue, comment);
|
|
1001
|
+
}
|
|
1002
|
+
catch (error) {
|
|
1003
|
+
this.#logger.warn?.('[factory] comment writeback skipped', error);
|
|
1004
|
+
}
|
|
1005
|
+
await this.#linear.setState(issue, implementingStateId);
|
|
1006
|
+
this.#emit('writeback-verified', { issue: dispatchDecision.issue, path: issue.path });
|
|
1007
|
+
}
|
|
1008
|
+
const result = {
|
|
1009
|
+
issue: dispatchDecision.issue,
|
|
1010
|
+
agents,
|
|
1011
|
+
comments: [comment],
|
|
1012
|
+
stateId: implementingStateId,
|
|
1013
|
+
dryRun,
|
|
1014
|
+
};
|
|
1015
|
+
record.result = result;
|
|
1016
|
+
this.#increment('dispatched');
|
|
1017
|
+
this.#emit('dispatched', { issue: dispatchDecision.issue, result });
|
|
1018
|
+
if (!dryRun) {
|
|
1019
|
+
await this.#ensureSlackDispatchThread(record, result);
|
|
1020
|
+
await this.#sendImplementerTask(record);
|
|
1021
|
+
await this.#sendCriticalReviewerMessage(record);
|
|
1022
|
+
}
|
|
1023
|
+
return result;
|
|
1024
|
+
}
|
|
1025
|
+
catch (error) {
|
|
1026
|
+
await this.#persistDispatchFailureReaperHandoff(record, spawnedForReaperHandoff);
|
|
1027
|
+
await this.#recordDispatchFailure(decision.issue);
|
|
1028
|
+
batch.abandon(decision.issue);
|
|
1029
|
+
this.#error(error, decision.issue);
|
|
1030
|
+
throw error;
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
status() {
|
|
1034
|
+
const batch = this.#batchView;
|
|
1035
|
+
return {
|
|
1036
|
+
inFlight: batch?.inFlight.map((record) => record.issue) ?? [],
|
|
1037
|
+
queued: batch?.queued.map((queued) => queued.issue) ?? [],
|
|
1038
|
+
counters: { ...this.#counters },
|
|
1039
|
+
slackDegraded: this.#slackDegraded,
|
|
1040
|
+
slackDegradedReason: this.#slackDegradedReason,
|
|
1041
|
+
};
|
|
1042
|
+
}
|
|
1043
|
+
on(event, listener) {
|
|
1044
|
+
let listeners = this.#listeners.get(event);
|
|
1045
|
+
if (!listeners) {
|
|
1046
|
+
listeners = new Set();
|
|
1047
|
+
this.#listeners.set(event, listeners);
|
|
1048
|
+
}
|
|
1049
|
+
listeners.add(listener);
|
|
1050
|
+
return () => {
|
|
1051
|
+
listeners?.delete(listener);
|
|
1052
|
+
};
|
|
1053
|
+
}
|
|
1054
|
+
#wireFleetEvents() {
|
|
1055
|
+
if (!this.#offAgentExit) {
|
|
1056
|
+
this.#offAgentExit = this.#fleet.onAgentExit((name, reason) => {
|
|
1057
|
+
void this.#handleAgentExit(name, reason);
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
1060
|
+
if (!this.#offDeliveryFailed) {
|
|
1061
|
+
this.#offDeliveryFailed = this.#fleet.onDeliveryFailed?.((info) => {
|
|
1062
|
+
void this.#handleDeliveryFailed(info);
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
if (!this.#offAgentMessage) {
|
|
1066
|
+
this.#offAgentMessage = this.#fleet.onAgentMessage?.((message) => {
|
|
1067
|
+
void this.#handleAgentMessage(message);
|
|
1068
|
+
});
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
async #backfillReadyIssues() {
|
|
1072
|
+
const page = await this.#mount.getEvents({ limit: READY_EVENTS_LIMIT });
|
|
1073
|
+
const eventPaths = page.events.map((event) => event.resource.path).filter(isIssueFilePath);
|
|
1074
|
+
const githubEventPaths = page.events.map((event) => event.resource.path).filter(isGithubIssueFilePath);
|
|
1075
|
+
for (const path of new Set(githubEventPaths)) {
|
|
1076
|
+
await this.#handleGithubIssueChange(path, { dryRun: this.#config.dryRun });
|
|
1077
|
+
}
|
|
1078
|
+
await this.#ingestGithubIssues({ dryRun: this.#config.dryRun });
|
|
1079
|
+
const treePaths = await this.#readyIssuePaths();
|
|
1080
|
+
for (const path of new Set([...eventPaths, ...treePaths])) {
|
|
1081
|
+
await this.#handleChange(path);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
async #handleChange(path, opts = {}) {
|
|
1085
|
+
if (!isIssueFilePath(path)) {
|
|
1086
|
+
return;
|
|
1087
|
+
}
|
|
1088
|
+
try {
|
|
1089
|
+
const issue = await this.#readIssue(path);
|
|
1090
|
+
if (issue) {
|
|
1091
|
+
await this.#recordCanonicalIssueState(issue);
|
|
1092
|
+
}
|
|
1093
|
+
if (!issue || !this.#states.isRole(issue.stateId, 'readyForAgent')) {
|
|
1094
|
+
return;
|
|
1095
|
+
}
|
|
1096
|
+
if (opts.requireRealIssue && !isRealLinearIssue(issue)) {
|
|
1097
|
+
return;
|
|
1098
|
+
}
|
|
1099
|
+
if (!isInFactoryScope(issue, this.#config.safety)) {
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
if (!isRealLinearIssue(issue)) {
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
const batch = await this.#batch();
|
|
1106
|
+
if (batch.isInFlight(issue) || batch.isQueued(issue)) {
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
if (await this.#dispatchBlockReason(issue)) {
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
const decision = await this.triageIssue(issue);
|
|
1113
|
+
const escalationReason = triageEscalationReason(decision);
|
|
1114
|
+
if (escalationReason) {
|
|
1115
|
+
await this.#escalateTriageToSlack(decision, escalationReason, this.#config.dryRun);
|
|
1116
|
+
this.#error(new Error(`${escalationReason}; escalation required`), decision.issue);
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
if (batch.canStart()) {
|
|
1120
|
+
await this.dispatch(decision, { dryRun: this.#config.dryRun });
|
|
1121
|
+
}
|
|
1122
|
+
else {
|
|
1123
|
+
if (batch.queue(decision, this.#config.dryRun)) {
|
|
1124
|
+
this.#emit('issue-queued', { issue: decision.issue });
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
catch (error) {
|
|
1129
|
+
this.#logger.error?.('[factory] failed to handle issue change', error);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
// Probes the GitHub issue sub-root at most once and caches the verdict so
|
|
1133
|
+
// repeated iterations skip listTree calls when the mount is absent.
|
|
1134
|
+
async #ensureGithubIngestionReady() {
|
|
1135
|
+
if (this.#githubIngestionEnabled !== undefined) {
|
|
1136
|
+
return this.#githubIngestionEnabled;
|
|
1137
|
+
}
|
|
1138
|
+
const githubReady = await this.#mount.ensureSubRoot(GITHUB_ISSUE_ROOT, { timeoutMs: 90_000 });
|
|
1139
|
+
this.#githubIngestionEnabled = githubReady === 'ready';
|
|
1140
|
+
if (!this.#githubIngestionEnabled) {
|
|
1141
|
+
this.#logger.warn?.(`[factory] ${GITHUB_ISSUE_ROOT} sub-root is not mounted; GitHub issue ingestion disabled`);
|
|
1142
|
+
}
|
|
1143
|
+
return this.#githubIngestionEnabled;
|
|
1144
|
+
}
|
|
1145
|
+
async #ingestGithubIssues(opts = {}) {
|
|
1146
|
+
if (!await this.#ensureGithubIngestionReady()) {
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
// Load the existing Linear mirror candidates once for the whole pass so
|
|
1150
|
+
// dedupe stays O(N + M) reads instead of re-scanning ISSUE_ROOT for every
|
|
1151
|
+
// GitHub issue (gemini perf finding on #findGithubIssueMirror).
|
|
1152
|
+
const candidates = this.#newMirrorCandidateCache();
|
|
1153
|
+
for (const path of await this.#githubIssuePaths()) {
|
|
1154
|
+
await this.#handleGithubIssueChange(path, { ...opts, candidates });
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
// Lazily lists+reads ISSUE_ROOT mirror candidates at most once, then memoizes
|
|
1158
|
+
// them for reuse across every GitHub issue handled in the same pass.
|
|
1159
|
+
#newMirrorCandidateCache() {
|
|
1160
|
+
let loaded;
|
|
1161
|
+
return {
|
|
1162
|
+
load: () => {
|
|
1163
|
+
if (!loaded) {
|
|
1164
|
+
loaded = this.#loadLinearMirrorCandidates();
|
|
1165
|
+
}
|
|
1166
|
+
return loaded;
|
|
1167
|
+
},
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
async #loadLinearMirrorCandidates() {
|
|
1171
|
+
const candidates = [];
|
|
1172
|
+
for (const path of await this.#mount.listTree(ISSUE_ROOT)) {
|
|
1173
|
+
if (!isLinearIssueMirrorCandidatePath(path)) {
|
|
1174
|
+
continue;
|
|
1175
|
+
}
|
|
1176
|
+
const issue = await this.#readIssue(path);
|
|
1177
|
+
if (issue) {
|
|
1178
|
+
candidates.push(issue);
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
return candidates;
|
|
1182
|
+
}
|
|
1183
|
+
async #githubIssuePaths() {
|
|
1184
|
+
try {
|
|
1185
|
+
const paths = await this.#mount.listTree(GITHUB_ISSUE_ROOT);
|
|
1186
|
+
const issuePaths = [];
|
|
1187
|
+
for (const path of paths) {
|
|
1188
|
+
if (githubIssuePathParts(path) !== undefined) {
|
|
1189
|
+
issuePaths.push(path);
|
|
1190
|
+
}
|
|
1191
|
+
else if (githubIssueDirectoryPathParts(path) !== undefined) {
|
|
1192
|
+
// listTree returns the issue directory entry alongside its
|
|
1193
|
+
// meta.json file; githubIssuePathParts() already collected the
|
|
1194
|
+
// file, so skip the directory to avoid reading the same issue
|
|
1195
|
+
// twice in one backfill pass. Directory paths are only meaningful
|
|
1196
|
+
// for live change events, not the tree scan.
|
|
1197
|
+
continue;
|
|
1198
|
+
}
|
|
1199
|
+
else if (isGithubIssueTreePath(path)) {
|
|
1200
|
+
this.#increment('githubIssuesIgnoredByPathRegex');
|
|
1201
|
+
this.#logger.debug?.('[factory] ignored GitHub issue path with unsupported relayfile shape', { path });
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
return issuePaths.sort();
|
|
1205
|
+
}
|
|
1206
|
+
catch (error) {
|
|
1207
|
+
this.#increment('githubIssueListFailures');
|
|
1208
|
+
this.#logger.warn?.('[factory] failed to list GitHub issue source tree', error);
|
|
1209
|
+
return [];
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
async #handleGithubIssueChange(path, opts = {}) {
|
|
1213
|
+
if (this.#githubIngestionEnabled === false || !isGithubIssueFilePath(path)) {
|
|
1214
|
+
return;
|
|
1215
|
+
}
|
|
1216
|
+
try {
|
|
1217
|
+
const ghIssue = await this.#readGithubIssue(path);
|
|
1218
|
+
if (!ghIssue) {
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
if (githubIssueIsClosed(ghIssue)) {
|
|
1222
|
+
const mirror = await this.#findGithubIssueMirror(ghIssue, opts.candidates);
|
|
1223
|
+
if (mirror && !this.#states.isRole(mirror.stateId, 'done')) {
|
|
1224
|
+
if (!opts.dryRun) {
|
|
1225
|
+
const record = (await this.#batch()).getIssue(mirror);
|
|
1226
|
+
if (record) {
|
|
1227
|
+
await this.#completeIssue(record);
|
|
1228
|
+
}
|
|
1229
|
+
else {
|
|
1230
|
+
await this.#linear.setState(mirror, this.#states.idFor(mirror.team, 'done'));
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
this.#increment('githubIssueMirrorsClosed');
|
|
1234
|
+
}
|
|
1235
|
+
return;
|
|
1236
|
+
}
|
|
1237
|
+
if (!githubIssueHasFactoryLabel(ghIssue)) {
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
const mirror = await this.#findGithubIssueMirror(ghIssue, opts.candidates);
|
|
1241
|
+
if (mirror) {
|
|
1242
|
+
this.#increment('githubIssueMirrorsDeduped');
|
|
1243
|
+
return;
|
|
1244
|
+
}
|
|
1245
|
+
const repoLabel = this.#repoLabelForGithubIssue(ghIssue);
|
|
1246
|
+
if (!repoLabel) {
|
|
1247
|
+
this.#increment('githubIssueMirrorsSkippedUnroutable');
|
|
1248
|
+
this.#logger.warn?.('[factory] skipped factory-labeled GitHub issue without repos.byLabel route', {
|
|
1249
|
+
path,
|
|
1250
|
+
repo: ghIssue.repo,
|
|
1251
|
+
url: ghIssue.url,
|
|
1252
|
+
});
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
if (!opts.dryRun) {
|
|
1256
|
+
await this.#linear.createIssue(githubIssueMirrorPayload(ghIssue, repoLabel, this.#config, this.#states.idFor(this.#config.safety.requireTeamKey, 'readyForAgent')));
|
|
1257
|
+
}
|
|
1258
|
+
this.#increment('githubIssueMirrorsCreated');
|
|
1259
|
+
}
|
|
1260
|
+
catch (error) {
|
|
1261
|
+
this.#logger.error?.('[factory] failed to ingest GitHub issue', error);
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
async #readGithubIssue(path) {
|
|
1265
|
+
const candidatePaths = githubIssueReadCandidatePaths(path);
|
|
1266
|
+
try {
|
|
1267
|
+
for (const candidatePath of candidatePaths) {
|
|
1268
|
+
try {
|
|
1269
|
+
const { content } = await this.#mount.readFile(candidatePath);
|
|
1270
|
+
return parseGithubIssue(candidatePath, content);
|
|
1271
|
+
}
|
|
1272
|
+
catch (error) {
|
|
1273
|
+
if (isMissingIssueFileError(error) && candidatePath !== candidatePaths.at(-1)) {
|
|
1274
|
+
continue;
|
|
1275
|
+
}
|
|
1276
|
+
throw error;
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
catch (error) {
|
|
1281
|
+
if (isMissingIssueFileError(error)) {
|
|
1282
|
+
this.#increment('githubIssuePhantomSkipped');
|
|
1283
|
+
this.#logger.debug?.('[factory] skipped missing GitHub issue file discovered from issue tree', { path });
|
|
1284
|
+
return undefined;
|
|
1285
|
+
}
|
|
1286
|
+
throw error;
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
async #findGithubIssueMirror(ghIssue, candidates) {
|
|
1290
|
+
const draftPath = githubIssueMirrorDraftPath(ghIssue);
|
|
1291
|
+
try {
|
|
1292
|
+
return parseLinearIssue(draftPath, (await this.#mount.readFile(draftPath)).content);
|
|
1293
|
+
}
|
|
1294
|
+
catch {
|
|
1295
|
+
// The draft path only exists before the Linear provider reconciles the
|
|
1296
|
+
// create writeback into a canonical AR-* issue. Fall through to persisted
|
|
1297
|
+
// source metadata on existing Linear issues for restart/replay dedupe.
|
|
1298
|
+
}
|
|
1299
|
+
// Cached resolved mirror path: skip the full ISSUE_ROOT scan for mirrors we
|
|
1300
|
+
// already located on an earlier cycle. Re-validate the cached path and fall
|
|
1301
|
+
// back to a scan if the mirror was deleted or renamed.
|
|
1302
|
+
const cacheKey = githubIssueMirrorId(ghIssue);
|
|
1303
|
+
const cachedPath = this.#githubMirrorPathCache.get(cacheKey);
|
|
1304
|
+
if (cachedPath) {
|
|
1305
|
+
const cached = await this.#readIssue(cachedPath);
|
|
1306
|
+
if (cached && linearIssueMirrorsGithubIssue(cached, ghIssue)) {
|
|
1307
|
+
return cached;
|
|
1308
|
+
}
|
|
1309
|
+
this.#githubMirrorPathCache.delete(cacheKey);
|
|
1310
|
+
}
|
|
1311
|
+
// During a full ingestion pass the candidate list is loaded once and shared
|
|
1312
|
+
// across every GitHub issue; the live single-event path passes none and
|
|
1313
|
+
// loads on demand.
|
|
1314
|
+
const mirrorCandidates = candidates
|
|
1315
|
+
? await candidates.load()
|
|
1316
|
+
: await this.#loadLinearMirrorCandidates();
|
|
1317
|
+
const mirror = mirrorCandidates.find((issue) => linearIssueMirrorsGithubIssue(issue, ghIssue));
|
|
1318
|
+
if (mirror) {
|
|
1319
|
+
this.#githubMirrorPathCache.set(cacheKey, mirror.path);
|
|
1320
|
+
}
|
|
1321
|
+
return mirror;
|
|
1322
|
+
}
|
|
1323
|
+
#repoLabelForGithubIssue(ghIssue) {
|
|
1324
|
+
const fullName = ghIssue.repo.toLowerCase();
|
|
1325
|
+
const bareName = ghIssue.repoName.toLowerCase();
|
|
1326
|
+
const entry = Object.entries(this.#config.repos.byLabel)
|
|
1327
|
+
.find(([, repo]) => {
|
|
1328
|
+
const configured = repo.toLowerCase();
|
|
1329
|
+
// ghIssue.repo is always owner/name, so byLabel entries configured with
|
|
1330
|
+
// just the bare repo name still match via repoName.
|
|
1331
|
+
return configured === fullName || configured === bareName;
|
|
1332
|
+
});
|
|
1333
|
+
return entry?.[0];
|
|
1334
|
+
}
|
|
1335
|
+
async #dispatchBlockReason(issue) {
|
|
1336
|
+
const key = issue.key;
|
|
1337
|
+
const state = await this.#state.getDispatchAttempts(this.#workspaceId, key);
|
|
1338
|
+
if (!state)
|
|
1339
|
+
return undefined;
|
|
1340
|
+
if (state.terminal)
|
|
1341
|
+
return 'dispatch already terminal';
|
|
1342
|
+
if (state.inFlight)
|
|
1343
|
+
return 'dispatch already in-flight';
|
|
1344
|
+
const now = this.#clock.now();
|
|
1345
|
+
if (state.backoffUntilMs > now) {
|
|
1346
|
+
return 'dispatch backoff active';
|
|
1347
|
+
}
|
|
1348
|
+
if (state.attempts >= this.#config.dispatch.maxAttempts) {
|
|
1349
|
+
state.terminal = true;
|
|
1350
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, key, state);
|
|
1351
|
+
return 'dispatch retry limit reached';
|
|
1352
|
+
}
|
|
1353
|
+
return undefined;
|
|
1354
|
+
}
|
|
1355
|
+
async #recordDispatchAttempt(issue) {
|
|
1356
|
+
const key = issue.key;
|
|
1357
|
+
const state = await this.#state.getDispatchAttempts(this.#workspaceId, key) ?? {
|
|
1358
|
+
attempts: 0,
|
|
1359
|
+
inFlight: false,
|
|
1360
|
+
terminal: false,
|
|
1361
|
+
backoffUntilMs: 0,
|
|
1362
|
+
};
|
|
1363
|
+
state.attempts += 1;
|
|
1364
|
+
state.inFlight = true;
|
|
1365
|
+
state.backoffUntilMs = 0;
|
|
1366
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, key, state);
|
|
1367
|
+
}
|
|
1368
|
+
async #clearDispatchInFlight(issue) {
|
|
1369
|
+
await this.#state.releaseInFlight(this.#workspaceId, issue.key);
|
|
1370
|
+
}
|
|
1371
|
+
async #recordDispatchFailure(issue) {
|
|
1372
|
+
const state = await this.#state.getDispatchAttempts(this.#workspaceId, issue.key);
|
|
1373
|
+
if (!state)
|
|
1374
|
+
return;
|
|
1375
|
+
state.inFlight = false;
|
|
1376
|
+
if (state.attempts >= this.#config.dispatch.maxAttempts) {
|
|
1377
|
+
state.terminal = true;
|
|
1378
|
+
state.backoffUntilMs = 0;
|
|
1379
|
+
this.#increment('dispatchTerminalFailures');
|
|
1380
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, issue.key, state);
|
|
1381
|
+
return;
|
|
1382
|
+
}
|
|
1383
|
+
state.backoffUntilMs = this.#clock.now() + this.#config.dispatch.errorCooldownMs;
|
|
1384
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, issue.key, state);
|
|
1385
|
+
this.#increment('dispatchBackoffs');
|
|
1386
|
+
}
|
|
1387
|
+
async #recordDispatchTerminal(issue) {
|
|
1388
|
+
const state = await this.#state.getDispatchAttempts(this.#workspaceId, issue.key) ?? {
|
|
1389
|
+
attempts: 0,
|
|
1390
|
+
inFlight: false,
|
|
1391
|
+
terminal: false,
|
|
1392
|
+
backoffUntilMs: 0,
|
|
1393
|
+
};
|
|
1394
|
+
state.inFlight = false;
|
|
1395
|
+
state.terminal = true;
|
|
1396
|
+
state.backoffUntilMs = 0;
|
|
1397
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, issue.key, state);
|
|
1398
|
+
}
|
|
1399
|
+
async #recordCanonicalIssueState(issue) {
|
|
1400
|
+
const previousStateId = await this.#state.getCanonicalState(this.#workspaceId, issue.key);
|
|
1401
|
+
const previousRole = this.#states.roleOf(previousStateId);
|
|
1402
|
+
const reopenedFromTerminal = previousRole === 'done' || previousRole === 'humanReview';
|
|
1403
|
+
if (reopenedFromTerminal && this.#states.isRole(issue.stateId, 'readyForAgent')) {
|
|
1404
|
+
const dispatchState = await this.#state.getDispatchAttempts(this.#workspaceId, issue.key);
|
|
1405
|
+
if (dispatchState?.terminal) {
|
|
1406
|
+
dispatchState.attempts = 0;
|
|
1407
|
+
dispatchState.inFlight = false;
|
|
1408
|
+
dispatchState.terminal = false;
|
|
1409
|
+
dispatchState.backoffUntilMs = 0;
|
|
1410
|
+
await this.#state.recordDispatchAttempt(this.#workspaceId, issue.key, dispatchState);
|
|
1411
|
+
this.#increment('dispatchTerminalReopened');
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
await this.#state.recordCanonicalState(this.#workspaceId, issue.key, issue.stateId);
|
|
1415
|
+
}
|
|
1416
|
+
async #writeLoopHeartbeat(path, registryPath, status, iteration, maxIterations) {
|
|
1417
|
+
const updatedAtMs = this.#clock.now();
|
|
1418
|
+
const heartbeat = {
|
|
1419
|
+
pid: process.pid,
|
|
1420
|
+
status,
|
|
1421
|
+
iteration,
|
|
1422
|
+
maxIterations,
|
|
1423
|
+
updatedAt: new Date(updatedAtMs).toISOString(),
|
|
1424
|
+
updatedAtMs,
|
|
1425
|
+
registryPath,
|
|
1426
|
+
};
|
|
1427
|
+
await mkdir(dirname(path), { recursive: true });
|
|
1428
|
+
await writeFile(path, `${JSON.stringify(heartbeat, null, 2)}\n`, 'utf8');
|
|
1429
|
+
await this.#writeInFlightRegistry(registryPath, path);
|
|
1430
|
+
}
|
|
1431
|
+
async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
|
|
1432
|
+
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
1433
|
+
if (handoffs.length === 0) {
|
|
1434
|
+
return;
|
|
1435
|
+
}
|
|
1436
|
+
try {
|
|
1437
|
+
const protectedPids = await this.#protectedPids();
|
|
1438
|
+
let registryChanged = false;
|
|
1439
|
+
for (const [key, handoff] of handoffs) {
|
|
1440
|
+
const roots = await this.#terminationRoots(handoff.name, handoff.tracked, protectedPids);
|
|
1441
|
+
if (roots.pids.length === 0 && roots.status === 'unresolved') {
|
|
1442
|
+
const unresolvedAgeMs = this.#clock.now() - handoff.persistedAtMs;
|
|
1443
|
+
this.#increment('agentTerminateMissingPid');
|
|
1444
|
+
this.#logger.error?.('[factory] no pid available to reap dispatch-failed handoff during loop catch', {
|
|
1445
|
+
agentName: handoff.name,
|
|
1446
|
+
issue: handoff.issue,
|
|
1447
|
+
sessionRef: handoff.tracked.sessionRef,
|
|
1448
|
+
unresolvedAgeMs,
|
|
1449
|
+
});
|
|
1450
|
+
if (unresolvedAgeMs >= DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS) {
|
|
1451
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
1452
|
+
registryChanged = true;
|
|
1453
|
+
this.#increment('dispatchFailureReaperHandoffsDroppedStaleUnresolved');
|
|
1454
|
+
this.#logger.warn?.('[factory] dropped stale unresolved dispatch-failed handoff', {
|
|
1455
|
+
agentName: handoff.name,
|
|
1456
|
+
issue: handoff.issue,
|
|
1457
|
+
unresolvedAgeMs,
|
|
1458
|
+
ttlMs: DISPATCH_FAILURE_HANDOFF_UNRESOLVED_TTL_MS,
|
|
1459
|
+
});
|
|
1460
|
+
try {
|
|
1461
|
+
await this.#fleet.release(handoff.name, 'dispatch failed');
|
|
1462
|
+
}
|
|
1463
|
+
catch (error) {
|
|
1464
|
+
this.#logger.warn?.('[factory] failed to release unresolved dispatch-failure handoff after pruning', {
|
|
1465
|
+
agentName: handoff.name,
|
|
1466
|
+
error,
|
|
1467
|
+
});
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
continue;
|
|
1471
|
+
}
|
|
1472
|
+
let blockingSkip = false;
|
|
1473
|
+
if (roots.pids.length > 0) {
|
|
1474
|
+
const report = await terminatePids(roots.pids, {
|
|
1475
|
+
kill: this.#kill,
|
|
1476
|
+
readChildPids: this.#readChildPids,
|
|
1477
|
+
sleep: this.#clock.sleep,
|
|
1478
|
+
termGraceMs: this.#terminationGraceMs,
|
|
1479
|
+
protectedPids,
|
|
1480
|
+
});
|
|
1481
|
+
if (report.terminated.length > 0) {
|
|
1482
|
+
this.#increment('loopDispatchFailureHandoffsReaped');
|
|
1483
|
+
}
|
|
1484
|
+
for (const skipped of report.skipped) {
|
|
1485
|
+
if (skipped.reason !== 'pid not running') {
|
|
1486
|
+
blockingSkip = true;
|
|
1487
|
+
this.#logger.warn?.('[factory] dispatch-failure handoff reap skipped during loop catch', {
|
|
1488
|
+
...skipped,
|
|
1489
|
+
agentName: handoff.name,
|
|
1490
|
+
});
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
if (!blockingSkip) {
|
|
1495
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, key);
|
|
1496
|
+
registryChanged = true;
|
|
1497
|
+
try {
|
|
1498
|
+
await this.#fleet.release(handoff.name, 'dispatch failed');
|
|
1499
|
+
}
|
|
1500
|
+
catch (error) {
|
|
1501
|
+
this.#logger.warn?.(`[factory] failed to release ${handoff.name} after dispatch-failure reap`, error);
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
if (registryChanged) {
|
|
1506
|
+
await this.#writeInFlightRegistry(registryPath, heartbeatPath);
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1509
|
+
catch (error) {
|
|
1510
|
+
this.#increment('loopDispatchFailureHandoffReapFailures');
|
|
1511
|
+
this.#error(error);
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
async #readyIssuePaths() {
|
|
1515
|
+
const pathsByKey = new Map();
|
|
1516
|
+
const canonicalPathsByKey = new Map();
|
|
1517
|
+
for (const path of await this.#mount.listTree(ISSUE_ROOT)) {
|
|
1518
|
+
if (isIssueFilePath(path)) {
|
|
1519
|
+
const key = keyFromPath(path);
|
|
1520
|
+
canonicalPathsByKey.set(key, path);
|
|
1521
|
+
pathsByKey.set(key, path);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
for (const path of await this.#mount.listTree(linearByStatePath('ready-for-agent'))) {
|
|
1525
|
+
if (isIssueAliasFilePath(path)) {
|
|
1526
|
+
const canonicalPath = canonicalPathsByKey.get(keyFromPath(path));
|
|
1527
|
+
if (canonicalPath) {
|
|
1528
|
+
pathsByKey.set(keyFromPath(path), canonicalPath);
|
|
1529
|
+
}
|
|
1530
|
+
else {
|
|
1531
|
+
this.#increment('readyAliasesWithoutCanonical');
|
|
1532
|
+
this.#logger.debug?.('[factory] skipped ready alias without canonical issue', { path });
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
return [...pathsByKey.values()].sort();
|
|
1537
|
+
}
|
|
1538
|
+
async #readIssue(path) {
|
|
1539
|
+
try {
|
|
1540
|
+
const { content } = await this.#mount.readFile(path);
|
|
1541
|
+
return parseLinearIssue(path, content);
|
|
1542
|
+
}
|
|
1543
|
+
catch (error) {
|
|
1544
|
+
if (isMissingIssueFileError(error) && isIssuePathUnderRoot(path)) {
|
|
1545
|
+
this.#increment('phantomSkipped');
|
|
1546
|
+
this.#logger.debug?.('[factory] skipped missing issue file discovered from issue tree', { path });
|
|
1547
|
+
return undefined;
|
|
1548
|
+
}
|
|
1549
|
+
this.#logger.warn?.(`Unable to read issue ${path}`, error);
|
|
1550
|
+
return undefined;
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
async #releaseInFlightAgents(reason) {
|
|
1554
|
+
const agents = new Map();
|
|
1555
|
+
for (const record of (await this.#batch()).inFlight) {
|
|
1556
|
+
if (record.dryRun) {
|
|
1557
|
+
continue;
|
|
1558
|
+
}
|
|
1559
|
+
for (const [agentName, tracked] of record.agents) {
|
|
1560
|
+
agents.set(agentName, tracked);
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
await this.#releaseAndTerminateAgents([...agents], reason, 'stop');
|
|
1564
|
+
await this.#writeInFlightRegistry(undefined, undefined, true);
|
|
1565
|
+
}
|
|
1566
|
+
async #releaseAndTerminateAgents(agents, reason, context) {
|
|
1567
|
+
const protectedPids = await this.#protectedPids();
|
|
1568
|
+
for (const [agentName, tracked] of agents) {
|
|
1569
|
+
if (context === 'stop') {
|
|
1570
|
+
await this.#refreshStoppingHeartbeat();
|
|
1571
|
+
}
|
|
1572
|
+
const roots = await this.#terminationRoots(agentName, tracked, protectedPids);
|
|
1573
|
+
if (roots.pids.length === 0 && roots.status === 'unresolved') {
|
|
1574
|
+
this.#increment('agentTerminateMissingPid');
|
|
1575
|
+
this.#logger.error?.(`[factory] no pid available to terminate ${agentName} during ${context}`, {
|
|
1576
|
+
agentName,
|
|
1577
|
+
reason,
|
|
1578
|
+
sessionRef: tracked.sessionRef,
|
|
1579
|
+
});
|
|
1580
|
+
}
|
|
1581
|
+
if (roots.pids.length > 0) {
|
|
1582
|
+
const report = await terminatePids(roots.pids, {
|
|
1583
|
+
kill: this.#kill,
|
|
1584
|
+
readChildPids: this.#readChildPids,
|
|
1585
|
+
sleep: this.#clock.sleep,
|
|
1586
|
+
termGraceMs: this.#terminationGraceMs,
|
|
1587
|
+
protectedPids,
|
|
1588
|
+
});
|
|
1589
|
+
for (const skipped of report.skipped) {
|
|
1590
|
+
if (skipped.reason !== 'pid not running') {
|
|
1591
|
+
this.#logger.warn?.(`[factory] failed to terminate pid ${skipped.pid} for ${agentName} during ${context}`, skipped.reason);
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
if (context === 'stop') {
|
|
1596
|
+
await this.#refreshStoppingHeartbeat();
|
|
1597
|
+
}
|
|
1598
|
+
try {
|
|
1599
|
+
await this.#fleet.release(agentName, reason);
|
|
1600
|
+
}
|
|
1601
|
+
catch (error) {
|
|
1602
|
+
this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error);
|
|
1603
|
+
}
|
|
1604
|
+
if (context === 'stop') {
|
|
1605
|
+
await this.#refreshStoppingHeartbeat();
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
async #terminationRoots(agentName, tracked, protectedPids = []) {
|
|
1610
|
+
const pids = pidsFromSpawnResult(tracked.result);
|
|
1611
|
+
if (!this.#fleet.resolveAgentPid) {
|
|
1612
|
+
return pids.length > 0 ? { pids, status: 'found' } : { pids: [], status: 'unresolved' };
|
|
1613
|
+
}
|
|
1614
|
+
const scan = await this.#processFinder(agentName, { protectedPids });
|
|
1615
|
+
if (scan.status === 'found' &&
|
|
1616
|
+
Number.isInteger(scan.identity.pid) &&
|
|
1617
|
+
scan.identity.pid > 0 &&
|
|
1618
|
+
scan.identity.cmdline.includes(agentName)) {
|
|
1619
|
+
return { pids: [scan.identity.pid], status: 'found' };
|
|
1620
|
+
}
|
|
1621
|
+
if (scan.status === 'ambiguous') {
|
|
1622
|
+
this.#logger.warn?.(`[factory] ambiguous process lookup for ${agentName}`);
|
|
1623
|
+
return { pids: [], status: 'unresolved' };
|
|
1624
|
+
}
|
|
1625
|
+
if (pids.length > 0) {
|
|
1626
|
+
return { pids, status: 'found' };
|
|
1627
|
+
}
|
|
1628
|
+
try {
|
|
1629
|
+
const resolution = await this.#fleet.resolveAgentPid?.(agentName);
|
|
1630
|
+
if (!resolution) {
|
|
1631
|
+
return { pids: [], status: 'unresolved' };
|
|
1632
|
+
}
|
|
1633
|
+
if (resolution.status === 'found' && Number.isInteger(resolution.pid) && resolution.pid > 0) {
|
|
1634
|
+
return { pids: [resolution.pid], status: 'found' };
|
|
1635
|
+
}
|
|
1636
|
+
if (resolution.status === 'unresolved' && scan.status === 'missing') {
|
|
1637
|
+
return { pids: [], status: 'missing' };
|
|
1638
|
+
}
|
|
1639
|
+
return { pids: [], status: resolution.status };
|
|
1640
|
+
}
|
|
1641
|
+
catch (error) {
|
|
1642
|
+
this.#logger.warn?.(`[factory] failed to resolve pid for ${agentName}`, error);
|
|
1643
|
+
return { pids: [], status: 'unresolved' };
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
async #protectedPids() {
|
|
1647
|
+
try {
|
|
1648
|
+
return await this.#fleet.protectedPids?.() ?? [];
|
|
1649
|
+
}
|
|
1650
|
+
catch (error) {
|
|
1651
|
+
this.#logger.warn?.('[factory] failed to resolve protected fleet pids', error);
|
|
1652
|
+
return [];
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
async #persistDispatchFailureReaperHandoff(record, handoffAgents) {
|
|
1656
|
+
if (record.dryRun || handoffAgents.length === 0) {
|
|
1657
|
+
return;
|
|
1658
|
+
}
|
|
1659
|
+
try {
|
|
1660
|
+
for (const agent of handoffAgents) {
|
|
1661
|
+
await this.#state.recordFailureHandoff(this.#workspaceId, registryHandoffKey(agent.issue, agent.name), agent);
|
|
1662
|
+
}
|
|
1663
|
+
await this.#writeInFlightRegistry();
|
|
1664
|
+
this.#increment('dispatchFailureReaperHandoffs');
|
|
1665
|
+
this.#logger.warn?.('[factory] persisted dispatch-failed agents for orphan reaper', {
|
|
1666
|
+
issue: record.issue,
|
|
1667
|
+
agents: handoffAgents.map((agent) => agent.name).sort(),
|
|
1668
|
+
});
|
|
1669
|
+
}
|
|
1670
|
+
catch (error) {
|
|
1671
|
+
this.#increment('dispatchFailureReaperHandoffFailures');
|
|
1672
|
+
for (const agent of handoffAgents) {
|
|
1673
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(agent.issue, agent.name));
|
|
1674
|
+
}
|
|
1675
|
+
this.#logger.error?.('[factory] failed to persist dispatch-failed agents for orphan reaper', {
|
|
1676
|
+
issue: record.issue,
|
|
1677
|
+
error,
|
|
1678
|
+
});
|
|
1679
|
+
this.#error(error, record.issue);
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
async #writeInFlightRegistry(path = this.#config.loop.registryPath, heartbeatPath = this.#config.loop.heartbeatPath, empty = false) {
|
|
1683
|
+
const updatedAtMs = this.#clock.now();
|
|
1684
|
+
const agents = [];
|
|
1685
|
+
const seenAgents = new Set();
|
|
1686
|
+
const appendAgent = async (issue, agentName, tracked) => {
|
|
1687
|
+
const key = registryHandoffKey(issue, agentName);
|
|
1688
|
+
if (seenAgents.has(key)) {
|
|
1689
|
+
return;
|
|
1690
|
+
}
|
|
1691
|
+
seenAgents.add(key);
|
|
1692
|
+
const { pids } = await this.#terminationRoots(agentName, tracked);
|
|
1693
|
+
const processes = [];
|
|
1694
|
+
for (const pid of pids) {
|
|
1695
|
+
const identity = await this.#processIdentityReader(pid);
|
|
1696
|
+
if (identity && identity.cmdline.includes(agentName)) {
|
|
1697
|
+
processes.push({ ...identity, agentName });
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
agents.push({
|
|
1701
|
+
name: agentName,
|
|
1702
|
+
role: tracked.spec.role,
|
|
1703
|
+
issue,
|
|
1704
|
+
sessionRef: tracked.sessionRef,
|
|
1705
|
+
pids,
|
|
1706
|
+
processes,
|
|
1707
|
+
});
|
|
1708
|
+
};
|
|
1709
|
+
if (!empty) {
|
|
1710
|
+
for (const record of (await this.#batch()).inFlight) {
|
|
1711
|
+
if (record.dryRun)
|
|
1712
|
+
continue;
|
|
1713
|
+
for (const [agentName, tracked] of record.agents) {
|
|
1714
|
+
await appendAgent(record.issue, agentName, tracked);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
for (const [, agent] of await this.#state.listFailureHandoffs(this.#workspaceId)) {
|
|
1719
|
+
await appendAgent(agent.issue, agent.name, agent.tracked);
|
|
1720
|
+
}
|
|
1721
|
+
const registry = {
|
|
1722
|
+
pid: process.pid,
|
|
1723
|
+
heartbeatPath,
|
|
1724
|
+
updatedAt: new Date(updatedAtMs).toISOString(),
|
|
1725
|
+
updatedAtMs,
|
|
1726
|
+
agents,
|
|
1727
|
+
};
|
|
1728
|
+
await mkdir(dirname(path), { recursive: true });
|
|
1729
|
+
await writeFile(path, `${JSON.stringify(registry, null, 2)}\n`, 'utf8');
|
|
1730
|
+
}
|
|
1731
|
+
async #spawnAgent(record, spec, dryRun) {
|
|
1732
|
+
const batch = await this.#batch();
|
|
1733
|
+
const invocationId = batch.invocationIdFor(record.issue, spec);
|
|
1734
|
+
const existing = record.agents.get(spec.name);
|
|
1735
|
+
if (existing) {
|
|
1736
|
+
return { name: existing.result?.name ?? spec.name };
|
|
1737
|
+
}
|
|
1738
|
+
if (!batch.shouldSpawn(record, invocationId)) {
|
|
1739
|
+
return { name: spec.name };
|
|
1740
|
+
}
|
|
1741
|
+
if (dryRun) {
|
|
1742
|
+
batch.recordDryRun(record, spec, invocationId);
|
|
1743
|
+
return { name: spec.name };
|
|
1744
|
+
}
|
|
1745
|
+
let roster;
|
|
1746
|
+
try {
|
|
1747
|
+
roster = await retryOnTimeout(() => this.#fleet.roster(), { attempts: 3, delayMs: 2000 });
|
|
1748
|
+
}
|
|
1749
|
+
catch (error) {
|
|
1750
|
+
throw contextualError(`Dispatch roster lookup failed for ${record.issue.key}`, error);
|
|
1751
|
+
}
|
|
1752
|
+
if (roster.agents.some((agent) => agent.name === spec.name)) {
|
|
1753
|
+
batch.recordSpawn(record, spec, invocationId, { name: spec.name, sessionRef: spec.sessionRef });
|
|
1754
|
+
return { name: spec.name };
|
|
1755
|
+
}
|
|
1756
|
+
let result;
|
|
1757
|
+
try {
|
|
1758
|
+
result = await this.#fleet.spawn({
|
|
1759
|
+
name: spec.name,
|
|
1760
|
+
capability: spec.capability,
|
|
1761
|
+
node: spec.node ?? 'self',
|
|
1762
|
+
task: spec.task,
|
|
1763
|
+
model: spec.model,
|
|
1764
|
+
cwd: spec.clonePath,
|
|
1765
|
+
sessionRef: spec.sessionRef,
|
|
1766
|
+
invocationId,
|
|
1767
|
+
restartPolicy: spec.restartPolicy ?? defaultRestartPolicy(spec),
|
|
1768
|
+
channel: spec.channel,
|
|
1769
|
+
});
|
|
1770
|
+
}
|
|
1771
|
+
catch (error) {
|
|
1772
|
+
throw contextualError(`Dispatch spawn failed for ${record.issue.key}/${spec.name} (${spec.capability}) cwd=${spec.clonePath ?? 'default'}`, error);
|
|
1773
|
+
}
|
|
1774
|
+
batch.recordSpawn(record, spec, invocationId, result);
|
|
1775
|
+
return { name: result.name };
|
|
1776
|
+
}
|
|
1777
|
+
async #handleAgentExit(name, reason) {
|
|
1778
|
+
if (this.#stopping) {
|
|
1779
|
+
return;
|
|
1780
|
+
}
|
|
1781
|
+
const batch = await this.#batch();
|
|
1782
|
+
const record = batch.getIssueByAgent(name);
|
|
1783
|
+
if (!record) {
|
|
1784
|
+
return;
|
|
1785
|
+
}
|
|
1786
|
+
const exiting = record.agents.get(name);
|
|
1787
|
+
if (isCompletionReason(reason)) {
|
|
1788
|
+
if (this.#config.babysitter.enabled && !record.dryRun) {
|
|
1789
|
+
// Babysitter path: an implementer/reviewer finishing does NOT mark the
|
|
1790
|
+
// issue done — it hands the open PR to the babysitter. The babysitter
|
|
1791
|
+
// itself finishing means it believes the PR is ready, so re-check and
|
|
1792
|
+
// advance to Human Review.
|
|
1793
|
+
if (exiting?.spec.role === 'babysitter') {
|
|
1794
|
+
await this.#maybeAdvanceToHumanReview(record);
|
|
1795
|
+
}
|
|
1796
|
+
else {
|
|
1797
|
+
await this.#ensureBabysitterForIssue(record);
|
|
1798
|
+
}
|
|
1799
|
+
return;
|
|
1800
|
+
}
|
|
1801
|
+
await this.#completeIssue(record);
|
|
1802
|
+
return;
|
|
1803
|
+
}
|
|
1804
|
+
const tracked = exiting;
|
|
1805
|
+
if (!tracked || record.dryRun) {
|
|
1806
|
+
return;
|
|
1807
|
+
}
|
|
1808
|
+
try {
|
|
1809
|
+
if (tracked.spec.role === 'implementer' && await this.#issueHasCompletionPr(record)) {
|
|
1810
|
+
if (this.#config.babysitter.enabled) {
|
|
1811
|
+
await this.#ensureBabysitterForIssue(record);
|
|
1812
|
+
return;
|
|
1813
|
+
}
|
|
1814
|
+
await this.#completeIssue(record);
|
|
1815
|
+
return;
|
|
1816
|
+
}
|
|
1817
|
+
if (tracked.sessionRef) {
|
|
1818
|
+
const resumeKey = `${issueKey(record.issue)}:${name}:${tracked.sessionRef}`;
|
|
1819
|
+
if (await this.#state.isResumed(this.#workspaceId, resumeKey)) {
|
|
1820
|
+
return;
|
|
1821
|
+
}
|
|
1822
|
+
const existing = this.#resumeInFlight.get(resumeKey);
|
|
1823
|
+
if (existing) {
|
|
1824
|
+
await existing;
|
|
1825
|
+
return;
|
|
1826
|
+
}
|
|
1827
|
+
const resume = this.#resumeTrackedAgent(record, name, tracked);
|
|
1828
|
+
this.#resumeInFlight.set(resumeKey, resume);
|
|
1829
|
+
try {
|
|
1830
|
+
await resume;
|
|
1831
|
+
await this.#state.markResumed(this.#workspaceId, resumeKey);
|
|
1832
|
+
}
|
|
1833
|
+
catch (error) {
|
|
1834
|
+
if (isAgentAlreadyExistsError(error)) {
|
|
1835
|
+
// The broker never released this agent's name on exit
|
|
1836
|
+
// (relay#1116-family), so re-registering collides with the stuck
|
|
1837
|
+
// name. The error is marked retryable but isn't — retrying just
|
|
1838
|
+
// re-collides forever. Treat it as terminal for this name: record
|
|
1839
|
+
// the resume key so subsequent exit events short-circuit, count it,
|
|
1840
|
+
// and warn once instead of spamming a 500 stack trace. The external
|
|
1841
|
+
// reaper / a broker restart reclaims the leaked name.
|
|
1842
|
+
await this.#state.markResumed(this.#workspaceId, resumeKey);
|
|
1843
|
+
this.#increment('resumeNameCollisions');
|
|
1844
|
+
this.#logger.warn?.('[factory] resume skipped: broker still holds agent name (relay#1116); not retrying', {
|
|
1845
|
+
issue: record.issue.key,
|
|
1846
|
+
name,
|
|
1847
|
+
});
|
|
1848
|
+
}
|
|
1849
|
+
else {
|
|
1850
|
+
throw error;
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
finally {
|
|
1854
|
+
this.#resumeInFlight.delete(resumeKey);
|
|
1855
|
+
}
|
|
1856
|
+
}
|
|
1857
|
+
else {
|
|
1858
|
+
const invocationId = `${batch.invocationIdFor(record.issue, tracked.spec)}:restart:${this.#clock.now()}`;
|
|
1859
|
+
const result = await this.#fleet.spawn({
|
|
1860
|
+
name: tracked.spec.name,
|
|
1861
|
+
capability: tracked.spec.capability,
|
|
1862
|
+
node: tracked.spec.node ?? 'self',
|
|
1863
|
+
task: tracked.spec.task,
|
|
1864
|
+
model: tracked.spec.model,
|
|
1865
|
+
cwd: tracked.spec.clonePath,
|
|
1866
|
+
sessionRef: tracked.spec.sessionRef,
|
|
1867
|
+
invocationId,
|
|
1868
|
+
restartPolicy: defaultRestartPolicy(tracked.spec),
|
|
1869
|
+
channel: tracked.spec.channel,
|
|
1870
|
+
});
|
|
1871
|
+
batch.recordSpawn(record, tracked.spec, invocationId, result);
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
catch (error) {
|
|
1875
|
+
this.#error(error, record.issue);
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
async #issueHasCompletionPr(record) {
|
|
1879
|
+
try {
|
|
1880
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
1881
|
+
if (!issue) {
|
|
1882
|
+
return false;
|
|
1883
|
+
}
|
|
1884
|
+
// Only a NON-DRAFT (ready) PR counts as completion. A draft PR means the
|
|
1885
|
+
// work isn't review-ready, so an implementer exiting with only a draft PR
|
|
1886
|
+
// must NOT mark the issue done / release agents — mirror the
|
|
1887
|
+
// #sweepPrStateCompletions draft guard, which keeps draft-PR issues in flight.
|
|
1888
|
+
const pr = await this.#completionPrForIssue(issue);
|
|
1889
|
+
return Boolean(pr && !pr.draft);
|
|
1890
|
+
}
|
|
1891
|
+
catch (error) {
|
|
1892
|
+
this.#logger.warn?.('[factory] PR probe failed after implementer exit; preserving restart behavior', {
|
|
1893
|
+
issue: record.issue.key,
|
|
1894
|
+
error: describeError(error).errorMessage,
|
|
1895
|
+
});
|
|
1896
|
+
this.#increment('exitPrProbeFailures');
|
|
1897
|
+
return false;
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
async #resumeTrackedAgent(record, name, tracked) {
|
|
1901
|
+
if (!tracked.sessionRef) {
|
|
1902
|
+
return;
|
|
1903
|
+
}
|
|
1904
|
+
const result = await this.#fleet.resume({
|
|
1905
|
+
name,
|
|
1906
|
+
sessionRef: tracked.sessionRef,
|
|
1907
|
+
node: tracked.spec.node ?? 'self',
|
|
1908
|
+
capability: tracked.spec.capability,
|
|
1909
|
+
});
|
|
1910
|
+
tracked.result = result;
|
|
1911
|
+
tracked.sessionRef = result.sessionRef ?? tracked.sessionRef;
|
|
1912
|
+
record.agents.delete(name);
|
|
1913
|
+
record.agents.set(result.name, tracked);
|
|
1914
|
+
}
|
|
1915
|
+
async #handleDeliveryFailed(info) {
|
|
1916
|
+
const critical = await this.#state.consumeCritical(this.#workspaceId, info.msgId ?? '');
|
|
1917
|
+
const record = (await this.#batch()).getIssueByAgent(info.to);
|
|
1918
|
+
const issue = critical?.issue ?? record?.issue;
|
|
1919
|
+
const error = new Error(`Critical delivery failed to ${info.to}${info.reason ? `: ${info.reason}` : ''}`);
|
|
1920
|
+
this.#error(error, issue);
|
|
1921
|
+
if (critical && this.#fleet.waitForInjected) {
|
|
1922
|
+
try {
|
|
1923
|
+
const ack = await this.#waitForInjectedAndSubmit(critical.input);
|
|
1924
|
+
await this.#state.recordCritical(this.#workspaceId, ack.eventId, critical);
|
|
1925
|
+
}
|
|
1926
|
+
catch (retryError) {
|
|
1927
|
+
this.#error(retryError, critical.issue);
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
async #handleAgentMessage(message) {
|
|
1932
|
+
// The babysitter signals "PR is green" by DMing factory. Confirm with an
|
|
1933
|
+
// authoritative readiness read before advancing to Human Review.
|
|
1934
|
+
if (this.#config.babysitter.enabled && isFactoryQuestionTarget(message.target)) {
|
|
1935
|
+
const ready = parsePrReadySignal(message);
|
|
1936
|
+
if (ready) {
|
|
1937
|
+
const record = (await this.#batch()).getIssueByAgent(ready.agentName);
|
|
1938
|
+
if (!record || record.dryRun) {
|
|
1939
|
+
this.#increment('prReadySignalsIgnoredNoInFlight');
|
|
1940
|
+
return;
|
|
1941
|
+
}
|
|
1942
|
+
if (ready.issueKey && ready.issueKey !== record.issue.key) {
|
|
1943
|
+
this.#increment('prReadySignalsIgnoredIssueMismatch');
|
|
1944
|
+
return;
|
|
1945
|
+
}
|
|
1946
|
+
await this.#maybeAdvanceToHumanReview(record);
|
|
1947
|
+
return;
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1950
|
+
const question = parseAgentQuestion(message);
|
|
1951
|
+
if (!question || !isFactoryQuestionTarget(message.target)) {
|
|
1952
|
+
return;
|
|
1953
|
+
}
|
|
1954
|
+
const record = (await this.#batch()).getIssueByAgent(question.agentName);
|
|
1955
|
+
if (!record || record.dryRun) {
|
|
1956
|
+
this.#increment('agentQuestionsIgnoredNoInFlight');
|
|
1957
|
+
return;
|
|
1958
|
+
}
|
|
1959
|
+
if (question.issueKey && question.issueKey !== record.issue.key) {
|
|
1960
|
+
this.#increment('agentQuestionsIgnoredIssueMismatch');
|
|
1961
|
+
this.#logger.warn?.('[factory] ignored agent question for mismatched issue', {
|
|
1962
|
+
from: question.agentName,
|
|
1963
|
+
requestedIssue: question.issueKey,
|
|
1964
|
+
activeIssue: record.issue.key,
|
|
1965
|
+
});
|
|
1966
|
+
return;
|
|
1967
|
+
}
|
|
1968
|
+
const dedupeKey = agentQuestionDedupeKey(record.issue, question);
|
|
1969
|
+
if (!await this.#state.claimAgentQuestion(this.#workspaceId, dedupeKey)) {
|
|
1970
|
+
this.#increment('agentQuestionDuplicatesSuppressed');
|
|
1971
|
+
this.#logger.debug?.('[factory] suppressed duplicate agent question', {
|
|
1972
|
+
from: question.agentName,
|
|
1973
|
+
issue: record.issue.key,
|
|
1974
|
+
});
|
|
1975
|
+
return;
|
|
1976
|
+
}
|
|
1977
|
+
if (!question.eventId) {
|
|
1978
|
+
this.#increment('agentQuestionsMissingIdentity');
|
|
1979
|
+
this.#logger.warn?.('[factory] agent question event missing stable identity; falling back to sender/content dedupe', {
|
|
1980
|
+
from: question.agentName,
|
|
1981
|
+
issue: record.issue.key,
|
|
1982
|
+
});
|
|
1983
|
+
}
|
|
1984
|
+
await this.#postAgentQuestionToSlack(record, question);
|
|
1985
|
+
}
|
|
1986
|
+
async #postAgentQuestionToSlack(record, question) {
|
|
1987
|
+
if (!this.#slack || !this.#config.slack) {
|
|
1988
|
+
this.#increment('agentQuestionsSkippedNoSlack');
|
|
1989
|
+
this.#logger.warn?.('[factory] agent question has no Slack channel configured', {
|
|
1990
|
+
issue: record.issue,
|
|
1991
|
+
from: question.agentName,
|
|
1992
|
+
});
|
|
1993
|
+
return;
|
|
1994
|
+
}
|
|
1995
|
+
if (await this.#shouldSkipSlackWriteback('agent-question')) {
|
|
1996
|
+
this.#increment('agentQuestionsSkippedSlackDegraded');
|
|
1997
|
+
return;
|
|
1998
|
+
}
|
|
1999
|
+
const key = issueKey(record.issue);
|
|
2000
|
+
try {
|
|
2001
|
+
await this.#slackWatcherStarts.get(key);
|
|
2002
|
+
}
|
|
2003
|
+
catch {
|
|
2004
|
+
// The initiator logs Slack watcher startup failures.
|
|
2005
|
+
}
|
|
2006
|
+
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
2007
|
+
if (!threadId) {
|
|
2008
|
+
this.#increment('agentQuestionsSkippedMissingThread');
|
|
2009
|
+
this.#logger.warn?.('[factory] agent question has no Slack dispatch thread', {
|
|
2010
|
+
issue: record.issue,
|
|
2011
|
+
from: question.agentName,
|
|
2012
|
+
});
|
|
2013
|
+
return;
|
|
2014
|
+
}
|
|
2015
|
+
try {
|
|
2016
|
+
await this.#slack.reply(threadId, agentQuestionSlackText(record.issue, question));
|
|
2017
|
+
this.#increment('agentQuestionsPostedToSlack');
|
|
2018
|
+
this.#recordSlackWritebackSuccess('agent-question');
|
|
2019
|
+
}
|
|
2020
|
+
catch (error) {
|
|
2021
|
+
this.#markSlackWritebackFailure('agent-question', error);
|
|
2022
|
+
this.#logger.warn?.(`[factory] failed to post agent question for ${record.issue.key}`, error);
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
#recordArrivalLatency(event) {
|
|
2026
|
+
const occurredAt = Date.parse(event.occurredAt);
|
|
2027
|
+
if (!Number.isFinite(occurredAt))
|
|
2028
|
+
return;
|
|
2029
|
+
const latencyMs = Math.max(0, this.#clock.now() - occurredAt);
|
|
2030
|
+
this.#counters.liveEvents = (this.#counters.liveEvents ?? 0) + 1;
|
|
2031
|
+
this.#counters.liveArrivalLatencyMsLast = latencyMs;
|
|
2032
|
+
this.#counters.liveArrivalLatencyMsMax = Math.max(this.#counters.liveArrivalLatencyMsMax ?? 0, latencyMs);
|
|
2033
|
+
this.#logger.debug?.('[factory] live issue event latency recorded', {
|
|
2034
|
+
eventId: event.id,
|
|
2035
|
+
path: event.resource.path,
|
|
2036
|
+
latencyMs,
|
|
2037
|
+
});
|
|
2038
|
+
}
|
|
2039
|
+
async #sendCriticalReviewerMessage(record) {
|
|
2040
|
+
if (!this.#fleet.waitForInjected) {
|
|
2041
|
+
return;
|
|
2042
|
+
}
|
|
2043
|
+
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
2044
|
+
if (!reviewer) {
|
|
2045
|
+
return;
|
|
2046
|
+
}
|
|
2047
|
+
const input = {
|
|
2048
|
+
to: reviewer.result?.name ?? reviewer.spec.name,
|
|
2049
|
+
text: `Review is queued for ${record.issue.key}. Watch implementer PR handoff and report readiness.`,
|
|
2050
|
+
from: 'factory',
|
|
2051
|
+
data: { issue: record.issue },
|
|
2052
|
+
};
|
|
2053
|
+
const ack = await this.#waitForInjectedAndSubmit(input);
|
|
2054
|
+
await this.#state.recordCritical(this.#workspaceId, ack.eventId, { issue: record.issue, input });
|
|
2055
|
+
}
|
|
2056
|
+
async #sendImplementerTask(record) {
|
|
2057
|
+
if (!this.#fleet.waitForInjected) {
|
|
2058
|
+
return;
|
|
2059
|
+
}
|
|
2060
|
+
const implementers = [...record.agents.values()].filter((agent) => agent.spec.role === 'implementer');
|
|
2061
|
+
if (implementers.length === 0) {
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
2065
|
+
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
2066
|
+
const reviewerName = reviewer?.result?.name ?? reviewer?.spec.name ?? 'reviewer';
|
|
2067
|
+
const implementerNames = implementers.map((agent) => agent.result?.name ?? agent.spec.name);
|
|
2068
|
+
for (const implementer of implementers) {
|
|
2069
|
+
const input = {
|
|
2070
|
+
to: implementer.result?.name ?? implementer.spec.name,
|
|
2071
|
+
text: renderAgentTask({
|
|
2072
|
+
issue: templateIssueFromRecord(record, issue),
|
|
2073
|
+
route: routeForImplementer(record, implementer.spec),
|
|
2074
|
+
role: 'implementer',
|
|
2075
|
+
config: { mergePolicy: this.#config.mergePolicy, terminalState: this.#config.terminalState },
|
|
2076
|
+
reviewerName,
|
|
2077
|
+
implementerNames,
|
|
2078
|
+
slackDispatchThread: await this.#slackDispatchThreadFor(record),
|
|
2079
|
+
}),
|
|
2080
|
+
from: 'factory',
|
|
2081
|
+
data: { issue: record.issue },
|
|
2082
|
+
};
|
|
2083
|
+
const ack = await this.#waitForInjectedAndSubmit(input);
|
|
2084
|
+
await this.#state.recordCritical(this.#workspaceId, ack.eventId, { issue: record.issue, input });
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
async #waitForInjectedAndSubmit(input) {
|
|
2088
|
+
if (!this.#fleet.waitForInjected) {
|
|
2089
|
+
throw new Error('Fleet client does not support confirmed task injection');
|
|
2090
|
+
}
|
|
2091
|
+
const ack = await this.#waitForInjectedWithRetry(input);
|
|
2092
|
+
await this.#submitInjectedTask(input, ack);
|
|
2093
|
+
return ack;
|
|
2094
|
+
}
|
|
2095
|
+
async #waitForInjectedWithRetry(input) {
|
|
2096
|
+
if (!this.#fleet.waitForInjected) {
|
|
2097
|
+
throw new Error('Fleet client does not support confirmed task injection');
|
|
2098
|
+
}
|
|
2099
|
+
const startedAt = this.#clock.now();
|
|
2100
|
+
let attempt = 0;
|
|
2101
|
+
let lastError;
|
|
2102
|
+
while (attempt < INJECTION_MAX_ATTEMPTS && this.#clock.now() - startedAt < INJECTION_CONFIRMATION_TIMEOUT_MS) {
|
|
2103
|
+
attempt += 1;
|
|
2104
|
+
const elapsed = Math.max(0, this.#clock.now() - startedAt);
|
|
2105
|
+
const remaining = Math.max(1, INJECTION_CONFIRMATION_TIMEOUT_MS - elapsed);
|
|
2106
|
+
try {
|
|
2107
|
+
return await this.#fleet.waitForInjected(input, {
|
|
2108
|
+
timeoutMs: Math.min(INJECTION_RETRY_ATTEMPT_TIMEOUT_MS, remaining),
|
|
2109
|
+
});
|
|
2110
|
+
}
|
|
2111
|
+
catch (error) {
|
|
2112
|
+
lastError = error;
|
|
2113
|
+
if (!isRegistrationLagInjectionError(error) ||
|
|
2114
|
+
remaining <= INJECTION_RETRY_DELAY_MS ||
|
|
2115
|
+
attempt >= INJECTION_MAX_ATTEMPTS) {
|
|
2116
|
+
throw error;
|
|
2117
|
+
}
|
|
2118
|
+
this.#increment('injectionRegistrationLagRetries');
|
|
2119
|
+
this.#logger.warn?.('[factory] task injection target not registered yet; retrying', {
|
|
2120
|
+
to: input.to,
|
|
2121
|
+
attempt,
|
|
2122
|
+
error: describeError(error).errorMessage,
|
|
2123
|
+
});
|
|
2124
|
+
await this.#clock.sleep(Math.min(INJECTION_RETRY_DELAY_MS, remaining));
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
throw lastError instanceof Error ? lastError : new Error(`Timed out waiting to inject task to ${input.to}`);
|
|
2128
|
+
}
|
|
2129
|
+
async #submitInjectedTask(input, ack) {
|
|
2130
|
+
if (!this.#fleet.sendInput) {
|
|
2131
|
+
return;
|
|
2132
|
+
}
|
|
2133
|
+
const targets = ack.targets && ack.targets.length > 0 ? ack.targets : [input.to];
|
|
2134
|
+
for (const target of new Set(targets)) {
|
|
2135
|
+
await this.#fleet.sendInput(target, '\r');
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
// ── PR babysitter ──────────────────────────────────────────────────────────
|
|
2139
|
+
// Webhook-driven: a change event on the PR's webhook-fed mount file
|
|
2140
|
+
// (/github/repos/<owner>/<repo>/pulls/<n>/meta.json) — PR opened, new commits,
|
|
2141
|
+
// draft toggle, closed/merged — routes here. The PR readiness *verdict* (CI
|
|
2142
|
+
// green, conflicts resolved, comments addressed) is owned by the babysitter
|
|
2143
|
+
// agent, which sees the same per-event webhook data in its sandbox and signals
|
|
2144
|
+
// `[factory-pr-ready]`; the orchestrator never runs `gh`. PR meta events here
|
|
2145
|
+
// only (a) spawn the babysitter on open and (b) carry the latest open/draft/
|
|
2146
|
+
// merged state used to guard the final transition.
|
|
2147
|
+
async #handlePrChange(path) {
|
|
2148
|
+
const parts = githubPullPathParts(path);
|
|
2149
|
+
if (!parts) {
|
|
2150
|
+
return;
|
|
2151
|
+
}
|
|
2152
|
+
let snapshot;
|
|
2153
|
+
try {
|
|
2154
|
+
snapshot = parsePullSnapshot((await this.#mount.readFile(path)).content, parts.number);
|
|
2155
|
+
}
|
|
2156
|
+
catch (error) {
|
|
2157
|
+
this.#logger.debug?.('[factory] babysitter could not read PR snapshot', { path, error: describeError(error).errorMessage });
|
|
2158
|
+
return;
|
|
2159
|
+
}
|
|
2160
|
+
if (!snapshot) {
|
|
2161
|
+
return;
|
|
2162
|
+
}
|
|
2163
|
+
// Map the PR to an in-flight issue by its head ref (reuses the existing
|
|
2164
|
+
// key matcher; no branch-naming assumption).
|
|
2165
|
+
const headRef = snapshot.headRef ?? '';
|
|
2166
|
+
const record = (await this.#batch()).inFlight.find((candidate) => !candidate.dryRun && headRef && containsIssueKey(headRef, candidate.issue.key));
|
|
2167
|
+
if (prMetaShowsMerged(snapshot)) {
|
|
2168
|
+
await this.#advanceMergedPrToDone(snapshot, record);
|
|
2169
|
+
return;
|
|
2170
|
+
}
|
|
2171
|
+
if (!this.#config.babysitter.enabled) {
|
|
2172
|
+
return;
|
|
2173
|
+
}
|
|
2174
|
+
if (!record) {
|
|
2175
|
+
return;
|
|
2176
|
+
}
|
|
2177
|
+
if (snapshot.state && snapshot.state.trim().toUpperCase() !== 'OPEN') {
|
|
2178
|
+
return;
|
|
2179
|
+
}
|
|
2180
|
+
if (snapshot.draft) {
|
|
2181
|
+
this.#increment('babysitterDraftPrSkipped');
|
|
2182
|
+
return;
|
|
2183
|
+
}
|
|
2184
|
+
await this.#ensureBabysitter(record, { repo: `${parts.owner}/${parts.repo}`, prNumber: snapshot.number, url: snapshot.url, path });
|
|
2185
|
+
}
|
|
2186
|
+
async #advanceMergedPrToDone(snapshot, record) {
|
|
2187
|
+
if (record) {
|
|
2188
|
+
await this.#completeIssue(record, { targetState: 'done', runMergeGate: false, completionReason: 'pr-merged' });
|
|
2189
|
+
return;
|
|
2190
|
+
}
|
|
2191
|
+
const issue = await this.#findMergeAdvanceIssueForPr(snapshot);
|
|
2192
|
+
if (!issue) {
|
|
2193
|
+
this.#increment('mergedPrAdvanceNoIssue');
|
|
2194
|
+
return;
|
|
2195
|
+
}
|
|
2196
|
+
const advanceKey = `${issue.key}:${snapshot.number}`;
|
|
2197
|
+
if (this.#postMergeDoneAdvances.has(advanceKey)) {
|
|
2198
|
+
this.#increment('mergedPrAdvanceDuplicatesSuppressed');
|
|
2199
|
+
return;
|
|
2200
|
+
}
|
|
2201
|
+
this.#postMergeDoneAdvances.add(advanceKey);
|
|
2202
|
+
try {
|
|
2203
|
+
const doneStateId = this.#states.idFor(issue.team, 'done');
|
|
2204
|
+
await this.#linear.setState(issue, doneStateId);
|
|
2205
|
+
await this.#recordCanonicalIssueState({ key: issue.key, stateId: doneStateId });
|
|
2206
|
+
this.#emit('writeback-verified', { issue: issueRef(issue), path: issue.path });
|
|
2207
|
+
this.#increment('mergedPrAdvancedDone');
|
|
2208
|
+
this.#increment('done');
|
|
2209
|
+
this.#logger.info?.('[factory] merged PR advanced issue to Done', {
|
|
2210
|
+
issue: issue.key,
|
|
2211
|
+
prNumber: snapshot.number,
|
|
2212
|
+
url: snapshot.url,
|
|
2213
|
+
});
|
|
2214
|
+
if (this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('merge-done-thread')) {
|
|
2215
|
+
try {
|
|
2216
|
+
const root = await this.#slack.postThread({
|
|
2217
|
+
channel: this.#config.slack.channel,
|
|
2218
|
+
text: `${issue.key}: PR merged; Linear state set to Done.`,
|
|
2219
|
+
});
|
|
2220
|
+
await this.#slack.reply(root.threadId, `${issue.key}: Linear state set to Done.`);
|
|
2221
|
+
this.#recordSlackWritebackSuccess('merge-done-thread');
|
|
2222
|
+
}
|
|
2223
|
+
catch (error) {
|
|
2224
|
+
this.#markSlackWritebackFailure('merge-done-thread', error);
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2228
|
+
catch (error) {
|
|
2229
|
+
this.#postMergeDoneAdvances.delete(advanceKey);
|
|
2230
|
+
this.#error(error, issueRef(issue));
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
2233
|
+
async #findMergeAdvanceIssueForPr(snapshot) {
|
|
2234
|
+
// An issue is "upstream" of a merge if it sits in the agent-implementing or
|
|
2235
|
+
// human-review role for its team. UUIDs are globally unique, so the reverse
|
|
2236
|
+
// role lookup covers every team without per-team scoping here.
|
|
2237
|
+
const isUpstreamState = (stateId) => {
|
|
2238
|
+
const role = this.#states.roleOf(stateId);
|
|
2239
|
+
return role === 'agentImplementing' || role === 'humanReview';
|
|
2240
|
+
};
|
|
2241
|
+
let best;
|
|
2242
|
+
const scanStartedAtMs = this.#clock.now();
|
|
2243
|
+
// This no-record path runs after agents are released, so there is no
|
|
2244
|
+
// tracked PR identity left. Keep the scan simple and prefer branch identity
|
|
2245
|
+
// over title/body references to avoid "related to AR-N" body false positives.
|
|
2246
|
+
for (const path of await this.#mount.listTree(ISSUE_ROOT)) {
|
|
2247
|
+
if (!isIssueFilePath(path)) {
|
|
2248
|
+
continue;
|
|
2249
|
+
}
|
|
2250
|
+
const issue = await this.#readIssue(path);
|
|
2251
|
+
if (!issue || !isUpstreamState(issue.stateId)) {
|
|
2252
|
+
continue;
|
|
2253
|
+
}
|
|
2254
|
+
if (!isRealLinearIssue(issue) || !isInFactoryScope(issue, this.#config.safety)) {
|
|
2255
|
+
continue;
|
|
2256
|
+
}
|
|
2257
|
+
const score = prSnapshotIssueMatchScore(snapshot, issue.key);
|
|
2258
|
+
if (score > 0 && (!best || score > best.score)) {
|
|
2259
|
+
best = { issue, score };
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
this.#logger.debug?.('[factory] scanned Linear issues for merged PR advance', {
|
|
2263
|
+
prNumber: snapshot.number,
|
|
2264
|
+
durationMs: this.#clock.now() - scanStartedAtMs,
|
|
2265
|
+
matchedIssue: best?.issue.key,
|
|
2266
|
+
matchScore: best?.score,
|
|
2267
|
+
});
|
|
2268
|
+
return best?.issue;
|
|
2269
|
+
}
|
|
2270
|
+
// Safety net for a missed PR-open mount event: resolve the PR via the existing
|
|
2271
|
+
// probe resolver and spawn the babysitter. Triggered by an implementer exiting
|
|
2272
|
+
// after opening its PR (an event, not a poll).
|
|
2273
|
+
async #ensureBabysitterForIssue(record) {
|
|
2274
|
+
if (this.#babysitterSpawned.has(record.issue.key)) {
|
|
2275
|
+
return;
|
|
2276
|
+
}
|
|
2277
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
2278
|
+
if (!issue) {
|
|
2279
|
+
return;
|
|
2280
|
+
}
|
|
2281
|
+
const pr = await this.#completionPrForIssue(issue);
|
|
2282
|
+
if (!pr || pr.draft) {
|
|
2283
|
+
return;
|
|
2284
|
+
}
|
|
2285
|
+
await this.#ensureBabysitter(record, { repo: pr.repo, prNumber: pr.prNumber });
|
|
2286
|
+
}
|
|
2287
|
+
async #ensureBabysitter(record, prRef) {
|
|
2288
|
+
this.#babysitterPr.set(record.issue.key, { repo: prRef.repo, prNumber: prRef.prNumber, path: prRef.path });
|
|
2289
|
+
if (this.#babysitterSpawned.has(record.issue.key)) {
|
|
2290
|
+
return;
|
|
2291
|
+
}
|
|
2292
|
+
if ([...record.agents.values()].some((agent) => agent.spec.role === 'babysitter')) {
|
|
2293
|
+
this.#babysitterSpawned.add(record.issue.key);
|
|
2294
|
+
return;
|
|
2295
|
+
}
|
|
2296
|
+
// Reserve up-front so concurrent PR events in a drain don't double-spawn.
|
|
2297
|
+
this.#babysitterSpawned.add(record.issue.key);
|
|
2298
|
+
try {
|
|
2299
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
2300
|
+
if (!issue) {
|
|
2301
|
+
this.#babysitterSpawned.delete(record.issue.key);
|
|
2302
|
+
return;
|
|
2303
|
+
}
|
|
2304
|
+
const route = record.decision.routes.find((candidate) => candidate.repo === prRef.repo)
|
|
2305
|
+
?? record.decision.routes[0];
|
|
2306
|
+
const spec = babysitterSpec(issue, this.#config, route);
|
|
2307
|
+
const reviewer = [...record.agents.values()].find((agent) => agent.spec.role === 'reviewer');
|
|
2308
|
+
const reviewerName = reviewer?.result?.name ?? reviewer?.spec.name ?? `${spec.name.replace(/-babysit$/, '')}-review`;
|
|
2309
|
+
const implementerNames = [...record.agents.values()]
|
|
2310
|
+
.filter((agent) => agent.spec.role === 'implementer')
|
|
2311
|
+
.map((agent) => agent.result?.name ?? agent.spec.name);
|
|
2312
|
+
const task = renderAgentTask({
|
|
2313
|
+
issue: templateIssueFromRecord(record, issue),
|
|
2314
|
+
route: route ?? { repo: prRef.repo },
|
|
2315
|
+
role: 'babysitter',
|
|
2316
|
+
config: { mergePolicy: this.#config.mergePolicy, terminalState: this.#config.terminalState },
|
|
2317
|
+
reviewerName,
|
|
2318
|
+
implementerNames,
|
|
2319
|
+
pr: { number: prRef.prNumber, url: prRef.url },
|
|
2320
|
+
slackDispatchThread: await this.#slackDispatchThreadFor(record),
|
|
2321
|
+
});
|
|
2322
|
+
const spawned = await this.#spawnAgent(record, { ...spec, task }, false);
|
|
2323
|
+
await this.#writeInFlightRegistry();
|
|
2324
|
+
this.#increment('babysittersSpawned');
|
|
2325
|
+
this.#logger.info?.('[factory] babysitter spawned for open PR', {
|
|
2326
|
+
issue: record.issue.key,
|
|
2327
|
+
repo: prRef.repo,
|
|
2328
|
+
prNumber: prRef.prNumber,
|
|
2329
|
+
babysitter: spawned.name,
|
|
2330
|
+
});
|
|
2331
|
+
if (this.#fleet.waitForInjected) {
|
|
2332
|
+
const tracked = record.agents.get(spawned.name);
|
|
2333
|
+
const input = {
|
|
2334
|
+
to: tracked?.result?.name ?? spawned.name,
|
|
2335
|
+
text: task,
|
|
2336
|
+
from: 'factory',
|
|
2337
|
+
data: { issue: record.issue },
|
|
2338
|
+
};
|
|
2339
|
+
const ack = await this.#waitForInjectedAndSubmit(input);
|
|
2340
|
+
await this.#state.recordCritical(this.#workspaceId, ack.eventId, { issue: record.issue, input });
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
catch (error) {
|
|
2344
|
+
// Allow a later event to retry the spawn.
|
|
2345
|
+
this.#babysitterSpawned.delete(record.issue.key);
|
|
2346
|
+
this.#increment('babysitterSpawnFailures');
|
|
2347
|
+
this.#error(error, record.issue);
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
// The babysitter owns the readiness verdict (CI green + conflicts resolved +
|
|
2351
|
+
// review comments addressed) — it sees the per-event PR webhook data in its
|
|
2352
|
+
// sandbox, exactly like AgentWorkforce/agents review. It signals readiness by
|
|
2353
|
+
// DMing `[factory-pr-ready]`. The orchestrator trusts that signal and only
|
|
2354
|
+
// guards on the PR's OWN webhook-fed meta (still open, not a draft, not already
|
|
2355
|
+
// merged) before flipping the issue to Human Review. No `gh` call.
|
|
2356
|
+
async #maybeAdvanceToHumanReview(record) {
|
|
2357
|
+
if (this.#completionInFlight.has(issueKey(record.issue))) {
|
|
2358
|
+
return;
|
|
2359
|
+
}
|
|
2360
|
+
const snapshot = await this.#readBabysatPrSnapshot(record);
|
|
2361
|
+
if (snapshot) {
|
|
2362
|
+
const guard = prMetaAllowsHumanReview(snapshot);
|
|
2363
|
+
if (!guard.ok) {
|
|
2364
|
+
this.#increment('babysitterReadinessGuardBlocked');
|
|
2365
|
+
this.#logger.info?.('[factory] babysitter ready signal ignored; PR meta not eligible', {
|
|
2366
|
+
issue: record.issue.key,
|
|
2367
|
+
reason: guard.reason,
|
|
2368
|
+
});
|
|
2369
|
+
return;
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
this.#increment('babysitterReadinessReady');
|
|
2373
|
+
this.#logger.info?.('[factory] babysitter signalled PR ready; advancing to human review', {
|
|
2374
|
+
issue: record.issue.key,
|
|
2375
|
+
prMetaChecked: Boolean(snapshot),
|
|
2376
|
+
});
|
|
2377
|
+
await this.#completeIssue(record);
|
|
2378
|
+
}
|
|
2379
|
+
// Re-read the babysat PR's webhook-fed meta from the mount (no gh). Prefers the
|
|
2380
|
+
// exact path captured when the babysitter was spawned; otherwise scans the
|
|
2381
|
+
// repo's pulls subtree for the PR number across known layout shapes.
|
|
2382
|
+
async #readBabysatPrSnapshot(record) {
|
|
2383
|
+
const ref = this.#babysitterPr.get(record.issue.key);
|
|
2384
|
+
if (!ref) {
|
|
2385
|
+
return undefined;
|
|
2386
|
+
}
|
|
2387
|
+
const candidatePaths = ref.path ? [ref.path] : await this.#pullMetaPathsFor(ref.repo, ref.prNumber);
|
|
2388
|
+
for (const path of candidatePaths) {
|
|
2389
|
+
try {
|
|
2390
|
+
const snapshot = parsePullSnapshot((await this.#mount.readFile(path)).content, ref.prNumber);
|
|
2391
|
+
if (snapshot) {
|
|
2392
|
+
return snapshot;
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
catch {
|
|
2396
|
+
// try the next candidate path
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
return undefined;
|
|
2400
|
+
}
|
|
2401
|
+
async #pullMetaPathsFor(repo, prNumber) {
|
|
2402
|
+
const [owner, name] = repo.split('/');
|
|
2403
|
+
if (!owner || !name) {
|
|
2404
|
+
return [];
|
|
2405
|
+
}
|
|
2406
|
+
// Scan both the nested <owner>/<repo> and the flat <owner>__<repo> pulls
|
|
2407
|
+
// roots so the readiness guard finds the PR meta regardless of mount layout.
|
|
2408
|
+
const roots = [`/github/repos/${owner}/${name}/pulls/`, `/github/repos/${owner}__${name}/pulls/`];
|
|
2409
|
+
// Match .../pulls/<n>(__slug)?(/...)? for this PR number, in either the
|
|
2410
|
+
// nested directory or flat <n>.json / by-id/<n>.json forms.
|
|
2411
|
+
const numberSegment = new RegExp(`/pulls/(?:by-id/)?${prNumber}(?:__[^/]*)?(?:/|\\.json$)`, 'u');
|
|
2412
|
+
const found = [];
|
|
2413
|
+
for (const root of roots) {
|
|
2414
|
+
try {
|
|
2415
|
+
const tree = await this.#mount.listTree(root);
|
|
2416
|
+
found.push(...tree.filter((path) => path.endsWith('.json') && numberSegment.test(path)));
|
|
2417
|
+
}
|
|
2418
|
+
catch {
|
|
2419
|
+
// try the next root
|
|
2420
|
+
}
|
|
2421
|
+
}
|
|
2422
|
+
return found;
|
|
2423
|
+
}
|
|
2424
|
+
async #completeIssue(record, opts = {}) {
|
|
2425
|
+
const completionKey = issueKey(record.issue);
|
|
2426
|
+
if (this.#completionInFlight.has(completionKey)) {
|
|
2427
|
+
return;
|
|
2428
|
+
}
|
|
2429
|
+
this.#completionInFlight.add(completionKey);
|
|
2430
|
+
try {
|
|
2431
|
+
const issue = await this.#readIssue(record.issue.path);
|
|
2432
|
+
// Resolve the terminal state for the issue's own team. Land in
|
|
2433
|
+
// `human-review` only when the operator opted into that terminal state AND
|
|
2434
|
+
// the team actually has a human-review state; otherwise fall back to `done`
|
|
2435
|
+
// (the legacy terminal) so the issue never gets stuck on an unconfigured
|
|
2436
|
+
// state.
|
|
2437
|
+
const issueTeam = issue?.team;
|
|
2438
|
+
const humanReview = opts.targetState !== 'done' &&
|
|
2439
|
+
this.#config.terminalState === 'human-review' &&
|
|
2440
|
+
this.#states.hasHumanReview(issueTeam);
|
|
2441
|
+
const targetState = humanReview
|
|
2442
|
+
? this.#states.idFor(issueTeam, 'humanReview')
|
|
2443
|
+
: this.#states.idFor(issueTeam, 'done');
|
|
2444
|
+
const statusLabel = humanReview ? 'In Human Review' : 'Done';
|
|
2445
|
+
if (issue) {
|
|
2446
|
+
await this.#linear.setState(issue, targetState);
|
|
2447
|
+
await this.#recordCanonicalIssueState({ key: issue.key, stateId: targetState });
|
|
2448
|
+
this.#emit('writeback-verified', { issue: record.issue, path: issue.path });
|
|
2449
|
+
}
|
|
2450
|
+
if (this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) {
|
|
2451
|
+
try {
|
|
2452
|
+
const merged = opts.completionReason === 'pr-merged';
|
|
2453
|
+
const completionText = merged
|
|
2454
|
+
? `${record.issue.key}: PR merged; Linear state set to ${statusLabel}.`
|
|
2455
|
+
: `${record.issue.key}: factory agents completed${humanReview ? '; awaiting human review' : ''}.\nStatus: ${statusLabel}\nMerge policy: ${this.#config.mergePolicy}`;
|
|
2456
|
+
const stateText = merged
|
|
2457
|
+
? `${record.issue.key}: PR merged; Linear state set to ${statusLabel}.`
|
|
2458
|
+
: humanReview
|
|
2459
|
+
? `${record.issue.key}: awaiting human review; Linear state set to ${statusLabel}.`
|
|
2460
|
+
: `${record.issue.key}: Linear state set to ${statusLabel}.`;
|
|
2461
|
+
const root = await this.#slack.postThread({
|
|
2462
|
+
channel: this.#config.slack.channel,
|
|
2463
|
+
text: completionText,
|
|
2464
|
+
});
|
|
2465
|
+
await this.#slack.reply(root.threadId, stateText);
|
|
2466
|
+
this.#recordSlackWritebackSuccess('completion-thread');
|
|
2467
|
+
}
|
|
2468
|
+
catch (error) {
|
|
2469
|
+
this.#markSlackWritebackFailure('completion-thread', error);
|
|
2470
|
+
}
|
|
2471
|
+
}
|
|
2472
|
+
// Only auto-merge on the `done` terminal path. Human Review parks the PR
|
|
2473
|
+
// for an operator — the merge gate (which requires an APPROVED review)
|
|
2474
|
+
// would refuse anyway, and we must not merge before the human has looked.
|
|
2475
|
+
if (issue && !humanReview && opts.runMergeGate !== false) {
|
|
2476
|
+
await this.#runCompletionMergeGate(issue);
|
|
2477
|
+
}
|
|
2478
|
+
await this.#releaseAndTerminateAgents([...record.agents], humanReview ? 'issue-human-review' : 'issue-done', 'completion');
|
|
2479
|
+
this.#increment(humanReview ? 'humanReview' : 'done');
|
|
2480
|
+
this.#emit('issue-done', { issue: record.issue });
|
|
2481
|
+
await this.#stopSlackWatcher(record.issue);
|
|
2482
|
+
await this.#recordDispatchTerminal(record.issue);
|
|
2483
|
+
const next = (await this.#batch()).complete(record.issue);
|
|
2484
|
+
if (next) {
|
|
2485
|
+
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
2486
|
+
}
|
|
2487
|
+
await this.#writeInFlightRegistry();
|
|
2488
|
+
}
|
|
2489
|
+
catch (error) {
|
|
2490
|
+
this.#error(error, record.issue);
|
|
2491
|
+
}
|
|
2492
|
+
finally {
|
|
2493
|
+
this.#completionInFlight.delete(completionKey);
|
|
2494
|
+
this.#probePrGhBackoffUntilMs.delete(completionKey);
|
|
2495
|
+
this.#probePrResolvedCache.delete(completionKey);
|
|
2496
|
+
this.#babysitterSpawned.delete(record.issue.key);
|
|
2497
|
+
this.#babysitterPr.delete(record.issue.key);
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
#emit(event, payload) {
|
|
2501
|
+
for (const listener of this.#listeners.get(event) ?? []) {
|
|
2502
|
+
listener(payload);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
#error(error, issue) {
|
|
2506
|
+
this.#increment('errors');
|
|
2507
|
+
this.#logger.error?.('[factory] error', error);
|
|
2508
|
+
this.#emit('error', { error, ...describeError(error), issue });
|
|
2509
|
+
}
|
|
2510
|
+
#increment(name) {
|
|
2511
|
+
this.#counters[name] = (this.#counters[name] ?? 0) + 1;
|
|
2512
|
+
}
|
|
2513
|
+
async #shouldSkipSlackWriteback(context) {
|
|
2514
|
+
if (!this.#config.slack)
|
|
2515
|
+
return false;
|
|
2516
|
+
const freshness = await this.#slackFreshness();
|
|
2517
|
+
if (this.#slackWritebackFailureDegraded) {
|
|
2518
|
+
if (this.#slackWritebackFailureBackoffUntilMs > this.#clock.now()) {
|
|
2519
|
+
this.#increment('slackWritebacksSkipped');
|
|
2520
|
+
return true;
|
|
2521
|
+
}
|
|
2522
|
+
return false;
|
|
2523
|
+
}
|
|
2524
|
+
if (!freshness.degraded && freshness.known) {
|
|
2525
|
+
if (this.#slackDegraded) {
|
|
2526
|
+
this.#logger.info?.('[factory] Slack sync recovered; resuming Slack writeback', { context });
|
|
2527
|
+
this.#increment('slackRecoveredEpisodes');
|
|
2528
|
+
}
|
|
2529
|
+
this.#slackDegraded = false;
|
|
2530
|
+
this.#slackDegradedReason = undefined;
|
|
2531
|
+
return false;
|
|
2532
|
+
}
|
|
2533
|
+
if (!freshness.degraded && this.#slackDegraded) {
|
|
2534
|
+
this.#increment('slackWritebacksSkipped');
|
|
2535
|
+
return true;
|
|
2536
|
+
}
|
|
2537
|
+
if (!freshness.degraded) {
|
|
2538
|
+
return false;
|
|
2539
|
+
}
|
|
2540
|
+
this.#slackDegradedReason = freshness.reason;
|
|
2541
|
+
this.#increment('slackWritebacksSkipped');
|
|
2542
|
+
if (!this.#slackDegraded) {
|
|
2543
|
+
this.#slackDegraded = true;
|
|
2544
|
+
this.#increment('slackDegradedEpisodes');
|
|
2545
|
+
this.#logger.warn?.('[factory] Slack sync degraded; skipping Slack writeback', {
|
|
2546
|
+
context,
|
|
2547
|
+
reason: freshness.reason,
|
|
2548
|
+
status: freshness.status,
|
|
2549
|
+
});
|
|
2550
|
+
}
|
|
2551
|
+
return true;
|
|
2552
|
+
}
|
|
2553
|
+
#markSlackWritebackFailure(context, error) {
|
|
2554
|
+
this.#slackWritebackFailureDegraded = true;
|
|
2555
|
+
this.#slackWritebackFailureBackoffUntilMs = this.#clock.now() + (this.#config.slack?.staleAfterMs ?? 10 * 60_000);
|
|
2556
|
+
this.#slackDegradedReason = `slack writeback failed: ${describeError(error).errorMessage}`;
|
|
2557
|
+
if (!this.#slackDegraded) {
|
|
2558
|
+
this.#slackDegraded = true;
|
|
2559
|
+
this.#increment('slackDegradedEpisodes');
|
|
2560
|
+
this.#logger.warn?.('[factory] Slack writeback failed; marking Slack degraded', {
|
|
2561
|
+
context,
|
|
2562
|
+
reason: this.#slackDegradedReason,
|
|
2563
|
+
});
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
#recordSlackWritebackSuccess(context) {
|
|
2567
|
+
if (this.#slackWritebackFailureDegraded) {
|
|
2568
|
+
this.#logger.info?.('[factory] Slack writeback recovered; clearing write-failure degradation', { context });
|
|
2569
|
+
this.#increment('slackRecoveredEpisodes');
|
|
2570
|
+
}
|
|
2571
|
+
this.#slackWritebackFailureDegraded = false;
|
|
2572
|
+
this.#slackWritebackFailureBackoffUntilMs = 0;
|
|
2573
|
+
if (this.#slackDegraded) {
|
|
2574
|
+
this.#slackDegraded = false;
|
|
2575
|
+
this.#slackDegradedReason = undefined;
|
|
2576
|
+
}
|
|
2577
|
+
}
|
|
2578
|
+
async #slackFreshness() {
|
|
2579
|
+
const staleAfterMs = this.#config.slack?.staleAfterMs ?? 10 * 60_000;
|
|
2580
|
+
let sawSlackStatus = false;
|
|
2581
|
+
let softStatusResult;
|
|
2582
|
+
let softStatus;
|
|
2583
|
+
try {
|
|
2584
|
+
const status = await this.#mount.getSyncStatus?.('slack');
|
|
2585
|
+
sawSlackStatus = status?.provider === 'slack';
|
|
2586
|
+
const statusResult = slackSyncStatusResult(status, this.#clock.now(), staleAfterMs);
|
|
2587
|
+
if (statusResult.known) {
|
|
2588
|
+
if (!statusResult.degraded || statusResult.severity === 'hard') {
|
|
2589
|
+
return { known: true, degraded: statusResult.degraded, reason: statusResult.reason, status };
|
|
2590
|
+
}
|
|
2591
|
+
softStatusResult = statusResult;
|
|
2592
|
+
softStatus = status;
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
catch (error) {
|
|
2596
|
+
this.#logger.warn?.('[factory] Slack sync freshness check failed; proceeding without degradation', error);
|
|
2597
|
+
}
|
|
2598
|
+
try {
|
|
2599
|
+
const watermark = await this.#slackEventWatermark();
|
|
2600
|
+
if (watermark.lastEventAtMs === undefined) {
|
|
2601
|
+
return sawSlackStatus
|
|
2602
|
+
? {
|
|
2603
|
+
known: true,
|
|
2604
|
+
degraded: true,
|
|
2605
|
+
reason: softStatusResult?.reason ?? 'slack sync has no recent event watermark',
|
|
2606
|
+
status: softStatus,
|
|
2607
|
+
}
|
|
2608
|
+
: { known: false, degraded: false };
|
|
2609
|
+
}
|
|
2610
|
+
const ageMs = this.#clock.now() - watermark.lastEventAtMs;
|
|
2611
|
+
if (ageMs <= staleAfterMs) {
|
|
2612
|
+
if (softStatusResult?.degraded) {
|
|
2613
|
+
this.#increment('slackGateBypassedByEventWatermark');
|
|
2614
|
+
this.#logger.info?.('[factory] Slack sync soft-degraded but event watermark is fresh; continuing Slack writeback', {
|
|
2615
|
+
reason: softStatusResult.reason,
|
|
2616
|
+
status: softStatus,
|
|
2617
|
+
lastSlackEventAtMs: watermark.lastEventAtMs,
|
|
2618
|
+
eventWatermarkAgeMs: ageMs,
|
|
2619
|
+
});
|
|
2620
|
+
}
|
|
2621
|
+
return { known: true, degraded: false };
|
|
2622
|
+
}
|
|
2623
|
+
return {
|
|
2624
|
+
known: true,
|
|
2625
|
+
degraded: true,
|
|
2626
|
+
reason: softStatusResult?.reason ?? `slack event watermark stale by ${ageMs}ms`,
|
|
2627
|
+
status: softStatus,
|
|
2628
|
+
};
|
|
2629
|
+
}
|
|
2630
|
+
catch (error) {
|
|
2631
|
+
this.#logger.warn?.(softStatusResult?.degraded
|
|
2632
|
+
? '[factory] Slack event freshness fallback failed; honoring soft sync degradation'
|
|
2633
|
+
: '[factory] Slack event freshness fallback failed; proceeding without degradation', error);
|
|
2634
|
+
return softStatusResult?.degraded
|
|
2635
|
+
? { known: true, degraded: true, reason: softStatusResult.reason, status: softStatus }
|
|
2636
|
+
: { known: false, degraded: false };
|
|
2637
|
+
}
|
|
2638
|
+
}
|
|
2639
|
+
async #slackEventWatermark() {
|
|
2640
|
+
const nowMs = this.#clock.now();
|
|
2641
|
+
if (this.#slackEventWatermarkCache && nowMs - this.#slackEventWatermarkCache.checkedAtMs <= SLACK_EVENT_WATERMARK_CACHE_MS) {
|
|
2642
|
+
return this.#slackEventWatermarkCache.result;
|
|
2643
|
+
}
|
|
2644
|
+
if (this.#slackEventWatermarkRefresh) {
|
|
2645
|
+
return this.#slackEventWatermarkRefresh;
|
|
2646
|
+
}
|
|
2647
|
+
this.#slackEventWatermarkRefresh = this.#refreshSlackEventWatermark();
|
|
2648
|
+
try {
|
|
2649
|
+
return await this.#slackEventWatermarkRefresh;
|
|
2650
|
+
}
|
|
2651
|
+
finally {
|
|
2652
|
+
this.#slackEventWatermarkRefresh = undefined;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
async #refreshSlackEventWatermark() {
|
|
2656
|
+
this.#increment('slackEventWatermarkRefreshes');
|
|
2657
|
+
const page = await this.#mount.getEvents({ provider: 'slack', last: 100, limit: 100 });
|
|
2658
|
+
const lastEventAtMs = page.events
|
|
2659
|
+
.filter((event) => eventProvider(event) === 'slack')
|
|
2660
|
+
.map((event) => eventOccurredAtMs(event))
|
|
2661
|
+
.filter((time) => time !== undefined && Number.isFinite(time))
|
|
2662
|
+
.sort((a, b) => b - a)[0];
|
|
2663
|
+
const result = lastEventAtMs === undefined
|
|
2664
|
+
? { known: true }
|
|
2665
|
+
: { known: true, lastEventAtMs };
|
|
2666
|
+
this.#slackEventWatermarkCache = { checkedAtMs: this.#clock.now(), result };
|
|
2667
|
+
return result;
|
|
2668
|
+
}
|
|
2669
|
+
async #ensureSlackDispatchThread(record, result) {
|
|
2670
|
+
if (!this.#slack || !this.#config.slack || result.dryRun) {
|
|
2671
|
+
return;
|
|
2672
|
+
}
|
|
2673
|
+
if (await this.#shouldSkipSlackWriteback('dispatch-thread')) {
|
|
2674
|
+
return;
|
|
2675
|
+
}
|
|
2676
|
+
const key = issueKey(record.issue);
|
|
2677
|
+
if (await this.#state.getSlackThread(this.#workspaceId, key) || this.#slackWatcherStarts.has(key)) {
|
|
2678
|
+
try {
|
|
2679
|
+
await this.#slackWatcherStarts.get(key);
|
|
2680
|
+
}
|
|
2681
|
+
catch {
|
|
2682
|
+
// The initiator logs Slack watcher startup failures.
|
|
2683
|
+
}
|
|
2684
|
+
return;
|
|
2685
|
+
}
|
|
2686
|
+
const start = this.#postAndWatchSlackDispatchThread(record, result);
|
|
2687
|
+
this.#slackWatcherStarts.set(key, start);
|
|
2688
|
+
try {
|
|
2689
|
+
await start;
|
|
2690
|
+
}
|
|
2691
|
+
catch (error) {
|
|
2692
|
+
this.#markSlackWritebackFailure('dispatch-thread', error);
|
|
2693
|
+
this.#logger.warn?.(`[factory] failed to establish Slack dispatch thread for ${record.issue.key}`, error);
|
|
2694
|
+
}
|
|
2695
|
+
finally {
|
|
2696
|
+
this.#slackWatcherStarts.delete(key);
|
|
2697
|
+
}
|
|
2698
|
+
}
|
|
2699
|
+
async #postAndWatchSlackDispatchThread(record, result) {
|
|
2700
|
+
if (!this.#slack || !this.#config.slack) {
|
|
2701
|
+
return;
|
|
2702
|
+
}
|
|
2703
|
+
const root = await this.#slack.postThread({
|
|
2704
|
+
channel: this.#config.slack.channel,
|
|
2705
|
+
text: [
|
|
2706
|
+
`${record.issue.key}: factory agents dispatched.`,
|
|
2707
|
+
`State: ${result.stateId ?? 'dispatching'}`,
|
|
2708
|
+
`Agents: ${result.agents.map((agent) => agent.name).join(', ') || 'none'}`,
|
|
2709
|
+
].join('\n'),
|
|
2710
|
+
});
|
|
2711
|
+
await this.#state.setSlackThread(this.#workspaceId, issueKey(record.issue), root.threadId);
|
|
2712
|
+
await this.#watchSlackThread(record, root.threadId);
|
|
2713
|
+
this.#recordSlackWritebackSuccess('dispatch-thread');
|
|
2714
|
+
}
|
|
2715
|
+
async #escalateTriageToSlack(decision, reason, dryRun) {
|
|
2716
|
+
if (dryRun) {
|
|
2717
|
+
return;
|
|
2718
|
+
}
|
|
2719
|
+
if (!this.#slack || !this.#config.slack) {
|
|
2720
|
+
this.#logger.warn?.('[factory] triage escalation has no Slack channel configured', {
|
|
2721
|
+
issue: decision.issue,
|
|
2722
|
+
reason,
|
|
2723
|
+
});
|
|
2724
|
+
return;
|
|
2725
|
+
}
|
|
2726
|
+
if (await this.#shouldSkipSlackWriteback('triage-escalation')) {
|
|
2727
|
+
return;
|
|
2728
|
+
}
|
|
2729
|
+
const key = issueKey(decision.issue);
|
|
2730
|
+
if (await this.#state.getSlackThread(this.#workspaceId, key) || this.#slackWatcherStarts.has(key)) {
|
|
2731
|
+
try {
|
|
2732
|
+
await this.#slackWatcherStarts.get(key);
|
|
2733
|
+
}
|
|
2734
|
+
catch {
|
|
2735
|
+
// The initiator logs Slack watcher startup failures.
|
|
2736
|
+
}
|
|
2737
|
+
return;
|
|
2738
|
+
}
|
|
2739
|
+
const start = this.#postAndWatchSlackEscalationThread(decision, reason);
|
|
2740
|
+
this.#slackWatcherStarts.set(key, start);
|
|
2741
|
+
try {
|
|
2742
|
+
await start;
|
|
2743
|
+
}
|
|
2744
|
+
catch (error) {
|
|
2745
|
+
this.#markSlackWritebackFailure('triage-escalation', error);
|
|
2746
|
+
this.#logger.warn?.(`[factory] failed to establish Slack escalation thread for ${decision.issue.key}`, error);
|
|
2747
|
+
}
|
|
2748
|
+
finally {
|
|
2749
|
+
this.#slackWatcherStarts.delete(key);
|
|
2750
|
+
}
|
|
2751
|
+
}
|
|
2752
|
+
async #postAndWatchSlackEscalationThread(decision, reason) {
|
|
2753
|
+
if (!this.#slack || !this.#config.slack) {
|
|
2754
|
+
return;
|
|
2755
|
+
}
|
|
2756
|
+
const issue = await this.#readIssue(decision.issue.path);
|
|
2757
|
+
const root = await this.#slack.postThread({
|
|
2758
|
+
channel: this.#config.slack.channel,
|
|
2759
|
+
text: [
|
|
2760
|
+
`${decision.issue.key}: factory triage escalation for ${issue?.title ?? decision.issue.key}`,
|
|
2761
|
+
`Reason: ${reason}`,
|
|
2762
|
+
`Question: Please clarify the intended repo/approach or add enough acceptance detail for the factory agent to proceed.`,
|
|
2763
|
+
].join('\n'),
|
|
2764
|
+
});
|
|
2765
|
+
await this.#state.setSlackThread(this.#workspaceId, issueKey(decision.issue), root.threadId);
|
|
2766
|
+
await this.#watchSlackThread({
|
|
2767
|
+
issue: decision.issue,
|
|
2768
|
+
decision,
|
|
2769
|
+
agents: new Map(),
|
|
2770
|
+
invocationIds: new Set(),
|
|
2771
|
+
dryRun: false,
|
|
2772
|
+
}, root.threadId);
|
|
2773
|
+
this.#recordSlackWritebackSuccess('triage-escalation');
|
|
2774
|
+
}
|
|
2775
|
+
async #watchSlackThread(record, threadId) {
|
|
2776
|
+
if (!this.#config.slack) {
|
|
2777
|
+
return;
|
|
2778
|
+
}
|
|
2779
|
+
const key = issueKey(record.issue);
|
|
2780
|
+
if (this.#slackWatchers.has(key)) {
|
|
2781
|
+
return;
|
|
2782
|
+
}
|
|
2783
|
+
const channelDir = this.#config.slack.channel;
|
|
2784
|
+
const messagesPrefix = slackChannelMessagesPrefix(channelDir);
|
|
2785
|
+
const preExistingPaths = new Set();
|
|
2786
|
+
const seenReplies = new Set();
|
|
2787
|
+
const seenReplyMessages = new Set();
|
|
2788
|
+
let missingIdentityLogged = false;
|
|
2789
|
+
let cursor;
|
|
2790
|
+
let stopped = false;
|
|
2791
|
+
let pollTimer;
|
|
2792
|
+
const markPreExisting = async () => {
|
|
2793
|
+
try {
|
|
2794
|
+
const page = await this.#mount.getEvents({ limit: SLACK_REPLY_EVENTS_LIMIT });
|
|
2795
|
+
cursor = page.nextCursor ?? undefined;
|
|
2796
|
+
for (const event of page.events) {
|
|
2797
|
+
if (event.resource.path.startsWith(messagesPrefix)) {
|
|
2798
|
+
preExistingPaths.add(event.resource.path);
|
|
2799
|
+
}
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2802
|
+
catch (error) {
|
|
2803
|
+
this.#logger.warn?.('[factory] unable to seed Slack reply watcher event cursor', error);
|
|
2804
|
+
}
|
|
2805
|
+
};
|
|
2806
|
+
const handle = async (event) => {
|
|
2807
|
+
try {
|
|
2808
|
+
if (stopped || !event.resource.path.startsWith(messagesPrefix)) {
|
|
2809
|
+
return;
|
|
2810
|
+
}
|
|
2811
|
+
const eventKey = eventIdentity(event);
|
|
2812
|
+
if (!eventKey) {
|
|
2813
|
+
if (!missingIdentityLogged) {
|
|
2814
|
+
missingIdentityLogged = true;
|
|
2815
|
+
this.#logger.warn?.('[factory] Slack reply event missing stable identity; falling back to path/content dedupe');
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2818
|
+
if (preExistingPaths.has(event.resource.path)) {
|
|
2819
|
+
return;
|
|
2820
|
+
}
|
|
2821
|
+
const reply = await this.#readSlackReply(event.resource.path);
|
|
2822
|
+
if (!reply || !reply.isThreadReply || reply.threadTs !== threadId || reply.channelDir !== channelDir) {
|
|
2823
|
+
return;
|
|
2824
|
+
}
|
|
2825
|
+
const replyMessageKey = `${reply.threadTs}:${reply.messageTs}`;
|
|
2826
|
+
if (seenReplyMessages.has(replyMessageKey)) {
|
|
2827
|
+
this.#logger.debug?.('[factory] suppressed duplicate Slack reply message', { issue: record.issue.key, path: event.resource.path });
|
|
2828
|
+
return;
|
|
2829
|
+
}
|
|
2830
|
+
const replyKey = `${eventKey ?? event.resource.path}:${stableHash(JSON.stringify(reply.raw))}`;
|
|
2831
|
+
if (seenReplies.has(replyKey)) {
|
|
2832
|
+
this.#logger.debug?.('[factory] suppressed duplicate Slack reply payload', { issue: record.issue.key, path: event.resource.path });
|
|
2833
|
+
return;
|
|
2834
|
+
}
|
|
2835
|
+
seenReplies.add(replyKey);
|
|
2836
|
+
if (reply.isBot) {
|
|
2837
|
+
return;
|
|
2838
|
+
}
|
|
2839
|
+
seenReplyMessages.add(replyMessageKey);
|
|
2840
|
+
await this.#routeSlackAnswerToImplementers(record, reply);
|
|
2841
|
+
}
|
|
2842
|
+
catch (error) {
|
|
2843
|
+
this.#logger.error?.('[factory] failed to handle Slack reply event', error);
|
|
2844
|
+
}
|
|
2845
|
+
};
|
|
2846
|
+
await markPreExisting();
|
|
2847
|
+
let subscription;
|
|
2848
|
+
try {
|
|
2849
|
+
subscription = this.#mount.subscribe([`${messagesPrefix}**`], (event) => {
|
|
2850
|
+
void handle(event);
|
|
2851
|
+
});
|
|
2852
|
+
}
|
|
2853
|
+
catch (error) {
|
|
2854
|
+
this.#logger.warn?.('[factory] Slack reply subscribe failed; relying on event polling', error);
|
|
2855
|
+
}
|
|
2856
|
+
const poll = async () => {
|
|
2857
|
+
if (stopped) {
|
|
2858
|
+
return;
|
|
2859
|
+
}
|
|
2860
|
+
try {
|
|
2861
|
+
const page = await this.#mount.getEvents({ cursor, limit: SLACK_REPLY_EVENTS_LIMIT });
|
|
2862
|
+
cursor = page.nextCursor ?? cursor;
|
|
2863
|
+
for (const event of page.events) {
|
|
2864
|
+
await handle(event);
|
|
2865
|
+
}
|
|
2866
|
+
}
|
|
2867
|
+
catch (error) {
|
|
2868
|
+
this.#logger.warn?.('[factory] Slack reply polling failed', error);
|
|
2869
|
+
}
|
|
2870
|
+
if (!stopped) {
|
|
2871
|
+
pollTimer = setTimeout(() => {
|
|
2872
|
+
void poll();
|
|
2873
|
+
}, SLACK_REPLY_POLL_INTERVAL_MS);
|
|
2874
|
+
pollTimer.unref?.();
|
|
2875
|
+
}
|
|
2876
|
+
};
|
|
2877
|
+
void poll();
|
|
2878
|
+
this.#slackWatchers.set(key, {
|
|
2879
|
+
stop: async () => {
|
|
2880
|
+
stopped = true;
|
|
2881
|
+
if (pollTimer) {
|
|
2882
|
+
clearTimeout(pollTimer);
|
|
2883
|
+
pollTimer = undefined;
|
|
2884
|
+
}
|
|
2885
|
+
await this.#boundedStopTeardown('Slack reply subscription unsubscribe', () => subscription?.unsubscribe());
|
|
2886
|
+
},
|
|
2887
|
+
});
|
|
2888
|
+
}
|
|
2889
|
+
async #stopSlackWatcher(issue) {
|
|
2890
|
+
const key = issueKey(issue);
|
|
2891
|
+
const watcher = this.#slackWatchers.get(key);
|
|
2892
|
+
this.#slackWatchers.delete(key);
|
|
2893
|
+
await this.#state.clearSlackThread(this.#workspaceId, key);
|
|
2894
|
+
await watcher?.stop();
|
|
2895
|
+
}
|
|
2896
|
+
async #readSlackReply(path) {
|
|
2897
|
+
try {
|
|
2898
|
+
const { content } = await this.#mount.readFile(path);
|
|
2899
|
+
return parseSlackReply(path, content, this.#config.slack?.botUserId ?? 'U0B2596R7EZ');
|
|
2900
|
+
}
|
|
2901
|
+
catch (error) {
|
|
2902
|
+
this.#logger.warn?.(`Unable to read Slack reply ${path}`, error);
|
|
2903
|
+
return undefined;
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2906
|
+
async #routeSlackAnswerToImplementers(record, reply) {
|
|
2907
|
+
if (!this.#config.slack || !this.#fleet.sendInput) {
|
|
2908
|
+
return;
|
|
2909
|
+
}
|
|
2910
|
+
const liveRecord = (await this.#batch()).getIssue(record.issue);
|
|
2911
|
+
if (!liveRecord || liveRecord.dryRun) {
|
|
2912
|
+
this.#increment('slackAnswersIgnoredNoInFlight');
|
|
2913
|
+
return;
|
|
2914
|
+
}
|
|
2915
|
+
const text = reply.text.trim();
|
|
2916
|
+
if (!text) {
|
|
2917
|
+
this.#increment('slackAnswersIgnoredEmpty');
|
|
2918
|
+
return;
|
|
2919
|
+
}
|
|
2920
|
+
// Route human replies to the implementers AND the babysitter — once a PR is
|
|
2921
|
+
// open the babysitter is the one chatting with the human about it.
|
|
2922
|
+
const recipients = [...liveRecord.agents.values()]
|
|
2923
|
+
.filter((agent) => agent.spec.role === 'implementer' || agent.spec.role === 'babysitter')
|
|
2924
|
+
.map((agent) => agent.result?.name ?? agent.spec.name)
|
|
2925
|
+
.filter((name) => Boolean(name));
|
|
2926
|
+
if (recipients.length === 0) {
|
|
2927
|
+
this.#increment('slackAnswersIgnoredNoImplementer');
|
|
2928
|
+
return;
|
|
2929
|
+
}
|
|
2930
|
+
const input = slackAnswerInput(liveRecord.issue, text);
|
|
2931
|
+
for (const recipient of new Set(recipients)) {
|
|
2932
|
+
await this.#fleet.sendInput(recipient, input);
|
|
2933
|
+
this.#increment('slackAnswersInjected');
|
|
2934
|
+
}
|
|
2935
|
+
}
|
|
2936
|
+
async #slackDispatchThreadFor(record) {
|
|
2937
|
+
if (!this.#config.slack) {
|
|
2938
|
+
return undefined;
|
|
2939
|
+
}
|
|
2940
|
+
const threadId = await this.#state.getSlackThread(this.#workspaceId, issueKey(record.issue));
|
|
2941
|
+
return threadId ? { channel: this.#config.slack.channel, threadId } : undefined;
|
|
2942
|
+
}
|
|
2943
|
+
async #runCompletionMergeGate(issue) {
|
|
2944
|
+
if (this.#isSyntheticProbeIssue(issue)) {
|
|
2945
|
+
await this.#closeSyntheticProbeIfPresent(issue);
|
|
2946
|
+
return;
|
|
2947
|
+
}
|
|
2948
|
+
if (!isRealLinearIssue(issue)) {
|
|
2949
|
+
this.#logger.warn?.('[factory] merge gate skipped non-real Linear issue', { issue: issue.key });
|
|
2950
|
+
this.#increment('mergeGateSkippedNonReal');
|
|
2951
|
+
return;
|
|
2952
|
+
}
|
|
2953
|
+
if (this.#config.mergePolicy !== 'on-green-with-review') {
|
|
2954
|
+
return;
|
|
2955
|
+
}
|
|
2956
|
+
const pr = await this.#probePrResolver(issue);
|
|
2957
|
+
if (!pr) {
|
|
2958
|
+
this.#logger.warn?.('[factory] merge gate found no PR for real issue', { issue: issue.key });
|
|
2959
|
+
this.#increment('mergeGateMissingPr');
|
|
2960
|
+
return;
|
|
2961
|
+
}
|
|
2962
|
+
const ready = await this.#waitForMergeReady(pr);
|
|
2963
|
+
const headSha = ready?.live.headRefOid;
|
|
2964
|
+
if (!ready || !headSha) {
|
|
2965
|
+
this.#increment('mergeGateNotMerged');
|
|
2966
|
+
return;
|
|
2967
|
+
}
|
|
2968
|
+
const result = await this.#mergeGate.merge({
|
|
2969
|
+
repo: pr.repo,
|
|
2970
|
+
number: pr.prNumber,
|
|
2971
|
+
expectedHeadSha: headSha,
|
|
2972
|
+
});
|
|
2973
|
+
if (!result.merged) {
|
|
2974
|
+
this.#logger.warn?.('[factory] merge gate aborted guarded merge', {
|
|
2975
|
+
issue: issue.key,
|
|
2976
|
+
repo: pr.repo,
|
|
2977
|
+
prNumber: pr.prNumber,
|
|
2978
|
+
headSha,
|
|
2979
|
+
reason: result.reason,
|
|
2980
|
+
});
|
|
2981
|
+
this.#increment('mergeGateMergeAborted');
|
|
2982
|
+
return;
|
|
2983
|
+
}
|
|
2984
|
+
this.#logger.info?.('[factory] merge gate merged PR', {
|
|
2985
|
+
issue: issue.key,
|
|
2986
|
+
repo: pr.repo,
|
|
2987
|
+
prNumber: pr.prNumber,
|
|
2988
|
+
headSha,
|
|
2989
|
+
});
|
|
2990
|
+
this.#increment('mergeGateMerged');
|
|
2991
|
+
}
|
|
2992
|
+
async #closeSyntheticProbeIfPresent(issue) {
|
|
2993
|
+
const probe = this.#customProbePrResolver
|
|
2994
|
+
? await this.#probePrResolver(issue)
|
|
2995
|
+
: await this.#resolveIssuePr(issue, {
|
|
2996
|
+
titleMarker: FACTORY_E2E_MARKER,
|
|
2997
|
+
});
|
|
2998
|
+
if (!probe) {
|
|
2999
|
+
return;
|
|
3000
|
+
}
|
|
3001
|
+
await this.#probeCloser({
|
|
3002
|
+
repo: probe.repo,
|
|
3003
|
+
prNumber: probe.prNumber,
|
|
3004
|
+
expectedIssueKey: issue.key,
|
|
3005
|
+
requireTitleMarker: false,
|
|
3006
|
+
});
|
|
3007
|
+
this.#increment('mergeGateSyntheticClosed');
|
|
3008
|
+
}
|
|
3009
|
+
async #waitForMergeReady(pr) {
|
|
3010
|
+
let lastReason = 'not checked';
|
|
3011
|
+
for (let attempt = 1; attempt <= MERGE_GATE_MAX_ATTEMPTS; attempt += 1) {
|
|
3012
|
+
const verdict = await this.#mergeGate.check({ repo: pr.repo, number: pr.prNumber });
|
|
3013
|
+
lastReason = verdict.reason;
|
|
3014
|
+
if (verdict.ready && verdict.live.headRefOid) {
|
|
3015
|
+
return verdict;
|
|
3016
|
+
}
|
|
3017
|
+
if (attempt < MERGE_GATE_MAX_ATTEMPTS) {
|
|
3018
|
+
await this.#clock.sleep(MERGE_GATE_POLL_DELAY_MS);
|
|
3019
|
+
}
|
|
3020
|
+
}
|
|
3021
|
+
this.#logger.warn?.('[factory] merge gate left PR open; readiness timeout', {
|
|
3022
|
+
repo: pr.repo,
|
|
3023
|
+
prNumber: pr.prNumber,
|
|
3024
|
+
attempts: MERGE_GATE_MAX_ATTEMPTS,
|
|
3025
|
+
reason: lastReason,
|
|
3026
|
+
});
|
|
3027
|
+
return undefined;
|
|
3028
|
+
}
|
|
3029
|
+
#isSyntheticProbeIssue(issue) {
|
|
3030
|
+
return hasTitlePrefix(issue.title, FACTORY_E2E_MARKER);
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
3033
|
+
export function parseLinearIssue(path, content) {
|
|
3034
|
+
const parsed = parseJsonContent(content);
|
|
3035
|
+
const payload = wrappedPayload(parsed);
|
|
3036
|
+
const wrapper = asRecord(parsed) ?? {};
|
|
3037
|
+
const state = asRecord(payload.state);
|
|
3038
|
+
const labels = Array.isArray(payload.labels)
|
|
3039
|
+
? payload.labels.map(labelName).filter((label) => Boolean(label))
|
|
3040
|
+
: [];
|
|
3041
|
+
const project = recordName(payload.project);
|
|
3042
|
+
const team = recordName(payload.team);
|
|
3043
|
+
const assignee = recordName(payload.assignee);
|
|
3044
|
+
const key = stringValue(payload.identifier) ?? keyFromPath(path);
|
|
3045
|
+
const uuid = stringValue(payload.id) ?? stringValue(wrapper.objectId) ?? uuidFromPath(path) ?? key;
|
|
3046
|
+
const stateName = stringValue(state?.name) ?? stringValue(payload.state_name);
|
|
3047
|
+
// Resolve the issue's state UUID from the payload only. The factory maps that
|
|
3048
|
+
// UUID to a role via the runtime state resolution (no hardcoded name->id), so
|
|
3049
|
+
// an unknown state simply matches no role rather than being mis-routed.
|
|
3050
|
+
const stateId = stringValue(payload.stateId) ?? stringValue(state?.id) ?? '';
|
|
3051
|
+
return {
|
|
3052
|
+
uuid,
|
|
3053
|
+
key,
|
|
3054
|
+
title: stringValue(payload.title) ?? '',
|
|
3055
|
+
description: stringValue(payload.description) ?? '',
|
|
3056
|
+
stateId,
|
|
3057
|
+
state: state || stateName ? { name: stateName ?? '' } : undefined,
|
|
3058
|
+
labels,
|
|
3059
|
+
project,
|
|
3060
|
+
team,
|
|
3061
|
+
assignee,
|
|
3062
|
+
path,
|
|
3063
|
+
raw: asRecord(parsed) ?? payload,
|
|
3064
|
+
};
|
|
3065
|
+
}
|
|
3066
|
+
export function parseGithubIssue(path, content) {
|
|
3067
|
+
const parsed = parseJsonContent(content);
|
|
3068
|
+
const payload = wrappedPayload(parsed);
|
|
3069
|
+
const pathParts = githubIssuePathParts(path);
|
|
3070
|
+
const repository = asRecord(payload.repository);
|
|
3071
|
+
const owner = stringValue(asRecord(repository?.owner)?.login) ?? stringValue(asRecord(repository?.owner)?.name) ?? pathParts?.owner;
|
|
3072
|
+
const repoName = stringValue(repository?.name) ?? pathParts?.repo;
|
|
3073
|
+
const number = numberValue(payload.number) ?? pathParts?.number;
|
|
3074
|
+
const url = stringValue(payload.html_url) ?? stringValue(payload.url) ?? stringValue(payload.issue_url);
|
|
3075
|
+
if (!owner || !repoName || typeof number !== 'number' || !url) {
|
|
3076
|
+
throw new Error(`GitHub issue ${path} is missing owner, repo, number, or url`);
|
|
3077
|
+
}
|
|
3078
|
+
return {
|
|
3079
|
+
owner,
|
|
3080
|
+
repoName,
|
|
3081
|
+
repo: `${owner}/${repoName}`,
|
|
3082
|
+
number,
|
|
3083
|
+
title: stringValue(payload.title) ?? '',
|
|
3084
|
+
body: stringValue(payload.body) ?? '',
|
|
3085
|
+
url,
|
|
3086
|
+
state: (stringValue(payload.state) ?? '').toLowerCase(),
|
|
3087
|
+
labels: Array.isArray(payload.labels)
|
|
3088
|
+
? payload.labels.map(labelName).filter((label) => Boolean(label))
|
|
3089
|
+
: [],
|
|
3090
|
+
path,
|
|
3091
|
+
raw: asRecord(parsed) ?? payload,
|
|
3092
|
+
};
|
|
3093
|
+
}
|
|
3094
|
+
export async function readFactoryLoopHeartbeat(path = DEFAULT_FACTORY_LOOP_HEARTBEAT_PATH) {
|
|
3095
|
+
try {
|
|
3096
|
+
return parseJsonContent(await readFile(path, 'utf8'));
|
|
3097
|
+
}
|
|
3098
|
+
catch {
|
|
3099
|
+
return undefined;
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
export function checkFactoryLoopLiveness(heartbeat, opts = {}) {
|
|
3103
|
+
if (!heartbeat) {
|
|
3104
|
+
return { ok: false, stale: true, reason: 'heartbeat missing' };
|
|
3105
|
+
}
|
|
3106
|
+
const nowMs = opts.nowMs ?? Date.now();
|
|
3107
|
+
const staleMs = opts.staleMs ?? 60_000;
|
|
3108
|
+
const ageMs = Math.max(0, nowMs - heartbeat.updatedAtMs);
|
|
3109
|
+
const stale = ageMs > staleMs;
|
|
3110
|
+
if (stale) {
|
|
3111
|
+
return { ok: false, stale: true, ageMs, heartbeat, reason: 'heartbeat stale' };
|
|
3112
|
+
}
|
|
3113
|
+
if (heartbeat.status === 'stopping') {
|
|
3114
|
+
return { ok: false, stale: false, ageMs, heartbeat, reason: 'loop stopping' };
|
|
3115
|
+
}
|
|
3116
|
+
return { ok: true, stale: false, ageMs, heartbeat };
|
|
3117
|
+
}
|
|
3118
|
+
export function isRealLinearIssue(issue) {
|
|
3119
|
+
const payload = wrappedPayload(issue.raw);
|
|
3120
|
+
const identifier = stringValue(payload.identifier) ?? issue.key;
|
|
3121
|
+
return identifier === issue.key &&
|
|
3122
|
+
/^[A-Z]+-\d+$/u.test(identifier) &&
|
|
3123
|
+
typeof payload.url === 'string' &&
|
|
3124
|
+
payload.url.length > 0;
|
|
3125
|
+
}
|
|
3126
|
+
const issueRef = (issue) => ({ uuid: issue.uuid, key: issue.key, path: issue.path });
|
|
3127
|
+
const pidsFromSpawnResult = (result) => {
|
|
3128
|
+
const pids = new Set();
|
|
3129
|
+
for (const pid of result?.pids ?? []) {
|
|
3130
|
+
if (Number.isInteger(pid) && pid > 0)
|
|
3131
|
+
pids.add(pid);
|
|
3132
|
+
}
|
|
3133
|
+
if (Number.isInteger(result?.pid) && result.pid > 0) {
|
|
3134
|
+
pids.add(result.pid);
|
|
3135
|
+
}
|
|
3136
|
+
return [...pids].sort((a, b) => a - b);
|
|
3137
|
+
};
|
|
3138
|
+
const dispatchComment = (decision, agents) => [
|
|
3139
|
+
`Factory dispatch for ${decision.issue.key}`,
|
|
3140
|
+
`Implementers: ${agents.filter((agent) => agent.role === 'implementer').map((agent) => agent.name).join(', ') || 'none'}`,
|
|
3141
|
+
`Reviewer: ${agents.find((agent) => agent.role === 'reviewer')?.name ?? 'none'}`,
|
|
3142
|
+
].join('\n');
|
|
3143
|
+
function labelDerivedDispatchDecision(liveIssue, decision, config) {
|
|
3144
|
+
const routesByLabel = labelRoutesForIssue(liveIssue, config);
|
|
3145
|
+
if (routesByLabel.labels.length === 0) {
|
|
3146
|
+
return {
|
|
3147
|
+
ok: false,
|
|
3148
|
+
reason: 'no-labels',
|
|
3149
|
+
offendingLabels: [],
|
|
3150
|
+
};
|
|
3151
|
+
}
|
|
3152
|
+
if (routesByLabel.routes.length === 0) {
|
|
3153
|
+
return {
|
|
3154
|
+
ok: false,
|
|
3155
|
+
reason: 'unmapped-labels',
|
|
3156
|
+
offendingLabels: routesByLabel.offendingLabels.length > 0 ? routesByLabel.offendingLabels : routesByLabel.labels,
|
|
3157
|
+
};
|
|
3158
|
+
}
|
|
3159
|
+
const maxImplementers = Math.min(config.triage.maxImplementers, MAX_LABEL_IMPLEMENTERS);
|
|
3160
|
+
if (routesByLabel.routes.length > maxImplementers) {
|
|
3161
|
+
return {
|
|
3162
|
+
ok: false,
|
|
3163
|
+
reason: 'too-many-labels',
|
|
3164
|
+
offendingLabels: routesByLabel.routes.map((assignment) => assignment.slug),
|
|
3165
|
+
maxImplementers,
|
|
3166
|
+
};
|
|
3167
|
+
}
|
|
3168
|
+
const implementers = routesByLabel.routes.map(({ slug, route }) => routeImplementerSpec(liveIssue, config, slug, route));
|
|
3169
|
+
return {
|
|
3170
|
+
ok: true,
|
|
3171
|
+
decision: {
|
|
3172
|
+
...decision,
|
|
3173
|
+
routes: routesByLabel.routes.map(({ route }) => route),
|
|
3174
|
+
scope: implementers.length >= 2 ? 'team' : 'single',
|
|
3175
|
+
implementers,
|
|
3176
|
+
reviewer: routeReviewerSpec(liveIssue, config, routesByLabel.routes[0].route, decision.reviewer),
|
|
3177
|
+
},
|
|
3178
|
+
};
|
|
3179
|
+
}
|
|
3180
|
+
function labelRoutesForIssue(issue, config) {
|
|
3181
|
+
const labels = uniqueNormalizedLabels(issue.labels);
|
|
3182
|
+
const routes = [];
|
|
3183
|
+
const offendingLabels = [];
|
|
3184
|
+
const seenRepos = new Set();
|
|
3185
|
+
for (const label of labels) {
|
|
3186
|
+
const entry = findLabelRoute(config.repos.byLabel, label);
|
|
3187
|
+
if (!entry) {
|
|
3188
|
+
offendingLabels.push(label);
|
|
3189
|
+
continue;
|
|
3190
|
+
}
|
|
3191
|
+
const repo = entry.repo;
|
|
3192
|
+
if (seenRepos.has(repo)) {
|
|
3193
|
+
continue;
|
|
3194
|
+
}
|
|
3195
|
+
seenRepos.add(repo);
|
|
3196
|
+
routes.push({
|
|
3197
|
+
slug: entry.label,
|
|
3198
|
+
route: {
|
|
3199
|
+
repo,
|
|
3200
|
+
clonePath: config.repos.clonePaths[repo],
|
|
3201
|
+
rationale: `Label "${entry.label}" routes to ${repo}.`,
|
|
3202
|
+
},
|
|
3203
|
+
});
|
|
3204
|
+
}
|
|
3205
|
+
return { labels, offendingLabels, routes };
|
|
3206
|
+
}
|
|
3207
|
+
function routeImplementerSpec(issue, config, slug, route) {
|
|
3208
|
+
return {
|
|
3209
|
+
name: `${agentBaseName(issue)}-impl-${sanitizeAgentSlug(slug)}`,
|
|
3210
|
+
role: 'implementer',
|
|
3211
|
+
capability: 'spawn:codex',
|
|
3212
|
+
model: config.models.implementer,
|
|
3213
|
+
task: taskForDispatch(issue, route, 'implementer'),
|
|
3214
|
+
repo: route.repo,
|
|
3215
|
+
clonePath: route.clonePath,
|
|
3216
|
+
node: 'self',
|
|
3217
|
+
};
|
|
3218
|
+
}
|
|
3219
|
+
function routeReviewerSpec(issue, config, route, reviewer) {
|
|
3220
|
+
return {
|
|
3221
|
+
...reviewer,
|
|
3222
|
+
name: `${agentBaseName(issue)}-review`,
|
|
3223
|
+
role: 'reviewer',
|
|
3224
|
+
capability: reviewer.capability ?? 'spawn:claude',
|
|
3225
|
+
model: reviewer.model ?? config.models.reviewer,
|
|
3226
|
+
task: taskForDispatch(issue, route, 'reviewer'),
|
|
3227
|
+
repo: route.repo,
|
|
3228
|
+
clonePath: route.clonePath,
|
|
3229
|
+
node: reviewer.node ?? 'self',
|
|
3230
|
+
};
|
|
3231
|
+
}
|
|
3232
|
+
function labelDispatchFailureSignature(resolution) {
|
|
3233
|
+
return `${resolution.reason}:${[...resolution.offendingLabels].sort().join(',')}`;
|
|
3234
|
+
}
|
|
3235
|
+
function labelDispatchFailureComment(issue, resolution) {
|
|
3236
|
+
const lines = [`Factory dispatch for ${issue.key} skipped`];
|
|
3237
|
+
if (resolution.reason === 'no-labels') {
|
|
3238
|
+
lines.push('No Linear labels were present.');
|
|
3239
|
+
lines.push('Add one repo label from factory.config.json repos.byLabel, then move the issue back to Ready for Agent.');
|
|
3240
|
+
}
|
|
3241
|
+
else if (resolution.reason === 'too-many-labels') {
|
|
3242
|
+
lines.push(`Too many repo labels matched dispatch: ${resolution.offendingLabels.join(', ')}.`);
|
|
3243
|
+
lines.push(`Dispatch is capped at ${resolution.maxImplementers} implementer(s); scope the issue or raise triage.maxImplementers within the dispatch cap.`);
|
|
3244
|
+
}
|
|
3245
|
+
else if (resolution.offendingLabels.length > 0) {
|
|
3246
|
+
lines.push(`No repo labels matched factory.config.json repos.byLabel. Unmapped label(s): ${resolution.offendingLabels.join(', ')}.`);
|
|
3247
|
+
lines.push('Update the labels or factory.config.json repos.byLabel, then move the issue back to Ready for Agent.');
|
|
3248
|
+
}
|
|
3249
|
+
else {
|
|
3250
|
+
lines.push('No repo labels matched factory.config.json repos.byLabel.');
|
|
3251
|
+
lines.push('Update the labels or factory.config.json repos.byLabel, then move the issue back to Ready for Agent.');
|
|
3252
|
+
}
|
|
3253
|
+
return lines.join('\n');
|
|
3254
|
+
}
|
|
3255
|
+
function findLabelRoute(map, label) {
|
|
3256
|
+
const exact = map[label];
|
|
3257
|
+
if (exact) {
|
|
3258
|
+
return { label, repo: exact };
|
|
3259
|
+
}
|
|
3260
|
+
const normalized = label.toLowerCase();
|
|
3261
|
+
const entry = Object.entries(map).find(([candidate]) => candidate.toLowerCase() === normalized);
|
|
3262
|
+
return entry ? { label: entry[0], repo: entry[1] } : undefined;
|
|
3263
|
+
}
|
|
3264
|
+
function uniqueNormalizedLabels(labels) {
|
|
3265
|
+
const seen = new Set();
|
|
3266
|
+
const unique = [];
|
|
3267
|
+
for (const label of labels) {
|
|
3268
|
+
const trimmed = label.trim();
|
|
3269
|
+
const normalized = trimmed.toLowerCase();
|
|
3270
|
+
if (!trimmed || seen.has(normalized)) {
|
|
3271
|
+
continue;
|
|
3272
|
+
}
|
|
3273
|
+
seen.add(normalized);
|
|
3274
|
+
unique.push(trimmed);
|
|
3275
|
+
}
|
|
3276
|
+
return unique;
|
|
3277
|
+
}
|
|
3278
|
+
function taskForDispatch(issue, route, role) {
|
|
3279
|
+
const verb = role === 'implementer' ? 'Implement' : role === 'babysitter' ? 'Babysit the PR for' : 'Review';
|
|
3280
|
+
return [
|
|
3281
|
+
`${verb} ${issue.key}: ${issue.title}`,
|
|
3282
|
+
`Repo: ${route.repo}`,
|
|
3283
|
+
`Route rationale: ${route.rationale}`,
|
|
3284
|
+
issue.description,
|
|
3285
|
+
].join('\n\n');
|
|
3286
|
+
}
|
|
3287
|
+
function agentBaseName(issue) {
|
|
3288
|
+
const number = issue.key.match(/\d+/)?.[0] ?? sanitizeAgentSlug(issue.key);
|
|
3289
|
+
return `ar-${number}`;
|
|
3290
|
+
}
|
|
3291
|
+
function sanitizeAgentSlug(slug) {
|
|
3292
|
+
return slug.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'scope';
|
|
3293
|
+
}
|
|
3294
|
+
const templateIssueFromRecord = (record, issue) => ({
|
|
3295
|
+
key: issue?.key ?? record.issue.key,
|
|
3296
|
+
title: issue?.title ?? record.issue.key,
|
|
3297
|
+
description: issue?.description ?? '',
|
|
3298
|
+
});
|
|
3299
|
+
const routeForImplementer = (record, spec) => {
|
|
3300
|
+
const route = record.decision.routes.find((candidate) => candidate.repo === spec.repo && candidate.clonePath === spec.clonePath) ?? record.decision.routes.find((candidate) => candidate.repo === spec.repo);
|
|
3301
|
+
return {
|
|
3302
|
+
repo: spec.repo,
|
|
3303
|
+
clonePath: spec.clonePath,
|
|
3304
|
+
rationale: route?.rationale,
|
|
3305
|
+
};
|
|
3306
|
+
};
|
|
3307
|
+
const repoMapFromConfig = (config) => {
|
|
3308
|
+
const repos = new Set([
|
|
3309
|
+
...Object.values(config.repos.byLabel),
|
|
3310
|
+
...Object.values(config.repos.byProject),
|
|
3311
|
+
...config.repos.keywordRules.map((rule) => rule.repo),
|
|
3312
|
+
config.repos.default,
|
|
3313
|
+
].filter((repo) => Boolean(repo)));
|
|
3314
|
+
return [...repos].map((repo) => ({
|
|
3315
|
+
repo,
|
|
3316
|
+
clonePath: config.repos.clonePaths[repo],
|
|
3317
|
+
source: 'default',
|
|
3318
|
+
}));
|
|
3319
|
+
};
|
|
3320
|
+
export const githubIssuePathParts = (path) => {
|
|
3321
|
+
// Canonical GitHub relayfile issue entries are nested as
|
|
3322
|
+
// /github/repos/<owner>/<repo>/issues/<number>__<slug>/meta.json. Keep the
|
|
3323
|
+
// legacy flat and by-id forms for older mount state, and accept metadata.json
|
|
3324
|
+
// as a read-only compatibility alias for historical local mount snapshots.
|
|
3325
|
+
const match = path.match(/^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/(?:(?:by-id\/)?(\d+)\.json|(\d+)(?:__([^/]+))?\/(?:meta|metadata)\.json)$/u);
|
|
3326
|
+
if (!match) {
|
|
3327
|
+
return undefined;
|
|
3328
|
+
}
|
|
3329
|
+
const number = Number(match[3] ?? match[4]);
|
|
3330
|
+
return {
|
|
3331
|
+
owner: match[1],
|
|
3332
|
+
repo: match[2],
|
|
3333
|
+
number,
|
|
3334
|
+
slug: match[5],
|
|
3335
|
+
};
|
|
3336
|
+
};
|
|
3337
|
+
const githubIssueReadCandidatePaths = (path) => {
|
|
3338
|
+
if (path.endsWith('/')) {
|
|
3339
|
+
return [`${path}meta.json`, `${path}metadata.json`];
|
|
3340
|
+
}
|
|
3341
|
+
if (path.endsWith('/meta.json')) {
|
|
3342
|
+
return [path, path.replace(/\/meta\.json$/u, '/metadata.json')];
|
|
3343
|
+
}
|
|
3344
|
+
if (path.endsWith('/metadata.json')) {
|
|
3345
|
+
return [path, path.replace(/\/metadata\.json$/u, '/meta.json')];
|
|
3346
|
+
}
|
|
3347
|
+
if (githubIssuePathParts(path)) {
|
|
3348
|
+
return [path];
|
|
3349
|
+
}
|
|
3350
|
+
if (githubIssueDirectoryPathParts(path)) {
|
|
3351
|
+
// meta.json is the canonical relayfile GitHub issue basename. metadata.json
|
|
3352
|
+
// remains a legacy read fallback for older local mount-state snapshots.
|
|
3353
|
+
return [`${path}/meta.json`, `${path}/metadata.json`];
|
|
3354
|
+
}
|
|
3355
|
+
return [path];
|
|
3356
|
+
};
|
|
3357
|
+
const githubIssueDirectoryPathParts = (path) => {
|
|
3358
|
+
const match = path.match(/^\/github\/repos\/([^/]+)\/([^/]+)\/issues\/(\d+)(?:__([^/]+))?$/u);
|
|
3359
|
+
if (!match) {
|
|
3360
|
+
return undefined;
|
|
3361
|
+
}
|
|
3362
|
+
return {
|
|
3363
|
+
owner: match[1],
|
|
3364
|
+
repo: match[2],
|
|
3365
|
+
number: Number(match[3]),
|
|
3366
|
+
slug: match[4],
|
|
3367
|
+
};
|
|
3368
|
+
};
|
|
3369
|
+
const githubIssueHasFactoryLabel = (issue) => issue.labels.some((label) => label.toLowerCase() === GITHUB_FACTORY_LABEL);
|
|
3370
|
+
const githubIssueIsClosed = (issue) => issue.state === 'closed';
|
|
3371
|
+
const githubIssueMirrorPayload = (issue, repoLabel, config, readyForAgentStateId) => ({
|
|
3372
|
+
id: githubIssueMirrorId(issue),
|
|
3373
|
+
title: `${GITHUB_MIRROR_TITLE_PREFIX} ${issue.title}`.trim(),
|
|
3374
|
+
description: githubIssueMirrorDescription(issue),
|
|
3375
|
+
stateId: readyForAgentStateId,
|
|
3376
|
+
labels: [{ name: repoLabel }],
|
|
3377
|
+
team: { key: config.safety.requireTeamKey },
|
|
3378
|
+
source: {
|
|
3379
|
+
provider: 'github',
|
|
3380
|
+
owner: issue.owner,
|
|
3381
|
+
repo: issue.repoName,
|
|
3382
|
+
number: issue.number,
|
|
3383
|
+
url: issue.url,
|
|
3384
|
+
path: issue.path,
|
|
3385
|
+
},
|
|
3386
|
+
});
|
|
3387
|
+
const githubIssueMirrorDescription = (issue) => {
|
|
3388
|
+
const body = issue.body.trim();
|
|
3389
|
+
const source = `${GITHUB_MIRROR_SOURCE_PREFIX}${issue.url}`;
|
|
3390
|
+
return body ? `${body}\n\n${source}` : source;
|
|
3391
|
+
};
|
|
3392
|
+
const githubIssueMirrorId = (issue) => `github-${stableHash(`${issue.repo.toLowerCase()}#${issue.number}:${issue.url}`)}`;
|
|
3393
|
+
const githubIssueMirrorDraftPath = (issue) => `/linear/issues/factory-create-${githubIssueMirrorId(issue)}.json`;
|
|
3394
|
+
const linearIssueMirrorsGithubIssue = (issue, ghIssue) => {
|
|
3395
|
+
const payload = wrappedPayload(issue.raw);
|
|
3396
|
+
const source = asRecord(payload.source);
|
|
3397
|
+
if (stringValue(source?.provider)?.toLowerCase() === 'github' &&
|
|
3398
|
+
stringValue(source?.url) === ghIssue.url) {
|
|
3399
|
+
return true;
|
|
3400
|
+
}
|
|
3401
|
+
if (stringValue(source?.provider)?.toLowerCase() === 'github' &&
|
|
3402
|
+
stringValue(source?.owner)?.toLowerCase() === ghIssue.owner.toLowerCase() &&
|
|
3403
|
+
stringValue(source?.repo)?.toLowerCase() === ghIssue.repoName.toLowerCase() &&
|
|
3404
|
+
numberValue(source?.number) === ghIssue.number) {
|
|
3405
|
+
return true;
|
|
3406
|
+
}
|
|
3407
|
+
return issue.description
|
|
3408
|
+
.split(/\r?\n/u)
|
|
3409
|
+
.some((line) => line.trim() === `${GITHUB_MIRROR_SOURCE_PREFIX}${ghIssue.url}`);
|
|
3410
|
+
};
|
|
3411
|
+
const resolveIssuePrFromMount = async (mount, config, issue, opts = {}) => {
|
|
3412
|
+
const candidates = [];
|
|
3413
|
+
for (const repo of reposFromConfig(config)) {
|
|
3414
|
+
for (const path of await mount.listTree(githubPullRoot(repo))) {
|
|
3415
|
+
if (!path.endsWith('.json'))
|
|
3416
|
+
continue;
|
|
3417
|
+
const pr = await readProbePrCandidate(mount, path);
|
|
3418
|
+
const score = pr
|
|
3419
|
+
? issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts)
|
|
3420
|
+
: 0;
|
|
3421
|
+
if (!pr || score <= 0)
|
|
3422
|
+
continue;
|
|
3423
|
+
candidates.push({ repo, prNumber: pr.number, draft: pr.draft, score });
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
return candidates.sort((a, b) => b.score - a.score || b.prNumber - a.prNumber)[0];
|
|
3427
|
+
};
|
|
3428
|
+
const resolveIssuePrFromGh = async (run, config, issue, opts = {}, logger) => {
|
|
3429
|
+
const candidates = [];
|
|
3430
|
+
for (const repo of reposFromConfig(config)) {
|
|
3431
|
+
let payload;
|
|
3432
|
+
try {
|
|
3433
|
+
const result = await run([
|
|
3434
|
+
'pr',
|
|
3435
|
+
'list',
|
|
3436
|
+
'--repo',
|
|
3437
|
+
repo,
|
|
3438
|
+
'--state',
|
|
3439
|
+
'all',
|
|
3440
|
+
'--json',
|
|
3441
|
+
'number,title,body,headRefName,isDraft,state',
|
|
3442
|
+
'--limit',
|
|
3443
|
+
String(PROBE_PR_GH_CANDIDATE_LIMIT),
|
|
3444
|
+
]);
|
|
3445
|
+
if (!result.stdout.trim()) {
|
|
3446
|
+
logger?.warn?.('[factory] gh PR resolver returned empty output', { issue: issue.key, repo });
|
|
3447
|
+
continue;
|
|
3448
|
+
}
|
|
3449
|
+
payload = parseJsonContent(result.stdout);
|
|
3450
|
+
}
|
|
3451
|
+
catch (error) {
|
|
3452
|
+
logger?.warn?.('[factory] gh PR resolver failed', { issue: issue.key, repo, error });
|
|
3453
|
+
continue;
|
|
3454
|
+
}
|
|
3455
|
+
if (!Array.isArray(payload)) {
|
|
3456
|
+
logger?.warn?.('[factory] gh PR resolver returned non-array payload', { issue: issue.key, repo });
|
|
3457
|
+
continue;
|
|
3458
|
+
}
|
|
3459
|
+
if (payload.length >= PROBE_PR_GH_CANDIDATE_LIMIT) {
|
|
3460
|
+
logger?.warn?.('[factory] gh PR resolver hit candidate limit', { issue: issue.key, repo, limit: PROBE_PR_GH_CANDIDATE_LIMIT });
|
|
3461
|
+
}
|
|
3462
|
+
for (const entry of payload) {
|
|
3463
|
+
const pr = ghProbePrCandidate(entry);
|
|
3464
|
+
if (!pr || !containsIssueKey(pr.headRef, issue.key))
|
|
3465
|
+
continue;
|
|
3466
|
+
const score = issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts);
|
|
3467
|
+
if (score <= 0)
|
|
3468
|
+
continue;
|
|
3469
|
+
candidates.push({
|
|
3470
|
+
repo,
|
|
3471
|
+
prNumber: pr.number,
|
|
3472
|
+
draft: pr.draft,
|
|
3473
|
+
score,
|
|
3474
|
+
open: normalizePrState(pr.state) === 'OPEN',
|
|
3475
|
+
});
|
|
3476
|
+
}
|
|
3477
|
+
}
|
|
3478
|
+
return candidates.sort((a, b) => b.score - a.score ||
|
|
3479
|
+
Number(b.open) - Number(a.open) ||
|
|
3480
|
+
b.prNumber - a.prNumber)[0];
|
|
3481
|
+
};
|
|
3482
|
+
const reposFromConfig = (config) => {
|
|
3483
|
+
const repos = new Set([
|
|
3484
|
+
...Object.values(config.repos.byLabel),
|
|
3485
|
+
...Object.values(config.repos.byProject),
|
|
3486
|
+
...config.repos.keywordRules.map((rule) => rule.repo),
|
|
3487
|
+
config.repos.default,
|
|
3488
|
+
].filter((repo) => Boolean(repo)));
|
|
3489
|
+
return [...repos];
|
|
3490
|
+
};
|
|
3491
|
+
const githubPullRoot = (repo) => {
|
|
3492
|
+
const [owner, name] = repo.split('/');
|
|
3493
|
+
return owner && name ? `/github/repos/${owner}__${name}/pulls/by-id/` : `/github/repos/${repo}/pulls/by-id/`;
|
|
3494
|
+
};
|
|
3495
|
+
const readProbePrCandidate = async (mount, path) => {
|
|
3496
|
+
try {
|
|
3497
|
+
const payload = wrappedPayload((await mount.readFile(path)).content);
|
|
3498
|
+
const number = typeof payload.number === 'number'
|
|
3499
|
+
? payload.number
|
|
3500
|
+
: Number(path.split('/').at(-1)?.replace(/\.json$/, ''));
|
|
3501
|
+
if (!Number.isInteger(number) || number <= 0)
|
|
3502
|
+
return undefined;
|
|
3503
|
+
return {
|
|
3504
|
+
number,
|
|
3505
|
+
title: stringValue(payload.title) ?? '',
|
|
3506
|
+
body: stringValue(payload.body) ?? '',
|
|
3507
|
+
headRef: refName(payload.headRef) ?? refName(payload.head) ?? stringValue(payload.head_ref) ?? '',
|
|
3508
|
+
draft: booleanValue(payload.isDraft) ?? booleanValue(payload.draft),
|
|
3509
|
+
};
|
|
3510
|
+
}
|
|
3511
|
+
catch {
|
|
3512
|
+
return undefined;
|
|
3513
|
+
}
|
|
3514
|
+
};
|
|
3515
|
+
const ghProbePrCandidate = (value) => {
|
|
3516
|
+
const payload = asRecord(value);
|
|
3517
|
+
if (!payload)
|
|
3518
|
+
return undefined;
|
|
3519
|
+
const number = numberValue(payload.number);
|
|
3520
|
+
if (typeof number !== 'number' || !Number.isInteger(number) || number <= 0)
|
|
3521
|
+
return undefined;
|
|
3522
|
+
return {
|
|
3523
|
+
number,
|
|
3524
|
+
title: stringValue(payload.title) ?? '',
|
|
3525
|
+
body: stringValue(payload.body) ?? '',
|
|
3526
|
+
headRef: stringValue(payload.headRefName) ?? '',
|
|
3527
|
+
draft: booleanValue(payload.isDraft),
|
|
3528
|
+
state: stringValue(payload.state),
|
|
3529
|
+
};
|
|
3530
|
+
};
|
|
3531
|
+
const issuePrMatchScore = (pr, issue, marker, opts = {}) => {
|
|
3532
|
+
if (opts.requireTitleMarker && !hasTitlePrefix(pr.title, marker))
|
|
3533
|
+
return 0;
|
|
3534
|
+
if (containsIssueKey(pr.headRef, issue.key))
|
|
3535
|
+
return 30;
|
|
3536
|
+
if (containsIssueKey(pr.title, issue.key))
|
|
3537
|
+
return 20;
|
|
3538
|
+
if (containsExplicitIssueReference(pr.body, issue.key))
|
|
3539
|
+
return 10;
|
|
3540
|
+
return 0;
|
|
3541
|
+
};
|
|
3542
|
+
const hasTitlePrefix = (title, marker) => title === marker || title.startsWith(`${marker} `);
|
|
3543
|
+
const normalizePrState = (state) => state?.toUpperCase();
|
|
3544
|
+
const failClosedGhRunner = async () => ({ stdout: '[]' });
|
|
3545
|
+
const ISSUE_KEY_PATTERN = /^[A-Z]+-\d+$/u;
|
|
3546
|
+
const isIssuePathUnderRoot = (path) => path.startsWith(`${ISSUE_ROOT}/`) && path.endsWith('.json');
|
|
3547
|
+
const isIssueFilePath = (path) => isIssuePathUnderRoot(path) &&
|
|
3548
|
+
!path.includes('/comments/') &&
|
|
3549
|
+
!path.includes('/by-state/') &&
|
|
3550
|
+
!path.includes('/by-id/') &&
|
|
3551
|
+
isCanonicalIssueFileBasename(path.split('/').at(-1) ?? '');
|
|
3552
|
+
const isLinearIssueMirrorCandidatePath = (path) => isIssueFilePath(path) || /^\/linear\/issues\/factory-create-[^/]+\.json$/u.test(path);
|
|
3553
|
+
const isGithubIssueFilePath = (path) => githubIssuePathParts(path) !== undefined || githubIssueDirectoryPathParts(path) !== undefined;
|
|
3554
|
+
const isGithubIssueTreePath = (path) => /^\/github\/repos\/[^/]+\/[^/]+\/issues\/.+/u.test(path);
|
|
3555
|
+
const githubPullPathParts = (path) => {
|
|
3556
|
+
// Tolerate every webhook-fed PR mount layout we've seen, across both the
|
|
3557
|
+
// nested <owner>/<repo> directory shape and the flat <owner>__<repo> shape
|
|
3558
|
+
// (the latter is what githubPullRoot / resolveIssuePrFromMount still use):
|
|
3559
|
+
// .../<owner>/<repo>/pulls/<n>__<slug>/meta.json (current adapters shape)
|
|
3560
|
+
// .../<owner>__<repo>/pulls/by-id/<n>.json (flat mount shape)
|
|
3561
|
+
// .../pulls/<n>/meta.json | metadata.json | .../pulls/<n>.json (variants)
|
|
3562
|
+
// Deliberately ignores siblings like pulls/_index.json and pulls/<n>/comments/*.
|
|
3563
|
+
const match = path.match(/^\/github\/repos\/(?:([^/]+)\/([^/]+)|([^/]+)__([^/]+))\/pulls\/(?:by-id\/)?(\d+)(?:__[^/]*)?(?:\/(?:meta|metadata)\.json|\.json)$/u);
|
|
3564
|
+
if (!match)
|
|
3565
|
+
return undefined;
|
|
3566
|
+
const owner = match[1] ?? match[3];
|
|
3567
|
+
const repo = match[2] ?? match[4];
|
|
3568
|
+
if (!owner || !repo)
|
|
3569
|
+
return undefined;
|
|
3570
|
+
return { owner, repo, number: Number(match[5]) };
|
|
3571
|
+
};
|
|
3572
|
+
const isGithubPullFilePath = (path) => githubPullPathParts(path) !== undefined;
|
|
3573
|
+
const parsePullSnapshot = (content, fallbackNumber) => {
|
|
3574
|
+
const payload = wrappedPayload(content);
|
|
3575
|
+
const number = typeof payload.number === 'number' ? payload.number : fallbackNumber;
|
|
3576
|
+
if (!Number.isInteger(number) || number <= 0)
|
|
3577
|
+
return undefined;
|
|
3578
|
+
return {
|
|
3579
|
+
number,
|
|
3580
|
+
state: stringValue(payload.state),
|
|
3581
|
+
headRef: refName(payload.headRef) ?? refName(payload.head) ?? stringValue(payload.head_ref) ?? stringValue(payload.headRefName),
|
|
3582
|
+
draft: booleanValue(payload.isDraft) ?? booleanValue(payload.draft),
|
|
3583
|
+
url: stringValue(payload.url) ?? stringValue(payload.html_url),
|
|
3584
|
+
title: stringValue(payload.title),
|
|
3585
|
+
body: stringValue(payload.body),
|
|
3586
|
+
merged: booleanValue(payload.merged),
|
|
3587
|
+
};
|
|
3588
|
+
};
|
|
3589
|
+
const prSnapshotIssueMatchScore = (snapshot, issueKey) => {
|
|
3590
|
+
if (containsIssueKey(snapshot.headRef ?? '', issueKey))
|
|
3591
|
+
return 30;
|
|
3592
|
+
if (containsIssueKey(snapshot.title ?? '', issueKey))
|
|
3593
|
+
return 20;
|
|
3594
|
+
if (containsExplicitIssueReference(snapshot.body ?? '', issueKey))
|
|
3595
|
+
return 10;
|
|
3596
|
+
return 0;
|
|
3597
|
+
};
|
|
3598
|
+
const prMetaShowsMerged = (snapshot) => snapshot.merged === true || snapshot.state?.trim().toUpperCase() === 'MERGED';
|
|
3599
|
+
// Guard applied to the babysitter's ready signal: the PR's own webhook-fed meta
|
|
3600
|
+
// must still be eligible for human review. A missing `state` is treated as open
|
|
3601
|
+
// (older mount layouts omit it). The CI/conflict/review verdict is NOT re-derived
|
|
3602
|
+
// here — the babysitter already owns it.
|
|
3603
|
+
const prMetaAllowsHumanReview = (snapshot) => {
|
|
3604
|
+
const state = snapshot.state?.trim().toUpperCase();
|
|
3605
|
+
if (state && state !== 'OPEN') {
|
|
3606
|
+
return { ok: false, reason: `pr state is ${state}` };
|
|
3607
|
+
}
|
|
3608
|
+
if (snapshot.merged === true) {
|
|
3609
|
+
return { ok: false, reason: 'pr already merged' };
|
|
3610
|
+
}
|
|
3611
|
+
if (snapshot.draft === true) {
|
|
3612
|
+
return { ok: false, reason: 'pr is a draft' };
|
|
3613
|
+
}
|
|
3614
|
+
return { ok: true, reason: 'pr open and not a draft' };
|
|
3615
|
+
};
|
|
3616
|
+
const isIssueAliasFilePath = (path) => path.startsWith(linearByStatePath('ready-for-agent')) &&
|
|
3617
|
+
path.endsWith('.json') &&
|
|
3618
|
+
!path.includes('/comments/') &&
|
|
3619
|
+
ISSUE_KEY_PATTERN.test(keyFromPath(path));
|
|
3620
|
+
const isCanonicalIssueFileBasename = (basename) => {
|
|
3621
|
+
const stem = basename.replace(/\.json$/u, '');
|
|
3622
|
+
const parts = stem.split('__');
|
|
3623
|
+
return parts.length === 2 && ISSUE_KEY_PATTERN.test(parts[0]) && parts[1].length > 0;
|
|
3624
|
+
};
|
|
3625
|
+
const isMissingIssueFileError = (error) => {
|
|
3626
|
+
const record = asRecord(error);
|
|
3627
|
+
const status = record?.status ?? record?.statusCode;
|
|
3628
|
+
if (status === 404)
|
|
3629
|
+
return true;
|
|
3630
|
+
return error instanceof Error && /(?:404|not found|file not found)/iu.test(error.message);
|
|
3631
|
+
};
|
|
3632
|
+
export const keyFromPath = (path) => path.split('/').at(-1)?.replace(/\.json$/, '').split('__')[0] ?? path;
|
|
3633
|
+
const uuidFromPath = (path) => path.split('__')[1]?.replace(/\.json$/, '');
|
|
3634
|
+
const stringValue = (value) => typeof value === 'string' ? value : undefined;
|
|
3635
|
+
const booleanValue = (value) => typeof value === 'boolean' ? value : undefined;
|
|
3636
|
+
const numberValue = (value) => typeof value === 'number' ? value : undefined;
|
|
3637
|
+
const liveHeartbeatIntervalMs = (staleMs) => Math.min(DEFAULT_LIVE_HEARTBEAT_INTERVAL_MS, Math.max(500, Math.floor(staleMs / 4)));
|
|
3638
|
+
const installFactoryDraftPredicate = (mount, config) => {
|
|
3639
|
+
mount.setDefaultAllowedDraftPredicate?.((path, content, opts) => isAllowedFactoryDraft(path, content, opts, mount, config));
|
|
3640
|
+
};
|
|
3641
|
+
const isAllowedFactoryDraft = async (path, content, opts, mount, config) => {
|
|
3642
|
+
if (!opts?.guarded)
|
|
3643
|
+
return false;
|
|
3644
|
+
// Comment writeback nested under its issue: /linear/issues/<ref>/comments/<draft>.json.
|
|
3645
|
+
// Scope-check the owning issue (the draft content is a comment, not an issue).
|
|
3646
|
+
const nestedComment = /^\/linear\/issues\/([^/]+)\/comments\/[^/]+$/u.exec(path);
|
|
3647
|
+
if (nestedComment) {
|
|
3648
|
+
return isIssuePathInFactoryScope(mount, `/linear/issues/${nestedComment[1]}.json`, config);
|
|
3649
|
+
}
|
|
3650
|
+
if (path.startsWith('/linear/issues/')) {
|
|
3651
|
+
if (isInFactoryScope(scopeIssueFromDraftContent(content), config.safety))
|
|
3652
|
+
return true;
|
|
3653
|
+
return isIssuePathInFactoryScope(mount, path, config);
|
|
3654
|
+
}
|
|
3655
|
+
if (/^\/slack\/channels\/[^/]+\/messages\/.+/u.test(path)) {
|
|
3656
|
+
return true;
|
|
3657
|
+
}
|
|
3658
|
+
return false;
|
|
3659
|
+
};
|
|
3660
|
+
const isIssuePathInFactoryScope = async (mount, path, config) => {
|
|
3661
|
+
try {
|
|
3662
|
+
return isInFactoryScope(parseLinearIssue(path, (await mount.readFile(path)).content), config.safety);
|
|
3663
|
+
}
|
|
3664
|
+
catch {
|
|
3665
|
+
return false;
|
|
3666
|
+
}
|
|
3667
|
+
};
|
|
3668
|
+
const scopeIssueFromDraftContent = (content) => ({
|
|
3669
|
+
title: typeof asRecord(content)?.title === 'string' ? asRecord(content)?.title : '',
|
|
3670
|
+
team: typeof asRecord(asRecord(content)?.team)?.key === 'string'
|
|
3671
|
+
? asRecord(asRecord(content)?.team)?.key
|
|
3672
|
+
: undefined,
|
|
3673
|
+
raw: asRecord(content) ?? {},
|
|
3674
|
+
});
|
|
3675
|
+
const eventCursorAfterPage = (cursor, events, nextCursor) => {
|
|
3676
|
+
if (nextCursor)
|
|
3677
|
+
return nextCursor;
|
|
3678
|
+
if (events.length === 0)
|
|
3679
|
+
return cursor;
|
|
3680
|
+
const numericCursor = cursor === undefined ? 0 : Number(cursor);
|
|
3681
|
+
if (Number.isInteger(numericCursor) && numericCursor >= 0) {
|
|
3682
|
+
return String(numericCursor + events.length);
|
|
3683
|
+
}
|
|
3684
|
+
return events.at(-1)?.id ?? cursor;
|
|
3685
|
+
};
|
|
3686
|
+
const liveEventDedupeKey = (event) => {
|
|
3687
|
+
if (!event.id)
|
|
3688
|
+
return undefined;
|
|
3689
|
+
const resource = asRecord(event.resource) ?? {};
|
|
3690
|
+
return [
|
|
3691
|
+
event.id,
|
|
3692
|
+
event.type,
|
|
3693
|
+
event.resource.path,
|
|
3694
|
+
stringValue(resource.revision) ?? '',
|
|
3695
|
+
event.digest ?? '',
|
|
3696
|
+
].join('\u001f');
|
|
3697
|
+
};
|
|
3698
|
+
const isBeforeLiveCutoff = (occurredAt, connectStartedAtMs, skewMarginMs) => {
|
|
3699
|
+
const occurredAtMs = Date.parse(occurredAt);
|
|
3700
|
+
if (!Number.isFinite(occurredAtMs))
|
|
3701
|
+
return false;
|
|
3702
|
+
return occurredAtMs < connectStartedAtMs - skewMarginMs;
|
|
3703
|
+
};
|
|
3704
|
+
const isAtOrBeforeHighWatermark = (eventId, highWatermark) => {
|
|
3705
|
+
if (!eventId || !highWatermark)
|
|
3706
|
+
return false;
|
|
3707
|
+
if (eventId === highWatermark)
|
|
3708
|
+
return true;
|
|
3709
|
+
const eventSequence = eventSequenceNumber(eventId);
|
|
3710
|
+
const watermarkSequence = eventSequenceNumber(highWatermark);
|
|
3711
|
+
if (eventSequence !== undefined && watermarkSequence !== undefined) {
|
|
3712
|
+
return eventSequence <= watermarkSequence;
|
|
3713
|
+
}
|
|
3714
|
+
return false;
|
|
3715
|
+
};
|
|
3716
|
+
const isHighWatermarkRouteUnavailable = (error) => {
|
|
3717
|
+
const details = asRecord(error) ?? {};
|
|
3718
|
+
const response = asRecord(details.response) ?? {};
|
|
3719
|
+
const status = details.status ?? details.statusCode ?? response.status ?? response.statusCode;
|
|
3720
|
+
if (status === 404 || status === '404') {
|
|
3721
|
+
return true;
|
|
3722
|
+
}
|
|
3723
|
+
const code = stringValue(details.code)?.toLowerCase();
|
|
3724
|
+
if (code === 'route_not_found') {
|
|
3725
|
+
return true;
|
|
3726
|
+
}
|
|
3727
|
+
return error instanceof Error && /route not found/i.test(error.message);
|
|
3728
|
+
};
|
|
3729
|
+
/**
|
|
3730
|
+
* Slack writeback liveness follows the provider signal hierarchy:
|
|
3731
|
+
* getEvents() provider watermarks are the authoritative ingest signal, sync
|
|
3732
|
+
* status is advisory, and only explicit error/failed states fail closed. Soft
|
|
3733
|
+
* sync states can be stale while Slack mount writes are still live, so
|
|
3734
|
+
* #slackFreshness lets fresh event watermarks override them instead of blocking
|
|
3735
|
+
* operator question writebacks.
|
|
3736
|
+
*/
|
|
3737
|
+
const slackSyncStatusResult = (status, nowMs, staleAfterMs) => {
|
|
3738
|
+
if (!status)
|
|
3739
|
+
return { known: false, degraded: false };
|
|
3740
|
+
const normalized = status.status?.toLowerCase();
|
|
3741
|
+
if (normalized && ['error', 'failed'].includes(normalized)) {
|
|
3742
|
+
return { known: true, degraded: true, reason: `slack sync status is ${status.status}`, severity: 'hard' };
|
|
3743
|
+
}
|
|
3744
|
+
if (normalized && ['lagging', 'stale', 'degraded'].includes(normalized)) {
|
|
3745
|
+
return { known: true, degraded: true, reason: `slack sync status is ${status.status}`, severity: 'soft' };
|
|
3746
|
+
}
|
|
3747
|
+
const lastEventAtMs = status.lastEventAtMs ??
|
|
3748
|
+
(status.lastEventAt ? Date.parse(status.lastEventAt) : undefined) ??
|
|
3749
|
+
(status.watermarkTs ? Date.parse(status.watermarkTs) : undefined);
|
|
3750
|
+
if (lastEventAtMs !== undefined && Number.isFinite(lastEventAtMs)) {
|
|
3751
|
+
const ageMs = nowMs - lastEventAtMs;
|
|
3752
|
+
return ageMs > staleAfterMs
|
|
3753
|
+
? { known: true, degraded: true, reason: `slack sync watermark stale by ${ageMs}ms`, severity: 'soft' }
|
|
3754
|
+
: { known: true, degraded: false };
|
|
3755
|
+
}
|
|
3756
|
+
if (status.lagSeconds !== undefined && Number.isFinite(status.lagSeconds)) {
|
|
3757
|
+
const lagMs = status.lagSeconds * 1000;
|
|
3758
|
+
return lagMs > staleAfterMs
|
|
3759
|
+
? { known: true, degraded: true, reason: `slack sync lag is ${lagMs}ms`, severity: 'soft' }
|
|
3760
|
+
: { known: true, degraded: false };
|
|
3761
|
+
}
|
|
3762
|
+
if (normalized && ['ok', 'healthy', 'fresh', 'synced', 'ready'].includes(normalized)) {
|
|
3763
|
+
return { known: true, degraded: false };
|
|
3764
|
+
}
|
|
3765
|
+
return { known: false, degraded: false };
|
|
3766
|
+
};
|
|
3767
|
+
const eventProvider = (event) => {
|
|
3768
|
+
const flat = asRecord(event) ?? {};
|
|
3769
|
+
return stringValue(asRecord(flat.resource)?.provider) ?? stringValue(flat.provider);
|
|
3770
|
+
};
|
|
3771
|
+
const eventOccurredAtMs = (event) => {
|
|
3772
|
+
const flat = asRecord(event) ?? {};
|
|
3773
|
+
const timestamp = stringValue(flat.occurredAt) ?? stringValue(flat.timestamp);
|
|
3774
|
+
const parsed = timestamp ? Date.parse(timestamp) : Number.NaN;
|
|
3775
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
3776
|
+
};
|
|
3777
|
+
const eventSequenceNumber = (eventId) => {
|
|
3778
|
+
const whole = Number(eventId);
|
|
3779
|
+
if (Number.isFinite(whole))
|
|
3780
|
+
return whole;
|
|
3781
|
+
const trailing = eventId.match(/(\d+)$/u)?.[1];
|
|
3782
|
+
if (!trailing)
|
|
3783
|
+
return undefined;
|
|
3784
|
+
const parsed = Number(trailing);
|
|
3785
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
3786
|
+
};
|
|
3787
|
+
const rememberLiveEvent = (seen, key) => {
|
|
3788
|
+
seen.add(key);
|
|
3789
|
+
if (seen.size <= LIVE_DEDUPE_LIMIT)
|
|
3790
|
+
return;
|
|
3791
|
+
const oldest = seen.values().next().value;
|
|
3792
|
+
if (oldest)
|
|
3793
|
+
seen.delete(oldest);
|
|
3794
|
+
};
|
|
3795
|
+
const recordName = (value) => {
|
|
3796
|
+
if (typeof value === 'string') {
|
|
3797
|
+
return value;
|
|
3798
|
+
}
|
|
3799
|
+
const record = asRecord(value);
|
|
3800
|
+
return stringValue(record?.name) ?? stringValue(record?.key) ?? stringValue(record?.id);
|
|
3801
|
+
};
|
|
3802
|
+
const refName = (value) => {
|
|
3803
|
+
if (typeof value === 'string') {
|
|
3804
|
+
return value;
|
|
3805
|
+
}
|
|
3806
|
+
const record = asRecord(value);
|
|
3807
|
+
return stringValue(record?.name) ?? stringValue(record?.ref);
|
|
3808
|
+
};
|
|
3809
|
+
const labelName = (value) => {
|
|
3810
|
+
if (typeof value === 'string') {
|
|
3811
|
+
return value;
|
|
3812
|
+
}
|
|
3813
|
+
const record = asRecord(value);
|
|
3814
|
+
return stringValue(record?.name);
|
|
3815
|
+
};
|
|
3816
|
+
const isCompletionReason = (reason) => reason === 'issue-done' || reason === 'done' || reason === 'completed';
|
|
3817
|
+
// The broker rejects re-registering a name it never released on exit
|
|
3818
|
+
// (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
|
|
3819
|
+
// the structured payload or the message so resume can treat it as terminal
|
|
3820
|
+
// rather than retrying the (falsely) "retryable" error forever.
|
|
3821
|
+
const isAgentAlreadyExistsError = (error) => {
|
|
3822
|
+
const record = asRecord(error);
|
|
3823
|
+
const data = asRecord(record?.data);
|
|
3824
|
+
const message = stringValue(data?.error) ?? (error instanceof Error ? error.message : '');
|
|
3825
|
+
return /already exists/iu.test(message);
|
|
3826
|
+
};
|
|
3827
|
+
const defaultRestartPolicy = (spec) =>
|
|
3828
|
+
// Implementers and babysitters are both long-running and resumable — the
|
|
3829
|
+
// babysitter shepherds an open PR over many CI/review cycles, so an abnormal
|
|
3830
|
+
// exit should resume its session rather than drop the PR. The reviewer is
|
|
3831
|
+
// short-lived and keeps the fleet default.
|
|
3832
|
+
spec.role === 'implementer' || spec.role === 'babysitter'
|
|
3833
|
+
? { maxRestarts: 3, strategy: 'resume' }
|
|
3834
|
+
: spec.restartPolicy;
|
|
3835
|
+
const slackPayloadTs = (threadId) => threadId.replace(/_/g, '.');
|
|
3836
|
+
const slackChannelMessagesPrefix = (channelDir) => `/slack/channels/${channelDir}/messages/`;
|
|
3837
|
+
const eventIdentity = (event) => {
|
|
3838
|
+
const record = event;
|
|
3839
|
+
const rawId = record.id ?? record.event_id ?? record.seq;
|
|
3840
|
+
const id = typeof rawId === 'string' || typeof rawId === 'number' ? String(rawId) : undefined;
|
|
3841
|
+
return id ? `event:${id}` : undefined;
|
|
3842
|
+
};
|
|
3843
|
+
const describeError = (error) => {
|
|
3844
|
+
if (error instanceof Error) {
|
|
3845
|
+
return {
|
|
3846
|
+
errorMessage: error.message || error.name || 'Error',
|
|
3847
|
+
errorStack: error.stack,
|
|
3848
|
+
};
|
|
3849
|
+
}
|
|
3850
|
+
if (typeof error === 'string') {
|
|
3851
|
+
return { errorMessage: error };
|
|
3852
|
+
}
|
|
3853
|
+
try {
|
|
3854
|
+
const serialized = JSON.stringify(error);
|
|
3855
|
+
if (serialized && serialized !== '{}') {
|
|
3856
|
+
return { errorMessage: serialized };
|
|
3857
|
+
}
|
|
3858
|
+
}
|
|
3859
|
+
catch {
|
|
3860
|
+
// Fall through to String(error).
|
|
3861
|
+
}
|
|
3862
|
+
return { errorMessage: String(error) };
|
|
3863
|
+
};
|
|
3864
|
+
const failedIterationReport = (error, dryRun) => {
|
|
3865
|
+
const details = describeError(error);
|
|
3866
|
+
return {
|
|
3867
|
+
pulled: [],
|
|
3868
|
+
triaged: [],
|
|
3869
|
+
dispatched: [],
|
|
3870
|
+
skipped: [],
|
|
3871
|
+
dryRun,
|
|
3872
|
+
error: {
|
|
3873
|
+
message: details.errorMessage,
|
|
3874
|
+
...(details.errorStack ? { stack: details.errorStack } : {}),
|
|
3875
|
+
},
|
|
3876
|
+
};
|
|
3877
|
+
};
|
|
3878
|
+
const isRegistrationLagInjectionError = (error) => {
|
|
3879
|
+
const { errorMessage } = describeError(error);
|
|
3880
|
+
return /recipient unavailable|not registered|unknown recipient|no such (agent|recipient)|timed out waiting for delivery_injected/i
|
|
3881
|
+
.test(errorMessage);
|
|
3882
|
+
};
|
|
3883
|
+
const isTimeoutError = (error) => error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError');
|
|
3884
|
+
const retryOnTimeout = async (fn, opts) => {
|
|
3885
|
+
let lastError;
|
|
3886
|
+
for (let attempt = 0; attempt < opts.attempts; attempt += 1) {
|
|
3887
|
+
try {
|
|
3888
|
+
return await fn();
|
|
3889
|
+
}
|
|
3890
|
+
catch (error) {
|
|
3891
|
+
lastError = error;
|
|
3892
|
+
if (!isTimeoutError(error) || attempt === opts.attempts - 1)
|
|
3893
|
+
throw error;
|
|
3894
|
+
await new Promise((resolve) => setTimeout(resolve, opts.delayMs));
|
|
3895
|
+
}
|
|
3896
|
+
}
|
|
3897
|
+
throw lastError;
|
|
3898
|
+
};
|
|
3899
|
+
const contextualError = (context, error) => {
|
|
3900
|
+
const details = describeError(error);
|
|
3901
|
+
const wrapped = new Error(`${context}: ${details.errorMessage}`);
|
|
3902
|
+
if (details.errorStack) {
|
|
3903
|
+
wrapped.stack = `${wrapped.stack ?? wrapped.message}\nCaused by: ${details.errorStack}`;
|
|
3904
|
+
}
|
|
3905
|
+
const withCause = wrapped;
|
|
3906
|
+
withCause.cause = error;
|
|
3907
|
+
return wrapped;
|
|
3908
|
+
};
|
|
3909
|
+
const registryHandoffKey = (issue, agentName) => `${issueKey(issue)}:${agentName}`;
|
|
3910
|
+
const cloneTrackedAgent = (tracked) => ({
|
|
3911
|
+
spec: { ...tracked.spec },
|
|
3912
|
+
result: tracked.result ? { ...tracked.result } : undefined,
|
|
3913
|
+
sessionRef: tracked.sessionRef,
|
|
3914
|
+
});
|
|
3915
|
+
const parseSlackReply = (path, content, botUserId) => {
|
|
3916
|
+
const raw = asRecord(parseJsonContent(content)) ?? {};
|
|
3917
|
+
const payload = wrappedPayload(raw);
|
|
3918
|
+
const channelDir = path.match(/^\/slack\/channels\/([^/]+)\//u)?.[1] ?? '';
|
|
3919
|
+
const pathMatch = path.match(/^\/slack\/channels\/[^/]+\/messages\/([^/]+)(?:\/replies\/([^/]+))?/u);
|
|
3920
|
+
const parentFromPath = pathMatch?.[2] ? slackPayloadTs(pathMatch[1]) : undefined;
|
|
3921
|
+
const messageFromPath = slackPayloadTs(pathMatch?.[2] ?? pathMatch?.[1] ?? '');
|
|
3922
|
+
const messageTs = stringValue(payload.ts) ?? messageFromPath;
|
|
3923
|
+
const threadTs = stringValue(payload.thread_ts) ?? parentFromPath;
|
|
3924
|
+
if (!channelDir || !threadTs || !messageTs) {
|
|
3925
|
+
return undefined;
|
|
3926
|
+
}
|
|
3927
|
+
return {
|
|
3928
|
+
channelDir,
|
|
3929
|
+
threadTs,
|
|
3930
|
+
messageTs,
|
|
3931
|
+
text: stringValue(payload.text) ?? '',
|
|
3932
|
+
isThreadReply: Boolean(parentFromPath) || threadTs !== messageTs,
|
|
3933
|
+
isBot: isOwnSlackBotReply(payload, botUserId),
|
|
3934
|
+
raw,
|
|
3935
|
+
};
|
|
3936
|
+
};
|
|
3937
|
+
const triageEscalationReason = (decision) => {
|
|
3938
|
+
const reasons = [];
|
|
3939
|
+
if (decision.confidence === 'low') {
|
|
3940
|
+
reasons.push('low-confidence triage');
|
|
3941
|
+
}
|
|
3942
|
+
if (decision.thin) {
|
|
3943
|
+
reasons.push('thin issue context');
|
|
3944
|
+
}
|
|
3945
|
+
if (reasons.length === 0) {
|
|
3946
|
+
return undefined;
|
|
3947
|
+
}
|
|
3948
|
+
return `${reasons.join(' and ')}${decision.rationale ? `: ${decision.rationale}` : ''}`;
|
|
3949
|
+
};
|
|
3950
|
+
const slackAnswerInput = (issue, text) => `Slack reply for ${issue.key}:\n${text}\r`;
|
|
3951
|
+
const isFactoryQuestionTarget = (target) => {
|
|
3952
|
+
const normalized = target.trim().replace(/^@/u, '').toLowerCase();
|
|
3953
|
+
return normalized === 'broker' || normalized === 'factory';
|
|
3954
|
+
};
|
|
3955
|
+
const parseAgentQuestion = (message) => {
|
|
3956
|
+
const markerPattern = new RegExp(`(^|\\n)\\s*(?:${escapeRegExp(AGENT_NEEDS_INPUT_MARKER)}|${escapeRegExp(LEGACY_AGENT_NEEDS_INPUT_MARKER)})\\s*(?::\\s*)?`, 'iu');
|
|
3957
|
+
const match = markerPattern.exec(message.body);
|
|
3958
|
+
if (!match || !message.from.trim()) {
|
|
3959
|
+
return undefined;
|
|
3960
|
+
}
|
|
3961
|
+
const markerEnd = match.index + match[0].length;
|
|
3962
|
+
const bodyAfterMarker = message.body.slice(markerEnd).trim();
|
|
3963
|
+
const issueKey = message.body.match(/(?:^|\n)\s*Issue:\s*([A-Z]+-\d+)\s*(?:\n|$)/iu)?.[1]?.toUpperCase();
|
|
3964
|
+
const question = extractQuestionText(bodyAfterMarker);
|
|
3965
|
+
if (!question) {
|
|
3966
|
+
return undefined;
|
|
3967
|
+
}
|
|
3968
|
+
return {
|
|
3969
|
+
agentName: message.from.trim(),
|
|
3970
|
+
issueKey,
|
|
3971
|
+
question,
|
|
3972
|
+
eventId: message.eventId,
|
|
3973
|
+
};
|
|
3974
|
+
};
|
|
3975
|
+
const parsePrReadySignal = (message) => {
|
|
3976
|
+
if (!message.from.trim()) {
|
|
3977
|
+
return undefined;
|
|
3978
|
+
}
|
|
3979
|
+
const markerPattern = new RegExp(`(^|\\n|\\s)${escapeRegExp(AGENT_PR_READY_MARKER)}\\s*(?::\\s*)?([A-Z]+-\\d+)?`, 'iu');
|
|
3980
|
+
const match = markerPattern.exec(message.body);
|
|
3981
|
+
if (!match) {
|
|
3982
|
+
return undefined;
|
|
3983
|
+
}
|
|
3984
|
+
return { agentName: message.from.trim(), issueKey: match[2]?.toUpperCase() };
|
|
3985
|
+
};
|
|
3986
|
+
const extractQuestionText = (bodyAfterMarker) => {
|
|
3987
|
+
const questionMatch = bodyAfterMarker.match(/(?:^|\n)\s*Question:\s*([\s\S]+)$/iu);
|
|
3988
|
+
const raw = questionMatch?.[1] ?? bodyAfterMarker;
|
|
3989
|
+
return raw
|
|
3990
|
+
.split(/\r?\n/u)
|
|
3991
|
+
.filter((line) => !/^\s*Issue:\s*[A-Z]+-\d+\s*$/iu.test(line))
|
|
3992
|
+
.join('\n')
|
|
3993
|
+
.trim();
|
|
3994
|
+
};
|
|
3995
|
+
const agentQuestionDedupeKey = (issue, question) => `${question.eventId ?? 'missing'}:${stableHash(JSON.stringify({
|
|
3996
|
+
issue: issue.key,
|
|
3997
|
+
from: question.agentName,
|
|
3998
|
+
question: question.question,
|
|
3999
|
+
}))}`;
|
|
4000
|
+
const agentQuestionSlackText = (issue, question) => [
|
|
4001
|
+
`${issue.key}: ${question.agentName} needs input.`,
|
|
4002
|
+
`Question: ${question.question}`,
|
|
4003
|
+
].join('\n');
|
|
4004
|
+
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|
|
4005
|
+
const isOwnSlackBotReply = (payload, botUserId) => payload.user_is_bot === true ||
|
|
4006
|
+
stringValue(payload.user) === botUserId;
|
|
4007
|
+
const unrefDelay = (ms) => new Promise((resolve) => {
|
|
4008
|
+
const timer = setTimeout(resolve, ms);
|
|
4009
|
+
timer.unref?.();
|
|
4010
|
+
});
|
|
4011
|
+
const liveEventYield = () => new Promise((resolve) => {
|
|
4012
|
+
if (typeof setImmediate === 'function') {
|
|
4013
|
+
setImmediate(resolve);
|
|
4014
|
+
return;
|
|
4015
|
+
}
|
|
4016
|
+
setTimeout(resolve, 0);
|
|
4017
|
+
});
|
|
4018
|
+
//# sourceMappingURL=factory.js.map
|