@agent-relay/factory 0.1.63 → 0.1.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/fleet.d.ts +4 -0
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +31 -7
- package/dist/cli/fleet.js.map +1 -1
- package/dist/fleet/control-plane-circuit.d.ts.map +1 -1
- package/dist/fleet/control-plane-circuit.js +13 -1
- package/dist/fleet/control-plane-circuit.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts +23 -1
- package/dist/mount/relayfile-cloud-mount-client.d.ts.map +1 -1
- package/dist/mount/relayfile-cloud-mount-client.js +38 -9
- package/dist/mount/relayfile-cloud-mount-client.js.map +1 -1
- package/dist/observability/cloud-reporter.d.ts +1 -0
- package/dist/observability/cloud-reporter.d.ts.map +1 -1
- package/dist/observability/cloud-reporter.js +182 -50
- package/dist/observability/cloud-reporter.js.map +1 -1
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +826 -52
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/state.d.ts +60 -3
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/document-store.d.ts +2 -1
- package/dist/state/document-store.d.ts.map +1 -1
- package/dist/state/document-store.js.map +1 -1
- package/dist/state/file-state-store.d.ts +16 -5
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +124 -8
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +13 -2
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +87 -4
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/state/watch-state-document.d.ts.map +1 -1
- package/dist/state/watch-state-document.js +45 -3
- package/dist/state/watch-state-document.js.map +1 -1
- package/dist/trajectory.d.ts +25 -0
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/trajectory.js +51 -0
- package/dist/trajectory.js.map +1 -0
- package/package.json +1 -1
|
@@ -31,6 +31,7 @@ import { readFactoryInFlightRegistry, terminatePids } from './reaper.js';
|
|
|
31
31
|
import { createFactoryCloudEventV1, factoryCloudReleaseReasonV1, } from '../observability/events.js';
|
|
32
32
|
import { boundedRunCostTotal, CostLedger } from '../cost/ledger.js';
|
|
33
33
|
import { createTicketDispatchDelivery } from '../delivery/ticket-dispatch.js';
|
|
34
|
+
import { canonicalTrajectorySessionRef, renderTrajectoryPointer, stripTrajectoryPointers, } from '../trajectory.js';
|
|
34
35
|
import { FleetControlPlaneCircuit, FleetControlPlaneCircuitOpenError, guardFleetControlPlane, } from '../fleet/control-plane-circuit.js';
|
|
35
36
|
class ClarificationWakeLeaseLostError extends Error {
|
|
36
37
|
}
|
|
@@ -146,8 +147,32 @@ const STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS = 30_000;
|
|
|
146
147
|
const RECONCILED_AGENT_EXIT_CONCURRENCY = 4;
|
|
147
148
|
const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000;
|
|
148
149
|
const SLACK_CONVERSATION_TURN_LEASE_MS = 60_000;
|
|
150
|
+
// Both receipt leases guard an in-flight Slack writeback, and no fixed lease can
|
|
151
|
+
// cover one: MountSlackWriteback budgets 90s for the confirm alone, on top of an
|
|
152
|
+
// unbounded writeFile. Sizing them past that worst case would only trade a stolen
|
|
153
|
+
// claim for a stranded one — a lease long enough to survive the slowest write is
|
|
154
|
+
// equally long enough to hold the receipt hostage to a dead holder. So these
|
|
155
|
+
// bound the *idle* claim and #withRenewedProviderLease extends them for exactly
|
|
156
|
+
// as long as the write they cover is still running.
|
|
157
|
+
const SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS = 60_000;
|
|
158
|
+
const SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS = 60_000;
|
|
159
|
+
// Renewal without a ceiling is the same defect from the other side: a heartbeat
|
|
160
|
+
// that extends the claim for as long as the write runs also extends it forever
|
|
161
|
+
// when the write never returns, and nothing else can reclaim the receipt short
|
|
162
|
+
// of a restart. So renewal is bounded past the slowest write this daemon budgets
|
|
163
|
+
// for — MountSlackWriteback's 90s confirm on top of its writeFile — and beyond
|
|
164
|
+
// that the write is not slow, it is wedged: the heartbeat stops, the idle lease
|
|
165
|
+
// runs out, and the retry that owns the queued replies can take them back.
|
|
166
|
+
const SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS = 5 * 60_000;
|
|
149
167
|
const SLACK_CONVERSATION_TURN_RETRY_MS = 1_000;
|
|
150
168
|
const SLACK_REPLY_ROUTE_RETRY_MS = 1_000;
|
|
169
|
+
// One pass drains the whole chain (#slackReplyRoutes holds only the newest
|
|
170
|
+
// route per key and every route awaits its predecessor). The extra passes only
|
|
171
|
+
// exist so the drain can prove quiescence rather than assume it.
|
|
172
|
+
const SLACK_REPLY_ROUTE_DRAIN_PASSES = 8;
|
|
173
|
+
const SLACK_TERMINAL_THREAD_GRACE_MS = 24 * 60 * 60_000;
|
|
174
|
+
const SLACK_TERMINAL_RECEIPT_RETRY_MS = 1_000;
|
|
175
|
+
const SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS = 5 * 60_000;
|
|
151
176
|
const MERGE_GATE_MAX_ATTEMPTS = 12;
|
|
152
177
|
const MERGE_GATE_POLL_DELAY_MS = 10_000;
|
|
153
178
|
const MAX_LABEL_IMPLEMENTERS = 4;
|
|
@@ -215,6 +240,25 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
|
|
|
215
240
|
this.leaseUntilMs = leaseUntilMs;
|
|
216
241
|
}
|
|
217
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* The durable dispatch-lifecycle claim was refused for one work unit: its
|
|
245
|
+
* record is already terminal, or another publisher currently holds the lease.
|
|
246
|
+
* Both are facts about that single unit — the rest of the pass is unaffected —
|
|
247
|
+
* so the readiness loop skips it and keeps going (#292).
|
|
248
|
+
*
|
|
249
|
+
* Typed rather than left as a plain `Error` so the loop can classify it by
|
|
250
|
+
* construction instead of by matching on `Refusing to dispatch ...` text.
|
|
251
|
+
*/
|
|
252
|
+
class DispatchLifecycleClaimRefusedError extends Error {
|
|
253
|
+
issueKey;
|
|
254
|
+
refusal;
|
|
255
|
+
constructor(issueKey, refusal, message) {
|
|
256
|
+
super(message);
|
|
257
|
+
this.issueKey = issueKey;
|
|
258
|
+
this.refusal = refusal;
|
|
259
|
+
this.name = 'DispatchLifecycleClaimRefusedError';
|
|
260
|
+
}
|
|
261
|
+
}
|
|
218
262
|
const realClock = {
|
|
219
263
|
now: () => Date.now(),
|
|
220
264
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -266,6 +310,19 @@ export class FactoryLoop {
|
|
|
266
310
|
#dispatchInFlight = new Map();
|
|
267
311
|
#slackWatchers = new Map();
|
|
268
312
|
#slackWatcherStarts = new Map();
|
|
313
|
+
#slackTerminalWatchExpiryTimers = new Map();
|
|
314
|
+
#slackTerminalReceiptRetryTimers = new Map();
|
|
315
|
+
#terminalSlackWatchIssues = new Set();
|
|
316
|
+
/**
|
|
317
|
+
* The one in-memory record of "this work unit has an in-flight Slack side
|
|
318
|
+
* effect". Both ordinary reply routes and the writebacks the terminal fence
|
|
319
|
+
* issues on their behalf register here, because this map is what the terminal
|
|
320
|
+
* drain waits on: anything that touches Slack for a work unit without
|
|
321
|
+
* registering is invisible to the drain, and the watcher teardown that follows
|
|
322
|
+
* a successful drain then pulls that effect's retry timer out from under it.
|
|
323
|
+
*/
|
|
324
|
+
#slackReplyRoutes = new Map();
|
|
325
|
+
#slackReplyRouteDrains = new Set();
|
|
269
326
|
#slackConversationTurns;
|
|
270
327
|
#slackConversationOwner = `${process.pid}:${randomUUID()}`;
|
|
271
328
|
#githubIssueCommentWatchers = new Map();
|
|
@@ -439,6 +496,11 @@ export class FactoryLoop {
|
|
|
439
496
|
#discoverySweepRenewTimer;
|
|
440
497
|
#discoverySweepRenewalInFlight;
|
|
441
498
|
#discoverySweepLeaseLost = false;
|
|
499
|
+
// Registry/heartbeat paths the in-flight runLoop iteration would use. A
|
|
500
|
+
// per-item dispatch failure now skips instead of aborting the pass (#292),
|
|
501
|
+
// so the loop's catch no longer runs the failure-handoff reaper for it; the
|
|
502
|
+
// pass reaps inline and must write to the same paths runLoop would.
|
|
503
|
+
#loopReapPaths;
|
|
442
504
|
#discoveryOverloadError;
|
|
443
505
|
#resolvedIssueSource;
|
|
444
506
|
#integrationInstructions;
|
|
@@ -907,6 +969,14 @@ export class FactoryLoop {
|
|
|
907
969
|
await this.#boundedStopTeardown('factory subscription unsubscribe', () => subscription?.unsubscribe());
|
|
908
970
|
await Promise.all([...this.#slackWatchers.values()].map((watcher) => watcher.stop()));
|
|
909
971
|
this.#slackWatchers.clear();
|
|
972
|
+
for (const timer of this.#slackTerminalWatchExpiryTimers.values())
|
|
973
|
+
clearTimeout(timer);
|
|
974
|
+
this.#slackTerminalWatchExpiryTimers.clear();
|
|
975
|
+
for (const timer of this.#slackTerminalReceiptRetryTimers.values())
|
|
976
|
+
clearTimeout(timer);
|
|
977
|
+
this.#slackTerminalReceiptRetryTimers.clear();
|
|
978
|
+
this.#terminalSlackWatchIssues.clear();
|
|
979
|
+
this.#slackReplyRouteDrains.clear();
|
|
910
980
|
await Promise.all([...this.#githubIssueCommentWatchers.values()].map((watcher) => watcher.stop()));
|
|
911
981
|
this.#githubIssueCommentWatchers.clear();
|
|
912
982
|
this.#githubIssueCommentWatchStates.clear();
|
|
@@ -2047,6 +2117,10 @@ export class FactoryLoop {
|
|
|
2047
2117
|
reason: entry.reason,
|
|
2048
2118
|
});
|
|
2049
2119
|
};
|
|
2120
|
+
// Backstop for the skip-by-default catch below: see #292. Reset only by
|
|
2121
|
+
// a completed dispatch, so the name says "since a dispatch" rather than
|
|
2122
|
+
// "consecutive" — a benign classified skip in between does not clear it.
|
|
2123
|
+
let unclassifiedFailuresSinceDispatch = 0;
|
|
2050
2124
|
let lastReadyReadProgressAtMs = this.#clock.now();
|
|
2051
2125
|
let readyIssueReads = 0;
|
|
2052
2126
|
const issueEntries = [];
|
|
@@ -2144,19 +2218,10 @@ export class FactoryLoop {
|
|
|
2144
2218
|
}
|
|
2145
2219
|
const decision = await this.triageIssue(issue);
|
|
2146
2220
|
triaged.push(decision);
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
catch (error) {
|
|
2152
|
-
if (!(error instanceof LiveDispatchStateChangedError))
|
|
2153
|
-
throw error;
|
|
2154
|
-
recordSkip({ issue: decision.issue, reason: 'live state changed during dispatch' });
|
|
2155
|
-
this.#logger.info?.('[factory] skipped issue whose live state changed during dispatch', {
|
|
2156
|
-
issue: decision.issue.key,
|
|
2157
|
-
});
|
|
2158
|
-
continue;
|
|
2159
|
-
}
|
|
2221
|
+
const result = await this.dispatch(decision, { dryRun });
|
|
2222
|
+
// A completed dispatch — even one that parks or escalates the issue —
|
|
2223
|
+
// proves the pipeline still works, so the fuse below starts over.
|
|
2224
|
+
unclassifiedFailuresSinceDispatch = 0;
|
|
2160
2225
|
if (result.agents.length === 0 && !dryRun) {
|
|
2161
2226
|
const reason = result.hold?.kind === 'dependency-cycle'
|
|
2162
2227
|
? `dependency cycle detected: ${result.hold.cycle?.join(' -> ') ?? 'unknown cycle'}`
|
|
@@ -2169,6 +2234,52 @@ export class FactoryLoop {
|
|
|
2169
2234
|
dispatched.push(result);
|
|
2170
2235
|
}
|
|
2171
2236
|
}
|
|
2237
|
+
catch (error) {
|
|
2238
|
+
// #292: issues in a pass are independent work units, so a failure
|
|
2239
|
+
// that is about ONE unit costs that unit and nothing else. Only the
|
|
2240
|
+
// conditions named in `#isPassFatalFailure` — the ones where
|
|
2241
|
+
// continuing the pass is meaningless — abort the whole sweep.
|
|
2242
|
+
if (this.#isPassFatalFailure(error, dryRun))
|
|
2243
|
+
throw error;
|
|
2244
|
+
if (!isClassifiedPerItemDispatchFailure(error)) {
|
|
2245
|
+
unclassifiedFailuresSinceDispatch += 1;
|
|
2246
|
+
// A pass-wide fault can arrive disguised as a run of per-item
|
|
2247
|
+
// faults. Skipping every unit would then hand back a green report
|
|
2248
|
+
// that dispatched nothing, which is the same silent wedge #292 is
|
|
2249
|
+
// about, wearing the opposite costume. Fail the pass loudly so
|
|
2250
|
+
// `readinessReconcile.lastError` carries the cause.
|
|
2251
|
+
if (unclassifiedFailuresSinceDispatch >= UNCLASSIFIED_DISPATCH_FAILURE_LIMIT) {
|
|
2252
|
+
throw contextualError(`Aborting readiness pass after ${unclassifiedFailuresSinceDispatch} unclassified dispatch failures without a successful dispatch`, error);
|
|
2253
|
+
}
|
|
2254
|
+
this.#increment('dispatchItemFailuresSkipped');
|
|
2255
|
+
// The raw message is operator-facing only; the run report carries
|
|
2256
|
+
// the sanitized classification from `perItemDispatchSkipReason`.
|
|
2257
|
+
this.#logger.warn?.('[factory] skipped a work unit whose dispatch failed; continuing the pass', {
|
|
2258
|
+
issue: issueRef(issue).key,
|
|
2259
|
+
unclassifiedFailuresSinceDispatch,
|
|
2260
|
+
error: describeError(error).errorMessage,
|
|
2261
|
+
});
|
|
2262
|
+
this.#error(error, issueRef(issue));
|
|
2263
|
+
// The failure may have left half-spawned agents behind. runLoop's
|
|
2264
|
+
// catch used to reap them because this error aborted the pass;
|
|
2265
|
+
// now that the pass survives, the reap has to happen here or the
|
|
2266
|
+
// agents leak until the next failed iteration.
|
|
2267
|
+
await this.#reapDispatchFailureHandoffsNow();
|
|
2268
|
+
}
|
|
2269
|
+
else {
|
|
2270
|
+
// Not an error — the unit simply cannot be dispatched right now —
|
|
2271
|
+
// so this stays out of `counters.errors` and gets its own counter
|
|
2272
|
+
// instead, or a terminal-lifecycle backlog would be invisible to
|
|
2273
|
+
// anyone watching only `dispatchItemFailuresSkipped`.
|
|
2274
|
+
this.#increment('dispatchItemsSkippedUndispatchable');
|
|
2275
|
+
this.#logger.info?.('[factory] skipped a work unit that cannot be dispatched right now', {
|
|
2276
|
+
issue: issueRef(issue).key,
|
|
2277
|
+
error: describeError(error).errorMessage,
|
|
2278
|
+
});
|
|
2279
|
+
}
|
|
2280
|
+
recordSkip({ issue: issueRef(issue), reason: perItemDispatchSkipReason(error) });
|
|
2281
|
+
continue;
|
|
2282
|
+
}
|
|
2172
2283
|
finally {
|
|
2173
2284
|
if (recoveredIdentity)
|
|
2174
2285
|
this.#reconciledGithubInProgress.delete(recoveredIdentity);
|
|
@@ -2202,6 +2313,79 @@ export class FactoryLoop {
|
|
|
2202
2313
|
}
|
|
2203
2314
|
}
|
|
2204
2315
|
}
|
|
2316
|
+
/**
|
|
2317
|
+
* Whether a failure raised while processing ONE work unit must abort the
|
|
2318
|
+
* whole readiness pass instead of skipping that unit.
|
|
2319
|
+
*
|
|
2320
|
+
* The default is the opposite, and that inversion is the fix for #292.
|
|
2321
|
+
* Issues in a pass are independent work units: a failure that is *about one
|
|
2322
|
+
* unit* — its dispatch-lifecycle record, its live state, a provider fault on
|
|
2323
|
+
* its own writeback — costs that unit and nothing else. Before this, every
|
|
2324
|
+
* error except `LiveDispatchStateChangedError` escaped the `for` loop, so a
|
|
2325
|
+
* single issue whose lifecycle record had gone terminal stopped all dispatch
|
|
2326
|
+
* indefinitely, every pass.
|
|
2327
|
+
*
|
|
2328
|
+
* A condition belongs here only when continuing the pass is meaningless or
|
|
2329
|
+
* actively harmful — when the failure is about the *pass*, not the item:
|
|
2330
|
+
*
|
|
2331
|
+
* - The discovery sweep lease is gone. Another process now owns this
|
|
2332
|
+
* workspace's sweep, so every remaining read throws the same way and each
|
|
2333
|
+
* one would be recorded as an ordinary per-issue skip. The run report
|
|
2334
|
+
* would then claim a clean pass over work this process no longer has the
|
|
2335
|
+
* right to touch.
|
|
2336
|
+
* - Relayfile signalled overload for this sweep. The backend is shedding
|
|
2337
|
+
* load; grinding through the remaining units makes it worse, and
|
|
2338
|
+
* `#runOnceWithDiscoveryFence` is going to rethrow this at the fence
|
|
2339
|
+
* anyway.
|
|
2340
|
+
* - The factory is stopping. Teardown is in progress and dispatching more
|
|
2341
|
+
* agents now leaks them past the shutdown deadline.
|
|
2342
|
+
* - The fleet control-plane circuit is no longer closed, **on a live pass**.
|
|
2343
|
+
* Dispatch is globally paused — the same condition
|
|
2344
|
+
* `#assertFleetControlPlaneAvailable` refuses to *start* a live pass on,
|
|
2345
|
+
* so it must also stop one already in flight. A dry run is exempt: it
|
|
2346
|
+
* never calls that admission gate and never spawns, so a paused control
|
|
2347
|
+
* plane is irrelevant to it rather than fatal to it. Without the
|
|
2348
|
+
* exemption, one live pass that trips the circuit would poison every
|
|
2349
|
+
* later dry run — including the boot gate's own `run-once --dry-run`
|
|
2350
|
+
* probe, turning a recoverable circuit-open condition into a failed boot.
|
|
2351
|
+
* That is a nastier version of the wedge this whole change removes.
|
|
2352
|
+
*
|
|
2353
|
+
* Deliberately NOT here: JavaScript builtin error types. Classifying
|
|
2354
|
+
* "programmer faults" such as `TypeError` as fatal is the obvious next rule
|
|
2355
|
+
* and it is a trap — Node reports a failed `fetch` as `TypeError: fetch
|
|
2356
|
+
* failed`, which is precisely the transient per-item roster lookup that
|
|
2357
|
+
* wedged the second instance (#291). A rule keyed on builtin types would
|
|
2358
|
+
* have preserved that outage verbatim.
|
|
2359
|
+
*
|
|
2360
|
+
* Everything else — a refused lifecycle claim, a terminal lifecycle record,
|
|
2361
|
+
* a transient network fault on one issue — is per-item: record a skip and
|
|
2362
|
+
* keep going. The unclassified-failure fuse in `#performRunOnce` is the
|
|
2363
|
+
* backstop for a pass-wide fault that does not announce itself as one.
|
|
2364
|
+
*/
|
|
2365
|
+
#isPassFatalFailure(error, dryRun) {
|
|
2366
|
+
// Sweep-scoped: these are about this process's right or ability to run the
|
|
2367
|
+
// pass at all, so they hold for a dry run exactly as for a live one.
|
|
2368
|
+
if (this.#discoverySweepLeaseLost || this.#discoveryOverloadError !== undefined || this.#stopping) {
|
|
2369
|
+
return true;
|
|
2370
|
+
}
|
|
2371
|
+
// Fleet-scoped, and therefore live-only. See the doc comment above.
|
|
2372
|
+
return !dryRun && this.#isFleetControlPlaneHalted(error);
|
|
2373
|
+
}
|
|
2374
|
+
/**
|
|
2375
|
+
* Whether dispatch is globally paused by the fleet control-plane circuit.
|
|
2376
|
+
*
|
|
2377
|
+
* Two reads, because the circuit announces itself two different ways. The
|
|
2378
|
+
* state read covers `guardedMutation`, which records a mutation's own
|
|
2379
|
+
* transport failure and rethrows the *original* error rather than the
|
|
2380
|
+
* circuit-open type — converting it there would be wrong, since the mutation
|
|
2381
|
+
* may already have reached the broker and callers key spawn-failure handling
|
|
2382
|
+
* off that original error. The type check covers a rejection raised without
|
|
2383
|
+
* any state transition, such as an already-open circuit refusing admission.
|
|
2384
|
+
*/
|
|
2385
|
+
#isFleetControlPlaneHalted(error) {
|
|
2386
|
+
return this.#fleetControlPlane.status().state !== 'closed' ||
|
|
2387
|
+
wrapsErrorOfType(error, FleetControlPlaneCircuitOpenError);
|
|
2388
|
+
}
|
|
2205
2389
|
#startDiscoverySweepRenewal(epoch) {
|
|
2206
2390
|
this.#discoverySweepRenewTimer = setInterval(() => {
|
|
2207
2391
|
if (this.#discoverySweepRenewalInFlight || this.#discoverySweepLeaseLost)
|
|
@@ -3085,6 +3269,7 @@ export class FactoryLoop {
|
|
|
3085
3269
|
const maxConsecutiveFailures = Math.min(5, Math.max(1, Math.trunc(opts.maxConsecutiveFailures ?? this.#config.loop.maxConsecutiveFailures)));
|
|
3086
3270
|
const heartbeatPath = opts.heartbeatPath ?? this.#config.loop.heartbeatPath;
|
|
3087
3271
|
const registryPath = opts.registryPath ?? this.#config.loop.registryPath;
|
|
3272
|
+
this.#loopReapPaths = { heartbeatPath, registryPath };
|
|
3088
3273
|
const reports = [];
|
|
3089
3274
|
let consecutiveFailures = 0;
|
|
3090
3275
|
let completed = false;
|
|
@@ -3140,6 +3325,7 @@ export class FactoryLoop {
|
|
|
3140
3325
|
return reports;
|
|
3141
3326
|
}
|
|
3142
3327
|
finally {
|
|
3328
|
+
this.#loopReapPaths = undefined;
|
|
3143
3329
|
if (!completed) {
|
|
3144
3330
|
await this.#writeLoopHeartbeat(heartbeatPath, registryPath, 'stopping', reports.length, maxIterations);
|
|
3145
3331
|
}
|
|
@@ -4142,10 +4328,11 @@ export class FactoryLoop {
|
|
|
4142
4328
|
seed.decision = decisionWithLifecycleBranches(seed.decision, seed.runId);
|
|
4143
4329
|
const claim = await this.#state.claimDispatchLifecycle(this.#workspaceId, key, seed, this.#dispatchLifecycleOwner, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
4144
4330
|
if (!claim.acquired || !claim.lease) {
|
|
4145
|
-
const
|
|
4331
|
+
const terminal = isTerminalDispatchLifecycle(claim.lifecycle);
|
|
4332
|
+
const reason = terminal
|
|
4146
4333
|
? 'dispatch lifecycle is already terminal'
|
|
4147
4334
|
: `dispatch lifecycle is owned by ${claim.lifecycle.lease?.owner ?? 'another publisher'}`;
|
|
4148
|
-
throw new
|
|
4335
|
+
throw new DispatchLifecycleClaimRefusedError(decision.issue.key, terminal ? 'terminal' : 'owned-elsewhere', `Refusing to dispatch ${decision.issue.key}: ${reason}`);
|
|
4149
4336
|
}
|
|
4150
4337
|
this.#dispatchLifecycleEpochs.set(claim.key ?? key, claim.lease.epoch);
|
|
4151
4338
|
this.#hydrateCostLedger(claim.lifecycle);
|
|
@@ -5008,7 +5195,7 @@ export class FactoryLoop {
|
|
|
5008
5195
|
for (const [name] of record.agents) {
|
|
5009
5196
|
this.#fleet.markAgentTerminal?.(name, 'durable-dispatch-abandoned');
|
|
5010
5197
|
}
|
|
5011
|
-
await this.#
|
|
5198
|
+
await this.#retireSlackWatcher(record);
|
|
5012
5199
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
5013
5200
|
await this.#writeInFlightRegistry();
|
|
5014
5201
|
this.#increment('dispatchLifecycleStaleIssuesAbandoned');
|
|
@@ -6131,7 +6318,7 @@ export class FactoryLoop {
|
|
|
6131
6318
|
},
|
|
6132
6319
|
});
|
|
6133
6320
|
}
|
|
6134
|
-
async #reapDispatchFailureHandoffsNow(heartbeatPath, registryPath) {
|
|
6321
|
+
async #reapDispatchFailureHandoffsNow(heartbeatPath = this.#loopReapPaths?.heartbeatPath ?? this.#config.loop.heartbeatPath, registryPath = this.#loopReapPaths?.registryPath ?? this.#config.loop.registryPath) {
|
|
6135
6322
|
const handoffs = await this.#state.listFailureHandoffs(this.#workspaceId);
|
|
6136
6323
|
if (handoffs.length === 0) {
|
|
6137
6324
|
return;
|
|
@@ -7149,6 +7336,7 @@ export class FactoryLoop {
|
|
|
7149
7336
|
}
|
|
7150
7337
|
async #publishImplementerPullRequest(record, implementer, opts = {}) {
|
|
7151
7338
|
const key = `${issueKey(record.issue)}:${implementer.spec.repo}`;
|
|
7339
|
+
const trajectorySessionRef = canonicalTrajectorySessionRef(implementer.sessionRef);
|
|
7152
7340
|
const expectedHeadRef = implementer.spec.branch;
|
|
7153
7341
|
if (!expectedHeadRef) {
|
|
7154
7342
|
throw new Error(`Refusing to publish ${record.issue.key}: implementer has no Factory-derived branch`);
|
|
@@ -7214,7 +7402,7 @@ export class FactoryLoop {
|
|
|
7214
7402
|
expectedHeadRef,
|
|
7215
7403
|
baseRef,
|
|
7216
7404
|
title: `${issue.key}: ${issue.title}`,
|
|
7217
|
-
body: githubPullRequestBody(issue, implementer.spec.preview),
|
|
7405
|
+
body: githubPullRequestBody(issue, implementer.spec.preview, trajectorySessionRef),
|
|
7218
7406
|
...(implementer.sessionRef ? { sessionRef: implementer.sessionRef } : {}),
|
|
7219
7407
|
});
|
|
7220
7408
|
const published = result.author
|
|
@@ -7803,7 +7991,7 @@ export class FactoryLoop {
|
|
|
7803
7991
|
await this.#recordDispatchTerminal(record.issue);
|
|
7804
7992
|
const next = (await this.#batch()).complete(record.issue);
|
|
7805
7993
|
await this.#drainReadyClarificationWake();
|
|
7806
|
-
await this.#
|
|
7994
|
+
await this.#retireSlackWatcher(record);
|
|
7807
7995
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
7808
7996
|
await this.#writeInFlightRegistry();
|
|
7809
7997
|
if (next) {
|
|
@@ -11763,7 +11951,7 @@ export class FactoryLoop {
|
|
|
11763
11951
|
}
|
|
11764
11952
|
if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, releaseReason))
|
|
11765
11953
|
return;
|
|
11766
|
-
await this.#
|
|
11954
|
+
await this.#retireSlackWatcher(record);
|
|
11767
11955
|
await this.#stopGithubIssueCommentWatcherForIssue(record.issue);
|
|
11768
11956
|
await this.#recordDispatchTerminal(record.issue);
|
|
11769
11957
|
await this.#finishDurableRelease(record, releaseReason);
|
|
@@ -12241,6 +12429,21 @@ export class FactoryLoop {
|
|
|
12241
12429
|
return;
|
|
12242
12430
|
}
|
|
12243
12431
|
const key = issueKey(record.issue);
|
|
12432
|
+
const previousWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
12433
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
12434
|
+
if (previousWatch?.kind === 'terminal-grace') {
|
|
12435
|
+
// A reopened work unit needs a fresh dispatch notification and a fresh
|
|
12436
|
+
// conversation. Do not let the old grace-period watcher (or its expiry
|
|
12437
|
+
// timer) capture and later tear down the new dispatch.
|
|
12438
|
+
if (!await this.#stopSlackWatcher(record.issue)) {
|
|
12439
|
+
// Fail closed. An undrained reply route still holds the retired thread
|
|
12440
|
+
// and would bind it to this dispatch, delivering a stale human reply to
|
|
12441
|
+
// fresh work. Leave the fence up; the next reconcile retries the drain.
|
|
12442
|
+
this.#logger.warn?.('[factory] deferring Slack dispatch thread for reopened work unit; in-flight reply route not drained', { issue: record.issue.key });
|
|
12443
|
+
this.#increment('slackDispatchThreadsDeferredUndrainedReply');
|
|
12444
|
+
return;
|
|
12445
|
+
}
|
|
12446
|
+
}
|
|
12244
12447
|
const existingThread = await this.#persistedSlackThread(key);
|
|
12245
12448
|
const watcherStart = this.#slackWatcherStarts.get(key);
|
|
12246
12449
|
if (existingThread || watcherStart) {
|
|
@@ -12323,9 +12526,10 @@ export class FactoryLoop {
|
|
|
12323
12526
|
if (existing) {
|
|
12324
12527
|
const sessionRef = owned?.tracked.sessionRef;
|
|
12325
12528
|
const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
|
|
12529
|
+
let rebound = false;
|
|
12326
12530
|
if (owned && sessionRef &&
|
|
12327
|
-
(agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
|
|
12328
|
-
|
|
12531
|
+
(!existing.agent || agentName !== existing.agent.name || (options.forceAgentRebind === true && sessionRef !== existing.agent.sessionRef))) {
|
|
12532
|
+
rebound = await this.#state.rebindConversationSession(this.#workspaceId, conversationId, {
|
|
12329
12533
|
name: agentName,
|
|
12330
12534
|
sessionRef,
|
|
12331
12535
|
role: owned.tracked.spec.role,
|
|
@@ -12339,40 +12543,39 @@ export class FactoryLoop {
|
|
|
12339
12543
|
}
|
|
12340
12544
|
if (existing.pending.length > 0 || existing.delivery) {
|
|
12341
12545
|
const waiting = await this.#state.getWaitingClarification(this.#workspaceId, issueKey(existing.issue));
|
|
12342
|
-
if (!waiting)
|
|
12546
|
+
if (!waiting && (existing.agent || rebound))
|
|
12343
12547
|
this.#slackConversationTurns.schedule(conversationId);
|
|
12344
12548
|
}
|
|
12345
12549
|
return;
|
|
12346
12550
|
}
|
|
12347
12551
|
const sessionRef = owned?.tracked.sessionRef;
|
|
12348
|
-
if (!owned || !sessionRef) {
|
|
12349
|
-
this.#increment('slackConversationSessionsSkippedMissingSession');
|
|
12350
|
-
return;
|
|
12351
|
-
}
|
|
12352
12552
|
const channelDir = await this.#slackChannelDir() ?? this.#config.slack?.channel;
|
|
12353
12553
|
if (!channelDir)
|
|
12354
12554
|
return;
|
|
12355
|
-
const agentName = owned.tracked.result?.name ?? owned.name;
|
|
12555
|
+
const agentName = owned ? (owned.tracked.result?.name ?? owned.name) : undefined;
|
|
12356
12556
|
const reserved = await this.#state.reserveConversationSession(this.#workspaceId, conversationId, {
|
|
12357
12557
|
provider: 'slack',
|
|
12358
12558
|
issue: { ...record.issue },
|
|
12359
12559
|
externalId: threadId,
|
|
12360
12560
|
context: { channelDir },
|
|
12361
|
-
agent: {
|
|
12362
|
-
|
|
12363
|
-
|
|
12364
|
-
|
|
12365
|
-
|
|
12366
|
-
|
|
12367
|
-
|
|
12368
|
-
|
|
12369
|
-
|
|
12561
|
+
...(owned && sessionRef && agentName ? { agent: {
|
|
12562
|
+
name: agentName,
|
|
12563
|
+
sessionRef,
|
|
12564
|
+
role: owned.tracked.spec.role,
|
|
12565
|
+
node: owned.tracked.result?.node ?? owned.tracked.spec.node,
|
|
12566
|
+
capability: owned.tracked.spec.capability,
|
|
12567
|
+
repo: owned.tracked.spec.repo,
|
|
12568
|
+
clonePath: owned.tracked.spec.clonePath,
|
|
12569
|
+
} } : {}),
|
|
12370
12570
|
history: [],
|
|
12371
12571
|
processedMessageIds: [],
|
|
12572
|
+
acknowledgedMessageIds: [],
|
|
12573
|
+
acknowledgementClaims: {},
|
|
12372
12574
|
pending: [],
|
|
12373
12575
|
});
|
|
12374
|
-
if (reserved)
|
|
12375
|
-
this.#increment('slackConversationSessionsOwned');
|
|
12576
|
+
if (reserved) {
|
|
12577
|
+
this.#increment(owned && sessionRef ? 'slackConversationSessionsOwned' : 'slackConversationSessionsReservedUnowned');
|
|
12578
|
+
}
|
|
12376
12579
|
}
|
|
12377
12580
|
// Called right after a babysitter is spawned/reattached for an issue's PR so
|
|
12378
12581
|
// an already-owned Slack conversation session (reserved earlier by the
|
|
@@ -12400,11 +12603,15 @@ export class FactoryLoop {
|
|
|
12400
12603
|
const claimed = await this.#state.claimConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId, this.#clock.now(), SLACK_CONVERSATION_TURN_LEASE_MS);
|
|
12401
12604
|
if (!claimed?.delivery) {
|
|
12402
12605
|
const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
12403
|
-
if (current && (current.pending.length > 0 || current.delivery)) {
|
|
12606
|
+
if (current?.agent && (current.pending.length > 0 || current.delivery)) {
|
|
12404
12607
|
this.#slackConversationTurns.schedule(conversationId, SLACK_CONVERSATION_TURN_RETRY_MS);
|
|
12405
12608
|
}
|
|
12406
12609
|
return;
|
|
12407
12610
|
}
|
|
12611
|
+
if (!claimed.agent) {
|
|
12612
|
+
await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
|
|
12613
|
+
return;
|
|
12614
|
+
}
|
|
12408
12615
|
if (!await this.#ownsActiveSlackConversationIssue(claimed.issue)) {
|
|
12409
12616
|
await this.#state.releaseConversationTurn(this.#workspaceId, conversationId, this.#slackConversationOwner, claimId);
|
|
12410
12617
|
this.#increment('slackConversationTurnsSuppressedStaleOwner');
|
|
@@ -12488,10 +12695,13 @@ export class FactoryLoop {
|
|
|
12488
12695
|
}
|
|
12489
12696
|
}
|
|
12490
12697
|
async #recordSlackConversationResume(session, result) {
|
|
12698
|
+
const sessionAgent = session.agent;
|
|
12699
|
+
if (!sessionAgent)
|
|
12700
|
+
return;
|
|
12491
12701
|
const record = (await this.#batch()).getIssue(session.issue);
|
|
12492
12702
|
if (!record)
|
|
12493
12703
|
return;
|
|
12494
|
-
const entry = [...record.agents.entries()].find(([name, tracked]) => name ===
|
|
12704
|
+
const entry = [...record.agents.entries()].find(([name, tracked]) => name === sessionAgent.name || tracked.result?.name === sessionAgent.name);
|
|
12495
12705
|
if (!entry)
|
|
12496
12706
|
return;
|
|
12497
12707
|
const [previousName, tracked] = entry;
|
|
@@ -12817,7 +13027,14 @@ export class FactoryLoop {
|
|
|
12817
13027
|
`Question: ${triageEscalationQuestion(decision, issue)}`,
|
|
12818
13028
|
].join('\n'),
|
|
12819
13029
|
});
|
|
12820
|
-
|
|
13030
|
+
const key = issueKey(decision.issue);
|
|
13031
|
+
await this.#state.setSlackThread(this.#workspaceId, key, root.threadId);
|
|
13032
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
|
|
13033
|
+
kind: 'triage',
|
|
13034
|
+
issue: { ...decision.issue },
|
|
13035
|
+
decision: structuredClone(decision),
|
|
13036
|
+
threadId: root.threadId,
|
|
13037
|
+
});
|
|
12821
13038
|
const replayedResult = await this.#watchSlackThread(escalationWatchRecord(decision), root.threadId);
|
|
12822
13039
|
this.#recordSlackWritebackSuccess('triage-escalation');
|
|
12823
13040
|
return replayedResult;
|
|
@@ -12883,6 +13100,11 @@ export class FactoryLoop {
|
|
|
12883
13100
|
if (!reply || !reply.isThreadReply || reply.threadTs !== threadId || reply.channelDir !== channelDir) {
|
|
12884
13101
|
return;
|
|
12885
13102
|
}
|
|
13103
|
+
if (allowPreExisting &&
|
|
13104
|
+
options.replayAfterMs !== undefined &&
|
|
13105
|
+
slackMessageReceivedAtMs(reply.messageTs, Number.MAX_SAFE_INTEGER) < options.replayAfterMs) {
|
|
13106
|
+
return;
|
|
13107
|
+
}
|
|
12886
13108
|
const replyMessageKey = `${reply.threadTs}:${reply.messageTs}`;
|
|
12887
13109
|
if (seenReplyMessages.has(replyMessageKey)) {
|
|
12888
13110
|
this.#logger.debug?.('[factory] suppressed duplicate Slack reply message', { issue: record.issue.key, path });
|
|
@@ -13061,6 +13283,47 @@ export class FactoryLoop {
|
|
|
13061
13283
|
this.#slackConversationTurns.schedule(conversationId);
|
|
13062
13284
|
}
|
|
13063
13285
|
}
|
|
13286
|
+
for (const [key, watch] of await this.#state.listSlackThreadWatches(this.#workspaceId)) {
|
|
13287
|
+
if (this.#slackWatchers.has(key) || this.#slackWatcherStarts.has(key))
|
|
13288
|
+
continue;
|
|
13289
|
+
if (watch.kind === 'terminal-grace' && watch.expiresAtMs <= this.#clock.now()) {
|
|
13290
|
+
await this.#stopSlackWatcher(watch.issue);
|
|
13291
|
+
continue;
|
|
13292
|
+
}
|
|
13293
|
+
await this.#state.setSlackThread(this.#workspaceId, key, watch.threadId);
|
|
13294
|
+
const watchRecord = escalationWatchRecord(watch.decision);
|
|
13295
|
+
if (watch.kind === 'terminal-grace') {
|
|
13296
|
+
const retiredAtMs = terminalSlackWatchRetiredAtMs(watch);
|
|
13297
|
+
if (watch.retiredAtMs !== retiredAtMs) {
|
|
13298
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, { ...watch, retiredAtMs });
|
|
13299
|
+
}
|
|
13300
|
+
this.#terminalSlackWatchIssues.add(key);
|
|
13301
|
+
const conversationId = slackConversationId(watch.threadId);
|
|
13302
|
+
await this.#slackConversationTurns.cancel(conversationId);
|
|
13303
|
+
try {
|
|
13304
|
+
await this.#surfaceUndeliveredSlackConversation(watch.threadId);
|
|
13305
|
+
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13306
|
+
}
|
|
13307
|
+
catch (error) {
|
|
13308
|
+
// The undelivered-reply receipt needs Slack writeback, which may be
|
|
13309
|
+
// unavailable at startup. That is retryable state maintenance for this
|
|
13310
|
+
// one thread, not a reason to abandon rehydration: aborting here would
|
|
13311
|
+
// leave every remaining thread watched by nobody. Keep the queued
|
|
13312
|
+
// replies (clearing them now would drop replies nobody was told about)
|
|
13313
|
+
// and carry on re-arming.
|
|
13314
|
+
this.#logger.warn?.('[factory] failed to settle undelivered Slack replies for terminal watch; will retry', { issue: watch.issue.key, error });
|
|
13315
|
+
this.#increment('slackTerminalWatchReceiptsDeferred');
|
|
13316
|
+
this.#scheduleSlackTerminalReceiptRetry(watch.issue, watch.threadId, watch.expiresAtMs);
|
|
13317
|
+
}
|
|
13318
|
+
await this.#rearmSlackWatcher(watchRecord, watch.threadId, {
|
|
13319
|
+
replayConversationReplies: true,
|
|
13320
|
+
replayAfterMs: retiredAtMs,
|
|
13321
|
+
});
|
|
13322
|
+
this.#scheduleSlackTerminalWatchExpiry(watch.issue, watch.expiresAtMs);
|
|
13323
|
+
continue;
|
|
13324
|
+
}
|
|
13325
|
+
await this.#rearmSlackWatcher(watchRecord, watch.threadId, { replayConversationReplies: true });
|
|
13326
|
+
}
|
|
13064
13327
|
await this.#sweepWaitingClarifications();
|
|
13065
13328
|
for (const [, waiting] of await this.#state.listWaitingClarifications(this.#workspaceId)) {
|
|
13066
13329
|
if (!waiting.threadId)
|
|
@@ -13206,8 +13469,73 @@ export class FactoryLoop {
|
|
|
13206
13469
|
this.#clarificationSweepTimer = timer;
|
|
13207
13470
|
this.#clarificationSweepDueAtMs = dueAtMs;
|
|
13208
13471
|
}
|
|
13472
|
+
// The terminal fence is the only thing that makes an in-flight reply route
|
|
13473
|
+
// answer "no active agent" instead of binding the retired thread to whatever
|
|
13474
|
+
// dispatch owns this key. Routes are chained per work unit, so awaiting the
|
|
13475
|
+
// newest one drains every reply queued behind it.
|
|
13476
|
+
async #drainSlackReplyRoutes(key) {
|
|
13477
|
+
// Snapshotting #slackReplyRoutes is not enough on its own. A reply handler
|
|
13478
|
+
// that is still inside its mount read when the drain starts registers its
|
|
13479
|
+
// route *after* the snapshot, so it would run once the fence is gone and
|
|
13480
|
+
// bind the retired thread to the next dispatch — the same escape one level
|
|
13481
|
+
// in. Bar *routing* for this key first (the bar and the registration are
|
|
13482
|
+
// both synchronous, so nothing can slip between them), then drain whatever
|
|
13483
|
+
// is already chained, then prove the set is empty before reporting success.
|
|
13484
|
+
// A barred reply still answers the human, and that writeback registers here
|
|
13485
|
+
// like any other effect, so the extra passes are what pick it up: quiescence
|
|
13486
|
+
// means every effect this work unit started has settled, not merely the ones
|
|
13487
|
+
// that existed when the drain began.
|
|
13488
|
+
const nested = this.#slackReplyRouteDrains.has(key);
|
|
13489
|
+
this.#slackReplyRouteDrains.add(key);
|
|
13490
|
+
try {
|
|
13491
|
+
for (let pass = 0; pass < SLACK_REPLY_ROUTE_DRAIN_PASSES; pass += 1) {
|
|
13492
|
+
const route = this.#slackReplyRoutes.get(key);
|
|
13493
|
+
if (!route)
|
|
13494
|
+
return true;
|
|
13495
|
+
try {
|
|
13496
|
+
await route;
|
|
13497
|
+
}
|
|
13498
|
+
catch (error) {
|
|
13499
|
+
// A route that *rejects* is not drained: the watcher replays it after
|
|
13500
|
+
// SLACK_REPLY_ROUTE_RETRY_MS, and that replay would land on the next
|
|
13501
|
+
// dispatch. Fail closed and let the caller keep the fence up rather
|
|
13502
|
+
// than leak a stale human reply onto fresh work.
|
|
13503
|
+
this.#logger.warn?.('[factory] in-flight Slack reply route did not drain; keeping terminal Slack fence', { issue: key, error });
|
|
13504
|
+
this.#increment('slackReplyRouteDrainsFailed');
|
|
13505
|
+
return false;
|
|
13506
|
+
}
|
|
13507
|
+
// The owner clears its own entry when it settles; retiring it here too
|
|
13508
|
+
// keeps the loop monotonic if that finally has not run yet.
|
|
13509
|
+
if (this.#slackReplyRoutes.get(key) === route)
|
|
13510
|
+
this.#slackReplyRoutes.delete(key);
|
|
13511
|
+
}
|
|
13512
|
+
// Not provably quiescent. Fail closed for the same reason as a rejection.
|
|
13513
|
+
this.#logger.warn?.('[factory] Slack reply routes did not quiesce; keeping terminal Slack fence', { issue: key });
|
|
13514
|
+
this.#increment('slackReplyRouteDrainsFailed');
|
|
13515
|
+
return false;
|
|
13516
|
+
}
|
|
13517
|
+
finally {
|
|
13518
|
+
if (!nested)
|
|
13519
|
+
this.#slackReplyRouteDrains.delete(key);
|
|
13520
|
+
}
|
|
13521
|
+
}
|
|
13209
13522
|
async #stopSlackWatcher(issue) {
|
|
13210
13523
|
const key = issueKey(issue);
|
|
13524
|
+
// Drain before clearing the fence. Clearing it first lets a reply that is
|
|
13525
|
+
// already mid-route — or one queued behind it — fall through the fence check
|
|
13526
|
+
// in #routeSlackConversationAnswerUnlocked and rebind the retired thread to
|
|
13527
|
+
// the next dispatch of this work unit.
|
|
13528
|
+
if (!await this.#drainSlackReplyRoutes(key))
|
|
13529
|
+
return false;
|
|
13530
|
+
this.#terminalSlackWatchIssues.delete(key);
|
|
13531
|
+
const expiryTimer = this.#slackTerminalWatchExpiryTimers.get(key);
|
|
13532
|
+
if (expiryTimer)
|
|
13533
|
+
clearTimeout(expiryTimer);
|
|
13534
|
+
this.#slackTerminalWatchExpiryTimers.delete(key);
|
|
13535
|
+
const receiptRetryTimer = this.#slackTerminalReceiptRetryTimers.get(key);
|
|
13536
|
+
if (receiptRetryTimer)
|
|
13537
|
+
clearTimeout(receiptRetryTimer);
|
|
13538
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
13211
13539
|
const watcher = this.#slackWatchers.get(key);
|
|
13212
13540
|
this.#slackWatchers.delete(key);
|
|
13213
13541
|
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
@@ -13218,6 +13546,259 @@ export class FactoryLoop {
|
|
|
13218
13546
|
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13219
13547
|
}
|
|
13220
13548
|
await this.#state.clearSlackThread(this.#workspaceId, key);
|
|
13549
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, key);
|
|
13550
|
+
return true;
|
|
13551
|
+
}
|
|
13552
|
+
async #retireSlackWatcher(record) {
|
|
13553
|
+
const key = issueKey(record.issue);
|
|
13554
|
+
const threadId = await this.#state.getSlackThread(this.#workspaceId, key);
|
|
13555
|
+
if (!threadId) {
|
|
13556
|
+
await this.#stopSlackWatcher(record.issue);
|
|
13557
|
+
return;
|
|
13558
|
+
}
|
|
13559
|
+
const existingWatch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
13560
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
13561
|
+
const retiredAtMs = existingWatch?.kind === 'terminal-grace'
|
|
13562
|
+
? terminalSlackWatchRetiredAtMs(existingWatch)
|
|
13563
|
+
: this.#clock.now();
|
|
13564
|
+
const expiresAtMs = existingWatch?.kind === 'terminal-grace'
|
|
13565
|
+
? existingWatch.expiresAtMs
|
|
13566
|
+
: retiredAtMs + SLACK_TERMINAL_THREAD_GRACE_MS;
|
|
13567
|
+
await this.#state.setSlackThreadWatch(this.#workspaceId, key, {
|
|
13568
|
+
kind: 'terminal-grace',
|
|
13569
|
+
issue: { ...record.issue },
|
|
13570
|
+
decision: structuredClone(record.decision),
|
|
13571
|
+
threadId,
|
|
13572
|
+
retiredAtMs,
|
|
13573
|
+
expiresAtMs,
|
|
13574
|
+
});
|
|
13575
|
+
// A terminal thread must never retain a resumable session for an agent that
|
|
13576
|
+
// has already exited. Keep only the exact-thread listener so a late human
|
|
13577
|
+
// reply receives the explicit no-active-agent writeback below.
|
|
13578
|
+
this.#terminalSlackWatchIssues.add(key);
|
|
13579
|
+
await this.#slackReplyRoutes.get(key)?.catch(() => undefined);
|
|
13580
|
+
const conversationId = slackConversationId(threadId);
|
|
13581
|
+
await this.#slackConversationTurns.cancel(conversationId);
|
|
13582
|
+
try {
|
|
13583
|
+
await this.#surfaceUndeliveredSlackConversation(threadId);
|
|
13584
|
+
await this.#state.clearConversationSession(this.#workspaceId, conversationId);
|
|
13585
|
+
}
|
|
13586
|
+
catch (error) {
|
|
13587
|
+
// The receipt fails whenever another handler holds the claim or Slack
|
|
13588
|
+
// writeback is down — neither is a reason to abort retirement. Callers
|
|
13589
|
+
// reach here having already committed the terminal phase and dropped the
|
|
13590
|
+
// pending abandon reason, so a rejection escaping would strand the
|
|
13591
|
+
// registry rewrite, the GitHub watcher stop, and the queued next dispatch
|
|
13592
|
+
// with nothing left to re-run them. Keep the queued replies and let the
|
|
13593
|
+
// retry that owns this receipt settle it inside the grace window.
|
|
13594
|
+
this.#logger.warn?.('[factory] failed to settle undelivered Slack replies while retiring the watcher; will retry', { issue: record.issue.key, error });
|
|
13595
|
+
this.#increment('slackTerminalWatchReceiptsDeferred');
|
|
13596
|
+
this.#scheduleSlackTerminalReceiptRetry(record.issue, threadId, expiresAtMs);
|
|
13597
|
+
}
|
|
13598
|
+
if (!this.#slackWatchers.has(key) && !this.#stopping) {
|
|
13599
|
+
await this.#rearmSlackWatcher(record, threadId);
|
|
13600
|
+
}
|
|
13601
|
+
this.#scheduleSlackTerminalWatchExpiry(record.issue, expiresAtMs);
|
|
13602
|
+
this.#increment('slackTerminalWatchersRetained');
|
|
13603
|
+
}
|
|
13604
|
+
// The durable half of the same record. Every caller here follows the receipt
|
|
13605
|
+
// with a state write (clearing the session), and those two cannot be one
|
|
13606
|
+
// durable step: when the state write fails, the retry that owns it must not
|
|
13607
|
+
// read "replies still queued" as "the human has not been told" and post the
|
|
13608
|
+
// notice again. So the receipt is claimed before the provider write and marked
|
|
13609
|
+
// posted after it, and a retry finds it already settled.
|
|
13610
|
+
async #surfaceUndeliveredSlackConversation(threadId) {
|
|
13611
|
+
const conversationId = slackConversationId(threadId);
|
|
13612
|
+
const session = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13613
|
+
const pendingCount = session
|
|
13614
|
+
? session.pending.length + (session.delivery?.messages.length ?? 0)
|
|
13615
|
+
: 0;
|
|
13616
|
+
if (pendingCount === 0)
|
|
13617
|
+
return;
|
|
13618
|
+
if (session?.terminalReceipt?.posted) {
|
|
13619
|
+
this.#increment('slackTerminalReceiptsAlreadySettled');
|
|
13620
|
+
return;
|
|
13621
|
+
}
|
|
13622
|
+
if (!this.#slack)
|
|
13623
|
+
throw new Error(`Slack thread ${threadId} cannot surface undelivered replies without writeback`);
|
|
13624
|
+
const claimId = randomUUID();
|
|
13625
|
+
if (!await this.#state.claimConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now(), SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS)) {
|
|
13626
|
+
const current = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13627
|
+
if (current?.terminalReceipt?.posted) {
|
|
13628
|
+
this.#increment('slackTerminalReceiptsAlreadySettled');
|
|
13629
|
+
return;
|
|
13630
|
+
}
|
|
13631
|
+
// Another handler is mid-write. Fail closed so the queued replies survive
|
|
13632
|
+
// for whoever settles them rather than racing a second notice onto the
|
|
13633
|
+
// same thread.
|
|
13634
|
+
throw new Error(`Slack thread ${threadId} terminal receipt is claimed by another handler; retrying`);
|
|
13635
|
+
}
|
|
13636
|
+
const noun = pendingCount === 1 ? 'reply' : 'replies';
|
|
13637
|
+
const slack = this.#slack;
|
|
13638
|
+
try {
|
|
13639
|
+
await this.#withRenewedProviderLease('terminal Slack receipt', SLACK_TERMINAL_RECEIPT_CLAIM_LEASE_MS, () => this.#state.renewConversationTerminalReceipt(this.#workspaceId, conversationId, claimId, this.#clock.now()), () => slack.reply(threadId, `Factory could not deliver ${pendingCount} queued ${noun} because this work unit no longer has an active agent. Please continue on the linked issue or pull request.`));
|
|
13640
|
+
}
|
|
13641
|
+
catch (error) {
|
|
13642
|
+
await this.#state.releaseConversationTerminalReceipt(this.#workspaceId, conversationId, claimId);
|
|
13643
|
+
throw error;
|
|
13644
|
+
}
|
|
13645
|
+
if (!await this.#state.completeConversationTerminalReceipt(this.#workspaceId, conversationId, claimId)) {
|
|
13646
|
+
throw new Error(`Slack thread ${threadId} terminal receipt could not be recorded`);
|
|
13647
|
+
}
|
|
13648
|
+
this.#increment('slackConversationRepliesSurfacedTerminal');
|
|
13649
|
+
}
|
|
13650
|
+
// A claim only means something for as long as it outlives the work it covers.
|
|
13651
|
+
// A provider write can legitimately run past a fixed lease, at which point the
|
|
13652
|
+
// claim stops protecting the write it was taken for and a second handler can
|
|
13653
|
+
// post the same thing to the same human. Renewing on a heartbeat scopes the
|
|
13654
|
+
// lease to the work instead of to a guessed duration, and leaves the idle
|
|
13655
|
+
// timeout short enough that a holder that dies mid-write still frees it.
|
|
13656
|
+
//
|
|
13657
|
+
// The heartbeat is bounded on both ends, because a renewal loop that never
|
|
13658
|
+
// stops is a lock with no owner check: a provider write that hangs would hold
|
|
13659
|
+
// the receipt past every retry and past shutdown, and the human whose reply is
|
|
13660
|
+
// queued behind it would be told nothing until the process restarts. So it
|
|
13661
|
+
// stops at the ceiling and it stops when this daemon is stopping, and either
|
|
13662
|
+
// way it says so — from there the claim ages out on its own idle lease and
|
|
13663
|
+
// becomes reclaimable. The write may still land afterwards and duplicate the
|
|
13664
|
+
// notice; a reply nobody can ever reclaim is the worse of the two.
|
|
13665
|
+
async #withRenewedProviderLease(label, leaseMs, renew, run) {
|
|
13666
|
+
const renewUntilMs = this.#clock.now() + SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS;
|
|
13667
|
+
let renewalStopped = false;
|
|
13668
|
+
let renewalInFlight = false;
|
|
13669
|
+
const stopRenewing = (counter, reason) => {
|
|
13670
|
+
renewalStopped = true;
|
|
13671
|
+
this.#increment(counter);
|
|
13672
|
+
this.#logger.warn?.(`[factory] ${label} lease will not be renewed further (${reason}); ` +
|
|
13673
|
+
'its claim expires and the queued replies return to whoever retries them');
|
|
13674
|
+
};
|
|
13675
|
+
const heartbeat = setInterval(() => {
|
|
13676
|
+
if (renewalInFlight || renewalStopped)
|
|
13677
|
+
return;
|
|
13678
|
+
if (this.#stopping) {
|
|
13679
|
+
stopRenewing('slackProviderReceiptLeaseRenewalsStoppedForShutdown', 'shutting down');
|
|
13680
|
+
return;
|
|
13681
|
+
}
|
|
13682
|
+
if (this.#clock.now() >= renewUntilMs) {
|
|
13683
|
+
stopRenewing('slackProviderReceiptLeaseRenewalsExpired', `provider write exceeded ${SLACK_PROVIDER_LEASE_MAX_RENEWAL_MS}ms`);
|
|
13684
|
+
return;
|
|
13685
|
+
}
|
|
13686
|
+
renewalInFlight = true;
|
|
13687
|
+
void renew()
|
|
13688
|
+
.then((renewed) => {
|
|
13689
|
+
if (renewed)
|
|
13690
|
+
return;
|
|
13691
|
+
// Losing the lease mid-write is not recoverable from in here: the
|
|
13692
|
+
// write may already have landed. Stop renewing and let the caller's
|
|
13693
|
+
// completion check fail closed, which keeps the queued replies for
|
|
13694
|
+
// whoever holds the claim now.
|
|
13695
|
+
renewalStopped = true;
|
|
13696
|
+
this.#increment('slackProviderReceiptLeasesLost');
|
|
13697
|
+
this.#logger.warn?.(`[factory] ${label} lease was lost while its provider write was in flight`);
|
|
13698
|
+
})
|
|
13699
|
+
.catch((error) => this.#logger.warn?.(`[factory] ${label} lease renewal failed`, {
|
|
13700
|
+
error: describeError(error).errorMessage,
|
|
13701
|
+
}))
|
|
13702
|
+
.finally(() => { renewalInFlight = false; });
|
|
13703
|
+
}, Math.max(1_000, Math.floor(leaseMs / 3)));
|
|
13704
|
+
heartbeat.unref?.();
|
|
13705
|
+
try {
|
|
13706
|
+
return await run();
|
|
13707
|
+
}
|
|
13708
|
+
finally {
|
|
13709
|
+
clearInterval(heartbeat);
|
|
13710
|
+
}
|
|
13711
|
+
}
|
|
13712
|
+
// A terminal receipt that could not be written leaves the queued replies
|
|
13713
|
+
// pending with the human who wrote them told nothing. That is retryable
|
|
13714
|
+
// maintenance this daemon owns, not work to leave for the next restart: the
|
|
13715
|
+
// grace watch is the only window in which the receipt can still land on the
|
|
13716
|
+
// retired thread, so keep reattempting inside it and give up when it closes.
|
|
13717
|
+
#scheduleSlackTerminalReceiptRetry(issue, threadId, expiresAtMs, attempt = 0) {
|
|
13718
|
+
if (this.#stopping)
|
|
13719
|
+
return;
|
|
13720
|
+
const key = issueKey(issue);
|
|
13721
|
+
const existing = this.#slackTerminalReceiptRetryTimers.get(key);
|
|
13722
|
+
if (existing)
|
|
13723
|
+
clearTimeout(existing);
|
|
13724
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
13725
|
+
const remainingMs = expiresAtMs - this.#clock.now();
|
|
13726
|
+
if (remainingMs <= 0) {
|
|
13727
|
+
this.#increment('slackTerminalWatchReceiptsAbandoned');
|
|
13728
|
+
return;
|
|
13729
|
+
}
|
|
13730
|
+
const backoffMs = Math.min(SLACK_TERMINAL_RECEIPT_RETRY_MAX_MS, SLACK_TERMINAL_RECEIPT_RETRY_MS * 2 ** Math.min(attempt, 16));
|
|
13731
|
+
const timer = setTimeout(() => {
|
|
13732
|
+
this.#slackTerminalReceiptRetryTimers.delete(key);
|
|
13733
|
+
void this.#retrySlackTerminalReceipt(issue, threadId, attempt);
|
|
13734
|
+
}, Math.max(0, Math.min(backoffMs, remainingMs)));
|
|
13735
|
+
timer.unref?.();
|
|
13736
|
+
this.#slackTerminalReceiptRetryTimers.set(key, timer);
|
|
13737
|
+
}
|
|
13738
|
+
async #retrySlackTerminalReceipt(issue, threadId, attempt) {
|
|
13739
|
+
if (this.#stopping)
|
|
13740
|
+
return;
|
|
13741
|
+
const key = issueKey(issue);
|
|
13742
|
+
const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
13743
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
13744
|
+
// The grace watch is gone (expired, or the work unit reopened): the thread
|
|
13745
|
+
// this receipt would settle no longer exists, so there is nothing to say.
|
|
13746
|
+
if (watch?.kind !== 'terminal-grace' || watch.threadId !== threadId)
|
|
13747
|
+
return;
|
|
13748
|
+
try {
|
|
13749
|
+
await this.#surfaceUndeliveredSlackConversation(threadId);
|
|
13750
|
+
await this.#state.clearConversationSession(this.#workspaceId, slackConversationId(threadId));
|
|
13751
|
+
this.#increment('slackTerminalWatchReceiptsRecovered');
|
|
13752
|
+
}
|
|
13753
|
+
catch (error) {
|
|
13754
|
+
this.#logger.warn?.('[factory] terminal Slack receipt retry failed; rescheduling', {
|
|
13755
|
+
issue: issue.key,
|
|
13756
|
+
error,
|
|
13757
|
+
});
|
|
13758
|
+
this.#increment('slackTerminalWatchReceiptRetryFailures');
|
|
13759
|
+
this.#scheduleSlackTerminalReceiptRetry(issue, threadId, watch.expiresAtMs, attempt + 1);
|
|
13760
|
+
}
|
|
13761
|
+
}
|
|
13762
|
+
#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, retryDelayMs) {
|
|
13763
|
+
if (this.#stopping)
|
|
13764
|
+
return;
|
|
13765
|
+
const key = issueKey(issue);
|
|
13766
|
+
const existing = this.#slackTerminalWatchExpiryTimers.get(key);
|
|
13767
|
+
if (existing)
|
|
13768
|
+
clearTimeout(existing);
|
|
13769
|
+
const timer = setTimeout(() => {
|
|
13770
|
+
this.#slackTerminalWatchExpiryTimers.delete(key);
|
|
13771
|
+
void this.#expireSlackTerminalWatcher(issue, expiresAtMs).catch((error) => {
|
|
13772
|
+
this.#logger.warn?.('[factory] failed to expire terminal Slack reply watcher; retrying', {
|
|
13773
|
+
issue: issue.key,
|
|
13774
|
+
error,
|
|
13775
|
+
});
|
|
13776
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
|
|
13777
|
+
});
|
|
13778
|
+
}, retryDelayMs ?? Math.max(0, expiresAtMs - this.#clock.now()));
|
|
13779
|
+
timer.unref?.();
|
|
13780
|
+
this.#slackTerminalWatchExpiryTimers.set(key, timer);
|
|
13781
|
+
}
|
|
13782
|
+
async #expireSlackTerminalWatcher(issue, expiresAtMs) {
|
|
13783
|
+
const key = issueKey(issue);
|
|
13784
|
+
const watch = (await this.#state.listSlackThreadWatches(this.#workspaceId))
|
|
13785
|
+
.find(([watchKey]) => watchKey === key)?.[1];
|
|
13786
|
+
if (watch?.kind !== 'terminal-grace' || watch.expiresAtMs !== expiresAtMs)
|
|
13787
|
+
return;
|
|
13788
|
+
if (watch.expiresAtMs > this.#clock.now()) {
|
|
13789
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, watch.expiresAtMs);
|
|
13790
|
+
return;
|
|
13791
|
+
}
|
|
13792
|
+
// #stopSlackWatcher fails closed when an in-flight reply route will not
|
|
13793
|
+
// drain, leaving the watch and its terminal fence in place. Counting that as
|
|
13794
|
+
// an expiration retires the watch in the metrics while the real one lives
|
|
13795
|
+
// on unwatched by any expiry timer, so reschedule and count only on success.
|
|
13796
|
+
if (!await this.#stopSlackWatcher(issue)) {
|
|
13797
|
+
this.#increment('slackTerminalWatchExpiriesDeferred');
|
|
13798
|
+
this.#scheduleSlackTerminalWatchExpiry(issue, expiresAtMs, SLACK_REPLY_ROUTE_RETRY_MS);
|
|
13799
|
+
return;
|
|
13800
|
+
}
|
|
13801
|
+
this.#increment('slackTerminalWatchersExpired');
|
|
13221
13802
|
}
|
|
13222
13803
|
async #readSlackReply(path) {
|
|
13223
13804
|
try {
|
|
@@ -13282,33 +13863,147 @@ export class FactoryLoop {
|
|
|
13282
13863
|
await this.#wakeWaitingClarification(clarificationKey, claimed);
|
|
13283
13864
|
return;
|
|
13284
13865
|
}
|
|
13866
|
+
return await this.#routeSlackConversationAnswer(record, reply, text, clarificationKey);
|
|
13867
|
+
}
|
|
13868
|
+
async #routeSlackConversationAnswer(record, reply, text, clarificationKey) {
|
|
13869
|
+
if (this.#slackReplyRouteDrains.has(clarificationKey)) {
|
|
13870
|
+
// The terminal fence for this work unit is being drained right now.
|
|
13871
|
+
// Registering an ordinary route here would put it past the drain's
|
|
13872
|
+
// snapshot and run it once the fence is gone. Answer it the way the fence
|
|
13873
|
+
// would have — but as a tracked effect, because this writeback is still a
|
|
13874
|
+
// side effect of this work unit. Left untracked it is the same escape one
|
|
13875
|
+
// level further in: the drain reports quiescence without it, the watcher
|
|
13876
|
+
// stop clears the retry timer that owns this reply, and a slow or failed
|
|
13877
|
+
// receipt leaves the human told nothing at all.
|
|
13878
|
+
this.#increment('slackReplyRoutesFencedDuringDrain');
|
|
13879
|
+
return await this.#trackSlackWorkUnitEffect(clarificationKey, async () => {
|
|
13880
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
13881
|
+
return undefined;
|
|
13882
|
+
});
|
|
13883
|
+
}
|
|
13884
|
+
return await this.#trackSlackWorkUnitEffect(clarificationKey, () => this.#routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey));
|
|
13885
|
+
}
|
|
13886
|
+
// Every Slack side effect a work unit makes on its own behalf runs through
|
|
13887
|
+
// here, so #slackReplyRoutes stays the single record the terminal drain
|
|
13888
|
+
// consults. Effects are chained per work unit: awaiting the newest one drains
|
|
13889
|
+
// everything queued behind it, and a rejection propagates to the drain, which
|
|
13890
|
+
// fails closed rather than tearing the effect's retry path down.
|
|
13891
|
+
async #trackSlackWorkUnitEffect(key, run) {
|
|
13892
|
+
const preceding = this.#slackReplyRoutes.get(key);
|
|
13893
|
+
const effect = (async () => {
|
|
13894
|
+
await preceding?.catch(() => undefined);
|
|
13895
|
+
return await run();
|
|
13896
|
+
})();
|
|
13897
|
+
this.#slackReplyRoutes.set(key, effect);
|
|
13898
|
+
try {
|
|
13899
|
+
return await effect;
|
|
13900
|
+
}
|
|
13901
|
+
finally {
|
|
13902
|
+
if (this.#slackReplyRoutes.get(key) === effect)
|
|
13903
|
+
this.#slackReplyRoutes.delete(key);
|
|
13904
|
+
}
|
|
13905
|
+
}
|
|
13906
|
+
async #routeSlackConversationAnswerUnlocked(record, reply, text, clarificationKey) {
|
|
13907
|
+
if (this.#terminalSlackWatchIssues.has(clarificationKey)) {
|
|
13908
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
13909
|
+
return;
|
|
13910
|
+
}
|
|
13285
13911
|
const conversationId = slackConversationId(reply.threadTs);
|
|
13286
|
-
|
|
13912
|
+
let conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13913
|
+
let liveRecord;
|
|
13914
|
+
if (!conversation) {
|
|
13915
|
+
liveRecord = (await this.#batch()).getIssue(record.issue);
|
|
13916
|
+
if (liveRecord && !liveRecord.dryRun) {
|
|
13917
|
+
await this.#ensureSlackConversationSession(liveRecord, reply.threadTs);
|
|
13918
|
+
conversation = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13919
|
+
}
|
|
13920
|
+
}
|
|
13287
13921
|
if (conversation && issueKey(conversation.issue) === clarificationKey) {
|
|
13922
|
+
const replyId = `${reply.threadTs}:${reply.messageTs}`;
|
|
13288
13923
|
const queued = await this.#state.appendConversationMessage(this.#workspaceId, conversationId, {
|
|
13289
|
-
id:
|
|
13924
|
+
id: replyId,
|
|
13290
13925
|
text,
|
|
13291
13926
|
receivedAtMs: slackMessageReceivedAtMs(reply.messageTs, this.#clock.now()),
|
|
13292
13927
|
providerSequence: reply.messageTs,
|
|
13293
13928
|
author: reply.author,
|
|
13294
13929
|
});
|
|
13295
|
-
|
|
13930
|
+
const durable = queued ?? await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13931
|
+
if (!durable || !durable.processedMessageIds.includes(replyId)) {
|
|
13932
|
+
throw new Error(`Slack reply ${replyId} was not durably queued`);
|
|
13933
|
+
}
|
|
13934
|
+
if (!(durable.acknowledgedMessageIds ?? []).includes(replyId)) {
|
|
13935
|
+
const acknowledgementClaimId = randomUUID();
|
|
13936
|
+
const acknowledgementClaimed = await this.#state.claimConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now(), SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS);
|
|
13937
|
+
if (acknowledgementClaimed) {
|
|
13938
|
+
try {
|
|
13939
|
+
if (!this.#slack)
|
|
13940
|
+
throw new Error(`Slack reply ${replyId} cannot be acknowledged without writeback`);
|
|
13941
|
+
const owner = durable.agent?.role === 'babysitter'
|
|
13942
|
+
? 'the PR babysitter'
|
|
13943
|
+
: durable.agent
|
|
13944
|
+
? 'the issue implementer'
|
|
13945
|
+
: 'an issue agent';
|
|
13946
|
+
const receipt = durable.agent
|
|
13947
|
+
? `Factory received this reply and durably queued it for ${owner}.`
|
|
13948
|
+
: 'Factory received and durably stored this reply; it will route when an issue agent is resumable.';
|
|
13949
|
+
const slack = this.#slack;
|
|
13950
|
+
// Same lease scope as the terminal receipt: this claim covers a
|
|
13951
|
+
// provider write that can outrun any fixed duration, so it is
|
|
13952
|
+
// renewed for as long as that write is actually running.
|
|
13953
|
+
await this.#withRenewedProviderLease('Slack reply acknowledgement', SLACK_REPLY_ACKNOWLEDGEMENT_LEASE_MS, () => this.#state.renewConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId, this.#clock.now()), () => slack.reply(reply.threadTs, receipt));
|
|
13954
|
+
if (!await this.#state.completeConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId)) {
|
|
13955
|
+
throw new Error(`Slack reply ${replyId} receipt could not be recorded`);
|
|
13956
|
+
}
|
|
13957
|
+
this.#increment('slackConversationRepliesAcknowledged');
|
|
13958
|
+
}
|
|
13959
|
+
catch (error) {
|
|
13960
|
+
await this.#state.releaseConversationMessageAcknowledgement(this.#workspaceId, conversationId, replyId, acknowledgementClaimId);
|
|
13961
|
+
throw error;
|
|
13962
|
+
}
|
|
13963
|
+
}
|
|
13964
|
+
else {
|
|
13965
|
+
const acknowledgementState = await this.#state.getConversationSession(this.#workspaceId, conversationId);
|
|
13966
|
+
if (!(acknowledgementState?.acknowledgedMessageIds ?? []).includes(replyId)) {
|
|
13967
|
+
throw new Error(`Slack reply ${replyId} receipt is claimed by another handler; retrying`);
|
|
13968
|
+
}
|
|
13969
|
+
}
|
|
13970
|
+
}
|
|
13971
|
+
if (queued) {
|
|
13972
|
+
this.#increment('slackConversationRepliesQueued');
|
|
13973
|
+
}
|
|
13974
|
+
else {
|
|
13296
13975
|
this.#increment('slackConversationDuplicateRepliesSuppressed');
|
|
13297
|
-
return;
|
|
13298
13976
|
}
|
|
13299
|
-
|
|
13300
|
-
|
|
13977
|
+
const pending = durable.pending.some((message) => message.id === replyId) ||
|
|
13978
|
+
Boolean(durable.delivery?.messages.some((message) => message.id === replyId));
|
|
13979
|
+
if (pending && durable.agent) {
|
|
13980
|
+
this.#slackConversationTurns.schedule(conversationId);
|
|
13981
|
+
}
|
|
13982
|
+
else if (pending) {
|
|
13983
|
+
this.#increment('slackConversationRepliesWaitingForOwner');
|
|
13984
|
+
}
|
|
13301
13985
|
return;
|
|
13302
13986
|
}
|
|
13303
|
-
|
|
13987
|
+
liveRecord ??= (await this.#batch()).getIssue(record.issue);
|
|
13304
13988
|
if (!liveRecord || liveRecord.dryRun) {
|
|
13305
13989
|
if (isTriageEscalationWatchRecord(record)) {
|
|
13306
13990
|
return await this.#handleTriageEscalationSlackAnswer(record, text);
|
|
13307
13991
|
}
|
|
13308
|
-
this.#
|
|
13992
|
+
await this.#writeUnroutableSlackReply(reply.threadTs);
|
|
13309
13993
|
return;
|
|
13310
13994
|
}
|
|
13311
13995
|
this.#increment('slackAnswersIgnoredNoConversationSession');
|
|
13996
|
+
if (this.#slack) {
|
|
13997
|
+
await this.#slack.reply(reply.threadTs, 'Factory received this reply but could not create a durable agent route. It will remain replayable; please also continue on the linked issue or pull request.');
|
|
13998
|
+
this.#increment('slackAnswersUnroutableVisible');
|
|
13999
|
+
}
|
|
14000
|
+
}
|
|
14001
|
+
async #writeUnroutableSlackReply(threadId) {
|
|
14002
|
+
this.#increment('slackAnswersIgnoredNoInFlight');
|
|
14003
|
+
if (!this.#slack)
|
|
14004
|
+
return;
|
|
14005
|
+
await this.#slack.reply(threadId, 'Factory received this reply but could not route it because this work unit no longer has an active agent. Please continue on the linked issue or pull request.');
|
|
14006
|
+
this.#increment('slackAnswersUnroutableVisible');
|
|
13312
14007
|
}
|
|
13313
14008
|
async #wakeWaitingClarification(key, waiting) {
|
|
13314
14009
|
const existing = this.#clarificationWakeInFlight.get(key);
|
|
@@ -13680,6 +14375,7 @@ export class FactoryLoop {
|
|
|
13680
14375
|
const batch = await this.#batch();
|
|
13681
14376
|
if (batch.isInFlight(record.issue) || batch.isQueued(record.issue)) {
|
|
13682
14377
|
this.#increment('slackTriageAnswersIgnoredAlreadyActive');
|
|
14378
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
13683
14379
|
return;
|
|
13684
14380
|
}
|
|
13685
14381
|
if (await this.#dispatchBlockReason(record.issue)) {
|
|
@@ -13696,6 +14392,10 @@ export class FactoryLoop {
|
|
|
13696
14392
|
if (hasDispatchableRoute(decision)) {
|
|
13697
14393
|
this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
|
|
13698
14394
|
const result = await this.#startOrQueueSlackClarifiedDecision(dispatchAfterSlackClarification(decision, escalationReason));
|
|
14395
|
+
const active = await this.#batch();
|
|
14396
|
+
if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
|
|
14397
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
14398
|
+
}
|
|
13699
14399
|
this.#increment('slackTriageAnswersDispatchedWithRemainingEscalation');
|
|
13700
14400
|
return result;
|
|
13701
14401
|
}
|
|
@@ -13708,6 +14408,10 @@ export class FactoryLoop {
|
|
|
13708
14408
|
}
|
|
13709
14409
|
this.#pendingSlackClarifications.set(issueKey(decision.issue), text);
|
|
13710
14410
|
const result = await this.#startOrQueueSlackClarifiedDecision(decision);
|
|
14411
|
+
const active = await this.#batch();
|
|
14412
|
+
if (result || active.isInFlight(decision.issue) || active.isQueued(decision.issue)) {
|
|
14413
|
+
await this.#state.clearSlackThreadWatch(this.#workspaceId, issueKey(record.issue));
|
|
14414
|
+
}
|
|
13711
14415
|
this.#increment('slackTriageAnswersDispatched');
|
|
13712
14416
|
return result;
|
|
13713
14417
|
}
|
|
@@ -16218,8 +16922,8 @@ const normalizeGithubRepo = (repo, defaultOwner) => {
|
|
|
16218
16922
|
}
|
|
16219
16923
|
return `${owner}/${repo}`;
|
|
16220
16924
|
};
|
|
16221
|
-
const githubPullRequestBody = (issue, preview) => [
|
|
16222
|
-
issue.description,
|
|
16925
|
+
const githubPullRequestBody = (issue, preview, sessionRef) => [
|
|
16926
|
+
stripTrajectoryPointers(issue.description),
|
|
16223
16927
|
'',
|
|
16224
16928
|
isGithubIssue(issue) && /^\d+$/u.test(issue.key)
|
|
16225
16929
|
? `Fixes #${issue.key}`
|
|
@@ -16229,7 +16933,25 @@ const githubPullRequestBody = (issue, preview) => [
|
|
|
16229
16933
|
`Live preview: ${preview.url}`,
|
|
16230
16934
|
'Access: Tailscale tailnet membership and the tailnet grants/ACLs are required; this URL is not public.',
|
|
16231
16935
|
] : []),
|
|
16936
|
+
'',
|
|
16937
|
+
renderTrajectoryPointer({
|
|
16938
|
+
...trajectoryWorkUnitForIssue(issue),
|
|
16939
|
+
sessionRef,
|
|
16940
|
+
}),
|
|
16232
16941
|
].join('\n').trim();
|
|
16942
|
+
const trajectoryWorkUnitForIssue = (issue) => {
|
|
16943
|
+
const github = githubIssueSourceRef(issue);
|
|
16944
|
+
if (github) {
|
|
16945
|
+
return {
|
|
16946
|
+
workUnitId: `${github.owner}/${github.repo}#${github.number}`,
|
|
16947
|
+
workUnitSurface: 'github',
|
|
16948
|
+
};
|
|
16949
|
+
}
|
|
16950
|
+
if (isRealLinearIssue(issue)) {
|
|
16951
|
+
return { workUnitId: issue.key, workUnitSurface: 'linear' };
|
|
16952
|
+
}
|
|
16953
|
+
return { workUnitId: `factory:${issue.uuid}`, workUnitSurface: 'factory' };
|
|
16954
|
+
};
|
|
16233
16955
|
// The broker rejects re-registering a name it never released on exit
|
|
16234
16956
|
// (relay#1116-family) with a 500 "agent '<name>' already exists". Detect it from
|
|
16235
16957
|
// the structured payload or the message so resume can treat it as terminal
|
|
@@ -16258,6 +16980,9 @@ const slackMessageReceivedAtMs = (messageTs, fallback) => {
|
|
|
16258
16980
|
const seconds = Number(messageTs);
|
|
16259
16981
|
return Number.isFinite(seconds) && seconds > 0 ? Math.floor(seconds * 1_000) : fallback;
|
|
16260
16982
|
};
|
|
16983
|
+
const terminalSlackWatchRetiredAtMs = (watch) => typeof watch.retiredAtMs === 'number' && Number.isFinite(watch.retiredAtMs)
|
|
16984
|
+
? watch.retiredAtMs
|
|
16985
|
+
: Math.max(0, watch.expiresAtMs - SLACK_TERMINAL_THREAD_GRACE_MS);
|
|
16261
16986
|
const eventIdentity = (event) => {
|
|
16262
16987
|
const record = event;
|
|
16263
16988
|
const rawId = record.id ?? record.event_id ?? record.seq;
|
|
@@ -16569,6 +17294,55 @@ export class LiveDispatchStateChangedError extends Error {
|
|
|
16569
17294
|
export function isLiveDispatchStateChangedError(error) {
|
|
16570
17295
|
return error instanceof LiveDispatchStateChangedError;
|
|
16571
17296
|
}
|
|
17297
|
+
/** How deep to follow `cause` when classifying a wrapped failure. */
|
|
17298
|
+
const PASS_FATAL_CAUSE_DEPTH = 4;
|
|
17299
|
+
/**
|
|
17300
|
+
* Whether `error`, or anything it wraps, is an instance of `type`.
|
|
17301
|
+
* `contextualError` and the fleet control-plane guard both rethrow wrapped, so
|
|
17302
|
+
* classification has to follow the cause chain rather than trust the outermost
|
|
17303
|
+
* type.
|
|
17304
|
+
*/
|
|
17305
|
+
const wrapsErrorOfType = (error, type, depth = 0) => {
|
|
17306
|
+
if (depth > PASS_FATAL_CAUSE_DEPTH || !(error instanceof Error))
|
|
17307
|
+
return false;
|
|
17308
|
+
if (error instanceof type)
|
|
17309
|
+
return true;
|
|
17310
|
+
return wrapsErrorOfType(error.cause, type, depth + 1);
|
|
17311
|
+
};
|
|
17312
|
+
/**
|
|
17313
|
+
* How many *unclassified* per-item failures without an intervening successful
|
|
17314
|
+
* dispatch end the pass. Named per-item conditions (a lifecycle claim refusal,
|
|
17315
|
+
* a live-state race) never count toward it and never reset it: those
|
|
17316
|
+
* legitimately affect many units at once and are exactly the benign case #292
|
|
17317
|
+
* asks the loop to survive, so they are neither evidence of a pass-wide fault
|
|
17318
|
+
* nor evidence against one.
|
|
17319
|
+
*/
|
|
17320
|
+
const UNCLASSIFIED_DISPATCH_FAILURE_LIMIT = 5;
|
|
17321
|
+
/**
|
|
17322
|
+
* Failures the loop recognizes as belonging to one work unit. They are always
|
|
17323
|
+
* skippable and are exempt from the consecutive-failure fuse.
|
|
17324
|
+
*/
|
|
17325
|
+
const isClassifiedPerItemDispatchFailure = (error) => error instanceof LiveDispatchStateChangedError ||
|
|
17326
|
+
error instanceof DispatchLifecycleClaimRefusedError;
|
|
17327
|
+
/**
|
|
17328
|
+
* The run-report reason recorded for a work unit the pass could not dispatch.
|
|
17329
|
+
*
|
|
17330
|
+
* `factory run-once` serializes the whole report to stdout, so this string is
|
|
17331
|
+
* a public surface: it stays a fixed classification plus an allowlisted error
|
|
17332
|
+
* class name, never raw provider text or filesystem paths. The full message
|
|
17333
|
+
* goes to the operator log instead, the same split
|
|
17334
|
+
* `describeControlPlaneError` makes for circuit state.
|
|
17335
|
+
*/
|
|
17336
|
+
const perItemDispatchSkipReason = (error) => {
|
|
17337
|
+
if (error instanceof LiveDispatchStateChangedError)
|
|
17338
|
+
return 'live state changed during dispatch';
|
|
17339
|
+
if (error instanceof DispatchLifecycleClaimRefusedError) {
|
|
17340
|
+
return error.refusal === 'terminal'
|
|
17341
|
+
? 'dispatch lifecycle already terminal'
|
|
17342
|
+
: 'dispatch lifecycle owned by another publisher';
|
|
17343
|
+
}
|
|
17344
|
+
return `dispatch failed (${telemetryErrorClass(error)})`;
|
|
17345
|
+
};
|
|
16572
17346
|
const triageEscalationQuestion = (decision, issue) => {
|
|
16573
17347
|
const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean);
|
|
16574
17348
|
const subject = issue?.title?.trim() || decision.issue.key;
|