@agent-relay/factory 0.1.79 → 0.1.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -5
- package/dist/cli/fleet.d.ts +8 -2
- package/dist/cli/fleet.d.ts.map +1 -1
- package/dist/cli/fleet.js +40 -4
- package/dist/cli/fleet.js.map +1 -1
- package/dist/config/schema.d.ts +9 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +14 -3
- package/dist/config/schema.js.map +1 -1
- package/dist/fleet/relay-fleet-client.d.ts +5 -0
- package/dist/fleet/relay-fleet-client.d.ts.map +1 -1
- package/dist/fleet/relay-fleet-client.js +57 -12
- package/dist/fleet/relay-fleet-client.js.map +1 -1
- package/dist/github/gh-identity.d.ts +44 -0
- package/dist/github/gh-identity.d.ts.map +1 -0
- package/dist/github/gh-identity.js +31 -0
- package/dist/github/gh-identity.js.map +1 -0
- package/dist/github/index.d.ts +3 -1
- package/dist/github/index.d.ts.map +1 -1
- package/dist/github/index.js +2 -1
- package/dist/github/index.js.map +1 -1
- package/dist/github/merge-gate.d.ts +22 -1
- package/dist/github/merge-gate.d.ts.map +1 -1
- package/dist/github/merge-gate.js +103 -24
- package/dist/github/merge-gate.js.map +1 -1
- package/dist/github/probe-closer.d.ts +4 -3
- package/dist/github/probe-closer.d.ts.map +1 -1
- package/dist/github/probe-closer.js +37 -28
- package/dist/github/probe-closer.js.map +1 -1
- package/dist/github/standalone-babysitter.d.ts +7 -3
- package/dist/github/standalone-babysitter.d.ts.map +1 -1
- package/dist/github/standalone-babysitter.js +16 -46
- package/dist/github/standalone-babysitter.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/intake/notion.d.ts +36 -6
- package/dist/intake/notion.d.ts.map +1 -1
- package/dist/intake/notion.js +74 -16
- package/dist/intake/notion.js.map +1 -1
- package/dist/orchestrator/factory.d.ts +32 -5
- package/dist/orchestrator/factory.d.ts.map +1 -1
- package/dist/orchestrator/factory.js +424 -243
- package/dist/orchestrator/factory.js.map +1 -1
- package/dist/ports/fleet.d.ts +6 -0
- package/dist/ports/fleet.d.ts.map +1 -1
- package/dist/ports/mount.d.ts +1 -0
- package/dist/ports/mount.d.ts.map +1 -1
- package/dist/ports/state.d.ts +17 -0
- package/dist/ports/state.d.ts.map +1 -1
- package/dist/state/file-state-store.d.ts +1 -0
- package/dist/state/file-state-store.d.ts.map +1 -1
- package/dist/state/file-state-store.js +22 -1
- package/dist/state/file-state-store.js.map +1 -1
- package/dist/state/in-memory-state-store.d.ts +1 -0
- package/dist/state/in-memory-state-store.d.ts.map +1 -1
- package/dist/state/in-memory-state-store.js +15 -1
- package/dist/state/in-memory-state-store.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -6,7 +6,7 @@ import { DEFAULT_DISCOVERY_SWEEP_BUDGET_MS, DEFAULT_READINESS_RECONCILE_TIMEOUT_
|
|
|
6
6
|
import { DEFAULT_RELAYFILE_OPERATION_TIMEOUT_MS, RelayfileOperationTimeoutError, relayfileTimeoutWithPhase, withRelayfileCallDeadline, } from '../mount/relayfile-operation-timeout.js';
|
|
7
7
|
import { linearByStatePath, linearByIdPath, linearByUuidPath } from '../constants/linear.js';
|
|
8
8
|
import { stateResolutionFromIds } from '../linear/state-resolver.js';
|
|
9
|
-
import {
|
|
9
|
+
import { GhCliGithubMergeGate, MountedGithubMergeGate, closeProbePr } from '../github/index.js';
|
|
10
10
|
import { factoryGithubIssueCommentDraftName, isFactoryGithubIssueCommentDraftName, isFactoryGithubOperationDraftName, } from '../github/writeback-paths.js';
|
|
11
11
|
import { VerificationPipeline } from '../environments/verification-pipeline.js';
|
|
12
12
|
import { factoryWorktreeIssueSlug, factoryWorktreePath } from '../git/agent-worktree.js';
|
|
@@ -64,8 +64,7 @@ const LIVE_EVENT_DRAIN_BATCH_SIZE = 5;
|
|
|
64
64
|
const COMPLETION_SWEEP_INTERVAL_MS = 15_000;
|
|
65
65
|
const COMPLETION_SWEEP_BATCH_SIZE = 2;
|
|
66
66
|
const PREVIEW_SWEEP_INTERVAL_MS = 60_000;
|
|
67
|
-
const
|
|
68
|
-
const PROBE_PR_GH_CANDIDATE_LIMIT = 200;
|
|
67
|
+
const PROBE_PR_RESOLVED_CACHE_MS = 60_000;
|
|
69
68
|
const PUBLISHED_PR_CONFIRM_ATTEMPTS = 20;
|
|
70
69
|
const PUBLISHED_PR_CONFIRM_DELAY_MS = 100;
|
|
71
70
|
const SLACK_REPLY_EVENTS_LIMIT = 100;
|
|
@@ -119,8 +118,8 @@ const BABYSITTER_PR_SNAPSHOT_DEAD_LETTER_LIMIT = 256;
|
|
|
119
118
|
const BABYSITTER_PR_SNAPSHOT_DRAIN_PER_SWEEP = 16;
|
|
120
119
|
// A durably faulted mount would otherwise log an error per path per sweep.
|
|
121
120
|
const BABYSITTER_PR_SNAPSHOT_ESCALATED_LOG_EVERY = 20;
|
|
122
|
-
// Adoption probes an issue's open PR through the
|
|
123
|
-
//
|
|
121
|
+
// Adoption probes an issue's open PR through the mounted projection, so its
|
|
122
|
+
// full-tree fallback runs on a slower cadence than the completion sweep.
|
|
124
123
|
const BABYSITTER_ORPHAN_SWEEP_INTERVAL_MS = 60_000;
|
|
125
124
|
// Warn once per PR identity that arrives unowned, then fall back to debug. The
|
|
126
125
|
// counter carries the true volume; the log only has to make it discoverable.
|
|
@@ -156,6 +155,8 @@ const STOP_REJECTED_DISPATCH_DRAIN_TIMEOUT_MS = 2_500;
|
|
|
156
155
|
const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000;
|
|
157
156
|
const DISPATCH_LIFECYCLE_RENEW_MS = 60_000;
|
|
158
157
|
const DISPATCH_LIFECYCLE_RETRY_MS = 1_000;
|
|
158
|
+
const REMOTE_AGENT_REGISTRATION_TIMEOUT_MS = 30_000;
|
|
159
|
+
const REMOTE_AGENT_REGISTRATION_POLL_MS = 500;
|
|
159
160
|
/**
|
|
160
161
|
* Ceiling on the durable capacity-wait re-arm (#303).
|
|
161
162
|
*
|
|
@@ -349,6 +350,26 @@ class DispatchLifecycleOwnedElsewhereError extends Error {
|
|
|
349
350
|
this.leaseUntilMs = leaseUntilMs;
|
|
350
351
|
}
|
|
351
352
|
}
|
|
353
|
+
class FleetPlacementUnavailableError extends Error {
|
|
354
|
+
capability;
|
|
355
|
+
constructor(capability) {
|
|
356
|
+
super(`Refusing remote dispatch: no live fleet node advertises ${capability}`);
|
|
357
|
+
this.capability = capability;
|
|
358
|
+
this.name = 'FleetPlacementUnavailableError';
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
class RemoteAgentRegistrationTimeoutError extends Error {
|
|
362
|
+
agentName;
|
|
363
|
+
cleanupConfirmed;
|
|
364
|
+
cleanupError;
|
|
365
|
+
constructor(agentName, cleanupConfirmed, cleanupError) {
|
|
366
|
+
super(`Remote agent ${agentName} did not register with the fleet before the startup deadline`);
|
|
367
|
+
this.agentName = agentName;
|
|
368
|
+
this.cleanupConfirmed = cleanupConfirmed;
|
|
369
|
+
this.cleanupError = cleanupError;
|
|
370
|
+
this.name = 'RemoteAgentRegistrationTimeoutError';
|
|
371
|
+
}
|
|
372
|
+
}
|
|
352
373
|
/**
|
|
353
374
|
* The durable dispatch-lifecycle claim was refused for one work unit: its
|
|
354
375
|
* record is already terminal, or another publisher currently holds the lease.
|
|
@@ -435,10 +456,9 @@ export class FactoryLoop {
|
|
|
435
456
|
#mergeGate;
|
|
436
457
|
#verificationGate;
|
|
437
458
|
#probeCloser;
|
|
459
|
+
#defaultProbeCloser;
|
|
438
460
|
#probePrResolver;
|
|
439
461
|
#customProbePrResolver;
|
|
440
|
-
#hasProbePrGhRunner;
|
|
441
|
-
#probePrGhRunner;
|
|
442
462
|
#logger;
|
|
443
463
|
#clock;
|
|
444
464
|
#processIdentityReader;
|
|
@@ -450,6 +470,7 @@ export class FactoryLoop {
|
|
|
450
470
|
#babysitterWakeUnreachableRetryMs;
|
|
451
471
|
#startupAgentExitDrainTimeoutMs;
|
|
452
472
|
#dispatchLifecycleRetryMs;
|
|
473
|
+
#dispatchLifecycleRenewMs;
|
|
453
474
|
#state;
|
|
454
475
|
#workspaceId;
|
|
455
476
|
#relayflows;
|
|
@@ -764,7 +785,6 @@ export class FactoryLoop {
|
|
|
764
785
|
#publishedPullRequests = new Map();
|
|
765
786
|
#previewReferences = new Map();
|
|
766
787
|
#removedPreviewIds = new Set();
|
|
767
|
-
#probePrGhBackoffUntilMs = new Map();
|
|
768
788
|
#probePrResolvedCache = new Map();
|
|
769
789
|
// GitHub issue mirror-id -> resolved Linear mirror path, so repeat ingestion
|
|
770
790
|
// cycles read the mirror directly instead of re-scanning all Linear issues.
|
|
@@ -857,7 +877,7 @@ export class FactoryLoop {
|
|
|
857
877
|
this.#githubWriteback = ports.githubWriteback ?? defaultGithubWriteback(config, ports.mount);
|
|
858
878
|
this.#slack = config.slack ? MountSlackWriteback(ports.mount, config.slack) : ports.slack;
|
|
859
879
|
this.#github = ports.github ?? MountGithubRead(ports.mount);
|
|
860
|
-
this.#mergeGate = ports.mergeGate ??
|
|
880
|
+
this.#mergeGate = ports.mergeGate ?? defaultMergeGate(config, ports.mount);
|
|
861
881
|
this.#verificationGate = ports.verificationGate ?? (config.verification.enabled
|
|
862
882
|
? new VerificationPipeline({
|
|
863
883
|
descriptorPath: config.verification.descriptorPath,
|
|
@@ -869,10 +889,9 @@ export class FactoryLoop {
|
|
|
869
889
|
maxTeardownTimeoutMs: config.verification.maxTeardownTimeoutMs,
|
|
870
890
|
})
|
|
871
891
|
: undefined);
|
|
892
|
+
this.#defaultProbeCloser = !ports.probeCloser;
|
|
872
893
|
this.#probeCloser = ports.probeCloser ?? closeProbePr;
|
|
873
894
|
this.#customProbePrResolver = Boolean(ports.probePrResolver);
|
|
874
|
-
this.#hasProbePrGhRunner = Boolean(ports.probePrGhRunner);
|
|
875
|
-
this.#probePrGhRunner = ports.probePrGhRunner ?? failClosedGhRunner;
|
|
876
895
|
this.#probePrResolver = ports.probePrResolver ??
|
|
877
896
|
((issue) => this.#resolveIssuePr(issue, { repo: this.#probeRepoForIssue(issue) }));
|
|
878
897
|
this.#logger = normalizeLogger(ports.logger ?? console);
|
|
@@ -896,6 +915,7 @@ export class FactoryLoop {
|
|
|
896
915
|
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
897
916
|
this.#startupAgentExitDrainTimeoutMs = ports.startupAgentExitDrainTimeoutMs ?? STARTUP_AGENT_EXIT_DRAIN_TIMEOUT_MS;
|
|
898
917
|
this.#dispatchLifecycleRetryMs = ports.dispatchLifecycleRetryMs ?? DISPATCH_LIFECYCLE_RETRY_MS;
|
|
918
|
+
this.#dispatchLifecycleRenewMs = ports.dispatchLifecycleRenewMs ?? DISPATCH_LIFECYCLE_RENEW_MS;
|
|
899
919
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
900
920
|
this.#relayflows = ports.relayflows;
|
|
901
921
|
this.#worktrees = ports.worktrees;
|
|
@@ -2397,11 +2417,6 @@ export class FactoryLoop {
|
|
|
2397
2417
|
}
|
|
2398
2418
|
if (pr.draft) {
|
|
2399
2419
|
this.#increment('completionSweepDraftPr');
|
|
2400
|
-
// Must match the key `#completionPrForIssue` resolves under —
|
|
2401
|
-
// same options, same repo scope — or this backoff is written
|
|
2402
|
-
// under a name nothing reads and the draft PR is re-fetched from
|
|
2403
|
-
// gh on every pass.
|
|
2404
|
-
this.#probePrGhBackoffUntilMs.set(this.#probePrCacheKey(issue, { repo: this.#probeRepoForIssue(issue) }), this.#clock.now() + PROBE_PR_GH_BACKOFF_MS);
|
|
2405
2420
|
return undefined;
|
|
2406
2421
|
}
|
|
2407
2422
|
if (record.decision.implementers.length > 1 && !await this.#allImplementersHaveCompletionPr(record)) {
|
|
@@ -2533,9 +2548,7 @@ export class FactoryLoop {
|
|
|
2533
2548
|
}
|
|
2534
2549
|
}
|
|
2535
2550
|
// Candidate PR meta paths for an ownerless record, newest PR first. Resolution
|
|
2536
|
-
// reuses the durable dispatch receipts and then the probe resolver
|
|
2537
|
-
// carries its own gh backoff, so a record whose implementer has not opened a
|
|
2538
|
-
// PR yet costs no more than a cached miss).
|
|
2551
|
+
// reuses the durable dispatch receipts and then the mounted probe resolver.
|
|
2539
2552
|
async #orphanedPrMetaPaths(record) {
|
|
2540
2553
|
const wanted = [];
|
|
2541
2554
|
if (this.#usesDurableDispatchLifecycle()) {
|
|
@@ -2684,19 +2697,13 @@ export class FactoryLoop {
|
|
|
2684
2697
|
return dependencyRepoForIssue(issue, undefined, this.#config, { allowDefault: false });
|
|
2685
2698
|
}
|
|
2686
2699
|
/**
|
|
2687
|
-
* The key for
|
|
2688
|
-
*
|
|
2689
|
-
* Shared rather than inlined because those two maps are written from more than
|
|
2690
|
-
* one place: `#resolveIssuePr` writes both, and the completion sweep writes a
|
|
2691
|
-
* draft-PR backoff directly. Those writes must agree on the key or the backoff
|
|
2692
|
-
* is set under a name nothing reads, which silently costs a gh call per pass.
|
|
2700
|
+
* The key for `#probePrResolvedCache`.
|
|
2693
2701
|
*
|
|
2694
2702
|
* `repo` is part of the key because it narrows which pull requests the
|
|
2695
2703
|
* resolution can even see — an issue whose route changes must not be served
|
|
2696
|
-
* the previous repository's PR,
|
|
2697
|
-
*
|
|
2698
|
-
*
|
|
2699
|
-
* must not share an entry with any single-repo one.
|
|
2704
|
+
* the previous repository's PR, because the completion path probes and
|
|
2705
|
+
* CLOSES what this returns. `*` marks the unscoped walk, a genuinely
|
|
2706
|
+
* different resolution that must not share an entry with any single-repo one.
|
|
2700
2707
|
*
|
|
2701
2708
|
* Every dimension is a trailing `:`-prefixed segment so the completion
|
|
2702
2709
|
* invalidation — which clears `stateKey` plus everything starting
|
|
@@ -2721,31 +2728,13 @@ export class FactoryLoop {
|
|
|
2721
2728
|
// method. The cache had a reader and no writer on the hot path, so the
|
|
2722
2729
|
// full tree walk repeated for every caller, on every sweep, forever.
|
|
2723
2730
|
//
|
|
2724
|
-
//
|
|
2725
|
-
//
|
|
2726
|
-
// property of the PR (its state is about to flip and the caller wants to
|
|
2727
|
-
// see that promptly), not of which resolver observed it.
|
|
2731
|
+
// Keep drafts uncached because their state is expected to flip and callers
|
|
2732
|
+
// need to observe that promptly.
|
|
2728
2733
|
if (!mountPr.draft) {
|
|
2729
|
-
this.#probePrResolvedCache.set(key, { pr: mountPr, expiresAtMs: now +
|
|
2734
|
+
this.#probePrResolvedCache.set(key, { pr: mountPr, expiresAtMs: now + PROBE_PR_RESOLVED_CACHE_MS });
|
|
2730
2735
|
}
|
|
2731
2736
|
return mountPr;
|
|
2732
2737
|
}
|
|
2733
|
-
const backoffUntil = this.#probePrGhBackoffUntilMs.get(key) ?? 0;
|
|
2734
|
-
if (backoffUntil > now) {
|
|
2735
|
-
this.#increment('probePrGhBackoffSkips');
|
|
2736
|
-
return undefined;
|
|
2737
|
-
}
|
|
2738
|
-
const ghPr = await resolveIssuePrFromGh(this.#probePrGhRunner, this.#config, issue, opts, this.#logger);
|
|
2739
|
-
this.#increment('probePrGhResolveAttempts');
|
|
2740
|
-
if (ghPr) {
|
|
2741
|
-
this.#probePrGhBackoffUntilMs.delete(key);
|
|
2742
|
-
if (!ghPr.draft) {
|
|
2743
|
-
this.#probePrResolvedCache.set(key, { pr: ghPr, expiresAtMs: now + PROBE_PR_GH_BACKOFF_MS });
|
|
2744
|
-
}
|
|
2745
|
-
this.#increment('probePrGhResolveHits');
|
|
2746
|
-
return ghPr;
|
|
2747
|
-
}
|
|
2748
|
-
this.#probePrGhBackoffUntilMs.set(key, now + PROBE_PR_GH_BACKOFF_MS);
|
|
2749
2738
|
return undefined;
|
|
2750
2739
|
}
|
|
2751
2740
|
async runOnce(opts = {}) {
|
|
@@ -2783,8 +2772,9 @@ export class FactoryLoop {
|
|
|
2783
2772
|
}
|
|
2784
2773
|
async #assertFleetControlPlaneAvailable() {
|
|
2785
2774
|
try {
|
|
2786
|
-
await this.#fleet.roster();
|
|
2775
|
+
const roster = await this.#fleet.roster();
|
|
2787
2776
|
this.#increment('fleetControlPlaneProbeSuccesses');
|
|
2777
|
+
return roster;
|
|
2788
2778
|
}
|
|
2789
2779
|
catch (error) {
|
|
2790
2780
|
const health = this.#fleetControlPlane.status();
|
|
@@ -5019,8 +5009,10 @@ export class FactoryLoop {
|
|
|
5019
5009
|
// consuming a dispatch attempt. The mutation proxy probes again at the
|
|
5020
5010
|
// actual spawn/resume boundary so a later control-plane fault still fails
|
|
5021
5011
|
// closed.
|
|
5022
|
-
|
|
5023
|
-
|
|
5012
|
+
const admissionRoster = !dryRun ? await this.#assertFleetControlPlaneAvailable() : undefined;
|
|
5013
|
+
if (admissionRoster && this.#fleet.placementLocality === 'remote') {
|
|
5014
|
+
dispatchDecision = decisionWithVerifiedRemotePlacements(dispatchDecision, admissionRoster);
|
|
5015
|
+
}
|
|
5024
5016
|
const durableDispatch = !dryRun && this.#usesDurableDispatchLifecycle();
|
|
5025
5017
|
// Local dispatches need the same deterministic branch identity as remote
|
|
5026
5018
|
// ones. Without it, every worker starts in the configured shared checkout
|
|
@@ -5437,6 +5429,13 @@ export class FactoryLoop {
|
|
|
5437
5429
|
throw error;
|
|
5438
5430
|
}
|
|
5439
5431
|
settlePostSpawnIssueObservation(!(error instanceof LiveDispatchStateChangedError));
|
|
5432
|
+
if ((error instanceof FleetPlacementUnavailableError ||
|
|
5433
|
+
(error instanceof RemoteAgentRegistrationTimeoutError && error.cleanupConfirmed)) &&
|
|
5434
|
+
await this.#rollbackUnregisteredRemoteDispatch(record, spawnedForReaperHandoff)) {
|
|
5435
|
+
this.#increment('remoteDispatchAdmissionRollbacks');
|
|
5436
|
+
this.#error(error, decision.issue);
|
|
5437
|
+
throw error;
|
|
5438
|
+
}
|
|
5440
5439
|
// A spawn can fail after the broker accepted it but before its ack
|
|
5441
5440
|
// reached Factory. Include every planned worktree agent, not only the
|
|
5442
5441
|
// acknowledged spawns, so cleanup never races a name-only survivor.
|
|
@@ -6188,7 +6187,7 @@ export class FactoryLoop {
|
|
|
6188
6187
|
return;
|
|
6189
6188
|
this.#dispatchLifecycleRenewTimer = setInterval(() => {
|
|
6190
6189
|
void this.#renewDispatchLifecycles();
|
|
6191
|
-
},
|
|
6190
|
+
}, this.#dispatchLifecycleRenewMs);
|
|
6192
6191
|
this.#dispatchLifecycleRenewTimer.unref?.();
|
|
6193
6192
|
}
|
|
6194
6193
|
/**
|
|
@@ -6385,6 +6384,18 @@ export class FactoryLoop {
|
|
|
6385
6384
|
}
|
|
6386
6385
|
async #renewDispatchLifecycles() {
|
|
6387
6386
|
for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) {
|
|
6387
|
+
// The snapshot above can outlive the ownership it records: a relinquish
|
|
6388
|
+
// that runs while this loop is awaiting the store for an earlier key
|
|
6389
|
+
// leaves a stale entry here, and renewing it would re-block a key this
|
|
6390
|
+
// process has already handed back (#391 review, P2). Re-read the live map
|
|
6391
|
+
// rather than trusting the snapshot.
|
|
6392
|
+
//
|
|
6393
|
+
// This narrows the window but does not close it on its own — the delete
|
|
6394
|
+
// can still land after this check and before the renew resolves. Closing
|
|
6395
|
+
// it is `renewDispatchLifecycle`'s expiry fence, which makes a
|
|
6396
|
+
// relinquished lease unrenewable no matter how the two race.
|
|
6397
|
+
if (this.#dispatchLifecycleEpochs.get(key) !== epoch)
|
|
6398
|
+
continue;
|
|
6388
6399
|
const renewed = await this.#state.renewDispatchLifecycle(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch, this.#clock.now(), DISPATCH_LIFECYCLE_LEASE_MS);
|
|
6389
6400
|
if (!renewed) {
|
|
6390
6401
|
this.#dispatchLifecycleEpochs.delete(key);
|
|
@@ -6985,8 +6996,17 @@ export class FactoryLoop {
|
|
|
6985
6996
|
* the only thing still running is the spin.
|
|
6986
6997
|
*/
|
|
6987
6998
|
#chargeReleaseAttempt(record, key, context) {
|
|
6988
|
-
if (this.#dispatchLifecycleReleaseAbandoned.has(key))
|
|
6999
|
+
if (this.#dispatchLifecycleReleaseAbandoned.has(key)) {
|
|
7000
|
+
// Re-entry, and the reason relinquishing the lease once is not enough.
|
|
7001
|
+
// `#driveDispatchLifecycle` re-claims the lease at the TOP of every
|
|
7002
|
+
// drive, before it has read the phase, so anything that drives an
|
|
7003
|
+
// already-dead-lettered key — the held-agent-deadline sweep, a registry
|
|
7004
|
+
// restore, a takeover — puts the epoch straight back into the renewal
|
|
7005
|
+
// map and re-arms the livelock this bound just escaped. Whenever the
|
|
7006
|
+
// budget declines a re-arm, ownership goes back too.
|
|
7007
|
+
this.#trackDispatchLifecycleDrive(this.#relinquishDispatchLifecycleLease(key, record.issue.key));
|
|
6989
7008
|
return false;
|
|
7009
|
+
}
|
|
6990
7010
|
const attempts = (this.#dispatchLifecycleReleaseAttempts.get(key) ?? 0) + 1;
|
|
6991
7011
|
this.#dispatchLifecycleReleaseAttempts.set(key, attempts);
|
|
6992
7012
|
if (attempts <= DISPATCH_LIFECYCLE_MAX_RELEASE_ATTEMPTS)
|
|
@@ -6995,6 +7015,19 @@ export class FactoryLoop {
|
|
|
6995
7015
|
this.#dispatchLifecycleReleaseAttempts.delete(key);
|
|
6996
7016
|
this.#increment('dispatchLifecycleReleaseAbandoned');
|
|
6997
7017
|
const durableLifecycleRetained = this.#usesDurableDispatchLifecycle();
|
|
7018
|
+
// Cleanup is armed BEFORE anything that can throw (#391 review, P1). The
|
|
7019
|
+
// only statements above are set/map writes and a counter, none of which can
|
|
7020
|
+
// reject; `this.#logger.error` below is caller-supplied and can. Ordering
|
|
7021
|
+
// the drive first means nothing between "this unit is abandoned" and "its
|
|
7022
|
+
// lease is handed back" is allowed to fail in a way that skips the handback
|
|
7023
|
+
// — which is precisely the defect shape this whole method exists to fix.
|
|
7024
|
+
this.#trackDispatchLifecycleDrive(this.#releaseDeadLetteredSlot(record, key)
|
|
7025
|
+
.catch((error) => {
|
|
7026
|
+
this.#logger.warn?.('[factory] dead-lettered release could not free its batch slot', {
|
|
7027
|
+
issue: record.issue.key,
|
|
7028
|
+
error: describeError(error).errorMessage,
|
|
7029
|
+
});
|
|
7030
|
+
}));
|
|
6998
7031
|
// `error`, not `warn`. Every previous layer of this failure was invisible
|
|
6999
7032
|
// until somebody read stderr by hand; a work unit whose cleanup this
|
|
7000
7033
|
// process has permanently given up on is exactly the event that must not
|
|
@@ -7009,16 +7042,63 @@ export class FactoryLoop {
|
|
|
7009
7042
|
// spin, it does not declare the work unit clean.
|
|
7010
7043
|
durableLifecycleRetained,
|
|
7011
7044
|
});
|
|
7012
|
-
|
|
7013
|
-
|
|
7014
|
-
|
|
7015
|
-
|
|
7045
|
+
return false;
|
|
7046
|
+
}
|
|
7047
|
+
/** Keeps a lifecycle-side effect awaitable by `stop()` without leaking the set entry. */
|
|
7048
|
+
#trackDispatchLifecycleDrive(promise) {
|
|
7049
|
+
const drive = promise.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
7050
|
+
this.#dispatchLifecycleDrives.add(drive);
|
|
7051
|
+
}
|
|
7052
|
+
/**
|
|
7053
|
+
* Stop asserting durable ownership of a work unit this process will not drive
|
|
7054
|
+
* again.
|
|
7055
|
+
*
|
|
7056
|
+
* `#dispatchLifecycleEpochs` is not merely a cache. `#renewDispatchLifecycles`
|
|
7057
|
+
* walks it every `DISPATCH_LIFECYCLE_RENEW_MS` and re-stamps a full
|
|
7058
|
+
* `DISPATCH_LIFECYCLE_LEASE_MS` onto every key it finds, unconditionally.
|
|
7059
|
+
*
|
|
7060
|
+
* A dead-lettered release deliberately leaves its row in the non-terminal
|
|
7061
|
+
* `releasing` phase so a successor or a restart can re-drive the cleanup with
|
|
7062
|
+
* a fresh budget. That is only a recovery path if the successor can CLAIM the
|
|
7063
|
+
* row — and a lease renewed forever by a process that has permanently given
|
|
7064
|
+
* up driving it is a claim nobody can ever win. #379 bounded the retry and
|
|
7065
|
+
* handed back the batch slot, then kept the key locked for the life of the
|
|
7066
|
+
* process, which is how production reached four issues (a dispatch canary
|
|
7067
|
+
* among them) all logging `durable dispatch is leased by another publisher`
|
|
7068
|
+
* at 1 Hz for three days while the holder logged nothing but 503
|
|
7069
|
+
* `agent_host_unavailable`.
|
|
7070
|
+
*
|
|
7071
|
+
* The epoch is dropped BEFORE the durable release, so a renewal tick that
|
|
7072
|
+
* STARTS after this point finds nothing to renew. That ordering alone is not
|
|
7073
|
+
* sufficient and an earlier version of this comment wrongly claimed it was
|
|
7074
|
+
* (#391 review, P2): `#renewDispatchLifecycles` iterates a snapshot array, so
|
|
7075
|
+
* a tick already in flight still holds this key and would restore the lease
|
|
7076
|
+
* for a full term. The guarantee comes from `renewDispatchLifecycle` fencing
|
|
7077
|
+
* on expiry as well as owner and epoch, which makes a relinquished lease
|
|
7078
|
+
* unrenewable however the two race; the epoch drop and the loop's live re-read
|
|
7079
|
+
* narrow the window ahead of it.
|
|
7080
|
+
*
|
|
7081
|
+
* If the durable release itself fails, the dropped epoch alone still ends the
|
|
7082
|
+
* livelock: nothing renews the lease any more, so it expires within
|
|
7083
|
+
* `DISPATCH_LIFECYCLE_LEASE_MS` instead of never.
|
|
7084
|
+
*/
|
|
7085
|
+
async #relinquishDispatchLifecycleLease(key, issueKey) {
|
|
7086
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
7087
|
+
if (epoch === undefined)
|
|
7088
|
+
return;
|
|
7089
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
7090
|
+
try {
|
|
7091
|
+
await this.#state.releaseDispatchLifecycleLease(this.#workspaceId, key, this.#dispatchLifecycleOwner, epoch);
|
|
7092
|
+
this.#increment('dispatchLifecycleLeasesRelinquished');
|
|
7093
|
+
}
|
|
7094
|
+
catch (error) {
|
|
7095
|
+
this.#logger.warn?.('[factory] could not relinquish the dispatch lease of an abandoned work unit', {
|
|
7096
|
+
issue: issueKey,
|
|
7097
|
+
// The lease still expires on its own now that nothing renews it.
|
|
7098
|
+
expiresWithinMs: DISPATCH_LIFECYCLE_LEASE_MS,
|
|
7016
7099
|
error: describeError(error).errorMessage,
|
|
7017
7100
|
});
|
|
7018
|
-
}
|
|
7019
|
-
.finally(() => this.#dispatchLifecycleDrives.delete(drive));
|
|
7020
|
-
this.#dispatchLifecycleDrives.add(drive);
|
|
7021
|
-
return false;
|
|
7101
|
+
}
|
|
7022
7102
|
}
|
|
7023
7103
|
/**
|
|
7024
7104
|
* Hand back the batch slot of a work unit whose release was dead-lettered.
|
|
@@ -7035,12 +7115,36 @@ export class FactoryLoop {
|
|
|
7035
7115
|
* local slot does not write a terminal phase. On a local lifecycle there is
|
|
7036
7116
|
* no durable record to retain, so the batch record is all there is and
|
|
7037
7117
|
* completing it is what keeps capacity honest.
|
|
7118
|
+
*
|
|
7119
|
+
* The DURABLE lease has to go back with the slot. Retaining the row for a
|
|
7120
|
+
* successor while renewing the lease that locks the successor out is not a
|
|
7121
|
+
* handoff, it is a permanent block on the key — see
|
|
7122
|
+
* `#relinquishDispatchLifecycleLease`.
|
|
7123
|
+
*
|
|
7124
|
+
* The handback lives in a `finally` for a reason worth stating plainly
|
|
7125
|
+
* (#391 review, P1). The FIRST version of this fix relinquished after
|
|
7126
|
+
* `#writeInFlightRegistry()`, on the happy path only — so a rejecting
|
|
7127
|
+
* registry write would skip it and leave the abandoned key renewing its lease
|
|
7128
|
+
* forever, with nothing but a `warn` to show for it. That is the identical
|
|
7129
|
+
* shape of the bug being fixed (#379 freed the slot but not the lease, on the
|
|
7130
|
+
* failure path), reproduced one level up. Cleanup that only runs when the
|
|
7131
|
+
* rest of cleanup succeeded is not cleanup. `#batch()` and
|
|
7132
|
+
* `#writeInFlightRegistry()` can both reject; neither may strand the key.
|
|
7038
7133
|
*/
|
|
7039
7134
|
async #releaseDeadLetteredSlot(record, key) {
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
|
|
7135
|
+
let next;
|
|
7136
|
+
try {
|
|
7137
|
+
const batch = await this.#batch();
|
|
7138
|
+
next = batch.complete(record.issue);
|
|
7139
|
+
this.#uncompensatedDispatchClaims.delete(key);
|
|
7140
|
+
await this.#writeInFlightRegistry();
|
|
7141
|
+
}
|
|
7142
|
+
finally {
|
|
7143
|
+
// Unconditional, and safe to put in a `finally` because
|
|
7144
|
+
// `#relinquishDispatchLifecycleLease` handles its own errors and cannot
|
|
7145
|
+
// throw — so it can never mask the failure that brought us here.
|
|
7146
|
+
await this.#relinquishDispatchLifecycleLease(key, record.issue.key);
|
|
7147
|
+
}
|
|
7044
7148
|
// A freed slot that nothing is admitted into is only half the repair.
|
|
7045
7149
|
if (next && !this.#stopping)
|
|
7046
7150
|
await this.dispatch(next.decision, { dryRun: next.dryRun });
|
|
@@ -8084,18 +8188,18 @@ export class FactoryLoop {
|
|
|
8084
8188
|
const roots = githubIssueRepoRoots(owner, repo);
|
|
8085
8189
|
const cachedBatches = roots.map((root) => this.#cachedDiscoveryTree(root));
|
|
8086
8190
|
const allRootsCached = cachedBatches.every((paths) => paths !== undefined);
|
|
8087
|
-
|
|
8088
|
-
|
|
8089
|
-
|
|
8191
|
+
// The issue index is the current eligibility authority, while the
|
|
8192
|
+
// durable trees are only an incremental fallback for mounts whose
|
|
8193
|
+
// index is absent or malformed. A checkpoint can legitimately retain
|
|
8194
|
+
// stale membership (for example across an older cache format or a
|
|
8195
|
+
// missed change window); letting two cached roots bypass a healthy
|
|
8196
|
+
// index makes that stale snapshot self-validating forever.
|
|
8197
|
+
const indexedPaths = await this.#githubIssuePathsFromIndex(owner, repo);
|
|
8090
8198
|
// Keep the fallback roots as separate batches. Flattening a very large
|
|
8091
8199
|
// provider result is synchronous work and can starve the durable loop
|
|
8092
8200
|
// heartbeat before the bounded scan below gets a chance to yield.
|
|
8093
8201
|
let pathBatches;
|
|
8094
|
-
if (
|
|
8095
|
-
pathBatches = cachedBatches;
|
|
8096
|
-
this.#increment('githubIssueDiscoveryCacheReposUsed');
|
|
8097
|
-
}
|
|
8098
|
-
else if (indexedPaths) {
|
|
8202
|
+
if (indexedPaths) {
|
|
8099
8203
|
// Do not feed this into the discovery cache: the index only covers
|
|
8100
8204
|
// open, labeled issues, so it is a filtered subset of the real
|
|
8101
8205
|
// tree (and an empty result for whichever root form the index
|
|
@@ -8105,6 +8209,10 @@ export class FactoryLoop {
|
|
|
8105
8209
|
pathBatches = [indexedPaths];
|
|
8106
8210
|
this.#increment('githubIssueIndexReposUsed');
|
|
8107
8211
|
}
|
|
8212
|
+
else if (allRootsCached) {
|
|
8213
|
+
pathBatches = cachedBatches;
|
|
8214
|
+
this.#increment('githubIssueDiscoveryCacheReposUsed');
|
|
8215
|
+
}
|
|
8108
8216
|
else {
|
|
8109
8217
|
pathBatches = [];
|
|
8110
8218
|
for (const root of roots) {
|
|
@@ -8642,9 +8750,9 @@ export class FactoryLoop {
|
|
|
8642
8750
|
const repo = dependencyRepoForIssue(issue, undefined, this.#config);
|
|
8643
8751
|
if (!repo)
|
|
8644
8752
|
return false;
|
|
8645
|
-
// This probe does
|
|
8646
|
-
//
|
|
8647
|
-
//
|
|
8753
|
+
// This specialized probe does not go through `#resolveIssuePr`, so it never
|
|
8754
|
+
// saw that method's cache, and `#terminalDependencyIdentities` only ever
|
|
8755
|
+
// memoises the TRUE answer. A dependency that is not merged was
|
|
8648
8756
|
// therefore re-walked in full for every issue declaring it, on every sweep;
|
|
8649
8757
|
// several issues blocked on one dependency multiplied a single tree walk by
|
|
8650
8758
|
// the number of blocked issues. Memoise the negative answer too, on exactly
|
|
@@ -9585,13 +9693,6 @@ export class FactoryLoop {
|
|
|
9585
9693
|
batch.recordDryRun(record, spec, invocationId);
|
|
9586
9694
|
return { name: spec.name };
|
|
9587
9695
|
}
|
|
9588
|
-
// Persist intent before the remote side effect. If the owner crashes after
|
|
9589
|
-
// the spawn ack but before recording its result, takeover retries the same
|
|
9590
|
-
// deterministic invocation id instead of inventing a second worker.
|
|
9591
|
-
batch.recordPlanned(record, { ...spec, invocationId });
|
|
9592
|
-
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
9593
|
-
throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
|
|
9594
|
-
}
|
|
9595
9696
|
let roster;
|
|
9596
9697
|
try {
|
|
9597
9698
|
roster = await retryOnTimeout(() => this.#fleet.roster(), { attempts: 3, delayMs: 2000 });
|
|
@@ -9601,6 +9702,15 @@ export class FactoryLoop {
|
|
|
9601
9702
|
}
|
|
9602
9703
|
const rosterAgent = roster.agents.find((agent) => agent.name === spec.name);
|
|
9603
9704
|
if (rosterAgent) {
|
|
9705
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9706
|
+
const host = rosterAgent.node
|
|
9707
|
+
? roster.nodes.find((node) => node.name === rosterAgent.node && node.live && node.capabilities.includes(spec.capability))
|
|
9708
|
+
: undefined;
|
|
9709
|
+
if (!host) {
|
|
9710
|
+
throw new FleetPlacementUnavailableError(spec.capability);
|
|
9711
|
+
}
|
|
9712
|
+
spec = { ...spec, node: host.name };
|
|
9713
|
+
}
|
|
9604
9714
|
const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name);
|
|
9605
9715
|
record.heldSinceAtMs ??= this.#clock.now();
|
|
9606
9716
|
batch.recordSpawn(record, spec, invocationId, {
|
|
@@ -9618,6 +9728,21 @@ export class FactoryLoop {
|
|
|
9618
9728
|
await this.#reportAgent(record, adopted, 'agent.adopted');
|
|
9619
9729
|
return { name: spec.name };
|
|
9620
9730
|
}
|
|
9731
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9732
|
+
const loads = new Map();
|
|
9733
|
+
for (const agent of roster.agents) {
|
|
9734
|
+
if (agent.node)
|
|
9735
|
+
loads.set(agent.node, (loads.get(agent.node) ?? 0) + 1);
|
|
9736
|
+
}
|
|
9737
|
+
spec = { ...spec, node: liveFleetNodeForSpec(spec, roster, loads) };
|
|
9738
|
+
}
|
|
9739
|
+
// Persist intent before the remote side effect. If the owner crashes after
|
|
9740
|
+
// the spawn ack but before recording its result, takeover retries the same
|
|
9741
|
+
// deterministic invocation id instead of inventing a second worker.
|
|
9742
|
+
batch.recordPlanned(record, { ...spec, invocationId });
|
|
9743
|
+
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
9744
|
+
throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`);
|
|
9745
|
+
}
|
|
9621
9746
|
await this.#prepareAgentWorktree(record, spec);
|
|
9622
9747
|
let result;
|
|
9623
9748
|
try {
|
|
@@ -9657,6 +9782,26 @@ export class FactoryLoop {
|
|
|
9657
9782
|
await this.#releaseOrphanedLatePlacement(record, spec, result);
|
|
9658
9783
|
throw new LatePlacementReleasedError(record.issue.key, result.name ?? spec.name);
|
|
9659
9784
|
}
|
|
9785
|
+
if (this.#fleet.placementLocality === 'remote') {
|
|
9786
|
+
const registered = await this.#awaitRemoteAgentRegistration(result.name, spec.capability, result.node);
|
|
9787
|
+
if (!registered) {
|
|
9788
|
+
try {
|
|
9789
|
+
await this.#fleet.release(result.name, 'spawn-registration-timeout');
|
|
9790
|
+
this.#fleet.markAgentTerminal?.(result.name, 'spawn-registration-timeout');
|
|
9791
|
+
throw new RemoteAgentRegistrationTimeoutError(result.name, true);
|
|
9792
|
+
}
|
|
9793
|
+
catch (error) {
|
|
9794
|
+
if (error instanceof RemoteAgentRegistrationTimeoutError)
|
|
9795
|
+
throw error;
|
|
9796
|
+
// Cleanup is unconfirmed. Persist the placement so a successor can
|
|
9797
|
+
// retry the release; forgetting it would be worse than retaining a
|
|
9798
|
+
// nonterminal lifecycle for a worker that may still be alive.
|
|
9799
|
+
batch.recordSpawn(record, spec, invocationId, result);
|
|
9800
|
+
await this.#saveDispatchLifecycle(record, 'dispatching');
|
|
9801
|
+
throw new RemoteAgentRegistrationTimeoutError(result.name, false, error);
|
|
9802
|
+
}
|
|
9803
|
+
}
|
|
9804
|
+
}
|
|
9660
9805
|
record.heldSinceAtMs ??= this.#clock.now();
|
|
9661
9806
|
batch.recordSpawn(record, spec, invocationId, result);
|
|
9662
9807
|
if (!await this.#saveDispatchLifecycle(record, 'dispatching')) {
|
|
@@ -9668,6 +9813,89 @@ export class FactoryLoop {
|
|
|
9668
9813
|
await this.#reportAgent(record, spawned, 'agent.spawned');
|
|
9669
9814
|
return { name: result.name };
|
|
9670
9815
|
}
|
|
9816
|
+
async #awaitRemoteAgentRegistration(name, capability, expectedNode) {
|
|
9817
|
+
const deadlineAtMs = this.#clock.now() + REMOTE_AGENT_REGISTRATION_TIMEOUT_MS;
|
|
9818
|
+
do {
|
|
9819
|
+
try {
|
|
9820
|
+
if (expectedNode) {
|
|
9821
|
+
if (this.#fleet.isAgentRegistered) {
|
|
9822
|
+
if (await this.#fleet.isAgentRegistered({ name, node: expectedNode, capability }))
|
|
9823
|
+
return true;
|
|
9824
|
+
}
|
|
9825
|
+
else {
|
|
9826
|
+
const roster = await this.#fleet.roster();
|
|
9827
|
+
const agent = roster.agents.find((candidate) => candidate.name === name && candidate.node === expectedNode);
|
|
9828
|
+
const node = agent
|
|
9829
|
+
? roster.nodes.find((candidate) => candidate.name === expectedNode && candidate.live && candidate.capabilities.includes(capability))
|
|
9830
|
+
: undefined;
|
|
9831
|
+
if (agent && node)
|
|
9832
|
+
return true;
|
|
9833
|
+
}
|
|
9834
|
+
}
|
|
9835
|
+
}
|
|
9836
|
+
catch (error) {
|
|
9837
|
+
this.#logger.warn?.('[factory] remote agent registration probe failed; retrying within startup bound', {
|
|
9838
|
+
agent: name,
|
|
9839
|
+
error: describeError(error).errorMessage,
|
|
9840
|
+
});
|
|
9841
|
+
}
|
|
9842
|
+
const remainingMs = deadlineAtMs - this.#clock.now();
|
|
9843
|
+
if (remainingMs <= 0)
|
|
9844
|
+
return false;
|
|
9845
|
+
await this.#clock.sleep(Math.min(REMOTE_AGENT_REGISTRATION_POLL_MS, remainingMs));
|
|
9846
|
+
} while (this.#clock.now() <= deadlineAtMs);
|
|
9847
|
+
return false;
|
|
9848
|
+
}
|
|
9849
|
+
async #rollbackUnregisteredRemoteDispatch(record, acknowledged) {
|
|
9850
|
+
const handoffs = this.#dispatchFailureHandoffs(record, acknowledged);
|
|
9851
|
+
await this.#persistDispatchFailureReaperHandoff(record, handoffs);
|
|
9852
|
+
const hasWorktrees = handoffs.some((handoff) => handoff.worktree);
|
|
9853
|
+
if (hasWorktrees) {
|
|
9854
|
+
if (!await this.#teardownFailedDispatchWorktrees(handoffs, 'spawn-registration-timeout', { skipNeverPlacedAgents: true }))
|
|
9855
|
+
return false;
|
|
9856
|
+
}
|
|
9857
|
+
else {
|
|
9858
|
+
const failed = await this.#releaseAndTerminateAgents(handoffs
|
|
9859
|
+
.filter((handoff) => handoff.tracked.result !== undefined)
|
|
9860
|
+
.map((handoff) => [handoff.name, handoff.tracked]), 'spawn-registration-timeout', 'completion');
|
|
9861
|
+
if (failed.length > 0)
|
|
9862
|
+
return false;
|
|
9863
|
+
for (const handoff of handoffs) {
|
|
9864
|
+
await this.#state.clearFailureHandoff(this.#workspaceId, registryHandoffKey(handoff.issue, handoff.name));
|
|
9865
|
+
}
|
|
9866
|
+
}
|
|
9867
|
+
try {
|
|
9868
|
+
await this.#teardownPreviews(record);
|
|
9869
|
+
}
|
|
9870
|
+
catch (error) {
|
|
9871
|
+
this.#logger.warn?.('[factory] retained unregistered spawn lifecycle after preview rollback failed', {
|
|
9872
|
+
issue: record.issue.key,
|
|
9873
|
+
error: describeError(error).errorMessage,
|
|
9874
|
+
});
|
|
9875
|
+
return false;
|
|
9876
|
+
}
|
|
9877
|
+
const key = dispatchLifecycleKey(record.issue);
|
|
9878
|
+
const epoch = this.#dispatchLifecycleEpochs.get(key);
|
|
9879
|
+
const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key);
|
|
9880
|
+
const lease = lifecycle?.lease;
|
|
9881
|
+
if (epoch === undefined ||
|
|
9882
|
+
!lease ||
|
|
9883
|
+
lease.owner !== this.#dispatchLifecycleOwner ||
|
|
9884
|
+
lease.epoch !== epoch ||
|
|
9885
|
+
!await this.#state.clearClaimedDispatchLifecycle(this.#workspaceId, key, lease))
|
|
9886
|
+
return false;
|
|
9887
|
+
const retryTimer = this.#dispatchLifecycleRetryTimers.get(key);
|
|
9888
|
+
if (retryTimer)
|
|
9889
|
+
clearTimeout(retryTimer);
|
|
9890
|
+
this.#dispatchLifecycleRetryTimers.delete(key);
|
|
9891
|
+
this.#dispatchLifecycleEpochs.delete(key);
|
|
9892
|
+
this.#abandonedDispatchReasons.delete(key);
|
|
9893
|
+
const batch = await this.#batch();
|
|
9894
|
+
batch.abandon(record.issue);
|
|
9895
|
+
await this.#writeInFlightRegistry();
|
|
9896
|
+
this.#resetDispatchCapacityBackoff();
|
|
9897
|
+
return true;
|
|
9898
|
+
}
|
|
9671
9899
|
/**
|
|
9672
9900
|
* Is this process still the owner of a lifecycle that is not already done?
|
|
9673
9901
|
*
|
|
@@ -10302,44 +10530,6 @@ export class FactoryLoop {
|
|
|
10302
10530
|
return true;
|
|
10303
10531
|
}
|
|
10304
10532
|
async #openPullRequestByHead(repo, expectedHeadRef) {
|
|
10305
|
-
if (this.#hasProbePrGhRunner) {
|
|
10306
|
-
try {
|
|
10307
|
-
const result = await this.#probePrGhRunner([
|
|
10308
|
-
'pr',
|
|
10309
|
-
'list',
|
|
10310
|
-
'--repo',
|
|
10311
|
-
repo,
|
|
10312
|
-
'--head',
|
|
10313
|
-
expectedHeadRef,
|
|
10314
|
-
'--state',
|
|
10315
|
-
'open',
|
|
10316
|
-
'--json',
|
|
10317
|
-
'number,url,headRefName,isDraft',
|
|
10318
|
-
'--limit',
|
|
10319
|
-
'10',
|
|
10320
|
-
]);
|
|
10321
|
-
const payload = parseJsonContent(result.stdout);
|
|
10322
|
-
if (Array.isArray(payload)) {
|
|
10323
|
-
const candidates = payload.flatMap((entry) => {
|
|
10324
|
-
const candidate = asRecord(entry);
|
|
10325
|
-
const number = numberValue(candidate?.number);
|
|
10326
|
-
const url = stringValue(candidate?.url);
|
|
10327
|
-
const headRef = stringValue(candidate?.headRefName);
|
|
10328
|
-
if (!number || !url || headRef !== expectedHeadRef || candidate?.isDraft !== false)
|
|
10329
|
-
return [];
|
|
10330
|
-
return [{ repo, number, url, headRef }];
|
|
10331
|
-
});
|
|
10332
|
-
return candidates.sort((a, b) => b.number - a.number)[0];
|
|
10333
|
-
}
|
|
10334
|
-
}
|
|
10335
|
-
catch (error) {
|
|
10336
|
-
this.#logger.warn?.('[factory] exact-head gh PR lookup failed; falling back to mounted metadata', {
|
|
10337
|
-
repo,
|
|
10338
|
-
headRef: expectedHeadRef,
|
|
10339
|
-
error: describeError(error).errorMessage,
|
|
10340
|
-
});
|
|
10341
|
-
}
|
|
10342
|
-
}
|
|
10343
10533
|
const parts = githubRepoParts(repo);
|
|
10344
10534
|
if (!parts)
|
|
10345
10535
|
return undefined;
|
|
@@ -15111,11 +15301,6 @@ export class FactoryLoop {
|
|
|
15111
15301
|
this.#probePrResolvedCache.delete(cacheKey);
|
|
15112
15302
|
}
|
|
15113
15303
|
}
|
|
15114
|
-
for (const backoffKey of [...this.#probePrGhBackoffUntilMs.keys()]) {
|
|
15115
|
-
if (backoffKey === stateKey || backoffKey.startsWith(`${stateKey}:`)) {
|
|
15116
|
-
this.#probePrGhBackoffUntilMs.delete(backoffKey);
|
|
15117
|
-
}
|
|
15118
|
-
}
|
|
15119
15304
|
// Cancellation must see the subscription identity so it can issue the
|
|
15120
15305
|
// idempotent Relayfile DELETE before clearing the local owner maps.
|
|
15121
15306
|
await this.#cancelBabysittersForIssue(record.issue);
|
|
@@ -17788,19 +17973,29 @@ export class FactoryLoop {
|
|
|
17788
17973
|
if (!probe) {
|
|
17789
17974
|
return;
|
|
17790
17975
|
}
|
|
17791
|
-
|
|
17976
|
+
const closeInput = {
|
|
17792
17977
|
repo: probe.repo,
|
|
17793
17978
|
prNumber: probe.prNumber,
|
|
17794
17979
|
expectedIssueKey: issue.key,
|
|
17795
17980
|
requireTitleMarker: false,
|
|
17796
|
-
|
|
17797
|
-
|
|
17981
|
+
};
|
|
17982
|
+
if (this.#defaultProbeCloser) {
|
|
17983
|
+
await closeProbePr({
|
|
17984
|
+
...closeInput,
|
|
17985
|
+
...(probe.path ? { path: probe.path } : {}),
|
|
17986
|
+
mount: this.#mount,
|
|
17987
|
+
...(this.#mount.githubWrite ? { githubWrite: this.#mount.githubWrite } : {}),
|
|
17988
|
+
});
|
|
17989
|
+
}
|
|
17990
|
+
else {
|
|
17991
|
+
await this.#probeCloser(closeInput);
|
|
17992
|
+
}
|
|
17798
17993
|
this.#increment('mergeGateSyntheticClosed');
|
|
17799
17994
|
}
|
|
17800
17995
|
async #waitForMergeReady(pr) {
|
|
17801
17996
|
let lastReason = 'not checked';
|
|
17802
17997
|
for (let attempt = 1; attempt <= MERGE_GATE_MAX_ATTEMPTS; attempt += 1) {
|
|
17803
|
-
const verdict = await this.#mergeGate.check({ repo: pr.repo, number: pr.prNumber });
|
|
17998
|
+
const verdict = await this.#mergeGate.check({ repo: pr.repo, number: pr.prNumber, ...(pr.path ? { path: pr.path } : {}) });
|
|
17804
17999
|
lastReason = verdict.reason;
|
|
17805
18000
|
if (verdict.ready && verdict.live.headRefOid) {
|
|
17806
18001
|
return verdict;
|
|
@@ -17821,14 +18016,44 @@ export class FactoryLoop {
|
|
|
17821
18016
|
return hasTitlePrefix(issue.title, FACTORY_E2E_MARKER);
|
|
17822
18017
|
}
|
|
17823
18018
|
}
|
|
18019
|
+
/**
|
|
18020
|
+
* The guarded merge is a GitHub mutation, so it answers to the same identity
|
|
18021
|
+
* policy as the lifecycle writeback. Under exact `github.identity: "app"` the
|
|
18022
|
+
* gate refuses the merge instead of squash-merging as the operator's local
|
|
18023
|
+
* `gh` user; `auto` and `user` keep today's behavior.
|
|
18024
|
+
*
|
|
18025
|
+
* Exported so the selection itself is testable rather than buried in the
|
|
18026
|
+
* constructor — the same shape as `defaultGithubWriteback` below. `run` is a
|
|
18027
|
+
* test seam only: production passes nothing and gets `defaultGhRunner`. No
|
|
18028
|
+
* test may reach the real `gh` binary here, because the operation it would
|
|
18029
|
+
* perform is an irreversible merge.
|
|
18030
|
+
*/
|
|
18031
|
+
export const defaultMergeGate = (config, mount, run) => new MountedGithubMergeGate(mount, new GhCliGithubMergeGate(run, config.github.identity));
|
|
18032
|
+
const hasAppGithubLifecycleWrite = (write) => Boolean(write?.postIssueComment &&
|
|
18033
|
+
write.ensureRepositoryLabel &&
|
|
18034
|
+
write.mutateIssueLabel &&
|
|
18035
|
+
write.updateIssue);
|
|
17824
18036
|
const defaultGithubWriteback = (config, mount) => {
|
|
17825
|
-
|
|
18037
|
+
const identity = config.github.identity;
|
|
18038
|
+
const cloudContainer = mount.writebackTransport === 'relayfile-cloud';
|
|
18039
|
+
if (identity === 'user') {
|
|
18040
|
+
if (cloudContainer) {
|
|
18041
|
+
throw new Error('GitHub identity "user" requires lifecycle writes through local gh, but the Factory cloud container does not contain gh. ' +
|
|
18042
|
+
'Use identity "auto" or "app" with a connected workspace GitHub App lifecycle write path.');
|
|
18043
|
+
}
|
|
17826
18044
|
return new GhCliGithubWriteback();
|
|
17827
18045
|
}
|
|
17828
|
-
if (
|
|
18046
|
+
if (hasAppGithubLifecycleWrite(mount.githubWrite)) {
|
|
18047
|
+
return new AppGithubWriteback(mount.githubWrite, mount.githubRead);
|
|
18048
|
+
}
|
|
18049
|
+
if (identity === 'app') {
|
|
17829
18050
|
throw new Error('GitHub identity "app" requires a connected workspace GitHub App lifecycle write path; refusing to fall back to the local gh user');
|
|
17830
18051
|
}
|
|
17831
|
-
|
|
18052
|
+
if (cloudContainer) {
|
|
18053
|
+
throw new Error('GitHub identity "auto" cannot select lifecycle writeback in the Factory cloud container: no connected workspace GitHub App lifecycle write path is available, ' +
|
|
18054
|
+
'and the container does not contain gh.');
|
|
18055
|
+
}
|
|
18056
|
+
return new GhCliGithubWriteback();
|
|
17832
18057
|
};
|
|
17833
18058
|
export function parseLinearIssue(path, content) {
|
|
17834
18059
|
const parsed = parseJsonContent(content);
|
|
@@ -18234,6 +18459,50 @@ function dispatchSpecs(decision) {
|
|
|
18234
18459
|
}
|
|
18235
18460
|
return [...decision.implementers, decision.reviewer];
|
|
18236
18461
|
}
|
|
18462
|
+
function liveFleetNodeForSpec(spec, roster, assignedLoads) {
|
|
18463
|
+
const eligible = roster.nodes.filter((node) => node.live && node.capabilities.includes(spec.capability));
|
|
18464
|
+
if (eligible.length === 0)
|
|
18465
|
+
throw new FleetPlacementUnavailableError(spec.capability);
|
|
18466
|
+
const explicitlyRequested = spec.node && spec.node !== 'self'
|
|
18467
|
+
? eligible.find((node) => node.name === spec.node)
|
|
18468
|
+
: undefined;
|
|
18469
|
+
const selected = explicitlyRequested ?? [...eligible].sort((left, right) => {
|
|
18470
|
+
const loadDifference = (assignedLoads.get(left.name) ?? 0) - (assignedLoads.get(right.name) ?? 0);
|
|
18471
|
+
return loadDifference || left.name.localeCompare(right.name);
|
|
18472
|
+
})[0];
|
|
18473
|
+
assignedLoads.set(selected.name, (assignedLoads.get(selected.name) ?? 0) + 1);
|
|
18474
|
+
return selected.name;
|
|
18475
|
+
}
|
|
18476
|
+
/**
|
|
18477
|
+
* Resolve every remote dispatch spec against one canonical roster snapshot.
|
|
18478
|
+
*
|
|
18479
|
+
* A configured node is a preference, not an entitlement: if it is offline or
|
|
18480
|
+
* no longer advertises the required capability, placement is re-selected from
|
|
18481
|
+
* the live fleet. This runs before durable lifecycle creation, so an empty
|
|
18482
|
+
* eligible set cannot leave a claim or consume a batch slot.
|
|
18483
|
+
*/
|
|
18484
|
+
function decisionWithVerifiedRemotePlacements(decision, roster) {
|
|
18485
|
+
const assignedLoads = new Map();
|
|
18486
|
+
for (const agent of roster.agents) {
|
|
18487
|
+
if (agent.node)
|
|
18488
|
+
assignedLoads.set(agent.node, (assignedLoads.get(agent.node) ?? 0) + 1);
|
|
18489
|
+
}
|
|
18490
|
+
const place = (spec) => ({
|
|
18491
|
+
...spec,
|
|
18492
|
+
node: liveFleetNodeForSpec(spec, roster, assignedLoads),
|
|
18493
|
+
});
|
|
18494
|
+
if (decision.scope === 'workflow') {
|
|
18495
|
+
return {
|
|
18496
|
+
...structuredClone(decision),
|
|
18497
|
+
...(decision.workflow ? { workflow: place(decision.workflow) } : {}),
|
|
18498
|
+
};
|
|
18499
|
+
}
|
|
18500
|
+
return {
|
|
18501
|
+
...structuredClone(decision),
|
|
18502
|
+
implementers: decision.implementers.map(place),
|
|
18503
|
+
reviewer: place(decision.reviewer),
|
|
18504
|
+
};
|
|
18505
|
+
}
|
|
18237
18506
|
function dispatchSessionOwner(decision) {
|
|
18238
18507
|
for (const spec of dispatchSpecs(decision)) {
|
|
18239
18508
|
const sessionOwner = spec.principal?.trim() || spec.owner?.trim();
|
|
@@ -19125,14 +19394,26 @@ const PULL_INDEX_DECISIVE_SCORE = 20;
|
|
|
19125
19394
|
*
|
|
19126
19395
|
* All-or-nothing per repository, and the discipline is copied verbatim from
|
|
19127
19396
|
* Factory's issue-side reader (`#githubIssuePathsFromIndex`): fall back for the
|
|
19128
|
-
* ENTIRE repository if any row is legacy or malformed.
|
|
19129
|
-
* the
|
|
19130
|
-
*
|
|
19131
|
-
*
|
|
19132
|
-
* the
|
|
19133
|
-
*
|
|
19134
|
-
*
|
|
19135
|
-
*
|
|
19397
|
+
* ENTIRE repository if any row is legacy or malformed. One legacy row is enough
|
|
19398
|
+
* to poison the whole conclusion: the primary match is a branch match worth 30,
|
|
19399
|
+
* so a row without `headRef` cannot be ruled out as the real winner, and ranking
|
|
19400
|
+
* the remaining rows against each other would answer a different question from
|
|
19401
|
+
* the one the walk answers.
|
|
19402
|
+
*
|
|
19403
|
+
* Mixed indexes are the expected state today, not a corruption. `headRef` landed
|
|
19404
|
+
* in `@relayfile/adapter-github@0.5.7` and there is no pull-side backfill: the
|
|
19405
|
+
* incremental writers replace exactly the row they touched and pass every other
|
|
19406
|
+
* row through verbatim, so a webhook update hydrates one row and leaves the rest
|
|
19407
|
+
* legacy. Only an eager re-ingest converges a repository, because it rebuilds
|
|
19408
|
+
* the file from a list-pulls response in which every row carries `head.ref`.
|
|
19409
|
+
*
|
|
19410
|
+
* Row facts this relies on, from that contract: `state` is GitHub's lowercase
|
|
19411
|
+
* `"open"`/`"closed"` and there is no `"merged"` state — `merged: true` (with
|
|
19412
|
+
* `mergedAt`) is the only merged signal and `merged: false` is never written.
|
|
19413
|
+
* `headRef` is the bare branch name (`head.ref`), matching what
|
|
19414
|
+
* `readProbePrCandidate` reads off a record, NOT the `owner:branch` form of
|
|
19415
|
+
* `head.label`. `id` and `updated` are deliberately not required here: the
|
|
19416
|
+
* oldest legacy rows carry neither.
|
|
19136
19417
|
*/
|
|
19137
19418
|
const readPullIndexForProbe = async (mount, repo) => {
|
|
19138
19419
|
const [owner, name] = repo.split('/');
|
|
@@ -19374,79 +19655,6 @@ export const resolveIssuePrFromMount = async (mount, config, issue, opts = {}, l
|
|
|
19374
19655
|
}
|
|
19375
19656
|
return resolved;
|
|
19376
19657
|
};
|
|
19377
|
-
const resolveIssuePrFromGh = async (run, config, issue, opts = {}, logger) => {
|
|
19378
|
-
const candidates = [];
|
|
19379
|
-
let lookupFailures = 0;
|
|
19380
|
-
for (const repo of reposFromConfig(config)) {
|
|
19381
|
-
let payload;
|
|
19382
|
-
try {
|
|
19383
|
-
const result = await run([
|
|
19384
|
-
'pr',
|
|
19385
|
-
'list',
|
|
19386
|
-
'--repo',
|
|
19387
|
-
repo,
|
|
19388
|
-
'--state',
|
|
19389
|
-
'all',
|
|
19390
|
-
'--json',
|
|
19391
|
-
'number,title,body,headRefName,headRepository,headRepositoryOwner,isCrossRepository,isDraft,state,url',
|
|
19392
|
-
'--limit',
|
|
19393
|
-
String(PROBE_PR_GH_CANDIDATE_LIMIT),
|
|
19394
|
-
]);
|
|
19395
|
-
if (!result.stdout.trim()) {
|
|
19396
|
-
lookupFailures += 1;
|
|
19397
|
-
logger?.warn?.('[factory] gh PR resolver returned empty output', { issue: issue.key, repo });
|
|
19398
|
-
continue;
|
|
19399
|
-
}
|
|
19400
|
-
payload = parseJsonContent(result.stdout);
|
|
19401
|
-
}
|
|
19402
|
-
catch (error) {
|
|
19403
|
-
lookupFailures += 1;
|
|
19404
|
-
logger?.warn?.('[factory] gh PR resolver failed', { issue: issue.key, repo, error });
|
|
19405
|
-
continue;
|
|
19406
|
-
}
|
|
19407
|
-
if (!Array.isArray(payload)) {
|
|
19408
|
-
lookupFailures += 1;
|
|
19409
|
-
logger?.warn?.('[factory] gh PR resolver returned non-array payload', { issue: issue.key, repo });
|
|
19410
|
-
continue;
|
|
19411
|
-
}
|
|
19412
|
-
if (payload.length >= PROBE_PR_GH_CANDIDATE_LIMIT) {
|
|
19413
|
-
logger?.warn?.('[factory] gh PR resolver hit candidate limit', { issue: issue.key, repo, limit: PROBE_PR_GH_CANDIDATE_LIMIT });
|
|
19414
|
-
if (opts.failOnLookupError)
|
|
19415
|
-
lookupFailures += 1;
|
|
19416
|
-
}
|
|
19417
|
-
for (const entry of payload) {
|
|
19418
|
-
const pr = ghProbePrCandidate(entry);
|
|
19419
|
-
if (!pr ||
|
|
19420
|
-
(!factoryBranchMatchesIssue(pr.headRef, issue.key) &&
|
|
19421
|
-
!(opts.allowLegacyGithubBranch && legacyGithubBranchMatchesIssue(pr.headRef, issue))))
|
|
19422
|
-
continue;
|
|
19423
|
-
if (opts.openOnly && normalizePrState(pr.state) !== 'OPEN')
|
|
19424
|
-
continue;
|
|
19425
|
-
const score = issuePrMatchScore(pr, issue, opts.titleMarker ?? config.safety.requireTitlePrefix, opts);
|
|
19426
|
-
if (score <= 0)
|
|
19427
|
-
continue;
|
|
19428
|
-
candidates.push({
|
|
19429
|
-
repo,
|
|
19430
|
-
prNumber: pr.number,
|
|
19431
|
-
draft: pr.draft,
|
|
19432
|
-
headRef: pr.headRef,
|
|
19433
|
-
headRepo: pr.headRepo,
|
|
19434
|
-
crossRepository: pr.crossRepository ?? (pr.headRepo ? pr.headRepo.toLowerCase() !== repo.toLowerCase() : undefined),
|
|
19435
|
-
state: pr.state,
|
|
19436
|
-
url: pr.url,
|
|
19437
|
-
score,
|
|
19438
|
-
open: normalizePrState(pr.state) === 'OPEN',
|
|
19439
|
-
});
|
|
19440
|
-
}
|
|
19441
|
-
}
|
|
19442
|
-
const resolved = candidates.sort((a, b) => b.score - a.score ||
|
|
19443
|
-
Number(b.open) - Number(a.open) ||
|
|
19444
|
-
b.prNumber - a.prNumber)[0];
|
|
19445
|
-
if (!resolved && opts.failOnLookupError && lookupFailures > 0) {
|
|
19446
|
-
throw new Error(`Unable to confirm open pull request state for ${issue.key} in ${lookupFailures} configured repository lookup(s)`);
|
|
19447
|
-
}
|
|
19448
|
-
return resolved;
|
|
19449
|
-
};
|
|
19450
19658
|
const reposFromConfig = (config) => {
|
|
19451
19659
|
const repos = new Set([
|
|
19452
19660
|
...Object.values(config.repos.byLabel),
|
|
@@ -19565,32 +19773,6 @@ const readProbePrCandidate = async (mount, path) => {
|
|
|
19565
19773
|
return undefined;
|
|
19566
19774
|
}
|
|
19567
19775
|
};
|
|
19568
|
-
const ghProbePrCandidate = (value) => {
|
|
19569
|
-
const payload = asRecord(value);
|
|
19570
|
-
if (!payload)
|
|
19571
|
-
return undefined;
|
|
19572
|
-
const number = numberValue(payload.number);
|
|
19573
|
-
if (typeof number !== 'number' || !Number.isInteger(number) || number <= 0)
|
|
19574
|
-
return undefined;
|
|
19575
|
-
const headRepository = asRecord(payload.headRepository);
|
|
19576
|
-
const headRepositoryOwner = asRecord(payload.headRepositoryOwner);
|
|
19577
|
-
const headRepo = githubRepositoryFullName(payload.headRepository) ?? (() => {
|
|
19578
|
-
const name = stringValue(headRepository?.name);
|
|
19579
|
-
const owner = stringValue(headRepositoryOwner?.login) ?? stringValue(headRepositoryOwner?.name);
|
|
19580
|
-
return name && owner ? `${owner}/${name}` : undefined;
|
|
19581
|
-
})();
|
|
19582
|
-
return {
|
|
19583
|
-
number,
|
|
19584
|
-
title: stringValue(payload.title) ?? '',
|
|
19585
|
-
body: stringValue(payload.body) ?? '',
|
|
19586
|
-
headRef: stringValue(payload.headRefName) ?? '',
|
|
19587
|
-
headRepo,
|
|
19588
|
-
crossRepository: booleanValue(payload.isCrossRepository),
|
|
19589
|
-
draft: booleanValue(payload.isDraft),
|
|
19590
|
-
state: stringValue(payload.state),
|
|
19591
|
-
url: stringValue(payload.url),
|
|
19592
|
-
};
|
|
19593
|
-
};
|
|
19594
19776
|
const issuePrMatchScore = (pr, issue, marker, opts = {}) => {
|
|
19595
19777
|
if (opts.requireTitleMarker && !hasTitlePrefix(pr.title, marker))
|
|
19596
19778
|
return 0;
|
|
@@ -19633,7 +19815,6 @@ const legacyGithubPrCanBeAdopted = (issue, pr) => {
|
|
|
19633
19815
|
return pr.crossRepository === false || pr.headRepo?.toLowerCase() === pr.repo.toLowerCase();
|
|
19634
19816
|
};
|
|
19635
19817
|
const normalizePrState = (state) => state?.toUpperCase();
|
|
19636
|
-
const failClosedGhRunner = async () => ({ stdout: '[]' });
|
|
19637
19818
|
const ISSUE_KEY_PATTERN = /^[A-Z]+-\d+$/u;
|
|
19638
19819
|
const isIssuePathUnderRoot = (path) => path.startsWith(`${ISSUE_ROOT}/`) && path.endsWith('.json');
|
|
19639
19820
|
const isIssueFilePath = (path) => isIssuePathUnderRoot(path) &&
|