@defrex/autobuild 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +192 -0
- package/SPEC.md +2269 -0
- package/bin/ab-build-runner.ts +47 -0
- package/bin/ab-dev.ts +27 -0
- package/bin/ab-dispatch-kernel.ts +92 -0
- package/bin/ab-harvest-runner.ts +29 -0
- package/bin/ab-init-probe.ts +19 -0
- package/bin/ab-process-group-reaper.ts +32 -0
- package/bin/ab.ts +6 -0
- package/bin/agent/ab +3 -0
- package/docs/answering-blocked-builds.md +89 -0
- package/docs/architecture.md +726 -0
- package/docs/assets/headline-wide.png +0 -0
- package/docs/configuration.md +1623 -0
- package/docs/hosted-dispatcher-cutover-validation.md +178 -0
- package/docs/hosted-dispatcher.md +204 -0
- package/docs/mcp.md +119 -0
- package/docs/operator-api.md +186 -0
- package/docs/remote-store-protocol.md +972 -0
- package/docs/remote-ticket-protocol.md +79 -0
- package/docs/setup.md +289 -0
- package/docs/spec-standard.md +63 -0
- package/docs/ticket-dependencies.md +59 -0
- package/docs/vercel-sandbox-rollout-probe.md +7 -0
- package/package.json +135 -0
- package/packages/core/src/cli/args.ts +74 -0
- package/packages/core/src/cli/artifact.ts +182 -0
- package/packages/core/src/cli/binary.ts +198 -0
- package/packages/core/src/cli/build-control.ts +789 -0
- package/packages/core/src/cli/build-progress.ts +55 -0
- package/packages/core/src/cli/bulk-control.ts +385 -0
- package/packages/core/src/cli/context.ts +470 -0
- package/packages/core/src/cli/dashboard/actions.ts +59 -0
- package/packages/core/src/cli/dashboard/cells.ts +92 -0
- package/packages/core/src/cli/dashboard/composer.ts +230 -0
- package/packages/core/src/cli/dashboard/detail.ts +100 -0
- package/packages/core/src/cli/dashboard/frame-image.ts +373 -0
- package/packages/core/src/cli/dashboard/live.ts +152 -0
- package/packages/core/src/cli/dashboard/model.ts +1173 -0
- package/packages/core/src/cli/dashboard/poll.ts +196 -0
- package/packages/core/src/cli/dashboard/render.ts +1568 -0
- package/packages/core/src/cli/dashboard/selection.ts +66 -0
- package/packages/core/src/cli/dashboard/session-feed.ts +244 -0
- package/packages/core/src/cli/dashboard/session-view.ts +506 -0
- package/packages/core/src/cli/dashboard/transcript.ts +108 -0
- package/packages/core/src/cli/dev.ts +46 -0
- package/packages/core/src/cli/dispatch-frontend.ts +1045 -0
- package/packages/core/src/cli/dispatch-process.ts +259 -0
- package/packages/core/src/cli/dispatch.ts +3692 -0
- package/packages/core/src/cli/dotenv.ts +48 -0
- package/packages/core/src/cli/env.ts +206 -0
- package/packages/core/src/cli/harvest.ts +698 -0
- package/packages/core/src/cli/help.ts +713 -0
- package/packages/core/src/cli/init-agent.ts +106 -0
- package/packages/core/src/cli/init-validation.ts +583 -0
- package/packages/core/src/cli/init.ts +756 -0
- package/packages/core/src/cli/installation.ts +333 -0
- package/packages/core/src/cli/keyboard.ts +261 -0
- package/packages/core/src/cli/main.ts +1399 -0
- package/packages/core/src/cli/mcp.ts +279 -0
- package/packages/core/src/cli/models.ts +65 -0
- package/packages/core/src/cli/observe.ts +55 -0
- package/packages/core/src/cli/plugin.ts +227 -0
- package/packages/core/src/cli/pr-attachments.ts +138 -0
- package/packages/core/src/cli/pr-summary.ts +124 -0
- package/packages/core/src/cli/publication-settlement.ts +188 -0
- package/packages/core/src/cli/repo-state.ts +213 -0
- package/packages/core/src/cli/repository-status.ts +80 -0
- package/packages/core/src/cli/self-update.ts +443 -0
- package/packages/core/src/cli/status.ts +895 -0
- package/packages/core/src/cli/store-opening.contract.ts +289 -0
- package/packages/core/src/cli/store-opening.ts +167 -0
- package/packages/core/src/cli/store-ref.ts +33 -0
- package/packages/core/src/cli/terminal-restore.ts +174 -0
- package/packages/core/src/cli/terminal.ts +361 -0
- package/packages/core/src/cli/terminals.ts +995 -0
- package/packages/core/src/cli/testkit.ts +194 -0
- package/packages/core/src/cli/ticket.ts +484 -0
- package/packages/core/src/cli/upgrade-agent.ts +180 -0
- package/packages/core/src/cli/upgrade-commit.ts +468 -0
- package/packages/core/src/cli/upgrade-notice.ts +91 -0
- package/packages/core/src/cli/upgrade-progress.ts +110 -0
- package/packages/core/src/cli/upgrade.ts +888 -0
- package/packages/core/src/cli/wait.ts +587 -0
- package/packages/core/src/cli/watch.ts +932 -0
- package/packages/core/src/config/live.ts +255 -0
- package/packages/core/src/config/load.ts +98 -0
- package/packages/core/src/config/roles.ts +438 -0
- package/packages/core/src/config/schema.ts +774 -0
- package/packages/core/src/distribution.ts +24 -0
- package/packages/core/src/events/catalog.ts +201 -0
- package/packages/core/src/events/envelope.ts +59 -0
- package/packages/core/src/events/payloads.ts +618 -0
- package/packages/core/src/events/repository.ts +559 -0
- package/packages/core/src/events/sessions.ts +180 -0
- package/packages/core/src/harvest/schema.ts +147 -0
- package/packages/core/src/ids.ts +45 -0
- package/packages/core/src/integration/harness.ts +681 -0
- package/packages/core/src/kernel/auto-merge.ts +259 -0
- package/packages/core/src/kernel/converge.ts +117 -0
- package/packages/core/src/kernel/dispatch-settings.ts +60 -0
- package/packages/core/src/kernel/dispatch-status.ts +141 -0
- package/packages/core/src/kernel/engine.ts +1047 -0
- package/packages/core/src/kernel/harvest.ts +823 -0
- package/packages/core/src/kernel/origin.ts +69 -0
- package/packages/core/src/kernel/phases.ts +196 -0
- package/packages/core/src/kernel/plan-verify-selection.ts +105 -0
- package/packages/core/src/kernel/pr-attachments.ts +82 -0
- package/packages/core/src/kernel/reducer.ts +752 -0
- package/packages/core/src/kernel/stall.ts +155 -0
- package/packages/core/src/kernel/verify-applicability.ts +52 -0
- package/packages/core/src/markdown.ts +67 -0
- package/packages/core/src/ontology.ts +288 -0
- package/packages/core/src/open-map.ts +203 -0
- package/packages/core/src/operator/annotations.ts +187 -0
- package/packages/core/src/operator/client.ts +311 -0
- package/packages/core/src/operator/control.ts +117 -0
- package/packages/core/src/operator/index.ts +62 -0
- package/packages/core/src/operator/presentation.ts +22 -0
- package/packages/core/src/operator/protocol.ts +98 -0
- package/packages/core/src/operator/query.ts +201 -0
- package/packages/core/src/operator/registry.ts +1043 -0
- package/packages/core/src/operator/requests.ts +138 -0
- package/packages/core/src/operator/sandbox.ts +458 -0
- package/packages/core/src/operator/server.ts +616 -0
- package/packages/core/src/operator/sessions.ts +223 -0
- package/packages/core/src/operator/tickets.ts +177 -0
- package/packages/core/src/plugin-sdk/index.ts +174 -0
- package/packages/core/src/plugins/contract-entry.ts +74 -0
- package/packages/core/src/plugins/contract-env.ts +3 -0
- package/packages/core/src/plugins/load.ts +259 -0
- package/packages/core/src/plugins/manifest.ts +236 -0
- package/packages/core/src/plugins/registry.ts +274 -0
- package/packages/core/src/plugins/runtimes.ts +75 -0
- package/packages/core/src/ports/forge/contract.ts +258 -0
- package/packages/core/src/ports/forge/create.ts +74 -0
- package/packages/core/src/ports/forge/fake.ts +488 -0
- package/packages/core/src/ports/forge/github-pr-attachments.ts +351 -0
- package/packages/core/src/ports/forge/github-transport.ts +352 -0
- package/packages/core/src/ports/forge/github.ts +1162 -0
- package/packages/core/src/ports/forge/local-git.ts +568 -0
- package/packages/core/src/ports/runner/claude.ts +873 -0
- package/packages/core/src/ports/runner/cli-stream.ts +90 -0
- package/packages/core/src/ports/runner/codex.ts +873 -0
- package/packages/core/src/ports/runner/contract.ts +492 -0
- package/packages/core/src/ports/runner/fake.ts +213 -0
- package/packages/core/src/ports/runner/json-record.ts +2 -0
- package/packages/core/src/ports/runner/live-contract-fixture.ts +176 -0
- package/packages/core/src/ports/runner/one-shot.ts +29 -0
- package/packages/core/src/ports/runner/pi-bridge-core.ts +136 -0
- package/packages/core/src/ports/runner/pi-bridge.js +6 -0
- package/packages/core/src/ports/runner/pi-rpc.ts +454 -0
- package/packages/core/src/ports/runner/pi.ts +617 -0
- package/packages/core/src/ports/runner/production.ts +74 -0
- package/packages/core/src/ports/runner/provider-error.ts +121 -0
- package/packages/core/src/ports/runner/routing.ts +234 -0
- package/packages/core/src/ports/runner/runtime.ts +214 -0
- package/packages/core/src/ports/runner/session-env.ts +34 -0
- package/packages/core/src/ports/runner/stream-parts.ts +103 -0
- package/packages/core/src/ports/tickets/contract.ts +570 -0
- package/packages/core/src/ports/tickets/create.ts +135 -0
- package/packages/core/src/ports/tickets/fake.ts +215 -0
- package/packages/core/src/ports/tickets/file.ts +559 -0
- package/packages/core/src/ports/tickets/linear.ts +811 -0
- package/packages/core/src/ports/tickets/operations.ts +161 -0
- package/packages/core/src/ports/tickets/remote-index.ts +13 -0
- package/packages/core/src/ports/tickets/remote-protocol.ts +80 -0
- package/packages/core/src/ports/tickets/remote-server.ts +185 -0
- package/packages/core/src/ports/tickets/remote.ts +130 -0
- package/packages/core/src/ports/tickets/update.ts +51 -0
- package/packages/core/src/ports/types.ts +528 -0
- package/packages/core/src/ports/workspace/build-execution.ts +123 -0
- package/packages/core/src/ports/workspace/contract.ts +136 -0
- package/packages/core/src/ports/workspace/create.ts +132 -0
- package/packages/core/src/ports/workspace/distribution-archive.ts +317 -0
- package/packages/core/src/ports/workspace/fake.ts +432 -0
- package/packages/core/src/ports/workspace/git-worktree.ts +848 -0
- package/packages/core/src/ports/workspace/harvest-execution.ts +111 -0
- package/packages/core/src/ports/workspace/in-process-build-execution.ts +30 -0
- package/packages/core/src/ports/workspace/local-build-execution.ts +88 -0
- package/packages/core/src/ports/workspace/operator-sandbox.ts +142 -0
- package/packages/core/src/ports/workspace/phase-scratch.ts +156 -0
- package/packages/core/src/ports/workspace/vercel-sandbox.ts +2541 -0
- package/packages/core/src/processes/build-child-exit.ts +78 -0
- package/packages/core/src/processes/build-child.ts +119 -0
- package/packages/core/src/processes/build-execution-state.ts +95 -0
- package/packages/core/src/processes/build-parent-watch.ts +35 -0
- package/packages/core/src/processes/build-runner.ts +2614 -0
- package/packages/core/src/processes/dispatcher.ts +2825 -0
- package/packages/core/src/processes/execution-settlement.ts +171 -0
- package/packages/core/src/processes/harvest-child.ts +115 -0
- package/packages/core/src/processes/harvest-execution-state.ts +121 -0
- package/packages/core/src/processes/harvest-runner.ts +1656 -0
- package/packages/core/src/processes/harvest.ts +595 -0
- package/packages/core/src/processes/infrastructure-failure-budget.ts +146 -0
- package/packages/core/src/processes/phase-failure-budget.ts +52 -0
- package/packages/core/src/processes/process-group-reaper.ts +35 -0
- package/packages/core/src/processes/process-group.ts +65 -0
- package/packages/core/src/processes/publication-loss.ts +62 -0
- package/packages/core/src/processes/publication-state.ts +93 -0
- package/packages/core/src/processes/sandbox-state.ts +101 -0
- package/packages/core/src/registry.ts +24 -0
- package/packages/core/src/sessions/reducer.ts +185 -0
- package/packages/core/src/skills.ts +7 -0
- package/packages/core/src/spec-standard.ts +47 -0
- package/packages/core/src/store/adapter.ts +80 -0
- package/packages/core/src/store/build-scope.ts +272 -0
- package/packages/core/src/store/contract.ts +2581 -0
- package/packages/core/src/store/local/blobs.ts +38 -0
- package/packages/core/src/store/local/schema.ts +180 -0
- package/packages/core/src/store/local/store.ts +1713 -0
- package/packages/core/src/store/memory.ts +1138 -0
- package/packages/core/src/store/phase-session.ts +425 -0
- package/packages/core/src/store/remote/client.ts +780 -0
- package/packages/core/src/store/remote/index.ts +12 -0
- package/packages/core/src/store/remote/protocol.ts +348 -0
- package/packages/core/src/store/remote/server.ts +923 -0
- package/packages/core/src/store/remote/token.ts +137 -0
- package/packages/core/src/store/remote/version.ts +10 -0
- package/packages/core/src/store/retention.ts +91 -0
- package/packages/core/src/store/session-handle.ts +265 -0
- package/packages/core/src/store/streams/assemble.ts +79 -0
- package/packages/core/src/store/streams/lock.ts +42 -0
- package/packages/core/src/store/streams/session-writer.ts +306 -0
- package/packages/core/src/store/streams/types.ts +171 -0
- package/packages/core/src/store/streams/wait.ts +127 -0
- package/packages/core/src/store/subscribe.ts +105 -0
- package/packages/core/src/store/types.ts +466 -0
- package/packages/core/src/testing/fixed.ts +44 -0
- package/packages/core/src/testing/store-failures.ts +32 -0
- package/packages/core/src/zod-issues.ts +91 -0
- package/skills/code-review/SKILL.md +100 -0
- package/skills/finalize/SKILL.md +44 -0
- package/skills/guide/SKILL.md +1781 -0
- package/skills/guide/references/operator-api.md +164 -0
- package/skills/guide/references/plugin-authoring.md +354 -0
- package/skills/guide/references/remote-store-protocol.md +972 -0
- package/skills/guide/references/remote-ticket-protocol.md +79 -0
- package/skills/guide/references/setup.md +289 -0
- package/skills/guide/references/spec-standard.md +63 -0
- package/skills/guide/references/ticket-dependencies.md +59 -0
- package/skills/harvest/SKILL.md +66 -0
- package/skills/harvest-review/SKILL.md +63 -0
- package/skills/implement/SKILL.md +82 -0
- package/skills/plan/SKILL.md +123 -0
- package/skills/plan-review/SKILL.md +121 -0
- package/skills/reconcile/SKILL.md +65 -0
- package/skills/spec/SKILL.md +132 -0
- package/skills/tickets/SKILL.md +139 -0
- package/templates/autobuild.toml +36 -0
|
@@ -0,0 +1,2825 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The dispatcher process (SPEC §3.3, §12): watches the TicketSource for
|
|
3
|
+
* Ready tickets, claims, provisions a workspace, and launches build-runners
|
|
4
|
+
* up to a capacity limit — plus janitor duty over the post-PR epilogue
|
|
5
|
+
* (§15.7, D1) and the stale-lease sweep (§15.6-C). Cron-friendly: one
|
|
6
|
+
* `tick()` does one pass over everything and is safe to re-run — every
|
|
7
|
+
* action is deduped against the reduced event log, and single-writer
|
|
8
|
+
* discipline (§12) means at most one dispatcher runs per repo.
|
|
9
|
+
*
|
|
10
|
+
* Order within a tick (deliberate):
|
|
11
|
+
* a. JANITOR — settles finished work first, releasing capacity that the
|
|
12
|
+
* dispatch step below can immediately reuse.
|
|
13
|
+
* b. DISPATCH RECOVERY — completes interrupted pre-run build setup.
|
|
14
|
+
* c. STARTUP RESUME (first CLI tick only) — attempts every actionable
|
|
15
|
+
* current build, including policy-exhausted infra failures.
|
|
16
|
+
* d. LEASE SWEEP — re-attaches runners to builds whose runner died.
|
|
17
|
+
* e. DISPATCH — fills remaining capacity from Ready tickets.
|
|
18
|
+
* f. HARVEST — observation back-pressure, independent of build capacity.
|
|
19
|
+
* A build launched in an earlier step is never launched again by a later
|
|
20
|
+
* one (per-tick launch dedupe); a build dispatched in step (d) is not swept
|
|
21
|
+
* in the same tick because the sweep already ran.
|
|
22
|
+
*
|
|
23
|
+
* A repository-wide pause (`dispatcher.pause-set`, written by the dashboard's
|
|
24
|
+
* pause-all) holds every QUEUED build across steps (b)–(e): none of them may
|
|
25
|
+
* attach a runner to a build that has no runner yet, whichever stage reaches
|
|
26
|
+
* it first. The janitor (a) is deliberately NOT gated, so abort and discard
|
|
27
|
+
* still settle while the repository is at rest, and running builds are left to
|
|
28
|
+
* the per-build pauses that pause-all writes alongside the flag. The flag is
|
|
29
|
+
* sampled once per tick, so a pause landing mid-tick takes effect on the next
|
|
30
|
+
* one — the same semantics intake already has.
|
|
31
|
+
*
|
|
32
|
+
* The dispatcher never runs pipeline agents (§15.7) — conflicted PRs and
|
|
33
|
+
* stale leases both resolve by re-attaching a build-runner via
|
|
34
|
+
* `launchRunner`. Optional pre-build judgment (spec authoring and short slug
|
|
35
|
+
* naming) lives behind bounded seams before a build exists; deterministic
|
|
36
|
+
* validation and fallback remain here. The supervising kernel reserves each
|
|
37
|
+
* execution lease before launch; the child renews that same-holder lease.
|
|
38
|
+
*/
|
|
39
|
+
import type { Config } from '../config/schema'
|
|
40
|
+
import { DISPATCHER, agentActor, humanActor } from '../events/envelope'
|
|
41
|
+
import type { AbEvent, EventWrite } from '../events/catalog'
|
|
42
|
+
import type { EventPayload } from '../events/payloads'
|
|
43
|
+
import type { IdSource } from '../ids'
|
|
44
|
+
import {
|
|
45
|
+
autoMergeApplicationType,
|
|
46
|
+
pendingAutoMerge,
|
|
47
|
+
recordAutoMergeDeferralObservation,
|
|
48
|
+
} from '../kernel/auto-merge'
|
|
49
|
+
import { reduceDispatchSettings } from '../kernel/dispatch-settings'
|
|
50
|
+
import { decideNext } from '../kernel/engine'
|
|
51
|
+
import {
|
|
52
|
+
DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS,
|
|
53
|
+
decideHarvestControl,
|
|
54
|
+
harvestCreationsDuringReadyScan,
|
|
55
|
+
reduceHarvest,
|
|
56
|
+
} from '../kernel/harvest'
|
|
57
|
+
import { pendingPrAttachmentReclaims } from '../kernel/pr-attachments'
|
|
58
|
+
import { reduceBuild, type BuildState } from '../kernel/reducer'
|
|
59
|
+
import type { ArtifactRef } from '../ontology'
|
|
60
|
+
import { BUILD_EXECUTION_LEASE_TTL_MS } from '../ports/workspace/build-execution'
|
|
61
|
+
import type {
|
|
62
|
+
DependencyState,
|
|
63
|
+
Forge,
|
|
64
|
+
Ticket,
|
|
65
|
+
TicketSource,
|
|
66
|
+
WorkspaceProvider,
|
|
67
|
+
} from '../ports/types'
|
|
68
|
+
import type { Exec } from '../ports/workspace/git-worktree'
|
|
69
|
+
import type { ArtifactMeta, BuildRecord, BuildStore, Clock } from '../store/types'
|
|
70
|
+
import { resolveRepoOrigin } from '../cli/repo-state'
|
|
71
|
+
import { specConformance } from '../spec-standard'
|
|
72
|
+
export { specConformance, type SpecConformance } from '../spec-standard'
|
|
73
|
+
import { recordInfrastructureFailure as appendInfrastructureFailure } from './infrastructure-failure-budget'
|
|
74
|
+
import { lastExecutionOutcome, openExecution, settleExecution } from './execution-settlement'
|
|
75
|
+
import { openHarvestExecutions } from './harvest-execution-state'
|
|
76
|
+
import type { RepositoryEvent } from '../events/repository'
|
|
77
|
+
import { sandboxStates } from './sandbox-state'
|
|
78
|
+
import { abandonedPublicationPending, publicationPending } from './publication-state'
|
|
79
|
+
import { settlePublicationBeforeRelease } from './publication-loss'
|
|
80
|
+
|
|
81
|
+
// ── Readiness resolution (SPEC §3.3) ─────────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* What "ready for dispatch" means, resolved against the ticket source.
|
|
85
|
+
*
|
|
86
|
+
* The state gate is no longer source-defaulted: `readyState` is a required,
|
|
87
|
+
* non-blank config value (src/config/schema.ts), so every source gates on the
|
|
88
|
+
* exact configured state and always return one. The removed branch — where the
|
|
89
|
+
* linear source left `state` unset when `readyState` was absent — was the
|
|
90
|
+
* AUT-10 hole: with no state filter, every labelled ticket was eligible in any
|
|
91
|
+
* workflow state, so a completed ticket still carrying the label got dispatched
|
|
92
|
+
* again. That branch is now unrepresentable.
|
|
93
|
+
*
|
|
94
|
+
* The only per-source difference left is the *label* default. Linear has no
|
|
95
|
+
* `ready/` directory, so a label is the only thing that can mark a ticket
|
|
96
|
+
* dispatchable: the historical `["autobuild"]` default. The file tracker's gate
|
|
97
|
+
* is otherwise the directory, so it defaults to no label gate. Plugin sources
|
|
98
|
+
* likewise receive no host-imposed label convention. An explicit
|
|
99
|
+
* `readyLabels` wins for every source.
|
|
100
|
+
*/
|
|
101
|
+
export function readyCriteria(config: Config): { labels: string[]; state: string } {
|
|
102
|
+
const { readyLabels, readyState } = config.tickets
|
|
103
|
+
if (config.tickets.source === 'linear') {
|
|
104
|
+
return { labels: readyLabels ?? ['autobuild'], state: readyState }
|
|
105
|
+
}
|
|
106
|
+
return { labels: readyLabels ?? [], state: readyState }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Where the dispatcher hands work back to a human — spec-gate bounces (§6.3),
|
|
111
|
+
* aborted builds, closed-unmerged PRs. Source-dependent for the same reason
|
|
112
|
+
* readiness is: the file tracker's grooming area IS the `triage/` directory,
|
|
113
|
+
* while a Linear team only has a "Triage" workflow state when the team's
|
|
114
|
+
* triage feature is enabled — Backlog is the state every Linear team has.
|
|
115
|
+
* The hosted source is the credential-bearing Linear proxy and shares that
|
|
116
|
+
* fallback. Plugin sources use the neutral `Triage` fallback unless configured.
|
|
117
|
+
*/
|
|
118
|
+
export function defaultTriageState(config: Config): string {
|
|
119
|
+
return (
|
|
120
|
+
config.tickets.triageState ??
|
|
121
|
+
(config.tickets.source === 'linear' || config.tickets.source === 'hosted'
|
|
122
|
+
? 'Backlog'
|
|
123
|
+
: 'Triage')
|
|
124
|
+
)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Where harvest files a synthesized proposal (§12). Defaults to the triage
|
|
129
|
+
* state, so the constitutional gate — nothing auto-generated reaches Ready
|
|
130
|
+
* without a human — holds unless a repository names another state.
|
|
131
|
+
*
|
|
132
|
+
* It is a field of its own rather than a reuse of `triageState` because the
|
|
133
|
+
* two answer different questions. Pointing triage at the ready state to get
|
|
134
|
+
* auto-dispatched proposals would also send spec-gate bounces there, and a
|
|
135
|
+
* bounce that lands back in Ready is claimed and bounced again every tick.
|
|
136
|
+
*/
|
|
137
|
+
export function defaultProposalState(config: Config): string {
|
|
138
|
+
return config.tickets.proposalState ?? defaultTriageState(config)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// ── Spec quality gate (SPEC §6.3, docs/spec-standard.md) ─────────────────────
|
|
142
|
+
// Shared with deterministic harvest filing; imported/re-exported above so the
|
|
143
|
+
// long-standing dispatcher API remains stable.
|
|
144
|
+
|
|
145
|
+
// ── Ticket dependency gate (SPEC §13) ────────────────────────────────────────
|
|
146
|
+
//
|
|
147
|
+
// Division of labor: the TicketSource answers FACTS (does this blocker exist,
|
|
148
|
+
// is it resolved by my native lifecycle, what does it declare as its own
|
|
149
|
+
// blockers); the dispatcher decides POLICY (an unresolved blocker means don't
|
|
150
|
+
// dispatch; a broken graph means skip this ticket, not the tick). Keeping the
|
|
151
|
+
// decision here means one tested implementation instead of one per adapter,
|
|
152
|
+
// and no adapter's state taxonomy leaks upward.
|
|
153
|
+
|
|
154
|
+
export interface DependencyVerdict {
|
|
155
|
+
/** Unresolved blocker ids — nonempty means DO NOT dispatch. */
|
|
156
|
+
unresolved: string[]
|
|
157
|
+
/** Actionable lines naming the affected ticket and the dependency. */
|
|
158
|
+
diagnostics: string[]
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Walks back from `from` looking for a node already on `path`; returns the
|
|
163
|
+
* cycle (first repeated node → … → itself) or null. `explored` prunes nodes
|
|
164
|
+
* proven acyclic on an earlier branch, so a wide graph cannot blow up.
|
|
165
|
+
*
|
|
166
|
+
* Note the repeat test is against `path`, not `nodes` — which is why closing a
|
|
167
|
+
* cycle back onto the ticket under analysis needs no entry for it in `nodes`.
|
|
168
|
+
*/
|
|
169
|
+
function findCycle(
|
|
170
|
+
from: string,
|
|
171
|
+
path: string[],
|
|
172
|
+
nodes: Map<string, DependencyState>,
|
|
173
|
+
explored: Set<string>,
|
|
174
|
+
): string[] | null {
|
|
175
|
+
for (const next of nodes.get(from)?.blockedBy ?? []) {
|
|
176
|
+
const seen = path.indexOf(next)
|
|
177
|
+
if (seen !== -1) return [...path.slice(seen), next]
|
|
178
|
+
if (explored.has(next)) continue
|
|
179
|
+
const cycle = findCycle(next, [...path, next], nodes, explored)
|
|
180
|
+
if (cycle) return cycle
|
|
181
|
+
explored.add(next)
|
|
182
|
+
}
|
|
183
|
+
return null
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Whether the ticket `ticketId`, which declares `blockedBy`, may be
|
|
188
|
+
* dispatched — given `nodes`, closed over its reachable blockers (see
|
|
189
|
+
* `loadDependencyGraph`).
|
|
190
|
+
*
|
|
191
|
+
* The ticket's own blockers are a PARAMETER, not a lookup in `nodes`: they
|
|
192
|
+
* come from the ticket itself, which is authoritative for them. `nodes` holds
|
|
193
|
+
* only facts fetched from the source, so nothing here has to fabricate an
|
|
194
|
+
* entry for the ticket under analysis — a fabrication that, in a cache shared
|
|
195
|
+
* across a tick, is indistinguishable from a fact about a *blocker* when the
|
|
196
|
+
* next ticket looks it up.
|
|
197
|
+
*
|
|
198
|
+
* Pure and total: a node the walk cannot find is treated as
|
|
199
|
+
* already-reported-missing rather than throwing. Exported for testing, like
|
|
200
|
+
* `specConformance`.
|
|
201
|
+
*/
|
|
202
|
+
export function analyzeDependencies(
|
|
203
|
+
ticketId: string,
|
|
204
|
+
blockedBy: string[],
|
|
205
|
+
nodes: Map<string, DependencyState>,
|
|
206
|
+
): DependencyVerdict {
|
|
207
|
+
const unresolved: string[] = []
|
|
208
|
+
const diagnostics: string[] = []
|
|
209
|
+
|
|
210
|
+
for (const blockerId of blockedBy) {
|
|
211
|
+
if (blockerId === ticketId) {
|
|
212
|
+
unresolved.push(blockerId)
|
|
213
|
+
diagnostics.push(`ticket ${ticketId} depends on itself`)
|
|
214
|
+
continue
|
|
215
|
+
}
|
|
216
|
+
const node = nodes.get(blockerId)
|
|
217
|
+
if (!node?.exists) {
|
|
218
|
+
unresolved.push(blockerId)
|
|
219
|
+
diagnostics.push(
|
|
220
|
+
`ticket ${ticketId} blocked by ${blockerId}, which does not exist in ` +
|
|
221
|
+
'this ticket source',
|
|
222
|
+
)
|
|
223
|
+
continue
|
|
224
|
+
}
|
|
225
|
+
// A resolved blocker imposes nothing, and its own history is irrelevant —
|
|
226
|
+
// never walked, so a cycle *behind* completed work is not a diagnostic.
|
|
227
|
+
if (node.resolved) continue
|
|
228
|
+
|
|
229
|
+
unresolved.push(blockerId)
|
|
230
|
+
// Gating never needs the transitive walk (an unresolved blocker already
|
|
231
|
+
// holds the ticket); the walk exists only to NAME a cycle, which direct
|
|
232
|
+
// blockers alone cannot express.
|
|
233
|
+
const cycle = findCycle(blockerId, [ticketId, blockerId], nodes, new Set<string>())
|
|
234
|
+
diagnostics.push(
|
|
235
|
+
cycle
|
|
236
|
+
? `ticket ${ticketId}: dependency cycle ${cycle.join(' → ')}`
|
|
237
|
+
: `ticket ${ticketId} blocked by ${blockerId} (not complete)`,
|
|
238
|
+
)
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
return { unresolved, diagnostics }
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** `Add rate limiting!` → `add-rate-limiting` (general title normalizer). */
|
|
245
|
+
export function kebab(title: string): string {
|
|
246
|
+
const slug = title
|
|
247
|
+
.toLowerCase()
|
|
248
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
249
|
+
.replace(/^-+|-+$/g, '')
|
|
250
|
+
return slug || 'build'
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Deterministic naming fallback: the first three normalized title tokens. */
|
|
254
|
+
export function fallbackSlug(title: string): string {
|
|
255
|
+
return kebab(title).split('-').slice(0, 3).join('-')
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Validate a judgment-produced slug base without repairing model prose. Outer
|
|
260
|
+
* whitespace is harmless; everything else must already be one-to-three
|
|
261
|
+
* lowercase ASCII kebab tokens. Collision suffixes are appended later and are
|
|
262
|
+
* intentionally outside this budget.
|
|
263
|
+
*/
|
|
264
|
+
export function validateSlugCandidate(candidate: string | null | undefined): string | null {
|
|
265
|
+
if (candidate === null || candidate === undefined) return null
|
|
266
|
+
const trimmed = candidate.trim()
|
|
267
|
+
return /^[a-z0-9]+(?:-[a-z0-9]+){0,2}$/.test(trimmed) ? trimmed : null
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// ── Tick report ──────────────────────────────────────────────────────────────
|
|
271
|
+
|
|
272
|
+
/** Counts per action, for observability and tests. An idempotent re-run of
|
|
273
|
+
* `tick()` over unchanged state reports all zeroes — except invalid-record,
|
|
274
|
+
* in-flight-creation, and dependency fields, which are standing queue reports rather than records of
|
|
275
|
+
* action. They repeat on every ready scan by design, because that is the only
|
|
276
|
+
* place the source problems are visible without provider inspection. */
|
|
277
|
+
export interface TickReport {
|
|
278
|
+
/** Janitor (§15.7): builds completed as merged. */
|
|
279
|
+
merged: number
|
|
280
|
+
/** Janitor: builds completed as closed-unmerged. */
|
|
281
|
+
closed: number
|
|
282
|
+
/** Janitor: `pr.conflicted` emitted + build-runner re-attached. */
|
|
283
|
+
conflicted: number
|
|
284
|
+
/** Janitor: aborted builds cleaned up and completed as abandoned. */
|
|
285
|
+
abandoned: number
|
|
286
|
+
/** Janitor: queued builds explicitly discarded and returned to Ready. */
|
|
287
|
+
discarded: number
|
|
288
|
+
/** Janitor failures contained to individual builds during this pass. */
|
|
289
|
+
janitorFailed: number
|
|
290
|
+
/** One actionable line per contained janitor failure, attributed by slug. */
|
|
291
|
+
janitorDiagnostics: string[]
|
|
292
|
+
/** Ordinary-tick recovery completed an interrupted post-creation dispatch. */
|
|
293
|
+
recovered: number
|
|
294
|
+
/** Post-creation dispatch attempts that durably recorded a failure. */
|
|
295
|
+
dispatchFailed: number
|
|
296
|
+
/** Dispatch-command startup: current builds for which a runner was launched. */
|
|
297
|
+
resumed: number
|
|
298
|
+
/** One line per current build left parked by open escalations on startup. */
|
|
299
|
+
blockedDiagnostics: string[]
|
|
300
|
+
/** Lease sweep (§15.6-C): runners re-attached to stale builds. */
|
|
301
|
+
swept: number
|
|
302
|
+
/** Settlement stage: foreign executions settled from durable facts plus
|
|
303
|
+
* provider liveness (a recorded completion or provider proof of end). */
|
|
304
|
+
settled: number
|
|
305
|
+
/** Builds whose remote workspace provisioning was (re-)kicked as tracked
|
|
306
|
+
* background work this tick. The tick returns without awaiting it; the
|
|
307
|
+
* continuation completes the remaining boundaries and the launch. */
|
|
308
|
+
provisioning: number
|
|
309
|
+
/** Dispatch (§12): builds created and launched. With durable supervision
|
|
310
|
+
* this counts ACCEPTED dispatches — durable boundaries recorded — not
|
|
311
|
+
* awaited launches: a dispatch whose remote provisioning continues in the
|
|
312
|
+
* background is still accepted. */
|
|
313
|
+
dispatched: number
|
|
314
|
+
/** Of `dispatched`: specs produced via the authorSpec seam (§6.3). */
|
|
315
|
+
authored: number
|
|
316
|
+
/** Dispatch quality gate (§6.3): tickets bounced back to Triage. */
|
|
317
|
+
bounced: number
|
|
318
|
+
/** Claim-before-launch (§12): claims lost to another dispatcher. */
|
|
319
|
+
claimRaces: number
|
|
320
|
+
/** Invalid source records excluded from this ready listing. Like dependency
|
|
321
|
+
* fields, this is a standing count and is re-reported until repaired. */
|
|
322
|
+
invalidTickets: number
|
|
323
|
+
/** Actionable source diagnostics for each excluded invalid record. */
|
|
324
|
+
ticketDiagnostics: string[]
|
|
325
|
+
/** Standing queue depth: ready tickets still waiting after this tick —
|
|
326
|
+
* excludes tickets already embodied by an active build, and ones that left
|
|
327
|
+
* the queue this tick (dispatched, bounced, or claimed elsewhere). Like the
|
|
328
|
+
* invalid/dependency fields it re-reports on every ready scan. */
|
|
329
|
+
queued: number
|
|
330
|
+
/** Ready tickets withheld while Autobuild's own external creation is still
|
|
331
|
+
* between durable reservation and settled blocker recording. */
|
|
332
|
+
creationWithheld: number
|
|
333
|
+
/** One line per creation hold, with Harvest run/proposal provenance. */
|
|
334
|
+
creationDiagnostics: string[]
|
|
335
|
+
/** Dependency gate (§13): ready tickets held back by unresolved blockers. */
|
|
336
|
+
dependencyBlocked: number
|
|
337
|
+
/** One line per held ticket naming its unresolved blockers — the operator's
|
|
338
|
+
* only view of the dependency queue short of provider inspection. */
|
|
339
|
+
dependencyDiagnostics: string[]
|
|
340
|
+
/** Repository observation workflow counters (independent of build capacity).
|
|
341
|
+
* The CLI's in-flight coordinator merges these when asynchronous harvest
|
|
342
|
+
* runs settle; Dispatcher.tick itself only initiates the work. */
|
|
343
|
+
harvestStarted: number
|
|
344
|
+
harvestResumed: number
|
|
345
|
+
harvestCompleted: number
|
|
346
|
+
harvestEscalated: number
|
|
347
|
+
harvestFailed: number
|
|
348
|
+
/** Operator sandboxes idly stopped this tick (AUT-340). */
|
|
349
|
+
sandboxIdleStops: number
|
|
350
|
+
/** Contained idle-settlement failures this tick; retried next tick. */
|
|
351
|
+
sandboxSettleFailures: number
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
export function emptyTickReport(): TickReport {
|
|
355
|
+
return {
|
|
356
|
+
merged: 0,
|
|
357
|
+
closed: 0,
|
|
358
|
+
conflicted: 0,
|
|
359
|
+
abandoned: 0,
|
|
360
|
+
discarded: 0,
|
|
361
|
+
janitorFailed: 0,
|
|
362
|
+
janitorDiagnostics: [],
|
|
363
|
+
recovered: 0,
|
|
364
|
+
dispatchFailed: 0,
|
|
365
|
+
resumed: 0,
|
|
366
|
+
blockedDiagnostics: [],
|
|
367
|
+
swept: 0,
|
|
368
|
+
settled: 0,
|
|
369
|
+
provisioning: 0,
|
|
370
|
+
dispatched: 0,
|
|
371
|
+
authored: 0,
|
|
372
|
+
bounced: 0,
|
|
373
|
+
claimRaces: 0,
|
|
374
|
+
invalidTickets: 0,
|
|
375
|
+
ticketDiagnostics: [],
|
|
376
|
+
queued: 0,
|
|
377
|
+
creationWithheld: 0,
|
|
378
|
+
creationDiagnostics: [],
|
|
379
|
+
dependencyBlocked: 0,
|
|
380
|
+
dependencyDiagnostics: [],
|
|
381
|
+
harvestStarted: 0,
|
|
382
|
+
harvestResumed: 0,
|
|
383
|
+
harvestCompleted: 0,
|
|
384
|
+
harvestEscalated: 0,
|
|
385
|
+
harvestFailed: 0,
|
|
386
|
+
sandboxIdleStops: 0,
|
|
387
|
+
sandboxSettleFailures: 0,
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// ── Dispatcher ───────────────────────────────────────────────────────────────
|
|
392
|
+
|
|
393
|
+
const DEFAULT_SLUG_NAMING_TIMEOUT_MS = 10_000
|
|
394
|
+
/** Heartbeat cadence of a background provisioning continuation's execution
|
|
395
|
+
* lease. A dead provisioner is therefore adopted within one lease TTL. */
|
|
396
|
+
const PROVISION_HEARTBEAT_MS = 15_000
|
|
397
|
+
const DEFAULT_PROVISION_STALE_MS = 20 * 60 * 1000
|
|
398
|
+
|
|
399
|
+
/** Outcome of one attempt to complete an interrupted dispatch. `deferred`
|
|
400
|
+
* means durable progress was made (background provisioning kicked or already
|
|
401
|
+
* in flight) and a later tick continues; the launch is never awaited here. */
|
|
402
|
+
export type DispatchOutcome = 'completed' | 'deferred' | 'parked' | 'failed'
|
|
403
|
+
|
|
404
|
+
/** One tracked background provisioning continuation (process-local). Its
|
|
405
|
+
* durable residue is the `workspace.provision-started` marker plus the
|
|
406
|
+
* execution lease it claims and heartbeats. */
|
|
407
|
+
interface ProvisionContinuation {
|
|
408
|
+
readonly slug: string
|
|
409
|
+
readonly instance: string
|
|
410
|
+
readonly controller: AbortController
|
|
411
|
+
promise: Promise<void>
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
export interface DispatcherOpts {
|
|
415
|
+
/**
|
|
416
|
+
* Grace for builds with NO lease at all: the sweep treats an absent lease
|
|
417
|
+
* as expired only once this long has passed since the build's last store
|
|
418
|
+
* update — a just-launched runner needs time to claim its first lease.
|
|
419
|
+
* Default 0 (the simple predicate: absent ≡ expired). Expired-but-present
|
|
420
|
+
* leases are always swept immediately.
|
|
421
|
+
*/
|
|
422
|
+
leaseTtlMs?: number
|
|
423
|
+
/**
|
|
424
|
+
* Ticket state that hands work back to a human. Aborted work and
|
|
425
|
+
* closed-unmerged PRs go back here rather than to a done-state — an abort
|
|
426
|
+
* or close is a human decision that the ticket needs human re-triage, not
|
|
427
|
+
* silent completion. Bounced tickets (§6.3) land here too. Absent =
|
|
428
|
+
* defaultTriageState(config): [tickets].triageState, else 'Backlog'
|
|
429
|
+
* (linear) / 'Triage' (file).
|
|
430
|
+
*/
|
|
431
|
+
triageState?: string
|
|
432
|
+
/** Ticket state for merged builds. Default 'Done'. */
|
|
433
|
+
doneState?: string
|
|
434
|
+
/** Internal deadline for the optional pre-build naming seam. Production is
|
|
435
|
+
* fixed; injection exists only so timeout behavior is deterministic in tests. */
|
|
436
|
+
slugNamingTimeoutMs?: number
|
|
437
|
+
/** Must match the HarvestRunner's durable outer recovery budget. */
|
|
438
|
+
maxHarvestRecoveryAttempts?: number
|
|
439
|
+
/** Backstop for the crash window between a `workspace.provision-started`
|
|
440
|
+
* append and its owner's lease claim: an open marker with NO lease is
|
|
441
|
+
* adopted only once it is older than this. The primary adoption gate is the
|
|
442
|
+
* execution lease itself — a dead heartbeating owner is adopted within one
|
|
443
|
+
* lease TTL (60 s). Default 20 minutes. */
|
|
444
|
+
provisionStaleMs?: number
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
export interface TickOpts {
|
|
448
|
+
/**
|
|
449
|
+
* Attempt every actionable, non-terminal build for this repo before the
|
|
450
|
+
* ordinary stale-lease sweep. `ab dispatch` sets this only on its first
|
|
451
|
+
* tick so crash recovery does not repeatedly probe healthy runners. Open
|
|
452
|
+
* escalations remain human gates and are only reported as parked.
|
|
453
|
+
*/
|
|
454
|
+
resumeCurrent?: boolean
|
|
455
|
+
/**
|
|
456
|
+
* Repository-derived intake gate supplied by the CLI for this tick. False
|
|
457
|
+
* skips only the ready-ticket list/claim/dispatch stage; janitor, startup
|
|
458
|
+
* resume, lease sweep, and in-flight runners continue. Absent defaults true.
|
|
459
|
+
*/
|
|
460
|
+
acceptNewWork?: boolean
|
|
461
|
+
/**
|
|
462
|
+
* Repository-derived claim-time default supplied by the CLI for this tick.
|
|
463
|
+
* True records the existing durable auto-merge request only on builds
|
|
464
|
+
* created by this tick's ticket claims. Absent/false preserves the sequence.
|
|
465
|
+
*/
|
|
466
|
+
defaultAutoMerge?: boolean
|
|
467
|
+
/** Human attribution for a claim-time auto-merge request. Required when
|
|
468
|
+
* `defaultAutoMerge` is true; the CLI resolves USER/USERNAME/fallback. */
|
|
469
|
+
autoMergeUser?: string
|
|
470
|
+
/** Invocation bound in epoch ms against `deps.clock`. Once the clock reads
|
|
471
|
+
* at or past it, no additional forge/store transport call is STARTED inside
|
|
472
|
+
* the tick: per-item loops gate each new unit of work on the remaining
|
|
473
|
+
* budget, so the tick's overrun past the bound is bounded to the single
|
|
474
|
+
* transport call already in flight when the deadline passed. In-flight
|
|
475
|
+
* calls are deliberately not cancelled — aborting mid-write risks tearing
|
|
476
|
+
* durable store state. Skipped work is simply left un-attempted: ticks are
|
|
477
|
+
* idempotent (§3.3 re-run safety) and the next invocation picks it up,
|
|
478
|
+
* exactly as capacity exhaustion already defers tickets. Absent ⇒ unbounded
|
|
479
|
+
* (today's exact behavior). */
|
|
480
|
+
deadlineAt?: number
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/** Process-local launch coordination result. Durable lease acquisition remains
|
|
484
|
+
* the BuildRunner's responsibility; "scheduled" means only that this process
|
|
485
|
+
* accepted a runner launch rather than suppressing a known in-flight slug. */
|
|
486
|
+
export type LaunchRunnerResult = 'scheduled' | 'already-active'
|
|
487
|
+
|
|
488
|
+
export interface DispatcherDeps {
|
|
489
|
+
store: BuildStore
|
|
490
|
+
tickets: TicketSource
|
|
491
|
+
workspaces: WorkspaceProvider
|
|
492
|
+
/** Providers that owned workspaces recorded under an earlier
|
|
493
|
+
* `[workspace].provider`, so their builds can still be released after a
|
|
494
|
+
* configuration switch. A workspace is only ever reaped or released by the
|
|
495
|
+
* provider named on its provisioned fact. */
|
|
496
|
+
retiredWorkspaces?: readonly WorkspaceProvider[]
|
|
497
|
+
forge: Forge
|
|
498
|
+
config: Config
|
|
499
|
+
/** Current process snapshot, captured once at the beginning of each tick.
|
|
500
|
+
* Omission preserves the cron/static-config API. */
|
|
501
|
+
getConfig?: () => Config
|
|
502
|
+
/** The repo this dispatcher serves — a local path (§15.7: `git ls-remote`
|
|
503
|
+
* against it needs no network) or the repository's normalized origin when
|
|
504
|
+
* there is no checkout (origin mode; base facts then come from the forge's
|
|
505
|
+
* `remoteBranchSha`). One dispatcher per repo (§12). */
|
|
506
|
+
repo: string
|
|
507
|
+
/** Explicit normalized origin for the served repository. When set (origin
|
|
508
|
+
* mode), the dispatcher never probes the checkout's git for its origin —
|
|
509
|
+
* there is no checkout — and stamps created builds directly with it. */
|
|
510
|
+
repoOrigin?: string
|
|
511
|
+
/** The physical main checkout when one exists (checkout mode). Local
|
|
512
|
+
* workspace providers (git-worktree) provision FROM it; origin mode has no
|
|
513
|
+
* checkout, and its remote providers never need one. */
|
|
514
|
+
checkout?: string
|
|
515
|
+
exec: Exec
|
|
516
|
+
/** Launch (or re-attach — §15.6-C, §15.7) a build-runner for `slug`. The
|
|
517
|
+
* dispatcher never runs pipeline agents itself. The launcher reports local
|
|
518
|
+
* single-flight suppression so resume/sweep counters describe real schedules. */
|
|
519
|
+
launchRunner: (slug: string) => Promise<LaunchRunnerResult>
|
|
520
|
+
/**
|
|
521
|
+
* Non-interactive spec authoring for thin-but-groomed tickets (§6.3):
|
|
522
|
+
* returns a candidate spec body, or null when the ticket cannot be
|
|
523
|
+
* expanded. Pragmatism, documented: the seam returns only the body — the
|
|
524
|
+
* dispatcher stamps the authoring agent's session id itself (via `ids`)
|
|
525
|
+
* for the `spec.authored` actor/payload, rather than widening the seam to
|
|
526
|
+
* return session metadata the fake would have to invent.
|
|
527
|
+
*/
|
|
528
|
+
authorSpec?: (ticket: Ticket) => Promise<string | null>
|
|
529
|
+
/** Start the repository-scoped observation workflow without awaiting it.
|
|
530
|
+
* The owner must single-flight and track the promise; Dispatcher invokes
|
|
531
|
+
* this after ready-ticket dispatch, independent of drain/build capacity. */
|
|
532
|
+
startHarvest?: () => void
|
|
533
|
+
/** Optional one-shot judgment over the final conforming spec. The dispatcher
|
|
534
|
+
* supplies cancellation and treats every absence/failure/invalid result as a
|
|
535
|
+
* local deterministic fallback, so naming can never prevent build creation. */
|
|
536
|
+
nameSlug?: (spec: string, signal: AbortSignal) => Promise<string | null>
|
|
537
|
+
/** Settle pending publication for `slug` after durable proof that the
|
|
538
|
+
* guest execution ended. Supplied by the DispatchLoop; the settlement stage
|
|
539
|
+
* invokes it only on a recorded completion. */
|
|
540
|
+
settlePublication?: (slug: string) => Promise<void>
|
|
541
|
+
/** Slugs THIS process supervises in memory. The settlement stage skips
|
|
542
|
+
* them: their completion chain owns the durable facts. */
|
|
543
|
+
activeExecutions?: () => ReadonlySet<string>
|
|
544
|
+
/** Hosted-harvest execution ids THIS process supervises in memory. The
|
|
545
|
+
* harvest settlement stage skips them: their completion chain owns the
|
|
546
|
+
* durable facts. */
|
|
547
|
+
activeHarvestExecutions?: () => ReadonlySet<string>
|
|
548
|
+
ids: IdSource
|
|
549
|
+
clock: Clock
|
|
550
|
+
opts?: DispatcherOpts
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/** Latest `workspace.provisioned` not followed by a `workspace.released` —
|
|
554
|
+
* the reducer deliberately ignores workspace events (liveness is the
|
|
555
|
+
* dispatcher's concern), so the janitor scans the raw log. */
|
|
556
|
+
function openWorkspace(
|
|
557
|
+
events: AbEvent[],
|
|
558
|
+
): { provider: string; ref: string; path?: string; localPath?: string; branch: string } | null {
|
|
559
|
+
let open: {
|
|
560
|
+
provider: string
|
|
561
|
+
ref: string
|
|
562
|
+
path?: string
|
|
563
|
+
localPath?: string
|
|
564
|
+
branch: string
|
|
565
|
+
} | null = null
|
|
566
|
+
for (const event of events) {
|
|
567
|
+
if (event.type === 'workspace.provisioned') open = event.payload
|
|
568
|
+
else if (event.type === 'workspace.released') open = null
|
|
569
|
+
}
|
|
570
|
+
return open
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/** The build's base branch, from its own `build.created` fact (§15.3);
|
|
574
|
+
* falls back to the repo config for logs missing one. */
|
|
575
|
+
function baseBranchOf(events: AbEvent[], config: Config): string {
|
|
576
|
+
for (const event of events) {
|
|
577
|
+
if (event.type === 'build.created') return event.payload.baseBranch
|
|
578
|
+
}
|
|
579
|
+
return config.baseBranch
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
function artifactRefOf(deposited: ArtifactMeta[]): ArtifactRef {
|
|
583
|
+
const meta = deposited[0]
|
|
584
|
+
if (!meta) throw new Error('spec deposit produced no artifact meta')
|
|
585
|
+
return { kind: meta.kind, rev: meta.revision }
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/**
|
|
589
|
+
* Whether a repository-wide pause holds this build back from being given a
|
|
590
|
+
* runner. Narrow on purpose, and the narrowness is load-bearing:
|
|
591
|
+
*
|
|
592
|
+
* - `queued` is exactly "a build record exists and no runner is attached" —
|
|
593
|
+
* the gap the repository pause exists to close. It is also the set bulk
|
|
594
|
+
* pause eligibility deliberately excludes, so the two halves of the control
|
|
595
|
+
* cover the whole space with no overlap.
|
|
596
|
+
* - `running` builds are NOT held here. They are parked by the per-build pause
|
|
597
|
+
* the same keypress writes, and holding them in the dispatcher would strand
|
|
598
|
+
* `pendingCommands` delivery — the lease sweep is the only thing that
|
|
599
|
+
* re-attaches a runner so a build whose runner died can receive its abort or
|
|
600
|
+
* resume (§15.2.7).
|
|
601
|
+
* - `done` / `aborted` / `blocked` / `paused` builds are the janitor's and the
|
|
602
|
+
* engine's business; the janitor is not gated, so abort and discard keep
|
|
603
|
+
* settling while the repository is at rest.
|
|
604
|
+
*/
|
|
605
|
+
export function heldByRepositoryPause(state: BuildState, paused: boolean): boolean {
|
|
606
|
+
return paused && state.status === 'queued'
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
export class Dispatcher {
|
|
610
|
+
/** Memoized normalized origin of the served repository (one git call per
|
|
611
|
+
* dispatcher), recorded on created builds so ambient reads can accept a
|
|
612
|
+
* differently located checkout of the same repository by origin equality.
|
|
613
|
+
* Origin mode supplies it explicitly (deps.repoOrigin) and never execs. */
|
|
614
|
+
private repoOrigin?: Promise<string | undefined>
|
|
615
|
+
|
|
616
|
+
private resolveRepoOriginOnce(): Promise<string | undefined> {
|
|
617
|
+
if (this.deps.repoOrigin !== undefined) {
|
|
618
|
+
this.repoOrigin ??= Promise.resolve(this.deps.repoOrigin)
|
|
619
|
+
return this.repoOrigin
|
|
620
|
+
}
|
|
621
|
+
// Probe the physical checkout, not the store identity: since the identity
|
|
622
|
+
// change `deps.repo` is the normalized origin (or a path only for
|
|
623
|
+
// origin-less fixtures), and `git remote get-url` in a URL cwd silently
|
|
624
|
+
// loses the origin. Origin mode returns above and never execs.
|
|
625
|
+
this.repoOrigin ??= resolveRepoOrigin(this.deps.checkout ?? this.deps.repo, this.deps.exec)
|
|
626
|
+
return this.repoOrigin
|
|
627
|
+
}
|
|
628
|
+
private readonly leaseTtlMs: number
|
|
629
|
+
private readonly doneState: string
|
|
630
|
+
private readonly slugNamingTimeoutMs: number
|
|
631
|
+
private readonly maxHarvestRecoveryAttempts: number
|
|
632
|
+
private readonly provisionStaleMs: number
|
|
633
|
+
/** Process-local background provisioning, keyed by slug. A tick that sees a
|
|
634
|
+
* slug here defers to the continuation; it never blocks on it. */
|
|
635
|
+
private readonly continuations = new Map<string, ProvisionContinuation>()
|
|
636
|
+
|
|
637
|
+
constructor(private readonly deps: DispatcherDeps) {
|
|
638
|
+
this.leaseTtlMs = deps.opts?.leaseTtlMs ?? 0
|
|
639
|
+
this.doneState = deps.opts?.doneState ?? 'Done'
|
|
640
|
+
this.slugNamingTimeoutMs = deps.opts?.slugNamingTimeoutMs ?? DEFAULT_SLUG_NAMING_TIMEOUT_MS
|
|
641
|
+
this.maxHarvestRecoveryAttempts =
|
|
642
|
+
deps.opts?.maxHarvestRecoveryAttempts ?? DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS
|
|
643
|
+
this.provisionStaleMs = deps.opts?.provisionStaleMs ?? DEFAULT_PROVISION_STALE_MS
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/**
|
|
647
|
+
* One cron-friendly pass (§3.3): janitor → interrupted dispatch recovery →
|
|
648
|
+
* optional startup resume → lease sweep → dispatch. Startup resume is opt-in
|
|
649
|
+
* because the CLI invokes it
|
|
650
|
+
* once per command, while ordinary watch ticks must preserve policy parks.
|
|
651
|
+
*/
|
|
652
|
+
/** Invocation-budget gate: true when a `deadlineAt` was supplied and the
|
|
653
|
+
* clock already reads at or past it. Pure clock arithmetic — the per-item
|
|
654
|
+
* loops consult this before starting each new forge/store transport call,
|
|
655
|
+
* bounding the tick's overrun to the one call already in flight. */
|
|
656
|
+
private outOfBudget(opts: TickOpts): boolean {
|
|
657
|
+
return opts.deadlineAt !== undefined && this.deps.clock().getTime() >= opts.deadlineAt
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
async tick(opts: TickOpts = {}): Promise<TickReport> {
|
|
661
|
+
// One immutable config for this complete decision pass. A reload racing the
|
|
662
|
+
// tick is observed by the next tick, never half-way through this one.
|
|
663
|
+
this.deps.config = this.deps.getConfig?.() ?? this.deps.config
|
|
664
|
+
let autoMergeUser: string | undefined
|
|
665
|
+
if (opts.defaultAutoMerge === true) {
|
|
666
|
+
autoMergeUser = opts.autoMergeUser?.trim()
|
|
667
|
+
if (autoMergeUser === undefined || autoMergeUser === '') {
|
|
668
|
+
throw new Error('defaultAutoMerge requires nonempty human attribution in autoMergeUser')
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
const report = emptyTickReport()
|
|
672
|
+
/** Builds already launched this tick — a janitor/startup re-attach must
|
|
673
|
+
* not be doubled by the sweep, nor a fresh dispatch by anything. */
|
|
674
|
+
const launched = new Set<string>()
|
|
675
|
+
// Sampled once, before any stage that could attach a runner: a pause that
|
|
676
|
+
// lands mid-tick takes effect on the next tick, matching intake (which the
|
|
677
|
+
// CLI likewise samples before calling `tick`). Neither control pretends to
|
|
678
|
+
// a serialization the store does not offer.
|
|
679
|
+
const paused = await this.repositoryPaused()
|
|
680
|
+
// Stage 0 — SETTLEMENT: observe foreign executions from durable facts plus
|
|
681
|
+
// provider liveness and settle their completion, lease, and publication.
|
|
682
|
+
// Runs before the janitor so its executionLeaseLive checks see freshly
|
|
683
|
+
// released or renewed leases.
|
|
684
|
+
// Stage entries are budget-gated so an already-spent budget does not even
|
|
685
|
+
// open the first record listing of the next stage; each stage's own
|
|
686
|
+
// per-iteration gate (below) covers a deadline that passes mid-stage.
|
|
687
|
+
if (this.outOfBudget(opts)) return report
|
|
688
|
+
await this.settleForeignExecutions(report, opts)
|
|
689
|
+
if (this.outOfBudget(opts)) return report
|
|
690
|
+
await this.settleHarvestExecutions(opts)
|
|
691
|
+
if (this.outOfBudget(opts)) return report
|
|
692
|
+
await this.settleOrchestratorSandboxes(report, opts)
|
|
693
|
+
if (this.outOfBudget(opts)) return report
|
|
694
|
+
await this.janitor(report, launched, opts)
|
|
695
|
+
if (this.outOfBudget(opts)) return report
|
|
696
|
+
await this.recoverDispatches(report, launched, paused, opts)
|
|
697
|
+
if (this.outOfBudget(opts)) return report
|
|
698
|
+
if (opts.resumeCurrent === true) await this.resumeCurrent(report, launched, paused, opts)
|
|
699
|
+
if (this.outOfBudget(opts)) return report
|
|
700
|
+
await this.leaseSweep(report, launched, paused, opts)
|
|
701
|
+
if (this.outOfBudget(opts)) return report
|
|
702
|
+
if (opts.acceptNewWork !== false) {
|
|
703
|
+
await this.dispatch(report, launched, autoMergeUser, paused, opts)
|
|
704
|
+
}
|
|
705
|
+
// Fire-and-forget by contract: long synthesize/review sessions must not
|
|
706
|
+
// stop janitor, lease sweep, ticket dispatch, or signal handling on later
|
|
707
|
+
// watch ticks. Durable pause and infrastructure-error stops suppress
|
|
708
|
+
// launch. Pending control commands still launch the runner so the kernel
|
|
709
|
+
// can settle them under the repository lease.
|
|
710
|
+
await this.triggerHarvest()
|
|
711
|
+
return report
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
/** The repository-wide quiescence flag, from durable state. Guarded like
|
|
715
|
+
* `triggerHarvest`: `getRepoEvents` throws for a repository row no tick has
|
|
716
|
+
* created yet, and an unpaused repository is the correct reading of one that
|
|
717
|
+
* has never recorded a setting. */
|
|
718
|
+
private triageState(): string {
|
|
719
|
+
return this.deps.opts?.triageState ?? defaultTriageState(this.deps.config)
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/** Observation-only ready listing for Store-backed frontends while intake is
|
|
723
|
+
* disabled. It shares the active-ticket exclusion with dispatch but performs
|
|
724
|
+
* no dependency fetch, claim, bounce, build, workspace, or launch side
|
|
725
|
+
* effect, and deliberately does not alter TickReport compatibility. */
|
|
726
|
+
async observeReady(): Promise<{
|
|
727
|
+
queued: number
|
|
728
|
+
invalidTickets: number
|
|
729
|
+
ticketDiagnostics: string[]
|
|
730
|
+
}> {
|
|
731
|
+
this.deps.config = this.deps.getConfig?.() ?? this.deps.config
|
|
732
|
+
const activeTicketIds = new Set<string>()
|
|
733
|
+
for (const record of await this.deps.store.listBuilds()) {
|
|
734
|
+
if (record.repo !== this.deps.repo || record.ticket === undefined) continue
|
|
735
|
+
const state = reduceBuild(await this.deps.store.getEvents(record.slug))
|
|
736
|
+
if (state.status !== 'done' && state.status !== 'aborted') {
|
|
737
|
+
activeTicketIds.add(record.ticket.id)
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
const listing = await this.deps.tickets.listReady(readyCriteria(this.deps.config))
|
|
741
|
+
return {
|
|
742
|
+
queued: listing.tickets.filter((ticket) => !activeTicketIds.has(ticket.ref.id)).length,
|
|
743
|
+
invalidTickets: listing.diagnostics.length,
|
|
744
|
+
ticketDiagnostics: [...listing.diagnostics],
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
private async repositoryEvents() {
|
|
749
|
+
if ((await this.deps.store.getRepo(this.deps.repo)) === null) return []
|
|
750
|
+
return this.deps.store.getRepoEvents(this.deps.repo)
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
private async repositoryPaused(): Promise<boolean> {
|
|
754
|
+
return reduceDispatchSettings(await this.repositoryEvents()).paused
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
private async triggerHarvest(): Promise<void> {
|
|
758
|
+
const start = this.deps.startHarvest
|
|
759
|
+
if (start === undefined) return
|
|
760
|
+
const record = await this.deps.store.getRepo(this.deps.repo)
|
|
761
|
+
if (record === null) {
|
|
762
|
+
start()
|
|
763
|
+
return
|
|
764
|
+
}
|
|
765
|
+
const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
|
|
766
|
+
if (decideHarvestControl(state, this.maxHarvestRecoveryAttempts).kind !== 'park') {
|
|
767
|
+
start()
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
private async launch(slug: string, launched: Set<string>): Promise<LaunchRunnerResult> {
|
|
772
|
+
if (launched.has(slug)) return 'already-active'
|
|
773
|
+
launched.add(slug)
|
|
774
|
+
return this.deps.launchRunner(slug)
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
// ── 0. Settlement stage (durable supervision) ──────────────────────────────
|
|
778
|
+
|
|
779
|
+
/** The provider that owns a recorded execution, by the provider name on its
|
|
780
|
+
* `execution.started` fact. An execution from a provider this process no
|
|
781
|
+
* longer knows cannot be observed and is left to the stale-lease path. */
|
|
782
|
+
private executionOwner(provider: string): WorkspaceProvider | null {
|
|
783
|
+
if (provider === this.deps.workspaces.name) return this.deps.workspaces
|
|
784
|
+
return this.deps.retiredWorkspaces?.find((candidate) => candidate.name === provider) ?? null
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/** Observe and settle every foreign open execution of this repo. A build
|
|
788
|
+
* this process supervises in memory is skipped — its completion chain owns
|
|
789
|
+
* the durable facts. One listBuilds plus at most two bounded provider round
|
|
790
|
+
* trips per unsettled build; a failing build is contained so the rest of
|
|
791
|
+
* the stage still settles. */
|
|
792
|
+
private async settleForeignExecutions(report: TickReport, opts: TickOpts = {}): Promise<void> {
|
|
793
|
+
const active = this.deps.activeExecutions?.() ?? new Set<string>()
|
|
794
|
+
for (const record of await this.deps.store.listBuilds()) {
|
|
795
|
+
if (record.repo !== this.deps.repo) continue
|
|
796
|
+
if (active.has(record.slug)) continue
|
|
797
|
+
// Budget gate: once spent, stop settling further foreign executions.
|
|
798
|
+
if (this.outOfBudget(opts)) break
|
|
799
|
+
try {
|
|
800
|
+
const events = await this.deps.store.getEvents(record.slug)
|
|
801
|
+
const open = openExecution(events)
|
|
802
|
+
if (open === null) continue
|
|
803
|
+
const owner = this.executionOwner(open.provider)
|
|
804
|
+
const execution = owner?.buildExecution
|
|
805
|
+
if (execution?.observe === undefined || open.commandId === undefined) continue
|
|
806
|
+
const settlement = await settleExecution(
|
|
807
|
+
{
|
|
808
|
+
store: this.deps.store,
|
|
809
|
+
execution,
|
|
810
|
+
...(this.deps.settlePublication !== undefined
|
|
811
|
+
? { settlePublication: this.deps.settlePublication }
|
|
812
|
+
: {}),
|
|
813
|
+
},
|
|
814
|
+
record.slug,
|
|
815
|
+
events,
|
|
816
|
+
)
|
|
817
|
+
if (settlement === 'settled') report.settled += 1
|
|
818
|
+
} catch {
|
|
819
|
+
// An unobservable build must not stall settlement of the rest; the
|
|
820
|
+
// next tick retries it.
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/** Durable settlement for hosted-harvest executions (AUT-305), next to the
|
|
826
|
+
* build pass above and run in the same settlement stage: every
|
|
827
|
+
* `harvest.execution.started` without a matching released fact whose
|
|
828
|
+
* execution is not supervised by this process is observed through the
|
|
829
|
+
* owning provider. `lost`/`ended` reaps the disposable environment and
|
|
830
|
+
* journals the close fact with the snapshot purge outcome; `running`
|
|
831
|
+
* leaves it — the guest is the live supervisor and holds the adopted
|
|
832
|
+
* repository lease, and the next pass reaps it after it ends. The run
|
|
833
|
+
* itself needs no settlement here: an interrupted run stays open in the
|
|
834
|
+
* journal and the next threshold-triggered harvest resumes it under the
|
|
835
|
+
* repository lease. */
|
|
836
|
+
private async settleHarvestExecutions(opts: TickOpts = {}): Promise<void> {
|
|
837
|
+
const active = this.deps.activeHarvestExecutions?.() ?? new Set<string>()
|
|
838
|
+
let events: RepositoryEvent[]
|
|
839
|
+
try {
|
|
840
|
+
events = await this.repositoryEvents()
|
|
841
|
+
} catch {
|
|
842
|
+
// A repository with no journal yet has nothing to settle.
|
|
843
|
+
return
|
|
844
|
+
}
|
|
845
|
+
for (const execution of openHarvestExecutions(events)) {
|
|
846
|
+
if (active.has(execution.execution)) continue
|
|
847
|
+
// Budget gate: once spent, stop settling further harvest executions;
|
|
848
|
+
// the next tick retries what remains.
|
|
849
|
+
if (this.outOfBudget(opts)) break
|
|
850
|
+
try {
|
|
851
|
+
const owner = this.executionOwner(execution.provider)
|
|
852
|
+
const capability = owner?.harvestExecution
|
|
853
|
+
if (
|
|
854
|
+
owner === null ||
|
|
855
|
+
capability?.observe === undefined ||
|
|
856
|
+
execution.commandId === undefined
|
|
857
|
+
)
|
|
858
|
+
continue
|
|
859
|
+
const observation = await capability.observe({
|
|
860
|
+
provider: execution.provider,
|
|
861
|
+
workspaceRef: execution.environmentId,
|
|
862
|
+
environmentId: execution.environmentId,
|
|
863
|
+
...(execution.sessionId !== undefined ? { sessionId: execution.sessionId } : {}),
|
|
864
|
+
commandId: execution.commandId,
|
|
865
|
+
})
|
|
866
|
+
// Conservatism: an unresolvable observation (provider error) throws
|
|
867
|
+
// and is contained below — it never reaps.
|
|
868
|
+
if (observation.state === 'running') continue
|
|
869
|
+
let snapshots: { outcome: 'confirmed' | 'unknown'; deleted?: number; error?: string } = {
|
|
870
|
+
outcome: 'unknown',
|
|
871
|
+
error: 'provider did not report the snapshot purge',
|
|
872
|
+
}
|
|
873
|
+
if (owner.recovery !== undefined) {
|
|
874
|
+
const reap = await owner.recovery.reap({
|
|
875
|
+
provider: owner.name,
|
|
876
|
+
ref: execution.environmentId,
|
|
877
|
+
path: execution.environmentId,
|
|
878
|
+
branch: '',
|
|
879
|
+
})
|
|
880
|
+
snapshots = {
|
|
881
|
+
outcome: reap.snapshots.outcome === 'unknown' ? 'unknown' : 'confirmed',
|
|
882
|
+
...(reap.snapshots.deleted !== undefined ? { deleted: reap.snapshots.deleted } : {}),
|
|
883
|
+
...(reap.snapshots.error !== undefined ? { error: reap.snapshots.error } : {}),
|
|
884
|
+
}
|
|
885
|
+
} else {
|
|
886
|
+
await owner.release({
|
|
887
|
+
provider: owner.name,
|
|
888
|
+
ref: execution.environmentId,
|
|
889
|
+
path: execution.environmentId,
|
|
890
|
+
branch: '',
|
|
891
|
+
})
|
|
892
|
+
}
|
|
893
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
894
|
+
actor: DISPATCHER,
|
|
895
|
+
type: 'harvest.execution.released',
|
|
896
|
+
payload: {
|
|
897
|
+
execution: execution.execution,
|
|
898
|
+
environmentId: execution.environmentId,
|
|
899
|
+
snapshots,
|
|
900
|
+
},
|
|
901
|
+
})
|
|
902
|
+
} catch {
|
|
903
|
+
// An unobservable harvest execution must not stall the rest of the
|
|
904
|
+
// settlement stage; the next tick retries it.
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/** Durable idle settlement for operator sandboxes (AUT-340), staged after
|
|
910
|
+
* harvest settlement: every journal-live environment whose latest evidence
|
|
911
|
+
* is older than `[orchestrator].sandbox.idleMinutes` is stopped through the
|
|
912
|
+
* provider (snapshot kept) and closed with a dispatcher-authored fact.
|
|
913
|
+
* Because activity evidence is durably journaled, this one rule also
|
|
914
|
+
* settles the orphan case — a dead `ab mcp` process stops appending
|
|
915
|
+
* activity, so the next tick idles its environment out, exactly as harvest
|
|
916
|
+
* executions are settled. A journal-live environment whose journaled
|
|
917
|
+
* provider no longer matches the wired provider is likewise dispositioned
|
|
918
|
+
* once idle: its trail is closed with an unconfirmed release fact (AUT-377)
|
|
919
|
+
* — never a stop, since this dispatcher cannot reach a provider it is not
|
|
920
|
+
* wired to — and if the old provider returns, the next operator ensure()
|
|
921
|
+
* re-provisions and re-brackets the environment, so the disposition is
|
|
922
|
+
* reachable again. The stage is a no-op unless `[orchestrator]` is
|
|
923
|
+
* enabled and the wired provider hosts the capability; `unsupported`
|
|
924
|
+
* providers (local worktrees) are skipped silently. */
|
|
925
|
+
private async settleOrchestratorSandboxes(
|
|
926
|
+
report: TickReport,
|
|
927
|
+
opts: TickOpts = {},
|
|
928
|
+
): Promise<void> {
|
|
929
|
+
const config = this.deps.config
|
|
930
|
+
if (config.orchestrator?.enabled !== true) return
|
|
931
|
+
const capability = this.deps.workspaces.orchestratorSandbox
|
|
932
|
+
if (capability === undefined) return
|
|
933
|
+
const providerName = this.deps.workspaces.name
|
|
934
|
+
let events: RepositoryEvent[]
|
|
935
|
+
try {
|
|
936
|
+
events = await this.repositoryEvents()
|
|
937
|
+
} catch {
|
|
938
|
+
// A repository with no journal yet has nothing to settle.
|
|
939
|
+
return
|
|
940
|
+
}
|
|
941
|
+
const idleMs = config.orchestrator.sandbox.idleMinutes * 60_000
|
|
942
|
+
for (const environment of sandboxStates(events)) {
|
|
943
|
+
if (environment.state !== 'live') continue
|
|
944
|
+
// Budget gate: once spent, stop settling further environments; the
|
|
945
|
+
// next tick retries what remains.
|
|
946
|
+
if (this.outOfBudget(opts)) break
|
|
947
|
+
const lastEvidence = Date.parse(environment.lastEvidenceTs)
|
|
948
|
+
if (Number.isFinite(lastEvidence) && this.deps.clock().getTime() - lastEvidence < idleMs) {
|
|
949
|
+
continue
|
|
950
|
+
}
|
|
951
|
+
try {
|
|
952
|
+
if (environment.provider !== providerName) {
|
|
953
|
+
// Provider-mismatch disposition (AUT-377): the journaled provider no
|
|
954
|
+
// longer matches the wired provider, so environmentIds are out of this
|
|
955
|
+
// dispatcher's reach and idle settlement would otherwise skip it on
|
|
956
|
+
// every tick forever. Close the trail with an unconfirmed release —
|
|
957
|
+
// never a stop: nothing was stopped through a provider this
|
|
958
|
+
// dispatcher no longer owns. If the provider returns, the next
|
|
959
|
+
// operator ensure() re-provisions and re-brackets the environment.
|
|
960
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
961
|
+
actor: DISPATCHER,
|
|
962
|
+
type: 'orchestrator.sandbox.released',
|
|
963
|
+
payload: {
|
|
964
|
+
operator: environment.operator,
|
|
965
|
+
environmentId: environment.environmentId,
|
|
966
|
+
snapshots: {
|
|
967
|
+
outcome: 'unknown',
|
|
968
|
+
error: `environment owned by provider "${environment.provider}", not the wired provider "${providerName}" — unreachable at idle settlement`,
|
|
969
|
+
},
|
|
970
|
+
},
|
|
971
|
+
})
|
|
972
|
+
report.sandboxIdleStops += 1
|
|
973
|
+
continue
|
|
974
|
+
}
|
|
975
|
+
const outcome = await capability.stop({
|
|
976
|
+
operator: environment.operator,
|
|
977
|
+
environmentId: environment.environmentId,
|
|
978
|
+
})
|
|
979
|
+
if (outcome.outcome === 'unsupported') continue
|
|
980
|
+
if (outcome.outcome === 'stopped') {
|
|
981
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
982
|
+
actor: DISPATCHER,
|
|
983
|
+
type: 'orchestrator.sandbox.stopped',
|
|
984
|
+
payload: {
|
|
985
|
+
operator: environment.operator,
|
|
986
|
+
environmentId: environment.environmentId,
|
|
987
|
+
reason: 'idle',
|
|
988
|
+
},
|
|
989
|
+
})
|
|
990
|
+
report.sandboxIdleStops += 1
|
|
991
|
+
} else {
|
|
992
|
+
// The environment vanished without a stop (expired, reaped, or a
|
|
993
|
+
// dead host): close the orphan's trail with an unconfirmed-purge
|
|
994
|
+
// release fact, dispatcher-authored exactly as
|
|
995
|
+
// `harvest.execution.released` is.
|
|
996
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
997
|
+
actor: DISPATCHER,
|
|
998
|
+
type: 'orchestrator.sandbox.released',
|
|
999
|
+
payload: {
|
|
1000
|
+
operator: environment.operator,
|
|
1001
|
+
environmentId: environment.environmentId,
|
|
1002
|
+
snapshots: {
|
|
1003
|
+
outcome: 'unknown',
|
|
1004
|
+
error: 'environment absent at idle settlement',
|
|
1005
|
+
},
|
|
1006
|
+
},
|
|
1007
|
+
})
|
|
1008
|
+
report.sandboxIdleStops += 1
|
|
1009
|
+
}
|
|
1010
|
+
} catch (error) {
|
|
1011
|
+
// Contained: the next tick retries this environment.
|
|
1012
|
+
report.sandboxSettleFailures += 1
|
|
1013
|
+
report.janitorDiagnostics.push(
|
|
1014
|
+
`operator sandbox ${environment.environmentId}: idle settlement failed — ${error instanceof Error ? error.message : String(error)}`,
|
|
1015
|
+
)
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
// ── Background provisioning (durable supervision) ──────────────────────────
|
|
1021
|
+
|
|
1022
|
+
/** The latest `workspace.provision-started` not yet followed by a
|
|
1023
|
+
* provisioned/released fact — an open provisioning marker. */
|
|
1024
|
+
private openProvisionMarker(
|
|
1025
|
+
events: AbEvent[],
|
|
1026
|
+
): { provider: string; branch: string; generation: number; ts: string; seq: number } | undefined {
|
|
1027
|
+
let marker:
|
|
1028
|
+
| { provider: string; branch: string; generation: number; ts: string; seq: number }
|
|
1029
|
+
| undefined
|
|
1030
|
+
for (const event of events) {
|
|
1031
|
+
if (event.type === 'workspace.provision-started') {
|
|
1032
|
+
marker = { ...event.payload, ts: event.ts, seq: event.seq }
|
|
1033
|
+
} else if (
|
|
1034
|
+
marker !== undefined &&
|
|
1035
|
+
(event.type === 'workspace.provisioned' || event.type === 'workspace.released')
|
|
1036
|
+
) {
|
|
1037
|
+
marker = undefined
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
return marker
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
/** Liveness of the lease behind an open provision marker: `live` means a
|
|
1044
|
+
* fresh heartbeat (someone is provisioning); `expired` means the owner died
|
|
1045
|
+
* within the last TTL and is adopted immediately; `absent` means the owner
|
|
1046
|
+
* may have crashed between marker and claim — adopted only after the
|
|
1047
|
+
* `provisionStaleMs` backstop. */
|
|
1048
|
+
private async provisionMarkerLiveness(slug: string): Promise<'live' | 'expired' | 'absent'> {
|
|
1049
|
+
const record = await this.deps.store.getBuild(slug)
|
|
1050
|
+
if (record?.lease === undefined) return 'absent'
|
|
1051
|
+
return new Date(record.lease.expiresAt).getTime() > this.deps.clock().getTime()
|
|
1052
|
+
? 'live'
|
|
1053
|
+
: 'expired'
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
/** Kick (or defer to) the background provisioning continuation for a remote
|
|
1057
|
+
* build with no open workspace. Returns what happened so the caller can
|
|
1058
|
+
* report progress without awaiting any of it. */
|
|
1059
|
+
private async ensureProvisionContinuation(
|
|
1060
|
+
record: BuildRecord,
|
|
1061
|
+
events: AbEvent[],
|
|
1062
|
+
tail: 'dispatch' | 'launch',
|
|
1063
|
+
seed?: {
|
|
1064
|
+
ticket: Ticket
|
|
1065
|
+
body: string
|
|
1066
|
+
authoredSession?: string
|
|
1067
|
+
autoMergeUser?: string
|
|
1068
|
+
},
|
|
1069
|
+
): Promise<'kicked' | 'in-flight' | 'foreign-live'> {
|
|
1070
|
+
const slug = record.slug
|
|
1071
|
+
if (this.continuations.has(slug)) return 'in-flight'
|
|
1072
|
+
const marker = this.openProvisionMarker(events)
|
|
1073
|
+
if (marker !== undefined) {
|
|
1074
|
+
const liveness = await this.provisionMarkerLiveness(slug)
|
|
1075
|
+
const failedAttempt = events.some(
|
|
1076
|
+
(event) =>
|
|
1077
|
+
event.type === 'dispatch.failed' &&
|
|
1078
|
+
event.payload.stage === 'workspace' &&
|
|
1079
|
+
event.seq > marker.seq,
|
|
1080
|
+
)
|
|
1081
|
+
if (liveness === 'live') return 'foreign-live'
|
|
1082
|
+
if (
|
|
1083
|
+
liveness === 'absent' &&
|
|
1084
|
+
!failedAttempt &&
|
|
1085
|
+
this.deps.clock().getTime() - new Date(marker.ts).getTime() < this.provisionStaleMs
|
|
1086
|
+
) {
|
|
1087
|
+
// A lease-less marker is the crash window this backstop exists for:
|
|
1088
|
+
// the owner may still be between the append and its claim. A failed
|
|
1089
|
+
// attempt (dispatch.failed after the marker) adopts immediately.
|
|
1090
|
+
return 'foreign-live'
|
|
1091
|
+
}
|
|
1092
|
+
// Expired lease, failed attempt, or an old lease-less marker: adopt.
|
|
1093
|
+
}
|
|
1094
|
+
const branch = record.branch ?? `ab/${slug}`
|
|
1095
|
+
const generation = events.filter((event) => event.type === 'workspace.provisioned').length
|
|
1096
|
+
await this.deps.store.append(slug, {
|
|
1097
|
+
actor: DISPATCHER,
|
|
1098
|
+
type: 'workspace.provision-started',
|
|
1099
|
+
payload: { provider: this.deps.workspaces.name, branch, generation },
|
|
1100
|
+
})
|
|
1101
|
+
const entry: ProvisionContinuation = {
|
|
1102
|
+
slug,
|
|
1103
|
+
instance: `provision-${this.deps.ids('inst')}`,
|
|
1104
|
+
controller: new AbortController(),
|
|
1105
|
+
promise: Promise.resolve(),
|
|
1106
|
+
}
|
|
1107
|
+
// Registered synchronously before any await of the continuation itself so
|
|
1108
|
+
// a concurrent re-entry defers to it instead of double-kicking.
|
|
1109
|
+
this.continuations.set(slug, entry)
|
|
1110
|
+
entry.promise = this.runProvisionContinuation(entry, {
|
|
1111
|
+
branch,
|
|
1112
|
+
generation,
|
|
1113
|
+
seed,
|
|
1114
|
+
tail,
|
|
1115
|
+
})
|
|
1116
|
+
return 'kicked'
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
/** The background continuation: claim and heartbeat the build's execution
|
|
1120
|
+
* lease, provision, append the durable fact, then complete the remaining
|
|
1121
|
+
* boundaries and launch. Every provider call and boundary write is fenced
|
|
1122
|
+
* by the cancellation token, so a superseded supervisor exits without
|
|
1123
|
+
* colliding with its replacement. Failures append
|
|
1124
|
+
* `dispatch.failed {stage: 'workspace'}` (plus the infrastructure failure
|
|
1125
|
+
* when applicable) and stay retryable. */
|
|
1126
|
+
private async runProvisionContinuation(
|
|
1127
|
+
entry: ProvisionContinuation,
|
|
1128
|
+
input: {
|
|
1129
|
+
branch: string
|
|
1130
|
+
generation: number
|
|
1131
|
+
seed?:
|
|
1132
|
+
| {
|
|
1133
|
+
ticket: Ticket
|
|
1134
|
+
body: string
|
|
1135
|
+
authoredSession?: string
|
|
1136
|
+
autoMergeUser?: string
|
|
1137
|
+
}
|
|
1138
|
+
| undefined
|
|
1139
|
+
tail: 'dispatch' | 'launch'
|
|
1140
|
+
},
|
|
1141
|
+
): Promise<void> {
|
|
1142
|
+
const { store, workspaces, config } = this.deps
|
|
1143
|
+
const slug = entry.slug
|
|
1144
|
+
let leaseClaimed = false
|
|
1145
|
+
let heartbeat: ReturnType<typeof setInterval> | undefined
|
|
1146
|
+
const stage: EventPayload<'dispatch.failed'>['stage'] = 'workspace'
|
|
1147
|
+
try {
|
|
1148
|
+
if (entry.controller.signal.aborted) {
|
|
1149
|
+
// Signalled before the body even started (teardown raced the kick):
|
|
1150
|
+
// exit before claiming anything; stopProvisioning already released
|
|
1151
|
+
// the marker's lease so the next supervisor adopts immediately.
|
|
1152
|
+
return
|
|
1153
|
+
}
|
|
1154
|
+
leaseClaimed = await store.claimLease(slug, entry.instance, BUILD_EXECUTION_LEASE_TTL_MS)
|
|
1155
|
+
if (!leaseClaimed) return // another live holder — the next tick re-evaluates
|
|
1156
|
+
heartbeat = setInterval(() => {
|
|
1157
|
+
void store.claimLease(slug, entry.instance, BUILD_EXECUTION_LEASE_TTL_MS).catch(() => {
|
|
1158
|
+
// Store unreachable: retry on the next beat; expiry fences.
|
|
1159
|
+
})
|
|
1160
|
+
}, PROVISION_HEARTBEAT_MS)
|
|
1161
|
+
heartbeat.unref?.()
|
|
1162
|
+
if (entry.controller.signal.aborted) {
|
|
1163
|
+
// Teardown: exit before provisioning. The finally releases the lease,
|
|
1164
|
+
// so the next supervisor adopts the marker immediately.
|
|
1165
|
+
return
|
|
1166
|
+
}
|
|
1167
|
+
const events = await store.getEvents(slug)
|
|
1168
|
+
const priorGenerations = events.filter(
|
|
1169
|
+
(event) => event.type === 'workspace.provisioned',
|
|
1170
|
+
).length
|
|
1171
|
+
const handle = await workspaces.provision({
|
|
1172
|
+
// Local providers provision FROM the physical checkout; remote
|
|
1173
|
+
// providers ignore the value (their host seams are injected).
|
|
1174
|
+
repo: this.deps.checkout ?? this.deps.repo,
|
|
1175
|
+
baseBranch: baseBranchOf(events, config),
|
|
1176
|
+
branch: input.branch,
|
|
1177
|
+
...(priorGenerations > 0 || input.generation > 0
|
|
1178
|
+
? {
|
|
1179
|
+
revision: this.recoveryCheckpoint(events),
|
|
1180
|
+
generation: Math.max(priorGenerations, input.generation),
|
|
1181
|
+
}
|
|
1182
|
+
: {}),
|
|
1183
|
+
})
|
|
1184
|
+
// The provisioned fact is true and welcome even when a teardown signal
|
|
1185
|
+
// landed mid-call; the token check below still prevents a superseded
|
|
1186
|
+
// supervisor from launching.
|
|
1187
|
+
await store.append(slug, {
|
|
1188
|
+
actor: DISPATCHER,
|
|
1189
|
+
type: 'workspace.provisioned',
|
|
1190
|
+
payload: {
|
|
1191
|
+
provider: handle.provider,
|
|
1192
|
+
ref: handle.ref,
|
|
1193
|
+
path: handle.path,
|
|
1194
|
+
...(handle.localPath !== undefined ? { localPath: handle.localPath } : {}),
|
|
1195
|
+
branch: handle.branch,
|
|
1196
|
+
base: handle.base,
|
|
1197
|
+
},
|
|
1198
|
+
} satisfies EventWrite<'workspace.provisioned'>)
|
|
1199
|
+
// Release before the launch: the launcher re-claims the lease under the
|
|
1200
|
+
// runner's own instance. The tiny gap is safe because every downstream
|
|
1201
|
+
// action is single-flight in-process and lease-fenced cross-process.
|
|
1202
|
+
if (heartbeat !== undefined) {
|
|
1203
|
+
clearInterval(heartbeat)
|
|
1204
|
+
heartbeat = undefined
|
|
1205
|
+
}
|
|
1206
|
+
await store.releaseLease(slug, entry.instance)
|
|
1207
|
+
leaseClaimed = false
|
|
1208
|
+
if (entry.controller.signal.aborted) return
|
|
1209
|
+
const record = await store.getBuild(slug)
|
|
1210
|
+
if (record === null) return
|
|
1211
|
+
await (input.tail === 'dispatch'
|
|
1212
|
+
? this.completeDispatchBoundaries(record, input.seed)
|
|
1213
|
+
: this.deps.launchRunner(slug))
|
|
1214
|
+
} catch (error) {
|
|
1215
|
+
const events = await store.getEvents(slug)
|
|
1216
|
+
if (workspaces.recovery !== undefined) {
|
|
1217
|
+
const open = openWorkspace(events)
|
|
1218
|
+
const latestRef = [...events].reverse().find((event) => event.type === 'workspace.released')
|
|
1219
|
+
await this.recordInfrastructureFailure(slug, events, {
|
|
1220
|
+
provider: workspaces.name,
|
|
1221
|
+
workspaceRef:
|
|
1222
|
+
open?.ref ??
|
|
1223
|
+
(latestRef?.type === 'workspace.released' && 'ref' in latestRef.payload
|
|
1224
|
+
? latestRef.payload.ref
|
|
1225
|
+
: slug),
|
|
1226
|
+
operation: 'provision',
|
|
1227
|
+
error,
|
|
1228
|
+
cleanupPending: true,
|
|
1229
|
+
})
|
|
1230
|
+
}
|
|
1231
|
+
const attempt = events.filter((event) => event.type === 'dispatch.failed').length + 1
|
|
1232
|
+
const message = (error instanceof Error ? error.message : String(error)).trim()
|
|
1233
|
+
await store.append(slug, {
|
|
1234
|
+
actor: DISPATCHER,
|
|
1235
|
+
type: 'dispatch.failed',
|
|
1236
|
+
payload: {
|
|
1237
|
+
stage,
|
|
1238
|
+
attempt,
|
|
1239
|
+
error: message || 'dispatch failed without an error message',
|
|
1240
|
+
},
|
|
1241
|
+
})
|
|
1242
|
+
} finally {
|
|
1243
|
+
if (heartbeat !== undefined) clearInterval(heartbeat)
|
|
1244
|
+
if (leaseClaimed) {
|
|
1245
|
+
try {
|
|
1246
|
+
await store.releaseLease(slug, entry.instance)
|
|
1247
|
+
} catch {
|
|
1248
|
+
// Expiry fences an ambiguous release.
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
if (this.continuations.get(slug)?.instance === entry.instance) {
|
|
1252
|
+
this.continuations.delete(slug)
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
/** Await every in-flight provisioning continuation. A test/observability
|
|
1258
|
+
* seam — a tick never awaits these; the durable facts remain the contract. */
|
|
1259
|
+
async drainProvisioning(): Promise<void> {
|
|
1260
|
+
await Promise.allSettled([...this.continuations.values()].map((entry) => entry.promise))
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
/** Whether any provisioning continuation is still running in this process.
|
|
1264
|
+
* A one-shot invocation consults it to keep awaiting (within its deadline)
|
|
1265
|
+
* instead of tearing the continuation down as abandoned. */
|
|
1266
|
+
provisioningInFlight(): boolean {
|
|
1267
|
+
return this.continuations.size > 0
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
/** Teardown: signal every live continuation, then record a durable failure
|
|
1271
|
+
* fact for any whose provisioning marker is still open (so the next
|
|
1272
|
+
* supervisor adopts it immediately rather than after the crash backstop),
|
|
1273
|
+
* then release each one's provisioning lease, in that order. Never awaited
|
|
1274
|
+
* by a tick. */
|
|
1275
|
+
async stopProvisioning(): Promise<void> {
|
|
1276
|
+
const entries = [...this.continuations.values()]
|
|
1277
|
+
for (const entry of entries) entry.controller.abort()
|
|
1278
|
+
for (const entry of entries) {
|
|
1279
|
+
try {
|
|
1280
|
+
const events = await this.deps.store.getEvents(entry.slug)
|
|
1281
|
+
if (this.openProvisionMarker(events) !== undefined) {
|
|
1282
|
+
const attempt = events.filter((event) => event.type === 'dispatch.failed').length + 1
|
|
1283
|
+
await this.deps.store.append(entry.slug, {
|
|
1284
|
+
actor: DISPATCHER,
|
|
1285
|
+
type: 'dispatch.failed',
|
|
1286
|
+
payload: {
|
|
1287
|
+
stage: 'workspace',
|
|
1288
|
+
attempt,
|
|
1289
|
+
error:
|
|
1290
|
+
'provisioning abandoned at dispatcher teardown; the next supervisor adopts the open marker',
|
|
1291
|
+
},
|
|
1292
|
+
})
|
|
1293
|
+
}
|
|
1294
|
+
} catch {
|
|
1295
|
+
// The durable marker plus the released lease still gate adoption.
|
|
1296
|
+
}
|
|
1297
|
+
try {
|
|
1298
|
+
await this.deps.store.releaseLease(entry.slug, entry.instance)
|
|
1299
|
+
} catch {
|
|
1300
|
+
// Expiry fences an ambiguous release.
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
// ── a. Janitor (SPEC §15.7, D1) ────────────────────────────────────────────
|
|
1306
|
+
|
|
1307
|
+
private async janitor(
|
|
1308
|
+
report: TickReport,
|
|
1309
|
+
launched: Set<string>,
|
|
1310
|
+
opts: TickOpts = {},
|
|
1311
|
+
): Promise<void> {
|
|
1312
|
+
// Listing is repository-wide work: if it fails there is no individual
|
|
1313
|
+
// build to attribute, so the caller's existing top-level boundary owns it.
|
|
1314
|
+
const records = await this.deps.store.listBuilds()
|
|
1315
|
+
for (const record of records) {
|
|
1316
|
+
// One dispatcher per repo (§12), but the store is shared by design
|
|
1317
|
+
// (§7.2) — another repo's builds are another dispatcher's duty. Acting
|
|
1318
|
+
// on them would poll foreign PRs and break single-writer discipline.
|
|
1319
|
+
if (record.repo !== this.deps.repo) continue
|
|
1320
|
+
// Budget gate: the janitor's per-build path issues forge/store transport
|
|
1321
|
+
// calls (PR probes, closePr, deleteBranch); once the budget is spent,
|
|
1322
|
+
// stop starting new ones. The remaining builds stay janitor due — the
|
|
1323
|
+
// next invocation re-runs them (§3.3 idempotency).
|
|
1324
|
+
if (this.outOfBudget(opts)) break
|
|
1325
|
+
try {
|
|
1326
|
+
// Once a record is known, contain its complete janitor path: loading
|
|
1327
|
+
// and reducing facts, cleanup, forge/ticket calls, and store writes.
|
|
1328
|
+
// Existing partial facts leave the same work due on the next tick.
|
|
1329
|
+
const events = await this.deps.store.getEvents(record.slug)
|
|
1330
|
+
const state = reduceBuild(events)
|
|
1331
|
+
const executionLeaseLive = this.hasLiveExecutionLease(record)
|
|
1332
|
+
// Pipeline/ticket/workspace state is settled, but release-asset cleanup
|
|
1333
|
+
// has its own crash window after build.completed. Revisit only pending
|
|
1334
|
+
// hosted handles; this never relaunches a runner or consumes capacity.
|
|
1335
|
+
if (state.status === 'done') {
|
|
1336
|
+
await this.reclaimPrAttachments(record.slug, events)
|
|
1337
|
+
continue
|
|
1338
|
+
}
|
|
1339
|
+
// Abort is destructive human judgment and therefore outranks a queued
|
|
1340
|
+
// discard when both requests raced into the same stream.
|
|
1341
|
+
if (
|
|
1342
|
+
!executionLeaseLive &&
|
|
1343
|
+
state.status === 'queued' &&
|
|
1344
|
+
state.pendingCommands.some((command) => command.command === 'abort')
|
|
1345
|
+
) {
|
|
1346
|
+
const acknowledged = await this.deps.store.append(record.slug, {
|
|
1347
|
+
actor: DISPATCHER,
|
|
1348
|
+
type: 'build.aborted',
|
|
1349
|
+
payload: {},
|
|
1350
|
+
} satisfies EventWrite<'build.aborted'>)
|
|
1351
|
+
events.push(acknowledged)
|
|
1352
|
+
await this.cleanupAborted(record, events, report)
|
|
1353
|
+
continue
|
|
1354
|
+
}
|
|
1355
|
+
// Discard is a queued-only recovery control. A request that raced with
|
|
1356
|
+
// runner attachment is deliberately inert: it must never tear a live
|
|
1357
|
+
// runner's workspace down or return its ticket for a duplicate build.
|
|
1358
|
+
if (
|
|
1359
|
+
!executionLeaseLive &&
|
|
1360
|
+
state.status === 'queued' &&
|
|
1361
|
+
state.discardRequest !== undefined
|
|
1362
|
+
) {
|
|
1363
|
+
await this.cleanupDiscarded(record, events, report)
|
|
1364
|
+
continue
|
|
1365
|
+
}
|
|
1366
|
+
if (!executionLeaseLive && state.status === 'aborted') {
|
|
1367
|
+
await this.cleanupAborted(record, events, report)
|
|
1368
|
+
continue
|
|
1369
|
+
}
|
|
1370
|
+
if (state.pr)
|
|
1371
|
+
await this.checkPr(record, events, state, report, launched, executionLeaseLive)
|
|
1372
|
+
} catch (error) {
|
|
1373
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
1374
|
+
report.janitorFailed += 1
|
|
1375
|
+
report.janitorDiagnostics.push(`build ${record.slug}: janitor failed — ${message}`)
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
private forgeWorkspacePath(events: AbEvent[]): string {
|
|
1381
|
+
const open = openWorkspace(events)
|
|
1382
|
+
if (open?.provider === 'vercel-sandbox') return this.deps.repo
|
|
1383
|
+
// Local forges run git with cwd = workspacePath: it must be the physical
|
|
1384
|
+
// checkout, never the store identity (an origin URL since the identity
|
|
1385
|
+
// change; a path only for origin-less fixtures).
|
|
1386
|
+
return open?.localPath ?? open?.path ?? open?.ref ?? this.deps.checkout ?? this.deps.repo
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
private hasLiveExecutionLease(record: BuildRecord): boolean {
|
|
1390
|
+
return (
|
|
1391
|
+
record.lease !== undefined &&
|
|
1392
|
+
new Date(record.lease.expiresAt).getTime() > this.deps.clock().getTime()
|
|
1393
|
+
)
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
/** Complete abort cleanup as an ordered checkpointed saga. External effects
|
|
1397
|
+
* are idempotent; a fact after each one prevents already-settled work from
|
|
1398
|
+
* repeating on later ticks. `build.completed` is deliberately last. */
|
|
1399
|
+
private async cleanupAborted(
|
|
1400
|
+
record: BuildRecord,
|
|
1401
|
+
events: AbEvent[],
|
|
1402
|
+
report: TickReport,
|
|
1403
|
+
): Promise<void> {
|
|
1404
|
+
const { store, tickets, forge } = this.deps
|
|
1405
|
+
const branch = record.branch
|
|
1406
|
+
// Whether this build ever owned a local checkout workspace, judged on the
|
|
1407
|
+
// provisioned facts (not the open one, which is null when the build never
|
|
1408
|
+
// provisioned or already released — only the former must skip the
|
|
1409
|
+
// local-branch git step; a released local workspace may still have left a
|
|
1410
|
+
// branch in the checkout). Remote workspaces (vercel-sandbox) never create
|
|
1411
|
+
// a local branch in the main checkout, and origin mode has no checkout at
|
|
1412
|
+
// all: the local-branch git step is skipped, and the saga's later steps
|
|
1413
|
+
// tolerate its absence.
|
|
1414
|
+
const hadLocalWorkspace = events.some(
|
|
1415
|
+
(event) =>
|
|
1416
|
+
event.type === 'workspace.provisioned' && event.payload.provider !== 'vercel-sandbox',
|
|
1417
|
+
)
|
|
1418
|
+
const has = (type: AbEvent['type']): boolean => events.some((event) => event.type === type)
|
|
1419
|
+
const append = async <T extends EventWrite['type']>(write: EventWrite<T>): Promise<void> => {
|
|
1420
|
+
const event = await store.append(record.slug, write)
|
|
1421
|
+
events.push(event as AbEvent)
|
|
1422
|
+
}
|
|
1423
|
+
const fail = (operation: string, identity: string, error: unknown): never => {
|
|
1424
|
+
const detail = error instanceof Error ? error.message : String(error)
|
|
1425
|
+
throw new Error(
|
|
1426
|
+
`abort cleanup ${operation} failed for ${identity}; remaining cleanup will retry on next dispatcher tick — ${detail}`,
|
|
1427
|
+
)
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
if (record.lease !== undefined) {
|
|
1431
|
+
try {
|
|
1432
|
+
await store.releaseLease(record.slug, record.lease.holder)
|
|
1433
|
+
} catch (error) {
|
|
1434
|
+
fail('lease release', `build ${record.slug}`, error)
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
const reduced = reduceBuild(events)
|
|
1439
|
+
if (reduced.pr !== undefined && !has('pr.closed') && !has('pr.merged')) {
|
|
1440
|
+
const closePr = forge.closePr
|
|
1441
|
+
if (closePr === undefined) {
|
|
1442
|
+
fail(
|
|
1443
|
+
'PR close',
|
|
1444
|
+
`${forge.name} PR #${reduced.pr.number}`,
|
|
1445
|
+
new Error(
|
|
1446
|
+
`selected forge "${forge.name}" does not implement closePr; upgrade its plugin`,
|
|
1447
|
+
),
|
|
1448
|
+
)
|
|
1449
|
+
}
|
|
1450
|
+
try {
|
|
1451
|
+
const result = await closePr!.call(
|
|
1452
|
+
forge,
|
|
1453
|
+
this.forgeWorkspacePath(events),
|
|
1454
|
+
reduced.pr.number,
|
|
1455
|
+
)
|
|
1456
|
+
await append(
|
|
1457
|
+
result.state === 'merged'
|
|
1458
|
+
? { actor: DISPATCHER, type: 'pr.merged', payload: { sha: result.sha } }
|
|
1459
|
+
: { actor: DISPATCHER, type: 'pr.closed', payload: {} },
|
|
1460
|
+
)
|
|
1461
|
+
} catch (error) {
|
|
1462
|
+
fail('PR close', `${forge.name} PR #${reduced.pr.number}`, error)
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
try {
|
|
1467
|
+
const released = await this.releaseWorkspace(record.slug, events, 'abort')
|
|
1468
|
+
if (released !== undefined) events.push(released)
|
|
1469
|
+
} catch (error) {
|
|
1470
|
+
fail('workspace release', `build ${record.slug}`, error)
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
// A provider push happens before its completion fact is appended. Always
|
|
1474
|
+
// attempt the exact idempotent delete so an abort in that crash window
|
|
1475
|
+
// cannot strand an otherwise unobservable remote branch.
|
|
1476
|
+
if (branch !== undefined && !has('abort.remote-branch-deleted')) {
|
|
1477
|
+
const deleteBranch = forge.deleteBranch
|
|
1478
|
+
if (deleteBranch === undefined) {
|
|
1479
|
+
fail(
|
|
1480
|
+
'remote branch deletion',
|
|
1481
|
+
`${forge.name} branch ${branch}`,
|
|
1482
|
+
new Error(
|
|
1483
|
+
`selected forge "${forge.name}" does not implement deleteBranch; upgrade its plugin`,
|
|
1484
|
+
),
|
|
1485
|
+
)
|
|
1486
|
+
}
|
|
1487
|
+
try {
|
|
1488
|
+
// Local forges treat workspacePath as a git cwd: pass the physical
|
|
1489
|
+
// checkout, not the store identity (an origin URL in checkout mode).
|
|
1490
|
+
// API forges ignore the argument.
|
|
1491
|
+
await deleteBranch!.call(forge, this.deps.checkout ?? this.deps.repo, branch)
|
|
1492
|
+
await append({
|
|
1493
|
+
actor: DISPATCHER,
|
|
1494
|
+
type: 'abort.remote-branch-deleted',
|
|
1495
|
+
payload: { branch },
|
|
1496
|
+
})
|
|
1497
|
+
} catch (error) {
|
|
1498
|
+
fail('remote branch deletion', `${forge.name} branch ${branch}`, error)
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
if (branch !== undefined && !has('abort.local-branch-deleted') && hadLocalWorkspace) {
|
|
1503
|
+
const ref = `refs/heads/${branch}`
|
|
1504
|
+
try {
|
|
1505
|
+
// Local-branch cleanup runs in the physical checkout — the store
|
|
1506
|
+
// identity is an origin URL (or a path only for origin-less fixtures),
|
|
1507
|
+
// and git with cwd = a URL fails the whole saga.
|
|
1508
|
+
const cwd = this.deps.checkout ?? this.deps.repo
|
|
1509
|
+
const valid = await this.deps.exec(['git', 'check-ref-format', ref], {
|
|
1510
|
+
cwd,
|
|
1511
|
+
})
|
|
1512
|
+
if (valid.exitCode !== 0) throw new Error(`invalid exact build ref ${ref}`)
|
|
1513
|
+
const deleted = await this.deps.exec(['git', 'update-ref', '-d', ref], {
|
|
1514
|
+
cwd,
|
|
1515
|
+
})
|
|
1516
|
+
if (deleted.exitCode !== 0) {
|
|
1517
|
+
throw new Error(
|
|
1518
|
+
deleted.stderr.trim() || deleted.stdout.trim() || `git exited ${deleted.exitCode}`,
|
|
1519
|
+
)
|
|
1520
|
+
}
|
|
1521
|
+
await append({
|
|
1522
|
+
actor: DISPATCHER,
|
|
1523
|
+
type: 'abort.local-branch-deleted',
|
|
1524
|
+
payload: { branch },
|
|
1525
|
+
})
|
|
1526
|
+
} catch (error) {
|
|
1527
|
+
fail('local branch deletion', ref, error)
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
if (record.ticket !== undefined && !has('abort.ticket-returned')) {
|
|
1532
|
+
const label = 'autobuild:aborted'
|
|
1533
|
+
try {
|
|
1534
|
+
const current = await tickets.get(record.ticket.id)
|
|
1535
|
+
if (current === null) throw new Error(`ticket no longer exists in ${tickets.name}`)
|
|
1536
|
+
if (!current.labels.includes(label)) {
|
|
1537
|
+
await tickets.update(record.ticket.id, { labels: [...current.labels, label] })
|
|
1538
|
+
}
|
|
1539
|
+
const triageState = this.triageState()
|
|
1540
|
+
await tickets.transition(record.ticket.id, triageState)
|
|
1541
|
+
await append({
|
|
1542
|
+
actor: DISPATCHER,
|
|
1543
|
+
type: 'abort.ticket-returned',
|
|
1544
|
+
payload: { ticket: record.ticket, state: triageState, label },
|
|
1545
|
+
})
|
|
1546
|
+
} catch (error) {
|
|
1547
|
+
fail('ticket handback', `${tickets.name} ticket ${record.ticket.id}`, error)
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
await append({
|
|
1552
|
+
actor: DISPATCHER,
|
|
1553
|
+
type: 'build.completed',
|
|
1554
|
+
payload: { outcome: 'abandoned' },
|
|
1555
|
+
})
|
|
1556
|
+
await this.reclaimPrAttachments(record.slug, events)
|
|
1557
|
+
report.abandoned += 1
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
/** A discard is recovery from dispatcher plumbing, not judgment about the
|
|
1561
|
+
* ticket. Return it to the configured Ready state and terminalize last so
|
|
1562
|
+
* every partial cleanup is retried after a crash. */
|
|
1563
|
+
private async cleanupDiscarded(
|
|
1564
|
+
record: BuildRecord,
|
|
1565
|
+
events: AbEvent[],
|
|
1566
|
+
report: TickReport,
|
|
1567
|
+
): Promise<void> {
|
|
1568
|
+
const { store, tickets } = this.deps
|
|
1569
|
+
await this.releaseWorkspace(record.slug, events, 'discard')
|
|
1570
|
+
if (record.lease !== undefined) {
|
|
1571
|
+
await store.releaseLease(record.slug, record.lease.holder)
|
|
1572
|
+
}
|
|
1573
|
+
if (record.ticket !== undefined) {
|
|
1574
|
+
await tickets.transition(record.ticket.id, this.deps.config.tickets.readyState)
|
|
1575
|
+
await tickets.comment(
|
|
1576
|
+
record.ticket.id,
|
|
1577
|
+
`build ${record.slug} was discarded during dispatch — returned to ${this.deps.config.tickets.readyState}`,
|
|
1578
|
+
)
|
|
1579
|
+
}
|
|
1580
|
+
await store.append(record.slug, {
|
|
1581
|
+
actor: DISPATCHER,
|
|
1582
|
+
type: 'build.completed',
|
|
1583
|
+
payload: { outcome: 'discarded' },
|
|
1584
|
+
} satisfies EventWrite<'build.completed'>)
|
|
1585
|
+
report.discarded += 1
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
/** Post-PR epilogue (§15.7): poll the forge, emit `pr.*` facts (deduped
|
|
1589
|
+
* against the reduced `prState`), release/complete, project to the ticket. */
|
|
1590
|
+
private async checkPr(
|
|
1591
|
+
record: BuildRecord,
|
|
1592
|
+
events: AbEvent[],
|
|
1593
|
+
state: BuildState,
|
|
1594
|
+
report: TickReport,
|
|
1595
|
+
launched: Set<string>,
|
|
1596
|
+
executionLeaseLive: boolean,
|
|
1597
|
+
): Promise<void> {
|
|
1598
|
+
const { store, tickets, forge } = this.deps
|
|
1599
|
+
const pr = state.pr
|
|
1600
|
+
if (!pr) return
|
|
1601
|
+
// Forge calls run from the workspace when it still exists (it does until
|
|
1602
|
+
// the build completes); fall back to the repo itself for odd logs.
|
|
1603
|
+
const workspacePath = this.forgeWorkspacePath(events)
|
|
1604
|
+
const prState = await forge.getPrState(workspacePath, pr.number)
|
|
1605
|
+
const autoMerge = prState.state === 'open' ? pendingAutoMerge(state) : undefined
|
|
1606
|
+
|
|
1607
|
+
// Revoking consent takes priority even while the PR is conflicted. A live
|
|
1608
|
+
// native request could otherwise merge immediately after reconcile pushes
|
|
1609
|
+
// and checks pass, before the next janitor poll gets a chance to cancel it.
|
|
1610
|
+
if (autoMerge?.enabled === false) {
|
|
1611
|
+
const result = await forge.setAutoMerge(workspacePath, pr.number, false)
|
|
1612
|
+
if (result.kind !== 'applied') {
|
|
1613
|
+
throw new Error(`forge returned ${result.kind} while disabling native auto-merge`)
|
|
1614
|
+
}
|
|
1615
|
+
await store.append(record.slug, {
|
|
1616
|
+
actor: DISPATCHER,
|
|
1617
|
+
type: autoMergeApplicationType(false),
|
|
1618
|
+
payload: { commandSeq: autoMerge.commandSeq },
|
|
1619
|
+
})
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
// A positive conflict re-enters reconcile before any enable attempt. In
|
|
1623
|
+
// particular, an ungated fallback can never race ahead of the conflict
|
|
1624
|
+
// event and its full post-reconcile verification cycle.
|
|
1625
|
+
if (prState.state === 'open' && prState.mergeable === false) {
|
|
1626
|
+
// Dedupe: a reduced prState of 'conflicted' means reconcile is already
|
|
1627
|
+
// pending (a `reconcile.completed` returns it to 'open').
|
|
1628
|
+
if (state.prState === 'conflicted') return
|
|
1629
|
+
const baseSha = await this.baseSha(baseBranchOf(events, this.deps.config))
|
|
1630
|
+
const conflicted = await store.append(record.slug, {
|
|
1631
|
+
actor: DISPATCHER,
|
|
1632
|
+
type: 'pr.conflicted',
|
|
1633
|
+
payload: { baseSha },
|
|
1634
|
+
} satisfies EventWrite<'pr.conflicted'>)
|
|
1635
|
+
events.push(conflicted)
|
|
1636
|
+
if (openWorkspace(events) === null && this.deps.workspaces.recovery !== undefined) {
|
|
1637
|
+
// The reconcile workspace is provisioned by the tracked background
|
|
1638
|
+
// continuation, whose tail re-attaches the build-runner; this tick
|
|
1639
|
+
// never blocks on guest bootstrap.
|
|
1640
|
+
const kicked = await this.ensureProvisionContinuation(record, events, 'launch')
|
|
1641
|
+
if (kicked === 'kicked') report.provisioning += 1
|
|
1642
|
+
report.conflicted += 1
|
|
1643
|
+
return
|
|
1644
|
+
}
|
|
1645
|
+
// The dispatcher never runs agents (§15.7): re-attach a build-runner,
|
|
1646
|
+
// which executes the reconcile epilogue phase.
|
|
1647
|
+
await this.launch(record.slug, launched)
|
|
1648
|
+
report.conflicted += 1
|
|
1649
|
+
return
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
// Enabling and the ungated fallback begin from the same durable intent
|
|
1653
|
+
// predicate. Only `applied` acknowledges native state. A direct merge is
|
|
1654
|
+
// owned solely by this janitor and only after the engine is parked at
|
|
1655
|
+
// awaiting-pr (all verify/finalize work complete).
|
|
1656
|
+
if (prState.state === 'open' && autoMerge?.enabled === true) {
|
|
1657
|
+
const result = await forge.setAutoMerge(workspacePath, pr.number, true)
|
|
1658
|
+
if (result.kind === 'applied') {
|
|
1659
|
+
await store.append(record.slug, {
|
|
1660
|
+
actor: DISPATCHER,
|
|
1661
|
+
type: autoMergeApplicationType(true),
|
|
1662
|
+
payload: { commandSeq: autoMerge.commandSeq },
|
|
1663
|
+
})
|
|
1664
|
+
} else if (result.kind === 'deferred' && result.reason !== undefined) {
|
|
1665
|
+
await recordAutoMergeDeferralObservation(
|
|
1666
|
+
store,
|
|
1667
|
+
record.slug,
|
|
1668
|
+
result.reason,
|
|
1669
|
+
pr.number,
|
|
1670
|
+
autoMerge.commandSeq,
|
|
1671
|
+
this.deps.ids('obs'),
|
|
1672
|
+
)
|
|
1673
|
+
} else if (result.kind === 'ungated' && prState.mergeable === true) {
|
|
1674
|
+
// Re-read at the last possible point. A cancellation, replacement
|
|
1675
|
+
// command, newly due pipeline work, or application fact suppresses
|
|
1676
|
+
// this attempt; the next tick reclassifies from fresh forge state.
|
|
1677
|
+
const latestEvents = await store.getEvents(record.slug)
|
|
1678
|
+
const latestState = reduceBuild(latestEvents)
|
|
1679
|
+
const latestIntent = pendingAutoMerge(latestState)
|
|
1680
|
+
const decision = decideNext(latestEvents, this.deps.config)
|
|
1681
|
+
if (
|
|
1682
|
+
latestState.pr?.number === pr.number &&
|
|
1683
|
+
latestIntent?.enabled === true &&
|
|
1684
|
+
latestIntent.commandSeq === autoMerge.commandSeq &&
|
|
1685
|
+
decision.kind === 'wait' &&
|
|
1686
|
+
decision.reason === 'awaiting-pr'
|
|
1687
|
+
) {
|
|
1688
|
+
await forge.squashMerge(workspacePath, pr.number, result.headSha)
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
switch (prState.state) {
|
|
1694
|
+
case 'merged': {
|
|
1695
|
+
if (executionLeaseLive) return
|
|
1696
|
+
// Emit the fact once (a crash between steps re-runs this block; the
|
|
1697
|
+
// reduced prState dedupes the event, the log dedupes the release).
|
|
1698
|
+
if (state.prState !== 'merged') {
|
|
1699
|
+
await store.append(record.slug, {
|
|
1700
|
+
actor: DISPATCHER,
|
|
1701
|
+
type: 'pr.merged',
|
|
1702
|
+
payload: { sha: prState.sha },
|
|
1703
|
+
} satisfies EventWrite<'pr.merged'>)
|
|
1704
|
+
}
|
|
1705
|
+
await this.releaseWorkspace(record.slug, events)
|
|
1706
|
+
if (record.ticket) {
|
|
1707
|
+
await tickets.transition(record.ticket.id, this.doneState)
|
|
1708
|
+
await tickets.comment(record.ticket.id, `build ${record.slug} merged: ${pr.url}`)
|
|
1709
|
+
}
|
|
1710
|
+
await store.append(record.slug, {
|
|
1711
|
+
actor: DISPATCHER,
|
|
1712
|
+
type: 'build.completed',
|
|
1713
|
+
payload: { outcome: 'merged' },
|
|
1714
|
+
} satisfies EventWrite<'build.completed'>)
|
|
1715
|
+
await this.reclaimPrAttachments(record.slug, events)
|
|
1716
|
+
report.merged += 1
|
|
1717
|
+
return
|
|
1718
|
+
}
|
|
1719
|
+
case 'closed': {
|
|
1720
|
+
if (executionLeaseLive) return
|
|
1721
|
+
// Closed without merge is a human decision — back to Triage (§15.7).
|
|
1722
|
+
if (state.prState !== 'closed') {
|
|
1723
|
+
await store.append(record.slug, {
|
|
1724
|
+
actor: DISPATCHER,
|
|
1725
|
+
type: 'pr.closed',
|
|
1726
|
+
payload: {},
|
|
1727
|
+
} satisfies EventWrite<'pr.closed'>)
|
|
1728
|
+
}
|
|
1729
|
+
await this.releaseWorkspace(record.slug, events)
|
|
1730
|
+
if (record.ticket) {
|
|
1731
|
+
const triageState = this.triageState()
|
|
1732
|
+
await tickets.transition(record.ticket.id, triageState)
|
|
1733
|
+
await tickets.comment(
|
|
1734
|
+
record.ticket.id,
|
|
1735
|
+
`build ${record.slug} PR closed without merging (${pr.url}) — returned to ${triageState}`,
|
|
1736
|
+
)
|
|
1737
|
+
}
|
|
1738
|
+
await store.append(record.slug, {
|
|
1739
|
+
actor: DISPATCHER,
|
|
1740
|
+
type: 'build.completed',
|
|
1741
|
+
payload: { outcome: 'closed-unmerged' },
|
|
1742
|
+
} satisfies EventWrite<'build.completed'>)
|
|
1743
|
+
await this.reclaimPrAttachments(record.slug, events)
|
|
1744
|
+
report.closed += 1
|
|
1745
|
+
return
|
|
1746
|
+
}
|
|
1747
|
+
case 'open':
|
|
1748
|
+
// Unknown mergeability and transient auto-merge classifications are
|
|
1749
|
+
// retried on a later poll; never guess.
|
|
1750
|
+
return
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
/** Reclaim review-window release copies after terminal completion. Cleanup
|
|
1755
|
+
* is deliberately post-terminal and best-effort: a provider, timeout, or
|
|
1756
|
+
* store failure cannot roll back build.completed/ticket/workspace work.
|
|
1757
|
+
* Pending handles remain derivable and are retried on every later tick. */
|
|
1758
|
+
private async reclaimPrAttachments(slug: string, events: AbEvent[]): Promise<void> {
|
|
1759
|
+
for (const hosted of pendingPrAttachmentReclaims(events)) {
|
|
1760
|
+
const priorAttempts = events.filter(
|
|
1761
|
+
(event) =>
|
|
1762
|
+
event.type === 'pr-attachment.reclaim-failed' && event.payload.hostedSeq === hosted.seq,
|
|
1763
|
+
).length
|
|
1764
|
+
try {
|
|
1765
|
+
const capability = this.deps.forge.prAttachments
|
|
1766
|
+
if (capability === undefined) {
|
|
1767
|
+
throw new Error(
|
|
1768
|
+
`forge ${this.deps.forge.name} does not support PR attachment reclamation`,
|
|
1769
|
+
)
|
|
1770
|
+
}
|
|
1771
|
+
await capability.reclaim({
|
|
1772
|
+
// Physical checkout for cwd-style reclaimers (local git), not the
|
|
1773
|
+
// store identity; API-based reclaimers ignore it.
|
|
1774
|
+
workspacePath: this.deps.checkout ?? this.deps.repo,
|
|
1775
|
+
asset: hosted.payload.asset,
|
|
1776
|
+
})
|
|
1777
|
+
await this.deps.store.append(slug, {
|
|
1778
|
+
actor: DISPATCHER,
|
|
1779
|
+
type: 'pr-attachment.reclaimed',
|
|
1780
|
+
payload: { hostedSeq: hosted.seq },
|
|
1781
|
+
})
|
|
1782
|
+
} catch (error) {
|
|
1783
|
+
try {
|
|
1784
|
+
await this.deps.store.append(slug, {
|
|
1785
|
+
actor: DISPATCHER,
|
|
1786
|
+
type: 'pr-attachment.reclaim-failed',
|
|
1787
|
+
payload: {
|
|
1788
|
+
hostedSeq: hosted.seq,
|
|
1789
|
+
attempt: priorAttempts + 1,
|
|
1790
|
+
error:
|
|
1791
|
+
(error instanceof Error ? error.message : String(error)).trim() ||
|
|
1792
|
+
'PR attachment reclamation failed without an error message',
|
|
1793
|
+
},
|
|
1794
|
+
})
|
|
1795
|
+
} catch {
|
|
1796
|
+
// The hosted fact remains pending; a later tick retries both the
|
|
1797
|
+
// idempotent delete and its durable acknowledgement.
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
private recoveryCheckpoint(events: AbEvent[]): string | undefined {
|
|
1804
|
+
let settled: string | undefined
|
|
1805
|
+
let original: string | undefined
|
|
1806
|
+
for (const event of events) {
|
|
1807
|
+
if (event.type === 'workspace.provisioned' && original === undefined) {
|
|
1808
|
+
original = event.payload.base.sha
|
|
1809
|
+
} else if (event.type === 'implement.completed') {
|
|
1810
|
+
settled = event.payload.commits.head
|
|
1811
|
+
} else if (event.type === 'reconcile.completed') {
|
|
1812
|
+
settled = event.payload.mergeCommit
|
|
1813
|
+
} else if (
|
|
1814
|
+
event.type === 'finalize.step-completed' &&
|
|
1815
|
+
event.payload.ok &&
|
|
1816
|
+
event.payload.headSha !== undefined
|
|
1817
|
+
) {
|
|
1818
|
+
settled = event.payload.headSha
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
return settled ?? original
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
private async recordInfrastructureFailure(
|
|
1825
|
+
slug: string,
|
|
1826
|
+
events: AbEvent[],
|
|
1827
|
+
input: {
|
|
1828
|
+
provider: string
|
|
1829
|
+
workspaceRef: string
|
|
1830
|
+
operation: 'provision' | 'start' | 'wait' | 'stop' | 'delete' | 'reconcile'
|
|
1831
|
+
error: unknown
|
|
1832
|
+
cleanupPending: boolean
|
|
1833
|
+
},
|
|
1834
|
+
): Promise<void> {
|
|
1835
|
+
const execution = [...events].reverse().find((event) => event.type === 'execution.started')
|
|
1836
|
+
const appended = await appendInfrastructureFailure(
|
|
1837
|
+
{ store: this.deps.store, ids: this.deps.ids },
|
|
1838
|
+
{
|
|
1839
|
+
slug,
|
|
1840
|
+
events,
|
|
1841
|
+
maxAttempts: this.deps.config.policy.maxInfrastructureAttempts,
|
|
1842
|
+
provider: input.provider,
|
|
1843
|
+
workspaceRef: input.workspaceRef,
|
|
1844
|
+
instance:
|
|
1845
|
+
execution?.type === 'execution.started'
|
|
1846
|
+
? execution.payload.instance
|
|
1847
|
+
: 'dispatcher-recovery',
|
|
1848
|
+
...(execution?.type === 'execution.started' && execution.payload.environmentId !== undefined
|
|
1849
|
+
? { environmentId: execution.payload.environmentId }
|
|
1850
|
+
: {}),
|
|
1851
|
+
...(execution?.type === 'execution.started' && execution.payload.sessionId !== undefined
|
|
1852
|
+
? { sessionId: execution.payload.sessionId }
|
|
1853
|
+
: {}),
|
|
1854
|
+
operation: input.operation,
|
|
1855
|
+
error: input.error,
|
|
1856
|
+
cleanupPending: input.cleanupPending,
|
|
1857
|
+
},
|
|
1858
|
+
)
|
|
1859
|
+
events.push(...appended)
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
/** The provider that owns a recorded workspace: the one named on its
|
|
1863
|
+
* provisioned fact. A handle from any other provider (a build that outlived
|
|
1864
|
+
* a `[workspace].provider` change) is never handed to the current provider,
|
|
1865
|
+
* whose reap or release would misread the ref. */
|
|
1866
|
+
private workspaceOwner(open: { provider: string }): WorkspaceProvider | null {
|
|
1867
|
+
if (open.provider === this.deps.workspaces.name) return this.deps.workspaces
|
|
1868
|
+
return this.deps.retiredWorkspaces?.find((provider) => provider.name === open.provider) ?? null
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
/** Fence and remove a stale remote environment. A replacement is created
|
|
1872
|
+
* only after exact-name cleanup is durably acknowledged. */
|
|
1873
|
+
private async reapStaleWorkspace(
|
|
1874
|
+
slug: string,
|
|
1875
|
+
events: AbEvent[],
|
|
1876
|
+
reason: 'pause' | 'blocked' | 'replacement',
|
|
1877
|
+
): Promise<boolean> {
|
|
1878
|
+
const open = openWorkspace(events)
|
|
1879
|
+
if (open === null) return false
|
|
1880
|
+
const recovery = this.workspaceOwner(open)?.recovery
|
|
1881
|
+
if (recovery === undefined) return false
|
|
1882
|
+
// Last-chance settlement boundary (AUT-328): record the pending
|
|
1883
|
+
// publication's fate BEFORE the destructive reap. The guard's append
|
|
1884
|
+
// failure propagates — a stale workspace with an unrecorded pending
|
|
1885
|
+
// request is never reaped (the next tick retries).
|
|
1886
|
+
events = await settlePublicationBeforeRelease(
|
|
1887
|
+
{ store: this.deps.store, settlePublication: this.deps.settlePublication },
|
|
1888
|
+
slug,
|
|
1889
|
+
events,
|
|
1890
|
+
reason,
|
|
1891
|
+
)
|
|
1892
|
+
const attempt =
|
|
1893
|
+
events.filter(
|
|
1894
|
+
(event) =>
|
|
1895
|
+
event.type === 'infrastructure.cleanup-attempted' &&
|
|
1896
|
+
event.payload.workspaceRef === open.ref,
|
|
1897
|
+
).length + 1
|
|
1898
|
+
try {
|
|
1899
|
+
const outcome = await recovery.reap({
|
|
1900
|
+
provider: open.provider,
|
|
1901
|
+
ref: open.ref,
|
|
1902
|
+
path: open.path ?? open.ref,
|
|
1903
|
+
...(open.localPath !== undefined ? { localPath: open.localPath } : {}),
|
|
1904
|
+
branch: open.branch,
|
|
1905
|
+
})
|
|
1906
|
+
const cleaned = await this.deps.store.append(slug, {
|
|
1907
|
+
actor: DISPATCHER,
|
|
1908
|
+
type: 'infrastructure.cleanup-attempted',
|
|
1909
|
+
payload: {
|
|
1910
|
+
provider: open.provider,
|
|
1911
|
+
workspaceRef: open.ref,
|
|
1912
|
+
operation: 'reconcile',
|
|
1913
|
+
attempt,
|
|
1914
|
+
outcome: outcome.outcome,
|
|
1915
|
+
snapshots: outcome.snapshots,
|
|
1916
|
+
},
|
|
1917
|
+
})
|
|
1918
|
+
events.push(cleaned)
|
|
1919
|
+
const released = await this.deps.store.append(slug, {
|
|
1920
|
+
actor: DISPATCHER,
|
|
1921
|
+
type: 'workspace.released',
|
|
1922
|
+
payload: { ref: open.ref, reason },
|
|
1923
|
+
})
|
|
1924
|
+
events.push(released)
|
|
1925
|
+
return true
|
|
1926
|
+
} catch (error) {
|
|
1927
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
1928
|
+
const failed = await this.deps.store.append(slug, {
|
|
1929
|
+
actor: DISPATCHER,
|
|
1930
|
+
type: 'infrastructure.cleanup-attempted',
|
|
1931
|
+
payload: {
|
|
1932
|
+
provider: open.provider,
|
|
1933
|
+
workspaceRef: open.ref,
|
|
1934
|
+
operation: 'reconcile',
|
|
1935
|
+
attempt,
|
|
1936
|
+
outcome: 'unknown',
|
|
1937
|
+
error: message || 'cleanup failed without an error message',
|
|
1938
|
+
},
|
|
1939
|
+
})
|
|
1940
|
+
events.push(failed)
|
|
1941
|
+
await this.recordInfrastructureFailure(slug, events, {
|
|
1942
|
+
provider: open.provider,
|
|
1943
|
+
workspaceRef: open.ref,
|
|
1944
|
+
operation: 'reconcile',
|
|
1945
|
+
error,
|
|
1946
|
+
cleanupPending: true,
|
|
1947
|
+
})
|
|
1948
|
+
return false
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
/** Release the build's workspace if the log shows one still provisioned;
|
|
1953
|
+
* append `workspace.released`. Log-deduped, so re-runs are no-ops. */
|
|
1954
|
+
private async releaseWorkspace(
|
|
1955
|
+
slug: string,
|
|
1956
|
+
events: AbEvent[],
|
|
1957
|
+
reason: 'completion' | 'abort' | 'pause' | 'blocked' | 'replacement' | 'discard' = 'completion',
|
|
1958
|
+
): Promise<AbEvent | undefined> {
|
|
1959
|
+
const open = openWorkspace(events)
|
|
1960
|
+
if (!open) return undefined
|
|
1961
|
+
const owner = this.workspaceOwner(open)
|
|
1962
|
+
// No live provider can act on this ref; leave the fact open rather than
|
|
1963
|
+
// claim a release that never happened.
|
|
1964
|
+
if (owner === null) return undefined
|
|
1965
|
+
const handle = {
|
|
1966
|
+
provider: open.provider,
|
|
1967
|
+
ref: open.ref,
|
|
1968
|
+
path: open.path ?? open.ref,
|
|
1969
|
+
branch: open.branch,
|
|
1970
|
+
}
|
|
1971
|
+
// Last-chance settlement boundary (AUT-328): settle or durably record the
|
|
1972
|
+
// pending publication BEFORE any provider cleanup call, while the
|
|
1973
|
+
// workspace still exists. The guard's append failure propagates — the
|
|
1974
|
+
// release never proceeds past an unrecorded pending request.
|
|
1975
|
+
events = await settlePublicationBeforeRelease(
|
|
1976
|
+
{ store: this.deps.store, settlePublication: this.deps.settlePublication },
|
|
1977
|
+
slug,
|
|
1978
|
+
events,
|
|
1979
|
+
reason,
|
|
1980
|
+
)
|
|
1981
|
+
const recovery = owner.recovery
|
|
1982
|
+
if (recovery !== undefined) {
|
|
1983
|
+
const attempt =
|
|
1984
|
+
events.filter(
|
|
1985
|
+
(event) =>
|
|
1986
|
+
event.type === 'infrastructure.cleanup-attempted' &&
|
|
1987
|
+
event.payload.workspaceRef === open.ref,
|
|
1988
|
+
).length + 1
|
|
1989
|
+
try {
|
|
1990
|
+
const outcome = await recovery.reap(handle)
|
|
1991
|
+
const cleaned = await this.deps.store.append(slug, {
|
|
1992
|
+
actor: DISPATCHER,
|
|
1993
|
+
type: 'infrastructure.cleanup-attempted',
|
|
1994
|
+
payload: {
|
|
1995
|
+
provider: open.provider,
|
|
1996
|
+
workspaceRef: open.ref,
|
|
1997
|
+
operation: 'delete',
|
|
1998
|
+
attempt,
|
|
1999
|
+
outcome: outcome.outcome,
|
|
2000
|
+
snapshots: outcome.snapshots,
|
|
2001
|
+
},
|
|
2002
|
+
})
|
|
2003
|
+
events.push(cleaned)
|
|
2004
|
+
} catch (error) {
|
|
2005
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
2006
|
+
const failed = await this.deps.store.append(slug, {
|
|
2007
|
+
actor: DISPATCHER,
|
|
2008
|
+
type: 'infrastructure.cleanup-attempted',
|
|
2009
|
+
payload: {
|
|
2010
|
+
provider: open.provider,
|
|
2011
|
+
workspaceRef: open.ref,
|
|
2012
|
+
operation: 'delete',
|
|
2013
|
+
attempt,
|
|
2014
|
+
outcome: 'unknown',
|
|
2015
|
+
error: message || 'cleanup failed without an error message',
|
|
2016
|
+
},
|
|
2017
|
+
})
|
|
2018
|
+
events.push(failed)
|
|
2019
|
+
await this.recordInfrastructureFailure(slug, events, {
|
|
2020
|
+
provider: open.provider,
|
|
2021
|
+
workspaceRef: open.ref,
|
|
2022
|
+
operation: 'delete',
|
|
2023
|
+
error,
|
|
2024
|
+
cleanupPending: true,
|
|
2025
|
+
})
|
|
2026
|
+
throw error
|
|
2027
|
+
}
|
|
2028
|
+
} else {
|
|
2029
|
+
await owner.release(handle)
|
|
2030
|
+
}
|
|
2031
|
+
return this.deps.store.append(slug, {
|
|
2032
|
+
actor: DISPATCHER,
|
|
2033
|
+
type: 'workspace.released',
|
|
2034
|
+
payload: { ref: open.ref, reason },
|
|
2035
|
+
} satisfies EventWrite<'workspace.released'>)
|
|
2036
|
+
}
|
|
2037
|
+
|
|
2038
|
+
/** Current tip of the base branch. A forge with the checkout-less
|
|
2039
|
+
* `remoteBranchSha` capability answers from the GitHub API (origin mode);
|
|
2040
|
+
* otherwise `git ls-remote <path> refs/heads/<b>` against the dispatcher's
|
|
2041
|
+
* physical checkout (§15.7: no network). The checkout, not the store
|
|
2042
|
+
* identity — `deps.repo` is the normalized origin since the identity
|
|
2043
|
+
* change, and ls-remote against it would hit the network from a URL.
|
|
2044
|
+
* Origin-less fixtures whose identity is a path keep working through the
|
|
2045
|
+
* fallback. */
|
|
2046
|
+
private async baseSha(baseBranch: string): Promise<string> {
|
|
2047
|
+
const remoteBranchSha = this.deps.forge.remoteBranchSha
|
|
2048
|
+
if (remoteBranchSha !== undefined) {
|
|
2049
|
+
return await remoteBranchSha.call(this.deps.forge, baseBranch)
|
|
2050
|
+
}
|
|
2051
|
+
const args = [
|
|
2052
|
+
'git',
|
|
2053
|
+
'ls-remote',
|
|
2054
|
+
this.deps.checkout ?? this.deps.repo,
|
|
2055
|
+
`refs/heads/${baseBranch}`,
|
|
2056
|
+
]
|
|
2057
|
+
const result = await this.deps.exec(args, {})
|
|
2058
|
+
const sha = result.stdout.trim().split(/\s+/)[0]
|
|
2059
|
+
if (result.exitCode !== 0 || !sha) {
|
|
2060
|
+
throw new Error(
|
|
2061
|
+
`${args.join(' ')} exited ${result.exitCode}: ${
|
|
2062
|
+
result.stderr.trim() || result.stdout.trim() || '(no output)'
|
|
2063
|
+
}`,
|
|
2064
|
+
)
|
|
2065
|
+
}
|
|
2066
|
+
return sha
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
// ── b. Interrupted dispatch recovery ───────────────────────────────────────
|
|
2070
|
+
|
|
2071
|
+
/** Every boundary before runner attachment is derived from durable facts.
|
|
2072
|
+
* Missing facts are the todo list; successful provider calls are followed
|
|
2073
|
+
* immediately by their facts, so an ordinary later tick can continue. */
|
|
2074
|
+
// ── b. Interrupted dispatch recovery ─────────────────────────────────────────
|
|
2075
|
+
|
|
2076
|
+
/** Every boundary before runner attachment is derived from durable facts.
|
|
2077
|
+
* Missing facts are the todo list; successful provider calls are followed
|
|
2078
|
+
* immediately by their facts, so an ordinary later tick can continue.
|
|
2079
|
+
*
|
|
2080
|
+
* Remote (recovery-capable) providers never provision inline: the workspace
|
|
2081
|
+
* stage kicks a tracked background continuation and returns `deferred`, so a
|
|
2082
|
+
* tick never blocks on guest bootstrap. The continuation provisions, then
|
|
2083
|
+
* completes the remaining boundaries and the launch. Local providers keep
|
|
2084
|
+
* the original inline provisioning byte-for-byte. */
|
|
2085
|
+
private async attemptDispatchCompletion(
|
|
2086
|
+
record: BuildRecord,
|
|
2087
|
+
launched: Set<string>,
|
|
2088
|
+
report: TickReport,
|
|
2089
|
+
seed?: {
|
|
2090
|
+
ticket: Ticket
|
|
2091
|
+
body: string
|
|
2092
|
+
authoredSession?: string
|
|
2093
|
+
autoMergeUser?: string
|
|
2094
|
+
},
|
|
2095
|
+
): Promise<DispatchOutcome> {
|
|
2096
|
+
const { store, config } = this.deps
|
|
2097
|
+
let stage: EventPayload<'dispatch.failed'>['stage'] = 'create'
|
|
2098
|
+
try {
|
|
2099
|
+
let events = await store.getEvents(record.slug)
|
|
2100
|
+
if (!events.some((event) => event.type === 'build.created')) {
|
|
2101
|
+
if (events.length !== 0 || record.ticket === undefined) {
|
|
2102
|
+
throw new Error(
|
|
2103
|
+
'build record lacks the immutable ticket facts needed to reconstruct build.created',
|
|
2104
|
+
)
|
|
2105
|
+
}
|
|
2106
|
+
await store.appendIfCurrent(record.slug, 0, {
|
|
2107
|
+
actor: DISPATCHER,
|
|
2108
|
+
type: 'build.created',
|
|
2109
|
+
payload: {
|
|
2110
|
+
ticket: record.ticket,
|
|
2111
|
+
repo: record.repo,
|
|
2112
|
+
baseBranch: config.baseBranch,
|
|
2113
|
+
...(seed?.autoMergeUser !== undefined
|
|
2114
|
+
? { autoMergeRequestedBy: seed.autoMergeUser }
|
|
2115
|
+
: {}),
|
|
2116
|
+
...(config.pr !== undefined ? { pr: config.pr } : {}),
|
|
2117
|
+
},
|
|
2118
|
+
})
|
|
2119
|
+
events = await store.getEvents(record.slug)
|
|
2120
|
+
if (!events.some((event) => event.type === 'build.created')) {
|
|
2121
|
+
throw new Error('build.created reconstruction lost its conditional append race')
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
const created = events.find((event) => event.type === 'build.created')
|
|
2126
|
+
const autoMergeUser = seed?.autoMergeUser ?? created?.payload.autoMergeRequestedBy
|
|
2127
|
+
if (
|
|
2128
|
+
autoMergeUser !== undefined &&
|
|
2129
|
+
!events.some((event) => event.type === 'build.auto-merge-requested')
|
|
2130
|
+
) {
|
|
2131
|
+
await store.append(record.slug, {
|
|
2132
|
+
actor: humanActor(autoMergeUser),
|
|
2133
|
+
type: 'build.auto-merge-requested',
|
|
2134
|
+
payload: {},
|
|
2135
|
+
})
|
|
2136
|
+
events = await store.getEvents(record.slug)
|
|
2137
|
+
}
|
|
2138
|
+
|
|
2139
|
+
stage = 'workspace'
|
|
2140
|
+
if (openWorkspace(events) === null) {
|
|
2141
|
+
if (this.deps.workspaces.recovery !== undefined) {
|
|
2142
|
+
// Recovery-capable provider: durable marker + tracked background
|
|
2143
|
+
// continuation. The tick makes durable progress and returns; the
|
|
2144
|
+
// continuation provisions, then completes the remaining boundaries
|
|
2145
|
+
// and the launch.
|
|
2146
|
+
const kicked = await this.ensureProvisionContinuation(record, events, 'dispatch', seed)
|
|
2147
|
+
if (kicked === 'kicked') report.provisioning += 1
|
|
2148
|
+
launched.add(record.slug)
|
|
2149
|
+
return 'deferred'
|
|
2150
|
+
}
|
|
2151
|
+
const priorGenerations = events.filter(
|
|
2152
|
+
(event) => event.type === 'workspace.provisioned',
|
|
2153
|
+
).length
|
|
2154
|
+
const handle = await this.deps.workspaces.provision({
|
|
2155
|
+
// Local providers provision FROM the physical checkout; remote
|
|
2156
|
+
// providers ignore the value (their host seams are injected).
|
|
2157
|
+
repo: this.deps.checkout ?? this.deps.repo,
|
|
2158
|
+
baseBranch: baseBranchOf(events, config),
|
|
2159
|
+
branch: record.branch ?? `ab/${record.slug}`,
|
|
2160
|
+
...(priorGenerations > 0
|
|
2161
|
+
? {
|
|
2162
|
+
revision: this.recoveryCheckpoint(events),
|
|
2163
|
+
generation: priorGenerations,
|
|
2164
|
+
}
|
|
2165
|
+
: {}),
|
|
2166
|
+
})
|
|
2167
|
+
await store.append(record.slug, {
|
|
2168
|
+
actor: DISPATCHER,
|
|
2169
|
+
type: 'workspace.provisioned',
|
|
2170
|
+
payload: {
|
|
2171
|
+
provider: handle.provider,
|
|
2172
|
+
ref: handle.ref,
|
|
2173
|
+
path: handle.path,
|
|
2174
|
+
...(handle.localPath !== undefined ? { localPath: handle.localPath } : {}),
|
|
2175
|
+
branch: handle.branch,
|
|
2176
|
+
base: handle.base,
|
|
2177
|
+
},
|
|
2178
|
+
} satisfies EventWrite<'workspace.provisioned'>)
|
|
2179
|
+
events = await store.getEvents(record.slug)
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
return await this.completeDispatchBoundaries(record, seed, report, launched)
|
|
2183
|
+
} catch (error) {
|
|
2184
|
+
const events = await store.getEvents(record.slug)
|
|
2185
|
+
if (stage === 'workspace' && this.deps.workspaces.recovery !== undefined) {
|
|
2186
|
+
const open = openWorkspace(events)
|
|
2187
|
+
await this.recordInfrastructureFailure(record.slug, events, {
|
|
2188
|
+
provider: this.deps.workspaces.name,
|
|
2189
|
+
workspaceRef: open?.ref ?? record.slug,
|
|
2190
|
+
operation: 'provision',
|
|
2191
|
+
error,
|
|
2192
|
+
cleanupPending: true,
|
|
2193
|
+
})
|
|
2194
|
+
}
|
|
2195
|
+
const attempt = events.filter((event) => event.type === 'dispatch.failed').length + 1
|
|
2196
|
+
const message = (error instanceof Error ? error.message : String(error)).trim()
|
|
2197
|
+
await store.append(record.slug, {
|
|
2198
|
+
actor: DISPATCHER,
|
|
2199
|
+
type: 'dispatch.failed',
|
|
2200
|
+
payload: {
|
|
2201
|
+
stage,
|
|
2202
|
+
attempt,
|
|
2203
|
+
error: message || 'dispatch failed without an error message',
|
|
2204
|
+
},
|
|
2205
|
+
})
|
|
2206
|
+
report.dispatchFailed += 1
|
|
2207
|
+
return 'failed'
|
|
2208
|
+
}
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
/** The post-workspace dispatch boundaries — spec, comment, launch — shared
|
|
2212
|
+
* by the synchronous local path and the background continuation. Stage
|
|
2213
|
+
* failures record their own durable `dispatch.failed` fact. */
|
|
2214
|
+
private async completeDispatchBoundaries(
|
|
2215
|
+
record: BuildRecord,
|
|
2216
|
+
seed?: {
|
|
2217
|
+
ticket: Ticket
|
|
2218
|
+
body: string
|
|
2219
|
+
authoredSession?: string
|
|
2220
|
+
autoMergeUser?: string
|
|
2221
|
+
},
|
|
2222
|
+
report?: TickReport,
|
|
2223
|
+
launched?: Set<string>,
|
|
2224
|
+
): Promise<Exclude<DispatchOutcome, 'deferred'>> {
|
|
2225
|
+
const { store } = this.deps
|
|
2226
|
+
let stage: EventPayload<'dispatch.failed'>['stage'] = 'spec'
|
|
2227
|
+
let authored = false
|
|
2228
|
+
try {
|
|
2229
|
+
const events = await store.getEvents(record.slug)
|
|
2230
|
+
if (
|
|
2231
|
+
!events.some((event) => event.type === 'spec.imported' || event.type === 'spec.authored')
|
|
2232
|
+
) {
|
|
2233
|
+
let ticket: Ticket
|
|
2234
|
+
if (seed?.ticket !== undefined) {
|
|
2235
|
+
ticket = seed.ticket
|
|
2236
|
+
} else {
|
|
2237
|
+
const ticketId = record.ticket?.id
|
|
2238
|
+
if (ticketId === undefined) {
|
|
2239
|
+
throw new Error('build record has no ticket to recover its spec from')
|
|
2240
|
+
}
|
|
2241
|
+
const recoveredTicket = await this.deps.tickets.get(ticketId)
|
|
2242
|
+
if (recoveredTicket === null) {
|
|
2243
|
+
throw new Error(`claimed ticket ${ticketId} is no longer readable`)
|
|
2244
|
+
}
|
|
2245
|
+
ticket = recoveredTicket
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
let body = seed?.body ?? ticket.body
|
|
2249
|
+
let conformance = specConformance(body)
|
|
2250
|
+
let authoredSession = seed?.authoredSession
|
|
2251
|
+
if (!conformance.conforms && this.deps.authorSpec !== undefined) {
|
|
2252
|
+
const candidate = await this.deps.authorSpec(ticket)
|
|
2253
|
+
if (candidate !== null) {
|
|
2254
|
+
conformance = specConformance(candidate)
|
|
2255
|
+
if (conformance.conforms) {
|
|
2256
|
+
body = candidate
|
|
2257
|
+
authoredSession = this.deps.ids('s')
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
if (!conformance.conforms) {
|
|
2262
|
+
throw new Error(
|
|
2263
|
+
`claimed ticket no longer conforms to the spec standard: ${conformance.missing.join('; ')}`,
|
|
2264
|
+
)
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
const spec = {
|
|
2268
|
+
kind: 'spec',
|
|
2269
|
+
content: body,
|
|
2270
|
+
metadata: { ticket: ticket.ref.id, source: ticket.ref.source },
|
|
2271
|
+
}
|
|
2272
|
+
if (authoredSession !== undefined) {
|
|
2273
|
+
const session = authoredSession
|
|
2274
|
+
await store.appendWithArtifacts(record.slug, [spec], (deposited) => ({
|
|
2275
|
+
actor: agentActor('spec', session),
|
|
2276
|
+
type: 'spec.authored' as const,
|
|
2277
|
+
payload: { artifact: artifactRefOf(deposited), session },
|
|
2278
|
+
}))
|
|
2279
|
+
authored = true
|
|
2280
|
+
} else {
|
|
2281
|
+
await store.appendWithArtifacts(record.slug, [spec], (deposited) => ({
|
|
2282
|
+
actor: DISPATCHER,
|
|
2283
|
+
type: 'spec.imported' as const,
|
|
2284
|
+
payload: { artifact: artifactRefOf(deposited), ticket: ticket!.ref },
|
|
2285
|
+
}))
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
|
|
2289
|
+
stage = 'comment'
|
|
2290
|
+
if (
|
|
2291
|
+
record.ticket !== undefined &&
|
|
2292
|
+
!events.some((event) => event.type === 'dispatch.comment-posted')
|
|
2293
|
+
) {
|
|
2294
|
+
await this.deps.tickets.comment(record.ticket.id, `build ${record.slug} dispatched`)
|
|
2295
|
+
await store.append(record.slug, {
|
|
2296
|
+
actor: DISPATCHER,
|
|
2297
|
+
type: 'dispatch.comment-posted',
|
|
2298
|
+
payload: {},
|
|
2299
|
+
})
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
stage = 'launch'
|
|
2303
|
+
// Discard validation and recovery selection happened before several
|
|
2304
|
+
// provider awaits. Re-read immediately before launch so a request made
|
|
2305
|
+
// during those boundaries parks cleanly for the next janitor pass.
|
|
2306
|
+
const launchState = reduceBuild(await store.getEvents(record.slug))
|
|
2307
|
+
if (launchState.status !== 'queued' || launchState.discardRequest !== undefined) {
|
|
2308
|
+
return 'parked'
|
|
2309
|
+
}
|
|
2310
|
+
if (launched === undefined) await this.deps.launchRunner(record.slug)
|
|
2311
|
+
else await this.launch(record.slug, launched)
|
|
2312
|
+
if (authored && report !== undefined) report.authored += 1
|
|
2313
|
+
return 'completed'
|
|
2314
|
+
} catch (error) {
|
|
2315
|
+
const events = await store.getEvents(record.slug)
|
|
2316
|
+
const attempt = events.filter((event) => event.type === 'dispatch.failed').length + 1
|
|
2317
|
+
const message = (error instanceof Error ? error.message : String(error)).trim()
|
|
2318
|
+
await store.append(record.slug, {
|
|
2319
|
+
actor: DISPATCHER,
|
|
2320
|
+
type: 'dispatch.failed',
|
|
2321
|
+
payload: {
|
|
2322
|
+
stage,
|
|
2323
|
+
attempt,
|
|
2324
|
+
error: message || 'dispatch failed without an error message',
|
|
2325
|
+
},
|
|
2326
|
+
})
|
|
2327
|
+
if (report !== undefined) report.dispatchFailed += 1
|
|
2328
|
+
return 'failed'
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
private async recoverDispatches(
|
|
2333
|
+
report: TickReport,
|
|
2334
|
+
launched: Set<string>,
|
|
2335
|
+
paused: boolean,
|
|
2336
|
+
opts: TickOpts = {},
|
|
2337
|
+
): Promise<void> {
|
|
2338
|
+
for (const record of await this.deps.store.listBuilds()) {
|
|
2339
|
+
if (
|
|
2340
|
+
record.repo !== this.deps.repo ||
|
|
2341
|
+
record.ticket === undefined ||
|
|
2342
|
+
launched.has(record.slug) ||
|
|
2343
|
+
this.continuations.has(record.slug)
|
|
2344
|
+
) {
|
|
2345
|
+
continue
|
|
2346
|
+
}
|
|
2347
|
+
// Budget gate: recovery issues store transport calls per build.
|
|
2348
|
+
if (this.outOfBudget(opts)) break
|
|
2349
|
+
const events = await this.deps.store.getEvents(record.slug)
|
|
2350
|
+
const state = reduceBuild(events)
|
|
2351
|
+
if (
|
|
2352
|
+
state.status !== 'queued' ||
|
|
2353
|
+
state.discardRequest !== undefined ||
|
|
2354
|
+
heldByRepositoryPause(state, paused)
|
|
2355
|
+
) {
|
|
2356
|
+
continue
|
|
2357
|
+
}
|
|
2358
|
+
// Queued itself means runner attachment is still missing. Always enter
|
|
2359
|
+
// the fact-driven helper: it skips every completed boundary and retries
|
|
2360
|
+
// only the final launch when setup is already durable.
|
|
2361
|
+
if ((await this.attemptDispatchCompletion(record, launched, report)) === 'completed') {
|
|
2362
|
+
report.recovered += 1
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2367
|
+
// ── c. Dispatch-command startup resume (SPEC §2.2, §15.6-C) ────────────────
|
|
2368
|
+
|
|
2369
|
+
/**
|
|
2370
|
+
* A fresh `ab dispatch` invocation attempts every current build for this
|
|
2371
|
+
* repo, even when its old lease has not expired yet. Lease claiming remains
|
|
2372
|
+
* the exclusivity gate: a genuinely live runner wins and the attempted
|
|
2373
|
+
* replacement harmlessly skips.
|
|
2374
|
+
*
|
|
2375
|
+
* `decideNext` keeps every human judgment gate intact while still selecting
|
|
2376
|
+
* pending operator-command acknowledgements before it returns a blocked
|
|
2377
|
+
* wait. Open escalations are reported for plain-mode observability but never
|
|
2378
|
+
* answered by process startup.
|
|
2379
|
+
*/
|
|
2380
|
+
private async resumeCurrent(
|
|
2381
|
+
report: TickReport,
|
|
2382
|
+
launched: Set<string>,
|
|
2383
|
+
paused: boolean,
|
|
2384
|
+
opts: TickOpts = {},
|
|
2385
|
+
): Promise<void> {
|
|
2386
|
+
const { store, config } = this.deps
|
|
2387
|
+
for (const record of await store.listBuilds()) {
|
|
2388
|
+
if (record.repo !== this.deps.repo || launched.has(record.slug)) continue
|
|
2389
|
+
// Budget gate: resume launches runners (transport-backed work).
|
|
2390
|
+
if (this.outOfBudget(opts)) break
|
|
2391
|
+
|
|
2392
|
+
const events = await store.getEvents(record.slug)
|
|
2393
|
+
const state = reduceBuild(events)
|
|
2394
|
+
if (
|
|
2395
|
+
state.status === 'done' ||
|
|
2396
|
+
state.status === 'aborted' ||
|
|
2397
|
+
(state.status === 'queued' && state.discardRequest !== undefined) ||
|
|
2398
|
+
heldByRepositoryPause(state, paused)
|
|
2399
|
+
) {
|
|
2400
|
+
continue
|
|
2401
|
+
}
|
|
2402
|
+
|
|
2403
|
+
const decision = decideNext(events, config)
|
|
2404
|
+
if (decision.kind === 'wait') {
|
|
2405
|
+
if (decision.reason === 'blocked' && state.openEscalations.length > 0) {
|
|
2406
|
+
report.blockedDiagnostics.push(
|
|
2407
|
+
`build ${record.slug} skipped: open escalations ${state.openEscalations
|
|
2408
|
+
.map((escalation) => escalation.id)
|
|
2409
|
+
.join(', ')}`,
|
|
2410
|
+
)
|
|
2411
|
+
}
|
|
2412
|
+
continue
|
|
2413
|
+
}
|
|
2414
|
+
// Disposable remote environments must pass through the stale-lease
|
|
2415
|
+
// fencing sweep below; startup must never attach directly to an old VM.
|
|
2416
|
+
if (this.deps.workspaces.recovery !== undefined) continue
|
|
2417
|
+
const result = await this.launch(record.slug, launched)
|
|
2418
|
+
if (result === 'scheduled') report.resumed += 1
|
|
2419
|
+
}
|
|
2420
|
+
}
|
|
2421
|
+
|
|
2422
|
+
// ── d. Lease sweep (SPEC §15.6-C) ──────────────────────────────────────────
|
|
2423
|
+
|
|
2424
|
+
/**
|
|
2425
|
+
* Predicate: lease expired or absent + the engine has runner work →
|
|
2426
|
+
* re-attach a runner. Actionability is `decideNext`'s, not a status
|
|
2427
|
+
* heuristic — the two must agree or work the engine would decide is never
|
|
2428
|
+
* executed (a runner is the only pendingCommands/decision consumer). Every
|
|
2429
|
+
* non-`wait` decision re-attaches; that covers phase work, but also:
|
|
2430
|
+
*
|
|
2431
|
+
* - pending operator commands on paused/blocked builds (D2, §15.2.7: "a
|
|
2432
|
+
* runner that is dead still receives its commands on resume" — the
|
|
2433
|
+
* request does not change status, only the kernel's acknowledgement
|
|
2434
|
+
* does, so a status gate would strand resume/abort forever);
|
|
2435
|
+
* - post-PR phase work while prState is 'open' (§15.7: the verify re-run
|
|
2436
|
+
* after `reconcile.completed`; §5: finalize post-steps still due after
|
|
2437
|
+
* `finalize.completed`) and the reconcile run while 'conflicted';
|
|
2438
|
+
* - engine-side escalation raises (stall/policy) a dead runner never
|
|
2439
|
+
* appended.
|
|
2440
|
+
*
|
|
2441
|
+
* Every `wait` reason is correctly parked: blocked/paused (human), awaiting
|
|
2442
|
+
* spec (human lands rev N+1; the next tick sees run-phase plan), awaiting
|
|
2443
|
+
* PR (janitor duty), done/aborted (janitor duty). The supervising launcher
|
|
2444
|
+
* reserves the execution lease and the child renews it; the reduced log tells it where to resume
|
|
2445
|
+
* (started-without-terminal work re-runs from the phase start).
|
|
2446
|
+
*/
|
|
2447
|
+
private async leaseSweep(
|
|
2448
|
+
report: TickReport,
|
|
2449
|
+
launched: Set<string>,
|
|
2450
|
+
paused: boolean,
|
|
2451
|
+
opts: TickOpts = {},
|
|
2452
|
+
): Promise<void> {
|
|
2453
|
+
const now = this.deps.clock().getTime()
|
|
2454
|
+
for (const record of await this.deps.store.listBuilds()) {
|
|
2455
|
+
if (record.repo !== this.deps.repo) continue // §12: not this dispatcher's build
|
|
2456
|
+
if (launched.has(record.slug)) continue
|
|
2457
|
+
// Budget gate: the sweep re-attaches runners per expired build.
|
|
2458
|
+
if (this.outOfBudget(opts)) break
|
|
2459
|
+
if (record.lease) {
|
|
2460
|
+
if (new Date(record.lease.expiresAt).getTime() > now) continue // healthy
|
|
2461
|
+
} else if (now - new Date(record.updatedAt).getTime() < this.leaseTtlMs) {
|
|
2462
|
+
continue // absent lease within the first-claim grace (see DispatcherOpts)
|
|
2463
|
+
}
|
|
2464
|
+
const events = await this.deps.store.getEvents(record.slug)
|
|
2465
|
+
const state = reduceBuild(events)
|
|
2466
|
+
if (
|
|
2467
|
+
(state.status === 'queued' && state.discardRequest !== undefined) ||
|
|
2468
|
+
heldByRepositoryPause(state, paused)
|
|
2469
|
+
) {
|
|
2470
|
+
continue
|
|
2471
|
+
}
|
|
2472
|
+
const decision = decideNext(events, this.deps.config)
|
|
2473
|
+
if (state.status === 'done' || state.status === 'aborted') continue
|
|
2474
|
+
// Liveness gate (durable supervision): a recorded execution the provider
|
|
2475
|
+
// observes as RUNNING is left completely alone — no settle, no reap, no
|
|
2476
|
+
// replacement, no re-attach. This is the change that stops a supervisor
|
|
2477
|
+
// restart from replacing a healthy environment generation. A provider-
|
|
2478
|
+
// proved end settles first (completion facts, lease release, publication)
|
|
2479
|
+
// and then falls through to the normal decision; a provider-proved loss
|
|
2480
|
+
// falls through to the existing stale path below.
|
|
2481
|
+
const foreignExecution = openExecution(events)
|
|
2482
|
+
// A provider- or log-proved end in the same generation re-attaches in
|
|
2483
|
+
// place: the guest finished its work, the workspace is still the
|
|
2484
|
+
// build's, and replacing the environment would discard a healthy
|
|
2485
|
+
// generation. Only an unproven stale lease or a proved loss reaps.
|
|
2486
|
+
const lastExecution = lastExecutionOutcome(events)
|
|
2487
|
+
let endedInPlace = lastExecution === 'completed'
|
|
2488
|
+
if (
|
|
2489
|
+
foreignExecution !== null &&
|
|
2490
|
+
foreignExecution.commandId !== undefined &&
|
|
2491
|
+
!launched.has(record.slug)
|
|
2492
|
+
) {
|
|
2493
|
+
const owner = this.executionOwner(foreignExecution.provider)
|
|
2494
|
+
const execution = owner?.buildExecution
|
|
2495
|
+
if (execution?.observe !== undefined) {
|
|
2496
|
+
const settlement = await settleExecution(
|
|
2497
|
+
{
|
|
2498
|
+
store: this.deps.store,
|
|
2499
|
+
execution,
|
|
2500
|
+
...(this.deps.settlePublication !== undefined
|
|
2501
|
+
? { settlePublication: this.deps.settlePublication }
|
|
2502
|
+
: {}),
|
|
2503
|
+
},
|
|
2504
|
+
record.slug,
|
|
2505
|
+
events,
|
|
2506
|
+
)
|
|
2507
|
+
if (settlement === 'running') continue
|
|
2508
|
+
if (settlement === 'settled') {
|
|
2509
|
+
report.settled += 1
|
|
2510
|
+
endedInPlace = true
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
}
|
|
2514
|
+
const publicationRecoveryDue =
|
|
2515
|
+
state.status !== 'paused' &&
|
|
2516
|
+
state.status !== 'blocked' &&
|
|
2517
|
+
(publicationPending(events) || abandonedPublicationPending(events))
|
|
2518
|
+
const open = openWorkspace(events)
|
|
2519
|
+
// A provider- or log-proved END in the same generation re-attaches in
|
|
2520
|
+
// place: the guest finished its work, the workspace is still the
|
|
2521
|
+
// build's, and replacing the environment would discard a healthy
|
|
2522
|
+
// generation. Only an unproven stale lease or a proved loss reaps.
|
|
2523
|
+
if (open !== null && this.workspaceOwner(open)?.recovery !== undefined && !endedInPlace) {
|
|
2524
|
+
const parked = decision.kind === 'wait' && !publicationRecoveryDue
|
|
2525
|
+
const reason =
|
|
2526
|
+
state.status === 'paused'
|
|
2527
|
+
? ('pause' as const)
|
|
2528
|
+
: state.status === 'blocked'
|
|
2529
|
+
? ('blocked' as const)
|
|
2530
|
+
: ('replacement' as const)
|
|
2531
|
+
const reaped = await this.reapStaleWorkspace(record.slug, events, reason)
|
|
2532
|
+
if (!reaped) continue
|
|
2533
|
+
if (!parked) {
|
|
2534
|
+
// Recovery-capable provider: reap inline (bounded), then kick the
|
|
2535
|
+
// same background continuation — it provisions and launches without
|
|
2536
|
+
// blocking this tick. Local providers never enter this branch.
|
|
2537
|
+
const kicked = await this.ensureProvisionContinuation(record, events, 'launch')
|
|
2538
|
+
if (kicked === 'kicked') {
|
|
2539
|
+
report.provisioning += 1
|
|
2540
|
+
report.swept += 1
|
|
2541
|
+
}
|
|
2542
|
+
continue
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
// Reachable via the reap block above: when the wait decision is parked
|
|
2546
|
+
// (no publication recovery due) the `if (!parked)` arm continues, but the
|
|
2547
|
+
// parked arm falls through here after a successful reap. Without this
|
|
2548
|
+
// guard the tick would reach the unconditional `launch` below and start a
|
|
2549
|
+
// fresh runner for a paused or blocked build whose workspace was just
|
|
2550
|
+
// reaped. Every other path that reaches this line with a wait decision
|
|
2551
|
+
// already continued earlier.
|
|
2552
|
+
if (decision.kind === 'wait' && !publicationRecoveryDue) continue
|
|
2553
|
+
if (
|
|
2554
|
+
openWorkspace(events) === null &&
|
|
2555
|
+
this.deps.workspaces.recovery !== undefined &&
|
|
2556
|
+
events.some((event) => event.type === 'workspace.provisioned')
|
|
2557
|
+
) {
|
|
2558
|
+
const kicked = await this.ensureProvisionContinuation(record, events, 'launch')
|
|
2559
|
+
if (kicked === 'kicked') {
|
|
2560
|
+
report.provisioning += 1
|
|
2561
|
+
report.swept += 1
|
|
2562
|
+
continue
|
|
2563
|
+
}
|
|
2564
|
+
// In-flight or foreign-live: whoever owns the marker completes the
|
|
2565
|
+
// boundaries and the launch; this tick performs neither.
|
|
2566
|
+
continue
|
|
2567
|
+
}
|
|
2568
|
+
const result = await this.launch(record.slug, launched)
|
|
2569
|
+
if (result === 'scheduled') report.swept += 1
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
|
|
2573
|
+
// ── e. Dispatch (SPEC §12, §6.3) ───────────────────────────────────────────
|
|
2574
|
+
|
|
2575
|
+
private async dispatch(
|
|
2576
|
+
report: TickReport,
|
|
2577
|
+
launched: Set<string>,
|
|
2578
|
+
autoMergeUser: string | undefined,
|
|
2579
|
+
paused: boolean,
|
|
2580
|
+
opts: TickOpts = {},
|
|
2581
|
+
): Promise<void> {
|
|
2582
|
+
const { store, tickets, config } = this.deps
|
|
2583
|
+
// Blocked and paused builds still occupy a slot: their workspaces and
|
|
2584
|
+
// pending work are live, only waiting on a human. Capacity is per repo
|
|
2585
|
+
// (§16.1) — another repo's builds never consume this repo's slots.
|
|
2586
|
+
let active = 0
|
|
2587
|
+
const activeTicketIds = new Set<string>()
|
|
2588
|
+
for (const record of await store.listBuilds()) {
|
|
2589
|
+
if (record.repo !== this.deps.repo) continue
|
|
2590
|
+
const state = reduceBuild(await store.getEvents(record.slug))
|
|
2591
|
+
if (state.status !== 'done' && state.status !== 'aborted') {
|
|
2592
|
+
active += 1
|
|
2593
|
+
if (record.ticket) activeTicketIds.add(record.ticket.id)
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
let capacity = config.capacity - active
|
|
2597
|
+
|
|
2598
|
+
// The ready scan runs even at full capacity: `queued` is the operator's
|
|
2599
|
+
// standing queue-depth report and must stay honest exactly when the
|
|
2600
|
+
// dispatcher is saturated. Sources that keep a claimed ticket in the
|
|
2601
|
+
// ready state are deduped against the active builds embodying them.
|
|
2602
|
+
const journalBeforeReady = await this.repositoryEvents()
|
|
2603
|
+
const listing = await tickets.listReady(readyCriteria(config))
|
|
2604
|
+
const journalAfterReady = await this.repositoryEvents()
|
|
2605
|
+
const creationHolds = new Map(
|
|
2606
|
+
harvestCreationsDuringReadyScan(journalBeforeReady, journalAfterReady).map((creation) => [
|
|
2607
|
+
creation.creationKey.toLowerCase(),
|
|
2608
|
+
creation,
|
|
2609
|
+
]),
|
|
2610
|
+
)
|
|
2611
|
+
const ready = listing.tickets.filter((ticket) => !activeTicketIds.has(ticket.ref.id))
|
|
2612
|
+
report.invalidTickets += listing.diagnostics.length
|
|
2613
|
+
report.ticketDiagnostics.push(...listing.diagnostics)
|
|
2614
|
+
report.queued = ready.length
|
|
2615
|
+
const withheld = new Set<string>()
|
|
2616
|
+
for (const ticket of ready) {
|
|
2617
|
+
if (ticket.creationKey === undefined) continue
|
|
2618
|
+
const creation = creationHolds.get(ticket.creationKey.toLowerCase())
|
|
2619
|
+
if (creation === undefined) continue
|
|
2620
|
+
withheld.add(ticket.ref.id)
|
|
2621
|
+
report.creationWithheld += 1
|
|
2622
|
+
report.creationDiagnostics.push(
|
|
2623
|
+
`ticket ${ticket.ref.id}: creation withheld — Harvest run ${creation.run}, ` +
|
|
2624
|
+
`proposal ${creation.proposalKey}, reservation repo seq ${creation.reservationSeq} ` +
|
|
2625
|
+
'has not reached a safe post-create listing boundary',
|
|
2626
|
+
)
|
|
2627
|
+
}
|
|
2628
|
+
// This stage creates a build and launches it in one step, so it is also a
|
|
2629
|
+
// way to attach a runner to a queued build. A pause-all turns intake off,
|
|
2630
|
+
// so in practice this stage does not run while paused; it becomes reachable
|
|
2631
|
+
// only if an operator toggles intake back on without resuming, or if the
|
|
2632
|
+
// intake write failed after the pause write landed. The ready scan above
|
|
2633
|
+
// still runs either way, so the standing queue-depth report stays honest —
|
|
2634
|
+
// this only declines to claim a ticket into a build that would immediately
|
|
2635
|
+
// sit held.
|
|
2636
|
+
if (capacity <= 0 || paused) return
|
|
2637
|
+
// One dependency-node cache per tick: blockers are commonly shared across
|
|
2638
|
+
// the ready set, and a blocker's resolution must be re-read every tick
|
|
2639
|
+
// (never cached across them) so a completion lands on the next pass.
|
|
2640
|
+
const nodes = new Map<string, DependencyState>()
|
|
2641
|
+
|
|
2642
|
+
for (const ticket of ready) {
|
|
2643
|
+
if (capacity <= 0) break
|
|
2644
|
+
// Budget gate before the dependency fetch / claim / createBuild: the
|
|
2645
|
+
// first ready read above stays (it is the standing queue-depth report);
|
|
2646
|
+
// once the budget is spent no additional transport call is started, so
|
|
2647
|
+
// the remaining tickets are left un-attempted for the next invocation.
|
|
2648
|
+
if (this.outOfBudget(opts)) break
|
|
2649
|
+
|
|
2650
|
+
// Autobuild's own in-flight create gate precedes even dependency reads:
|
|
2651
|
+
// the ready projection may have been captured before blocker recording.
|
|
2652
|
+
// It spends no capacity and leaves queue depth unchanged.
|
|
2653
|
+
if (withheld.has(ticket.ref.id)) continue
|
|
2654
|
+
|
|
2655
|
+
// The dependency gate runs FIRST — before the claim, before the spec
|
|
2656
|
+
// gate. A blocked ticket must not be claimed, must not be bounced, must
|
|
2657
|
+
// create no build or workspace, and must not spend capacity; it is
|
|
2658
|
+
// simply not this tick's work. Note this sits *above* [tickets]'s
|
|
2659
|
+
// readyLabels / readyState gate: those criteria produced `ready`, and the
|
|
2660
|
+
// gate subtracts from it — labels and state can make a ticket a
|
|
2661
|
+
// candidate but can never override an unresolved blocker.
|
|
2662
|
+
if ((ticket.blockedBy ?? []).length > 0) {
|
|
2663
|
+
let verdict: DependencyVerdict
|
|
2664
|
+
try {
|
|
2665
|
+
await this.loadDependencyGraph(ticket, nodes)
|
|
2666
|
+
verdict = analyzeDependencies(ticket.ref.id, ticket.blockedBy ?? [], nodes)
|
|
2667
|
+
} catch (error) {
|
|
2668
|
+
// A broken graph is this ticket's problem, not the tick's: skip it
|
|
2669
|
+
// and let unrelated eligible tickets dispatch normally.
|
|
2670
|
+
verdict = {
|
|
2671
|
+
unresolved: [ticket.ref.id],
|
|
2672
|
+
diagnostics: [
|
|
2673
|
+
`ticket ${ticket.ref.id}: dependency check failed — ${
|
|
2674
|
+
error instanceof Error ? error.message : String(error)
|
|
2675
|
+
}`,
|
|
2676
|
+
],
|
|
2677
|
+
}
|
|
2678
|
+
}
|
|
2679
|
+
if (verdict.unresolved.length > 0) {
|
|
2680
|
+
report.dependencyBlocked += 1
|
|
2681
|
+
report.dependencyDiagnostics.push(...verdict.diagnostics)
|
|
2682
|
+
continue
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
// Claim-before-launch (§12): losing the claim means another dispatcher
|
|
2687
|
+
// (or an earlier tick) owns this ticket.
|
|
2688
|
+
if (!(await tickets.claim(ticket.ref.id))) {
|
|
2689
|
+
report.claimRaces += 1
|
|
2690
|
+
report.queued -= 1
|
|
2691
|
+
continue
|
|
2692
|
+
}
|
|
2693
|
+
|
|
2694
|
+
// Quality gate (§6.3): import a conforming ticket body as the spec;
|
|
2695
|
+
// author one for thin-but-groomed tickets; otherwise bounce.
|
|
2696
|
+
let body = ticket.body
|
|
2697
|
+
let conformance = specConformance(body)
|
|
2698
|
+
let authoredSession: string | undefined
|
|
2699
|
+
if (!conformance.conforms && this.deps.authorSpec) {
|
|
2700
|
+
const authored = await this.deps.authorSpec(ticket)
|
|
2701
|
+
if (authored !== null) {
|
|
2702
|
+
conformance = specConformance(authored)
|
|
2703
|
+
if (conformance.conforms) {
|
|
2704
|
+
body = authored
|
|
2705
|
+
authoredSession = this.deps.ids('s')
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
if (!conformance.conforms) {
|
|
2710
|
+
await this.bounce(ticket, conformance.missing)
|
|
2711
|
+
report.bounced += 1
|
|
2712
|
+
report.queued -= 1
|
|
2713
|
+
continue
|
|
2714
|
+
}
|
|
2715
|
+
|
|
2716
|
+
const baseSlug = await this.chooseSlugBase(ticket.title, body)
|
|
2717
|
+
const slug = await this.uniqueSlug(baseSlug)
|
|
2718
|
+
const branch = `ab/${slug}`
|
|
2719
|
+
const repoOrigin = await this.resolveRepoOriginOnce()
|
|
2720
|
+
const record = await store.createBuild({
|
|
2721
|
+
slug,
|
|
2722
|
+
repo: this.deps.repo,
|
|
2723
|
+
ticket: ticket.ref,
|
|
2724
|
+
branch,
|
|
2725
|
+
...(repoOrigin !== undefined ? { repoOrigin } : {}),
|
|
2726
|
+
})
|
|
2727
|
+
report.queued -= 1
|
|
2728
|
+
capacity -= 1
|
|
2729
|
+
const outcome = await this.attemptDispatchCompletion(record, launched, report, {
|
|
2730
|
+
ticket,
|
|
2731
|
+
body,
|
|
2732
|
+
...(authoredSession !== undefined ? { authoredSession } : {}),
|
|
2733
|
+
...(autoMergeUser !== undefined ? { autoMergeUser } : {}),
|
|
2734
|
+
})
|
|
2735
|
+
// Accepted dispatches: durable boundaries recorded. A deferred outcome
|
|
2736
|
+
// means the remote provisioning continuation owns the rest.
|
|
2737
|
+
if (outcome === 'completed' || outcome === 'deferred') report.dispatched += 1
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
|
|
2741
|
+
/**
|
|
2742
|
+
* Close `nodes` over everything reachable from `ticket`'s blockers, so the
|
|
2743
|
+
* analyzer can both gate and name a cycle. Breadth-first, one port call per
|
|
2744
|
+
* level, skipping ids already cached — the common case (a blocker or two,
|
|
2745
|
+
* none of them blocked themselves) is a single call.
|
|
2746
|
+
*
|
|
2747
|
+
* INVARIANT: `nodes` contains only states actually returned by the source.
|
|
2748
|
+
* The cache is shared across the tick's tickets, so a node this ticket
|
|
2749
|
+
* invents is a node the *next* ticket will trust as a fact about its own
|
|
2750
|
+
* blocker. Nothing is seeded here for that reason (f_8bc9ee0c); the
|
|
2751
|
+
* analyzer takes the ticket's own blockers as an argument instead, and
|
|
2752
|
+
* `findCycle` closes a cycle via its path rather than the map.
|
|
2753
|
+
*
|
|
2754
|
+
* A provider-side cycle terminates the walk: an id already in `nodes` is
|
|
2755
|
+
* never re-queued, so every reachable node is fetched exactly once.
|
|
2756
|
+
*/
|
|
2757
|
+
private async loadDependencyGraph(
|
|
2758
|
+
ticket: Ticket,
|
|
2759
|
+
nodes: Map<string, DependencyState>,
|
|
2760
|
+
): Promise<void> {
|
|
2761
|
+
let frontier = [...new Set(ticket.blockedBy ?? [])].filter((id) => !nodes.has(id))
|
|
2762
|
+
while (frontier.length > 0) {
|
|
2763
|
+
const states = await this.deps.tickets.dependencyStates(frontier)
|
|
2764
|
+
for (const state of states) nodes.set(state.id, state)
|
|
2765
|
+
frontier = [...new Set(states.flatMap((state) => state.blockedBy))].filter(
|
|
2766
|
+
(id) => !nodes.has(id),
|
|
2767
|
+
)
|
|
2768
|
+
}
|
|
2769
|
+
}
|
|
2770
|
+
|
|
2771
|
+
/** Bounce (§6.3): back to Triage with a comment citing the standard and
|
|
2772
|
+
* WHICH parts are missing — failure at the cheapest point, not a build
|
|
2773
|
+
* that thrashes and escalates. No build is created. */
|
|
2774
|
+
private async bounce(ticket: Ticket, missing: string[]): Promise<void> {
|
|
2775
|
+
const triageState = this.triageState()
|
|
2776
|
+
await this.deps.tickets.transition(ticket.ref.id, triageState)
|
|
2777
|
+
await this.deps.tickets.comment(
|
|
2778
|
+
ticket.ref.id,
|
|
2779
|
+
`Bounced back to ${triageState}: this ticket does not conform to ` +
|
|
2780
|
+
`the spec standard (.agents/skills/ab-guide/references/spec-standard.md) and cannot be dispatched.\n` +
|
|
2781
|
+
`Missing: ${missing.join('; ')}.`,
|
|
2782
|
+
)
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2785
|
+
/**
|
|
2786
|
+
* Ask for a spec-aware base behind a hard deadline. Every failure mode is
|
|
2787
|
+
* deliberately indistinguishable here: naming is optional judgment, while
|
|
2788
|
+
* successful dispatch is deterministic policy.
|
|
2789
|
+
*/
|
|
2790
|
+
private async chooseSlugBase(title: string, spec: string): Promise<string> {
|
|
2791
|
+
const fallback = fallbackSlug(title)
|
|
2792
|
+
const nameSlug = this.deps.nameSlug
|
|
2793
|
+
if (nameSlug === undefined) return fallback
|
|
2794
|
+
|
|
2795
|
+
const controller = new AbortController()
|
|
2796
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
2797
|
+
const deadline = new Promise<never>((_resolve, reject) => {
|
|
2798
|
+
timer = setTimeout(
|
|
2799
|
+
() => {
|
|
2800
|
+
controller.abort()
|
|
2801
|
+
reject(new Error('build slug naming deadline exceeded'))
|
|
2802
|
+
},
|
|
2803
|
+
Math.max(0, this.slugNamingTimeoutMs),
|
|
2804
|
+
)
|
|
2805
|
+
})
|
|
2806
|
+
|
|
2807
|
+
try {
|
|
2808
|
+
const candidate = await Promise.race([nameSlug(spec, controller.signal), deadline])
|
|
2809
|
+
return validateSlugCandidate(candidate) ?? fallback
|
|
2810
|
+
} catch {
|
|
2811
|
+
return fallback
|
|
2812
|
+
} finally {
|
|
2813
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
2814
|
+
}
|
|
2815
|
+
}
|
|
2816
|
+
|
|
2817
|
+
/** Valid bounded base, deduped store-wide with -2/-3/… suffixes. */
|
|
2818
|
+
private async uniqueSlug(base: string): Promise<string> {
|
|
2819
|
+
let slug = base
|
|
2820
|
+
for (let n = 2; (await this.deps.store.getBuild(slug)) !== null; n += 1) {
|
|
2821
|
+
slug = `${base}-${n}`
|
|
2822
|
+
}
|
|
2823
|
+
return slug
|
|
2824
|
+
}
|
|
2825
|
+
}
|