@defrex/autobuild 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/SPEC.md +323 -26
- package/bin/ab-build-runner.ts +8 -2
- package/docs/architecture.md +166 -14
- package/docs/assets/headline-wide.png +0 -0
- package/docs/configuration.md +175 -28
- package/docs/hosted-dispatcher.md +69 -13
- package/docs/operator-api.md +2 -2
- package/docs/remote-store-protocol.md +204 -16
- package/docs/remote-ticket-protocol.md +1 -1
- package/docs/setup.md +12 -9
- package/docs/website.md +14 -0
- package/package.json +45 -23
- package/packages/core/src/cli/artifact.ts +130 -6
- package/packages/core/src/cli/binary.ts +9 -1
- package/packages/core/src/cli/build-control.ts +110 -13
- package/packages/core/src/cli/context.ts +46 -0
- package/packages/core/src/cli/dashboard/facts.ts +215 -0
- package/packages/core/src/cli/dashboard/frame-parse.ts +223 -0
- package/packages/core/src/cli/dashboard/model.ts +141 -153
- package/packages/core/src/cli/dashboard/poll.ts +76 -12
- package/packages/core/src/cli/dashboard/render.ts +34 -5
- package/packages/core/src/cli/dispatch-frontend.ts +54 -5
- package/packages/core/src/cli/dispatch-process.ts +1 -13
- package/packages/core/src/cli/dispatch.ts +392 -75
- package/packages/core/src/cli/harvest.ts +122 -8
- package/packages/core/src/cli/help.ts +62 -9
- package/packages/core/src/cli/init-readiness-remote.ts +110 -0
- package/packages/core/src/cli/init-readiness-shared.ts +72 -0
- package/packages/core/src/cli/init-validation.ts +266 -222
- package/packages/core/src/cli/installation.ts +50 -8
- package/packages/core/src/cli/main.ts +70 -11
- package/packages/core/src/cli/mcp.ts +23 -1
- package/packages/core/src/cli/plugin.ts +7 -4
- package/packages/core/src/cli/remote-poll.ts +277 -0
- package/packages/core/src/cli/repo-state.ts +22 -0
- package/packages/core/src/cli/repository-status.ts +39 -1
- package/packages/core/src/cli/self-update.ts +77 -2
- package/packages/core/src/cli/status.ts +150 -28
- package/packages/core/src/cli/terminals.ts +2 -1
- package/packages/core/src/cli/ticket-assets.ts +247 -0
- package/packages/core/src/cli/ticket.ts +453 -8
- package/packages/core/src/cli/wait.ts +157 -41
- package/packages/core/src/cli/watch.ts +138 -168
- package/packages/core/src/config/live.ts +23 -0
- package/packages/core/src/config/pipeline-source.ts +388 -0
- package/packages/core/src/config/schema.ts +251 -19
- package/packages/core/src/events/catalog.ts +1 -0
- package/packages/core/src/events/globs.ts +47 -0
- package/packages/core/src/events/payloads.ts +52 -2
- package/packages/core/src/events/repository.ts +52 -1
- package/packages/core/src/events/sessions.ts +69 -10
- package/packages/core/src/events/workspace-remote.ts +12 -0
- package/packages/core/src/integration/harness.ts +115 -2
- package/packages/core/src/kernel/__snapshots__/reducer-registry.test.ts.snap +1369 -0
- package/packages/core/src/kernel/auto-merge-default.ts +157 -0
- package/packages/core/src/kernel/auto-merge.ts +99 -22
- package/packages/core/src/kernel/dispatch-settings.ts +42 -17
- package/packages/core/src/kernel/dispatch-status.ts +23 -0
- package/packages/core/src/kernel/engine.ts +9 -438
- package/packages/core/src/kernel/generators/build-log.ts +259 -0
- package/packages/core/src/kernel/generators/repository-journals.ts +206 -0
- package/packages/core/src/kernel/generators/session-log.ts +72 -0
- package/packages/core/src/kernel/harvest.ts +62 -18
- package/packages/core/src/kernel/incremental-contract.ts +75 -0
- package/packages/core/src/kernel/incremental-fixtures.ts +211 -0
- package/packages/core/src/kernel/incremental.ts +53 -0
- package/packages/core/src/kernel/log-index.ts +620 -0
- package/packages/core/src/kernel/phases.ts +16 -2
- package/packages/core/src/kernel/pr-attachments.ts +164 -46
- package/packages/core/src/kernel/reducer-registry.ts +164 -0
- package/packages/core/src/kernel/reducer.ts +254 -73
- package/packages/core/src/operator/annotations.ts +6 -0
- package/packages/core/src/operator/control.ts +19 -5
- package/packages/core/src/operator/index.ts +71 -6
- package/packages/core/src/operator/query.ts +253 -24
- package/packages/core/src/operator/read-view.ts +98 -0
- package/packages/core/src/operator/registry.ts +55 -10
- package/packages/core/src/operator/requests.ts +9 -6
- package/packages/core/src/operator/sandbox-host.ts +140 -0
- package/packages/core/src/operator/sandbox.ts +247 -14
- package/packages/core/src/operator/ticket-source-backend.ts +47 -0
- package/packages/core/src/orchestrator/approvals.ts +115 -0
- package/packages/core/src/orchestrator/conversation.ts +167 -0
- package/packages/core/src/orchestrator/model.ts +86 -0
- package/packages/core/src/orchestrator/prompt.ts +61 -0
- package/packages/core/src/orchestrator/tools.ts +117 -0
- package/packages/core/src/orchestrator/turn-runner.ts +457 -0
- package/packages/core/src/plugin-sdk/index.ts +51 -1
- package/packages/core/src/plugins/load.ts +135 -7
- package/packages/core/src/plugins/manifest.ts +183 -10
- package/packages/core/src/plugins/registry.ts +73 -4
- package/packages/core/src/ports/forge/create.ts +1 -1
- package/packages/core/src/ports/forge/fake.ts +31 -0
- package/packages/core/src/ports/forge/github-transport.ts +13 -2
- package/packages/core/src/ports/forge/github.ts +13 -2
- package/packages/core/src/ports/forge/local-git.ts +31 -1
- package/packages/core/src/ports/runner/claude.ts +2 -2
- package/packages/core/src/ports/runner/codex.ts +6 -2
- package/packages/core/src/ports/runner/fake.ts +7 -1
- package/packages/core/src/ports/runner/pi.ts +8 -6
- package/packages/core/src/ports/runner/provider-error.ts +13 -3
- package/packages/core/src/ports/tickets/fake.ts +5 -1
- package/packages/core/src/ports/tickets/file.ts +169 -48
- package/packages/core/src/ports/tickets/{remote-index.ts → hosted-index.ts} +3 -2
- package/packages/core/src/ports/tickets/linear.ts +33 -1
- package/packages/core/src/ports/tickets/operations.ts +1 -1
- package/packages/core/src/ports/tickets/remote.ts +8 -1
- package/packages/core/src/ports/types.ts +49 -3
- package/packages/core/src/ports/workspace/builtin-capabilities.ts +114 -0
- package/packages/core/src/ports/workspace/create.ts +74 -39
- package/packages/core/src/ports/workspace/distribution-archive.ts +171 -12
- package/packages/core/src/ports/workspace/fake.ts +169 -4
- package/packages/core/src/ports/workspace/git-worktree.ts +246 -26
- package/packages/core/src/ports/workspace/github-origin.ts +36 -0
- package/packages/core/src/ports/workspace/operator-sandbox.ts +7 -1
- package/packages/core/src/ports/workspace/provider-capabilities.ts +171 -0
- package/packages/core/src/ports/workspace/vercel-capabilities.ts +110 -0
- package/packages/core/src/ports/workspace/vercel-sandbox.ts +142 -114
- package/packages/core/src/processes/build-child.ts +17 -2
- package/packages/core/src/processes/build-execution-state.ts +68 -15
- package/packages/core/src/processes/build-runner.ts +238 -93
- package/packages/core/src/processes/dispatcher-selectors.ts +123 -0
- package/packages/core/src/processes/dispatcher.ts +501 -201
- package/packages/core/src/processes/execution-settlement.ts +72 -39
- package/packages/core/src/processes/harvest-child.ts +5 -0
- package/packages/core/src/processes/harvest-execution-state.ts +41 -29
- package/packages/core/src/processes/harvest-runner.ts +66 -19
- package/packages/core/src/processes/harvest.ts +196 -24
- package/packages/core/src/processes/infrastructure-failure-budget.ts +31 -18
- package/packages/core/src/processes/orchestrator-tick.ts +360 -0
- package/packages/core/src/processes/publication-state.ts +164 -52
- package/packages/core/src/processes/repo-view.ts +1436 -0
- package/packages/core/src/processes/sandbox-state.ts +36 -2
- package/packages/core/src/processes/tick-state.ts +33 -0
- package/packages/core/src/store/adapter.ts +41 -0
- package/packages/core/src/store/build-scope.ts +90 -0
- package/packages/core/src/store/contract.ts +1205 -0
- package/packages/core/src/store/digest.ts +102 -0
- package/packages/core/src/store/local/schema.ts +46 -2
- package/packages/core/src/store/local/store.ts +445 -4
- package/packages/core/src/store/memory.ts +246 -0
- package/packages/core/src/store/new-build.ts +17 -0
- package/packages/core/src/store/phase-session.ts +94 -0
- package/packages/core/src/store/remote/client.ts +310 -2
- package/packages/core/src/store/remote/index.ts +14 -2
- package/packages/core/src/store/remote/protocol.ts +115 -12
- package/packages/core/src/store/remote/version.ts +6 -2
- package/packages/core/src/store/repo-state-events.ts +205 -0
- package/packages/core/src/store/retention.ts +7 -0
- package/packages/core/src/store/session-handle.ts +94 -0
- package/packages/core/src/{sessions/reducer.ts → store/session-reducer.ts} +79 -19
- package/packages/core/src/store/snapshots.ts +88 -0
- package/packages/core/src/store/streams/session-writer.ts +42 -2
- package/packages/core/src/store/streams/wait.ts +4 -5
- package/packages/core/src/store/ticket-assets.ts +458 -0
- package/packages/core/src/store/types.ts +160 -1
- package/packages/core/src/testing/index.ts +59 -0
- package/skills/code-review/SKILL.md +5 -1
- package/skills/guide/SKILL.md +152 -40
- package/skills/guide/references/operator-api.md +2 -2
- package/skills/guide/references/plugin-authoring.md +17 -0
- package/skills/guide/references/remote-store-protocol.md +204 -16
- package/skills/guide/references/remote-ticket-protocol.md +1 -1
- package/skills/guide/references/setup.md +12 -9
- package/skills/implement/SKILL.md +3 -0
- package/skills/operate/SKILL.md +126 -0
- package/skills/plan/SKILL.md +4 -0
- package/skills/plan-review/SKILL.md +5 -1
- package/skills/spec/SKILL.md +46 -3
- package/skills/tickets/SKILL.md +57 -5
- package/packages/core/src/cli/dashboard/frame-image.ts +0 -373
- package/packages/core/src/cli/store-opening.contract.ts +0 -289
- package/packages/core/src/cli/testkit.ts +0 -194
- package/packages/core/src/markdown.ts +0 -67
- package/packages/core/src/operator/client.ts +0 -311
- package/packages/core/src/operator/server.ts +0 -616
- package/packages/core/src/operator/sessions.ts +0 -223
- package/packages/core/src/ports/runner/live-contract-fixture.ts +0 -176
- package/packages/core/src/ports/tickets/remote-server.ts +0 -185
- package/packages/core/src/store/remote/server.ts +0 -923
- package/packages/core/src/testing/store-failures.ts +0 -32
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The dispatcher tick's orchestrator step (AUT-342): resume suspended turns,
|
|
3
|
+
* reap crashed ones, and wake idle sessions from build attention events.
|
|
4
|
+
*
|
|
5
|
+
* Two independent gates keep this inert where it does not belong, and BOTH
|
|
6
|
+
* are required:
|
|
7
|
+
*
|
|
8
|
+
* 1. The step is constructed only in origin mode — the gate lives in
|
|
9
|
+
* `processes/dispatcher.ts` on `DispatcherDeps.repoOrigin` (set
|
|
10
|
+
* exclusively by origin mode), so a local `ab dispatch` never constructs
|
|
11
|
+
* the step at all: an `enabled = true` in a local repository's
|
|
12
|
+
* autobuild.toml lists no sessions, builds no registry, and calls no
|
|
13
|
+
* model.
|
|
14
|
+
* 2. In origin mode the step returns immediately unless the repository's
|
|
15
|
+
* effective config enables the orchestrator — the disabled-repository
|
|
16
|
+
* no-op.
|
|
17
|
+
*
|
|
18
|
+
* The wake cursor advances only through the recorded trigger (the reducer
|
|
19
|
+
* takes `max`), so a crash between scan and start cannot skip or duplicate a
|
|
20
|
+
* wake; other builds' — and the journal's — matching events re-trigger on
|
|
21
|
+
* later ticks. One turn per session per tick, never concurrent.
|
|
22
|
+
*/
|
|
23
|
+
import { ORCHESTRATOR_MIN_TURN_SECONDS, type Config } from '../config/schema'
|
|
24
|
+
import { compileEventGlobs } from '../events/globs'
|
|
25
|
+
import type { RepositoryEvent } from '../events/repository'
|
|
26
|
+
import type { TicketSource } from '../ports/types'
|
|
27
|
+
import { buildRegistry, type OperatorToolRegistry } from '../operator/registry'
|
|
28
|
+
import type { OperatorSandboxService } from '../operator/sandbox'
|
|
29
|
+
import { ticketBackendFromSource } from '../operator/ticket-source-backend'
|
|
30
|
+
import { reduceSession, type SessionTurnTrigger } from '../store/session-reducer'
|
|
31
|
+
import { reduceBuild } from '../kernel/reducer'
|
|
32
|
+
import type { AbEvent } from '../events/catalog'
|
|
33
|
+
import type { BuildStore, Clock } from '../store/types'
|
|
34
|
+
import type { RepoViewStore } from './repo-view'
|
|
35
|
+
import type { IdSource } from '../ids'
|
|
36
|
+
import type { LanguageModel } from 'ai'
|
|
37
|
+
import {
|
|
38
|
+
createOrchestratorTurnRunner,
|
|
39
|
+
type OrchestratorTurnRunner,
|
|
40
|
+
type WakeTurnInput,
|
|
41
|
+
} from '../orchestrator/turn-runner'
|
|
42
|
+
|
|
43
|
+
/** Minutes an open, unsuspended turn's stream may stay silent before the
|
|
44
|
+
* reaper fails it: a crashed invocation leaves the session `running` with an
|
|
45
|
+
* open stream and no runner. Suspended turns are NEVER reaped — a turn
|
|
46
|
+
* suspended for approval is precisely an open stream that goes quiet for
|
|
47
|
+
* however long the operator takes. */
|
|
48
|
+
export const ORCHESTRATOR_STALE_TURN_MINUTES = 15
|
|
49
|
+
|
|
50
|
+
export interface OrchestratorTickOptions {
|
|
51
|
+
store: BuildStore
|
|
52
|
+
repo: string
|
|
53
|
+
config: Config
|
|
54
|
+
clock: Clock
|
|
55
|
+
ids: IdSource
|
|
56
|
+
/** The dispatcher's ticket source, adapted into the registry's ticket
|
|
57
|
+
* backend so wake and resumed turns keep the full ticket surface. */
|
|
58
|
+
tickets: TicketSource
|
|
59
|
+
/** Injected language model; defaults to the configured gateway model. */
|
|
60
|
+
model?: LanguageModel
|
|
61
|
+
/** The operator-sandbox backend (AUT-584): when supplied, the tick's turns
|
|
62
|
+
* serve the `sandbox.*` tools under the same configuration/approval gating
|
|
63
|
+
* every binding applies. Absent (the compatibility case) → the registry
|
|
64
|
+
* filters them from advertisement AND dispatch. The origin-mode dispatcher
|
|
65
|
+
* builds it from its own wired provider; every other caller omits it. */
|
|
66
|
+
sandbox?: OperatorSandboxService
|
|
67
|
+
/** Remaining tick budget in seconds at step entry; every turn's deadline
|
|
68
|
+
* is recomputed from this against the clock, so N sessions draw down the
|
|
69
|
+
* SAME budget instead of each getting a fresh full slice (f_9c1161ae).
|
|
70
|
+
* Less than ORCHESTRATOR_MIN_TURN_SECONDS skips the whole step
|
|
71
|
+
* (suspension state is durable — skipping is safe). */
|
|
72
|
+
remainingBudgetSeconds?: number
|
|
73
|
+
/** The dispatcher's repository view, when the tick runs under one: the wake
|
|
74
|
+
* pass reads its full build state through it (no replay of history the view
|
|
75
|
+
* already folded) and releases consumed window history through it. */
|
|
76
|
+
view?: Pick<RepoViewStore, 'buildState' | 'releaseWindowsBelow'>
|
|
77
|
+
log?: (message: string) => void
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface OrchestratorTickReport {
|
|
81
|
+
resumed: number
|
|
82
|
+
reaped: number
|
|
83
|
+
woken: number
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** The registry the tick's turns run against: the in-process binding over
|
|
87
|
+
* the dispatcher's store, with the ticket source adapted into the backend
|
|
88
|
+
* `buildRegistry`'s ticket tools require (`buildMcpTicketBackend` reads a
|
|
89
|
+
* checkout's autobuild.toml and cannot run in origin mode). */
|
|
90
|
+
export function orchestratorTickRegistry(options: OrchestratorTickOptions): OperatorToolRegistry {
|
|
91
|
+
return buildRegistry({
|
|
92
|
+
store: options.store,
|
|
93
|
+
tickets: ticketBackendFromSource({ source: options.tickets, config: options.config }),
|
|
94
|
+
...(options.sandbox !== undefined ? { sandbox: options.sandbox } : {}),
|
|
95
|
+
clock: options.clock,
|
|
96
|
+
allowedRepo: options.repo,
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function orchestratorTickRunner(options: OrchestratorTickOptions): OrchestratorTurnRunner {
|
|
101
|
+
return createOrchestratorTurnRunner({
|
|
102
|
+
store: options.store,
|
|
103
|
+
registry: orchestratorTickRegistry(options),
|
|
104
|
+
repo: options.repo,
|
|
105
|
+
config: options.config,
|
|
106
|
+
clock: options.clock,
|
|
107
|
+
ids: options.ids,
|
|
108
|
+
...(options.model !== undefined ? { model: options.model } : {}),
|
|
109
|
+
...(options.remainingBudgetSeconds !== undefined
|
|
110
|
+
? { remainingBudgetSeconds: options.remainingBudgetSeconds }
|
|
111
|
+
: {}),
|
|
112
|
+
})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** One pass of the orchestrator step. The caller has already evaluated both
|
|
116
|
+
* gates (origin mode and `enabled`); this function assumes it may run. */
|
|
117
|
+
export async function runOrchestratorTickStep(
|
|
118
|
+
options: OrchestratorTickOptions,
|
|
119
|
+
): Promise<OrchestratorTickReport> {
|
|
120
|
+
const { store, repo, config, clock } = options
|
|
121
|
+
const report: OrchestratorTickReport = { resumed: 0, reaped: 0, woken: 0 }
|
|
122
|
+
const entry =
|
|
123
|
+
options.remainingBudgetSeconds === undefined
|
|
124
|
+
? config.orchestrator.invocationBudgetSeconds
|
|
125
|
+
: options.remainingBudgetSeconds
|
|
126
|
+
if (entry < ORCHESTRATOR_MIN_TURN_SECONDS) return report
|
|
127
|
+
|
|
128
|
+
// The step's deadline, fixed at entry: every turn's slice is recomputed
|
|
129
|
+
// from the clock against THIS deadline, so the step can only consume the
|
|
130
|
+
// remaining slack no matter how many sessions it serves. (A fixed
|
|
131
|
+
// per-session value handed to each turn would let N sessions run
|
|
132
|
+
// N × remaining seconds past the tick deadline — f_9c1161ae.)
|
|
133
|
+
const deadlineMs = clock().getTime() + entry * 1000
|
|
134
|
+
const remainingNow = (): number =>
|
|
135
|
+
Math.max(0, Math.floor((deadlineMs - clock().getTime()) / 1000))
|
|
136
|
+
|
|
137
|
+
const runner = orchestratorTickRunner(options)
|
|
138
|
+
const sessions = await store.listSessions(repo)
|
|
139
|
+
|
|
140
|
+
// ── Resume pass ─────────────────────────────────────────────────────────
|
|
141
|
+
// The lowest cursors only cover sessions visited; a budget break leaves later
|
|
142
|
+
// sessions (possibly with lower cursors) uncounted, so it skips the trim.
|
|
143
|
+
let visitedAll = true
|
|
144
|
+
for (const record of sessions) {
|
|
145
|
+
const events = await store.getSessionEvents(record.id)
|
|
146
|
+
const state = reduceSession(events)
|
|
147
|
+
if (state.status !== 'suspended' || state.openTurn === undefined) continue
|
|
148
|
+
// Budget suspensions resume always; approval suspensions only once the
|
|
149
|
+
// answer has landed (`pendingApproval` gone — the answered-approval
|
|
150
|
+
// fallback when the resuming operator-route invocation died). An
|
|
151
|
+
// unanswered approval stays put until the operator answers.
|
|
152
|
+
if (state.pendingApproval !== undefined) continue
|
|
153
|
+
// An answered approval resumes with the recorded decision, so the
|
|
154
|
+
// runner recovers the wire approvalId and appends the response part.
|
|
155
|
+
let approval: { decision: 'approve' | 'deny'; toolCallId: string } | undefined
|
|
156
|
+
if (state.suspendedCause === 'approval') {
|
|
157
|
+
for (const event of [...events].reverse()) {
|
|
158
|
+
if (event.type === 'approval.answered' && event.payload.turn === state.openTurn.turn) {
|
|
159
|
+
approval = { decision: event.payload.decision, toolCallId: event.payload.toolCallId }
|
|
160
|
+
break
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
if (approval === undefined) continue
|
|
164
|
+
}
|
|
165
|
+
// Draw down the shared deadline: a session is skipped — suspension state
|
|
166
|
+
// is durable, so skipping is always safe — when the step's remaining
|
|
167
|
+
// slack no longer leaves the floor for a meaningful turn slice.
|
|
168
|
+
const remaining = remainingNow() - ORCHESTRATOR_MIN_TURN_SECONDS
|
|
169
|
+
if (remaining < ORCHESTRATOR_MIN_TURN_SECONDS) break
|
|
170
|
+
const resumed = await runner.resumeTurn(record.id, {
|
|
171
|
+
...(approval !== undefined ? { approval } : {}),
|
|
172
|
+
remainingBudgetSeconds: remaining,
|
|
173
|
+
})
|
|
174
|
+
if (resumed.resumed) {
|
|
175
|
+
report.resumed += 1
|
|
176
|
+
// One turn per session per tick, never concurrent: the loop outcome
|
|
177
|
+
// runs to its suspension/completion before the next session.
|
|
178
|
+
await resumed.outcome
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ── Crash reaper ────────────────────────────────────────────────────────
|
|
183
|
+
const staleMs = ORCHESTRATOR_STALE_TURN_MINUTES * 60_000
|
|
184
|
+
for (const record of sessions) {
|
|
185
|
+
const events = await store.getSessionEvents(record.id)
|
|
186
|
+
const state = reduceSession(events)
|
|
187
|
+
// Reap only a session whose reduced status is exactly `running` — an
|
|
188
|
+
// open turn with no pending approval and no suspension. A late approve
|
|
189
|
+
// answer must still find its pending approval and its turn.
|
|
190
|
+
if (state.status !== 'running' || state.openTurn === undefined) continue
|
|
191
|
+
const lastChunkTs = await lastStreamActivity(store, state.openTurn.stream)
|
|
192
|
+
if (lastChunkTs === null) continue
|
|
193
|
+
if (clock().getTime() - Date.parse(lastChunkTs) < staleMs) continue
|
|
194
|
+
const lastSeq = events[events.length - 1]?.seq ?? 0
|
|
195
|
+
const reaped = await store
|
|
196
|
+
.appendSessionEventIfCurrent(record.id, lastSeq, {
|
|
197
|
+
actor: { kind: 'agent', role: 'orchestrator', session: state.openTurn.turn },
|
|
198
|
+
type: 'turn.failed',
|
|
199
|
+
payload: {
|
|
200
|
+
turn: state.openTurn.turn,
|
|
201
|
+
kind: 'internal',
|
|
202
|
+
error: `turn invocation ended without an outcome (no stream activity for ${ORCHESTRATOR_STALE_TURN_MINUTES} minutes)`,
|
|
203
|
+
},
|
|
204
|
+
})
|
|
205
|
+
.catch(() => null)
|
|
206
|
+
if (reaped === null) continue // a resume landed first
|
|
207
|
+
await store.closeStream(state.openTurn.stream, 'aborted').catch(() => undefined)
|
|
208
|
+
report.reaped += 1
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// ── Wake pass ───────────────────────────────────────────────────────────
|
|
212
|
+
const builds = (await store.listBuilds()).filter((candidate) => candidate.repo === repo)
|
|
213
|
+
/** The lowest wake cursor across every session that scans, per source: the
|
|
214
|
+
* view may drop window history at or below it once the pass ends. */
|
|
215
|
+
const lowestBuildCursor = new Map<string, number>()
|
|
216
|
+
let lowestJournalCursor: number | undefined
|
|
217
|
+
let scanners = 0
|
|
218
|
+
for (const record of sessions) {
|
|
219
|
+
const events = await store.getSessionEvents(record.id)
|
|
220
|
+
const state = reduceSession(events)
|
|
221
|
+
if (state.wakeGlobs.length > 0) {
|
|
222
|
+
scanners += 1
|
|
223
|
+
for (const build of builds) {
|
|
224
|
+
const cursor = state.wakeCursors[build.slug] ?? 0
|
|
225
|
+
lowestBuildCursor.set(
|
|
226
|
+
build.slug,
|
|
227
|
+
Math.min(lowestBuildCursor.get(build.slug) ?? cursor, cursor),
|
|
228
|
+
)
|
|
229
|
+
}
|
|
230
|
+
lowestJournalCursor = Math.min(
|
|
231
|
+
lowestJournalCursor ?? state.journalWakeCursor,
|
|
232
|
+
state.journalWakeCursor,
|
|
233
|
+
)
|
|
234
|
+
}
|
|
235
|
+
if (state.status !== 'idle') continue
|
|
236
|
+
if (state.wakeGlobs.length === 0) continue
|
|
237
|
+
let filters: RegExp[]
|
|
238
|
+
try {
|
|
239
|
+
filters = compileEventGlobs(state.wakeGlobs, {
|
|
240
|
+
repository: true,
|
|
241
|
+
usage: '[orchestrator].wake',
|
|
242
|
+
})
|
|
243
|
+
} catch {
|
|
244
|
+
continue // unmatchable settings wake nothing
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// Scan every build's events after this session's per-build wake cursor,
|
|
248
|
+
// then the repository journal after its own cursor; select the single
|
|
249
|
+
// newest matching event across all scanned sources.
|
|
250
|
+
let newest:
|
|
251
|
+
| { build: string; event: AbEvent }
|
|
252
|
+
| { journal: true; event: RepositoryEvent }
|
|
253
|
+
| undefined
|
|
254
|
+
for (const build of builds) {
|
|
255
|
+
const cursor = state.wakeCursors[build.slug] ?? 0
|
|
256
|
+
const buildEvents = await store.getEvents(build.slug, cursor)
|
|
257
|
+
for (const event of buildEvents) {
|
|
258
|
+
if (event.seq <= cursor) continue
|
|
259
|
+
if (!filters.some((regex) => regex.test(event.type))) continue
|
|
260
|
+
if (
|
|
261
|
+
newest === undefined ||
|
|
262
|
+
event.ts > newest.event.ts ||
|
|
263
|
+
(event.ts === newest.event.ts && event.seq > newest.event.seq)
|
|
264
|
+
) {
|
|
265
|
+
newest = { build: build.slug, event }
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
// The journal's seq space is independent of every build's; a `seq`
|
|
270
|
+
// tie-break across sources compares numbers from different spaces —
|
|
271
|
+
// deterministic, and only matters when two sources' newest matches share
|
|
272
|
+
// a millisecond timestamp.
|
|
273
|
+
const journalCursor = state.journalWakeCursor
|
|
274
|
+
const journalEvents = await store.getRepoEvents(repo, journalCursor)
|
|
275
|
+
for (const event of journalEvents) {
|
|
276
|
+
if (event.seq <= journalCursor) continue
|
|
277
|
+
if (!filters.some((regex) => regex.test(event.type))) continue
|
|
278
|
+
if (
|
|
279
|
+
newest === undefined ||
|
|
280
|
+
event.ts > newest.event.ts ||
|
|
281
|
+
(event.ts === newest.event.ts && event.seq > newest.event.seq)
|
|
282
|
+
) {
|
|
283
|
+
newest = { journal: true, event }
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
if (newest === undefined) continue
|
|
287
|
+
|
|
288
|
+
// Same drawdown for wake turns: one turn per session per tick, bounded
|
|
289
|
+
// by the step's shared deadline; past the floor the wake defers to a
|
|
290
|
+
// later tick.
|
|
291
|
+
const remaining = remainingNow() - ORCHESTRATOR_MIN_TURN_SECONDS
|
|
292
|
+
if (remaining < ORCHESTRATOR_MIN_TURN_SECONDS) {
|
|
293
|
+
visitedAll = false
|
|
294
|
+
break
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// A journal wake delivers the frozen event record only — the turn's
|
|
298
|
+
// operator registry already exposes bounded repository reads, so no
|
|
299
|
+
// reduced state is coupled in. A build wake delivers the build's reduced
|
|
300
|
+
// state as before.
|
|
301
|
+
const event = {
|
|
302
|
+
seq: newest.event.seq,
|
|
303
|
+
ts: newest.event.ts,
|
|
304
|
+
type: newest.event.type,
|
|
305
|
+
payload: newest.event.payload,
|
|
306
|
+
}
|
|
307
|
+
const trigger: SessionTurnTrigger =
|
|
308
|
+
'journal' in newest
|
|
309
|
+
? { kind: 'wake', journal: true, seq: newest.event.seq, type: newest.event.type }
|
|
310
|
+
: { kind: 'wake', build: newest.build, seq: newest.event.seq, type: newest.event.type }
|
|
311
|
+
const wake: WakeTurnInput =
|
|
312
|
+
'journal' in newest
|
|
313
|
+
? { event }
|
|
314
|
+
: {
|
|
315
|
+
event,
|
|
316
|
+
buildState: (options.view !== undefined
|
|
317
|
+
? await options.view.buildState(newest.build)
|
|
318
|
+
: reduceBuild(await store.getEvents(newest.build))) as unknown as Record<
|
|
319
|
+
string,
|
|
320
|
+
unknown
|
|
321
|
+
>,
|
|
322
|
+
}
|
|
323
|
+
const start = await runner.startTurn(record.id, trigger, wake, {
|
|
324
|
+
remainingBudgetSeconds: remaining,
|
|
325
|
+
})
|
|
326
|
+
if (start.started) {
|
|
327
|
+
report.woken += 1
|
|
328
|
+
// The cursor advances only through the recorded trigger (the reducer
|
|
329
|
+
// takes `max`), so a crash between scan and start cannot skip or
|
|
330
|
+
// duplicate a wake — and other builds' matching events re-trigger on
|
|
331
|
+
// later ticks.
|
|
332
|
+
await start.outcome
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (options.view !== undefined && scanners > 0 && visitedAll) {
|
|
337
|
+
for (const [slug, cursor] of lowestBuildCursor) {
|
|
338
|
+
if (cursor > 0) options.view.releaseWindowsBelow({ build: slug }, cursor)
|
|
339
|
+
}
|
|
340
|
+
if (lowestJournalCursor !== undefined && lowestJournalCursor > 0) {
|
|
341
|
+
options.view.releaseWindowsBelow('journal', lowestJournalCursor)
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
options.log?.(
|
|
346
|
+
`orchestrator step: resumed=${report.resumed} reaped=${report.reaped} woken=${report.woken}`,
|
|
347
|
+
)
|
|
348
|
+
return report
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** The newest chunk timestamp on a stream, or its creation time when the
|
|
352
|
+
* stream has no chunks yet — the liveness signal the reaper reads. */
|
|
353
|
+
async function lastStreamActivity(store: BuildStore, streamId: string): Promise<string | null> {
|
|
354
|
+
const read = await store.readStream(streamId).catch(() => null)
|
|
355
|
+
if (read === null) return null
|
|
356
|
+
const last = read.chunks[read.chunks.length - 1]
|
|
357
|
+
if (last !== undefined) return last.ts
|
|
358
|
+
const record = await store.getStream(streamId)
|
|
359
|
+
return record?.createdAt ?? null
|
|
360
|
+
}
|
|
@@ -1,32 +1,170 @@
|
|
|
1
1
|
import type { AbEvent } from '../events/catalog'
|
|
2
|
+
import { defineReducer, type IncrementalReducer } from '../kernel/incremental'
|
|
2
3
|
|
|
3
4
|
export type PublicationRequest = Extract<AbEvent, { type: 'publication.requested' }>
|
|
4
5
|
|
|
6
|
+
/** One entry of the order-preserving publication ledger: the fields the
|
|
7
|
+
* publication selectors read from each relevant event, in array order. */
|
|
8
|
+
type LedgerItem =
|
|
9
|
+
| { t: 'request'; seq: number; event: PublicationRequest }
|
|
10
|
+
| { t: 'lost'; seq: number; request: number }
|
|
11
|
+
| { t: 'provisioned'; seq: number; ref: string }
|
|
12
|
+
| { t: 'released'; seq: number; ref?: string | null }
|
|
13
|
+
| { t: 'implement'; seq: number; round: number; base: string; head: string }
|
|
14
|
+
| { t: 'reconcile'; seq: number; mergeCommit: string }
|
|
15
|
+
| { t: 'finalize'; seq: number; headSha: string }
|
|
16
|
+
| { t: 'step'; seq: number; step: string; ok: boolean; headSha?: string }
|
|
17
|
+
/** A maximal run of consecutive irrelevant events. Only its max seq
|
|
18
|
+
* matters: the workspace scan's `seq > request.seq` break test. */
|
|
19
|
+
| { t: 'barrier'; seq: number }
|
|
20
|
+
|
|
21
|
+
/** The publication family's accumulator: the whole event sequence in array
|
|
22
|
+
* order, relevant events compacted and each run of irrelevant ones collapsed
|
|
23
|
+
* to a barrier. Filtering to relevant types alone would be wrong, because
|
|
24
|
+
* `publicationRequestSettled` stops its workspace scan at the first event (of
|
|
25
|
+
* any type) whose seq exceeds the request's. */
|
|
26
|
+
export interface PublicationLedger {
|
|
27
|
+
ledger: LedgerItem[]
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Bump when `PublicationLedger` or its fold changes. */
|
|
31
|
+
export const PUBLICATION_STATE_REDUCER_VERSION = 1
|
|
32
|
+
|
|
33
|
+
function compact(event: AbEvent): LedgerItem | undefined {
|
|
34
|
+
switch (event.type) {
|
|
35
|
+
case 'publication.requested':
|
|
36
|
+
return { t: 'request', seq: event.seq, event }
|
|
37
|
+
case 'publication.lost':
|
|
38
|
+
return { t: 'lost', seq: event.seq, request: event.payload.request }
|
|
39
|
+
case 'workspace.provisioned':
|
|
40
|
+
return { t: 'provisioned', seq: event.seq, ref: event.payload.ref }
|
|
41
|
+
case 'workspace.released': {
|
|
42
|
+
const payload = event.payload as { ref?: string }
|
|
43
|
+
return 'ref' in payload
|
|
44
|
+
? { t: 'released', seq: event.seq, ref: payload.ref ?? null }
|
|
45
|
+
: { t: 'released', seq: event.seq }
|
|
46
|
+
}
|
|
47
|
+
case 'implement.completed':
|
|
48
|
+
return {
|
|
49
|
+
t: 'implement',
|
|
50
|
+
seq: event.seq,
|
|
51
|
+
round: event.payload.round,
|
|
52
|
+
base: event.payload.commits.base,
|
|
53
|
+
head: event.payload.commits.head,
|
|
54
|
+
}
|
|
55
|
+
case 'reconcile.completed':
|
|
56
|
+
return { t: 'reconcile', seq: event.seq, mergeCommit: event.payload.mergeCommit }
|
|
57
|
+
case 'finalize.completed':
|
|
58
|
+
return { t: 'finalize', seq: event.seq, headSha: event.payload.pr.headSha }
|
|
59
|
+
case 'finalize.step-completed': {
|
|
60
|
+
const item: LedgerItem = {
|
|
61
|
+
t: 'step',
|
|
62
|
+
seq: event.seq,
|
|
63
|
+
step: event.payload.step,
|
|
64
|
+
ok: event.payload.ok,
|
|
65
|
+
}
|
|
66
|
+
const headSha = (event.payload as { headSha?: string }).headSha
|
|
67
|
+
if (headSha !== undefined) item.headSha = headSha
|
|
68
|
+
return item
|
|
69
|
+
}
|
|
70
|
+
default:
|
|
71
|
+
return undefined
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function itemCompletes(item: LedgerItem, request: PublicationRequest): boolean {
|
|
76
|
+
if (item.seq <= request.seq) return false
|
|
77
|
+
const payload = request.payload
|
|
78
|
+
if (payload.operation === 'implement')
|
|
79
|
+
return (
|
|
80
|
+
item.t === 'implement' &&
|
|
81
|
+
item.round === payload.round &&
|
|
82
|
+
item.base === payload.base &&
|
|
83
|
+
item.head === payload.sha
|
|
84
|
+
)
|
|
85
|
+
if (payload.operation === 'reconcile')
|
|
86
|
+
return item.t === 'reconcile' && item.mergeCommit === payload.sha
|
|
87
|
+
if (payload.operation === 'finalize') return item.t === 'finalize' && item.headSha === payload.sha
|
|
88
|
+
return (
|
|
89
|
+
item.t === 'step' && item.step === payload.step && (!item.ok || item.headSha === payload.sha)
|
|
90
|
+
)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The publication selectors, derived from one ledger. */
|
|
94
|
+
export interface PublicationView {
|
|
95
|
+
requestCompleted(request: PublicationRequest): boolean
|
|
96
|
+
requestSettled(request: PublicationRequest): boolean
|
|
97
|
+
latestUncompletedRequest(): PublicationRequest | undefined
|
|
98
|
+
lostRecorded(request: PublicationRequest): boolean
|
|
99
|
+
abandonedPending(): boolean
|
|
100
|
+
pending(): boolean
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function viewOf(ledger: readonly LedgerItem[]): PublicationView {
|
|
104
|
+
const requestCompleted = (request: PublicationRequest): boolean =>
|
|
105
|
+
ledger.some((item) => itemCompletes(item, request))
|
|
106
|
+
|
|
107
|
+
const requestSettled = (request: PublicationRequest): boolean => {
|
|
108
|
+
let workspaceRef: string | undefined
|
|
109
|
+
for (const item of ledger) {
|
|
110
|
+
if (item.seq > request.seq) break
|
|
111
|
+
if (item.t === 'provisioned') workspaceRef = item.ref
|
|
112
|
+
else if (item.t === 'released') workspaceRef = undefined
|
|
113
|
+
}
|
|
114
|
+
return (
|
|
115
|
+
requestCompleted(request) ||
|
|
116
|
+
ledger.some(
|
|
117
|
+
(item) =>
|
|
118
|
+
item.seq > request.seq &&
|
|
119
|
+
item.t === 'released' &&
|
|
120
|
+
(item.ref === undefined || workspaceRef === undefined || item.ref === workspaceRef),
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const latestUncompletedRequest = (): PublicationRequest | undefined => {
|
|
126
|
+
const item = ledger.findLast((entry) => entry.t === 'request' && !requestCompleted(entry.event))
|
|
127
|
+
return item?.t === 'request' ? item.event : undefined
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
requestCompleted,
|
|
132
|
+
requestSettled,
|
|
133
|
+
latestUncompletedRequest,
|
|
134
|
+
lostRecorded: (request) =>
|
|
135
|
+
ledger.some((item) => item.t === 'lost' && item.request === request.seq),
|
|
136
|
+
abandonedPending() {
|
|
137
|
+
const request = latestUncompletedRequest()
|
|
138
|
+
return request !== undefined && requestSettled(request)
|
|
139
|
+
},
|
|
140
|
+
pending: () => ledger.some((item) => item.t === 'request' && !requestSettled(item.event)),
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export const publicationStateReducer: IncrementalReducer<
|
|
145
|
+
PublicationLedger,
|
|
146
|
+
AbEvent,
|
|
147
|
+
PublicationView
|
|
148
|
+
> = defineReducer<PublicationLedger, AbEvent, PublicationView>({
|
|
149
|
+
version: PUBLICATION_STATE_REDUCER_VERSION,
|
|
150
|
+
initial: () => ({ ledger: [] }),
|
|
151
|
+
fold(acc, events) {
|
|
152
|
+
for (const event of events) {
|
|
153
|
+
const item = compact(event)
|
|
154
|
+
const last = acc.ledger[acc.ledger.length - 1]
|
|
155
|
+
if (item !== undefined) acc.ledger.push(item)
|
|
156
|
+
else if (last?.t === 'barrier') last.seq = Math.max(last.seq, event.seq)
|
|
157
|
+
else acc.ledger.push({ t: 'barrier', seq: event.seq })
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
finish: (acc) => viewOf(acc.ledger),
|
|
161
|
+
})
|
|
162
|
+
|
|
5
163
|
export function publicationRequestCompleted(
|
|
6
164
|
events: readonly AbEvent[],
|
|
7
165
|
request: PublicationRequest,
|
|
8
166
|
): boolean {
|
|
9
|
-
return events.
|
|
10
|
-
if (event.seq <= request.seq) return false
|
|
11
|
-
if (request.payload.operation === 'implement')
|
|
12
|
-
return (
|
|
13
|
-
event.type === 'implement.completed' &&
|
|
14
|
-
event.payload.round === request.payload.round &&
|
|
15
|
-
event.payload.commits.base === request.payload.base &&
|
|
16
|
-
event.payload.commits.head === request.payload.sha
|
|
17
|
-
)
|
|
18
|
-
if (request.payload.operation === 'reconcile')
|
|
19
|
-
return (
|
|
20
|
-
event.type === 'reconcile.completed' && event.payload.mergeCommit === request.payload.sha
|
|
21
|
-
)
|
|
22
|
-
if (request.payload.operation === 'finalize')
|
|
23
|
-
return event.type === 'finalize.completed' && event.payload.pr.headSha === request.payload.sha
|
|
24
|
-
return (
|
|
25
|
-
event.type === 'finalize.step-completed' &&
|
|
26
|
-
event.payload.step === request.payload.step &&
|
|
27
|
-
(!event.payload.ok || event.payload.headSha === request.payload.sha)
|
|
28
|
-
)
|
|
29
|
-
})
|
|
167
|
+
return publicationStateReducer.reduce(events).requestCompleted(request)
|
|
30
168
|
}
|
|
31
169
|
|
|
32
170
|
/** A request is no longer publishable after its workspace is released. The
|
|
@@ -36,23 +174,7 @@ export function publicationRequestSettled(
|
|
|
36
174
|
events: readonly AbEvent[],
|
|
37
175
|
request: PublicationRequest,
|
|
38
176
|
): boolean {
|
|
39
|
-
|
|
40
|
-
for (const event of events) {
|
|
41
|
-
if (event.seq > request.seq) break
|
|
42
|
-
if (event.type === 'workspace.provisioned') workspaceRef = event.payload.ref
|
|
43
|
-
else if (event.type === 'workspace.released') workspaceRef = undefined
|
|
44
|
-
}
|
|
45
|
-
return (
|
|
46
|
-
publicationRequestCompleted(events, request) ||
|
|
47
|
-
events.some(
|
|
48
|
-
(event) =>
|
|
49
|
-
event.seq > request.seq &&
|
|
50
|
-
event.type === 'workspace.released' &&
|
|
51
|
-
(!('ref' in event.payload) ||
|
|
52
|
-
workspaceRef === undefined ||
|
|
53
|
-
event.payload.ref === workspaceRef),
|
|
54
|
-
)
|
|
55
|
-
)
|
|
177
|
+
return publicationStateReducer.reduce(events).requestSettled(request)
|
|
56
178
|
}
|
|
57
179
|
|
|
58
180
|
/** The latest `publication.requested` with no settling completion fact — the
|
|
@@ -61,11 +183,7 @@ export function publicationRequestSettled(
|
|
|
61
183
|
export function latestUncompletedPublicationRequest(
|
|
62
184
|
events: readonly AbEvent[],
|
|
63
185
|
): PublicationRequest | undefined {
|
|
64
|
-
|
|
65
|
-
(event) =>
|
|
66
|
-
event.type === 'publication.requested' && !publicationRequestCompleted(events, event),
|
|
67
|
-
)
|
|
68
|
-
return request?.type === 'publication.requested' ? request : undefined
|
|
186
|
+
return publicationStateReducer.reduce(events).latestUncompletedRequest()
|
|
69
187
|
}
|
|
70
188
|
|
|
71
189
|
/** Whether the durable loss record (AUT-328) has already been appended for
|
|
@@ -75,19 +193,13 @@ export function publicationLostRecorded(
|
|
|
75
193
|
events: readonly AbEvent[],
|
|
76
194
|
request: PublicationRequest,
|
|
77
195
|
): boolean {
|
|
78
|
-
return events.
|
|
79
|
-
(event) => event.type === 'publication.lost' && event.payload.request === request.seq,
|
|
80
|
-
)
|
|
196
|
+
return publicationStateReducer.reduce(events).lostRecorded(request)
|
|
81
197
|
}
|
|
82
198
|
|
|
83
199
|
export function abandonedPublicationPending(events: readonly AbEvent[]): boolean {
|
|
84
|
-
|
|
85
|
-
return request !== undefined && publicationRequestSettled(events, request)
|
|
200
|
+
return publicationStateReducer.reduce(events).abandonedPending()
|
|
86
201
|
}
|
|
87
202
|
|
|
88
203
|
export function publicationPending(events: readonly AbEvent[]): boolean {
|
|
89
|
-
return events.
|
|
90
|
-
(request) =>
|
|
91
|
-
request.type === 'publication.requested' && !publicationRequestSettled(events, request),
|
|
92
|
-
)
|
|
204
|
+
return publicationStateReducer.reduce(events).pending()
|
|
93
205
|
}
|