@defrex/autobuild 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/SPEC.md +19 -1
- package/bin/ab-dispatch-kernel.ts +4 -6
- package/docs/architecture.md +96 -70
- package/docs/hosted-service.md +2 -2
- package/docs/operator-api.md +1 -1
- package/docs/remote-store-protocol.md +84 -2
- package/package.json +4 -3
- package/packages/core/src/cli/attention.ts +18 -164
- package/packages/core/src/cli/build-control.ts +11 -15
- package/packages/core/src/cli/bulk-control.ts +3 -2
- package/packages/core/src/cli/context.ts +16 -78
- package/packages/core/src/cli/dashboard/model.ts +36 -22
- package/packages/core/src/cli/dispatch-frontend.ts +24 -67
- package/packages/core/src/cli/dispatch-process.ts +14 -7
- package/packages/core/src/cli/dispatch-repository-feed.ts +196 -0
- package/packages/core/src/cli/dispatch.ts +148 -79
- package/packages/core/src/cli/dispatcher-liveness.ts +26 -19
- package/packages/core/src/cli/harvest.ts +71 -39
- package/packages/core/src/cli/historical-build-state.ts +27 -0
- package/packages/core/src/cli/main.ts +3 -3
- package/packages/core/src/cli/pr-attachments.ts +10 -13
- package/packages/core/src/cli/pr-summary-publication.ts +27 -60
- package/packages/core/src/cli/pr-summary.ts +43 -13
- package/packages/core/src/cli/publication-settlement.ts +10 -20
- package/packages/core/src/cli/repository-status.ts +38 -2
- package/packages/core/src/cli/status.ts +62 -32
- package/packages/core/src/cli/terminals.ts +25 -142
- package/packages/core/src/cli/ticket.ts +2 -2
- package/packages/core/src/cli/wait.ts +91 -84
- package/packages/core/src/cli/watch.ts +60 -97
- package/packages/core/src/config/pipeline-source.ts +4 -3
- package/packages/core/src/events/repository.ts +7 -3
- package/packages/core/src/integration/harness.ts +1 -2
- package/packages/core/src/kernel/__snapshots__/reducer-registry.test.ts.snap +2429 -9
- package/packages/core/src/kernel/auto-merge.ts +19 -3
- package/packages/core/src/kernel/dispatcher-liveness.ts +10 -4
- package/packages/core/src/kernel/generators/repository-journals.ts +14 -3
- package/packages/core/src/kernel/harvest.ts +25 -23
- package/packages/core/src/kernel/log-index-value.ts +38 -0
- package/packages/core/src/kernel/read-facts.ts +637 -0
- package/packages/core/src/kernel/reducer-registry.ts +137 -3
- package/packages/core/src/kernel/repository-attention.ts +179 -0
- package/packages/core/src/kernel/service-facts.ts +135 -0
- package/packages/core/src/kernel/terminal-guard.ts +133 -0
- package/packages/core/src/operator/control.ts +5 -6
- package/packages/core/src/operator/index.ts +2 -0
- package/packages/core/src/operator/query.ts +37 -20
- package/packages/core/src/operator/read-view.ts +4 -13
- package/packages/core/src/operator/requests.ts +2 -14
- package/packages/core/src/operator/sandbox.ts +10 -9
- package/packages/core/src/operator/tickets.ts +2 -2
- package/packages/core/src/plugin-sdk/index.ts +13 -0
- package/packages/core/src/processes/build-child.ts +6 -6
- package/packages/core/src/processes/build-runner.ts +62 -70
- package/packages/core/src/processes/dispatch-failure-budget.ts +5 -4
- package/packages/core/src/processes/dispatcher.ts +113 -173
- package/packages/core/src/processes/harvest-execution-state.ts +15 -9
- package/packages/core/src/processes/harvest-runner.ts +34 -70
- package/packages/core/src/processes/harvest.ts +38 -15
- package/packages/core/src/processes/infrastructure-failure-budget.ts +13 -15
- package/packages/core/src/processes/orchestrator-tick.ts +2 -2
- package/packages/core/src/processes/publication-loss.ts +35 -21
- package/packages/core/src/processes/repo-view.ts +229 -1134
- package/packages/core/src/processes/tick-publication.ts +110 -0
- package/packages/core/src/store/adapter.ts +23 -0
- package/packages/core/src/store/build-scope.ts +17 -0
- package/packages/core/src/store/contract.ts +251 -0
- package/packages/core/src/store/local/schema.ts +1 -0
- package/packages/core/src/store/local/store.ts +179 -54
- package/packages/core/src/store/memory.ts +119 -29
- package/packages/core/src/store/phase-session.ts +17 -0
- package/packages/core/src/store/reduced-reads.ts +239 -0
- package/packages/core/src/store/reducer-value-shapes.json +14056 -0
- package/packages/core/src/store/reducer-value-validation.ts +96 -0
- package/packages/core/src/store/reducer-values.ts +145 -0
- package/packages/core/src/store/remote/client.ts +41 -0
- package/packages/core/src/store/remote/compatibility.ts +2 -0
- package/packages/core/src/store/remote/index.ts +19 -0
- package/packages/core/src/store/remote/protocol.ts +16 -0
- package/packages/core/src/store/remote/version.ts +3 -1
- package/packages/core/src/store/repo-state-events.ts +1 -1
- package/packages/core/src/store/repo-tick.ts +72 -0
- package/packages/core/src/store/session-handle.ts +19 -0
- package/packages/core/src/store/snapshots.ts +10 -0
- package/packages/core/src/store/ticket-assets.ts +4 -4
- package/packages/core/src/store/types.ts +39 -2
- package/packages/core/src/testing/dispatch-read-budget.ts +215 -0
- package/packages/core/src/testing/index.ts +8 -0
- package/packages/core/src/testing/reduced-read-budget.ts +462 -0
- package/skills/guide/SKILL.md +6 -1
- package/skills/guide/references/operator-api.md +1 -1
- package/skills/guide/references/remote-store-protocol.md +84 -2
- package/templates/hosted-service/package.json +2 -2
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { type RepositoryEvent, validateRepositoryEventWrite } from '../events/repository'
|
|
2
|
+
import { defineReducer } from '../kernel/incremental'
|
|
3
|
+
import { loadReducerSnapshot, persistReducerSnapshot } from '../store/snapshots'
|
|
4
|
+
import type { BuildStore, Clock } from '../store/types'
|
|
5
|
+
|
|
6
|
+
export function isDashboardRepositoryEvent(event: RepositoryEvent): boolean {
|
|
7
|
+
return (
|
|
8
|
+
event.type.startsWith('harvest.') ||
|
|
9
|
+
event.type === 'dispatcher.intake-set' ||
|
|
10
|
+
event.type === 'dispatcher.pause-set' ||
|
|
11
|
+
event.type === 'dispatcher.auto-merge-default-set'
|
|
12
|
+
)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export const dispatchFrontendReducer = defineReducer({
|
|
16
|
+
version: 2,
|
|
17
|
+
initial: (): { retained: RepositoryEvent[]; incompleteIdentities: boolean } => ({
|
|
18
|
+
retained: [],
|
|
19
|
+
incompleteIdentities: true,
|
|
20
|
+
}),
|
|
21
|
+
fold(acc, events: readonly RepositoryEvent[]) {
|
|
22
|
+
const facts = new Map(acc.retained.map((event) => [event.seq, event]))
|
|
23
|
+
for (const event of events) {
|
|
24
|
+
if (isDashboardRepositoryEvent(event) || event.type === 'dispatcher.run-started')
|
|
25
|
+
facts.set(event.seq, event)
|
|
26
|
+
}
|
|
27
|
+
acc.retained = [...facts.values()].sort((a, b) => a.seq - b.seq)
|
|
28
|
+
},
|
|
29
|
+
finish: (acc) => acc.retained,
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
/** One serialized, frontend-only repository projection. The consumed cursor is
|
|
33
|
+
* independent of retained facts: quiet ticks must never survive in the cache. */
|
|
34
|
+
export class DispatchRepositoryFeed {
|
|
35
|
+
private cursor = 0
|
|
36
|
+
private ready = false
|
|
37
|
+
private retained: RepositoryEvent[] = []
|
|
38
|
+
private tail: Promise<void> = Promise.resolve()
|
|
39
|
+
private savedAt = -Infinity
|
|
40
|
+
private boundary: number | undefined
|
|
41
|
+
private recovery = false
|
|
42
|
+
incompleteIdentities = true
|
|
43
|
+
|
|
44
|
+
constructor(
|
|
45
|
+
private readonly store: BuildStore,
|
|
46
|
+
private readonly repo: string,
|
|
47
|
+
private readonly clock: Clock,
|
|
48
|
+
) {}
|
|
49
|
+
|
|
50
|
+
get events(): RepositoryEvent[] {
|
|
51
|
+
return this.retained.filter(isDashboardRepositoryEvent)
|
|
52
|
+
}
|
|
53
|
+
get identities(): RepositoryEvent[] {
|
|
54
|
+
return this.retained.filter((e) => e.type === 'dispatcher.run-started')
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
private validate(events: RepositoryEvent[], cursor: number, contiguous: boolean): void {
|
|
58
|
+
let previous = cursor
|
|
59
|
+
for (const event of events) {
|
|
60
|
+
if (
|
|
61
|
+
event.repo !== this.repo ||
|
|
62
|
+
!Number.isInteger(event.seq) ||
|
|
63
|
+
event.seq <= previous ||
|
|
64
|
+
(contiguous && event.seq !== previous + 1) ||
|
|
65
|
+
typeof event.ts !== 'string' ||
|
|
66
|
+
!Number.isFinite(Date.parse(event.ts))
|
|
67
|
+
) {
|
|
68
|
+
throw new Error('invalid dashboard repository delta')
|
|
69
|
+
}
|
|
70
|
+
validateRepositoryEventWrite(event)
|
|
71
|
+
previous = event.seq
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async warm(): Promise<RepositoryEvent[]> {
|
|
76
|
+
this.recovery = true
|
|
77
|
+
this.boundary = await this.store.getRepoHighWater(this.repo)
|
|
78
|
+
this.recovery = false
|
|
79
|
+
try {
|
|
80
|
+
return await this.refresh()
|
|
81
|
+
} catch (error) {
|
|
82
|
+
this.recovery = true
|
|
83
|
+
throw error
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
refresh(persist = true): Promise<RepositoryEvent[]> {
|
|
88
|
+
let result: RepositoryEvent[] = []
|
|
89
|
+
const next = this.tail.then(async () => {
|
|
90
|
+
let cursor = this.cursor
|
|
91
|
+
let retained = this.retained
|
|
92
|
+
let incompleteIdentities = this.incompleteIdentities
|
|
93
|
+
let delta: RepositoryEvent[]
|
|
94
|
+
if (!this.ready) {
|
|
95
|
+
if ((await this.store.getRepo(this.repo)) === null) {
|
|
96
|
+
this.ready = true
|
|
97
|
+
this.recovery = false
|
|
98
|
+
return
|
|
99
|
+
}
|
|
100
|
+
const high = await this.store.getRepoHighWater(this.repo)
|
|
101
|
+
const snapshot = await loadReducerSnapshot(
|
|
102
|
+
this.store,
|
|
103
|
+
{ kind: 'repo', repo: this.repo },
|
|
104
|
+
'dispatchFrontend',
|
|
105
|
+
dispatchFrontendReducer,
|
|
106
|
+
)
|
|
107
|
+
let restored = false
|
|
108
|
+
if (snapshot && snapshot.cursor <= high) {
|
|
109
|
+
try {
|
|
110
|
+
const state = snapshot.state as {
|
|
111
|
+
retained: RepositoryEvent[]
|
|
112
|
+
incompleteIdentities: boolean
|
|
113
|
+
}
|
|
114
|
+
if (!Array.isArray(state.retained) || typeof state.incompleteIdentities !== 'boolean')
|
|
115
|
+
throw new Error('invalid projection')
|
|
116
|
+
this.validate(state.retained, 0, false)
|
|
117
|
+
if (
|
|
118
|
+
state.retained.some(
|
|
119
|
+
(e) =>
|
|
120
|
+
e.seq > snapshot.cursor ||
|
|
121
|
+
!(isDashboardRepositoryEvent(e) || e.type === 'dispatcher.run-started'),
|
|
122
|
+
)
|
|
123
|
+
)
|
|
124
|
+
throw new Error('invalid retained fact')
|
|
125
|
+
incompleteIdentities = state.incompleteIdentities
|
|
126
|
+
retained = state.retained
|
|
127
|
+
cursor = snapshot.cursor
|
|
128
|
+
restored = true
|
|
129
|
+
} catch {
|
|
130
|
+
/* A malformed cache uses the bounded seed. */
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (!restored) {
|
|
134
|
+
const seed = await this.store.getRepoStateEvents(this.repo)
|
|
135
|
+
this.validate(seed, 0, false)
|
|
136
|
+
incompleteIdentities = true
|
|
137
|
+
retained = dispatchFrontendReducer.reduce(seed)
|
|
138
|
+
cursor = Math.max(high, seed.at(-1)?.seq ?? 0)
|
|
139
|
+
}
|
|
140
|
+
delta = await this.store.getRepoEvents(this.repo, cursor)
|
|
141
|
+
this.validate(delta, cursor, true)
|
|
142
|
+
// A failed prelaunch seed must not skip this invocation's config/start
|
|
143
|
+
// even if a competing run advanced the bounded seed's anchor.
|
|
144
|
+
if (this.recovery) {
|
|
145
|
+
const recovery = await this.store.getRepoEvents(this.repo, this.boundary ?? 0)
|
|
146
|
+
this.validate(recovery, this.boundary ?? 0, true)
|
|
147
|
+
delta = [...new Map([...delta, ...recovery].map((e) => [e.seq, e])).values()].sort(
|
|
148
|
+
(a, b) => a.seq - b.seq,
|
|
149
|
+
)
|
|
150
|
+
}
|
|
151
|
+
} else {
|
|
152
|
+
delta = await this.store.getRepoEvents(this.repo, cursor)
|
|
153
|
+
this.validate(delta, cursor, true)
|
|
154
|
+
// A standby correlation may have supplied an older identity while the
|
|
155
|
+
// delta read was pending. Keep those memoized facts in this commit.
|
|
156
|
+
retained = this.retained
|
|
157
|
+
incompleteIdentities = this.incompleteIdentities
|
|
158
|
+
}
|
|
159
|
+
retained = dispatchFrontendReducer.advance({ retained, incompleteIdentities }, delta).retained
|
|
160
|
+
this.cursor = Math.max(cursor, delta.at(-1)?.seq ?? 0)
|
|
161
|
+
this.incompleteIdentities = incompleteIdentities
|
|
162
|
+
this.retained = retained
|
|
163
|
+
this.ready = true
|
|
164
|
+
this.recovery = false
|
|
165
|
+
result = delta
|
|
166
|
+
if (persist) await this.save()
|
|
167
|
+
})
|
|
168
|
+
this.tail = next.catch(() => {})
|
|
169
|
+
return next.then(() => result)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
remember(events: readonly RepositoryEvent[], complete = false): void {
|
|
173
|
+
this.retained = dispatchFrontendReducer.advance(
|
|
174
|
+
{ retained: this.retained, incompleteIdentities: this.incompleteIdentities },
|
|
175
|
+
events.filter((e) => e.seq <= this.cursor),
|
|
176
|
+
).retained
|
|
177
|
+
if (complete) this.incompleteIdentities = false
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async save(force = false): Promise<void> {
|
|
181
|
+
if (!this.ready || (!force && this.clock().getTime() - this.savedAt < 5000)) return
|
|
182
|
+
// A rejected equal-cursor put and an unavailable store still cost a
|
|
183
|
+
// request. Bound attempts, not just successful writes, on quiet polls.
|
|
184
|
+
this.savedAt = this.clock().getTime()
|
|
185
|
+
await persistReducerSnapshot(
|
|
186
|
+
this.store,
|
|
187
|
+
{ kind: 'repo', repo: this.repo },
|
|
188
|
+
'dispatchFrontend',
|
|
189
|
+
{
|
|
190
|
+
version: dispatchFrontendReducer.version,
|
|
191
|
+
cursor: this.cursor,
|
|
192
|
+
state: { retained: this.retained, incompleteIdentities: this.incompleteIdentities },
|
|
193
|
+
},
|
|
194
|
+
)
|
|
195
|
+
}
|
|
196
|
+
}
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
import type { BuildState } from '../kernel/reducer'
|
|
2
|
+
import {
|
|
3
|
+
readBuildFacts,
|
|
4
|
+
readRepositoryFacts,
|
|
5
|
+
readBuildSelector,
|
|
6
|
+
readDispatcherRunFacts,
|
|
7
|
+
} from '../store/reduced-reads'
|
|
8
|
+
import { shouldPublishTickFromFacts } from '../processes/tick-publication'
|
|
1
9
|
import { issueTicketGrant, ticketGrantRequest } from '../ports/tickets/grants'
|
|
2
10
|
import { RemoteBuildStore } from '../store/remote/client'
|
|
3
11
|
import { delegateCredential } from '../store/remote/credentials'
|
|
@@ -54,17 +62,14 @@ import { loadPlugins, pluginResolutionKind } from '../plugins/load'
|
|
|
54
62
|
import type { PluginRegistry } from '../plugins/registry'
|
|
55
63
|
import { materializePluginRuntimes } from '../plugins/runtimes'
|
|
56
64
|
import { DISPATCHER, KERNEL, humanActor } from '../events/envelope'
|
|
57
|
-
import type { AbEvent } from '../events/catalog'
|
|
58
65
|
import type { RepositoryEventWrite } from '../events/repository'
|
|
59
66
|
import { randomIds, randomUuids, type IdSource, type UuidSource } from '../ids'
|
|
60
|
-
import { reduceDispatchSettings } from '../kernel/dispatch-settings'
|
|
67
|
+
import type { reduceDispatchSettings } from '../kernel/dispatch-settings'
|
|
61
68
|
import {
|
|
62
69
|
DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS,
|
|
63
70
|
actionableHarvestRun,
|
|
64
71
|
decideHarvestControl,
|
|
65
|
-
reduceHarvest,
|
|
66
72
|
} from '../kernel/harvest'
|
|
67
|
-
import { reduceBuild } from '../kernel/reducer'
|
|
68
73
|
import { dashboardBuildControl } from './dashboard/actions'
|
|
69
74
|
import {
|
|
70
75
|
buildDashboardFromProjected,
|
|
@@ -127,12 +132,10 @@ import {
|
|
|
127
132
|
parseBuildConfigMetadata,
|
|
128
133
|
parseDiagnostic,
|
|
129
134
|
parseEffectiveBuildConfig,
|
|
130
|
-
selectOpenWorkspace,
|
|
131
135
|
} from '../processes/build-execution-state'
|
|
132
136
|
import { HarvestRunner, type HarvestRunnerResult } from '../processes/harvest-runner'
|
|
133
137
|
import {
|
|
134
138
|
sampleUnclaimedObservationCount,
|
|
135
|
-
readRepoEventsIfRecorded,
|
|
136
139
|
evaluateHarvestPressureFromStore,
|
|
137
140
|
} from '../processes/harvest'
|
|
138
141
|
import { classifyHarvestOutcome } from '../processes/harvest-execution-state'
|
|
@@ -164,7 +167,13 @@ import {
|
|
|
164
167
|
} from './repo-state'
|
|
165
168
|
import { openStoreForRepoState } from './store-opening'
|
|
166
169
|
import { DispatchFrontend } from './dispatch-frontend'
|
|
167
|
-
import {
|
|
170
|
+
import {
|
|
171
|
+
systemClock,
|
|
172
|
+
type BuildStore,
|
|
173
|
+
type Clock,
|
|
174
|
+
type RepoTickCompletion,
|
|
175
|
+
type RepoTickFinish,
|
|
176
|
+
} from '../store/types'
|
|
168
177
|
import { availableRelease } from './self-update'
|
|
169
178
|
import {
|
|
170
179
|
startUpgradeNotice,
|
|
@@ -648,6 +657,7 @@ class DispatchLoop {
|
|
|
648
657
|
* manufacturing a zero queue measurement or losing diagnostics. */
|
|
649
658
|
private lastTickStatus = {
|
|
650
659
|
queued: 0,
|
|
660
|
+
blockedDiagnostics: [] as string[],
|
|
651
661
|
janitorDiagnostics: [] as string[],
|
|
652
662
|
ticketDiagnostics: [] as string[],
|
|
653
663
|
creationDiagnostics: [] as string[],
|
|
@@ -850,7 +860,7 @@ class DispatchLoop {
|
|
|
850
860
|
private async resolveBuildPipeline(slug: string): Promise<{
|
|
851
861
|
source: Awaited<ReturnType<typeof resolvePipelineSource>>
|
|
852
862
|
failures: PipelineSourceFailure[]
|
|
853
|
-
|
|
863
|
+
observations: BuildState['observations']
|
|
854
864
|
}> {
|
|
855
865
|
const failures: PipelineSourceFailure[] = []
|
|
856
866
|
try {
|
|
@@ -858,12 +868,12 @@ class DispatchLoop {
|
|
|
858
868
|
// the record's `branch` (when present) is the same fact. Using the event
|
|
859
869
|
// keeps this read to the events the caller already needs and avoids an
|
|
860
870
|
// extra build-record lookup on the hot launch path.
|
|
861
|
-
const
|
|
862
|
-
const workspace =
|
|
871
|
+
const facts = (await readBuildFacts(this.repoView, slug)).value
|
|
872
|
+
const workspace = facts.workspace
|
|
863
873
|
const source = await resolvePipelineSource({
|
|
864
874
|
slug,
|
|
865
875
|
record: { ...(workspace !== null ? { branch: workspace.branch } : {}) },
|
|
866
|
-
|
|
876
|
+
recordedBase: await readBuildSelector(this.repoView, slug, 'recordedBaseSha'),
|
|
867
877
|
mode: this.opts.repository !== undefined ? 'origin' : 'checkout',
|
|
868
878
|
forge: this.wiring.forge,
|
|
869
879
|
checkout: this.opts.targetRepo,
|
|
@@ -871,13 +881,13 @@ class DispatchLoop {
|
|
|
871
881
|
...(workspace !== null ? { workspacePath: workspace.path } : {}),
|
|
872
882
|
onFailure: (failure) => failures.push(failure),
|
|
873
883
|
})
|
|
874
|
-
return { source, failures,
|
|
884
|
+
return { source, failures, observations: facts.observations }
|
|
875
885
|
} catch (error) {
|
|
876
886
|
failures.push({
|
|
877
887
|
kind: 'store',
|
|
878
888
|
detail: `build-event read failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
879
889
|
})
|
|
880
|
-
return { source: undefined, failures,
|
|
890
|
+
return { source: undefined, failures, observations: [] }
|
|
881
891
|
}
|
|
882
892
|
}
|
|
883
893
|
|
|
@@ -901,7 +911,7 @@ class DispatchLoop {
|
|
|
901
911
|
* naming the failed attempts; the documented fallbacks (an unpublished
|
|
902
912
|
* branch, an unreadable legacy base) stay quiet. */
|
|
903
913
|
private async publishBuildConfig(slug: string, snapshot = this.currentConfig()): Promise<void> {
|
|
904
|
-
const { source, failures,
|
|
914
|
+
const { source, failures, observations } = await this.resolveBuildPipeline(slug)
|
|
905
915
|
if (source !== undefined) {
|
|
906
916
|
await this.depositBuildConfig(
|
|
907
917
|
slug,
|
|
@@ -931,7 +941,7 @@ class DispatchLoop {
|
|
|
931
941
|
// option left that keeps the launch alive.
|
|
932
942
|
await this.depositBuildConfig(slug, snapshot, snapshot.config)
|
|
933
943
|
}
|
|
934
|
-
await this.filePipelineFallbackObservation(slug, reused, failures,
|
|
944
|
+
await this.filePipelineFallbackObservation(slug, reused, failures, observations)
|
|
935
945
|
}
|
|
936
946
|
|
|
937
947
|
/** File a build-scoped observation when a deposit hit an actionable
|
|
@@ -944,7 +954,7 @@ class DispatchLoop {
|
|
|
944
954
|
slug: string,
|
|
945
955
|
reused: PipelineSourceMeta | undefined,
|
|
946
956
|
failures: PipelineSourceFailure[],
|
|
947
|
-
|
|
957
|
+
observations: BuildState['observations'],
|
|
948
958
|
): Promise<void> {
|
|
949
959
|
const actionable = failures.filter(isActionablePipelineFailure)
|
|
950
960
|
if (actionable.length === 0) return
|
|
@@ -961,11 +971,7 @@ class DispatchLoop {
|
|
|
961
971
|
}`
|
|
962
972
|
// Deduplicated against the build's log: a parked build re-attaches every
|
|
963
973
|
// sweep, and an outage must not harvest one identical diagnostic per tick.
|
|
964
|
-
if (
|
|
965
|
-
events.some(
|
|
966
|
-
(event) => event.type === 'observation.recorded' && event.payload.summary === summary,
|
|
967
|
-
)
|
|
968
|
-
) {
|
|
974
|
+
if (observations.some((observation) => observation.summary === summary)) {
|
|
969
975
|
return
|
|
970
976
|
}
|
|
971
977
|
try {
|
|
@@ -1106,9 +1112,11 @@ class DispatchLoop {
|
|
|
1106
1112
|
// The view's bounded subset (AUT-489): the settings reducer consumes
|
|
1107
1113
|
// durable types only. `refresh()` ran first in the tick, so this is the
|
|
1108
1114
|
// delta-current journal, never a store read.
|
|
1109
|
-
return
|
|
1115
|
+
return this.repoView.repositoryFacts().settings
|
|
1110
1116
|
}
|
|
1111
1117
|
|
|
1118
|
+
private tickStartedAt: string | undefined
|
|
1119
|
+
|
|
1112
1120
|
private dispatcherTick(resumeCurrent: boolean): Promise<Awaited<ReturnType<Dispatcher['tick']>>> {
|
|
1113
1121
|
return this.serialize(async () => {
|
|
1114
1122
|
// Refresh before every watch decision. The owner publishes atomically;
|
|
@@ -1121,11 +1129,12 @@ class DispatchLoop {
|
|
|
1121
1129
|
await this.recordTickYielded()
|
|
1122
1130
|
return emptyTickReport()
|
|
1123
1131
|
}
|
|
1124
|
-
await this.
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
}
|
|
1132
|
+
const startedAt = await this.repoView.beginRepoTick(this.repoIdentity, this.runId)
|
|
1133
|
+
if (startedAt === null) {
|
|
1134
|
+
this.superseded = true
|
|
1135
|
+
return emptyTickReport()
|
|
1136
|
+
}
|
|
1137
|
+
this.tickStartedAt = startedAt
|
|
1129
1138
|
|
|
1130
1139
|
// Unclaimed observations are display-only and sampled once per interactive
|
|
1131
1140
|
// dispatcher tick from the journal and one repo-scoped digest read
|
|
@@ -1174,6 +1183,7 @@ class DispatchLoop {
|
|
|
1174
1183
|
this.queuedCount = publishedReport.queued
|
|
1175
1184
|
this.lastTickStatus = {
|
|
1176
1185
|
queued: publishedReport.queued,
|
|
1186
|
+
blockedDiagnostics: [...publishedReport.blockedDiagnostics],
|
|
1177
1187
|
janitorDiagnostics: [...publishedReport.janitorDiagnostics],
|
|
1178
1188
|
ticketDiagnostics: [...publishedReport.ticketDiagnostics],
|
|
1179
1189
|
creationDiagnostics: [...publishedReport.creationDiagnostics],
|
|
@@ -1610,8 +1620,8 @@ class DispatchLoop {
|
|
|
1610
1620
|
const { store } = this.wiring
|
|
1611
1621
|
const repo = this.repoIdentity
|
|
1612
1622
|
await store.ensureRepo(repo)
|
|
1613
|
-
const events = await store
|
|
1614
|
-
const state =
|
|
1623
|
+
const events = (await readRepositoryFacts(store, repo)).value
|
|
1624
|
+
const state = events.harvest
|
|
1615
1625
|
const projected = projectHarvest(events)
|
|
1616
1626
|
if (expectedRun === undefined || projected === undefined || projected.run !== expectedRun) {
|
|
1617
1627
|
this.say('harvest run action ignored: selected run is no longer active')
|
|
@@ -2502,8 +2512,8 @@ class DispatchLoop {
|
|
|
2502
2512
|
// reduces durable types only. Take the journal delta first — guests and
|
|
2503
2513
|
// other dispatchers write harvest facts between ticks.
|
|
2504
2514
|
await this.repoView.refreshJournal()
|
|
2505
|
-
const events = this.repoView.
|
|
2506
|
-
const state =
|
|
2515
|
+
const events = this.repoView.repositoryFacts().harvest
|
|
2516
|
+
const state = events
|
|
2507
2517
|
const control = decideHarvestControl(state, this.maxHarvestRecoveryAttempts)
|
|
2508
2518
|
const resumePending =
|
|
2509
2519
|
actionableHarvestRun(state) !== undefined ||
|
|
@@ -2692,7 +2702,7 @@ class DispatchLoop {
|
|
|
2692
2702
|
// Bounded subset (AUT-489): classifyHarvestOutcome reduces harvest facts
|
|
2693
2703
|
// only. A guest wrote the facts through its own handle, so read the delta.
|
|
2694
2704
|
await this.repoView.refreshJournal()
|
|
2695
|
-
const events = this.repoView.
|
|
2705
|
+
const events = this.repoView.repositoryFacts()
|
|
2696
2706
|
const record = await store.getRepo(repo)
|
|
2697
2707
|
return classifyHarvestOutcome(events, {
|
|
2698
2708
|
executionStartedSeq: startedSeq,
|
|
@@ -2831,15 +2841,38 @@ class DispatchLoop {
|
|
|
2831
2841
|
dependencyDiagnostics: _dependencyDiagnostics,
|
|
2832
2842
|
...counters
|
|
2833
2843
|
} = merged
|
|
2834
|
-
|
|
2844
|
+
const completed = {
|
|
2835
2845
|
actor: DISPATCHER,
|
|
2836
|
-
type: 'dispatcher.tick-completed',
|
|
2837
|
-
payload: {
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2846
|
+
type: 'dispatcher.tick-completed' as const,
|
|
2847
|
+
payload: { run: this.runId, ...this.lastTickStatus, counters },
|
|
2848
|
+
}
|
|
2849
|
+
if (settlementOnly) {
|
|
2850
|
+
await this.appendStatus(completed)
|
|
2851
|
+
} else {
|
|
2852
|
+
const startedAt = this.tickStartedAt!
|
|
2853
|
+
const facts = this.repoView.repositoryFacts()
|
|
2854
|
+
const recent = facts.recentRuns.find(([run]) => run === this.runId)?.[1]
|
|
2855
|
+
const runFacts = recent?.start
|
|
2856
|
+
? recent
|
|
2857
|
+
: (await readDispatcherRunFacts(this.repoView, this.repoIdentity, this.runId)).value
|
|
2858
|
+
const publish = shouldPublishTickFromFacts(merged, facts, runFacts, startedAt)
|
|
2859
|
+
const events: RepositoryEventWrite[] = publish
|
|
2860
|
+
? [
|
|
2861
|
+
{
|
|
2862
|
+
actor: DISPATCHER,
|
|
2863
|
+
type: 'dispatcher.tick-started',
|
|
2864
|
+
payload: { run: this.runId, startedAt },
|
|
2865
|
+
},
|
|
2866
|
+
completed,
|
|
2867
|
+
]
|
|
2868
|
+
: []
|
|
2869
|
+
const result = await this.finishTick({ outcome: 'completed', counters }, events)
|
|
2870
|
+
if (result === null) {
|
|
2871
|
+
this.superseded = true
|
|
2872
|
+
throw new Error('repository tick lease lost')
|
|
2873
|
+
}
|
|
2874
|
+
this.tickStartedAt = undefined
|
|
2875
|
+
}
|
|
2843
2876
|
|
|
2844
2877
|
this.pendingHarvest.harvestStarted -= snapshot.harvestStarted
|
|
2845
2878
|
this.pendingHarvest.harvestResumed -= snapshot.harvestResumed
|
|
@@ -2849,10 +2882,72 @@ class DispatchLoop {
|
|
|
2849
2882
|
return merged
|
|
2850
2883
|
}
|
|
2851
2884
|
|
|
2885
|
+
/** A remote finish can commit before its response is lost. Confirm that
|
|
2886
|
+
* exact tick through durable telemetry before acknowledging its counters;
|
|
2887
|
+
* never try to close an already committed boundary as a second failure. */
|
|
2888
|
+
private async finishTick(
|
|
2889
|
+
completion: RepoTickCompletion,
|
|
2890
|
+
events: RepositoryEventWrite[],
|
|
2891
|
+
): Promise<RepoTickFinish | null> {
|
|
2892
|
+
try {
|
|
2893
|
+
return await this.repoView.finishRepoTick(this.repoIdentity, this.runId, completion, events)
|
|
2894
|
+
} catch (error) {
|
|
2895
|
+
const observation = (await this.wiring.store.getRepo(this.repoIdentity).catch(() => null))
|
|
2896
|
+
?.tick
|
|
2897
|
+
const finished = observation?.finished
|
|
2898
|
+
if (
|
|
2899
|
+
observation?.run !== this.runId ||
|
|
2900
|
+
observation.open ||
|
|
2901
|
+
finished?.run !== this.runId ||
|
|
2902
|
+
finished.startedAt !== this.tickStartedAt ||
|
|
2903
|
+
finished.outcome !== completion.outcome ||
|
|
2904
|
+
(finished.outcome === 'failed' &&
|
|
2905
|
+
completion.outcome === 'failed' &&
|
|
2906
|
+
finished.error !== completion.error) ||
|
|
2907
|
+
(finished.outcome === 'completed' &&
|
|
2908
|
+
completion.outcome === 'completed' &&
|
|
2909
|
+
Object.entries(completion.counters).some(
|
|
2910
|
+
([key, value]) => finished.counters[key as keyof typeof finished.counters] !== value,
|
|
2911
|
+
))
|
|
2912
|
+
)
|
|
2913
|
+
throw error
|
|
2914
|
+
// The failed response bypassed the view's own-write fold. Refresh its
|
|
2915
|
+
// cursor from the journal, including any committed boundary pair.
|
|
2916
|
+
await this.repoView.refresh()
|
|
2917
|
+
return { observation, events: [] }
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
private async publishTickFailure(error: unknown): Promise<void> {
|
|
2922
|
+
const failed = {
|
|
2923
|
+
actor: DISPATCHER,
|
|
2924
|
+
type: 'dispatcher.tick-failed' as const,
|
|
2925
|
+
payload: { run: this.runId, error: error instanceof Error ? error.message : String(error) },
|
|
2926
|
+
}
|
|
2927
|
+
if (this.tickStartedAt === undefined) {
|
|
2928
|
+
await this.appendStatus(failed)
|
|
2929
|
+
return
|
|
2930
|
+
}
|
|
2931
|
+
const result = await this.finishTick({ outcome: 'failed', error: failed.payload.error }, [
|
|
2932
|
+
{
|
|
2933
|
+
actor: DISPATCHER,
|
|
2934
|
+
type: 'dispatcher.tick-started',
|
|
2935
|
+
payload: { run: this.runId, startedAt: this.tickStartedAt },
|
|
2936
|
+
},
|
|
2937
|
+
failed,
|
|
2938
|
+
])
|
|
2939
|
+
if (result === null) {
|
|
2940
|
+
this.superseded = true
|
|
2941
|
+
return
|
|
2942
|
+
}
|
|
2943
|
+
this.tickStartedAt = undefined
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2852
2946
|
/** Settlement-only completions are serialized after operational ticks. Loop
|
|
2853
2947
|
* so a result arriving during an awaited status append gets its own report. */
|
|
2854
2948
|
private publishSettlementReports(): Promise<TickReport[]> {
|
|
2855
2949
|
return this.serialize(async () => {
|
|
2950
|
+
if (this.superseded) return []
|
|
2856
2951
|
const reports: TickReport[] = []
|
|
2857
2952
|
while (this.hasPendingHarvestResults()) {
|
|
2858
2953
|
const report = await this.publishTickReport(emptyTickReport(), true)
|
|
@@ -2918,7 +3013,7 @@ class DispatchLoop {
|
|
|
2918
3013
|
{ store: this.repoView, ids: this.wiring.ids },
|
|
2919
3014
|
{
|
|
2920
3015
|
slug: input.slug,
|
|
2921
|
-
events: await this.repoView
|
|
3016
|
+
events: (await readBuildFacts(this.repoView, input.slug)).value,
|
|
2922
3017
|
maxAttempts: this.currentConfig().config.policy.maxInfrastructureAttempts,
|
|
2923
3018
|
provider: input.identity?.provider ?? this.wiring.workspaces.name,
|
|
2924
3019
|
workspaceRef: input.workspaceRef,
|
|
@@ -2960,18 +3055,9 @@ class DispatchLoop {
|
|
|
2960
3055
|
this.failureNotice(`build ${slug} already held by another runner — skipped`)
|
|
2961
3056
|
return 'already-active'
|
|
2962
3057
|
}
|
|
2963
|
-
const
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
for (const event of launchEvents) {
|
|
2967
|
-
if (event.type === 'workspace.provisioned') {
|
|
2968
|
-
workspaceRef = event.payload.ref
|
|
2969
|
-
workspaceProvider = event.payload.provider
|
|
2970
|
-
} else if (event.type === 'workspace.released') {
|
|
2971
|
-
workspaceRef = undefined
|
|
2972
|
-
workspaceProvider = undefined
|
|
2973
|
-
}
|
|
2974
|
-
}
|
|
3058
|
+
const workspace = (await readBuildFacts(this.repoView, slug)).value.workspace
|
|
3059
|
+
const workspaceRef = workspace?.ref
|
|
3060
|
+
const workspaceProvider = workspace?.provider
|
|
2975
3061
|
if (workspaceRef === undefined) throw new Error(`build ${slug} has no open workspace`)
|
|
2976
3062
|
// Execute through the runtime that owns the workspace: a build
|
|
2977
3063
|
// provisioned before a provider switch still runs where it lives.
|
|
@@ -3054,7 +3140,7 @@ class DispatchLoop {
|
|
|
3054
3140
|
}
|
|
3055
3141
|
|
|
3056
3142
|
await this.repoView.refreshBuild(slug)
|
|
3057
|
-
const state =
|
|
3143
|
+
const state = (await readBuildFacts(this.repoView, slug)).value.state
|
|
3058
3144
|
if (diagnostic === null && exit.exitCode === 0) {
|
|
3059
3145
|
await this.appendStatus({
|
|
3060
3146
|
actor: DISPATCHER,
|
|
@@ -3121,12 +3207,7 @@ class DispatchLoop {
|
|
|
3121
3207
|
if (leaseClaimed) {
|
|
3122
3208
|
// The runner appended through its own handle: take the delta.
|
|
3123
3209
|
await this.repoView.refreshBuild(slug)
|
|
3124
|
-
const
|
|
3125
|
-
let workspaceRef = ''
|
|
3126
|
-
for (const event of events) {
|
|
3127
|
-
if (event.type === 'workspace.provisioned') workspaceRef = event.payload.ref
|
|
3128
|
-
else if (event.type === 'workspace.released') workspaceRef = ''
|
|
3129
|
-
}
|
|
3210
|
+
const workspaceRef = (await readBuildFacts(this.repoView, slug)).value.workspace?.ref ?? ''
|
|
3130
3211
|
if (workspaceRef !== '') {
|
|
3131
3212
|
await this.recordInfrastructureFailure({
|
|
3132
3213
|
slug,
|
|
@@ -3402,7 +3483,7 @@ class DispatchLoop {
|
|
|
3402
3483
|
// no anchor dependence (anchor analysis recorded; the frontend seed needed one
|
|
3403
3484
|
// only because reduceDispatchStatus folds here too — it does not here).
|
|
3404
3485
|
// Missing journal record stays a read-only empty journal (AUT-524).
|
|
3405
|
-
const repositoryEvents = await
|
|
3486
|
+
const repositoryEvents = (await readRepositoryFacts(this.wiring.store, this.repoIdentity)).value
|
|
3406
3487
|
|
|
3407
3488
|
// Action-triggered and timer refreshes share the cache but may finish their
|
|
3408
3489
|
// repository reads out of order. Never let an older build snapshot replace
|
|
@@ -3802,14 +3883,7 @@ class DispatchLoop {
|
|
|
3802
3883
|
// invocation.
|
|
3803
3884
|
if (this.pastDeadline()) return
|
|
3804
3885
|
const initial = await this.dispatcherTick(true).catch(async (error: unknown) => {
|
|
3805
|
-
await this.
|
|
3806
|
-
actor: DISPATCHER,
|
|
3807
|
-
type: 'dispatcher.tick-failed',
|
|
3808
|
-
payload: {
|
|
3809
|
-
run: this.runId,
|
|
3810
|
-
error: error instanceof Error ? error.message : String(error),
|
|
3811
|
-
},
|
|
3812
|
-
})
|
|
3886
|
+
await this.publishTickFailure(error)
|
|
3813
3887
|
throw error
|
|
3814
3888
|
})
|
|
3815
3889
|
const initialPrinted = this.printReport(initial, false)
|
|
@@ -3856,11 +3930,7 @@ class DispatchLoop {
|
|
|
3856
3930
|
startup = false
|
|
3857
3931
|
} catch (error) {
|
|
3858
3932
|
const message = error instanceof Error ? error.message : String(error)
|
|
3859
|
-
await this.
|
|
3860
|
-
actor: DISPATCHER,
|
|
3861
|
-
type: 'dispatcher.tick-failed',
|
|
3862
|
-
payload: { run: this.runId, error: message },
|
|
3863
|
-
})
|
|
3933
|
+
await this.publishTickFailure(error)
|
|
3864
3934
|
this.warn(`tick failed: ${message}`)
|
|
3865
3935
|
}
|
|
3866
3936
|
if (this.stopped || this.superseded) break
|
|
@@ -4314,11 +4384,10 @@ export async function abDispatch(opts: DispatchOpts): Promise<void> {
|
|
|
4314
4384
|
const loop = new DispatchLoop(liveConfig, wiring, resolvedOpts, view)
|
|
4315
4385
|
try {
|
|
4316
4386
|
await loop.run()
|
|
4317
|
-
const stopRequest =
|
|
4387
|
+
const stopRequest =
|
|
4318
4388
|
resolvedOpts.kernelRunId && resolvedOpts.signal?.aborted
|
|
4319
|
-
? await wiring.store
|
|
4320
|
-
:
|
|
4321
|
-
).find((event) => event.type === 'dispatcher.stop-requested' && event.payload.run === runId)
|
|
4389
|
+
? (await readDispatcherRunFacts(wiring.store, state.repo, runId)).value.stopRequest
|
|
4390
|
+
: undefined
|
|
4322
4391
|
const stoppedSignal =
|
|
4323
4392
|
stopRequest?.type === 'dispatcher.stop-requested'
|
|
4324
4393
|
? stopRequest.payload.signal
|
|
@@ -4356,7 +4425,7 @@ export async function abDispatch(opts: DispatchOpts): Promise<void> {
|
|
|
4356
4425
|
// Every append this invocation made, `run-stopped` included, is in the
|
|
4357
4426
|
// view; leave the reduced state behind so the next cold process reads
|
|
4358
4427
|
// only what is newer than it.
|
|
4359
|
-
|
|
4428
|
+
|
|
4360
4429
|
if (resolvedOpts.wire === undefined) await wiring.store.close()
|
|
4361
4430
|
}
|
|
4362
4431
|
}
|