@defrex/autobuild 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/SPEC.md +10 -0
- package/docs/architecture.md +109 -0
- package/docs/configuration.md +24 -3
- package/docs/remote-store-protocol.md +35 -0
- package/docs/website.md +4 -0
- package/package.json +6 -4
- package/packages/core/src/cli/dashboard/facts.ts +215 -0
- package/packages/core/src/cli/dashboard/frame-parse.ts +223 -0
- package/packages/core/src/cli/dashboard/model.ts +85 -147
- package/packages/core/src/cli/dashboard/poll.ts +7 -2
- package/packages/core/src/cli/dispatch-process.ts +1 -13
- package/packages/core/src/cli/dispatch.ts +80 -37
- package/packages/core/src/cli/help.ts +15 -1
- package/packages/core/src/cli/init-validation.ts +18 -0
- package/packages/core/src/cli/installation.ts +50 -8
- package/packages/core/src/cli/self-update.ts +77 -2
- package/packages/core/src/cli/status.ts +1 -1
- package/packages/core/src/cli/ticket.ts +146 -1
- package/packages/core/src/cli/wait.ts +10 -2
- package/packages/core/src/cli/watch.ts +10 -2
- package/packages/core/src/config/live.ts +1 -0
- package/packages/core/src/config/pipeline-source.ts +17 -5
- package/packages/core/src/config/schema.ts +21 -0
- package/packages/core/src/integration/harness.ts +113 -1
- package/packages/core/src/kernel/__snapshots__/reducer-registry.test.ts.snap +1369 -0
- package/packages/core/src/kernel/auto-merge-default.ts +43 -9
- package/packages/core/src/kernel/auto-merge.ts +99 -22
- package/packages/core/src/kernel/dispatch-settings.ts +42 -17
- package/packages/core/src/kernel/dispatch-status.ts +23 -0
- package/packages/core/src/kernel/engine.ts +9 -438
- package/packages/core/src/kernel/generators/build-log.ts +259 -0
- package/packages/core/src/kernel/generators/repository-journals.ts +206 -0
- package/packages/core/src/kernel/generators/session-log.ts +72 -0
- package/packages/core/src/kernel/harvest.ts +62 -18
- package/packages/core/src/kernel/incremental-contract.ts +75 -0
- package/packages/core/src/kernel/incremental-fixtures.ts +211 -0
- package/packages/core/src/kernel/incremental.ts +53 -0
- package/packages/core/src/kernel/log-index.ts +620 -0
- package/packages/core/src/kernel/pr-attachments.ts +164 -46
- package/packages/core/src/kernel/reducer-registry.ts +164 -0
- package/packages/core/src/kernel/reducer.ts +216 -74
- package/packages/core/src/operator/query.ts +68 -28
- package/packages/core/src/operator/read-view.ts +98 -0
- package/packages/core/src/ports/tickets/fake.ts +5 -1
- package/packages/core/src/ports/tickets/file.ts +169 -48
- package/packages/core/src/ports/tickets/linear.ts +33 -1
- package/packages/core/src/ports/tickets/operations.ts +1 -1
- package/packages/core/src/ports/tickets/remote.ts +8 -1
- package/packages/core/src/ports/types.ts +32 -1
- package/packages/core/src/processes/build-execution-state.ts +31 -15
- package/packages/core/src/processes/build-runner.ts +184 -90
- package/packages/core/src/processes/dispatcher-selectors.ts +123 -0
- package/packages/core/src/processes/dispatcher.ts +270 -176
- package/packages/core/src/processes/execution-settlement.ts +72 -39
- package/packages/core/src/processes/harvest-execution-state.ts +41 -29
- package/packages/core/src/processes/infrastructure-failure-budget.ts +31 -18
- package/packages/core/src/processes/orchestrator-tick.ts +44 -3
- package/packages/core/src/processes/publication-state.ts +164 -52
- package/packages/core/src/processes/repo-view.ts +1436 -0
- package/packages/core/src/processes/sandbox-state.ts +26 -2
- package/packages/core/src/processes/tick-state.ts +33 -0
- package/packages/core/src/store/adapter.ts +10 -1
- package/packages/core/src/store/build-scope.ts +26 -0
- package/packages/core/src/store/contract.ts +337 -0
- package/packages/core/src/store/digest.ts +71 -18
- package/packages/core/src/store/local/schema.ts +17 -0
- package/packages/core/src/store/local/store.ts +111 -4
- package/packages/core/src/store/memory.ts +68 -1
- package/packages/core/src/store/phase-session.ts +23 -0
- package/packages/core/src/store/remote/client.ts +50 -1
- package/packages/core/src/store/remote/index.ts +2 -0
- package/packages/core/src/store/remote/protocol.ts +15 -0
- package/packages/core/src/store/repo-state-events.ts +72 -7
- package/packages/core/src/store/session-handle.ts +30 -0
- package/packages/core/src/store/session-reducer.ts +52 -11
- package/packages/core/src/store/snapshots.ts +88 -0
- package/packages/core/src/store/ticket-assets.ts +41 -8
- package/packages/core/src/store/types.ts +38 -0
- package/skills/guide/SKILL.md +26 -7
- package/skills/guide/references/remote-store-protocol.md +35 -0
- package/skills/tickets/SKILL.md +57 -5
|
@@ -55,7 +55,7 @@ import {
|
|
|
55
55
|
type AutoMergeDefaultFact,
|
|
56
56
|
} from '../kernel/auto-merge-default'
|
|
57
57
|
import { reduceDispatchSettings } from '../kernel/dispatch-settings'
|
|
58
|
-
import {
|
|
58
|
+
import { decideNextFromFacts } from '../kernel/engine'
|
|
59
59
|
import {
|
|
60
60
|
DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS,
|
|
61
61
|
decideHarvestControl,
|
|
@@ -75,18 +75,25 @@ import type {
|
|
|
75
75
|
WorkspaceProvider,
|
|
76
76
|
} from '../ports/types'
|
|
77
77
|
import type { Exec } from '../ports/workspace/git-worktree'
|
|
78
|
-
import type { ArtifactMeta, BuildRecord, BuildStore, Clock } from '../store/types'
|
|
78
|
+
import type { ArtifactMeta, BuildDigest, BuildRecord, BuildStore, Clock } from '../store/types'
|
|
79
79
|
import { samplePinnedAssets } from '../store/ticket-assets'
|
|
80
80
|
import { resolveRepoOrigin } from '../cli/repo-state'
|
|
81
81
|
import { specConformance } from '../spec-standard'
|
|
82
82
|
export { specConformance, type SpecConformance } from '../spec-standard'
|
|
83
83
|
import { recordInfrastructureFailure as appendInfrastructureFailure } from './infrastructure-failure-budget'
|
|
84
|
-
import {
|
|
84
|
+
import { settleOpenExecution } from './execution-settlement'
|
|
85
85
|
import { openHarvestExecutions } from './harvest-execution-state'
|
|
86
|
-
import {
|
|
86
|
+
import { isWorkDigest, RepoViewStore } from './repo-view'
|
|
87
|
+
import {
|
|
88
|
+
baseBranchReducer,
|
|
89
|
+
openBuildWorkspace,
|
|
90
|
+
provisionMarkerReducer,
|
|
91
|
+
recoveryCheckpointReducer,
|
|
92
|
+
type OpenBuildWorkspace,
|
|
93
|
+
type ProvisionMarker,
|
|
94
|
+
} from './dispatcher-selectors'
|
|
87
95
|
import type { RepositoryEvent } from '../events/repository'
|
|
88
96
|
import { sandboxStates } from './sandbox-state'
|
|
89
|
-
import { abandonedPublicationPending, publicationPending } from './publication-state'
|
|
90
97
|
import { settlePublicationBeforeRelease } from './publication-loss'
|
|
91
98
|
|
|
92
99
|
// ── Readiness resolution (SPEC §3.3) ─────────────────────────────────────────
|
|
@@ -108,11 +115,22 @@ import { settlePublicationBeforeRelease } from './publication-loss'
|
|
|
108
115
|
* is otherwise the directory, so it defaults to no label gate. Plugin sources
|
|
109
116
|
* likewise receive no host-imposed label convention. An explicit
|
|
110
117
|
* `readyLabels` wins for every source.
|
|
118
|
+
*
|
|
119
|
+
* `[tickets].readyAssignee` (Linear only) adds a third conjunctive gate: the
|
|
120
|
+
* ticket must be assigned to that user.
|
|
111
121
|
*/
|
|
112
|
-
export function readyCriteria(config: Config): {
|
|
113
|
-
|
|
122
|
+
export function readyCriteria(config: Config): {
|
|
123
|
+
labels: string[]
|
|
124
|
+
state: string
|
|
125
|
+
assignee?: string
|
|
126
|
+
} {
|
|
127
|
+
const { readyLabels, readyState, readyAssignee } = config.tickets
|
|
114
128
|
if (config.tickets.source === 'linear') {
|
|
115
|
-
return {
|
|
129
|
+
return {
|
|
130
|
+
labels: readyLabels ?? ['autobuild'],
|
|
131
|
+
state: readyState,
|
|
132
|
+
...(readyAssignee !== undefined ? { assignee: readyAssignee } : {}),
|
|
133
|
+
}
|
|
116
134
|
}
|
|
117
135
|
return { labels: readyLabels ?? [], state: readyState }
|
|
118
136
|
}
|
|
@@ -469,6 +487,10 @@ export interface TickOpts {
|
|
|
469
487
|
* escalations remain human gates and are only reported as parked.
|
|
470
488
|
*/
|
|
471
489
|
resumeCurrent?: boolean
|
|
490
|
+
/** The caller refreshed the repository view for this tick already (the CLI
|
|
491
|
+
* does, to sample the repository-wide controls from the same state). The
|
|
492
|
+
* tick then skips its own refresh so one tick performs one discovery read. */
|
|
493
|
+
viewRefreshed?: boolean
|
|
472
494
|
/**
|
|
473
495
|
* Repository-derived intake gate supplied by the CLI for this tick. False
|
|
474
496
|
* skips only the ready-ticket list/claim/dispatch stage; janitor, startup
|
|
@@ -575,30 +597,14 @@ export interface DispatcherDeps {
|
|
|
575
597
|
/** Latest `workspace.provisioned` not followed by a `workspace.released` —
|
|
576
598
|
* the reducer deliberately ignores workspace events (liveness is the
|
|
577
599
|
* dispatcher's concern), so the janitor scans the raw log. */
|
|
578
|
-
function openWorkspace(
|
|
579
|
-
events
|
|
580
|
-
): { provider: string; ref: string; path?: string; localPath?: string; branch: string } | null {
|
|
581
|
-
let open: {
|
|
582
|
-
provider: string
|
|
583
|
-
ref: string
|
|
584
|
-
path?: string
|
|
585
|
-
localPath?: string
|
|
586
|
-
branch: string
|
|
587
|
-
} | null = null
|
|
588
|
-
for (const event of events) {
|
|
589
|
-
if (event.type === 'workspace.provisioned') open = event.payload
|
|
590
|
-
else if (event.type === 'workspace.released') open = null
|
|
591
|
-
}
|
|
592
|
-
return open
|
|
600
|
+
function openWorkspace(events: AbEvent[]): OpenBuildWorkspace | null {
|
|
601
|
+
return openBuildWorkspace(events)
|
|
593
602
|
}
|
|
594
603
|
|
|
595
604
|
/** The build's base branch, from its own `build.created` fact (§15.3);
|
|
596
605
|
* falls back to the repo config for logs missing one. */
|
|
597
606
|
function baseBranchOf(events: AbEvent[], config: Config): string {
|
|
598
|
-
|
|
599
|
-
if (event.type === 'build.created') return event.payload.baseBranch
|
|
600
|
-
}
|
|
601
|
-
return config.baseBranch
|
|
607
|
+
return baseBranchReducer.reduce(events) ?? config.baseBranch
|
|
602
608
|
}
|
|
603
609
|
|
|
604
610
|
function artifactRefOf(deposited: ArtifactMeta[]): ArtifactRef {
|
|
@@ -656,7 +662,27 @@ export class Dispatcher {
|
|
|
656
662
|
* slug here defers to the continuation; it never blocks on it. */
|
|
657
663
|
private readonly continuations = new Map<string, ProvisionContinuation>()
|
|
658
664
|
|
|
665
|
+
/** The incremental repository view every stage reads and writes through
|
|
666
|
+
* (AUT-647): `deps.store` itself when the caller already wrapped it. */
|
|
667
|
+
private readonly store: RepoViewStore
|
|
668
|
+
|
|
669
|
+
/** `deps.settlePublication`, followed by a delta read of the build: the
|
|
670
|
+
* guest-side settlement appends through its own store handle, which the view
|
|
671
|
+
* only learns of by reading. */
|
|
672
|
+
private get settlePublication(): ((slug: string) => Promise<void>) | undefined {
|
|
673
|
+
const settle = this.deps.settlePublication
|
|
674
|
+
if (settle === undefined) return undefined
|
|
675
|
+
return async (slug) => {
|
|
676
|
+
await settle(slug)
|
|
677
|
+
await this.store.refreshBuild(slug)
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
659
681
|
constructor(private readonly deps: DispatcherDeps) {
|
|
682
|
+
this.store =
|
|
683
|
+
deps.store instanceof RepoViewStore
|
|
684
|
+
? deps.store
|
|
685
|
+
: new RepoViewStore(deps.store, { repo: deps.repo })
|
|
660
686
|
this.leaseTtlMs = deps.opts?.leaseTtlMs ?? 0
|
|
661
687
|
this.doneState = deps.opts?.doneState ?? 'Done'
|
|
662
688
|
this.slugNamingTimeoutMs = deps.opts?.slugNamingTimeoutMs ?? DEFAULT_SLUG_NAMING_TIMEOUT_MS
|
|
@@ -680,9 +706,21 @@ export class Dispatcher {
|
|
|
680
706
|
}
|
|
681
707
|
|
|
682
708
|
async tick(opts: TickOpts = {}): Promise<TickReport> {
|
|
709
|
+
try {
|
|
710
|
+
return await this.runTick(opts)
|
|
711
|
+
} finally {
|
|
712
|
+
// Leave this tick's reduced state behind for the next process (a cache;
|
|
713
|
+
// a failure to write it is never a tick failure). Throttled in a
|
|
714
|
+
// resident dispatcher, always run by a cold single-tick process.
|
|
715
|
+
await this.store.persistSnapshots()
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
private async runTick(opts: TickOpts): Promise<TickReport> {
|
|
683
720
|
// One immutable config for this complete decision pass. A reload racing the
|
|
684
721
|
// tick is observed by the next tick, never half-way through this one.
|
|
685
722
|
this.deps.config = this.deps.getConfig?.() ?? this.deps.config
|
|
723
|
+
if (opts.viewRefreshed !== true) await this.store.refresh()
|
|
686
724
|
let autoMergeUser: string | undefined
|
|
687
725
|
if (opts.defaultAutoMerge === true) {
|
|
688
726
|
autoMergeUser = opts.autoMergeUser?.trim()
|
|
@@ -699,7 +737,7 @@ export class Dispatcher {
|
|
|
699
737
|
// likewise samples before calling `tick`); the auto-merge default fact is
|
|
700
738
|
// the durable driver of this tick's fan-out below. Neither control
|
|
701
739
|
// pretends to a serialization the store does not offer.
|
|
702
|
-
const repoEvents =
|
|
740
|
+
const repoEvents = this.store.recordedJournal()
|
|
703
741
|
const paused = reduceDispatchSettings(repoEvents).paused
|
|
704
742
|
const autoMergeDefault = latestAutoMergeDefault(repoEvents)
|
|
705
743
|
// The claim-time seed cites the newest fact whose enabled state matches
|
|
@@ -776,7 +814,7 @@ export class Dispatcher {
|
|
|
776
814
|
let sandbox: OperatorSandboxService | undefined
|
|
777
815
|
try {
|
|
778
816
|
sandbox = await createOperatorSandboxService({
|
|
779
|
-
store: this.
|
|
817
|
+
store: this.store,
|
|
780
818
|
repo: this.deps.repo,
|
|
781
819
|
provider: this.deps.workspaces,
|
|
782
820
|
sandbox: this.deps.config.orchestrator.sandbox,
|
|
@@ -789,7 +827,8 @@ export class Dispatcher {
|
|
|
789
827
|
sandbox = undefined
|
|
790
828
|
}
|
|
791
829
|
await runOrchestratorTickStep({
|
|
792
|
-
store: this.
|
|
830
|
+
store: this.store,
|
|
831
|
+
view: this.store,
|
|
793
832
|
repo: this.deps.repo,
|
|
794
833
|
config: this.deps.config,
|
|
795
834
|
clock: this.deps.clock,
|
|
@@ -814,6 +853,46 @@ export class Dispatcher {
|
|
|
814
853
|
return this.deps.opts?.triageState ?? defaultTriageState(this.deps.config)
|
|
815
854
|
}
|
|
816
855
|
|
|
856
|
+
/** This repository's builds paired with their digests, from one record
|
|
857
|
+
* listing and one digest read — the tick loops' per-build event reads start
|
|
858
|
+
* from here so a settled terminal build costs none. Records are read before
|
|
859
|
+
* digests (as `operator/query.ts` does) so a concurrent create cannot fail
|
|
860
|
+
* the completeness check. */
|
|
861
|
+
private async repoBuildDigests(): Promise<{ record: BuildRecord; digest: BuildDigest }[]> {
|
|
862
|
+
const store = this.store
|
|
863
|
+
const { repo } = this.deps
|
|
864
|
+
const records = (await store.listBuilds()).filter((record) => record.repo === repo)
|
|
865
|
+
const digests = await store.getRepoBuildDigests(repo)
|
|
866
|
+
return records.map((record) => {
|
|
867
|
+
const digest = digests.get(record.slug)
|
|
868
|
+
if (digest === undefined) {
|
|
869
|
+
throw new Error(
|
|
870
|
+
`build digest missing for ${record.slug}: the digest read broke completeness`,
|
|
871
|
+
)
|
|
872
|
+
}
|
|
873
|
+
return { record, digest }
|
|
874
|
+
})
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
/** Builds with no terminal fact: the only ones whose tick duties change
|
|
878
|
+
* their state (decide, resume, lease sweep, capacity). */
|
|
879
|
+
private async nonterminalBuilds(): Promise<BuildRecord[]> {
|
|
880
|
+
return (await this.repoBuildDigests())
|
|
881
|
+
.filter(({ digest }) => digest.terminal === undefined)
|
|
882
|
+
.map(({ record }) => record)
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
/** Builds that still have work: nonterminal ones, plus terminal builds whose
|
|
886
|
+
* post-terminal duty is outstanding — an `aborted` build awaiting its
|
|
887
|
+
* cleanup's `build.completed`, a `done` build with an unreclaimed release
|
|
888
|
+
* asset, or any build with an execution the provider has not been seen to
|
|
889
|
+
* end. A terminal, settled build costs no event reads. */
|
|
890
|
+
private async workBuilds(): Promise<BuildRecord[]> {
|
|
891
|
+
return (await this.repoBuildDigests())
|
|
892
|
+
.filter(({ digest }) => isWorkDigest(digest))
|
|
893
|
+
.map(({ record }) => record)
|
|
894
|
+
}
|
|
895
|
+
|
|
817
896
|
/** Observation-only ready listing for Store-backed frontends while intake is
|
|
818
897
|
* disabled. It shares the active-ticket exclusion with dispatch but performs
|
|
819
898
|
* no dependency fetch, claim, bounce, build, workspace, or launch side
|
|
@@ -824,11 +903,10 @@ export class Dispatcher {
|
|
|
824
903
|
ticketDiagnostics: string[]
|
|
825
904
|
}> {
|
|
826
905
|
this.deps.config = this.deps.getConfig?.() ?? this.deps.config
|
|
906
|
+
await this.store.refresh()
|
|
827
907
|
const activeTicketIds = new Set<string>()
|
|
828
|
-
for (const record of await this.
|
|
829
|
-
if (record.
|
|
830
|
-
const state = reduceBuild(await this.deps.store.getEvents(record.slug))
|
|
831
|
-
if (state.status !== 'done' && state.status !== 'aborted') {
|
|
908
|
+
for (const { record, digest } of await this.repoBuildDigests()) {
|
|
909
|
+
if (record.ticket !== undefined && digest.terminal === undefined) {
|
|
832
910
|
activeTicketIds.add(record.ticket.id)
|
|
833
911
|
}
|
|
834
912
|
}
|
|
@@ -857,13 +935,14 @@ export class Dispatcher {
|
|
|
857
935
|
* report shape stays compatible. */
|
|
858
936
|
private async applyAutoMergeDefault(fact: AutoMergeDefaultFact | undefined): Promise<void> {
|
|
859
937
|
if (fact === undefined) return
|
|
860
|
-
const
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
938
|
+
const store = this.store
|
|
939
|
+
|
|
940
|
+
// Terminal builds are never eligible (`autoMergeDefaultEligible`).
|
|
941
|
+
for (const record of (await this.nonterminalBuilds()).sort((a, b) =>
|
|
942
|
+
a.slug.localeCompare(b.slug),
|
|
943
|
+
)) {
|
|
864
944
|
while (true) {
|
|
865
|
-
const
|
|
866
|
-
const state = reduceBuild(events)
|
|
945
|
+
const { state } = await store.buildFacts(record.slug)
|
|
867
946
|
const target = autoMergeDefaultTarget(state, fact)
|
|
868
947
|
if (target === undefined) break
|
|
869
948
|
if (target === 'observed') {
|
|
@@ -890,13 +969,15 @@ export class Dispatcher {
|
|
|
890
969
|
private async triggerHarvest(): Promise<void> {
|
|
891
970
|
const start = this.deps.startHarvest
|
|
892
971
|
if (start === undefined) return
|
|
893
|
-
|
|
894
|
-
|
|
972
|
+
// The journal moves under foreign harvest writers while the stages run, so
|
|
973
|
+
// take the delta before deciding.
|
|
974
|
+
await this.store.refreshJournal()
|
|
975
|
+
if (!this.store.journalRecorded()) {
|
|
895
976
|
start()
|
|
896
977
|
return
|
|
897
978
|
}
|
|
898
|
-
// Bounded
|
|
899
|
-
const state = reduceHarvest(
|
|
979
|
+
// Bounded subset (AUT-489): the harvest reducer consumes durable types only.
|
|
980
|
+
const state = reduceHarvest(this.store.recordedJournal())
|
|
900
981
|
if (decideHarvestControl(state, this.maxHarvestRecoveryAttempts).kind !== 'park') {
|
|
901
982
|
start()
|
|
902
983
|
}
|
|
@@ -925,28 +1006,26 @@ export class Dispatcher {
|
|
|
925
1006
|
* the stage still settles. */
|
|
926
1007
|
private async settleForeignExecutions(report: TickReport, opts: TickOpts = {}): Promise<void> {
|
|
927
1008
|
const active = this.deps.activeExecutions?.() ?? new Set<string>()
|
|
928
|
-
for (const record of await this.
|
|
929
|
-
if (record.repo !== this.deps.repo) continue
|
|
1009
|
+
for (const record of await this.workBuilds()) {
|
|
930
1010
|
if (active.has(record.slug)) continue
|
|
931
1011
|
// Budget gate: once spent, stop settling further foreign executions.
|
|
932
1012
|
if (this.outOfBudget(opts)) break
|
|
933
1013
|
try {
|
|
934
|
-
const
|
|
935
|
-
const open = openExecution(events)
|
|
1014
|
+
const { open } = await this.store.buildFacts(record.slug)
|
|
936
1015
|
if (open === null) continue
|
|
937
1016
|
const owner = this.executionOwner(open.provider)
|
|
938
1017
|
const execution = owner?.buildExecution
|
|
939
1018
|
if (execution?.observe === undefined || open.commandId === undefined) continue
|
|
940
|
-
const settlement = await
|
|
1019
|
+
const settlement = await settleOpenExecution(
|
|
941
1020
|
{
|
|
942
|
-
store: this.
|
|
1021
|
+
store: this.store,
|
|
943
1022
|
execution,
|
|
944
|
-
...(this.
|
|
945
|
-
? { settlePublication: this.
|
|
1023
|
+
...(this.settlePublication !== undefined
|
|
1024
|
+
? { settlePublication: this.settlePublication }
|
|
946
1025
|
: {}),
|
|
947
1026
|
},
|
|
948
1027
|
record.slug,
|
|
949
|
-
|
|
1028
|
+
open,
|
|
950
1029
|
)
|
|
951
1030
|
if (settlement === 'settled') report.settled += 1
|
|
952
1031
|
} catch {
|
|
@@ -971,7 +1050,11 @@ export class Dispatcher {
|
|
|
971
1050
|
const active = this.deps.activeHarvestExecutions?.() ?? new Set<string>()
|
|
972
1051
|
let events: RepositoryEvent[]
|
|
973
1052
|
try {
|
|
974
|
-
|
|
1053
|
+
// The earlier stages awaited providers; take the journal delta so a
|
|
1054
|
+
// foreign append (operator activity, harvest facts) made meanwhile is
|
|
1055
|
+
// seen before any destructive decision below.
|
|
1056
|
+
await this.store.refreshJournal()
|
|
1057
|
+
events = this.store.recordedJournal()
|
|
975
1058
|
} catch {
|
|
976
1059
|
// The read itself failed (a transient store error — a missing record
|
|
977
1060
|
// answers `[]` inside the helper, so it never reaches this arm):
|
|
@@ -1027,7 +1110,7 @@ export class Dispatcher {
|
|
|
1027
1110
|
branch: '',
|
|
1028
1111
|
})
|
|
1029
1112
|
}
|
|
1030
|
-
await this.
|
|
1113
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
1031
1114
|
actor: DISPATCHER,
|
|
1032
1115
|
type: 'harvest.execution.released',
|
|
1033
1116
|
payload: {
|
|
@@ -1070,7 +1153,11 @@ export class Dispatcher {
|
|
|
1070
1153
|
const providerName = this.deps.workspaces.name
|
|
1071
1154
|
let events: RepositoryEvent[]
|
|
1072
1155
|
try {
|
|
1073
|
-
|
|
1156
|
+
// The earlier stages awaited providers; take the journal delta so a
|
|
1157
|
+
// foreign append (operator activity, harvest facts) made meanwhile is
|
|
1158
|
+
// seen before any destructive decision below.
|
|
1159
|
+
await this.store.refreshJournal()
|
|
1160
|
+
events = this.store.recordedJournal()
|
|
1074
1161
|
} catch {
|
|
1075
1162
|
// The read itself failed (a transient store error — a missing record
|
|
1076
1163
|
// answers `[]` inside the helper, so it never reaches this arm):
|
|
@@ -1097,7 +1184,7 @@ export class Dispatcher {
|
|
|
1097
1184
|
// never a stop: nothing was stopped through a provider this
|
|
1098
1185
|
// dispatcher no longer owns. If the provider returns, the next
|
|
1099
1186
|
// operator ensure() re-provisions and re-brackets the environment.
|
|
1100
|
-
await this.
|
|
1187
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
1101
1188
|
actor: DISPATCHER,
|
|
1102
1189
|
type: 'orchestrator.sandbox.released',
|
|
1103
1190
|
payload: {
|
|
@@ -1118,7 +1205,7 @@ export class Dispatcher {
|
|
|
1118
1205
|
})
|
|
1119
1206
|
if (outcome.outcome === 'unsupported') continue
|
|
1120
1207
|
if (outcome.outcome === 'stopped') {
|
|
1121
|
-
await this.
|
|
1208
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
1122
1209
|
actor: DISPATCHER,
|
|
1123
1210
|
type: 'orchestrator.sandbox.stopped',
|
|
1124
1211
|
payload: {
|
|
@@ -1133,7 +1220,7 @@ export class Dispatcher {
|
|
|
1133
1220
|
// dead host): close the orphan's trail with an unconfirmed-purge
|
|
1134
1221
|
// release fact, dispatcher-authored exactly as
|
|
1135
1222
|
// `harvest.execution.released` is.
|
|
1136
|
-
await this.
|
|
1223
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
1137
1224
|
actor: DISPATCHER,
|
|
1138
1225
|
type: 'orchestrator.sandbox.released',
|
|
1139
1226
|
payload: {
|
|
@@ -1161,23 +1248,8 @@ export class Dispatcher {
|
|
|
1161
1248
|
|
|
1162
1249
|
/** The latest `workspace.provision-started` not yet followed by a
|
|
1163
1250
|
* provisioned/released fact — an open provisioning marker. */
|
|
1164
|
-
private openProvisionMarker(
|
|
1165
|
-
events
|
|
1166
|
-
): { provider: string; branch: string; generation: number; ts: string; seq: number } | undefined {
|
|
1167
|
-
let marker:
|
|
1168
|
-
| { provider: string; branch: string; generation: number; ts: string; seq: number }
|
|
1169
|
-
| undefined
|
|
1170
|
-
for (const event of events) {
|
|
1171
|
-
if (event.type === 'workspace.provision-started') {
|
|
1172
|
-
marker = { ...event.payload, ts: event.ts, seq: event.seq }
|
|
1173
|
-
} else if (
|
|
1174
|
-
marker !== undefined &&
|
|
1175
|
-
(event.type === 'workspace.provisioned' || event.type === 'workspace.released')
|
|
1176
|
-
) {
|
|
1177
|
-
marker = undefined
|
|
1178
|
-
}
|
|
1179
|
-
}
|
|
1180
|
-
return marker
|
|
1251
|
+
private openProvisionMarker(events: AbEvent[]): ProvisionMarker | undefined {
|
|
1252
|
+
return provisionMarkerReducer.reduce(events)
|
|
1181
1253
|
}
|
|
1182
1254
|
|
|
1183
1255
|
/** Liveness of the lease behind an open provision marker: `live` means a
|
|
@@ -1186,7 +1258,7 @@ export class Dispatcher {
|
|
|
1186
1258
|
* may have crashed between marker and claim — adopted only after the
|
|
1187
1259
|
* `provisionStaleMs` backstop. */
|
|
1188
1260
|
private async provisionMarkerLiveness(slug: string): Promise<'live' | 'expired' | 'absent'> {
|
|
1189
|
-
const record = await this.
|
|
1261
|
+
const record = await this.store.getBuild(slug)
|
|
1190
1262
|
if (record?.lease === undefined) return 'absent'
|
|
1191
1263
|
return new Date(record.lease.expiresAt).getTime() > this.deps.clock().getTime()
|
|
1192
1264
|
? 'live'
|
|
@@ -1235,7 +1307,7 @@ export class Dispatcher {
|
|
|
1235
1307
|
}
|
|
1236
1308
|
const branch = record.branch ?? `ab/${slug}`
|
|
1237
1309
|
const generation = events.filter((event) => event.type === 'workspace.provisioned').length
|
|
1238
|
-
await this.
|
|
1310
|
+
await this.store.append(slug, {
|
|
1239
1311
|
actor: DISPATCHER,
|
|
1240
1312
|
type: 'workspace.provision-started',
|
|
1241
1313
|
payload: { provider: this.deps.workspaces.name, branch, generation },
|
|
@@ -1283,7 +1355,8 @@ export class Dispatcher {
|
|
|
1283
1355
|
tail: 'dispatch' | 'launch'
|
|
1284
1356
|
},
|
|
1285
1357
|
): Promise<void> {
|
|
1286
|
-
const
|
|
1358
|
+
const store = this.store
|
|
1359
|
+
const { workspaces, config } = this.deps
|
|
1287
1360
|
const slug = entry.slug
|
|
1288
1361
|
let leaseClaimed = false
|
|
1289
1362
|
let heartbeat: ReturnType<typeof setInterval> | undefined
|
|
@@ -1422,10 +1495,10 @@ export class Dispatcher {
|
|
|
1422
1495
|
for (const entry of entries) entry.controller.abort()
|
|
1423
1496
|
for (const entry of entries) {
|
|
1424
1497
|
try {
|
|
1425
|
-
const events = await this.
|
|
1498
|
+
const events = await this.store.getEvents(entry.slug)
|
|
1426
1499
|
if (this.openProvisionMarker(events) !== undefined) {
|
|
1427
1500
|
const attempt = events.filter((event) => event.type === 'dispatch.failed').length + 1
|
|
1428
|
-
await this.
|
|
1501
|
+
await this.store.append(entry.slug, {
|
|
1429
1502
|
actor: DISPATCHER,
|
|
1430
1503
|
type: 'dispatch.failed',
|
|
1431
1504
|
payload: {
|
|
@@ -1440,7 +1513,7 @@ export class Dispatcher {
|
|
|
1440
1513
|
// The durable marker plus the released lease still gate adoption.
|
|
1441
1514
|
}
|
|
1442
1515
|
try {
|
|
1443
|
-
await this.
|
|
1516
|
+
await this.store.releaseLease(entry.slug, entry.instance)
|
|
1444
1517
|
} catch {
|
|
1445
1518
|
// Expiry fences an ambiguous release.
|
|
1446
1519
|
}
|
|
@@ -1456,12 +1529,11 @@ export class Dispatcher {
|
|
|
1456
1529
|
): Promise<void> {
|
|
1457
1530
|
// Listing is repository-wide work: if it fails there is no individual
|
|
1458
1531
|
// build to attribute, so the caller's existing top-level boundary owns it.
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
if (record.repo !== this.deps.repo) continue
|
|
1532
|
+
// Only this repo's builds that still have work (§12: another repo's
|
|
1533
|
+
// builds are another dispatcher's duty; settled terminal builds cost no
|
|
1534
|
+
// event reads).
|
|
1535
|
+
const entries = (await this.repoBuildDigests()).filter(({ digest }) => isWorkDigest(digest))
|
|
1536
|
+
for (const { record, digest } of entries) {
|
|
1465
1537
|
// Budget gate: the janitor's per-build path issues forge/store transport
|
|
1466
1538
|
// calls (PR probes, closePr, deleteBranch); once the budget is spent,
|
|
1467
1539
|
// stop starting new ones. The remaining builds stay janitor due — the
|
|
@@ -1471,14 +1543,22 @@ export class Dispatcher {
|
|
|
1471
1543
|
// Once a record is known, contain its complete janitor path: loading
|
|
1472
1544
|
// and reducing facts, cleanup, forge/ticket calls, and store writes.
|
|
1473
1545
|
// Existing partial facts leave the same work due on the next tick.
|
|
1474
|
-
const
|
|
1475
|
-
const state = reduceBuild(events)
|
|
1546
|
+
const { state, workspace } = await this.store.buildFacts(record.slug)
|
|
1476
1547
|
const executionLeaseLive = this.hasLiveExecutionLease(record)
|
|
1548
|
+
// Raw history, read at most once and only by a branch that acts on it:
|
|
1549
|
+
// the facts above answer every quiet-path question without it.
|
|
1550
|
+
let loaded: AbEvent[] | undefined
|
|
1551
|
+
const history = async (): Promise<AbEvent[]> => {
|
|
1552
|
+
loaded ??= await this.store.getEvents(record.slug)
|
|
1553
|
+
return loaded
|
|
1554
|
+
}
|
|
1477
1555
|
// Pipeline/ticket/workspace state is settled, but release-asset cleanup
|
|
1478
1556
|
// has its own crash window after build.completed. Revisit only pending
|
|
1479
1557
|
// hosted handles; this never relaunches a runner or consumes capacity.
|
|
1480
1558
|
if (state.status === 'done') {
|
|
1481
|
-
|
|
1559
|
+
if (digest.reclaimPending === true) {
|
|
1560
|
+
await this.reclaimPrAttachments(record.slug, await history())
|
|
1561
|
+
}
|
|
1482
1562
|
continue
|
|
1483
1563
|
}
|
|
1484
1564
|
// Abort is destructive human judgment and therefore outranks a queued
|
|
@@ -1488,13 +1568,15 @@ export class Dispatcher {
|
|
|
1488
1568
|
state.status === 'queued' &&
|
|
1489
1569
|
state.pendingCommands.some((command) => command.command === 'abort')
|
|
1490
1570
|
) {
|
|
1491
|
-
const acknowledged = await this.
|
|
1571
|
+
const acknowledged = await this.store.append(record.slug, {
|
|
1492
1572
|
actor: DISPATCHER,
|
|
1493
1573
|
type: 'build.aborted',
|
|
1494
1574
|
payload: {},
|
|
1495
1575
|
} satisfies EventWrite<'build.aborted'>)
|
|
1496
|
-
|
|
1497
|
-
|
|
1576
|
+
void acknowledged
|
|
1577
|
+
// The view folded the acknowledgement it just appended.
|
|
1578
|
+
loaded = undefined
|
|
1579
|
+
await this.cleanupAborted(record, await history(), report)
|
|
1498
1580
|
continue
|
|
1499
1581
|
}
|
|
1500
1582
|
// Discard is a queued-only recovery control. A request that raced with
|
|
@@ -1505,15 +1587,23 @@ export class Dispatcher {
|
|
|
1505
1587
|
state.status === 'queued' &&
|
|
1506
1588
|
state.discardRequest !== undefined
|
|
1507
1589
|
) {
|
|
1508
|
-
await this.cleanupDiscarded(record,
|
|
1590
|
+
await this.cleanupDiscarded(record, await history(), report)
|
|
1509
1591
|
continue
|
|
1510
1592
|
}
|
|
1511
1593
|
if (!executionLeaseLive && state.status === 'aborted') {
|
|
1512
|
-
await this.cleanupAborted(record,
|
|
1594
|
+
await this.cleanupAborted(record, await history(), report)
|
|
1513
1595
|
continue
|
|
1514
1596
|
}
|
|
1515
|
-
if (state.pr)
|
|
1516
|
-
await this.checkPr(
|
|
1597
|
+
if (state.pr) {
|
|
1598
|
+
await this.checkPr(
|
|
1599
|
+
record,
|
|
1600
|
+
{ history, workspace },
|
|
1601
|
+
state,
|
|
1602
|
+
report,
|
|
1603
|
+
launched,
|
|
1604
|
+
executionLeaseLive,
|
|
1605
|
+
)
|
|
1606
|
+
}
|
|
1517
1607
|
} catch (error) {
|
|
1518
1608
|
const message = error instanceof Error ? error.message : String(error)
|
|
1519
1609
|
report.janitorFailed += 1
|
|
@@ -1523,7 +1613,10 @@ export class Dispatcher {
|
|
|
1523
1613
|
}
|
|
1524
1614
|
|
|
1525
1615
|
private forgeWorkspacePath(events: AbEvent[]): string {
|
|
1526
|
-
|
|
1616
|
+
return this.forgeWorkspacePathOf(openWorkspace(events))
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
private forgeWorkspacePathOf(open: OpenBuildWorkspace | null): string {
|
|
1527
1620
|
if (open !== null && isRemoteWorkspace(open)) return this.deps.repo
|
|
1528
1621
|
// Local forges run git with cwd = workspacePath: it must be the physical
|
|
1529
1622
|
// checkout, never the store identity (an origin URL since the identity
|
|
@@ -1546,7 +1639,8 @@ export class Dispatcher {
|
|
|
1546
1639
|
events: AbEvent[],
|
|
1547
1640
|
report: TickReport,
|
|
1548
1641
|
): Promise<void> {
|
|
1549
|
-
const
|
|
1642
|
+
const store = this.store
|
|
1643
|
+
const { tickets, forge } = this.deps
|
|
1550
1644
|
const branch = record.branch
|
|
1551
1645
|
// Whether this build ever owned a local checkout workspace, judged on the
|
|
1552
1646
|
// provisioned facts (not the open one, which is null when the build never
|
|
@@ -1709,7 +1803,8 @@ export class Dispatcher {
|
|
|
1709
1803
|
events: AbEvent[],
|
|
1710
1804
|
report: TickReport,
|
|
1711
1805
|
): Promise<void> {
|
|
1712
|
-
const
|
|
1806
|
+
const store = this.store
|
|
1807
|
+
const { tickets } = this.deps
|
|
1713
1808
|
await this.releaseWorkspace(record.slug, events, 'discard')
|
|
1714
1809
|
if (record.lease !== undefined) {
|
|
1715
1810
|
await store.releaseLease(record.slug, record.lease.holder)
|
|
@@ -1733,18 +1828,21 @@ export class Dispatcher {
|
|
|
1733
1828
|
* against the reduced `prState`), release/complete, project to the ticket. */
|
|
1734
1829
|
private async checkPr(
|
|
1735
1830
|
record: BuildRecord,
|
|
1736
|
-
|
|
1831
|
+
// `workspace` is the open workspace fact; `history` loads the raw log, which
|
|
1832
|
+
// only the branches that act on a PR transition need.
|
|
1833
|
+
source: { history: () => Promise<AbEvent[]>; workspace: OpenBuildWorkspace | null },
|
|
1737
1834
|
state: BuildState,
|
|
1738
1835
|
report: TickReport,
|
|
1739
1836
|
launched: Set<string>,
|
|
1740
1837
|
executionLeaseLive: boolean,
|
|
1741
1838
|
): Promise<void> {
|
|
1742
|
-
const
|
|
1839
|
+
const store = this.store
|
|
1840
|
+
const { tickets, forge } = this.deps
|
|
1743
1841
|
const pr = state.pr
|
|
1744
1842
|
if (!pr) return
|
|
1745
1843
|
// Forge calls run from the workspace when it still exists (it does until
|
|
1746
1844
|
// the build completes); fall back to the repo itself for odd logs.
|
|
1747
|
-
const workspacePath = this.
|
|
1845
|
+
const workspacePath = this.forgeWorkspacePathOf(source.workspace)
|
|
1748
1846
|
const prState = await forge.getPrState(workspacePath, pr.number)
|
|
1749
1847
|
const autoMerge = prState.state === 'open' ? pendingAutoMerge(state) : undefined
|
|
1750
1848
|
|
|
@@ -1770,6 +1868,7 @@ export class Dispatcher {
|
|
|
1770
1868
|
// Dedupe: a reduced prState of 'conflicted' means reconcile is already
|
|
1771
1869
|
// pending (a `reconcile.completed` returns it to 'open').
|
|
1772
1870
|
if (state.prState === 'conflicted') return
|
|
1871
|
+
const events = await source.history()
|
|
1773
1872
|
const baseSha = await this.baseSha(baseBranchOf(events, this.deps.config))
|
|
1774
1873
|
const conflicted = await store.append(record.slug, {
|
|
1775
1874
|
actor: DISPATCHER,
|
|
@@ -1818,10 +1917,11 @@ export class Dispatcher {
|
|
|
1818
1917
|
// Re-read at the last possible point. A cancellation, replacement
|
|
1819
1918
|
// command, newly due pipeline work, or application fact suppresses
|
|
1820
1919
|
// this attempt; the next tick reclassifies from fresh forge state.
|
|
1821
|
-
|
|
1822
|
-
const
|
|
1920
|
+
await store.refreshBuild(record.slug)
|
|
1921
|
+
const latest = await store.buildFacts(record.slug)
|
|
1922
|
+
const latestState = latest.state
|
|
1823
1923
|
const latestIntent = pendingAutoMerge(latestState)
|
|
1824
|
-
const decision =
|
|
1924
|
+
const decision = decideNextFromFacts(latest.state, latest.log, this.deps.config)
|
|
1825
1925
|
if (
|
|
1826
1926
|
latestState.pr?.number === pr.number &&
|
|
1827
1927
|
latestIntent?.enabled === true &&
|
|
@@ -1837,6 +1937,7 @@ export class Dispatcher {
|
|
|
1837
1937
|
switch (prState.state) {
|
|
1838
1938
|
case 'merged': {
|
|
1839
1939
|
if (executionLeaseLive) return
|
|
1940
|
+
const events = await source.history()
|
|
1840
1941
|
// Emit the fact once (a crash between steps re-runs this block; the
|
|
1841
1942
|
// reduced prState dedupes the event, the log dedupes the release).
|
|
1842
1943
|
if (state.prState !== 'merged') {
|
|
@@ -1862,6 +1963,7 @@ export class Dispatcher {
|
|
|
1862
1963
|
}
|
|
1863
1964
|
case 'closed': {
|
|
1864
1965
|
if (executionLeaseLive) return
|
|
1966
|
+
const events = await source.history()
|
|
1865
1967
|
// Closed without merge is a human decision — back to Triage (§15.7).
|
|
1866
1968
|
if (state.prState !== 'closed') {
|
|
1867
1969
|
await store.append(record.slug, {
|
|
@@ -1918,14 +2020,14 @@ export class Dispatcher {
|
|
|
1918
2020
|
workspacePath: this.deps.checkout ?? this.deps.repo,
|
|
1919
2021
|
asset: hosted.payload.asset,
|
|
1920
2022
|
})
|
|
1921
|
-
await this.
|
|
2023
|
+
await this.store.append(slug, {
|
|
1922
2024
|
actor: DISPATCHER,
|
|
1923
2025
|
type: 'pr-attachment.reclaimed',
|
|
1924
2026
|
payload: { hostedSeq: hosted.seq },
|
|
1925
2027
|
})
|
|
1926
2028
|
} catch (error) {
|
|
1927
2029
|
try {
|
|
1928
|
-
await this.
|
|
2030
|
+
await this.store.append(slug, {
|
|
1929
2031
|
actor: DISPATCHER,
|
|
1930
2032
|
type: 'pr-attachment.reclaim-failed',
|
|
1931
2033
|
payload: {
|
|
@@ -1945,24 +2047,7 @@ export class Dispatcher {
|
|
|
1945
2047
|
}
|
|
1946
2048
|
|
|
1947
2049
|
private recoveryCheckpoint(events: AbEvent[]): string | undefined {
|
|
1948
|
-
|
|
1949
|
-
let original: string | undefined
|
|
1950
|
-
for (const event of events) {
|
|
1951
|
-
if (event.type === 'workspace.provisioned' && original === undefined) {
|
|
1952
|
-
original = event.payload.base.sha
|
|
1953
|
-
} else if (event.type === 'implement.completed') {
|
|
1954
|
-
settled = event.payload.commits.head
|
|
1955
|
-
} else if (event.type === 'reconcile.completed') {
|
|
1956
|
-
settled = event.payload.mergeCommit
|
|
1957
|
-
} else if (
|
|
1958
|
-
event.type === 'finalize.step-completed' &&
|
|
1959
|
-
event.payload.ok &&
|
|
1960
|
-
event.payload.headSha !== undefined
|
|
1961
|
-
) {
|
|
1962
|
-
settled = event.payload.headSha
|
|
1963
|
-
}
|
|
1964
|
-
}
|
|
1965
|
-
return settled ?? original
|
|
2050
|
+
return recoveryCheckpointReducer.reduce(events)
|
|
1966
2051
|
}
|
|
1967
2052
|
|
|
1968
2053
|
private async recordInfrastructureFailure(
|
|
@@ -1978,7 +2063,7 @@ export class Dispatcher {
|
|
|
1978
2063
|
): Promise<void> {
|
|
1979
2064
|
const execution = [...events].reverse().find((event) => event.type === 'execution.started')
|
|
1980
2065
|
const appended = await appendInfrastructureFailure(
|
|
1981
|
-
{ store: this.
|
|
2066
|
+
{ store: this.store, ids: this.deps.ids },
|
|
1982
2067
|
{
|
|
1983
2068
|
slug,
|
|
1984
2069
|
events,
|
|
@@ -2028,7 +2113,7 @@ export class Dispatcher {
|
|
|
2028
2113
|
// failure propagates — a stale workspace with an unrecorded pending
|
|
2029
2114
|
// request is never reaped (the next tick retries).
|
|
2030
2115
|
events = await settlePublicationBeforeRelease(
|
|
2031
|
-
{ store: this.
|
|
2116
|
+
{ store: this.store, settlePublication: this.settlePublication },
|
|
2032
2117
|
slug,
|
|
2033
2118
|
events,
|
|
2034
2119
|
reason,
|
|
@@ -2047,7 +2132,7 @@ export class Dispatcher {
|
|
|
2047
2132
|
...(open.localPath !== undefined ? { localPath: open.localPath } : {}),
|
|
2048
2133
|
branch: open.branch,
|
|
2049
2134
|
})
|
|
2050
|
-
const cleaned = await this.
|
|
2135
|
+
const cleaned = await this.store.append(slug, {
|
|
2051
2136
|
actor: DISPATCHER,
|
|
2052
2137
|
type: 'infrastructure.cleanup-attempted',
|
|
2053
2138
|
payload: {
|
|
@@ -2060,7 +2145,7 @@ export class Dispatcher {
|
|
|
2060
2145
|
},
|
|
2061
2146
|
})
|
|
2062
2147
|
events.push(cleaned)
|
|
2063
|
-
const released = await this.
|
|
2148
|
+
const released = await this.store.append(slug, {
|
|
2064
2149
|
actor: DISPATCHER,
|
|
2065
2150
|
type: 'workspace.released',
|
|
2066
2151
|
payload: { ref: open.ref, reason },
|
|
@@ -2069,7 +2154,7 @@ export class Dispatcher {
|
|
|
2069
2154
|
return true
|
|
2070
2155
|
} catch (error) {
|
|
2071
2156
|
const message = error instanceof Error ? error.message : String(error)
|
|
2072
|
-
const failed = await this.
|
|
2157
|
+
const failed = await this.store.append(slug, {
|
|
2073
2158
|
actor: DISPATCHER,
|
|
2074
2159
|
type: 'infrastructure.cleanup-attempted',
|
|
2075
2160
|
payload: {
|
|
@@ -2117,7 +2202,7 @@ export class Dispatcher {
|
|
|
2117
2202
|
// workspace still exists. The guard's append failure propagates — the
|
|
2118
2203
|
// release never proceeds past an unrecorded pending request.
|
|
2119
2204
|
events = await settlePublicationBeforeRelease(
|
|
2120
|
-
{ store: this.
|
|
2205
|
+
{ store: this.store, settlePublication: this.settlePublication },
|
|
2121
2206
|
slug,
|
|
2122
2207
|
events,
|
|
2123
2208
|
reason,
|
|
@@ -2132,7 +2217,7 @@ export class Dispatcher {
|
|
|
2132
2217
|
).length + 1
|
|
2133
2218
|
try {
|
|
2134
2219
|
const outcome = await recovery.reap(handle)
|
|
2135
|
-
const cleaned = await this.
|
|
2220
|
+
const cleaned = await this.store.append(slug, {
|
|
2136
2221
|
actor: DISPATCHER,
|
|
2137
2222
|
type: 'infrastructure.cleanup-attempted',
|
|
2138
2223
|
payload: {
|
|
@@ -2147,7 +2232,7 @@ export class Dispatcher {
|
|
|
2147
2232
|
events.push(cleaned)
|
|
2148
2233
|
} catch (error) {
|
|
2149
2234
|
const message = error instanceof Error ? error.message : String(error)
|
|
2150
|
-
const failed = await this.
|
|
2235
|
+
const failed = await this.store.append(slug, {
|
|
2151
2236
|
actor: DISPATCHER,
|
|
2152
2237
|
type: 'infrastructure.cleanup-attempted',
|
|
2153
2238
|
payload: {
|
|
@@ -2172,7 +2257,7 @@ export class Dispatcher {
|
|
|
2172
2257
|
} else {
|
|
2173
2258
|
await owner.release(handle)
|
|
2174
2259
|
}
|
|
2175
|
-
return this.
|
|
2260
|
+
return this.store.append(slug, {
|
|
2176
2261
|
actor: DISPATCHER,
|
|
2177
2262
|
type: 'workspace.released',
|
|
2178
2263
|
payload: { ref: open.ref, reason },
|
|
@@ -2270,7 +2355,8 @@ export class Dispatcher {
|
|
|
2270
2355
|
assets?: PinnedAsset[]
|
|
2271
2356
|
},
|
|
2272
2357
|
): Promise<DispatchOutcome> {
|
|
2273
|
-
const
|
|
2358
|
+
const store = this.store
|
|
2359
|
+
const { config } = this.deps
|
|
2274
2360
|
let stage: EventPayload<'dispatch.failed'>['stage'] = 'create'
|
|
2275
2361
|
try {
|
|
2276
2362
|
let events = await store.getEvents(record.slug)
|
|
@@ -2406,7 +2492,8 @@ export class Dispatcher {
|
|
|
2406
2492
|
report?: TickReport,
|
|
2407
2493
|
launched?: Set<string>,
|
|
2408
2494
|
): Promise<Exclude<DispatchOutcome, 'deferred'>> {
|
|
2409
|
-
const
|
|
2495
|
+
const store = this.store
|
|
2496
|
+
|
|
2410
2497
|
let stage: EventPayload<'dispatch.failed'>['stage'] = 'spec'
|
|
2411
2498
|
let authored = false
|
|
2412
2499
|
try {
|
|
@@ -2487,6 +2574,7 @@ export class Dispatcher {
|
|
|
2487
2574
|
// Discard validation and recovery selection happened before several
|
|
2488
2575
|
// provider awaits. Re-read immediately before launch so a request made
|
|
2489
2576
|
// during those boundaries parks cleanly for the next janitor pass.
|
|
2577
|
+
await store.refreshBuild(record.slug)
|
|
2490
2578
|
const launchState = reduceBuild(await store.getEvents(record.slug))
|
|
2491
2579
|
if (launchState.status !== 'queued' || launchState.discardRequest !== undefined) {
|
|
2492
2580
|
return 'parked'
|
|
@@ -2519,9 +2607,8 @@ export class Dispatcher {
|
|
|
2519
2607
|
paused: boolean,
|
|
2520
2608
|
opts: TickOpts = {},
|
|
2521
2609
|
): Promise<void> {
|
|
2522
|
-
for (const record of await this.
|
|
2610
|
+
for (const record of await this.nonterminalBuilds()) {
|
|
2523
2611
|
if (
|
|
2524
|
-
record.repo !== this.deps.repo ||
|
|
2525
2612
|
record.ticket === undefined ||
|
|
2526
2613
|
launched.has(record.slug) ||
|
|
2527
2614
|
this.continuations.has(record.slug)
|
|
@@ -2530,8 +2617,7 @@ export class Dispatcher {
|
|
|
2530
2617
|
}
|
|
2531
2618
|
// Budget gate: recovery issues store transport calls per build.
|
|
2532
2619
|
if (this.outOfBudget(opts)) break
|
|
2533
|
-
const
|
|
2534
|
-
const state = reduceBuild(events)
|
|
2620
|
+
const { state } = await this.store.buildFacts(record.slug)
|
|
2535
2621
|
if (
|
|
2536
2622
|
state.status !== 'queued' ||
|
|
2537
2623
|
state.discardRequest !== undefined ||
|
|
@@ -2567,14 +2653,15 @@ export class Dispatcher {
|
|
|
2567
2653
|
paused: boolean,
|
|
2568
2654
|
opts: TickOpts = {},
|
|
2569
2655
|
): Promise<void> {
|
|
2570
|
-
const
|
|
2571
|
-
|
|
2572
|
-
|
|
2656
|
+
const store = this.store
|
|
2657
|
+
const { config } = this.deps
|
|
2658
|
+
for (const record of await this.nonterminalBuilds()) {
|
|
2659
|
+
if (launched.has(record.slug)) continue
|
|
2573
2660
|
// Budget gate: resume launches runners (transport-backed work).
|
|
2574
2661
|
if (this.outOfBudget(opts)) break
|
|
2575
2662
|
|
|
2576
|
-
const
|
|
2577
|
-
const state =
|
|
2663
|
+
const facts = await store.buildFacts(record.slug)
|
|
2664
|
+
const state = facts.state
|
|
2578
2665
|
if (
|
|
2579
2666
|
state.status === 'done' ||
|
|
2580
2667
|
state.status === 'aborted' ||
|
|
@@ -2584,7 +2671,7 @@ export class Dispatcher {
|
|
|
2584
2671
|
continue
|
|
2585
2672
|
}
|
|
2586
2673
|
|
|
2587
|
-
const decision =
|
|
2674
|
+
const decision = decideNextFromFacts(state, facts.log, config)
|
|
2588
2675
|
if (decision.kind === 'wait') {
|
|
2589
2676
|
if (decision.reason === 'blocked' && state.openEscalations.length > 0) {
|
|
2590
2677
|
report.blockedDiagnostics.push(
|
|
@@ -2635,8 +2722,9 @@ export class Dispatcher {
|
|
|
2635
2722
|
opts: TickOpts = {},
|
|
2636
2723
|
): Promise<void> {
|
|
2637
2724
|
const now = this.deps.clock().getTime()
|
|
2638
|
-
|
|
2639
|
-
|
|
2725
|
+
// Terminal builds are filtered on the digest before the lease and TTL
|
|
2726
|
+
// checks, so a zero TTL cannot fall through to a log read.
|
|
2727
|
+
for (const record of await this.nonterminalBuilds()) {
|
|
2640
2728
|
if (launched.has(record.slug)) continue
|
|
2641
2729
|
// Budget gate: the sweep re-attaches runners per expired build.
|
|
2642
2730
|
if (this.outOfBudget(opts)) break
|
|
@@ -2645,16 +2733,22 @@ export class Dispatcher {
|
|
|
2645
2733
|
} else if (now - new Date(record.updatedAt).getTime() < this.leaseTtlMs) {
|
|
2646
2734
|
continue // absent lease within the first-claim grace (see DispatcherOpts)
|
|
2647
2735
|
}
|
|
2648
|
-
const
|
|
2649
|
-
const state =
|
|
2736
|
+
const facts = await this.store.buildFacts(record.slug)
|
|
2737
|
+
const state = facts.state
|
|
2650
2738
|
if (
|
|
2651
2739
|
(state.status === 'queued' && state.discardRequest !== undefined) ||
|
|
2652
2740
|
heldByRepositoryPause(state, paused)
|
|
2653
2741
|
) {
|
|
2654
2742
|
continue
|
|
2655
2743
|
}
|
|
2656
|
-
const decision =
|
|
2744
|
+
const decision = decideNextFromFacts(state, facts.log, this.deps.config)
|
|
2657
2745
|
if (state.status === 'done' || state.status === 'aborted') continue
|
|
2746
|
+
// Raw history, read at most once and only by a branch that acts on it.
|
|
2747
|
+
let loaded: AbEvent[] | undefined
|
|
2748
|
+
const history = async (): Promise<AbEvent[]> => {
|
|
2749
|
+
loaded ??= await this.store.getEvents(record.slug)
|
|
2750
|
+
return loaded
|
|
2751
|
+
}
|
|
2658
2752
|
// Liveness gate (durable supervision): a recorded execution the provider
|
|
2659
2753
|
// observes as RUNNING is left completely alone — no settle, no reap, no
|
|
2660
2754
|
// replacement, no re-attach. This is the change that stops a supervisor
|
|
@@ -2662,13 +2756,12 @@ export class Dispatcher {
|
|
|
2662
2756
|
// proved end settles first (completion facts, lease release, publication)
|
|
2663
2757
|
// and then falls through to the normal decision; a provider-proved loss
|
|
2664
2758
|
// falls through to the existing stale path below.
|
|
2665
|
-
const foreignExecution =
|
|
2759
|
+
const foreignExecution = facts.open
|
|
2666
2760
|
// A provider- or log-proved end in the same generation re-attaches in
|
|
2667
2761
|
// place: the guest finished its work, the workspace is still the
|
|
2668
2762
|
// build's, and replacing the environment would discard a healthy
|
|
2669
2763
|
// generation. Only an unproven stale lease or a proved loss reaps.
|
|
2670
|
-
|
|
2671
|
-
let endedInPlace = lastExecution === 'completed'
|
|
2764
|
+
let endedInPlace = facts.lastExecution === 'completed'
|
|
2672
2765
|
if (
|
|
2673
2766
|
foreignExecution !== null &&
|
|
2674
2767
|
foreignExecution.commandId !== undefined &&
|
|
@@ -2677,16 +2770,16 @@ export class Dispatcher {
|
|
|
2677
2770
|
const owner = this.executionOwner(foreignExecution.provider)
|
|
2678
2771
|
const execution = owner?.buildExecution
|
|
2679
2772
|
if (execution?.observe !== undefined) {
|
|
2680
|
-
const settlement = await
|
|
2773
|
+
const settlement = await settleOpenExecution(
|
|
2681
2774
|
{
|
|
2682
|
-
store: this.
|
|
2775
|
+
store: this.store,
|
|
2683
2776
|
execution,
|
|
2684
|
-
...(this.
|
|
2685
|
-
? { settlePublication: this.
|
|
2777
|
+
...(this.settlePublication !== undefined
|
|
2778
|
+
? { settlePublication: this.settlePublication }
|
|
2686
2779
|
: {}),
|
|
2687
2780
|
},
|
|
2688
2781
|
record.slug,
|
|
2689
|
-
|
|
2782
|
+
foreignExecution,
|
|
2690
2783
|
)
|
|
2691
2784
|
if (settlement === 'running') continue
|
|
2692
2785
|
if (settlement === 'settled') {
|
|
@@ -2698,8 +2791,8 @@ export class Dispatcher {
|
|
|
2698
2791
|
const publicationRecoveryDue =
|
|
2699
2792
|
state.status !== 'paused' &&
|
|
2700
2793
|
state.status !== 'blocked' &&
|
|
2701
|
-
(
|
|
2702
|
-
const open =
|
|
2794
|
+
(facts.publication.pending() || facts.publication.abandonedPending())
|
|
2795
|
+
const open = facts.workspace
|
|
2703
2796
|
// A provider- or log-proved END in the same generation re-attaches in
|
|
2704
2797
|
// place: the guest finished its work, the workspace is still the
|
|
2705
2798
|
// build's, and replacing the environment would discard a healthy
|
|
@@ -2712,13 +2805,13 @@ export class Dispatcher {
|
|
|
2712
2805
|
: state.status === 'blocked'
|
|
2713
2806
|
? ('blocked' as const)
|
|
2714
2807
|
: ('replacement' as const)
|
|
2715
|
-
const reaped = await this.reapStaleWorkspace(record.slug,
|
|
2808
|
+
const reaped = await this.reapStaleWorkspace(record.slug, await history(), reason)
|
|
2716
2809
|
if (!reaped) continue
|
|
2717
2810
|
if (!parked) {
|
|
2718
2811
|
// Recovery-capable provider: reap inline (bounded), then kick the
|
|
2719
2812
|
// same background continuation — it provisions and launches without
|
|
2720
2813
|
// blocking this tick. Local providers never enter this branch.
|
|
2721
|
-
const kicked = await this.ensureProvisionContinuation(record,
|
|
2814
|
+
const kicked = await this.ensureProvisionContinuation(record, await history(), 'launch')
|
|
2722
2815
|
if (kicked === 'kicked') {
|
|
2723
2816
|
report.provisioning += 1
|
|
2724
2817
|
report.swept += 1
|
|
@@ -2735,11 +2828,11 @@ export class Dispatcher {
|
|
|
2735
2828
|
// already continued earlier.
|
|
2736
2829
|
if (decision.kind === 'wait' && !publicationRecoveryDue) continue
|
|
2737
2830
|
if (
|
|
2738
|
-
|
|
2831
|
+
facts.workspace === null &&
|
|
2739
2832
|
this.deps.workspaces.recovery !== undefined &&
|
|
2740
|
-
|
|
2833
|
+
(await history()).some((event) => event.type === 'workspace.provisioned')
|
|
2741
2834
|
) {
|
|
2742
|
-
const kicked = await this.ensureProvisionContinuation(record,
|
|
2835
|
+
const kicked = await this.ensureProvisionContinuation(record, await history(), 'launch')
|
|
2743
2836
|
if (kicked === 'kicked') {
|
|
2744
2837
|
report.provisioning += 1
|
|
2745
2838
|
report.swept += 1
|
|
@@ -2764,16 +2857,15 @@ export class Dispatcher {
|
|
|
2764
2857
|
opts: TickOpts = {},
|
|
2765
2858
|
autoMergeSeedFact: AutoMergeDefaultFact | undefined = undefined,
|
|
2766
2859
|
): Promise<void> {
|
|
2767
|
-
const
|
|
2860
|
+
const store = this.store
|
|
2861
|
+
const { tickets, config } = this.deps
|
|
2768
2862
|
// Blocked and paused builds still occupy a slot: their workspaces and
|
|
2769
2863
|
// pending work are live, only waiting on a human. Capacity is per repo
|
|
2770
2864
|
// (§16.1) — another repo's builds never consume this repo's slots.
|
|
2771
2865
|
let active = 0
|
|
2772
2866
|
const activeTicketIds = new Set<string>()
|
|
2773
|
-
for (const record of await
|
|
2774
|
-
if (
|
|
2775
|
-
const state = reduceBuild(await store.getEvents(record.slug))
|
|
2776
|
-
if (state.status !== 'done' && state.status !== 'aborted') {
|
|
2867
|
+
for (const { record, digest } of await this.repoBuildDigests()) {
|
|
2868
|
+
if (digest.terminal === undefined) {
|
|
2777
2869
|
active += 1
|
|
2778
2870
|
if (record.ticket) activeTicketIds.add(record.ticket.id)
|
|
2779
2871
|
}
|
|
@@ -2784,9 +2876,11 @@ export class Dispatcher {
|
|
|
2784
2876
|
// standing queue-depth report and must stay honest exactly when the
|
|
2785
2877
|
// dispatcher is saturated. Sources that keep a claimed ticket in the
|
|
2786
2878
|
// ready state are deduped against the active builds embodying them.
|
|
2787
|
-
|
|
2879
|
+
await this.store.refreshJournal()
|
|
2880
|
+
const journalBeforeReady = this.store.recordedJournal()
|
|
2788
2881
|
const listing = await tickets.listReady(readyCriteria(config))
|
|
2789
|
-
|
|
2882
|
+
await this.store.refreshJournal()
|
|
2883
|
+
const journalAfterReady = this.store.recordedJournal()
|
|
2790
2884
|
const creationHolds = new Map(
|
|
2791
2885
|
harvestCreationsDuringReadyScan(journalBeforeReady, journalAfterReady).map((creation) => [
|
|
2792
2886
|
creation.creationKey.toLowerCase(),
|
|
@@ -3023,7 +3117,7 @@ export class Dispatcher {
|
|
|
3023
3117
|
/** Valid bounded base, deduped store-wide with -2/-3/… suffixes. */
|
|
3024
3118
|
private async uniqueSlug(base: string): Promise<string> {
|
|
3025
3119
|
let slug = base
|
|
3026
|
-
for (let n = 2; (await this.
|
|
3120
|
+
for (let n = 2; (await this.store.getBuild(slug)) !== null; n += 1) {
|
|
3027
3121
|
slug = `${base}-${n}`
|
|
3028
3122
|
}
|
|
3029
3123
|
return slug
|