@defrex/autobuild 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/SPEC.md +237 -23
- package/bin/ab-dispatch-kernel.ts +1 -1
- package/docs/architecture.md +66 -4
- package/docs/configuration.md +137 -19
- package/docs/hosted-dispatcher.md +17 -2
- package/docs/hosted-service.md +214 -0
- package/docs/mcp.md +8 -2
- package/docs/operator-api.md +107 -7
- package/docs/remote-store-protocol.md +189 -144
- package/docs/remote-ticket-protocol.md +111 -18
- package/docs/setup.md +173 -16
- package/docs/ticket-dependencies.md +1 -1
- package/package.json +7 -6
- package/packages/core/src/cli/attention.ts +209 -0
- package/packages/core/src/cli/binary.ts +3 -3
- package/packages/core/src/cli/configured-store.ts +17 -0
- package/packages/core/src/cli/connect.ts +261 -0
- package/packages/core/src/cli/dashboard/model.ts +8 -5
- package/packages/core/src/cli/dashboard/render.ts +37 -7
- package/packages/core/src/cli/dispatch-frontend.ts +16 -1
- package/packages/core/src/cli/dispatch-process.ts +25 -7
- package/packages/core/src/cli/dispatch.ts +247 -47
- package/packages/core/src/cli/dispatcher-liveness.ts +91 -0
- package/packages/core/src/cli/help.ts +45 -13
- package/packages/core/src/cli/init-validation.ts +27 -2
- package/packages/core/src/cli/main.ts +41 -15
- package/packages/core/src/cli/mcp.ts +7 -1
- package/packages/core/src/cli/pr-summary-publication.ts +220 -0
- package/packages/core/src/cli/pr-summary.ts +2 -0
- package/packages/core/src/cli/project-credentials.ts +132 -0
- package/packages/core/src/cli/repo-state.ts +15 -2
- package/packages/core/src/cli/repository-status.ts +45 -5
- package/packages/core/src/cli/self-update.ts +10 -0
- package/packages/core/src/cli/store-opening.ts +22 -14
- package/packages/core/src/cli/terminals.ts +0 -10
- package/packages/core/src/cli/ticket.ts +8 -1
- package/packages/core/src/cli/upgrade-compatibility.ts +80 -0
- package/packages/core/src/cli/wait.ts +84 -17
- package/packages/core/src/cli/watch.ts +350 -68
- package/packages/core/src/config/live.ts +4 -0
- package/packages/core/src/config/schema.ts +65 -11
- package/packages/core/src/connection.ts +86 -0
- package/packages/core/src/events/catalog.ts +10 -2
- package/packages/core/src/events/globs.ts +8 -3
- package/packages/core/src/events/payloads.ts +44 -0
- package/packages/core/src/events/repository.ts +64 -2
- package/packages/core/src/integration/harness.ts +2 -1
- package/packages/core/src/kernel/__snapshots__/reducer-registry.test.ts.snap +148 -83
- package/packages/core/src/kernel/dispatch-status.ts +9 -1
- package/packages/core/src/kernel/dispatcher-liveness.ts +70 -0
- package/packages/core/src/kernel/engine.ts +24 -0
- package/packages/core/src/kernel/generators/build-log.ts +10 -0
- package/packages/core/src/kernel/log-index.ts +17 -0
- package/packages/core/src/kernel/pr-summary.ts +89 -0
- package/packages/core/src/kernel/reducer-registry.ts +12 -0
- package/packages/core/src/kernel/reducer.ts +8 -0
- package/packages/core/src/kernel/ticket-source-health.ts +30 -0
- package/packages/core/src/ontology.ts +1 -0
- package/packages/core/src/operator/index.ts +7 -0
- package/packages/core/src/operator/query.ts +44 -19
- package/packages/core/src/operator/ticket-source-backend.ts +1 -0
- package/packages/core/src/operator/tickets.ts +6 -8
- package/packages/core/src/plugin-sdk/index.ts +1 -0
- package/packages/core/src/plugins/manifest.ts +5 -3
- package/packages/core/src/plugins/runtimes.ts +3 -1
- package/packages/core/src/ports/forge/fake.ts +13 -1
- package/packages/core/src/ports/forge/github-transport.ts +36 -1
- package/packages/core/src/ports/forge/github.ts +29 -5
- package/packages/core/src/ports/forge/local-git.ts +12 -4
- package/packages/core/src/ports/forge/summary-marker.ts +8 -0
- package/packages/core/src/ports/runner/session-env.ts +1 -0
- package/packages/core/src/ports/tickets/contract.ts +25 -0
- package/packages/core/src/ports/tickets/create.ts +11 -3
- package/packages/core/src/ports/tickets/fake.ts +10 -8
- package/packages/core/src/ports/tickets/file.ts +1 -0
- package/packages/core/src/ports/tickets/hosted-index.ts +0 -3
- package/packages/core/src/ports/tickets/linear.ts +241 -49
- package/packages/core/src/ports/tickets/remote-protocol.ts +3 -0
- package/packages/core/src/ports/tickets/remote.ts +1 -0
- package/packages/core/src/ports/types.ts +17 -0
- package/packages/core/src/ports/workspace/build-execution.ts +5 -1
- package/packages/core/src/ports/workspace/create.ts +11 -2
- package/packages/core/src/ports/workspace/distribution-archive.ts +3 -5
- package/packages/core/src/ports/workspace/harvest-execution.ts +3 -1
- package/packages/core/src/ports/workspace/local-build-execution.ts +8 -1
- package/packages/core/src/ports/workspace/vercel-sandbox.ts +47 -19
- package/packages/core/src/processes/build-execution-state.ts +1 -1
- package/packages/core/src/processes/build-runner.ts +87 -26
- package/packages/core/src/processes/dispatch-failure-budget.ts +83 -0
- package/packages/core/src/processes/dispatcher.ts +336 -83
- package/packages/core/src/processes/harvest-child.ts +1 -0
- package/packages/core/src/processes/harvest-execution-state.ts +19 -2
- package/packages/core/src/processes/harvest-runner.ts +142 -100
- package/packages/core/src/processes/phase-failure-budget.ts +1 -0
- package/packages/core/src/processes/publication-loss.ts +18 -1
- package/packages/core/src/processes/repo-view.ts +2 -1
- package/packages/core/src/store/adapter.ts +1 -0
- package/packages/core/src/store/contract.ts +109 -0
- package/packages/core/src/store/digest.ts +14 -2
- package/packages/core/src/store/local/store.ts +18 -5
- package/packages/core/src/store/remote/client.ts +1 -1
- package/packages/core/src/store/remote/compatibility.ts +90 -0
- package/packages/core/src/store/remote/credentials.ts +28 -0
- package/packages/core/src/store/remote/index.ts +14 -1
- package/packages/core/src/store/remote/protocol.ts +26 -0
- package/packages/core/src/store/remote/token.ts +74 -13
- package/packages/core/src/store/repo-state-events.ts +49 -59
- package/packages/core/src/store/types.ts +5 -1
- package/packages/core/src/testing/index.ts +7 -0
- package/skills/guide/SKILL.md +175 -26
- package/skills/guide/references/operator-api.md +135 -13
- package/skills/guide/references/plugin-authoring.md +10 -2
- package/skills/guide/references/remote-store-protocol.md +189 -144
- package/skills/guide/references/remote-ticket-protocol.md +111 -18
- package/skills/guide/references/setup.md +173 -16
- package/skills/guide/references/ticket-dependencies.md +1 -1
- package/skills/operate/SKILL.md +159 -34
- package/templates/hosted-service/package.json +13 -0
- package/templates/hosted-service/vercel.json +7 -0
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import { ensureDispatchBudget } from './dispatch-failure-budget'
|
|
2
|
+
import {
|
|
3
|
+
reduceTicketSourceHealth,
|
|
4
|
+
type TicketSourceDegradation,
|
|
5
|
+
} from '../kernel/ticket-source-health'
|
|
1
6
|
/**
|
|
2
7
|
* The dispatcher process (SPEC §3.3, §12): watches the TicketSource for
|
|
3
8
|
* Ready tickets, claims, provisions a workspace, and launches build-runners
|
|
@@ -36,6 +41,7 @@
|
|
|
36
41
|
* validation and fallback remain here. The supervising kernel reserves each
|
|
37
42
|
* execution lease before launch; the child renews that same-holder lease.
|
|
38
43
|
*/
|
|
44
|
+
import { createHash } from 'node:crypto'
|
|
39
45
|
import type { Config } from '../config/schema'
|
|
40
46
|
import type { LanguageModel } from 'ai'
|
|
41
47
|
import { runOrchestratorTickStep } from './orchestrator-tick'
|
|
@@ -74,6 +80,7 @@ import type {
|
|
|
74
80
|
TicketSource,
|
|
75
81
|
WorkspaceProvider,
|
|
76
82
|
} from '../ports/types'
|
|
83
|
+
import { TicketSourceRateLimitError } from '../ports/types'
|
|
77
84
|
import type { Exec } from '../ports/workspace/git-worktree'
|
|
78
85
|
import type { ArtifactMeta, BuildDigest, BuildRecord, BuildStore, Clock } from '../store/types'
|
|
79
86
|
import { samplePinnedAssets } from '../store/ticket-assets'
|
|
@@ -201,7 +208,9 @@ function findCycle(
|
|
|
201
208
|
nodes: Map<string, DependencyState>,
|
|
202
209
|
explored: Set<string>,
|
|
203
210
|
): string[] | null {
|
|
204
|
-
|
|
211
|
+
const node = nodes.get(from)
|
|
212
|
+
if (!node?.exists || node.resolved) return null
|
|
213
|
+
for (const next of node.blockedBy) {
|
|
205
214
|
const seen = path.indexOf(next)
|
|
206
215
|
if (seen !== -1) return [...path.slice(seen), next]
|
|
207
216
|
if (explored.has(next)) continue
|
|
@@ -301,7 +310,7 @@ export function validateSlugCandidate(candidate: string | null | undefined): str
|
|
|
301
310
|
/** Counts per action, for observability and tests. An idempotent re-run of
|
|
302
311
|
* `tick()` over unchanged state reports all zeroes — except invalid-record,
|
|
303
312
|
* in-flight-creation, and dependency fields, which are standing queue reports rather than records of
|
|
304
|
-
* action. They repeat
|
|
313
|
+
* action. They repeat between scans and refresh on completed scans, because that is the only
|
|
305
314
|
* place the source problems are visible without provider inspection. */
|
|
306
315
|
export interface TickReport {
|
|
307
316
|
/** Janitor (§15.7): builds completed as merged. */
|
|
@@ -354,13 +363,15 @@ export interface TickReport {
|
|
|
354
363
|
/** Standing queue depth: ready tickets still waiting after this tick —
|
|
355
364
|
* excludes tickets already embodied by an active build, and ones that left
|
|
356
365
|
* the queue this tick (dispatched, bounced, or claimed elsewhere). Like the
|
|
357
|
-
* invalid/dependency fields it
|
|
366
|
+
* invalid/dependency fields it repeats between completed scans. */
|
|
358
367
|
queued: number
|
|
359
368
|
/** Ready tickets withheld while Autobuild's own external creation is still
|
|
360
369
|
* between durable reservation and settled blocker recording. */
|
|
361
370
|
creationWithheld: number
|
|
362
371
|
/** One line per creation hold, with Harvest run/proposal provenance. */
|
|
363
372
|
creationDiagnostics: string[]
|
|
373
|
+
/** Ready tickets whose dependency reads failed rather than yielding a verdict. */
|
|
374
|
+
dependencyCheckFailed: number
|
|
364
375
|
/** Dependency gate (§13): ready tickets held back by unresolved blockers. */
|
|
365
376
|
dependencyBlocked: number
|
|
366
377
|
/** One line per held ticket naming its unresolved blockers — the operator's
|
|
@@ -410,6 +421,7 @@ export function emptyTickReport(): TickReport {
|
|
|
410
421
|
queued: 0,
|
|
411
422
|
creationWithheld: 0,
|
|
412
423
|
creationDiagnostics: [],
|
|
424
|
+
dependencyCheckFailed: 0,
|
|
413
425
|
dependencyBlocked: 0,
|
|
414
426
|
dependencyDiagnostics: [],
|
|
415
427
|
harvestStarted: 0,
|
|
@@ -654,7 +666,15 @@ export class Dispatcher {
|
|
|
654
666
|
return this.repoOrigin
|
|
655
667
|
}
|
|
656
668
|
private readonly leaseTtlMs: number
|
|
657
|
-
private
|
|
669
|
+
private get doneState(): string {
|
|
670
|
+
return (
|
|
671
|
+
this.deps.opts?.doneState ??
|
|
672
|
+
(this.deps.config.tickets.source === 'hosted'
|
|
673
|
+
? this.deps.config.tickets.doneState
|
|
674
|
+
: undefined) ??
|
|
675
|
+
'Done'
|
|
676
|
+
)
|
|
677
|
+
}
|
|
658
678
|
private readonly slugNamingTimeoutMs: number
|
|
659
679
|
private readonly maxHarvestRecoveryAttempts: number
|
|
660
680
|
private readonly provisionStaleMs: number
|
|
@@ -684,7 +704,6 @@ export class Dispatcher {
|
|
|
684
704
|
? deps.store
|
|
685
705
|
: new RepoViewStore(deps.store, { repo: deps.repo })
|
|
686
706
|
this.leaseTtlMs = deps.opts?.leaseTtlMs ?? 0
|
|
687
|
-
this.doneState = deps.opts?.doneState ?? 'Done'
|
|
688
707
|
this.slugNamingTimeoutMs = deps.opts?.slugNamingTimeoutMs ?? DEFAULT_SLUG_NAMING_TIMEOUT_MS
|
|
689
708
|
this.maxHarvestRecoveryAttempts =
|
|
690
709
|
deps.opts?.maxHarvestRecoveryAttempts ?? DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS
|
|
@@ -729,6 +748,7 @@ export class Dispatcher {
|
|
|
729
748
|
}
|
|
730
749
|
}
|
|
731
750
|
const report = emptyTickReport()
|
|
751
|
+
this.restoreScan(report)
|
|
732
752
|
/** Builds already launched this tick — a janitor/startup re-attach must
|
|
733
753
|
* not be doubled by the sweep, nor a fresh dispatch by anything. */
|
|
734
754
|
const launched = new Set<string>()
|
|
@@ -893,6 +913,81 @@ export class Dispatcher {
|
|
|
893
913
|
.map(({ record }) => record)
|
|
894
914
|
}
|
|
895
915
|
|
|
916
|
+
private async sourceCondition(
|
|
917
|
+
next?: TicketSourceDegradation,
|
|
918
|
+
successfulOperation?: 'ready-scan' | 'dependency-check',
|
|
919
|
+
): Promise<void> {
|
|
920
|
+
await this.store.refreshJournal()
|
|
921
|
+
const current = reduceTicketSourceHealth(this.store.recordedJournal())
|
|
922
|
+
if (next !== undefined && current?.cause === next.cause && current.source === next.source) {
|
|
923
|
+
if (
|
|
924
|
+
next.cause === 'rate-limited' &&
|
|
925
|
+
next.operation === 'dependency-check' &&
|
|
926
|
+
current.operation !== 'dependency-check'
|
|
927
|
+
) {
|
|
928
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
929
|
+
actor: DISPATCHER,
|
|
930
|
+
type: 'dispatcher.ticket-source-dependency-limited',
|
|
931
|
+
payload: { source: current.source },
|
|
932
|
+
})
|
|
933
|
+
}
|
|
934
|
+
return
|
|
935
|
+
}
|
|
936
|
+
if (
|
|
937
|
+
next === undefined &&
|
|
938
|
+
current?.operation === 'dependency-check' &&
|
|
939
|
+
successfulOperation !== 'dependency-check'
|
|
940
|
+
)
|
|
941
|
+
return
|
|
942
|
+
if (current !== undefined) {
|
|
943
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
944
|
+
actor: DISPATCHER,
|
|
945
|
+
type: 'dispatcher.ticket-source-recovered',
|
|
946
|
+
payload: { source: current.source, cause: current.cause, operation: current.operation },
|
|
947
|
+
})
|
|
948
|
+
}
|
|
949
|
+
if (next !== undefined) {
|
|
950
|
+
await this.store.ensureRepo(this.deps.repo)
|
|
951
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
952
|
+
actor: DISPATCHER,
|
|
953
|
+
type: 'dispatcher.ticket-source-degraded',
|
|
954
|
+
payload: next,
|
|
955
|
+
})
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
private async sourceFailure(
|
|
960
|
+
error: unknown,
|
|
961
|
+
operation: 'ready-scan' | 'dependency-check',
|
|
962
|
+
): Promise<void> {
|
|
963
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
964
|
+
await this.sourceCondition(
|
|
965
|
+
error instanceof TicketSourceRateLimitError
|
|
966
|
+
? {
|
|
967
|
+
source: error.source,
|
|
968
|
+
cause: 'rate-limited',
|
|
969
|
+
operation,
|
|
970
|
+
message,
|
|
971
|
+
resetAt: new Date(error.resetAt).toISOString(),
|
|
972
|
+
}
|
|
973
|
+
: {
|
|
974
|
+
source: this.deps.tickets.name,
|
|
975
|
+
cause: 'ready-scan-failed',
|
|
976
|
+
operation,
|
|
977
|
+
message: message || 'Ready scan failed',
|
|
978
|
+
},
|
|
979
|
+
)
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
private async listReady() {
|
|
983
|
+
try {
|
|
984
|
+
return await this.deps.tickets.listReady(readyCriteria(this.deps.config))
|
|
985
|
+
} catch (error) {
|
|
986
|
+
await this.sourceFailure(error, 'ready-scan')
|
|
987
|
+
throw error
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
|
|
896
991
|
/** Observation-only ready listing for Store-backed frontends while intake is
|
|
897
992
|
* disabled. It shares the active-ticket exclusion with dispatch but performs
|
|
898
993
|
* no dependency fetch, claim, bounce, build, workspace, or launch side
|
|
@@ -910,14 +1005,109 @@ export class Dispatcher {
|
|
|
910
1005
|
activeTicketIds.add(record.ticket.id)
|
|
911
1006
|
}
|
|
912
1007
|
}
|
|
913
|
-
const
|
|
1008
|
+
const report = emptyTickReport()
|
|
1009
|
+
this.restoreScan(report)
|
|
1010
|
+
const checkpoint = await this.scanCheckpoint('observation')
|
|
1011
|
+
if (checkpoint.due) {
|
|
1012
|
+
await this.store.refreshJournal()
|
|
1013
|
+
const listing = await this.listReady()
|
|
1014
|
+
await this.store.refreshJournal()
|
|
1015
|
+
report.queued = listing.tickets.filter((ticket) => !activeTicketIds.has(ticket.ref.id)).length
|
|
1016
|
+
report.invalidTickets = listing.diagnostics.length
|
|
1017
|
+
report.ticketDiagnostics = [...listing.diagnostics]
|
|
1018
|
+
report.dependencyBlocked = 0
|
|
1019
|
+
report.dependencyDiagnostics = []
|
|
1020
|
+
report.creationWithheld = 0
|
|
1021
|
+
report.creationDiagnostics = []
|
|
1022
|
+
await this.sourceCondition(undefined, 'ready-scan')
|
|
1023
|
+
await this.completeScan(checkpoint, report)
|
|
1024
|
+
}
|
|
1025
|
+
return {
|
|
1026
|
+
queued: report.queued,
|
|
1027
|
+
invalidTickets: report.invalidTickets,
|
|
1028
|
+
ticketDiagnostics: report.ticketDiagnostics,
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
private latestScan() {
|
|
1033
|
+
return this.store
|
|
1034
|
+
.recordedJournal()
|
|
1035
|
+
.findLast(
|
|
1036
|
+
(event): event is Extract<RepositoryEvent, { type: 'dispatcher.ticket-scan-completed' }> =>
|
|
1037
|
+
event.type === 'dispatcher.ticket-scan-completed',
|
|
1038
|
+
)?.payload
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
private restoreScan(report: TickReport): void {
|
|
1042
|
+
const previous = this.latestScan()
|
|
1043
|
+
if (previous === undefined) return
|
|
1044
|
+
for (const key of [
|
|
1045
|
+
'queued',
|
|
1046
|
+
'invalidTickets',
|
|
1047
|
+
'dependencyBlocked',
|
|
1048
|
+
'creationWithheld',
|
|
1049
|
+
] as const)
|
|
1050
|
+
report[key] = previous[key]
|
|
1051
|
+
for (const key of [
|
|
1052
|
+
'ticketDiagnostics',
|
|
1053
|
+
'dependencyDiagnostics',
|
|
1054
|
+
'creationDiagnostics',
|
|
1055
|
+
] as const)
|
|
1056
|
+
report[key] = [...previous[key]]
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
private async scanCheckpoint(mode: 'admission' | 'observation') {
|
|
1060
|
+
const digest = (await this.repoBuildDigests())
|
|
1061
|
+
.filter(({ digest }) => digest.readinessChangeSeq !== undefined)
|
|
1062
|
+
.map(({ record, digest }) => [record.slug, digest.readinessChangeSeq] as const)
|
|
1063
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
1064
|
+
const hash = (value: unknown) =>
|
|
1065
|
+
createHash('sha256').update(JSON.stringify(value)).digest('hex')
|
|
1066
|
+
const readinessFingerprint = hash(digest)
|
|
1067
|
+
const criteriaFingerprint = hash(readyCriteria(this.deps.config))
|
|
1068
|
+
const startedAt = this.deps.clock().toISOString()
|
|
1069
|
+
const previous = this.latestScan()
|
|
1070
|
+
const elapsed =
|
|
1071
|
+
previous === undefined ? Infinity : Date.parse(startedAt) - Date.parse(previous.startedAt)
|
|
914
1072
|
return {
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
1073
|
+
startedAt,
|
|
1074
|
+
mode,
|
|
1075
|
+
readinessFingerprint,
|
|
1076
|
+
criteriaFingerprint,
|
|
1077
|
+
due:
|
|
1078
|
+
previous === undefined ||
|
|
1079
|
+
elapsed < 0 ||
|
|
1080
|
+
elapsed >= this.deps.config.ticketScanIntervalSeconds * 1000 ||
|
|
1081
|
+
previous.readinessFingerprint !== readinessFingerprint ||
|
|
1082
|
+
previous.criteriaFingerprint !== criteriaFingerprint ||
|
|
1083
|
+
(mode === 'admission' && previous.mode === 'observation'),
|
|
918
1084
|
}
|
|
919
1085
|
}
|
|
920
1086
|
|
|
1087
|
+
private async completeScan(
|
|
1088
|
+
checkpoint: Awaited<ReturnType<Dispatcher['scanCheckpoint']>>,
|
|
1089
|
+
report: TickReport,
|
|
1090
|
+
): Promise<void> {
|
|
1091
|
+
await this.store.ensureRepo(this.deps.repo)
|
|
1092
|
+
await this.store.appendRepo(this.deps.repo, {
|
|
1093
|
+
actor: DISPATCHER,
|
|
1094
|
+
type: 'dispatcher.ticket-scan-completed',
|
|
1095
|
+
payload: {
|
|
1096
|
+
startedAt: checkpoint.startedAt,
|
|
1097
|
+
mode: checkpoint.mode,
|
|
1098
|
+
readinessFingerprint: checkpoint.readinessFingerprint,
|
|
1099
|
+
criteriaFingerprint: checkpoint.criteriaFingerprint,
|
|
1100
|
+
queued: report.queued,
|
|
1101
|
+
invalidTickets: report.invalidTickets,
|
|
1102
|
+
ticketDiagnostics: report.ticketDiagnostics.slice(0, 1000),
|
|
1103
|
+
dependencyBlocked: report.dependencyBlocked,
|
|
1104
|
+
dependencyDiagnostics: report.dependencyDiagnostics.slice(0, 1000),
|
|
1105
|
+
creationWithheld: report.creationWithheld,
|
|
1106
|
+
creationDiagnostics: report.creationDiagnostics.slice(0, 1000),
|
|
1107
|
+
},
|
|
1108
|
+
})
|
|
1109
|
+
}
|
|
1110
|
+
|
|
921
1111
|
/** Apply the newest durable `dispatcher.auto-merge-default-set` fact to every
|
|
922
1112
|
* current non-terminal build of this repository: on — request consent; off —
|
|
923
1113
|
* cancel it. Attribution is the toggling human's own actor, exactly as if
|
|
@@ -1326,6 +1516,10 @@ export class Dispatcher {
|
|
|
1326
1516
|
generation,
|
|
1327
1517
|
seed,
|
|
1328
1518
|
tail,
|
|
1519
|
+
}).catch(() => {
|
|
1520
|
+
// Detached work must never reject into the dispatcher process. Failure
|
|
1521
|
+
// recording can itself fail on a store outage; durable boundaries and
|
|
1522
|
+
// lease expiry let the next tick repair or retry the unfinished work.
|
|
1329
1523
|
})
|
|
1330
1524
|
return 'kicked'
|
|
1331
1525
|
}
|
|
@@ -1360,6 +1554,7 @@ export class Dispatcher {
|
|
|
1360
1554
|
const slug = entry.slug
|
|
1361
1555
|
let leaseClaimed = false
|
|
1362
1556
|
let heartbeat: ReturnType<typeof setInterval> | undefined
|
|
1557
|
+
let boundary: 'workspace' | 'read' | 'dispatch' | 'launch' = 'workspace'
|
|
1363
1558
|
const stage: EventPayload<'dispatch.failed'>['stage'] = 'workspace'
|
|
1364
1559
|
try {
|
|
1365
1560
|
if (entry.controller.signal.aborted) {
|
|
@@ -1424,12 +1619,19 @@ export class Dispatcher {
|
|
|
1424
1619
|
await store.releaseLease(slug, entry.instance)
|
|
1425
1620
|
leaseClaimed = false
|
|
1426
1621
|
if (entry.controller.signal.aborted) return
|
|
1622
|
+
boundary = 'read'
|
|
1427
1623
|
const record = await store.getBuild(slug)
|
|
1428
1624
|
if (record === null) return
|
|
1429
|
-
await (
|
|
1430
|
-
|
|
1431
|
-
|
|
1625
|
+
const queued = reduceBuild(await store.getEvents(slug)).status === 'queued'
|
|
1626
|
+
boundary = input.tail === 'dispatch' || queued ? 'dispatch' : 'launch'
|
|
1627
|
+
if (boundary === 'dispatch') await this.completeDispatchBoundaries(record, input.seed)
|
|
1628
|
+
else await this.deps.launchRunner(slug)
|
|
1432
1629
|
} catch (error) {
|
|
1630
|
+
// Dispatch boundaries own their failure facts and exhaustion repair.
|
|
1631
|
+
// A failed tail read/write leaves durable unfinished work for the next
|
|
1632
|
+
// tick; it is not another provider workspace failure. Replacement
|
|
1633
|
+
// runner launch errors retain the existing infrastructure-budget path.
|
|
1634
|
+
if (boundary === 'read' || boundary === 'dispatch') return
|
|
1433
1635
|
const events = await store.getEvents(slug)
|
|
1434
1636
|
if (workspaces.recovery !== undefined) {
|
|
1435
1637
|
const open = openWorkspace(events)
|
|
@@ -2303,6 +2505,16 @@ export class Dispatcher {
|
|
|
2303
2505
|
return sha
|
|
2304
2506
|
}
|
|
2305
2507
|
|
|
2508
|
+
private dispatchBudget(slug: string): Promise<boolean> {
|
|
2509
|
+
return ensureDispatchBudget(
|
|
2510
|
+
this.store,
|
|
2511
|
+
this.deps.ids,
|
|
2512
|
+
slug,
|
|
2513
|
+
this.deps.config.policy.maxDispatchAttempts,
|
|
2514
|
+
this.deps.workspaces.recovery !== undefined,
|
|
2515
|
+
)
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2306
2518
|
// ── b. Interrupted dispatch recovery ───────────────────────────────────────
|
|
2307
2519
|
|
|
2308
2520
|
/** The claim-time `build.created` payload. Assets are omitted when the ticket
|
|
@@ -2357,6 +2569,7 @@ export class Dispatcher {
|
|
|
2357
2569
|
): Promise<DispatchOutcome> {
|
|
2358
2570
|
const store = this.store
|
|
2359
2571
|
const { config } = this.deps
|
|
2572
|
+
if (!(await this.dispatchBudget(record.slug))) return 'parked'
|
|
2360
2573
|
let stage: EventPayload<'dispatch.failed'>['stage'] = 'create'
|
|
2361
2574
|
try {
|
|
2362
2575
|
let events = await store.getEvents(record.slug)
|
|
@@ -2446,8 +2659,6 @@ export class Dispatcher {
|
|
|
2446
2659
|
} satisfies EventWrite<'workspace.provisioned'>)
|
|
2447
2660
|
events = await store.getEvents(record.slug)
|
|
2448
2661
|
}
|
|
2449
|
-
|
|
2450
|
-
return await this.completeDispatchBoundaries(record, seed, report, launched)
|
|
2451
2662
|
} catch (error) {
|
|
2452
2663
|
const events = await store.getEvents(record.slug)
|
|
2453
2664
|
if (stage === 'workspace' && this.deps.workspaces.recovery !== undefined) {
|
|
@@ -2472,8 +2683,10 @@ export class Dispatcher {
|
|
|
2472
2683
|
},
|
|
2473
2684
|
})
|
|
2474
2685
|
report.dispatchFailed += 1
|
|
2686
|
+
await this.dispatchBudget(record.slug)
|
|
2475
2687
|
return 'failed'
|
|
2476
2688
|
}
|
|
2689
|
+
return this.completeDispatchBoundaries(record, seed, report, launched)
|
|
2477
2690
|
}
|
|
2478
2691
|
|
|
2479
2692
|
/** The post-workspace dispatch boundaries — spec, comment, launch — shared
|
|
@@ -2494,10 +2707,13 @@ export class Dispatcher {
|
|
|
2494
2707
|
): Promise<Exclude<DispatchOutcome, 'deferred'>> {
|
|
2495
2708
|
const store = this.store
|
|
2496
2709
|
|
|
2710
|
+
if (!(await this.dispatchBudget(record.slug))) return 'parked'
|
|
2497
2711
|
let stage: EventPayload<'dispatch.failed'>['stage'] = 'spec'
|
|
2498
2712
|
let authored = false
|
|
2499
2713
|
try {
|
|
2500
2714
|
const events = await store.getEvents(record.slug)
|
|
2715
|
+
const state = reduceBuild(events)
|
|
2716
|
+
if (state.status !== 'queued' || state.discardRequest !== undefined) return 'parked'
|
|
2501
2717
|
if (
|
|
2502
2718
|
!events.some((event) => event.type === 'spec.imported' || event.type === 'spec.authored')
|
|
2503
2719
|
) {
|
|
@@ -2597,6 +2813,7 @@ export class Dispatcher {
|
|
|
2597
2813
|
},
|
|
2598
2814
|
})
|
|
2599
2815
|
if (report !== undefined) report.dispatchFailed += 1
|
|
2816
|
+
await this.dispatchBudget(record.slug)
|
|
2600
2817
|
return 'failed'
|
|
2601
2818
|
}
|
|
2602
2819
|
}
|
|
@@ -2628,9 +2845,11 @@ export class Dispatcher {
|
|
|
2628
2845
|
// Queued itself means runner attachment is still missing. Always enter
|
|
2629
2846
|
// the fact-driven helper: it skips every completed boundary and retries
|
|
2630
2847
|
// only the final launch when setup is already durable.
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2848
|
+
const outcome = await this.attemptDispatchCompletion(record, launched, report)
|
|
2849
|
+
if (outcome === 'completed') report.recovered += 1
|
|
2850
|
+
// A failed boundary owns this tick's attempt; startup and lease recovery
|
|
2851
|
+
// must not bypass the unfinished dispatch boundary later in the tick.
|
|
2852
|
+
if (outcome === 'failed') launched.add(record.slug)
|
|
2634
2853
|
}
|
|
2635
2854
|
}
|
|
2636
2855
|
|
|
@@ -2671,6 +2890,12 @@ export class Dispatcher {
|
|
|
2671
2890
|
continue
|
|
2672
2891
|
}
|
|
2673
2892
|
|
|
2893
|
+
if (
|
|
2894
|
+
state.status === 'queued' &&
|
|
2895
|
+
!state.pendingCommands.some((command) => command.command === 'abort') &&
|
|
2896
|
+
!(await this.dispatchBudget(record.slug))
|
|
2897
|
+
)
|
|
2898
|
+
continue
|
|
2674
2899
|
const decision = decideNextFromFacts(state, facts.log, config)
|
|
2675
2900
|
if (decision.kind === 'wait') {
|
|
2676
2901
|
if (decision.reason === 'blocked' && state.openEscalations.length > 0) {
|
|
@@ -2741,6 +2966,12 @@ export class Dispatcher {
|
|
|
2741
2966
|
) {
|
|
2742
2967
|
continue
|
|
2743
2968
|
}
|
|
2969
|
+
if (
|
|
2970
|
+
state.status === 'queued' &&
|
|
2971
|
+
!state.pendingCommands.some((command) => command.command === 'abort') &&
|
|
2972
|
+
!(await this.dispatchBudget(record.slug))
|
|
2973
|
+
)
|
|
2974
|
+
continue
|
|
2744
2975
|
const decision = decideNextFromFacts(state, facts.log, this.deps.config)
|
|
2745
2976
|
if (state.status === 'done' || state.status === 'aborted') continue
|
|
2746
2977
|
// Raw history, read at most once and only by a branch that acts on it.
|
|
@@ -2859,6 +3090,13 @@ export class Dispatcher {
|
|
|
2859
3090
|
): Promise<void> {
|
|
2860
3091
|
const store = this.store
|
|
2861
3092
|
const { tickets, config } = this.deps
|
|
3093
|
+
const checkpoint = await this.scanCheckpoint('admission')
|
|
3094
|
+
if (!checkpoint.due) return
|
|
3095
|
+
// Clear only standing measurement fields; action counters belong to this tick.
|
|
3096
|
+
report.queued = report.invalidTickets = report.dependencyBlocked = report.creationWithheld = 0
|
|
3097
|
+
report.ticketDiagnostics = []
|
|
3098
|
+
report.dependencyDiagnostics = []
|
|
3099
|
+
report.creationDiagnostics = []
|
|
2862
3100
|
// Blocked and paused builds still occupy a slot: their workspaces and
|
|
2863
3101
|
// pending work are live, only waiting on a human. Capacity is per repo
|
|
2864
3102
|
// (§16.1) — another repo's builds never consume this repo's slots.
|
|
@@ -2878,9 +3116,12 @@ export class Dispatcher {
|
|
|
2878
3116
|
// ready state are deduped against the active builds embodying them.
|
|
2879
3117
|
await this.store.refreshJournal()
|
|
2880
3118
|
const journalBeforeReady = this.store.recordedJournal()
|
|
2881
|
-
const listing = await
|
|
3119
|
+
const listing = await this.listReady()
|
|
2882
3120
|
await this.store.refreshJournal()
|
|
2883
3121
|
const journalAfterReady = this.store.recordedJournal()
|
|
3122
|
+
if (reduceTicketSourceHealth(journalAfterReady)?.cause === 'ready-scan-failed') {
|
|
3123
|
+
await this.sourceCondition(undefined, 'ready-scan')
|
|
3124
|
+
}
|
|
2884
3125
|
const creationHolds = new Map(
|
|
2885
3126
|
harvestCreationsDuringReadyScan(journalBeforeReady, journalAfterReady).map((creation) => [
|
|
2886
3127
|
creation.creationKey.toLowerCase(),
|
|
@@ -2912,55 +3153,67 @@ export class Dispatcher {
|
|
|
2912
3153
|
// still runs either way, so the standing queue-depth report stays honest —
|
|
2913
3154
|
// this only declines to claim a ticket into a build that would immediately
|
|
2914
3155
|
// sit held.
|
|
2915
|
-
if (capacity <= 0 || paused)
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
3156
|
+
if (capacity <= 0 || paused) {
|
|
3157
|
+
await this.sourceCondition(undefined, 'ready-scan')
|
|
3158
|
+
if (!this.outOfBudget(opts)) await this.completeScan(checkpoint, report)
|
|
3159
|
+
return
|
|
3160
|
+
}
|
|
3161
|
+
// One dependency-node cache per scan: blockers are commonly shared across
|
|
3162
|
+
// the ready set. Never reuse ticket or dependency objects between scans.
|
|
3163
|
+
const nodes = new Map<string, DependencyState>(
|
|
3164
|
+
listing.tickets.map((ticket) => [
|
|
3165
|
+
ticket.ref.id,
|
|
3166
|
+
{
|
|
3167
|
+
id: ticket.ref.id,
|
|
3168
|
+
exists: true,
|
|
3169
|
+
resolved: ticket.resolved,
|
|
3170
|
+
blockedBy: [...(ticket.blockedBy ?? [])],
|
|
3171
|
+
},
|
|
3172
|
+
]),
|
|
3173
|
+
)
|
|
2920
3174
|
|
|
3175
|
+
// Finish read-only evaluation before any intake claim: a later source
|
|
3176
|
+
// failure must leave the entire ready set unclaimed. Publish dependency
|
|
3177
|
+
// diagnostics only after the complete preflight succeeds.
|
|
3178
|
+
let scanSuccessful = true
|
|
3179
|
+
const verdicts = new Map<string, DependencyVerdict>()
|
|
3180
|
+
const failed = new Set<string>()
|
|
2921
3181
|
for (const ticket of ready) {
|
|
2922
|
-
if (
|
|
2923
|
-
// Budget gate before the dependency fetch / claim / createBuild: the
|
|
2924
|
-
// first ready read above stays (it is the standing queue-depth report);
|
|
2925
|
-
// once the budget is spent no additional transport call is started, so
|
|
2926
|
-
// the remaining tickets are left un-attempted for the next invocation.
|
|
2927
|
-
if (this.outOfBudget(opts)) break
|
|
2928
|
-
|
|
2929
|
-
// Autobuild's own in-flight create gate precedes even dependency reads:
|
|
2930
|
-
// the ready projection may have been captured before blocker recording.
|
|
2931
|
-
// It spends no capacity and leaves queue depth unchanged.
|
|
3182
|
+
if (this.outOfBudget(opts)) return
|
|
2932
3183
|
if (withheld.has(ticket.ref.id)) continue
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
let verdict: DependencyVerdict
|
|
2943
|
-
try {
|
|
2944
|
-
await this.loadDependencyGraph(ticket, nodes)
|
|
2945
|
-
verdict = analyzeDependencies(ticket.ref.id, ticket.blockedBy ?? [], nodes)
|
|
2946
|
-
} catch (error) {
|
|
2947
|
-
// A broken graph is this ticket's problem, not the tick's: skip it
|
|
2948
|
-
// and let unrelated eligible tickets dispatch normally.
|
|
2949
|
-
verdict = {
|
|
2950
|
-
unresolved: [ticket.ref.id],
|
|
2951
|
-
diagnostics: [
|
|
2952
|
-
`ticket ${ticket.ref.id}: dependency check failed — ${
|
|
2953
|
-
error instanceof Error ? error.message : String(error)
|
|
2954
|
-
}`,
|
|
2955
|
-
],
|
|
2956
|
-
}
|
|
2957
|
-
}
|
|
2958
|
-
if (verdict.unresolved.length > 0) {
|
|
2959
|
-
report.dependencyBlocked += 1
|
|
2960
|
-
report.dependencyDiagnostics.push(...verdict.diagnostics)
|
|
2961
|
-
continue
|
|
3184
|
+
if (ticket.blockersResolved === true || (ticket.blockedBy ?? []).length === 0) continue
|
|
3185
|
+
let verdict: DependencyVerdict
|
|
3186
|
+
try {
|
|
3187
|
+
if (!(await this.loadDependencyGraph(ticket, nodes, opts))) return
|
|
3188
|
+
verdict = analyzeDependencies(ticket.ref.id, ticket.blockedBy ?? [], nodes)
|
|
3189
|
+
} catch (error) {
|
|
3190
|
+
if (error instanceof TicketSourceRateLimitError) {
|
|
3191
|
+
await this.sourceFailure(error, 'dependency-check')
|
|
3192
|
+
throw error
|
|
2962
3193
|
}
|
|
3194
|
+
scanSuccessful = false
|
|
3195
|
+
failed.add(ticket.ref.id)
|
|
3196
|
+
report.dependencyCheckFailed += 1
|
|
3197
|
+
report.dependencyDiagnostics.push(
|
|
3198
|
+
`ticket ${ticket.ref.id}: dependency check failed — ${error instanceof Error ? error.message : String(error)}`,
|
|
3199
|
+
)
|
|
3200
|
+
continue
|
|
3201
|
+
}
|
|
3202
|
+
verdicts.set(ticket.ref.id, verdict)
|
|
3203
|
+
}
|
|
3204
|
+
if (this.outOfBudget(opts)) return
|
|
3205
|
+
if (failed.size === 0) await this.sourceCondition(undefined, 'dependency-check')
|
|
3206
|
+
for (const verdict of verdicts.values()) {
|
|
3207
|
+
if (verdict.unresolved.length > 0) {
|
|
3208
|
+
report.dependencyBlocked += 1
|
|
3209
|
+
report.dependencyDiagnostics.push(...verdict.diagnostics)
|
|
2963
3210
|
}
|
|
3211
|
+
}
|
|
3212
|
+
for (const ticket of ready) {
|
|
3213
|
+
if (capacity <= 0 || this.outOfBudget(opts)) break
|
|
3214
|
+
if (withheld.has(ticket.ref.id)) continue
|
|
3215
|
+
if (failed.has(ticket.ref.id)) continue
|
|
3216
|
+
if ((verdicts.get(ticket.ref.id)?.unresolved.length ?? 0) > 0) continue
|
|
2964
3217
|
|
|
2965
3218
|
// Claim-before-launch (§12): losing the claim means another dispatcher
|
|
2966
3219
|
// (or an earlier tick) owns this ticket.
|
|
@@ -3036,36 +3289,36 @@ export class Dispatcher {
|
|
|
3036
3289
|
// means the remote provisioning continuation owns the rest.
|
|
3037
3290
|
if (outcome === 'completed' || outcome === 'deferred') report.dispatched += 1
|
|
3038
3291
|
}
|
|
3292
|
+
if (scanSuccessful && !this.outOfBudget(opts)) await this.completeScan(checkpoint, report)
|
|
3039
3293
|
}
|
|
3040
3294
|
|
|
3041
3295
|
/**
|
|
3042
|
-
*
|
|
3043
|
-
*
|
|
3044
|
-
* level, skipping ids already cached — the common case (a blocker or two,
|
|
3045
|
-
* none of them blocked themselves) is a single call.
|
|
3046
|
-
*
|
|
3047
|
-
* INVARIANT: `nodes` contains only states actually returned by the source.
|
|
3048
|
-
* The cache is shared across the tick's tickets, so a node this ticket
|
|
3049
|
-
* invents is a node the *next* ticket will trust as a fact about its own
|
|
3050
|
-
* blocker. Nothing is seeded here for that reason (f_8bc9ee0c); the
|
|
3051
|
-
* analyzer takes the ticket's own blockers as an argument instead, and
|
|
3052
|
-
* `findCycle` closes a cycle via its path rather than the map.
|
|
3053
|
-
*
|
|
3054
|
-
* A provider-side cycle terminates the walk: an id already in `nodes` is
|
|
3055
|
-
* never re-queued, so every reachable node is fetched exactly once.
|
|
3296
|
+
* Walk unresolved branches in provider order. Listing facts and dependency
|
|
3297
|
+
* reads share a tick-local cache; traversal visits are local to this root.
|
|
3056
3298
|
*/
|
|
3057
3299
|
private async loadDependencyGraph(
|
|
3058
3300
|
ticket: Ticket,
|
|
3059
3301
|
nodes: Map<string, DependencyState>,
|
|
3060
|
-
|
|
3061
|
-
|
|
3302
|
+
opts: TickOpts = {},
|
|
3303
|
+
): Promise<boolean> {
|
|
3304
|
+
const visited = new Set<string>()
|
|
3305
|
+
let frontier = [...new Set(ticket.blockedBy ?? [])]
|
|
3062
3306
|
while (frontier.length > 0) {
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3307
|
+
if (this.outOfBudget(opts)) return false
|
|
3308
|
+
const uncached = frontier.filter((id) => !nodes.has(id))
|
|
3309
|
+
if (uncached.length > 0) {
|
|
3310
|
+
const states = await this.deps.tickets.dependencyStates(uncached)
|
|
3311
|
+
for (const state of states) nodes.set(state.id, state)
|
|
3312
|
+
}
|
|
3313
|
+
const next: string[] = []
|
|
3314
|
+
for (const id of frontier) {
|
|
3315
|
+
visited.add(id)
|
|
3316
|
+
const node = nodes.get(id)
|
|
3317
|
+
if (node?.exists && !node.resolved) next.push(...node.blockedBy)
|
|
3318
|
+
}
|
|
3319
|
+
frontier = [...new Set(next)].filter((id) => !visited.has(id))
|
|
3068
3320
|
}
|
|
3321
|
+
return true
|
|
3069
3322
|
}
|
|
3070
3323
|
|
|
3071
3324
|
/** Bounce (§6.3): back to Triage with a comment citing the standard and
|
|
@@ -97,6 +97,7 @@ export async function runHarvestChild(
|
|
|
97
97
|
uuids: randomUuids(),
|
|
98
98
|
clock: systemClock,
|
|
99
99
|
instance: input.instance,
|
|
100
|
+
...(input.preparedRun ? { preparedRun: input.preparedRun } : {}),
|
|
100
101
|
// Adopted holder: the guest skips its own claim, heartbeats the owning
|
|
101
102
|
// dispatch loop's repository lease, and never releases it.
|
|
102
103
|
...(input.leaseHolder !== undefined ? { leaseHolder: input.leaseHolder } : {}),
|
|
@@ -92,9 +92,26 @@ export function classifyHarvestOutcome(
|
|
|
92
92
|
input: HarvestExecutionClassificationInput,
|
|
93
93
|
): HarvestRunnerResult {
|
|
94
94
|
const state = reduceHarvest(events)
|
|
95
|
-
const
|
|
95
|
+
const started = events.find(
|
|
96
|
+
(event) =>
|
|
97
|
+
event.type === 'harvest.execution.started' && event.seq === input.executionStartedSeq,
|
|
98
|
+
)
|
|
99
|
+
const authorization =
|
|
100
|
+
started?.type === 'harvest.execution.started'
|
|
101
|
+
? events.find(
|
|
102
|
+
(event) =>
|
|
103
|
+
event.type === 'harvest.execution.authorized' &&
|
|
104
|
+
event.payload.execution === started.payload.execution,
|
|
105
|
+
)
|
|
106
|
+
: undefined
|
|
107
|
+
const selected =
|
|
108
|
+
authorization?.type === 'harvest.execution.authorized' ? authorization.payload : undefined
|
|
109
|
+
const latestRun = selected
|
|
110
|
+
? state.runs.find((run) => run.run === selected.run)
|
|
111
|
+
: state.runs.at(-1)
|
|
96
112
|
if (latestRun !== undefined) {
|
|
97
|
-
const launch =
|
|
113
|
+
const launch =
|
|
114
|
+
selected?.launch ?? (latestRun.startedSeq > input.executionStartedSeq ? 'started' : 'resumed')
|
|
98
115
|
// Recency gate: a terminal outcome counts only if its fact — the run's
|
|
99
116
|
// terminal fact, or for an unresolved failure the failure fact itself
|
|
100
117
|
// (`failureSeq`), which also covers exhaustion via `terminalSeq` —
|