@defrex/autobuild 0.10.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +3 -1
  2. package/SPEC.md +237 -23
  3. package/bin/ab-dispatch-kernel.ts +1 -1
  4. package/docs/architecture.md +66 -4
  5. package/docs/configuration.md +137 -19
  6. package/docs/hosted-dispatcher.md +17 -2
  7. package/docs/hosted-service.md +214 -0
  8. package/docs/mcp.md +8 -2
  9. package/docs/operator-api.md +107 -7
  10. package/docs/remote-store-protocol.md +189 -144
  11. package/docs/remote-ticket-protocol.md +111 -18
  12. package/docs/setup.md +173 -16
  13. package/docs/ticket-dependencies.md +1 -1
  14. package/package.json +7 -6
  15. package/packages/core/src/cli/attention.ts +209 -0
  16. package/packages/core/src/cli/binary.ts +3 -3
  17. package/packages/core/src/cli/configured-store.ts +17 -0
  18. package/packages/core/src/cli/connect.ts +261 -0
  19. package/packages/core/src/cli/dashboard/model.ts +8 -5
  20. package/packages/core/src/cli/dashboard/render.ts +37 -7
  21. package/packages/core/src/cli/dispatch-frontend.ts +16 -1
  22. package/packages/core/src/cli/dispatch-process.ts +25 -7
  23. package/packages/core/src/cli/dispatch.ts +247 -47
  24. package/packages/core/src/cli/dispatcher-liveness.ts +91 -0
  25. package/packages/core/src/cli/help.ts +45 -13
  26. package/packages/core/src/cli/init-validation.ts +27 -2
  27. package/packages/core/src/cli/main.ts +41 -15
  28. package/packages/core/src/cli/mcp.ts +7 -1
  29. package/packages/core/src/cli/pr-summary-publication.ts +220 -0
  30. package/packages/core/src/cli/pr-summary.ts +2 -0
  31. package/packages/core/src/cli/project-credentials.ts +132 -0
  32. package/packages/core/src/cli/repo-state.ts +15 -2
  33. package/packages/core/src/cli/repository-status.ts +45 -5
  34. package/packages/core/src/cli/self-update.ts +10 -0
  35. package/packages/core/src/cli/store-opening.ts +22 -14
  36. package/packages/core/src/cli/terminals.ts +0 -10
  37. package/packages/core/src/cli/ticket.ts +8 -1
  38. package/packages/core/src/cli/upgrade-compatibility.ts +80 -0
  39. package/packages/core/src/cli/wait.ts +84 -17
  40. package/packages/core/src/cli/watch.ts +350 -68
  41. package/packages/core/src/config/live.ts +4 -0
  42. package/packages/core/src/config/schema.ts +65 -11
  43. package/packages/core/src/connection.ts +86 -0
  44. package/packages/core/src/events/catalog.ts +10 -2
  45. package/packages/core/src/events/globs.ts +8 -3
  46. package/packages/core/src/events/payloads.ts +44 -0
  47. package/packages/core/src/events/repository.ts +64 -2
  48. package/packages/core/src/integration/harness.ts +2 -1
  49. package/packages/core/src/kernel/__snapshots__/reducer-registry.test.ts.snap +148 -83
  50. package/packages/core/src/kernel/dispatch-status.ts +9 -1
  51. package/packages/core/src/kernel/dispatcher-liveness.ts +70 -0
  52. package/packages/core/src/kernel/engine.ts +24 -0
  53. package/packages/core/src/kernel/generators/build-log.ts +10 -0
  54. package/packages/core/src/kernel/log-index.ts +17 -0
  55. package/packages/core/src/kernel/pr-summary.ts +89 -0
  56. package/packages/core/src/kernel/reducer-registry.ts +12 -0
  57. package/packages/core/src/kernel/reducer.ts +8 -0
  58. package/packages/core/src/kernel/ticket-source-health.ts +30 -0
  59. package/packages/core/src/ontology.ts +1 -0
  60. package/packages/core/src/operator/index.ts +7 -0
  61. package/packages/core/src/operator/query.ts +44 -19
  62. package/packages/core/src/operator/ticket-source-backend.ts +1 -0
  63. package/packages/core/src/operator/tickets.ts +6 -8
  64. package/packages/core/src/plugin-sdk/index.ts +1 -0
  65. package/packages/core/src/plugins/manifest.ts +5 -3
  66. package/packages/core/src/plugins/runtimes.ts +3 -1
  67. package/packages/core/src/ports/forge/fake.ts +13 -1
  68. package/packages/core/src/ports/forge/github-transport.ts +36 -1
  69. package/packages/core/src/ports/forge/github.ts +29 -5
  70. package/packages/core/src/ports/forge/local-git.ts +12 -4
  71. package/packages/core/src/ports/forge/summary-marker.ts +8 -0
  72. package/packages/core/src/ports/runner/session-env.ts +1 -0
  73. package/packages/core/src/ports/tickets/contract.ts +25 -0
  74. package/packages/core/src/ports/tickets/create.ts +11 -3
  75. package/packages/core/src/ports/tickets/fake.ts +10 -8
  76. package/packages/core/src/ports/tickets/file.ts +1 -0
  77. package/packages/core/src/ports/tickets/hosted-index.ts +0 -3
  78. package/packages/core/src/ports/tickets/linear.ts +241 -49
  79. package/packages/core/src/ports/tickets/remote-protocol.ts +3 -0
  80. package/packages/core/src/ports/tickets/remote.ts +1 -0
  81. package/packages/core/src/ports/types.ts +17 -0
  82. package/packages/core/src/ports/workspace/build-execution.ts +5 -1
  83. package/packages/core/src/ports/workspace/create.ts +11 -2
  84. package/packages/core/src/ports/workspace/distribution-archive.ts +3 -5
  85. package/packages/core/src/ports/workspace/harvest-execution.ts +3 -1
  86. package/packages/core/src/ports/workspace/local-build-execution.ts +8 -1
  87. package/packages/core/src/ports/workspace/vercel-sandbox.ts +47 -19
  88. package/packages/core/src/processes/build-execution-state.ts +1 -1
  89. package/packages/core/src/processes/build-runner.ts +87 -26
  90. package/packages/core/src/processes/dispatch-failure-budget.ts +83 -0
  91. package/packages/core/src/processes/dispatcher.ts +336 -83
  92. package/packages/core/src/processes/harvest-child.ts +1 -0
  93. package/packages/core/src/processes/harvest-execution-state.ts +19 -2
  94. package/packages/core/src/processes/harvest-runner.ts +142 -100
  95. package/packages/core/src/processes/phase-failure-budget.ts +1 -0
  96. package/packages/core/src/processes/publication-loss.ts +18 -1
  97. package/packages/core/src/processes/repo-view.ts +2 -1
  98. package/packages/core/src/store/adapter.ts +1 -0
  99. package/packages/core/src/store/contract.ts +109 -0
  100. package/packages/core/src/store/digest.ts +14 -2
  101. package/packages/core/src/store/local/store.ts +18 -5
  102. package/packages/core/src/store/remote/client.ts +1 -1
  103. package/packages/core/src/store/remote/compatibility.ts +90 -0
  104. package/packages/core/src/store/remote/credentials.ts +28 -0
  105. package/packages/core/src/store/remote/index.ts +14 -1
  106. package/packages/core/src/store/remote/protocol.ts +26 -0
  107. package/packages/core/src/store/remote/token.ts +74 -13
  108. package/packages/core/src/store/repo-state-events.ts +49 -59
  109. package/packages/core/src/store/types.ts +5 -1
  110. package/packages/core/src/testing/index.ts +7 -0
  111. package/skills/guide/SKILL.md +175 -26
  112. package/skills/guide/references/operator-api.md +135 -13
  113. package/skills/guide/references/plugin-authoring.md +10 -2
  114. package/skills/guide/references/remote-store-protocol.md +189 -144
  115. package/skills/guide/references/remote-ticket-protocol.md +111 -18
  116. package/skills/guide/references/setup.md +173 -16
  117. package/skills/guide/references/ticket-dependencies.md +1 -1
  118. package/skills/operate/SKILL.md +159 -34
  119. package/templates/hosted-service/package.json +13 -0
  120. package/templates/hosted-service/vercel.json +7 -0
@@ -1,3 +1,8 @@
1
+ import { ensureDispatchBudget } from './dispatch-failure-budget'
2
+ import {
3
+ reduceTicketSourceHealth,
4
+ type TicketSourceDegradation,
5
+ } from '../kernel/ticket-source-health'
1
6
  /**
2
7
  * The dispatcher process (SPEC §3.3, §12): watches the TicketSource for
3
8
  * Ready tickets, claims, provisions a workspace, and launches build-runners
@@ -36,6 +41,7 @@
36
41
  * validation and fallback remain here. The supervising kernel reserves each
37
42
  * execution lease before launch; the child renews that same-holder lease.
38
43
  */
44
+ import { createHash } from 'node:crypto'
39
45
  import type { Config } from '../config/schema'
40
46
  import type { LanguageModel } from 'ai'
41
47
  import { runOrchestratorTickStep } from './orchestrator-tick'
@@ -74,6 +80,7 @@ import type {
74
80
  TicketSource,
75
81
  WorkspaceProvider,
76
82
  } from '../ports/types'
83
+ import { TicketSourceRateLimitError } from '../ports/types'
77
84
  import type { Exec } from '../ports/workspace/git-worktree'
78
85
  import type { ArtifactMeta, BuildDigest, BuildRecord, BuildStore, Clock } from '../store/types'
79
86
  import { samplePinnedAssets } from '../store/ticket-assets'
@@ -201,7 +208,9 @@ function findCycle(
201
208
  nodes: Map<string, DependencyState>,
202
209
  explored: Set<string>,
203
210
  ): string[] | null {
204
- for (const next of nodes.get(from)?.blockedBy ?? []) {
211
+ const node = nodes.get(from)
212
+ if (!node?.exists || node.resolved) return null
213
+ for (const next of node.blockedBy) {
205
214
  const seen = path.indexOf(next)
206
215
  if (seen !== -1) return [...path.slice(seen), next]
207
216
  if (explored.has(next)) continue
@@ -301,7 +310,7 @@ export function validateSlugCandidate(candidate: string | null | undefined): str
301
310
  /** Counts per action, for observability and tests. An idempotent re-run of
302
311
  * `tick()` over unchanged state reports all zeroes — except invalid-record,
303
312
  * in-flight-creation, and dependency fields, which are standing queue reports rather than records of
304
- * action. They repeat on every ready scan by design, because that is the only
313
+ * action. They repeat between scans and refresh on completed scans, because that is the only
305
314
  * place the source problems are visible without provider inspection. */
306
315
  export interface TickReport {
307
316
  /** Janitor (§15.7): builds completed as merged. */
@@ -354,13 +363,15 @@ export interface TickReport {
354
363
  /** Standing queue depth: ready tickets still waiting after this tick —
355
364
  * excludes tickets already embodied by an active build, and ones that left
356
365
  * the queue this tick (dispatched, bounced, or claimed elsewhere). Like the
357
- * invalid/dependency fields it re-reports on every ready scan. */
366
+ * invalid/dependency fields it repeats between completed scans. */
358
367
  queued: number
359
368
  /** Ready tickets withheld while Autobuild's own external creation is still
360
369
  * between durable reservation and settled blocker recording. */
361
370
  creationWithheld: number
362
371
  /** One line per creation hold, with Harvest run/proposal provenance. */
363
372
  creationDiagnostics: string[]
373
+ /** Ready tickets whose dependency reads failed rather than yielding a verdict. */
374
+ dependencyCheckFailed: number
364
375
  /** Dependency gate (§13): ready tickets held back by unresolved blockers. */
365
376
  dependencyBlocked: number
366
377
  /** One line per held ticket naming its unresolved blockers — the operator's
@@ -410,6 +421,7 @@ export function emptyTickReport(): TickReport {
410
421
  queued: 0,
411
422
  creationWithheld: 0,
412
423
  creationDiagnostics: [],
424
+ dependencyCheckFailed: 0,
413
425
  dependencyBlocked: 0,
414
426
  dependencyDiagnostics: [],
415
427
  harvestStarted: 0,
@@ -654,7 +666,15 @@ export class Dispatcher {
654
666
  return this.repoOrigin
655
667
  }
656
668
  private readonly leaseTtlMs: number
657
- private readonly doneState: string
669
+ private get doneState(): string {
670
+ return (
671
+ this.deps.opts?.doneState ??
672
+ (this.deps.config.tickets.source === 'hosted'
673
+ ? this.deps.config.tickets.doneState
674
+ : undefined) ??
675
+ 'Done'
676
+ )
677
+ }
658
678
  private readonly slugNamingTimeoutMs: number
659
679
  private readonly maxHarvestRecoveryAttempts: number
660
680
  private readonly provisionStaleMs: number
@@ -684,7 +704,6 @@ export class Dispatcher {
684
704
  ? deps.store
685
705
  : new RepoViewStore(deps.store, { repo: deps.repo })
686
706
  this.leaseTtlMs = deps.opts?.leaseTtlMs ?? 0
687
- this.doneState = deps.opts?.doneState ?? 'Done'
688
707
  this.slugNamingTimeoutMs = deps.opts?.slugNamingTimeoutMs ?? DEFAULT_SLUG_NAMING_TIMEOUT_MS
689
708
  this.maxHarvestRecoveryAttempts =
690
709
  deps.opts?.maxHarvestRecoveryAttempts ?? DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS
@@ -729,6 +748,7 @@ export class Dispatcher {
729
748
  }
730
749
  }
731
750
  const report = emptyTickReport()
751
+ this.restoreScan(report)
732
752
  /** Builds already launched this tick — a janitor/startup re-attach must
733
753
  * not be doubled by the sweep, nor a fresh dispatch by anything. */
734
754
  const launched = new Set<string>()
@@ -893,6 +913,81 @@ export class Dispatcher {
893
913
  .map(({ record }) => record)
894
914
  }
895
915
 
916
+ private async sourceCondition(
917
+ next?: TicketSourceDegradation,
918
+ successfulOperation?: 'ready-scan' | 'dependency-check',
919
+ ): Promise<void> {
920
+ await this.store.refreshJournal()
921
+ const current = reduceTicketSourceHealth(this.store.recordedJournal())
922
+ if (next !== undefined && current?.cause === next.cause && current.source === next.source) {
923
+ if (
924
+ next.cause === 'rate-limited' &&
925
+ next.operation === 'dependency-check' &&
926
+ current.operation !== 'dependency-check'
927
+ ) {
928
+ await this.store.appendRepo(this.deps.repo, {
929
+ actor: DISPATCHER,
930
+ type: 'dispatcher.ticket-source-dependency-limited',
931
+ payload: { source: current.source },
932
+ })
933
+ }
934
+ return
935
+ }
936
+ if (
937
+ next === undefined &&
938
+ current?.operation === 'dependency-check' &&
939
+ successfulOperation !== 'dependency-check'
940
+ )
941
+ return
942
+ if (current !== undefined) {
943
+ await this.store.appendRepo(this.deps.repo, {
944
+ actor: DISPATCHER,
945
+ type: 'dispatcher.ticket-source-recovered',
946
+ payload: { source: current.source, cause: current.cause, operation: current.operation },
947
+ })
948
+ }
949
+ if (next !== undefined) {
950
+ await this.store.ensureRepo(this.deps.repo)
951
+ await this.store.appendRepo(this.deps.repo, {
952
+ actor: DISPATCHER,
953
+ type: 'dispatcher.ticket-source-degraded',
954
+ payload: next,
955
+ })
956
+ }
957
+ }
958
+
959
+ private async sourceFailure(
960
+ error: unknown,
961
+ operation: 'ready-scan' | 'dependency-check',
962
+ ): Promise<void> {
963
+ const message = error instanceof Error ? error.message : String(error)
964
+ await this.sourceCondition(
965
+ error instanceof TicketSourceRateLimitError
966
+ ? {
967
+ source: error.source,
968
+ cause: 'rate-limited',
969
+ operation,
970
+ message,
971
+ resetAt: new Date(error.resetAt).toISOString(),
972
+ }
973
+ : {
974
+ source: this.deps.tickets.name,
975
+ cause: 'ready-scan-failed',
976
+ operation,
977
+ message: message || 'Ready scan failed',
978
+ },
979
+ )
980
+ }
981
+
982
+ private async listReady() {
983
+ try {
984
+ return await this.deps.tickets.listReady(readyCriteria(this.deps.config))
985
+ } catch (error) {
986
+ await this.sourceFailure(error, 'ready-scan')
987
+ throw error
988
+ }
989
+ }
990
+
896
991
  /** Observation-only ready listing for Store-backed frontends while intake is
897
992
  * disabled. It shares the active-ticket exclusion with dispatch but performs
898
993
  * no dependency fetch, claim, bounce, build, workspace, or launch side
@@ -910,14 +1005,109 @@ export class Dispatcher {
910
1005
  activeTicketIds.add(record.ticket.id)
911
1006
  }
912
1007
  }
913
- const listing = await this.deps.tickets.listReady(readyCriteria(this.deps.config))
1008
+ const report = emptyTickReport()
1009
+ this.restoreScan(report)
1010
+ const checkpoint = await this.scanCheckpoint('observation')
1011
+ if (checkpoint.due) {
1012
+ await this.store.refreshJournal()
1013
+ const listing = await this.listReady()
1014
+ await this.store.refreshJournal()
1015
+ report.queued = listing.tickets.filter((ticket) => !activeTicketIds.has(ticket.ref.id)).length
1016
+ report.invalidTickets = listing.diagnostics.length
1017
+ report.ticketDiagnostics = [...listing.diagnostics]
1018
+ report.dependencyBlocked = 0
1019
+ report.dependencyDiagnostics = []
1020
+ report.creationWithheld = 0
1021
+ report.creationDiagnostics = []
1022
+ await this.sourceCondition(undefined, 'ready-scan')
1023
+ await this.completeScan(checkpoint, report)
1024
+ }
1025
+ return {
1026
+ queued: report.queued,
1027
+ invalidTickets: report.invalidTickets,
1028
+ ticketDiagnostics: report.ticketDiagnostics,
1029
+ }
1030
+ }
1031
+
1032
+ private latestScan() {
1033
+ return this.store
1034
+ .recordedJournal()
1035
+ .findLast(
1036
+ (event): event is Extract<RepositoryEvent, { type: 'dispatcher.ticket-scan-completed' }> =>
1037
+ event.type === 'dispatcher.ticket-scan-completed',
1038
+ )?.payload
1039
+ }
1040
+
1041
+ private restoreScan(report: TickReport): void {
1042
+ const previous = this.latestScan()
1043
+ if (previous === undefined) return
1044
+ for (const key of [
1045
+ 'queued',
1046
+ 'invalidTickets',
1047
+ 'dependencyBlocked',
1048
+ 'creationWithheld',
1049
+ ] as const)
1050
+ report[key] = previous[key]
1051
+ for (const key of [
1052
+ 'ticketDiagnostics',
1053
+ 'dependencyDiagnostics',
1054
+ 'creationDiagnostics',
1055
+ ] as const)
1056
+ report[key] = [...previous[key]]
1057
+ }
1058
+
1059
+ private async scanCheckpoint(mode: 'admission' | 'observation') {
1060
+ const digest = (await this.repoBuildDigests())
1061
+ .filter(({ digest }) => digest.readinessChangeSeq !== undefined)
1062
+ .map(({ record, digest }) => [record.slug, digest.readinessChangeSeq] as const)
1063
+ .sort(([a], [b]) => a.localeCompare(b))
1064
+ const hash = (value: unknown) =>
1065
+ createHash('sha256').update(JSON.stringify(value)).digest('hex')
1066
+ const readinessFingerprint = hash(digest)
1067
+ const criteriaFingerprint = hash(readyCriteria(this.deps.config))
1068
+ const startedAt = this.deps.clock().toISOString()
1069
+ const previous = this.latestScan()
1070
+ const elapsed =
1071
+ previous === undefined ? Infinity : Date.parse(startedAt) - Date.parse(previous.startedAt)
914
1072
  return {
915
- queued: listing.tickets.filter((ticket) => !activeTicketIds.has(ticket.ref.id)).length,
916
- invalidTickets: listing.diagnostics.length,
917
- ticketDiagnostics: [...listing.diagnostics],
1073
+ startedAt,
1074
+ mode,
1075
+ readinessFingerprint,
1076
+ criteriaFingerprint,
1077
+ due:
1078
+ previous === undefined ||
1079
+ elapsed < 0 ||
1080
+ elapsed >= this.deps.config.ticketScanIntervalSeconds * 1000 ||
1081
+ previous.readinessFingerprint !== readinessFingerprint ||
1082
+ previous.criteriaFingerprint !== criteriaFingerprint ||
1083
+ (mode === 'admission' && previous.mode === 'observation'),
918
1084
  }
919
1085
  }
920
1086
 
1087
+ private async completeScan(
1088
+ checkpoint: Awaited<ReturnType<Dispatcher['scanCheckpoint']>>,
1089
+ report: TickReport,
1090
+ ): Promise<void> {
1091
+ await this.store.ensureRepo(this.deps.repo)
1092
+ await this.store.appendRepo(this.deps.repo, {
1093
+ actor: DISPATCHER,
1094
+ type: 'dispatcher.ticket-scan-completed',
1095
+ payload: {
1096
+ startedAt: checkpoint.startedAt,
1097
+ mode: checkpoint.mode,
1098
+ readinessFingerprint: checkpoint.readinessFingerprint,
1099
+ criteriaFingerprint: checkpoint.criteriaFingerprint,
1100
+ queued: report.queued,
1101
+ invalidTickets: report.invalidTickets,
1102
+ ticketDiagnostics: report.ticketDiagnostics.slice(0, 1000),
1103
+ dependencyBlocked: report.dependencyBlocked,
1104
+ dependencyDiagnostics: report.dependencyDiagnostics.slice(0, 1000),
1105
+ creationWithheld: report.creationWithheld,
1106
+ creationDiagnostics: report.creationDiagnostics.slice(0, 1000),
1107
+ },
1108
+ })
1109
+ }
1110
+
921
1111
  /** Apply the newest durable `dispatcher.auto-merge-default-set` fact to every
922
1112
  * current non-terminal build of this repository: on — request consent; off —
923
1113
  * cancel it. Attribution is the toggling human's own actor, exactly as if
@@ -1326,6 +1516,10 @@ export class Dispatcher {
1326
1516
  generation,
1327
1517
  seed,
1328
1518
  tail,
1519
+ }).catch(() => {
1520
+ // Detached work must never reject into the dispatcher process. Failure
1521
+ // recording can itself fail on a store outage; durable boundaries and
1522
+ // lease expiry let the next tick repair or retry the unfinished work.
1329
1523
  })
1330
1524
  return 'kicked'
1331
1525
  }
@@ -1360,6 +1554,7 @@ export class Dispatcher {
1360
1554
  const slug = entry.slug
1361
1555
  let leaseClaimed = false
1362
1556
  let heartbeat: ReturnType<typeof setInterval> | undefined
1557
+ let boundary: 'workspace' | 'read' | 'dispatch' | 'launch' = 'workspace'
1363
1558
  const stage: EventPayload<'dispatch.failed'>['stage'] = 'workspace'
1364
1559
  try {
1365
1560
  if (entry.controller.signal.aborted) {
@@ -1424,12 +1619,19 @@ export class Dispatcher {
1424
1619
  await store.releaseLease(slug, entry.instance)
1425
1620
  leaseClaimed = false
1426
1621
  if (entry.controller.signal.aborted) return
1622
+ boundary = 'read'
1427
1623
  const record = await store.getBuild(slug)
1428
1624
  if (record === null) return
1429
- await (input.tail === 'dispatch'
1430
- ? this.completeDispatchBoundaries(record, input.seed)
1431
- : this.deps.launchRunner(slug))
1625
+ const queued = reduceBuild(await store.getEvents(slug)).status === 'queued'
1626
+ boundary = input.tail === 'dispatch' || queued ? 'dispatch' : 'launch'
1627
+ if (boundary === 'dispatch') await this.completeDispatchBoundaries(record, input.seed)
1628
+ else await this.deps.launchRunner(slug)
1432
1629
  } catch (error) {
1630
+ // Dispatch boundaries own their failure facts and exhaustion repair.
1631
+ // A failed tail read/write leaves durable unfinished work for the next
1632
+ // tick; it is not another provider workspace failure. Replacement
1633
+ // runner launch errors retain the existing infrastructure-budget path.
1634
+ if (boundary === 'read' || boundary === 'dispatch') return
1433
1635
  const events = await store.getEvents(slug)
1434
1636
  if (workspaces.recovery !== undefined) {
1435
1637
  const open = openWorkspace(events)
@@ -2303,6 +2505,16 @@ export class Dispatcher {
2303
2505
  return sha
2304
2506
  }
2305
2507
 
2508
+ private dispatchBudget(slug: string): Promise<boolean> {
2509
+ return ensureDispatchBudget(
2510
+ this.store,
2511
+ this.deps.ids,
2512
+ slug,
2513
+ this.deps.config.policy.maxDispatchAttempts,
2514
+ this.deps.workspaces.recovery !== undefined,
2515
+ )
2516
+ }
2517
+
2306
2518
  // ── b. Interrupted dispatch recovery ───────────────────────────────────────
2307
2519
 
2308
2520
  /** The claim-time `build.created` payload. Assets are omitted when the ticket
@@ -2357,6 +2569,7 @@ export class Dispatcher {
2357
2569
  ): Promise<DispatchOutcome> {
2358
2570
  const store = this.store
2359
2571
  const { config } = this.deps
2572
+ if (!(await this.dispatchBudget(record.slug))) return 'parked'
2360
2573
  let stage: EventPayload<'dispatch.failed'>['stage'] = 'create'
2361
2574
  try {
2362
2575
  let events = await store.getEvents(record.slug)
@@ -2446,8 +2659,6 @@ export class Dispatcher {
2446
2659
  } satisfies EventWrite<'workspace.provisioned'>)
2447
2660
  events = await store.getEvents(record.slug)
2448
2661
  }
2449
-
2450
- return await this.completeDispatchBoundaries(record, seed, report, launched)
2451
2662
  } catch (error) {
2452
2663
  const events = await store.getEvents(record.slug)
2453
2664
  if (stage === 'workspace' && this.deps.workspaces.recovery !== undefined) {
@@ -2472,8 +2683,10 @@ export class Dispatcher {
2472
2683
  },
2473
2684
  })
2474
2685
  report.dispatchFailed += 1
2686
+ await this.dispatchBudget(record.slug)
2475
2687
  return 'failed'
2476
2688
  }
2689
+ return this.completeDispatchBoundaries(record, seed, report, launched)
2477
2690
  }
2478
2691
 
2479
2692
  /** The post-workspace dispatch boundaries — spec, comment, launch — shared
@@ -2494,10 +2707,13 @@ export class Dispatcher {
2494
2707
  ): Promise<Exclude<DispatchOutcome, 'deferred'>> {
2495
2708
  const store = this.store
2496
2709
 
2710
+ if (!(await this.dispatchBudget(record.slug))) return 'parked'
2497
2711
  let stage: EventPayload<'dispatch.failed'>['stage'] = 'spec'
2498
2712
  let authored = false
2499
2713
  try {
2500
2714
  const events = await store.getEvents(record.slug)
2715
+ const state = reduceBuild(events)
2716
+ if (state.status !== 'queued' || state.discardRequest !== undefined) return 'parked'
2501
2717
  if (
2502
2718
  !events.some((event) => event.type === 'spec.imported' || event.type === 'spec.authored')
2503
2719
  ) {
@@ -2597,6 +2813,7 @@ export class Dispatcher {
2597
2813
  },
2598
2814
  })
2599
2815
  if (report !== undefined) report.dispatchFailed += 1
2816
+ await this.dispatchBudget(record.slug)
2600
2817
  return 'failed'
2601
2818
  }
2602
2819
  }
@@ -2628,9 +2845,11 @@ export class Dispatcher {
2628
2845
  // Queued itself means runner attachment is still missing. Always enter
2629
2846
  // the fact-driven helper: it skips every completed boundary and retries
2630
2847
  // only the final launch when setup is already durable.
2631
- if ((await this.attemptDispatchCompletion(record, launched, report)) === 'completed') {
2632
- report.recovered += 1
2633
- }
2848
+ const outcome = await this.attemptDispatchCompletion(record, launched, report)
2849
+ if (outcome === 'completed') report.recovered += 1
2850
+ // A failed boundary owns this tick's attempt; startup and lease recovery
2851
+ // must not bypass the unfinished dispatch boundary later in the tick.
2852
+ if (outcome === 'failed') launched.add(record.slug)
2634
2853
  }
2635
2854
  }
2636
2855
 
@@ -2671,6 +2890,12 @@ export class Dispatcher {
2671
2890
  continue
2672
2891
  }
2673
2892
 
2893
+ if (
2894
+ state.status === 'queued' &&
2895
+ !state.pendingCommands.some((command) => command.command === 'abort') &&
2896
+ !(await this.dispatchBudget(record.slug))
2897
+ )
2898
+ continue
2674
2899
  const decision = decideNextFromFacts(state, facts.log, config)
2675
2900
  if (decision.kind === 'wait') {
2676
2901
  if (decision.reason === 'blocked' && state.openEscalations.length > 0) {
@@ -2741,6 +2966,12 @@ export class Dispatcher {
2741
2966
  ) {
2742
2967
  continue
2743
2968
  }
2969
+ if (
2970
+ state.status === 'queued' &&
2971
+ !state.pendingCommands.some((command) => command.command === 'abort') &&
2972
+ !(await this.dispatchBudget(record.slug))
2973
+ )
2974
+ continue
2744
2975
  const decision = decideNextFromFacts(state, facts.log, this.deps.config)
2745
2976
  if (state.status === 'done' || state.status === 'aborted') continue
2746
2977
  // Raw history, read at most once and only by a branch that acts on it.
@@ -2859,6 +3090,13 @@ export class Dispatcher {
2859
3090
  ): Promise<void> {
2860
3091
  const store = this.store
2861
3092
  const { tickets, config } = this.deps
3093
+ const checkpoint = await this.scanCheckpoint('admission')
3094
+ if (!checkpoint.due) return
3095
+ // Clear only standing measurement fields; action counters belong to this tick.
3096
+ report.queued = report.invalidTickets = report.dependencyBlocked = report.creationWithheld = 0
3097
+ report.ticketDiagnostics = []
3098
+ report.dependencyDiagnostics = []
3099
+ report.creationDiagnostics = []
2862
3100
  // Blocked and paused builds still occupy a slot: their workspaces and
2863
3101
  // pending work are live, only waiting on a human. Capacity is per repo
2864
3102
  // (§16.1) — another repo's builds never consume this repo's slots.
@@ -2878,9 +3116,12 @@ export class Dispatcher {
2878
3116
  // ready state are deduped against the active builds embodying them.
2879
3117
  await this.store.refreshJournal()
2880
3118
  const journalBeforeReady = this.store.recordedJournal()
2881
- const listing = await tickets.listReady(readyCriteria(config))
3119
+ const listing = await this.listReady()
2882
3120
  await this.store.refreshJournal()
2883
3121
  const journalAfterReady = this.store.recordedJournal()
3122
+ if (reduceTicketSourceHealth(journalAfterReady)?.cause === 'ready-scan-failed') {
3123
+ await this.sourceCondition(undefined, 'ready-scan')
3124
+ }
2884
3125
  const creationHolds = new Map(
2885
3126
  harvestCreationsDuringReadyScan(journalBeforeReady, journalAfterReady).map((creation) => [
2886
3127
  creation.creationKey.toLowerCase(),
@@ -2912,55 +3153,67 @@ export class Dispatcher {
2912
3153
  // still runs either way, so the standing queue-depth report stays honest —
2913
3154
  // this only declines to claim a ticket into a build that would immediately
2914
3155
  // sit held.
2915
- if (capacity <= 0 || paused) return
2916
- // One dependency-node cache per tick: blockers are commonly shared across
2917
- // the ready set, and a blocker's resolution must be re-read every tick
2918
- // (never cached across them) so a completion lands on the next pass.
2919
- const nodes = new Map<string, DependencyState>()
3156
+ if (capacity <= 0 || paused) {
3157
+ await this.sourceCondition(undefined, 'ready-scan')
3158
+ if (!this.outOfBudget(opts)) await this.completeScan(checkpoint, report)
3159
+ return
3160
+ }
3161
+ // One dependency-node cache per scan: blockers are commonly shared across
3162
+ // the ready set. Never reuse ticket or dependency objects between scans.
3163
+ const nodes = new Map<string, DependencyState>(
3164
+ listing.tickets.map((ticket) => [
3165
+ ticket.ref.id,
3166
+ {
3167
+ id: ticket.ref.id,
3168
+ exists: true,
3169
+ resolved: ticket.resolved,
3170
+ blockedBy: [...(ticket.blockedBy ?? [])],
3171
+ },
3172
+ ]),
3173
+ )
2920
3174
 
3175
+ // Finish read-only evaluation before any intake claim: a later source
3176
+ // failure must leave the entire ready set unclaimed. Publish dependency
3177
+ // diagnostics only after the complete preflight succeeds.
3178
+ let scanSuccessful = true
3179
+ const verdicts = new Map<string, DependencyVerdict>()
3180
+ const failed = new Set<string>()
2921
3181
  for (const ticket of ready) {
2922
- if (capacity <= 0) break
2923
- // Budget gate before the dependency fetch / claim / createBuild: the
2924
- // first ready read above stays (it is the standing queue-depth report);
2925
- // once the budget is spent no additional transport call is started, so
2926
- // the remaining tickets are left un-attempted for the next invocation.
2927
- if (this.outOfBudget(opts)) break
2928
-
2929
- // Autobuild's own in-flight create gate precedes even dependency reads:
2930
- // the ready projection may have been captured before blocker recording.
2931
- // It spends no capacity and leaves queue depth unchanged.
3182
+ if (this.outOfBudget(opts)) return
2932
3183
  if (withheld.has(ticket.ref.id)) continue
2933
-
2934
- // The dependency gate runs FIRST — before the claim, before the spec
2935
- // gate. A blocked ticket must not be claimed, must not be bounced, must
2936
- // create no build or workspace, and must not spend capacity; it is
2937
- // simply not this tick's work. Note this sits *above* [tickets]'s
2938
- // readyLabels / readyState gate: those criteria produced `ready`, and the
2939
- // gate subtracts from it — labels and state can make a ticket a
2940
- // candidate but can never override an unresolved blocker.
2941
- if ((ticket.blockedBy ?? []).length > 0) {
2942
- let verdict: DependencyVerdict
2943
- try {
2944
- await this.loadDependencyGraph(ticket, nodes)
2945
- verdict = analyzeDependencies(ticket.ref.id, ticket.blockedBy ?? [], nodes)
2946
- } catch (error) {
2947
- // A broken graph is this ticket's problem, not the tick's: skip it
2948
- // and let unrelated eligible tickets dispatch normally.
2949
- verdict = {
2950
- unresolved: [ticket.ref.id],
2951
- diagnostics: [
2952
- `ticket ${ticket.ref.id}: dependency check failed — ${
2953
- error instanceof Error ? error.message : String(error)
2954
- }`,
2955
- ],
2956
- }
2957
- }
2958
- if (verdict.unresolved.length > 0) {
2959
- report.dependencyBlocked += 1
2960
- report.dependencyDiagnostics.push(...verdict.diagnostics)
2961
- continue
3184
+ if (ticket.blockersResolved === true || (ticket.blockedBy ?? []).length === 0) continue
3185
+ let verdict: DependencyVerdict
3186
+ try {
3187
+ if (!(await this.loadDependencyGraph(ticket, nodes, opts))) return
3188
+ verdict = analyzeDependencies(ticket.ref.id, ticket.blockedBy ?? [], nodes)
3189
+ } catch (error) {
3190
+ if (error instanceof TicketSourceRateLimitError) {
3191
+ await this.sourceFailure(error, 'dependency-check')
3192
+ throw error
2962
3193
  }
3194
+ scanSuccessful = false
3195
+ failed.add(ticket.ref.id)
3196
+ report.dependencyCheckFailed += 1
3197
+ report.dependencyDiagnostics.push(
3198
+ `ticket ${ticket.ref.id}: dependency check failed — ${error instanceof Error ? error.message : String(error)}`,
3199
+ )
3200
+ continue
3201
+ }
3202
+ verdicts.set(ticket.ref.id, verdict)
3203
+ }
3204
+ if (this.outOfBudget(opts)) return
3205
+ if (failed.size === 0) await this.sourceCondition(undefined, 'dependency-check')
3206
+ for (const verdict of verdicts.values()) {
3207
+ if (verdict.unresolved.length > 0) {
3208
+ report.dependencyBlocked += 1
3209
+ report.dependencyDiagnostics.push(...verdict.diagnostics)
2963
3210
  }
3211
+ }
3212
+ for (const ticket of ready) {
3213
+ if (capacity <= 0 || this.outOfBudget(opts)) break
3214
+ if (withheld.has(ticket.ref.id)) continue
3215
+ if (failed.has(ticket.ref.id)) continue
3216
+ if ((verdicts.get(ticket.ref.id)?.unresolved.length ?? 0) > 0) continue
2964
3217
 
2965
3218
  // Claim-before-launch (§12): losing the claim means another dispatcher
2966
3219
  // (or an earlier tick) owns this ticket.
@@ -3036,36 +3289,36 @@ export class Dispatcher {
3036
3289
  // means the remote provisioning continuation owns the rest.
3037
3290
  if (outcome === 'completed' || outcome === 'deferred') report.dispatched += 1
3038
3291
  }
3292
+ if (scanSuccessful && !this.outOfBudget(opts)) await this.completeScan(checkpoint, report)
3039
3293
  }
3040
3294
 
3041
3295
  /**
3042
- * Close `nodes` over everything reachable from `ticket`'s blockers, so the
3043
- * analyzer can both gate and name a cycle. Breadth-first, one port call per
3044
- * level, skipping ids already cached — the common case (a blocker or two,
3045
- * none of them blocked themselves) is a single call.
3046
- *
3047
- * INVARIANT: `nodes` contains only states actually returned by the source.
3048
- * The cache is shared across the tick's tickets, so a node this ticket
3049
- * invents is a node the *next* ticket will trust as a fact about its own
3050
- * blocker. Nothing is seeded here for that reason (f_8bc9ee0c); the
3051
- * analyzer takes the ticket's own blockers as an argument instead, and
3052
- * `findCycle` closes a cycle via its path rather than the map.
3053
- *
3054
- * A provider-side cycle terminates the walk: an id already in `nodes` is
3055
- * never re-queued, so every reachable node is fetched exactly once.
3296
+ * Walk unresolved branches in provider order. Listing facts and dependency
3297
+ * reads share a tick-local cache; traversal visits are local to this root.
3056
3298
  */
3057
3299
  private async loadDependencyGraph(
3058
3300
  ticket: Ticket,
3059
3301
  nodes: Map<string, DependencyState>,
3060
- ): Promise<void> {
3061
- let frontier = [...new Set(ticket.blockedBy ?? [])].filter((id) => !nodes.has(id))
3302
+ opts: TickOpts = {},
3303
+ ): Promise<boolean> {
3304
+ const visited = new Set<string>()
3305
+ let frontier = [...new Set(ticket.blockedBy ?? [])]
3062
3306
  while (frontier.length > 0) {
3063
- const states = await this.deps.tickets.dependencyStates(frontier)
3064
- for (const state of states) nodes.set(state.id, state)
3065
- frontier = [...new Set(states.flatMap((state) => state.blockedBy))].filter(
3066
- (id) => !nodes.has(id),
3067
- )
3307
+ if (this.outOfBudget(opts)) return false
3308
+ const uncached = frontier.filter((id) => !nodes.has(id))
3309
+ if (uncached.length > 0) {
3310
+ const states = await this.deps.tickets.dependencyStates(uncached)
3311
+ for (const state of states) nodes.set(state.id, state)
3312
+ }
3313
+ const next: string[] = []
3314
+ for (const id of frontier) {
3315
+ visited.add(id)
3316
+ const node = nodes.get(id)
3317
+ if (node?.exists && !node.resolved) next.push(...node.blockedBy)
3318
+ }
3319
+ frontier = [...new Set(next)].filter((id) => !visited.has(id))
3068
3320
  }
3321
+ return true
3069
3322
  }
3070
3323
 
3071
3324
  /** Bounce (§6.3): back to Triage with a comment citing the standard and
@@ -97,6 +97,7 @@ export async function runHarvestChild(
97
97
  uuids: randomUuids(),
98
98
  clock: systemClock,
99
99
  instance: input.instance,
100
+ ...(input.preparedRun ? { preparedRun: input.preparedRun } : {}),
100
101
  // Adopted holder: the guest skips its own claim, heartbeats the owning
101
102
  // dispatch loop's repository lease, and never releases it.
102
103
  ...(input.leaseHolder !== undefined ? { leaseHolder: input.leaseHolder } : {}),
@@ -92,9 +92,26 @@ export function classifyHarvestOutcome(
92
92
  input: HarvestExecutionClassificationInput,
93
93
  ): HarvestRunnerResult {
94
94
  const state = reduceHarvest(events)
95
- const latestRun = state.runs.at(-1)
95
+ const started = events.find(
96
+ (event) =>
97
+ event.type === 'harvest.execution.started' && event.seq === input.executionStartedSeq,
98
+ )
99
+ const authorization =
100
+ started?.type === 'harvest.execution.started'
101
+ ? events.find(
102
+ (event) =>
103
+ event.type === 'harvest.execution.authorized' &&
104
+ event.payload.execution === started.payload.execution,
105
+ )
106
+ : undefined
107
+ const selected =
108
+ authorization?.type === 'harvest.execution.authorized' ? authorization.payload : undefined
109
+ const latestRun = selected
110
+ ? state.runs.find((run) => run.run === selected.run)
111
+ : state.runs.at(-1)
96
112
  if (latestRun !== undefined) {
97
- const launch = latestRun.startedSeq > input.executionStartedSeq ? 'started' : 'resumed'
113
+ const launch =
114
+ selected?.launch ?? (latestRun.startedSeq > input.executionStartedSeq ? 'started' : 'resumed')
98
115
  // Recency gate: a terminal outcome counts only if its fact — the run's
99
116
  // terminal fact, or for an unresolved failure the failure fact itself
100
117
  // (`failureSeq`), which also covers exhaustion via `terminalSeq` —