@jr2/orchestrator 0.4.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (361) hide show
  1. package/README.md +20 -2
  2. package/console/components/app.ts +2 -0
  3. package/console/components/fleet.ts +14 -1
  4. package/console/components/machine-pane.ts +41 -2
  5. package/console/main.ts +13 -1
  6. package/console/store.ts +102 -6
  7. package/console/style.css +21 -0
  8. package/docs/agents-block.md +5 -0
  9. package/docs/concepts/agents-and-turns.md +478 -0
  10. package/docs/concepts/architecture.md +243 -0
  11. package/docs/concepts/charts.md +146 -0
  12. package/docs/concepts/composition.md +357 -0
  13. package/docs/concepts/converge.md +257 -0
  14. package/docs/concepts/durable-runs.md +189 -0
  15. package/docs/concepts/effects.md +658 -0
  16. package/docs/concepts/gates.md +208 -0
  17. package/docs/concepts/machines-and-workflows.md +298 -0
  18. package/docs/concepts/pools-and-sources.md +230 -0
  19. package/docs/concepts/secrets-and-trust.md +306 -0
  20. package/docs/concepts/the-menu.md +381 -0
  21. package/docs/concepts/workspaces.md +756 -0
  22. package/docs/get-started/a-model-decides.md +563 -0
  23. package/docs/get-started/an-agent-in-a-sandbox.md +383 -0
  24. package/docs/get-started/connect-a-model.md +248 -0
  25. package/docs/get-started/create-an-instance.md +194 -0
  26. package/docs/get-started/have-your-assistant-write-a-machine.md +993 -0
  27. package/docs/get-started/install.md +130 -0
  28. package/docs/guides/add-tools-to-the-sandbox.md +308 -0
  29. package/docs/guides/answer-a-gate.md +304 -0
  30. package/docs/guides/bind-open-parts.md +224 -0
  31. package/docs/guides/cancel-a-run.md +146 -0
  32. package/docs/guides/clean-up.md +206 -0
  33. package/docs/guides/cut-a-worktree.md +560 -0
  34. package/docs/guides/fix-a-failed-converge.md +361 -0
  35. package/docs/guides/give-the-cluster-git-credentials.md +222 -0
  36. package/docs/guides/iterate-on-a-machine-on-kind.md +190 -0
  37. package/docs/guides/let-an-agent-write-a-chart.md +230 -0
  38. package/docs/guides/look-inside-a-sandbox.md +177 -0
  39. package/docs/guides/mirror-the-kit-images.md +176 -0
  40. package/docs/guides/move-from-kind-to-a-shared-cluster.md +318 -0
  41. package/docs/guides/open-the-console.md +91 -0
  42. package/docs/guides/publish-a-machine-as-an-npm-package.md +350 -0
  43. package/docs/guides/push-and-sign-a-branch.md +532 -0
  44. package/docs/guides/run-a-chart-by-name.md +270 -0
  45. package/docs/guides/run-a-command-or-call-a-url.md +587 -0
  46. package/docs/guides/run-one-worker-per-item.md +322 -0
  47. package/docs/guides/set-an-agents-access-and-dials.md +420 -0
  48. package/docs/guides/size-a-workspace.md +217 -0
  49. package/docs/guides/start-a-run-from-a-run.md +237 -0
  50. package/docs/guides/start-a-run.md +207 -0
  51. package/docs/guides/upgrade-the-kit-version.md +240 -0
  52. package/docs/guides/use-a-hosted-provider.md +214 -0
  53. package/docs/guides/use-a-local-model.md +313 -0
  54. package/docs/guides/watch-a-run.md +349 -0
  55. package/docs/guides/write-a-workflow.md +318 -0
  56. package/docs/index.md +44 -0
  57. package/docs/llms.txt +237 -0
  58. package/docs/reference/authoring-surface/agent.md +151 -0
  59. package/docs/reference/authoring-surface/answer.md +156 -0
  60. package/docs/reference/authoring-surface/arg.md +82 -0
  61. package/docs/reference/authoring-surface/basic.md +62 -0
  62. package/docs/reference/authoring-surface/bearer.md +53 -0
  63. package/docs/reference/authoring-surface/branchname.md +15 -0
  64. package/docs/reference/authoring-surface/chartschema.md +23 -0
  65. package/docs/reference/authoring-surface/chartschemafor.md +48 -0
  66. package/docs/reference/authoring-surface/command.md +206 -0
  67. package/docs/reference/authoring-surface/customize.md +64 -0
  68. package/docs/reference/authoring-surface/defineconfig.md +41 -0
  69. package/docs/reference/authoring-surface/defineevent.md +114 -0
  70. package/docs/reference/authoring-surface/dispatch.md +126 -0
  71. package/docs/reference/authoring-surface/fail.md +96 -0
  72. package/docs/reference/authoring-surface/file.md +142 -0
  73. package/docs/reference/authoring-surface/frompick.md +277 -0
  74. package/docs/reference/authoring-surface/http.md +259 -0
  75. package/docs/reference/authoring-surface/isfailure.md +58 -0
  76. package/docs/reference/authoring-surface/isready.md +14 -0
  77. package/docs/reference/authoring-surface/jr2setup.md +237 -0
  78. package/docs/reference/authoring-surface/open.md +105 -0
  79. package/docs/reference/authoring-surface/openbody.md +86 -0
  80. package/docs/reference/authoring-surface/paramsof.md +143 -0
  81. package/docs/reference/authoring-surface/pool.md +186 -0
  82. package/docs/reference/authoring-surface/push.md +130 -0
  83. package/docs/reference/authoring-surface/readyof.md +21 -0
  84. package/docs/reference/authoring-surface/secret.md +139 -0
  85. package/docs/reference/authoring-surface/sentby.md +83 -0
  86. package/docs/reference/authoring-surface/source.md +51 -0
  87. package/docs/reference/authoring-surface/text.md +62 -0
  88. package/docs/reference/authoring-surface/type-agentdeclaration.md +27 -0
  89. package/docs/reference/authoring-surface/type-agentdefinition.md +80 -0
  90. package/docs/reference/authoring-surface/type-agentlogic.md +15 -0
  91. package/docs/reference/authoring-surface/type-agentslots.md +15 -0
  92. package/docs/reference/authoring-surface/type-answeraction.md +12 -0
  93. package/docs/reference/authoring-surface/type-anyframepath.md +14 -0
  94. package/docs/reference/authoring-surface/type-anyworktreepath.md +15 -0
  95. package/docs/reference/authoring-surface/type-arg.md +19 -0
  96. package/docs/reference/authoring-surface/type-argnames.md +12 -0
  97. package/docs/reference/authoring-surface/type-bindactors.md +14 -0
  98. package/docs/reference/authoring-surface/type-binding.md +61 -0
  99. package/docs/reference/authoring-surface/type-bindingof.md +12 -0
  100. package/docs/reference/authoring-surface/type-bodyacceptsdoor.md +15 -0
  101. package/docs/reference/authoring-surface/type-bodyacceptsslots.md +17 -0
  102. package/docs/reference/authoring-surface/type-ceilings.md +41 -0
  103. package/docs/reference/authoring-surface/type-chainlink.md +19 -0
  104. package/docs/reference/authoring-surface/type-chartaction.md +46 -0
  105. package/docs/reference/authoring-surface/type-chartagent.md +28 -0
  106. package/docs/reference/authoring-surface/type-chartevent.md +20 -0
  107. package/docs/reference/authoring-surface/type-chartguard.md +20 -0
  108. package/docs/reference/authoring-surface/type-chartinvoke.md +28 -0
  109. package/docs/reference/authoring-surface/type-chartjson.md +14 -0
  110. package/docs/reference/authoring-surface/type-chartstate.md +46 -0
  111. package/docs/reference/authoring-surface/type-charttext.md +40 -0
  112. package/docs/reference/authoring-surface/type-charttransition.md +22 -0
  113. package/docs/reference/authoring-surface/type-charttransitions.md +14 -0
  114. package/docs/reference/authoring-surface/type-chartvalue.md +15 -0
  115. package/docs/reference/authoring-surface/type-childat.md +12 -0
  116. package/docs/reference/authoring-surface/type-childslots.md +12 -0
  117. package/docs/reference/authoring-surface/type-commandenv.md +15 -0
  118. package/docs/reference/authoring-surface/type-commandinput.md +20 -0
  119. package/docs/reference/authoring-surface/type-commandoptions.md +56 -0
  120. package/docs/reference/authoring-surface/type-commandoutput.md +122 -0
  121. package/docs/reference/authoring-surface/type-customize.md +111 -0
  122. package/docs/reference/authoring-surface/type-dispatchdefinition.md +39 -0
  123. package/docs/reference/authoring-surface/type-dispatchinput.md +14 -0
  124. package/docs/reference/authoring-surface/type-dispatchopenparts.md +14 -0
  125. package/docs/reference/authoring-surface/type-dispatchoutput.md +72 -0
  126. package/docs/reference/authoring-surface/type-doorlessinput.md +19 -0
  127. package/docs/reference/authoring-surface/type-duration.md +15 -0
  128. package/docs/reference/authoring-surface/type-effectbinding.md +93 -0
  129. package/docs/reference/authoring-surface/type-effectlogic.md +12 -0
  130. package/docs/reference/authoring-surface/type-envopenparts.md +14 -0
  131. package/docs/reference/authoring-surface/type-eventaudience.md +27 -0
  132. package/docs/reference/authoring-surface/type-eventdef.md +45 -0
  133. package/docs/reference/authoring-surface/type-eventfrom.md +16 -0
  134. package/docs/reference/authoring-surface/type-eventsemantics.md +26 -0
  135. package/docs/reference/authoring-surface/type-execoutcome.md +34 -0
  136. package/docs/reference/authoring-surface/type-failslot.md +14 -0
  137. package/docs/reference/authoring-surface/type-failure.md +19 -0
  138. package/docs/reference/authoring-surface/type-failureinput.md +20 -0
  139. package/docs/reference/authoring-surface/type-filebody.md +37 -0
  140. package/docs/reference/authoring-surface/type-framepath.md +14 -0
  141. package/docs/reference/authoring-surface/type-frominput.md +14 -0
  142. package/docs/reference/authoring-surface/type-hostinjectedinput.md +56 -0
  143. package/docs/reference/authoring-surface/type-httpdefinition.md +104 -0
  144. package/docs/reference/authoring-surface/type-httpfileinput.md +20 -0
  145. package/docs/reference/authoring-surface/type-httpinput.md +12 -0
  146. package/docs/reference/authoring-surface/type-httpmethod.md +14 -0
  147. package/docs/reference/authoring-surface/type-httpopenparts.md +12 -0
  148. package/docs/reference/authoring-surface/type-httpoutput.md +95 -0
  149. package/docs/reference/authoring-surface/type-httprequestparts.md +33 -0
  150. package/docs/reference/authoring-surface/type-jr2config.md +126 -0
  151. package/docs/reference/authoring-surface/type-lateroutcome.md +21 -0
  152. package/docs/reference/authoring-surface/type-lends.md +38 -0
  153. package/docs/reference/authoring-surface/type-logicat.md +12 -0
  154. package/docs/reference/authoring-surface/type-markedif.md +12 -0
  155. package/docs/reference/authoring-surface/type-mergedactors.md +12 -0
  156. package/docs/reference/authoring-surface/type-namedsecret.md +25 -0
  157. package/docs/reference/authoring-surface/type-namestart.md +77 -0
  158. package/docs/reference/authoring-surface/type-noslotworkspaceoptions.md +27 -0
  159. package/docs/reference/authoring-surface/type-notworktreekeys.md +12 -0
  160. package/docs/reference/authoring-surface/type-openbodydeclaration.md +40 -0
  161. package/docs/reference/authoring-surface/type-openbodylogic.md +14 -0
  162. package/docs/reference/authoring-surface/type-openbodyoutput.md +42 -0
  163. package/docs/reference/authoring-surface/type-openbodyslot.md +18 -0
  164. package/docs/reference/authoring-surface/type-opencommandinput.md +22 -0
  165. package/docs/reference/authoring-surface/type-openeffect.md +12 -0
  166. package/docs/reference/authoring-surface/type-openeffectslots.md +12 -0
  167. package/docs/reference/authoring-surface/type-openof.md +12 -0
  168. package/docs/reference/authoring-surface/type-opensecret.md +23 -0
  169. package/docs/reference/authoring-surface/type-opensecretmark.md +12 -0
  170. package/docs/reference/authoring-surface/type-permissivenoslotworkspaceoptions.md +26 -0
  171. package/docs/reference/authoring-surface/type-permissiveworkspaceoptions.md +27 -0
  172. package/docs/reference/authoring-surface/type-poolmachine.md +28 -0
  173. package/docs/reference/authoring-surface/type-poolnextinput.md +22 -0
  174. package/docs/reference/authoring-surface/type-poolspec.md +60 -0
  175. package/docs/reference/authoring-surface/type-pushinput.md +34 -0
  176. package/docs/reference/authoring-surface/type-pushoptions.md +79 -0
  177. package/docs/reference/authoring-surface/type-pushoutput.md +149 -0
  178. package/docs/reference/authoring-surface/type-pushwireoutput.md +12 -0
  179. package/docs/reference/authoring-surface/type-quantity.md +14 -0
  180. package/docs/reference/authoring-surface/type-reached.md +12 -0
  181. package/docs/reference/authoring-surface/type-reposlot.md +70 -0
  182. package/docs/reference/authoring-surface/type-reposlotsof.md +16 -0
  183. package/docs/reference/authoring-surface/type-sandboxoptions.md +84 -0
  184. package/docs/reference/authoring-surface/type-secret.md +25 -0
  185. package/docs/reference/authoring-surface/type-sender.md +19 -0
  186. package/docs/reference/authoring-surface/type-sentbyguards.md +12 -0
  187. package/docs/reference/authoring-surface/type-size.md +85 -0
  188. package/docs/reference/authoring-surface/type-slothandles.md +24 -0
  189. package/docs/reference/authoring-surface/type-sourcenextresult.md +17 -0
  190. package/docs/reference/authoring-surface/type-sourcespec.md +44 -0
  191. package/docs/reference/authoring-surface/type-syncstate.md +12 -0
  192. package/docs/reference/authoring-surface/type-text.md +19 -0
  193. package/docs/reference/authoring-surface/type-textpart.md +24 -0
  194. package/docs/reference/authoring-surface/type-thinkinglevel.md +39 -0
  195. package/docs/reference/authoring-surface/type-toparameterizedobject.md +14 -0
  196. package/docs/reference/authoring-surface/type-toprovidedactor.md +14 -0
  197. package/docs/reference/authoring-surface/type-unknown.md +21 -0
  198. package/docs/reference/authoring-surface/type-usercontainer.md +77 -0
  199. package/docs/reference/authoring-surface/type-withoutemptyindex.md +20 -0
  200. package/docs/reference/authoring-surface/type-workflowevent.md +46 -0
  201. package/docs/reference/authoring-surface/type-workspaceaccess.md +16 -0
  202. package/docs/reference/authoring-surface/type-workspaced.md +115 -0
  203. package/docs/reference/authoring-surface/type-workspacehandles.md +22 -0
  204. package/docs/reference/authoring-surface/type-workspacemachine.md +29 -0
  205. package/docs/reference/authoring-surface/type-workspaceof.md +13 -0
  206. package/docs/reference/authoring-surface/type-workspaceoptions.md +27 -0
  207. package/docs/reference/authoring-surface/type-worktreefound.md +16 -0
  208. package/docs/reference/authoring-surface/type-worktreeinput.md +56 -0
  209. package/docs/reference/authoring-surface/type-worktreekey.md +14 -0
  210. package/docs/reference/authoring-surface/type-worktreeoptions.md +29 -0
  211. package/docs/reference/authoring-surface/type-worktreeoutput.md +45 -0
  212. package/docs/reference/authoring-surface/type-worktreepath.md +16 -0
  213. package/docs/reference/authoring-surface/type-worktreeready.md +37 -0
  214. package/docs/reference/authoring-surface/type-worktreerefused.md +21 -0
  215. package/docs/reference/authoring-surface/workspace.md +287 -0
  216. package/docs/reference/authoring-surface/worktree.md +71 -0
  217. package/docs/reference/authoring-surface.md +217 -0
  218. package/docs/reference/cli.md +433 -0
  219. package/docs/reference/config.md +765 -0
  220. package/docs/reference/decisions/0001-operator-owned-generic-sandbox.md +114 -0
  221. package/docs/reference/decisions/0002-duplex-streaming-actor.md +117 -0
  222. package/docs/reference/decisions/0004-worktree-provisioning-model.md +130 -0
  223. package/docs/reference/decisions/0005-sandbox-pod-composition.md +240 -0
  224. package/docs/reference/decisions/0006-agent-control-surface.md +50 -0
  225. package/docs/reference/decisions/0007-durable-machine-state.md +109 -0
  226. package/docs/reference/decisions/0008-orchestrator-as-deployed-app.md +60 -0
  227. package/docs/reference/decisions/0009-cli-and-instance-interface.md +275 -0
  228. package/docs/reference/decisions/0010-bdd-acceptance-tests.md +160 -0
  229. package/docs/reference/decisions/0011-workflow-defined-events.md +141 -0
  230. package/docs/reference/decisions/0012-workspace-wrapper-machine.md +103 -0
  231. package/docs/reference/decisions/0013-the-agent-reaches-its-machine-through-a-container-it-cannot-read.md +175 -0
  232. package/docs/reference/decisions/0014-observation-is-open-run-state-is-not.md +95 -0
  233. package/docs/reference/decisions/0015-authoring-surface-absorbs-the-mechanism.md +271 -0
  234. package/docs/reference/decisions/0016-agent-turn-mechanics-are-internal.md +126 -0
  235. package/docs/reference/decisions/0017-pool-over-a-source-port.md +71 -0
  236. package/docs/reference/decisions/0018-instance-agents-are-definitions-jr2-assembles-the-harness.md +111 -0
  237. package/docs/reference/decisions/0019-one-converging-command-against-the-current-context.md +287 -0
  238. package/docs/reference/decisions/0020-private-ca-trust-is-a-named-concept.md +60 -0
  239. package/docs/reference/decisions/0021-workspace-continuity-is-a-lease-that-answers-back.md +101 -0
  240. package/docs/reference/decisions/0022-observation-is-a-level-triggered-feed.md +103 -0
  241. package/docs/reference/decisions/0023-the-harness-prints-the-conversation.md +175 -0
  242. package/docs/reference/decisions/0024-an-agents-turn-ends-with-the-state-that-asked-for-it.md +436 -0
  243. package/docs/reference/decisions/0025-cancel-ends-a-run-stop-parks-it.md +65 -0
  244. package/docs/reference/decisions/0026-a-turn-that-is-over-is-never-prompted.md +62 -0
  245. package/docs/reference/decisions/0027-the-harness-is-jr2s-own-server-flue-retires-the-wire-stays.md +245 -0
  246. package/docs/reference/decisions/0028-what-an-agent-may-do-to-the-workspace-is-part-of-its-definition.md +90 -0
  247. package/docs/reference/decisions/0029-a-menu-is-fixed-for-a-conversation-and-a-turn-is-told-its-allowed-picks.md +154 -0
  248. package/docs/reference/decisions/0030-a-snapshot-names-the-machine-it-was-written-under.md +100 -0
  249. package/docs/reference/decisions/0031-menu-only-agents-run-on-the-instance-harness.md +193 -0
  250. package/docs/reference/decisions/0032-the-console-unlocks-with-the-instance-token.md +61 -0
  251. package/docs/reference/decisions/0033-a-machine-declares-the-input-that-starts-it.md +126 -0
  252. package/docs/reference/decisions/0034-the-console-renders-with-preact-and-the-server-erases-the-types.md +57 -0
  253. package/docs/reference/decisions/0035-a-runaway-turn-is-ended-rerolled-once-then-a-fault.md +92 -0
  254. package/docs/reference/decisions/0036-a-turn-compacts-its-own-context-mid-flight.md +138 -0
  255. package/docs/reference/decisions/0037-an-instance-builds-its-sandbox-images-jr2-injects-the-harness.md +196 -0
  256. package/docs/reference/decisions/0038-jr2-up-builds-every-image-it-deploys.md +170 -0
  257. package/docs/reference/decisions/0039-image-garbage-collects-by-reachability.md +115 -0
  258. package/docs/reference/decisions/0041-a-build-the-host-already-holds-is-not-spent-again.md +64 -0
  259. package/docs/reference/decisions/0042-ready-is-not-routable-an-admission-retries-its-connection.md +142 -0
  260. package/docs/reference/decisions/0043-the-kit-is-tested-as-installed-a-local-registry-stands-in-for-npm.md +170 -0
  261. package/docs/reference/decisions/0044-kit-images-live-at-a-canonical-home-a-self-host-mirrors-it.md +85 -0
  262. package/docs/reference/decisions/0045-the-platform-joins-the-image-address-and-the-cluster-chooses-it.md +82 -0
  263. package/docs/reference/decisions/0046-a-converge-failure-carries-its-diagnosis.md +34 -0
  264. package/docs/reference/decisions/0047-the-git-ssh-key-source-is-the-users-choice.md +47 -0
  265. package/docs/reference/decisions/0048-the-orchestrator-boots-without-its-repos.md +36 -0
  266. package/docs/reference/decisions/0049-a-machine-carries-its-parts-and-composes-by-invoke.md +126 -0
  267. package/docs/reference/decisions/0050-what-code-names-config-declares-what-the-cli-names-discovery-finds.md +73 -0
  268. package/docs/reference/decisions/0051-a-repo-is-a-slot-on-the-workspace-and-a-cache-on-the-node.md +238 -0
  269. package/docs/reference/decisions/0052-a-sandbox-lands-where-an-ordinary-pod-lands-and-the-cache-follows-it.md +86 -0
  270. package/docs/reference/decisions/0053-a-fetch-inside-the-pod-asks-the-node-cache-and-the-cache-asks-the-remote.md +123 -0
  271. package/docs/reference/decisions/0054-the-kit-ships-machines-and-a-shipped-machine-leaves-its-parts-open.md +126 -0
  272. package/docs/reference/decisions/0055-a-release-is-a-pushed-tag-and-the-publish-guard-is-the-credential.md +107 -0
  273. package/docs/reference/decisions/0056-a-global-jr2-hands-off-to-the-instances-own-and-an-instance-has-one-kit-version.md +62 -0
  274. package/docs/reference/decisions/0057-a-turn-is-its-frame-its-dials-and-whether-it-continues.md +152 -0
  275. package/docs/reference/decisions/0058-a-harness-answers-the-orchestrator-alone.md +108 -0
  276. package/docs/reference/decisions/0059-a-harness-holds-stand-ins-and-the-custodian-holds-the-keys.md +427 -0
  277. package/docs/reference/decisions/0060-a-workspace-states-its-size-and-jr2-reserves-all-of-it.md +111 -0
  278. package/docs/reference/decisions/0061-a-memory-kill-ends-the-agents-processes-not-the-harness.md +83 -0
  279. package/docs/reference/decisions/0062-jr2-briefs-the-agent-on-its-seat-never-on-its-task.md +239 -0
  280. package/docs/reference/decisions/0063-the-orchestrator-watches-the-cluster.md +82 -0
  281. package/docs/reference/decisions/0064-a-workspace-waits-for-capacity.md +79 -0
  282. package/docs/reference/decisions/0065-a-persist-survives-a-crash-and-history-is-a-week.md +65 -0
  283. package/docs/reference/decisions/0066-a-model-tier-runs-a-real-model-and-a-change-to-what-it-reads-runs-the-tier.md +552 -0
  284. package/docs/reference/decisions/0067-an-effect-is-an-actor-slot-and-runs-at-most-once-unless-marked-idempotent.md +458 -0
  285. package/docs/reference/decisions/0068-an-effector-holds-an-effects-secrets-and-signs-its-wire.md +475 -0
  286. package/docs/reference/decisions/0069-the-orchestrator-is-a-kit-image-and-an-instance-publishes-its-assembly.md +180 -0
  287. package/docs/reference/decisions/0070-a-machine-lends-its-parts-to-a-chart.md +334 -0
  288. package/docs/reference/decisions/0071-a-run-starts-a-run-through-a-dispatch-effect.md +183 -0
  289. package/docs/reference/decisions/0072-the-docs-ship-inside-the-kit-and-an-instances-agents-md-indexes-them.md +144 -0
  290. package/docs/reference/decisions/0073-a-machine-fails-a-run-with-fail-and-an-uncaught-failure-ends-it-failed.md +122 -0
  291. package/docs/reference/decisions/0074-an-unrouted-mechanism-event-is-refused-at-load-and-faults-the-run.md +76 -0
  292. package/docs/reference/decisions/0075-a-machine-cuts-a-worktree-and-the-workspace-only-clones.md +295 -0
  293. package/docs/reference/decisions/0076-an-agents-control-credential-lives-only-where-no-workspace-code-runs.md +320 -0
  294. package/docs/reference/decisions/0077-jr2-requires-landlock-on-every-sandbox-node.md +79 -0
  295. package/docs/reference/decisions.md +86 -0
  296. package/docs/reference/glossary.md +687 -0
  297. package/docs/reference/http-api.md +1386 -0
  298. package/docs/reference/machines.md +189 -0
  299. package/docs/reference/mechanism-events.md +72 -0
  300. package/docs/why.md +207 -0
  301. package/package.json +5 -3
  302. package/src/actor.ts +694 -106
  303. package/src/agent.ts +114 -38
  304. package/src/ambient.ts +3 -4
  305. package/src/assembly-checks.ts +189 -7
  306. package/src/cause.ts +234 -0
  307. package/src/chart.ts +345 -46
  308. package/src/charts.ts +43 -8
  309. package/src/config.ts +440 -178
  310. package/src/custodian.lua +17 -5
  311. package/src/custodian.ts +325 -99
  312. package/src/customize.ts +199 -49
  313. package/src/dispatch-effect.ts +945 -0
  314. package/src/door-compat.ts +217 -0
  315. package/src/durability.ts +11 -4
  316. package/src/effect-checks.ts +50 -3
  317. package/src/effect.ts +641 -128
  318. package/src/effector-wire.ts +165 -13
  319. package/src/fail.ts +222 -0
  320. package/src/harness-client.ts +587 -50
  321. package/src/harness-env.ts +33 -0
  322. package/src/held-secrets.ts +125 -16
  323. package/src/holds.ts +3 -0
  324. package/src/http-effect.ts +556 -58
  325. package/src/http.ts +480 -30
  326. package/src/index.ts +74 -82
  327. package/src/instance.ts +68 -8
  328. package/src/internal.ts +85 -0
  329. package/src/kube-client.ts +6 -3
  330. package/src/machine-doc.ts +24 -7
  331. package/src/mechanism-routes.ts +168 -0
  332. package/src/model-keys.generated.ts +53 -0
  333. package/src/model-keys.ts +42 -58
  334. package/src/names.ts +19 -6
  335. package/src/notices.ts +483 -41
  336. package/src/open-body.ts +242 -40
  337. package/src/open-effect.ts +81 -8
  338. package/src/open.ts +90 -6
  339. package/src/parts.ts +213 -30
  340. package/src/party.ts +101 -2
  341. package/src/pick.ts +497 -11
  342. package/src/pool.ts +282 -50
  343. package/src/registration.ts +133 -12
  344. package/src/repo-identity.ts +7 -3
  345. package/src/repos.ts +22 -3
  346. package/src/run-host.ts +665 -45
  347. package/src/run-output.ts +38 -0
  348. package/src/sandbox-effect.ts +457 -71
  349. package/src/sandbox-kube.ts +66 -17
  350. package/src/secret-placement.ts +59 -17
  351. package/src/server.ts +3 -0
  352. package/src/setup.ts +325 -62
  353. package/src/size.ts +121 -9
  354. package/src/snapshot-store.ts +54 -10
  355. package/src/stub-harness.ts +44 -16
  356. package/src/timers.ts +14 -1
  357. package/src/tokens.ts +107 -32
  358. package/src/vocabulary.ts +88 -14
  359. package/src/wire.ts +100 -26
  360. package/src/workspace.ts +771 -238
  361. package/src/worktree.ts +249 -0
package/src/run-host.ts CHANGED
@@ -34,8 +34,9 @@ import { SENDER_KEY, type EventDef, type EventSemantics, type Sender } from "@jr
34
34
  import { partiesAt, seatOf, senderText } from "./party.ts";
35
35
  import { placeAnswers, type AnswerSeat } from "./pick.ts";
36
36
  import { inputSchemaOf } from "./vocabulary.ts";
37
- import type { EchoEvent, EchoStatusChild, LiveConversation, PickAnswer } from "./wire.ts";
37
+ import type { EchoEvent, EchoStatusChild, LiveConversation, Notice, PickAnswer } from "./wire.ts";
38
38
  import { answerOf, childErrorOf, replyOf, type Hold } from "./holds.ts";
39
+ import { PickCauses } from "./cause.ts";
39
40
  import { restOf, runningBelow } from "./rest.ts";
40
41
  import {
41
42
  acceptsNow,
@@ -52,6 +53,8 @@ import {
52
53
  wouldMove,
53
54
  targetsOf,
54
55
  leftAgain,
56
+ type DispatchedBy,
57
+ type DispatchStart,
55
58
  type Registration,
56
59
  type RetryTelemetry,
57
60
  type RunBinding,
@@ -69,6 +72,8 @@ import {
69
72
  } from "./workspace.ts";
70
73
  import { INSTANCE_HARNESS_SERVICE } from "./names.ts";
71
74
  import { composesSandbox } from "./parts.ts";
75
+ import { failurePayload, isFailure, type FailureInput } from "./fail.ts";
76
+ import { unroutedMechanismEvent, type MicrostepView } from "./mechanism-routes.ts";
72
77
  import { fingerprintOf } from "./fingerprint.ts";
73
78
  import {
74
79
  serializeMachine,
@@ -214,8 +219,44 @@ function chartDrift(machine: AnyStateMachine, routes: readonly BoundChartRoute[]
214
219
  return undefined;
215
220
  }
216
221
 
217
- /** The serializable identity of a run — what reconcile sees and what restore rebuilds from. */
218
- export type RunRecord = { runId: string; workflow: string; instanceId: string };
222
+ /** The serializable identity of a run — what reconcile sees and what restore rebuilds from.
223
+ * `dispatchedBy` is the run that started it through a `dispatch` Effect, and this run's depth in
224
+ * that chain, as the caller sent it (ADR-0071); absent for a run a human or a client started. */
225
+ export type RunRecord = { runId: string; workflow: string; instanceId: string; dispatchedBy?: DispatchedBy };
226
+
227
+ /** A start of a name no Workflow is registered under: the door answers 404 (ADR-0009), and an
228
+ * in-process `dispatch` settles `refused` with status 404 (ADR-0071). */
229
+ export class UnknownWorkflowError extends Error {
230
+ override name = "UnknownWorkflowError";
231
+ }
232
+
233
+ /** A start whose idempotency key a run of ANOTHER Workflow already carries (ADR-0071): the door
234
+ * answers 409, and an in-process `dispatch` settles `refused` with status 409. Never answered as
235
+ * the existing run: the caller asked for a run of one Workflow and would wait on a run of another. */
236
+ export class DispatchKeyConflictError extends Error {
237
+ override name = "DispatchKeyConflictError";
238
+ }
239
+
240
+ /** The run a key names (ADR-0071), with its Workflow so a start of another can be refused. */
241
+ export type DispatchedRun = { runId: string; instanceId: string; workflow: string };
242
+
243
+ /** The run `found` as the answer to a start of `workflow` under its key, or the 409 (ADR-0071). */
244
+ export function keyAnswer(
245
+ found: DispatchedRun,
246
+ workflow: string,
247
+ effect: string,
248
+ ): { runId: string; instanceId: string } {
249
+ if (found.workflow !== workflow) throw new DispatchKeyConflictError(keyConflict(found, workflow, effect));
250
+ return { runId: found.runId, instanceId: found.instanceId };
251
+ }
252
+
253
+ /** Why a start of `workflow` under `effect` is refused, when the key's run is of another. */
254
+ export function keyConflict(found: DispatchedRun, workflow: string, effect: string): string {
255
+ return (
256
+ `the key "${effect}" already started run ${found.runId} of "${found.workflow}", not of "${workflow}" — ` +
257
+ "one key is one start of one Workflow (ADR-0071)"
258
+ );
259
+ }
219
260
 
220
261
  /**
221
262
  * One live child MACHINE of a run, and its own children below it. A workflow's root machine is
@@ -306,24 +347,31 @@ function runWaiting(snapshot: unknown, logic?: unknown, path: string[] = []): Ru
306
347
  *
307
348
  * Left out: a Workspace in `placing`, which deletes its own Sandbox when its wait is aborted — and
308
349
  * only once any write in flight has settled, so a delete from here could race the apply and leave
309
- * the CR behind (ADR-0064); and one that is `done` or ended, whose `teardown` already ran.
350
+ * the CR behind (ADR-0064); and one that is `done` or ended, whose `teardown` or `failing` already
351
+ * ran. An actor that ended in `error` is not a Workspace here, but its children are read: a failed
352
+ * run's root (ADR-0073) is in `error`, and its active children are still in its snapshot, as
353
+ * `stopTree` reads them.
354
+ *
355
+ * With `placing`, a Workspace in `placing` is also named: the names that a live Workspace holds
356
+ * now, which a delete must leave alone.
310
357
  */
311
- function runSandboxes(actor: AnyActorRef, runId: string): string[] {
358
+ function runSandboxes(actor: AnyActorRef, runId: string, opts: { placing?: boolean } = {}): string[] {
312
359
  const out: string[] = [];
313
360
  const snap = actor.getSnapshot() as
314
361
  | { status?: string; value?: unknown; context?: { wsId?: unknown }; children?: Record<string, AnyActorRef> }
315
362
  | undefined;
316
- if (snap?.status !== "active") return out;
363
+ if (snap?.status !== "active" && snap?.status !== "error") return out;
317
364
  const logic = (actor as AnyActorRef & { logic?: unknown }).logic;
318
365
  if (
366
+ snap.status === "active" &&
319
367
  logic != null &&
320
368
  composesSandbox(logic as AnyStateMachine) &&
321
- snap.value !== "placing" &&
369
+ (opts.placing === true || snap.value !== "placing") &&
322
370
  typeof snap.context?.wsId === "string"
323
371
  ) {
324
372
  out.push(workspaceName(runId, snap.context.wsId));
325
373
  }
326
- for (const child of Object.values(snap.children ?? {})) out.push(...runSandboxes(child, runId));
374
+ for (const child of Object.values(snap.children ?? {})) out.push(...runSandboxes(child, runId, opts));
327
375
  return out;
328
376
  }
329
377
 
@@ -356,7 +404,7 @@ type Sending = {
356
404
 
357
405
  /**
358
406
  * Whose pick started a child (ADR-0067): the Agent's slot, and the invoke id of the Agent's state the
359
- * pick was made at, which tells one slot's two regions apart. `invoke` is absent when the event that
407
+ * pick was made at, which tells apart two states that invoke one slot, one after the other. `invoke` is absent when the event that
360
408
  * started the child named the Agent and not where (a pick taken outside its own send).
361
409
  */
362
410
  type StartedBy = { agent: string; invoke?: string };
@@ -443,7 +491,8 @@ function heldOf(holds: Iterable<Hold>): RunHeld[] {
443
491
  }
444
492
 
445
493
  /** A run's current observable state. `fault` carries the error message when status is "error"
446
- * (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). */
494
+ * (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). A run that ends
495
+ * "failed" (ADR-0073) has no `fault`: its `output` is the Failure's `{ reason, detail? }`. */
447
496
  export type RunStatus = RunRecord & {
448
497
  status: string;
449
498
  value: unknown;
@@ -475,8 +524,33 @@ export type RunStatus = RunRecord & {
475
524
  * second Chart in its life (a planner whose first Chart reaches a ceiling picks again), so these,
476
525
  * not the run id, say which bound Machine `runMachine` draws. Absent when it binds none. */
477
526
  chartHashes?: string[];
527
+ /**
528
+ * The Machine's output, for a run that is no longer live (ADR-0071): what a `dispatch` with
529
+ * `wait` settles with, and what `GET /runs/<id>` serves its foreign caller. Never for a live run,
530
+ * whose Machine has none yet. For a `failed` run, the Failure's `{ reason, detail? }` (ADR-0073),
531
+ * not a value of the Machine's output type. Behind the token, like `context`: it is the
532
+ * workflow's data.
533
+ */
534
+ output?: unknown;
535
+ /** The runs this run's `dispatch` Effects started, oldest first (ADR-0071): the chain's other
536
+ * direction, read off the caller's own ledger rows. A foreign dispatch's run is a run of the
537
+ * Orchestrator its target names. Absent when it started none. */
538
+ dispatched?: Dispatched[];
539
+ /** The Worktrees this run's cuts made, oldest first (ADR-0075), read off its ledger's settled
540
+ * `ready` rows, so an Agent's `git switch` does not change them. A re-cut of one path and branch
541
+ * is one entry. Behind the token, like `context`: a branch name is the workflow's data. Absent
542
+ * when the run cut none. */
543
+ worktrees?: RunWorktree[];
478
544
  };
479
545
 
546
+ /** One Worktree a `worktree()` Effect of this run cut (ADR-0075): its Repo Slot, its path, and the
547
+ * branch of the cut. `branch` is `null` for a detached Worktree. */
548
+ export type RunWorktree = { slot: string; path: string; branch: string | null };
549
+
550
+ /** One run a `dispatch` Effect of this run started (ADR-0071): its slot, the run, and when the
551
+ * ledger row was written. */
552
+ export type Dispatched = { slot: string; run: string; at: string };
553
+
480
554
  /**
481
555
  * A run as an UNAUTHENTICATED observer may see it: which run, of what workflow, and where in the
482
556
  * Machine it is. That is the whole set — enough to light up a state in the Console, and nothing
@@ -506,6 +580,13 @@ export type RunObservation = {
506
580
  /** The hashes of the Charts it binds now (see {@link RunStatus.chartHashes}). Structure: a hash
507
581
  * of a Chart's text, which `runMachine` already serves on the open band. */
508
582
  chartHashes?: string[];
583
+ /** The run that dispatched this one, its slot, and the chain's depth (ADR-0071). Routing — two
584
+ * ids, a slot name and a number — the class of thing `runId` and `value` are, so ADR-0014's line
585
+ * is unmoved. The idempotency key is not: it names the caller's ledger, which is the caller's. */
586
+ dispatchedBy?: Omit<DispatchedBy, "effect">;
587
+ /** The runs it dispatched, by slot (ADR-0071): the same routing as `dispatchedBy`, the other
588
+ * direction. */
589
+ dispatched?: Array<{ slot: string; run: string }>;
509
590
  };
510
591
 
511
592
  /** Project a full status down to what an observer may see. The one place the line is drawn. */
@@ -518,9 +599,17 @@ export function observe(status: RunStatus): RunObservation {
518
599
  children: status.children,
519
600
  ...(status.bound ? { bound: true as const } : {}),
520
601
  ...(status.chartHashes ? { chartHashes: status.chartHashes } : {}),
602
+ ...(status.dispatchedBy ? { dispatchedBy: routingOf(status.dispatchedBy) } : {}),
603
+ ...(status.dispatched ? { dispatched: status.dispatched.map(({ slot, run }) => ({ slot, run })) } : {}),
521
604
  };
522
605
  }
523
606
 
607
+ /** A {@link DispatchedBy} without its idempotency key (ADR-0071): what the open band carries. */
608
+ function routingOf(by: DispatchedBy): Omit<DispatchedBy, "effect"> {
609
+ const { effect: _key, ...routing } = by;
610
+ return routing;
611
+ }
612
+
524
613
  /** An Agent registration's Menu (ADR-0029): the one it registered, or — registered with none, by a
525
614
  * plain `setup()` machine — its derived set. */
526
615
  function menuOf(reg: Registration): Map<string, EventDef> {
@@ -588,6 +677,9 @@ export type AgentSurfaceView = {
588
677
  instanceId: string;
589
678
  runId: string;
590
679
  sandbox?: string;
680
+ /** The slot this surface belongs to (ADR-0076): its Agent's actor path and agent name — the slot
681
+ * the Harness must state to speak here. Absent on a registration no pod hosts. */
682
+ slot?: string;
591
683
  menu: SurfaceEvent[];
592
684
  allowed: string[];
593
685
  };
@@ -805,6 +897,9 @@ export type RunHostOptions = {
805
897
  sandbox?: SandboxPort;
806
898
  /** The Instance Harness base URL (ADR-0031) — where `workspace: "none"` Turns are admitted. */
807
899
  instanceHarness?: string;
900
+ /** The name this Orchestrator gives itself at another Orchestrator's door, as a foreign
901
+ * dispatch's `dispatchedBy.instance` (ADR-0071): stable and not secret. Deployed, its namespace. */
902
+ orchestratorName?: string;
808
903
  /** The Harness bearer for a placement (ADR-0058) — `startInstance` closes `harnessToken` over the
809
904
  * signing key. Handed to every run's binding, and to the echo pusher below. Absent, no Harness
810
905
  * call bears one: only the stub Harness answers such a host. */
@@ -812,6 +907,13 @@ export type RunHostOptions = {
812
907
  /** Where an Effector placement is, and its key (ADR-0068) — `startInstance` derives both, the key
813
908
  * from the signing key. Handed to every run's binding; absent, an http Effect faults pointedly. */
814
909
  effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
910
+ /**
911
+ * Why a listed named Chart cannot run now, or undefined (ADR-0070): the registry of named Charts
912
+ * keeps the unrunnable ones, never the host. An in-process `dispatch` of one is refused 409 with
913
+ * this reason, as its start through the door is (ADR-0071). Absent, such a name answers 404:
914
+ * the host registers only the runnable ones.
915
+ */
916
+ unrunnableChart?: (name: string) => string | undefined;
815
917
  /**
816
918
  * Build the run-narrative echo pusher for one Workspace's Harness (ADR-0023) — the seam a fake
817
919
  * echo server rides in tests; `startInstance` binds the real wire push (harness-client.ts).
@@ -840,6 +942,9 @@ export type RunHostOptions = {
840
942
  type RunBlob = {
841
943
  workflow: string;
842
944
  instanceId: string;
945
+ /** The run that dispatched this one (ADR-0071), its key included: the store finds a run by its
946
+ * key here (`findByDispatch`), so a re-sent start answers the run it already made. */
947
+ dispatchedBy?: DispatchedBy;
843
948
  /** The shape of the Machine that WROTE this snapshot (ADR-0030). `workflow` says which def to
844
949
  * look up; this says whether the def found there is still the one this snapshot can be read by.
845
950
  * Absent means written before the stamp existed, which restore treats as drift — the point is to
@@ -868,8 +973,10 @@ type RunBlob = {
868
973
  * The pending NOTICES (ADR-0062): what the next Turn in a scope must be told — a memory kill in
869
974
  * its Workspace, a conversation that is new. The Orchestrator is their only keeper, so they ride
870
975
  * this blob beside the two ledgers, for the same reasons: the run's exact lifetime, and the
871
- * snapshot's atomicity. A notice leaves it in the save that ledgers the admission that carried
872
- * it, so a restart before that save delivers it again and a restart after does not.
976
+ * snapshot's atomicity. A notice is cleared for a conversation in the save that ledgers the
977
+ * admission that carried it there, so a restart before that save delivers it again and a restart
978
+ * after does not. A conversation notice then leaves it; a Workspace notice stays, with the
979
+ * conversations that heard it, for the Workspace's other conversations.
873
980
  */
874
981
  notices?: NoticeLedgerState;
875
982
  /**
@@ -906,6 +1013,9 @@ type RunBlob = {
906
1013
  */
907
1014
  startedBy?: Record<string, StartedBy>;
908
1015
  fault?: string;
1016
+ /** The Failure that ended the run `failed` (ADR-0073), as plain data: the run's output. xstate
1017
+ * keeps the error of an errored snapshot, and serializes it to `{}`, so it is kept here. */
1018
+ failure?: FailureInput;
909
1019
  };
910
1020
 
911
1021
  type LiveRun = {
@@ -930,6 +1040,8 @@ type LiveRun = {
930
1040
  /** The error that killed the run, if it errored (xstate serializes Error to `{}`, so the
931
1041
  * message is captured here at the observer and persisted onto the blob for `read`). */
932
1042
  fault?: string;
1043
+ /** The Failure that ended the run, if it failed (ADR-0073): persisted as `RunBlob.failure`. */
1044
+ failure?: FailureInput;
933
1045
  /** Per-run observers fed by `persist()` (status) and the actor's `emit` (emit) — SSE/CLI watch. */
934
1046
  listeners: Set<(e: RunFeedEvent) => void>;
935
1047
  /** The run's feed-so-far (ADR-0023): what a Workspace attaching mid-run gets replayed as its
@@ -954,8 +1066,19 @@ export class RunHost {
954
1066
  private readonly resolveDef?: RunHostOptions["resolveDef"];
955
1067
  private readonly sandbox?: SandboxPort;
956
1068
  private readonly instanceHarness?: string;
1069
+ private readonly orchestratorName?: string;
957
1070
  private readonly harnessBearer?: (placement: string) => string;
958
1071
  private readonly effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
1072
+ private readonly unrunnableChart?: (name: string) => string | undefined;
1073
+ /**
1074
+ * Run id → what waits for that run to settle (ADR-0071): a `dispatch` with `wait`. Keyed by the
1075
+ * run, not kept on its {@link LiveRun}, because a restored caller may attach before its callee is
1076
+ * hosted again (`store.live()` row order) or while the callee is not hosted at all.
1077
+ */
1078
+ private readonly settleListeners = new Map<string, Set<(status: RunStatus) => void>>();
1079
+ /** Idempotency key → the start in flight under it (ADR-0071): a second start with the same key
1080
+ * before the first has spawned its run answers the first's run, never a second. */
1081
+ private readonly dispatching = new Map<string, Promise<DispatchedRun>>();
959
1082
  private readonly echoFactory?: (
960
1083
  endpoint: string,
961
1084
  bearer: string | undefined,
@@ -1025,8 +1148,10 @@ export class RunHost {
1025
1148
  this.resolveDef = opts.resolveDef;
1026
1149
  this.sandbox = opts.sandbox;
1027
1150
  this.instanceHarness = opts.instanceHarness;
1151
+ this.orchestratorName = opts.orchestratorName;
1028
1152
  this.harnessBearer = opts.harnessBearer;
1029
1153
  this.effector = opts.effector;
1154
+ this.unrunnableChart = opts.unrunnableChart;
1030
1155
  this.echoFactory = opts.echo;
1031
1156
  this.liveSetPush =
1032
1157
  opts.instanceHarness && opts.liveSet
@@ -1200,10 +1325,107 @@ export class RunHost {
1200
1325
  return schema ? (z.toJSONSchema(schema, { io: "input" }) as Record<string, unknown>) : null;
1201
1326
  }
1202
1327
 
1203
- /** Start a fresh run of a registered workflow; returns its durable ids. */
1204
- async start(workflow: string, input: Record<string, unknown> = {}): Promise<{ runId: string; instanceId: string }> {
1328
+ /**
1329
+ * Start a fresh run of a registered workflow; returns its durable ids.
1330
+ *
1331
+ * With `dispatchedBy` (ADR-0071), the start is a `dispatch` Effect's, and its `effect` is the
1332
+ * idempotency key: a run that already carries the key is answered, `existing`, and none is
1333
+ * started. That is what lets a caller re-send a start whose answer it lost. The key is looked for
1334
+ * among the runs this process hosts first — a run's first save is a microtask after its spawn —
1335
+ * then in the store. A key on a row the weekly sweep removed (ADR-0065) starts a new run: known,
1336
+ * and not guarded. The depth is recorded as sent: the caller states the callee's.
1337
+ */
1338
+ async start(
1339
+ workflow: string,
1340
+ input: Record<string, unknown> = {},
1341
+ opts: { dispatchedBy?: DispatchedBy } = {},
1342
+ ): Promise<{ runId: string; instanceId: string; existing?: true }> {
1343
+ const by = opts.dispatchedBy;
1344
+ if (!by) return this.startFresh(workflow, input);
1345
+ const inFlight = this.dispatching.get(by.effect);
1346
+ if (inFlight) return { ...keyAnswer(await inFlight, workflow, by.effect), existing: true };
1347
+ let existing = false;
1348
+ const started = (async (): Promise<DispatchedRun> => {
1349
+ const found = await this.dispatched(by.effect);
1350
+ if (found) {
1351
+ existing = true;
1352
+ return found;
1353
+ }
1354
+ return { ...(await this.startFresh(workflow, input, by)), workflow };
1355
+ })();
1356
+ this.dispatching.set(by.effect, started);
1357
+ try {
1358
+ const ids = keyAnswer(await started, workflow, by.effect);
1359
+ return existing ? { ...ids, existing: true } : ids;
1360
+ } finally {
1361
+ this.dispatching.delete(by.effect);
1362
+ }
1363
+ }
1364
+
1365
+ /**
1366
+ * The run that carries the idempotency key `effect` (ADR-0071), or undefined: a hosted run first,
1367
+ * then a stored one, in any status.
1368
+ */
1369
+ async dispatched(effect: string): Promise<DispatchedRun | undefined> {
1370
+ for (const run of this.runs.values()) {
1371
+ if (run.record.dispatchedBy?.effect === effect) {
1372
+ return { runId: run.record.runId, instanceId: run.record.instanceId, workflow: run.record.workflow };
1373
+ }
1374
+ }
1375
+ const runId = await this.store.findByDispatch(effect);
1376
+ if (runId === undefined) return undefined;
1377
+ const blob = (await this.store.load(runId))?.snapshot as RunBlob | null | undefined;
1378
+ return blob ? { runId, instanceId: blob.instanceId, workflow: blob.workflow } : undefined;
1379
+ }
1380
+
1381
+ /**
1382
+ * The in-process start of a `dispatch` Effect (ADR-0071, the binding's `dispatch.start`): every
1383
+ * refusal the door would answer, with its status, as output — the body may have come from a
1384
+ * model, and the report must reach it through the Chain. Any other throw is a defect, and is the
1385
+ * invoke's error.
1386
+ */
1387
+ private async dispatchStart(
1388
+ workflow: string,
1389
+ input: Record<string, unknown>,
1390
+ by: DispatchedBy,
1391
+ ): Promise<DispatchStart> {
1392
+ // The key first: a re-sent start answers its run whatever the Assembly holds now.
1393
+ const found = await this.dispatched(by.effect);
1394
+ if (found && found.workflow !== workflow) {
1395
+ return { ok: false, status: 409, report: [keyConflict(found, workflow, by.effect)] };
1396
+ }
1397
+ if (found) return { ok: true, runId: found.runId, existing: true };
1398
+ const unrunnable = this.unrunnableChart?.(workflow);
1399
+ if (unrunnable !== undefined) {
1400
+ return { ok: false, status: 409, report: [`named Chart "${workflow}" cannot run: ${unrunnable}`] };
1401
+ }
1402
+ try {
1403
+ const started = await this.start(workflow, input, { dispatchedBy: by });
1404
+ return { ok: true, runId: started.runId, existing: started.existing === true };
1405
+ } catch (err) {
1406
+ if (err instanceof ChartRefusal) return { ok: false, status: 422, report: err.report };
1407
+ if (err instanceof EventValidationError) {
1408
+ return {
1409
+ ok: false,
1410
+ status: 400,
1411
+ report: [err.message, `the door of "${workflow}" takes ${JSON.stringify(this.inputSchema(workflow))}`],
1412
+ };
1413
+ }
1414
+ if (err instanceof UnknownWorkflowError) return { ok: false, status: 404, report: [err.message] };
1415
+ if (err instanceof DispatchKeyConflictError) return { ok: false, status: 409, report: [err.message] };
1416
+ throw err;
1417
+ }
1418
+ }
1419
+
1420
+ /** A fresh run: the door, the Chart admission, the spawn. Synchronous after its call, so the run
1421
+ * is hosted — and its key findable in {@link dispatched} — before anything else runs. */
1422
+ private async startFresh(
1423
+ workflow: string,
1424
+ input: Record<string, unknown>,
1425
+ dispatchedBy?: DispatchedBy,
1426
+ ): Promise<{ runId: string; instanceId: string }> {
1205
1427
  const def = this.workflowDefs.get(workflow);
1206
- if (!def) throw new Error(`no workflow registered as "${workflow}"`);
1428
+ if (!def) throw new UnknownWorkflowError(`no workflow registered as "${workflow}"`);
1207
1429
 
1208
1430
  // A named Chart supplies every Chart field of its wrapper's door, and the caller the rest
1209
1431
  // (ADR-0070): what the caller sent under a Chart field is not the run's.
@@ -1228,7 +1450,7 @@ export class RunHost {
1228
1450
 
1229
1451
  const runId = this.newId();
1230
1452
  const instanceId = this.newId();
1231
- const record: RunRecord = { runId, workflow, instanceId };
1453
+ const record: RunRecord = { runId, workflow, instanceId, ...(dispatchedBy ? { dispatchedBy } : {}) };
1232
1454
 
1233
1455
  const machine = this.assemble(def, instanceId);
1234
1456
  // Every Chart field of a Chart wrapper's door is admitted BEFORE the run exists (ADR-0070): a
@@ -1319,6 +1541,7 @@ export class RunHost {
1319
1541
  const blob = stored.snapshot as RunBlob | null;
1320
1542
  if (!blob) {
1321
1543
  await this.store.markLost(stored.runId, "empty snapshot");
1544
+ this.settledUnhosted(stored.runId, "lost", "empty snapshot");
1322
1545
  lost.push(stored.runId);
1323
1546
  continue;
1324
1547
  }
@@ -1333,11 +1556,13 @@ export class RunHost {
1333
1556
  const found = await this.defFor(blob);
1334
1557
  if ("lost" in found) {
1335
1558
  await this.store.markLost(stored.runId, found.lost);
1559
+ this.settledUnhosted(stored.runId, "lost", found.lost, blob);
1336
1560
  lost.push(stored.runId);
1337
1561
  continue;
1338
1562
  }
1339
1563
  if ("drifted" in found) {
1340
1564
  await this.store.markDrifted(stored.runId, found.drifted);
1565
+ this.settledUnhosted(stored.runId, "drifted", found.drifted, blob);
1341
1566
  drifted.push(stored.runId);
1342
1567
  continue;
1343
1568
  }
@@ -1357,11 +1582,11 @@ export class RunHost {
1357
1582
  // a run that cannot be read.
1358
1583
  const expected = fingerprintOf(def.machine);
1359
1584
  if (blob.machine !== expected) {
1360
- await this.store.markDrifted(
1361
- stored.runId,
1585
+ const reason =
1362
1586
  `workflow "${blob.workflow}"${def.assembly !== undefined ? ` in Assembly ${def.assembly}` : ""} changed shape since this run was saved ` +
1363
- `(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`,
1364
- );
1587
+ `(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`;
1588
+ await this.store.markDrifted(stored.runId, reason);
1589
+ this.settledUnhosted(stored.runId, "drifted", reason, blob);
1365
1590
  drifted.push(stored.runId);
1366
1591
  continue;
1367
1592
  }
@@ -1372,15 +1597,24 @@ export class RunHost {
1372
1597
  const charts = chartDrift(def.machine, boundChartRoutesOf(blob.snapshot));
1373
1598
  if (charts !== undefined) {
1374
1599
  await this.store.markDrifted(stored.runId, charts);
1600
+ this.settledUnhosted(stored.runId, "drifted", charts, blob);
1375
1601
  drifted.push(stored.runId);
1376
1602
  continue;
1377
1603
  }
1378
1604
 
1379
- const record: RunRecord = { runId: stored.runId, workflow: blob.workflow, instanceId: blob.instanceId };
1605
+ // The run's place in a chain of dispatches comes back with it (ADR-0071): its key keeps
1606
+ // answering a re-sent start, and its depth is the one it started at.
1607
+ const record: RunRecord = {
1608
+ runId: stored.runId,
1609
+ workflow: blob.workflow,
1610
+ instanceId: blob.instanceId,
1611
+ ...(blob.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
1612
+ };
1380
1613
 
1381
1614
  try {
1382
1615
  if (!(await this.reconcile(record))) {
1383
1616
  await this.store.markLost(stored.runId, "reconcile: live world absent");
1617
+ this.settledUnhosted(stored.runId, "lost", "reconcile: live world absent", blob);
1384
1618
  lost.push(stored.runId);
1385
1619
  continue;
1386
1620
  }
@@ -1520,6 +1754,7 @@ export class RunHost {
1520
1754
  instanceId,
1521
1755
  runId: reg.runId,
1522
1756
  sandbox: reg.sandbox,
1757
+ ...(reg.slot !== undefined ? { slot: reg.slot } : {}),
1523
1758
  menu: [...menuOf(reg).values()].map((def) => ({
1524
1759
  name: def.name,
1525
1760
  description: def.description,
@@ -1537,6 +1772,19 @@ export class RunHost {
1537
1772
  this.table.lookup(agentAddress(instanceId))?.served?.();
1538
1773
  }
1539
1774
 
1775
+ /** The notices this Turn's conversation hears at a step boundary (ADR-0062): its pending notices
1776
+ * in their audience, reserved for this read until the Harness acknowledges them
1777
+ * ({@link noticesHeard}) or the Turn ends. Empty for a registration that hears none. */
1778
+ async boundaryNotices(instanceId: string): Promise<Array<{ id: string; notice: Notice }>> {
1779
+ return (await this.table.lookup(agentAddress(instanceId))?.boundaryNotices?.()) ?? [];
1780
+ }
1781
+
1782
+ /** The Harness appended these step-boundary notices to the Turn (ADR-0062): they are heard in its
1783
+ * conversation, ledgered in one save. Answers how many were this conversation's reservations. */
1784
+ noticesHeard(instanceId: string, ids: readonly string[]): number {
1785
+ return this.table.lookup(agentAddress(instanceId))?.noticesHeard?.(ids) ?? 0;
1786
+ }
1787
+
1540
1788
  /**
1541
1789
  * Deliver one event from an Agent's Harness (`POST /agents/:iid/events` — ADR-0013). Validation
1542
1790
  * and delivery are the table's; this only agent-scopes the address and mints the receipt. (A
@@ -1670,6 +1918,7 @@ export class RunHost {
1670
1918
  agent: invoking.sender?.id ?? "",
1671
1919
  since,
1672
1920
  ...(invoking.sandbox !== undefined ? { sandbox: invoking.sandbox } : {}),
1921
+ ...(invoking.slot !== undefined ? { slot: invoking.slot } : {}),
1673
1922
  ...(invoking.invokeId !== undefined ? { invokeId: invoking.invokeId } : {}),
1674
1923
  ...(conversation !== undefined ? { conversation } : {}),
1675
1924
  ...(sending && sending.chain.length > 0 ? { chain: [...sending.chain] } : {}),
@@ -1746,6 +1995,46 @@ export class RunHost {
1746
1995
  };
1747
1996
  }
1748
1997
 
1998
+ /**
1999
+ * The registrations whose Agents' picks `event` comes from (ADR-0024, `cause.ts`): the one whose
2000
+ * pick it is, in its own send; and each one whose pick, in its own send or out, it answers — an
2001
+ * event that the Machine invoking that Agent takes from below the Agent's state, the sources of a
2002
+ * reply or a child's error (a nested Agent's pick, a Gate's answer, a child's output or error).
2003
+ */
2004
+ private pickCause(
2005
+ runId: string,
2006
+ holds: Map<string, Hold>,
2007
+ target: AnyActorRef,
2008
+ source: AnyActorRef | undefined,
2009
+ event: AnyEventObject,
2010
+ ): Set<Registration> | undefined {
2011
+ const out = new Set<Registration>();
2012
+ const sending = this.sending;
2013
+ if (sending && sending.reg.runId === runId && sending.reg.invoker === target && event.type === sending.pick) {
2014
+ out.add(sending.reg);
2015
+ }
2016
+ const picking: Registration[] = [];
2017
+ if (sending && sending.reg.runId === runId) picking.push(sending.reg);
2018
+ for (const hold of holds.values()) {
2019
+ if (hold.answer !== undefined) continue;
2020
+ const live = this.table.lookup(agentAddress(hold.instanceId));
2021
+ const reg = this.holdRegs.get(hold) ?? live;
2022
+ if (reg !== undefined && reg === live) picking.push(reg);
2023
+ }
2024
+ for (const reg of picking) {
2025
+ if (reg.invoker !== target || !reg.invokeId) continue;
2026
+ const seat = seatOf(target.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
2027
+ if (!seat) continue;
2028
+ if (
2029
+ replyOf(event, source, target, seat) !== undefined ||
2030
+ childErrorOf(event, source, target, seat) !== undefined
2031
+ ) {
2032
+ out.add(reg);
2033
+ }
2034
+ }
2035
+ return out.size > 0 ? out : undefined;
2036
+ }
2037
+
1749
2038
  /**
1750
2039
  * Answer every outstanding pick of `run` that can be answered now (ADR-0024). Called by the persist that
1751
2040
  * follows every transition — a microtask after the snapshot, so the mailbox has drained — and
@@ -1813,8 +2102,8 @@ export class RunHost {
1813
2102
  * its own send, else the one outstanding. On a transition below no Agent's state (`null`): the
1814
2103
  * event that took the transition names the Agent — the sender of a pick, or the Agent whose pick
1815
2104
  * started the child whose done or error event it is — and that Agent's pick is the one shaped, and
1816
- * no other; none when that Agent has no pick being answered. When one slot is invoked in two
1817
- * regions, the event names an invoke too, and only the pick at that invoke is shaped: the sending
2105
+ * no other; none when that Agent has no pick being answered. When one slot is invoked by two
2106
+ * states, the event names an invoke too, and only the pick at that invoke is shaped: the sending
1818
2107
  * pick, or the pick at the invoke whose pick started the child; none when that pick is not being
1819
2108
  * answered (its Turn ended, and its child runs on). An event that names no Agent (a timer,
1820
2109
  * a Gate's answer, an outside send) shapes the pick in its own send. Outside one, it shapes the one
@@ -1846,7 +2135,7 @@ export class RunHost {
1846
2135
  const { agent } = named;
1847
2136
  const own = sending && sending.reg.sender?.id === agent ? sending : undefined;
1848
2137
  // That Agent's picks being answered: the one in its own send, and the ones outstanding at
1849
- // its other invokes (one slot invoked in two regions). The event names the invoke of the
2138
+ // its other invokes (one slot invoked by two states). The event names the invoke of the
1850
2139
  // pick that sent it or started its child, and only the pick at that invoke is shaped; none
1851
2140
  // when that pick is not being answered, and each other pick keeps its default answer.
1852
2141
  const held = open.filter((hold) => hold.agent === agent && hold.instanceId !== own?.reg.id);
@@ -1917,10 +2206,16 @@ export class RunHost {
1917
2206
  return this.parked.has(`${instanceId} ${deliveryId}`);
1918
2207
  }
1919
2208
 
1920
- /** The Sandbox that may read a delivery's answer (ADR-0013), or undefined for an unknown one. */
1921
- deliveryScope(instanceId: string, deliveryId: string): { sandbox?: string } | undefined {
2209
+ /** The pod and slot that may read a delivery's answer (ADR-0013, ADR-0076), or undefined for an
2210
+ * unknown one. */
2211
+ deliveryScope(instanceId: string, deliveryId: string): { sandbox?: string; slot?: string } | undefined {
1922
2212
  const found = this.holdAt(instanceId, deliveryId);
1923
- return found && { ...(found.hold.sandbox !== undefined ? { sandbox: found.hold.sandbox } : {}) };
2213
+ return (
2214
+ found && {
2215
+ ...(found.hold.sandbox !== undefined ? { sandbox: found.hold.sandbox } : {}),
2216
+ ...(found.hold.slot !== undefined ? { slot: found.hold.slot } : {}),
2217
+ }
2218
+ );
1924
2219
  }
1925
2220
 
1926
2221
  /**
@@ -2060,6 +2355,105 @@ export class RunHost {
2060
2355
  return () => run.listeners.delete(listener);
2061
2356
  }
2062
2357
 
2358
+ /**
2359
+ * Hear once that a run is no longer live (ADR-0071): `listener` gets its last status — `done`,
2360
+ * `failed`, `error` or `cancelled` with its `output`, or the store's `lost` or `drifted` — and is
2361
+ * then dropped. Returns the detach. What an in-process `dispatch` with `wait` waits on.
2362
+ *
2363
+ * Keyed by the run id, not by the run's hosting, so the order of a restore does not matter: a
2364
+ * caller restored before its callee attaches to a run this process does not host yet, and hears
2365
+ * it when the restored callee settles, or when restore marks it lost or drifted. A run that has
2366
+ * settled already is answered at once, from {@link read}. `subscribe` is the feed, and unchanged:
2367
+ * it attaches only to a hosted run.
2368
+ */
2369
+ onSettled(runId: string, listener: (status: RunStatus) => void): () => void {
2370
+ let listeners = this.settleListeners.get(runId);
2371
+ if (!listeners) this.settleListeners.set(runId, (listeners = new Set()));
2372
+ // Its own entry, so one listener attached twice is heard twice and detached once each.
2373
+ const entry = (status: RunStatus): void => listener(status);
2374
+ listeners.add(entry);
2375
+ const attached = (): boolean => this.settleListeners.get(runId)?.has(entry) === true;
2376
+ const detach = (): void => {
2377
+ const now = this.settleListeners.get(runId);
2378
+ now?.delete(entry);
2379
+ if (now?.size === 0) this.settleListeners.delete(runId);
2380
+ };
2381
+ if (!this.runs.has(runId)) {
2382
+ // Not hosted: settled already, or a live row a restore has not reached. Only the first is
2383
+ // answered here; a settle heard in the meantime has detached the entry.
2384
+ this.read(runId).then(
2385
+ (status) => {
2386
+ if (!status || !attached() || !settledStatus(status.status)) return;
2387
+ detach();
2388
+ hear(entry, status);
2389
+ },
2390
+ () => {},
2391
+ );
2392
+ }
2393
+ return detach;
2394
+ }
2395
+
2396
+ /** A hosted run's depth in a chain of dispatches (ADR-0071): 0 for a run nothing dispatched, and
2397
+ * undefined for a run this process does not host — not hosted is not a root run. */
2398
+ depthOf(runId: string): number | undefined {
2399
+ const run = this.runs.get(runId);
2400
+ return run && (run.record.dispatchedBy?.depth ?? 0);
2401
+ }
2402
+
2403
+ /** Tell everything waiting on `runId` that it settled, once (ADR-0071). */
2404
+ private fireSettled(runId: string, status: RunStatus): void {
2405
+ const listeners = this.settleListeners.get(runId);
2406
+ if (!listeners) return;
2407
+ this.settleListeners.delete(runId);
2408
+ for (const listener of listeners) hear(listener, status);
2409
+ }
2410
+
2411
+ /**
2412
+ * A run restore did not host settled: marked `lost` (its status is the row's identity and reason
2413
+ * alone: a lost run is not read back, though a row that was saved keeps its blob and its key) or
2414
+ * `drifted` (its snapshot is read back).
2415
+ * Only when something waits on it (ADR-0071).
2416
+ */
2417
+ private settledUnhosted(runId: string, status: "lost" | "drifted", reason: string, blob?: RunBlob): void {
2418
+ if (!this.settleListeners.has(runId)) return;
2419
+ const bare: RunStatus = {
2420
+ runId,
2421
+ workflow: blob?.workflow ?? "",
2422
+ instanceId: blob?.instanceId ?? "",
2423
+ ...(blob?.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
2424
+ status,
2425
+ value: undefined,
2426
+ context: undefined,
2427
+ children: [],
2428
+ waiting: [],
2429
+ held: [],
2430
+ reason,
2431
+ };
2432
+ if (status === "lost") return this.fireSettled(runId, bare);
2433
+ this.read(runId).then(
2434
+ (read) => this.fireSettled(runId, read ?? bare),
2435
+ () => this.fireSettled(runId, bare),
2436
+ );
2437
+ }
2438
+
2439
+ /** The binding's `dispatch.settled` (ADR-0071): {@link onSettled} as a promise that an abort —
2440
+ * the Effect's state exiting, its wait timing out — rejects and detaches. */
2441
+ private settledOf(runId: string, signal: AbortSignal): Promise<RunStatus> {
2442
+ return new Promise((resolve, reject) => {
2443
+ if (signal.aborted) return reject(signal.reason);
2444
+ let detach = (): void => {};
2445
+ const onAbort = (): void => {
2446
+ detach();
2447
+ reject(signal.reason);
2448
+ };
2449
+ signal.addEventListener("abort", onAbort, { once: true });
2450
+ detach = this.onSettled(runId, (status) => {
2451
+ signal.removeEventListener("abort", onAbort);
2452
+ resolve(status);
2453
+ });
2454
+ });
2455
+ }
2456
+
2063
2457
  /**
2064
2458
  * Observe a whole WORKFLOW (the `GET /workflows/:name/events` SSE — ADR-0022): every run of it
2065
2459
  * appearing, moving, emitting and leaving, for as long as the caller stays attached.
@@ -2111,8 +2505,9 @@ export class RunHost {
2111
2505
  * Read a run's status, **reading through to the store** when it is no longer live (ADR-0009). A
2112
2506
  * completed run's final snapshot is persisted before `persist()` drops it from the registry, so a
2113
2507
  * terminal run reports its `done`/`error` status + final context here rather than 404-ing. Returns
2114
- * undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared) —
2115
- * and a swept one is unknown: a finished run's row lives a week (ADR-0065).
2508
+ * undefined only for a genuinely unknown run (or one marked `lost` with no row saved before, so no
2509
+ * snapshot: `markLost` keeps a saved row's blob, whose key must keep answering, ADR-0071) — and a
2510
+ * swept one is unknown: a finished run's row lives a week (ADR-0065).
2116
2511
  */
2117
2512
  async read(runId: string): Promise<RunStatus | undefined> {
2118
2513
  const live = this.status(runId);
@@ -2120,7 +2515,7 @@ export class RunHost {
2120
2515
  const stored = await this.store.load(runId);
2121
2516
  const blob = stored?.snapshot as RunBlob | null | undefined;
2122
2517
  if (!stored || !blob) return undefined;
2123
- const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown };
2518
+ const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown; output?: unknown };
2124
2519
  // The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
2125
2520
  // cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
2126
2521
  // (ADR-0025). While the row still says "live", the Machine's own status is the answer.
@@ -2139,6 +2534,15 @@ export class RunHost {
2139
2534
  reason: stored.reason,
2140
2535
  ...(blob.assembly !== undefined ? { assembly: blob.assembly } : {}),
2141
2536
  ...boundFields(snap),
2537
+ ...(blob.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
2538
+ // The Machine's output once the row says the run is over (ADR-0071), never while it is live;
2539
+ // for a failed run, its Failure (ADR-0073).
2540
+ ...(stored.status !== "live" && blob.failure !== undefined ? { output: blob.failure } : {}),
2541
+ ...(stored.status !== "live" && blob.failure === undefined && snap.output !== undefined
2542
+ ? { output: snap.output }
2543
+ : {}),
2544
+ ...dispatchedField(blob.effects ?? {}),
2545
+ ...worktreesField(blob.effects ?? {}),
2142
2546
  };
2143
2547
  }
2144
2548
 
@@ -2275,15 +2679,40 @@ export class RunHost {
2275
2679
  // The run is over, so nothing can use its Workspaces, and each holds its whole Size (ADR-0025).
2276
2680
  // An actor stop skips the wrapper's `teardown` state, so the host runs the same idempotent
2277
2681
  // delete here, after the stop, so no lease renews what was deleted. `keep` leaves them to the
2278
- // operator's idle timeout, for inspection. A delete that fails is the idle GC's, as in
2279
- // `teardown` (ADR-0012 backstop): the cancel has already happened.
2682
+ // operator's idle timeout, for inspection.
2683
+ await this.destroySandboxes(runId, sandboxes, "the cancel");
2684
+ }
2685
+
2686
+ /**
2687
+ * End a run `error` because a Machine of its tree dropped a mechanism event (ADR-0074): `fault`
2688
+ * says which. The actor stops as a cancel stops it, so each live Agent's Turn ends, and the run
2689
+ * is persisted terminal. Its Sandboxes stay as evidence, as a fault's do (ADR-0012): the Machine
2690
+ * did not choose this end. Once per run: a second dropped event in the same step adds nothing.
2691
+ */
2692
+ private faultUnrouted(runId: string, binding: RunBinding, fault: string): void {
2693
+ const run = this.runs.get(runId);
2694
+ if (!run || run.binding !== binding || run.fault !== undefined) return;
2695
+ run.fault = fault;
2696
+ run.actor.stop();
2697
+ this.persist(run, "error");
2698
+ }
2699
+
2700
+ /**
2701
+ * Delete the Sandboxes of a run that is over and stopped: a cancelled one (ADR-0025) and a failed
2702
+ * one (ADR-0073). After the stop, so no lease renews what was deleted. The delete is idempotent:
2703
+ * for a failed run, the Workspace a Failure went through deleted its own already, so this is a
2704
+ * backstop for the others. A delete that fails is the idle GC's, as in `teardown` (ADR-0012
2705
+ * backstop): the run has already ended. `why` names that end in the log line of a delete that
2706
+ * fails.
2707
+ */
2708
+ private async destroySandboxes(runId: string, sandboxes: readonly string[], why: string): Promise<void> {
2280
2709
  const port = this.sandbox;
2281
2710
  if (!port) return;
2282
2711
  await Promise.all(
2283
2712
  sandboxes.map((name) =>
2284
2713
  port.destroy(name).catch((err: unknown) => {
2285
2714
  console.error(
2286
- `run ${runId}: cancel could not delete Sandbox ${name} (the idle timeout will): ` +
2715
+ `run ${runId}: Sandbox ${name} was not deleted at ${why} (the idle timeout will delete it): ` +
2287
2716
  (err instanceof Error ? err.message : String(err)),
2288
2717
  );
2289
2718
  }),
@@ -2362,6 +2791,14 @@ export class RunHost {
2362
2791
  fault: run.fault,
2363
2792
  ...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
2364
2793
  ...boundFields(snap),
2794
+ // A Machine that reached its final state has its output (ADR-0071); an active one has none.
2795
+ // A failed run's output is its Failure (ADR-0073).
2796
+ ...(run.failure !== undefined ? { output: run.failure } : {}),
2797
+ ...(run.failure === undefined && snap.status !== "active" && snap.output !== undefined
2798
+ ? { output: snap.output }
2799
+ : {}),
2800
+ ...dispatchedField(run.effects),
2801
+ ...worktreesField(run.effects),
2365
2802
  };
2366
2803
  }
2367
2804
 
@@ -2393,6 +2830,9 @@ export class RunHost {
2393
2830
  ): AnyActor {
2394
2831
  const notices = new NoticeLedger(noticeState, this.newId);
2395
2832
  const holds = new Map(holdState.map((hold) => [hold.instanceId, hold]));
2833
+ // What took the Machine out of each Agent's state: that Agent's own pick, or something else
2834
+ // (ADR-0024). Read off the inspection stream below.
2835
+ const causes = new PickCauses();
2396
2836
  let live: LiveRun | undefined;
2397
2837
  let scheduled = false;
2398
2838
  const schedule = () => {
@@ -2406,9 +2846,20 @@ export class RunHost {
2406
2846
  const binding: RunBinding = {
2407
2847
  runId: record.runId,
2408
2848
  workflow: record.workflow,
2849
+ ...(this.orchestratorName !== undefined ? { orchestratorName: this.orchestratorName } : {}),
2850
+ depth: record.dispatchedBy?.depth ?? 0,
2851
+ // A `dispatch` with no target (ADR-0071): the run host itself, with no wire and no token.
2852
+ dispatch: {
2853
+ start: (workflow, input, by) => this.dispatchStart(workflow, input, by),
2854
+ settled: (runId, signal) => this.settledOf(runId, signal),
2855
+ },
2409
2856
  ...(def.assembly !== undefined ? { assembly: def.assembly } : {}),
2410
2857
  table: this.table,
2411
2858
  sandbox: this.sandbox,
2859
+ heldSandboxes: () => {
2860
+ const run = this.runs.get(record.runId);
2861
+ return new Set(run ? runSandboxes(run.actor, record.runId, { placing: true }) : []);
2862
+ },
2412
2863
  instanceHarness: this.instanceHarness,
2413
2864
  ...(this.harnessBearer ? { harnessBearer: this.harnessBearer } : {}),
2414
2865
  ...(this.effector ? { effector: this.effector } : {}),
@@ -2416,10 +2867,20 @@ export class RunHost {
2416
2867
  // moment the Harness admits it, and the ledger hits the store in the same RunBlob save. An
2417
2868
  // admission arriving around stop/untrack still lands in `agents` but skips the save,
2418
2869
  // exactly like the persist scheduler's tracked-run guard.
2419
- recordAdmission: (instanceId, admission, delivered = []) => {
2870
+ recordAdmission: (instanceId, admission, delivered = [], taker, member) => {
2420
2871
  agents[instanceId] = admission;
2421
- // The notices this admission carried are delivered by THIS save (ADR-0062).
2422
- notices.delivered(delivered);
2872
+ // The notices this admission carried are delivered by THIS save, in the conversation it
2873
+ // rode (ADR-0062): a Workspace notice stays pending for the rest of its audience. And the
2874
+ // conversation is a member of its Workspace's audiences from now on, in the same save.
2875
+ notices.delivered(delivered, taker);
2876
+ if (member !== undefined && taker !== undefined) {
2877
+ notices.admitted(member.workspace, {
2878
+ taker,
2879
+ slot: member.slot,
2880
+ continued: member.continued,
2881
+ since: new Date().toISOString(),
2882
+ });
2883
+ }
2423
2884
  const run = this.runs.get(record.runId);
2424
2885
  if (run && run.agents === agents) this.persist(run);
2425
2886
  },
@@ -2431,6 +2892,9 @@ export class RunHost {
2431
2892
  // invoke would otherwise restore into the burned conversation.
2432
2893
  epochOf: (conversation) => epochs[conversation] ?? 0,
2433
2894
  bumpEpoch: (conversation, reason) => {
2895
+ // The burned conversation can take no Turn again: it leaves every audience (ADR-0062).
2896
+ const burned = epochs[conversation] ? `${conversation}/${epochs[conversation]}` : conversation;
2897
+ notices.forget(burned);
2434
2898
  epochs[conversation] = (epochs[conversation] ?? 0) + 1;
2435
2899
  // The next `continue` lands on a virgin conversation and is told so, with the fault's own
2436
2900
  // words (ADR-0062) — raised in the SAME save as the bump, so no restart can split them.
@@ -2440,13 +2904,36 @@ export class RunHost {
2440
2904
  },
2441
2905
  // The notices' other halves (ADR-0062). A raise is saved at the write, like the ledgers; a
2442
2906
  // take is a reservation in memory, delivered by `recordAdmission` or released on failure.
2443
- raiseNotice: (notice, to, source) => {
2444
- if (!notices.raise(notice, to, source)) return;
2907
+ raiseNotice: (notice, to, source, older, raiseOpts) => {
2908
+ if (!notices.raise(notice, to, source, older, raiseOpts)) return;
2909
+ const run = this.runs.get(record.runId);
2910
+ if (run && run.notices === notices) this.persist(run);
2911
+ },
2912
+ restoreNotice: (notice, to, source, taker) => {
2913
+ if (!notices.restore(notice, to, source, taker)) return;
2445
2914
  const run = this.runs.get(record.runId);
2446
2915
  if (run && run.notices === notices) this.persist(run);
2447
2916
  },
2448
2917
  takeNotices: (scope) => notices.take(scope),
2449
- releaseNotices: (ids) => notices.release(ids),
2918
+ releaseNotices: (ids, taker) => notices.release(ids, taker),
2919
+ peekNotices: (scope) => notices.peek(scope),
2920
+ acknowledgeNotices: (ids, taker) => {
2921
+ const heard = notices.acknowledge(ids, taker);
2922
+ const run = this.runs.get(record.runId);
2923
+ if (heard > 0 && run && run.notices === notices) this.persist(run);
2924
+ return heard;
2925
+ },
2926
+ noticesTold: (told) => {
2927
+ let changed = false;
2928
+ for (const { source, takers } of told) changed = notices.told(source, takers) || changed;
2929
+ const run = this.runs.get(record.runId);
2930
+ if (changed && run && run.notices === notices) this.persist(run);
2931
+ },
2932
+ noticeTurnEnded: (workspace, takers) => {
2933
+ if (!notices.ended(workspace, takers)) return;
2934
+ const run = this.runs.get(record.runId);
2935
+ if (run && run.notices === notices) this.persist(run);
2936
+ },
2450
2937
  // Absorbed-retry attempts go straight to the run's observers (SSE/CLI watch) — they are
2451
2938
  // feed events, not machine events (ADR-0016: the workflow sees only the terminal fault).
2452
2939
  telemetry: (event) => {
@@ -2521,6 +3008,11 @@ export class RunHost {
2521
3008
  const hold = holds.get(instanceId);
2522
3009
  if (hold) hold.fetched = true;
2523
3010
  },
3011
+ // Read as the invocation stops, while its registration is still there (ADR-0024).
3012
+ turnEndedByPick: (instanceId) => {
3013
+ const reg = this.table.lookup(agentAddress(instanceId));
3014
+ return reg !== undefined && reg.runId === record.runId && causes.endedByPick(reg);
3015
+ },
2524
3016
  // A Chart by name (ADR-0070, D-C): among the Charts of the wrapper that is the run's Workflow.
2525
3017
  chartNamed: (name) => this.namedChart(wrapperName(record.workflow), name)?.text,
2526
3018
  };
@@ -2532,7 +3024,8 @@ export class RunHost {
2532
3024
  const childRefsOf = (snapshot: unknown): Set<AnyActorRef> =>
2533
3025
  new Set(Object.values((snapshot as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {}));
2534
3026
  const actor = createActor(machine, {
2535
- clock: this.clock,
3027
+ // Each timer keeps the cause of the relay that armed it (`cause.ts`, ADR-0024).
3028
+ clock: causes.clockOver(this.clock),
2536
3029
  ...options,
2537
3030
  inspect: (ev) => {
2538
3031
  // Bind the run's actor SYSTEM on the ROOT's creation event — the first inspection event,
@@ -2545,7 +3038,10 @@ export class RunHost {
2545
3038
  bindRun((ev.actorRef as AnyActorRef).system, binding);
2546
3039
  // …and watch its scheduler before anything is armed: whether an event the Machine took
2547
3040
  // was a timer it set, or an event an actor sent at once, decides `cutShort` below.
2548
- firedTimer = watchTimers((ev.actorRef as AnyActorRef).system);
3041
+ // The same watch keeps the cause of each timer: the relay that armed it (ADR-0024).
3042
+ firedTimer = watchTimers((ev.actorRef as AnyActorRef).system, {
3043
+ arming: (source) => causes.armingFrom(source),
3044
+ });
2549
3045
  }
2550
3046
  // A restored hold is the Turn of the FIRST registration of its address in this tree: bind
2551
3047
  // it as soon as one exists, before an event can end that Turn and start another at the same
@@ -2661,8 +3157,63 @@ export class RunHost {
2661
3157
  const reply = sending && replyTo(sending.reg);
2662
3158
  if (sending && reply) sending.chain.push(reply);
2663
3159
  if (sending) errorTo(sending.reg, sending.pick);
3160
+ // The cause of this relay (`cause.ts`, ADR-0024): read before the Machine takes it.
3161
+ const target = ev.actorRef as AnyActorRef;
3162
+ const source = ev.sourceRef as AnyActorRef | undefined;
3163
+ const event = ev.event as AnyEventObject;
3164
+ causes.relayed(
3165
+ source,
3166
+ target,
3167
+ event,
3168
+ () => this.pickCause(record.runId, holds, target, source, event),
3169
+ () => this.table.byRun(record.runId).filter((reg) => reg.kind === "agent"),
3170
+ );
3171
+ // A Failure ended a Machine that a Machine above can catch (ADR-0073). xstate does not stop
3172
+ // the children of a Machine that ends `error`, so the host stops them, as `stopTree` does for
3173
+ // the run: an Agent in that Machine settles its Turn, and a Gate or a timer there does not
3174
+ // wait for nobody. After the parent takes the event, so its exit is recorded first. A stop
3175
+ // skips `teardown`, so a Workspace in another region of that Machine does not delete its
3176
+ // Sandbox: the host names them before the stop and deletes them after it, as for a failed run.
3177
+ // A name that a live Workspace of the run holds after the stop is not deleted: a parent that
3178
+ // invokes the same Machine again gives its new Workspace the same invoke id, and so the same
3179
+ // name, and that Workspace is in `placing` by now.
3180
+ if (
3181
+ source !== undefined &&
3182
+ source._parent === target &&
3183
+ event.type === `xstate.error.actor.${source.id}` &&
3184
+ isFailure((event as { error?: unknown }).error)
3185
+ ) {
3186
+ queueMicrotask(() => {
3187
+ const sandboxes = runSandboxes(source, record.runId);
3188
+ stopTree(source);
3189
+ const held = this.runs.get(record.runId)?.binding.heldSandboxes?.() ?? new Set<string>();
3190
+ const ended = sandboxes.filter((name) => !held.has(name));
3191
+ void this.destroySandboxes(record.runId, ended, "the end of the failed Machine");
3192
+ });
3193
+ }
3194
+ }
3195
+ // A mechanism event that no transition took (ADR-0074): `agent.fault` or `workspace.lost`
3196
+ // that a Machine got and dropped, as a guard that was false when it came. The run would wait
3197
+ // for ever, so it faults. After the macrostep, so the stop does not cut xstate's own step.
3198
+ if (ev.type === "@xstate.microstep") {
3199
+ const unrouted = unroutedMechanismEvent(ev as unknown as MicrostepView);
3200
+ if (unrouted !== undefined) queueMicrotask(() => this.faultUnrouted(record.runId, binding, unrouted));
3201
+ }
3202
+ // The exit of each Agent's state (`cause.ts`, ADR-0024).
3203
+ if (ev.type === "@xstate.microstep") {
3204
+ const machine = ev.actorRef as AnyActorRef;
3205
+ const agents = this.table.byRun(record.runId).filter((reg) => reg.kind === "agent");
3206
+ causes.microstep(
3207
+ machine,
3208
+ (ev as unknown as { snapshot: AnyMachineSnapshot }).snapshot,
3209
+ ev.event as AnyEventObject,
3210
+ agents,
3211
+ );
3212
+ }
3213
+ if (ev.type === "@xstate.snapshot") {
3214
+ causes.taken(ev.actorRef as AnyActorRef, ev.event as AnyEventObject);
3215
+ schedule();
2664
3216
  }
2665
- if (ev.type === "@xstate.snapshot") schedule();
2666
3217
  },
2667
3218
  });
2668
3219
  live = this.track(record, actor, def, agents, epochs, notices, binding, holds, effects, startedBy);
@@ -2702,6 +3253,17 @@ export class RunHost {
2702
3253
  // `{}`), then persist: the snapshot's "error" status stores the run and untracks it.
2703
3254
  actor.subscribe({
2704
3255
  error: (err) => {
3256
+ // A Failure that no state caught (ADR-0073): the end the Machine chose, so the run ends
3257
+ // `failed`, with no fault and the Failure as its output, and its Sandboxes go as a cancel's
3258
+ // do (ADR-0025). Named BEFORE the stop, while the tree still says which Workspaces are live.
3259
+ if (isFailure(err)) {
3260
+ run.failure = failurePayload(err);
3261
+ const sandboxes = runSandboxes(actor, record.runId);
3262
+ this.persist(run, "failed");
3263
+ stopTree(actor);
3264
+ void this.destroySandboxes(record.runId, sandboxes, "the end of the failed run");
3265
+ return;
3266
+ }
2705
3267
  run.fault = err instanceof Error ? err.message : String(err);
2706
3268
  this.persist(run);
2707
3269
  // A faulted run stops its actors — each workspace's lease among them (ADR-0021), and a
@@ -2734,9 +3296,12 @@ export class RunHost {
2734
3296
  /**
2735
3297
  * Persist the run's snapshot after a transition; drop it from the registry once final.
2736
3298
  *
2737
- * `terminal` is the run-lifecycle verdict the MACHINE cannot supply: `cancel()` passes
2738
- * "cancelled", because xstate only knows its actor was stopped (ADR-0025). It is the stored
2739
- * status and the one the feeds report.
3299
+ * `terminal` is the run-lifecycle verdict the MACHINE cannot supply. Three callers pass it:
3300
+ * `cancel()` passes "cancelled", because xstate only knows its actor was stopped (ADR-0025); the
3301
+ * error observer passes "failed" for a Failure that no state caught, because xstate gives a
3302
+ * Failure and a defect the same `error` status (ADR-0073); and `faultUnrouted()` passes "error"
3303
+ * for a mechanism event that no transition took, because the host stops the actor and xstate
3304
+ * then knows only that it was stopped (ADR-0074). It is the stored status and the one the feeds report.
2740
3305
  */
2741
3306
  private persist(run: LiveRun, terminal?: string): void {
2742
3307
  // The holds first: an answer read at Rest, and a notice a route out raised, ride this same save.
@@ -2750,6 +3315,7 @@ export class RunHost {
2750
3315
  const blob: RunBlob = {
2751
3316
  workflow: run.record.workflow,
2752
3317
  instanceId: run.record.instanceId,
3318
+ ...(run.record.dispatchedBy ? { dispatchedBy: run.record.dispatchedBy } : {}),
2753
3319
  // Stamped on every save, off the registered TEMPLATE (never the per-run `.provide()` result —
2754
3320
  // providers do not change shape, and `fingerprintOf` memoizes per machine object). ADR-0030.
2755
3321
  machine: fingerprintOf(run.def.machine),
@@ -2766,6 +3332,7 @@ export class RunHost {
2766
3332
  effects: run.effects,
2767
3333
  startedBy: liveStartedBy(run.startedBy, run.actor),
2768
3334
  fault: run.fault,
3335
+ ...(run.failure !== undefined ? { failure: run.failure } : {}),
2769
3336
  };
2770
3337
  const status = terminal ?? (machineStatus === "active" ? "live" : machineStatus);
2771
3338
  // A save that fails is this run's fault, never the process's: an unhandled rejection would end
@@ -2794,12 +3361,30 @@ export class RunHost {
2794
3361
  this.announceGone(run, runStatus);
2795
3362
  this.untrack(run.record);
2796
3363
  run.listeners.clear();
3364
+ // After the save and the untrack: a listener that reads the run back reads it settled.
3365
+ this.fireSettled(run.record.runId, runStatus);
2797
3366
  } else {
2798
3367
  this.announce(run.record.workflow, { kind: "status", status: runStatus });
2799
3368
  }
2800
3369
  }
2801
3370
  }
2802
3371
 
3372
+ /** A status that is no longer live (ADR-0071): the store's terminal words. A live row reads its
3373
+ * Machine's `active`; the store's own `live` is never a status a reader is handed, but is not
3374
+ * settled either. */
3375
+ function settledStatus(status: string): boolean {
3376
+ return status !== "active" && status !== "live";
3377
+ }
3378
+
3379
+ /** Call a settle listener; one that throws is its own fault, never the run's or the host's. */
3380
+ function hear(listener: (status: RunStatus) => void, status: RunStatus): void {
3381
+ try {
3382
+ listener(status);
3383
+ } catch (err) {
3384
+ console.error(`run ${status.runId}: a settle listener threw: ${err instanceof Error ? err.message : String(err)}`);
3385
+ }
3386
+ }
3387
+
2803
3388
  /** Does work still run below the Agent's state of `reg` — a live child of an active state under it?
2804
3389
  * A timer that holds Rest is no work: when the state exits, nothing it started is stopped. */
2805
3390
  function stillRunning(reg: Registration): boolean {
@@ -2814,6 +3399,41 @@ function atOrBelow(actor: AnyActorRef, ref: AnyActorRef): boolean {
2814
3399
  return false;
2815
3400
  }
2816
3401
 
3402
+ /** The runs a run's `dispatch` Effects started (ADR-0071), from its ledger: an intent row the
3403
+ * start's answer re-wrote with its run, a settled row whose output names one (`started`,
3404
+ * `settled`), and an aborted row that kept it. A row that never learned its run names none. */
3405
+ function dispatchedField(effects: Record<string, EffectRow>): { dispatched?: Dispatched[] } {
3406
+ const out: Dispatched[] = [];
3407
+ for (const row of Object.values(effects)) {
3408
+ if (row.kind !== "dispatch") continue;
3409
+ const run =
3410
+ row.state === "settled"
3411
+ ? (row.output as { run?: unknown } | null | undefined)?.run
3412
+ : (row as { run?: unknown }).run;
3413
+ if (typeof run === "string" && run !== "") out.push({ slot: row.slot, run, at: row.at });
3414
+ }
3415
+ out.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
3416
+ return out.length ? { dispatched: out } : {};
3417
+ }
3418
+
3419
+ /** The Worktrees a run's cuts made (ADR-0075), from the settled `ready` rows of its ledger, in the
3420
+ * order of each Worktree's first cut. A cut also reads a Worktree, so one path and branch has many
3421
+ * rows and one entry. */
3422
+ function worktreesField(effects: Record<string, EffectRow>): { worktrees?: RunWorktree[] } {
3423
+ const rows = Object.values(effects).flatMap((row) =>
3424
+ row.state === "settled" && row.cut !== undefined && row.repo !== undefined
3425
+ ? [{ at: row.at, slot: row.repo, ...row.cut }]
3426
+ : [],
3427
+ );
3428
+ rows.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
3429
+ const out = new Map<string, RunWorktree>();
3430
+ for (const { slot, path, branch } of rows) {
3431
+ const key = JSON.stringify([path, branch]);
3432
+ if (!out.has(key)) out.set(key, { slot, path, branch });
3433
+ }
3434
+ return out.size ? { worktrees: [...out.values()] } : {};
3435
+ }
3436
+
2817
3437
  /** A status's bound fields (ADR-0070): `bound`, and the hashes of the Charts bound, distinct and
2818
3438
  * sorted. Both absent when the snapshot binds no Chart. */
2819
3439
  function boundFields(snap: unknown): { bound?: true; chartHashes?: string[] } {