@jr2/orchestrator 0.3.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (354) hide show
  1. package/README.md +20 -2
  2. package/bin/server.ts +7 -4
  3. package/console/canvas.ts +27 -14
  4. package/console/components/app.ts +12 -1
  5. package/console/components/fleet.ts +52 -6
  6. package/console/components/machine-pane.ts +128 -2
  7. package/console/components/nav.ts +1 -1
  8. package/console/components/schema-form.ts +12 -4
  9. package/console/json-field.ts +23 -0
  10. package/console/main.ts +122 -39
  11. package/console/store.ts +321 -21
  12. package/console/style.css +111 -0
  13. package/docs/agents-block.md +5 -0
  14. package/docs/concepts/agents-and-turns.md +442 -0
  15. package/docs/concepts/architecture.md +230 -0
  16. package/docs/concepts/charts.md +143 -0
  17. package/docs/concepts/composition.md +357 -0
  18. package/docs/concepts/converge.md +254 -0
  19. package/docs/concepts/durable-runs.md +189 -0
  20. package/docs/concepts/effects.md +603 -0
  21. package/docs/concepts/gates.md +208 -0
  22. package/docs/concepts/machines-and-workflows.md +289 -0
  23. package/docs/concepts/pools-and-sources.md +230 -0
  24. package/docs/concepts/secrets-and-trust.md +222 -0
  25. package/docs/concepts/the-menu.md +378 -0
  26. package/docs/concepts/workspaces.md +678 -0
  27. package/docs/get-started/a-model-decides.md +563 -0
  28. package/docs/get-started/an-agent-in-a-sandbox.md +381 -0
  29. package/docs/get-started/connect-a-model.md +248 -0
  30. package/docs/get-started/create-an-instance.md +194 -0
  31. package/docs/get-started/have-your-assistant-write-a-machine.md +922 -0
  32. package/docs/get-started/install.md +126 -0
  33. package/docs/guides/add-tools-to-the-sandbox.md +295 -0
  34. package/docs/guides/answer-a-gate.md +304 -0
  35. package/docs/guides/bind-open-parts.md +223 -0
  36. package/docs/guides/cancel-a-run.md +146 -0
  37. package/docs/guides/clean-up.md +206 -0
  38. package/docs/guides/fix-a-failed-converge.md +361 -0
  39. package/docs/guides/give-the-cluster-git-credentials.md +222 -0
  40. package/docs/guides/iterate-on-a-machine-on-kind.md +190 -0
  41. package/docs/guides/let-an-agent-write-a-chart.md +226 -0
  42. package/docs/guides/look-inside-a-sandbox.md +174 -0
  43. package/docs/guides/mirror-the-kit-images.md +176 -0
  44. package/docs/guides/move-from-kind-to-a-shared-cluster.md +302 -0
  45. package/docs/guides/open-the-console.md +91 -0
  46. package/docs/guides/publish-a-machine-as-an-npm-package.md +350 -0
  47. package/docs/guides/push-and-sign-a-branch.md +486 -0
  48. package/docs/guides/run-a-chart-by-name.md +270 -0
  49. package/docs/guides/run-a-command-or-call-a-url.md +544 -0
  50. package/docs/guides/run-one-worker-per-item.md +321 -0
  51. package/docs/guides/set-an-agents-access-and-dials.md +375 -0
  52. package/docs/guides/size-a-workspace.md +204 -0
  53. package/docs/guides/start-a-run-from-a-run.md +237 -0
  54. package/docs/guides/start-a-run.md +207 -0
  55. package/docs/guides/upgrade-the-kit-version.md +240 -0
  56. package/docs/guides/use-a-hosted-provider.md +208 -0
  57. package/docs/guides/use-a-local-model.md +313 -0
  58. package/docs/guides/watch-a-run.md +326 -0
  59. package/docs/guides/write-a-workflow.md +318 -0
  60. package/docs/index.md +44 -0
  61. package/docs/llms.txt +216 -0
  62. package/docs/reference/authoring-surface/agent.md +128 -0
  63. package/docs/reference/authoring-surface/answer.md +140 -0
  64. package/docs/reference/authoring-surface/arg.md +60 -0
  65. package/docs/reference/authoring-surface/basic.md +62 -0
  66. package/docs/reference/authoring-surface/bearer.md +53 -0
  67. package/docs/reference/authoring-surface/chartschema.md +23 -0
  68. package/docs/reference/authoring-surface/chartschemafor.md +48 -0
  69. package/docs/reference/authoring-surface/command.md +191 -0
  70. package/docs/reference/authoring-surface/customize.md +64 -0
  71. package/docs/reference/authoring-surface/defineconfig.md +41 -0
  72. package/docs/reference/authoring-surface/defineevent.md +114 -0
  73. package/docs/reference/authoring-surface/dispatch.md +126 -0
  74. package/docs/reference/authoring-surface/fail.md +83 -0
  75. package/docs/reference/authoring-surface/file.md +122 -0
  76. package/docs/reference/authoring-surface/frompick.md +277 -0
  77. package/docs/reference/authoring-surface/http.md +159 -0
  78. package/docs/reference/authoring-surface/isfailure.md +58 -0
  79. package/docs/reference/authoring-surface/jr2setup.md +237 -0
  80. package/docs/reference/authoring-surface/open.md +74 -0
  81. package/docs/reference/authoring-surface/openbody.md +86 -0
  82. package/docs/reference/authoring-surface/paramsof.md +125 -0
  83. package/docs/reference/authoring-surface/pool.md +186 -0
  84. package/docs/reference/authoring-surface/push.md +112 -0
  85. package/docs/reference/authoring-surface/secret.md +118 -0
  86. package/docs/reference/authoring-surface/sentby.md +83 -0
  87. package/docs/reference/authoring-surface/source.md +51 -0
  88. package/docs/reference/authoring-surface/text.md +62 -0
  89. package/docs/reference/authoring-surface/type-agentdeclaration.md +27 -0
  90. package/docs/reference/authoring-surface/type-agentdefinition.md +80 -0
  91. package/docs/reference/authoring-surface/type-agentlogic.md +15 -0
  92. package/docs/reference/authoring-surface/type-agentslots.md +12 -0
  93. package/docs/reference/authoring-surface/type-answeraction.md +12 -0
  94. package/docs/reference/authoring-surface/type-arg.md +19 -0
  95. package/docs/reference/authoring-surface/type-argnames.md +12 -0
  96. package/docs/reference/authoring-surface/type-binding.md +62 -0
  97. package/docs/reference/authoring-surface/type-bindingof.md +12 -0
  98. package/docs/reference/authoring-surface/type-bodyacceptsdoor.md +15 -0
  99. package/docs/reference/authoring-surface/type-bodyacceptsslots.md +17 -0
  100. package/docs/reference/authoring-surface/type-ceilings.md +41 -0
  101. package/docs/reference/authoring-surface/type-chainlink.md +19 -0
  102. package/docs/reference/authoring-surface/type-chartaction.md +46 -0
  103. package/docs/reference/authoring-surface/type-chartagent.md +28 -0
  104. package/docs/reference/authoring-surface/type-chartevent.md +20 -0
  105. package/docs/reference/authoring-surface/type-chartguard.md +20 -0
  106. package/docs/reference/authoring-surface/type-chartinvoke.md +28 -0
  107. package/docs/reference/authoring-surface/type-chartjson.md +14 -0
  108. package/docs/reference/authoring-surface/type-chartstate.md +46 -0
  109. package/docs/reference/authoring-surface/type-charttext.md +40 -0
  110. package/docs/reference/authoring-surface/type-charttransition.md +22 -0
  111. package/docs/reference/authoring-surface/type-charttransitions.md +14 -0
  112. package/docs/reference/authoring-surface/type-chartvalue.md +15 -0
  113. package/docs/reference/authoring-surface/type-childat.md +12 -0
  114. package/docs/reference/authoring-surface/type-childslots.md +12 -0
  115. package/docs/reference/authoring-surface/type-commandenv.md +15 -0
  116. package/docs/reference/authoring-surface/type-commandinput.md +19 -0
  117. package/docs/reference/authoring-surface/type-commandoptions.md +78 -0
  118. package/docs/reference/authoring-surface/type-commandoutput.md +104 -0
  119. package/docs/reference/authoring-surface/type-customize.md +109 -0
  120. package/docs/reference/authoring-surface/type-dispatchdefinition.md +39 -0
  121. package/docs/reference/authoring-surface/type-dispatchinput.md +14 -0
  122. package/docs/reference/authoring-surface/type-dispatchopenparts.md +14 -0
  123. package/docs/reference/authoring-surface/type-dispatchoutput.md +65 -0
  124. package/docs/reference/authoring-surface/type-duration.md +15 -0
  125. package/docs/reference/authoring-surface/type-effectbinding.md +93 -0
  126. package/docs/reference/authoring-surface/type-effectlogic.md +12 -0
  127. package/docs/reference/authoring-surface/type-envopenparts.md +14 -0
  128. package/docs/reference/authoring-surface/type-eventaudience.md +27 -0
  129. package/docs/reference/authoring-surface/type-eventdef.md +45 -0
  130. package/docs/reference/authoring-surface/type-eventfrom.md +16 -0
  131. package/docs/reference/authoring-surface/type-eventsemantics.md +26 -0
  132. package/docs/reference/authoring-surface/type-execoutcome.md +34 -0
  133. package/docs/reference/authoring-surface/type-failslot.md +14 -0
  134. package/docs/reference/authoring-surface/type-failure.md +19 -0
  135. package/docs/reference/authoring-surface/type-failureinput.md +20 -0
  136. package/docs/reference/authoring-surface/type-filebody.md +41 -0
  137. package/docs/reference/authoring-surface/type-frominput.md +14 -0
  138. package/docs/reference/authoring-surface/type-hostinjectedinput.md +58 -0
  139. package/docs/reference/authoring-surface/type-httpdefinition.md +101 -0
  140. package/docs/reference/authoring-surface/type-httpinput.md +12 -0
  141. package/docs/reference/authoring-surface/type-httpmethod.md +14 -0
  142. package/docs/reference/authoring-surface/type-httpopenparts.md +12 -0
  143. package/docs/reference/authoring-surface/type-httpoutput.md +89 -0
  144. package/docs/reference/authoring-surface/type-httprequestparts.md +33 -0
  145. package/docs/reference/authoring-surface/type-jr2config.md +126 -0
  146. package/docs/reference/authoring-surface/type-lends.md +38 -0
  147. package/docs/reference/authoring-surface/type-logicat.md +12 -0
  148. package/docs/reference/authoring-surface/type-markedif.md +12 -0
  149. package/docs/reference/authoring-surface/type-mergedactors.md +12 -0
  150. package/docs/reference/authoring-surface/type-namedsecret.md +25 -0
  151. package/docs/reference/authoring-surface/type-noslotspec.md +26 -0
  152. package/docs/reference/authoring-surface/type-noslotworkspaceoptions.md +32 -0
  153. package/docs/reference/authoring-surface/type-openbodydeclaration.md +40 -0
  154. package/docs/reference/authoring-surface/type-openbodylogic.md +14 -0
  155. package/docs/reference/authoring-surface/type-openbodyoutput.md +42 -0
  156. package/docs/reference/authoring-surface/type-openbodyslot.md +18 -0
  157. package/docs/reference/authoring-surface/type-opencommandinput.md +17 -0
  158. package/docs/reference/authoring-surface/type-openeffect.md +12 -0
  159. package/docs/reference/authoring-surface/type-openeffectslots.md +12 -0
  160. package/docs/reference/authoring-surface/type-openof.md +12 -0
  161. package/docs/reference/authoring-surface/type-opensecret.md +23 -0
  162. package/docs/reference/authoring-surface/type-opensecretmark.md +12 -0
  163. package/docs/reference/authoring-surface/type-permissivenoslotworkspaceoptions.md +31 -0
  164. package/docs/reference/authoring-surface/type-permissiveworkspaceoptions.md +32 -0
  165. package/docs/reference/authoring-surface/type-poolmachine.md +28 -0
  166. package/docs/reference/authoring-surface/type-poolspec.md +60 -0
  167. package/docs/reference/authoring-surface/type-pushinput.md +15 -0
  168. package/docs/reference/authoring-surface/type-pushoptions.md +79 -0
  169. package/docs/reference/authoring-surface/type-pushoutput.md +124 -0
  170. package/docs/reference/authoring-surface/type-pushwireoutput.md +12 -0
  171. package/docs/reference/authoring-surface/type-quantity.md +14 -0
  172. package/docs/reference/authoring-surface/type-reached.md +12 -0
  173. package/docs/reference/authoring-surface/type-reposlot.md +70 -0
  174. package/docs/reference/authoring-surface/type-reposlotsof.md +16 -0
  175. package/docs/reference/authoring-surface/type-sandboxoptions.md +77 -0
  176. package/docs/reference/authoring-surface/type-secret.md +25 -0
  177. package/docs/reference/authoring-surface/type-sender.md +19 -0
  178. package/docs/reference/authoring-surface/type-sentbyguards.md +12 -0
  179. package/docs/reference/authoring-surface/type-size.md +85 -0
  180. package/docs/reference/authoring-surface/type-slothandles.md +32 -0
  181. package/docs/reference/authoring-surface/type-sourcenextresult.md +17 -0
  182. package/docs/reference/authoring-surface/type-sourcespec.md +44 -0
  183. package/docs/reference/authoring-surface/type-syncstate.md +12 -0
  184. package/docs/reference/authoring-surface/type-text.md +19 -0
  185. package/docs/reference/authoring-surface/type-textpart.md +24 -0
  186. package/docs/reference/authoring-surface/type-thinkinglevel.md +39 -0
  187. package/docs/reference/authoring-surface/type-toparameterizedobject.md +14 -0
  188. package/docs/reference/authoring-surface/type-toprovidedactor.md +14 -0
  189. package/docs/reference/authoring-surface/type-unknown.md +21 -0
  190. package/docs/reference/authoring-surface/type-usercontainer.md +77 -0
  191. package/docs/reference/authoring-surface/type-withoutemptyindex.md +20 -0
  192. package/docs/reference/authoring-surface/type-workflowevent.md +46 -0
  193. package/docs/reference/authoring-surface/type-workspaceaccess.md +16 -0
  194. package/docs/reference/authoring-surface/type-workspaced.md +94 -0
  195. package/docs/reference/authoring-surface/type-workspacehandles.md +23 -0
  196. package/docs/reference/authoring-surface/type-workspacemachine.md +29 -0
  197. package/docs/reference/authoring-surface/type-workspaceof.md +13 -0
  198. package/docs/reference/authoring-surface/type-workspaceoptions.md +39 -0
  199. package/docs/reference/authoring-surface/type-workspacespec.md +37 -0
  200. package/docs/reference/authoring-surface/workspace.md +259 -0
  201. package/docs/reference/authoring-surface.md +197 -0
  202. package/docs/reference/cli.md +433 -0
  203. package/docs/reference/config.md +756 -0
  204. package/docs/reference/decisions/0001-operator-owned-generic-sandbox.md +113 -0
  205. package/docs/reference/decisions/0002-duplex-streaming-actor.md +110 -0
  206. package/docs/reference/decisions/0004-worktree-provisioning-model.md +124 -0
  207. package/docs/reference/decisions/0005-sandbox-pod-composition.md +230 -0
  208. package/docs/reference/decisions/0006-agent-control-surface.md +47 -0
  209. package/docs/reference/decisions/0007-durable-machine-state.md +109 -0
  210. package/docs/reference/decisions/0008-orchestrator-as-deployed-app.md +60 -0
  211. package/docs/reference/decisions/0009-cli-and-instance-interface.md +275 -0
  212. package/docs/reference/decisions/0010-bdd-acceptance-tests.md +160 -0
  213. package/docs/reference/decisions/0011-workflow-defined-events.md +141 -0
  214. package/docs/reference/decisions/0012-workspace-wrapper-machine.md +103 -0
  215. package/docs/reference/decisions/0013-the-agent-reaches-its-machine-through-a-container-it-cannot-read.md +157 -0
  216. package/docs/reference/decisions/0014-observation-is-open-run-state-is-not.md +95 -0
  217. package/docs/reference/decisions/0015-authoring-surface-absorbs-the-mechanism.md +265 -0
  218. package/docs/reference/decisions/0016-agent-turn-mechanics-are-internal.md +124 -0
  219. package/docs/reference/decisions/0017-pool-over-a-source-port.md +69 -0
  220. package/docs/reference/decisions/0018-instance-agents-are-definitions-jr2-assembles-the-harness.md +111 -0
  221. package/docs/reference/decisions/0019-one-converging-command-against-the-current-context.md +275 -0
  222. package/docs/reference/decisions/0020-private-ca-trust-is-a-named-concept.md +60 -0
  223. package/docs/reference/decisions/0021-workspace-continuity-is-a-lease-that-answers-back.md +100 -0
  224. package/docs/reference/decisions/0022-observation-is-a-level-triggered-feed.md +103 -0
  225. package/docs/reference/decisions/0023-the-harness-prints-the-conversation.md +176 -0
  226. package/docs/reference/decisions/0024-an-agents-turn-ends-with-the-state-that-asked-for-it.md +439 -0
  227. package/docs/reference/decisions/0025-cancel-ends-a-run-stop-parks-it.md +65 -0
  228. package/docs/reference/decisions/0026-a-turn-that-is-over-is-never-prompted.md +62 -0
  229. package/docs/reference/decisions/0027-the-harness-is-jr2s-own-server-flue-retires-the-wire-stays.md +237 -0
  230. package/docs/reference/decisions/0028-what-an-agent-may-do-to-the-workspace-is-part-of-its-definition.md +89 -0
  231. package/docs/reference/decisions/0029-a-menu-is-fixed-for-a-conversation-and-a-turn-is-told-its-allowed-picks.md +154 -0
  232. package/docs/reference/decisions/0030-a-snapshot-names-the-machine-it-was-written-under.md +100 -0
  233. package/docs/reference/decisions/0031-menu-only-agents-run-on-the-instance-harness.md +190 -0
  234. package/docs/reference/decisions/0032-the-console-unlocks-with-the-instance-token.md +61 -0
  235. package/docs/reference/decisions/0033-a-machine-declares-the-input-that-starts-it.md +127 -0
  236. package/docs/reference/decisions/0034-the-console-renders-with-preact-and-the-server-erases-the-types.md +57 -0
  237. package/docs/reference/decisions/0035-a-runaway-turn-is-ended-rerolled-once-then-a-fault.md +92 -0
  238. package/docs/reference/decisions/0036-a-turn-compacts-its-own-context-mid-flight.md +138 -0
  239. package/docs/reference/decisions/0037-an-instance-builds-its-sandbox-images-jr2-injects-the-harness.md +196 -0
  240. package/docs/reference/decisions/0038-jr2-up-builds-every-image-it-deploys.md +170 -0
  241. package/docs/reference/decisions/0039-image-garbage-collects-by-reachability.md +115 -0
  242. package/docs/reference/decisions/0041-a-build-the-host-already-holds-is-not-spent-again.md +64 -0
  243. package/docs/reference/decisions/0042-ready-is-not-routable-an-admission-retries-its-connection.md +142 -0
  244. package/docs/reference/decisions/0043-the-kit-is-tested-as-installed-a-local-registry-stands-in-for-npm.md +170 -0
  245. package/docs/reference/decisions/0044-kit-images-live-at-a-canonical-home-a-self-host-mirrors-it.md +85 -0
  246. package/docs/reference/decisions/0045-the-platform-joins-the-image-address-and-the-cluster-chooses-it.md +82 -0
  247. package/docs/reference/decisions/0046-a-converge-failure-carries-its-diagnosis.md +34 -0
  248. package/docs/reference/decisions/0047-the-git-ssh-key-source-is-the-users-choice.md +47 -0
  249. package/docs/reference/decisions/0048-the-orchestrator-boots-without-its-repos.md +36 -0
  250. package/docs/reference/decisions/0049-a-machine-carries-its-parts-and-composes-by-invoke.md +126 -0
  251. package/docs/reference/decisions/0050-what-code-names-config-declares-what-the-cli-names-discovery-finds.md +73 -0
  252. package/docs/reference/decisions/0051-a-repo-is-a-slot-on-the-workspace-and-a-cache-on-the-node.md +227 -0
  253. package/docs/reference/decisions/0052-a-sandbox-lands-where-an-ordinary-pod-lands-and-the-cache-follows-it.md +81 -0
  254. package/docs/reference/decisions/0053-a-fetch-inside-the-pod-asks-the-node-cache-and-the-cache-asks-the-remote.md +111 -0
  255. package/docs/reference/decisions/0054-the-kit-ships-machines-and-a-shipped-machine-leaves-its-parts-open.md +126 -0
  256. package/docs/reference/decisions/0055-a-release-is-a-pushed-tag-and-the-publish-guard-is-the-credential.md +107 -0
  257. package/docs/reference/decisions/0056-a-global-jr2-hands-off-to-the-instances-own-and-an-instance-has-one-kit-version.md +62 -0
  258. package/docs/reference/decisions/0057-a-turn-is-its-frame-its-dials-and-whether-it-continues.md +152 -0
  259. package/docs/reference/decisions/0058-a-harness-answers-the-orchestrator-alone.md +106 -0
  260. package/docs/reference/decisions/0059-a-harness-holds-stand-ins-and-the-custodian-holds-the-keys.md +384 -0
  261. package/docs/reference/decisions/0060-a-workspace-states-its-size-and-jr2-reserves-all-of-it.md +111 -0
  262. package/docs/reference/decisions/0061-a-memory-kill-ends-the-agents-processes-not-the-harness.md +83 -0
  263. package/docs/reference/decisions/0062-jr2-briefs-the-agent-on-its-seat-never-on-its-task.md +212 -0
  264. package/docs/reference/decisions/0063-the-orchestrator-watches-the-cluster.md +83 -0
  265. package/docs/reference/decisions/0064-a-workspace-waits-for-capacity.md +79 -0
  266. package/docs/reference/decisions/0065-a-persist-survives-a-crash-and-history-is-a-week.md +65 -0
  267. package/docs/reference/decisions/0066-a-model-tier-runs-a-real-model-and-a-change-to-what-it-reads-runs-the-tier.md +550 -0
  268. package/docs/reference/decisions/0067-an-effect-is-an-actor-slot-and-runs-at-most-once-unless-marked-idempotent.md +430 -0
  269. package/docs/reference/decisions/0068-an-effector-holds-an-effects-secrets-and-signs-its-wire.md +464 -0
  270. package/docs/reference/decisions/0069-the-orchestrator-is-a-kit-image-and-an-instance-publishes-its-assembly.md +180 -0
  271. package/docs/reference/decisions/0070-a-machine-lends-its-parts-to-a-chart.md +323 -0
  272. package/docs/reference/decisions/0071-a-run-starts-a-run-through-a-dispatch-effect.md +183 -0
  273. package/docs/reference/decisions/0072-the-docs-ship-inside-the-kit-and-an-instances-agents-md-indexes-them.md +144 -0
  274. package/docs/reference/decisions/0073-a-machine-fails-a-run-with-fail-and-an-uncaught-failure-ends-it-failed.md +122 -0
  275. package/docs/reference/decisions/0074-an-unrouted-mechanism-event-is-refused-at-load-and-faults-the-run.md +76 -0
  276. package/docs/reference/decisions.md +83 -0
  277. package/docs/reference/glossary.md +648 -0
  278. package/docs/reference/http-api.md +1348 -0
  279. package/docs/reference/machines.md +190 -0
  280. package/docs/reference/mechanism-events.md +72 -0
  281. package/docs/why.md +206 -0
  282. package/package.json +6 -4
  283. package/src/actor.ts +976 -61
  284. package/src/agent.ts +114 -38
  285. package/src/ambient.ts +4 -1
  286. package/src/assembly-checks.ts +347 -0
  287. package/src/assembly-tar.ts +494 -0
  288. package/src/assembly.ts +721 -0
  289. package/src/cause.ts +234 -0
  290. package/src/chart.ts +2182 -0
  291. package/src/charts.ts +748 -0
  292. package/src/config.ts +520 -162
  293. package/src/content-hash.ts +105 -0
  294. package/src/custodian.lua +6 -5
  295. package/src/custodian.ts +143 -39
  296. package/src/customize.ts +388 -55
  297. package/src/dispatch-effect.ts +945 -0
  298. package/src/door-compat.ts +217 -0
  299. package/src/durability.ts +11 -4
  300. package/src/effect-checks.ts +557 -0
  301. package/src/effect.ts +1202 -0
  302. package/src/effector-wire.ts +674 -0
  303. package/src/fail.ts +209 -0
  304. package/src/fingerprint.ts +4 -3
  305. package/src/gate.ts +8 -0
  306. package/src/git-clone.ts +65 -0
  307. package/src/git-push.ts +336 -0
  308. package/src/harness-client.ts +764 -48
  309. package/src/held-secrets.ts +82 -26
  310. package/src/holds.ts +156 -0
  311. package/src/http-effect.ts +1423 -0
  312. package/src/http.ts +916 -113
  313. package/src/images.ts +47 -21
  314. package/src/index.ts +62 -43
  315. package/src/instance.ts +505 -59
  316. package/src/internal.ts +84 -0
  317. package/src/kube-client.ts +333 -0
  318. package/src/machine-doc.ts +106 -6
  319. package/src/mechanism-routes.ts +168 -0
  320. package/src/model-keys.generated.ts +53 -0
  321. package/src/model-keys.ts +18 -58
  322. package/src/mount-digest.ts +53 -0
  323. package/src/names.ts +66 -8
  324. package/src/notices.ts +141 -0
  325. package/src/open-body.ts +1580 -0
  326. package/src/open-effect.ts +217 -0
  327. package/src/open.ts +59 -6
  328. package/src/parts.ts +709 -41
  329. package/src/party.ts +534 -0
  330. package/src/pick.ts +761 -0
  331. package/src/pool.ts +248 -37
  332. package/src/registration.ts +382 -19
  333. package/src/repo-fetch.ts +60 -85
  334. package/src/repos.ts +65 -75
  335. package/src/rest.ts +133 -0
  336. package/src/run-host.ts +2383 -115
  337. package/src/run-output.ts +38 -0
  338. package/src/sandbox-effect.ts +1126 -0
  339. package/src/sandbox-effector.ts +111 -0
  340. package/src/sandbox-kube.ts +1721 -0
  341. package/src/sandbox-watch.ts +260 -0
  342. package/src/secret-placement.ts +529 -0
  343. package/src/server.ts +284 -84
  344. package/src/setup.ts +819 -117
  345. package/src/size.ts +454 -0
  346. package/src/snapshot-store.ts +115 -20
  347. package/src/stub-harness.ts +49 -16
  348. package/src/sync-client.ts +54 -0
  349. package/src/timers.ts +191 -0
  350. package/src/tokens.ts +35 -4
  351. package/src/vocabulary.ts +135 -15
  352. package/src/wire.ts +238 -10
  353. package/src/workspace.ts +1195 -338
  354. package/src/sandbox-kubectl.ts +0 -1108
package/src/run-host.ts CHANGED
@@ -21,29 +21,87 @@
21
21
 
22
22
  import { randomUUID } from "node:crypto";
23
23
  import { z } from "zod";
24
- import { createActor, type AnyActor, type AnyActorLogic, type AnyActorRef, type AnyStateMachine } from "xstate";
25
- import type { EventSemantics } from "@jr2/agent-protocol";
24
+ import {
25
+ createActor,
26
+ type AnyActor,
27
+ type AnyActorLogic,
28
+ type AnyActorRef,
29
+ type AnyEventObject,
30
+ type AnyMachineSnapshot,
31
+ type AnyStateMachine,
32
+ } from "xstate";
33
+ import { SENDER_KEY, type EventDef, type EventSemantics, type Sender } from "@jr2/agent-protocol";
34
+ import { partiesAt, seatOf, senderText } from "./party.ts";
35
+ import { placeAnswers, type AnswerSeat } from "./pick.ts";
26
36
  import { inputSchemaOf } from "./vocabulary.ts";
27
- import type { EchoEvent, EchoStatusChild } from "./wire.ts";
37
+ import type { EchoEvent, EchoStatusChild, LiveConversation, PickAnswer } from "./wire.ts";
38
+ import { answerOf, childErrorOf, replyOf, type Hold } from "./holds.ts";
39
+ import { PickCauses } from "./cause.ts";
40
+ import { restOf, runningBelow } from "./rest.ts";
28
41
  import {
42
+ acceptsNow,
29
43
  agentAddress,
44
+ allowedPicks,
30
45
  bindRun,
31
46
  EventValidationError,
32
47
  gateAddress,
33
- mayMove,
48
+ NotTakenError,
49
+ OutstandingError,
34
50
  RegistrationTable,
51
+ refuseSenderField,
35
52
  UnknownAddressError,
36
53
  wouldMove,
54
+ targetsOf,
55
+ leftAgain,
56
+ type DispatchedBy,
57
+ type DispatchStart,
58
+ type Registration,
37
59
  type RetryTelemetry,
38
60
  type RunBinding,
39
- type TurnMarker,
61
+ type RunMarker,
40
62
  } from "./registration.ts";
41
- import type { SandboxPort } from "./workspace.ts";
63
+ import { NoticeLedger, type NoticeLedgerState } from "./notices.ts";
64
+ import { pendingTimers, startAndRearm, watchTimers, type PendingTimer } from "./timers.ts";
65
+ import {
66
+ DEFAULT_LEASE_INTERVAL_MS,
67
+ WORKSPACE_WAIT_KEY,
68
+ leaseDelay,
69
+ workspaceName,
70
+ type SandboxPort,
71
+ type WorkspaceWait,
72
+ } from "./workspace.ts";
73
+ import { INSTANCE_HARNESS_SERVICE } from "./names.ts";
74
+ import { composesSandbox } from "./parts.ts";
75
+ import { failurePayload, isFailure, type FailureInput } from "./fail.ts";
76
+ import { unroutedMechanismEvent, type MicrostepView } from "./mechanism-routes.ts";
42
77
  import { fingerprintOf } from "./fingerprint.ts";
43
- import { serializeMachine, type MachineDoc } from "./machine-doc.ts";
78
+ import {
79
+ serializeMachine,
80
+ type ChildMachineDoc,
81
+ type MachineBodyDoc,
82
+ type MachineDoc,
83
+ type MachineStateDoc,
84
+ } from "./machine-doc.ts";
44
85
  import type { SnapshotStore } from "./snapshot-store.ts";
45
86
  import type { AgentAdmission } from "./actor.ts";
87
+ import { dropSavedTails, type EffectRow } from "./effect.ts";
88
+ import type { EffectorPlacement, EffectorTarget } from "./http-effect.ts";
46
89
  import { reattachAgentRuns, serializeSnapshot } from "./durability.ts";
90
+ import { admitChart, chartBriefingOf, chartHash, ChartRefusal, isChartSchema, type ChartText } from "./chart.ts";
91
+ import type { ChartOrigin, NamedChart } from "./charts.ts";
92
+ import {
93
+ admitBodyChart,
94
+ admitWrapperChart,
95
+ boundChartRoutesOf,
96
+ chartWrapperMachineOf,
97
+ boundMachineOf,
98
+ compileChart,
99
+ isOpenBody,
100
+ lentIn,
101
+ openBodiesOf,
102
+ openBodyDeclarationOf,
103
+ type BoundChartRoute,
104
+ } from "./open-body.ts";
47
105
 
48
106
  /** The actors filled into a run's named slots — `.provide()` is the unit-test seam (ADR-0015);
49
107
  * production instances inject nothing. Built fresh at start and at restore. */
@@ -56,10 +114,149 @@ export type WorkflowDef = {
56
114
  machine: AnyStateMachine;
57
115
  /** Build this run's live providers. A test seam (ADR-0015): discovery injects nothing. */
58
116
  provide: (ctx: { instanceId: string }) => RunProviders;
117
+ /**
118
+ * The hash of the Assembly this Machine came from (ADR-0069): set at register for the current
119
+ * Assembly, and by `resolveDef` for a held one at restore. A run holds the Assembly it started
120
+ * under, so every save of the run stamps this hash. Absent only for a def that no Assembly
121
+ * loaded: an in-process test host, which stamps nothing.
122
+ */
123
+ assembly?: string;
124
+ /**
125
+ * The def is a Chart wrapper (ADR-0070): `body` is its (first) Open body's slot key, and
126
+ * `chartFields` are the door's fields that are `chartSchema` — empty for a wrapper whose Charts
127
+ * only its own states supply. `register` reads both off the Machine when the caller leaves this
128
+ * out.
129
+ */
130
+ wrapper?: { body: string; chartFields: string[] };
131
+ /**
132
+ * The def is a named Chart (ADR-0070): the Workflow `<wrapper>/<name>`, whose `machine` is its
133
+ * wrapper's. A start supplies every Chart field of the wrapper's door from `text` and takes the
134
+ * rest from the caller.
135
+ */
136
+ chart?: { wrapper: string; name: string; text: ChartText; hash: string; origin: ChartOrigin };
137
+ };
138
+
139
+ /**
140
+ * One entry of `GET /workflows` (ADR-0070): a Workflow and where it came from. A Chart wrapper's
141
+ * entry carries its Open body, whether its door takes a Chart (`oneShot`), and its named Charts.
142
+ */
143
+ export type WorkflowListing = {
144
+ name: string;
145
+ origin: "machine" | ChartOrigin;
146
+ wrapper?: {
147
+ body: string;
148
+ oneShot: boolean;
149
+ charts: Array<{ name: string; origin: ChartOrigin; unrunnable?: string }>;
150
+ };
151
+ unrunnable?: string;
59
152
  };
60
153
 
61
- /** The serializable identity of a run — what reconcile sees and what restore rebuilds from. */
62
- export type RunRecord = { runId: string; workflow: string; instanceId: string };
154
+ /** What makes a Machine a Chart wrapper (ADR-0070), read off it: its first Open body's slot key and
155
+ * the door's fields that are `chartSchema`. The bodies are seen through jr2's transparent wrappers
156
+ * (a Chart wrapper that is a `workspace()`'s body), and the door is the registered Machine's.
157
+ * Undefined for a Machine with no Open body. */
158
+ function wrapperOf(machine: AnyStateMachine): WorkflowDef["wrapper"] | undefined {
159
+ const bodies = openBodiesOf(chartWrapperMachineOf(machine).machine);
160
+ if (!bodies.length) return undefined;
161
+ const shape = (inputSchemaOf(machine)?.shape ?? {}) as Record<string, unknown>;
162
+ return { body: bodies[0]!.slot, chartFields: Object.keys(shape).filter((k) => isChartSchema(shape[k])) };
163
+ }
164
+
165
+ /** A pick one of whose parameters is a Chart (by schema identity, ADR-0070) is marked with that
166
+ * grammar's briefing, for the Turn part. */
167
+ function briefingOf(input: z.ZodObject): { briefing?: string } {
168
+ const shape = input.shape as Record<string, unknown>;
169
+ for (const key of Object.keys(shape)) {
170
+ const briefing = chartBriefingOf(shape[key]);
171
+ if (briefing !== undefined) return { briefing };
172
+ }
173
+ return {};
174
+ }
175
+
176
+ /** The short wrapper name of a Workflow name: `<wrapper>` of `<wrapper>/<name>`, else the name. */
177
+ function wrapperName(workflow: string): string {
178
+ const at = workflow.indexOf("/");
179
+ return at > 0 ? workflow.slice(0, at) : workflow;
180
+ }
181
+
182
+ /**
183
+ * Why a run's bound Charts no longer read under `machine` (ADR-0070, D-B), or undefined when each
184
+ * does: every bound Chart is admitted again against the Open body slot it binds, resolved on the
185
+ * Machine through `implementations.actors` (a lent child Machine's own slots included), with the
186
+ * lent set that Machine has now. A text whose hash does not match is drift too: a hand-edited
187
+ * volume.
188
+ */
189
+ function chartDrift(machine: AnyStateMachine, routes: readonly BoundChartRoute[]): string | undefined {
190
+ for (const route of routes) {
191
+ const where = `chart "${route.path.join("/")}" under body "${route.src}"`;
192
+ let at: AnyStateMachine = machine;
193
+ let resolved = true;
194
+ for (const src of route.srcs.slice(0, -1)) {
195
+ const logic = (at.implementations.actors as Record<string, unknown>)[src];
196
+ // The lent parts a bound body's Chart invokes are its wrapper's: the next hop is resolved there.
197
+ if (isOpenBody(logic)) continue;
198
+ const child = (logic as AnyStateMachine | undefined)?.root ? (logic as AnyStateMachine) : undefined;
199
+ if (!child) {
200
+ resolved = false;
201
+ break;
202
+ }
203
+ at = child;
204
+ }
205
+ const logic = resolved ? (at.implementations.actors as Record<string, unknown>)[route.src] : undefined;
206
+ if (!isOpenBody(logic)) return `${where}: the Machine has no Open body "${route.src}" now`;
207
+ const hash = chartHash(route.chart);
208
+ if (hash !== route.chartHash) {
209
+ return `${where}: its hash ${route.chartHash} does not match its text (${hash}); the stored run was edited`;
210
+ }
211
+ const decl = openBodyDeclarationOf(logic);
212
+ try {
213
+ admitBodyChart(route.chart, decl, lentIn(at.implementations as never, decl));
214
+ } catch (err) {
215
+ if (!(err instanceof ChartRefusal)) throw err;
216
+ return `${where}: ${err.report.join("; ")}`;
217
+ }
218
+ }
219
+ return undefined;
220
+ }
221
+
222
+ /** The serializable identity of a run — what reconcile sees and what restore rebuilds from.
223
+ * `dispatchedBy` is the run that started it through a `dispatch` Effect, and this run's depth in
224
+ * that chain, as the caller sent it (ADR-0071); absent for a run a human or a client started. */
225
+ export type RunRecord = { runId: string; workflow: string; instanceId: string; dispatchedBy?: DispatchedBy };
226
+
227
+ /** A start of a name no Workflow is registered under: the door answers 404 (ADR-0009), and an
228
+ * in-process `dispatch` settles `refused` with status 404 (ADR-0071). */
229
+ export class UnknownWorkflowError extends Error {
230
+ override name = "UnknownWorkflowError";
231
+ }
232
+
233
+ /** A start whose idempotency key a run of ANOTHER Workflow already carries (ADR-0071): the door
234
+ * answers 409, and an in-process `dispatch` settles `refused` with status 409. Never answered as
235
+ * the existing run: the caller asked for a run of one Workflow and would wait on a run of another. */
236
+ export class DispatchKeyConflictError extends Error {
237
+ override name = "DispatchKeyConflictError";
238
+ }
239
+
240
+ /** The run a key names (ADR-0071), with its Workflow so a start of another can be refused. */
241
+ export type DispatchedRun = { runId: string; instanceId: string; workflow: string };
242
+
243
+ /** The run `found` as the answer to a start of `workflow` under its key, or the 409 (ADR-0071). */
244
+ export function keyAnswer(
245
+ found: DispatchedRun,
246
+ workflow: string,
247
+ effect: string,
248
+ ): { runId: string; instanceId: string } {
249
+ if (found.workflow !== workflow) throw new DispatchKeyConflictError(keyConflict(found, workflow, effect));
250
+ return { runId: found.runId, instanceId: found.instanceId };
251
+ }
252
+
253
+ /** Why a start of `workflow` under `effect` is refused, when the key's run is of another. */
254
+ export function keyConflict(found: DispatchedRun, workflow: string, effect: string): string {
255
+ return (
256
+ `the key "${effect}" already started run ${found.runId} of "${found.workflow}", not of "${workflow}" — ` +
257
+ "one key is one start of one Workflow (ADR-0071)"
258
+ );
259
+ }
63
260
 
64
261
  /**
65
262
  * One live child MACHINE of a run, and its own children below it. A workflow's root machine is
@@ -101,14 +298,215 @@ function runChildren(snapshot: unknown): RunChild[] {
101
298
  return out;
102
299
  }
103
300
 
301
+ /**
302
+ * One Workspace of a run waiting for capacity (ADR-0064): its Sandbox has no node. `child` is the
303
+ * Workspace's actor path below the run root — the {@link RunChild} ids down to it, joined by `/`,
304
+ * and `""` when the run's root IS the Workspace. `on` is what it waits for: a node from the
305
+ * scheduler, or room under the cluster owner's quota; `message` is the scheduler's or the API
306
+ * server's own words; `since` (ISO) is when the wait began, kept across a changed reason and a
307
+ * restart.
308
+ */
309
+ export type RunWaiting = { child: string; on: "node" | "quota"; message: string; since: string };
310
+
311
+ /**
312
+ * Every Workspace in `placing` with a reason, at any depth, read off the snapshot tree in the two
313
+ * shapes {@link runChildren} walks — the wait rides each Workspace's own persisted context, so a
314
+ * restored run reports it before its watch has said a word. A live child is held to being a
315
+ * `workspace()` by its logic; a persisted one has none, and the state key and the context's shape
316
+ * are the test. Only an ACTIVE run and an active Workspace wait: a run that ended — faulted,
317
+ * cancelled, done — has deleted what never ran, so a `placing` left in its snapshot waits for nothing.
318
+ */
319
+ function runWaiting(snapshot: unknown, logic?: unknown, path: string[] = []): RunWaiting[] {
320
+ const out: RunWaiting[] = [];
321
+ const snap = snapshot as { status?: unknown; value?: unknown; context?: Record<string, unknown> } | undefined;
322
+ if (snap?.status !== undefined && snap.status !== "active") return out;
323
+ const wait = snap?.context?.[WORKSPACE_WAIT_KEY] as WorkspaceWait | undefined;
324
+ const isWorkspace = logic == null || composesSandbox(logic as AnyStateMachine);
325
+ if (isWorkspace && snap?.value === "placing" && (wait?.on === "node" || wait?.on === "quota")) {
326
+ out.push({ child: path.join("/"), on: wait.on, message: String(wait.message), since: String(wait.since) });
327
+ }
328
+ const children = (snapshot as { children?: Record<string, unknown> } | undefined)?.children ?? {};
329
+ for (const [id, entry] of Object.entries(children)) {
330
+ const child = entry as { snapshot?: unknown; getSnapshot?: () => unknown; logic?: unknown };
331
+ const live = typeof child.getSnapshot === "function";
332
+ const childSnap = live ? child.getSnapshot!() : child.snapshot;
333
+ if ((childSnap as { value?: unknown } | undefined)?.value === undefined) continue;
334
+ out.push(...runWaiting(childSnap, live ? child.logic : undefined, [...path, id]));
335
+ }
336
+ return out;
337
+ }
338
+
339
+ /**
340
+ * The Sandbox of every live Workspace under `actor`, at any depth, by CR name — what a cancel
341
+ * destroys (ADR-0025). Read off the run's own live actor tree: each Workspace holds its `wsId` in
342
+ * context, and the name is a function of the run and that id, the same one its `teardown` deletes.
343
+ * Correct after a restart by construction: `restore()` rebuilds this tree from the persisted
344
+ * snapshot, so a restored run names its Sandboxes before its watch has listed. The Sandbox watch's
345
+ * set, filtered by the run's label, is not: before the watch has listed it holds nothing, and
346
+ * unknown is never absent (ADR-0063).
347
+ *
348
+ * Left out: a Workspace in `placing`, which deletes its own Sandbox when its wait is aborted — and
349
+ * only once any write in flight has settled, so a delete from here could race the apply and leave
350
+ * the CR behind (ADR-0064); and one that is `done` or ended, whose `teardown` or `failing` already
351
+ * ran. An actor that ended in `error` is not a Workspace here, but its children are read: a failed
352
+ * run's root (ADR-0073) is in `error`, and its active children are still in its snapshot, as
353
+ * `stopTree` reads them.
354
+ *
355
+ * With `placing`, a Workspace in `placing` is also named: the names that a live Workspace holds
356
+ * now, which a delete must leave alone.
357
+ */
358
+ function runSandboxes(actor: AnyActorRef, runId: string, opts: { placing?: boolean } = {}): string[] {
359
+ const out: string[] = [];
360
+ const snap = actor.getSnapshot() as
361
+ | { status?: string; value?: unknown; context?: { wsId?: unknown }; children?: Record<string, AnyActorRef> }
362
+ | undefined;
363
+ if (snap?.status !== "active" && snap?.status !== "error") return out;
364
+ const logic = (actor as AnyActorRef & { logic?: unknown }).logic;
365
+ if (
366
+ snap.status === "active" &&
367
+ logic != null &&
368
+ composesSandbox(logic as AnyStateMachine) &&
369
+ (opts.placing === true || snap.value !== "placing") &&
370
+ typeof snap.context?.wsId === "string"
371
+ ) {
372
+ out.push(workspaceName(runId, snap.context.wsId));
373
+ }
374
+ for (const child of Object.values(snap.children ?? {})) out.push(...runSandboxes(child, runId, opts));
375
+ return out;
376
+ }
377
+
378
+ /**
379
+ * Stop every actor under `actor`, at any depth, deepest first. A machine that ended in `error` has
380
+ * stopped processing — its own stop is a no-op — so its children are reached through its snapshot
381
+ * rather than asked to stop through it.
382
+ */
383
+ function stopTree(actor: AnyActorRef): void {
384
+ const children = (actor.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
385
+ for (const child of Object.values(children)) {
386
+ stopTree(child);
387
+ // `_stop` is what xstate's own `stopChild` calls: a child cannot be stopped through `stop()`.
388
+ (child as AnyActorRef & { _stop?: () => void })._stop?.();
389
+ }
390
+ }
391
+
392
+ /** One outstanding pick, as `RunStatus.held` reports it: the Agent slot, its pick, and the ISO time
393
+ * it was delivered. */
394
+ export type RunHeld = { agent: string; pick: string; since: string };
395
+
396
+ /** The pick being delivered now (ADR-0024): its registration, every reply from below its Agent's
397
+ * state inside its own send, and what an `answer` action set there (ADR-0067). */
398
+ type Sending = {
399
+ reg: Registration;
400
+ pick: string;
401
+ chain: PickAnswer[];
402
+ shaped?: { value: unknown };
403
+ };
404
+
405
+ /**
406
+ * Whose pick started a child (ADR-0067): the Agent's slot, and the invoke id of the Agent's state the
407
+ * pick was made at, which tells one slot's two regions apart. `invoke` is absent when the event that
408
+ * started the child named the Agent and not where (a pick taken outside its own send).
409
+ */
410
+ type StartedBy = { agent: string; invoke?: string };
411
+
412
+ /**
413
+ * The key of the child `childId` of `parent` in the run's record of which Agent started each child
414
+ * (`startedBy`, ADR-0067): the invoke ids from below the root down to `parent`, then `childId`, each
415
+ * encoded and joined by `/`. An invoke id is local to its Machine — two Machines of one tree, and
416
+ * every worker of a pool, may each have a child `job` — and the chain is not. A child of the root
417
+ * keeps its bare id. The ids are the snapshot's, so a restore gives the same key.
418
+ */
419
+ function childKey(parent: AnyActorRef, childId: string): string {
420
+ const ids = [childId];
421
+ for (let at: AnyActorRef | undefined = parent; at?._parent !== undefined; at = at._parent) ids.unshift(at.id);
422
+ return ids.map(encodeURIComponent).join("/");
423
+ }
424
+
425
+ /**
426
+ * The Agent whose event `event` is, as `taker` takes it (ADR-0024, ADR-0067): a pick's sender, or
427
+ * the Agent whose pick started the child of `taker` whose done or error event it is (`startedBy`).
428
+ * Undefined when the event names no Agent: a timer, a Gate's answer, an outside send, a child no
429
+ * Agent's pick started. `sending`'s invoke is where a pick in its own send was made.
430
+ */
431
+ function agentOfEvent(
432
+ event: AnyEventObject,
433
+ startedBy: Record<string, StartedBy>,
434
+ taker: AnyActorRef,
435
+ sending?: Sending,
436
+ ): StartedBy | undefined {
437
+ const sender = (event as { [SENDER_KEY]?: Sender })[SENDER_KEY];
438
+ if (sender?.party === "agent") {
439
+ const invoke = sending?.reg.sender?.id === sender.id ? sending.reg.invokeId : undefined;
440
+ return invoke !== undefined ? { agent: sender.id, invoke } : { agent: sender.id };
441
+ }
442
+ const child = /^xstate\.(?:done|error)\.actor\.(.+)$/.exec(event.type)?.[1];
443
+ if (child === undefined) return undefined;
444
+ const key = childKey(taker, child);
445
+ return Object.hasOwn(startedBy, key) ? startedBy[key] : undefined;
446
+ }
447
+
448
+ /** How many of `snapshot`'s active states invoke an Agent. With two or more, an event that names no
449
+ * Agent cannot say whose pick an `answer` below no Agent's state shapes (ADR-0024). */
450
+ function activeAgentStates(snapshot: AnyMachineSnapshot): number {
451
+ const machine = snapshot.machine as AnyStateMachine;
452
+ const active = (snapshot as { _nodes?: AnyStateMachine["root"][] })._nodes ?? [];
453
+ return active.filter((node) => partiesAt(node, machine).some((party) => party.party === "agent")).length;
454
+ }
455
+
456
+ /** The `childKey` of every child in `actor`'s tree, at any depth. */
457
+ function liveChildKeys(actor: AnyActorRef, into = new Set<string>()): Set<string> {
458
+ const children = (actor.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
459
+ for (const child of Object.values(children)) {
460
+ if (!child) continue;
461
+ into.add(childKey(actor, child.id));
462
+ liveChildKeys(child, into);
463
+ }
464
+ return into;
465
+ }
466
+
467
+ /** `startedBy` as the save keeps it: only children still in the run's tree (ADR-0067). Read at the
468
+ * save, a microtask after the snapshot, so a done event the Machine took has run its route. */
469
+ function liveStartedBy(startedBy: Record<string, StartedBy>, actor: AnyActorRef): Record<string, StartedBy> {
470
+ const live = liveChildKeys(actor);
471
+ for (const id of Object.keys(startedBy)) if (!live.has(id)) delete startedBy[id];
472
+ return startedBy;
473
+ }
474
+
475
+ /** An `answer` action's value as the hold keeps it: JSON, because it rides the run blob and the
476
+ * wire (ADR-0067). `undefined`, or a value with no JSON form, sets no answer — the default stands. */
477
+ function jsonOf(value: unknown): { value: unknown } | undefined {
478
+ try {
479
+ const text = JSON.stringify(value);
480
+ return text === undefined ? undefined : { value: JSON.parse(text) as unknown };
481
+ } catch {
482
+ return undefined;
483
+ }
484
+ }
485
+
486
+ /** The outstanding picks among a run's holds. */
487
+ function heldOf(holds: Iterable<Hold>): RunHeld[] {
488
+ return [...holds]
489
+ .filter((hold) => hold.answer === undefined)
490
+ .map((hold) => ({ agent: hold.agent, pick: hold.pick, since: hold.since }));
491
+ }
492
+
104
493
  /** A run's current observable state. `fault` carries the error message when status is "error"
105
- * (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). */
494
+ * (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). A run that ends
495
+ * "failed" (ADR-0073) has no `fault`: its `output` is the Failure's `{ reason, detail? }`. */
106
496
  export type RunStatus = RunRecord & {
107
497
  status: string;
108
498
  value: unknown;
109
499
  context: unknown;
110
500
  /** The live child machines beneath the root — where most of a run actually is. */
111
501
  children: RunChild[];
502
+ /** Every Workspace waiting for capacity, and why (ADR-0064). Behind the token: the reason is
503
+ * the scheduler's or the quota's own words, which can name nodes and taints, so {@link observe}
504
+ * never carries it — the open band sees only that a Workspace is in `placing`. */
505
+ waiting: RunWaiting[];
506
+ /** Every pick of this run that is outstanding (ADR-0024): the Agent, its pick, and since when —
507
+ * the Console's "held since". There is no default time bound on a held Turn; this is how a
508
+ * human sees one. Behind the token, like `context`: a pick names what the Agent did. */
509
+ held: RunHeld[];
112
510
  fault?: string;
113
511
  /**
114
512
  * Why the HOST set this status, for the statuses the Machine did not choose — today `drifted`
@@ -117,8 +515,33 @@ export type RunStatus = RunRecord & {
117
515
  * route could read it.
118
516
  */
119
517
  reason?: string;
518
+ /** The hash of the Assembly the run started under (ADR-0069). Absent for a run no Assembly
519
+ * started. */
520
+ assembly?: string;
521
+ /** The run binds at least one Chart (ADR-0070): its Machine is the bound one, `runMachine`. */
522
+ bound?: true;
523
+ /** The hashes of the Charts the run binds now, distinct and sorted (ADR-0070). A run can bind a
524
+ * second Chart in its life (a planner whose first Chart reaches a ceiling picks again), so these,
525
+ * not the run id, say which bound Machine `runMachine` draws. Absent when it binds none. */
526
+ chartHashes?: string[];
527
+ /**
528
+ * The Machine's output, for a run that is no longer live (ADR-0071): what a `dispatch` with
529
+ * `wait` settles with, and what `GET /runs/<id>` serves its foreign caller. Never for a live run,
530
+ * whose Machine has none yet. For a `failed` run, the Failure's `{ reason, detail? }` (ADR-0073),
531
+ * not a value of the Machine's output type. Behind the token, like `context`: it is the
532
+ * workflow's data.
533
+ */
534
+ output?: unknown;
535
+ /** The runs this run's `dispatch` Effects started, oldest first (ADR-0071): the chain's other
536
+ * direction, read off the caller's own ledger rows. A foreign dispatch's run is a run of the
537
+ * Orchestrator its target names. Absent when it started none. */
538
+ dispatched?: Dispatched[];
120
539
  };
121
540
 
541
+ /** One run a `dispatch` Effect of this run started (ADR-0071): its slot, the run, and when the
542
+ * ledger row was written. */
543
+ export type Dispatched = { slot: string; run: string; at: string };
544
+
122
545
  /**
123
546
  * A run as an UNAUTHENTICATED observer may see it: which run, of what workflow, and where in the
124
547
  * Machine it is. That is the whole set — enough to light up a state in the Console, and nothing
@@ -142,6 +565,19 @@ export type RunObservation = {
142
565
  status: string;
143
566
  value: unknown;
144
567
  children: RunChild[];
568
+ /** The run binds a Chart (ADR-0070): the Console draws `GET /workflows/:name/runs/:runId/machine`.
569
+ * Structure, like `value`. */
570
+ bound?: true;
571
+ /** The hashes of the Charts it binds now (see {@link RunStatus.chartHashes}). Structure: a hash
572
+ * of a Chart's text, which `runMachine` already serves on the open band. */
573
+ chartHashes?: string[];
574
+ /** The run that dispatched this one, its slot, and the chain's depth (ADR-0071). Routing — two
575
+ * ids, a slot name and a number — the class of thing `runId` and `value` are, so ADR-0014's line
576
+ * is unmoved. The idempotency key is not: it names the caller's ledger, which is the caller's. */
577
+ dispatchedBy?: Omit<DispatchedBy, "effect">;
578
+ /** The runs it dispatched, by slot (ADR-0071): the same routing as `dispatchedBy`, the other
579
+ * direction. */
580
+ dispatched?: Array<{ slot: string; run: string }>;
145
581
  };
146
582
 
147
583
  /** Project a full status down to what an observer may see. The one place the line is drawn. */
@@ -152,9 +588,39 @@ export function observe(status: RunStatus): RunObservation {
152
588
  status: status.status,
153
589
  value: status.value,
154
590
  children: status.children,
591
+ ...(status.bound ? { bound: true as const } : {}),
592
+ ...(status.chartHashes ? { chartHashes: status.chartHashes } : {}),
593
+ ...(status.dispatchedBy ? { dispatchedBy: routingOf(status.dispatchedBy) } : {}),
594
+ ...(status.dispatched ? { dispatched: status.dispatched.map(({ slot, run }) => ({ slot, run })) } : {}),
155
595
  };
156
596
  }
157
597
 
598
+ /** A {@link DispatchedBy} without its idempotency key (ADR-0071): what the open band carries. */
599
+ function routingOf(by: DispatchedBy): Omit<DispatchedBy, "effect"> {
600
+ const { effect: _key, ...routing } = by;
601
+ return routing;
602
+ }
603
+
604
+ /** An Agent registration's Menu (ADR-0029): the one it registered, or — registered with none, by a
605
+ * plain `setup()` machine — its derived set. */
606
+ function menuOf(reg: Registration): Map<string, EventDef> {
607
+ return reg.menu ?? reg.defs;
608
+ }
609
+
610
+ /** The invoking Machine's state value, as one comparable string; undefined when it cannot be read. */
611
+ function stateValueOf(invoker: AnyActorRef | undefined): string | undefined {
612
+ try {
613
+ return invoker ? JSON.stringify(invoker.getSnapshot().value) : undefined;
614
+ } catch {
615
+ return undefined;
616
+ }
617
+ }
618
+
619
+ /** The Allowed picks (ADR-0029), as the Agent's nudge and wake read them too. */
620
+ function allowedOf(reg: Registration): string[] {
621
+ return allowedPicks(reg);
622
+ }
623
+
158
624
  /** One open gate as external callers discover it (`GET /runs/:id` — ADR-0011): the accepted
159
625
  * events with their input schemas as JSON Schema (what drives a form or a `jr2 send` prompt),
160
626
  * plus the workflow-supplied `meta` (what a UI renders and a webhook translator matches on). */
@@ -165,15 +631,35 @@ export type GateView = {
165
631
  * id was authored ("F-12") or derived. Never parse `gate` for this. */
166
632
  path: string[];
167
633
  accepts: Array<{ name: string; description?: string; input: unknown }>;
634
+ /** The names of `accepts` the run would take from this Gate NOW, payload-blind (ADR-0011,
635
+ * ADR-0029): a send of another name is refused 409. `accepts` stays whole — it drives the forms. */
636
+ acceptsNow: string[];
168
637
  meta?: Record<string, unknown>;
169
638
  };
170
639
 
640
+ /** One event on an Agent's Menu, as the surface serves it: a tool, its `input` schema the tool's
641
+ * parameters. `semantics` rides along so the Harness refuses a reserved one (ADR-0013). */
642
+ /** One pick on the surface. `briefing` marks a pick whose parameter is a Chart: jr2's text on how a
643
+ * Chart runs (`CHART_BRIEFING`, ADR-0070), which the Harness says in the Turn part when the pick is
644
+ * Allowed, so only the Turns that may write a Chart carry it. */
645
+ export type SurfaceEvent = {
646
+ name: string;
647
+ description?: string;
648
+ input: unknown;
649
+ semantics: EventSemantics;
650
+ briefing?: string;
651
+ };
652
+
171
653
  /**
172
654
  * One live agent surface, as the Harness reads it through its Custodian (`GET /agents/:iid/surface`
173
- * — ADR-0013). The Harness presents it to its model as the Menu: each accepted event becomes a
174
- * tool, its `input` schema the tool's parameters. `semantics` rides along so the Harness can tell
175
- * an awaiting tool from a fire-and-forget one — the room ADR-0006's deferred results will land in,
176
- * unbuilt today.
655
+ * — ADR-0013, ADR-0029). Two answers, because they change at different rates:
656
+ *
657
+ * - `menu` is the Agent's MENU — every pick any state invoking this Agent offers, in a fixed
658
+ * order. The Harness presents it as the tools block, which is then byte-identical for every
659
+ * Turn of the conversation and never breaks the provider's prompt cache.
660
+ * - `allowed` is this Turn's ALLOWED PICKS — the invoking state's derived set, narrowed to what its
661
+ * guards would accept now, in Menu order. The Harness states it in the Briefing's Turn part,
662
+ * after the Frame's prompt (ADR-0062). Enforced at delivery, not here.
177
663
  *
178
664
  * `sandbox` is the Sandbox that may deliver here. It is not a secret from that pod (the pod IS the
179
665
  * sandbox), and serving it lets a caller fail loudly on a surface that is not its own.
@@ -182,7 +668,8 @@ export type AgentSurfaceView = {
182
668
  instanceId: string;
183
669
  runId: string;
184
670
  sandbox?: string;
185
- accepts: Array<{ name: string; description?: string; input: unknown; semantics: EventSemantics }>;
671
+ menu: SurfaceEvent[];
672
+ allowed: string[];
186
673
  };
187
674
 
188
675
  /**
@@ -195,10 +682,30 @@ export type AgentSurfaceView = {
195
682
  * turn has stopped waiting. It fails conservatively — anything that made the read unreliable reads
196
683
  * `false`, which is today's behavior, never a false claim.
197
684
  *
198
- * `deliveryId` is unchanged from ADR-0013: an outcome stays ADDRESSABLE after the fact, the room a
199
- * deferred result needs when it lands. Nothing polls it today, deliberately.
685
+ * `deliveryId` keeps the outcome ADDRESSABLE after the fact (ADR-0013): a pick the workflow is still
686
+ * running when the POST answers is OUTSTANDING (`outstanding: true`), and its answer is read at
687
+ * `GET /agents/:iid/deliveries/:id` — the same shape, `answered: true`, once the workflow is back at
688
+ * Rest in the Agent's state or has left it (ADR-0024).
200
689
  */
201
690
  export type AgentDeliveryReceipt = {
691
+ /** The workflow is running the pick: the answer comes on the delivery's poll. `allowed` is empty:
692
+ * no other pick is taken until it answers. */
693
+ outstanding?: true;
694
+ /** ISO time the pick was delivered — with `outstanding`. */
695
+ since?: string;
696
+ /** This receipt is a pick's answer, read at Rest (or once the Turn left the Agent's state): an outstanding
697
+ * pick's, a reply inside the pick's own send, or a guard straight back with no reply. */
698
+ answered?: true;
699
+ /** With `answered`: a guard straight back (ADR-0024) — the pick entered a state below the Agent's
700
+ * and came back inside its own send, nothing replying. The one answer that says "moved; nothing
701
+ * ran"; an outstanding pick's answer never carries it. */
702
+ straightBack?: true;
703
+ /** The last reply since the pick (ADR-0024); absent with `answered`: no reply came. */
704
+ answer?: PickAnswer;
705
+ /** With `answered` and `turnComplete`: something other than the pick's route left the Agent's
706
+ * state while the pick's work ran, and that exit stopped the work before it finished. `answer`,
707
+ * if any, is an earlier link's reply, kept (ADR-0024). */
708
+ stopped?: true;
202
709
  delivered: true;
203
710
  /** The event delivered — the Agent reads its own pick back, by name. */
204
711
  event: string;
@@ -213,6 +720,12 @@ export type AgentDeliveryReceipt = {
213
720
  moved: boolean;
214
721
  /** The invoking state stopped waiting: this Agent's turn is over. */
215
722
  turnComplete: boolean;
723
+ /**
724
+ * The Allowed picks read AFTER the delivery (ADR-0029): what this Turn may pick now — narrowed by
725
+ * a pick that moved the Machine within the invoking state, empty once the Turn is over. The
726
+ * Harness renders it ("Allowed now: `finish`"); it does not pre-check against it.
727
+ */
728
+ allowed: string[];
216
729
  deliveryId: string;
217
730
  };
218
731
 
@@ -227,7 +740,7 @@ export type AgentDeliveryReceipt = {
227
740
  export type RunFeedEvent =
228
741
  | { kind: "status"; status: RunStatus }
229
742
  | { kind: "emit"; event: { type: string } & Record<string, unknown> }
230
- | TurnMarker
743
+ | RunMarker
231
744
  // Absorbed-retry telemetry (ADR-0016): `{ child, attempt }` is state-key-class data — no iids
232
745
  // ride the feed (ADR-0014). `reason` is mechanism text, guarded like `fault`.
233
746
  | RetryTelemetry
@@ -235,9 +748,10 @@ export type RunFeedEvent =
235
748
 
236
749
  /**
237
750
  * The feed-so-far cap. The buffer exists so a Workspace attaching mid-run can open its log with
238
- * the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST frames, which
239
- * is the same trade kubelet's log rotation already makes on the printed side — the feed remains
240
- * the record, the buffer serves a courtesy view.
751
+ * the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST markers and
752
+ * Emits, which is the same trade kubelet's log rotation already makes on the printed side — the
753
+ * feed remains the record, the buffer serves a courtesy view. Statuses do not count against it:
754
+ * the buffer keeps only the latest (a status is a level — ADR-0023).
241
755
  */
242
756
  const FEED_SO_FAR_CAP = 1000;
243
757
 
@@ -265,6 +779,51 @@ function echoEventOf(event: RunFeedEvent, target: string): EchoEvent | undefined
265
779
  return undefined;
266
780
  }
267
781
 
782
+ /**
783
+ * The echo's outbox (ADR-0023): one push in flight per attachment, and whatever the feed produces
784
+ * meanwhile rides the NEXT push, together — a burst of transitions costs the Harness one push per
785
+ * flush, not one per transition.
786
+ *
787
+ * A status is a level, so a pending status is REPLACED by a newer one, never queued behind it. The
788
+ * ordering rule, exactly: markers and Emits keep feed order among themselves, and the one pending
789
+ * status sits where the NEWEST status arrived — after every marker and Emit fed before it, before
790
+ * every one fed after it. A superseded status drops out; nothing else moves. So a Harness that
791
+ * answers slowly holds at most one status plus the markers since its last push.
792
+ *
793
+ * Fire-and-forget: a failed push goes to `report` and the outbox drains on — nothing here can
794
+ * fail a turn, a state, or a run. Exported for its unit test; {@link RunHost} is its one caller.
795
+ */
796
+ export function echoOutbox(
797
+ push: (events: EchoEvent[]) => Promise<void>,
798
+ report: (err: unknown) => void,
799
+ ): (events: EchoEvent[]) => void {
800
+ let pending: EchoEvent[] = [];
801
+ let flushing = false;
802
+ const flush = async (): Promise<void> => {
803
+ while (pending.length > 0) {
804
+ const batch = pending;
805
+ pending = [];
806
+ try {
807
+ await push(batch);
808
+ } catch (err) {
809
+ report(err);
810
+ }
811
+ }
812
+ flushing = false;
813
+ };
814
+ return (events) => {
815
+ for (const event of events) {
816
+ if (event.kind === "status") pending = pending.filter((p) => p.kind !== "status");
817
+ pending.push(event);
818
+ }
819
+ if (flushing || pending.length === 0) return;
820
+ flushing = true;
821
+ // A microtask, so a synchronous burst (the replay; one transition's Emits and status) leaves
822
+ // as one push rather than as its first event alone.
823
+ queueMicrotask(() => void flush());
824
+ };
825
+ }
826
+
268
827
  /** The child-machine tree for the echo's status: spawn ids and state values alone — a root's
269
828
  * value says almost nothing about where a run is, and a {@link RunChild} is already context-free
270
829
  * by construction. `src`/`status` stay behind: they are join keys for the Console, not story. */
@@ -305,6 +864,19 @@ export type RunHostOptions = {
305
864
  reconcile?: (run: RunRecord) => boolean | Promise<boolean>;
306
865
  /** Injectable id generator (deterministic ids in tests). Default: `crypto.randomUUID`. */
307
866
  newId?: () => string;
867
+ /**
868
+ * Resolve a stamped blob's Workflow inside the Assembly it names (ADR-0069): the Machine that
869
+ * Assembly's `workflows/<name>.ts` exports, as a def stamped with that hash, or why it cannot be
870
+ * read (`drifted`: not on the volume, published for another Kit version, does not import). The
871
+ * host never learns the volume layout; `startInstance` supplies this over `assemblies/<hash>`.
872
+ * A def it returns is NOT registered: the registry is the current Assembly's alone. A throw is a
873
+ * fault, not a verdict: the row stays live for the next boot (`failed`).
874
+ *
875
+ * Absent (an in-process host): a stamped blob resolves to the registered def of its name only
876
+ * when that def came from the same Assembly hash, since one hash names one tree and so one
877
+ * Machine; otherwise the run is `drifted`.
878
+ */
879
+ resolveDef?: (blob: { workflow: string; assembly: string }) => Promise<WorkflowDef | { drifted: string }>;
308
880
  /** Report a run that threw during restore (ADR-0030). The row is left `live` for the next boot,
309
881
  * so this is the only account of why — the entrypoint logs it. Default: silent. */
310
882
  onRestoreError?: (runId: string, err: unknown) => void;
@@ -313,10 +885,23 @@ export type RunHostOptions = {
313
885
  sandbox?: SandboxPort;
314
886
  /** The Instance Harness base URL (ADR-0031) — where `workspace: "none"` Turns are admitted. */
315
887
  instanceHarness?: string;
888
+ /** The name this Orchestrator gives itself at another Orchestrator's door, as a foreign
889
+ * dispatch's `dispatchedBy.instance` (ADR-0071): stable and not secret. Deployed, its namespace. */
890
+ orchestratorName?: string;
316
891
  /** The Harness bearer for a placement (ADR-0058) — `startInstance` closes `harnessToken` over the
317
892
  * signing key. Handed to every run's binding, and to the echo pusher below. Absent, no Harness
318
893
  * call bears one: only the stub Harness answers such a host. */
319
894
  harnessBearer?: (placement: string) => string;
895
+ /** Where an Effector placement is, and its key (ADR-0068) — `startInstance` derives both, the key
896
+ * from the signing key. Handed to every run's binding; absent, an http Effect faults pointedly. */
897
+ effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
898
+ /**
899
+ * Why a listed named Chart cannot run now, or undefined (ADR-0070): the registry of named Charts
900
+ * keeps the unrunnable ones, never the host. An in-process `dispatch` of one is refused 409 with
901
+ * this reason, as its start through the door is (ADR-0071). Absent, such a name answers 404:
902
+ * the host registers only the runnable ones.
903
+ */
904
+ unrunnableChart?: (name: string) => string | undefined;
320
905
  /**
321
906
  * Build the run-narrative echo pusher for one Workspace's Harness (ADR-0023) — the seam a fake
322
907
  * echo server rides in tests; `startInstance` binds the real wire push (harness-client.ts).
@@ -325,19 +910,38 @@ export type RunHostOptions = {
325
910
  * never a dependency of the run.
326
911
  */
327
912
  echo?: (endpoint: string, bearer: string | undefined) => (events: EchoEvent[]) => Promise<void>;
913
+ /**
914
+ * Build the live-set statement to the Instance Harness (ADR-0031) — the seam a fake rides in
915
+ * tests; `startInstance` binds the real wire push (harness-client.ts). `bearer` is the Instance
916
+ * Harness's (ADR-0058). Absent, or with no `instanceHarness`, nothing is stated.
917
+ */
918
+ liveSet?: (endpoint: string, bearer: string | undefined) => (live: LiveConversation[]) => Promise<unknown>;
919
+ /** How often the live set is stated, before jitter. Default: the Lease's cadence (ADR-0021).
920
+ * Tests shrink it. */
921
+ liveSetIntervalMs?: number;
922
+ /** The wall clock a run's timer table is written and read on (ADR-0007, timers.ts): each pending
923
+ * timer's deadline is a time on it. Default: `Date.now`. Tests pass a clock that does not step. */
924
+ now?: () => number;
328
925
  };
329
926
 
330
927
  /** What we persist per run: the machine snapshot wrapped with the run metadata restore needs.
331
- * `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent actor
928
+ * `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent's Actor
332
929
  * through the run binding and saved in the same blob (same store, same atomicity). */
333
930
  type RunBlob = {
334
931
  workflow: string;
335
932
  instanceId: string;
933
+ /** The run that dispatched this one (ADR-0071), its key included: the store finds a run by its
934
+ * key here (`findByDispatch`), so a re-sent start answers the run it already made. */
935
+ dispatchedBy?: DispatchedBy;
336
936
  /** The shape of the Machine that WROTE this snapshot (ADR-0030). `workflow` says which def to
337
937
  * look up; this says whether the def found there is still the one this snapshot can be read by.
338
938
  * Absent means written before the stamp existed, which restore treats as drift — the point is to
339
939
  * never interpret a snapshot whose Machine cannot be vouched for. */
340
940
  machine?: string;
941
+ /** The hash of the Assembly the run started under (ADR-0069), stamped on every save. A blob
942
+ * without one cannot be vouched for, so restore treats it as drift, as it treats a blob with no
943
+ * `machine` stamp. */
944
+ assembly?: string;
341
945
  snapshot: unknown;
342
946
  agents?: Record<string, AgentAdmission>;
343
947
  /**
@@ -353,7 +957,51 @@ type RunBlob = {
353
957
  * `<id>/<epoch>`), so one conversation has exactly one counter however often it burns.
354
958
  */
355
959
  epochs?: Record<string, number>;
960
+ /**
961
+ * The pending NOTICES (ADR-0062): what the next Turn in a scope must be told — a memory kill in
962
+ * its Workspace, a conversation that is new. The Orchestrator is their only keeper, so they ride
963
+ * this blob beside the two ledgers, for the same reasons: the run's exact lifetime, and the
964
+ * snapshot's atomicity. A notice leaves it in the save that ledgers the admission that carried
965
+ * it, so a restart before that save delivers it again and a restart after does not.
966
+ */
967
+ notices?: NoticeLedgerState;
968
+ /**
969
+ * The pending TIMERS (`after`, named delays, delayed `raise` and `sendTo`), each with its
970
+ * absolute deadline (ADR-0007, timers.ts). xstate 5 keeps them on the actor system, not in the
971
+ * snapshot, so a restore without this table loses every one. Same blob for the snapshot's
972
+ * atomicity: the timers saved are exactly the ones the saved states scheduled.
973
+ */
974
+ timers?: PendingTimer[];
975
+ /**
976
+ * The outstanding picks (ADR-0024, holds.ts): each conversation's latest pick that the workflow ran, its
977
+ * delivery id, the last reply since it, and its answer once read at Rest. Same blob for the
978
+ * snapshot's atomicity: a restored host answers the same poll from the same Machine.
979
+ */
980
+ holds?: Hold[];
981
+ /**
982
+ * The Effect LEDGER (ADR-0067): one row per Effect, by its id — `intent` before the call,
983
+ * `settled` with its output, `aborted` when its state exited while it ran. Same blob for the
984
+ * snapshot's atomicity: an intent row is saved with the snapshot that holds its Effect, so a restore
985
+ * reads which Effects may have acted and runs none of them twice. A row is never dropped: the
986
+ * settled and aborted rows keep id, kind and outcome, the record of what the run's Effects did; a
987
+ * settled row's output tail is dropped once a save carries its transition (`dropSavedTails`).
988
+ */
989
+ effects?: Record<string, EffectRow>;
990
+ /**
991
+ * Which Agent started each live child (ADR-0067), by the child's key — the invoke ids from below
992
+ * the root down to the child, encoded and joined by `/` (`childKey`), so a child of one Machine is
993
+ * never another Machine's child of the same local id: the Agent whose pick's
994
+ * own send started it, or whose event the Machine was taking when it started. A done event of that
995
+ * child names that Agent, so an `answer` on a transition below no Agent's state that the done event
996
+ * takes shapes that Agent's pick and no other — whether the pick that started it is still
997
+ * outstanding or not. Same blob, so a restore names the same Agent. A child no longer in the tree
998
+ * is dropped at the save.
999
+ */
1000
+ startedBy?: Record<string, StartedBy>;
356
1001
  fault?: string;
1002
+ /** The Failure that ended the run `failed` (ADR-0073), as plain data: the run's output. xstate
1003
+ * keeps the error of an errored snapshot, and serializes it to `{}`, so it is kept here. */
1004
+ failure?: FailureInput;
357
1005
  };
358
1006
 
359
1007
  type LiveRun = {
@@ -367,15 +1015,29 @@ type LiveRun = {
367
1015
  agents: Record<string, AgentAdmission>;
368
1016
  /** The live epoch ledger (ADR-0057): persisted as `RunBlob.epochs`, seeded on restore. */
369
1017
  epochs: Record<string, number>;
1018
+ /** The pending notices (ADR-0062): persisted as `RunBlob.notices`, seeded on restore. */
1019
+ notices: NoticeLedger;
1020
+ /** The outstanding picks by Agent address (ADR-0024): persisted as `RunBlob.holds`, seeded on restore. */
1021
+ holds: Map<string, Hold>;
1022
+ /** The Effect ledger (ADR-0067): persisted as `RunBlob.effects`, seeded on restore. */
1023
+ effects: Record<string, EffectRow>;
1024
+ /** Which Agent started each live child (ADR-0067): persisted as `RunBlob.startedBy`, seeded on restore. */
1025
+ startedBy: Record<string, StartedBy>;
370
1026
  /** The error that killed the run, if it errored (xstate serializes Error to `{}`, so the
371
1027
  * message is captured here at the observer and persisted onto the blob for `read`). */
372
1028
  fault?: string;
1029
+ /** The Failure that ended the run, if it failed (ADR-0073): persisted as `RunBlob.failure`. */
1030
+ failure?: FailureInput;
373
1031
  /** Per-run observers fed by `persist()` (status) and the actor's `emit` (emit) — SSE/CLI watch. */
374
1032
  listeners: Set<(e: RunFeedEvent) => void>;
375
1033
  /** The run's feed-so-far (ADR-0023): what a Workspace attaching mid-run gets replayed as its
376
- * log's preamble. In-memory and this-boot only — a restored run's preamble starts at restore,
377
- * the same live-only contract the printed log already has. Capped ({@link FEED_SO_FAR_CAP}). */
1034
+ * log's preamble — its markers and Emits, in feed order. In-memory and this-boot only — a
1035
+ * restored run's preamble starts at restore, the same live-only contract the printed log already
1036
+ * has. Capped ({@link FEED_SO_FAR_CAP}). */
378
1037
  feedSoFar: RunFeedEvent[];
1038
+ /** The preamble's last line: where the run stands now. A status is a level (ADR-0023), so the
1039
+ * buffer keeps the latest alone — old statuses are not interleaved between old markers. */
1040
+ latestStatus?: RunFeedEvent & { kind: "status" };
379
1041
  };
380
1042
 
381
1043
  export class RunHost {
@@ -387,15 +1049,41 @@ export class RunHost {
387
1049
  private readonly reconcile: (run: RunRecord) => boolean | Promise<boolean>;
388
1050
  private readonly newId: () => string;
389
1051
  private readonly onRestoreError?: (runId: string, err: unknown) => void;
1052
+ private readonly resolveDef?: RunHostOptions["resolveDef"];
390
1053
  private readonly sandbox?: SandboxPort;
391
1054
  private readonly instanceHarness?: string;
1055
+ private readonly orchestratorName?: string;
392
1056
  private readonly harnessBearer?: (placement: string) => string;
1057
+ private readonly effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
1058
+ private readonly unrunnableChart?: (name: string) => string | undefined;
1059
+ /**
1060
+ * Run id → what waits for that run to settle (ADR-0071): a `dispatch` with `wait`. Keyed by the
1061
+ * run, not kept on its {@link LiveRun}, because a restored caller may attach before its callee is
1062
+ * hosted again (`store.live()` row order) or while the callee is not hosted at all.
1063
+ */
1064
+ private readonly settleListeners = new Map<string, Set<(status: RunStatus) => void>>();
1065
+ /** Idempotency key → the start in flight under it (ADR-0071): a second start with the same key
1066
+ * before the first has spawned its run answers the first's run, never a second. */
1067
+ private readonly dispatching = new Map<string, Promise<DispatchedRun>>();
393
1068
  private readonly echoFactory?: (
394
1069
  endpoint: string,
395
1070
  bearer: string | undefined,
396
1071
  ) => (events: EchoEvent[]) => Promise<void>;
1072
+ private readonly liveSetPush?: (live: LiveConversation[]) => Promise<unknown>;
1073
+ private readonly liveSetIntervalMs: number;
1074
+ private readonly now: () => number;
1075
+ private liveSetTimer?: ReturnType<typeof setTimeout>;
1076
+ /** Whether the last live-set statement failed — a failing streak logs one line (ADR-0031). */
1077
+ private liveSetFailing = false;
397
1078
  private readonly workflowDefs = new Map<string, WorkflowDef>();
398
1079
  private readonly runs = new Map<string, LiveRun>();
1080
+ /**
1081
+ * The admission ledgers of the runs restore could not host this boot (a `reconcile` that threw).
1082
+ * Their rows stay `live` for the next boot to retry, so their conversations on the Instance
1083
+ * Harness are still held: the live set names them from here (ADR-0031), or the retry would find
1084
+ * nothing to re-attach to.
1085
+ */
1086
+ private readonly unhosted = new Map<string, Record<string, AgentAdmission>>();
399
1087
  /**
400
1088
  * Workflow name → its observers. Deliberately on the HOST and not on `LiveRun`: a workflow's
401
1089
  * watcher outlives every individual run it watches, and `persist()` clears a settled run's own
@@ -403,16 +1091,60 @@ export class RunHost {
403
1091
  * than a comment asking the next reader not to clear the wrong Set.
404
1092
  */
405
1093
  private readonly workflowListeners = new Map<string, Set<WorkflowListener>>();
1094
+ /** Every timer a run's actor system has armed and not yet fired or cleared: an `after`, a Chart's
1095
+ * wall clock (ADR-0070). `close()` unrefs them, so a timer of hours never holds the process. */
1096
+ private readonly timers = new Set<ReturnType<typeof setTimeout>>();
1097
+ /** The clock every run's actor system arms its timers on: xstate's own, with each timer kept in
1098
+ * {@link timers} and unref'd once the host is closing. The scheduled events stay in the system's
1099
+ * snapshot, so a restore arms them again. */
1100
+ private readonly clock = {
1101
+ setTimeout: (fn: (...args: unknown[]) => void, ms: number) => {
1102
+ const timer = setTimeout(() => {
1103
+ this.timers.delete(timer);
1104
+ fn();
1105
+ }, ms);
1106
+ this.timers.add(timer);
1107
+ if (this.#stopping) (timer as { unref?: () => void }).unref?.();
1108
+ return timer;
1109
+ },
1110
+ clearTimeout: (timer: ReturnType<typeof setTimeout>) => {
1111
+ this.timers.delete(timer);
1112
+ clearTimeout(timer);
1113
+ },
1114
+ };
1115
+ /** The registration each hold was opened under, this boot: the same address re-registered by
1116
+ * another invocation (`continue: true`) is another Turn. A restored hold adopts the first. */
1117
+ private readonly holdRegs = new WeakMap<Hold, Registration>();
1118
+ /** The long polls waiting on each open hold (ADR-0024). */
1119
+ private readonly holdWaiters = new Map<Hold, Set<() => void>>();
1120
+ /** The pick being delivered now, and the last reply from below its Agent's state inside its own
1121
+ * send (ADR-0024). A child that settles in the same macrostep as the pick — a decision Machine
1122
+ * that is final as it starts — replies before a hold can exist, so the chain records it here. */
1123
+ private sending: Sending | undefined;
1124
+ /** `<iid> <deliveryId>` of every outstanding pick of a run this host parked (`stop`). */
1125
+ private readonly parked = new Set<string>();
1126
+ #stopping = false;
1127
+ #restorePending = false;
406
1128
 
407
1129
  constructor(opts: RunHostOptions) {
408
1130
  this.store = opts.store;
409
1131
  this.reconcile = opts.reconcile ?? (() => true);
410
1132
  this.newId = opts.newId ?? (() => randomUUID());
411
1133
  this.onRestoreError = opts.onRestoreError;
1134
+ this.resolveDef = opts.resolveDef;
412
1135
  this.sandbox = opts.sandbox;
413
1136
  this.instanceHarness = opts.instanceHarness;
1137
+ this.orchestratorName = opts.orchestratorName;
414
1138
  this.harnessBearer = opts.harnessBearer;
1139
+ this.effector = opts.effector;
1140
+ this.unrunnableChart = opts.unrunnableChart;
415
1141
  this.echoFactory = opts.echo;
1142
+ this.liveSetPush =
1143
+ opts.instanceHarness && opts.liveSet
1144
+ ? opts.liveSet(opts.instanceHarness, opts.harnessBearer?.(INSTANCE_HARNESS_SERVICE))
1145
+ : undefined;
1146
+ this.liveSetIntervalMs = opts.liveSetIntervalMs ?? DEFAULT_LEASE_INTERVAL_MS;
1147
+ this.now = opts.now ?? Date.now;
416
1148
  }
417
1149
 
418
1150
  /** Register a workflow so `start`/`restore` can run it. Re-registering replaces (dev reload).
@@ -420,7 +1152,56 @@ export class RunHost {
420
1152
  * declared them and resolved at invoke time off the invoking Machine (ADR-0011, ADR-0049), so
421
1153
  * a nested Machine's defs are never the root's — or this host's — to hold. */
422
1154
  register(def: WorkflowDef): void {
423
- this.workflowDefs.set(def.name, def);
1155
+ const wrapper = def.wrapper ?? wrapperOf(def.machine);
1156
+ this.workflowDefs.set(def.name, wrapper ? { ...def, wrapper } : def);
1157
+ }
1158
+
1159
+ /**
1160
+ * Register a named Chart as the Workflow `<wrapper>/<name>` (ADR-0070). The registry of named
1161
+ * Charts (charts.ts) calls this for each runnable one; it keeps where the Charts are stored, and
1162
+ * the unrunnable ones, itself. Replacing a Chart a live run holds is fine: the run holds its text.
1163
+ */
1164
+ registerChart(def: WorkflowDef & { chart: NonNullable<WorkflowDef["chart"]> }): void {
1165
+ this.register(def);
1166
+ }
1167
+
1168
+ /** Drop a named Chart's Workflow. A live run of it keeps its text and runs on. */
1169
+ unregisterChart(wrapper: string, name: string): void {
1170
+ const key = `${wrapper}/${name}`;
1171
+ if (this.workflowDefs.get(key)?.chart) this.workflowDefs.delete(key);
1172
+ }
1173
+
1174
+ /** The registered named Chart `<wrapper>/<name>`, or undefined. */
1175
+ namedChart(wrapper: string, name: string): NamedChart | undefined {
1176
+ const chart = this.workflowDefs.get(`${wrapper}/${name}`)?.chart;
1177
+ return (
1178
+ chart && { wrapper: chart.wrapper, name: chart.name, text: chart.text, hash: chart.hash, origin: chart.origin }
1179
+ );
1180
+ }
1181
+
1182
+ /**
1183
+ * Every registered Workflow with its origin (ADR-0070): a Machine, or a named Chart from a file or
1184
+ * at run time. A Chart wrapper's entry carries its Open body, whether its door takes a Chart, and
1185
+ * its named Charts: a named Chart lists inside its wrapper's entry, not beside it, and has an
1186
+ * entry of its own only when its wrapper is not registered.
1187
+ */
1188
+ workflowList(): WorkflowListing[] {
1189
+ const defs = [...this.workflowDefs.values()];
1190
+ const wrappers = new Set(defs.filter((d) => !d.chart && d.wrapper).map((d) => d.name));
1191
+ return defs.flatMap((def): WorkflowListing[] => {
1192
+ if (def.chart) return wrappers.has(def.chart.wrapper) ? [] : [{ name: def.name, origin: def.chart.origin }];
1193
+ if (!def.wrapper) return [{ name: def.name, origin: "machine" }];
1194
+ const charts = defs
1195
+ .filter((d) => d.chart?.wrapper === def.name)
1196
+ .map((d) => ({ name: d.chart!.name, origin: d.chart!.origin }));
1197
+ return [
1198
+ {
1199
+ name: def.name,
1200
+ origin: "machine",
1201
+ wrapper: { body: def.wrapper.body, oneShot: def.wrapper.chartFields.length > 0, charts },
1202
+ },
1203
+ ];
1204
+ });
424
1205
  }
425
1206
 
426
1207
  /** Drop a workflow's registration (a workflow the host no longer discovers). In-flight
@@ -438,12 +1219,79 @@ export class RunHost {
438
1219
  return [...this.workflowDefs.keys()];
439
1220
  }
440
1221
 
1222
+ /** The runs this process hosts, by the hash of the Assembly each started under (ADR-0069). A
1223
+ * run from a def with no Assembly holds none. The store's live rows are the hold
1224
+ * (`SnapshotStore.liveAssemblies`); this adds a run started a moment ago, before its first save. */
1225
+ assemblyHolds(): Map<string, string[]> {
1226
+ const holds = new Map<string, string[]>();
1227
+ for (const [runId, run] of this.runs) {
1228
+ const hash = run.def.assembly;
1229
+ if (hash === undefined) continue;
1230
+ holds.set(hash, [...(holds.get(hash) ?? []), runId]);
1231
+ }
1232
+ return holds;
1233
+ }
1234
+
441
1235
  /** The registered template Machine's serialized structure (`GET /workflows/:name/machine`). */
442
1236
  machine(name: string): MachineDoc | undefined {
443
1237
  const def = this.workflowDefs.get(name);
444
1238
  return def && serializeMachine(name, def.machine);
445
1239
  }
446
1240
 
1241
+ /**
1242
+ * The run's bound Machine (ADR-0070): the Workflow's Machine with each bound Open body's subgraph
1243
+ * the compiled Chart's (its `src` stays the slot key, so the live child joins it as before) —
1244
+ * from the live actor's bound Machine, or, for a stored run, compiled again from the text its
1245
+ * blob holds. Undefined for an unknown run, and for a stored run whose Workflow is not registered.
1246
+ */
1247
+ async runMachine(runId: string): Promise<MachineDoc | undefined> {
1248
+ const live = this.runs.get(runId);
1249
+ let workflow: string;
1250
+ let machine: AnyStateMachine;
1251
+ let bodies: Array<{ route: BoundChartRoute; compiled: AnyStateMachine | string }>;
1252
+ if (live) {
1253
+ workflow = live.record.workflow;
1254
+ machine = live.def.machine;
1255
+ const refs = new Map<string, AnyActorRef>();
1256
+ const collect = (ref: AnyActorRef, path: string[]): void => {
1257
+ for (const [id, child] of Object.entries(childrenOfRef(ref))) {
1258
+ refs.set([...path, id].join("/"), child);
1259
+ collect(child, [...path, id]);
1260
+ }
1261
+ };
1262
+ collect(live.actor, []);
1263
+ bodies = boundChartRoutesOf(live.actor.getSnapshot()).map((route) => {
1264
+ const ref = refs.get(route.path.join("/"));
1265
+ return { route, compiled: (ref && boundMachineOf(ref)) ?? "the live body holds no compiled Chart" };
1266
+ });
1267
+ } else {
1268
+ const stored = await this.store.load(runId);
1269
+ const blob = stored?.snapshot as RunBlob | null | undefined;
1270
+ if (!blob) return undefined;
1271
+ const def = this.workflowDefs.get(blob.workflow) ?? this.workflowDefs.get(wrapperName(blob.workflow));
1272
+ if (!def) return undefined;
1273
+ workflow = blob.workflow;
1274
+ machine = def.machine;
1275
+ bodies = boundChartRoutesOf(blob.snapshot).map((route) => ({
1276
+ route,
1277
+ compiled: compileStored(def.machine, route),
1278
+ }));
1279
+ }
1280
+ const doc = serializeMachine(workflow, machine);
1281
+ const notDrawn: NonNullable<MachineDoc["chartsNotDrawn"]> = [];
1282
+ for (const { route, compiled } of bodies) {
1283
+ if (typeof compiled === "string") {
1284
+ // The doc keeps the slot's own subgraph, and says so: a doc that drew the wrapper's body
1285
+ // with no word would disagree with the run's `bound` (ADR-0070, D-B).
1286
+ notDrawn.push({ path: route.path.join("/"), src: route.src, reason: compiled });
1287
+ continue;
1288
+ }
1289
+ const { workflow: _w, opaqueStates: _o, ...body } = serializeMachine(route.src, compiled);
1290
+ placeBody(doc, route.srcs, body);
1291
+ }
1292
+ return notDrawn.length ? { ...doc, chartsNotDrawn: notDrawn } : doc;
1293
+ }
1294
+
447
1295
  /**
448
1296
  * The workflow's declared run-input contract as JSON Schema (ADR-0033) — what the workflow
449
1297
  * detail serves and the Console's start form generates from. `null` for a machine that
@@ -453,16 +1301,124 @@ export class RunHost {
453
1301
  inputSchema(name: string): Record<string, unknown> | null | undefined {
454
1302
  const def = this.workflowDefs.get(name);
455
1303
  if (!def) return undefined;
456
- const schema = inputSchemaOf(def.machine);
1304
+ let schema = inputSchemaOf(def.machine);
1305
+ // A named Chart's door is its wrapper's minus the Chart fields, which the Chart supplies (ADR-0070).
1306
+ if (schema && def.chart && def.wrapper?.chartFields.length) {
1307
+ schema = schema.omit(Object.fromEntries(def.wrapper.chartFields.map((f) => [f, true])) as never);
1308
+ }
457
1309
  // `io: "input"`: this schema describes what a caller SENDS — a defaulted field is optional
458
1310
  // at the door (the default is applied by `start`'s parse), not required of the form.
459
1311
  return schema ? (z.toJSONSchema(schema, { io: "input" }) as Record<string, unknown>) : null;
460
1312
  }
461
1313
 
462
- /** Start a fresh run of a registered workflow; returns its durable ids. */
463
- async start(workflow: string, input: Record<string, unknown> = {}): Promise<{ runId: string; instanceId: string }> {
1314
+ /**
1315
+ * Start a fresh run of a registered workflow; returns its durable ids.
1316
+ *
1317
+ * With `dispatchedBy` (ADR-0071), the start is a `dispatch` Effect's, and its `effect` is the
1318
+ * idempotency key: a run that already carries the key is answered, `existing`, and none is
1319
+ * started. That is what lets a caller re-send a start whose answer it lost. The key is looked for
1320
+ * among the runs this process hosts first — a run's first save is a microtask after its spawn —
1321
+ * then in the store. A key on a row the weekly sweep removed (ADR-0065) starts a new run: known,
1322
+ * and not guarded. The depth is recorded as sent: the caller states the callee's.
1323
+ */
1324
+ async start(
1325
+ workflow: string,
1326
+ input: Record<string, unknown> = {},
1327
+ opts: { dispatchedBy?: DispatchedBy } = {},
1328
+ ): Promise<{ runId: string; instanceId: string; existing?: true }> {
1329
+ const by = opts.dispatchedBy;
1330
+ if (!by) return this.startFresh(workflow, input);
1331
+ const inFlight = this.dispatching.get(by.effect);
1332
+ if (inFlight) return { ...keyAnswer(await inFlight, workflow, by.effect), existing: true };
1333
+ let existing = false;
1334
+ const started = (async (): Promise<DispatchedRun> => {
1335
+ const found = await this.dispatched(by.effect);
1336
+ if (found) {
1337
+ existing = true;
1338
+ return found;
1339
+ }
1340
+ return { ...(await this.startFresh(workflow, input, by)), workflow };
1341
+ })();
1342
+ this.dispatching.set(by.effect, started);
1343
+ try {
1344
+ const ids = keyAnswer(await started, workflow, by.effect);
1345
+ return existing ? { ...ids, existing: true } : ids;
1346
+ } finally {
1347
+ this.dispatching.delete(by.effect);
1348
+ }
1349
+ }
1350
+
1351
+ /**
1352
+ * The run that carries the idempotency key `effect` (ADR-0071), or undefined: a hosted run first,
1353
+ * then a stored one, in any status.
1354
+ */
1355
+ async dispatched(effect: string): Promise<DispatchedRun | undefined> {
1356
+ for (const run of this.runs.values()) {
1357
+ if (run.record.dispatchedBy?.effect === effect) {
1358
+ return { runId: run.record.runId, instanceId: run.record.instanceId, workflow: run.record.workflow };
1359
+ }
1360
+ }
1361
+ const runId = await this.store.findByDispatch(effect);
1362
+ if (runId === undefined) return undefined;
1363
+ const blob = (await this.store.load(runId))?.snapshot as RunBlob | null | undefined;
1364
+ return blob ? { runId, instanceId: blob.instanceId, workflow: blob.workflow } : undefined;
1365
+ }
1366
+
1367
+ /**
1368
+ * The in-process start of a `dispatch` Effect (ADR-0071, the binding's `dispatch.start`): every
1369
+ * refusal the door would answer, with its status, as output — the body may have come from a
1370
+ * model, and the report must reach it through the Chain. Any other throw is a defect, and is the
1371
+ * invoke's error.
1372
+ */
1373
+ private async dispatchStart(
1374
+ workflow: string,
1375
+ input: Record<string, unknown>,
1376
+ by: DispatchedBy,
1377
+ ): Promise<DispatchStart> {
1378
+ // The key first: a re-sent start answers its run whatever the Assembly holds now.
1379
+ const found = await this.dispatched(by.effect);
1380
+ if (found && found.workflow !== workflow) {
1381
+ return { ok: false, status: 409, report: [keyConflict(found, workflow, by.effect)] };
1382
+ }
1383
+ if (found) return { ok: true, runId: found.runId, existing: true };
1384
+ const unrunnable = this.unrunnableChart?.(workflow);
1385
+ if (unrunnable !== undefined) {
1386
+ return { ok: false, status: 409, report: [`named Chart "${workflow}" cannot run: ${unrunnable}`] };
1387
+ }
1388
+ try {
1389
+ const started = await this.start(workflow, input, { dispatchedBy: by });
1390
+ return { ok: true, runId: started.runId, existing: started.existing === true };
1391
+ } catch (err) {
1392
+ if (err instanceof ChartRefusal) return { ok: false, status: 422, report: err.report };
1393
+ if (err instanceof EventValidationError) {
1394
+ return {
1395
+ ok: false,
1396
+ status: 400,
1397
+ report: [err.message, `the door of "${workflow}" takes ${JSON.stringify(this.inputSchema(workflow))}`],
1398
+ };
1399
+ }
1400
+ if (err instanceof UnknownWorkflowError) return { ok: false, status: 404, report: [err.message] };
1401
+ if (err instanceof DispatchKeyConflictError) return { ok: false, status: 409, report: [err.message] };
1402
+ throw err;
1403
+ }
1404
+ }
1405
+
1406
+ /** A fresh run: the door, the Chart admission, the spawn. Synchronous after its call, so the run
1407
+ * is hosted — and its key findable in {@link dispatched} — before anything else runs. */
1408
+ private async startFresh(
1409
+ workflow: string,
1410
+ input: Record<string, unknown>,
1411
+ dispatchedBy?: DispatchedBy,
1412
+ ): Promise<{ runId: string; instanceId: string }> {
464
1413
  const def = this.workflowDefs.get(workflow);
465
- if (!def) throw new Error(`no workflow registered as "${workflow}"`);
1414
+ if (!def) throw new UnknownWorkflowError(`no workflow registered as "${workflow}"`);
1415
+
1416
+ // A named Chart supplies every Chart field of its wrapper's door, and the caller the rest
1417
+ // (ADR-0070): what the caller sent under a Chart field is not the run's.
1418
+ if (def.chart && def.wrapper) {
1419
+ input = { ...input };
1420
+ for (const field of def.wrapper.chartFields) input[field] = def.chart.text;
1421
+ }
466
1422
 
467
1423
  // A declared schema is enforced at the door (ADR-0033), and what starts the run is the
468
1424
  // PARSED shape (defaults applied, unknown keys stripped) — exactly what a gate delivery
@@ -480,9 +1436,13 @@ export class RunHost {
480
1436
 
481
1437
  const runId = this.newId();
482
1438
  const instanceId = this.newId();
483
- const record: RunRecord = { runId, workflow, instanceId };
1439
+ const record: RunRecord = { runId, workflow, instanceId, ...(dispatchedBy ? { dispatchedBy } : {}) };
484
1440
 
485
1441
  const machine = this.assemble(def, instanceId);
1442
+ // Every Chart field of a Chart wrapper's door is admitted BEFORE the run exists (ADR-0070): a
1443
+ // refusal throws `ChartRefusal`, which the wire answers `422 { error: "refused", report }`. The
1444
+ // lent set is the run's own Machine's, with its providers in.
1445
+ this.admitDoorCharts(def, machine, runInput);
486
1446
  // The root machine gets the parsed door PLUS what the host injects beside it —
487
1447
  // `HostInjectedInput` (vocabulary.ts): the seed Instance ID, added after the parse because no
488
1448
  // caller sends it and nothing serves it (ADR-0033). Only the root: a machine invoked below is
@@ -493,6 +1453,46 @@ export class RunHost {
493
1453
  return { runId, instanceId };
494
1454
  }
495
1455
 
1456
+ /**
1457
+ * Admit every Chart field present in a Chart wrapper's parsed door (ADR-0070): the whole door,
1458
+ * the compile included, against each Open body (the door cannot tell which body a field feeds).
1459
+ * `input.*` references are the body's to check at its start: the Chart's `input` is what the
1460
+ * wrapper's invoke sends, which is not the door. Throws ChartRefusal.
1461
+ */
1462
+ private admitDoorCharts(def: WorkflowDef, machine: AnyStateMachine, input: Record<string, unknown>): void {
1463
+ for (const field of def.wrapper?.chartFields ?? []) {
1464
+ if (input[field] === undefined) continue;
1465
+ admitWrapperChart(machine, input[field]);
1466
+ }
1467
+ }
1468
+
1469
+ /**
1470
+ * The def that reads `blob` (ADR-0069), or the verdict when none can. A stamped blob resolves
1471
+ * inside the Assembly it names, through `resolveDef`; an unstamped blob resolves by name in the
1472
+ * registry, and only on a host that loads no Assembly. A blob that names no Workflow the
1473
+ * registry holds is `lost`: there is nothing it could ever be read by.
1474
+ */
1475
+ private async defFor(blob: RunBlob): Promise<WorkflowDef | { drifted: string } | { lost: string }> {
1476
+ const registered = this.workflowDefs.get(blob.workflow);
1477
+ if (blob.assembly !== undefined) {
1478
+ const ref = { workflow: blob.workflow, assembly: blob.assembly };
1479
+ if (this.resolveDef) {
1480
+ const found = await this.resolveDef(ref);
1481
+ return "drifted" in found ? found : { ...found, assembly: blob.assembly };
1482
+ }
1483
+ if (registered?.assembly === blob.assembly) return registered;
1484
+ return {
1485
+ drifted: `workflow "${blob.workflow}" saved under Assembly ${blob.assembly}, which this host does not hold`,
1486
+ };
1487
+ }
1488
+ // A blob that names no Assembly cannot be vouched for on a host that loads Assemblies:
1489
+ // greenfield, no migration (ADR-0069).
1490
+ if (this.resolveDef || registered?.assembly !== undefined) {
1491
+ return { drifted: `workflow "${blob.workflow}": saved under no Assembly` };
1492
+ }
1493
+ return registered ?? { lost: `no workflow "${blob.workflow}"` };
1494
+ }
1495
+
496
1496
  /**
497
1497
  * Restore every persisted live run: hydrate, reconcile against the live world, and either
498
1498
  * re-attach (re-spawn from the rewritten child input), mark lost, or refuse as drifted. Returns
@@ -504,8 +1504,10 @@ export class RunHost {
504
1504
  * later run in the store never restored at all. One run that cannot be resumed is one run, not an
505
1505
  * outage.
506
1506
  *
507
- * `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict — the
508
- * Machine changed, and it will still have changed on the next boot — so it is written to the row.
1507
+ * `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict — this
1508
+ * Orchestrator cannot read what it saved (the Assembly is gone, at another Kit version, does not
1509
+ * load, or its Machine's shape does not match), and the next boot will not read it either — so it
1510
+ * is written to the row.
509
1511
  * A throw is not: `reconcile` talks to a cluster, and a kubectl blip must not permanently condemn
510
1512
  * a run. Those rows are left `live` to be retried on the next boot, and reported every time until
511
1513
  * they stop failing.
@@ -516,35 +1518,89 @@ export class RunHost {
516
1518
  const drifted: string[] = [];
517
1519
  const failed: string[] = [];
518
1520
 
519
- for (const stored of await this.store.list()) {
520
- if (stored.status !== "live") continue;
1521
+ for (const stored of await this.store.live()) {
1522
+ // A run this process hosts already is not restored a second time. A deferred restore
1523
+ // (ADR-0069) that throws as a whole stays pending, and the next publish tries it again; a run
1524
+ // started under the current Assembly between those two publishes has a live row and a live
1525
+ // actor, and must not get a second actor.
1526
+ if (this.runs.has(stored.runId)) continue;
521
1527
  const blob = stored.snapshot as RunBlob | null;
522
- const def = blob ? this.workflowDefs.get(blob.workflow) : undefined;
523
- if (!blob || !def) {
524
- await this.store.markLost(stored.runId, blob ? `no workflow "${blob.workflow}"` : "empty snapshot");
1528
+ if (!blob) {
1529
+ await this.store.markLost(stored.runId, "empty snapshot");
1530
+ this.settledUnhosted(stored.runId, "lost", "empty snapshot");
525
1531
  lost.push(stored.runId);
526
1532
  continue;
527
1533
  }
528
1534
 
529
- // The workflow name resolved a def; this asks whether that def is still the Machine this
530
- // snapshot was written by (ADR-0030). Refused BEFORE `reconcile`, which talks to the cluster:
531
- // there is no point proving a Sandbox is alive for a run that cannot be read.
1535
+ // Which Machine reads this snapshot (ADR-0069). One rule: a stamped blob restores from its
1536
+ // Assembly; an unstamped blob restores from the registry; a stamped host never has unstamped
1537
+ // blobs it can vouch for. So a publish strands nothing: the def comes from the Assembly the
1538
+ // run started under, never from the current one. Drift is only "I can no longer read what I
1539
+ // saved", and it keeps the snapshot (`markDrifted`), so `jr2 status` still reads the run.
1540
+ let def: WorkflowDef;
1541
+ try {
1542
+ const found = await this.defFor(blob);
1543
+ if ("lost" in found) {
1544
+ await this.store.markLost(stored.runId, found.lost);
1545
+ this.settledUnhosted(stored.runId, "lost", found.lost, blob);
1546
+ lost.push(stored.runId);
1547
+ continue;
1548
+ }
1549
+ if ("drifted" in found) {
1550
+ await this.store.markDrifted(stored.runId, found.drifted);
1551
+ this.settledUnhosted(stored.runId, "drifted", found.drifted, blob);
1552
+ drifted.push(stored.runId);
1553
+ continue;
1554
+ }
1555
+ def = found;
1556
+ } catch (err) {
1557
+ // A fault while reading the Assembly, not a verdict on it: left live, as below.
1558
+ this.onRestoreError?.(stored.runId, err);
1559
+ failed.push(stored.runId);
1560
+ this.unhosted.set(stored.runId, blob.agents ?? {});
1561
+ continue;
1562
+ }
1563
+
1564
+ // The def is the Machine this snapshot names; this asks whether it is still the shape this
1565
+ // snapshot was written by (ADR-0030). Inside one Assembly the same bytes give the same
1566
+ // Machine, so this guards a hash collision or a hand-edited volume. Refused BEFORE
1567
+ // `reconcile`, which talks to the cluster: there is no point proving a Sandbox is alive for
1568
+ // a run that cannot be read.
532
1569
  const expected = fingerprintOf(def.machine);
533
1570
  if (blob.machine !== expected) {
534
- await this.store.markDrifted(
535
- stored.runId,
536
- `workflow "${blob.workflow}" changed shape since this run was saved ` +
537
- `(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`,
538
- );
1571
+ const reason =
1572
+ `workflow "${blob.workflow}"${def.assembly !== undefined ? ` in Assembly ${def.assembly}` : ""} changed shape since this run was saved ` +
1573
+ `(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`;
1574
+ await this.store.markDrifted(stored.runId, reason);
1575
+ this.settledUnhosted(stored.runId, "drifted", reason, blob);
1576
+ drifted.push(stored.runId);
1577
+ continue;
1578
+ }
1579
+
1580
+ // The Charts the run binds (ADR-0070, D-B): the shape stamp is the wrapper's, and each Chart's
1581
+ // identity is its text, kept in its body's snapshot. Each is admitted again against the Open
1582
+ // body it binds and the lent set the Machine has now; one that no longer admits is drift.
1583
+ const charts = chartDrift(def.machine, boundChartRoutesOf(blob.snapshot));
1584
+ if (charts !== undefined) {
1585
+ await this.store.markDrifted(stored.runId, charts);
1586
+ this.settledUnhosted(stored.runId, "drifted", charts, blob);
539
1587
  drifted.push(stored.runId);
540
1588
  continue;
541
1589
  }
542
1590
 
543
- const record: RunRecord = { runId: stored.runId, workflow: blob.workflow, instanceId: blob.instanceId };
1591
+ // The run's place in a chain of dispatches comes back with it (ADR-0071): its key keeps
1592
+ // answering a re-sent start, and its depth is the one it started at.
1593
+ const record: RunRecord = {
1594
+ runId: stored.runId,
1595
+ workflow: blob.workflow,
1596
+ instanceId: blob.instanceId,
1597
+ ...(blob.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
1598
+ };
544
1599
 
545
1600
  try {
546
1601
  if (!(await this.reconcile(record))) {
547
1602
  await this.store.markLost(stored.runId, "reconcile: live world absent");
1603
+ this.settledUnhosted(stored.runId, "lost", "reconcile: live world absent", blob);
548
1604
  lost.push(stored.runId);
549
1605
  continue;
550
1606
  }
@@ -558,37 +1614,124 @@ export class RunHost {
558
1614
  this.assemble(def, blob.instanceId),
559
1615
  { snapshot: hydrated as never },
560
1616
  record,
1617
+ // The def of the Assembly the run started under (ADR-0069), so every later save names
1618
+ // that hash. The config the run provisions under is the CURRENT Assembly's all the same:
1619
+ // the ports read it through the binding a publish replaces, and a held Assembly lends
1620
+ // only its Machines (ADR-0069).
561
1621
  def,
562
1622
  agents,
563
1623
  // The burned conversations come back with the admissions (ADR-0057): a restored run's
564
1624
  // next `continue` must land where this run's last one did, not on a dead conversation.
565
1625
  blob.epochs ?? {},
1626
+ // …and so do the undelivered notices (ADR-0062), reservations forgotten: a notice whose
1627
+ // admission never reached the ledger is delivered again.
1628
+ blob.notices,
1629
+ // …and the outstanding picks (ADR-0024): the restored Machine says when each is answered.
1630
+ blob.holds,
1631
+ // …and the Effect ledger (ADR-0067): a restored Effect reads its row and never runs twice.
1632
+ blob.effects ?? {},
1633
+ // …and which Agent started each child (ADR-0067): its done event names the same Agent.
1634
+ blob.startedBy ?? {},
566
1635
  );
567
- actor.start();
1636
+ // `start()`, then the pending timers again (ADR-0007, see `startAndRearm`). The next save
1637
+ // records them again.
1638
+ startAndRearm(actor, blob.timers ?? [], this.now());
1639
+ // A run an earlier restore left live to retry is hosted now.
1640
+ this.unhosted.delete(stored.runId);
568
1641
  reattached.push(stored.runId);
569
1642
  } catch (err) {
570
1643
  // Left `live` on purpose — see the note above. The row is unchanged, so the next boot tries
571
1644
  // again; what must not happen is this taking the remaining runs down with it.
572
1645
  this.onRestoreError?.(stored.runId, err);
573
1646
  failed.push(stored.runId);
1647
+ this.unhosted.set(stored.runId, blob.agents ?? {});
574
1648
  }
575
1649
  }
576
1650
 
1651
+ // Every run this boot will host is live now, so the set is whole: state it, and keep stating it
1652
+ // (ADR-0031). Not before — a statement ahead of restore would free the conversations of the
1653
+ // runs about to be re-attached.
1654
+ this.stateLiveSet();
1655
+ this.scheduleLiveSet();
1656
+
1657
+ this.#restorePending = false;
577
1658
  return { reattached, lost, drifted, failed };
578
1659
  }
579
1660
 
1661
+ /**
1662
+ * The conversations the live runs hold on the Instance Harness (ADR-0031): every admission in
1663
+ * their ledgers whose endpoint is the Instance Harness, as the `(agent, instanceId)` pair it runs
1664
+ * under. From MEMORY, never the store: the store also holds runs this process does not host, and
1665
+ * a run this process hosts is exactly one that may still continue its conversation — and so is a
1666
+ * run restore left `live` to retry (`unhosted`). A Sandbox's Harness is not in it — its
1667
+ * Workspace's teardown frees everything there.
1668
+ */
1669
+ liveConversations(): LiveConversation[] {
1670
+ const live = new Map<string, LiveConversation>();
1671
+ const ledgers = [...[...this.runs.values()].map((run) => run.agents), ...this.unhosted.values()];
1672
+ for (const agents of ledgers) {
1673
+ for (const [iid, admission] of Object.entries(agents)) {
1674
+ if (admission.endpoint !== this.instanceHarness || admission.agent === undefined) continue;
1675
+ const instanceId = admission.instanceId ?? iid;
1676
+ live.set(JSON.stringify([admission.agent, instanceId]), { agent: admission.agent, instanceId });
1677
+ }
1678
+ }
1679
+ return [...live.values()];
1680
+ }
1681
+
1682
+ /**
1683
+ * State the live set to the Instance Harness (ADR-0031), which frees every other conversation.
1684
+ * FIRE-AND-FORGET: level-triggered and idempotent, so a missed statement costs nothing and the
1685
+ * next one says it all again. A failing streak logs one line; nothing here can touch a run.
1686
+ */
1687
+ private stateLiveSet(): void {
1688
+ const push = this.liveSetPush;
1689
+ if (!push || this.#stopping) return;
1690
+ push(this.liveConversations()).then(
1691
+ () => {
1692
+ this.liveSetFailing = false;
1693
+ },
1694
+ (err: unknown) => {
1695
+ if (this.liveSetFailing) return;
1696
+ this.liveSetFailing = true;
1697
+ console.error(
1698
+ `live-set statement to ${this.instanceHarness} failed (the Instance Harness keeps what it holds): ` +
1699
+ (err instanceof Error ? err.message : String(err)),
1700
+ );
1701
+ },
1702
+ );
1703
+ }
1704
+
1705
+ /** Every Lease interval, ±20% (ADR-0021's cadence, which ADR-0031 names). The timer holds no
1706
+ * process alive, and `close()` clears it. */
1707
+ private scheduleLiveSet(): void {
1708
+ if (!this.liveSetPush || this.liveSetTimer !== undefined || this.#stopping) return;
1709
+ const next = () => {
1710
+ this.liveSetTimer = setTimeout(
1711
+ () => {
1712
+ if (this.#stopping) return;
1713
+ this.stateLiveSet();
1714
+ next();
1715
+ },
1716
+ leaseDelay(this.liveSetIntervalMs, "next", Math.random()),
1717
+ );
1718
+ this.liveSetTimer.unref?.();
1719
+ };
1720
+ next();
1721
+ }
1722
+
580
1723
  /**
581
1724
  * An agent instance's LIVE surface (`GET /agents/:iid/surface` — ADR-0013): what the state that
582
1725
  * invoked this Agent accepts, right now. Undefined once nothing is registered (the state exited,
583
1726
  * the run settled, the iid is unknown) — the one catch point, and the reason the Harness never
584
1727
  * has to learn which turn is live: it asks, per turn, and the answer IS the turn.
585
1728
  *
586
- * "Right now" is load-bearing (ADR-0029): the registered defs are the state's VOCABULARY, derived
587
- * statically from its transitions, and the guards on those transitions are asked here — so an
588
- * event the Machine cannot currently accept is not offered. The pick has not happened yet, so the
589
- * question is `mayMove`, not `wouldMove`: a guard that would have judged the Agent's arguments is
590
- * left on the menu and settled at delivery. The Harness reads this afresh per Submission, so the filter lands at turn boundaries and never moves under
591
- * an Agent mid-turn.
1729
+ * "Right now" is the Allowed picks' half (ADR-0029): the registered defs are the state's
1730
+ * VOCABULARY, derived statically from its transitions, and the guards on those transitions are
1731
+ * asked here, as this Agent's sender (`allowedPicks`, ADR-0015). The pick has not happened yet, so
1732
+ * the question is payload-blind: a guard that would have judged the Agent's arguments is left
1733
+ * allowed and settled at delivery. The Menu does not move with them — it is the conversation's,
1734
+ * fixed at the Machine's build.
592
1735
  */
593
1736
  agentSurface(instanceId: string): AgentSurfaceView | undefined {
594
1737
  const reg = this.table.lookup(agentAddress(instanceId));
@@ -597,30 +1740,79 @@ export class RunHost {
597
1740
  instanceId,
598
1741
  runId: reg.runId,
599
1742
  sandbox: reg.sandbox,
600
- accepts: [...reg.defs.values()]
601
- .filter((def) => mayMove(reg.invoker, def.name))
602
- .map((def) => ({
603
- name: def.name,
604
- description: def.description,
605
- input: z.toJSONSchema(def.input),
606
- semantics: def.semantics,
607
- })),
1743
+ menu: [...menuOf(reg).values()].map((def) => ({
1744
+ name: def.name,
1745
+ description: def.description,
1746
+ input: z.toJSONSchema(def.input),
1747
+ semantics: def.semantics,
1748
+ ...briefingOf(def.input),
1749
+ })),
1750
+ allowed: allowedOf(reg),
608
1751
  };
609
1752
  }
610
1753
 
1754
+ /** The Harness read this Turn's surface and was answered (ADR-0026): the notices its admission
1755
+ * carried reach a model now, so an end to the Turn no longer raises them again (ADR-0062). */
1756
+ agentSurfaceServed(instanceId: string): void {
1757
+ this.table.lookup(agentAddress(instanceId))?.served?.();
1758
+ }
1759
+
611
1760
  /**
612
1761
  * Deliver one event from an Agent's Harness (`POST /agents/:iid/events` — ADR-0013). Validation
613
1762
  * and delivery are the table's; this only agent-scopes the address and mints the receipt. (A
614
- * `deferred` event never reaches here: `jr2Setup` refuses to build a Machine that declares one —
1763
+ * `poll` event never reaches here: `jr2Setup` refuses to build a Machine that declares one —
615
1764
  * ADR-0013.)
616
1765
  */
617
1766
  sendToAgent(instanceId: string, event: { type?: unknown } & Record<string, unknown>): AgentDeliveryReceipt {
618
1767
  const { type, ...payload } = event;
619
1768
  if (typeof type !== "string" || !type) {
620
- throw new EventValidationError(`event body must carry a string "type" (one of the surface's accepted names)`);
1769
+ throw new EventValidationError(`event body must carry a string "type" (one of the surface's Allowed picks)`);
621
1770
  }
1771
+ refuseSenderField(payload);
622
1772
  const address = agentAddress(instanceId);
623
1773
  const invoking = this.table.lookup(address);
1774
+ const run = invoking ? this.runs.get(invoking.runId) : undefined;
1775
+ // One pick at a time per Turn (ADR-0024): while a pick is outstanding, or the Agent's state is
1776
+ // busy with work it started in, every pick is refused — `finish` included, and before any other
1777
+ // judgment. Decided from the Machine (the Rest rule) and the run's holds, both of which a
1778
+ // restore brings back, so it holds after a restart. xstate itself would take the pick, and one
1779
+ // the Agent's state handles would exit the child state and stop the work running there.
1780
+ if (invoking && run) {
1781
+ this.settleHolds(run);
1782
+ const hold = run.holds.get(instanceId);
1783
+ const out = hold?.answer === undefined ? hold?.pick : undefined;
1784
+ if (out !== undefined || restOf(invoking) === "busy") {
1785
+ throw new OutstandingError(
1786
+ out !== undefined
1787
+ ? `"${type}" was not delivered: the pick "${out}" is still running in the workflow, and no other pick is taken until it answers`
1788
+ : `"${type}" was not delivered: the workflow is still running work in the state that asked for this Turn, and no pick is taken until it is back at rest`,
1789
+ out,
1790
+ );
1791
+ }
1792
+ }
1793
+ if (invoking && !invoking.defs.has(type)) {
1794
+ // A pick on the Menu that another state of this Agent handles (ADR-0029): answered, not
1795
+ // refused — `moved: false` is the correction receipt, whatever its arguments, so the payload
1796
+ // is not read. It never reaches the Machine: the invoking state does not handle it, so it
1797
+ // could move nothing of this Turn's, and only a state the Agent does not own (a nested
1798
+ // Agent's, a Gate's) could act on it — a widening the audience rule forbids (ADR-0015).
1799
+ if (menuOf(invoking).has(type)) {
1800
+ return {
1801
+ delivered: true,
1802
+ event: type,
1803
+ moved: false,
1804
+ turnComplete: false,
1805
+ allowed: allowedOf(invoking),
1806
+ deliveryId: this.newId(),
1807
+ };
1808
+ }
1809
+ // A name off the Menu is refused HERE, in the Agent's own terms: the table's refusal lists the
1810
+ // derived set, bare and unnarrowed, where the Harness is about to write the Allowed picks in
1811
+ // the model's tool names (ADR-0029, ADR-0062) — two lists that disagree.
1812
+ const refused = new EventValidationError(`the workflow's current state does not accept "${type}"`);
1813
+ refused.allowed = allowedOf(invoking);
1814
+ throw refused;
1815
+ }
624
1816
 
625
1817
  // Ask the guard question BEFORE delivering, and ask it with the VALIDATED payload — this is the
626
1818
  // exact form of the check the surface build can only approximate payload-blind (ADR-0029).
@@ -630,9 +1822,45 @@ export class RunHost {
630
1822
  // makes this unreliable and `deliver` throws on the next line anyway — so it fails open.
631
1823
  const def = invoking?.defs.get(type);
632
1824
  const parsed = def?.input.safeParse(payload ?? {});
633
- const moved = wouldMove(invoking?.invoker, parsed?.success ? { type, ...parsed.data } : { type, ...payload });
634
-
635
- this.table.deliver(address, type, payload);
1825
+ const probe = {
1826
+ ...(parsed?.success ? { type, ...parsed.data } : { type, ...payload }),
1827
+ ...(invoking?.sender ? { [SENDER_KEY]: invoking.sender } : {}),
1828
+ };
1829
+ const moved = wouldMove(invoking?.invoker, probe);
1830
+ // The states the pick names as targets, and the state value it leaves: after the send, the same
1831
+ // value with a target no longer active means it entered a state and came straight back
1832
+ // (ADR-0024); a new value, an in-Turn mode move.
1833
+ const targets = moved ? targetsOf(invoking?.invoker, probe) : [];
1834
+ const before = targets.length > 0 ? stateValueOf(invoking?.invoker) : undefined;
1835
+
1836
+ const outer = this.sending;
1837
+ const sending: Sending | undefined = invoking && { reg: invoking, pick: type, chain: [] };
1838
+ this.sending = sending;
1839
+ try {
1840
+ this.table.deliver(address, type, payload);
1841
+ } catch (err) {
1842
+ // The writing xstate would run for this pick is another party's (ADR-0015, ADR-0029): the
1843
+ // pick is ANSWERED, never delivered — `moved: false`, and the Harness picks the receipt by
1844
+ // whether the name is in `allowed`. Not in it: the correction receipt. In it: a guard that
1845
+ // read the payload turned the selection outward, and the receipt says to change the arguments.
1846
+ if (err instanceof NotTakenError && invoking) {
1847
+ return {
1848
+ delivered: true,
1849
+ event: type,
1850
+ moved: false,
1851
+ turnComplete: false,
1852
+ allowed: allowedOf(invoking),
1853
+ deliveryId: this.newId(),
1854
+ };
1855
+ }
1856
+ // A refused pick names the Allowed picks, read after the refusal (ADR-0029): one refusal
1857
+ // naming them is what brings a wrong pick back — the statement in the Briefing carries the
1858
+ // everyday case, and this is the net under it.
1859
+ if (err instanceof EventValidationError && invoking) err.allowed = allowedOf(invoking);
1860
+ throw err;
1861
+ } finally {
1862
+ this.sending = outer;
1863
+ }
636
1864
  // Read AFTER the delivery, off the SAME table the ADR-0024 guarantee uses — so the receipt
637
1865
  // reports what happened rather than what was hoped. `deliver` reached the invoking state's
638
1866
  // `sendBack` synchronously, so a pick that moved the Machine out of that state has already
@@ -640,15 +1868,371 @@ export class RunHost {
640
1868
  //
641
1869
  // IDENTITY, not existence: under `continue: true` the next state re-registers the SAME
642
1870
  // address for its own turn, and that is a new turn — this one still ended.
1871
+ const turnComplete = this.table.lookup(address) !== invoking;
1872
+ // A reply from below the Agent's state inside the pick's own send (ADR-0024): a child that
1873
+ // settled in the same macrostep. It is the answer's default as any later reply would be.
1874
+ const replied = moved && sending ? answerOf(sending) : undefined;
1875
+ // Where the answer goes if the route leaves the Agent's state: the conversation the picking
1876
+ // Turn continues. A fresh Turn has none — its conversation is never heard again, and a later
1877
+ // continued Turn of the same slot never made this pick.
1878
+ const conversation = invoking?.conversation;
1879
+ // The pick moved the workflow to work inside the Agent's state (ADR-0024): it is OUTSTANDING,
1880
+ // answered once the workflow is back at Rest there — read after the mailbox drains, by the
1881
+ // persist that follows every transition (`settleHolds`). The POST answers now, with the delivery
1882
+ // id the Harness polls; no single request waits on the work (ADR-0013).
1883
+ if (!turnComplete && invoking && run && moved && restOf(invoking) === "busy") {
1884
+ const deliveryId = this.newId();
1885
+ const since = new Date(this.now()).toISOString();
1886
+ const hold: Hold = {
1887
+ instanceId,
1888
+ deliveryId,
1889
+ pick: type,
1890
+ agent: invoking.sender?.id ?? "",
1891
+ since,
1892
+ ...(invoking.sandbox !== undefined ? { sandbox: invoking.sandbox } : {}),
1893
+ ...(invoking.invokeId !== undefined ? { invokeId: invoking.invokeId } : {}),
1894
+ ...(conversation !== undefined ? { conversation } : {}),
1895
+ ...(sending && sending.chain.length > 0 ? { chain: [...sending.chain] } : {}),
1896
+ ...(sending?.shaped !== undefined ? { shaped: sending.shaped } : {}),
1897
+ };
1898
+ this.closeHold(run.holds.get(instanceId));
1899
+ run.holds.set(instanceId, hold);
1900
+ this.holdRegs.set(hold, invoking);
1901
+ this.persist(run);
1902
+ return {
1903
+ delivered: true,
1904
+ event: type,
1905
+ moved: true,
1906
+ turnComplete: false,
1907
+ allowed: [],
1908
+ deliveryId,
1909
+ outstanding: true,
1910
+ since,
1911
+ };
1912
+ }
1913
+ // The child replied and the route ran inside the send: the pick is answered on its own POST,
1914
+ // as its poll would have answered it — back at Rest with the Allowed picks read after the
1915
+ // route, or out of the Agent's state with the answer as a notice to the conversation the picking
1916
+ // Turn continued (none for a fresh Turn).
1917
+ if (replied !== undefined && invoking) {
1918
+ if (turnComplete && conversation !== undefined && run) {
1919
+ run.notices.raise(
1920
+ { kind: "pick-answer", scope: "conversation", pick: type, cut: false, answer: replied },
1921
+ conversation,
1922
+ );
1923
+ }
1924
+ return {
1925
+ delivered: true,
1926
+ event: type,
1927
+ moved: true,
1928
+ turnComplete,
1929
+ allowed: turnComplete ? [] : allowedOf(invoking),
1930
+ deliveryId: this.newId(),
1931
+ answered: true,
1932
+ answer: replied,
1933
+ };
1934
+ }
1935
+ // It entered a state below the Agent's and came straight back inside its own send — a guard
1936
+ // back (an `always`), nothing replying: answered, "moved; nothing ran" (ADR-0024). A pick that
1937
+ // changed the state value moved to a mode, and one whose target is still active (the state it
1938
+ // was taken in) entered nothing it left: both get the plain receipt below.
1939
+ if (
1940
+ !turnComplete &&
1941
+ invoking &&
1942
+ before !== undefined &&
1943
+ stateValueOf(invoking.invoker) === before &&
1944
+ leftAgain(invoking.invoker, targets)
1945
+ ) {
1946
+ return {
1947
+ delivered: true,
1948
+ event: type,
1949
+ moved: true,
1950
+ turnComplete: false,
1951
+ allowed: allowedOf(invoking),
1952
+ deliveryId: this.newId(),
1953
+ answered: true,
1954
+ straightBack: true,
1955
+ };
1956
+ }
643
1957
  return {
644
1958
  delivered: true,
645
1959
  event: type,
646
1960
  moved,
647
- turnComplete: this.table.lookup(address) !== invoking,
1961
+ turnComplete,
1962
+ // Read off the SAME registration, after the delivery: a pick that moved the Machine within
1963
+ // the invoking state may have changed its guards. A Turn that is over may pick nothing.
1964
+ allowed: turnComplete || !invoking ? [] : allowedOf(invoking),
648
1965
  deliveryId: this.newId(),
649
1966
  };
650
1967
  }
651
1968
 
1969
+ /**
1970
+ * The registrations whose Agents' picks `event` comes from (ADR-0024, `cause.ts`): the one whose
1971
+ * pick it is, in its own send; and each one whose pick, in its own send or out, it answers — an
1972
+ * event that the Machine invoking that Agent takes from below the Agent's state, the sources of a
1973
+ * reply or a child's error (a nested Agent's pick, a Gate's answer, a child's output or error).
1974
+ */
1975
+ private pickCause(
1976
+ runId: string,
1977
+ holds: Map<string, Hold>,
1978
+ target: AnyActorRef,
1979
+ source: AnyActorRef | undefined,
1980
+ event: AnyEventObject,
1981
+ ): Set<Registration> | undefined {
1982
+ const out = new Set<Registration>();
1983
+ const sending = this.sending;
1984
+ if (sending && sending.reg.runId === runId && sending.reg.invoker === target && event.type === sending.pick) {
1985
+ out.add(sending.reg);
1986
+ }
1987
+ const picking: Registration[] = [];
1988
+ if (sending && sending.reg.runId === runId) picking.push(sending.reg);
1989
+ for (const hold of holds.values()) {
1990
+ if (hold.answer !== undefined) continue;
1991
+ const live = this.table.lookup(agentAddress(hold.instanceId));
1992
+ const reg = this.holdRegs.get(hold) ?? live;
1993
+ if (reg !== undefined && reg === live) picking.push(reg);
1994
+ }
1995
+ for (const reg of picking) {
1996
+ if (reg.invoker !== target || !reg.invokeId) continue;
1997
+ const seat = seatOf(target.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
1998
+ if (!seat) continue;
1999
+ if (
2000
+ replyOf(event, source, target, seat) !== undefined ||
2001
+ childErrorOf(event, source, target, seat) !== undefined
2002
+ ) {
2003
+ out.add(reg);
2004
+ }
2005
+ }
2006
+ return out.size > 0 ? out : undefined;
2007
+ }
2008
+
2009
+ /**
2010
+ * Answer every outstanding pick of `run` that can be answered now (ADR-0024). Called by the persist that
2011
+ * follows every transition — a microtask after the snapshot, so the mailbox has drained — and
2012
+ * before any read of a hold, so no reader sees a hold the Machine already answered.
2013
+ *
2014
+ * - The Agent's state is busy: still outstanding.
2015
+ * - Back at Rest in it: answered with the last reply since the pick and the Allowed picks read
2016
+ * AFTER the route — the guards read the context the route wrote (ADR-0029).
2017
+ * - It left the Agent's state (the registration is gone, or is another invocation's): the Turn
2018
+ * is over, and the answer goes as a notice to the next Turn of the conversation the picking
2019
+ * Turn continued; a fresh picking Turn's answer goes nowhere (ADR-0062).
2020
+ */
2021
+ private settleHolds(run: LiveRun): void {
2022
+ for (const hold of run.holds.values()) {
2023
+ if (hold.answer !== undefined) continue;
2024
+ const reg = this.table.lookup(agentAddress(hold.instanceId));
2025
+ let same = reg !== undefined && reg.runId === run.record.runId && reg.invokeId === hold.invokeId;
2026
+ if (same) {
2027
+ const opened = this.holdRegs.get(hold);
2028
+ if (opened === undefined) this.holdRegs.set(hold, reg!);
2029
+ else same = opened === reg;
2030
+ }
2031
+ const reading = same ? restOf(reg!) : "gone";
2032
+ if (reading === "busy") continue;
2033
+ // The answer's default is the last reply since the pick, whatever moved the Machine after it
2034
+ // (ADR-0024); with none, the answer carries no reply. Never `straightBack`: an outstanding
2035
+ // pick's work was running, so "nothing ran" is not this answer's to say.
2036
+ const answered = answerOf(hold);
2037
+ const answer = answered ? { answer: answered } : {};
2038
+ if (reading === "gone") {
2039
+ // Left by the route of the last reply — its own events and the events it sent itself at once
2040
+ // included — or by something else while work below still ran (a timer the Machine set, an
2041
+ // outside event): then the exit stopped that work, and the answer says so. It keeps the last
2042
+ // reply an earlier link gave.
2043
+ const stopped = hold.cutShort === true ? { stopped: true as const } : {};
2044
+ hold.answer = { turnComplete: true, allowed: [], ...answer, ...stopped };
2045
+ if (hold.conversation !== undefined) {
2046
+ run.notices.raise(
2047
+ {
2048
+ kind: "pick-answer",
2049
+ scope: "conversation",
2050
+ pick: hold.pick,
2051
+ cut: false,
2052
+ ...answer,
2053
+ ...stopped,
2054
+ },
2055
+ hold.conversation,
2056
+ );
2057
+ }
2058
+ } else {
2059
+ hold.answer = { turnComplete: false, allowed: allowedOf(reg!), ...answer };
2060
+ }
2061
+ // The answer is fixed: nothing reads the chain or the shaped value again, so the next save
2062
+ // drops them. A loop below the Agent's state grows the blob by its tails only while one pick
2063
+ // is outstanding (ADR-0067, ADR-0065).
2064
+ delete hold.chain;
2065
+ delete hold.shaped;
2066
+ this.closeHold(hold);
2067
+ }
2068
+ }
2069
+
2070
+ /**
2071
+ * The pick an `answer` action in `machine` shapes (ADR-0067), from `seat`, the one place it runs
2072
+ * from (`placeAnswers`). At an Agent's state: only a pick of the Agent seated there — the one in
2073
+ * its own send, else the one outstanding. On a transition below no Agent's state (`null`): the
2074
+ * event that took the transition names the Agent — the sender of a pick, or the Agent whose pick
2075
+ * started the child whose done or error event it is — and that Agent's pick is the one shaped, and
2076
+ * no other; none when that Agent has no pick being answered. When one slot is invoked in two
2077
+ * regions, the event names an invoke too, and only the pick at that invoke is shaped: the sending
2078
+ * pick, or the pick at the invoke whose pick started the child; none when that pick is not being
2079
+ * answered (its Turn ended, and its child runs on). An event that names no Agent (a timer,
2080
+ * a Gate's answer, an outside send) shapes the pick in its own send. Outside one, it shapes the one
2081
+ * outstanding pick only while one Agent's state is active; while two Agents' states are active it
2082
+ * shapes none, whatever is outstanding: the event cannot say whose pick it is, and the other
2083
+ * Agent's pick keeps its default answer.
2084
+ */
2085
+ private answeringIn(
2086
+ machine: AnyActorRef,
2087
+ seat: AnswerSeat,
2088
+ runId: string,
2089
+ holds: Map<string, Hold>,
2090
+ startedBy: Record<string, StartedBy>,
2091
+ event?: AnyEventObject,
2092
+ ): Sending | Hold | undefined {
2093
+ const seatedFor = (reg: Registration | undefined): boolean => {
2094
+ if (reg?.invoker !== machine) return false;
2095
+ if (seat === null) return true;
2096
+ const snapshot = machine.getSnapshot() as AnyMachineSnapshot;
2097
+ return reg.invokeId !== undefined && seatOf(snapshot, reg.invokeId) === seat;
2098
+ };
2099
+ const regOf = (hold: Hold): Registration | undefined =>
2100
+ this.holdRegs.get(hold) ?? this.table.lookup(agentAddress(hold.instanceId));
2101
+ const sending = this.sending && this.sending.reg.runId === runId ? this.sending : undefined;
2102
+ const open = [...holds.values()].filter((hold) => hold.answer === undefined && seatedFor(regOf(hold)));
2103
+ if (seat === null && event !== undefined) {
2104
+ const named = agentOfEvent(event, startedBy, machine, sending);
2105
+ if (named !== undefined) {
2106
+ const { agent } = named;
2107
+ const own = sending && sending.reg.sender?.id === agent ? sending : undefined;
2108
+ // That Agent's picks being answered: the one in its own send, and the ones outstanding at
2109
+ // its other invokes (one slot invoked in two regions). The event names the invoke of the
2110
+ // pick that sent it or started its child, and only the pick at that invoke is shaped; none
2111
+ // when that pick is not being answered, and each other pick keeps its default answer.
2112
+ const held = open.filter((hold) => hold.agent === agent && hold.instanceId !== own?.reg.id);
2113
+ const picks = [...(own ? [own] : []), ...held];
2114
+ const at = (pick: Sending | Hold): string | undefined =>
2115
+ pick === own ? own.reg.invokeId : (pick as Hold).invokeId;
2116
+ const same = named.invoke === undefined ? picks : picks.filter((pick) => at(pick) === named.invoke);
2117
+ if (same.length !== 1) return undefined;
2118
+ const pick = same[0]!;
2119
+ if (pick === own) return seatedFor(own.reg) ? own : undefined;
2120
+ return pick;
2121
+ }
2122
+ }
2123
+ if (sending && seatedFor(sending.reg)) return sending;
2124
+ if (seat === null && activeAgentStates(machine.getSnapshot() as AnyMachineSnapshot) > 1) return undefined;
2125
+ return open.length === 1 ? open[0] : undefined;
2126
+ }
2127
+
2128
+ /**
2129
+ * The Agent whose outstanding pick holds the child `childId` that `machine` just started, or
2130
+ * undefined: the child is invoked on a state strictly below that Agent's state, so by the Rest rule
2131
+ * it is that pick's work (ADR-0024). Below two such states (one Agent's state inside another's),
2132
+ * it is the work of the nearer one's pick. `snapshot` is `machine`'s snapshot after the step.
2133
+ */
2134
+ private outstandingAbove(
2135
+ machine: AnyActorRef,
2136
+ snapshot: AnyMachineSnapshot,
2137
+ childId: string,
2138
+ holds: Map<string, Hold>,
2139
+ ): StartedBy | undefined {
2140
+ const node = seatOf(snapshot, childId);
2141
+ if (node === undefined) return undefined;
2142
+ const byState = new Map<unknown, StartedBy>();
2143
+ for (const hold of holds.values()) {
2144
+ if (hold.answer !== undefined) continue;
2145
+ const reg = this.holdRegs.get(hold) ?? this.table.lookup(agentAddress(hold.instanceId));
2146
+ if (reg?.invoker !== machine || reg.invokeId === undefined) continue;
2147
+ const agentState = seatOf(snapshot, reg.invokeId);
2148
+ if (agentState !== undefined) byState.set(agentState, { agent: hold.agent, invoke: reg.invokeId });
2149
+ }
2150
+ // The nearest Agent's state above the child: with B's state inside A's, the child is B's work.
2151
+ for (let n = node.parent; n; n = n.parent) {
2152
+ const starter = byState.get(n);
2153
+ if (starter !== undefined) return starter;
2154
+ }
2155
+ return undefined;
2156
+ }
2157
+
2158
+ /** Wake every long poll on a hold: it was answered, replaced, or the host is going away. */
2159
+ private closeHold(hold: Hold | undefined): void {
2160
+ if (hold === undefined) return;
2161
+ const waiters = this.holdWaiters.get(hold);
2162
+ this.holdWaiters.delete(hold);
2163
+ for (const wake of waiters ?? []) wake();
2164
+ }
2165
+
2166
+ /** An outstanding pick's delivery: its run and hold, by Agent address and delivery id. */
2167
+ private holdAt(instanceId: string, deliveryId: string): { run: LiveRun; hold: Hold } | undefined {
2168
+ for (const run of this.runs.values()) {
2169
+ const hold = run.holds.get(instanceId);
2170
+ if (hold?.deliveryId === deliveryId) return { run, hold };
2171
+ }
2172
+ return undefined;
2173
+ }
2174
+
2175
+ /** Is this delivery an outstanding pick of a run this host parked? Its poll is asked again. */
2176
+ parkedDelivery(instanceId: string, deliveryId: string): boolean {
2177
+ return this.parked.has(`${instanceId} ${deliveryId}`);
2178
+ }
2179
+
2180
+ /** The Sandbox that may read a delivery's answer (ADR-0013), or undefined for an unknown one. */
2181
+ deliveryScope(instanceId: string, deliveryId: string): { sandbox?: string } | undefined {
2182
+ const found = this.holdAt(instanceId, deliveryId);
2183
+ return found && { ...(found.hold.sandbox !== undefined ? { sandbox: found.hold.sandbox } : {}) };
2184
+ }
2185
+
2186
+ /**
2187
+ * One long poll of an outstanding pick's answer (`GET /agents/:iid/deliveries/:id` — ADR-0013, ADR-0024):
2188
+ * the answer when the workflow is back at Rest in the Agent's state or has left it, else
2189
+ * `outstanding` after at most `waitMs`, and the Harness asks again. Undefined for a delivery this
2190
+ * host does not hold — a pick that was never outstanding, or a run that ended.
2191
+ *
2192
+ * `signal` is the poll's connection. A poll whose caller went away (a killed Harness) ends then,
2193
+ * `outstanding`, and never marks the answer read: no model read it, so the re-admission after
2194
+ * the restart must still carry it (ADR-0031).
2195
+ */
2196
+ async awaitDelivery(
2197
+ instanceId: string,
2198
+ deliveryId: string,
2199
+ waitMs = 25_000,
2200
+ signal?: AbortSignal,
2201
+ ): Promise<AgentDeliveryReceipt | undefined> {
2202
+ const found = this.holdAt(instanceId, deliveryId);
2203
+ if (!found) return undefined;
2204
+ const { run, hold } = found;
2205
+ this.settleHolds(run);
2206
+ if (hold.answer === undefined && !this.#stopping && !signal?.aborted) {
2207
+ await new Promise<void>((resolve) => {
2208
+ let waiters = this.holdWaiters.get(hold);
2209
+ if (!waiters) this.holdWaiters.set(hold, (waiters = new Set()));
2210
+ const timer = setTimeout(wake, waitMs);
2211
+ signal?.addEventListener("abort", wake, { once: true });
2212
+ function wake(): void {
2213
+ clearTimeout(timer);
2214
+ signal?.removeEventListener("abort", wake);
2215
+ waiters!.delete(wake);
2216
+ resolve();
2217
+ }
2218
+ waiters.add(wake);
2219
+ });
2220
+ }
2221
+ const base = { delivered: true as const, event: hold.pick, moved: true, deliveryId };
2222
+ if (hold.answer === undefined || signal?.aborted) {
2223
+ return { ...base, turnComplete: false, allowed: [], outstanding: true, since: hold.since };
2224
+ }
2225
+ hold.fetched = true;
2226
+ return {
2227
+ ...base,
2228
+ turnComplete: hold.answer.turnComplete,
2229
+ allowed: hold.answer.allowed,
2230
+ answered: true,
2231
+ ...(hold.answer.answer !== undefined ? { answer: hold.answer.answer } : {}),
2232
+ ...(hold.answer.stopped ? { stopped: true as const } : {}),
2233
+ };
2234
+ }
2235
+
652
2236
  /** A run's open gates as callers discover them (`GET /runs/:id` — ADR-0011). Settled/unknown
653
2237
  * run → empty: a gate is a LIVE surface, it does not outlive its state. */
654
2238
  gates(runId: string): GateView[] {
@@ -663,20 +2247,50 @@ export class RunHost {
663
2247
  description: def.description,
664
2248
  input: z.toJSONSchema(def.input),
665
2249
  })),
2250
+ acceptsNow: acceptsNow(reg),
666
2251
  meta: reg.meta,
667
2252
  }));
668
2253
  }
669
2254
 
670
- /** Deliver one external event to a run's open gate (`POST /runs/:id/gates/:gate/events`).
671
- * Validation and delivery are the table's — this only run-scopes the address. */
2255
+ /**
2256
+ * Deliver one external event to a run's open gate (`POST /runs/:id/gates/:gate/events`).
2257
+ * Validation and delivery are the table's — this run-scopes the address, and returns only when
2258
+ * the run TOOK the event (ADR-0013): one that would run another party's writing, or that no
2259
+ * writing would take with this payload, throws {@link NotTakenError} naming what the Gate takes
2260
+ * now, and nothing is delivered.
2261
+ */
672
2262
  sendToGate(runId: string, gate: string, event: { type?: unknown } & Record<string, unknown>): void {
673
2263
  const { type, ...payload } = event;
674
2264
  if (typeof type !== "string" || !type) {
675
2265
  throw new EventValidationError(`event body must carry a string "type" (one of the gate's accepted names)`);
676
2266
  }
2267
+ const reg = this.table.lookup(gateAddress(runId, gate));
2268
+ const refused = (why: string): NotTakenError =>
2269
+ new NotTakenError(
2270
+ `gate "${gate}" is open, and the run does not take "${type}" now: ${why} — not delivered ` +
2271
+ `(takes now: ${reg ? acceptsNow(reg).join(", ") || "nothing" : "nothing"})`,
2272
+ );
677
2273
  try {
2274
+ // Exact, with the validated payload: a send no writing would move on is refused, never
2275
+ // answered `{ ok: true }`. A bad payload or an unaccepted name falls through to the table,
2276
+ // which refuses it with its own words.
2277
+ const def = reg?.defs.get(type);
2278
+ const parsed = def && !(payload && SENDER_KEY in payload) ? def.input.safeParse(payload ?? {}) : undefined;
2279
+ if (reg && parsed?.success && !wouldMove(reg.invoker, { type, ...parsed.data, [SENDER_KEY]: reg.sender })) {
2280
+ throw refused("no writing of it would move the run with this payload");
2281
+ }
678
2282
  this.table.deliver(gateAddress(runId, gate), type, payload);
679
2283
  } catch (err) {
2284
+ if (err instanceof NotTakenError && err.writings.length) {
2285
+ throw refused(
2286
+ err.writings
2287
+ .map(
2288
+ (w) =>
2289
+ `it would run the writing on "${w.state}"${w.owners.length ? `, which is ${w.owners.map(senderText).join(" and ")}'s` : ""}`,
2290
+ )
2291
+ .join("; "),
2292
+ );
2293
+ }
680
2294
  if (err instanceof UnknownAddressError) {
681
2295
  const open = this.gates(runId).map((g) => g.gate);
682
2296
  throw new UnknownAddressError(
@@ -699,10 +2313,112 @@ export class RunHost {
699
2313
  const run = this.runs.get(runId);
700
2314
  if (!run) return () => {};
701
2315
  run.listeners.add(listener);
702
- listener({ kind: "status", status: this.liveStatus(run) });
2316
+ // The status the feed last carried, the SAME object for every attach, so http.ts renders the
2317
+ // replay once however many readers attach — a reconnect after a close is the common attach
2318
+ // (ADR-0022). A transition not yet persisted follows on the feed within the microtask.
2319
+ listener(run.latestStatus ?? { kind: "status", status: this.liveStatus(run) });
703
2320
  return () => run.listeners.delete(listener);
704
2321
  }
705
2322
 
2323
+ /**
2324
+ * Hear once that a run is no longer live (ADR-0071): `listener` gets its last status — `done`,
2325
+ * `failed`, `error` or `cancelled` with its `output`, or the store's `lost` or `drifted` — and is
2326
+ * then dropped. Returns the detach. What an in-process `dispatch` with `wait` waits on.
2327
+ *
2328
+ * Keyed by the run id, not by the run's hosting, so the order of a restore does not matter: a
2329
+ * caller restored before its callee attaches to a run this process does not host yet, and hears
2330
+ * it when the restored callee settles, or when restore marks it lost or drifted. A run that has
2331
+ * settled already is answered at once, from {@link read}. `subscribe` is the feed, and unchanged:
2332
+ * it attaches only to a hosted run.
2333
+ */
2334
+ onSettled(runId: string, listener: (status: RunStatus) => void): () => void {
2335
+ let listeners = this.settleListeners.get(runId);
2336
+ if (!listeners) this.settleListeners.set(runId, (listeners = new Set()));
2337
+ // Its own entry, so one listener attached twice is heard twice and detached once each.
2338
+ const entry = (status: RunStatus): void => listener(status);
2339
+ listeners.add(entry);
2340
+ const attached = (): boolean => this.settleListeners.get(runId)?.has(entry) === true;
2341
+ const detach = (): void => {
2342
+ const now = this.settleListeners.get(runId);
2343
+ now?.delete(entry);
2344
+ if (now?.size === 0) this.settleListeners.delete(runId);
2345
+ };
2346
+ if (!this.runs.has(runId)) {
2347
+ // Not hosted: settled already, or a live row a restore has not reached. Only the first is
2348
+ // answered here; a settle heard in the meantime has detached the entry.
2349
+ this.read(runId).then(
2350
+ (status) => {
2351
+ if (!status || !attached() || !settledStatus(status.status)) return;
2352
+ detach();
2353
+ hear(entry, status);
2354
+ },
2355
+ () => {},
2356
+ );
2357
+ }
2358
+ return detach;
2359
+ }
2360
+
2361
+ /** A hosted run's depth in a chain of dispatches (ADR-0071): 0 for a run nothing dispatched, and
2362
+ * undefined for a run this process does not host — not hosted is not a root run. */
2363
+ depthOf(runId: string): number | undefined {
2364
+ const run = this.runs.get(runId);
2365
+ return run && (run.record.dispatchedBy?.depth ?? 0);
2366
+ }
2367
+
2368
+ /** Tell everything waiting on `runId` that it settled, once (ADR-0071). */
2369
+ private fireSettled(runId: string, status: RunStatus): void {
2370
+ const listeners = this.settleListeners.get(runId);
2371
+ if (!listeners) return;
2372
+ this.settleListeners.delete(runId);
2373
+ for (const listener of listeners) hear(listener, status);
2374
+ }
2375
+
2376
+ /**
2377
+ * A run restore did not host settled: marked `lost` (its status is the row's identity and reason
2378
+ * alone: a lost run is not read back, though a row that was saved keeps its blob and its key) or
2379
+ * `drifted` (its snapshot is read back).
2380
+ * Only when something waits on it (ADR-0071).
2381
+ */
2382
+ private settledUnhosted(runId: string, status: "lost" | "drifted", reason: string, blob?: RunBlob): void {
2383
+ if (!this.settleListeners.has(runId)) return;
2384
+ const bare: RunStatus = {
2385
+ runId,
2386
+ workflow: blob?.workflow ?? "",
2387
+ instanceId: blob?.instanceId ?? "",
2388
+ ...(blob?.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
2389
+ status,
2390
+ value: undefined,
2391
+ context: undefined,
2392
+ children: [],
2393
+ waiting: [],
2394
+ held: [],
2395
+ reason,
2396
+ };
2397
+ if (status === "lost") return this.fireSettled(runId, bare);
2398
+ this.read(runId).then(
2399
+ (read) => this.fireSettled(runId, read ?? bare),
2400
+ () => this.fireSettled(runId, bare),
2401
+ );
2402
+ }
2403
+
2404
+ /** The binding's `dispatch.settled` (ADR-0071): {@link onSettled} as a promise that an abort —
2405
+ * the Effect's state exiting, its wait timing out — rejects and detaches. */
2406
+ private settledOf(runId: string, signal: AbortSignal): Promise<RunStatus> {
2407
+ return new Promise((resolve, reject) => {
2408
+ if (signal.aborted) return reject(signal.reason);
2409
+ let detach = (): void => {};
2410
+ const onAbort = (): void => {
2411
+ detach();
2412
+ reject(signal.reason);
2413
+ };
2414
+ signal.addEventListener("abort", onAbort, { once: true });
2415
+ detach = this.onSettled(runId, (status) => {
2416
+ signal.removeEventListener("abort", onAbort);
2417
+ resolve(status);
2418
+ });
2419
+ });
2420
+ }
2421
+
706
2422
  /**
707
2423
  * Observe a whole WORKFLOW (the `GET /workflows/:name/events` SSE — ADR-0022): every run of it
708
2424
  * appearing, moving, emitting and leaving, for as long as the caller stays attached.
@@ -754,7 +2470,9 @@ export class RunHost {
754
2470
  * Read a run's status, **reading through to the store** when it is no longer live (ADR-0009). A
755
2471
  * completed run's final snapshot is persisted before `persist()` drops it from the registry, so a
756
2472
  * terminal run reports its `done`/`error` status + final context here rather than 404-ing. Returns
757
- * undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared).
2473
+ * undefined only for a genuinely unknown run (or one marked `lost` with no row saved before, so no
2474
+ * snapshot: `markLost` keeps a saved row's blob, whose key must keep answering, ADR-0071) — and a
2475
+ * swept one is unknown: a finished run's row lives a week (ADR-0065).
758
2476
  */
759
2477
  async read(runId: string): Promise<RunStatus | undefined> {
760
2478
  const live = this.status(runId);
@@ -762,20 +2480,33 @@ export class RunHost {
762
2480
  const stored = await this.store.load(runId);
763
2481
  const blob = stored?.snapshot as RunBlob | null | undefined;
764
2482
  if (!stored || !blob) return undefined;
765
- const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown };
2483
+ const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown; output?: unknown };
2484
+ // The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
2485
+ // cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
2486
+ // (ADR-0025). While the row still says "live", the Machine's own status is the answer.
2487
+ const status = stored.status === "live" ? (snap.status ?? stored.status) : stored.status;
766
2488
  return {
767
2489
  runId,
768
2490
  workflow: blob.workflow,
769
2491
  instanceId: blob.instanceId,
770
- // The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
771
- // cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
772
- // (ADR-0025). While the row still says "live", the Machine's own status is the answer.
773
- status: stored.status === "live" ? (snap.status ?? stored.status) : stored.status,
2492
+ status,
774
2493
  value: snap.value,
775
2494
  context: snap.context,
776
2495
  children: runChildren(snap), // the persisted `children` map — same tree, off the store
2496
+ waiting: status === "active" ? runWaiting(snap) : [],
2497
+ held: status === "active" ? heldOf(blob.holds ?? []) : [],
777
2498
  fault: blob.fault,
778
2499
  reason: stored.reason,
2500
+ ...(blob.assembly !== undefined ? { assembly: blob.assembly } : {}),
2501
+ ...boundFields(snap),
2502
+ ...(blob.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
2503
+ // The Machine's output once the row says the run is over (ADR-0071), never while it is live;
2504
+ // for a failed run, its Failure (ADR-0073).
2505
+ ...(stored.status !== "live" && blob.failure !== undefined ? { output: blob.failure } : {}),
2506
+ ...(stored.status !== "live" && blob.failure === undefined && snap.output !== undefined
2507
+ ? { output: snap.output }
2508
+ : {}),
2509
+ ...dispatchedField(blob.effects ?? {}),
779
2510
  };
780
2511
  }
781
2512
 
@@ -805,6 +2536,25 @@ export class RunHost {
805
2536
  .map(observe);
806
2537
  }
807
2538
 
2539
+ /** The host is going away (`close()` has begun): its Turns are not over, and the Agents' surface
2540
+ * must not say they are (ADR-0026). */
2541
+ get stopping(): boolean {
2542
+ return this.#stopping;
2543
+ }
2544
+
2545
+ /** The live rows of a prior process are not restored yet: a boot with no current Assembly
2546
+ * deferred their restore to the first publish (ADR-0069). Those Turns are not over, so the
2547
+ * Agents' surface must not say they are (ADR-0026). Set by `deferRestore`, cleared when a
2548
+ * `restore()` completes; a restore that throws as a whole leaves it set. */
2549
+ get restorePending(): boolean {
2550
+ return this.#restorePending;
2551
+ }
2552
+
2553
+ /** Mark the restore deferred: until a `restore()` completes, `restorePending` is true. */
2554
+ deferRestore(): void {
2555
+ this.#restorePending = true;
2556
+ }
2557
+
808
2558
  /**
809
2559
  * End every open feed because the host is going away — the shutdown counterpart to `subscribe`.
810
2560
  *
@@ -815,6 +2565,15 @@ export class RunHost {
815
2565
  * OBSERVATION, not the work — the snapshots are already durable, and `restore()` picks them up.
816
2566
  */
817
2567
  async close(): Promise<void> {
2568
+ // From here on the Orchestrator is going away, not its Turns: `/agents/*` answers 503, which a
2569
+ // Harness re-asks on ADR-0042's ladder, and never 404, which says a Turn is over (ADR-0026).
2570
+ this.#stopping = true;
2571
+ clearTimeout(this.liveSetTimer);
2572
+ // A run's timers stay armed (a run is untouched by a close), but none holds the process alive.
2573
+ for (const timer of this.timers) (timer as { unref?: () => void }).unref?.();
2574
+ // Every long poll answers `outstanding` now, so the server can close; the Harness asks again,
2575
+ // and a restarted Orchestrator answers it from the same hold (ADR-0024).
2576
+ for (const hold of [...this.holdWaiters.keys()]) this.closeHold(hold);
818
2577
  for (const run of this.runs.values()) {
819
2578
  for (const listener of run.listeners) listener({ kind: "closed" });
820
2579
  run.listeners.clear();
@@ -841,6 +2600,14 @@ export class RunHost {
841
2600
  const run = this.runs.get(runId);
842
2601
  if (!run) return;
843
2602
  run.binding.hostStopping = true;
2603
+ // A parked run's outstanding picks are not over (ADR-0024): their polls are answered `503` —
2604
+ // ask again — never `404`, which tells the Harness its Turn ended. The host that restores the
2605
+ // run answers them from the saved holds.
2606
+ for (const hold of run.holds.values()) {
2607
+ if (hold.answer !== undefined) continue;
2608
+ this.parked.add(`${hold.instanceId} ${hold.deliveryId}`);
2609
+ this.closeHold(hold);
2610
+ }
844
2611
  // Say so BEFORE the actor stops, while there is still a status to read. A stopped run is not a
845
2612
  // settled one — `persist()` never runs for it (the tracked-run guard drops the scheduled save,
846
2613
  // keeping the stored status "live" for restore — see `spawn`), so nothing else on this path
@@ -859,14 +2626,62 @@ export class RunHost {
859
2626
  * Ending the turns and refusing to restore are one decision, not two: a cancelled run that came
860
2627
  * back would re-attach to submissions that settled `aborted`, and `settle`'s rejection would
861
2628
  * fault a run whose Agents were stopped on purpose.
2629
+ *
2630
+ * It then destroys every Sandbox the run owns, unless `keep` (`jr2 send <run> --event CANCEL
2631
+ * --keep`): kept, they age out at the operator's idle timeout, and `jr2 ssh` reaches them until
2632
+ * then. A faulted run never comes through here, and keeps its Workspaces (ADR-0012).
862
2633
  */
863
- async cancel(runId: string): Promise<void> {
2634
+ async cancel(runId: string, opts: { keep?: boolean } = {}): Promise<void> {
864
2635
  const run = this.runs.get(runId);
865
2636
  if (!run) return;
2637
+ // Named BEFORE the stop, while the tree still says which Workspaces are live.
2638
+ const sandboxes = opts.keep ? [] : runSandboxes(run.actor, runId);
866
2639
  run.actor.stop();
867
2640
  // Persist AFTER the stop: the snapshot is final, and `persist` fans out the last status,
868
2641
  // announces `gone`, and untracks — the same terminal path a run that settled on its own takes.
869
2642
  this.persist(run, "cancelled");
2643
+ // The run is over, so nothing can use its Workspaces, and each holds its whole Size (ADR-0025).
2644
+ // An actor stop skips the wrapper's `teardown` state, so the host runs the same idempotent
2645
+ // delete here, after the stop, so no lease renews what was deleted. `keep` leaves them to the
2646
+ // operator's idle timeout, for inspection.
2647
+ await this.destroySandboxes(runId, sandboxes, "the cancel");
2648
+ }
2649
+
2650
+ /**
2651
+ * End a run `error` because a Machine of its tree dropped a mechanism event (ADR-0074): `fault`
2652
+ * says which. The actor stops as a cancel stops it, so each live Agent's Turn ends, and the run
2653
+ * is persisted terminal. Its Sandboxes stay as evidence, as a fault's do (ADR-0012): the Machine
2654
+ * did not choose this end. Once per run: a second dropped event in the same step adds nothing.
2655
+ */
2656
+ private faultUnrouted(runId: string, binding: RunBinding, fault: string): void {
2657
+ const run = this.runs.get(runId);
2658
+ if (!run || run.binding !== binding || run.fault !== undefined) return;
2659
+ run.fault = fault;
2660
+ run.actor.stop();
2661
+ this.persist(run, "error");
2662
+ }
2663
+
2664
+ /**
2665
+ * Delete the Sandboxes of a run that is over and stopped: a cancelled one (ADR-0025) and a failed
2666
+ * one (ADR-0073). After the stop, so no lease renews what was deleted. The delete is idempotent:
2667
+ * for a failed run, the Workspace a Failure went through deleted its own already, so this is a
2668
+ * backstop for the others. A delete that fails is the idle GC's, as in `teardown` (ADR-0012
2669
+ * backstop): the run has already ended. `why` names that end in the log line of a delete that
2670
+ * fails.
2671
+ */
2672
+ private async destroySandboxes(runId: string, sandboxes: readonly string[], why: string): Promise<void> {
2673
+ const port = this.sandbox;
2674
+ if (!port) return;
2675
+ await Promise.all(
2676
+ sandboxes.map((name) =>
2677
+ port.destroy(name).catch((err: unknown) => {
2678
+ console.error(
2679
+ `run ${runId}: Sandbox ${name} was not deleted at ${why} (the idle timeout will delete it): ` +
2680
+ (err instanceof Error ? err.message : String(err)),
2681
+ );
2682
+ }),
2683
+ ),
2684
+ );
870
2685
  }
871
2686
 
872
2687
  /** Feed a run's final word to both granularities: where it ended, then that it is gone. */
@@ -879,19 +2694,25 @@ export class RunHost {
879
2694
 
880
2695
  /** Put one event on the run's feed: buffer it for ADR-0023's backfill, then fan it out. The
881
2696
  * buffer and the fan-out share one seat so an attached echo and a later attach see the SAME
882
- * feed — a projection cannot drift from a record it is read out of. */
2697
+ * feed — a projection cannot drift from a record it is read out of. Every listener is handed the
2698
+ * SAME event object, which is what lets http.ts render each frame once (ADR-0022). */
883
2699
  private feed(run: LiveRun, event: RunFeedEvent): void {
884
- run.feedSoFar.push(event);
885
- if (run.feedSoFar.length > FEED_SO_FAR_CAP) run.feedSoFar.shift();
2700
+ if (event.kind === "status") {
2701
+ run.latestStatus = event;
2702
+ } else {
2703
+ run.feedSoFar.push(event);
2704
+ if (run.feedSoFar.length > FEED_SO_FAR_CAP) run.feedSoFar.shift();
2705
+ }
886
2706
  for (const listener of run.listeners) listener(event);
887
2707
  }
888
2708
 
889
2709
  /**
890
2710
  * Attach the run-narrative echo (ADR-0023): replay the run's feed-so-far to the Workspace's
891
- * Harness at `endpoint` — the backfilled preamble, why this Workspace exists — then tee live
892
- * until detached. FIRE-AND-FORGET is this method's contract: pushes are chained so events
893
- * arrive in feed order, and a failure is logged once and swallowed — the feed remains the
894
- * record, the log is a courtesy view, and nothing here can fail a turn, a state, or a run.
2711
+ * Harness at `endpoint` — the backfilled preamble, why this Workspace exists: its markers and
2712
+ * Emits in feed order, then where the run stands now — then tee live until detached, through an
2713
+ * {@link echoOutbox}, which coalesces statuses. FIRE-AND-FORGET is this method's contract: a
2714
+ * failure is logged once and swallowed — the feed remains the record, the log is a courtesy
2715
+ * view, and nothing here can fail a turn, a state, or a run.
895
2716
  * Scoped to the OWNING run's lineage by construction: it reads one run's buffer and listeners
896
2717
  * and nothing else — a sibling run's events cannot reach this endpoint through here.
897
2718
  */
@@ -899,25 +2720,19 @@ export class RunHost {
899
2720
  const run = this.runs.get(runId);
900
2721
  const factory = this.echoFactory;
901
2722
  if (!run || !factory) return () => {};
902
- const push = factory(endpoint, this.harnessBearer?.(placement));
903
- let chain = Promise.resolve();
904
2723
  let reported = false;
905
- const enqueue = (events: EchoEvent[]): void => {
906
- if (events.length === 0) return;
907
- chain = chain
908
- .then(() => push(events))
909
- .catch((err) => {
910
- // Log-and-continue, ONCE per attachment — an unreachable Harness must not turn every
911
- // transition into an error line, and must not surface anywhere a run could trip on.
912
- if (reported) return;
913
- reported = true;
914
- console.error(
915
- `run ${runId}: echo to ${endpoint} failed (log only — the run is unaffected): ` +
916
- (err instanceof Error ? err.message : String(err)),
917
- );
918
- });
919
- };
920
- enqueue(run.feedSoFar.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
2724
+ const enqueue = echoOutbox(factory(endpoint, this.harnessBearer?.(placement)), (err) => {
2725
+ // Log-and-continue, ONCE per attachment — an unreachable Harness must not turn every
2726
+ // transition into an error line, and must not surface anywhere a run could trip on.
2727
+ if (reported) return;
2728
+ reported = true;
2729
+ console.error(
2730
+ `run ${runId}: echo to ${endpoint} failed (log only — the run is unaffected): ` +
2731
+ (err instanceof Error ? err.message : String(err)),
2732
+ );
2733
+ });
2734
+ const preamble = run.latestStatus ? [...run.feedSoFar, run.latestStatus] : run.feedSoFar;
2735
+ enqueue(preamble.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
921
2736
  const listener = (ev: RunFeedEvent): void => {
922
2737
  const projected = echoEventOf(ev, endpoint);
923
2738
  if (projected) enqueue([projected]);
@@ -935,7 +2750,18 @@ export class RunHost {
935
2750
  value: snap.value,
936
2751
  context: snap.context,
937
2752
  children: runChildren(snap),
2753
+ waiting: runWaiting(snap, run.actor.logic),
2754
+ held: heldOf(run.holds.values()),
938
2755
  fault: run.fault,
2756
+ ...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
2757
+ ...boundFields(snap),
2758
+ // A Machine that reached its final state has its output (ADR-0071); an active one has none.
2759
+ // A failed run's output is its Failure (ADR-0073).
2760
+ ...(run.failure !== undefined ? { output: run.failure } : {}),
2761
+ ...(run.failure === undefined && snap.status !== "active" && snap.output !== undefined
2762
+ ? { output: snap.output }
2763
+ : {}),
2764
+ ...dispatchedField(run.effects),
939
2765
  };
940
2766
  }
941
2767
 
@@ -960,7 +2786,16 @@ export class RunHost {
960
2786
  def: WorkflowDef,
961
2787
  agents: Record<string, AgentAdmission> = {},
962
2788
  epochs: Record<string, number> = {},
2789
+ noticeState?: NoticeLedgerState,
2790
+ holdState: Hold[] = [],
2791
+ effects: Record<string, EffectRow> = {},
2792
+ startedBy: Record<string, StartedBy> = {},
963
2793
  ): AnyActor {
2794
+ const notices = new NoticeLedger(noticeState, this.newId);
2795
+ const holds = new Map(holdState.map((hold) => [hold.instanceId, hold]));
2796
+ // What took the Machine out of each Agent's state: that Agent's own pick, or something else
2797
+ // (ADR-0024). Read off the inspection stream below.
2798
+ const causes = new PickCauses();
964
2799
  let live: LiveRun | undefined;
965
2800
  let scheduled = false;
966
2801
  const schedule = () => {
@@ -974,30 +2809,63 @@ export class RunHost {
974
2809
  const binding: RunBinding = {
975
2810
  runId: record.runId,
976
2811
  workflow: record.workflow,
2812
+ ...(this.orchestratorName !== undefined ? { orchestratorName: this.orchestratorName } : {}),
2813
+ depth: record.dispatchedBy?.depth ?? 0,
2814
+ // A `dispatch` with no target (ADR-0071): the run host itself, with no wire and no token.
2815
+ dispatch: {
2816
+ start: (workflow, input, by) => this.dispatchStart(workflow, input, by),
2817
+ settled: (runId, signal) => this.settledOf(runId, signal),
2818
+ },
2819
+ ...(def.assembly !== undefined ? { assembly: def.assembly } : {}),
977
2820
  table: this.table,
978
2821
  sandbox: this.sandbox,
2822
+ heldSandboxes: () => {
2823
+ const run = this.runs.get(record.runId);
2824
+ return new Set(run ? runSandboxes(run.actor, record.runId, { placing: true }) : []);
2825
+ },
979
2826
  instanceHarness: this.instanceHarness,
980
2827
  ...(this.harnessBearer ? { harnessBearer: this.harnessBearer } : {}),
981
- // The admission ledger's write half (ADR-0016): the Agent actor reports the durable handle the
2828
+ ...(this.effector ? { effector: this.effector } : {}),
2829
+ // The admission ledger's write half (ADR-0016): the Agent's Actor reports the durable handle the
982
2830
  // moment the Harness admits it, and the ledger hits the store in the same RunBlob save. An
983
2831
  // admission arriving around stop/untrack still lands in `agents` but skips the save,
984
2832
  // exactly like the persist scheduler's tracked-run guard.
985
- recordAdmission: (instanceId, admission) => {
2833
+ recordAdmission: (instanceId, admission, delivered = []) => {
986
2834
  agents[instanceId] = admission;
2835
+ // The notices this admission carried are delivered by THIS save (ADR-0062).
2836
+ notices.delivered(delivered);
987
2837
  const run = this.runs.get(record.runId);
988
2838
  if (run && run.agents === agents) this.persist(run);
989
2839
  },
2840
+ admissionOf: (instanceId) => agents[instanceId],
990
2841
  // The epoch ledger's two halves (ADR-0057). The read is what the input mapper mints from;
991
2842
  // the write is the terminal `agent.fault` burning the conversation it ended, saved in the
992
2843
  // same RunBlob as the admission ledger — and, like it, saved AT THE WRITE rather than
993
2844
  // waiting for the next transition, because a crash between the fault and the next state's
994
2845
  // invoke would otherwise restore into the burned conversation.
995
2846
  epochOf: (conversation) => epochs[conversation] ?? 0,
996
- bumpEpoch: (conversation) => {
2847
+ bumpEpoch: (conversation, reason) => {
997
2848
  epochs[conversation] = (epochs[conversation] ?? 0) + 1;
2849
+ // The next `continue` lands on a virgin conversation and is told so, with the fault's own
2850
+ // words (ADR-0062) — raised in the SAME save as the bump, so no restart can split them.
2851
+ notices.raise({ kind: "conversation-new", scope: "conversation", reason }, conversation);
998
2852
  const run = this.runs.get(record.runId);
999
2853
  if (run && run.epochs === epochs) this.persist(run);
1000
2854
  },
2855
+ // The notices' other halves (ADR-0062). A raise is saved at the write, like the ledgers; a
2856
+ // take is a reservation in memory, delivered by `recordAdmission` or released on failure.
2857
+ raiseNotice: (notice, to, source, older) => {
2858
+ if (!notices.raise(notice, to, source, older)) return;
2859
+ const run = this.runs.get(record.runId);
2860
+ if (run && run.notices === notices) this.persist(run);
2861
+ },
2862
+ restoreNotice: (notice, to, source) => {
2863
+ if (!notices.restore(notice, to, source)) return;
2864
+ const run = this.runs.get(record.runId);
2865
+ if (run && run.notices === notices) this.persist(run);
2866
+ },
2867
+ takeNotices: (scope) => notices.take(scope),
2868
+ releaseNotices: (ids) => notices.release(ids),
1001
2869
  // Absorbed-retry attempts go straight to the run's observers (SSE/CLI watch) — they are
1002
2870
  // feed events, not machine events (ADR-0016: the workflow sees only the terminal fault).
1003
2871
  telemetry: (event) => {
@@ -1007,15 +2875,89 @@ export class RunHost {
1007
2875
  // Turn markers (ADR-0023) ride the per-run feed alone — the prompt is Instance-token-class
1008
2876
  // data, so they never touch the workflow feed (ADR-0014's open band strips even Emit
1009
2877
  // payloads there).
2878
+ // An Effect's line that comes after its run ended — a stop's answer on a cancel or a final
2879
+ // state (ADR-0067) — has no feed left to ride, so it goes to the Orchestrator's log.
1010
2880
  marker: (event) => {
1011
2881
  const run = this.runs.get(record.runId);
1012
2882
  if (run && run.binding === binding) this.feed(run, event);
2883
+ else if (!run && event.kind === "effect") {
2884
+ console.error(`run ${record.runId}: effect ${event.slot} (${event.effect}) ${event.stage}: ${event.text}`);
2885
+ }
1013
2886
  },
1014
2887
  // The run-narrative echo attach (ADR-0023), called from `workspace()`'s registrar.
1015
2888
  echo: (endpoint, placement) => this.attachEcho(record.runId, endpoint, placement),
2889
+ // The outstanding pick of the Turn now at this address, as the Agent's Actor reads it when its
2890
+ // Harness restarted (ADR-0024/0031). Settled first, so a hold whose Turn is over reads
2891
+ // `turnComplete`: that is an EARLIER Turn's pick (a route left its state, and the answer went as
2892
+ // a notice to the next Turn, or nowhere) — never this Turn's, whatever it continues.
2893
+ outstandingOf: (instanceId) => {
2894
+ const run = this.runs.get(record.runId);
2895
+ if (run && run.holds === holds) this.settleHolds(run);
2896
+ const hold = holds.get(instanceId);
2897
+ if (hold?.answer?.turnComplete) return undefined;
2898
+ return (
2899
+ hold && {
2900
+ pick: hold.pick,
2901
+ answered: hold.answer !== undefined,
2902
+ turnComplete: hold.answer?.turnComplete ?? false,
2903
+ ...(hold.answer?.answer !== undefined ? { answer: hold.answer.answer } : {}),
2904
+ fetched: hold.fetched === true,
2905
+ }
2906
+ );
2907
+ },
2908
+ // The `answer` action's two halves (ADR-0067): the chain of the one pick being answered of an
2909
+ // Agent whose state the action is written below — the one in its own send first, else the one
2910
+ // outstanding — and the value that replaces its default answer.
2911
+ chainOf: (machine, seat, event) => {
2912
+ const pick = this.answeringIn(machine, seat, record.runId, holds, startedBy, event);
2913
+ if (pick === undefined) return undefined;
2914
+ return "reg" in pick ? [...pick.chain] : [...(pick.chain ?? [])];
2915
+ },
2916
+ setAnswer: (machine, seat, value, event) => {
2917
+ const pick = this.answeringIn(machine, seat, record.runId, holds, startedBy, event);
2918
+ if (pick === undefined) return;
2919
+ // Not saved here: the action runs inside a transition, and the persist that follows it carries
2920
+ // the value — a save now would read the Machine before its route, and answer too early.
2921
+ pick.shaped = jsonOf(value);
2922
+ },
2923
+ // The Effect ledger's two halves (ADR-0067). A write is saved AT THE WRITE, like an admission:
2924
+ // an intent row must be on disk, with the snapshot that holds its Effect, before the call
2925
+ // leaves. A row written inside a transition (an abort) waits for the save that carries that
2926
+ // transition — a cancel's final save included, which untracks the run — and is saved after
2927
+ // it only when nothing else did. A row written around a park lands in memory and skips the save.
2928
+ effectRow: (effectId) => effects[effectId],
2929
+ effectRows: () => effects,
2930
+ recordEffect: (effectId, row, save = "now") => {
2931
+ effects[effectId] = row;
2932
+ const persist = (): void => {
2933
+ const run = this.runs.get(record.runId);
2934
+ if (run && run.effects === effects) this.persist(run);
2935
+ };
2936
+ if (save === "now") persist();
2937
+ else queueMicrotask(persist);
2938
+ },
2939
+ heldAnswerDelivered: (instanceId) => {
2940
+ const hold = holds.get(instanceId);
2941
+ if (hold) hold.fetched = true;
2942
+ },
2943
+ // Read as the invocation stops, while its registration is still there (ADR-0024).
2944
+ turnEndedByPick: (instanceId) => {
2945
+ const reg = this.table.lookup(agentAddress(instanceId));
2946
+ return reg !== undefined && reg.runId === record.runId && causes.endedByPick(reg);
2947
+ },
2948
+ // A Chart by name (ADR-0070, D-C): among the Charts of the wrapper that is the run's Workflow.
2949
+ chartNamed: (name) => this.namedChart(wrapperName(record.workflow), name)?.text,
1016
2950
  };
1017
2951
  let bound = false;
2952
+ let firedTimer: ((source: unknown, event: unknown) => boolean) | undefined;
2953
+ // The children each Machine of the tree had after its last microstep: what the next one started
2954
+ // is the difference (ADR-0067's attribution of a child to the pick whose work started it).
2955
+ const childrenAfter = new WeakMap<AnyActorRef, Set<AnyActorRef>>();
2956
+ const childRefsOf = (snapshot: unknown): Set<AnyActorRef> =>
2957
+ new Set(Object.values((snapshot as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {}));
1018
2958
  const actor = createActor(machine, {
2959
+ // Each timer keeps the cause of the relay that armed it (`cause.ts`, ADR-0024).
2960
+ clock: causes.clockOver(this.clock),
1019
2961
  ...options,
1020
2962
  inspect: (ev) => {
1021
2963
  // Bind the run's actor SYSTEM on the ROOT's creation event — the first inspection event,
@@ -1026,12 +2968,67 @@ export class RunHost {
1026
2968
  if (!bound && ev.type === "@xstate.actor") {
1027
2969
  bound = true;
1028
2970
  bindRun((ev.actorRef as AnyActorRef).system, binding);
2971
+ // …and watch its scheduler before anything is armed: whether an event the Machine took
2972
+ // was a timer it set, or an event an actor sent at once, decides `cutShort` below.
2973
+ // The same watch keeps the cause of each timer: the relay that armed it (ADR-0024).
2974
+ firedTimer = watchTimers((ev.actorRef as AnyActorRef).system, {
2975
+ arming: (source) => causes.armingFrom(source),
2976
+ });
2977
+ }
2978
+ // A restored hold is the Turn of the FIRST registration of its address in this tree: bind
2979
+ // it as soon as one exists, before an event can end that Turn and start another at the same
2980
+ // address (`continue: true` re-registers it). Every inspection event, so it is bound before
2981
+ // the restored Machine processes anything: the root's snapshot comes after its children
2982
+ // start and before its mailbox drains.
2983
+ for (const hold of holds.values()) {
2984
+ if (hold.answer !== undefined || this.holdRegs.has(hold)) continue;
2985
+ const reg = this.table.lookup(agentAddress(hold.instanceId));
2986
+ if (reg && reg.runId === record.runId && reg.invokeId === hold.invokeId) this.holdRegs.set(hold, reg);
1029
2987
  }
1030
2988
  // Forward the workflow author's `emit({...})` onto the feeds, from EVERY actor in the
1031
2989
  // tree as it is created: xstate scopes emitted events to the actor that emits them — no
1032
2990
  // bubbling — and a run's Emits mostly happen in child machines (a `workspace()` body IS
1033
2991
  // one). A root-only subscription silently dropped exactly the Emits ADR-0022/0023 exist
1034
2992
  // to surface. The subscription dies with its actor; restore re-creates both.
2993
+ // The children a pick's own send starts are its Agent's (ADR-0067): their done events name
2994
+ // the Agent an `answer` below no Agent's state shapes. Each place an `answer` is written runs
2995
+ // as its own seat: each Machine of the tree is placed as its actor is created, before its
2996
+ // first state is entered (`placeAnswers`).
2997
+ if (ev.type === "@xstate.actor") {
2998
+ const sending = this.sending;
2999
+ const child = ev.actorRef as AnyActorRef;
3000
+ const logic = (child as { logic?: unknown }).logic as AnyStateMachine | undefined;
3001
+ if (logic && typeof logic === "object" && "root" in logic) placeAnswers(logic);
3002
+ if (sending && child._parent !== undefined && child._parent === sending.reg.invoker) {
3003
+ const agent = sending.reg.sender?.id;
3004
+ const invoke = sending.reg.invokeId;
3005
+ const key = childKey(child._parent, child.id);
3006
+ if (agent !== undefined) startedBy[key] = invoke !== undefined ? { agent, invoke } : { agent };
3007
+ else delete startedBy[key];
3008
+ }
3009
+ }
3010
+ // …and so are the children its later work starts: a child started while the Machine takes an
3011
+ // event that is already that Agent's — a done event of a child it started, or its own event —
3012
+ // is that Agent's too, so the work its pick began stays its own (ADR-0067). A child started
3013
+ // on an event that names no Agent (a timer) below the state of an Agent whose pick is
3014
+ // outstanding is that pick's work by the Rest rule, so it is that Agent's (ADR-0024); any
3015
+ // other child started on such an event is no Agent's.
3016
+ if (ev.type === "@xstate.microstep") {
3017
+ const taker = ev.actorRef as AnyActorRef;
3018
+ const before = childrenAfter.get(taker) ?? childRefsOf(taker.getSnapshot());
3019
+ const snapshot = (ev as { snapshot?: unknown }).snapshot;
3020
+ const after = childRefsOf(snapshot);
3021
+ childrenAfter.set(taker, after);
3022
+ const started = [...after].filter((c) => !before.has(c)).map((c) => c.id);
3023
+ if (started.length > 0 && !this.sending) {
3024
+ const agent = agentOfEvent(ev.event as AnyEventObject, startedBy, taker);
3025
+ for (const id of started) {
3026
+ const owner = agent ?? this.outstandingAbove(taker, snapshot as AnyMachineSnapshot, id, holds);
3027
+ if (owner !== undefined) startedBy[childKey(taker, id)] = owner;
3028
+ else delete startedBy[childKey(taker, id)];
3029
+ }
3030
+ }
3031
+ }
1035
3032
  if (ev.type === "@xstate.actor") {
1036
3033
  const ref = ev.actorRef as AnyActorRef & { on?: (type: string, handler: (e: unknown) => void) => unknown };
1037
3034
  ref.on?.("*", (emitted) => {
@@ -1042,10 +3039,116 @@ export class RunHost {
1042
3039
  this.announce(record.workflow, { kind: "emit", runId: record.runId, event });
1043
3040
  });
1044
3041
  }
1045
- if (ev.type === "@xstate.snapshot") schedule();
3042
+ // The chain (ADR-0024): every reply from below the Agent's state while one of its picks is
3043
+ // outstanding — inspected as it is relayed, before the Machine processes it, so the states
3044
+ // it came from are still active — and the last is the answer's default. The Machine's own
3045
+ // events, and events from outside the states below the Agent's, are skipped (`replyOf`).
3046
+ // A reply inside the pick's own send comes before any hold exists: `sending` records it.
3047
+ if (ev.type === "@xstate.event") {
3048
+ const replyTo = (reg: Registration | undefined): PickAnswer | undefined => {
3049
+ if (!reg?.invoker || !reg.invokeId || reg.invoker !== ev.actorRef) return undefined;
3050
+ const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
3051
+ return seat && replyOf(ev.event, ev.sourceRef as AnyActorRef | undefined, reg.invoker, seat);
3052
+ };
3053
+ // A child's error from below the Agent's state is no reply: the pick's answer carries none,
3054
+ // and the operator reads it on the run's feed (ADR-0024).
3055
+ const errorTo = (reg: Registration | undefined, pick: string): void => {
3056
+ if (!reg?.invoker || !reg.invokeId || reg.invoker !== ev.actorRef) return;
3057
+ const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
3058
+ const failed = seat && childErrorOf(ev.event, ev.sourceRef as AnyActorRef | undefined, reg.invoker, seat);
3059
+ const run = this.runs.get(record.runId);
3060
+ if (!failed || !run || run.binding !== binding) return;
3061
+ this.feed(run, { kind: "child-error", agent: reg.sender?.id ?? "", pick, ...failed });
3062
+ };
3063
+ // Every relayed event is asked, so the watch forgets each timer as it fires.
3064
+ const timer = firedTimer?.(ev.sourceRef, ev.event) === true;
3065
+ for (const hold of holds.values()) {
3066
+ if (hold.answer !== undefined) continue;
3067
+ const live = this.table.lookup(agentAddress(hold.instanceId));
3068
+ const reg = this.holdRegs.get(hold) ?? live;
3069
+ const reply = replyTo(reg);
3070
+ if (reply) hold.chain = [...(hold.chain ?? []), reply];
3071
+ errorTo(reg, hold.pick);
3072
+ // While the Agent's state is still the hold's, each event the Machine takes says who would
3073
+ // move it out (`cutShort`), when the Machine that invokes the Agent or one above it takes
3074
+ // it — events between the children below do not move it; once the state is gone, the next
3075
+ // state's events are not the hold's:
3076
+ // - a reply: its route;
3077
+ // - an event a Machine sent itself at once (`sendTo(self)`; a `raise` is never relayed):
3078
+ // part of the route of the event whose actions sent it — no change;
3079
+ // - anything else — a timer the Machine set, an event from outside the states below, a
3080
+ // party's event, the Agent's own end: it stops the work below, if any still runs.
3081
+ const taker = ev.actorRef as AnyActorRef;
3082
+ if (reg === undefined || reg !== live || !reg.invoker || !atOrBelow(reg.invoker, taker)) continue;
3083
+ if (reply) delete hold.cutShort;
3084
+ else if (!timer && ev.sourceRef === taker) continue;
3085
+ else if (stillRunning(reg)) hold.cutShort = true;
3086
+ else delete hold.cutShort;
3087
+ }
3088
+ const sending = this.sending;
3089
+ const reply = sending && replyTo(sending.reg);
3090
+ if (sending && reply) sending.chain.push(reply);
3091
+ if (sending) errorTo(sending.reg, sending.pick);
3092
+ // The cause of this relay (`cause.ts`, ADR-0024): read before the Machine takes it.
3093
+ const target = ev.actorRef as AnyActorRef;
3094
+ const source = ev.sourceRef as AnyActorRef | undefined;
3095
+ const event = ev.event as AnyEventObject;
3096
+ causes.relayed(
3097
+ source,
3098
+ target,
3099
+ event,
3100
+ () => this.pickCause(record.runId, holds, target, source, event),
3101
+ () => this.table.byRun(record.runId).filter((reg) => reg.kind === "agent"),
3102
+ );
3103
+ // A Failure ended a Machine that a Machine above can catch (ADR-0073). xstate does not stop
3104
+ // the children of a Machine that ends `error`, so the host stops them, as `stopTree` does for
3105
+ // the run: an Agent in that Machine settles its Turn, and a Gate or a timer there does not
3106
+ // wait for nobody. After the parent takes the event, so its exit is recorded first. A stop
3107
+ // skips `teardown`, so a Workspace in another region of that Machine does not delete its
3108
+ // Sandbox: the host names them before the stop and deletes them after it, as for a failed run.
3109
+ // A name that a live Workspace of the run holds after the stop is not deleted: a parent that
3110
+ // invokes the same Machine again gives its new Workspace the same invoke id, and so the same
3111
+ // name, and that Workspace is in `placing` by now.
3112
+ if (
3113
+ source !== undefined &&
3114
+ source._parent === target &&
3115
+ event.type === `xstate.error.actor.${source.id}` &&
3116
+ isFailure((event as { error?: unknown }).error)
3117
+ ) {
3118
+ queueMicrotask(() => {
3119
+ const sandboxes = runSandboxes(source, record.runId);
3120
+ stopTree(source);
3121
+ const held = this.runs.get(record.runId)?.binding.heldSandboxes?.() ?? new Set<string>();
3122
+ const ended = sandboxes.filter((name) => !held.has(name));
3123
+ void this.destroySandboxes(record.runId, ended, "the end of the failed Machine");
3124
+ });
3125
+ }
3126
+ }
3127
+ // A mechanism event that no transition took (ADR-0074): `agent.fault` or `workspace.lost`
3128
+ // that a Machine got and dropped, as a guard that was false when it came. The run would wait
3129
+ // for ever, so it faults. After the macrostep, so the stop does not cut xstate's own step.
3130
+ if (ev.type === "@xstate.microstep") {
3131
+ const unrouted = unroutedMechanismEvent(ev as unknown as MicrostepView);
3132
+ if (unrouted !== undefined) queueMicrotask(() => this.faultUnrouted(record.runId, binding, unrouted));
3133
+ }
3134
+ // The exit of each Agent's state (`cause.ts`, ADR-0024).
3135
+ if (ev.type === "@xstate.microstep") {
3136
+ const machine = ev.actorRef as AnyActorRef;
3137
+ const agents = this.table.byRun(record.runId).filter((reg) => reg.kind === "agent");
3138
+ causes.microstep(
3139
+ machine,
3140
+ (ev as unknown as { snapshot: AnyMachineSnapshot }).snapshot,
3141
+ ev.event as AnyEventObject,
3142
+ agents,
3143
+ );
3144
+ }
3145
+ if (ev.type === "@xstate.snapshot") {
3146
+ causes.taken(ev.actorRef as AnyActorRef, ev.event as AnyEventObject);
3147
+ schedule();
3148
+ }
1046
3149
  },
1047
3150
  });
1048
- live = this.track(record, actor, def, agents, epochs, binding);
3151
+ live = this.track(record, actor, def, agents, epochs, notices, binding, holds, effects, startedBy);
1049
3152
  return actor;
1050
3153
  }
1051
3154
 
@@ -1055,9 +3158,26 @@ export class RunHost {
1055
3158
  def: WorkflowDef,
1056
3159
  agents: Record<string, AgentAdmission>,
1057
3160
  epochs: Record<string, number>,
3161
+ notices: NoticeLedger,
1058
3162
  binding: RunBinding,
3163
+ holds: Map<string, Hold>,
3164
+ effects: Record<string, EffectRow>,
3165
+ startedBy: Record<string, StartedBy>,
1059
3166
  ): LiveRun {
1060
- const run: LiveRun = { record, actor, def, agents, epochs, binding, listeners: new Set(), feedSoFar: [] };
3167
+ const run: LiveRun = {
3168
+ record,
3169
+ actor,
3170
+ def,
3171
+ agents,
3172
+ epochs,
3173
+ notices,
3174
+ holds,
3175
+ effects,
3176
+ startedBy,
3177
+ binding,
3178
+ listeners: new Set(),
3179
+ feedSoFar: [],
3180
+ };
1061
3181
  this.runs.set(record.runId, run);
1062
3182
  // Ordinary persistence rides the inspection stream (see `spawn`); the subscription exists
1063
3183
  // for the ERROR channel: an errored actor (an invoke threw — e.g. ADR-0011's invoke-time
@@ -1065,11 +3185,26 @@ export class RunHost {
1065
3185
  // `{}`), then persist: the snapshot's "error" status stores the run and untracks it.
1066
3186
  actor.subscribe({
1067
3187
  error: (err) => {
3188
+ // A Failure that no state caught (ADR-0073): the end the Machine chose, so the run ends
3189
+ // `failed`, with no fault and the Failure as its output, and its Sandboxes go as a cancel's
3190
+ // do (ADR-0025). Named BEFORE the stop, while the tree still says which Workspaces are live.
3191
+ if (isFailure(err)) {
3192
+ run.failure = failurePayload(err);
3193
+ const sandboxes = runSandboxes(actor, record.runId);
3194
+ this.persist(run, "failed");
3195
+ stopTree(actor);
3196
+ void this.destroySandboxes(record.runId, sandboxes, "the end of the failed run");
3197
+ return;
3198
+ }
1068
3199
  run.fault = err instanceof Error ? err.message : String(err);
1069
3200
  this.persist(run);
1070
- // Nothing to release: a faulted run stops its actors, and each workspace's lease is one
1071
- // of them (ADR-0021). The pod stays up for inspection (ADR-0012's destroy-less terminal)
1072
- // and ages out of the operator's idle timeout on its own.
3201
+ // A faulted run stops its actors — each workspace's lease among them (ADR-0021), and a
3202
+ // Workspace still `placing` deletes the Sandbox that never ran (ADR-0064). Said here, not
3203
+ // assumed: xstate ends a machine on an unhandled child error WITHOUT stopping its other
3204
+ // children, so a lease beside the actor that failed would renew for nobody, forever. The
3205
+ // pod that was placed stays up for inspection (ADR-0012's destroy-less terminal) and ages
3206
+ // out of the operator's idle timeout on its own.
3207
+ stopTree(actor);
1073
3208
  },
1074
3209
  });
1075
3210
  // (The author's `emit({...})` forwarding lives in `spawn`'s inspect handler — per actor,
@@ -1093,11 +3228,16 @@ export class RunHost {
1093
3228
  /**
1094
3229
  * Persist the run's snapshot after a transition; drop it from the registry once final.
1095
3230
  *
1096
- * `terminal` is the run-lifecycle verdict the MACHINE cannot supply: `cancel()` passes
1097
- * "cancelled", because xstate only knows its actor was stopped (ADR-0025). It is the stored
1098
- * status and the one the feeds report.
3231
+ * `terminal` is the run-lifecycle verdict the MACHINE cannot supply. Three callers pass it:
3232
+ * `cancel()` passes "cancelled", because xstate only knows its actor was stopped (ADR-0025); the
3233
+ * error observer passes "failed" for a Failure that no state caught, because xstate gives a
3234
+ * Failure and a defect the same `error` status (ADR-0073); and `faultUnrouted()` passes "error"
3235
+ * for a mechanism event that no transition took, because the host stops the actor and xstate
3236
+ * then knows only that it was stopped (ADR-0074). It is the stored status and the one the feeds report.
1099
3237
  */
1100
3238
  private persist(run: LiveRun, terminal?: string): void {
3239
+ // The holds first: an answer read at Rest, and a notice a route out raised, ride this same save.
3240
+ this.settleHolds(run);
1101
3241
  const snapshot = run.actor.getPersistedSnapshot();
1102
3242
  const machineStatus = (snapshot as { status?: string }).status ?? "active";
1103
3243
  const serialized = serializeSnapshot(snapshot, {
@@ -1107,16 +3247,36 @@ export class RunHost {
1107
3247
  const blob: RunBlob = {
1108
3248
  workflow: run.record.workflow,
1109
3249
  instanceId: run.record.instanceId,
3250
+ ...(run.record.dispatchedBy ? { dispatchedBy: run.record.dispatchedBy } : {}),
1110
3251
  // Stamped on every save, off the registered TEMPLATE (never the per-run `.provide()` result —
1111
3252
  // providers do not change shape, and `fingerprintOf` memoizes per machine object). ADR-0030.
1112
3253
  machine: fingerprintOf(run.def.machine),
3254
+ // The Assembly the run started under (ADR-0069): the def it was spawned with, never the
3255
+ // current one, so a publish mid-run does not restamp it, and a restore spawns it with the
3256
+ // hash its blob named.
3257
+ ...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
1113
3258
  snapshot: serialized,
1114
3259
  agents: run.agents,
1115
3260
  epochs: run.epochs,
3261
+ notices: run.notices.state(),
3262
+ timers: pendingTimers(run.actor, this.now()),
3263
+ holds: [...run.holds.values()],
3264
+ effects: run.effects,
3265
+ startedBy: liveStartedBy(run.startedBy, run.actor),
1116
3266
  fault: run.fault,
3267
+ ...(run.failure !== undefined ? { failure: run.failure } : {}),
1117
3268
  };
1118
3269
  const status = terminal ?? (machineStatus === "active" ? "live" : machineStatus);
1119
- void this.store.save(run.record.runId, blob, status);
3270
+ // A save that fails is this run's fault, never the process's: an unhandled rejection would end
3271
+ // the Orchestrator and every run on it. The run keeps its last saved snapshot.
3272
+ this.store.save(run.record.runId, blob, status).catch((err: unknown) => {
3273
+ console.error(
3274
+ `run ${run.record.runId}: its snapshot was not saved: ${err instanceof Error ? err.message : String(err)}`,
3275
+ );
3276
+ });
3277
+ // The save is on disk (the store writes synchronously, ADR-0065): a settled row whose transition
3278
+ // it carries keeps no tail in the next one (ADR-0067).
3279
+ dropSavedTails(run.effects, snapshot);
1120
3280
 
1121
3281
  // Feed per-run observers (SSE/CLI watch). On the terminal transition emit the final status
1122
3282
  // BEFORE untrack drops the run from the registry, then drop the now-useless listener set.
@@ -1133,8 +3293,116 @@ export class RunHost {
1133
3293
  this.announceGone(run, runStatus);
1134
3294
  this.untrack(run.record);
1135
3295
  run.listeners.clear();
3296
+ // After the save and the untrack: a listener that reads the run back reads it settled.
3297
+ this.fireSettled(run.record.runId, runStatus);
1136
3298
  } else {
1137
3299
  this.announce(run.record.workflow, { kind: "status", status: runStatus });
1138
3300
  }
1139
3301
  }
1140
3302
  }
3303
+
3304
+ /** A status that is no longer live (ADR-0071): the store's terminal words. A live row reads its
3305
+ * Machine's `active`; the store's own `live` is never a status a reader is handed, but is not
3306
+ * settled either. */
3307
+ function settledStatus(status: string): boolean {
3308
+ return status !== "active" && status !== "live";
3309
+ }
3310
+
3311
+ /** Call a settle listener; one that throws is its own fault, never the run's or the host's. */
3312
+ function hear(listener: (status: RunStatus) => void, status: RunStatus): void {
3313
+ try {
3314
+ listener(status);
3315
+ } catch (err) {
3316
+ console.error(`run ${status.runId}: a settle listener threw: ${err instanceof Error ? err.message : String(err)}`);
3317
+ }
3318
+ }
3319
+
3320
+ /** Does work still run below the Agent's state of `reg` — a live child of an active state under it?
3321
+ * A timer that holds Rest is no work: when the state exits, nothing it started is stopped. */
3322
+ function stillRunning(reg: Registration): boolean {
3323
+ if (!reg.invoker || !reg.invokeId) return false;
3324
+ const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
3325
+ return seat !== undefined && runningBelow(reg.invoker, seat).length > 0;
3326
+ }
3327
+
3328
+ /** `actor` is `ref`, or `ref` is its parent, its parent's parent, and so on up to the root. */
3329
+ function atOrBelow(actor: AnyActorRef, ref: AnyActorRef): boolean {
3330
+ for (let at: AnyActorRef | undefined = actor; at !== undefined; at = at._parent) if (at === ref) return true;
3331
+ return false;
3332
+ }
3333
+
3334
+ /** A status's bound fields (ADR-0070): `bound`, and the hashes of the Charts bound, distinct and
3335
+ * sorted. Both absent when the snapshot binds no Chart. */
3336
+ /** The runs a run's `dispatch` Effects started (ADR-0071), from its ledger: an intent row the
3337
+ * start's answer re-wrote with its run, a settled row whose output names one (`started`,
3338
+ * `settled`), and an aborted row that kept it. A row that never learned its run names none. */
3339
+ function dispatchedField(effects: Record<string, EffectRow>): { dispatched?: Dispatched[] } {
3340
+ const out: Dispatched[] = [];
3341
+ for (const row of Object.values(effects)) {
3342
+ if (row.kind !== "dispatch") continue;
3343
+ const run =
3344
+ row.state === "settled"
3345
+ ? (row.output as { run?: unknown } | null | undefined)?.run
3346
+ : (row as { run?: unknown }).run;
3347
+ if (typeof run === "string" && run !== "") out.push({ slot: row.slot, run, at: row.at });
3348
+ }
3349
+ out.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
3350
+ return out.length ? { dispatched: out } : {};
3351
+ }
3352
+
3353
+ function boundFields(snap: unknown): { bound?: true; chartHashes?: string[] } {
3354
+ const hashes = [...new Set(boundChartRoutesOf(snap).map((route) => route.chartHash))].sort();
3355
+ return hashes.length ? { bound: true, chartHashes: hashes } : {};
3356
+ }
3357
+
3358
+ /** A live actor's children, by id. */
3359
+ function childrenOfRef(ref: AnyActorRef): Record<string, AnyActorRef> {
3360
+ return (ref.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
3361
+ }
3362
+
3363
+ /** A stored run's bound Chart, compiled again from its text under the Open body it binds, for the
3364
+ * run's doc — or why it is not: the doc then keeps the slot's subgraph and names the reason. */
3365
+ function compileStored(machine: AnyStateMachine, route: BoundChartRoute): AnyStateMachine | string {
3366
+ let at: AnyStateMachine = machine;
3367
+ for (const src of route.srcs.slice(0, -1)) {
3368
+ const logic = (at.implementations.actors as Record<string, unknown>)[src];
3369
+ if (isOpenBody(logic)) continue;
3370
+ if (!(logic as AnyStateMachine | undefined)?.root) return `no Machine at slot "${src}"`;
3371
+ at = logic as AnyStateMachine;
3372
+ }
3373
+ const logic = (at.implementations.actors as Record<string, unknown>)[route.src];
3374
+ if (!isOpenBody(logic)) return `slot "${route.src}" is no Open body`;
3375
+ const decl = openBodyDeclarationOf(logic);
3376
+ const lent = lentIn(at.implementations as never, decl);
3377
+ try {
3378
+ return compileChart(admitChart(route.chart, decl, lent), lent, decl, { id: route.src });
3379
+ } catch (err) {
3380
+ return err instanceof ChartRefusal
3381
+ ? `the door refuses the Chart: ${err.report.join("; ")}`
3382
+ : err instanceof Error
3383
+ ? err.message
3384
+ : String(err);
3385
+ }
3386
+ }
3387
+
3388
+ /** Put a compiled Chart's body under the states of `doc` that invoke the Open body at the end of
3389
+ * `srcs`, descending through the child Machine docs the earlier slots name. */
3390
+ function placeBody(doc: MachineBodyDoc, srcs: readonly string[], body: MachineBodyDoc): void {
3391
+ const [src, ...rest] = srcs;
3392
+ if (src === undefined) return;
3393
+ const visit = (state: MachineStateDoc): void => {
3394
+ if (rest.length === 0) {
3395
+ for (const inv of state.invoke) {
3396
+ if (inv.src !== src) continue;
3397
+ const child: ChildMachineDoc = { src, label: inv.id, via: "invoke", machine: body };
3398
+ const at = state.children.findIndex((c) => c.src === src);
3399
+ if (at >= 0) state.children[at] = child;
3400
+ else state.children.push(child);
3401
+ }
3402
+ } else {
3403
+ for (const child of state.children) if (child.src === src && child.machine) placeBody(child.machine, rest, body);
3404
+ }
3405
+ for (const nested of state.states) visit(nested);
3406
+ };
3407
+ visit(doc.root);
3408
+ }