@jr2/orchestrator 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/console/components/app.ts +55 -15
- package/console/components/bottom-drawer.ts +259 -0
- package/console/components/drawer.ts +14 -6
- package/console/components/fleet.ts +36 -13
- package/console/components/machine-pane.ts +61 -5
- package/console/components/nav.ts +54 -8
- package/console/conversation.ts +161 -0
- package/console/login.ts +123 -0
- package/console/main.ts +600 -60
- package/console/store.ts +611 -64
- package/console/style.css +231 -0
- package/docs/agents-block.md +2 -2
- package/docs/concepts/agents-and-turns.md +86 -9
- package/docs/concepts/architecture.md +41 -28
- package/docs/concepts/callers-and-hooks.md +205 -0
- package/docs/concepts/charts.md +7 -5
- package/docs/concepts/composition.md +10 -1
- package/docs/concepts/converge.md +4 -2
- package/docs/concepts/durable-runs.md +43 -16
- package/docs/concepts/effects.md +50 -22
- package/docs/concepts/gates.md +95 -25
- package/docs/concepts/machines-and-workflows.md +5 -4
- package/docs/concepts/pools-and-sources.md +187 -11
- package/docs/concepts/secrets-and-trust.md +73 -33
- package/docs/concepts/the-menu.md +11 -0
- package/docs/concepts/workspaces.md +6 -5
- package/docs/get-started/a-model-decides.md +14 -13
- package/docs/get-started/an-agent-in-a-sandbox.md +43 -34
- package/docs/get-started/connect-a-model.md +1 -1
- package/docs/get-started/create-an-instance.md +17 -14
- package/docs/guides/add-tools-to-the-sandbox.md +55 -11
- package/docs/guides/answer-a-gate-from-a-forge.md +304 -0
- package/docs/guides/answer-a-gate.md +74 -37
- package/docs/guides/bind-open-parts.md +24 -8
- package/docs/guides/call-an-mcp-tool-from-a-machine.md +572 -0
- package/docs/guides/cancel-a-run.md +26 -16
- package/docs/guides/fix-a-failed-converge.md +18 -6
- package/docs/guides/give-a-caller-a-scope.md +203 -0
- package/docs/guides/give-an-agent-mcp-tools.md +422 -0
- package/docs/guides/iterate-on-a-machine-on-kind.md +10 -9
- package/docs/guides/let-an-agent-write-a-chart.md +9 -6
- package/docs/guides/look-inside-a-sandbox.md +101 -31
- package/docs/guides/open-the-console.md +63 -14
- package/docs/guides/read-a-conversation.md +225 -0
- package/docs/guides/run-a-chart-by-name.md +17 -16
- package/docs/guides/run-one-worker-per-item.md +15 -9
- package/docs/guides/set-an-agents-access-and-dials.md +2 -0
- package/docs/guides/start-a-run-for-each-alert.md +290 -0
- package/docs/guides/start-a-run-from-a-run.md +18 -16
- package/docs/guides/start-a-run.md +24 -18
- package/docs/guides/steer-an-agent.md +332 -0
- package/docs/guides/use-a-hosted-provider.md +1 -1
- package/docs/guides/watch-a-run.md +127 -121
- package/docs/guides/write-a-hook-kind.md +238 -0
- package/docs/guides/write-a-tool-as-an-mcp-server.md +351 -0
- package/docs/guides/write-a-workflow.md +17 -7
- package/docs/llms.txt +81 -15
- package/docs/reference/authoring-surface/agent.md +1 -1
- package/docs/reference/authoring-surface/answer.md +1 -1
- package/docs/reference/authoring-surface/arg.md +1 -1
- package/docs/reference/authoring-surface/basic.md +1 -1
- package/docs/reference/authoring-surface/bearer.md +1 -1
- package/docs/reference/authoring-surface/branchname.md +1 -1
- package/docs/reference/authoring-surface/carry.md +23 -0
- package/docs/reference/authoring-surface/chartschema.md +1 -1
- package/docs/reference/authoring-surface/chartschemafor.md +1 -1
- package/docs/reference/authoring-surface/command.md +1 -1
- package/docs/reference/authoring-surface/customize.md +4 -1
- package/docs/reference/authoring-surface/defineconfig.md +1 -1
- package/docs/reference/authoring-surface/defineevent.md +1 -1
- package/docs/reference/authoring-surface/dispatch.md +5 -4
- package/docs/reference/authoring-surface/fail.md +1 -1
- package/docs/reference/authoring-surface/file.md +1 -1
- package/docs/reference/authoring-surface/frompick.md +1 -1
- package/docs/reference/authoring-surface/gate.md +118 -0
- package/docs/reference/authoring-surface/group-agents.md +15 -0
- package/docs/reference/authoring-surface/group-charts.md +16 -0
- package/docs/reference/authoring-surface/group-composition.md +25 -0
- package/docs/reference/authoring-surface/group-config.md +17 -0
- package/docs/reference/authoring-surface/group-effects.md +48 -0
- package/docs/reference/authoring-surface/group-failure.md +13 -0
- package/docs/reference/authoring-surface/group-gates.md +16 -0
- package/docs/reference/authoring-surface/group-machine-and-events.md +15 -0
- package/docs/reference/authoring-surface/group-steering.md +12 -0
- package/docs/reference/authoring-surface/group-types-these-signatures-name.md +137 -0
- package/docs/reference/authoring-surface/group-workspaces-and-repos.md +15 -0
- package/docs/reference/authoring-surface/http.md +1 -1
- package/docs/reference/authoring-surface/isfailure.md +1 -1
- package/docs/reference/authoring-surface/isready.md +1 -1
- package/docs/reference/authoring-surface/jr2setup.md +13 -3
- package/docs/reference/authoring-surface/mcp.md +107 -0
- package/docs/reference/authoring-surface/open.md +5 -3
- package/docs/reference/authoring-surface/openbody.md +1 -1
- package/docs/reference/authoring-surface/paramsof.md +1 -1
- package/docs/reference/authoring-surface/pool.md +77 -9
- package/docs/reference/authoring-surface/push.md +1 -1
- package/docs/reference/authoring-surface/readyof.md +1 -1
- package/docs/reference/authoring-surface/reread.md +19 -0
- package/docs/reference/authoring-surface/secret.md +1 -1
- package/docs/reference/authoring-surface/sentby.md +1 -1
- package/docs/reference/authoring-surface/skip.md +22 -0
- package/docs/reference/authoring-surface/source.md +56 -1
- package/docs/reference/authoring-surface/steer.md +93 -0
- package/docs/reference/authoring-surface/text.md +1 -1
- package/docs/reference/authoring-surface/type-agentdefinition.md +14 -1
- package/docs/reference/authoring-surface/type-agentmcpserver.md +54 -0
- package/docs/reference/authoring-surface/type-anyframepath.md +1 -1
- package/docs/reference/authoring-surface/type-anyworktreepath.md +1 -1
- package/docs/reference/authoring-surface/type-binding.md +1 -1
- package/docs/reference/authoring-surface/type-caller.md +40 -0
- package/docs/reference/authoring-surface/type-callerrecord.md +32 -0
- package/docs/reference/authoring-surface/type-carry.md +19 -0
- package/docs/reference/authoring-surface/type-chainlink.md +3 -3
- package/docs/reference/authoring-surface/type-commandoutput.md +1 -1
- package/docs/reference/authoring-surface/type-customize.md +16 -1
- package/docs/reference/authoring-surface/type-delivered.md +17 -0
- package/docs/reference/authoring-surface/type-deliveredevent.md +18 -0
- package/docs/reference/authoring-surface/type-dispatchdefinition.md +4 -3
- package/docs/reference/authoring-surface/type-dispatchoutput.md +1 -1
- package/docs/reference/authoring-surface/type-effectbinding.md +12 -2
- package/docs/reference/authoring-surface/type-framepath.md +1 -1
- package/docs/reference/authoring-surface/type-gatebinding.md +15 -0
- package/docs/reference/authoring-surface/type-gatehooks.md +12 -0
- package/docs/reference/authoring-surface/type-gatelook.md +37 -0
- package/docs/reference/authoring-surface/type-gateoutcome.md +32 -0
- package/docs/reference/authoring-surface/type-gatereply.md +19 -0
- package/docs/reference/authoring-surface/type-gateslot.md +21 -0
- package/docs/reference/authoring-surface/type-gateslotinput.md +13 -0
- package/docs/reference/authoring-surface/type-gateslotparts.md +18 -0
- package/docs/reference/authoring-surface/type-gateslots.md +12 -0
- package/docs/reference/authoring-surface/type-gatetypesat.md +13 -0
- package/docs/reference/authoring-surface/type-hostinjectedinput.md +1 -1
- package/docs/reference/authoring-surface/type-httpoutput.md +1 -1
- package/docs/reference/authoring-surface/type-jr2config.md +37 -2
- package/docs/reference/authoring-surface/type-lateroutcome.md +1 -1
- package/docs/reference/authoring-surface/type-lends.md +5 -1
- package/docs/reference/authoring-surface/type-lookoutput.md +14 -0
- package/docs/reference/authoring-surface/type-mcpcalloutput.md +78 -0
- package/docs/reference/authoring-surface/type-mcpnotreturned.md +42 -0
- package/docs/reference/authoring-surface/type-mcpoptions.md +47 -0
- package/docs/reference/authoring-surface/type-mcpoutput.md +50 -0
- package/docs/reference/authoring-surface/type-mcpreadoutput.md +53 -0
- package/docs/reference/authoring-surface/type-mcpremoteserver.md +50 -0
- package/docs/reference/authoring-surface/type-mcpserver.md +41 -0
- package/docs/reference/authoring-surface/type-mcpstdioserver.md +42 -0
- package/docs/reference/authoring-surface/type-namedsecret.md +4 -4
- package/docs/reference/authoring-surface/type-openbodydeclaration.md +1 -1
- package/docs/reference/authoring-surface/type-openbodyoutput.md +1 -1
- package/docs/reference/authoring-surface/type-poolnextinput.md +1 -1
- package/docs/reference/authoring-surface/type-poolspec.md +11 -6
- package/docs/reference/authoring-surface/type-pushoutput.md +1 -1
- package/docs/reference/authoring-surface/type-pushsourcespec.md +38 -0
- package/docs/reference/authoring-surface/type-readsourcespec.md +47 -0
- package/docs/reference/authoring-surface/type-reposlot.md +1 -1
- package/docs/reference/authoring-surface/type-reread.md +18 -0
- package/docs/reference/authoring-surface/type-runsconfig.md +41 -0
- package/docs/reference/authoring-surface/type-sender.md +26 -1
- package/docs/reference/authoring-surface/type-sendercallerfield.md +14 -0
- package/docs/reference/authoring-surface/type-size.md +1 -1
- package/docs/reference/authoring-surface/type-skip.md +19 -0
- package/docs/reference/authoring-surface/type-sourcehooks.md +23 -0
- package/docs/reference/authoring-surface/type-sourcein.md +25 -0
- package/docs/reference/authoring-surface/type-sourcereply.md +18 -0
- package/docs/reference/authoring-surface/type-sourcespec.md +16 -7
- package/docs/reference/authoring-surface/type-sourceverdict.md +49 -0
- package/docs/reference/authoring-surface/type-standingpoolmachine.md +12 -0
- package/docs/reference/authoring-surface/type-standingpoolspec.md +51 -0
- package/docs/reference/authoring-surface/type-steerargs.md +20 -0
- package/docs/reference/authoring-surface/type-thinkinglevel.md +1 -1
- package/docs/reference/authoring-surface/type-usercontainer.md +1 -1
- package/docs/reference/authoring-surface/type-workspaced.md +1 -1
- package/docs/reference/authoring-surface/type-worktreefound.md +3 -3
- package/docs/reference/authoring-surface/type-worktreeinput.md +1 -1
- package/docs/reference/authoring-surface/type-worktreeoptions.md +1 -1
- package/docs/reference/authoring-surface/type-worktreeoutput.md +1 -1
- package/docs/reference/authoring-surface/type-worktreepath.md +1 -1
- package/docs/reference/authoring-surface/type-worktreeready.md +1 -1
- package/docs/reference/authoring-surface/type-worktreerefused.md +4 -4
- package/docs/reference/authoring-surface/workspace.md +1 -1
- package/docs/reference/authoring-surface/worktree.md +1 -1
- package/docs/reference/authoring-surface.md +88 -167
- package/docs/reference/cli.md +413 -74
- package/docs/reference/config.md +235 -2
- package/docs/reference/decisions/0002-duplex-streaming-actor.md +71 -11
- package/docs/reference/decisions/0005-sandbox-pod-composition.md +14 -10
- package/docs/reference/decisions/0006-agent-control-surface.md +4 -4
- package/docs/reference/decisions/0007-durable-machine-state.md +1 -1
- package/docs/reference/decisions/0009-cli-and-instance-interface.md +150 -49
- package/docs/reference/decisions/0010-bdd-acceptance-tests.md +5 -3
- package/docs/reference/decisions/0011-workflow-defined-events.md +256 -12
- package/docs/reference/decisions/0012-workspace-wrapper-machine.md +9 -2
- package/docs/reference/decisions/0013-the-agent-reaches-its-machine-through-a-container-it-cannot-read.md +20 -14
- package/docs/reference/decisions/0014-observation-is-open-run-state-is-not.md +53 -27
- package/docs/reference/decisions/0015-authoring-surface-absorbs-the-mechanism.md +18 -14
- package/docs/reference/decisions/0017-pool-over-a-source-port.md +226 -12
- package/docs/reference/decisions/0018-instance-agents-are-definitions-jr2-assembles-the-harness.md +15 -7
- package/docs/reference/decisions/0019-one-converging-command-against-the-current-context.md +32 -18
- package/docs/reference/decisions/0021-workspace-continuity-is-a-lease-that-answers-back.md +14 -1
- package/docs/reference/decisions/0022-observation-is-a-level-triggered-feed.md +4 -4
- package/docs/reference/decisions/0023-the-harness-prints-the-conversation.md +116 -34
- package/docs/reference/decisions/0024-an-agents-turn-ends-with-the-state-that-asked-for-it.md +38 -37
- package/docs/reference/decisions/0025-cancel-ends-a-run-stop-parks-it.md +10 -5
- package/docs/reference/decisions/0027-the-harness-is-jr2s-own-server-flue-retires-the-wire-stays.md +7 -6
- package/docs/reference/decisions/0028-what-an-agent-may-do-to-the-workspace-is-part-of-its-definition.md +23 -7
- package/docs/reference/decisions/0029-a-menu-is-fixed-for-a-conversation-and-a-turn-is-told-its-allowed-picks.md +18 -10
- package/docs/reference/decisions/0030-a-snapshot-names-the-machine-it-was-written-under.md +3 -2
- package/docs/reference/decisions/0031-menu-only-agents-run-on-the-instance-harness.md +23 -18
- package/docs/reference/decisions/0032-the-console-unlocks-with-a-callers-credential.md +101 -0
- package/docs/reference/decisions/0033-a-machine-declares-the-input-that-starts-it.md +4 -3
- package/docs/reference/decisions/0034-the-console-renders-with-preact-and-the-server-erases-the-types.md +6 -6
- package/docs/reference/decisions/0037-an-instance-builds-its-sandbox-images-jr2-injects-the-harness.md +5 -4
- package/docs/reference/decisions/0049-a-machine-carries-its-parts-and-composes-by-invoke.md +19 -19
- package/docs/reference/decisions/0050-what-code-names-config-declares-what-the-cli-names-discovery-finds.md +6 -1
- package/docs/reference/decisions/0054-the-kit-ships-machines-and-a-shipped-machine-leaves-its-parts-open.md +55 -40
- package/docs/reference/decisions/0055-a-release-is-a-pushed-tag-and-the-publish-guard-is-the-credential.md +14 -13
- package/docs/reference/decisions/0057-a-turn-is-its-frame-its-dials-and-whether-it-continues.md +7 -0
- package/docs/reference/decisions/0058-a-harness-answers-the-orchestrator-alone.md +13 -7
- package/docs/reference/decisions/0059-a-harness-holds-stand-ins-and-the-custodian-holds-the-keys.md +38 -30
- package/docs/reference/decisions/0062-jr2-briefs-the-agent-on-its-seat-never-on-its-task.md +64 -17
- package/docs/reference/decisions/0063-the-orchestrator-watches-the-cluster.md +2 -1
- package/docs/reference/decisions/0064-a-workspace-waits-for-capacity.md +2 -2
- package/docs/reference/decisions/0065-a-persist-survives-a-crash-and-history-is-kept-for-a-set-time.md +109 -0
- package/docs/reference/decisions/0066-a-model-tier-runs-a-real-model-and-a-change-to-what-it-reads-runs-the-tier.md +51 -23
- package/docs/reference/decisions/0067-an-effect-is-an-actor-slot-and-runs-at-most-once-unless-marked-idempotent.md +94 -54
- package/docs/reference/decisions/0068-an-effector-holds-an-effects-secrets-and-signs-its-wire.md +101 -47
- package/docs/reference/decisions/0069-the-orchestrator-is-a-kit-image-and-an-instance-publishes-its-assembly.md +32 -9
- package/docs/reference/decisions/0070-a-machine-lends-its-parts-to-a-chart.md +17 -13
- package/docs/reference/decisions/0071-a-run-starts-a-run-through-a-dispatch-effect.md +56 -30
- package/docs/reference/decisions/0072-the-docs-ship-inside-the-kit-and-an-instances-agents-md-indexes-them.md +35 -25
- package/docs/reference/decisions/0075-a-machine-cuts-a-worktree-and-the-workspace-only-clones.md +4 -3
- package/docs/reference/decisions/0076-an-agents-control-credential-lives-only-where-no-workspace-code-runs.md +37 -10
- package/docs/reference/decisions/0078-a-caller-comes-from-one-authenticator-seam-and-acts-within-its-scopes.md +285 -0
- package/docs/reference/decisions/0079-an-agent-holds-mcp-tools-a-machine-calls-them-as-an-effect-and-the-machine-is-the-approval.md +198 -0
- package/docs/reference/decisions.md +6 -4
- package/docs/reference/glossary.md +96 -42
- package/docs/reference/hooks.md +1077 -0
- package/docs/reference/http-api.md +976 -149
- package/docs/why.md +2 -1
- package/package.json +10 -3
- package/src/actor.ts +36 -10
- package/src/agent.ts +50 -0
- package/src/assembly-checks.ts +530 -4
- package/src/callers.ts +245 -0
- package/src/chart.ts +52 -7
- package/src/config.ts +550 -1
- package/src/conversations.ts +453 -0
- package/src/custodian.ts +8 -2
- package/src/customize.ts +143 -4
- package/src/dispatch-effect.ts +142 -26
- package/src/durability.ts +2 -0
- package/src/duration.ts +20 -0
- package/src/effect-checks.ts +2 -1
- package/src/effect.ts +60 -7
- package/src/effector-wire.ts +272 -27
- package/src/gate-brand.ts +16 -0
- package/src/gate.ts +634 -48
- package/src/harness-client.ts +10 -4
- package/src/held-secrets.ts +72 -23
- package/src/holds.ts +2 -2
- package/src/hook-kinds/discord.ts +92 -0
- package/src/hook-kinds/door.ts +107 -0
- package/src/hook-kinds/github.ts +89 -0
- package/src/hook-kinds/gitlab.ts +72 -0
- package/src/hook-kinds/hook.ts +7 -0
- package/src/hook-kinds/index.ts +465 -0
- package/src/hook-kinds/slack.ts +151 -0
- package/src/hook-kinds/webhook.ts +113 -0
- package/src/hooks.ts +506 -0
- package/src/http-effect.ts +76 -38
- package/src/http.ts +889 -94
- package/src/index.ts +16 -5
- package/src/instance.ts +196 -26
- package/src/internal.ts +22 -3
- package/src/machine-doc.ts +26 -1
- package/src/mcp-effect.ts +424 -0
- package/src/mount-digest.ts +41 -7
- package/src/names.ts +64 -2
- package/src/notices.ts +138 -10
- package/src/open-body.ts +25 -11
- package/src/open-effect.ts +17 -4
- package/src/open.ts +4 -2
- package/src/parts.ts +87 -9
- package/src/party.ts +2 -1
- package/src/pick.ts +2 -2
- package/src/pool.ts +1142 -163
- package/src/registration.ts +203 -13
- package/src/run-host.ts +1509 -171
- package/src/run-list.ts +191 -0
- package/src/sandbox-kube.ts +53 -5
- package/src/secret-placement.ts +239 -33
- package/src/server.ts +53 -4
- package/src/setup.ts +72 -32
- package/src/snapshot-store.ts +565 -48
- package/src/steer.ts +111 -0
- package/src/stub-harness.ts +6 -2
- package/src/tokens.ts +263 -22
- package/src/wire.ts +17 -0
- package/src/workspace.ts +50 -6
- package/src/worktree.ts +3 -3
- package/docs/reference/decisions/0032-the-console-unlocks-with-the-instance-token.md +0 -61
- package/docs/reference/decisions/0065-a-persist-survives-a-crash-and-history-is-a-week.md +0 -65
package/src/run-host.ts
CHANGED
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
// persisted — so the snapshot is JSON-safe and restore re-attaches by rewriting the child's
|
|
20
20
|
// persisted input (drop `prompt`, set `attach` from the ledger) rather than re-POSTing the prompt.
|
|
21
21
|
|
|
22
|
-
import { randomUUID } from "node:crypto";
|
|
22
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
23
23
|
import { z } from "zod";
|
|
24
24
|
import {
|
|
25
25
|
createActor,
|
|
@@ -30,8 +30,15 @@ import {
|
|
|
30
30
|
type AnyMachineSnapshot,
|
|
31
31
|
type AnyStateMachine,
|
|
32
32
|
} from "xstate";
|
|
33
|
-
import {
|
|
33
|
+
import {
|
|
34
|
+
SENDER_KEY,
|
|
35
|
+
type EventDef,
|
|
36
|
+
type EventSemantics,
|
|
37
|
+
type Sender,
|
|
38
|
+
type SenderCallerField,
|
|
39
|
+
} from "@jr2/agent-protocol";
|
|
34
40
|
import { partiesAt, seatOf, senderText } from "./party.ts";
|
|
41
|
+
import { GATE_ANSWERS_KEPT, type CallerRecord, type GateAnswer, type RunStarter } from "./callers.ts";
|
|
35
42
|
import { placeAnswers, type AnswerSeat } from "./pick.ts";
|
|
36
43
|
import { inputSchemaOf } from "./vocabulary.ts";
|
|
37
44
|
import type { EchoEvent, EchoStatusChild, LiveConversation, Notice, PickAnswer } from "./wire.ts";
|
|
@@ -41,14 +48,18 @@ import { restOf, runningBelow } from "./rest.ts";
|
|
|
41
48
|
import {
|
|
42
49
|
acceptsNow,
|
|
43
50
|
agentAddress,
|
|
51
|
+
agentSlot,
|
|
44
52
|
allowedPicks,
|
|
45
53
|
bindRun,
|
|
54
|
+
dispatchKeyOf,
|
|
46
55
|
EventValidationError,
|
|
47
56
|
gateAddress,
|
|
48
57
|
NotTakenError,
|
|
49
58
|
OutstandingError,
|
|
50
59
|
RegistrationTable,
|
|
51
60
|
refuseSenderField,
|
|
61
|
+
slotPath,
|
|
62
|
+
type SenderCaller,
|
|
52
63
|
UnknownAddressError,
|
|
53
64
|
wouldMove,
|
|
54
65
|
targetsOf,
|
|
@@ -59,8 +70,10 @@ import {
|
|
|
59
70
|
type RetryTelemetry,
|
|
60
71
|
type RunBinding,
|
|
61
72
|
type RunMarker,
|
|
73
|
+
type SourceVerdictLine,
|
|
62
74
|
} from "./registration.ts";
|
|
63
|
-
import { NoticeLedger, type NoticeLedgerState } from "./notices.ts";
|
|
75
|
+
import { NoticeLedger, type LedgerBody, type NoticeLedgerState, type SteeringRecord } from "./notices.ts";
|
|
76
|
+
import { isAgent } from "./agent.ts";
|
|
64
77
|
import { pendingTimers, startAndRearm, watchTimers, type PendingTimer } from "./timers.ts";
|
|
65
78
|
import {
|
|
66
79
|
DEFAULT_LEASE_INTERVAL_MS,
|
|
@@ -82,8 +95,24 @@ import {
|
|
|
82
95
|
type MachineDoc,
|
|
83
96
|
type MachineStateDoc,
|
|
84
97
|
} from "./machine-doc.ts";
|
|
85
|
-
import type { SnapshotStore } from "./snapshot-store.ts";
|
|
98
|
+
import type { EndedRow, SnapshotStore, StandingHead } from "./snapshot-store.ts";
|
|
99
|
+
import {
|
|
100
|
+
encodeCursor,
|
|
101
|
+
isAfter,
|
|
102
|
+
LIVE_STATUS,
|
|
103
|
+
newestFirst,
|
|
104
|
+
placingOf,
|
|
105
|
+
shownOfRow,
|
|
106
|
+
type EndedStatus,
|
|
107
|
+
type RunListQuery,
|
|
108
|
+
type RunPage,
|
|
109
|
+
type RunSummary,
|
|
110
|
+
type ShownStatus,
|
|
111
|
+
} from "./run-list.ts";
|
|
112
|
+
import { grantOpens } from "./tokens.ts";
|
|
113
|
+
import { HAND_OVER, standingCarryOf, standingPoolOf, standingTaken, standingTook } from "./pool.ts";
|
|
86
114
|
import type { AgentAdmission } from "./actor.ts";
|
|
115
|
+
import type { AnyHandler } from "./gate.ts";
|
|
87
116
|
import { dropSavedTails, type EffectRow } from "./effect.ts";
|
|
88
117
|
import type { EffectorPlacement, EffectorTarget } from "./http-effect.ts";
|
|
89
118
|
import { reattachAgentRuns, serializeSnapshot } from "./durability.ts";
|
|
@@ -221,8 +250,21 @@ function chartDrift(machine: AnyStateMachine, routes: readonly BoundChartRoute[]
|
|
|
221
250
|
|
|
222
251
|
/** The serializable identity of a run — what reconcile sees and what restore rebuilds from.
|
|
223
252
|
* `dispatchedBy` is the run that started it through a `dispatch` Effect, and this run's depth in
|
|
224
|
-
* that chain, as the caller sent it (ADR-0071); absent for a run a human or a client started.
|
|
225
|
-
|
|
253
|
+
* that chain, as the caller sent it (ADR-0071); absent for a run a human or a client started.
|
|
254
|
+
* `startedBy` is who started it (ADR-0078): a Caller, a run, or jr2. Behind the token, like
|
|
255
|
+
* `context`: {@link observe} never carries it. Absent for a run that a host started with no
|
|
256
|
+
* starter, as a test does. */
|
|
257
|
+
export type RunRecord = {
|
|
258
|
+
runId: string;
|
|
259
|
+
workflow: string;
|
|
260
|
+
instanceId: string;
|
|
261
|
+
/** When the run started, ISO-8601 (ADR-0009): the order of `GET /runs`. */
|
|
262
|
+
startedAt: string;
|
|
263
|
+
dispatchedBy?: DispatchedBy;
|
|
264
|
+
startedBy?: RunStarter;
|
|
265
|
+
/** The Caller whose `jr2 run` resumed this standing run (ADR-0017, ADR-0078). */
|
|
266
|
+
resumedBy?: CallerRecord;
|
|
267
|
+
};
|
|
226
268
|
|
|
227
269
|
/** A start of a name no Workflow is registered under: the door answers 404 (ADR-0009), and an
|
|
228
270
|
* in-process `dispatch` settles `refused` with status 404 (ADR-0071). */
|
|
@@ -230,33 +272,9 @@ export class UnknownWorkflowError extends Error {
|
|
|
230
272
|
override name = "UnknownWorkflowError";
|
|
231
273
|
}
|
|
232
274
|
|
|
233
|
-
/**
|
|
234
|
-
*
|
|
235
|
-
|
|
236
|
-
export class DispatchKeyConflictError extends Error {
|
|
237
|
-
override name = "DispatchKeyConflictError";
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
/** The run a key names (ADR-0071), with its Workflow so a start of another can be refused. */
|
|
241
|
-
export type DispatchedRun = { runId: string; instanceId: string; workflow: string };
|
|
242
|
-
|
|
243
|
-
/** The run `found` as the answer to a start of `workflow` under its key, or the 409 (ADR-0071). */
|
|
244
|
-
export function keyAnswer(
|
|
245
|
-
found: DispatchedRun,
|
|
246
|
-
workflow: string,
|
|
247
|
-
effect: string,
|
|
248
|
-
): { runId: string; instanceId: string } {
|
|
249
|
-
if (found.workflow !== workflow) throw new DispatchKeyConflictError(keyConflict(found, workflow, effect));
|
|
250
|
-
return { runId: found.runId, instanceId: found.instanceId };
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/** Why a start of `workflow` under `effect` is refused, when the key's run is of another. */
|
|
254
|
-
export function keyConflict(found: DispatchedRun, workflow: string, effect: string): string {
|
|
255
|
-
return (
|
|
256
|
-
`the key "${effect}" already started run ${found.runId} of "${found.workflow}", not of "${workflow}" — ` +
|
|
257
|
-
"one key is one start of one Workflow (ADR-0071)"
|
|
258
|
-
);
|
|
259
|
-
}
|
|
275
|
+
/** The run a key names (ADR-0071). The key is scoped to the callee Workflow, so the run is always
|
|
276
|
+
* of the Workflow that was asked. */
|
|
277
|
+
export type DispatchedRun = { runId: string; instanceId: string };
|
|
260
278
|
|
|
261
279
|
/**
|
|
262
280
|
* One live child MACHINE of a run, and its own children below it. A workflow's root machine is
|
|
@@ -375,6 +393,69 @@ function runSandboxes(actor: AnyActorRef, runId: string, opts: { placing?: boole
|
|
|
375
393
|
return out;
|
|
376
394
|
}
|
|
377
395
|
|
|
396
|
+
/** One Agent slot of a live run (ADR-0002), as `GET /runs/:runId/agents` lists it: `agent` is the
|
|
397
|
+
* slot (`<machine path>/<agent name>`), `open` says that a Turn of it is open now, and `steerable`
|
|
398
|
+
* that a Caller may send it a Steering message. */
|
|
399
|
+
export type RunAgent = { agent: string; open: boolean; steerable: boolean };
|
|
400
|
+
|
|
401
|
+
/** The `steering` field of a status: the record, when the run got a message. */
|
|
402
|
+
function steeringOf(record: readonly SteeringRecord[]): { steering?: SteeringRecord[] } {
|
|
403
|
+
return record.length > 0 ? { steering: record.map((r) => ({ ...r })) } : {};
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** Is this actor a Machine (not a promise, a callback, or an Agent)? */
|
|
407
|
+
function isMachineActor(ref: AnyActorRef): boolean {
|
|
408
|
+
const logic = (ref as { logic?: unknown }).logic;
|
|
409
|
+
return typeof logic === "object" && logic !== null && "root" in logic;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* Every Agent slot of every live Machine instance under `actor`, at any depth (ADR-0002): the
|
|
414
|
+
* Agents that each Machine carries in its `actors`, by the slot their Turns are registered with
|
|
415
|
+
* (`agentSlot`). A Machine that is not active holds none: it can take no Turn.
|
|
416
|
+
*/
|
|
417
|
+
function liveSlots(actor: AnyActorRef): Array<{ agent: string; steerable: boolean }> {
|
|
418
|
+
const out: Array<{ agent: string; steerable: boolean }> = [];
|
|
419
|
+
const walk = (ref: AnyActorRef): void => {
|
|
420
|
+
const snap = ref.getSnapshot() as { status?: string; children?: Record<string, AnyActorRef> } | undefined;
|
|
421
|
+
if (snap?.status !== "active") return;
|
|
422
|
+
if (isMachineActor(ref)) {
|
|
423
|
+
const actors = ((ref as { logic?: AnyStateMachine }).logic?.implementations.actors ?? {}) as Record<
|
|
424
|
+
string,
|
|
425
|
+
unknown
|
|
426
|
+
>;
|
|
427
|
+
for (const [name, logic] of Object.entries(actors)) {
|
|
428
|
+
if (isAgent(logic)) out.push({ agent: agentSlot(ref, name), steerable: logic.definition.steerable !== false });
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
for (const child of Object.values(snap.children ?? {})) if (child) walk(child);
|
|
432
|
+
};
|
|
433
|
+
walk(actor);
|
|
434
|
+
return out;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** Why `POST /runs/:runId/steering` refuses a message (ADR-0002): `status` is the HTTP status, and
|
|
438
|
+
* `agents` the run's slots when the slot named none of them. */
|
|
439
|
+
export class SteeringRefused extends Error {
|
|
440
|
+
readonly status: 403 | 409;
|
|
441
|
+
readonly agents?: RunAgent[];
|
|
442
|
+
constructor(message: string, status: 403 | 409, agents?: RunAgent[]) {
|
|
443
|
+
super(message);
|
|
444
|
+
this.status = status;
|
|
445
|
+
if (agents !== undefined) this.agents = agents;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** Why `POST /runs/:runId/gates/:gate/events` refuses a Caller (ADR-0078): the Gate is open, and
|
|
450
|
+
* the Caller's `answer` scope does not open it. `gate` is the Gate's name, its invoke id. */
|
|
451
|
+
export class GateRefused extends Error {
|
|
452
|
+
readonly gate: string;
|
|
453
|
+
constructor(gate: string) {
|
|
454
|
+
super(`the "answer" scope does not open Gate "${gate}"`);
|
|
455
|
+
this.gate = gate;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
378
459
|
/**
|
|
379
460
|
* Stop every actor under `actor`, at any depth, deepest first. A machine that ended in `error` has
|
|
380
461
|
* stopped processing — its own stop is a no-op — so its children are reached through its snapshot
|
|
@@ -407,11 +488,11 @@ type Sending = {
|
|
|
407
488
|
* pick was made at, which tells apart two states that invoke one slot, one after the other. `invoke` is absent when the event that
|
|
408
489
|
* started the child named the Agent and not where (a pick taken outside its own send).
|
|
409
490
|
*/
|
|
410
|
-
type
|
|
491
|
+
type ChildAgent = { agent: string; invoke?: string };
|
|
411
492
|
|
|
412
493
|
/**
|
|
413
494
|
* The key of the child `childId` of `parent` in the run's record of which Agent started each child
|
|
414
|
-
* (`
|
|
495
|
+
* (`childAgents`, ADR-0067): the invoke ids from below the root down to `parent`, then `childId`, each
|
|
415
496
|
* encoded and joined by `/`. An invoke id is local to its Machine — two Machines of one tree, and
|
|
416
497
|
* every worker of a pool, may each have a child `job` — and the chain is not. A child of the root
|
|
417
498
|
* keeps its bare id. The ids are the snapshot's, so a restore gives the same key.
|
|
@@ -424,16 +505,16 @@ function childKey(parent: AnyActorRef, childId: string): string {
|
|
|
424
505
|
|
|
425
506
|
/**
|
|
426
507
|
* The Agent whose event `event` is, as `taker` takes it (ADR-0024, ADR-0067): a pick's sender, or
|
|
427
|
-
* the Agent whose pick started the child of `taker` whose done or error event it is (`
|
|
508
|
+
* the Agent whose pick started the child of `taker` whose done or error event it is (`childAgents`).
|
|
428
509
|
* Undefined when the event names no Agent: a timer, a Gate's answer, an outside send, a child no
|
|
429
510
|
* Agent's pick started. `sending`'s invoke is where a pick in its own send was made.
|
|
430
511
|
*/
|
|
431
512
|
function agentOfEvent(
|
|
432
513
|
event: AnyEventObject,
|
|
433
|
-
|
|
514
|
+
childAgents: Record<string, ChildAgent>,
|
|
434
515
|
taker: AnyActorRef,
|
|
435
516
|
sending?: Sending,
|
|
436
|
-
):
|
|
517
|
+
): ChildAgent | undefined {
|
|
437
518
|
const sender = (event as { [SENDER_KEY]?: Sender })[SENDER_KEY];
|
|
438
519
|
if (sender?.party === "agent") {
|
|
439
520
|
const invoke = sending?.reg.sender?.id === sender.id ? sending.reg.invokeId : undefined;
|
|
@@ -442,7 +523,7 @@ function agentOfEvent(
|
|
|
442
523
|
const child = /^xstate\.(?:done|error)\.actor\.(.+)$/.exec(event.type)?.[1];
|
|
443
524
|
if (child === undefined) return undefined;
|
|
444
525
|
const key = childKey(taker, child);
|
|
445
|
-
return Object.hasOwn(
|
|
526
|
+
return Object.hasOwn(childAgents, key) ? childAgents[key] : undefined;
|
|
446
527
|
}
|
|
447
528
|
|
|
448
529
|
/** How many of `snapshot`'s active states invoke an Agent. With two or more, an event that names no
|
|
@@ -464,12 +545,12 @@ function liveChildKeys(actor: AnyActorRef, into = new Set<string>()): Set<string
|
|
|
464
545
|
return into;
|
|
465
546
|
}
|
|
466
547
|
|
|
467
|
-
/** `
|
|
548
|
+
/** `childAgents` as the save keeps it: only children still in the run's tree (ADR-0067). Read at the
|
|
468
549
|
* save, a microtask after the snapshot, so a done event the Machine took has run its route. */
|
|
469
|
-
function
|
|
550
|
+
function liveChildAgents(childAgents: Record<string, ChildAgent>, actor: AnyActorRef): Record<string, ChildAgent> {
|
|
470
551
|
const live = liveChildKeys(actor);
|
|
471
|
-
for (const id of Object.keys(
|
|
472
|
-
return
|
|
552
|
+
for (const id of Object.keys(childAgents)) if (!live.has(id)) delete childAgents[id];
|
|
553
|
+
return childAgents;
|
|
473
554
|
}
|
|
474
555
|
|
|
475
556
|
/** An `answer` action's value as the hold keeps it: JSON, because it rides the run blob and the
|
|
@@ -490,11 +571,84 @@ function heldOf(holds: Iterable<Hold>): RunHeld[] {
|
|
|
490
571
|
.map((hold) => ({ agent: hold.agent, pick: hold.pick, since: hold.since }));
|
|
491
572
|
}
|
|
492
573
|
|
|
574
|
+
/** The Caller of a Gate's event as its Sender carries it (ADR-0011): `via` and `name` are always
|
|
575
|
+
* checked, because an authenticator made them; `onBehalfOf` only for a Hook, whose signature covers
|
|
576
|
+
* the payload that it comes from. */
|
|
577
|
+
function senderCallerOf(by: CallerRecord): SenderCaller {
|
|
578
|
+
const checked: SenderCallerField[] = ["via", "name"];
|
|
579
|
+
if (by.via === "hook" && by.onBehalfOf !== undefined) checked.push("onBehalfOf");
|
|
580
|
+
return {
|
|
581
|
+
via: by.via,
|
|
582
|
+
name: by.name,
|
|
583
|
+
...(by.onBehalfOf !== undefined ? { onBehalfOf: by.onBehalfOf } : {}),
|
|
584
|
+
checked,
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/** What a Hook's fan-in did (ADR-0011): the Gates that took the delivery, the ones that did not and
|
|
589
|
+
* why, and how many open Gates of its kind and key gave nothing. */
|
|
590
|
+
export type HookFanIn = {
|
|
591
|
+
delivered: Array<{ run: string; workflow: string; gate: string; event: string }>;
|
|
592
|
+
refused: Array<{ run: string; gate: string; event?: string; reason: string }>;
|
|
593
|
+
ignored: number;
|
|
594
|
+
/** Each Source's verdict (ADR-0017), one line for each Pool whose Source hears the kind and key. */
|
|
595
|
+
sources: SourceLine[];
|
|
596
|
+
/** The standing Workflows whose Source hears the kind and key, and that have no live run and are
|
|
597
|
+
* not paused (ADR-0017): a hand-over waits out its back-off. The door answers 503. */
|
|
598
|
+
waiting?: string[];
|
|
599
|
+
/** For a delivery whose answer a person reads: the `reply` of each target that has one, in the
|
|
600
|
+
* order of the fan-out (ADR-0011). The door joins them; it never puts them in the outcome. */
|
|
601
|
+
replies?: HookReply[];
|
|
602
|
+
};
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* What one target's `reply` gave (ADR-0011): its place in the join (a Source or a Gate, its
|
|
606
|
+
* Workflow and its run), and its text, or the error of a `reply` that threw or gave no text. A
|
|
607
|
+
* `skipped` Source's text shows only when no other target gave one.
|
|
608
|
+
*/
|
|
609
|
+
export type HookReply = {
|
|
610
|
+
target: "source" | "gate";
|
|
611
|
+
workflow: string;
|
|
612
|
+
run: string;
|
|
613
|
+
/** The Gate's id, or the Pool's `source` Gate's id. */
|
|
614
|
+
gate: string;
|
|
615
|
+
text?: string;
|
|
616
|
+
skipped?: true;
|
|
617
|
+
error?: string;
|
|
618
|
+
};
|
|
619
|
+
|
|
620
|
+
/** Run one `reply`: plain text, or nothing, or the error that the hook log keeps. */
|
|
621
|
+
function runReply(place: Omit<HookReply, "text" | "error">, run: () => unknown): HookReply {
|
|
622
|
+
let given: unknown;
|
|
623
|
+
try {
|
|
624
|
+
given = run();
|
|
625
|
+
} catch (err) {
|
|
626
|
+
return { ...place, error: `its reply threw: ${err instanceof Error ? err.message : String(err)}` };
|
|
627
|
+
}
|
|
628
|
+
if (given === undefined || given === "") return place;
|
|
629
|
+
if (typeof given !== "string") return { ...place, error: `its reply gave ${typeof given}, not text` };
|
|
630
|
+
return { ...place, text: given };
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/** One Source's verdict on one delivery (ADR-0017): the Pool's run, its Workflow, the `source`
|
|
634
|
+
* Gate's id, and the verdict. */
|
|
635
|
+
export type SourceLine = { pool: string; workflow: string; source: string } & SourceVerdictLine;
|
|
636
|
+
|
|
637
|
+
/** Where a kept conversation's entries may come from ({@link RunHost.conversationPlace}). No
|
|
638
|
+
* `agent` when the run's ledger does not name the Instance ID. */
|
|
639
|
+
export type ConversationPlace = {
|
|
640
|
+
runId: string;
|
|
641
|
+
live: boolean;
|
|
642
|
+
agent?: string;
|
|
643
|
+
sandbox?: string;
|
|
644
|
+
slot?: string;
|
|
645
|
+
};
|
|
646
|
+
|
|
493
647
|
/** A run's current observable state. `fault` carries the error message when status is "error"
|
|
494
648
|
* (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). A run that ends
|
|
495
649
|
* "failed" (ADR-0073) has no `fault`: its `output` is the Failure's `{ reason, detail? }`. */
|
|
496
650
|
export type RunStatus = RunRecord & {
|
|
497
|
-
status:
|
|
651
|
+
status: ShownStatus;
|
|
498
652
|
value: unknown;
|
|
499
653
|
context: unknown;
|
|
500
654
|
/** The live child machines beneath the root — where most of a run actually is. */
|
|
@@ -504,7 +658,7 @@ export type RunStatus = RunRecord & {
|
|
|
504
658
|
* never carries it — the open band sees only that a Workspace is in `placing`. */
|
|
505
659
|
waiting: RunWaiting[];
|
|
506
660
|
/** Every pick of this run that is outstanding (ADR-0024): the Agent, its pick, and since when —
|
|
507
|
-
* the Console's "
|
|
661
|
+
* the Console's "outstanding since". There is no default time bound on a held Turn; this is how a
|
|
508
662
|
* human sees one. Behind the token, like `context`: a pick names what the Agent did. */
|
|
509
663
|
held: RunHeld[];
|
|
510
664
|
fault?: string;
|
|
@@ -518,6 +672,9 @@ export type RunStatus = RunRecord & {
|
|
|
518
672
|
/** The hash of the Assembly the run started under (ADR-0069). Absent for a run no Assembly
|
|
519
673
|
* started. */
|
|
520
674
|
assembly?: string;
|
|
675
|
+
/** The space guard deleted the run's kept conversations (ADR-0065, ADR-0023): `jr2 logs` and
|
|
676
|
+
* the Console show this in place of them. The run's row stays. */
|
|
677
|
+
swept?: "conversation swept: no space";
|
|
521
678
|
/** The run binds at least one Chart (ADR-0070): its Machine is the bound one, `runMachine`. */
|
|
522
679
|
bound?: true;
|
|
523
680
|
/** The hashes of the Charts the run binds now, distinct and sorted (ADR-0070). A run can bind a
|
|
@@ -536,6 +693,19 @@ export type RunStatus = RunRecord & {
|
|
|
536
693
|
* direction, read off the caller's own ledger rows. A foreign dispatch's run is a run of the
|
|
537
694
|
* Orchestrator its target names. Absent when it started none. */
|
|
538
695
|
dispatched?: Dispatched[];
|
|
696
|
+
/** The Caller that cancelled the run (ADR-0078). Behind the token: {@link observe} never
|
|
697
|
+
* carries it. */
|
|
698
|
+
cancelledBy?: CallerRecord;
|
|
699
|
+
/** The run that took over from this standing run (ADR-0017): a hand-over ends the run `done`. */
|
|
700
|
+
handedOverTo?: string;
|
|
701
|
+
/** The answers to the run's Gates, oldest first, each with the Caller that gave it (ADR-0011,
|
|
702
|
+
* ADR-0078): the last {@link GATE_ANSWERS_KEPT}. An event that a Gate's `look` gives is not an
|
|
703
|
+
* answer, and is not here. Behind the token. Absent when no Gate was answered. */
|
|
704
|
+
answers?: GateAnswer[];
|
|
705
|
+
/** The run's Steering messages, oldest first (ADR-0002): each one's Agent slot, author, text,
|
|
706
|
+
* time and outcome — the newest 100, and each one still held. Behind the token, like `context`.
|
|
707
|
+
* Absent when the run got none. */
|
|
708
|
+
steering?: SteeringRecord[];
|
|
539
709
|
/** The Worktrees this run's cuts made, oldest first (ADR-0075), read off its ledger's settled
|
|
540
710
|
* `ready` rows, so an Agent's `git switch` does not change them. A re-cut of one path and branch
|
|
541
711
|
* is one entry. Behind the token, like `context`: a branch name is the workflow's data. Absent
|
|
@@ -583,7 +753,7 @@ export type RunObservation = {
|
|
|
583
753
|
/** The run that dispatched this one, its slot, and the chain's depth (ADR-0071). Routing — two
|
|
584
754
|
* ids, a slot name and a number — the class of thing `runId` and `value` are, so ADR-0014's line
|
|
585
755
|
* is unmoved. The idempotency key is not: it names the caller's ledger, which is the caller's. */
|
|
586
|
-
dispatchedBy?: Omit<DispatchedBy, "effect">;
|
|
756
|
+
dispatchedBy?: Omit<DispatchedBy, "effect" | "key" | "hook">;
|
|
587
757
|
/** The runs it dispatched, by slot (ADR-0071): the same routing as `dispatchedBy`, the other
|
|
588
758
|
* direction. */
|
|
589
759
|
dispatched?: Array<{ slot: string; run: string }>;
|
|
@@ -605,8 +775,10 @@ export function observe(status: RunStatus): RunObservation {
|
|
|
605
775
|
}
|
|
606
776
|
|
|
607
777
|
/** A {@link DispatchedBy} without its idempotency key (ADR-0071): what the open band carries. */
|
|
608
|
-
function routingOf(by: DispatchedBy): Omit<DispatchedBy, "effect"> {
|
|
609
|
-
|
|
778
|
+
function routingOf(by: DispatchedBy): Omit<DispatchedBy, "effect" | "key" | "hook"> {
|
|
779
|
+
// The key names the caller's ledger, or a Pool's item, and the Hook delivery is the workflow's
|
|
780
|
+
// data: neither is routing (ADR-0014, ADR-0017).
|
|
781
|
+
const { effect: _effect, key: _key, hook: _hook, ...routing } = by;
|
|
610
782
|
return routing;
|
|
611
783
|
}
|
|
612
784
|
|
|
@@ -646,6 +818,34 @@ export type GateView = {
|
|
|
646
818
|
meta?: Record<string, unknown>;
|
|
647
819
|
};
|
|
648
820
|
|
|
821
|
+
/** One entry of the Gate inbox (`GET /gates` — ADR-0011): an open Gate as {@link GateView} gives it,
|
|
822
|
+
* with its run, the run's Workflow, and when it opened, ISO-8601. `hook` names the Hook kinds that
|
|
823
|
+
* the Gate hears, when it hears one. Never the run's context (ADR-0014). */
|
|
824
|
+
export type GateInboxEntry = { runId: string; workflow: string } & GateView & {
|
|
825
|
+
openedAt: string;
|
|
826
|
+
hook?: string[];
|
|
827
|
+
};
|
|
828
|
+
|
|
829
|
+
/** The name of a Gate (ADR-0078): its invoke id, which its events carry as their sender. */
|
|
830
|
+
function gateNameOf(reg: Registration): string {
|
|
831
|
+
return reg.sender?.id ?? reg.invokeId ?? reg.id;
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
/** A Gate's registration as callers discover it (ADR-0011). */
|
|
835
|
+
function gateViewOf(reg: Registration): GateView {
|
|
836
|
+
return {
|
|
837
|
+
gate: reg.id,
|
|
838
|
+
path: reg.path ?? [],
|
|
839
|
+
accepts: [...reg.defs.values()].map((def) => ({
|
|
840
|
+
name: def.name,
|
|
841
|
+
description: def.description,
|
|
842
|
+
input: z.toJSONSchema(def.input),
|
|
843
|
+
})),
|
|
844
|
+
acceptsNow: acceptsNow(reg),
|
|
845
|
+
meta: reg.meta,
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
|
|
649
849
|
/** One event on an Agent's Menu, as the surface serves it: a tool, its `input` schema the tool's
|
|
650
850
|
* parameters. `semantics` rides along so the Harness refuses a reserved one (ADR-0013). */
|
|
651
851
|
/** One pick on the surface. `briefing` marks a pick whose parameter is a Chart: jr2's text on how a
|
|
@@ -802,6 +1002,10 @@ function echoEventOf(event: RunFeedEvent, target: string): EchoEvent | undefined
|
|
|
802
1002
|
* every one fed after it. A superseded status drops out; nothing else moves. So a Harness that
|
|
803
1003
|
* answers slowly holds at most one status plus the markers since its last push.
|
|
804
1004
|
*
|
|
1005
|
+
* A status the same as the last one pushed is dropped, with the pending one: the feed sends a
|
|
1006
|
+
* status on each inspection event, and an Agent actor's own transitions move nothing that the echo
|
|
1007
|
+
* shows, so without this each line of the log would print two times or more.
|
|
1008
|
+
*
|
|
805
1009
|
* Fire-and-forget: a failed push goes to `report` and the outbox drains on — nothing here can
|
|
806
1010
|
* fail a turn, a state, or a run. Exported for its unit test; {@link RunHost} is its one caller.
|
|
807
1011
|
*/
|
|
@@ -811,10 +1015,15 @@ export function echoOutbox(
|
|
|
811
1015
|
): (events: EchoEvent[]) => void {
|
|
812
1016
|
let pending: EchoEvent[] = [];
|
|
813
1017
|
let flushing = false;
|
|
1018
|
+
// The last status that left in a push, as its JSON: the level the log shows now.
|
|
1019
|
+
let shown: string | undefined;
|
|
814
1020
|
const flush = async (): Promise<void> => {
|
|
815
1021
|
while (pending.length > 0) {
|
|
816
1022
|
const batch = pending;
|
|
817
1023
|
pending = [];
|
|
1024
|
+
// A batch holds at most one status (see the enqueue below).
|
|
1025
|
+
const status = batch.find((p) => p.kind === "status");
|
|
1026
|
+
if (status) shown = JSON.stringify(status);
|
|
818
1027
|
try {
|
|
819
1028
|
await push(batch);
|
|
820
1029
|
} catch (err) {
|
|
@@ -825,7 +1034,10 @@ export function echoOutbox(
|
|
|
825
1034
|
};
|
|
826
1035
|
return (events) => {
|
|
827
1036
|
for (const event of events) {
|
|
828
|
-
if (event.kind === "status")
|
|
1037
|
+
if (event.kind === "status") {
|
|
1038
|
+
pending = pending.filter((p) => p.kind !== "status");
|
|
1039
|
+
if (JSON.stringify(event) === shown) continue;
|
|
1040
|
+
}
|
|
829
1041
|
pending.push(event);
|
|
830
1042
|
}
|
|
831
1043
|
if (flushing || pending.length === 0) return;
|
|
@@ -934,17 +1146,87 @@ export type RunHostOptions = {
|
|
|
934
1146
|
/** The wall clock a run's timer table is written and read on (ADR-0007, timers.ts): each pending
|
|
935
1147
|
* timer's deadline is a time on it. Default: `Date.now`. Tests pass a clock that does not step. */
|
|
936
1148
|
now?: () => number;
|
|
1149
|
+
/** The bound of a standing run and the back-off after its `error` (ADR-0017). Tests shrink them. */
|
|
1150
|
+
standing?: StandingOptions;
|
|
1151
|
+
};
|
|
1152
|
+
|
|
1153
|
+
/** The bound of a standing run, and the wait before a hand-over from a run in which no item that
|
|
1154
|
+
* it took started a run (ADR-0017). */
|
|
1155
|
+
export type StandingOptions = {
|
|
1156
|
+
/** Hand the run over once it has taken this many items, each one a dispatch. When no item that it
|
|
1157
|
+
* took itself started a run (a carried key does not count), the hand-over waits out the
|
|
1158
|
+
* back-off, as after `error`. Default: 500. */
|
|
1159
|
+
dispatches?: number;
|
|
1160
|
+
/** Hand the run over once it is this old, in milliseconds. Default: one day. */
|
|
1161
|
+
ageMs?: number;
|
|
1162
|
+
/** The wait before the hand-over after the `n`-th run in a row that ended `error`, ended
|
|
1163
|
+
* `failed` (ADR-0073, as `error`) or took its bound, and in which no item that it took itself
|
|
1164
|
+
* started a run (a carried key does not count). n is from 1; a run in which such an item started
|
|
1165
|
+
* a run waits as n = 1. Default: 1 s, 4 s, then 16 s. */
|
|
1166
|
+
backoffMs?: (n: number) => number;
|
|
1167
|
+
};
|
|
1168
|
+
|
|
1169
|
+
/** The bound of a standing run (ADR-0017): 500 dispatches or one day. Not measured past the size
|
|
1170
|
+
* of a save: each dispatch keeps a ledger row of about 217 bytes, so 500 is about 110 KB. */
|
|
1171
|
+
export const STANDING_BOUND = { dispatches: 500, ageMs: 24 * 60 * 60 * 1000 } as const;
|
|
1172
|
+
/** How many hand-overs jr2 makes from runs in a row that end `error`, end `failed` or take their
|
|
1173
|
+
* bound, in which no item that the run took itself started a run (ADR-0017). When the next such
|
|
1174
|
+
* run ends, jr2 pauses the standing Pool. Not measured. */
|
|
1175
|
+
export const STANDING_ERRORS_PAUSE = 3;
|
|
1176
|
+
/** How many times one delivery follows a standing Pool's hand-overs to the new run's Source. After
|
|
1177
|
+
* more hand-overs in one delivery's fan-in, the door answers 503, as for a Pool with no live run. */
|
|
1178
|
+
const STANDING_HOPS = 3;
|
|
1179
|
+
const standingBackoff = (n: number): number => Math.min(60_000, 1000 * 4 ** (n - 1));
|
|
1180
|
+
|
|
1181
|
+
/** A standing Workflow as `jr2 up` and `jr2 status` print it (ADR-0017): its Pool, its callee, its
|
|
1182
|
+
* state, its run, and how many items it holds. `live`: a run is live. `paused`: a cancel (`why`:
|
|
1183
|
+
* `cancel`), or a fourth run in a row in which no item that the run took started a run, after 3
|
|
1184
|
+
* hand-overs: the last one ended `error` or `failed` (`errors`), or took its bound (`bound`). The
|
|
1185
|
+
* held items are in the blob of `run`.
|
|
1186
|
+
* `waiting`: no run is live, and jr2 starts one after the back-off. Structure and counts only: no
|
|
1187
|
+
* key and no item, so it is on the open band with the Assembly. */
|
|
1188
|
+
export type StandingView = {
|
|
1189
|
+
workflow: string;
|
|
1190
|
+
pool: string;
|
|
1191
|
+
callee: string;
|
|
1192
|
+
state: "live" | "paused" | "waiting";
|
|
1193
|
+
run?: string;
|
|
1194
|
+
held: number;
|
|
1195
|
+
why?: "cancel" | "errors" | "bound";
|
|
937
1196
|
};
|
|
938
1197
|
|
|
1198
|
+
/** jr2 starts a standing Workflow's run, and a Caller's start only resumes a paused one (ADR-0017):
|
|
1199
|
+
* the door answers 409 with the message. */
|
|
1200
|
+
export class StandingRefused extends Error {
|
|
1201
|
+
override name = "StandingRefused";
|
|
1202
|
+
}
|
|
1203
|
+
|
|
939
1204
|
/** What we persist per run: the machine snapshot wrapped with the run metadata restore needs.
|
|
940
1205
|
* `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent's Actor
|
|
941
1206
|
* through the run binding and saved in the same blob (same store, same atomicity). */
|
|
942
1207
|
type RunBlob = {
|
|
943
1208
|
workflow: string;
|
|
944
1209
|
instanceId: string;
|
|
1210
|
+
/** When the run started (ADR-0009). The store copies it into a column of its own, which orders
|
|
1211
|
+
* `GET /runs`. Absent in a blob written before the field. */
|
|
1212
|
+
startedAt?: string;
|
|
945
1213
|
/** The run that dispatched this one (ADR-0071), its key included: the store finds a run by its
|
|
946
1214
|
* key here (`findByDispatch`), so a re-sent start answers the run it already made. */
|
|
947
1215
|
dispatchedBy?: DispatchedBy;
|
|
1216
|
+
/** Who started the run (ADR-0078). */
|
|
1217
|
+
startedBy?: RunStarter;
|
|
1218
|
+
/** The Caller that cancelled the run (ADR-0078). */
|
|
1219
|
+
cancelledBy?: CallerRecord;
|
|
1220
|
+
/** The Caller whose `jr2 run` resumed this standing run (ADR-0017). */
|
|
1221
|
+
resumedBy?: CallerRecord;
|
|
1222
|
+
/** The run that took over from this standing run at a hand-over (ADR-0017). */
|
|
1223
|
+
handedOverTo?: string;
|
|
1224
|
+
/** The last answers to the run's Gates, with their Callers (ADR-0078). */
|
|
1225
|
+
answers?: GateAnswer[];
|
|
1226
|
+
/** When each open Gate opened, ISO-8601, by its gate id (ADR-0011): what `GET /gates` orders by.
|
|
1227
|
+
* Kept in the blob so that a restored run's Gate keeps the time it first opened, not the time of
|
|
1228
|
+
* the restore. A Gate that closed is not in it. */
|
|
1229
|
+
gatesOpenedAt?: Record<string, string>;
|
|
948
1230
|
/** The shape of the Machine that WROTE this snapshot (ADR-0030). `workflow` says which def to
|
|
949
1231
|
* look up; this says whether the def found there is still the one this snapshot can be read by.
|
|
950
1232
|
* Absent means written before the stamp existed, which restore treats as drift — the point is to
|
|
@@ -1011,7 +1293,7 @@ type RunBlob = {
|
|
|
1011
1293
|
* outstanding or not. Same blob, so a restore names the same Agent. A child no longer in the tree
|
|
1012
1294
|
* is dropped at the save.
|
|
1013
1295
|
*/
|
|
1014
|
-
|
|
1296
|
+
childAgents?: Record<string, ChildAgent>;
|
|
1015
1297
|
fault?: string;
|
|
1016
1298
|
/** The Failure that ended the run `failed` (ADR-0073), as plain data: the run's output. xstate
|
|
1017
1299
|
* keeps the error of an errored snapshot, and serializes it to `{}`, so it is kept here. */
|
|
@@ -1035,13 +1317,33 @@ type LiveRun = {
|
|
|
1035
1317
|
holds: Map<string, Hold>;
|
|
1036
1318
|
/** The Effect ledger (ADR-0067): persisted as `RunBlob.effects`, seeded on restore. */
|
|
1037
1319
|
effects: Record<string, EffectRow>;
|
|
1038
|
-
/** Which Agent started each live child (ADR-0067): persisted as `RunBlob.
|
|
1039
|
-
|
|
1320
|
+
/** Which Agent started each live child (ADR-0067): persisted as `RunBlob.childAgents`, seeded on restore. */
|
|
1321
|
+
childAgents: Record<string, ChildAgent>;
|
|
1040
1322
|
/** The error that killed the run, if it errored (xstate serializes Error to `{}`, so the
|
|
1041
1323
|
* message is captured here at the observer and persisted onto the blob for `read`). */
|
|
1042
1324
|
fault?: string;
|
|
1043
1325
|
/** The Failure that ended the run, if it failed (ADR-0073): persisted as `RunBlob.failure`. */
|
|
1044
1326
|
failure?: FailureInput;
|
|
1327
|
+
/** The bound Machine as it was while the run last bound a Chart (ADR-0069, ADR-0070), and the
|
|
1328
|
+
* hashes of those Charts: the terminal write keeps this doc, because the terminal snapshot binds
|
|
1329
|
+
* no Chart. Not persisted: a restored run takes it again at its next save. */
|
|
1330
|
+
lastBound?: { hashes: string[]; doc: MachineDoc };
|
|
1331
|
+
/** The Caller that cancelled the run (ADR-0078): persisted as `RunBlob.cancelledBy`. */
|
|
1332
|
+
cancelledBy?: CallerRecord;
|
|
1333
|
+
/** The end that the host gives the run while it stops its actor ({@link RunHost.end}). Not
|
|
1334
|
+
* persisted: the save that follows the stop writes it as the run's status. */
|
|
1335
|
+
ending?: EndedStatus;
|
|
1336
|
+
/** The run that took over from this standing run (ADR-0017): persisted as `RunBlob.handedOverTo`. */
|
|
1337
|
+
handedOverTo?: string;
|
|
1338
|
+
/** This standing run took its bound of items and none of them started a run (ADR-0017): jr2 ended
|
|
1339
|
+
* it `done`, and its hand-over waits out the back-off. Not persisted: the head's streak keeps it. */
|
|
1340
|
+
boundWithNoRun?: boolean;
|
|
1341
|
+
/** The last answers to the run's Gates (ADR-0078): persisted as `RunBlob.answers`, seeded on
|
|
1342
|
+
* restore. */
|
|
1343
|
+
answers: GateAnswer[];
|
|
1344
|
+
/** When each open Gate opened, by its gate id (ADR-0011): persisted as `RunBlob.gatesOpenedAt`,
|
|
1345
|
+
* seeded on restore before the actor starts, so a Gate that registers again keeps its time. */
|
|
1346
|
+
gatesOpenedAt: Map<string, string>;
|
|
1045
1347
|
/** Per-run observers fed by `persist()` (status) and the actor's `emit` (emit) — SSE/CLI watch. */
|
|
1046
1348
|
listeners: Set<(e: RunFeedEvent) => void>;
|
|
1047
1349
|
/** The run's feed-so-far (ADR-0023): what a Workspace attaching mid-run gets replayed as its
|
|
@@ -1057,7 +1359,19 @@ type LiveRun = {
|
|
|
1057
1359
|
export class RunHost {
|
|
1058
1360
|
/** The internal registration table both delivery surfaces share (ADR-0011). Callback actors
|
|
1059
1361
|
* (`gate`, the Agent slots) reach it via the run binding, not this field. */
|
|
1060
|
-
|
|
1362
|
+
// A Gate's `openedAt` (ADR-0011): recorded when it registers, unless its run's blob already
|
|
1363
|
+
// holds one (a restore), and dropped when it closes. A run that a host parks keeps the time in
|
|
1364
|
+
// its last save, because a stopped run is not saved again.
|
|
1365
|
+
private readonly table = new RegistrationTable({
|
|
1366
|
+
opened: (reg) => {
|
|
1367
|
+
if (reg.kind !== "gate") return;
|
|
1368
|
+
const opened = this.runs.get(reg.runId)?.gatesOpenedAt;
|
|
1369
|
+
if (opened && !opened.has(reg.id)) opened.set(reg.id, new Date(this.now()).toISOString());
|
|
1370
|
+
},
|
|
1371
|
+
closed: (reg) => {
|
|
1372
|
+
if (reg.kind === "gate") this.runs.get(reg.runId)?.gatesOpenedAt.delete(reg.id);
|
|
1373
|
+
},
|
|
1374
|
+
});
|
|
1061
1375
|
|
|
1062
1376
|
private readonly store: SnapshotStore;
|
|
1063
1377
|
private readonly reconcile: (run: RunRecord) => boolean | Promise<boolean>;
|
|
@@ -1076,8 +1390,9 @@ export class RunHost {
|
|
|
1076
1390
|
* hosted again (`store.live()` row order) or while the callee is not hosted at all.
|
|
1077
1391
|
*/
|
|
1078
1392
|
private readonly settleListeners = new Map<string, Set<(status: RunStatus) => void>>();
|
|
1079
|
-
/**
|
|
1080
|
-
* before the first has spawned its run answers the first's run,
|
|
1393
|
+
/** (Workflow, idempotency key) → the start in flight under it (ADR-0071): a second start of the
|
|
1394
|
+
* same Workflow with the same key before the first has spawned its run answers the first's run,
|
|
1395
|
+
* never a second. */
|
|
1081
1396
|
private readonly dispatching = new Map<string, Promise<DispatchedRun>>();
|
|
1082
1397
|
private readonly echoFactory?: (
|
|
1083
1398
|
endpoint: string,
|
|
@@ -1139,6 +1454,14 @@ export class RunHost {
|
|
|
1139
1454
|
private readonly parked = new Set<string>();
|
|
1140
1455
|
#stopping = false;
|
|
1141
1456
|
#restorePending = false;
|
|
1457
|
+
/** The bound and the back-off of standing runs (ADR-0017). */
|
|
1458
|
+
private readonly standingOpts: Required<StandingOptions>;
|
|
1459
|
+
/** The head of each standing Workflow (ADR-0017), read from the store at first use and written
|
|
1460
|
+
* through to it. */
|
|
1461
|
+
#heads: Map<string, StandingHead> | undefined;
|
|
1462
|
+
/** Each standing Workflow's timer: the back-off before a hand-over after `error`, or the age
|
|
1463
|
+
* bound of its live run. One a Workflow. */
|
|
1464
|
+
private readonly standingTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
1142
1465
|
|
|
1143
1466
|
constructor(opts: RunHostOptions) {
|
|
1144
1467
|
this.store = opts.store;
|
|
@@ -1159,6 +1482,11 @@ export class RunHost {
|
|
|
1159
1482
|
: undefined;
|
|
1160
1483
|
this.liveSetIntervalMs = opts.liveSetIntervalMs ?? DEFAULT_LEASE_INTERVAL_MS;
|
|
1161
1484
|
this.now = opts.now ?? Date.now;
|
|
1485
|
+
this.standingOpts = {
|
|
1486
|
+
dispatches: opts.standing?.dispatches ?? STANDING_BOUND.dispatches,
|
|
1487
|
+
ageMs: opts.standing?.ageMs ?? STANDING_BOUND.ageMs,
|
|
1488
|
+
backoffMs: opts.standing?.backoffMs ?? standingBackoff,
|
|
1489
|
+
};
|
|
1162
1490
|
}
|
|
1163
1491
|
|
|
1164
1492
|
/** Register a workflow so `start`/`restore` can run it. Re-registering replaces (dev reload).
|
|
@@ -1255,42 +1583,52 @@ export class RunHost {
|
|
|
1255
1583
|
/**
|
|
1256
1584
|
* The run's bound Machine (ADR-0070): the Workflow's Machine with each bound Open body's subgraph
|
|
1257
1585
|
* the compiled Chart's (its `src` stays the slot key, so the live child joins it as before) —
|
|
1258
|
-
* from the live actor's bound Machine
|
|
1259
|
-
*
|
|
1586
|
+
* from the live actor's bound Machine; for an ended run, the copy its terminal write kept
|
|
1587
|
+
* (ADR-0069); for a stored run with no copy, compiled again from the text its blob holds. Undefined
|
|
1588
|
+
* for an unknown run, and for a stored run with no copy whose Workflow is not registered.
|
|
1260
1589
|
*/
|
|
1261
1590
|
async runMachine(runId: string): Promise<MachineDoc | undefined> {
|
|
1262
1591
|
const live = this.runs.get(runId);
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
}
|
|
1592
|
+
if (live) return this.liveMachineDoc(live);
|
|
1593
|
+
// An ended run's Machine as its terminal write kept it (ADR-0069): the Machine it ran on, which
|
|
1594
|
+
// can be of an Assembly that is no longer held.
|
|
1595
|
+
const kept = this.store.keptMachine(runId);
|
|
1596
|
+
if (kept) return { ...(kept.doc as MachineDoc), workflow: kept.workflow };
|
|
1597
|
+
const stored = await this.store.load(runId);
|
|
1598
|
+
const blob = stored?.snapshot as RunBlob | null | undefined;
|
|
1599
|
+
if (!blob) return undefined;
|
|
1600
|
+
const def = this.workflowDefs.get(blob.workflow) ?? this.workflowDefs.get(wrapperName(blob.workflow));
|
|
1601
|
+
if (!def) return undefined;
|
|
1602
|
+
return this.boundDoc(
|
|
1603
|
+
blob.workflow,
|
|
1604
|
+
def.machine,
|
|
1605
|
+
boundChartRoutesOf(blob.snapshot).map((route) => ({ route, compiled: compileStored(def.machine, route) })),
|
|
1606
|
+
);
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
/** A live run's bound Machine, from its own def (the Assembly it holds) and its live actor. */
|
|
1610
|
+
private liveMachineDoc(live: LiveRun): MachineDoc {
|
|
1611
|
+
const refs = new Map<string, AnyActorRef>();
|
|
1612
|
+
const collect = (ref: AnyActorRef, path: string[]): void => {
|
|
1613
|
+
for (const [id, child] of Object.entries(childrenOfRef(ref))) {
|
|
1614
|
+
refs.set([...path, id].join("/"), child);
|
|
1615
|
+
collect(child, [...path, id]);
|
|
1616
|
+
}
|
|
1617
|
+
};
|
|
1618
|
+
collect(live.actor, []);
|
|
1619
|
+
const bodies = boundChartRoutesOf(live.actor.getSnapshot()).map((route) => {
|
|
1620
|
+
const ref = refs.get(route.path.join("/"));
|
|
1621
|
+
return { route, compiled: (ref && boundMachineOf(ref)) ?? "the live body holds no compiled Chart" };
|
|
1622
|
+
});
|
|
1623
|
+
return this.boundDoc(live.record.workflow, live.def.machine, bodies);
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
/** The Workflow's Machine doc with each bound body's Chart compiled in. */
|
|
1627
|
+
private boundDoc(
|
|
1628
|
+
workflow: string,
|
|
1629
|
+
machine: AnyStateMachine,
|
|
1630
|
+
bodies: Array<{ route: BoundChartRoute; compiled: AnyStateMachine | string }>,
|
|
1631
|
+
): MachineDoc {
|
|
1294
1632
|
const doc = serializeMachine(workflow, machine);
|
|
1295
1633
|
const notDrawn: NonNullable<MachineDoc["chartsNotDrawn"]> = [];
|
|
1296
1634
|
for (const { route, compiled } of bodies) {
|
|
@@ -1329,53 +1667,65 @@ export class RunHost {
|
|
|
1329
1667
|
* Start a fresh run of a registered workflow; returns its durable ids.
|
|
1330
1668
|
*
|
|
1331
1669
|
* With `dispatchedBy` (ADR-0071), the start is a `dispatch` Effect's, and its `effect` is the
|
|
1332
|
-
* idempotency key
|
|
1333
|
-
* started. That is what lets a caller re-send a start whose answer it lost. The key is looked for
|
|
1670
|
+
* idempotency key, scoped to `workflow`: a run of `workflow` that already carries the key is
|
|
1671
|
+
* answered, `existing`, and none is started. The same key in another Workflow is another run. That is what lets a caller re-send a start whose answer it lost. The key is looked for
|
|
1334
1672
|
* among the runs this process hosts first — a run's first save is a microtask after its spawn —
|
|
1335
|
-
* then in the store. A key on a row the
|
|
1673
|
+
* then in the store. A key on a row the sweep removed (ADR-0065) starts a new run: known,
|
|
1336
1674
|
* and not guarded. The depth is recorded as sent: the caller states the callee's.
|
|
1675
|
+
*
|
|
1676
|
+
* `startedBy` is who starts it (ADR-0078), which the run records. Absent with `dispatchedBy`, the
|
|
1677
|
+
* start is a local dispatch's, and the run records the caller's run and slot.
|
|
1337
1678
|
*/
|
|
1338
1679
|
async start(
|
|
1339
1680
|
workflow: string,
|
|
1340
1681
|
input: Record<string, unknown> = {},
|
|
1341
|
-
opts: { dispatchedBy?: DispatchedBy } = {},
|
|
1682
|
+
opts: { dispatchedBy?: DispatchedBy; startedBy?: RunStarter } = {},
|
|
1342
1683
|
): Promise<{ runId: string; instanceId: string; existing?: true }> {
|
|
1343
1684
|
const by = opts.dispatchedBy;
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
if (
|
|
1685
|
+
const startedBy: RunStarter | undefined =
|
|
1686
|
+
opts.startedBy ?? (by ? { kind: "run", run: by.run, slot: by.slot } : undefined);
|
|
1687
|
+
if (!by) return this.startFresh(workflow, input, undefined, startedBy);
|
|
1688
|
+
// The starts in flight are held by the key in its Workflow's scope.
|
|
1689
|
+
const scoped = JSON.stringify([workflow, dispatchKeyOf(by)]);
|
|
1690
|
+
const inFlight = this.dispatching.get(scoped);
|
|
1691
|
+
if (inFlight) return { ...(await inFlight), existing: true };
|
|
1347
1692
|
let existing = false;
|
|
1348
1693
|
const started = (async (): Promise<DispatchedRun> => {
|
|
1349
|
-
const found = await this.dispatched(by
|
|
1694
|
+
const found = await this.dispatched(workflow, dispatchKeyOf(by));
|
|
1350
1695
|
if (found) {
|
|
1351
1696
|
existing = true;
|
|
1352
1697
|
return found;
|
|
1353
1698
|
}
|
|
1354
|
-
return
|
|
1699
|
+
return this.startFresh(workflow, input, by, startedBy);
|
|
1355
1700
|
})();
|
|
1356
|
-
this.dispatching.set(
|
|
1701
|
+
this.dispatching.set(scoped, started);
|
|
1357
1702
|
try {
|
|
1358
|
-
const ids =
|
|
1703
|
+
const ids = await started;
|
|
1359
1704
|
return existing ? { ...ids, existing: true } : ids;
|
|
1360
1705
|
} finally {
|
|
1361
|
-
this.dispatching.delete(
|
|
1706
|
+
this.dispatching.delete(scoped);
|
|
1362
1707
|
}
|
|
1363
1708
|
}
|
|
1364
1709
|
|
|
1365
1710
|
/**
|
|
1366
|
-
* The run that carries the idempotency key `
|
|
1367
|
-
* then a stored one, in any status.
|
|
1711
|
+
* The run of `workflow` that carries the idempotency key `key` (ADR-0071), or undefined: a hosted
|
|
1712
|
+
* run first, then a stored one, in any status, while its row lives. The key is scoped to the
|
|
1713
|
+
* Workflow: a run of another Workflow under the same key is not this one.
|
|
1368
1714
|
*/
|
|
1369
|
-
async dispatched(
|
|
1715
|
+
async dispatched(workflow: string, key: string): Promise<DispatchedRun | undefined> {
|
|
1370
1716
|
for (const run of this.runs.values()) {
|
|
1371
|
-
if (
|
|
1372
|
-
|
|
1717
|
+
if (
|
|
1718
|
+
run.record.workflow === workflow &&
|
|
1719
|
+
run.record.dispatchedBy !== undefined &&
|
|
1720
|
+
dispatchKeyOf(run.record.dispatchedBy) === key
|
|
1721
|
+
) {
|
|
1722
|
+
return { runId: run.record.runId, instanceId: run.record.instanceId };
|
|
1373
1723
|
}
|
|
1374
1724
|
}
|
|
1375
|
-
const runId = await this.store.findByDispatch(
|
|
1725
|
+
const runId = await this.store.findByDispatch(workflow, key);
|
|
1376
1726
|
if (runId === undefined) return undefined;
|
|
1377
1727
|
const blob = (await this.store.load(runId))?.snapshot as RunBlob | null | undefined;
|
|
1378
|
-
return blob ? { runId, instanceId: blob.instanceId
|
|
1728
|
+
return blob ? { runId, instanceId: blob.instanceId } : undefined;
|
|
1379
1729
|
}
|
|
1380
1730
|
|
|
1381
1731
|
/**
|
|
@@ -1390,11 +1740,16 @@ export class RunHost {
|
|
|
1390
1740
|
by: DispatchedBy,
|
|
1391
1741
|
): Promise<DispatchStart> {
|
|
1392
1742
|
// The key first: a re-sent start answers its run whatever the Assembly holds now.
|
|
1393
|
-
const found = await this.dispatched(by
|
|
1394
|
-
if (found && found.workflow !== workflow) {
|
|
1395
|
-
return { ok: false, status: 409, report: [keyConflict(found, workflow, by.effect)] };
|
|
1396
|
-
}
|
|
1743
|
+
const found = await this.dispatched(workflow, dispatchKeyOf(by));
|
|
1397
1744
|
if (found) return { ok: true, runId: found.runId, existing: true };
|
|
1745
|
+
// jr2 starts a standing Workflow's run (ADR-0017): a dispatch never starts one.
|
|
1746
|
+
if (this.isStanding(workflow)) {
|
|
1747
|
+
return {
|
|
1748
|
+
ok: false,
|
|
1749
|
+
status: 409,
|
|
1750
|
+
report: [`"${workflow}" is a standing Workflow: jr2 starts its run, and a dispatch cannot (ADR-0017)`],
|
|
1751
|
+
};
|
|
1752
|
+
}
|
|
1398
1753
|
const unrunnable = this.unrunnableChart?.(workflow);
|
|
1399
1754
|
if (unrunnable !== undefined) {
|
|
1400
1755
|
return { ok: false, status: 409, report: [`named Chart "${workflow}" cannot run: ${unrunnable}`] };
|
|
@@ -1412,7 +1767,6 @@ export class RunHost {
|
|
|
1412
1767
|
};
|
|
1413
1768
|
}
|
|
1414
1769
|
if (err instanceof UnknownWorkflowError) return { ok: false, status: 404, report: [err.message] };
|
|
1415
|
-
if (err instanceof DispatchKeyConflictError) return { ok: false, status: 409, report: [err.message] };
|
|
1416
1770
|
throw err;
|
|
1417
1771
|
}
|
|
1418
1772
|
}
|
|
@@ -1423,7 +1777,20 @@ export class RunHost {
|
|
|
1423
1777
|
workflow: string,
|
|
1424
1778
|
input: Record<string, unknown>,
|
|
1425
1779
|
dispatchedBy?: DispatchedBy,
|
|
1780
|
+
startedBy?: RunStarter,
|
|
1426
1781
|
): Promise<{ runId: string; instanceId: string }> {
|
|
1782
|
+
return this.startNow(workflow, input, dispatchedBy, startedBy);
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
/** {@link startFresh}, synchronous: a standing Pool's hand-over starts the new run and ends the old
|
|
1786
|
+
* one in one step (ADR-0017). Throws what the door refuses. */
|
|
1787
|
+
private startNow(
|
|
1788
|
+
workflow: string,
|
|
1789
|
+
input: Record<string, unknown>,
|
|
1790
|
+
dispatchedBy?: DispatchedBy,
|
|
1791
|
+
startedBy?: RunStarter,
|
|
1792
|
+
resumedBy?: CallerRecord,
|
|
1793
|
+
): { runId: string; instanceId: string } {
|
|
1427
1794
|
const def = this.workflowDefs.get(workflow);
|
|
1428
1795
|
if (!def) throw new UnknownWorkflowError(`no workflow registered as "${workflow}"`);
|
|
1429
1796
|
|
|
@@ -1450,7 +1817,15 @@ export class RunHost {
|
|
|
1450
1817
|
|
|
1451
1818
|
const runId = this.newId();
|
|
1452
1819
|
const instanceId = this.newId();
|
|
1453
|
-
const record: RunRecord = {
|
|
1820
|
+
const record: RunRecord = {
|
|
1821
|
+
runId,
|
|
1822
|
+
workflow,
|
|
1823
|
+
instanceId,
|
|
1824
|
+
startedAt: new Date(this.now()).toISOString(),
|
|
1825
|
+
...(dispatchedBy ? { dispatchedBy } : {}),
|
|
1826
|
+
...(startedBy ? { startedBy } : {}),
|
|
1827
|
+
...(resumedBy ? { resumedBy } : {}),
|
|
1828
|
+
};
|
|
1454
1829
|
|
|
1455
1830
|
const machine = this.assemble(def, instanceId);
|
|
1456
1831
|
// Every Chart field of a Chart wrapper's door is admitted BEFORE the run exists (ADR-0070): a
|
|
@@ -1608,7 +1983,11 @@ export class RunHost {
|
|
|
1608
1983
|
runId: stored.runId,
|
|
1609
1984
|
workflow: blob.workflow,
|
|
1610
1985
|
instanceId: blob.instanceId,
|
|
1986
|
+
// A blob written before the field has its row's start: its first save.
|
|
1987
|
+
startedAt: blob.startedAt ?? stored.startedAt ?? new Date(this.now()).toISOString(),
|
|
1611
1988
|
...(blob.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
|
|
1989
|
+
...(blob.startedBy ? { startedBy: blob.startedBy } : {}),
|
|
1990
|
+
...(blob.resumedBy ? { resumedBy: blob.resumedBy } : {}),
|
|
1612
1991
|
};
|
|
1613
1992
|
|
|
1614
1993
|
try {
|
|
@@ -1645,8 +2024,14 @@ export class RunHost {
|
|
|
1645
2024
|
// …and the Effect ledger (ADR-0067): a restored Effect reads its row and never runs twice.
|
|
1646
2025
|
blob.effects ?? {},
|
|
1647
2026
|
// …and which Agent started each child (ADR-0067): its done event names the same Agent.
|
|
1648
|
-
blob.
|
|
2027
|
+
blob.childAgents ?? {},
|
|
1649
2028
|
);
|
|
2029
|
+
// …and the answers to its Gates, with their Callers (ADR-0078): the record of who acted.
|
|
2030
|
+
const restored = this.runs.get(stored.runId);
|
|
2031
|
+
if (restored) restored.answers = [...(blob.answers ?? [])];
|
|
2032
|
+
// …and when each of its open Gates opened (ADR-0011), before the actor starts and the
|
|
2033
|
+
// Gates register again.
|
|
2034
|
+
if (restored) restored.gatesOpenedAt = new Map(Object.entries(blob.gatesOpenedAt ?? {}));
|
|
1650
2035
|
// `start()`, then the pending timers again (ADR-0007, see `startAndRearm`). The next save
|
|
1651
2036
|
// records them again.
|
|
1652
2037
|
startAndRearm(actor, blob.timers ?? [], this.now());
|
|
@@ -1734,6 +2119,62 @@ export class RunHost {
|
|
|
1734
2119
|
next();
|
|
1735
2120
|
}
|
|
1736
2121
|
|
|
2122
|
+
/**
|
|
2123
|
+
* Where a kept conversation's entries may come from (ADR-0023, ADR-0076): the run, the Agent, the
|
|
2124
|
+
* pod and the slot that the run's admission ledger names for the Instance ID. The live ledger, the
|
|
2125
|
+
* ledger of a run that restore left unhosted, or the ledger in an ended run's row: the ledger
|
|
2126
|
+
* outlives the Turn and the run, so a batch that comes after the Turn's sweep or the run's end is
|
|
2127
|
+
* still placed. `live` says whether the run is live, so a batch that comes before its admission is
|
|
2128
|
+
* ledgered is asked again, not dropped. Undefined for a run that is not known.
|
|
2129
|
+
*/
|
|
2130
|
+
async conversationPlace(instanceId: string): Promise<ConversationPlace | undefined> {
|
|
2131
|
+
const runId = instanceId.split("/")[0] ?? "";
|
|
2132
|
+
let agents: Record<string, AgentAdmission> | undefined = this.runs.get(runId)?.agents ?? this.unhosted.get(runId);
|
|
2133
|
+
let live = agents !== undefined;
|
|
2134
|
+
if (!agents) {
|
|
2135
|
+
const stored = await this.store.load(runId);
|
|
2136
|
+
if (!stored) return undefined;
|
|
2137
|
+
live = stored.status === "live";
|
|
2138
|
+
agents = (stored.snapshot as RunBlob | null | undefined)?.agents ?? {};
|
|
2139
|
+
}
|
|
2140
|
+
// A row is keyed by the invocation's Instance ID and names the newest conversation of it. A
|
|
2141
|
+
// runaway reroll (ADR-0035) replaces the conversation (`<key>-r<n>`), so a late batch of a
|
|
2142
|
+
// conversation that the reroll ended (the key itself, or an earlier `<key>-r<m>`) is placed by
|
|
2143
|
+
// the same row: that batch tells how the run went wrong. An exact match wins.
|
|
2144
|
+
const rows = Object.entries(agents);
|
|
2145
|
+
const row =
|
|
2146
|
+
rows.find(([key, a]) => (a.instanceId ?? key) === instanceId) ??
|
|
2147
|
+
rows.find(
|
|
2148
|
+
([key]) =>
|
|
2149
|
+
instanceId === key || (instanceId.startsWith(`${key}-r`) && /^\d+$/.test(instanceId.slice(key.length + 2))),
|
|
2150
|
+
);
|
|
2151
|
+
if (row) {
|
|
2152
|
+
const [, admission] = row;
|
|
2153
|
+
if (admission.agent === undefined) return { runId, live };
|
|
2154
|
+
return {
|
|
2155
|
+
runId,
|
|
2156
|
+
live,
|
|
2157
|
+
agent: admission.agent,
|
|
2158
|
+
...(admission.sandbox !== undefined ? { sandbox: admission.sandbox } : {}),
|
|
2159
|
+
...(admission.slot !== undefined ? { slot: admission.slot } : {}),
|
|
2160
|
+
};
|
|
2161
|
+
}
|
|
2162
|
+
return { runId, live };
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
/** Whether a run is live: hosted, left unhosted by a restore, or a `live` row (ADR-0065). The
|
|
2166
|
+
* space guard never sweeps such a run's conversations. */
|
|
2167
|
+
async runIsLive(runId: string): Promise<boolean> {
|
|
2168
|
+
if (this.runs.has(runId) || this.unhosted.has(runId)) return true;
|
|
2169
|
+
return (await this.store.load(runId))?.status === "live";
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
/** The run of the Turn that is registered for this Instance ID, or undefined when none is. A
|
|
2173
|
+
* cheap read: it asks no guard and builds no Menu. */
|
|
2174
|
+
agentRun(instanceId: string): string | undefined {
|
|
2175
|
+
return this.table.lookup(agentAddress(instanceId))?.runId;
|
|
2176
|
+
}
|
|
2177
|
+
|
|
1737
2178
|
/**
|
|
1738
2179
|
* An agent instance's LIVE surface (`GET /agents/:iid/surface` — ADR-0013): what the state that
|
|
1739
2180
|
* invoked this Agent accepts, right now. Undefined once nothing is registered (the state exited,
|
|
@@ -1775,7 +2216,7 @@ export class RunHost {
|
|
|
1775
2216
|
/** The notices this Turn's conversation hears at a step boundary (ADR-0062): its pending notices
|
|
1776
2217
|
* in their audience, reserved for this read until the Harness acknowledges them
|
|
1777
2218
|
* ({@link noticesHeard}) or the Turn ends. Empty for a registration that hears none. */
|
|
1778
|
-
async boundaryNotices(instanceId: string): Promise<Array<{ id: string; notice:
|
|
2219
|
+
async boundaryNotices(instanceId: string): Promise<Array<{ id: string; notice: LedgerBody }>> {
|
|
1779
2220
|
return (await this.table.lookup(agentAddress(instanceId))?.boundaryNotices?.()) ?? [];
|
|
1780
2221
|
}
|
|
1781
2222
|
|
|
@@ -1785,6 +2226,67 @@ export class RunHost {
|
|
|
1785
2226
|
return this.table.lookup(agentAddress(instanceId))?.noticesHeard?.(ids) ?? 0;
|
|
1786
2227
|
}
|
|
1787
2228
|
|
|
2229
|
+
/**
|
|
2230
|
+
* The Agent slots of a live run (ADR-0002): each slot of each live Machine instance, whether a
|
|
2231
|
+
* Turn of it is open, and whether a Caller may steer it. Undefined for a run that is not live.
|
|
2232
|
+
*/
|
|
2233
|
+
agents(runId: string): RunAgent[] | undefined {
|
|
2234
|
+
const run = this.runs.get(runId);
|
|
2235
|
+
if (!run) return undefined;
|
|
2236
|
+
const open = new Set(
|
|
2237
|
+
this.table
|
|
2238
|
+
.byRun(runId)
|
|
2239
|
+
.filter((reg) => reg.kind === "agent" && reg.slot !== undefined)
|
|
2240
|
+
.map((reg) => reg.slot!),
|
|
2241
|
+
);
|
|
2242
|
+
return liveSlots(run.actor).map((s) => ({ agent: s.agent, open: open.has(s.agent), steerable: s.steerable }));
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
/**
|
|
2246
|
+
* A Caller's Steering message to one Agent slot of a live run (ADR-0002, ADR-0062). It is held
|
|
2247
|
+
* in the run's notice ledger, saved at once: the slot's open Turn hears it at its next step
|
|
2248
|
+
* boundary, or its next Turn in the same Machine instance at admission. Refused (409) when the
|
|
2249
|
+
* run is not live or no live Machine instance holds the slot, and (403) when the Agent is not
|
|
2250
|
+
* steerable. Answers the message's id, and whether a Turn of the slot is open now.
|
|
2251
|
+
*/
|
|
2252
|
+
steer(runId: string, agent: string, text: string, by: CallerRecord): { id: string; agent: string; open: boolean } {
|
|
2253
|
+
const run = this.runs.get(runId);
|
|
2254
|
+
if (!run) throw new SteeringRefused(`run "${runId}" is not live: no Agent of it can hear a message`, 409);
|
|
2255
|
+
const agents = this.agents(runId) ?? [];
|
|
2256
|
+
const slot = agents.find((a) => a.agent === agent);
|
|
2257
|
+
if (!slot) {
|
|
2258
|
+
throw new SteeringRefused(
|
|
2259
|
+
`no live Machine instance of run "${runId}" holds the Agent slot "${agent}"` +
|
|
2260
|
+
(agents.length > 0 ? ` (its slots: ${agents.map((a) => a.agent).join(", ")})` : " (it has none now)"),
|
|
2261
|
+
409,
|
|
2262
|
+
agents,
|
|
2263
|
+
);
|
|
2264
|
+
}
|
|
2265
|
+
if (!slot.steerable) {
|
|
2266
|
+
throw new SteeringRefused(
|
|
2267
|
+
`the Agent slot "${agent}" is not steerable: its definition says steerable: false`,
|
|
2268
|
+
403,
|
|
2269
|
+
);
|
|
2270
|
+
}
|
|
2271
|
+
const id = run.notices.steer(
|
|
2272
|
+
{
|
|
2273
|
+
kind: "steering",
|
|
2274
|
+
scope: "slot",
|
|
2275
|
+
text,
|
|
2276
|
+
author: {
|
|
2277
|
+
kind: "caller",
|
|
2278
|
+
via: by.via,
|
|
2279
|
+
name: by.shown ?? by.name,
|
|
2280
|
+
...(by.onBehalfOf !== undefined ? { onBehalfOf: by.onBehalfOf } : {}),
|
|
2281
|
+
},
|
|
2282
|
+
at: new Date(this.now()).toISOString(),
|
|
2283
|
+
},
|
|
2284
|
+
agent,
|
|
2285
|
+
);
|
|
2286
|
+
this.persist(run);
|
|
2287
|
+
return { id, agent, open: slot.open };
|
|
2288
|
+
}
|
|
2289
|
+
|
|
1788
2290
|
/**
|
|
1789
2291
|
* Deliver one event from an Agent's Harness (`POST /agents/:iid/events` — ADR-0013). Validation
|
|
1790
2292
|
* and delivery are the table's; this only agent-scopes the address and mints the receipt. (A
|
|
@@ -2116,7 +2618,7 @@ export class RunHost {
|
|
|
2116
2618
|
seat: AnswerSeat,
|
|
2117
2619
|
runId: string,
|
|
2118
2620
|
holds: Map<string, Hold>,
|
|
2119
|
-
|
|
2621
|
+
childAgents: Record<string, ChildAgent>,
|
|
2120
2622
|
event?: AnyEventObject,
|
|
2121
2623
|
): Sending | Hold | undefined {
|
|
2122
2624
|
const seatedFor = (reg: Registration | undefined): boolean => {
|
|
@@ -2130,7 +2632,7 @@ export class RunHost {
|
|
|
2130
2632
|
const sending = this.sending && this.sending.reg.runId === runId ? this.sending : undefined;
|
|
2131
2633
|
const open = [...holds.values()].filter((hold) => hold.answer === undefined && seatedFor(regOf(hold)));
|
|
2132
2634
|
if (seat === null && event !== undefined) {
|
|
2133
|
-
const named = agentOfEvent(event,
|
|
2635
|
+
const named = agentOfEvent(event, childAgents, machine, sending);
|
|
2134
2636
|
if (named !== undefined) {
|
|
2135
2637
|
const { agent } = named;
|
|
2136
2638
|
const own = sending && sending.reg.sender?.id === agent ? sending : undefined;
|
|
@@ -2165,10 +2667,10 @@ export class RunHost {
|
|
|
2165
2667
|
snapshot: AnyMachineSnapshot,
|
|
2166
2668
|
childId: string,
|
|
2167
2669
|
holds: Map<string, Hold>,
|
|
2168
|
-
):
|
|
2670
|
+
): ChildAgent | undefined {
|
|
2169
2671
|
const node = seatOf(snapshot, childId);
|
|
2170
2672
|
if (node === undefined) return undefined;
|
|
2171
|
-
const byState = new Map<unknown,
|
|
2673
|
+
const byState = new Map<unknown, ChildAgent>();
|
|
2172
2674
|
for (const hold of holds.values()) {
|
|
2173
2675
|
if (hold.answer !== undefined) continue;
|
|
2174
2676
|
const reg = this.holdRegs.get(hold) ?? this.table.lookup(agentAddress(hold.instanceId));
|
|
@@ -2269,22 +2771,59 @@ export class RunHost {
|
|
|
2269
2771
|
}
|
|
2270
2772
|
|
|
2271
2773
|
/** A run's open gates as callers discover them (`GET /runs/:id` — ADR-0011). Settled/unknown
|
|
2272
|
-
* run → empty: a gate is a LIVE surface, it does not outlive its state.
|
|
2273
|
-
|
|
2774
|
+
* run → empty: a gate is a LIVE surface, it does not outlive its state. `opens`, when given, keeps
|
|
2775
|
+
* only the Gates whose name (their invoke id) it opens: the Gates that a Caller may answer
|
|
2776
|
+
* (ADR-0078). */
|
|
2777
|
+
gates(runId: string, opens?: (name: string) => boolean): GateView[] {
|
|
2274
2778
|
return this.table
|
|
2275
2779
|
.byRun(runId)
|
|
2276
|
-
.filter((reg) => reg.kind === "gate")
|
|
2277
|
-
.map(
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2780
|
+
.filter((reg) => reg.kind === "gate" && (opens === undefined || opens(gateNameOf(reg))))
|
|
2781
|
+
.map(gateViewOf);
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
/** The name of a run's open Gate (ADR-0078): its invoke id, the static name that the `answer`
|
|
2785
|
+
* scope names it by. Undefined when the run has no open Gate `gate`. */
|
|
2786
|
+
gateName(runId: string, gate: string): string | undefined {
|
|
2787
|
+
const reg = this.table.lookup(gateAddress(runId, gate));
|
|
2788
|
+
return reg?.kind === "gate" ? gateNameOf(reg) : undefined;
|
|
2789
|
+
}
|
|
2790
|
+
|
|
2791
|
+
/**
|
|
2792
|
+
* The Gate inbox (`GET /gates`, ADR-0011): every open Gate of every live run that `opens` opens
|
|
2793
|
+
* (the Gates that the Caller may answer, by the run's Workflow and the Gate's name, ADR-0078),
|
|
2794
|
+
* of one Workflow when `workflow` names it, oldest first. Never the run's context.
|
|
2795
|
+
*/
|
|
2796
|
+
gateInbox(opens: (workflow: string, name: string) => boolean, workflow?: string): GateInboxEntry[] {
|
|
2797
|
+
const out: GateInboxEntry[] = [];
|
|
2798
|
+
for (const run of this.runs.values()) {
|
|
2799
|
+
const runWorkflow = run.record.workflow;
|
|
2800
|
+
if (workflow !== undefined && runWorkflow !== workflow) continue;
|
|
2801
|
+
for (const reg of this.table.byRun(run.record.runId)) {
|
|
2802
|
+
// A Gate that hears only Hooks (a Pool's `source` Gate) takes no event from a Caller: it is no
|
|
2803
|
+
// entry of a person's inbox.
|
|
2804
|
+
if (reg.kind !== "gate" || reg.defs.size === 0 || !opens(runWorkflow, gateNameOf(reg))) continue;
|
|
2805
|
+
let openedAt = run.gatesOpenedAt.get(reg.id);
|
|
2806
|
+
if (openedAt === undefined) {
|
|
2807
|
+
openedAt = new Date(this.now()).toISOString();
|
|
2808
|
+
run.gatesOpenedAt.set(reg.id, openedAt);
|
|
2809
|
+
}
|
|
2810
|
+
const hooks = Object.entries(reg.hooked?.hooks ?? {})
|
|
2811
|
+
.filter(([, keys]) => Object.keys(keys).length > 0)
|
|
2812
|
+
.map(([kind]) => kind)
|
|
2813
|
+
.sort();
|
|
2814
|
+
out.push({
|
|
2815
|
+
runId: run.record.runId,
|
|
2816
|
+
workflow: runWorkflow,
|
|
2817
|
+
...gateViewOf(reg),
|
|
2818
|
+
openedAt,
|
|
2819
|
+
...(hooks.length ? { hook: hooks } : {}),
|
|
2820
|
+
});
|
|
2821
|
+
}
|
|
2822
|
+
}
|
|
2823
|
+
// Oldest first; one time is ordered by run and Gate, so two reads give one order.
|
|
2824
|
+
return out.sort(
|
|
2825
|
+
(a, b) => a.openedAt.localeCompare(b.openedAt) || a.runId.localeCompare(b.runId) || a.gate.localeCompare(b.gate),
|
|
2826
|
+
);
|
|
2288
2827
|
}
|
|
2289
2828
|
|
|
2290
2829
|
/**
|
|
@@ -2293,13 +2832,29 @@ export class RunHost {
|
|
|
2293
2832
|
* the run TOOK the event (ADR-0013): one that would run another party's writing, or that no
|
|
2294
2833
|
* writing would take with this payload, throws {@link NotTakenError} naming what the Gate takes
|
|
2295
2834
|
* now, and nothing is delivered.
|
|
2835
|
+
*
|
|
2836
|
+
* `by` is the Caller that answers (ADR-0078). The event's Sender carries it, with the fields that
|
|
2837
|
+
* jr2 checked (ADR-0011), and the run records it as the answer's `answeredBy`.
|
|
2838
|
+
*
|
|
2839
|
+
* `opens`, when given, is the Caller's `answer` scope for the run's Workflow, by the Gate's name
|
|
2840
|
+
* (its invoke id). An open Gate that it does not open throws {@link GateRefused}, and the 404 lists
|
|
2841
|
+
* only the Gates that it opens. The check reads the registration that the delivery goes to, in the
|
|
2842
|
+
* same synchronous step, so a Gate that opens while the request is in flight is checked too.
|
|
2296
2843
|
*/
|
|
2297
|
-
sendToGate(
|
|
2844
|
+
sendToGate(
|
|
2845
|
+
runId: string,
|
|
2846
|
+
gate: string,
|
|
2847
|
+
event: { type?: unknown } & Record<string, unknown>,
|
|
2848
|
+
by?: CallerRecord,
|
|
2849
|
+
delivery?: string,
|
|
2850
|
+
opens?: (name: string) => boolean,
|
|
2851
|
+
): void {
|
|
2852
|
+
const reg = this.table.lookup(gateAddress(runId, gate));
|
|
2853
|
+
if (reg?.kind === "gate" && opens !== undefined && !opens(gateNameOf(reg))) throw new GateRefused(gateNameOf(reg));
|
|
2298
2854
|
const { type, ...payload } = event;
|
|
2299
2855
|
if (typeof type !== "string" || !type) {
|
|
2300
2856
|
throw new EventValidationError(`event body must carry a string "type" (one of the gate's accepted names)`);
|
|
2301
2857
|
}
|
|
2302
|
-
const reg = this.table.lookup(gateAddress(runId, gate));
|
|
2303
2858
|
const refused = (why: string): NotTakenError =>
|
|
2304
2859
|
new NotTakenError(
|
|
2305
2860
|
`gate "${gate}" is open, and the run does not take "${type}" now: ${why} — not delivered ` +
|
|
@@ -2311,10 +2866,24 @@ export class RunHost {
|
|
|
2311
2866
|
// which refuses it with its own words.
|
|
2312
2867
|
const def = reg?.defs.get(type);
|
|
2313
2868
|
const parsed = def && !(payload && SENDER_KEY in payload) ? def.input.safeParse(payload ?? {}) : undefined;
|
|
2314
|
-
|
|
2869
|
+
const caller = by ? senderCallerOf(by) : undefined;
|
|
2870
|
+
const sender = reg?.sender && caller ? { ...reg.sender, ...caller } : reg?.sender;
|
|
2871
|
+
if (reg && parsed?.success && !wouldMove(reg.invoker, { type, ...parsed.data, [SENDER_KEY]: sender })) {
|
|
2315
2872
|
throw refused("no writing of it would move the run with this payload");
|
|
2316
2873
|
}
|
|
2317
|
-
this.table.deliver(gateAddress(runId, gate), type, payload);
|
|
2874
|
+
this.table.deliver(gateAddress(runId, gate), type, payload, caller);
|
|
2875
|
+
// Taken: the run records who answered (ADR-0078), in the save that this transition schedules.
|
|
2876
|
+
const run = this.runs.get(runId);
|
|
2877
|
+
if (run && by) {
|
|
2878
|
+
run.answers.push({
|
|
2879
|
+
gate,
|
|
2880
|
+
type,
|
|
2881
|
+
at: new Date(this.now()).toISOString(),
|
|
2882
|
+
answeredBy: by,
|
|
2883
|
+
...(delivery !== undefined ? { delivery } : {}),
|
|
2884
|
+
});
|
|
2885
|
+
if (run.answers.length > GATE_ANSWERS_KEPT) run.answers.splice(0, run.answers.length - GATE_ANSWERS_KEPT);
|
|
2886
|
+
}
|
|
2318
2887
|
} catch (err) {
|
|
2319
2888
|
if (err instanceof NotTakenError && err.writings.length) {
|
|
2320
2889
|
throw refused(
|
|
@@ -2327,7 +2896,7 @@ export class RunHost {
|
|
|
2327
2896
|
);
|
|
2328
2897
|
}
|
|
2329
2898
|
if (err instanceof UnknownAddressError) {
|
|
2330
|
-
const open = this.gates(runId).map((g) => g.gate);
|
|
2899
|
+
const open = this.gates(runId, opens).map((g) => g.gate);
|
|
2331
2900
|
throw new UnknownAddressError(
|
|
2332
2901
|
`no open gate "${gate}" on run "${runId}"${open.length ? ` (open: ${open.join(", ")})` : ""}`,
|
|
2333
2902
|
);
|
|
@@ -2336,6 +2905,162 @@ export class RunHost {
|
|
|
2336
2905
|
}
|
|
2337
2906
|
}
|
|
2338
2907
|
|
|
2908
|
+
/**
|
|
2909
|
+
* A Hook's fan-in (ADR-0011): give one delivery to every open Gate, in every live run of a Workflow
|
|
2910
|
+
* that `may` names, whose handlers hear `kind` and `key`, and to nothing else. Each handler reads
|
|
2911
|
+
* `{ payload, meta, caller }` and gives one event of its Gate's `delivers`, or `undefined` (not a
|
|
2912
|
+
* target: `ignored`). The event goes in as `jr2 send` sends it, with the Hook as its Caller, and
|
|
2913
|
+
* the run records the answer with the delivery's id. A handler that throws, an event that is not
|
|
2914
|
+
* in `delivers`, and an event that the run does not take now are each a line in `refused` and a
|
|
2915
|
+
* `hook-refused` mark on that run's feed: the run goes on. A Gate that already took this delivery
|
|
2916
|
+
* (a pair in the run's answers) is passed over, so a restore never gives it twice.
|
|
2917
|
+
*
|
|
2918
|
+
* Synchronous, as a handler is: the registrations are read once, and a Gate that a delivery before
|
|
2919
|
+
* it in the same fan-in closed is passed over.
|
|
2920
|
+
*/
|
|
2921
|
+
async deliverHook(d: {
|
|
2922
|
+
hook: string;
|
|
2923
|
+
kind: string;
|
|
2924
|
+
key: string;
|
|
2925
|
+
delivery: string;
|
|
2926
|
+
payload: unknown;
|
|
2927
|
+
caller: CallerRecord;
|
|
2928
|
+
may: (workflow: string) => boolean;
|
|
2929
|
+
/** A person reads the answer (ADR-0011): run the `reply` of each target. */
|
|
2930
|
+
person?: boolean;
|
|
2931
|
+
}): Promise<HookFanIn> {
|
|
2932
|
+
const out: HookFanIn = { delivered: [], refused: [], ignored: 0, sources: [] };
|
|
2933
|
+
const replies: HookReply[] = [];
|
|
2934
|
+
const targets: Array<{ run: LiveRun; reg: Registration; handler: AnyHandler }> = [];
|
|
2935
|
+
const sources: Array<{ run: LiveRun; reg: Registration }> = [];
|
|
2936
|
+
for (const run of this.runs.values()) {
|
|
2937
|
+
if (!d.may(run.record.workflow)) continue;
|
|
2938
|
+
for (const reg of this.table.byRun(run.record.runId)) {
|
|
2939
|
+
if (reg.kind !== "gate") continue;
|
|
2940
|
+
const handler = reg.hooked?.hooks[d.kind]?.[d.key];
|
|
2941
|
+
if (handler) targets.push({ run, reg, handler });
|
|
2942
|
+
if (reg.source?.hears(d.kind, d.key)) sources.push({ run, reg });
|
|
2943
|
+
}
|
|
2944
|
+
}
|
|
2945
|
+
for (const { run, reg, handler } of targets) {
|
|
2946
|
+
const runId = run.record.runId;
|
|
2947
|
+
// Closed by an earlier target of this fan-in, or the run ended.
|
|
2948
|
+
if (this.table.lookup(reg.address) !== reg || this.runs.get(runId) !== run) continue;
|
|
2949
|
+
if (run.answers.some((a) => a.delivery === d.delivery && a.gate === reg.id)) continue;
|
|
2950
|
+
// The Gate's `reply` runs after the run answers, from that Gate's own outcome (ADR-0011).
|
|
2951
|
+
const gateReply = d.person ? reg.hooked?.reply : undefined;
|
|
2952
|
+
const place = { target: "gate" as const, workflow: run.record.workflow, run: runId, gate: reg.id };
|
|
2953
|
+
const meta = reg.meta;
|
|
2954
|
+
const refuse = (reason: string, event?: string, given?: unknown): void => {
|
|
2955
|
+
out.refused.push({ run: runId, gate: reg.id, ...(event !== undefined ? { event } : {}), reason });
|
|
2956
|
+
if (gateReply) {
|
|
2957
|
+
const outcome = {
|
|
2958
|
+
state: "refused" as const,
|
|
2959
|
+
...(given !== undefined ? { event: given } : {}),
|
|
2960
|
+
reason,
|
|
2961
|
+
run: runId,
|
|
2962
|
+
};
|
|
2963
|
+
replies.push(runReply(place, () => gateReply(outcome as never, { meta, caller: d.caller })));
|
|
2964
|
+
}
|
|
2965
|
+
this.feed(run, {
|
|
2966
|
+
kind: "hook-refused",
|
|
2967
|
+
hook: d.hook,
|
|
2968
|
+
delivery: d.delivery,
|
|
2969
|
+
gate: reg.id,
|
|
2970
|
+
...(event !== undefined ? { event } : {}),
|
|
2971
|
+
reason,
|
|
2972
|
+
});
|
|
2973
|
+
};
|
|
2974
|
+
let event: unknown;
|
|
2975
|
+
try {
|
|
2976
|
+
event = handler({ payload: d.payload, meta: reg.meta, caller: d.caller });
|
|
2977
|
+
} catch (err) {
|
|
2978
|
+
refuse(`its handler threw: ${err instanceof Error ? err.message : String(err)}`);
|
|
2979
|
+
continue;
|
|
2980
|
+
}
|
|
2981
|
+
if (event === undefined) {
|
|
2982
|
+
out.ignored++;
|
|
2983
|
+
continue;
|
|
2984
|
+
}
|
|
2985
|
+
const type = (event as { type?: unknown } | null)?.type;
|
|
2986
|
+
if (typeof type !== "string" || !reg.hooked?.delivers.includes(type)) {
|
|
2987
|
+
refuse(
|
|
2988
|
+
`its handler gave ${typeof type === "string" ? `"${type}"` : "no event type"}, which is not in the Gate's ` +
|
|
2989
|
+
`delivers (${reg.hooked?.delivers.join(", ")})`,
|
|
2990
|
+
typeof type === "string" ? type : undefined,
|
|
2991
|
+
);
|
|
2992
|
+
continue;
|
|
2993
|
+
}
|
|
2994
|
+
try {
|
|
2995
|
+
this.sendToGate(runId, reg.id, event as { type: string } & Record<string, unknown>, d.caller, d.delivery);
|
|
2996
|
+
} catch (err) {
|
|
2997
|
+
refuse(err instanceof Error ? err.message : String(err), type, event);
|
|
2998
|
+
continue;
|
|
2999
|
+
}
|
|
3000
|
+
out.delivered.push({ run: runId, workflow: run.record.workflow, gate: reg.id, event: type });
|
|
3001
|
+
if (gateReply) {
|
|
3002
|
+
const outcome = { state: "delivered" as const, event, run: runId };
|
|
3003
|
+
replies.push(runReply(place, () => gateReply(outcome as never, { meta, caller: d.caller })));
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
// The Sources (ADR-0017), after the Gates: each one's handler, its key check and its hold. A
|
|
3007
|
+
// held item is durable before the answer leaves, so the run is saved before the next line.
|
|
3008
|
+
// A hand-over can end a standing run while a Source awaits (its key lookup, or an earlier
|
|
3009
|
+
// Source's save): the bound, a publish or the age timer. The start and the end of a hand-over are
|
|
3010
|
+
// one synchronous step, so the delivery goes on to the new run's Source; with no live run, the
|
|
3011
|
+
// last step below answers `paused` or `waiting` (503), and the delivery claims nothing.
|
|
3012
|
+
const handedOn: string[] = [];
|
|
3013
|
+
for (const first of sources) {
|
|
3014
|
+
let { run, reg } = first;
|
|
3015
|
+
for (let hops = 0; ; hops++) {
|
|
3016
|
+
const runId = run.record.runId;
|
|
3017
|
+
if (this.table.lookup(reg.address) !== reg || this.runs.get(runId) !== run) {
|
|
3018
|
+
const next = this.standingSourceAfter(run, d.kind, d.key);
|
|
3019
|
+
if (!next) break;
|
|
3020
|
+
if (hops >= STANDING_HOPS) {
|
|
3021
|
+
handedOn.push(run.record.workflow); // the door answers 503, and the delivery claims nothing
|
|
3022
|
+
break;
|
|
3023
|
+
}
|
|
3024
|
+
({ run, reg } = next);
|
|
3025
|
+
continue;
|
|
3026
|
+
}
|
|
3027
|
+
const seat = reg.source!;
|
|
3028
|
+
const { item, ...line } = await seat.hear({
|
|
3029
|
+
hook: d.hook,
|
|
3030
|
+
kind: d.kind,
|
|
3031
|
+
key: d.key,
|
|
3032
|
+
delivery: d.delivery,
|
|
3033
|
+
payload: d.payload,
|
|
3034
|
+
caller: d.caller,
|
|
3035
|
+
});
|
|
3036
|
+
// A skip from a run that ended while it was heard: ask the new run's Source.
|
|
3037
|
+
if (line.verdict === "skipped" && this.runs.get(runId) !== run && this.isStanding(run.record.workflow)) {
|
|
3038
|
+
continue;
|
|
3039
|
+
}
|
|
3040
|
+
out.sources.push({ pool: runId, workflow: run.record.workflow, source: reg.id, ...line });
|
|
3041
|
+
if (line.verdict === "held" && this.runs.get(runId) === run) await this.persist(run);
|
|
3042
|
+
if (d.person && seat.reply) {
|
|
3043
|
+
const place = { target: "source" as const, workflow: run.record.workflow, run: runId, gate: reg.id };
|
|
3044
|
+
const verdict = { ...line, ...(item !== undefined ? { item } : {}) };
|
|
3045
|
+
const said = runReply(place, () => seat.reply!(verdict, d.caller));
|
|
3046
|
+
replies.push(line.verdict === "skipped" ? { ...said, skipped: true } : said);
|
|
3047
|
+
}
|
|
3048
|
+
break;
|
|
3049
|
+
}
|
|
3050
|
+
}
|
|
3051
|
+
// A standing Pool with no live run (ADR-0017): a paused one says `paused` and holds nothing, so a
|
|
3052
|
+
// read Source finds the item at the resume; one in its back-off makes the door answer 503, so
|
|
3053
|
+
// the service sends the delivery again.
|
|
3054
|
+
const without = this.standingWithoutRun(d.kind, d.key, d.may);
|
|
3055
|
+
for (const p of without.paused) {
|
|
3056
|
+
out.sources.push({ pool: p.run, workflow: p.workflow, source: "source", verdict: "paused" });
|
|
3057
|
+
}
|
|
3058
|
+
const waiting = [...new Set([...without.waiting, ...handedOn])];
|
|
3059
|
+
if (waiting.length) out.waiting = waiting;
|
|
3060
|
+
if (d.person) out.replies = replies;
|
|
3061
|
+
return out;
|
|
3062
|
+
}
|
|
3063
|
+
|
|
2339
3064
|
/**
|
|
2340
3065
|
* Observe a live run's feed (the `GET /runs/:id/events` SSE — ADR-0009): status deltas after every
|
|
2341
3066
|
* transition, plus the author's `emit`s. Returns an unsubscribe fn. The current status is **replayed
|
|
@@ -2415,12 +3140,22 @@ export class RunHost {
|
|
|
2415
3140
|
* Only when something waits on it (ADR-0071).
|
|
2416
3141
|
*/
|
|
2417
3142
|
private settledUnhosted(runId: string, status: "lost" | "drifted", reason: string, blob?: RunBlob): void {
|
|
3143
|
+
// No Lease runs for a run that a restore ended without hosting it, so its Sandboxes (kept, as a
|
|
3144
|
+
// drift keeps them) are marked here, and `jr2 workspace rm` can remove them (ADR-0021).
|
|
3145
|
+
this.sandbox?.releaseRun?.(runId).catch((err: unknown) => {
|
|
3146
|
+
console.error(
|
|
3147
|
+
`run ${runId}: its Sandboxes were not marked as held by no run: ` +
|
|
3148
|
+
(err instanceof Error ? err.message : String(err)),
|
|
3149
|
+
);
|
|
3150
|
+
});
|
|
2418
3151
|
if (!this.settleListeners.has(runId)) return;
|
|
2419
3152
|
const bare: RunStatus = {
|
|
2420
3153
|
runId,
|
|
2421
3154
|
workflow: blob?.workflow ?? "",
|
|
2422
3155
|
instanceId: blob?.instanceId ?? "",
|
|
3156
|
+
startedAt: blob?.startedAt ?? "",
|
|
2423
3157
|
...(blob?.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
|
|
3158
|
+
...(blob?.startedBy ? { startedBy: blob.startedBy } : {}),
|
|
2424
3159
|
status,
|
|
2425
3160
|
value: undefined,
|
|
2426
3161
|
context: undefined,
|
|
@@ -2507,7 +3242,7 @@ export class RunHost {
|
|
|
2507
3242
|
* terminal run reports its `done`/`error` status + final context here rather than 404-ing. Returns
|
|
2508
3243
|
* undefined only for a genuinely unknown run (or one marked `lost` with no row saved before, so no
|
|
2509
3244
|
* snapshot: `markLost` keeps a saved row's blob, whose key must keep answering, ADR-0071) — and a
|
|
2510
|
-
* swept one is unknown: a finished run's row lives
|
|
3245
|
+
* swept one is unknown: a finished run's row lives for `runs.retain` (ADR-0065).
|
|
2511
3246
|
*/
|
|
2512
3247
|
async read(runId: string): Promise<RunStatus | undefined> {
|
|
2513
3248
|
const live = this.status(runId);
|
|
@@ -2519,11 +3254,14 @@ export class RunHost {
|
|
|
2519
3254
|
// The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
|
|
2520
3255
|
// cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
|
|
2521
3256
|
// (ADR-0025). While the row still says "live", the Machine's own status is the answer.
|
|
2522
|
-
|
|
3257
|
+
// A live row's snapshot is `active`: the save that ends a run writes its row's end too.
|
|
3258
|
+
const status =
|
|
3259
|
+
stored.status === "live" ? shownOfMachine(snap.status ?? "active", undefined) : shownOfRow(stored.status);
|
|
2523
3260
|
return {
|
|
2524
3261
|
runId,
|
|
2525
3262
|
workflow: blob.workflow,
|
|
2526
3263
|
instanceId: blob.instanceId,
|
|
3264
|
+
startedAt: blob.startedAt ?? stored.startedAt ?? "",
|
|
2527
3265
|
status,
|
|
2528
3266
|
value: snap.value,
|
|
2529
3267
|
context: snap.context,
|
|
@@ -2535,6 +3273,12 @@ export class RunHost {
|
|
|
2535
3273
|
...(blob.assembly !== undefined ? { assembly: blob.assembly } : {}),
|
|
2536
3274
|
...boundFields(snap),
|
|
2537
3275
|
...(blob.dispatchedBy ? { dispatchedBy: blob.dispatchedBy } : {}),
|
|
3276
|
+
...(blob.startedBy ? { startedBy: blob.startedBy } : {}),
|
|
3277
|
+
...(blob.resumedBy ? { resumedBy: blob.resumedBy } : {}),
|
|
3278
|
+
...(blob.cancelledBy ? { cancelledBy: blob.cancelledBy } : {}),
|
|
3279
|
+
...(blob.handedOverTo ? { handedOverTo: blob.handedOverTo } : {}),
|
|
3280
|
+
...(blob.answers?.length ? { answers: blob.answers } : {}),
|
|
3281
|
+
...steeringOf(blob.notices?.steering ?? []),
|
|
2538
3282
|
// The Machine's output once the row says the run is over (ADR-0071), never while it is live;
|
|
2539
3283
|
// for a failed run, its Failure (ADR-0073).
|
|
2540
3284
|
...(stored.status !== "live" && blob.failure !== undefined ? { output: blob.failure } : {}),
|
|
@@ -2546,10 +3290,109 @@ export class RunHost {
|
|
|
2546
3290
|
};
|
|
2547
3291
|
}
|
|
2548
3292
|
|
|
3293
|
+
/**
|
|
3294
|
+
* The first Caller of a run's chain of starts (ADR-0078): its `startedBy` when that is a Caller,
|
|
3295
|
+
* else the first Caller of the run that dispatched it, and so on, live or stored. Undefined when
|
|
3296
|
+
* jr2 started the chain, or a run of it is no longer known. `jr2 status` walks the same chain.
|
|
3297
|
+
*/
|
|
3298
|
+
async firstCaller(runId: string): Promise<CallerRecord | undefined> {
|
|
3299
|
+
const seen = new Set<string>();
|
|
3300
|
+
for (let id: string | undefined = runId; id !== undefined && !seen.has(id); ) {
|
|
3301
|
+
seen.add(id);
|
|
3302
|
+
const by: RunStarter | undefined = (await this.read(id))?.startedBy;
|
|
3303
|
+
if (by?.kind === "caller") {
|
|
3304
|
+
const { kind: _kind, ...caller } = by;
|
|
3305
|
+
return caller;
|
|
3306
|
+
}
|
|
3307
|
+
id = by?.kind === "run" ? by.run : undefined;
|
|
3308
|
+
}
|
|
3309
|
+
return undefined;
|
|
3310
|
+
}
|
|
3311
|
+
|
|
2549
3312
|
list(): RunStatus[] {
|
|
2550
3313
|
return [...this.runs.keys()].map((id) => this.status(id)).filter((s): s is RunStatus => s !== undefined);
|
|
2551
3314
|
}
|
|
2552
3315
|
|
|
3316
|
+
/**
|
|
3317
|
+
* One page of the run listing (`GET /runs`, ADR-0009): the live runs from the registry, the
|
|
3318
|
+
* store's `live` rows that the registry does not hold (`active`, waiting for restore), and the
|
|
3319
|
+
* ended runs from the store's listing columns, in one order, newest start first. `readable` is
|
|
3320
|
+
* the Workflows that the Caller may read (`"all"`, or the names of its grant, which also open
|
|
3321
|
+
* their Charts): a run of another Workflow is left out. The store is asked for one row past the
|
|
3322
|
+
* page, so `next` is known with no second query.
|
|
3323
|
+
*/
|
|
3324
|
+
async history(query: RunListQuery, readable: "all" | readonly string[]): Promise<RunPage> {
|
|
3325
|
+
// The rule of every other read route (`callerMay`); the store's SQL is the same rule.
|
|
3326
|
+
const opens = (workflow: string): boolean => grantOpens(readable, workflow);
|
|
3327
|
+
const live = query.live
|
|
3328
|
+
? this.list()
|
|
3329
|
+
.filter((s) => s.status === LIVE_STATUS && opens(s.workflow))
|
|
3330
|
+
.filter((s) => query.workflow === undefined || s.workflow === query.workflow)
|
|
3331
|
+
.filter((s) => query.since === undefined || s.startedAt >= query.since)
|
|
3332
|
+
.filter((s) => query.after === undefined || isAfter(s, query.after))
|
|
3333
|
+
.map((s) => this.summaryOf(s))
|
|
3334
|
+
: [];
|
|
3335
|
+
const filter = {
|
|
3336
|
+
...(readable !== "all" ? { workflows: readable } : {}),
|
|
3337
|
+
...(query.workflow !== undefined ? { workflow: query.workflow } : {}),
|
|
3338
|
+
...(query.since !== undefined ? { since: query.since } : {}),
|
|
3339
|
+
...(query.after !== undefined ? { after: query.after } : {}),
|
|
3340
|
+
};
|
|
3341
|
+
const summary = (row: EndedRow, ended: boolean): RunSummary => ({
|
|
3342
|
+
runId: row.runId,
|
|
3343
|
+
workflow: row.workflow,
|
|
3344
|
+
status: ended ? shownOfRow(row.status) : LIVE_STATUS,
|
|
3345
|
+
...(row.reason !== undefined ? { reason: row.reason } : ended ? {} : { reason: "waiting for restore" }),
|
|
3346
|
+
...(row.startedBy !== undefined ? { startedBy: row.startedBy as RunStarter } : {}),
|
|
3347
|
+
startedAt: row.startedAt,
|
|
3348
|
+
...(ended ? { endedAt: row.endedAt } : {}),
|
|
3349
|
+
...(row.assembly !== undefined ? { assembly: row.assembly } : {}),
|
|
3350
|
+
openGates: 0,
|
|
3351
|
+
...(row.dispatchedBy !== undefined ? { dispatchedBy: row.dispatchedBy as DispatchedBy } : {}),
|
|
3352
|
+
});
|
|
3353
|
+
// A `live` row that the registry does not hold: a restore deferred or failed (ADR-0069,
|
|
3354
|
+
// ADR-0030). Each hosted run takes at most one of the rows asked for, so asking for that many
|
|
3355
|
+
// more leaves a full page.
|
|
3356
|
+
const waiting = query.live
|
|
3357
|
+
? (await this.store.liveRows({ ...filter, limit: query.limit + 1 + this.runs.size }))
|
|
3358
|
+
.filter((row) => !this.runs.has(row.runId))
|
|
3359
|
+
.map((row) => summary(row, false))
|
|
3360
|
+
: [];
|
|
3361
|
+
const ended = query.ended
|
|
3362
|
+
? (await this.store.ended({ statuses: query.ended, ...filter, limit: query.limit + 1 })).map((row) =>
|
|
3363
|
+
summary(row, true),
|
|
3364
|
+
)
|
|
3365
|
+
: [];
|
|
3366
|
+
// A run is in one half: `persist()` writes the terminal row and drops the run from the registry
|
|
3367
|
+
// in one synchronous step. The live one wins all the same.
|
|
3368
|
+
const liveIds = new Set(live.map((r) => r.runId));
|
|
3369
|
+
const merged = [...live, ...[...waiting, ...ended].filter((r) => !liveIds.has(r.runId))].sort(newestFirst);
|
|
3370
|
+
const runs = merged.slice(0, query.limit);
|
|
3371
|
+
const last = runs.at(-1);
|
|
3372
|
+
return merged.length > query.limit && last ? { runs, next: encodeCursor(last) } : { runs };
|
|
3373
|
+
}
|
|
3374
|
+
|
|
3375
|
+
/** A live run's summary (ADR-0009): its open Gates counted now, its Workspaces in `placing`. */
|
|
3376
|
+
private summaryOf(status: RunStatus): RunSummary {
|
|
3377
|
+
const placing = placingOf(status.value, status.children);
|
|
3378
|
+
return {
|
|
3379
|
+
runId: status.runId,
|
|
3380
|
+
workflow: status.workflow,
|
|
3381
|
+
status: status.status,
|
|
3382
|
+
...(status.reason !== undefined
|
|
3383
|
+
? { reason: status.reason }
|
|
3384
|
+
: status.fault !== undefined
|
|
3385
|
+
? { reason: status.fault }
|
|
3386
|
+
: {}),
|
|
3387
|
+
...(status.startedBy ? { startedBy: status.startedBy } : {}),
|
|
3388
|
+
startedAt: status.startedAt,
|
|
3389
|
+
...(status.assembly !== undefined ? { assembly: status.assembly } : {}),
|
|
3390
|
+
openGates: this.gates(status.runId).length,
|
|
3391
|
+
...(placing.length ? { placing } : {}),
|
|
3392
|
+
...(status.dispatchedBy ? { dispatchedBy: status.dispatchedBy } : {}),
|
|
3393
|
+
};
|
|
3394
|
+
}
|
|
3395
|
+
|
|
2553
3396
|
/**
|
|
2554
3397
|
* Run ids sharing a prefix — the live registry unioned with the store — backing abbreviated run
|
|
2555
3398
|
* ids in the CLI (ADR-0009). The live half is not belt-and-braces: `persist()` is scheduled on a
|
|
@@ -2618,6 +3461,21 @@ export class RunHost {
|
|
|
2618
3461
|
for (const listener of listeners) listener({ kind: "closed" });
|
|
2619
3462
|
}
|
|
2620
3463
|
this.workflowListeners.clear();
|
|
3464
|
+
for (const hook of [...this.closeHooks]) hook();
|
|
3465
|
+
this.closeHooks.clear();
|
|
3466
|
+
}
|
|
3467
|
+
|
|
3468
|
+
private readonly closeHooks = new Set<() => void>();
|
|
3469
|
+
|
|
3470
|
+
/** Run `fn` when `close()` begins, so a request that has no natural end (a follow of a kept
|
|
3471
|
+
* conversation) can end and the server can close. Returns the call that removes `fn`. */
|
|
3472
|
+
onClose(fn: () => void): () => void {
|
|
3473
|
+
if (this.#stopping) {
|
|
3474
|
+
fn();
|
|
3475
|
+
return () => {};
|
|
3476
|
+
}
|
|
3477
|
+
this.closeHooks.add(fn);
|
|
3478
|
+
return () => this.closeHooks.delete(fn);
|
|
2621
3479
|
}
|
|
2622
3480
|
|
|
2623
3481
|
/**
|
|
@@ -2664,18 +3522,17 @@ export class RunHost {
|
|
|
2664
3522
|
* fault a run whose Agents were stopped on purpose.
|
|
2665
3523
|
*
|
|
2666
3524
|
* It then destroys every Sandbox the run owns, unless `keep` (`jr2 send <run> --event CANCEL
|
|
2667
|
-
* --keep`): kept, they age out at the operator's idle timeout, and `jr2
|
|
3525
|
+
* --keep`): kept, they age out at the operator's idle timeout, and `jr2 workspace shell` reaches them until
|
|
2668
3526
|
* then. A faulted run never comes through here, and keeps its Workspaces (ADR-0012).
|
|
2669
3527
|
*/
|
|
2670
|
-
async cancel(runId: string, opts: { keep?: boolean } = {}): Promise<void> {
|
|
3528
|
+
async cancel(runId: string, opts: { keep?: boolean; by?: CallerRecord } = {}): Promise<void> {
|
|
2671
3529
|
const run = this.runs.get(runId);
|
|
2672
3530
|
if (!run) return;
|
|
3531
|
+
// The Caller that cancelled it (ADR-0078), saved with the terminal snapshot below.
|
|
3532
|
+
if (opts.by) run.cancelledBy = opts.by;
|
|
2673
3533
|
// Named BEFORE the stop, while the tree still says which Workspaces are live.
|
|
2674
3534
|
const sandboxes = opts.keep ? [] : runSandboxes(run.actor, runId);
|
|
2675
|
-
|
|
2676
|
-
// Persist AFTER the stop: the snapshot is final, and `persist` fans out the last status,
|
|
2677
|
-
// announces `gone`, and untracks — the same terminal path a run that settled on its own takes.
|
|
2678
|
-
this.persist(run, "cancelled");
|
|
3535
|
+
this.end(run, "cancelled");
|
|
2679
3536
|
// The run is over, so nothing can use its Workspaces, and each holds its whole Size (ADR-0025).
|
|
2680
3537
|
// An actor stop skips the wrapper's `teardown` state, so the host runs the same idempotent
|
|
2681
3538
|
// delete here, after the stop, so no lease renews what was deleted. `keep` leaves them to the
|
|
@@ -2693,8 +3550,24 @@ export class RunHost {
|
|
|
2693
3550
|
const run = this.runs.get(runId);
|
|
2694
3551
|
if (!run || run.binding !== binding || run.fault !== undefined) return;
|
|
2695
3552
|
run.fault = fault;
|
|
3553
|
+
this.end(run, "error");
|
|
3554
|
+
}
|
|
3555
|
+
|
|
3556
|
+
/**
|
|
3557
|
+
* End a live run that the HOST ends, not its Machine: stop its actor, then save it with `status`.
|
|
3558
|
+
* The save comes AFTER the stop, so the snapshot is final; it feeds the last status, announces
|
|
3559
|
+
* `gone` and untracks, as for a run that ended on its own.
|
|
3560
|
+
*
|
|
3561
|
+
* The run is marked first, because the stop is not quiet. xstate gives the root the status
|
|
3562
|
+
* `stopped` before it stops the children, and each Agent invocation that stops ends its Turn in
|
|
3563
|
+
* the notice ledger, whose hook saves the run (ADR-0024, ADR-0062). That save would write and
|
|
3564
|
+
* feed `stopped`, and untrack the run before this end is said. While the mark is set, `persist`
|
|
3565
|
+
* writes nothing: the save below carries what the stop changed.
|
|
3566
|
+
*/
|
|
3567
|
+
private end(run: LiveRun, status: EndedStatus): void {
|
|
3568
|
+
run.ending = status;
|
|
2696
3569
|
run.actor.stop();
|
|
2697
|
-
this.persist(run,
|
|
3570
|
+
this.persist(run, status);
|
|
2698
3571
|
}
|
|
2699
3572
|
|
|
2700
3573
|
/**
|
|
@@ -2726,6 +3599,346 @@ export class RunHost {
|
|
|
2726
3599
|
this.announce(run.record.workflow, { kind: "gone", runId: run.record.runId });
|
|
2727
3600
|
}
|
|
2728
3601
|
|
|
3602
|
+
// --- standing Pools (ADR-0017) ---
|
|
3603
|
+
|
|
3604
|
+
/** The heads of the standing Workflows, by Workflow. */
|
|
3605
|
+
private get heads(): Map<string, StandingHead> {
|
|
3606
|
+
if (!this.#heads) this.#heads = new Map(this.store.standingHeads().map((h) => [h.workflow, h]));
|
|
3607
|
+
return this.#heads;
|
|
3608
|
+
}
|
|
3609
|
+
|
|
3610
|
+
private writeHead(head: StandingHead): void {
|
|
3611
|
+
this.heads.set(head.workflow, head);
|
|
3612
|
+
this.store.writeStanding(head);
|
|
3613
|
+
}
|
|
3614
|
+
|
|
3615
|
+
/** Is `workflow` a standing Workflow of the registry: its Machine a standing Pool (ADR-0017)? */
|
|
3616
|
+
isStanding(workflow: string): boolean {
|
|
3617
|
+
return standingPoolOf(this.workflowDefs.get(workflow)?.machine) !== undefined;
|
|
3618
|
+
}
|
|
3619
|
+
|
|
3620
|
+
/** The hosted runs of `workflow`, oldest first. */
|
|
3621
|
+
private hostedOf(workflow: string): LiveRun[] {
|
|
3622
|
+
return [...this.runs.values()].filter((r) => r.record.workflow === workflow);
|
|
3623
|
+
}
|
|
3624
|
+
|
|
3625
|
+
/** The live standing run of `workflow`: the head when this process hosts it, else the newest
|
|
3626
|
+
* hosted run. */
|
|
3627
|
+
private standingRun(workflow: string): LiveRun | undefined {
|
|
3628
|
+
const hosted = this.hostedOf(workflow);
|
|
3629
|
+
const head = this.heads.get(workflow);
|
|
3630
|
+
return hosted.find((r) => r.record.runId === head?.run) ?? hosted.at(-1);
|
|
3631
|
+
}
|
|
3632
|
+
|
|
3633
|
+
private clearStandingTimer(workflow: string): void {
|
|
3634
|
+
const timer = this.standingTimers.get(workflow);
|
|
3635
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
3636
|
+
this.standingTimers.delete(workflow);
|
|
3637
|
+
}
|
|
3638
|
+
|
|
3639
|
+
private setStandingTimer(workflow: string, ms: number, fn: () => void): void {
|
|
3640
|
+
this.clearStandingTimer(workflow);
|
|
3641
|
+
const timer = setTimeout(
|
|
3642
|
+
() => {
|
|
3643
|
+
this.standingTimers.delete(workflow);
|
|
3644
|
+
if (!this.#stopping) fn();
|
|
3645
|
+
},
|
|
3646
|
+
Math.max(0, ms),
|
|
3647
|
+
);
|
|
3648
|
+
// A day's bound never holds the process: the next boot arms it again from the head's `since`.
|
|
3649
|
+
(timer as { unref?: () => void }).unref?.();
|
|
3650
|
+
this.standingTimers.set(workflow, timer);
|
|
3651
|
+
}
|
|
3652
|
+
|
|
3653
|
+
/** Arm the age bound of a standing Workflow's live run: at `since` + the bound, hand it over. */
|
|
3654
|
+
private armAge(workflow: string, run: LiveRun): void {
|
|
3655
|
+
const since = Date.parse(this.heads.get(workflow)?.since ?? "") || this.now();
|
|
3656
|
+
this.setStandingTimer(workflow, since + this.standingOpts.ageMs - this.now(), () => {
|
|
3657
|
+
if (this.runs.get(run.record.runId) === run && this.heads.get(workflow)?.run === run.record.runId) {
|
|
3658
|
+
this.handOverLive(run, "hand-over");
|
|
3659
|
+
}
|
|
3660
|
+
});
|
|
3661
|
+
}
|
|
3662
|
+
|
|
3663
|
+
/**
|
|
3664
|
+
* Keep one live run of each standing Workflow under the current registry (ADR-0017). At boot,
|
|
3665
|
+
* after the restore, and at each publish, after the swap. A live run of an older Assembly (or of
|
|
3666
|
+
* another Machine) is handed over. A run that ended `error`, `drifted` or `lost` is handed over
|
|
3667
|
+
* from its blob. A paused Workflow gets no run, and nor does one in its back-off. A Workflow with
|
|
3668
|
+
* no head gets its first run, which records `why`.
|
|
3669
|
+
*/
|
|
3670
|
+
async keepStanding(why: "boot" | "publish"): Promise<void> {
|
|
3671
|
+
for (const [workflow, def] of [...this.workflowDefs]) {
|
|
3672
|
+
if (!standingPoolOf(def.machine)) continue;
|
|
3673
|
+
try {
|
|
3674
|
+
await this.keepOne(workflow, def, why);
|
|
3675
|
+
} catch (err) {
|
|
3676
|
+
// One standing Workflow that cannot start is one Workflow, not the boot (ADR-0030).
|
|
3677
|
+
console.error(
|
|
3678
|
+
`standing workflow "${workflow}": no run was started: ${err instanceof Error ? err.message : String(err)}`,
|
|
3679
|
+
);
|
|
3680
|
+
}
|
|
3681
|
+
}
|
|
3682
|
+
}
|
|
3683
|
+
|
|
3684
|
+
private async keepOne(workflow: string, def: WorkflowDef, why: "boot" | "publish"): Promise<void> {
|
|
3685
|
+
const head = this.heads.get(workflow);
|
|
3686
|
+
const live = this.standingRun(workflow);
|
|
3687
|
+
if (live) {
|
|
3688
|
+
if (head?.run !== live.record.runId) {
|
|
3689
|
+
this.writeHead({ workflow, run: live.record.runId, since: new Date(this.now()).toISOString(), streak: 0 });
|
|
3690
|
+
}
|
|
3691
|
+
// A hand-over at every publish: the Assembly is one hash, and a handler has no stamp of its own.
|
|
3692
|
+
if (live.def.assembly !== def.assembly || live.def.machine !== def.machine) {
|
|
3693
|
+
this.handOverLive(live, "hand-over");
|
|
3694
|
+
} else {
|
|
3695
|
+
this.armAge(workflow, live);
|
|
3696
|
+
}
|
|
3697
|
+
return;
|
|
3698
|
+
}
|
|
3699
|
+
if (head?.paused) return;
|
|
3700
|
+
if (this.standingTimers.has(workflow)) return; // in its back-off
|
|
3701
|
+
if (head) {
|
|
3702
|
+
if (this.unhosted.has(head.run)) return; // restore left it live to try again
|
|
3703
|
+
const stored = await this.store.load(head.run);
|
|
3704
|
+
const blob = stored?.snapshot as RunBlob | null | undefined;
|
|
3705
|
+
// A live row that this process does not host is not ended: never a second run beside it.
|
|
3706
|
+
if (stored?.status === "live") return;
|
|
3707
|
+
if (stored && blob) {
|
|
3708
|
+
if (stored.status === "cancelled") {
|
|
3709
|
+
this.writeHead({ ...head, paused: "cancel" });
|
|
3710
|
+
return;
|
|
3711
|
+
}
|
|
3712
|
+
this.handOverFrom(workflow, blob.snapshot, "hand-over", { streak: head.streak });
|
|
3713
|
+
return;
|
|
3714
|
+
}
|
|
3715
|
+
}
|
|
3716
|
+
this.startStanding(workflow, why, {});
|
|
3717
|
+
}
|
|
3718
|
+
|
|
3719
|
+
/** Start a standing run with what a hand-over carries, and make it the head. */
|
|
3720
|
+
private startStanding(
|
|
3721
|
+
workflow: string,
|
|
3722
|
+
why: "boot" | "publish" | "hand-over" | "resume",
|
|
3723
|
+
opts: { context?: unknown; dropHeld?: boolean; resumedBy?: CallerRecord; streak?: number },
|
|
3724
|
+
): { runId: string; instanceId: string } {
|
|
3725
|
+
this.clearStandingTimer(workflow);
|
|
3726
|
+
const carry = standingCarryOf(opts.context, { dropHeld: opts.dropHeld === true });
|
|
3727
|
+
const ids = this.startNow(workflow, { [HAND_OVER]: carry }, undefined, { kind: "jr2", why }, opts.resumedBy);
|
|
3728
|
+
this.writeHead({
|
|
3729
|
+
workflow,
|
|
3730
|
+
run: ids.runId,
|
|
3731
|
+
since: new Date(this.now()).toISOString(),
|
|
3732
|
+
streak: opts.streak ?? 0,
|
|
3733
|
+
});
|
|
3734
|
+
const run = this.runs.get(ids.runId);
|
|
3735
|
+
if (run) this.armAge(workflow, run);
|
|
3736
|
+
return ids;
|
|
3737
|
+
}
|
|
3738
|
+
|
|
3739
|
+
/**
|
|
3740
|
+
* Hand a live standing run over (ADR-0017): start the new run with its held items and active keys,
|
|
3741
|
+
* and end it `done` with `handedOverTo`, in one synchronous step, so no delivery and no settle
|
|
3742
|
+
* falls between the two runs.
|
|
3743
|
+
*/
|
|
3744
|
+
private handOverLive(run: LiveRun, why: "hand-over"): void {
|
|
3745
|
+
const workflow = run.record.workflow;
|
|
3746
|
+
const ids = this.startStanding(workflow, why, { context: run.actor.getSnapshot().context });
|
|
3747
|
+
run.handedOverTo = ids.runId;
|
|
3748
|
+
this.end(run, "done");
|
|
3749
|
+
}
|
|
3750
|
+
|
|
3751
|
+
/** Hand over from a run that is no longer live, by its saved context: `error`, `drifted`, `lost`,
|
|
3752
|
+
* or a paused run that a Caller resumes. */
|
|
3753
|
+
private handOverFrom(
|
|
3754
|
+
workflow: string,
|
|
3755
|
+
snapshot: unknown,
|
|
3756
|
+
why: "hand-over" | "resume",
|
|
3757
|
+
opts: { dropHeld?: boolean; resumedBy?: CallerRecord; streak?: number },
|
|
3758
|
+
): { runId: string; instanceId: string } {
|
|
3759
|
+
const context = (snapshot as { context?: unknown } | null | undefined)?.context;
|
|
3760
|
+
return this.startStanding(workflow, why, { ...opts, context });
|
|
3761
|
+
}
|
|
3762
|
+
|
|
3763
|
+
/**
|
|
3764
|
+
* What the end of a standing run means (ADR-0017), from `persist`: a cancel pauses the Workflow;
|
|
3765
|
+
* an `error` or a `failed` (ADR-0073) hands it over after the back-off, and so does a run that
|
|
3766
|
+
* took its bound with no run started. One rule for each of these ends: a run in which an item that
|
|
3767
|
+
* it took itself (not a carried key) started a run starts the count again, and any other run
|
|
3768
|
+
* counts. After 3 hand-overs from counted runs in a row, the next counted run pauses it. A
|
|
3769
|
+
* hand-over's own end, and a run that is not the head, mean nothing more.
|
|
3770
|
+
*/
|
|
3771
|
+
private standingEnded(run: LiveRun, status: EndedStatus): void {
|
|
3772
|
+
const workflow = run.record.workflow;
|
|
3773
|
+
const head = this.heads.get(workflow);
|
|
3774
|
+
if (head?.run !== run.record.runId || run.handedOverTo !== undefined) return;
|
|
3775
|
+
this.clearStandingTimer(workflow);
|
|
3776
|
+
if (status === "cancelled") {
|
|
3777
|
+
this.writeHead({ ...head, paused: "cancel" });
|
|
3778
|
+
return;
|
|
3779
|
+
}
|
|
3780
|
+
const atBound = run.boundWithNoRun === true;
|
|
3781
|
+
if (!atBound && status !== "error" && status !== "failed") return;
|
|
3782
|
+
// One rule for each end that counts: only a run in which an item that it took itself (not a
|
|
3783
|
+
// carried key) started a run starts the count again. Any other run is part of a poison loop.
|
|
3784
|
+
const streak = atBound || !tookOf(run).started ? head.streak + 1 : 0;
|
|
3785
|
+
if (streak > STANDING_ERRORS_PAUSE) {
|
|
3786
|
+
this.writeHead({ ...head, streak, paused: atBound ? "bound" : "errors" });
|
|
3787
|
+
const last = atBound ? `took its bound of ${this.standingOpts.dispatches} items` : `ended ${status}`;
|
|
3788
|
+
console.error(
|
|
3789
|
+
`standing workflow "${workflow}": paused — in ${streak} runs in a row, no item that the run took started a ` +
|
|
3790
|
+
`run, after ${STANDING_ERRORS_PAUSE} hand-overs; the last one ${last}; \`jr2 run ${workflow}\` resumes it (ADR-0017)`,
|
|
3791
|
+
);
|
|
3792
|
+
return;
|
|
3793
|
+
}
|
|
3794
|
+
this.writeHead({ ...head, streak });
|
|
3795
|
+
const context = run.actor.getSnapshot().context;
|
|
3796
|
+
this.setStandingTimer(workflow, this.standingOpts.backoffMs(Math.max(1, streak)), () => {
|
|
3797
|
+
if (this.heads.get(workflow)?.run !== run.record.runId || this.hostedOf(workflow).length > 0) return;
|
|
3798
|
+
if (!this.isStanding(workflow)) return;
|
|
3799
|
+
try {
|
|
3800
|
+
this.startStanding(workflow, "hand-over", { context, streak });
|
|
3801
|
+
} catch (err) {
|
|
3802
|
+
console.error(
|
|
3803
|
+
`standing workflow "${workflow}": the hand-over after ${atBound ? "the bound with no run started" : status} failed: ` +
|
|
3804
|
+
(err instanceof Error ? err.message : String(err)),
|
|
3805
|
+
);
|
|
3806
|
+
}
|
|
3807
|
+
});
|
|
3808
|
+
}
|
|
3809
|
+
|
|
3810
|
+
/**
|
|
3811
|
+
* A live standing run that took its bound of items, each one a dispatch, is handed over after the
|
|
3812
|
+
* save that counted the last one (ADR-0017). When none of the items it took itself started a run
|
|
3813
|
+
* (a carried key's run does not count: it was started before), the
|
|
3814
|
+
* run ends `done` and the hand-over waits out the back-off, as after `error`: an item that ends
|
|
3815
|
+
* with no run is known only in its run's life, so a backlog of such items fills the bound again
|
|
3816
|
+
* in each new run, and a hand-over at once would loop with no pause and no end. While no item has
|
|
3817
|
+
* an answer yet (each one in flight), jr2 waits: the save of the first answer asks again.
|
|
3818
|
+
*/
|
|
3819
|
+
private checkBound(run: LiveRun): void {
|
|
3820
|
+
const workflow = run.record.workflow;
|
|
3821
|
+
if (this.heads.get(workflow)?.run !== run.record.runId) return;
|
|
3822
|
+
// The items it took, not the keys a hand-over carried: each carried key is sent again, so a
|
|
3823
|
+
// count of its ledger's rows would hand a run with `cap` at the bound over again at once.
|
|
3824
|
+
if (standingTaken(run.actor.getSnapshot().context) < this.standingOpts.dispatches) return;
|
|
3825
|
+
const took = tookOf(run);
|
|
3826
|
+
if (!took.started && took.answered === 0) return;
|
|
3827
|
+
queueMicrotask(() => {
|
|
3828
|
+
if (this.runs.get(run.record.runId) !== run || this.heads.get(workflow)?.run !== run.record.runId) return;
|
|
3829
|
+
if (!this.isStanding(workflow)) return;
|
|
3830
|
+
if (tookOf(run).started) {
|
|
3831
|
+
this.handOverLive(run, "hand-over");
|
|
3832
|
+
return;
|
|
3833
|
+
}
|
|
3834
|
+
run.boundWithNoRun = true;
|
|
3835
|
+
this.end(run, "done");
|
|
3836
|
+
});
|
|
3837
|
+
}
|
|
3838
|
+
|
|
3839
|
+
/**
|
|
3840
|
+
* Resume a paused standing Workflow (ADR-0017): a hand-over from the run that the pause ended,
|
|
3841
|
+
* with its held items (none with `dropHeld`) and its active keys. The new run records the Caller
|
|
3842
|
+
* as `resumedBy`. Throws {@link StandingRefused} when the Workflow is live, or not paused.
|
|
3843
|
+
*/
|
|
3844
|
+
async resumeStanding(
|
|
3845
|
+
workflow: string,
|
|
3846
|
+
by: CallerRecord,
|
|
3847
|
+
opts: { dropHeld?: boolean } = {},
|
|
3848
|
+
): Promise<{ runId: string; instanceId: string }> {
|
|
3849
|
+
if (!this.isStanding(workflow)) throw new StandingRefused(`"${workflow}" is not a standing Workflow`);
|
|
3850
|
+
const live = this.standingRun(workflow);
|
|
3851
|
+
if (live) {
|
|
3852
|
+
throw new StandingRefused(
|
|
3853
|
+
`"${workflow}" is a standing Workflow, and its run ${live.record.runId} is live: jr2 keeps its run, and ` +
|
|
3854
|
+
"`jr2 run` only resumes a paused one (ADR-0017)",
|
|
3855
|
+
);
|
|
3856
|
+
}
|
|
3857
|
+
const head = this.heads.get(workflow);
|
|
3858
|
+
if (!head?.paused) {
|
|
3859
|
+
throw new StandingRefused(
|
|
3860
|
+
`"${workflow}" is a standing Workflow, and it is not paused: jr2 starts its next run after the back-off, and ` +
|
|
3861
|
+
"`jr2 run` only resumes a paused one (ADR-0017)",
|
|
3862
|
+
);
|
|
3863
|
+
}
|
|
3864
|
+
const blob = (await this.store.load(head.run))?.snapshot as RunBlob | null | undefined;
|
|
3865
|
+
// Asked again after the read: another resume may have started the run.
|
|
3866
|
+
if (this.standingRun(workflow) || !this.heads.get(workflow)?.paused) {
|
|
3867
|
+
throw new StandingRefused(`"${workflow}" was resumed a moment ago`);
|
|
3868
|
+
}
|
|
3869
|
+
return this.handOverFrom(workflow, blob?.snapshot, "resume", {
|
|
3870
|
+
resumedBy: by,
|
|
3871
|
+
dropHeld: opts.dropHeld === true,
|
|
3872
|
+
streak: 0,
|
|
3873
|
+
});
|
|
3874
|
+
}
|
|
3875
|
+
|
|
3876
|
+
/** Each standing Workflow of the registry, as `jr2 up` and `jr2 status` print it (ADR-0017). */
|
|
3877
|
+
async standingView(): Promise<StandingView[]> {
|
|
3878
|
+
const out: StandingView[] = [];
|
|
3879
|
+
for (const [workflow, def] of this.workflowDefs) {
|
|
3880
|
+
const info = standingPoolOf(def.machine);
|
|
3881
|
+
if (!info) continue;
|
|
3882
|
+
const head = this.heads.get(workflow);
|
|
3883
|
+
const live = this.standingRun(workflow);
|
|
3884
|
+
const base = { workflow, pool: info.id, callee: info.callee };
|
|
3885
|
+
if (live) {
|
|
3886
|
+
const held = standingCarryOf(live.actor.getSnapshot().context).held.length;
|
|
3887
|
+
out.push({ ...base, state: "live", run: live.record.runId, held });
|
|
3888
|
+
continue;
|
|
3889
|
+
}
|
|
3890
|
+
let held = 0;
|
|
3891
|
+
if (head) {
|
|
3892
|
+
const blob = (await this.store.load(head.run).catch(() => null))?.snapshot as RunBlob | null | undefined;
|
|
3893
|
+
held = standingCarryOf((blob?.snapshot as { context?: unknown } | undefined)?.context).held.length;
|
|
3894
|
+
}
|
|
3895
|
+
out.push({
|
|
3896
|
+
...base,
|
|
3897
|
+
state: head?.paused ? "paused" : "waiting",
|
|
3898
|
+
...(head ? { run: head.run } : {}),
|
|
3899
|
+
held,
|
|
3900
|
+
...(head?.paused ? { why: head.paused } : {}),
|
|
3901
|
+
});
|
|
3902
|
+
}
|
|
3903
|
+
return out;
|
|
3904
|
+
}
|
|
3905
|
+
|
|
3906
|
+
/** The live run that took over from a standing run that a hand-over ended (ADR-0017), and its
|
|
3907
|
+
* Source's registration, or undefined when the Workflow does not stand or has no live run. */
|
|
3908
|
+
private standingSourceAfter(
|
|
3909
|
+
ended: LiveRun,
|
|
3910
|
+
kind: string,
|
|
3911
|
+
key: string,
|
|
3912
|
+
): { run: LiveRun; reg: Registration } | undefined {
|
|
3913
|
+
const workflow = ended.record.workflow;
|
|
3914
|
+
if (!this.isStanding(workflow)) return undefined;
|
|
3915
|
+
const run = this.standingRun(workflow);
|
|
3916
|
+
if (!run || run === ended) return undefined;
|
|
3917
|
+
for (const reg of this.table.byRun(run.record.runId)) {
|
|
3918
|
+
if (reg.kind === "gate" && reg.source?.hears(kind, key)) return { run, reg };
|
|
3919
|
+
}
|
|
3920
|
+
return undefined;
|
|
3921
|
+
}
|
|
3922
|
+
|
|
3923
|
+
/** The standing Workflows that hear a Hook's kind and key and have no live run (ADR-0017): the
|
|
3924
|
+
* paused ones, by their head, and the ones waiting out a back-off. */
|
|
3925
|
+
private standingWithoutRun(
|
|
3926
|
+
kind: string,
|
|
3927
|
+
key: string,
|
|
3928
|
+
may: (workflow: string) => boolean,
|
|
3929
|
+
): { paused: Array<{ workflow: string; run: string }>; waiting: string[] } {
|
|
3930
|
+
const paused: Array<{ workflow: string; run: string }> = [];
|
|
3931
|
+
const waiting: string[] = [];
|
|
3932
|
+
for (const [workflow, def] of this.workflowDefs) {
|
|
3933
|
+
const info = standingPoolOf(def.machine);
|
|
3934
|
+
if (!info || !info.hears(kind, key) || !may(workflow) || this.standingRun(workflow)) continue;
|
|
3935
|
+
const head = this.heads.get(workflow);
|
|
3936
|
+
if (head?.paused) paused.push({ workflow, run: head.run });
|
|
3937
|
+
else waiting.push(workflow);
|
|
3938
|
+
}
|
|
3939
|
+
return { paused, waiting };
|
|
3940
|
+
}
|
|
3941
|
+
|
|
2729
3942
|
// --- internals ---
|
|
2730
3943
|
|
|
2731
3944
|
/** Put one event on the run's feed: buffer it for ADR-0023's backfill, then fan it out. The
|
|
@@ -2782,13 +3995,17 @@ export class RunHost {
|
|
|
2782
3995
|
const snap = run.actor.getSnapshot();
|
|
2783
3996
|
return {
|
|
2784
3997
|
...run.record,
|
|
2785
|
-
status: snap.status,
|
|
3998
|
+
status: shownOfMachine(snap.status, run.ending),
|
|
2786
3999
|
value: snap.value,
|
|
2787
4000
|
context: snap.context,
|
|
2788
4001
|
children: runChildren(snap),
|
|
2789
4002
|
waiting: runWaiting(snap, run.actor.logic),
|
|
2790
4003
|
held: heldOf(run.holds.values()),
|
|
2791
4004
|
fault: run.fault,
|
|
4005
|
+
...(run.cancelledBy ? { cancelledBy: run.cancelledBy } : {}),
|
|
4006
|
+
...(run.handedOverTo ? { handedOverTo: run.handedOverTo } : {}),
|
|
4007
|
+
...(run.answers.length ? { answers: [...run.answers] } : {}),
|
|
4008
|
+
...steeringOf(run.notices.steeringRecord()),
|
|
2792
4009
|
...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
|
|
2793
4010
|
...boundFields(snap),
|
|
2794
4011
|
// A Machine that reached its final state has its output (ADR-0071); an active one has none.
|
|
@@ -2826,7 +4043,7 @@ export class RunHost {
|
|
|
2826
4043
|
noticeState?: NoticeLedgerState,
|
|
2827
4044
|
holdState: Hold[] = [],
|
|
2828
4045
|
effects: Record<string, EffectRow> = {},
|
|
2829
|
-
|
|
4046
|
+
childAgents: Record<string, ChildAgent> = {},
|
|
2830
4047
|
): AnyActor {
|
|
2831
4048
|
const notices = new NoticeLedger(noticeState, this.newId);
|
|
2832
4049
|
const holds = new Map(holdState.map((hold) => [hold.instanceId, hold]));
|
|
@@ -2848,10 +4065,12 @@ export class RunHost {
|
|
|
2848
4065
|
workflow: record.workflow,
|
|
2849
4066
|
...(this.orchestratorName !== undefined ? { orchestratorName: this.orchestratorName } : {}),
|
|
2850
4067
|
depth: record.dispatchedBy?.depth ?? 0,
|
|
4068
|
+
firstCaller: () => this.firstCaller(record.runId),
|
|
2851
4069
|
// A `dispatch` with no target (ADR-0071): the run host itself, with no wire and no token.
|
|
2852
4070
|
dispatch: {
|
|
2853
4071
|
start: (workflow, input, by) => this.dispatchStart(workflow, input, by),
|
|
2854
4072
|
settled: (runId, signal) => this.settledOf(runId, signal),
|
|
4073
|
+
lookup: (workflow, key) => this.dispatched(workflow, key),
|
|
2855
4074
|
},
|
|
2856
4075
|
...(def.assembly !== undefined ? { assembly: def.assembly } : {}),
|
|
2857
4076
|
table: this.table,
|
|
@@ -2914,6 +4133,30 @@ export class RunHost {
|
|
|
2914
4133
|
const run = this.runs.get(record.runId);
|
|
2915
4134
|
if (run && run.notices === notices) this.persist(run);
|
|
2916
4135
|
},
|
|
4136
|
+
// The Machine's `steer` action (ADR-0002): saved with the transition that runs it.
|
|
4137
|
+
steer: (machine, agentName, text, from) => {
|
|
4138
|
+
const actors = ((machine as { logic?: AnyStateMachine }).logic?.implementations.actors ?? {}) as Record<
|
|
4139
|
+
string,
|
|
4140
|
+
unknown
|
|
4141
|
+
>;
|
|
4142
|
+
if (!isAgent(actors[agentName])) {
|
|
4143
|
+
throw new Error(
|
|
4144
|
+
`steer("${agentName}", …): this Machine carries no Agent "${agentName}" — a Steering message goes to ` +
|
|
4145
|
+
"an Agent slot of the Machine whose transition sends it (ADR-0002)",
|
|
4146
|
+
);
|
|
4147
|
+
}
|
|
4148
|
+
notices.steer(
|
|
4149
|
+
{
|
|
4150
|
+
kind: "steering",
|
|
4151
|
+
scope: "slot",
|
|
4152
|
+
text,
|
|
4153
|
+
author: { kind: "machine", ...(from !== undefined ? { from: { party: from.party, id: from.id } } : {}) },
|
|
4154
|
+
at: new Date(this.now()).toISOString(),
|
|
4155
|
+
},
|
|
4156
|
+
agentSlot(machine, agentName),
|
|
4157
|
+
);
|
|
4158
|
+
schedule();
|
|
4159
|
+
},
|
|
2917
4160
|
takeNotices: (scope) => notices.take(scope),
|
|
2918
4161
|
releaseNotices: (ids, taker) => notices.release(ids, taker),
|
|
2919
4162
|
peekNotices: (scope) => notices.peek(scope),
|
|
@@ -2977,12 +4220,12 @@ export class RunHost {
|
|
|
2977
4220
|
// Agent whose state the action is written below — the one in its own send first, else the one
|
|
2978
4221
|
// outstanding — and the value that replaces its default answer.
|
|
2979
4222
|
chainOf: (machine, seat, event) => {
|
|
2980
|
-
const pick = this.answeringIn(machine, seat, record.runId, holds,
|
|
4223
|
+
const pick = this.answeringIn(machine, seat, record.runId, holds, childAgents, event);
|
|
2981
4224
|
if (pick === undefined) return undefined;
|
|
2982
4225
|
return "reg" in pick ? [...pick.chain] : [...(pick.chain ?? [])];
|
|
2983
4226
|
},
|
|
2984
4227
|
setAnswer: (machine, seat, value, event) => {
|
|
2985
|
-
const pick = this.answeringIn(machine, seat, record.runId, holds,
|
|
4228
|
+
const pick = this.answeringIn(machine, seat, record.runId, holds, childAgents, event);
|
|
2986
4229
|
if (pick === undefined) return;
|
|
2987
4230
|
// Not saved here: the action runs inside a transition, and the persist that follows it carries
|
|
2988
4231
|
// the value — a save now would read the Machine before its route, and answer too early.
|
|
@@ -3071,8 +4314,8 @@ export class RunHost {
|
|
|
3071
4314
|
const agent = sending.reg.sender?.id;
|
|
3072
4315
|
const invoke = sending.reg.invokeId;
|
|
3073
4316
|
const key = childKey(child._parent, child.id);
|
|
3074
|
-
if (agent !== undefined)
|
|
3075
|
-
else delete
|
|
4317
|
+
if (agent !== undefined) childAgents[key] = invoke !== undefined ? { agent, invoke } : { agent };
|
|
4318
|
+
else delete childAgents[key];
|
|
3076
4319
|
}
|
|
3077
4320
|
}
|
|
3078
4321
|
// …and so are the children its later work starts: a child started while the Machine takes an
|
|
@@ -3088,12 +4331,17 @@ export class RunHost {
|
|
|
3088
4331
|
const after = childRefsOf(snapshot);
|
|
3089
4332
|
childrenAfter.set(taker, after);
|
|
3090
4333
|
const started = [...after].filter((c) => !before.has(c)).map((c) => c.id);
|
|
4334
|
+
// A Machine instance that left the tree ended (ADR-0002): the Steering messages held for
|
|
4335
|
+
// its slots, and for the slots of the Machines below it, retire "not delivered".
|
|
4336
|
+
for (const gone of before) {
|
|
4337
|
+
if (!after.has(gone) && isMachineActor(gone) && notices.retire(slotPath(gone))) schedule();
|
|
4338
|
+
}
|
|
3091
4339
|
if (started.length > 0 && !this.sending) {
|
|
3092
|
-
const agent = agentOfEvent(ev.event as AnyEventObject,
|
|
4340
|
+
const agent = agentOfEvent(ev.event as AnyEventObject, childAgents, taker);
|
|
3093
4341
|
for (const id of started) {
|
|
3094
4342
|
const owner = agent ?? this.outstandingAbove(taker, snapshot as AnyMachineSnapshot, id, holds);
|
|
3095
|
-
if (owner !== undefined)
|
|
3096
|
-
else delete
|
|
4343
|
+
if (owner !== undefined) childAgents[childKey(taker, id)] = owner;
|
|
4344
|
+
else delete childAgents[childKey(taker, id)];
|
|
3097
4345
|
}
|
|
3098
4346
|
}
|
|
3099
4347
|
}
|
|
@@ -3212,11 +4460,19 @@ export class RunHost {
|
|
|
3212
4460
|
}
|
|
3213
4461
|
if (ev.type === "@xstate.snapshot") {
|
|
3214
4462
|
causes.taken(ev.actorRef as AnyActorRef, ev.event as AnyEventObject);
|
|
4463
|
+
// A Machine instance that reached its end — `done`, or `error` — takes no Turn again, even
|
|
4464
|
+
// while its parent still holds it (ADR-0002). A stop is not an end here: a host that stops
|
|
4465
|
+
// parks the run, and its restore goes on.
|
|
4466
|
+
const ended = (ev as { snapshot?: { status?: string } }).snapshot?.status;
|
|
4467
|
+
const ref = ev.actorRef as AnyActorRef;
|
|
4468
|
+
if ((ended === "done" || ended === "error") && isMachineActor(ref)) {
|
|
4469
|
+
notices.retire(ref._parent === undefined ? undefined : slotPath(ref));
|
|
4470
|
+
}
|
|
3215
4471
|
schedule();
|
|
3216
4472
|
}
|
|
3217
4473
|
},
|
|
3218
4474
|
});
|
|
3219
|
-
live = this.track(record, actor, def, agents, epochs, notices, binding, holds, effects,
|
|
4475
|
+
live = this.track(record, actor, def, agents, epochs, notices, binding, holds, effects, childAgents);
|
|
3220
4476
|
return actor;
|
|
3221
4477
|
}
|
|
3222
4478
|
|
|
@@ -3230,7 +4486,7 @@ export class RunHost {
|
|
|
3230
4486
|
binding: RunBinding,
|
|
3231
4487
|
holds: Map<string, Hold>,
|
|
3232
4488
|
effects: Record<string, EffectRow>,
|
|
3233
|
-
|
|
4489
|
+
childAgents: Record<string, ChildAgent>,
|
|
3234
4490
|
): LiveRun {
|
|
3235
4491
|
const run: LiveRun = {
|
|
3236
4492
|
record,
|
|
@@ -3241,8 +4497,10 @@ export class RunHost {
|
|
|
3241
4497
|
notices,
|
|
3242
4498
|
holds,
|
|
3243
4499
|
effects,
|
|
3244
|
-
|
|
4500
|
+
childAgents,
|
|
3245
4501
|
binding,
|
|
4502
|
+
answers: [],
|
|
4503
|
+
gatesOpenedAt: new Map(),
|
|
3246
4504
|
listeners: new Set(),
|
|
3247
4505
|
feedSoFar: [],
|
|
3248
4506
|
};
|
|
@@ -3296,18 +4554,38 @@ export class RunHost {
|
|
|
3296
4554
|
/**
|
|
3297
4555
|
* Persist the run's snapshot after a transition; drop it from the registry once final.
|
|
3298
4556
|
*
|
|
3299
|
-
* `terminal` is the run-lifecycle verdict the MACHINE cannot supply.
|
|
3300
|
-
*
|
|
3301
|
-
*
|
|
3302
|
-
*
|
|
3303
|
-
* for a
|
|
3304
|
-
*
|
|
3305
|
-
*/
|
|
3306
|
-
private persist(run: LiveRun, terminal?:
|
|
4557
|
+
* `terminal` is the run-lifecycle verdict the MACHINE cannot supply. {@link end} passes the end
|
|
4558
|
+
* of a run whose actor the host stops, because xstate then knows only that it was stopped:
|
|
4559
|
+
* "cancelled" for a cancel (ADR-0025), "error" for a mechanism event that no transition took
|
|
4560
|
+
* (ADR-0074), and "done" for a standing run's hand-over (ADR-0017). The error observer passes
|
|
4561
|
+
* "failed" for a Failure that no state caught, because xstate gives a Failure and a defect the
|
|
4562
|
+
* same `error` status (ADR-0073). It is the stored status and the one the feeds report.
|
|
4563
|
+
*/
|
|
4564
|
+
private persist(run: LiveRun, terminal?: EndedStatus): Promise<void> {
|
|
4565
|
+
// A save that names no end, of an actor that the host stops, writes nothing. When the host ends
|
|
4566
|
+
// the run, its own save names the end ({@link end}). When it parks the run, the last save stays,
|
|
4567
|
+
// `live`, for restore (ADR-0025). xstate's `stopped` is never a run status.
|
|
4568
|
+
const machineStatus = (run.actor.getSnapshot() as { status?: string }).status ?? "active";
|
|
4569
|
+
if (terminal === undefined && (run.ending !== undefined || machineStatus === "stopped")) return Promise.resolve();
|
|
3307
4570
|
// The holds first: an answer read at Rest, and a notice a route out raised, ride this same save.
|
|
3308
4571
|
this.settleHolds(run);
|
|
3309
4572
|
const snapshot = run.actor.getPersistedSnapshot();
|
|
3310
|
-
|
|
4573
|
+
// A Machine that ends on its own ends `done` or `error`.
|
|
4574
|
+
const status: "live" | EndedStatus =
|
|
4575
|
+
terminal ?? (machineStatus === "active" ? "live" : machineStatus === "done" ? "done" : "error");
|
|
4576
|
+
// A run that ended holds no Steering message: each one still held retires, "not delivered"
|
|
4577
|
+
// (ADR-0002). Before the blob is built, so this save records it.
|
|
4578
|
+
if (status !== "live") run.notices.retire();
|
|
4579
|
+
// While a Chart is bound, take the bound Machine: the terminal snapshot binds none, because
|
|
4580
|
+
// xstate stops the Open body first, so the terminal write keeps this one (ADR-0069). Taken
|
|
4581
|
+
// again only when the Charts change, or when the last doc could not draw one of them.
|
|
4582
|
+
const hashes = status === "live" ? boundFields(snapshot).chartHashes : undefined;
|
|
4583
|
+
if (
|
|
4584
|
+
hashes &&
|
|
4585
|
+
(hashes.join(":") !== run.lastBound?.hashes.join(":") || run.lastBound.doc.chartsNotDrawn !== undefined)
|
|
4586
|
+
) {
|
|
4587
|
+
run.lastBound = { hashes, doc: this.liveMachineDoc(run) };
|
|
4588
|
+
}
|
|
3311
4589
|
const serialized = serializeSnapshot(snapshot, {
|
|
3312
4590
|
stripContext: (ctx) => ctx, // see ADR-0007: live infra lives in `.provide` closures, not context
|
|
3313
4591
|
stripChildInput: (input) => input,
|
|
@@ -3315,7 +4593,14 @@ export class RunHost {
|
|
|
3315
4593
|
const blob: RunBlob = {
|
|
3316
4594
|
workflow: run.record.workflow,
|
|
3317
4595
|
instanceId: run.record.instanceId,
|
|
4596
|
+
startedAt: run.record.startedAt,
|
|
3318
4597
|
...(run.record.dispatchedBy ? { dispatchedBy: run.record.dispatchedBy } : {}),
|
|
4598
|
+
...(run.record.startedBy ? { startedBy: run.record.startedBy } : {}),
|
|
4599
|
+
...(run.record.resumedBy ? { resumedBy: run.record.resumedBy } : {}),
|
|
4600
|
+
...(run.cancelledBy ? { cancelledBy: run.cancelledBy } : {}),
|
|
4601
|
+
...(run.handedOverTo ? { handedOverTo: run.handedOverTo } : {}),
|
|
4602
|
+
...(run.answers.length ? { answers: run.answers } : {}),
|
|
4603
|
+
...(run.gatesOpenedAt.size ? { gatesOpenedAt: Object.fromEntries(run.gatesOpenedAt) } : {}),
|
|
3319
4604
|
// Stamped on every save, off the registered TEMPLATE (never the per-run `.provide()` result —
|
|
3320
4605
|
// providers do not change shape, and `fingerprintOf` memoizes per machine object). ADR-0030.
|
|
3321
4606
|
machine: fingerprintOf(run.def.machine),
|
|
@@ -3330,14 +4615,13 @@ export class RunHost {
|
|
|
3330
4615
|
timers: pendingTimers(run.actor, this.now()),
|
|
3331
4616
|
holds: [...run.holds.values()],
|
|
3332
4617
|
effects: run.effects,
|
|
3333
|
-
|
|
4618
|
+
childAgents: liveChildAgents(run.childAgents, run.actor),
|
|
3334
4619
|
fault: run.fault,
|
|
3335
4620
|
...(run.failure !== undefined ? { failure: run.failure } : {}),
|
|
3336
4621
|
};
|
|
3337
|
-
const status = terminal ?? (machineStatus === "active" ? "live" : machineStatus);
|
|
3338
4622
|
// A save that fails is this run's fault, never the process's: an unhandled rejection would end
|
|
3339
4623
|
// the Orchestrator and every run on it. The run keeps its last saved snapshot.
|
|
3340
|
-
this.store.save(run.record.runId, blob, status).catch((err: unknown) => {
|
|
4624
|
+
const saved = this.store.save(run.record.runId, blob, status).catch((err: unknown) => {
|
|
3341
4625
|
console.error(
|
|
3342
4626
|
`run ${run.record.runId}: its snapshot was not saved: ${err instanceof Error ? err.message : String(err)}`,
|
|
3343
4627
|
);
|
|
@@ -3345,6 +4629,19 @@ export class RunHost {
|
|
|
3345
4629
|
// The save is on disk (the store writes synchronously, ADR-0065): a settled row whose transition
|
|
3346
4630
|
// it carries keeps no tail in the next one (ADR-0067).
|
|
3347
4631
|
dropSavedTails(run.effects, snapshot);
|
|
4632
|
+
// An ended run holds no Assembly, so its row keeps the Machine it ran on (ADR-0069): the Console
|
|
4633
|
+
// draws an ended run on its own Machine after a publish collects that Assembly. A failure here
|
|
4634
|
+
// costs the drawing, never the run's end.
|
|
4635
|
+
if (status !== "live") {
|
|
4636
|
+
try {
|
|
4637
|
+
const doc = run.lastBound?.doc ?? this.liveMachineDoc(run);
|
|
4638
|
+
this.store.keepMachine(run.record.runId, keptMachineKey(doc), doc);
|
|
4639
|
+
} catch (err) {
|
|
4640
|
+
console.error(
|
|
4641
|
+
`run ${run.record.runId}: its Machine was not kept: ${err instanceof Error ? err.message : String(err)}`,
|
|
4642
|
+
);
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
3348
4645
|
|
|
3349
4646
|
// Feed per-run observers (SSE/CLI watch). On the terminal transition emit the final status
|
|
3350
4647
|
// BEFORE untrack drops the run from the registry, then drop the now-useless listener set.
|
|
@@ -3363,12 +4660,28 @@ export class RunHost {
|
|
|
3363
4660
|
run.listeners.clear();
|
|
3364
4661
|
// After the save and the untrack: a listener that reads the run back reads it settled.
|
|
3365
4662
|
this.fireSettled(run.record.runId, runStatus);
|
|
4663
|
+
// A standing run's end (ADR-0017): a pause, or a hand-over after the back-off.
|
|
4664
|
+
this.standingEnded(run, status);
|
|
3366
4665
|
} else {
|
|
3367
4666
|
this.announce(run.record.workflow, { kind: "status", status: runStatus });
|
|
4667
|
+
this.checkBound(run);
|
|
3368
4668
|
}
|
|
4669
|
+
return saved;
|
|
3369
4670
|
}
|
|
3370
4671
|
}
|
|
3371
4672
|
|
|
4673
|
+
/**
|
|
4674
|
+
* The shown status of a hosted run, off its Machine's status (ADR-0025). A Machine that ends on its
|
|
4675
|
+
* own is `done` or `error`. xstate's `stopped` is mechanism: the run shows the end that the host
|
|
4676
|
+
* gives it, and with none, it is a park, which leaves the run live.
|
|
4677
|
+
*/
|
|
4678
|
+
function shownOfMachine(machine: string, ending: EndedStatus | undefined): ShownStatus {
|
|
4679
|
+
if (machine === "active") return LIVE_STATUS;
|
|
4680
|
+
if (machine === "done") return "done";
|
|
4681
|
+
if (machine === "stopped") return ending ?? LIVE_STATUS;
|
|
4682
|
+
return "error";
|
|
4683
|
+
}
|
|
4684
|
+
|
|
3372
4685
|
/** A status that is no longer live (ADR-0071): the store's terminal words. A live row reads its
|
|
3373
4686
|
* Machine's `active`; the store's own `live` is never a status a reader is handed, but is not
|
|
3374
4687
|
* settled either. */
|
|
@@ -3399,6 +4712,20 @@ function atOrBelow(actor: AnyActorRef, ref: AnyActorRef): boolean {
|
|
|
3399
4712
|
return false;
|
|
3400
4713
|
}
|
|
3401
4714
|
|
|
4715
|
+
/** What the items that a standing run took itself did (ADR-0017): whether one started a run, and
|
|
4716
|
+
* how many ended. A live worker's run is on its ledger row (ADR-0071). */
|
|
4717
|
+
function tookOf(run: LiveRun): { started: boolean; answered: number } {
|
|
4718
|
+
return standingTook(run.actor.getSnapshot(), (effectId) => {
|
|
4719
|
+
const row = run.effects[effectId];
|
|
4720
|
+
if (row?.kind !== "dispatch") return undefined;
|
|
4721
|
+
const r =
|
|
4722
|
+
row.state === "settled"
|
|
4723
|
+
? (row.output as { run?: unknown } | null | undefined)?.run
|
|
4724
|
+
: (row as { run?: unknown }).run;
|
|
4725
|
+
return typeof r === "string" && r !== "" ? r : undefined;
|
|
4726
|
+
});
|
|
4727
|
+
}
|
|
4728
|
+
|
|
3402
4729
|
/** The runs a run's `dispatch` Effects started (ADR-0071), from its ledger: an intent row the
|
|
3403
4730
|
* start's answer re-wrote with its run, a settled row whose output names one (`started`,
|
|
3404
4731
|
* `settled`), and an aborted row that kept it. A row that never learned its run names none. */
|
|
@@ -3434,6 +4761,17 @@ function worktreesField(effects: Record<string, EffectRow>): { worktrees?: RunWo
|
|
|
3434
4761
|
return out.size ? { worktrees: [...out.values()] } : {};
|
|
3435
4762
|
}
|
|
3436
4763
|
|
|
4764
|
+
/**
|
|
4765
|
+
* The key an ended run's kept Machine is stored under (ADR-0069): a digest of the doc itself, with
|
|
4766
|
+
* the Workflow name blanked, so a Chart wrapper and its named Chart share one copy, and two Machines
|
|
4767
|
+
* that differ in any part the doc draws (a guard, a description, an Effect's argv) keep two.
|
|
4768
|
+
*/
|
|
4769
|
+
function keptMachineKey(doc: MachineDoc): string {
|
|
4770
|
+
return createHash("sha256")
|
|
4771
|
+
.update(JSON.stringify({ ...doc, workflow: "" }))
|
|
4772
|
+
.digest("hex");
|
|
4773
|
+
}
|
|
4774
|
+
|
|
3437
4775
|
/** A status's bound fields (ADR-0070): `bound`, and the hashes of the Charts bound, distinct and
|
|
3438
4776
|
* sorted. Both absent when the snapshot binds no Chart. */
|
|
3439
4777
|
function boundFields(snap: unknown): { bound?: true; chartHashes?: string[] } {
|