@intx/workflow-host 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -10
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +53 -53
- package/dist/adapters/spawn-child.d.ts +71 -42
- package/dist/adapters/spawn-child.js +83 -77
- package/dist/adapters/step-invoker.js +84 -7
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/index.d.ts +2 -1
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +107 -13
- package/dist/child/run-child.js +290 -108
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/conversation-text.d.ts +23 -0
- package/dist/conversation-text.js +56 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/ipc/control-channel.d.ts +58 -0
- package/dist/ipc/control-channel.js +94 -1
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/seams/scheduler.d.ts +4 -6
- package/dist/seams/scheduler.js +74 -93
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +1 -1
- package/dist/supervisor/credentials.d.ts +11 -10
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +5 -2
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +5 -5
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +82 -25
- package/dist/supervisor/supervisor.js +1313 -410
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +134 -0
- package/dist/supervisor/types.d.ts +150 -23
- package/dist/workflow-definition-loader.d.ts +131 -0
- package/dist/workflow-definition-loader.js +316 -0
- package/package.json +12 -11
|
@@ -53,6 +53,7 @@
|
|
|
53
53
|
// Multi-step steps pass no cache and keep instantiate-send-teardown.
|
|
54
54
|
import { createAgent, } from "@intx/agent";
|
|
55
55
|
import { getLogger } from "@intx/log";
|
|
56
|
+
import { createInboundMessage } from "@intx/mime";
|
|
56
57
|
const logger = getLogger(["workflow-host", "step-invoker"]);
|
|
57
58
|
/**
|
|
58
59
|
* Construct the production `WorkflowRuntimeEnv.StepInvoker` adapter.
|
|
@@ -101,8 +102,7 @@ async function invokeColdStep(opts, agentFactory, req) {
|
|
|
101
102
|
// flow through by default.
|
|
102
103
|
const eventForward = subscribeAgentEvents(agent, opts.onEvent);
|
|
103
104
|
try {
|
|
104
|
-
|
|
105
|
-
return { output: { reply: sendResult.reply, turn: sendResult.turn } };
|
|
105
|
+
return stepResultFromSend(await sendWithAbort(agent, req, { closeOnAbort: true }));
|
|
106
106
|
}
|
|
107
107
|
finally {
|
|
108
108
|
// `close` is idempotent: a second call after the send already
|
|
@@ -177,8 +177,7 @@ async function invokeWarmStep(opts, warmCache, agentFactory, req) {
|
|
|
177
177
|
warmCache.setEventSink(key, opts.onEvent);
|
|
178
178
|
}
|
|
179
179
|
try {
|
|
180
|
-
|
|
181
|
-
return { output: { reply: sendResult.reply, turn: sendResult.turn } };
|
|
180
|
+
return stepResultFromSend(await sendWithAbort(agent, req, { closeOnAbort: false }));
|
|
182
181
|
}
|
|
183
182
|
finally {
|
|
184
183
|
// Do NOT close the agent or drain its forwarder: both span
|
|
@@ -213,6 +212,18 @@ async function buildStepAgent(opts, agentFactory, req) {
|
|
|
213
212
|
/**
|
|
214
213
|
* Drive one `agent.send`, racing it against the step's abort signal.
|
|
215
214
|
*
|
|
215
|
+
* The message sent depends on `req.resume` and its kind. A first invocation
|
|
216
|
+
* sends the synthesized `req.input` content. An `"approval"` resume sends the
|
|
217
|
+
* full correlated `InboundMessage` built from `req.resume`, whose
|
|
218
|
+
* `headers.interchangeCorrelationId` routes through the reactor's
|
|
219
|
+
* `tryCorrelate` to match the rehydrated gate and resume the parked cycle --
|
|
220
|
+
* no second inference cycle. An `"input"` resume sends the decision as plain
|
|
221
|
+
* synthesized content, exactly like a first invocation: it is the step's next
|
|
222
|
+
* turn, with no gate to correlate. Because every path goes through
|
|
223
|
+
* `agent.send`, the returned `SendResult` carries the reactor's full settle
|
|
224
|
+
* arm set: a cycle that parks on a gate settles as `"suspended"` regardless
|
|
225
|
+
* of how the turn was delivered.
|
|
226
|
+
*
|
|
216
227
|
* `closeOnAbort` selects the abort semantics:
|
|
217
228
|
* - `true` (cold path): the in-flight send is left to settle via
|
|
218
229
|
* `agent.close()` in the caller's `finally`, which aborts the
|
|
@@ -253,16 +264,41 @@ async function sendWithAbort(agent, req, cfg) {
|
|
|
253
264
|
};
|
|
254
265
|
abortListener = onAbort;
|
|
255
266
|
req.signal.addEventListener("abort", onAbort, { once: true });
|
|
256
|
-
let
|
|
267
|
+
let message;
|
|
257
268
|
try {
|
|
258
|
-
|
|
269
|
+
// How the resumed input is delivered depends on the park kind:
|
|
270
|
+
//
|
|
271
|
+
// - `"approval"`: the reactor is parked mid-turn on a tool/authz gate.
|
|
272
|
+
// Build the full `InboundMessage` stamped with `resume.correlationId`
|
|
273
|
+
// so the header reaches the reactor's `tryCorrelate` and matches the
|
|
274
|
+
// rehydrated gate. The object form is load-bearing -- a plain string
|
|
275
|
+
// would drop the correlation id and the resumed cycle would never
|
|
276
|
+
// match.
|
|
277
|
+
// - `"input"`: the step re-armed between turns; the decision is simply
|
|
278
|
+
// the next user turn, with NO gate to correlate. Deliver it as the
|
|
279
|
+
// plain synthesized content, exactly as a first invocation does.
|
|
280
|
+
//
|
|
281
|
+
// A first invocation (no resume) sends the plain synthesized input;
|
|
282
|
+
// `agent.send` stamps its own synthetic addressing.
|
|
283
|
+
message =
|
|
284
|
+
req.resume === undefined
|
|
285
|
+
? synthesizeInputContent(req.input)
|
|
286
|
+
: req.resume.kind === "input"
|
|
287
|
+
? synthesizeInputContent(req.resume.decision)
|
|
288
|
+
: createInboundMessage({
|
|
289
|
+
from: "signal@local",
|
|
290
|
+
to: "agent@local",
|
|
291
|
+
content: synthesizeInputContent(req.resume.decision),
|
|
292
|
+
interchangeType: "conversation.message",
|
|
293
|
+
correlationId: req.resume.correlationId,
|
|
294
|
+
});
|
|
259
295
|
}
|
|
260
296
|
catch (cause) {
|
|
261
297
|
reject(cause instanceof Error ? cause : new Error(String(cause)));
|
|
262
298
|
return;
|
|
263
299
|
}
|
|
264
300
|
const sendOpts = cfg.closeOnAbort ? undefined : { signal: req.signal };
|
|
265
|
-
agent.send(
|
|
301
|
+
agent.send(message, sendOpts).then(resolve, (cause) => {
|
|
266
302
|
reject(cause instanceof Error ? cause : new Error(String(cause)));
|
|
267
303
|
});
|
|
268
304
|
});
|
|
@@ -323,6 +359,47 @@ function subscribeAgentEvents(agent, onEvent) {
|
|
|
323
359
|
function wrapAuthorize(workflowAuthorize, authzContext) {
|
|
324
360
|
return async (resource, action) => workflowAuthorize(resource, action, authzContext);
|
|
325
361
|
}
|
|
362
|
+
/**
|
|
363
|
+
* Translate a settled `SendResult` into the step's `StepInvokeResult`.
|
|
364
|
+
*
|
|
365
|
+
* A `"reply"` outcome carries the assistant's reply and full-fidelity
|
|
366
|
+
* turn, which become the step output so downstream consumers can read
|
|
367
|
+
* either shape.
|
|
368
|
+
*
|
|
369
|
+
* A `"suspended"` outcome hands the workflow runtime the parked reactor's
|
|
370
|
+
* `correlationId`: the reactor parked on a gate awaiting an external
|
|
371
|
+
* decision. The runtime parks the step on the reserved signal channel for
|
|
372
|
+
* that correlation and, when the decision is delivered, re-invokes with
|
|
373
|
+
* `resume` so `sendWithAbort` sends the correlated inbound and drives the
|
|
374
|
+
* resumed reactor to a real reply -- or, when the resumed cycle re-parks
|
|
375
|
+
* on a second gate, to another `"suspended"` outcome that flows back
|
|
376
|
+
* through here unchanged.
|
|
377
|
+
*/
|
|
378
|
+
function stepResultFromSend(result) {
|
|
379
|
+
if (result.type === "suspended") {
|
|
380
|
+
// The reactor parks only on a tool/authz gate -- an APPROVAL. It never
|
|
381
|
+
// parks awaiting the next mail (that "input" park is the workflow-host's
|
|
382
|
+
// decision to re-arm a conversational step, not a reactor outcome), so a
|
|
383
|
+
// suspended `SendResult` is always an approval and MUST carry a snapshot.
|
|
384
|
+
// A snapshot-less suspend (e.g. a director `caps.suspend` wired with no
|
|
385
|
+
// tool definitions) is not a supported approval; classify the failure here
|
|
386
|
+
// at the producer rather than emitting an ambiguous suspend that the
|
|
387
|
+
// runtime would have to reject three hops downstream.
|
|
388
|
+
if (result.approvalSnapshot === undefined) {
|
|
389
|
+
throw new Error(`reactor suspended on correlation ${result.correlationId} with no ` +
|
|
390
|
+
`approval snapshot; a snapshot-less suspend is not a supported ` +
|
|
391
|
+
`approval park`);
|
|
392
|
+
}
|
|
393
|
+
return {
|
|
394
|
+
suspend: {
|
|
395
|
+
correlationId: result.correlationId,
|
|
396
|
+
kind: "approval",
|
|
397
|
+
approvalSnapshot: result.approvalSnapshot,
|
|
398
|
+
},
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
return { output: { reply: result.reply, turn: result.turn } };
|
|
402
|
+
}
|
|
326
403
|
/**
|
|
327
404
|
* Encode the step's resolved `input` as the synthetic inbound message
|
|
328
405
|
* content. The workflow runtime resolves `input` from the step's input
|
|
@@ -13,9 +13,11 @@
|
|
|
13
13
|
export declare const REQUIRED_SPAWN_ENV_KEYS: readonly ["IPC_CHANNEL_ID", "IPC_HMAC_KEY", "HOST_PUBKEY", "DEPLOYMENT_ID", "DEFINITION_HASH", "MAILBOX_ADDRESS", "STEP_COUNT"];
|
|
14
14
|
export type RequiredSpawnEnvKey = (typeof REQUIRED_SPAWN_ENV_KEYS)[number];
|
|
15
15
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* The parsed spawn-time env. The hex-encoded trust anchors decode to their raw
|
|
17
|
+
* byte representations so the IPC channel constructors can consume them without
|
|
18
|
+
* re-validating the hex shape. Source-ref is the only deploy lineage, so every
|
|
19
|
+
* child evaluates the pinned code closure at `closurePackageDir`; the field is
|
|
20
|
+
* always present.
|
|
19
21
|
*/
|
|
20
22
|
export interface SpawnTimeEnv {
|
|
21
23
|
/** Channel identifier minted by the supervisor for this spawn. */
|
|
@@ -24,9 +26,15 @@ export interface SpawnTimeEnv {
|
|
|
24
26
|
hmacKey: Uint8Array;
|
|
25
27
|
/** Supervisor's 32-byte Ed25519 public key for control-frame verification. */
|
|
26
28
|
hostPublicKey: Uint8Array;
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
-
/**
|
|
29
|
+
/** Anchor run id the supervisor manages. */
|
|
30
|
+
anchorRunId: string;
|
|
31
|
+
/**
|
|
32
|
+
* Content hash of the deployed `WorkflowDefinition`. This is the
|
|
33
|
+
* hub-approved wire hash the deploy frame carried
|
|
34
|
+
* (`AgentDeployWorkflow.approvedWireHash`), not a sidecar recompute -- the
|
|
35
|
+
* hub is the authority, so the child re-verifies its own recompute against
|
|
36
|
+
* this value.
|
|
37
|
+
*/
|
|
30
38
|
definitionHash: string;
|
|
31
39
|
/** Mail address the deployment registered on the bus. */
|
|
32
40
|
mailboxAddress: string;
|
|
@@ -44,6 +52,12 @@ export interface SpawnTimeEnv {
|
|
|
44
52
|
* cache when set and keeps cold instantiate-send-teardown otherwise.
|
|
45
53
|
*/
|
|
46
54
|
warmKeep: boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Sidecar-local dir of the materialized workflow-definition closure the child
|
|
57
|
+
* evaluates to a live definition and re-verifies by project-then-hash.
|
|
58
|
+
* Source-ref is the only deploy lineage, so it is always present.
|
|
59
|
+
*/
|
|
60
|
+
closurePackageDir: string;
|
|
47
61
|
}
|
|
48
62
|
/**
|
|
49
63
|
* Parse and validate `process.env`-shaped input into the typed
|
|
@@ -64,6 +64,13 @@ const SpawnTimeEnvShape = type({
|
|
|
64
64
|
// so the warm-keep decision is deterministic and a multi-step agent is
|
|
65
65
|
// never warm-kept by a silent default.
|
|
66
66
|
"WARM_KEEP?": "string",
|
|
67
|
+
// Sidecar-local directory of the materialized workflow-definition closure the
|
|
68
|
+
// deployment evaluates. Source-ref is the only deploy lineage, so the child
|
|
69
|
+
// always evaluates a pinned code closure to a LIVE definition and re-verifies
|
|
70
|
+
// it by project-then-hash; there is nothing to evaluate without this dir, so
|
|
71
|
+
// it is required. The sidecar computes it when it applies the frozen closure
|
|
72
|
+
// and threads it here; it never travels on the hub deploy frame.
|
|
73
|
+
CLOSURE_PACKAGE_DIR: "string > 0",
|
|
67
74
|
}).onUndeclaredKey("ignore");
|
|
68
75
|
/**
|
|
69
76
|
* Parse and validate `process.env`-shaped input into the typed
|
|
@@ -108,7 +115,7 @@ export function parseSpawnTimeEnv(rawEnv) {
|
|
|
108
115
|
channelId: validated.IPC_CHANNEL_ID,
|
|
109
116
|
hmacKey,
|
|
110
117
|
hostPublicKey,
|
|
111
|
-
|
|
118
|
+
anchorRunId: validated.DEPLOYMENT_ID,
|
|
112
119
|
definitionHash: validated.DEFINITION_HASH,
|
|
113
120
|
mailboxAddress: validated.MAILBOX_ADDRESS,
|
|
114
121
|
stepCount,
|
|
@@ -116,5 +123,6 @@ export function parseSpawnTimeEnv(rawEnv) {
|
|
|
116
123
|
// absence) reads false. Warm-keep is opt-in and deterministic; a
|
|
117
124
|
// typo'd or partial value must not silently enable it.
|
|
118
125
|
warmKeep: validated.WARM_KEEP === "true",
|
|
126
|
+
closurePackageDir: validated.CLOSURE_PACKAGE_DIR,
|
|
119
127
|
};
|
|
120
128
|
}
|
package/dist/child/index.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
export { createCredentialsBackedAuthorize, hashGrants, runWorkflowChild, type ChildStepInvoker, type CredentialsSnapshotRef, type DrainController, type GrantEvaluator, type RunWorkflowChildBindings, type RunWorkflowChildOpts, type RunWorkflowChildResult, type SourcesSnapshotRef, type SubstrateWriteResponseSink, } from "./run-child.js";
|
|
1
|
+
export { createCredentialsBackedAuthorize, hashGrants, runWorkflowChild, type ChildStepInvoker, type CredentialsSnapshotRef, type CredentialWiring, type DrainController, type GrantEvaluator, type RunWorkflowChildBindings, type RunWorkflowChildOpts, type RunWorkflowChildResult, type SourcesSnapshotRef, type SubstrateWriteResponseSink, } from "./run-child.js";
|
|
2
2
|
export { createChildSubstrateWriteBridge, type ChildSubstrateWriteBridge, type CreateChildSubstrateWriteBridgeOpts, type SubstrateWriteRequest, } from "./substrate-write-bridge.js";
|
|
3
3
|
export { createChildOutboundMailBridge, type ChildOutboundMailBridge, type CreateChildOutboundMailBridgeOpts, } from "./outbound-mail-bridge.js";
|
|
4
4
|
export { createSupervisorBackedTransport } from "./supervisor-backed-transport.js";
|
|
5
5
|
export { createProxyWorkflowRunRepoStore, type CreateProxyWorkflowRunRepoStoreOpts, } from "./proxy-repo-store.js";
|
|
6
6
|
export { parseSpawnTimeEnv, type SpawnTimeEnv } from "./env-bootstrap.js";
|
|
7
7
|
export { discoverInFlightRuns, type DiscoverRunsOpts, type DiscoveredRun, } from "./self-discovery.js";
|
|
8
|
+
export type { LoadParkedApproval } from "./parked-correlations.js";
|
|
8
9
|
export { createWarmAgentCache, type WarmAgentCache, type WarmEventSinkRef, } from "./warm-agent-cache.js";
|
|
9
10
|
export { EVENT_CHANNEL_FD, runWorkflowChildFromProcessEnv, type RunWorkflowChildFromProcessEnvOpts, type SubstrateFactory, type SubstrateFactoryEnv, } from "./from-process-env.js";
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
|
|
2
|
+
import { type RepoStore as RuntimeRepoStore } from "@intx/workflow";
|
|
3
|
+
import type { ApprovalSnapshot, ControlParkKind } from "@intx/types/runtime";
|
|
4
|
+
/**
|
|
5
|
+
* Recover the durable approval snapshot for one parked control-plane
|
|
6
|
+
* correlation. The child owns enumeration; the host owns the per-step
|
|
7
|
+
* on-disk layout (cold vs warm), so the snapshot read is a host binding.
|
|
8
|
+
* Returns `undefined` when no pending operation for the correlation carries
|
|
9
|
+
* a snapshot.
|
|
10
|
+
*/
|
|
11
|
+
export type LoadParkedApproval = (args: {
|
|
12
|
+
runId: string;
|
|
13
|
+
stepId: string;
|
|
14
|
+
attempt: number;
|
|
15
|
+
correlationId: string;
|
|
16
|
+
}) => Promise<ApprovalSnapshot | undefined>;
|
|
17
|
+
/**
|
|
18
|
+
* One parked control-plane correlation: the child-supplied half of a
|
|
19
|
+
* suspension registration. `parkKind` discriminates approval parks
|
|
20
|
+
* (which carry a snapshot) from input parks (which do not).
|
|
21
|
+
*/
|
|
22
|
+
export interface ParkedApprovalCorrelation {
|
|
23
|
+
runId: string;
|
|
24
|
+
correlationId: string;
|
|
25
|
+
parkKind: ControlParkKind;
|
|
26
|
+
snapshot?: ApprovalSnapshot;
|
|
27
|
+
}
|
|
28
|
+
export interface CollectParkedApprovalCorrelationsOpts {
|
|
29
|
+
substrate: SubstrateRepoStore;
|
|
30
|
+
repoId: RepoId;
|
|
31
|
+
runtimeRepoStore: RuntimeRepoStore;
|
|
32
|
+
loadParkedApproval?: LoadParkedApproval;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Enumerate every in-flight run's reduced state and return one entry per step
|
|
36
|
+
* parked on a control-plane approval channel. Throws when a park is found but
|
|
37
|
+
* no `loadParkedApproval` binding is wired to recover its snapshot, or when
|
|
38
|
+
* the binding returns no snapshot for an enumerated park -- both are
|
|
39
|
+
* disagreements between the reduced state and the durable store that must not
|
|
40
|
+
* silently drop a correlation the hub is waiting to register.
|
|
41
|
+
*/
|
|
42
|
+
export declare function collectParkedApprovalCorrelations(opts: CollectParkedApprovalCorrelationsOpts): Promise<ParkedApprovalCorrelation[]>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Enumerate a child's currently-parked approval correlations from durable
|
|
2
|
+
// state, so the child can answer a supervisor `parked-correlations.request`.
|
|
3
|
+
//
|
|
4
|
+
// Enumeration keys on REDUCED step state, never on raw `SignalAwaited` log
|
|
5
|
+
// events. `parkOnSignal` commits `SignalAwaited` to the durable log before it
|
|
6
|
+
// checks for the approval snapshot, so a snapshot-less correlated suspend (a
|
|
7
|
+
// director `caps.suspend`, an unwired authz gate) leaves a control-plane
|
|
8
|
+
// `SignalAwaited` in the log yet reduces to `phase === "failed"` -- never
|
|
9
|
+
// `awaiting-signal`, and never a hub row. Filtering on the reduced
|
|
10
|
+
// `awaiting-signal` phase therefore surfaces only the parks that carry a
|
|
11
|
+
// durable snapshot by construction; a snapshot-less enumerated step is a
|
|
12
|
+
// disagreement between the log and the step store, which this module surfaces
|
|
13
|
+
// loudly rather than dropping.
|
|
14
|
+
import { controlParkKindOf, } from "@intx/workflow";
|
|
15
|
+
import { correlationIdFromSignalName } from "@intx/types";
|
|
16
|
+
import { discoverInFlightRuns } from "./self-discovery.js";
|
|
17
|
+
/**
|
|
18
|
+
* Enumerate every in-flight run's reduced state and return one entry per step
|
|
19
|
+
* parked on a control-plane approval channel. Throws when a park is found but
|
|
20
|
+
* no `loadParkedApproval` binding is wired to recover its snapshot, or when
|
|
21
|
+
* the binding returns no snapshot for an enumerated park -- both are
|
|
22
|
+
* disagreements between the reduced state and the durable store that must not
|
|
23
|
+
* silently drop a correlation the hub is waiting to register.
|
|
24
|
+
*/
|
|
25
|
+
export async function collectParkedApprovalCorrelations(opts) {
|
|
26
|
+
const discovered = await discoverInFlightRuns({
|
|
27
|
+
substrate: opts.substrate,
|
|
28
|
+
repoId: opts.repoId,
|
|
29
|
+
runtimeRepoStore: opts.runtimeRepoStore,
|
|
30
|
+
});
|
|
31
|
+
const out = [];
|
|
32
|
+
for (const run of discovered) {
|
|
33
|
+
for (const step of run.resumedState.steps.values()) {
|
|
34
|
+
if (step.phase !== "awaiting-signal")
|
|
35
|
+
continue;
|
|
36
|
+
const awaited = step.awaitingSignal;
|
|
37
|
+
if (awaited === undefined)
|
|
38
|
+
continue;
|
|
39
|
+
const correlationId = correlationIdFromSignalName(awaited.name);
|
|
40
|
+
if (correlationId === undefined)
|
|
41
|
+
continue;
|
|
42
|
+
// An `"input"` park (a long-lived run awaiting its next mail) reduces to
|
|
43
|
+
// the same `awaiting-signal` on a reserved channel as an approval, but it
|
|
44
|
+
// carries NO snapshot and is never hub-registered -- the run's owner
|
|
45
|
+
// delivers the input directly. Skip it: enumerating it would call
|
|
46
|
+
// loadParkedApproval, get no snapshot, and throw below, taking the whole
|
|
47
|
+
// deployment's approval re-registration down on every reconnect.
|
|
48
|
+
// `controlParkKindOf` is the single point that reads a reserved-channel
|
|
49
|
+
// park's kind; an absent kind is a legacy approval, not an input park.
|
|
50
|
+
const parkKind = controlParkKindOf(awaited);
|
|
51
|
+
if (parkKind === "input") {
|
|
52
|
+
out.push({
|
|
53
|
+
runId: run.runId,
|
|
54
|
+
correlationId,
|
|
55
|
+
parkKind: "input",
|
|
56
|
+
});
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (opts.loadParkedApproval === undefined) {
|
|
60
|
+
throw new Error(`workflow-child parked-correlations: run ${run.runId} step ${step.stepId} is parked on control-plane correlation ${correlationId}, but no loadParkedApproval binding is wired to recover its snapshot`);
|
|
61
|
+
}
|
|
62
|
+
const snapshot = await opts.loadParkedApproval({
|
|
63
|
+
runId: run.runId,
|
|
64
|
+
stepId: step.stepId,
|
|
65
|
+
attempt: step.currentAttempt,
|
|
66
|
+
correlationId,
|
|
67
|
+
});
|
|
68
|
+
if (snapshot === undefined) {
|
|
69
|
+
throw new Error(`workflow-child parked-correlations: reduced state shows run ${run.runId} step ${step.stepId} awaiting control-plane correlation ${correlationId} (attempt ${String(step.currentAttempt)}), but durable storage carries no approval snapshot for it; the run log and the step store disagree`);
|
|
70
|
+
}
|
|
71
|
+
out.push({
|
|
72
|
+
runId: run.runId,
|
|
73
|
+
correlationId,
|
|
74
|
+
parkKind: "approval",
|
|
75
|
+
snapshot,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
@@ -5,8 +5,9 @@ export interface CreateProxyWorkflowRunRepoStoreOpts {
|
|
|
5
5
|
* Bare substrate handle the child opens against the shared on-disk
|
|
6
6
|
* data dir. Used for the read-only methods that consult the
|
|
7
7
|
* substrate's local state -- `getRepoDir` (path computation, no
|
|
8
|
-
* I/O), `resolveRef`, `listRefs`, `resolveHead`, `
|
|
9
|
-
* bare store is never
|
|
8
|
+
* I/O), `resolveRef`, `listRefs`, `resolveHead`, `openCommittedReads`,
|
|
9
|
+
* `openCommittedReadsAtCommit`, `createPack`. The bare store is never
|
|
10
|
+
* used as a writer here; its
|
|
10
11
|
* `writeTreePreservingPrefix` / `writeTree` / `receivePack` are not
|
|
11
12
|
* reachable through this proxy.
|
|
12
13
|
*/
|
|
@@ -117,6 +117,8 @@ export function createProxyWorkflowRunRepoStore(opts) {
|
|
|
117
117
|
listRefs: bareStore.listRefs.bind(bareStore),
|
|
118
118
|
resolveHead: bareStore.resolveHead.bind(bareStore),
|
|
119
119
|
getRepoDir: bareStore.getRepoDir.bind(bareStore),
|
|
120
|
+
openCommittedReads: bareStore.openCommittedReads.bind(bareStore),
|
|
121
|
+
openCommittedReadsAtCommit: bareStore.openCommittedReadsAtCommit.bind(bareStore),
|
|
120
122
|
subscribe(_principal, repoId, ref, subOpts) {
|
|
121
123
|
// Synthesizing the subscribe surface in the proxy: the bare
|
|
122
124
|
// store's `subscribe` would only fire from its own writes, but
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import type { Principal, RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
|
|
2
|
-
import type { DirectorRegistry } from "@intx/agent";
|
|
3
2
|
import type { AuthzCallResult } from "@intx/inference";
|
|
4
|
-
import type { RunResult, Scheduler, StepInvokeRequest, StepInvokeResult, SpawnChildWorkflow, WorkflowAuthorizeFn } from "@intx/workflow";
|
|
3
|
+
import type { RunResult, Scheduler, ReadParkedApprovalOps, StepInvokeRequest, StepInvokeResult, SpawnChildWorkflow, WorkflowAuthorizeFn, WorkflowPark } from "@intx/workflow";
|
|
5
4
|
import { type WorkflowHostDrainController } from "../drain-controller.js";
|
|
6
5
|
import type { InferenceSource } from "@intx/types/runtime";
|
|
6
|
+
import type { CredentialDelivery } from "@intx/types/sidecar";
|
|
7
|
+
import type { RunSuspendableChild, RunChildWorkflow } from "../adapters/spawn-child.js";
|
|
7
8
|
import { type ControlChannelSender, type ControlPayload, type EventPayload, type FrameWriter, type NdjsonReader, type NdjsonWriter } from "../ipc/index.js";
|
|
8
9
|
import type { CredentialsSnapshot } from "../supervisor/credentials.js";
|
|
9
10
|
import { hashGrants } from "../supervisor/credentials.js";
|
|
10
11
|
import type { SpawnTimeEnv } from "./env-bootstrap.js";
|
|
12
|
+
import { type LoadParkedApproval } from "./parked-correlations.js";
|
|
11
13
|
import type { ChildOutboundMailBridge } from "./outbound-mail-bridge.js";
|
|
12
14
|
import { type WarmAgentCache } from "./warm-agent-cache.js";
|
|
13
15
|
/**
|
|
@@ -27,6 +29,16 @@ import { type WarmAgentCache } from "./warm-agent-cache.js";
|
|
|
27
29
|
export type CredentialsSnapshotRef = {
|
|
28
30
|
current: CredentialsSnapshot | null;
|
|
29
31
|
};
|
|
32
|
+
/**
|
|
33
|
+
* The deployment's decrypted credential material and per-handle descriptors,
|
|
34
|
+
* held through a mutable reference and swapped wholesale on a rotation push (a
|
|
35
|
+
* revoked credential arrives by omission, so the swap evicts it). The secret
|
|
36
|
+
* lives ONLY here -- read at tool-invoke time through the gated capability --
|
|
37
|
+
* and is never copied into a snapshot, event, or state.
|
|
38
|
+
*/
|
|
39
|
+
export type CredentialMaterialRef = {
|
|
40
|
+
current: CredentialDelivery | null;
|
|
41
|
+
};
|
|
30
42
|
/**
|
|
31
43
|
* Per-step inference-source table the build path reads through a mutable
|
|
32
44
|
* reference, keyed by stepId. Each value is the step's ordered failover
|
|
@@ -92,7 +104,24 @@ export type DrainController = WorkflowHostDrainController;
|
|
|
92
104
|
* by the run-loop (`runWorkflowChild`), not the binding: the binding
|
|
93
105
|
* only reads it through to the adapter.
|
|
94
106
|
*/
|
|
95
|
-
|
|
107
|
+
/**
|
|
108
|
+
* Per-run credential inputs the top-level step invoker carries to the
|
|
109
|
+
* substrate: the live material cell the control channel writes each delivery
|
|
110
|
+
* into, and a resolver for a step's grants (which the substrate gates
|
|
111
|
+
* credential use against). The substrate combines these with its own static
|
|
112
|
+
* provider registry to assemble each tool bundle's `credentials` capability.
|
|
113
|
+
*
|
|
114
|
+
* Grants are typed `readonly unknown[]` here: this package owns no grant
|
|
115
|
+
* grammar (the credentials snapshot's grants are `unknown[]` throughout), so
|
|
116
|
+
* the substrate casts to its `GrantRule` shape at its own boundary, exactly
|
|
117
|
+
* as the grant evaluator does. The cell is read live per use, so a rotation or
|
|
118
|
+
* a revoking re-push reaches an already-shaped handle without a rebuild.
|
|
119
|
+
*/
|
|
120
|
+
export interface CredentialWiring {
|
|
121
|
+
readonly materialRef: CredentialMaterialRef;
|
|
122
|
+
readonly resolveStepGrants: (stepId: string) => readonly unknown[];
|
|
123
|
+
}
|
|
124
|
+
export type ChildStepInvoker = (req: StepInvokeRequest, onEvent: (event: EventPayload) => void, authorize: WorkflowAuthorizeFn, warmCache: WarmAgentCache | undefined, sourcesRef: SourcesSnapshotRef, credentialWiring: CredentialWiring) => Promise<StepInvokeResult>;
|
|
96
125
|
/**
|
|
97
126
|
* Bindings the binary owns: per-deployment substrate identity,
|
|
98
127
|
* principal credentials, the runtime-supplied callbacks the
|
|
@@ -114,10 +143,6 @@ export interface RunWorkflowChildBindings {
|
|
|
114
143
|
* the host's substrate accepts for `runs/<runId>/` writes.
|
|
115
144
|
*/
|
|
116
145
|
principal: Principal;
|
|
117
|
-
/** Workflow-asset repo identity (used to load `workflow.json`). */
|
|
118
|
-
workflowDefinitionRepoId: RepoId;
|
|
119
|
-
/** Workflow-asset ref the deploy orchestrator wrote to. */
|
|
120
|
-
workflowDefinitionRef: string;
|
|
121
146
|
/**
|
|
122
147
|
* Step-invoker callback the runtime body invokes per step. The
|
|
123
148
|
* shape is the workflow-runtime `StepInvoker` widened with an
|
|
@@ -128,11 +153,33 @@ export interface RunWorkflowChildBindings {
|
|
|
128
153
|
*/
|
|
129
154
|
invokeStep: ChildStepInvoker;
|
|
130
155
|
/**
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
* `
|
|
156
|
+
* Terminal child-spawn callback the runtime body invokes for a
|
|
157
|
+
* `childWorkflow` primitive when the deployment embeds NO inline child
|
|
158
|
+
* import (the map `run-child` lifts is empty). Optional and, in practice,
|
|
159
|
+
* only a test seam: a production deployment that carries a childWorkflow
|
|
160
|
+
* always has a non-empty lifted-body map and routes through the in-memory
|
|
161
|
+
* resolver built from `runChild` below, and one that carries none never
|
|
162
|
+
* invokes this. A workflow that reaches a childWorkflow with neither this
|
|
163
|
+
* nor `runChild` wired fails loud at spawn.
|
|
164
|
+
*/
|
|
165
|
+
spawnChild?: SpawnChildWorkflow;
|
|
166
|
+
/**
|
|
167
|
+
* Raw in-process terminal child executor. `run-child` builds the in-memory
|
|
168
|
+
* childWorkflow resolver from this executor plus the lifted-body map it
|
|
169
|
+
* extracts after loading the definition -- the parent's own re-verified
|
|
170
|
+
* closure -- so an owned inline child resolves with NO on-disk read. Parallel
|
|
171
|
+
* to `runSuspendableChild` for onTrigger bodies. Optional for the same
|
|
172
|
+
* reason: a child that embeds no childWorkflow import omits it.
|
|
173
|
+
*/
|
|
174
|
+
runChild?: RunChildWorkflow;
|
|
175
|
+
/**
|
|
176
|
+
* Raw in-process suspendable-child executor. `run-child` builds the in-memory
|
|
177
|
+
* onTrigger-body resolver from this executor plus the bodies map it extracts
|
|
178
|
+
* AFTER re-evaluating the closure -- the substrate factory cannot build that
|
|
179
|
+
* resolver because the bodies map does not exist pre-eval. Optional: a child
|
|
180
|
+
* that runs no onTrigger section omits it.
|
|
134
181
|
*/
|
|
135
|
-
|
|
182
|
+
runSuspendableChild?: RunSuspendableChild;
|
|
136
183
|
/** Host-process scheduler singleton. The child consumes the same instance. */
|
|
137
184
|
scheduler: Scheduler;
|
|
138
185
|
/** Grant evaluator wired against the host's grant-rule grammar. */
|
|
@@ -152,8 +199,32 @@ export interface RunWorkflowChildBindings {
|
|
|
152
199
|
* adapter (which roots no per-run scratch of its own) can omit it.
|
|
153
200
|
*/
|
|
154
201
|
cleanupRunStorage?: (runId: string) => Promise<void>;
|
|
155
|
-
/**
|
|
156
|
-
|
|
202
|
+
/**
|
|
203
|
+
* Recover the durable approval snapshot for a parked control-plane
|
|
204
|
+
* correlation, so the child can answer a supervisor
|
|
205
|
+
* `parked-correlations.request` (the supervisor's re-registration path
|
|
206
|
+
* after a re-establishment). The child owns enumeration -- it walks its
|
|
207
|
+
* own reduced run state for `awaiting-signal` steps on control-plane
|
|
208
|
+
* channels -- but the snapshot lives in per-step durable storage whose
|
|
209
|
+
* on-disk layout (cold vs warm) the host owns, so the read is a host
|
|
210
|
+
* binding next to `cleanupRunStorage`. Optional so tests inject a stub and
|
|
211
|
+
* the recursive child-workflow adapter (which roots no per-step approval
|
|
212
|
+
* storage) can omit it; a production child that enumerates a parked
|
|
213
|
+
* control-plane step with no binding wired throws rather than silently
|
|
214
|
+
* dropping the correlation the hub is waiting to register.
|
|
215
|
+
*/
|
|
216
|
+
loadParkedApproval?: LoadParkedApproval;
|
|
217
|
+
/**
|
|
218
|
+
* Enumerate the durable pending approval operations a crashed-mid-invocation
|
|
219
|
+
* step left behind, so the resume classifier can recover a step that crashed
|
|
220
|
+
* across the park boundary (durable `StepStarted`, unflushed `SignalAwaited`)
|
|
221
|
+
* as `awaiting-signal` rather than failing the run. Reads the same per-step
|
|
222
|
+
* durable storage as `loadParkedApproval` (cold isogit / warm substrate), so
|
|
223
|
+
* it is a host binding for the same reason. Optional so tests inject a stub
|
|
224
|
+
* and the recursive child-workflow adapter can omit it; absent, a crashed
|
|
225
|
+
* invocation step settles as a terminal failure, the pre-recovery behavior.
|
|
226
|
+
*/
|
|
227
|
+
readParkedApprovalOps?: ReadParkedApprovalOps;
|
|
157
228
|
/** Optional clock override; production wires `() => new Date()`. */
|
|
158
229
|
clock?: () => Date;
|
|
159
230
|
/** Optional id generator override; production wires a monotonic one. */
|
|
@@ -173,6 +244,14 @@ export interface RunWorkflowChildBindings {
|
|
|
173
244
|
* rather than resolving a default.
|
|
174
245
|
*/
|
|
175
246
|
initialSources?: Record<string, InferenceSource[]>;
|
|
247
|
+
/**
|
|
248
|
+
* Bootstrap credential material for the deployment's tools, decrypted
|
|
249
|
+
* hub-side and delivered on the deploy frame so it is resident before any
|
|
250
|
+
* step runs. Seeds the mutable `credentialMaterialRef` the gated capability
|
|
251
|
+
* reads. Absent when the deployment binds no credentials; a later
|
|
252
|
+
* `credentials-updated` control frame refreshes it on rotation.
|
|
253
|
+
*/
|
|
254
|
+
initialCredentialMaterial?: CredentialDelivery;
|
|
176
255
|
/**
|
|
177
256
|
* Optional override for the child's Ed25519 keypair factory. The
|
|
178
257
|
* child mints a fresh keypair at startup, holds the private half
|
|
@@ -286,6 +365,21 @@ export interface RunWorkflowChildResult {
|
|
|
286
365
|
* exits cleanly).
|
|
287
366
|
*/
|
|
288
367
|
export declare function runWorkflowChild(opts: RunWorkflowChildOpts): Promise<RunWorkflowChildResult>;
|
|
368
|
+
/**
|
|
369
|
+
* Forward a control-plane suspension to the supervisor over the upstream
|
|
370
|
+
* control channel. Fired from `env.onPark` each time a workflow agent step
|
|
371
|
+
* parks on a reserved `signalName(correlationId)` channel. The supervisor's
|
|
372
|
+
* `park.notify` arm stamps the deployment identity it owns and sends a
|
|
373
|
+
* `signal.correlation.register` frame to the hub.
|
|
374
|
+
*
|
|
375
|
+
* Best-effort like `emitTerminalEvent`'s send: a transport failure is logged,
|
|
376
|
+
* not rethrown. A lost frame means the correlation is not registered and the
|
|
377
|
+
* parked run cannot be resumed until it is re-registered; the failure surfaces
|
|
378
|
+
* structurally as a run that never resumes rather than a silent lifecycle
|
|
379
|
+
* corruption. The register at the hub is idempotent, so a re-park resume's
|
|
380
|
+
* re-emit is safe.
|
|
381
|
+
*/
|
|
382
|
+
export declare function emitParkNotify(upstreamSender: ControlChannelSender, park: WorkflowPark): Promise<void>;
|
|
289
383
|
/**
|
|
290
384
|
* Mirror a run's terminal status back to the supervisor over the
|
|
291
385
|
* upstream control channel. Fired once per run from the resume and
|