@intx/workflow-host 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -14
- package/dist/adapters/mail-part-store.d.ts +46 -0
- package/dist/adapters/mail-part-store.js +251 -0
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +56 -65
- package/dist/adapters/spawn-child.d.ts +109 -44
- package/dist/adapters/spawn-child.js +77 -81
- package/dist/adapters/step-invoker.d.ts +52 -2
- package/dist/adapters/step-invoker.js +284 -37
- package/dist/adapters/substrate-mailbox-store.d.ts +80 -0
- package/dist/adapters/substrate-mailbox-store.js +404 -0
- package/dist/child/child-mailbox-reader.d.ts +10 -0
- package/dist/child/child-mailbox-reader.js +23 -0
- package/dist/child/credential-cell.d.ts +8 -0
- package/dist/child/credential-cell.js +66 -0
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/from-process-env.d.ts +12 -0
- package/dist/child/from-process-env.js +6 -0
- package/dist/child/index.d.ts +6 -2
- package/dist/child/index.js +4 -1
- package/dist/child/mailbox-mutation-bridge.d.ts +61 -0
- package/dist/child/mailbox-mutation-bridge.js +101 -0
- package/dist/child/mailbox-watch-registry.d.ts +17 -0
- package/dist/child/mailbox-watch-registry.js +61 -0
- package/dist/child/outbound-mail-bridge.d.ts +3 -2
- package/dist/child/outbound-mail-bridge.js +20 -32
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/pending-request.d.ts +89 -0
- package/dist/child/pending-request.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +170 -14
- package/dist/child/run-child.js +569 -155
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/substrate-write-bridge.d.ts +3 -2
- package/dist/child/substrate-write-bridge.js +21 -38
- package/dist/child/supervisor-backed-transport.d.ts +52 -6
- package/dist/child/supervisor-backed-transport.js +205 -62
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/child/warm-agent-cache.d.ts +44 -4
- package/dist/child/warm-agent-cache.js +41 -10
- package/dist/index.d.ts +6 -4
- package/dist/index.js +6 -4
- package/dist/ipc/control-channel.d.ts +151 -2
- package/dist/ipc/control-channel.js +222 -29
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/ipc/index.d.ts +1 -1
- package/dist/ipc/index.js +1 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/run-body-then-cleanup.d.ts +17 -0
- package/dist/run-body-then-cleanup.js +38 -0
- package/dist/seams/scheduler.d.ts +16 -6
- package/dist/seams/scheduler.js +87 -97
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +4 -8
- package/dist/supervisor/credentials.d.ts +28 -15
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +10 -3
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +14 -19
- package/dist/supervisor/run-event-recovery.d.ts +34 -0
- package/dist/supervisor/run-event-recovery.js +45 -0
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +106 -26
- package/dist/supervisor/supervisor.js +1903 -414
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +130 -0
- package/dist/supervisor/types.d.ts +180 -23
- package/dist/testing/change-notifier.d.ts +12 -0
- package/dist/testing/change-notifier.js +63 -0
- package/dist/testing/index.d.ts +8 -0
- package/dist/testing/index.js +16 -0
- package/dist/testing/log-capture.d.ts +52 -0
- package/dist/testing/log-capture.js +124 -0
- package/dist/testing/mail-bus.d.ts +22 -0
- package/dist/testing/mail-bus.js +78 -0
- package/dist/testing/memory-streams.d.ts +43 -0
- package/dist/testing/memory-streams.js +211 -0
- package/dist/testing/spawn-observer.d.ts +12 -0
- package/dist/testing/spawn-observer.js +36 -0
- package/dist/testing/stub-repo-store.d.ts +10 -0
- package/dist/testing/stub-repo-store.js +39 -0
- package/dist/testing/supervisor-reaper.d.ts +24 -0
- package/dist/testing/supervisor-reaper.js +49 -0
- package/dist/testing/upstream-frames.d.ts +47 -0
- package/dist/testing/upstream-frames.js +94 -0
- package/dist/workflow-definition-loader.d.ts +187 -0
- package/dist/workflow-definition-loader.js +422 -0
- package/package.json +18 -11
package/dist/child/index.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
export { createCredentialsBackedAuthorize, hashGrants, runWorkflowChild, } from "./run-child.js";
|
|
2
2
|
export { createChildSubstrateWriteBridge, } from "./substrate-write-bridge.js";
|
|
3
3
|
export { createChildOutboundMailBridge, } from "./outbound-mail-bridge.js";
|
|
4
|
-
export {
|
|
4
|
+
export { createChildMailboxMutationBridge, } from "./mailbox-mutation-bridge.js";
|
|
5
|
+
export { createSupervisorBackedTransport, } from "./supervisor-backed-transport.js";
|
|
6
|
+
export { createMailboxWatchRegistry, } from "./mailbox-watch-registry.js";
|
|
7
|
+
export { createChildMailboxReader, } from "./child-mailbox-reader.js";
|
|
5
8
|
export { createProxyWorkflowRunRepoStore, } from "./proxy-repo-store.js";
|
|
6
9
|
export { parseSpawnTimeEnv } from "./env-bootstrap.js";
|
|
7
10
|
export { discoverInFlightRuns, } from "./self-discovery.js";
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { ControlChannelSender, ControlPayload } from "../ipc/control-channel.js";
|
|
2
|
+
/**
|
|
3
|
+
* A mailbox mutation the child asks the supervisor to apply. A flag
|
|
4
|
+
* mutation carries the target `uid` and the `flags` to add or remove; an
|
|
5
|
+
* `expunge` sweeps every `\Deleted` message in the mailbox and so carries
|
|
6
|
+
* neither.
|
|
7
|
+
*/
|
|
8
|
+
export type MailboxMutation = {
|
|
9
|
+
runId: string;
|
|
10
|
+
mailbox: string;
|
|
11
|
+
op: "addFlags" | "removeFlags";
|
|
12
|
+
uid: number;
|
|
13
|
+
flags: string[];
|
|
14
|
+
} | {
|
|
15
|
+
runId: string;
|
|
16
|
+
mailbox: string;
|
|
17
|
+
op: "expunge";
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* The supervisor's applied-mutation result. `expungedUids` is present
|
|
21
|
+
* only for an `expunge` and lists the uids the sweep removed, so the
|
|
22
|
+
* agent tool can report how many messages it consumed.
|
|
23
|
+
*/
|
|
24
|
+
export type MailboxMutationResult = {
|
|
25
|
+
expungedUids?: number[];
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Bridge surface the child's supervisor-backed transport reaches into.
|
|
29
|
+
* `submit` sends a `mailbox.mutate.request` upstream and resolves once
|
|
30
|
+
* the supervisor's matching `mailbox.mutate.response` lands.
|
|
31
|
+
* `handleResult` is the receiver-side entry point the child's control
|
|
32
|
+
* loop invokes when the downstream `mailbox.mutate.response` frame
|
|
33
|
+
* arrives. `cancelAll` is the cleanup hook the control loop invokes on
|
|
34
|
+
* any exit path so a pending mutation does not leak an awaiter when the
|
|
35
|
+
* supervisor has torn the IPC down.
|
|
36
|
+
*/
|
|
37
|
+
export interface ChildMailboxMutationBridge {
|
|
38
|
+
submit(mutation: MailboxMutation): Promise<MailboxMutationResult>;
|
|
39
|
+
handleResult(data: Extract<ControlPayload, {
|
|
40
|
+
type: "mailbox.mutate.response";
|
|
41
|
+
}>["data"]): void;
|
|
42
|
+
cancelAll(reason: string): void;
|
|
43
|
+
readonly pendingCount: number;
|
|
44
|
+
}
|
|
45
|
+
export interface CreateChildMailboxMutationBridgeOpts {
|
|
46
|
+
upstreamSender: ControlChannelSender;
|
|
47
|
+
/**
|
|
48
|
+
* Optional `requestId` allocator. Production wires a per-instance
|
|
49
|
+
* monotonic counter plus a random suffix; tests inject a
|
|
50
|
+
* deterministic factory so the upstream frame's `requestId` is
|
|
51
|
+
* predictable.
|
|
52
|
+
*/
|
|
53
|
+
allocateRequestId?: () => string;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Construct the child-side mailbox-mutation bridge. Pending mutations
|
|
57
|
+
* live in the shared pending-request core keyed by `requestId`; the
|
|
58
|
+
* bridge resolves the awaiter when the supervisor's matching
|
|
59
|
+
* `mailbox.mutate.response` lands.
|
|
60
|
+
*/
|
|
61
|
+
export declare function createChildMailboxMutationBridge(opts: CreateChildMailboxMutationBridgeOpts): ChildMailboxMutationBridge;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// Child-side mailbox-mutation bridge (INBOUND half of mailbox ownership,
|
|
2
|
+
// §3b).
|
|
3
|
+
//
|
|
4
|
+
// The supervisor is the sole mail owner: it holds the long-lived
|
|
5
|
+
// substrate mailbox store and is the only writer to the workflow-run
|
|
6
|
+
// ref. A step agent reads its INBOX locally (the supervisor-backed
|
|
7
|
+
// transport's read surface opens fresh committed snapshots), but every
|
|
8
|
+
// MUTATION of the mailbox -- flag writes (`\Seen`, `\Deleted`, ...) and
|
|
9
|
+
// `expunge` -- routes up to the supervisor through this bridge rather
|
|
10
|
+
// than being flushed from the child. A second writer flushing the same
|
|
11
|
+
// ref from the child would race the supervisor's in-memory mirror and
|
|
12
|
+
// break uid / modseq monotonicity, so the child never writes the
|
|
13
|
+
// mailbox directly.
|
|
14
|
+
//
|
|
15
|
+
// Lifecycle of one mutation:
|
|
16
|
+
//
|
|
17
|
+
// 1. The agent's mail tool (flag or expunge) calls the
|
|
18
|
+
// supervisor-backed transport's `setFlags` / `clearFlags` /
|
|
19
|
+
// `expunge`. The transport calls `bridge.submit(mutation)`.
|
|
20
|
+
// 2. `submit` mints a `requestId`, registers a pending awaiter, and
|
|
21
|
+
// emits `mailbox.mutate.request` upstream carrying the op and its
|
|
22
|
+
// operands.
|
|
23
|
+
// 3. The supervisor applies the op to its owned mailbox store,
|
|
24
|
+
// flushes, and replies with `mailbox.mutate.response`. The reply is
|
|
25
|
+
// sent only after the flush, so the child's next committed read
|
|
26
|
+
// observes the mutation (the same flush-before-signal ordering
|
|
27
|
+
// `mailbox.notify` relies on).
|
|
28
|
+
// 4. The bridge resolves / rejects the pending awaiter; the
|
|
29
|
+
// transport method returns to the mail tool. A supervisor-side
|
|
30
|
+
// failure (unknown uid, substrate fault) surfaces as a rejection so
|
|
31
|
+
// the agent's mail-tool call fails loudly rather than silently
|
|
32
|
+
// dropping the mutation.
|
|
33
|
+
import { getLogger } from "@intx/log";
|
|
34
|
+
import { createPendingRequestCore } from "./pending-request.js";
|
|
35
|
+
const logger = getLogger(["workflow-host", "child", "mailbox-mutation-bridge"]);
|
|
36
|
+
/**
|
|
37
|
+
* Construct the child-side mailbox-mutation bridge. Pending mutations
|
|
38
|
+
* live in the shared pending-request core keyed by `requestId`; the
|
|
39
|
+
* bridge resolves the awaiter when the supervisor's matching
|
|
40
|
+
* `mailbox.mutate.response` lands.
|
|
41
|
+
*/
|
|
42
|
+
export function createChildMailboxMutationBridge(opts) {
|
|
43
|
+
const pending = createPendingRequestCore({
|
|
44
|
+
label: "workflow-child mailbox mutation",
|
|
45
|
+
allocatorPrefix: "mm",
|
|
46
|
+
allocateRequestId: opts.allocateRequestId,
|
|
47
|
+
});
|
|
48
|
+
return {
|
|
49
|
+
get pendingCount() {
|
|
50
|
+
return pending.pendingCount;
|
|
51
|
+
},
|
|
52
|
+
async submit(mutation) {
|
|
53
|
+
const { requestId, promise } = pending.register(undefined);
|
|
54
|
+
const data = mutation.op === "expunge"
|
|
55
|
+
? {
|
|
56
|
+
requestId,
|
|
57
|
+
runId: mutation.runId,
|
|
58
|
+
mailbox: mutation.mailbox,
|
|
59
|
+
op: mutation.op,
|
|
60
|
+
}
|
|
61
|
+
: {
|
|
62
|
+
requestId,
|
|
63
|
+
runId: mutation.runId,
|
|
64
|
+
mailbox: mutation.mailbox,
|
|
65
|
+
op: mutation.op,
|
|
66
|
+
uid: mutation.uid,
|
|
67
|
+
flags: mutation.flags,
|
|
68
|
+
};
|
|
69
|
+
try {
|
|
70
|
+
await opts.upstreamSender.send({
|
|
71
|
+
type: "mailbox.mutate.request",
|
|
72
|
+
data,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
catch (cause) {
|
|
76
|
+
pending.discard(requestId);
|
|
77
|
+
throw pending.sendFailedError(requestId, cause);
|
|
78
|
+
}
|
|
79
|
+
return promise;
|
|
80
|
+
},
|
|
81
|
+
handleResult(data) {
|
|
82
|
+
const entry = pending.settle(data.requestId);
|
|
83
|
+
if (entry === undefined) {
|
|
84
|
+
logger.warn `mailbox.mutate.response landed with no pending entry; requestId=${data.requestId} dropped`;
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
if (data.result.ok) {
|
|
88
|
+
const result = {};
|
|
89
|
+
if (data.result.expungedUids !== undefined) {
|
|
90
|
+
result.expungedUids = data.result.expungedUids;
|
|
91
|
+
}
|
|
92
|
+
entry.resolve(result);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
entry.reject(pending.rejectedError(data.requestId, data.result.reason));
|
|
96
|
+
},
|
|
97
|
+
cancelAll(reason) {
|
|
98
|
+
pending.cancelAll(reason);
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { MailboxEvent, Unsubscribe } from "@intx/types/runtime";
|
|
2
|
+
export interface MailboxWatchRegistry {
|
|
3
|
+
/**
|
|
4
|
+
* Register a callback for a mailbox. Returns an `Unsubscribe` that removes
|
|
5
|
+
* it; after unsubscribe the callback observes no further events, including
|
|
6
|
+
* one whose `fire` preceded the unsubscribe but whose asynchronous delivery
|
|
7
|
+
* had not yet run.
|
|
8
|
+
*/
|
|
9
|
+
watch(mailbox: string, callback: (event: MailboxEvent) => void): Unsubscribe;
|
|
10
|
+
/**
|
|
11
|
+
* Deliver a `MailboxEvent` to every callback currently registered for the
|
|
12
|
+
* mailbox, each on its own microtask. A no-op when no callback is registered
|
|
13
|
+
* for the mailbox.
|
|
14
|
+
*/
|
|
15
|
+
fire(mailbox: string, event: MailboxEvent): void;
|
|
16
|
+
}
|
|
17
|
+
export declare function createMailboxWatchRegistry(): MailboxWatchRegistry;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Child-side mailbox watch registry (INBOUND half of mailbox ownership,
|
|
2
|
+
// design §3b).
|
|
3
|
+
//
|
|
4
|
+
// The supervisor is the sole mail owner: it commits an arrived message to the
|
|
5
|
+
// workflow-run substrate mailbox and fires a `mailbox.notify` control frame.
|
|
6
|
+
// The child's control loop routes that frame to this registry's `fire`, which
|
|
7
|
+
// delivers a typed `exists` `MailboxEvent` to every callback registered for the
|
|
8
|
+
// mailbox through `watch`. The step agent's supervisor-backed transport
|
|
9
|
+
// implements `MessageTransport.watch` over this registry, so `mail_wait`
|
|
10
|
+
// unblocks when new mail lands -- decoupled from the FIFO trigger dispatch that
|
|
11
|
+
// resolves a run's first input.
|
|
12
|
+
//
|
|
13
|
+
// Delivery is ASYNCHRONOUS. A `fire` never invokes a callback synchronously on
|
|
14
|
+
// the delivering call stack: it schedules each callback on a microtask, per the
|
|
15
|
+
// IMAP IDLE contract the `MailboxEvent` watcher models (MESSAGE.md § Real-Time
|
|
16
|
+
// Notification). Delivery re-checks registration at the microtask, so a watcher
|
|
17
|
+
// that unsubscribes between `fire` and delivery observes no event.
|
|
18
|
+
export function createMailboxWatchRegistry() {
|
|
19
|
+
const watchers = new Map();
|
|
20
|
+
return {
|
|
21
|
+
watch(mailbox, callback) {
|
|
22
|
+
let set = watchers.get(mailbox);
|
|
23
|
+
if (set === undefined) {
|
|
24
|
+
set = new Set();
|
|
25
|
+
watchers.set(mailbox, set);
|
|
26
|
+
}
|
|
27
|
+
set.add(callback);
|
|
28
|
+
let active = true;
|
|
29
|
+
return () => {
|
|
30
|
+
// Idempotent: a double-unsubscribe must not remove a same-identity
|
|
31
|
+
// callback a later `watch` re-registered.
|
|
32
|
+
if (!active)
|
|
33
|
+
return;
|
|
34
|
+
active = false;
|
|
35
|
+
const current = watchers.get(mailbox);
|
|
36
|
+
if (current === undefined)
|
|
37
|
+
return;
|
|
38
|
+
current.delete(callback);
|
|
39
|
+
if (current.size === 0)
|
|
40
|
+
watchers.delete(mailbox);
|
|
41
|
+
};
|
|
42
|
+
},
|
|
43
|
+
fire(mailbox, event) {
|
|
44
|
+
const set = watchers.get(mailbox);
|
|
45
|
+
if (set === undefined)
|
|
46
|
+
return;
|
|
47
|
+
// Snapshot the callbacks registered at fire time, then deliver each on
|
|
48
|
+
// its own microtask so no callback runs synchronously on this call
|
|
49
|
+
// stack. Re-check membership at delivery so a callback unsubscribed
|
|
50
|
+
// between now and its microtask does not receive the event.
|
|
51
|
+
for (const callback of [...set]) {
|
|
52
|
+
queueMicrotask(() => {
|
|
53
|
+
const current = watchers.get(mailbox);
|
|
54
|
+
if (current === undefined || !current.has(callback))
|
|
55
|
+
return;
|
|
56
|
+
callback(event);
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
}
|
|
@@ -30,7 +30,8 @@ export interface CreateChildOutboundMailBridgeOpts {
|
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
32
|
* Construct the child-side outbound-mail bridge. Pending sends live in
|
|
33
|
-
*
|
|
34
|
-
* supervisor's matching `outbound.result`
|
|
33
|
+
* the shared pending-request core keyed by `requestId`; the bridge
|
|
34
|
+
* resolves the awaiter when the supervisor's matching `outbound.result`
|
|
35
|
+
* lands.
|
|
35
36
|
*/
|
|
36
37
|
export declare function createChildOutboundMailBridge(opts: CreateChildOutboundMailBridgeOpts): ChildOutboundMailBridge;
|
|
@@ -32,24 +32,26 @@
|
|
|
32
32
|
// send.
|
|
33
33
|
import { getLogger } from "@intx/log";
|
|
34
34
|
import { base64Encode } from "@intx/types";
|
|
35
|
+
import { createPendingRequestCore } from "./pending-request.js";
|
|
35
36
|
const logger = getLogger(["workflow-host", "child", "outbound-mail-bridge"]);
|
|
36
37
|
/**
|
|
37
38
|
* Construct the child-side outbound-mail bridge. Pending sends live in
|
|
38
|
-
*
|
|
39
|
-
* supervisor's matching `outbound.result`
|
|
39
|
+
* the shared pending-request core keyed by `requestId`; the bridge
|
|
40
|
+
* resolves the awaiter when the supervisor's matching `outbound.result`
|
|
41
|
+
* lands.
|
|
40
42
|
*/
|
|
41
43
|
export function createChildOutboundMailBridge(opts) {
|
|
42
|
-
const pending =
|
|
43
|
-
|
|
44
|
+
const pending = createPendingRequestCore({
|
|
45
|
+
label: "workflow-child outbound mail",
|
|
46
|
+
allocatorPrefix: "om",
|
|
47
|
+
allocateRequestId: opts.allocateRequestId,
|
|
48
|
+
});
|
|
44
49
|
return {
|
|
45
50
|
get pendingCount() {
|
|
46
|
-
return pending.
|
|
51
|
+
return pending.pendingCount;
|
|
47
52
|
},
|
|
48
53
|
async submit(senderAddress, message) {
|
|
49
|
-
const requestId =
|
|
50
|
-
const resultPromise = new Promise((resolve, reject) => {
|
|
51
|
-
pending.set(requestId, { resolve, reject });
|
|
52
|
-
});
|
|
54
|
+
const { requestId, promise } = pending.register(undefined);
|
|
53
55
|
try {
|
|
54
56
|
await opts.upstreamSender.send({
|
|
55
57
|
type: "outbound.message",
|
|
@@ -61,19 +63,17 @@ export function createChildOutboundMailBridge(opts) {
|
|
|
61
63
|
});
|
|
62
64
|
}
|
|
63
65
|
catch (cause) {
|
|
64
|
-
pending.
|
|
65
|
-
|
|
66
|
-
throw new Error(`workflow-child outbound mail: upstream send failed for requestId ${requestId}: ${reason}`, { cause });
|
|
66
|
+
pending.discard(requestId);
|
|
67
|
+
throw pending.sendFailedError(requestId, cause);
|
|
67
68
|
}
|
|
68
|
-
return
|
|
69
|
+
return promise;
|
|
69
70
|
},
|
|
70
71
|
handleResult(data) {
|
|
71
|
-
const entry = pending.
|
|
72
|
+
const entry = pending.settle(data.requestId);
|
|
72
73
|
if (entry === undefined) {
|
|
73
74
|
logger.warn `outbound.result landed with no pending entry; requestId=${data.requestId} dropped`;
|
|
74
75
|
return;
|
|
75
76
|
}
|
|
76
|
-
pending.delete(data.requestId);
|
|
77
77
|
if (data.result.ok) {
|
|
78
78
|
entry.resolve({
|
|
79
79
|
messageId: data.result.messageId,
|
|
@@ -81,13 +81,10 @@ export function createChildOutboundMailBridge(opts) {
|
|
|
81
81
|
});
|
|
82
82
|
return;
|
|
83
83
|
}
|
|
84
|
-
entry.reject(
|
|
84
|
+
entry.reject(pending.rejectedError(data.requestId, data.result.reason));
|
|
85
85
|
},
|
|
86
86
|
cancelAll(reason) {
|
|
87
|
-
|
|
88
|
-
entry.reject(new Error(`workflow-child outbound mail (requestId=${requestId}) cancelled: ${reason}`));
|
|
89
|
-
}
|
|
90
|
-
pending.clear();
|
|
87
|
+
pending.cancelAll(reason);
|
|
91
88
|
},
|
|
92
89
|
};
|
|
93
90
|
}
|
|
@@ -114,6 +111,8 @@ function projectOutboundMessage(message) {
|
|
|
114
111
|
payload.summary = message.summary;
|
|
115
112
|
if (message.inReplyTo !== undefined)
|
|
116
113
|
payload.inReplyTo = message.inReplyTo;
|
|
114
|
+
if (message.references !== undefined)
|
|
115
|
+
payload.references = message.references;
|
|
117
116
|
if (message.correlationId !== undefined) {
|
|
118
117
|
payload.correlationId = message.correlationId;
|
|
119
118
|
}
|
|
@@ -125,19 +124,8 @@ function projectOutboundMessage(message) {
|
|
|
125
124
|
payload.attachments = message.attachments.map((a) => ({
|
|
126
125
|
name: a.name,
|
|
127
126
|
contentType: a.contentType,
|
|
128
|
-
dataBase64:
|
|
127
|
+
dataBase64: base64Encode(a.data),
|
|
129
128
|
}));
|
|
130
129
|
}
|
|
131
130
|
return payload;
|
|
132
131
|
}
|
|
133
|
-
function bytesToBase64(bytes) {
|
|
134
|
-
return base64Encode(bytes);
|
|
135
|
-
}
|
|
136
|
-
function defaultRequestIdAllocator() {
|
|
137
|
-
let counter = 0;
|
|
138
|
-
return () => {
|
|
139
|
-
counter += 1;
|
|
140
|
-
const rand = Math.random().toString(36).slice(2, 10);
|
|
141
|
-
return `om-${String(counter)}-${rand}`;
|
|
142
|
-
};
|
|
143
|
-
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
|
|
2
|
+
import { type RepoStore as RuntimeRepoStore } from "@intx/workflow";
|
|
3
|
+
import type { ApprovalSnapshot, ControlParkKind } from "@intx/types/runtime";
|
|
4
|
+
/**
|
|
5
|
+
* Recover the durable approval snapshot for one parked control-plane
|
|
6
|
+
* correlation. The child owns enumeration; the host owns the per-step
|
|
7
|
+
* on-disk layout (cold vs warm), so the snapshot read is a host binding.
|
|
8
|
+
* Returns `undefined` when no pending operation for the correlation carries
|
|
9
|
+
* a snapshot.
|
|
10
|
+
*/
|
|
11
|
+
export type LoadParkedApproval = (args: {
|
|
12
|
+
runId: string;
|
|
13
|
+
stepId: string;
|
|
14
|
+
attempt: number;
|
|
15
|
+
correlationId: string;
|
|
16
|
+
}) => Promise<ApprovalSnapshot | undefined>;
|
|
17
|
+
/**
|
|
18
|
+
* One parked control-plane correlation: the child-supplied half of a
|
|
19
|
+
* suspension registration. `parkKind` discriminates approval parks
|
|
20
|
+
* (which carry a snapshot) from input parks (which do not).
|
|
21
|
+
*/
|
|
22
|
+
export interface ParkedApprovalCorrelation {
|
|
23
|
+
runId: string;
|
|
24
|
+
correlationId: string;
|
|
25
|
+
parkKind: ControlParkKind;
|
|
26
|
+
snapshot?: ApprovalSnapshot;
|
|
27
|
+
}
|
|
28
|
+
export interface CollectParkedApprovalCorrelationsOpts {
|
|
29
|
+
substrate: SubstrateRepoStore;
|
|
30
|
+
repoId: RepoId;
|
|
31
|
+
runtimeRepoStore: RuntimeRepoStore;
|
|
32
|
+
loadParkedApproval?: LoadParkedApproval;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Enumerate every in-flight run's reduced state and return one entry per step
|
|
36
|
+
* parked on a control-plane approval channel. Throws when a park is found but
|
|
37
|
+
* no `loadParkedApproval` binding is wired to recover its snapshot, or when
|
|
38
|
+
* the binding returns no snapshot for an enumerated park -- both are
|
|
39
|
+
* disagreements between the reduced state and the durable store that must not
|
|
40
|
+
* silently drop a correlation the hub is waiting to register.
|
|
41
|
+
*/
|
|
42
|
+
export declare function collectParkedApprovalCorrelations(opts: CollectParkedApprovalCorrelationsOpts): Promise<ParkedApprovalCorrelation[]>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Enumerate a child's currently-parked approval correlations from durable
|
|
2
|
+
// state, so the child can answer a supervisor `parked-correlations.request`.
|
|
3
|
+
//
|
|
4
|
+
// Enumeration keys on REDUCED step state, never on raw `SignalAwaited` log
|
|
5
|
+
// events. `parkOnSignal` commits `SignalAwaited` to the durable log before it
|
|
6
|
+
// checks for the approval snapshot, so a snapshot-less correlated suspend (a
|
|
7
|
+
// director `caps.suspend`, an unwired authz gate) leaves a control-plane
|
|
8
|
+
// `SignalAwaited` in the log yet reduces to `phase === "failed"` -- never
|
|
9
|
+
// `awaiting-signal`, and never a hub row. Filtering on the reduced
|
|
10
|
+
// `awaiting-signal` phase therefore surfaces only the parks that carry a
|
|
11
|
+
// durable snapshot by construction; a snapshot-less enumerated step is a
|
|
12
|
+
// disagreement between the log and the step store, which this module surfaces
|
|
13
|
+
// loudly rather than dropping.
|
|
14
|
+
import { controlParkKindOf, } from "@intx/workflow";
|
|
15
|
+
import { correlationIdFromSignalName } from "@intx/types";
|
|
16
|
+
import { discoverInFlightRuns } from "./self-discovery.js";
|
|
17
|
+
/**
|
|
18
|
+
* Enumerate every in-flight run's reduced state and return one entry per step
|
|
19
|
+
* parked on a control-plane approval channel. Throws when a park is found but
|
|
20
|
+
* no `loadParkedApproval` binding is wired to recover its snapshot, or when
|
|
21
|
+
* the binding returns no snapshot for an enumerated park -- both are
|
|
22
|
+
* disagreements between the reduced state and the durable store that must not
|
|
23
|
+
* silently drop a correlation the hub is waiting to register.
|
|
24
|
+
*/
|
|
25
|
+
export async function collectParkedApprovalCorrelations(opts) {
|
|
26
|
+
const discovered = await discoverInFlightRuns({
|
|
27
|
+
substrate: opts.substrate,
|
|
28
|
+
repoId: opts.repoId,
|
|
29
|
+
runtimeRepoStore: opts.runtimeRepoStore,
|
|
30
|
+
});
|
|
31
|
+
const out = [];
|
|
32
|
+
for (const run of discovered) {
|
|
33
|
+
for (const step of run.resumedState.steps.values()) {
|
|
34
|
+
if (step.phase !== "awaiting-signal")
|
|
35
|
+
continue;
|
|
36
|
+
const awaited = step.awaitingSignal;
|
|
37
|
+
if (awaited === undefined)
|
|
38
|
+
continue;
|
|
39
|
+
const correlationId = correlationIdFromSignalName(awaited.name);
|
|
40
|
+
if (correlationId === undefined)
|
|
41
|
+
continue;
|
|
42
|
+
// An `"input"` park (a long-lived run awaiting its next mail) reduces to
|
|
43
|
+
// the same `awaiting-signal` on a reserved channel as an approval, but it
|
|
44
|
+
// carries NO snapshot and is never hub-registered -- the run's owner
|
|
45
|
+
// delivers the input directly. Skip it: enumerating it would call
|
|
46
|
+
// loadParkedApproval, get no snapshot, and throw below, taking the whole
|
|
47
|
+
// deployment's approval re-registration down on every reconnect.
|
|
48
|
+
// `controlParkKindOf` is the single point that reads a reserved-channel
|
|
49
|
+
// park's kind; an absent kind is a legacy approval, not an input park.
|
|
50
|
+
const parkKind = controlParkKindOf(awaited);
|
|
51
|
+
if (parkKind === "input") {
|
|
52
|
+
out.push({
|
|
53
|
+
runId: run.runId,
|
|
54
|
+
correlationId,
|
|
55
|
+
parkKind: "input",
|
|
56
|
+
});
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (opts.loadParkedApproval === undefined) {
|
|
60
|
+
throw new Error(`workflow-child parked-correlations: run ${run.runId} step ${step.stepId} is parked on control-plane correlation ${correlationId}, but no loadParkedApproval binding is wired to recover its snapshot`);
|
|
61
|
+
}
|
|
62
|
+
const snapshot = await opts.loadParkedApproval({
|
|
63
|
+
runId: run.runId,
|
|
64
|
+
stepId: step.stepId,
|
|
65
|
+
attempt: step.currentAttempt,
|
|
66
|
+
correlationId,
|
|
67
|
+
});
|
|
68
|
+
if (snapshot === undefined) {
|
|
69
|
+
throw new Error(`workflow-child parked-correlations: reduced state shows run ${run.runId} step ${step.stepId} awaiting control-plane correlation ${correlationId} (attempt ${String(step.currentAttempt)}), but durable storage carries no approval snapshot for it; the run log and the step store disagree`);
|
|
70
|
+
}
|
|
71
|
+
out.push({
|
|
72
|
+
runId: run.runId,
|
|
73
|
+
correlationId,
|
|
74
|
+
parkKind: "approval",
|
|
75
|
+
snapshot,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A live pending entry as returned by {@link PendingRequestCore.get} and
|
|
3
|
+
* {@link PendingRequestCore.settle}. `meta` is the per-bridge payload the
|
|
4
|
+
* entry was registered with (e.g. the substrate-write request whose merge
|
|
5
|
+
* closure the merge round-trip invokes).
|
|
6
|
+
*/
|
|
7
|
+
export type PendingEntryHandle<Value, Meta> = {
|
|
8
|
+
meta: Meta;
|
|
9
|
+
resolve: (value: Value) => void;
|
|
10
|
+
reject: (err: Error) => void;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Options for {@link createPendingRequestCore}. `label` prefixes every
|
|
14
|
+
* error the core builds (`<label>: upstream send failed for requestId
|
|
15
|
+
* <id>: ...`, `<label> (requestId=<id>) rejected by supervisor: ...`,
|
|
16
|
+
* `<label> (requestId=<id>) cancelled: ...`, `<label>: no pending entry
|
|
17
|
+
* for requestId <id>`), so each bridge's observable error strings stay
|
|
18
|
+
* its own. `allocatorPrefix` seeds the default requestId allocator
|
|
19
|
+
* (`<prefix>-<counter>-<rand>`); tests inject `allocateRequestId` for a
|
|
20
|
+
* deterministic id.
|
|
21
|
+
*/
|
|
22
|
+
export type PendingRequestCoreOptions = {
|
|
23
|
+
label: string;
|
|
24
|
+
allocatorPrefix: string;
|
|
25
|
+
/** `undefined` is accepted explicitly so callers can forward an optional option verbatim. */
|
|
26
|
+
allocateRequestId?: (() => string) | undefined;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* The pending-request lifecycle surface a bridge composes into its own
|
|
30
|
+
* `submit` / `handle*` / `cancelAll` methods.
|
|
31
|
+
*/
|
|
32
|
+
export type PendingRequestCore<Value, Meta = undefined> = {
|
|
33
|
+
/** Number of registered-but-unsettled requests. */
|
|
34
|
+
readonly pendingCount: number;
|
|
35
|
+
/**
|
|
36
|
+
* Mint a requestId, register an awaiter keyed by it, and return the id
|
|
37
|
+
* plus the promise the bridge returns from its `submit`. The entry
|
|
38
|
+
* stays pending until `settle` or `cancelAll` acts on it, or `discard`
|
|
39
|
+
* removes it without settling (the upstream-send-failure path).
|
|
40
|
+
*/
|
|
41
|
+
register(meta: Meta): {
|
|
42
|
+
requestId: string;
|
|
43
|
+
promise: Promise<Value>;
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Remove a pending entry without settling its promise. Used when the
|
|
47
|
+
* upstream send itself fails, so the abandoned awaiter does not leak.
|
|
48
|
+
*/
|
|
49
|
+
discard(requestId: string): void;
|
|
50
|
+
/**
|
|
51
|
+
* Look up a pending entry WITHOUT removing it. Used by the
|
|
52
|
+
* substrate-write bridge's merge round-trip, which must reach the
|
|
53
|
+
* entry's merge closure while the entry stays alive for the terminal
|
|
54
|
+
* write response.
|
|
55
|
+
*/
|
|
56
|
+
get(requestId: string): PendingEntryHandle<Value, Meta> | undefined;
|
|
57
|
+
/**
|
|
58
|
+
* Look up a pending entry and remove it. Used by the response-frame
|
|
59
|
+
* handlers: the entry is gone before its promise resolves/rejects, so
|
|
60
|
+
* a later stale response or `cancelAll` finds nothing to act on.
|
|
61
|
+
*/
|
|
62
|
+
settle(requestId: string): PendingEntryHandle<Value, Meta> | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Reject every pending entry with `<label> (requestId=<id>) cancelled:
|
|
65
|
+
* <reason>` and clear the map. The control loop invokes this on any
|
|
66
|
+
* exit path so no awaiter leaks when the supervisor has torn the IPC
|
|
67
|
+
* down.
|
|
68
|
+
*/
|
|
69
|
+
cancelAll(reason: string): void;
|
|
70
|
+
/** `<label>: upstream send failed for requestId <id>: <cause message>` with `{ cause }`. */
|
|
71
|
+
sendFailedError(requestId: string, cause: unknown): Error;
|
|
72
|
+
/** `<label> (requestId=<id>) rejected by supervisor: <reason>`. */
|
|
73
|
+
rejectedError(requestId: string, reason: string): Error;
|
|
74
|
+
/** `<label>: no pending entry for requestId <id>` (substrate merge failure reply). */
|
|
75
|
+
noPendingError(requestId: string): Error;
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Construct a pending-request core for one bridge. Pending entries live in
|
|
79
|
+
* a map keyed by `requestId`; the bridge's response handler settles the
|
|
80
|
+
* awaiter when the matching response frame lands.
|
|
81
|
+
*/
|
|
82
|
+
export declare function createPendingRequestCore<Value, Meta = undefined>(opts: PendingRequestCoreOptions): PendingRequestCore<Value, Meta>;
|
|
83
|
+
/**
|
|
84
|
+
* Default `requestId` allocator: a per-instance monotonic counter plus a
|
|
85
|
+
* random suffix, so ids are unique across bridge instances without any
|
|
86
|
+
* cross-instance coordination. The `prefix` names the owning bridge
|
|
87
|
+
* (`sw-`, `om-`, `mm-`) for triage in supervisor logs.
|
|
88
|
+
*/
|
|
89
|
+
export declare function defaultRequestIdAllocator(prefix: string): () => string;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Generic pending-request core shared by the three child-side control-IPC
|
|
2
|
+
// bridges (substrate-write, outbound-mail, mailbox-mutation).
|
|
3
|
+
//
|
|
4
|
+
// Every bridge runs the same request/response round-trip over the control
|
|
5
|
+
// channel: `submit` mints a `requestId`, registers a pending awaiter keyed
|
|
6
|
+
// by that id, emits a request frame upstream, and resolves/rejects the
|
|
7
|
+
// awaiter when the supervisor's matching response frame lands. The three
|
|
8
|
+
// bridges differ only in the request payload they send, the value they
|
|
9
|
+
// resolve, the label their error messages carry, and (for the
|
|
10
|
+
// substrate-write bridge) an intermediate merge round-trip that must peek
|
|
11
|
+
// at the pending entry without settling it.
|
|
12
|
+
//
|
|
13
|
+
// This core owns that lifecycle in one place so the per-bridge files
|
|
14
|
+
// contain only their wire-specific parts. It never touches the control
|
|
15
|
+
// channel itself: each bridge performs its own `upstreamSender.send` so
|
|
16
|
+
// the frame payloads stay byte-identical to what the supervisor's
|
|
17
|
+
// `ControlPayload` validator expects.
|
|
18
|
+
/**
|
|
19
|
+
* Construct a pending-request core for one bridge. Pending entries live in
|
|
20
|
+
* a map keyed by `requestId`; the bridge's response handler settles the
|
|
21
|
+
* awaiter when the matching response frame lands.
|
|
22
|
+
*/
|
|
23
|
+
export function createPendingRequestCore(opts) {
|
|
24
|
+
const pending = new Map();
|
|
25
|
+
const allocate = opts.allocateRequestId ?? defaultRequestIdAllocator(opts.allocatorPrefix);
|
|
26
|
+
return {
|
|
27
|
+
get pendingCount() {
|
|
28
|
+
return pending.size;
|
|
29
|
+
},
|
|
30
|
+
register(meta) {
|
|
31
|
+
const requestId = allocate();
|
|
32
|
+
const promise = new Promise((resolve, reject) => {
|
|
33
|
+
pending.set(requestId, { meta, resolve, reject });
|
|
34
|
+
});
|
|
35
|
+
return { requestId, promise };
|
|
36
|
+
},
|
|
37
|
+
discard(requestId) {
|
|
38
|
+
pending.delete(requestId);
|
|
39
|
+
},
|
|
40
|
+
get(requestId) {
|
|
41
|
+
return pending.get(requestId);
|
|
42
|
+
},
|
|
43
|
+
settle(requestId) {
|
|
44
|
+
const entry = pending.get(requestId);
|
|
45
|
+
if (entry !== undefined)
|
|
46
|
+
pending.delete(requestId);
|
|
47
|
+
return entry;
|
|
48
|
+
},
|
|
49
|
+
cancelAll(reason) {
|
|
50
|
+
for (const [requestId, entry] of pending) {
|
|
51
|
+
entry.reject(new Error(`${opts.label} (requestId=${requestId}) cancelled: ${reason}`));
|
|
52
|
+
}
|
|
53
|
+
pending.clear();
|
|
54
|
+
},
|
|
55
|
+
sendFailedError(requestId, cause) {
|
|
56
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
57
|
+
return new Error(`${opts.label}: upstream send failed for requestId ${requestId}: ${reason}`, { cause });
|
|
58
|
+
},
|
|
59
|
+
rejectedError(requestId, reason) {
|
|
60
|
+
return new Error(`${opts.label} (requestId=${requestId}) rejected by supervisor: ${reason}`);
|
|
61
|
+
},
|
|
62
|
+
noPendingError(requestId) {
|
|
63
|
+
return new Error(`${opts.label}: no pending entry for requestId ${requestId}`);
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Default `requestId` allocator: a per-instance monotonic counter plus a
|
|
69
|
+
* random suffix, so ids are unique across bridge instances without any
|
|
70
|
+
* cross-instance coordination. The `prefix` names the owning bridge
|
|
71
|
+
* (`sw-`, `om-`, `mm-`) for triage in supervisor logs.
|
|
72
|
+
*/
|
|
73
|
+
export function defaultRequestIdAllocator(prefix) {
|
|
74
|
+
let counter = 0;
|
|
75
|
+
return () => {
|
|
76
|
+
counter += 1;
|
|
77
|
+
const rand = Math.random().toString(36).slice(2, 10);
|
|
78
|
+
return `${prefix}-${String(counter)}-${rand}`;
|
|
79
|
+
};
|
|
80
|
+
}
|