@intx/hub-sessions 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/agent-repo.d.ts +23 -7
- package/dist/agent-repo.js +19 -6
- package/dist/agent-state-kind.js +18 -63
- package/dist/asset-service.d.ts +1 -20
- package/dist/asset-service.js +19 -97
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +52 -7
- package/dist/credential-push.js +178 -22
- package/dist/event-collector-registry.d.ts +3 -2
- package/dist/event-collector-registry.js +42 -13
- package/dist/event-collector.d.ts +12 -2
- package/dist/event-collector.js +45 -4
- package/dist/hub-session-lookups.d.ts +125 -7
- package/dist/hub-session-lookups.js +541 -86
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +22 -56
- package/dist/index.d.ts +19 -9
- package/dist/index.js +16 -7
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +2 -1
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/store.d.ts +1 -1
- package/dist/repo-store/store.js +138 -1
- package/dist/repo-store/subscribe-kind.d.ts +6 -3
- package/dist/repo-store/subscribe-kind.js +42 -77
- package/dist/repo-store/types.d.ts +94 -6
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +252 -126
- package/dist/session-service.js +851 -624
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +101 -0
- package/dist/sidecar-allocation/contracts.js +26 -0
- package/dist/sidecar-allocation/index.d.ts +5 -0
- package/dist/sidecar-allocation/index.js +4 -0
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +24 -0
- package/dist/sidecar-allocation/plugin-registry.js +61 -0
- package/dist/sidecar-allocation/reconciler.d.ts +54 -0
- package/dist/sidecar-allocation/reconciler.js +825 -0
- package/dist/skill-kind.js +12 -62
- package/dist/substrate.d.ts +3 -3
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +64 -0
- package/dist/workflow-allocation-service.js +554 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +42 -0
- package/dist/workflow-dispatch-service.js +209 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +29 -1
- package/dist/workflow-kind.js +143 -139
- package/dist/workflow-probe-gate.d.ts +286 -0
- package/dist/workflow-probe-gate.js +382 -0
- package/dist/workflow-run-kind.d.ts +239 -32
- package/dist/workflow-run-kind.js +874 -188
- package/dist/workflow-run-reader.d.ts +1 -1
- package/dist/workflow-run-reader.js +3 -7
- package/dist/workflow-run-restore.d.ts +16 -0
- package/dist/workflow-run-restore.js +30 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +3 -3
- package/dist/ws/index.js +2 -2
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +139 -37
- package/dist/ws/sidecar-events.js +2 -2
- package/dist/ws/sidecar-handler.d.ts +218 -60
- package/dist/ws/sidecar-handler.js +1713 -732
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
- package/dist/ws/sidecar-token-authenticator.js +78 -7
- package/package.json +14 -13
- package/dist/available-skills-stanza.d.ts +0 -21
- package/dist/available-skills-stanza.js +0 -32
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Hub-side dependency-closure resolution for a code-sourced workflow
|
|
2
|
+
// definition.
|
|
3
|
+
//
|
|
4
|
+
// Given a `WorkflowDefinitionSource` and a pin, walk the full dependency
|
|
5
|
+
// closure to concrete versions plus integrity SRIs, reusing the tool-packaging
|
|
6
|
+
// closure resolver. This is the "what exactly would we install" step: packument
|
|
7
|
+
// and package.json metadata reads only, no code execution. The returned
|
|
8
|
+
// `ToolPackageManifest` is the frozen closure the deploy path later ships by
|
|
9
|
+
// source-ref.
|
|
10
|
+
//
|
|
11
|
+
// There are three arms, one per `(source.kind, package.format)`:
|
|
12
|
+
// - `registry` names an npm registry; its URL and credentials are supplied by
|
|
13
|
+
// the caller as a `RegistryConfig`, since the caller owns the registry
|
|
14
|
+
// configuration.
|
|
15
|
+
// - `asset` + `tarball` names a hub asset that publishes the definition
|
|
16
|
+
// package as a tarball; the caller supplies bound `readBlob`/`listBlobs`
|
|
17
|
+
// closures over that asset (the same shape `session-service` mints for
|
|
18
|
+
// `AssetRegistrySource`).
|
|
19
|
+
// - `asset` + `source` names a hub asset whose definition package lives as a
|
|
20
|
+
// git subtree at a pinned commit; the caller supplies a `SourceTreeReads`
|
|
21
|
+
// pinned to that commit plus a `RegistryConfig` for the external npm deps.
|
|
22
|
+
// This arm delegates to `resolveSourceWorkflowClosure`, which reads the
|
|
23
|
+
// tree directly and needs no `name@range` pin (it selects the member from
|
|
24
|
+
// the source's `packageName`).
|
|
25
|
+
// Every arm takes pre-bound read capabilities, keeping this module free of any
|
|
26
|
+
// asset-service or repo-store dependency.
|
|
27
|
+
import { base64Encode } from "@intx/types";
|
|
28
|
+
import { AssetRegistrySource, HttpRegistrySource, createClosureResolver, parsePin, } from "@intx/tool-packaging";
|
|
29
|
+
import { resolveSourceWorkflowClosure, } from "./workflow-source-closure.js";
|
|
30
|
+
// Both asset arms carry an identical `source` field type, so narrow on the
|
|
31
|
+
// source's own `package.format` discriminant rather than adding a redundant
|
|
32
|
+
// discriminant to the args.
|
|
33
|
+
function isAssetSourceArgs(args) {
|
|
34
|
+
return (args.source.kind === "asset" && args.source.package.format === "source");
|
|
35
|
+
}
|
|
36
|
+
function isAssetTarballArgs(args) {
|
|
37
|
+
return (args.source.kind === "asset" && args.source.package.format === "tarball");
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Resolve a workflow definition's dependency closure to concrete versions
|
|
41
|
+
* and integrity SRIs, reusing the tool-packaging closure resolver.
|
|
42
|
+
*
|
|
43
|
+
* @returns the frozen closure: a `ToolPackageManifest` whose `topLevel`
|
|
44
|
+
* pins the definition package and whose `entries` carry the concrete
|
|
45
|
+
* version and integrity SRI of every transitive dependency.
|
|
46
|
+
*/
|
|
47
|
+
export async function resolveWorkflowClosure(args) {
|
|
48
|
+
if (isAssetSourceArgs(args)) {
|
|
49
|
+
return resolveSourceWorkflowClosure({
|
|
50
|
+
source: args.source,
|
|
51
|
+
reads: args.reads,
|
|
52
|
+
registryName: args.registryName,
|
|
53
|
+
registryConfig: args.registryConfig,
|
|
54
|
+
...(args.fetchPackument !== undefined
|
|
55
|
+
? { fetchPackument: args.fetchPackument }
|
|
56
|
+
: {}),
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
if (isAssetTarballArgs(args)) {
|
|
60
|
+
// The assetId is the source name so every walker resolution error is
|
|
61
|
+
// traceable to the exact asset; it is the sole and default source.
|
|
62
|
+
const name = args.source.assetId;
|
|
63
|
+
const assetSource = new AssetRegistrySource({
|
|
64
|
+
name,
|
|
65
|
+
assetId: args.source.assetId,
|
|
66
|
+
readBlob: args.readBlob,
|
|
67
|
+
listBlobs: args.listBlobs,
|
|
68
|
+
});
|
|
69
|
+
const resolver = createClosureResolver({
|
|
70
|
+
registries: new Map([[name, assetSource]]),
|
|
71
|
+
defaultRegistry: name,
|
|
72
|
+
});
|
|
73
|
+
return resolver.resolveClosure([parsePin(args.pin)]);
|
|
74
|
+
}
|
|
75
|
+
const registrySource = new HttpRegistrySource({
|
|
76
|
+
name: args.source.registry,
|
|
77
|
+
config: args.registryConfig,
|
|
78
|
+
...(args.fetchPackument !== undefined
|
|
79
|
+
? { fetchPackument: args.fetchPackument }
|
|
80
|
+
: {}),
|
|
81
|
+
});
|
|
82
|
+
const resolver = createClosureResolver({
|
|
83
|
+
registries: new Map([[registrySource.name, registrySource]]),
|
|
84
|
+
defaultRegistry: registrySource.name,
|
|
85
|
+
});
|
|
86
|
+
return resolver.resolveClosure([parsePin(args.pin)]);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* The workspace-relative mount path a workflow definition's source asset is
|
|
90
|
+
* delivered under, keyed by asset id. Both the probe (inline delivery) and the
|
|
91
|
+
* deploy (streamed fan-out) build their `assetId -> mountPath` maps through
|
|
92
|
+
* this one helper, so the same asset lands at the same path on both paths.
|
|
93
|
+
*/
|
|
94
|
+
export function workflowSourceAssetMountPath(assetId) {
|
|
95
|
+
return `source-assets/${assetId}/`;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Build the inline asset mounts a source-ref frame delivers for an asset-sourced
|
|
99
|
+
* closure: one per distinct asset the closure's `kind:"asset"` entries name (so
|
|
100
|
+
* a multi-asset closure delivers every backing asset, not just the source), each
|
|
101
|
+
* carrying the git pack the caller resolves for that asset. `mountPath` is
|
|
102
|
+
* derived through the shared helper so probe and deploy agree on it.
|
|
103
|
+
*/
|
|
104
|
+
export async function buildSourceAssetMounts(closure, resolveAttachment) {
|
|
105
|
+
const assetIds = new Set();
|
|
106
|
+
for (const entry of closure.entries) {
|
|
107
|
+
if (entry.source.kind === "asset") {
|
|
108
|
+
assetIds.add(entry.source.assetId);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const mounts = [];
|
|
112
|
+
for (const assetId of assetIds) {
|
|
113
|
+
const attachment = await resolveAttachment(assetId);
|
|
114
|
+
mounts.push({
|
|
115
|
+
assetId,
|
|
116
|
+
mountPath: workflowSourceAssetMountPath(assetId),
|
|
117
|
+
pack: base64Encode(attachment.pack),
|
|
118
|
+
ref: attachment.ref,
|
|
119
|
+
commitSha: attachment.commitSha,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return mounts;
|
|
123
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type DBExecutor, type WorkflowDefinitionSelector } from "@intx/db";
|
|
2
|
+
/**
|
|
3
|
+
* The first-class `workflow_definition` a selector names, created if absent. A
|
|
4
|
+
* native workflow carries its body in the asset it points at, so this only
|
|
5
|
+
* projects a definition row -- plus its version "1" -- over that asset, keyed
|
|
6
|
+
* by the selector's wire hash, so a run can anchor on a first-class definition.
|
|
7
|
+
* A single asset backs many definitions distinguished by their wire hash, so
|
|
8
|
+
* distinct hashes over one asset yield distinct definitions.
|
|
9
|
+
* `currentVersion`/`status` are left to the table defaults; the definition
|
|
10
|
+
* carries no `modelRequirements` manifest, so it deploys as a workflow rather
|
|
11
|
+
* than launching as an instance.
|
|
12
|
+
*
|
|
13
|
+
* Idempotent and concurrency-safe: the definition insert conflicts on the
|
|
14
|
+
* unique `(assetId, wireHash)` and the version on `(definitionId, version)`, so
|
|
15
|
+
* two concurrent deploys of the same selector still yield exactly one
|
|
16
|
+
* definition and one version. The returned id is read back after the conflict
|
|
17
|
+
* -- the winner's id, not a locally-minted one -- so a losing insert anchors
|
|
18
|
+
* its version on the real row. `created` distinguishes a fresh projection from
|
|
19
|
+
* a pre-existing one for callers that report fold counts.
|
|
20
|
+
*/
|
|
21
|
+
export declare function ensureWorkflowDefinitionForAsset(db: DBExecutor, selector: WorkflowDefinitionSelector): Promise<{
|
|
22
|
+
definitionId: string;
|
|
23
|
+
created: boolean;
|
|
24
|
+
}>;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { eq } from "drizzle-orm";
|
|
2
|
+
import { resolveDefinitionIdForAsset, } from "@intx/db";
|
|
3
|
+
import { asset, workflowDefinition, workflowDefinitionVersion, } from "@intx/db/schema";
|
|
4
|
+
import { generateId } from "@intx/hub-common";
|
|
5
|
+
/**
|
|
6
|
+
* The first-class `workflow_definition` a selector names, created if absent. A
|
|
7
|
+
* native workflow carries its body in the asset it points at, so this only
|
|
8
|
+
* projects a definition row -- plus its version "1" -- over that asset, keyed
|
|
9
|
+
* by the selector's wire hash, so a run can anchor on a first-class definition.
|
|
10
|
+
* A single asset backs many definitions distinguished by their wire hash, so
|
|
11
|
+
* distinct hashes over one asset yield distinct definitions.
|
|
12
|
+
* `currentVersion`/`status` are left to the table defaults; the definition
|
|
13
|
+
* carries no `modelRequirements` manifest, so it deploys as a workflow rather
|
|
14
|
+
* than launching as an instance.
|
|
15
|
+
*
|
|
16
|
+
* Idempotent and concurrency-safe: the definition insert conflicts on the
|
|
17
|
+
* unique `(assetId, wireHash)` and the version on `(definitionId, version)`, so
|
|
18
|
+
* two concurrent deploys of the same selector still yield exactly one
|
|
19
|
+
* definition and one version. The returned id is read back after the conflict
|
|
20
|
+
* -- the winner's id, not a locally-minted one -- so a losing insert anchors
|
|
21
|
+
* its version on the real row. `created` distinguishes a fresh projection from
|
|
22
|
+
* a pre-existing one for callers that report fold counts.
|
|
23
|
+
*/
|
|
24
|
+
export async function ensureWorkflowDefinitionForAsset(db, selector) {
|
|
25
|
+
const { assetId, wireHash } = selector;
|
|
26
|
+
const assetRow = await db
|
|
27
|
+
.select({
|
|
28
|
+
tenantId: asset.tenantId,
|
|
29
|
+
creatorPrincipalId: asset.creatorPrincipalId,
|
|
30
|
+
name: asset.name,
|
|
31
|
+
displayName: asset.displayName,
|
|
32
|
+
})
|
|
33
|
+
.from(asset)
|
|
34
|
+
.where(eq(asset.id, assetId))
|
|
35
|
+
.limit(1)
|
|
36
|
+
.then((rows) => rows[0]);
|
|
37
|
+
if (assetRow === undefined) {
|
|
38
|
+
throw new Error(`ensureWorkflowDefinitionForAsset: no asset found for id "${assetId}"`);
|
|
39
|
+
}
|
|
40
|
+
const inserted = await db
|
|
41
|
+
.insert(workflowDefinition)
|
|
42
|
+
.values({
|
|
43
|
+
id: generateId("workflowDefinition"),
|
|
44
|
+
tenantId: assetRow.tenantId,
|
|
45
|
+
creatorPrincipalId: assetRow.creatorPrincipalId,
|
|
46
|
+
assetId,
|
|
47
|
+
wireHash,
|
|
48
|
+
name: assetRow.name,
|
|
49
|
+
description: assetRow.displayName,
|
|
50
|
+
})
|
|
51
|
+
.onConflictDoNothing({
|
|
52
|
+
target: [workflowDefinition.assetId, workflowDefinition.wireHash],
|
|
53
|
+
})
|
|
54
|
+
.returning({ id: workflowDefinition.id });
|
|
55
|
+
const created = inserted.length > 0;
|
|
56
|
+
const insertedId = inserted[0]?.id;
|
|
57
|
+
const definitionId = insertedId ?? (await resolveDefinitionIdForAsset(db, selector));
|
|
58
|
+
if (definitionId === null || definitionId === undefined) {
|
|
59
|
+
throw new Error(`ensureWorkflowDefinitionForAsset: definition for asset "${assetId}" (wire hash "${wireHash}") missing after insert`);
|
|
60
|
+
}
|
|
61
|
+
await db
|
|
62
|
+
.insert(workflowDefinitionVersion)
|
|
63
|
+
.values({
|
|
64
|
+
id: generateId("workflowDefinitionVersion"),
|
|
65
|
+
definitionId,
|
|
66
|
+
version: "1",
|
|
67
|
+
})
|
|
68
|
+
.onConflictDoNothing({
|
|
69
|
+
target: [
|
|
70
|
+
workflowDefinitionVersion.definitionId,
|
|
71
|
+
workflowDefinitionVersion.version,
|
|
72
|
+
],
|
|
73
|
+
});
|
|
74
|
+
return { definitionId, created };
|
|
75
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { DBExecutor, EnqueueWorkflowRunDispatchArgs, EnqueueWorkflowRunDispatchResult, EnqueueWorkflowSignalDispatchArgs, SidecarAllocationStore, WorkflowRunDispatchStore } from "@intx/db";
|
|
2
|
+
import type { SidecarAllocationRouter } from "./ws/sidecar-handler.js";
|
|
3
|
+
type DispatchStore = Pick<WorkflowRunDispatchStore, "acknowledge" | "claimNextPending" | "enqueue" | "enqueueSignal" | "requeueUnsettled" | "scheduleRetry" | "settle">;
|
|
4
|
+
type AllocationStore = Pick<SidecarAllocationStore, "findByAnchorRunId">;
|
|
5
|
+
type DispatchRouter = Pick<SidecarAllocationRouter, "sendSignalDeliverToAllocation" | "sendWorkflowRunDispatchToAllocation">;
|
|
6
|
+
export type WorkflowDispatchAcknowledgement = {
|
|
7
|
+
readonly allocationId: string;
|
|
8
|
+
readonly anchorRunId: string;
|
|
9
|
+
readonly generation: number;
|
|
10
|
+
readonly messageId: string;
|
|
11
|
+
};
|
|
12
|
+
export type WorkflowDispatchService = {
|
|
13
|
+
enqueue(args: EnqueueWorkflowRunDispatchArgs, tx?: DBExecutor): Promise<EnqueueWorkflowRunDispatchResult>;
|
|
14
|
+
enqueueSignal(args: EnqueueWorkflowSignalDispatchArgs, tx?: DBExecutor): Promise<EnqueueWorkflowRunDispatchResult>;
|
|
15
|
+
acknowledge(args: WorkflowDispatchAcknowledgement): Promise<void>;
|
|
16
|
+
settle(anchorRunId: string, messageId: string): Promise<void>;
|
|
17
|
+
requeueForReadyAllocation(anchorRunId: string): Promise<number>;
|
|
18
|
+
reconcileNext(): Promise<boolean>;
|
|
19
|
+
reconcileUntilIdle(maxIterations?: number): Promise<number>;
|
|
20
|
+
wake(): void;
|
|
21
|
+
};
|
|
22
|
+
export type WorkflowDispatchServiceDeps = {
|
|
23
|
+
readonly dispatchStore: DispatchStore;
|
|
24
|
+
readonly allocationStore: AllocationStore;
|
|
25
|
+
readonly router: DispatchRouter;
|
|
26
|
+
/** Resolve the deployment anchor's durable routing address. */
|
|
27
|
+
readonly resolveAnchorAddress: (anchorRunId: string) => Promise<string | null>;
|
|
28
|
+
readonly leaseDurationMs?: number;
|
|
29
|
+
/** Bounds pending claims and unfinished deliveries across all drains. */
|
|
30
|
+
readonly maxConcurrentDispatches?: number;
|
|
31
|
+
readonly retryDelayMs?: (attempt: number) => number;
|
|
32
|
+
readonly now?: () => Date;
|
|
33
|
+
readonly createLeaseId?: () => string;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Drives Hub-owned workflow triggers onto provisioned sidecars. The database
|
|
37
|
+
* row is the delivery authority: websocket acceptance never deletes the raw
|
|
38
|
+
* payload, and a generation replacement requeues every row that has not been
|
|
39
|
+
* settled by the workflow-run Git claim-check.
|
|
40
|
+
*/
|
|
41
|
+
export declare function createWorkflowDispatchService({ dispatchStore, allocationStore, router, resolveAnchorAddress, leaseDurationMs, maxConcurrentDispatches, retryDelayMs, now, createLeaseId, }: WorkflowDispatchServiceDeps): WorkflowDispatchService;
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { getLogger } from "@intx/log";
|
|
2
|
+
import { base64Encode, deriveWorkflowRunId, hexEncode } from "@intx/types";
|
|
3
|
+
import { SignalDeliverFrame } from "@intx/types/sidecar";
|
|
4
|
+
import { runSidecarOperation, SidecarOperationTimeoutError, } from "./sidecar-allocation/operation.js";
|
|
5
|
+
const logger = getLogger(["hub", "workflow-dispatch"]);
|
|
6
|
+
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
|
7
|
+
const DEFAULT_MAX_CONCURRENT_DISPATCHES = 8;
|
|
8
|
+
function defaultRetryDelay(attempt) {
|
|
9
|
+
return Math.min(500 * 2 ** Math.min(attempt, 6), 30_000);
|
|
10
|
+
}
|
|
11
|
+
function randomLeaseId() {
|
|
12
|
+
return `dispatch_lease_${hexEncode(crypto.getRandomValues(new Uint8Array(16)))}`;
|
|
13
|
+
}
|
|
14
|
+
function targetForReadyAllocation(allocation) {
|
|
15
|
+
if (allocation.status !== "allocated" ||
|
|
16
|
+
allocation.ensureAcceptedGeneration !== allocation.generation ||
|
|
17
|
+
allocation.initializationLeaseId !== undefined ||
|
|
18
|
+
allocation.connectDeadline !== undefined) {
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
return {
|
|
22
|
+
allocationId: allocation.id,
|
|
23
|
+
generation: allocation.generation,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Drives Hub-owned workflow triggers onto provisioned sidecars. The database
|
|
28
|
+
* row is the delivery authority: websocket acceptance never deletes the raw
|
|
29
|
+
* payload, and a generation replacement requeues every row that has not been
|
|
30
|
+
* settled by the workflow-run Git claim-check.
|
|
31
|
+
*/
|
|
32
|
+
export function createWorkflowDispatchService({ dispatchStore, allocationStore, router, resolveAnchorAddress, leaseDurationMs = DEFAULT_LEASE_DURATION_MS, maxConcurrentDispatches = DEFAULT_MAX_CONCURRENT_DISPATCHES, retryDelayMs = defaultRetryDelay, now = () => new Date(), createLeaseId = randomLeaseId, }) {
|
|
33
|
+
if (!Number.isSafeInteger(leaseDurationMs) || leaseDurationMs <= 0) {
|
|
34
|
+
throw new Error("leaseDurationMs must be a positive integer");
|
|
35
|
+
}
|
|
36
|
+
if (!Number.isSafeInteger(maxConcurrentDispatches) ||
|
|
37
|
+
maxConcurrentDispatches <= 0) {
|
|
38
|
+
throw new Error("maxConcurrentDispatches must be a positive integer");
|
|
39
|
+
}
|
|
40
|
+
let drainPromise = null;
|
|
41
|
+
let admittedDispatches = 0;
|
|
42
|
+
const activeDispatches = new Set();
|
|
43
|
+
function retryAt(attempt) {
|
|
44
|
+
return new Date(now().getTime() + retryDelayMs(attempt));
|
|
45
|
+
}
|
|
46
|
+
async function retry(dispatch, leaseId, code, message) {
|
|
47
|
+
await dispatchStore.scheduleRetry({
|
|
48
|
+
dispatchId: dispatch.id,
|
|
49
|
+
nextAttemptAt: retryAt(dispatch.attemptCount),
|
|
50
|
+
code,
|
|
51
|
+
message,
|
|
52
|
+
expectedLeaseId: leaseId,
|
|
53
|
+
now: now(),
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
async function reconcileNext() {
|
|
57
|
+
if (admittedDispatches >= maxConcurrentDispatches)
|
|
58
|
+
return false;
|
|
59
|
+
admittedDispatches += 1;
|
|
60
|
+
let delivery;
|
|
61
|
+
let activeDispatchId;
|
|
62
|
+
try {
|
|
63
|
+
const leaseId = createLeaseId();
|
|
64
|
+
const claimStartedAt = performance.now();
|
|
65
|
+
const dispatch = await dispatchStore.claimNextPending({
|
|
66
|
+
leaseId,
|
|
67
|
+
leaseDurationMs,
|
|
68
|
+
excludedDispatchIds: [...activeDispatches],
|
|
69
|
+
});
|
|
70
|
+
if (dispatch === null)
|
|
71
|
+
return false;
|
|
72
|
+
// A concurrent claim can outlive both its exclusion snapshot and the old
|
|
73
|
+
// lease. Do not start another delivery while its original I/O is pending.
|
|
74
|
+
if (activeDispatches.has(dispatch.id))
|
|
75
|
+
return true;
|
|
76
|
+
const remaining = leaseDurationMs - (performance.now() - claimStartedAt);
|
|
77
|
+
if (remaining <= 0)
|
|
78
|
+
return true;
|
|
79
|
+
activeDispatchId = dispatch.id;
|
|
80
|
+
activeDispatches.add(dispatch.id);
|
|
81
|
+
try {
|
|
82
|
+
await runSidecarOperation("Workflow dispatch", Math.ceil(remaining), (signal) => {
|
|
83
|
+
delivery = deliver(dispatch, leaseId, signal);
|
|
84
|
+
return delivery;
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
if (!(error instanceof SidecarOperationTimeoutError))
|
|
89
|
+
throw error;
|
|
90
|
+
// The durable lease expires independently. Do not start another database
|
|
91
|
+
// write here: a stuck retry write must not occupy the freed drain either.
|
|
92
|
+
logger.warn `Dispatch ${dispatch.id} stopped at its delivery deadline`;
|
|
93
|
+
}
|
|
94
|
+
return true;
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
const release = () => {
|
|
98
|
+
admittedDispatches -= 1;
|
|
99
|
+
if (activeDispatchId !== undefined)
|
|
100
|
+
activeDispatches.delete(activeDispatchId);
|
|
101
|
+
};
|
|
102
|
+
// Timeouts free the drain, but the underlying I/O retains its admission
|
|
103
|
+
// reservation and dispatch exclusion until it actually settles.
|
|
104
|
+
if (delivery === undefined)
|
|
105
|
+
release();
|
|
106
|
+
else
|
|
107
|
+
void delivery.then(release, release);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
async function deliver(dispatch, leaseId, signal) {
|
|
111
|
+
const allocation = await allocationStore.findByAnchorRunId(dispatch.anchorRunId);
|
|
112
|
+
signal.throwIfAborted();
|
|
113
|
+
if (allocation === null) {
|
|
114
|
+
await retry(dispatch, leaseId, "allocation_missing", `No sidecar allocation exists for workflow anchor ${dispatch.anchorRunId}`);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
const target = targetForReadyAllocation(allocation);
|
|
118
|
+
if (target === null) {
|
|
119
|
+
await retry(dispatch, leaseId, "allocation_not_ready", `Sidecar allocation ${allocation.id} is not ready for delivery`);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
const agentAddress = await resolveAnchorAddress(dispatch.anchorRunId);
|
|
123
|
+
signal.throwIfAborted();
|
|
124
|
+
if (agentAddress === null) {
|
|
125
|
+
await retry(dispatch, leaseId, "anchor_address_missing", `Workflow anchor ${dispatch.anchorRunId} has no routing address`);
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
try {
|
|
129
|
+
if (dispatch.kind === "signal") {
|
|
130
|
+
const frame = SignalDeliverFrame.assert(JSON.parse(new TextDecoder().decode(dispatch.rawMessage)));
|
|
131
|
+
await router.sendSignalDeliverToAllocation(target, {
|
|
132
|
+
agentAddress: frame.agentAddress,
|
|
133
|
+
runId: frame.runId,
|
|
134
|
+
signalName: frame.signalName,
|
|
135
|
+
signalId: frame.signalId,
|
|
136
|
+
payload: frame.payload,
|
|
137
|
+
}, signal);
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
// A deliverable mail dispatch always carries the sender persisted at
|
|
141
|
+
// enqueue (the workflow_run_dispatch mail-sender check enforces it).
|
|
142
|
+
// A null here means the row bypassed that invariant, so fail loudly
|
|
143
|
+
// rather than deliver with no authenticated sender or fall back to the
|
|
144
|
+
// MIME From.
|
|
145
|
+
if (dispatch.senderAddress === null) {
|
|
146
|
+
throw new Error(`mail dispatch ${dispatch.id} has no persisted authenticated sender`);
|
|
147
|
+
}
|
|
148
|
+
await router.sendWorkflowRunDispatchToAllocation(target, agentAddress, deriveWorkflowRunId(agentAddress), dispatch.stepGrants, base64Encode(dispatch.rawMessage), dispatch.senderAddress, dispatch.messageId, signal);
|
|
149
|
+
}
|
|
150
|
+
// Keep the delivery lease until the sidecar acknowledges its durable
|
|
151
|
+
// inbox write. If that ack never arrives, lease expiry makes the same
|
|
152
|
+
// immutable payload claimable again.
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
signal.throwIfAborted();
|
|
156
|
+
await retry(dispatch, leaseId, "dispatch_unroutable", error instanceof Error ? error.message : String(error));
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
async function reconcileUntilIdle(maxIterations = 100) {
|
|
160
|
+
let reconciled = 0;
|
|
161
|
+
while (reconciled < maxIterations && (await reconcileNext())) {
|
|
162
|
+
reconciled += 1;
|
|
163
|
+
}
|
|
164
|
+
return reconciled;
|
|
165
|
+
}
|
|
166
|
+
function wake() {
|
|
167
|
+
if (drainPromise !== null)
|
|
168
|
+
return;
|
|
169
|
+
drainPromise = Promise.resolve()
|
|
170
|
+
.then(async () => {
|
|
171
|
+
await reconcileUntilIdle();
|
|
172
|
+
})
|
|
173
|
+
.catch((error) => {
|
|
174
|
+
logger.error `Workflow dispatch reconciliation failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
175
|
+
})
|
|
176
|
+
.finally(() => {
|
|
177
|
+
drainPromise = null;
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
return {
|
|
181
|
+
async enqueue(args, tx) {
|
|
182
|
+
const result = await dispatchStore.enqueue(args, tx);
|
|
183
|
+
wake();
|
|
184
|
+
return result;
|
|
185
|
+
},
|
|
186
|
+
async enqueueSignal(args, tx) {
|
|
187
|
+
const result = await dispatchStore.enqueueSignal(args, tx);
|
|
188
|
+
wake();
|
|
189
|
+
return result;
|
|
190
|
+
},
|
|
191
|
+
async acknowledge(args) {
|
|
192
|
+
await dispatchStore.acknowledge({
|
|
193
|
+
...args,
|
|
194
|
+
now: now(),
|
|
195
|
+
});
|
|
196
|
+
},
|
|
197
|
+
async settle(anchorRunId, messageId) {
|
|
198
|
+
await dispatchStore.settle(anchorRunId, messageId, now());
|
|
199
|
+
},
|
|
200
|
+
async requeueForReadyAllocation(anchorRunId) {
|
|
201
|
+
const count = await dispatchStore.requeueUnsettled(anchorRunId);
|
|
202
|
+
wake();
|
|
203
|
+
return count;
|
|
204
|
+
},
|
|
205
|
+
reconcileNext,
|
|
206
|
+
reconcileUntilIdle,
|
|
207
|
+
wake,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { CommittedReads } from "./repo-store/index.js";
|
|
2
|
+
export type ConsumedWorkflowDispatch = {
|
|
3
|
+
readonly messageId: string;
|
|
4
|
+
readonly address: string;
|
|
5
|
+
readonly rejection?: {
|
|
6
|
+
readonly code: string;
|
|
7
|
+
readonly message: string;
|
|
8
|
+
};
|
|
9
|
+
};
|
|
10
|
+
export type ReceivedWorkflowSignal = {
|
|
11
|
+
readonly runId: string;
|
|
12
|
+
readonly signalId: string;
|
|
13
|
+
};
|
|
14
|
+
export type AcceptedWorkflowDispatch = {
|
|
15
|
+
readonly runId: string;
|
|
16
|
+
readonly messageId: string;
|
|
17
|
+
readonly kind: "mail" | "signal";
|
|
18
|
+
};
|
|
19
|
+
/** Enumerate dispatches durably accepted by one live or sealed run log. */
|
|
20
|
+
export declare function listAcceptedWorkflowDispatches(reads: CommittedReads, runId: string, targetMessageIds?: ReadonlySet<string>): Promise<AcceptedWorkflowDispatch[]>;
|
|
21
|
+
/** Enumerate durable SignalReceived ids from one live or sealed run log. */
|
|
22
|
+
export declare function listReceivedWorkflowSignals(reads: CommittedReads, runId: string): Promise<ReceivedWorkflowSignal[]>;
|
|
23
|
+
/**
|
|
24
|
+
* Enumerate the workflow-run claim-check's retained consumed index. The kind
|
|
25
|
+
* validator guarantees this tree shape before the ref advances; the runtime
|
|
26
|
+
* validator here keeps the Git-to-database projection fail-closed if that
|
|
27
|
+
* invariant is ever violated.
|
|
28
|
+
*/
|
|
29
|
+
export declare function listConsumedWorkflowDispatches(reads: CommittedReads): Promise<ConsumedWorkflowDispatch[]>;
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { type } from "arktype";
|
|
2
|
+
import { splitCombinedEventLog, WORKFLOW_RUN_EVENTS_FILE, } from "./workflow-run-event-log.js";
|
|
3
|
+
import { WORKFLOW_RUN_ADDRESSES_PREFIX, WORKFLOW_RUN_CONSUMED_DIR, WORKFLOW_RUN_EVENTS_DIR, WORKFLOW_RUN_RUNS_PREFIX, } from "./workflow-run-kind.js";
|
|
4
|
+
const ConsumedEnvelope = type({
|
|
5
|
+
messageId: "string",
|
|
6
|
+
address: "string",
|
|
7
|
+
"rejection?": {
|
|
8
|
+
code: "string > 0",
|
|
9
|
+
message: "string > 0",
|
|
10
|
+
},
|
|
11
|
+
"+": "ignore",
|
|
12
|
+
});
|
|
13
|
+
const SignalReceived = type({
|
|
14
|
+
type: "'SignalReceived'",
|
|
15
|
+
signalId: "string",
|
|
16
|
+
"+": "ignore",
|
|
17
|
+
});
|
|
18
|
+
const RunStarted = type({
|
|
19
|
+
type: "'RunStarted'",
|
|
20
|
+
"consumedMessageId?": "string",
|
|
21
|
+
"+": "ignore",
|
|
22
|
+
});
|
|
23
|
+
function projectAcceptedDispatch(raw, path) {
|
|
24
|
+
let decoded;
|
|
25
|
+
try {
|
|
26
|
+
decoded = JSON.parse(raw);
|
|
27
|
+
}
|
|
28
|
+
catch (cause) {
|
|
29
|
+
throw new Error(`workflow_dispatch_event_invalid_json: ${path}`, {
|
|
30
|
+
cause,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
if (typeof decoded !== "object" || decoded === null || !("type" in decoded)) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
if (decoded.type === "SignalReceived") {
|
|
37
|
+
const signal = SignalReceived(decoded);
|
|
38
|
+
if (signal instanceof type.errors) {
|
|
39
|
+
throw new Error(`workflow_dispatch_signal_invalid: ${path}: ${signal.summary}`);
|
|
40
|
+
}
|
|
41
|
+
return { kind: "signal", messageId: signal.signalId };
|
|
42
|
+
}
|
|
43
|
+
if (decoded.type === "RunStarted") {
|
|
44
|
+
const started = RunStarted(decoded);
|
|
45
|
+
if (started instanceof type.errors) {
|
|
46
|
+
throw new Error(`workflow_dispatch_run_started_invalid: ${path}: ${started.summary}`);
|
|
47
|
+
}
|
|
48
|
+
return started.consumedMessageId === undefined
|
|
49
|
+
? null
|
|
50
|
+
: { kind: "mail", messageId: started.consumedMessageId };
|
|
51
|
+
}
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
/** Enumerate dispatches durably accepted by one live or sealed run log. */
|
|
55
|
+
export async function listAcceptedWorkflowDispatches(reads, runId, targetMessageIds) {
|
|
56
|
+
if (targetMessageIds?.size === 0)
|
|
57
|
+
return [];
|
|
58
|
+
const accepted = [];
|
|
59
|
+
const remaining = targetMessageIds === undefined ? undefined : new Set(targetMessageIds);
|
|
60
|
+
const accept = (dispatch) => {
|
|
61
|
+
if (dispatch === null)
|
|
62
|
+
return;
|
|
63
|
+
if (remaining !== undefined && !remaining.delete(dispatch.messageId))
|
|
64
|
+
return;
|
|
65
|
+
accepted.push({ runId, ...dispatch });
|
|
66
|
+
};
|
|
67
|
+
const runPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`;
|
|
68
|
+
const children = await reads.listDir(runPath);
|
|
69
|
+
const combined = children.find((entry) => entry.type === "blob" && entry.name === WORKFLOW_RUN_EVENTS_FILE);
|
|
70
|
+
if (combined !== undefined) {
|
|
71
|
+
const raw = new TextDecoder().decode(await reads.readBlobByOid(combined.oid));
|
|
72
|
+
const lines = splitCombinedEventLog(raw);
|
|
73
|
+
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
74
|
+
const line = lines[index];
|
|
75
|
+
if (line === undefined)
|
|
76
|
+
continue;
|
|
77
|
+
accept(projectAcceptedDispatch(line, `${runPath}/${WORKFLOW_RUN_EVENTS_FILE}:${String(index + 1)}`));
|
|
78
|
+
if (remaining?.size === 0)
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
return accepted.reverse();
|
|
82
|
+
}
|
|
83
|
+
const eventsPath = `${runPath}/${WORKFLOW_RUN_EVENTS_DIR}`;
|
|
84
|
+
const eventEntries = (await reads.listDir(eventsPath))
|
|
85
|
+
.filter((entry) => entry.type === "blob" && entry.name.endsWith(".json"))
|
|
86
|
+
.sort((left, right) => right.name.localeCompare(left.name, undefined, {
|
|
87
|
+
numeric: true,
|
|
88
|
+
}));
|
|
89
|
+
for (const eventEntry of eventEntries) {
|
|
90
|
+
const raw = new TextDecoder().decode(await reads.readBlobByOid(eventEntry.oid));
|
|
91
|
+
accept(projectAcceptedDispatch(raw, `${eventsPath}/${eventEntry.name}`));
|
|
92
|
+
if (remaining?.size === 0)
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
return accepted.reverse();
|
|
96
|
+
}
|
|
97
|
+
/** Enumerate durable SignalReceived ids from one live or sealed run log. */
|
|
98
|
+
export async function listReceivedWorkflowSignals(reads, runId) {
|
|
99
|
+
return (await listAcceptedWorkflowDispatches(reads, runId))
|
|
100
|
+
.filter((dispatch) => dispatch.kind === "signal")
|
|
101
|
+
.map((dispatch) => ({ runId, signalId: dispatch.messageId }));
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Enumerate the workflow-run claim-check's retained consumed index. The kind
|
|
105
|
+
* validator guarantees this tree shape before the ref advances; the runtime
|
|
106
|
+
* validator here keeps the Git-to-database projection fail-closed if that
|
|
107
|
+
* invariant is ever violated.
|
|
108
|
+
*/
|
|
109
|
+
export async function listConsumedWorkflowDispatches(reads) {
|
|
110
|
+
const consumed = [];
|
|
111
|
+
for (const addressEntry of await reads.listDir(WORKFLOW_RUN_ADDRESSES_PREFIX)) {
|
|
112
|
+
if (addressEntry.type !== "tree")
|
|
113
|
+
continue;
|
|
114
|
+
const consumedPath = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressEntry.name}/${WORKFLOW_RUN_CONSUMED_DIR}`;
|
|
115
|
+
for (const entry of await reads.listDir(consumedPath)) {
|
|
116
|
+
if (entry.type !== "blob" || !entry.name.endsWith(".json"))
|
|
117
|
+
continue;
|
|
118
|
+
const raw = await reads.readBlobByOid(entry.oid);
|
|
119
|
+
let decoded;
|
|
120
|
+
try {
|
|
121
|
+
decoded = JSON.parse(new TextDecoder().decode(raw));
|
|
122
|
+
}
|
|
123
|
+
catch (cause) {
|
|
124
|
+
throw new Error(`workflow_dispatch_consumed_invalid_json: ${consumedPath}/${entry.name}`, { cause });
|
|
125
|
+
}
|
|
126
|
+
const envelope = ConsumedEnvelope(decoded);
|
|
127
|
+
if (envelope instanceof type.errors) {
|
|
128
|
+
throw new Error(`workflow_dispatch_consumed_invalid: ${consumedPath}/${entry.name}: ${envelope.summary}`);
|
|
129
|
+
}
|
|
130
|
+
consumed.push({
|
|
131
|
+
messageId: envelope.messageId,
|
|
132
|
+
address: envelope.address,
|
|
133
|
+
...(envelope.rejection !== undefined
|
|
134
|
+
? { rejection: envelope.rejection }
|
|
135
|
+
: {}),
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return consumed;
|
|
140
|
+
}
|