@loomwork/dispatcher 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +8 -0
- package/README.md +83 -2
- package/dist/LocalDispatcher.d.ts +56 -0
- package/dist/LocalDispatcher.d.ts.map +1 -0
- package/dist/LocalDispatcher.js +104 -0
- package/dist/LocalDispatcher.js.map +1 -0
- package/dist/PlacementDispatcher.d.ts +45 -0
- package/dist/PlacementDispatcher.d.ts.map +1 -0
- package/dist/PlacementDispatcher.js +95 -0
- package/dist/PlacementDispatcher.js.map +1 -0
- package/dist/SupervisingDispatcher.d.ts +57 -0
- package/dist/SupervisingDispatcher.d.ts.map +1 -0
- package/dist/SupervisingDispatcher.js +119 -0
- package/dist/SupervisingDispatcher.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/placement.d.ts +5 -0
- package/dist/placement.d.ts.map +1 -0
- package/dist/placement.js +28 -0
- package/dist/placement.js.map +1 -0
- package/dist/types.d.ts +69 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +21 -0
- package/dist/types.js.map +1 -0
- package/package.json +53 -3
- package/src/LocalDispatcher.ts +141 -0
- package/src/PlacementDispatcher.ts +124 -0
- package/src/SupervisingDispatcher.ts +160 -0
- package/src/index.ts +27 -0
- package/src/placement.ts +26 -0
- package/src/types.ts +77 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { type ExecutionPackage, type ExecutionResult } from "@loomwork/worker-runtime";
|
|
2
|
+
import type { DispatchMetadata, DispatcherHealth, ExecutionDispatcher } from "./types";
|
|
3
|
+
export interface CircuitPolicy {
|
|
4
|
+
/** Consecutive dispatch failures before the circuit opens. */
|
|
5
|
+
threshold: number;
|
|
6
|
+
/** How long the circuit stays open before a half-open trial, in ms. */
|
|
7
|
+
cooldownMs: number;
|
|
8
|
+
/** Circuit key for a package. Default: `pkg.flowId` (per-flow breaker). */
|
|
9
|
+
key?: (pkg: ExecutionPackage) => string;
|
|
10
|
+
}
|
|
11
|
+
export interface SupervisionPolicy {
|
|
12
|
+
/** Restart a *throwing* dispatch up to N extra times (default 0). A returned
|
|
13
|
+
* "failed" result is a normal outcome, not a restartable error — and neither is a
|
|
14
|
+
* refusal because another holder owns the execution ({@link isSupersededRefusal},
|
|
15
|
+
* L20): it is rethrown at once, without a restart and without counting against the
|
|
16
|
+
* circuit, because restarting it would fence out the holder that won. */
|
|
17
|
+
maxRestarts?: number;
|
|
18
|
+
/** Base backoff between restarts, in ms (default 0). */
|
|
19
|
+
backoffMs?: number;
|
|
20
|
+
/** Exponential growth factor for the backoff (default 2). */
|
|
21
|
+
backoffFactor?: number;
|
|
22
|
+
/** Optional circuit breaker — fail fast when a target is consistently down. */
|
|
23
|
+
circuit?: CircuitPolicy;
|
|
24
|
+
}
|
|
25
|
+
export declare class CircuitOpenError extends Error {
|
|
26
|
+
readonly key: string;
|
|
27
|
+
constructor(key: string);
|
|
28
|
+
}
|
|
29
|
+
export interface SupervisingDispatcherOptions {
|
|
30
|
+
policy?: SupervisionPolicy;
|
|
31
|
+
now?: () => number;
|
|
32
|
+
sleep?: (ms: number) => Promise<void>;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Fault supervision (Loom 2.0 K4). A composite {@link ExecutionDispatcher} that wraps
|
|
36
|
+
* a child with an explicit restart/backoff policy and a per-key **circuit breaker**:
|
|
37
|
+
* after `threshold` consecutive dispatch failures the circuit opens and further
|
|
38
|
+
* dispatches fail fast (CircuitOpenError) until a `cooldownMs` half-open trial — so a
|
|
39
|
+
* flaky model/tool/worker is not hammered. Promotes the runtime's ad-hoc
|
|
40
|
+
* retry/redispatch into a named supervision seam; composes over LocalDispatcher /
|
|
41
|
+
* PlacementDispatcher unchanged.
|
|
42
|
+
*/
|
|
43
|
+
export declare class SupervisingDispatcher implements ExecutionDispatcher {
|
|
44
|
+
private readonly child;
|
|
45
|
+
private readonly circuits;
|
|
46
|
+
private readonly policy;
|
|
47
|
+
private readonly now;
|
|
48
|
+
private readonly sleep;
|
|
49
|
+
constructor(child: ExecutionDispatcher, options?: SupervisingDispatcherOptions);
|
|
50
|
+
dispatch(pkg: ExecutionPackage, meta?: DispatchMetadata): Promise<ExecutionResult>;
|
|
51
|
+
cancel(executionId: string): Promise<void>;
|
|
52
|
+
health(): Promise<DispatcherHealth>;
|
|
53
|
+
private guardCircuit;
|
|
54
|
+
private recordSuccess;
|
|
55
|
+
private recordFailure;
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=SupervisingDispatcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SupervisingDispatcher.d.ts","sourceRoot":"","sources":["../src/SupervisingDispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EAErB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAEvF,MAAM,WAAW,aAAa;IAC5B,8DAA8D;IAC9D,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB,2EAA2E;IAC3E,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,MAAM,CAAC;CACzC;AAED,MAAM,WAAW,iBAAiB;IAChC;;;;8EAI0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBACT,GAAG,EAAE,MAAM;CAKxB;AAOD,MAAM,WAAW,4BAA4B;IAC3C,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED;;;;;;;;GAQG;AACH,qBAAa,qBAAsB,YAAW,mBAAmB;IAO7D,OAAO,CAAC,QAAQ,CAAC,KAAK;IANxB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA8B;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;gBAGnC,KAAK,EAAE,mBAAmB,EAC3C,OAAO,GAAE,4BAAiC;IAOtC,QAAQ,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;IA8BlF,MAAM,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1C,MAAM,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAczC,OAAO,CAAC,YAAY;IAYpB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,aAAa;CAUtB"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SupervisingDispatcher = exports.CircuitOpenError = void 0;
|
|
4
|
+
const worker_runtime_1 = require("@loomwork/worker-runtime");
|
|
5
|
+
class CircuitOpenError extends Error {
|
|
6
|
+
key;
|
|
7
|
+
constructor(key) {
|
|
8
|
+
super(`Circuit open for "${key}" — failing fast`);
|
|
9
|
+
this.name = "CircuitOpenError";
|
|
10
|
+
this.key = key;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
exports.CircuitOpenError = CircuitOpenError;
|
|
14
|
+
/**
|
|
15
|
+
* Fault supervision (Loom 2.0 K4). A composite {@link ExecutionDispatcher} that wraps
|
|
16
|
+
* a child with an explicit restart/backoff policy and a per-key **circuit breaker**:
|
|
17
|
+
* after `threshold` consecutive dispatch failures the circuit opens and further
|
|
18
|
+
* dispatches fail fast (CircuitOpenError) until a `cooldownMs` half-open trial — so a
|
|
19
|
+
* flaky model/tool/worker is not hammered. Promotes the runtime's ad-hoc
|
|
20
|
+
* retry/redispatch into a named supervision seam; composes over LocalDispatcher /
|
|
21
|
+
* PlacementDispatcher unchanged.
|
|
22
|
+
*/
|
|
23
|
+
class SupervisingDispatcher {
|
|
24
|
+
child;
|
|
25
|
+
circuits = new Map();
|
|
26
|
+
policy;
|
|
27
|
+
now;
|
|
28
|
+
sleep;
|
|
29
|
+
constructor(child, options = {}) {
|
|
30
|
+
this.child = child;
|
|
31
|
+
this.policy = options.policy ?? {};
|
|
32
|
+
this.now = options.now ?? (() => Date.now());
|
|
33
|
+
this.sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
34
|
+
}
|
|
35
|
+
async dispatch(pkg, meta) {
|
|
36
|
+
const key = this.policy.circuit?.key?.(pkg) ?? pkg.flowId;
|
|
37
|
+
this.guardCircuit(key); // throws CircuitOpenError if open within cooldown
|
|
38
|
+
const maxRestarts = Math.max(0, this.policy.maxRestarts ?? 0);
|
|
39
|
+
const base = this.policy.backoffMs ?? 0;
|
|
40
|
+
const factor = this.policy.backoffFactor ?? 2;
|
|
41
|
+
let lastErr;
|
|
42
|
+
for (let attempt = 0; attempt <= maxRestarts; attempt++) {
|
|
43
|
+
try {
|
|
44
|
+
const result = await this.child.dispatch(pkg, meta);
|
|
45
|
+
this.recordSuccess(key);
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
catch (err) {
|
|
49
|
+
// L20 — superseded, not failed: not restartable, and not the target's fault. But an
|
|
50
|
+
// EARLIER restart of this dispatch that failed genuinely still counts against the
|
|
51
|
+
// circuit — the refusal ends the dispatch, it does not erase what preceded it.
|
|
52
|
+
if ((0, worker_runtime_1.isSupersededRefusal)(err, pkg.executionId)) {
|
|
53
|
+
if (attempt > 0)
|
|
54
|
+
this.recordFailure(key);
|
|
55
|
+
throw err;
|
|
56
|
+
}
|
|
57
|
+
lastErr = err;
|
|
58
|
+
if (attempt < maxRestarts)
|
|
59
|
+
await this.sleep(base * factor ** attempt);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
this.recordFailure(key);
|
|
63
|
+
throw lastErr;
|
|
64
|
+
}
|
|
65
|
+
async cancel(executionId) {
|
|
66
|
+
return this.child.cancel(executionId);
|
|
67
|
+
}
|
|
68
|
+
async health() {
|
|
69
|
+
const h = await this.child.health();
|
|
70
|
+
const open = [...this.circuits.entries()]
|
|
71
|
+
.filter(([, c]) => c.openedAt !== null)
|
|
72
|
+
.map(([k]) => k);
|
|
73
|
+
return {
|
|
74
|
+
healthy: h.healthy && open.length === 0,
|
|
75
|
+
inFlight: h.inFlight,
|
|
76
|
+
...(open.length > 0
|
|
77
|
+
? { detail: `circuits-open: ${open.join(",")}` }
|
|
78
|
+
: h.detail !== undefined && { detail: h.detail }),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
guardCircuit(key) {
|
|
82
|
+
const c = this.circuits.get(key);
|
|
83
|
+
if (c === undefined || c.openedAt === null)
|
|
84
|
+
return;
|
|
85
|
+
const cooldown = this.policy.circuit?.cooldownMs ?? 0;
|
|
86
|
+
if (this.now() - c.openedAt < cooldown) {
|
|
87
|
+
throw new CircuitOpenError(key);
|
|
88
|
+
}
|
|
89
|
+
// Cooldown elapsed → half-open: allow one trial. Keep the failure count so a
|
|
90
|
+
// fresh failure re-opens immediately; a success resets it.
|
|
91
|
+
c.openedAt = null;
|
|
92
|
+
}
|
|
93
|
+
recordSuccess(key) {
|
|
94
|
+
// DELETE rather than reset. A circuit with `failures: 0, openedAt: null` is
|
|
95
|
+
// indistinguishable from an absent one everywhere it is read — `guardCircuit`
|
|
96
|
+
// returns on either, `recordFailure` defaults to exactly that shape, and
|
|
97
|
+
// `health()` counts only open circuits — so dropping it is observationally
|
|
98
|
+
// identical and bounds the map to circuits actually accumulating failures.
|
|
99
|
+
//
|
|
100
|
+
// Without this the map only grows. The default key is `flowId` (bounded by
|
|
101
|
+
// deployed flows), but `policy.circuit.key` receives the whole ExecutionPackage,
|
|
102
|
+
// so keying per tenant — or per execution, which reads as natural — turned a
|
|
103
|
+
// supervisor into a leak for the lifetime of the process.
|
|
104
|
+
this.circuits.delete(key);
|
|
105
|
+
}
|
|
106
|
+
recordFailure(key) {
|
|
107
|
+
const circuit = this.policy.circuit;
|
|
108
|
+
if (circuit === undefined)
|
|
109
|
+
return;
|
|
110
|
+
const c = this.circuits.get(key) ?? { failures: 0, openedAt: null };
|
|
111
|
+
c.failures += 1;
|
|
112
|
+
if (c.failures >= circuit.threshold) {
|
|
113
|
+
c.openedAt = this.now();
|
|
114
|
+
}
|
|
115
|
+
this.circuits.set(key, c);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
exports.SupervisingDispatcher = SupervisingDispatcher;
|
|
119
|
+
//# sourceMappingURL=SupervisingDispatcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SupervisingDispatcher.js","sourceRoot":"","sources":["../src/SupervisingDispatcher.ts"],"names":[],"mappings":";;;AAAA,6DAIkC;AA2BlC,MAAa,gBAAiB,SAAQ,KAAK;IAChC,GAAG,CAAS;IACrB,YAAY,GAAW;QACrB,KAAK,CAAC,qBAAqB,GAAG,kBAAkB,CAAC,CAAC;QAClD,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAPD,4CAOC;AAaD;;;;;;;;GAQG;AACH,MAAa,qBAAqB;IAOb;IANF,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IACtC,MAAM,CAAoB;IAC1B,GAAG,CAAe;IAClB,KAAK,CAAgC;IAEtD,YACmB,KAA0B,EAC3C,UAAwC,EAAE;QADzB,UAAK,GAAL,KAAK,CAAqB;QAG3C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,GAAqB,EAAE,IAAuB;QAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC;QAC1D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,kDAAkD;QAE1E,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;QAE9C,IAAI,OAAgB,CAAC;QACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACpD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACxB,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,oFAAoF;gBACpF,kFAAkF;gBAClF,+EAA+E;gBAC/E,IAAI,IAAA,oCAAmB,EAAC,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC9C,IAAI,OAAO,GAAG,CAAC;wBAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;oBACzC,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,OAAO,GAAG,GAAG,CAAC;gBACd,IAAI,OAAO,GAAG,WAAW;oBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,IAAI,OAAO,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,OAAO,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,WAAmB;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;aACtC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC;aACtC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACnB,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACjB,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE;gBAChD,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,GAAG,QAAQ,EAAE,CAAC;YACvC,MAAM,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QACD,6EAA6E;QAC7E,2DAA2D;QAC3D,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,4EAA4E;QAC5E,8EAA8E;QAC9E,yEAAyE;QACzE,2EAA2E;QAC3E,2EAA2E;QAC3E,EAAE;QACF,2EAA2E;QAC3E,iFAAiF;QACjF,6EAA6E;QAC7E,0DAA0D;QAC1D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;QACpC,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACpE,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5B,CAAC;CACF;AAnGD,sDAmGC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export type { DispatchMetadata, DispatcherHealth, ExecutionDispatcher, PlacementPolicy, } from "./types";
|
|
2
|
+
export { DISPATCHER_SPI_VERSION } from "./types";
|
|
3
|
+
export { LocalDispatcher, type LocalDispatcherOptions } from "./LocalDispatcher";
|
|
4
|
+
export { PlacementDispatcher, type PlacementDispatcherOptions, } from "./PlacementDispatcher";
|
|
5
|
+
export { dispatchMetadataFromPolicy, defaultPlacementPolicy } from "./placement";
|
|
6
|
+
export { SupervisingDispatcher, CircuitOpenError, type SupervisionPolicy, type CircuitPolicy, type SupervisingDispatcherOptions, } from "./SupervisingDispatcher";
|
|
7
|
+
export type { ExecutionOutcome, ExecutionPackage, ExecutionResult, } from "@loomwork/worker-runtime";
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,GAChB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EACL,mBAAmB,EACnB,KAAK,0BAA0B,GAChC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,0BAA0B,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EACL,qBAAqB,EACrB,gBAAgB,EAChB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,4BAA4B,GAClC,MAAM,yBAAyB,CAAC;AAGjC,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,GAChB,MAAM,0BAA0B,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CircuitOpenError = exports.SupervisingDispatcher = exports.defaultPlacementPolicy = exports.dispatchMetadataFromPolicy = exports.PlacementDispatcher = exports.LocalDispatcher = exports.DISPATCHER_SPI_VERSION = void 0;
|
|
4
|
+
var types_1 = require("./types");
|
|
5
|
+
Object.defineProperty(exports, "DISPATCHER_SPI_VERSION", { enumerable: true, get: function () { return types_1.DISPATCHER_SPI_VERSION; } });
|
|
6
|
+
var LocalDispatcher_1 = require("./LocalDispatcher");
|
|
7
|
+
Object.defineProperty(exports, "LocalDispatcher", { enumerable: true, get: function () { return LocalDispatcher_1.LocalDispatcher; } });
|
|
8
|
+
var PlacementDispatcher_1 = require("./PlacementDispatcher");
|
|
9
|
+
Object.defineProperty(exports, "PlacementDispatcher", { enumerable: true, get: function () { return PlacementDispatcher_1.PlacementDispatcher; } });
|
|
10
|
+
var placement_1 = require("./placement");
|
|
11
|
+
Object.defineProperty(exports, "dispatchMetadataFromPolicy", { enumerable: true, get: function () { return placement_1.dispatchMetadataFromPolicy; } });
|
|
12
|
+
Object.defineProperty(exports, "defaultPlacementPolicy", { enumerable: true, get: function () { return placement_1.defaultPlacementPolicy; } });
|
|
13
|
+
var SupervisingDispatcher_1 = require("./SupervisingDispatcher");
|
|
14
|
+
Object.defineProperty(exports, "SupervisingDispatcher", { enumerable: true, get: function () { return SupervisingDispatcher_1.SupervisingDispatcher; } });
|
|
15
|
+
Object.defineProperty(exports, "CircuitOpenError", { enumerable: true, get: function () { return SupervisingDispatcher_1.CircuitOpenError; } });
|
|
16
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAMA,iCAAiD;AAAxC,+GAAA,sBAAsB,OAAA;AAC/B,qDAAiF;AAAxE,kHAAA,eAAe,OAAA;AACxB,6DAG+B;AAF7B,0HAAA,mBAAmB,OAAA;AAGrB,yCAAiF;AAAxE,uHAAA,0BAA0B,OAAA;AAAE,mHAAA,sBAAsB,OAAA;AAC3D,iEAMiC;AAL/B,8HAAA,qBAAqB,OAAA;AACrB,yHAAA,gBAAgB,OAAA"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ExecutionPolicy } from "@loomwork/core";
|
|
2
|
+
import type { DispatchMetadata, PlacementPolicy } from "./types";
|
|
3
|
+
export declare function dispatchMetadataFromPolicy(policy: ExecutionPolicy): DispatchMetadata;
|
|
4
|
+
export declare function defaultPlacementPolicy(defaultTarget: string): PlacementPolicy;
|
|
5
|
+
//# sourceMappingURL=placement.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"placement.d.ts","sourceRoot":"","sources":["../src/placement.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAMjE,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,eAAe,GAAG,gBAAgB,CAQpF;AAKD,wBAAgB,sBAAsB,CAAC,aAAa,EAAE,MAAM,GAAG,eAAe,CAK7E"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.dispatchMetadataFromPolicy = dispatchMetadataFromPolicy;
|
|
4
|
+
exports.defaultPlacementPolicy = defaultPlacementPolicy;
|
|
5
|
+
// Map the out-of-core ExecutionPolicy (the Loom 2.0 PCB sidecar) onto the advisory
|
|
6
|
+
// DispatchMetadata the compute layer already understands. This is the seam by which
|
|
7
|
+
// a flow's declared placement/priority/resources drive WHERE it runs — without the
|
|
8
|
+
// kernel ever depending on the compute layer (P10).
|
|
9
|
+
function dispatchMetadataFromPolicy(policy) {
|
|
10
|
+
return {
|
|
11
|
+
...(policy.computeTarget !== undefined && { target: policy.computeTarget }),
|
|
12
|
+
...(policy.priority !== undefined && { priority: policy.priority }),
|
|
13
|
+
...(policy.affinity !== undefined && { lane: policy.affinity }),
|
|
14
|
+
...(policy.resources?.cpu !== undefined && { estimatedCpu: policy.resources.cpu }),
|
|
15
|
+
...(policy.resources?.memoryMb !== undefined && { estimatedMemory: policy.resources.memoryMb }),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
// Default placement: honour an explicit `meta.target` when it names a registered
|
|
19
|
+
// target, otherwise fall through to the dispatcher's default. Deterministic and
|
|
20
|
+
// dependency-free — the advisory plane never affects correctness (P10), only WHERE.
|
|
21
|
+
function defaultPlacementPolicy(defaultTarget) {
|
|
22
|
+
return (_pkg, meta, targets) => {
|
|
23
|
+
if (meta?.target !== undefined && targets.includes(meta.target))
|
|
24
|
+
return meta.target;
|
|
25
|
+
return defaultTarget;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=placement.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"placement.js","sourceRoot":"","sources":["../src/placement.ts"],"names":[],"mappings":";;AAOA,gEAQC;AAKD,wDAKC;AAtBD,mFAAmF;AACnF,oFAAoF;AACpF,mFAAmF;AACnF,oDAAoD;AACpD,SAAgB,0BAA0B,CAAC,MAAuB;IAChE,OAAO;QACL,GAAG,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC;QAC3E,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACnE,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC/D,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;QAClF,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,KAAK,SAAS,IAAI,EAAE,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;KAChG,CAAC;AACJ,CAAC;AAED,iFAAiF;AACjF,gFAAgF;AAChF,oFAAoF;AACpF,SAAgB,sBAAsB,CAAC,aAAqB;IAC1D,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAC7B,IAAI,IAAI,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QACpF,OAAO,aAAa,CAAC;IACvB,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type { ExecutionPackage, ExecutionResult } from "@loomwork/worker-runtime";
|
|
2
|
+
/**
|
|
3
|
+
* SPI version for the {@link ExecutionDispatcher} contract (ADR 0033). Pinned by
|
|
4
|
+
* `runExecutionDispatcherConformance` in `@loomwork/testing`.
|
|
5
|
+
*
|
|
6
|
+
* 1.1.0 (L20, ADR 0004 amendment) — a dispatcher MUST NOT retry or redispatch when the worker
|
|
7
|
+
* was refused because another holder owns the execution (`isSupersededRefusal` from
|
|
8
|
+
* `@loomwork/worker-runtime`); it settles the dispatch with that refusal. MINOR: no method changed,
|
|
9
|
+
* a non-retrying dispatcher already conforms, and the kit case runs only when the caller opts in
|
|
10
|
+
* with `overWorker`, so no 1.0 dispatcher newly fails the kit.
|
|
11
|
+
*
|
|
12
|
+
* 1.2.0 (R3, ADR 0111) — `ExecutionPackage.principal` and `WorkerRequest.principal` (both
|
|
13
|
+
* optional JSON) carry the authenticated caller across the dispatch seam, and `WorkerRuntime`
|
|
14
|
+
* gains the optional `runsAsSystem` marker. MINOR: every field is optional, a dispatcher that
|
|
15
|
+
* forwards the package whole (all four first-party ones do) already conforms, and a worker that
|
|
16
|
+
* ignores `principal` behaves as before.
|
|
17
|
+
*/
|
|
18
|
+
export declare const DISPATCHER_SPI_VERSION = "1.2.0";
|
|
19
|
+
/**
|
|
20
|
+
* Placement hints the scheduler/orchestrator attaches to a dispatch. The
|
|
21
|
+
* Dispatcher maps these to a concrete placement decision (e.g. Cloud Run CPU/
|
|
22
|
+
* memory in Step 3). Everything is optional and advisory — correctness never
|
|
23
|
+
* depends on it.
|
|
24
|
+
*/
|
|
25
|
+
export interface DispatchMetadata {
|
|
26
|
+
lane?: string;
|
|
27
|
+
priority?: number;
|
|
28
|
+
estimatedCpu?: number;
|
|
29
|
+
estimatedMemory?: number;
|
|
30
|
+
estimatedDuration?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Named compute placement target (e.g. "local" | "cloudrun"). Advisory: a
|
|
33
|
+
* {@link PlacementDispatcher} routes to the matching target, falling back to its
|
|
34
|
+
* default when absent/unknown. Derived from `ExecutionPolicy.computeTarget` (M4).
|
|
35
|
+
*/
|
|
36
|
+
target?: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Chooses which named target handles a dispatch. Pure and synchronous so placement
|
|
40
|
+
* is deterministic and cheap. Given the package, its advisory metadata, and the set
|
|
41
|
+
* of registered target names, return the chosen name (a PlacementDispatcher falls
|
|
42
|
+
* back to its default if the returned name is not registered).
|
|
43
|
+
*/
|
|
44
|
+
export type PlacementPolicy = (pkg: ExecutionPackage, meta: DispatchMetadata | undefined, targets: readonly string[]) => string;
|
|
45
|
+
/** A snapshot of dispatcher liveness for health checks / observability. */
|
|
46
|
+
export interface DispatcherHealth {
|
|
47
|
+
healthy: boolean;
|
|
48
|
+
inFlight: number;
|
|
49
|
+
detail?: string;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Separates execution *placement* from execution *scheduling*. The scheduler
|
|
53
|
+
* decides WHAT runs; the Dispatcher decides WHERE, and owns worker allocation,
|
|
54
|
+
* result collection, retries and (from Step 4) timeout + failure redispatch. It
|
|
55
|
+
* talks only to a {@link WorkerRuntime} — never to a specific cloud.
|
|
56
|
+
*/
|
|
57
|
+
export interface ExecutionDispatcher {
|
|
58
|
+
/**
|
|
59
|
+
* Run `pkg` to suspend-or-complete. MAY retry a failed attempt; MUST NOT retry or redispatch a
|
|
60
|
+
* superseded refusal (`isSupersededRefusal`, 1.1.0) — it rejects with that refusal instead.
|
|
61
|
+
* Such a rejection is not the execution's failure: another holder owns it, possibly this
|
|
62
|
+
* caller's own earlier timed-out attempt. A caller that needs the outcome awaits it by id
|
|
63
|
+
* (`loadState` / `subscribe`); a holder that has died is re-driven by `recoverAbandoned`.
|
|
64
|
+
*/
|
|
65
|
+
dispatch(pkg: ExecutionPackage, meta?: DispatchMetadata): Promise<ExecutionResult>;
|
|
66
|
+
cancel(executionId: string): Promise<void>;
|
|
67
|
+
health(): Promise<DispatcherHealth>;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAElF;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,sBAAsB,UAAU,CAAC;AAE9C;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,CAC5B,GAAG,EAAE,gBAAgB,EACrB,IAAI,EAAE,gBAAgB,GAAG,SAAS,EAClC,OAAO,EAAE,SAAS,MAAM,EAAE,KACvB,MAAM,CAAC;AAEZ,2EAA2E;AAC3E,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;;OAMG;IACH,QAAQ,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IACnF,MAAM,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACrC"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DISPATCHER_SPI_VERSION = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* SPI version for the {@link ExecutionDispatcher} contract (ADR 0033). Pinned by
|
|
6
|
+
* `runExecutionDispatcherConformance` in `@loomwork/testing`.
|
|
7
|
+
*
|
|
8
|
+
* 1.1.0 (L20, ADR 0004 amendment) — a dispatcher MUST NOT retry or redispatch when the worker
|
|
9
|
+
* was refused because another holder owns the execution (`isSupersededRefusal` from
|
|
10
|
+
* `@loomwork/worker-runtime`); it settles the dispatch with that refusal. MINOR: no method changed,
|
|
11
|
+
* a non-retrying dispatcher already conforms, and the kit case runs only when the caller opts in
|
|
12
|
+
* with `overWorker`, so no 1.0 dispatcher newly fails the kit.
|
|
13
|
+
*
|
|
14
|
+
* 1.2.0 (R3, ADR 0111) — `ExecutionPackage.principal` and `WorkerRequest.principal` (both
|
|
15
|
+
* optional JSON) carry the authenticated caller across the dispatch seam, and `WorkerRuntime`
|
|
16
|
+
* gains the optional `runsAsSystem` marker. MINOR: every field is optional, a dispatcher that
|
|
17
|
+
* forwards the package whole (all four first-party ones do) already conforms, and a worker that
|
|
18
|
+
* ignores `principal` behaves as before.
|
|
19
|
+
*/
|
|
20
|
+
exports.DISPATCHER_SPI_VERSION = "1.2.0";
|
|
21
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";;;AAEA;;;;;;;;;;;;;;;GAeG;AACU,QAAA,sBAAsB,GAAG,OAAO,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,56 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loomwork/dispatcher",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Execution dispatcher for Loom — decides WHERE an execution runs and owns worker allocation, retries, timeout and result collection, sitting between the orchestrator and a WorkerRuntime",
|
|
5
|
+
"main": "./dist/index.js",
|
|
6
|
+
"types": "./dist/index.d.ts",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"require": "./dist/index.js",
|
|
11
|
+
"default": "./dist/index.js"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"dependencies": {
|
|
15
|
+
"@loomwork/core": "^0.1.0",
|
|
16
|
+
"@loomwork/worker-runtime": "^0.1.0"
|
|
17
|
+
},
|
|
18
|
+
"devDependencies": {
|
|
19
|
+
"@types/node": "^22.0.0",
|
|
20
|
+
"typescript": "^5.6.3",
|
|
21
|
+
"vitest": "^2.1.8"
|
|
22
|
+
},
|
|
23
|
+
"license": "Apache-2.0",
|
|
24
|
+
"repository": {
|
|
25
|
+
"type": "git",
|
|
26
|
+
"url": "git+https://github.com/chhetrisushil/loom.git",
|
|
27
|
+
"directory": "packages/dispatcher"
|
|
28
|
+
},
|
|
29
|
+
"files": [
|
|
30
|
+
"dist",
|
|
31
|
+
"src",
|
|
32
|
+
"!src/**/*.test.ts",
|
|
33
|
+
"!src/**/*.test.tsx",
|
|
34
|
+
"!src/**/__tests__",
|
|
35
|
+
"!src/**/__fixtures__",
|
|
36
|
+
"!src/**/__snapshots__",
|
|
37
|
+
"!src/**/__golden__",
|
|
38
|
+
"LICENSE",
|
|
39
|
+
"NOTICE"
|
|
40
|
+
],
|
|
41
|
+
"publishConfig": {
|
|
42
|
+
"access": "public",
|
|
43
|
+
"tag": "next"
|
|
44
|
+
},
|
|
45
|
+
"engines": {
|
|
46
|
+
"node": ">=20.0.0"
|
|
47
|
+
},
|
|
48
|
+
"scripts": {
|
|
49
|
+
"build": "tsc --build",
|
|
50
|
+
"dev": "tsc --build --watch",
|
|
51
|
+
"test": "vitest run",
|
|
52
|
+
"test:watch": "vitest",
|
|
53
|
+
"clean": "rm -rf dist tsconfig.tsbuildinfo",
|
|
54
|
+
"typecheck:tests": "tsc -p tsconfig.test.json"
|
|
55
|
+
}
|
|
6
56
|
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type ExecutionPackage,
|
|
3
|
+
type ExecutionResult,
|
|
4
|
+
type WorkerLease,
|
|
5
|
+
type WorkerRuntime,
|
|
6
|
+
isSupersededRefusal,
|
|
7
|
+
} from "@loomwork/worker-runtime";
|
|
8
|
+
import type { DispatchMetadata, DispatcherHealth, ExecutionDispatcher } from "./types";
|
|
9
|
+
|
|
10
|
+
export interface LocalDispatcherOptions {
|
|
11
|
+
/**
|
|
12
|
+
* How many times to attempt a dispatch before giving up. An attempt fails when
|
|
13
|
+
* the worker throws (transport/allocation error or a crashed instance) or when
|
|
14
|
+
* it exceeds {@link LocalDispatcherOptions.executeTimeoutMs} — NOT when the flow
|
|
15
|
+
* returns a "failed" result, which is a normal terminal outcome. Default 1.
|
|
16
|
+
*
|
|
17
|
+
* NOR when the worker was refused because another holder owns the execution — a lost
|
|
18
|
+
* start race or a fenced-out attempt ({@link isSupersededRefusal}, L20). That refusal is
|
|
19
|
+
* rethrown at once, never retried: `allocate` fences every attempt, so a retry would
|
|
20
|
+
* supersede the holder that won, and the two would ping-pong, re-running the in-flight
|
|
21
|
+
* step on every bounce. Declining it keeps ADR 0004's bound — the one interrupted step
|
|
22
|
+
* runs at most twice — whatever this is set to.
|
|
23
|
+
*
|
|
24
|
+
* Redispatch is safe: the worker is idempotent (it inspects durable state and
|
|
25
|
+
* resumes a crashed execution from its latest snapshot rather than restarting),
|
|
26
|
+
* so a retry continues where the failed attempt left off without re-running
|
|
27
|
+
* completed steps.
|
|
28
|
+
*/
|
|
29
|
+
maxAttempts?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Per-attempt deadline. A worker that hangs (a wedged/lost Cloud Run instance)
|
|
32
|
+
* is treated as a failed attempt and redispatched. Omit to wait indefinitely.
|
|
33
|
+
*/
|
|
34
|
+
executeTimeoutMs?: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
class DispatchTimeoutError extends Error {}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The default Dispatcher. It wraps a single {@link WorkerRuntime} — allocate a
|
|
41
|
+
* lease, execute the package, release the lease — and exposes the placement seam
|
|
42
|
+
* without any cloud awareness. Swapping the worker (Local → Remote → CloudRun)
|
|
43
|
+
* is the only change needed to move execution off-box; callers see the same
|
|
44
|
+
* {@link ExecutionResult} contract.
|
|
45
|
+
*/
|
|
46
|
+
export class LocalDispatcher implements ExecutionDispatcher {
|
|
47
|
+
private inFlight = 0;
|
|
48
|
+
private readonly maxAttempts: number;
|
|
49
|
+
private readonly executeTimeoutMs: number | undefined;
|
|
50
|
+
private readonly leases = new Map<string, WorkerLease>();
|
|
51
|
+
private readonly cancelled = new Set<string>();
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
private readonly worker: WorkerRuntime,
|
|
55
|
+
options: LocalDispatcherOptions = {}
|
|
56
|
+
) {
|
|
57
|
+
this.maxAttempts = Math.max(1, options.maxAttempts ?? 1);
|
|
58
|
+
this.executeTimeoutMs = options.executeTimeoutMs;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async dispatch(pkg: ExecutionPackage, meta?: DispatchMetadata): Promise<ExecutionResult> {
|
|
62
|
+
this.cancelled.delete(pkg.executionId);
|
|
63
|
+
this.inFlight++;
|
|
64
|
+
try {
|
|
65
|
+
let lastErr: unknown;
|
|
66
|
+
for (let attempt = 1; attempt <= this.maxAttempts; attempt++) {
|
|
67
|
+
const lease = await this.worker.allocate({
|
|
68
|
+
executionId: pkg.executionId,
|
|
69
|
+
...(meta !== undefined && { metadata: meta as Record<string, unknown> }),
|
|
70
|
+
...(pkg.principal !== undefined && { principal: pkg.principal }),
|
|
71
|
+
});
|
|
72
|
+
this.leases.set(pkg.executionId, lease);
|
|
73
|
+
try {
|
|
74
|
+
const run = this.worker.execute(lease, pkg);
|
|
75
|
+
return await (this.executeTimeoutMs
|
|
76
|
+
? withTimeout(run, this.executeTimeoutMs, pkg.executionId)
|
|
77
|
+
: run);
|
|
78
|
+
} catch (err) {
|
|
79
|
+
// L20 — superseded, not failed: someone else is driving this execution now.
|
|
80
|
+
if (isSupersededRefusal(err, pkg.executionId)) throw err;
|
|
81
|
+
lastErr = err;
|
|
82
|
+
} finally {
|
|
83
|
+
this.leases.delete(pkg.executionId);
|
|
84
|
+
await this.worker.release(lease);
|
|
85
|
+
}
|
|
86
|
+
// A cancel during this attempt aborts it; DON'T redispatch a cancelled execution
|
|
87
|
+
// (that would re-run what the caller just cancelled). Also stops the flag leaking.
|
|
88
|
+
if (this.cancelled.has(pkg.executionId)) break;
|
|
89
|
+
}
|
|
90
|
+
throw lastErr;
|
|
91
|
+
} finally {
|
|
92
|
+
this.inFlight--;
|
|
93
|
+
this.cancelled.delete(pkg.executionId);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Best-effort cancellation. Releases any lease held for the execution, and
|
|
99
|
+
* (K8.5) forwards the kill to the worker when it supports mid-flight
|
|
100
|
+
* interruption — in-process this aborts the in-flight effect and records the
|
|
101
|
+
* durable EXECUTION_CANCELLED; over HTTP it reaches the remote worker's
|
|
102
|
+
* `POST /cancel`. Delivery stays best-effort: durable settlement belongs to
|
|
103
|
+
* the store-side cancel (fence-then-cancel), never to this call landing.
|
|
104
|
+
*/
|
|
105
|
+
async cancel(executionId: string): Promise<void> {
|
|
106
|
+
const lease = this.leases.get(executionId);
|
|
107
|
+
if (lease) {
|
|
108
|
+
// Only flag as cancelled when a dispatch is actually in flight (a lease is held) —
|
|
109
|
+
// the flag exists solely to stop that dispatch's retry loop, and gating it here keeps
|
|
110
|
+
// `cancelled` bounded (it's cleared in dispatch's finally). A cancel with no in-flight
|
|
111
|
+
// dispatch has nothing to short-circuit.
|
|
112
|
+
this.cancelled.add(executionId);
|
|
113
|
+
this.leases.delete(executionId);
|
|
114
|
+
await this.worker.release(lease);
|
|
115
|
+
}
|
|
116
|
+
await this.worker.cancel?.(executionId);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async health(): Promise<DispatcherHealth> {
|
|
120
|
+
return { healthy: true, inFlight: this.inFlight };
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function withTimeout<T>(p: Promise<T>, ms: number, executionId: string): Promise<T> {
|
|
125
|
+
return new Promise<T>((resolve, reject) => {
|
|
126
|
+
const timer = setTimeout(
|
|
127
|
+
() => reject(new DispatchTimeoutError(`dispatch timed out after ${ms}ms for ${executionId}`)),
|
|
128
|
+
ms
|
|
129
|
+
);
|
|
130
|
+
p.then(
|
|
131
|
+
(v) => {
|
|
132
|
+
clearTimeout(timer);
|
|
133
|
+
resolve(v);
|
|
134
|
+
},
|
|
135
|
+
(err: unknown) => {
|
|
136
|
+
clearTimeout(timer);
|
|
137
|
+
reject(err as Error);
|
|
138
|
+
}
|
|
139
|
+
);
|
|
140
|
+
});
|
|
141
|
+
}
|