@loomwork/dispatcher 0.0.0-stage → 0.1.1

package/src/PlacementDispatcher.ts

@@ -0,0 +1,124 @@
+import type { ExecutionPackage, ExecutionResult } from "@loomwork/worker-runtime";
+import { defaultPlacementPolicy } from "./placement";
+import type {
+  DispatchMetadata,
+  DispatcherHealth,
+  ExecutionDispatcher,
+  PlacementPolicy,
+} from "./types";
+
+export interface PlacementDispatcherOptions {
+  /** Named compute targets. Each is a full ExecutionDispatcher (e.g. a LocalDispatcher
+   *  wrapping a local / remote-HTTP / Cloud Run WorkerRuntime), so retry/timeout/lease
+   *  handling is reused per target. */
+  targets: Record<string, ExecutionDispatcher>;
+  /** Target used when the policy returns an unregistered name. Must be a key of `targets`. */
+  default: string;
+  /** How to choose a target. Defaults to "honour meta.target, else default". */
+  policy?: PlacementPolicy;
+  /**
+   * Where to report a dispatch that named an unregistered target (#392). Defaults to
+   * `console.warn`. Falling back is correct; doing it silently is how an execution placed
+   * on sized compute ends up running somewhere else without anyone noticing.
+   */
+  onWarn?: (message: string) => void;
+}
+
+/**
+ * Policy-driven placement (Loom 2.0 M4). A composite {@link ExecutionDispatcher} that
+ * routes each dispatch to one of several named targets according to a
+ * {@link PlacementPolicy} — the WHERE decision, driven by `ExecutionPolicy`
+ * (`dispatchMetadataFromPolicy`). It owns no cloud knowledge: every target is just
+ * another ExecutionDispatcher, so local / remote-HTTP / Cloud Run workers compose
+ * behind one seam. With a single target it behaves exactly like that target, so
+ * existing single-placement dispatch is unchanged.
+ */
+export class PlacementDispatcher implements ExecutionDispatcher {
+  private readonly targets: Record<string, ExecutionDispatcher>;
+  private readonly names: string[];
+  private readonly defaultTarget: string;
+  private readonly policy: PlacementPolicy;
+  // Remember where each execution was placed so cancel() reaches the right target.
+  // Bounded (LRU-by-insertion) so it can't grow without limit in a long-lived dispatcher.
+  private static readonly ROUTED_CAP = 10_000;
+  private readonly routed = new Map<string, string>();
+  private inFlight = 0;
+  private readonly onWarn: (message: string) => void;
+  private readonly warnedTargets = new Set<string>();
+
+  constructor(options: PlacementDispatcherOptions) {
+    this.names = Object.keys(options.targets);
+    if (this.names.length === 0) {
+      throw new Error("PlacementDispatcher requires at least one target");
+    }
+    if (!(options.default in options.targets)) {
+      throw new Error(`PlacementDispatcher default target "${options.default}" is not registered`);
+    }
+    this.targets = options.targets;
+    this.defaultTarget = options.default;
+    this.policy = options.policy ?? defaultPlacementPolicy(options.default);
+    this.onWarn = options.onWarn ?? ((m: string) => console.warn(`[loom] ${m}`));
+  }
+
+  /** The target names currently registered. */
+  get targetNames(): readonly string[] {
+    return this.names;
+  }
+
+  async dispatch(pkg: ExecutionPackage, meta?: DispatchMetadata): Promise<ExecutionResult> {
+    // #392 — a placement hint can outlive the lane it names: an execution suspended for days
+    // wakes into a deployment where its target was renamed or retired. Falling back is
+    // right (a retired lane must not strand what named it) but doing it SILENTLY means an
+    // execution deliberately placed on sized compute quietly runs somewhere else forever.
+    // Warned once per unknown name — this is a deployment fact, not a per-dispatch event.
+    if (meta?.target !== undefined && !this.names.includes(meta.target)) {
+      if (!this.warnedTargets.has(meta.target)) {
+        this.warnedTargets.add(meta.target);
+        this.onWarn(
+          `placement target "${meta.target}" is not registered [${this.names.join(", ")}] — falling back to "${this.defaultTarget}". An execution that named it is now running somewhere else.`
+        );
+      }
+    }
+    const chosen = this.policy(pkg, meta, this.names);
+    const name = chosen in this.targets ? chosen : this.defaultTarget;
+    const target = this.targets[name] as ExecutionDispatcher;
+    this.routed.set(pkg.executionId, name);
+    // Bound `routed` (it would otherwise grow one entry per execution forever). Recent
+    // executions — the ones a cancel can still act on — keep their precise target; only a
+    // very old execution (evicted here) falls back to the all-targets cancel path, which is
+    // harmless (a finished execution ignores a stray cancel). Insertion-ordered eviction.
+    if (this.routed.size > PlacementDispatcher.ROUTED_CAP) {
+      const oldest = this.routed.keys().next().value;
+      if (oldest !== undefined) this.routed.delete(oldest);
+    }
+    this.inFlight++;
+    try {
+      return await target.dispatch(pkg, meta);
+    } finally {
+      this.inFlight--;
+    }
+  }
+
+  async cancel(executionId: string): Promise<void> {
+    const name = this.routed.get(executionId);
+    if (name !== undefined) {
+      await (this.targets[name] as ExecutionDispatcher).cancel(executionId);
+      return;
+    }
+    // Placement unknown (never dispatched here, or restarted): ask every target.
+    await Promise.all(
+      this.names.map((n) => (this.targets[n] as ExecutionDispatcher).cancel(executionId))
+    );
+  }
+
+  async health(): Promise<DispatcherHealth> {
+    const healths = await Promise.all(
+      this.names.map((n) => (this.targets[n] as ExecutionDispatcher).health())
+    );
+    return {
+      healthy: healths.every((h) => h.healthy),
+      inFlight: this.inFlight,
+      detail: this.names.map((n, i) => `${n}:${healths[i]?.inFlight ?? 0}`).join(","),
+    };
+  }
+}

package/src/SupervisingDispatcher.ts

@@ -0,0 +1,160 @@
+import {
+  type ExecutionPackage,
+  type ExecutionResult,
+  isSupersededRefusal,
+} from "@loomwork/worker-runtime";
+import type { DispatchMetadata, DispatcherHealth, ExecutionDispatcher } from "./types";
+
+export interface CircuitPolicy {
+  /** Consecutive dispatch failures before the circuit opens. */
+  threshold: number;
+  /** How long the circuit stays open before a half-open trial, in ms. */
+  cooldownMs: number;
+  /** Circuit key for a package. Default: `pkg.flowId` (per-flow breaker). */
+  key?: (pkg: ExecutionPackage) => string;
+}
+
+export interface SupervisionPolicy {
+  /** Restart a *throwing* dispatch up to N extra times (default 0). A returned
+   *  "failed" result is a normal outcome, not a restartable error — and neither is a
+   *  refusal because another holder owns the execution ({@link isSupersededRefusal},
+   *  L20): it is rethrown at once, without a restart and without counting against the
+   *  circuit, because restarting it would fence out the holder that won. */
+  maxRestarts?: number;
+  /** Base backoff between restarts, in ms (default 0). */
+  backoffMs?: number;
+  /** Exponential growth factor for the backoff (default 2). */
+  backoffFactor?: number;
+  /** Optional circuit breaker — fail fast when a target is consistently down. */
+  circuit?: CircuitPolicy;
+}
+
+export class CircuitOpenError extends Error {
+  readonly key: string;
+  constructor(key: string) {
+    super(`Circuit open for "${key}" — failing fast`);
+    this.name = "CircuitOpenError";
+    this.key = key;
+  }
+}
+
+interface Circuit {
+  failures: number;
+  openedAt: number | null;
+}
+
+export interface SupervisingDispatcherOptions {
+  policy?: SupervisionPolicy;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+}
+
+/**
+ * Fault supervision (Loom 2.0 K4). A composite {@link ExecutionDispatcher} that wraps
+ * a child with an explicit restart/backoff policy and a per-key **circuit breaker**:
+ * after `threshold` consecutive dispatch failures the circuit opens and further
+ * dispatches fail fast (CircuitOpenError) until a `cooldownMs` half-open trial — so a
+ * flaky model/tool/worker is not hammered. Promotes the runtime's ad-hoc
+ * retry/redispatch into a named supervision seam; composes over LocalDispatcher /
+ * PlacementDispatcher unchanged.
+ */
+export class SupervisingDispatcher implements ExecutionDispatcher {
+  private readonly circuits = new Map<string, Circuit>();
+  private readonly policy: SupervisionPolicy;
+  private readonly now: () => number;
+  private readonly sleep: (ms: number) => Promise<void>;
+
+  constructor(
+    private readonly child: ExecutionDispatcher,
+    options: SupervisingDispatcherOptions = {}
+  ) {
+    this.policy = options.policy ?? {};
+    this.now = options.now ?? (() => Date.now());
+    this.sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  }
+
+  async dispatch(pkg: ExecutionPackage, meta?: DispatchMetadata): Promise<ExecutionResult> {
+    const key = this.policy.circuit?.key?.(pkg) ?? pkg.flowId;
+    this.guardCircuit(key); // throws CircuitOpenError if open within cooldown
+
+    const maxRestarts = Math.max(0, this.policy.maxRestarts ?? 0);
+    const base = this.policy.backoffMs ?? 0;
+    const factor = this.policy.backoffFactor ?? 2;
+
+    let lastErr: unknown;
+    for (let attempt = 0; attempt <= maxRestarts; attempt++) {
+      try {
+        const result = await this.child.dispatch(pkg, meta);
+        this.recordSuccess(key);
+        return result;
+      } catch (err) {
+        // L20 — superseded, not failed: not restartable, and not the target's fault. But an
+        // EARLIER restart of this dispatch that failed genuinely still counts against the
+        // circuit — the refusal ends the dispatch, it does not erase what preceded it.
+        if (isSupersededRefusal(err, pkg.executionId)) {
+          if (attempt > 0) this.recordFailure(key);
+          throw err;
+        }
+        lastErr = err;
+        if (attempt < maxRestarts) await this.sleep(base * factor ** attempt);
+      }
+    }
+    this.recordFailure(key);
+    throw lastErr;
+  }
+
+  async cancel(executionId: string): Promise<void> {
+    return this.child.cancel(executionId);
+  }
+
+  async health(): Promise<DispatcherHealth> {
+    const h = await this.child.health();
+    const open = [...this.circuits.entries()]
+      .filter(([, c]) => c.openedAt !== null)
+      .map(([k]) => k);
+    return {
+      healthy: h.healthy && open.length === 0,
+      inFlight: h.inFlight,
+      ...(open.length > 0
+        ? { detail: `circuits-open: ${open.join(",")}` }
+        : h.detail !== undefined && { detail: h.detail }),
+    };
+  }
+
+  private guardCircuit(key: string): void {
+    const c = this.circuits.get(key);
+    if (c === undefined || c.openedAt === null) return;
+    const cooldown = this.policy.circuit?.cooldownMs ?? 0;
+    if (this.now() - c.openedAt < cooldown) {
+      throw new CircuitOpenError(key);
+    }
+    // Cooldown elapsed → half-open: allow one trial. Keep the failure count so a
+    // fresh failure re-opens immediately; a success resets it.
+    c.openedAt = null;
+  }
+
+  private recordSuccess(key: string): void {
+    // DELETE rather than reset. A circuit with `failures: 0, openedAt: null` is
+    // indistinguishable from an absent one everywhere it is read — `guardCircuit`
+    // returns on either, `recordFailure` defaults to exactly that shape, and
+    // `health()` counts only open circuits — so dropping it is observationally
+    // identical and bounds the map to circuits actually accumulating failures.
+    //
+    // Without this the map only grows. The default key is `flowId` (bounded by
+    // deployed flows), but `policy.circuit.key` receives the whole ExecutionPackage,
+    // so keying per tenant — or per execution, which reads as natural — turned a
+    // supervisor into a leak for the lifetime of the process.
+    this.circuits.delete(key);
+  }
+
+  private recordFailure(key: string): void {
+    const circuit = this.policy.circuit;
+    if (circuit === undefined) return;
+    const c = this.circuits.get(key) ?? { failures: 0, openedAt: null };
+    c.failures += 1;
+    if (c.failures >= circuit.threshold) {
+      c.openedAt = this.now();
+    }
+    this.circuits.set(key, c);
+  }
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+export type {
+  DispatchMetadata,
+  DispatcherHealth,
+  ExecutionDispatcher,
+  PlacementPolicy,
+} from "./types";
+export { DISPATCHER_SPI_VERSION } from "./types";
+export { LocalDispatcher, type LocalDispatcherOptions } from "./LocalDispatcher";
+export {
+  PlacementDispatcher,
+  type PlacementDispatcherOptions,
+} from "./PlacementDispatcher";
+export { dispatchMetadataFromPolicy, defaultPlacementPolicy } from "./placement";
+export {
+  SupervisingDispatcher,
+  CircuitOpenError,
+  type SupervisionPolicy,
+  type CircuitPolicy,
+  type SupervisingDispatcherOptions,
+} from "./SupervisingDispatcher";
+// Re-export the transport contract so consumers can import package + dispatcher
+// types from one place.
+export type {
+  ExecutionOutcome,
+  ExecutionPackage,
+  ExecutionResult,
+} from "@loomwork/worker-runtime";

package/src/placement.ts

@@ -0,0 +1,26 @@
+import type { ExecutionPolicy } from "@loomwork/core";
+import type { DispatchMetadata, PlacementPolicy } from "./types";
+
+// Map the out-of-core ExecutionPolicy (the Loom 2.0 PCB sidecar) onto the advisory
+// DispatchMetadata the compute layer already understands. This is the seam by which
+// a flow's declared placement/priority/resources drive WHERE it runs — without the
+// kernel ever depending on the compute layer (P10).
+export function dispatchMetadataFromPolicy(policy: ExecutionPolicy): DispatchMetadata {
+  return {
+    ...(policy.computeTarget !== undefined && { target: policy.computeTarget }),
+    ...(policy.priority !== undefined && { priority: policy.priority }),
+    ...(policy.affinity !== undefined && { lane: policy.affinity }),
+    ...(policy.resources?.cpu !== undefined && { estimatedCpu: policy.resources.cpu }),
+    ...(policy.resources?.memoryMb !== undefined && { estimatedMemory: policy.resources.memoryMb }),
+  };
+}
+
+// Default placement: honour an explicit `meta.target` when it names a registered
+// target, otherwise fall through to the dispatcher's default. Deterministic and
+// dependency-free — the advisory plane never affects correctness (P10), only WHERE.
+export function defaultPlacementPolicy(defaultTarget: string): PlacementPolicy {
+  return (_pkg, meta, targets) => {
+    if (meta?.target !== undefined && targets.includes(meta.target)) return meta.target;
+    return defaultTarget;
+  };
+}

package/src/types.ts

@@ -0,0 +1,77 @@
+import type { ExecutionPackage, ExecutionResult } from "@loomwork/worker-runtime";
+
+/**
+ * SPI version for the {@link ExecutionDispatcher} contract (ADR 0033). Pinned by
+ * `runExecutionDispatcherConformance` in `@loomwork/testing`.
+ *
+ * 1.1.0 (L20, ADR 0004 amendment) — a dispatcher MUST NOT retry or redispatch when the worker
+ * was refused because another holder owns the execution (`isSupersededRefusal` from
+ * `@loomwork/worker-runtime`); it settles the dispatch with that refusal. MINOR: no method changed,
+ * a non-retrying dispatcher already conforms, and the kit case runs only when the caller opts in
+ * with `overWorker`, so no 1.0 dispatcher newly fails the kit.
+ *
+ * 1.2.0 (R3, ADR 0111) — `ExecutionPackage.principal` and `WorkerRequest.principal` (both
+ * optional JSON) carry the authenticated caller across the dispatch seam, and `WorkerRuntime`
+ * gains the optional `runsAsSystem` marker. MINOR: every field is optional, a dispatcher that
+ * forwards the package whole (all four first-party ones do) already conforms, and a worker that
+ * ignores `principal` behaves as before.
+ */
+export const DISPATCHER_SPI_VERSION = "1.2.0";
+
+/**
+ * Placement hints the scheduler/orchestrator attaches to a dispatch. The
+ * Dispatcher maps these to a concrete placement decision (e.g. Cloud Run CPU/
+ * memory in Step 3). Everything is optional and advisory — correctness never
+ * depends on it.
+ */
+export interface DispatchMetadata {
+  lane?: string;
+  priority?: number;
+  estimatedCpu?: number;
+  estimatedMemory?: number;
+  estimatedDuration?: number;
+  /**
+   * Named compute placement target (e.g. "local" | "cloudrun"). Advisory: a
+   * {@link PlacementDispatcher} routes to the matching target, falling back to its
+   * default when absent/unknown. Derived from `ExecutionPolicy.computeTarget` (M4).
+   */
+  target?: string;
+}
+
+/**
+ * Chooses which named target handles a dispatch. Pure and synchronous so placement
+ * is deterministic and cheap. Given the package, its advisory metadata, and the set
+ * of registered target names, return the chosen name (a PlacementDispatcher falls
+ * back to its default if the returned name is not registered).
+ */
+export type PlacementPolicy = (
+  pkg: ExecutionPackage,
+  meta: DispatchMetadata | undefined,
+  targets: readonly string[]
+) => string;
+
+/** A snapshot of dispatcher liveness for health checks / observability. */
+export interface DispatcherHealth {
+  healthy: boolean;
+  inFlight: number;
+  detail?: string;
+}
+
+/**
+ * Separates execution *placement* from execution *scheduling*. The scheduler
+ * decides WHAT runs; the Dispatcher decides WHERE, and owns worker allocation,
+ * result collection, retries and (from Step 4) timeout + failure redispatch. It
+ * talks only to a {@link WorkerRuntime} — never to a specific cloud.
+ */
+export interface ExecutionDispatcher {
+  /**
+   * Run `pkg` to suspend-or-complete. MAY retry a failed attempt; MUST NOT retry or redispatch a
+   * superseded refusal (`isSupersededRefusal`, 1.1.0) — it rejects with that refusal instead.
+   * Such a rejection is not the execution's failure: another holder owns it, possibly this
+   * caller's own earlier timed-out attempt. A caller that needs the outcome awaits it by id
+   * (`loadState` / `subscribe`); a holder that has died is re-driven by `recoverAbandoned`.
+   */
+  dispatch(pkg: ExecutionPackage, meta?: DispatchMetadata): Promise<ExecutionResult>;
+  cancel(executionId: string): Promise<void>;
+  health(): Promise<DispatcherHealth>;
+}