@lostgradient/weft 0.2.0

package/dist/core/worker-execution-strategy.js

@@ -0,0 +1,356 @@
+import { WorkerCheckpointResumeState } from "./worker-checkpoint-resume-state.js";
+import {
+  WorkerExecutionDispatcher
+} from "./worker-execution-dispatcher.js";
+import { WorkerExecutionOwnership } from "./worker-execution-ownership.js";
+import { WorkerListenerRegistry } from "./worker-listener-registry.js";
+import {
+  emitWorkerMessageToEngine,
+  isParkableWaitSignalCheckpoint
+} from "./worker-message-helpers.js";
+import { WorkerProtocolGuard } from "./worker-protocol-guard.js";
+import { WORKER_PROTOCOL_VERSION } from "./worker-protocol.js";
+import { WorkerTurnWatchdog } from "./worker-turn-watchdog.js";
+
+export class WorkerExecutionStrategy {
+  #pool;
+  #ownership;
+  #workerListeners;
+  #broadcastChannel;
+  #broadcastListener;
+  #checkpointResumeState;
+  #workflowTurnTimeoutMs;
+  #maxProtocolMessageBytes;
+  #requireProtocolVersion;
+  #discardOnCancel;
+  #turnWatchdog;
+  #protocolGuard;
+  #dispatcher;
+  #messageHandler;
+  #disposed;
+  #nextTurnId;
+  constructor(pool, options) {
+    this.#pool = pool;
+    this.#ownership = new WorkerExecutionOwnership;
+    this.#workerListeners = new WorkerListenerRegistry;
+    this.#checkpointResumeState = new WorkerCheckpointResumeState;
+    this.#workflowTurnTimeoutMs = options?.workflowTurnTimeoutMs;
+    this.#maxProtocolMessageBytes = options?.maxProtocolMessageBytes;
+    this.#requireProtocolVersion = options?.requireProtocolVersion ?? !1;
+    this.#discardOnCancel = options?.discardOnCancel ?? !1;
+    this.#turnWatchdog = new WorkerTurnWatchdog(this.#workflowTurnTimeoutMs, (turn) => {
+      this.#handleTurnTimeout(turn);
+    });
+    this.#protocolGuard = new WorkerProtocolGuard(this.#maxProtocolMessageBytes, this.#requireProtocolVersion, this.#turnWatchdog);
+    this.#dispatcher = new WorkerExecutionDispatcher({
+      pool: this.#pool,
+      ownership: this.#ownership,
+      isDisposed: () => this.#disposed,
+      requireProtocolVersion: () => this.#requireProtocolVersion,
+      validateHostToWorkerMessage: (workflowId, message, worker) => this.#assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
+      attachWorkerListeners: (worker) => {
+        this.#attachWorkerListeners(worker);
+      },
+      detachWorkerListenersIfIdle: (worker) => {
+        this.#detachWorkerListenersIfIdle(worker);
+      },
+      beginTurn: (worker, workflowId, turnId, kind) => {
+        this.#turnWatchdog.begin(worker, workflowId, turnId, kind);
+      },
+      clearTurn: (worker) => {
+        this.#turnWatchdog.clear(worker);
+      },
+      discardWorkerAndFailWorkflows: (worker, discardOptions) => {
+        this.#discardWorkerAndFailWorkflows(worker, discardOptions);
+      },
+      emit: (message) => {
+        this.#emit(message);
+      }
+    });
+    this.#messageHandler = null;
+    this.#broadcastChannel = null;
+    this.#broadcastListener = null;
+    this.#disposed = !1;
+    this.#nextTurnId = 1;
+    if (options?.broadcastEvents)
+      try {
+        this.#broadcastChannel = new BroadcastChannel("weft:events");
+        this.#broadcastListener = (event) => {
+          this.#handleBroadcastMessage(event.data);
+        };
+        this.#broadcastChannel.addEventListener("message", this.#broadcastListener);
+      } catch {}
+  }
+  onMessage(handler) {
+    this.#messageHandler = handler;
+  }
+  startWorkflow(parameters) {
+    this.#ownership.resetWorkflow(parameters.workflowId);
+    this.#checkpointResumeState.resetWorkflow(parameters.workflowId);
+    const message = {
+      type: "run",
+      protocolVersion: WORKER_PROTOCOL_VERSION,
+      turnId: this.#nextTurnId++,
+      workflowId: parameters.workflowId,
+      workflowType: parameters.workflowType,
+      checkpoint: parameters.checkpoint,
+      input: parameters.input,
+      executionStateOwnerId: parameters.executionStateOwnerId ?? parameters.workflowId
+    };
+    if (this.#maxProtocolMessageBytes !== void 0)
+      message.maxProtocolMessageBytes = this.#maxProtocolMessageBytes;
+    if (parameters.deadline !== void 0)
+      message.deadline = parameters.deadline;
+    if (parameters.headers)
+      message.headers = parameters.headers;
+    if (!this.#assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
+      return;
+    this.#dispatcher.acquireAndSend(parameters.workflowId, message);
+  }
+  resumeWorkflow(parameters) {
+    const worker = this.#ownership.getActiveWorker(parameters.workflowId);
+    if (worker) {
+      this.#checkpointResumeState.recordResume(parameters.workflowId);
+      this.#dispatcher.postResumeMessage(worker, parameters, this.#createResumeMessage(parameters));
+      return;
+    }
+    const parkedWorker = this.#ownership.getParkedWorker(parameters.workflowId);
+    if (parkedWorker) {
+      this.#dispatcher.resumeParkedWorkflow(parameters, parkedWorker, this.#createResumeMessage(parameters));
+      return;
+    }
+    if (this.#ownership.consumeCancelled(parameters.workflowId))
+      return;
+    if (!this.#disposed)
+      this.#emit({
+        type: "failed",
+        workflowId: parameters.workflowId,
+        error: `No worker assigned for workflow: ${parameters.workflowId}`
+      });
+  }
+  #createResumeMessage(parameters) {
+    const message = {
+      type: "resume",
+      protocolVersion: WORKER_PROTOCOL_VERSION,
+      turnId: this.#nextTurnId++,
+      workflowId: parameters.workflowId,
+      checkpoint: parameters.checkpoint,
+      operationResult: parameters.operationResult
+    };
+    if (this.#maxProtocolMessageBytes !== void 0)
+      message.maxProtocolMessageBytes = this.#maxProtocolMessageBytes;
+    return message;
+  }
+  cancelWorkflow(workflowId) {
+    const worker = this.#ownership.getActiveWorker(workflowId);
+    if (worker) {
+      this.#ownership.markCancelled(workflowId);
+      if (this.#discardOnCancel) {
+        this.#discardWorkerAndFailWorkflows(worker, {
+          targetWorkflowId: workflowId,
+          skipTarget: !0,
+          otherCategory: "system",
+          otherError: `Worker discarded during cancellation of workflow: ${workflowId}`
+        });
+        return;
+      }
+      const message = {
+        type: "cancel",
+        workflowId
+      };
+      if (this.#requireProtocolVersion)
+        message.protocolVersion = WORKER_PROTOCOL_VERSION;
+      worker.postMessage(message);
+      this.#releaseActiveWorker(workflowId);
+      return;
+    }
+    const parkedWorker = this.#ownership.getParkedWorker(workflowId);
+    if (!parkedWorker)
+      return;
+    this.#ownership.markCancelled(workflowId);
+    this.#ownership.deleteParked(workflowId);
+    if (this.#discardOnCancel) {
+      this.#discardWorkerAndFailWorkflows(parkedWorker, {
+        targetWorkflowId: workflowId,
+        skipTarget: !0,
+        otherCategory: "system",
+        otherError: `Worker discarded during cancellation of parked workflow: ${workflowId}`
+      });
+      return;
+    }
+    this.#dispatcher.cancelParkedWorkflow(workflowId, parkedWorker);
+  }
+  [Symbol.dispose]() {
+    this.#teardown();
+    this.#pool[Symbol.dispose]();
+  }
+  async[Symbol.asyncDispose]() {
+    this.#teardown();
+    await this.#pool[Symbol.asyncDispose]();
+  }
+  #teardown() {
+    this.#disposed = !0;
+    if (this.#broadcastChannel) {
+      if (this.#broadcastListener)
+        this.#broadcastChannel.removeEventListener("message", this.#broadcastListener);
+      this.#broadcastChannel.close();
+    }
+    this.#turnWatchdog.clearAll();
+    const activeWorkflowIds = this.#ownership.activeWorkflowIds();
+    for (const workflowId of activeWorkflowIds)
+      this.#releaseActiveWorker(workflowId);
+    this.#workerListeners.detachAll();
+    this.#ownership.clear();
+    this.#checkpointResumeState.clear();
+    this.#messageHandler = null;
+  }
+  async#handleWorkerMessage(worker, message) {
+    if (!this.#acceptWorkerMessage(worker, message))
+      return;
+    this.#turnWatchdog.clear(worker);
+    const resumeVersionBeforeCheckpointHandling = this.#checkpointResumeState.beginCheckpointHandling(message), emitResult = emitWorkerMessageToEngine(this.#messageHandler, message), handlerFailed = emitResult instanceof Promise ? await emitResult : emitResult;
+    try {
+      if (this.#settleTerminalWorkerMessage(worker, message))
+        return;
+      if (handlerFailed)
+        return;
+      this.#parkCheckpointIfStillWaiting(worker, message, resumeVersionBeforeCheckpointHandling);
+    } finally {
+      this.#checkpointResumeState.finishCheckpointHandling(message.workflowId, resumeVersionBeforeCheckpointHandling, this.#ownership.isWorkflowClosed(message.workflowId));
+    }
+  }
+  #settleTerminalWorkerMessage(worker, message) {
+    if (message.type !== "completed" && message.type !== "failed")
+      return !1;
+    this.#ownership.consumeCancelled(message.workflowId);
+    this.#ownership.deleteParked(message.workflowId);
+    this.#releaseActiveWorker(message.workflowId);
+    this.#detachWorkerListenersIfIdle(worker);
+    this.#checkpointResumeState.forgetWorkflowIfClosed(message.workflowId, this.#ownership.isWorkflowClosed(message.workflowId));
+    return !0;
+  }
+  #parkCheckpointIfStillWaiting(worker, message, resumeVersionBeforeCheckpointHandling) {
+    if (message.type !== "checkpoint" || !isParkableWaitSignalCheckpoint(message))
+      return;
+    if (this.#checkpointResumeState.wasResumedDuringCheckpointHandling(message.workflowId, resumeVersionBeforeCheckpointHandling))
+      return;
+    if (this.#ownership.parkActive(message.workflowId, worker))
+      this.#pool.release(worker);
+  }
+  #handleWorkerError(worker, errorEvent) {
+    this.#discardWorkerAndFailWorkflows(worker, {
+      targetCategory: "system",
+      targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
+      otherCategory: "system",
+      otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
+    });
+  }
+  #releaseActiveWorker(workflowId) {
+    const worker = this.#ownership.releaseActive(workflowId);
+    if (worker) {
+      this.#turnWatchdog.clear(worker);
+      this.#detachWorkerListenersIfIdle(worker);
+      this.#pool.release(worker);
+    }
+  }
+  #attachWorkerListeners(worker) {
+    this.#workerListeners.attach(worker, {
+      message: (message) => {
+        this.#handleWorkerMessage(worker, message).catch(() => {});
+      },
+      error: (errorEvent) => {
+        this.#handleWorkerError(worker, errorEvent);
+      },
+      messageerror: () => {
+        this.#discardWorkerAndFailWorkflows(worker, {
+          targetCategory: "system",
+          targetError: "Worker messageerror event",
+          otherCategory: "system",
+          otherError: "Worker messageerror event"
+        });
+      }
+    });
+  }
+  #detachWorkerListenersIfIdle(worker) {
+    this.#workerListeners.detachIfIdle(worker, (candidate) => this.#ownership.workerIsIdle(candidate));
+  }
+  #assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
+    const failure = this.#protocolGuard.validateHostToWorkerMessage(message);
+    if (!failure)
+      return !0;
+    if (worker) {
+      this.#discardWorkerAndFailWorkflows(worker, {
+        targetWorkflowId: workflowId,
+        targetCategory: failure.failureCategory,
+        targetError: failure.error,
+        otherCategory: "system",
+        otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
+      });
+      return !1;
+    }
+    this.#emit({
+      type: "failed",
+      workflowId,
+      error: failure.error,
+      failureCategory: failure.failureCategory
+    });
+    return !1;
+  }
+  #acceptWorkerMessage(worker, message) {
+    const result = this.#protocolGuard.acceptWorkerMessage(worker, message);
+    if (result.accepted)
+      return !0;
+    this.#discardWorkerAndFailWorkflows(worker, {
+      ...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
+      targetCategory: result.failure.failureCategory,
+      targetError: result.failure.error,
+      otherCategory: "system",
+      otherError: result.failure.otherError
+    });
+    return !1;
+  }
+  #handleTurnTimeout(turn) {
+    this.#discardWorkerAndFailWorkflows(turn.worker, {
+      targetWorkflowId: turn.workflowId,
+      targetCategory: "timeout",
+      targetError: `Worker workflow turn timed out after ${this.#workflowTurnTimeoutMs}ms`,
+      otherCategory: "timeout",
+      otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
+    });
+  }
+  #discardWorkerAndFailWorkflows(worker, options) {
+    const workflowIds = this.#ownership.workflowIdsForWorker(worker);
+    if (workflowIds.length === 0) {
+      this.#turnWatchdog.clear(worker);
+      return;
+    }
+    for (const workflowId of workflowIds) {
+      const isTarget = workflowId === options.targetWorkflowId;
+      this.#ownership.forgetWorkflow(workflowId);
+      this.#checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
+      if (isTarget && options.skipTarget)
+        continue;
+      this.#emit({
+        type: "failed",
+        workflowId,
+        error: isTarget ? options.targetError ?? options.otherError : options.otherError,
+        failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
+      });
+    }
+    this.#turnWatchdog.clear(worker);
+    this.#workerListeners.detach(worker);
+    this.#pool.discard(worker);
+  }
+  #handleBroadcastMessage(data) {
+    if (data.type === "signal:received" && typeof data.workflowId === "string") {
+      const targetWorker = this.#ownership.getTargetWorker(data.workflowId);
+      if (targetWorker)
+        targetWorker.postMessage(data);
+    }
+  }
+  #emit(message) {
+    const result = this.#messageHandler?.(message);
+    if (result instanceof Promise)
+      result.catch(() => {});
+  }
+}

package/dist/core/worker-listener-registry.d.ts

@@ -0,0 +1,11 @@
+export declare class WorkerListenerRegistry {
+    #private;
+    attach(worker: Worker, handlers: {
+        message: (message: unknown) => void;
+        error: (errorEvent: ErrorEvent) => void;
+        messageerror: () => void;
+    }): void;
+    detach(worker: Worker): void;
+    detachIfIdle(worker: Worker, workerIsIdle: (worker: Worker) => boolean): void;
+    detachAll(): void;
+}

package/dist/core/worker-listener-registry.js

@@ -0,0 +1,35 @@
+export class WorkerListenerRegistry {
+  #workerListeners = new Map;
+  attach(worker, handlers) {
+    if (this.#workerListeners.has(worker))
+      return;
+    const listeners = {
+      message: (event) => {
+        handlers.message(event.data);
+      },
+      error: handlers.error,
+      messageerror: handlers.messageerror
+    };
+    this.#workerListeners.set(worker, listeners);
+    worker.addEventListener("message", listeners.message);
+    worker.addEventListener("error", listeners.error);
+    worker.addEventListener("messageerror", listeners.messageerror);
+  }
+  detach(worker) {
+    const listeners = this.#workerListeners.get(worker);
+    if (!listeners)
+      return;
+    worker.removeEventListener("message", listeners.message);
+    worker.removeEventListener("error", listeners.error);
+    worker.removeEventListener("messageerror", listeners.messageerror);
+    this.#workerListeners.delete(worker);
+  }
+  detachIfIdle(worker, workerIsIdle) {
+    if (workerIsIdle(worker))
+      this.detach(worker);
+  }
+  detachAll() {
+    for (const worker of this.#workerListeners.keys())
+      this.detach(worker);
+  }
+}

package/dist/core/worker-message-helpers.d.ts

@@ -0,0 +1,5 @@
+import type { WorkerOutboundMessage } from './types.ts';
+export declare function emitWorkerMessageToEngine(handler: ((message: WorkerOutboundMessage) => void | Promise<void>) | null, message: WorkerOutboundMessage): boolean | Promise<boolean>;
+export declare function isParkableWaitSignalCheckpoint(message: Extract<WorkerOutboundMessage, {
+    type: 'checkpoint';
+}>): boolean;

package/dist/core/worker-message-helpers.js

@@ -0,0 +1,14 @@
+export function emitWorkerMessageToEngine(handler, message) {
+  try {
+    const emitResult = handler?.(message);
+    if (emitResult instanceof Promise)
+      return emitResult.then(() => !1, () => !0);
+    return !1;
+  } catch {
+    return !0;
+  }
+}
+export function isParkableWaitSignalCheckpoint(message) {
+  const operationRequest = message.operationRequest;
+  return operationRequest.type === "wait-signal" || operationRequest.kind === "signal-wait";
+}

package/dist/core/worker-protocol-guard.d.ts

@@ -0,0 +1,20 @@
+import type { FailureCategory, WorkerInboundMessage, WorkerOutboundMessage } from './types.ts';
+import type { WorkerTurnWatchdog } from './worker-turn-watchdog.ts';
+export interface WorkerProtocolFailure {
+    failureCategory: FailureCategory;
+    error: string;
+    targetWorkflowId?: string;
+    otherError: string;
+}
+export declare class WorkerProtocolGuard {
+    #private;
+    constructor(maxProtocolMessageBytes: number | undefined, requireProtocolVersion: boolean, turnWatchdog: WorkerTurnWatchdog);
+    validateHostToWorkerMessage(message: WorkerInboundMessage): WorkerProtocolFailure | null;
+    acceptWorkerMessage(worker: Worker, message: unknown): {
+        accepted: true;
+        message: WorkerOutboundMessage;
+    } | {
+        accepted: false;
+        failure: WorkerProtocolFailure;
+    };
+}

package/dist/core/worker-protocol-guard.js

@@ -0,0 +1,69 @@
+import {
+  assertWorkerOutboundMessageShape,
+  assertWorkerProtocolMessageWithinLimit,
+  WORKER_PROTOCOL_VERSION,
+  WorkerProtocolError,
+  WorkerProtocolMessageSizeError
+} from "./worker-protocol.js";
+
+export class WorkerProtocolGuard {
+  #maxProtocolMessageBytes;
+  #requireProtocolVersion;
+  #turnWatchdog;
+  constructor(maxProtocolMessageBytes, requireProtocolVersion, turnWatchdog) {
+    this.#maxProtocolMessageBytes = maxProtocolMessageBytes;
+    this.#requireProtocolVersion = requireProtocolVersion;
+    this.#turnWatchdog = turnWatchdog;
+  }
+  validateHostToWorkerMessage(message) {
+    try {
+      assertWorkerProtocolMessageWithinLimit(message, this.#maxProtocolMessageBytes);
+      return null;
+    } catch (error) {
+      return protocolFailure(error, {
+        otherError: "Worker protocol send failed"
+      });
+    }
+  }
+  acceptWorkerMessage(worker, message) {
+    try {
+      assertWorkerOutboundMessageShape(message);
+      assertWorkerProtocolMessageWithinLimit(message, this.#maxProtocolMessageBytes);
+      if (this.#requireProtocolVersion)
+        this.#assertExpectedWorkerTurn(worker, message);
+      return { accepted: !0, message };
+    } catch (error) {
+      const targetWorkflowId = workflowIdFromWorkerMessage(message);
+      return {
+        accepted: !1,
+        failure: protocolFailure(error, {
+          ...targetWorkflowId === void 0 ? {} : { targetWorkflowId },
+          otherError: targetWorkflowId === void 0 ? "Worker discarded after protocol violation" : `Worker discarded after protocol violation for workflow: ${targetWorkflowId}`
+        })
+      };
+    }
+  }
+  #assertExpectedWorkerTurn(worker, message) {
+    if (message.protocolVersion !== WORKER_PROTOCOL_VERSION)
+      throw new WorkerProtocolError("Worker protocol version mismatch");
+    const turn = this.#turnWatchdog.get(worker);
+    if (!turn)
+      throw new WorkerProtocolError("Worker message arrived outside an active turn");
+    if (message.turnId !== turn.turnId || message.workflowId !== turn.workflowId)
+      throw new WorkerProtocolError("Worker message did not match the active turn");
+  }
+}
+function protocolFailure(error, options) {
+  return {
+    ...options.targetWorkflowId === void 0 ? {} : { targetWorkflowId: options.targetWorkflowId },
+    failureCategory: error instanceof WorkerProtocolMessageSizeError ? "resource" : "system",
+    error: error instanceof Error ? error.message : String(error),
+    otherError: options.otherError
+  };
+}
+function workflowIdFromWorkerMessage(message) {
+  if (typeof message !== "object" || message === null)
+    return;
+  const workflowId = message.workflowId;
+  return typeof workflowId === "string" ? workflowId : void 0;
+}

package/dist/core/worker-protocol.d.ts

@@ -0,0 +1,27 @@
+import type { ContextOperationRequest } from './context.ts';
+import type { FailureCategory, OperationRequest, WorkerOutboundMessage } from './types.ts';
+import { type WorkerReplayOperationSignature } from './types/checkpoint.ts';
+export { WORKER_REPLAY_SIGNATURE_FORMAT, type WorkerReplayOperationSignature, } from './types/checkpoint.ts';
+export declare const WORKER_PROTOCOL_VERSION = 1;
+export declare const DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000;
+export declare const DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576;
+export declare const MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096;
+export declare class WorkerProtocolError extends Error {
+    constructor(message: string);
+}
+export declare class WorkerProtocolMessageSizeError extends WorkerProtocolError {
+    readonly actualBytes: number;
+    readonly maxBytes: number;
+    constructor(actualBytes: number, maxBytes: number);
+}
+export declare function createBoundedWorkerFailureMessage(parameters: {
+    workflowId: string;
+    error: string;
+    failureCategory: FailureCategory;
+    turnId?: number;
+}): WorkerOutboundMessage;
+export declare function estimateWorkerProtocolMessageBytes(message: unknown): number;
+export declare function assertWorkerProtocolMessageWithinLimit(message: unknown, maxBytes: number | undefined): number;
+export declare function assertWorkerOutboundMessageShape(message: unknown): asserts message is WorkerOutboundMessage;
+export declare function createWorkerReplayOperationSignature(operation: OperationRequest | ContextOperationRequest, maxStableFieldsBytes: number): Promise<WorkerReplayOperationSignature>;
+export declare function workerReplayOperationSignaturesEqual(left: WorkerReplayOperationSignature, right: WorkerReplayOperationSignature): boolean;