@swirls/looms 0.4.0 → 0.5.0

package/dist/runtime/server/routes/effects.js

@@ -1,4 +1,5 @@
 import { Effect, Schema } from 'effect';
+import { JsonValueSchema, } from '@swirls/looms/core';
 import { EventInputBodySchema, provideStore, readJson } from '../http.js';
 import { integer, oneOf, route } from '../route-table.js';
 const WORKER_CALLBACK_KINDS = [
@@ -11,16 +12,46 @@ const WORKER_CALLBACK_KINDS = [
 const WorkerCallbackBodySchema = Schema.Struct({
     events: Schema.optional(Schema.Array(EventInputBodySchema)),
     error: Schema.optional(Schema.String),
+    nonRetryable: Schema.optional(Schema.Boolean),
+    retryAfterMs: Schema.optional(Schema.Finite),
+    details: Schema.optional(JsonValueSchema),
 });
 function workerCallbackInput(params, body) {
     const callback = { effectId: params.effectId, attempt: params.attempt };
     switch (params.action) {
         case 'complete':
             return { ...callback, kind: 'complete', events: body.events ?? [] };
-        case 'fail':
-            return body.error ? { ...callback, kind: 'fail', error: body.error } : null;
-        default:
+        case 'fail': {
+            if (body.error === undefined) {
+                return null;
+            }
+            const failure = {
+                ...callback,
+                kind: 'fail',
+                error: body.error,
+            };
+            if (body.nonRetryable !== undefined) {
+                failure.nonRetryable = body.nonRetryable;
+            }
+            if (body.retryAfterMs !== undefined) {
+                failure.retryAfterMs = body.retryAfterMs;
+            }
+            return failure;
+        }
+        case 'heartbeat': {
+            const heartbeat = { ...callback, kind: 'heartbeat' };
+            if (body.details !== undefined) {
+                heartbeat.details = body.details;
+            }
+            return heartbeat;
+        }
+        case 'started':
+        case 'cancelled':
             return { ...callback, kind: params.action };
+        default: {
+            const exhaustive = params.action;
+            return exhaustive;
+        }
     }
 }
 export function effectRoutes({ runtime, store }) {
@@ -39,6 +70,17 @@ export function effectRoutes({ runtime, store }) {
             access: 'operations',
             handle: ({ params }) => provideStore(runtime.retryEffect(params.runId, params.effectId), store).pipe(Effect.map((state) => Response.json({ runId: params.runId, effectId: params.effectId, state }))),
         }),
+        route({
+            method: 'POST',
+            path: '/runs/:runId/effects/:effectId/:attempt/secrets',
+            name: 'effects.workerSecrets',
+            access: 'worker',
+            params: { attempt: integer },
+            handle: ({ params }) => provideStore(runtime.workerSecrets(params.runId, {
+                effectId: params.effectId,
+                attempt: params.attempt,
+            }), store).pipe(Effect.map((result) => Response.json(result, { headers: { 'cache-control': 'no-store' } }))),
+        }),
         route({
             method: 'POST',
             path: '/runs/:runId/effects/:effectId/:attempt/:action',
@@ -51,8 +93,8 @@ export function effectRoutes({ runtime, store }) {
                 if (!input) {
                     return Response.json({ error: 'error required' }, { status: 400 });
                 }
-                const state = yield* provideStore(runtime.workerCallback(params.runId, input), store);
-                return Response.json({ runId: params.runId, state });
+                const result = yield* provideStore(runtime.workerCallback(params.runId, input), store);
+                return Response.json(result);
             }),
         }),
     ];

package/dist/runtime/server/routes/runs.js

@@ -1,5 +1,5 @@
 import { Effect, Schema } from 'effect';
-import { createRunId, EventStoreTag, JsonValueSchema, stringifyJson, } from '@swirls/looms/core';
+import { createRunId, EventStoreTag, JsonValueSchema, RunAttributesSchema, readTraceHeaders, SecretContextSchema, stringifyJson, } from '@swirls/looms/core';
 import { createEventStreamResponse } from '../../sse.js';
 import { acceptsEventStream, EventInputBodySchema, extractIdempotencyKey, provideStore, readIntegerQuery, readJson, readOptionalJson, RequestQueryError, } from '../http.js';
 import { route } from '../route-table.js';
@@ -11,6 +11,8 @@ const StartRunBodySchema = Schema.Struct({
     input: Schema.optional(JsonValueSchema),
     runId: Schema.optional(Schema.String),
     idempotencyKey: Schema.optional(Schema.String),
+    attributes: Schema.optional(RunAttributesSchema),
+    secrets: Schema.optional(SecretContextSchema),
 });
 const SignalBodySchema = Schema.Struct({
     events: Schema.optional(Schema.Array(EventInputBodySchema)),
@@ -108,6 +110,9 @@ export function runRoutes({ runtime, store }) {
                     input: body.input ?? null,
                     runId,
                     idempotencyKey,
+                    attributes: body.attributes,
+                    secrets: body.secrets,
+                    trace: readTraceHeaders(req.headers),
                 };
                 if (wantsRunStream(req, url)) {
                     return streamStartRun(req, runtime, store, args);

package/dist/runtime/server/run-route.d.ts

@@ -0,0 +1,18 @@
+import { Effect } from 'effect';
+import { type EventStore, type RunAttributes } from '@swirls/looms/core';
+import type { LoomsRuntime } from '../runtime';
+import type { AuthorizedAccess, RouteInfo } from './auth';
+/**
+ * Attributes are immutable and recorded on `runtime.run.started`, which is seq 1 of every
+ * untrimmed log, forks included. Only a trimmed log needs the snapshot-backed fold.
+ */
+export declare function readStoredAttributes(runtime: LoomsRuntime, store: EventStore, runId: string): Effect.Effect<RunAttributes, Error>;
+/**
+ * What `authorize` sees for a request about one run: the run's attributes are included when it has
+ * any. Every path that serves a run, including a WebSocket upgrade, authorizes through this.
+ */
+export declare function runRouteInfo(runtime: LoomsRuntime, store: EventStore, route: {
+    readonly access: AuthorizedAccess;
+    readonly name: string;
+    readonly runId: string;
+}): Effect.Effect<RouteInfo, Error>;

package/dist/runtime/server/run-route.js

@@ -0,0 +1,26 @@
+import { Effect } from 'effect';
+import { EventStoreTag, hasEntries, isJsonObject, readAttributes, } from '@swirls/looms/core';
+const noAttributes = {};
+/**
+ * Attributes are immutable and recorded on `runtime.run.started`, which is seq 1 of every
+ * untrimmed log, forks included. Only a trimmed log needs the snapshot-backed fold.
+ */
+export function readStoredAttributes(runtime, store, runId) {
+    const folded = runtime.getRun(runId).pipe(Effect.provideService(EventStoreTag, store), Effect.map((state) => state.attributes));
+    return store.read(runId, { fromSeq: 1, limit: 1 }).pipe(Effect.flatMap(([first]) => {
+        if (first === undefined) {
+            return Effect.succeed(noAttributes);
+        }
+        if (first.type === 'runtime.run.started' && isJsonObject(first.payload)) {
+            return Effect.succeed(readAttributes(first.payload.attributes));
+        }
+        return folded;
+    }), Effect.catch(() => folded));
+}
+/**
+ * What `authorize` sees for a request about one run: the run's attributes are included when it has
+ * any. Every path that serves a run, including a WebSocket upgrade, authorizes through this.
+ */
+export function runRouteInfo(runtime, store, route) {
+    return readStoredAttributes(runtime, store, route.runId).pipe(Effect.map((attributes) => (hasEntries(attributes) ? { ...route, attributes } : route)));
+}

package/dist/runtime/server.d.ts

@@ -1,7 +1,9 @@
 export { createFetchHandler, handlesPath, type FetchHandlerOptions } from './server/fetch-handler';
 export { isLoomsApiPath } from './server/http';
 export { mountDebugger, type DebuggerMount } from './server/debugger-host';
-export { authAllowed, authDenied, bearerAuth, defaultAuthorize, type AuthDecision, type AuthorizedAccess, type Authorize, type BearerTokens, type RouteAccess, type RouteInfo, } from './server/auth';
+export { runRouteInfo } from './server/run-route';
+export { authAllowed, authDenied, authResponse, bearerAuth, defaultAuthorize, type AuthDecision, type AuthorizedAccess, type Authorize, type BearerTokens, type RouteAccess, type RouteInfo, } from './server/auth';
+export { grantAuth, issueRunGrant, grantSubprotocol, runGrantToken, verifyRunGrant, GrantError, type GrantAuthOptions, type GrantFailure, type RunGrant, } from './server/grant';
 export interface RunningServer {
     port: number;
     stop: () => void;

package/dist/runtime/server.js

@@ -1,7 +1,9 @@
 export { createFetchHandler, handlesPath } from './server/fetch-handler.js';
 export { isLoomsApiPath } from './server/http.js';
 export { mountDebugger } from './server/debugger-host.js';
-export { authAllowed, authDenied, bearerAuth, defaultAuthorize, } from './server/auth.js';
+export { runRouteInfo } from './server/run-route.js';
+export { authAllowed, authDenied, authResponse, bearerAuth, defaultAuthorize, } from './server/auth.js';
+export { grantAuth, issueRunGrant, grantSubprotocol, runGrantToken, verifyRunGrant, GrantError, } from './server/grant.js';
 export function serveHttp(fetchHandler, options) {
     const port = options?.port ?? 8787;
     const hostname = options?.hostname ?? '0.0.0.0';

package/dist/runtime/sse.js

@@ -1,5 +1,6 @@
 import { Effect, Fiber, Schedule, Stream } from 'effect';
-import { encodeLoomsEvent, stringifyJson } from '@swirls/looms/core';
+import { encodeLoomsEvent, encodeSubscriptionFrame, } from '@swirls/looms/core';
+import { readSubscriptionBounds } from './subscription.js';
 function disableBunSocketTimeout(request) {
     if (!request) {
         return;
@@ -64,12 +65,33 @@ export function createEventStreamResponse(options) {
                     close();
                 }
             };
-            subscriptionFiber = Effect.runFork(store.subscribe(runId, { fromSeq }).pipe(Stream.tap((event) => Effect.sync(() => {
-                write(`id: ${event.seq}\ndata: ${stringifyJson({ batch: [encodeLoomsEvent(event)] })}\n\n`);
-            })), Stream.runDrain));
             // Send an immediate keepalive so bytes and headers flush over the wire immediately,
             // preventing dev server proxies and Bun.serve from aborting quiet connections.
+            // It has to be enqueued before the subscription fiber, which writes hello synchronously.
             write(': keepalive\n\n');
+            subscriptionFiber = Effect.runFork(Effect.gen(function* () {
+                const bounds = yield* readSubscriptionBounds(store, runId);
+                if (fromSeq < bounds.head) {
+                    write(`data: ${encodeSubscriptionFrame({
+                        type: 'resync',
+                        head: bounds.head,
+                        reason: 'trimmed',
+                    })}\n\n`);
+                    close();
+                    return;
+                }
+                write(`data: ${encodeSubscriptionFrame({
+                    type: 'hello',
+                    head: bounds.head,
+                    tail: bounds.tail,
+                })}\n\n`);
+                yield* store.subscribe(runId, { fromSeq }).pipe(Stream.tap((event) => Effect.sync(() => {
+                    write(`id: ${event.seq}\ndata: ${encodeSubscriptionFrame({
+                        type: 'batch',
+                        batch: [encodeLoomsEvent(event)],
+                    })}\n\n`);
+                })), Stream.runDrain);
+            }));
             heartbeatFiber = Effect.runFork(Effect.repeat(Effect.sleep(heartbeatMs).pipe(Effect.andThen(Effect.sync(() => write(': keepalive\n\n')))), Schedule.forever).pipe(Effect.asVoid));
             const startResult = onStart?.(write, close);
             const startEffect = Effect.isEffect(startResult)

package/dist/runtime/subscription.d.ts

@@ -0,0 +1,8 @@
+import { Effect } from 'effect';
+import { type EventStore, type EventStoreError, type SubscriptionOpening } from '@swirls/looms/core';
+export declare function readSubscriptionBounds(store: EventStore, runId: string): Effect.Effect<{
+    readonly head: number;
+    readonly tail: number;
+}, EventStoreError>;
+/** Hello plus the retained events, or a resync when `fromSeq` is below the log head. */
+export declare function loadSubscriptionOpening(store: EventStore, runId: string, fromSeq: number): Effect.Effect<SubscriptionOpening, EventStoreError>;

package/dist/runtime/subscription.js

@@ -0,0 +1,23 @@
+import { Effect } from 'effect';
+import { subscriptionReplay, } from '@swirls/looms/core';
+export function readSubscriptionBounds(store, runId) {
+    if (store.bounds) {
+        return store.bounds(runId);
+    }
+    return store.tail(runId).pipe(Effect.map((tail) => ({ head: 1, tail })));
+}
+/** Hello plus the retained events, or a resync when `fromSeq` is below the log head. */
+export function loadSubscriptionOpening(store, runId, fromSeq) {
+    return Effect.gen(function* () {
+        const bounds = yield* readSubscriptionBounds(store, runId);
+        const decision = subscriptionReplay(bounds.head, fromSeq);
+        if (decision.kind === 'resync') {
+            return { kind: 'resync', head: decision.head };
+        }
+        const events = yield* store.read(runId, {
+            fromSeq: decision.fromSeq,
+        });
+        const hello = { type: 'hello', head: bounds.head, tail: bounds.tail };
+        return { kind: 'open', hello, events };
+    });
+}

package/dist/runtime/types.d.ts

@@ -1,5 +1,5 @@
 import type { Effect } from 'effect';
-import type { AnyRuntimeModule, ComposedRegistry, EventEnvelope, EventInput, WorkerCallbackInput, EventStore, EventStoreAppendError, EventStoreError, EventStoreFencedError, EventStoreTag, EventStoreTrimCoverage, EventStoreTruncationError, JsonValue, ProjectionDefinition, ReplayStep, RunState, RunSummary, SnapshotStore, SnapshotStoreError } from '@swirls/looms/core';
+import type { AnyRuntimeModule, ComposedRegistry, DefinitionResolver, EventEnvelope, EventInput, WorkerCallback, WorkerCallbackInput, DefinitionResolveError, WorkerCallbackResult, WorkerSecretsResult, EventStore, EventStoreAppendError, EventStoreError, EventStoreFencedError, EventStoreTag, EventStoreTrimCoverage, RunLauncher, RunLaunchSource, RunAttributes, SecretContext, TraceContext, EventStoreTruncationError, JsonValue, ProjectionDefinition, ReplayStep, RunState, RunSummary, SnapshotStore, SnapshotStoreError } from '@swirls/looms/core';
 import type { EventStoreTrimmedError } from '@swirls/looms/core';
 import type { DuplicateEffectDispatchError, MaxWakeIterationsError } from './errors';
 import type { RuntimeObserver } from './observer';
@@ -7,6 +7,12 @@ import type { WakeScheduler } from './wake-scheduler';
 export type WakeError = DuplicateEffectDispatchError | MaxWakeIterationsError | EventStoreTruncationError | EventStoreTrimmedError | EventStoreFencedError | EventStoreError | Error;
 export interface CreateRuntimeOptions<TModules extends readonly AnyRuntimeModule[] = readonly AnyRuntimeModule[]> {
     readonly modules: TModules;
+    /**
+     * Loads a definition that was not registered on a module. Registered definitions are checked
+     * first. Lookups are cached, including misses. Start fails with `UnknownDefinitionError`
+     * and writes no events when the definition is missing.
+     */
+    readonly definitionResolver?: DefinitionResolver;
     readonly store?: EventStore;
     /**
      * Durable events between mid-wake snapshots. Default 200; `0` disables.
@@ -15,6 +21,8 @@ export interface CreateRuntimeOptions<TModules extends readonly AnyRuntimeModule
      */
     readonly snapshotEvery?: number;
     readonly maxWakeIterations?: number;
+    /** Maximum concurrent in-flight worker effects dispatched per run (fairness guard). */
+    readonly maxInFlightWorkerEffects?: number;
     /** Maximum active and queued mutation operations per run. Default 256. */
     readonly maxPendingRunOperations?: number;
     /** Defaults to the EventStore's attached store or an in-memory SnapshotStore. */
@@ -39,6 +47,13 @@ export interface CreateRuntimeOptions<TModules extends readonly AnyRuntimeModule
      * Defaults to an in-process Effect fiber scheduler.
      */
     readonly scheduler?: WakeScheduler;
+    /**
+     * Starts runs from effect handlers. Required for schedules on an actor or Durable Object host.
+     * Single-store `createLooms` installs one that launches into its own store.
+     */
+    readonly launcher?: RunLauncher;
+    /** Install a launcher that starts runs in this runtime's store. `createLooms` sets this. */
+    readonly localLauncher?: boolean;
     readonly worker?: EffectWorker;
     /** Failure-isolated, framework-neutral operational event sink. */
     readonly observer?: RuntimeObserver;
@@ -49,6 +64,24 @@ export interface EffectWorkerTask {
     readonly attempt: number;
     readonly type: string;
     readonly input: JsonValue;
+    /** Progress from the previous attempt's last heartbeat, when it reported any. */
+    readonly details?: JsonValue;
+    /** The run's W3C trace context, for parenting the worker's spans. */
+    readonly trace?: TraceContext;
+    /** The attempt's deadlines from the effect's retry policy, when it sets any. */
+    readonly deadlines?: EffectWorkerDeadlines;
+    /**
+     * Present only on the in-memory dispatch. A transport that persists the task must leave this out
+     * and let the worker read the secret context through an authenticated callback.
+     */
+    readonly secrets?: SecretContext;
+}
+/** Looms enforces these; a worker can use them to pace heartbeats or give up early. */
+export interface EffectWorkerDeadlines {
+    /** Epoch ms. A `started` callback after this is stale. */
+    readonly scheduleToStartAt?: number;
+    readonly startToCloseTimeoutMs?: number;
+    readonly heartbeatTimeoutMs?: number;
 }
 export interface EffectWorker {
     /** Effect types this worker claims, even when a local handler exists. */
@@ -57,6 +90,14 @@ export interface EffectWorker {
     readonly canDispatch?: (task: EffectWorkerTask) => boolean;
     dispatch(task: EffectWorkerTask): void | Promise<void>;
     cancel?(task: EffectWorkerTask): void | Promise<void>;
+    /**
+     * What a rejected `dispatch` means. With `'unacknowledged'` (the default) the task may still have
+     * arrived, so the attempt waits for its schedule-to-start timeout or an operator. With
+     * `'acknowledged'` a rejection means the transport refused it, so the attempt fails under the
+     * retry policy. A transport can still lose an accepted message, so every effect such a worker
+     * claims must set `retry.scheduleToStartTimeoutMs`.
+     */
+    readonly dispatchSemantics?: 'unacknowledged' | 'acknowledged';
 }
 export interface StartRunArgs {
     kind: string;
@@ -66,6 +107,22 @@ export interface StartRunArgs {
     runId?: string;
     threadId?: string;
     idempotencyKey?: string;
+    /** The effect that launched this run. The runtime sets it on launches made from effects. */
+    launchedBy?: RunLaunchSource;
+    /** Recorded on `runtime.run.started` to group related runs. */
+    correlationId?: string;
+    /** Immutable labels stored on the start event and the run summary. */
+    attributes?: RunAttributes;
+    /**
+     * Bearer material stored beside the log. A retry that sends a different context conflicts.
+     * Omitting it on a retry keeps the context already stored.
+     */
+    secrets?: SecretContext;
+    /**
+     * The caller's W3C trace context, recorded on `runtime.run.started` when it is a valid version-00
+     * `traceparent`. It is not part of the start identity, so a retry from another trace is not a conflict.
+     */
+    trace?: TraceContext;
 }
 export interface StartResult {
     runId: string;
@@ -91,18 +148,32 @@ export interface LoomsRuntime<TModules extends readonly AnyRuntimeModule[] = rea
     readonly modules: TModules;
     readonly registry: ComposedRegistry;
     startRun(args: StartRunArgs): Effect.Effect<StartResult, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
+    /**
+     * Commit a run's start, then schedule a wake instead of waiting for it. A duplicate start with
+     * the same identity is a no-op aside from the scheduled wake.
+     */
+    launchRun(args: StartRunArgs): Effect.Effect<StartResult, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
     signal(runId: string, events: ReadonlyArray<EventInput>, options?: {
         idempotencyKey?: string;
+        coalesce?: boolean;
+    }): Effect.Effect<RunState, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
+    /**
+     * Append events the runtime itself produced for another run. Unlike `signal`, this does not
+     * reject reserved runtime event types or rewrite their origin.
+     */
+    accept(runId: string, events: ReadonlyArray<EventInput>, options?: {
+        idempotencyKey?: string;
+        coalesce?: boolean;
     }): Effect.Effect<RunState, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
     wake(runId: string): Effect.Effect<RunState, WakeError, EventStoreTag>;
-    getRun(runId: string): Effect.Effect<RunState, EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, EventStoreTag>;
+    getRun(runId: string): Effect.Effect<RunState, EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag>;
     getEvents(runId: string, options?: {
         fromSeq?: number;
         limit?: number;
     }): Effect.Effect<EventEnvelope[], EventStoreError, EventStoreTag>;
     project<S>(runId: string, definition: ProjectionDefinition<S>): Effect.Effect<S, EventStoreError, EventStoreTag>;
     replayTo(runId: string, seq: number): Effect.Effect<ReplayStep | null, EventStoreError, EventStoreTag>;
-    inspectRun(runId: string): Effect.Effect<RunOperationalStatus, EventStoreError | EventStoreTrimmedError | EventStoreTruncationError, EventStoreTag>;
+    inspectRun(runId: string): Effect.Effect<RunOperationalStatus, EventStoreError | EventStoreTrimmedError | EventStoreTruncationError | DefinitionResolveError, EventStoreTag>;
     retryEffect(runId: string, effectId: string): Effect.Effect<RunState, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
     cancel(runId: string, threadId?: string): Effect.Effect<RunState, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
     /**
@@ -113,13 +184,21 @@ export interface LoomsRuntime<TModules extends readonly AnyRuntimeModule[] = rea
     fork(sourceRunId: string, options?: {
         runId?: string;
     }): Effect.Effect<StartResult, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
-    /** Applies a worker's report about one effect attempt; stale or foreign reports are ignored. */
-    workerCallback(runId: string, input: WorkerCallbackInput): Effect.Effect<RunState, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
+    /**
+     * Applies a worker's report about one effect attempt. A report for another attempt, or one that
+     * arrives after the attempt settled, is `stale` and changes nothing.
+     */
+    workerCallback(runId: string, input: WorkerCallbackInput): Effect.Effect<WorkerCallbackResult, Error | EventStoreAppendError | EventStoreError, EventStoreTag>;
+    /**
+     * The run's secret context for a worker attempt that has started and not settled. Any other
+     * attempt is `stale`, so a redelivered or superseded message never sees the secrets.
+     */
+    workerSecrets(runId: string, callback: WorkerCallback): Effect.Effect<WorkerSecretsResult, Error | EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, EventStoreTag>;
     readonly listRuns: Effect.Effect<string[], EventStoreError, EventStoreTag>;
     /** Indexed run rows. Folds a run only when its header has not been written yet. */
-    readonly listRunSummaries: Effect.Effect<readonly RunSummary[], EventStoreError | EventStoreTrimmedError | EventStoreTruncationError | SnapshotStoreError, EventStoreTag>;
-    readonly rescanTimers: Effect.Effect<number, EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, EventStoreTag>;
+    readonly listRunSummaries: Effect.Effect<readonly RunSummary[], EventStoreError | EventStoreTrimmedError | EventStoreTruncationError | SnapshotStoreError | DefinitionResolveError, EventStoreTag>;
+    readonly rescanTimers: Effect.Effect<number, EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag>;
     /** Reschedule every durable timer and effect deadline after restart or repair. */
-    readonly recoverDeadlines: Effect.Effect<number, EventStoreTruncationError | EventStoreTrimmedError | EventStoreError, EventStoreTag>;
+    readonly recoverDeadlines: Effect.Effect<number, EventStoreTruncationError | EventStoreTrimmedError | EventStoreError | DefinitionResolveError, EventStoreTag>;
     readonly dispose: Effect.Effect<void>;
 }

package/dist/runtime/wake-engine.d.ts

@@ -1,5 +1,5 @@
 import { type Context, type ManagedRuntime } from 'effect';
-import { type ComposedRegistry, type RegisteredDefinition } from '@swirls/looms/core';
+import { type ComposedRegistry, type DefinitionResolver, type RunLauncher } from '@swirls/looms/core';
 import { type RuntimeObserver } from './observer';
 import type { RuntimeRepository } from './repository';
 import type { RunExecutionContexts } from './run-execution-context';
@@ -8,13 +8,15 @@ import type { WakeScheduler } from './wake-scheduler';
 export interface WakeEngineOptions {
     readonly registry: ComposedRegistry;
     readonly services: ManagedRuntime.ManagedRuntime<Context.Service.Any, never>;
-    readonly definitions: ReadonlyMap<string, RegisteredDefinition>;
+    readonly definitionResolver: DefinitionResolver;
     readonly executionContexts: RunExecutionContexts;
     readonly repository: RuntimeRepository;
     readonly snapshotEvery: number;
     readonly maxWakeIterations: number;
+    readonly maxInFlightWorkerEffects?: number;
     readonly worker?: EffectWorker;
     readonly observer?: RuntimeObserver;
     readonly scheduler: () => WakeScheduler;
+    readonly launcher: () => RunLauncher;
 }
 export declare function createWakeEngine(options: WakeEngineOptions): LoomsRuntime['wake'];

package/dist/runtime/wake-engine.js

@@ -1,5 +1,6 @@
 import { Clock, Effect, Exit, Option } from 'effect';
-import { createEvent, EventStoreTag, foldRun, isRunTerminal, } from '@swirls/looms/core';
+import { createEvent, deferredUntil, EventStoreTag, foldRun, isRunTerminal, } from '@swirls/looms/core';
+import { childCloseCancellations, notifyLaunchingParent } from './child-run.js';
 import { nextRuntimeDeadline, planDeadlineTransitions, planTerminalCancellationTransitions, } from './deadline-transition-planner.js';
 import { DuplicateEffectDispatchError, MaxWakeIterationsError, RuntimeExecutionError, } from './errors.js';
 import { cascadedCancellations, stripSeq, synthesizedThreadFailed, waitSatisfiedEvents, } from './helpers.js';
@@ -10,13 +11,36 @@ function currentTimeMillis() {
     return Math.floor(performance.timeOrigin + performance.now());
 }
 export function createWakeEngine(options) {
-    const { registry, services, definitions, executionContexts, repository, snapshotEvery, maxWakeIterations, } = options;
+    const { registry, services, definitionResolver, executionContexts, repository, snapshotEvery, maxWakeIterations, } = options;
     const { appendObserved, applyAppend, liveAppends, loadCursor, persistSnapshot } = repository;
     const runCache = {
         delete: repository.deleteCachedCursor,
         set: repository.setCachedCursor,
     };
     const observe = (event) => notifyObserver(options.observer, event);
+    const closeLaunchedRuns = (store, runId, cursor) => Effect.gen(function* () {
+        let next = cursor;
+        const state = next.state;
+        if (isRunTerminal(state) && !state.childCloseSent && state.launchedRuns.length > 0) {
+            const requested = createEvent(runId, {
+                type: 'runtime.child.close.requested',
+                payload: { policy: 'terminate' },
+                threadId: state.rootThreadId,
+                origin: { type: 'system' },
+                idempotencyKey: `child-close:${runId}`,
+            });
+            const appended = yield* appendObserved(store, runId, stripSeq([requested]));
+            next = yield* applyAppend(store, runId, next, stripSeq([requested]), appended);
+            yield* Effect.forEach(state.launchedRuns, (child) => options.launcher().cancel(child.runId), {
+                concurrency: 1,
+                discard: true,
+            });
+        }
+        if (isRunTerminal(next.state)) {
+            yield* notifyLaunchingParent(options.launcher(), runId, next.state);
+        }
+        return next;
+    });
     return (runId) => Effect.gen(function* () {
         observe({ type: 'run.wake', runId, at: currentTimeMillis() });
         const executionContext = executionContexts.get(runId);
@@ -64,11 +88,18 @@ export function createWakeEngine(options) {
         };
         return yield* Effect.gen(function* () {
             const store = yield* EventStoreTag;
+            const secrets = store.runSecrets ? yield* store.runSecrets.get(runId) : {};
             const runInContext = Effect.runPromiseWith(yield* Effect.context());
             executionContext.liveStore = store;
             let finalState;
             do {
                 let cursor = yield* loadCursor(store, runId);
+                const closeEvents = childCloseCancellations(cursor.state);
+                if (closeEvents.length > 0) {
+                    const created = closeEvents.map((input) => createEvent(runId, input));
+                    const appended = yield* appendObserved(store, runId, stripSeq(created));
+                    cursor = yield* applyAppend(store, runId, cursor, stripSeq(created), appended);
+                }
                 if (isRunTerminal(cursor.state)) {
                     const now = yield* Clock.currentTimeMillis;
                     const events = planTerminalCancellationTransitions({
@@ -80,6 +111,7 @@ export function createWakeEngine(options) {
                         const appended = yield* appendObserved(store, runId, stripSeq(events));
                         cursor = yield* applyAppend(store, runId, cursor, stripSeq(events), appended);
                     }
+                    cursor = yield* closeLaunchedRuns(store, runId, cursor);
                     const nextDeadline = nextRuntimeDeadline(cursor.state, now);
                     if (nextDeadline !== undefined) {
                         yield* options.scheduler().schedule(runId, nextDeadline);
@@ -144,6 +176,8 @@ export function createWakeEngine(options) {
                         registry,
                         worker: options.worker,
                         now,
+                        maxInFlightWorkerEffects: options.maxInFlightWorkerEffects,
+                        secrets,
                     });
                     if (workerPlans.length > 0) {
                         for (const workerPlan of workerPlans) {
@@ -151,7 +185,7 @@ export function createWakeEngine(options) {
                                 const queuedAppend = yield* appendObserved(store, runId, stripSeq([workerPlan.queuedEvent]));
                                 cursor = yield* applyAppend(store, runId, cursor, stripSeq([workerPlan.queuedEvent]), queuedAppend);
                             }
-                            const dispatchEvent = yield* dispatchWorkerEffect(runId, workerPlan, options.worker);
+                            const dispatchEvent = yield* dispatchWorkerEffect(runId, workerPlan, options.worker, now);
                             const dispatchAppend = yield* appendObserved(store, runId, stripSeq([dispatchEvent]));
                             cursor = yield* applyAppend(store, runId, cursor, stripSeq([dispatchEvent]), dispatchAppend);
                         }
@@ -162,7 +196,9 @@ export function createWakeEngine(options) {
                         : []));
                     const outstanding = state.outstandingEffects.filter((item) => {
                         const execution = state.effectExecutions[item.effectId];
-                        return !((cancellationTargets.has(item.threadId) && item.effect.type !== 'runtime.cancel') ||
+                        const dueAt = deferredUntil(item.effect);
+                        return !((dueAt !== undefined && dueAt > now + 5) ||
+                            (cancellationTargets.has(item.threadId) && item.effect.type !== 'runtime.cancel') ||
                             isWorkerEffect({ runId, item, execution, registry, worker: options.worker }) ||
                             (execution?.status === 'retry_wait' &&
                                 execution.nextAttemptAt !== null &&
@@ -213,12 +249,24 @@ export function createWakeEngine(options) {
                     const produced = yield* executeLocalEffects({
                         registry,
                         services,
-                        definitions,
+                        definitionResolver,
                         executionContext,
                         liveAppends,
                         runInContext,
                         observe,
-                    }, { runId, outstanding, attempts, now });
+                        launcher: options.launcher(),
+                    }, {
+                        runId,
+                        context: {
+                            correlationId: cursor.state.startIdentity?.correlationId ?? runId,
+                            attributes: cursor.state.attributes ?? {},
+                            secrets,
+                            trace: cursor.state.startIdentity?.trace ?? null,
+                        },
+                        outstanding,
+                        attempts,
+                        now,
+                    });
                     if (produced.length === 0) {
                         break;
                     }
@@ -271,6 +319,9 @@ export function createWakeEngine(options) {
                     const appendResult = yield* appendObserved(store, runId, stripSeq([completed]));
                     cursor = yield* applyAppend(store, runId, cursor, stripSeq([completed]), appendResult);
                 }
+                if (isRunTerminal(cursor.state)) {
+                    cursor = yield* closeLaunchedRuns(store, runId, cursor);
+                }
                 // Snapshot on park: the next wake (on any instance) starts from here.
                 cursor = yield* persistSnapshot(store, runId, cursor);
                 const nextDeadline = nextRuntimeDeadline(cursor.state, plannedAt);

package/dist/runtime/worker-callbacks.d.ts

@@ -1,5 +1,5 @@
 import { Effect } from 'effect';
-import { EventStoreTag, type ComposedRegistry, type RunState, type WorkerCallbackInput } from '@swirls/looms/core';
+import { EventStoreTag, type ComposedRegistry, type WorkerCallback, type WorkerCallbackInput, type WorkerCallbackResult } from '@swirls/looms/core';
 import type { RunIngress } from './ingress';
 import { type RuntimeObserver } from './observer';
 import type { RuntimeRepository } from './repository';
@@ -12,5 +12,15 @@ export interface WorkerCallbacksOptions {
     readonly wake: LoomsRuntime['wake'];
 }
 export declare function createWorkerCallbacks(options: WorkerCallbacksOptions): {
-    workerCallback: (runId: string, callback: WorkerCallbackInput) => Effect.Effect<RunState, import("./errors").DuplicateEffectDispatchError | Error | import("@swirls/looms/core").EventPayloadTooLargeError | import("@swirls/looms/core").EventStoreConflictError | import("@swirls/looms/core").EventStoreError | import("@swirls/looms/core").EventStoreFencedError | import("@swirls/looms/core").EventStoreTrimmedError | import("@swirls/looms/core").EventStoreTruncationError | import("./errors").MaxWakeIterationsError | import("./ingress").RunOverloadedError | import("./errors").RuntimeExecutionError, EventStoreTag>;
+    workerCallback: (runId: string, callback: WorkerCallbackInput) => Effect.Effect<WorkerCallbackResult, import("@swirls/looms/core").DefinitionResolveError | import("./errors").DuplicateEffectDispatchError | Error | import("@swirls/looms/core").EventPayloadTooLargeError | import("@swirls/looms/core").EventStoreConflictError | import("@swirls/looms/core").EventStoreError | import("@swirls/looms/core").EventStoreFencedError | import("@swirls/looms/core").EventStoreTrimmedError | import("@swirls/looms/core").EventStoreTruncationError | import("./errors").MaxWakeIterationsError | import("./ingress").RunOverloadedError | import("./errors").RuntimeExecutionError, EventStoreTag>;
+    workerSecrets: (runId: string, callback: WorkerCallback) => Effect.Effect<{
+        status: "stale";
+        secrets?: undefined;
+    } | {
+        status: "cancelled";
+        secrets?: undefined;
+    } | {
+        status: "accepted";
+        secrets: import("@swirls/looms/core").SecretContext;
+    }, Error | import("@swirls/looms/core").EventStoreError, EventStoreTag>;
 };