@powerhousedao/reactor-workflow 6.2.3-dev.54 → 6.2.3-dev.56

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,13 +1,12 @@
1
- import { $ as isPrivateAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerError, H as PieceWorkerPool, I as SecretDeletedError, J as redact, K as PieceWorkerTimeoutError, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as DEFAULT_EGRESS_POLICY, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerDisposedError, X as rememberSecrets, Y as redactMessage, Z as secretsFor, _ as resolvedBlock, _t as isIndeterminateError, at as pieceModuleRef, b as resolutionOf, c as canonicalJson, ct as UnsupportedPieceFeatureError, d as DynamicPropertiesError, dt as unsupportedTrigger, et as parseAddress, f as checkDynamicProperties, g as isHostBound, gt as setPieceRegistryUrl, h as CompositeBlockExecutor, ht as pieceRegistrySource, i as packagePieces, it as bundleResolver, j as declaredReactorConnectionIds, l as hashOf, lt as authMethodFor, m as ActivepiecesBlockExecutor, mt as npmRegistryBaseUrl, n as installedPiece, nt as FileTooLargeError, o as errorNameOf, ot as sourcedResolver, p as effectiveRetryPolicy, q as containsRedactedMarker, r as installedPieces, rt as maxFileBytes, s as runWorkflow, t as PieceRegistry, tt as extractDedupeKey, u as stepConfigHash, ut as unsupportedAuth, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-0pdOQQdk.js";
1
+ import { $ as rememberSecrets, A as AttachmentCache, B as SecretNotFoundError, C as blockLabel, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as PieceWorkerPool, H as secretRefFromId, I as resolveStepInput, J as PieceWorkerError, K as PieceWorker, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as redactMessage, S as storeHandlers, T as blockPorts, U as stepBlock, V as parseSecretRef, W as triggerBlock, X as containsRedactedMarker, Y as PieceWorkerTimeoutError, Z as redact, _ as effectiveRetryPolicy, _t as npmRegistryBaseUrl, a as PieceRegistry, at as FileTooLargeError, b as isHostBound, bt as isIndeterminateError, c as packagePieces, ct as pieceModuleRef, d as runWorkflow, dt as UnsupportedPieceFeatureError, et as secretsFor, f as canonicalJson, ft as authMethodFor, g as checkDynamicProperties, h as DynamicPropertiesError, i as tracedSteps, it as extractDedupeKey, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as sourcedResolver, m as stepConfigHash, mt as unsupportedTrigger, n as WorkflowTelemetry, nt as isPrivateAddress, o as installedPiece, ot as maxFileBytes, p as hashOf, pt as unsupportedAuth, q as PieceWorkerDisposedError, r as tracedMethods, rt as parseAddress, s as installedPieces, st as bundleResolver, t as TELEMETRY_SCOPE, tt as DEFAULT_EGRESS_POLICY, u as errorNameOf, v as ActivepiecesBlockExecutor, vt as pieceRegistrySource, w as unpinnedNote, x as resolvedBlock, y as CompositeBlockExecutor, yt as setPieceRegistryUrl, z as SecretDeletedError } from "./telemetry-CM82x7pC.js";
2
2
  import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
3
- import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
3
+ import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
4
4
  import { open, readdir, rm, stat } from "node:fs/promises";
5
- import path, { dirname, join } from "node:path";
5
+ import { dirname, join } from "node:path";
6
6
  import { childLogger, generateId } from "document-model";
7
7
  import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
8
8
  import { Cron } from "croner";
9
9
  import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
10
- import { Readable } from "node:stream";
11
10
  import dns from "node:dns";
12
11
  import net from "node:net";
13
12
  import { AsyncLocalStorage } from "node:async_hooks";
@@ -19,9 +18,10 @@ import https from "node:https";
19
18
  import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
20
19
  import { Kind, parse } from "graphql";
21
20
  import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
22
- import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
21
+ import { SpanStatusCode, context } from "@opentelemetry/api";
23
22
  import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
24
23
  import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
24
+ import { performance as performance$1 } from "node:perf_hooks";
25
25
  //#region src/pieces/activepieces/piece-versions.ts
26
26
  const LISTED_TTL_MS = 3e5;
27
27
  const ABSENT_TTL_MS = 3e4;
@@ -101,21 +101,6 @@ function npmVersions(name, options = {}) {
101
101
  //#endregion
102
102
  //#region src/reactor/attachment-port.ts
103
103
  const logger$7 = childLogger(["workflow", "attachments"]);
104
- function fileStream(filePath) {
105
- return Readable.toWeb(createReadStream(filePath));
106
- }
107
- async function sha256File(filePath) {
108
- const hash = createHash("sha256");
109
- for await (const chunk of createReadStream(filePath)) hash.update(chunk);
110
- return hash.digest("hex");
111
- }
112
- function refOf(result) {
113
- const direct = result.ref;
114
- if (typeof direct === "string") return direct;
115
- const nested = result.reservation?.ref;
116
- if (typeof nested === "string") return nested;
117
- throw new Error("The attachment store returned no reference for the upload");
118
- }
119
104
  async function writeCapped(body, destPath, limit) {
120
105
  const reader = body.getReader();
121
106
  const handle = await open(destPath, "w");
@@ -162,34 +147,19 @@ function createAttachmentPort(client, access) {
162
147
  throw new FileTooLargeError(header.sizeBytes, limit);
163
148
  }
164
149
  await writeCapped(body, destPath, limit);
165
- const contentType = header.mimeType !== void 0 && header.mimeType !== "" ? header.mimeType : void 0;
150
+ const contentType = header.mimeType !== "" ? header.mimeType : void 0;
166
151
  return {
167
152
  fileName: header.fileName,
168
153
  contentType
169
154
  };
170
155
  },
171
156
  async write(file, signal) {
172
- const hash = await sha256File(file.path);
173
- const mimeType = file.contentType ?? "application/octet-stream";
174
- const extension = path.extname(file.fileName).slice(1);
175
- const preprocessed = {
176
- ref: `attachment://v1:${hash}`,
177
- hash,
178
- sizeBytes: file.size,
179
- options: {
180
- mimeType,
181
- fileName: file.fileName,
182
- ...extension ? { extension } : {},
183
- clientHash: hash,
184
- sizeBytes: file.size
185
- },
186
- data: fileStream(file.path),
187
- stream: () => fileStream(file.path)
188
- };
189
- const ref = refOf(await client.upload({
190
- preprocessed,
157
+ const { ref } = await client.upload({
158
+ file: await openAsBlob(file.path),
159
+ fileName: file.fileName,
160
+ mimeType: file.contentType ?? "application/octet-stream",
191
161
  ...signal ? { signal } : {}
192
- }));
162
+ });
193
163
  access.onWritten?.(ref);
194
164
  logger$7.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
195
165
  return ref;
@@ -1982,227 +1952,6 @@ async function buildReactorRunScope(host, request) {
1982
1952
  connection: { access: parsed.config.access ?? "write" }
1983
1953
  };
1984
1954
  }
1985
- //#endregion
1986
- //#region src/telemetry.ts
1987
- const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
1988
- var WorkflowTelemetry = class {
1989
- tracer;
1990
- meter;
1991
- runs;
1992
- runDuration;
1993
- stepDuration;
1994
- phaseDuration;
1995
- poolGauges;
1996
- constructor(options = {}) {
1997
- this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
1998
- this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
1999
- this.runs = this.meter.createCounter("workflow.runs", {
2000
- description: "Runs finished, by status and trigger kind",
2001
- unit: "{run}"
2002
- });
2003
- this.runDuration = this.meter.createHistogram("workflow.run.duration", {
2004
- description: "Run duration from fire to the journal closing it",
2005
- unit: "ms"
2006
- });
2007
- this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
2008
- description: "Step duration, by piece and status",
2009
- unit: "ms"
2010
- });
2011
- this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
2012
- description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
2013
- unit: "ms"
2014
- });
2015
- }
2016
- phase(name, attributes, fn, parent) {
2017
- const started = performance.now();
2018
- return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
2019
- }
2020
- workerTimings(request, timings, forkedAt, ipc) {
2021
- const under = (span) => trace.setSpan(context.active(), span);
2022
- const child = (name, start, end, parent = request, attributes) => {
2023
- const span = this.tracer.startSpan(`workflow.worker.${name}`, {
2024
- startTime: start,
2025
- ...attributes ? { attributes } : {}
2026
- }, under(parent));
2027
- span.end(end);
2028
- this.recordPhase(`worker.${name}`, end - start);
2029
- return span;
2030
- };
2031
- const ready = epoch(timings.ready);
2032
- if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
2033
- const received = epoch(timings.received);
2034
- const sent = epoch(timings.sent);
2035
- const delivered = Math.max(ipc.sent, ready ?? 0);
2036
- if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
2037
- const byName = /* @__PURE__ */ new Map();
2038
- const phases = Array.isArray(timings.phases) ? timings.phases : [];
2039
- for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
2040
- const parent = phase.parent ? byName.get(phase.parent) : void 0;
2041
- byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
2042
- }
2043
- if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
2044
- }
2045
- recordPhase(name, ms) {
2046
- this.phaseDuration.record(ms, { phase: name });
2047
- }
2048
- span(name, attributes, fn, parent = context.active()) {
2049
- return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
2050
- try {
2051
- return await fn(span);
2052
- } catch (error) {
2053
- failSpan(span, error);
2054
- throw error;
2055
- } finally {
2056
- span.end();
2057
- }
2058
- });
2059
- }
2060
- recordRun(ms, status, triggerKind) {
2061
- const attributes = {
2062
- "run.status": status,
2063
- "trigger.kind": metricTriggerKind(triggerKind)
2064
- };
2065
- this.runs.add(1, attributes);
2066
- this.runDuration.record(ms, attributes);
2067
- }
2068
- recordStep(ms, pieceName, status) {
2069
- this.stepDuration.record(ms, {
2070
- "piece.name": pieceName,
2071
- "step.status": status
2072
- });
2073
- }
2074
- observeWorkerPool(stats) {
2075
- this.poolGauges ??= this.createPoolGauges();
2076
- const { size, active, waiting } = this.poolGauges;
2077
- const observables = [
2078
- size,
2079
- active,
2080
- waiting
2081
- ];
2082
- const callback = (result) => {
2083
- const current = stats();
2084
- result.observe(size, current.size);
2085
- result.observe(active, current.active);
2086
- result.observe(waiting, current.waiting);
2087
- };
2088
- this.meter.addBatchObservableCallback(callback, observables);
2089
- return () => this.meter.removeBatchObservableCallback(callback, observables);
2090
- }
2091
- createPoolGauges() {
2092
- const gauge = (name, description) => this.meter.createObservableGauge(name, {
2093
- description,
2094
- unit: "{worker}"
2095
- });
2096
- return {
2097
- size: gauge("workflow.worker.pool.size", "Worker slots"),
2098
- active: gauge("workflow.worker.pool.active", "Slots held by runs"),
2099
- waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
2100
- };
2101
- }
2102
- };
2103
- const WORKER_PHASES = /* @__PURE__ */ new Set([
2104
- "piece.load",
2105
- "reactor.open",
2106
- "models",
2107
- "action"
2108
- ]);
2109
- const WORKER_ATTRIBUTES = {
2110
- "piece.cached": "boolean",
2111
- "document.type": "string"
2112
- };
2113
- const MAX_ATTRIBUTE_LENGTH = 256;
2114
- function epoch(value) {
2115
- return typeof value === "number" && Number.isFinite(value) ? value : void 0;
2116
- }
2117
- function knownPhase(phase) {
2118
- if (typeof phase !== "object" || phase === null) return false;
2119
- const { name, start, end, parent } = phase;
2120
- return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
2121
- }
2122
- function phaseAttributes(attributes) {
2123
- if (typeof attributes !== "object" || attributes === null) return void 0;
2124
- const kept = {};
2125
- for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
2126
- const value = attributes[key];
2127
- if (typeof value !== type) continue;
2128
- kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
2129
- }
2130
- return Object.keys(kept).length > 0 ? kept : void 0;
2131
- }
2132
- function metricTriggerKind(triggerKind) {
2133
- return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
2134
- }
2135
- function metricPieceName(step, resolution) {
2136
- return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
2137
- }
2138
- function tracedSteps(executor, telemetry) {
2139
- return { execute(execution) {
2140
- const { step } = execution;
2141
- const started = performance.now();
2142
- let status = "FAILED";
2143
- let resolution;
2144
- return telemetry.span("workflow.step", {
2145
- "step.id": step.id,
2146
- "step.key": step.key,
2147
- "piece.name": step.pieceName,
2148
- "action.name": step.actionName
2149
- }, async (span) => {
2150
- try {
2151
- const result = await executor.execute(execution);
2152
- resolution = result.resolution;
2153
- status = "SUCCEEDED";
2154
- const version = resolution?.resolved?.version;
2155
- if (version) span.setAttribute("piece.version", version);
2156
- return result;
2157
- } catch (error) {
2158
- resolution = resolutionOf(error);
2159
- throw error;
2160
- }
2161
- }).finally(() => {
2162
- telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
2163
- });
2164
- } };
2165
- }
2166
- function failSpan(span, error) {
2167
- span.recordException(error instanceof Error ? error : String(error));
2168
- span.setStatus({
2169
- code: SpanStatusCode.ERROR,
2170
- message: error instanceof Error ? error.message : String(error)
2171
- });
2172
- }
2173
- function tracedMethods(target, telemetry, prefix, parent) {
2174
- return new Proxy(target, { get(object, property, receiver) {
2175
- const value = Reflect.get(object, property, receiver);
2176
- if (typeof value !== "function" || typeof property !== "string") return value;
2177
- const phase = `${prefix}.${property}`;
2178
- return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
2179
- const started = performance.now();
2180
- const end = (error) => {
2181
- if (error !== void 0) failSpan(span, error);
2182
- span.end();
2183
- telemetry.recordPhase(phase, performance.now() - started);
2184
- };
2185
- let result;
2186
- try {
2187
- result = value.apply(object, args);
2188
- } catch (error) {
2189
- end(error);
2190
- throw error;
2191
- }
2192
- if (!(result instanceof Promise)) {
2193
- end();
2194
- return result;
2195
- }
2196
- return result.then((resolved) => {
2197
- end();
2198
- return resolved;
2199
- }, (error) => {
2200
- end(error);
2201
- throw error;
2202
- });
2203
- });
2204
- } });
2205
- }
2206
1955
  const JOB_NOT_FOUND = "Job not found";
2207
1956
  const MIN_DEADLINE_MARGIN_MS = 250;
2208
1957
  const MAX_DEADLINE_MARGIN_MS = 2e3;
@@ -7606,6 +7355,9 @@ function triggerBlockColumns(block) {
7606
7355
  }
7607
7356
  const FIRED_WITHOUT_RUN_ID = "fired-without-run";
7608
7357
  const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
7358
+ function abandonedAdmissionError(attempts) {
7359
+ return `This run was admitted at ${attempts} boots and the reactor stopped each time before it started, so it is no longer admitted on boot. Fix the cause, then rerun it to fire the workflow with the same payload.`;
7360
+ }
7609
7361
  function abandonedFireError(attempts) {
7610
7362
  return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
7611
7363
  }
@@ -7623,7 +7375,6 @@ const logger$4 = childLogger([
7623
7375
  "store"
7624
7376
  ]);
7625
7377
  const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
7626
- const ABANDONED_PENDING_RUN_ERROR = "Reactor stopped before the matched trigger started its run; rerun it to fire the workflow with the same payload";
7627
7378
  const PENDING_RUN_STATUS = "PENDING";
7628
7379
  function errorMessage(error) {
7629
7380
  return error instanceof Error ? error.message : String(error);
@@ -7707,6 +7458,9 @@ async function up(db) {
7707
7458
  await db.schema.alterTable("run").addColumn("enqueued_at", "text").execute();
7708
7459
  } catch {}
7709
7460
  await db.updateTable("run").set({ enqueued_at: sql.ref("started_at") }).where("enqueued_at", "is", null).execute();
7461
+ try {
7462
+ await db.schema.alterTable("run").addColumn("admit_attempts", "integer", (col) => col.notNull().defaultTo(0)).execute();
7463
+ } catch {}
7710
7464
  await db.schema.dropIndex("run_workflow_started").ifExists().execute();
7711
7465
  await db.schema.dropIndex("run_started").ifExists().execute();
7712
7466
  await db.schema.createIndex("run_workflow_enqueued").ifNotExists().on("run").columns([
@@ -7718,6 +7472,7 @@ async function up(db) {
7718
7472
  await db.schema.createIndex("trigger_dedupe_workflow_created").ifNotExists().on("trigger_dedupe").columns(["workflow_id", "created_at"]).execute();
7719
7473
  await db.schema.createIndex("run_ended").ifNotExists().on("run").column("ended_at").execute();
7720
7474
  await db.schema.createIndex("trigger_dedupe_created").ifNotExists().on("trigger_dedupe").column("created_at").execute();
7475
+ await db.schema.createIndex("run_pending").ifNotExists().on("run").columns(["enqueued_at", "id"]).where(sql.ref("status"), "=", PENDING_RUN_STATUS).execute();
7721
7476
  await db.schema.createIndex("trigger_state_due").ifNotExists().on("trigger_state").column("next_poll_at").where(sql.ref("status"), "=", "ENABLED").execute();
7722
7477
  try {
7723
7478
  await db.schema.dropTable("webhook_endpoint").ifExists().execute();
@@ -7925,6 +7680,10 @@ const LEGACY_MARKER_KEYS = [
7925
7680
  * ambiguous, and nothing can resolve that for a row already written; the
7926
7681
  * window is as narrow as the stored data allows.
7927
7682
  */
7683
+ function unreplayablePayload(payload) {
7684
+ if (isTruncatedStepPayload(payload)) return "truncated";
7685
+ if (containsRedactedMarker(payload)) return "redacted";
7686
+ }
7928
7687
  function isTruncatedStepPayload(value) {
7929
7688
  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
7930
7689
  const record = value;
@@ -7955,6 +7714,9 @@ function cappedPayload(value) {
7955
7714
  };
7956
7715
  return JSON.stringify(marker);
7957
7716
  }
7717
+ function rowHash(values) {
7718
+ return createHash("sha1").update(JSON.stringify(values)).digest("base64");
7719
+ }
7958
7720
  function stepValues(runId, ordinal, step) {
7959
7721
  return {
7960
7722
  run_id: runId,
@@ -8072,6 +7834,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8072
7834
  db;
8073
7835
  unmigrated;
8074
7836
  runsInFlight = /* @__PURE__ */ new Set();
7837
+ journaledSteps = /* @__PURE__ */ new Map();
8075
7838
  constructor(db, unmigrated) {
8076
7839
  this.db = db;
8077
7840
  this.unmigrated = unmigrated;
@@ -8081,7 +7844,6 @@ var WorkflowRunStore = class WorkflowRunStore {
8081
7844
  const unmigrated = await up(db);
8082
7845
  const store = new WorkflowRunStore(db, unmigrated);
8083
7846
  await store.recoverOrphanedRuns();
8084
- await store.recoverAbandonedRuns();
8085
7847
  return store;
8086
7848
  }
8087
7849
  hasUnmigratedTriggerState(workflowId) {
@@ -8101,16 +7863,56 @@ var WorkflowRunStore = class WorkflowRunStore {
8101
7863
  if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
8102
7864
  return recovered;
8103
7865
  }
8104
- async recoverAbandonedRuns() {
8105
- let query = this.db.updateTable("run").set({
8106
- status: "FAILED",
8107
- error: ABANDONED_PENDING_RUN_ERROR,
8108
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8109
- }).where("status", "=", PENDING_RUN_STATUS);
8110
- if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
8111
- const recovered = (await query.returning("id").execute()).length;
8112
- if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
8113
- return recovered;
7866
+ async admitAbandonedRuns(options = {}) {
7867
+ const admitted = await this.db.transaction().execute(async (trx) => {
7868
+ let query = trx.selectFrom("run").select([
7869
+ "id",
7870
+ "workflow_id",
7871
+ "trigger_kind",
7872
+ "trigger_payload",
7873
+ "admit_attempts"
7874
+ ]).where("status", "=", PENDING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc");
7875
+ if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
7876
+ if (options.enqueuedBefore) query = query.where("enqueued_at", "<", options.enqueuedBefore.toISOString());
7877
+ const rows = await query.execute();
7878
+ const runs = [];
7879
+ const now = (/* @__PURE__ */ new Date()).toISOString();
7880
+ for (const row of rows) {
7881
+ if (row.admit_attempts >= 3) {
7882
+ await trx.updateTable("run").set({
7883
+ status: "FAILED",
7884
+ error: abandonedAdmissionError(row.admit_attempts),
7885
+ ended_at: now
7886
+ }).where("id", "=", row.id).execute();
7887
+ continue;
7888
+ }
7889
+ const triggerPayload = row.trigger_payload === null ? void 0 : JSON.parse(row.trigger_payload);
7890
+ const unreplayable = unreplayablePayload(triggerPayload);
7891
+ if (unreplayable) {
7892
+ await trx.updateTable("run").set({
7893
+ status: "FAILED",
7894
+ error: `The trigger payload was ${unreplayable} by the journal and cannot be run after a restart; fire the workflow again`,
7895
+ ended_at: now
7896
+ }).where("id", "=", row.id).execute();
7897
+ continue;
7898
+ }
7899
+ await trx.updateTable("run").set({ admit_attempts: row.admit_attempts + 1 }).where("id", "=", row.id).execute();
7900
+ runs.push({
7901
+ runId: row.id,
7902
+ workflowId: row.workflow_id,
7903
+ triggerKind: row.trigger_kind,
7904
+ triggerPayload
7905
+ });
7906
+ }
7907
+ return runs;
7908
+ });
7909
+ for (const run of admitted) this.runsInFlight.add(run.runId);
7910
+ if (admitted.length > 0) logger$4.info(`Admitting ${admitted.length} workflow run(s) journaled by a stopped reactor`);
7911
+ return admitted;
7912
+ }
7913
+ async releaseAdmission(runId) {
7914
+ this.runsInFlight.delete(runId);
7915
+ await this.db.updateTable("run").set({ admit_attempts: sql`greatest(admit_attempts - 1, 0)` }).where("id", "=", runId).where("status", "=", PENDING_RUN_STATUS).execute();
8114
7916
  }
8115
7917
  /**
8116
7918
  * Closes a run out as CANCELLED, with the reason as its error.
@@ -8122,6 +7924,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8122
7924
  */
8123
7925
  async cancelRun(runId, reason) {
8124
7926
  this.runsInFlight.delete(runId);
7927
+ this.journaledSteps.delete(runId);
8125
7928
  if (erasedRuns.delete(runId)) return;
8126
7929
  await this.db.updateTable("run").set({
8127
7930
  status: "CANCELLED",
@@ -8131,10 +7934,20 @@ var WorkflowRunStore = class WorkflowRunStore {
8131
7934
  }
8132
7935
  async enqueueRun(options) {
8133
7936
  const id = randomUUID();
8134
- await this.insertPendingRun(this.db, id, options);
8135
- this.runsInFlight.add(id);
7937
+ await this.unmarkOnFailure([id], () => this.insertPendingRun(this.db, id, options));
8136
7938
  return id;
8137
7939
  }
7940
+ async unmarkOnFailure(ids, write) {
7941
+ try {
7942
+ return await write();
7943
+ } catch (error) {
7944
+ for (const id of ids) {
7945
+ this.runsInFlight.delete(id);
7946
+ this.journaledSteps.delete(id);
7947
+ }
7948
+ throw error;
7949
+ }
7950
+ }
8138
7951
  /**
8139
7952
  * Claims the dedupe key and journals the PENDING run, bounding crash replays.
8140
7953
  *
@@ -8178,38 +7991,39 @@ var WorkflowRunStore = class WorkflowRunStore {
8178
7991
  attempts: attempt.attempts
8179
7992
  };
8180
7993
  const id = randomUUID();
8181
- if (!await this.db.transaction().execute(async (trx) => {
7994
+ if (!await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
8182
7995
  if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
8183
7996
  await this.insertPendingRun(trx, id, options);
8184
7997
  return true;
8185
- })) return { outcome: "duplicate" };
8186
- this.runsInFlight.add(id);
7998
+ }))) return { outcome: "duplicate" };
8187
7999
  return {
8188
8000
  outcome: "claimed",
8189
8001
  runId: id
8190
8002
  };
8191
8003
  }
8192
- async journalTriggerItems(items, ttlMs, nowIso) {
8193
- const ids = await this.db.transaction().execute(async (trx) => {
8004
+ async journalTriggerItems(items, ttlMs, nowIso, success) {
8005
+ const marked = [];
8006
+ return this.unmarkOnFailure(marked, () => this.db.transaction().execute(async (trx) => {
8007
+ if (success) await pollSucceededIn(trx, success.workflowId, success.storeState, nowIso, success.nextPollAtIso);
8194
8008
  const journaled = [];
8195
8009
  for (const { dedupeKey, options } of items) {
8196
8010
  const id = randomUUID();
8197
8011
  const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
8198
- if (won) await this.insertPendingRun(trx, id, options);
8012
+ if (won) {
8013
+ marked.push(id);
8014
+ await this.insertPendingRun(trx, id, options);
8015
+ }
8199
8016
  journaled.push(won ? id : void 0);
8200
8017
  }
8201
8018
  return journaled;
8202
- });
8203
- for (const id of ids) if (id) this.runsInFlight.add(id);
8204
- return ids;
8019
+ }));
8205
8020
  }
8206
8021
  async recordScheduleFire(nowIso, nextPollAtIso, options) {
8207
8022
  const id = randomUUID();
8208
- await this.db.transaction().execute(async (trx) => {
8023
+ await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
8209
8024
  await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
8210
8025
  await this.insertPendingRun(trx, id, options);
8211
- });
8212
- this.runsInFlight.add(id);
8026
+ }));
8213
8027
  return id;
8214
8028
  }
8215
8029
  /**
@@ -8245,6 +8059,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8245
8059
  });
8246
8060
  for (const runId of redelivered.runIds) {
8247
8061
  this.runsInFlight.delete(runId);
8062
+ this.journaledSteps.delete(runId);
8248
8063
  erasedRuns.delete(runId);
8249
8064
  }
8250
8065
  }
@@ -8301,6 +8116,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8301
8116
  return id;
8302
8117
  }
8303
8118
  async insertPendingRun(db, id, options) {
8119
+ this.runsInFlight.add(id);
8304
8120
  const now = (/* @__PURE__ */ new Date()).toISOString();
8305
8121
  await db.insertInto("run").values({
8306
8122
  id,
@@ -8321,6 +8137,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8321
8137
  }
8322
8138
  async beginRun(runId, details) {
8323
8139
  this.runsInFlight.add(runId);
8140
+ this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
8324
8141
  await this.db.updateTable("run").set({
8325
8142
  status: "RUNNING",
8326
8143
  workflow_name: details.workflowName,
@@ -8348,6 +8165,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8348
8165
  warning_notes: null
8349
8166
  }).execute();
8350
8167
  this.runsInFlight.add(id);
8168
+ this.journaledSteps.set(id, /* @__PURE__ */ new Map());
8351
8169
  return id;
8352
8170
  }
8353
8171
  async recordStep(runId, ordinal, step) {
@@ -8358,11 +8176,26 @@ var WorkflowRunStore = class WorkflowRunStore {
8358
8176
  id: randomUUID(),
8359
8177
  ...values
8360
8178
  }).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
8179
+ this.journaledSteps.get(runId)?.set(step.stepId, {
8180
+ ordinal,
8181
+ hash: rowHash(values)
8182
+ });
8183
+ }
8184
+ sweepIsNoop(runId, result) {
8185
+ const journaled = this.journaledSteps.get(runId);
8186
+ if (!journaled) return false;
8187
+ return result.steps.every((step) => {
8188
+ const row = journaled.get(step.stepId);
8189
+ if (row === void 0) return false;
8190
+ return row.hash === rowHash(stepValues(runId, row.ordinal, step));
8191
+ });
8361
8192
  }
8362
8193
  async finishRun(runId, result, executionOrder) {
8363
8194
  this.runsInFlight.delete(runId);
8195
+ const sweep = !this.sweepIsNoop(runId, result);
8196
+ this.journaledSteps.delete(runId);
8364
8197
  if (erasedRuns.delete(runId)) return;
8365
- if (result.steps.length > 0) try {
8198
+ if (sweep && result.steps.length > 0) try {
8366
8199
  await this.sweepSteps(runId, result, executionOrder);
8367
8200
  } catch (error) {
8368
8201
  logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
@@ -8413,6 +8246,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8413
8246
  }
8414
8247
  async failRun(runId, error, errorName) {
8415
8248
  this.runsInFlight.delete(runId);
8249
+ this.journaledSteps.delete(runId);
8416
8250
  if (erasedRuns.delete(runId)) return;
8417
8251
  await this.db.updateTable("run").set({
8418
8252
  status: "FAILED",
@@ -8795,7 +8629,7 @@ var WorkflowRunGate = class {
8795
8629
  * journals the refusal rather than dropping it silently, because a firing
8796
8630
  * that vanished is indistinguishable from a trigger that never fired.
8797
8631
  */
8798
- async admit(workflowId, policy) {
8632
+ async admit(workflowId, policy, onQueue) {
8799
8633
  if (this.closed) return {
8800
8634
  admitted: false,
8801
8635
  reason: CLOSED_REASON,
@@ -8827,6 +8661,7 @@ var WorkflowRunGate = class {
8827
8661
  refusal: "queue-full",
8828
8662
  reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
8829
8663
  };
8664
+ onQueue?.();
8830
8665
  if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
8831
8666
  admitted: false,
8832
8667
  reason: CLOSED_REASON,
@@ -8862,6 +8697,117 @@ var WorkflowRunGate = class {
8862
8697
  }
8863
8698
  };
8864
8699
  //#endregion
8700
+ //#region src/reactor/shutdown.ts
8701
+ /** A firing refused because this runtime has shut down: retryable elsewhere. */
8702
+ var WorkflowRuntimeClosedError = class extends Error {
8703
+ workflowId;
8704
+ constructor(workflowId) {
8705
+ super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
8706
+ this.workflowId = workflowId;
8707
+ this.name = "WorkflowRuntimeClosedError";
8708
+ }
8709
+ };
8710
+ var TriggerSupervisorStoppedError = class extends Error {
8711
+ constructor(options) {
8712
+ super("The trigger supervisor has stopped", options);
8713
+ this.name = "TriggerSupervisorStoppedError";
8714
+ }
8715
+ };
8716
+ function isShutdownRefusal(error) {
8717
+ return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
8718
+ }
8719
+ //#endregion
8720
+ //#region src/reactor/run-dispatcher.ts
8721
+ const DEFAULT_ADMIT_MAX_ELU = .8;
8722
+ var RunDispatcher = class {
8723
+ maxInFlight;
8724
+ maxElu;
8725
+ loop;
8726
+ waiters = [];
8727
+ inFlight = 0;
8728
+ closed = false;
8729
+ unsubscribe;
8730
+ constructor(options) {
8731
+ this.maxInFlight = Math.max(1, Math.trunc(options.maxInFlight));
8732
+ this.maxElu = options.maxElu ?? .8;
8733
+ this.loop = options.loop;
8734
+ this.unsubscribe = this.loop.onSample(() => this.drain());
8735
+ }
8736
+ admit(workflowId, onQueue) {
8737
+ if (this.closed) return Promise.reject(new WorkflowRuntimeClosedError(workflowId));
8738
+ if (this.waiters.length === 0 && this.room()) return Promise.resolve(this.take());
8739
+ onQueue?.();
8740
+ return new Promise((resolve, reject) => {
8741
+ this.waiters.push({
8742
+ workflowId,
8743
+ resolve,
8744
+ reject
8745
+ });
8746
+ });
8747
+ }
8748
+ stats() {
8749
+ return {
8750
+ inFlight: this.inFlight,
8751
+ waiting: this.waiters.length
8752
+ };
8753
+ }
8754
+ close() {
8755
+ if (this.closed) return;
8756
+ this.closed = true;
8757
+ this.unsubscribe();
8758
+ for (const waiter of this.waiters.splice(0)) waiter.reject(new WorkflowRuntimeClosedError(waiter.workflowId));
8759
+ }
8760
+ room() {
8761
+ if (this.inFlight === 0) return true;
8762
+ return this.inFlight < this.maxInFlight && this.loop.utilization() <= this.maxElu;
8763
+ }
8764
+ take() {
8765
+ this.inFlight++;
8766
+ let released = false;
8767
+ return () => {
8768
+ if (released) return;
8769
+ released = true;
8770
+ this.inFlight--;
8771
+ this.drain();
8772
+ };
8773
+ }
8774
+ drain() {
8775
+ while (this.waiters.length > 0 && !this.closed && this.room()) this.waiters.shift().resolve(this.take());
8776
+ }
8777
+ };
8778
+ //#endregion
8779
+ //#region src/reactor/event-loop.ts
8780
+ const SAMPLE_MS = 1e3;
8781
+ var EventLoopMonitor = class {
8782
+ last = performance$1.eventLoopUtilization();
8783
+ current = 0;
8784
+ timer;
8785
+ listeners = /* @__PURE__ */ new Set();
8786
+ start() {
8787
+ if (this.timer) return;
8788
+ this.last = performance$1.eventLoopUtilization();
8789
+ this.timer = setInterval(() => this.sample(), SAMPLE_MS);
8790
+ this.timer.unref();
8791
+ }
8792
+ stop() {
8793
+ clearInterval(this.timer);
8794
+ this.timer = void 0;
8795
+ }
8796
+ utilization() {
8797
+ return this.current;
8798
+ }
8799
+ onSample(listener) {
8800
+ this.listeners.add(listener);
8801
+ return () => this.listeners.delete(listener);
8802
+ }
8803
+ sample() {
8804
+ const now = performance$1.eventLoopUtilization();
8805
+ this.current = performance$1.eventLoopUtilization(now, this.last).utilization;
8806
+ this.last = now;
8807
+ for (const listener of this.listeners) listener();
8808
+ }
8809
+ };
8810
+ //#endregion
8865
8811
  //#region src/reactor/run-retention.ts
8866
8812
  const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
8867
8813
  const RETENTION_SWEEP_INTERVAL_MS = 36e5;
@@ -8897,26 +8843,6 @@ async function sweepRetention(store, options) {
8897
8843
  dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
8898
8844
  };
8899
8845
  }
8900
- //#endregion
8901
- //#region src/reactor/shutdown.ts
8902
- /** A firing refused because this runtime has shut down: retryable elsewhere. */
8903
- var WorkflowRuntimeClosedError = class extends Error {
8904
- workflowId;
8905
- constructor(workflowId) {
8906
- super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
8907
- this.workflowId = workflowId;
8908
- this.name = "WorkflowRuntimeClosedError";
8909
- }
8910
- };
8911
- var TriggerSupervisorStoppedError = class extends Error {
8912
- constructor(options) {
8913
- super("The trigger supervisor has stopped", options);
8914
- this.name = "TriggerSupervisorStoppedError";
8915
- }
8916
- };
8917
- function isShutdownRefusal(error) {
8918
- return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
8919
- }
8920
8846
  childLogger(["workflow", "trigger-binding"]);
8921
8847
  function pieceTriggerKind(block) {
8922
8848
  return `piece:${block.pieceName}:${block.name}`;
@@ -9007,7 +8933,9 @@ var TriggerSupervisor = class {
9007
8933
  egress;
9008
8934
  parks;
9009
8935
  timer;
9010
- ops = Promise.resolve();
8936
+ lanes = /* @__PURE__ */ new Map();
8937
+ ticked = /* @__PURE__ */ new Set();
8938
+ hookSlots;
9011
8939
  ticking = false;
9012
8940
  warnedMissingJournal = false;
9013
8941
  stopped = false;
@@ -9020,6 +8948,7 @@ var TriggerSupervisor = class {
9020
8948
  this.tickMs = options.tickMs ?? 15e3;
9021
8949
  this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
9022
8950
  this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
8951
+ this.hookSlots = new Slots(options.hookConcurrency ?? 1);
9023
8952
  this.now = options.now ?? (() => /* @__PURE__ */ new Date());
9024
8953
  this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
9025
8954
  this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
@@ -9046,8 +8975,8 @@ var TriggerSupervisor = class {
9046
8975
  if (!this.options.worker) this.worker.dispose();
9047
8976
  logger$2.info("Trigger supervisor stopped");
9048
8977
  }
9049
- enqueue(task) {
9050
- const run = this.ops.then(async () => {
8978
+ enqueue(workflowId, task) {
8979
+ const run = (this.lanes.get(workflowId) ?? Promise.resolve()).then(async () => {
9051
8980
  this.throwIfStopped();
9052
8981
  try {
9053
8982
  return await task();
@@ -9056,7 +8985,11 @@ var TriggerSupervisor = class {
9056
8985
  throw error;
9057
8986
  }
9058
8987
  });
9059
- this.ops = run.catch(() => void 0);
8988
+ const settled = run.catch(() => void 0);
8989
+ this.lanes.set(workflowId, settled);
8990
+ settled.then(() => {
8991
+ if (this.lanes.get(workflowId) === settled) this.lanes.delete(workflowId);
8992
+ });
9060
8993
  return run;
9061
8994
  }
9062
8995
  throwIfStopped() {
@@ -9094,12 +9027,12 @@ var TriggerSupervisor = class {
9094
9027
  this.bindings.set(binding.workflowId, binding);
9095
9028
  const park = this.parks.known(binding.workflowId)?.park;
9096
9029
  if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
9097
- return this.enqueue(() => this.enable(binding, previous));
9030
+ return this.enqueue(binding.workflowId, () => this.enable(binding, previous));
9098
9031
  }
9099
9032
  /** Parks a workflow; true when a trigger row was parked with it. */
9100
9033
  park(workflowId, publishedVersion, reason) {
9101
9034
  this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
9102
- return this.enqueue(async () => {
9035
+ return this.enqueue(workflowId, async () => {
9103
9036
  const store = await this.store();
9104
9037
  if (!store) return false;
9105
9038
  return store.parkWorkflow(workflowId, publishedVersion, reason);
@@ -9108,7 +9041,7 @@ var TriggerSupervisor = class {
9108
9041
  /** Undoes {@link park} for that version only: a later park stands. */
9109
9042
  liftPark(workflowId, publishedVersion, trigger) {
9110
9043
  if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
9111
- return this.enqueue(async () => {
9044
+ return this.enqueue(workflowId, async () => {
9112
9045
  const store = await this.store();
9113
9046
  if (!store) return;
9114
9047
  if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
@@ -9119,7 +9052,7 @@ var TriggerSupervisor = class {
9119
9052
  * row through onDisable (a boot), the row is turned DISABLED here. */
9120
9053
  releasePark(workflowId, flipRow) {
9121
9054
  this.parks.delete(workflowId);
9122
- return this.enqueue(async () => {
9055
+ return this.enqueue(workflowId, async () => {
9123
9056
  const store = await this.store();
9124
9057
  if (!store) return;
9125
9058
  if (flipRow) await store.clearParkOnDisable(workflowId);
@@ -9137,7 +9070,7 @@ var TriggerSupervisor = class {
9137
9070
  if (retry) retry.at = 0;
9138
9071
  const park = this.parks.known(workflowId)?.park;
9139
9072
  if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
9140
- return this.enqueue(async () => {
9073
+ return this.enqueue(workflowId, async () => {
9141
9074
  const store = await this.store();
9142
9075
  if (!store) return;
9143
9076
  const park = await store.getWorkflowPark(workflowId);
@@ -9147,12 +9080,12 @@ var TriggerSupervisor = class {
9147
9080
  }
9148
9081
  remove(workflowId) {
9149
9082
  const binding = this.unbind(workflowId);
9150
- return this.enqueue(() => this.disable(workflowId, binding));
9083
+ return this.enqueue(workflowId, () => this.disable(workflowId, binding));
9151
9084
  }
9152
9085
  forget(workflowId) {
9153
9086
  const binding = this.unbind(workflowId);
9154
9087
  this.parks.delete(workflowId);
9155
- return this.enqueue(async () => {
9088
+ return this.enqueue(workflowId, async () => {
9156
9089
  const store = await this.store();
9157
9090
  if (!store) return;
9158
9091
  await this.disable(workflowId, binding);
@@ -9170,7 +9103,7 @@ var TriggerSupervisor = class {
9170
9103
  }
9171
9104
  reject(workflowId, block, config, message, retryAt) {
9172
9105
  this.unbind(workflowId);
9173
- return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
9106
+ return this.enqueue(workflowId, () => this.recordRejection(workflowId, block, config, message, retryAt));
9174
9107
  }
9175
9108
  async recordRejection(workflowId, block, config, message, retryAt) {
9176
9109
  const store = await this.store();
@@ -9203,7 +9136,7 @@ var TriggerSupervisor = class {
9203
9136
  });
9204
9137
  }
9205
9138
  test(binding, options = {}) {
9206
- return this.enqueue(async () => {
9139
+ return this.enqueue(binding.workflowId, async () => {
9207
9140
  const store = await this.store();
9208
9141
  try {
9209
9142
  return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
@@ -9222,10 +9155,10 @@ var TriggerSupervisor = class {
9222
9155
  }
9223
9156
  }
9224
9157
  handshake(binding, payload) {
9225
- return this.enqueue(() => this.hook(binding, "onHandshake", { payload }));
9158
+ return this.enqueue(binding.workflowId, () => this.hook(binding, "onHandshake", { payload }));
9226
9159
  }
9227
9160
  deliverWebhook(workflowId, payload) {
9228
- return this.enqueue(async () => {
9161
+ return this.enqueue(workflowId, async () => {
9229
9162
  const binding = this.bindings.get(workflowId);
9230
9163
  if (!binding || isSchedule(binding)) {
9231
9164
  logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
@@ -9243,8 +9176,12 @@ var TriggerSupervisor = class {
9243
9176
  throw error;
9244
9177
  });
9245
9178
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
9246
- if (row?.status === "ENABLED") await store.recordPollSuccess(workflowId, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
9247
- await this.fireItems(store, binding, result.output, now, rewind);
9179
+ const success = row?.status === "ENABLED" ? {
9180
+ workflowId,
9181
+ storeState: VESTIGIAL_STORE_STATE,
9182
+ nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
9183
+ } : void 0;
9184
+ await this.fireItems(store, binding, result.output, now, rewind, success);
9248
9185
  } catch (error) {
9249
9186
  if (!refused) await rewind();
9250
9187
  throw error;
@@ -9576,22 +9513,38 @@ var TriggerSupervisor = class {
9576
9513
  async tick() {
9577
9514
  if (this.ticking || this.stopped) return;
9578
9515
  this.ticking = true;
9516
+ const started = [];
9517
+ let listing;
9579
9518
  try {
9580
- await this.enqueue(() => this.pollDue());
9519
+ await this.startDue(started);
9520
+ } catch (error) {
9521
+ listing = error;
9581
9522
  } finally {
9582
9523
  this.ticking = false;
9583
9524
  }
9525
+ const outcomes = await Promise.allSettled(started.filter((run) => run !== void 0));
9526
+ if (listing !== void 0) throw listing instanceof Error ? listing : new Error(String(listing));
9527
+ const failed = outcomes.find((outcome) => outcome.status === "rejected");
9528
+ if (failed) throw failed.reason;
9584
9529
  }
9585
- async retryOneEnable() {
9530
+ startOnLane(workflowId, task, hook) {
9531
+ if (this.ticked.has(workflowId)) return void 0;
9532
+ this.ticked.add(workflowId);
9533
+ return (hook ? this.hookSlots.run(() => this.enqueue(workflowId, task)) : this.enqueue(workflowId, task)).finally(() => this.ticked.delete(workflowId));
9534
+ }
9535
+ startOneEnableRetry() {
9586
9536
  const now = this.now().getTime();
9587
- const next = [...this.enableRetries].filter(([, retry]) => retry.at <= now).sort((a, b) => a[1].at - b[1].at).at(0);
9588
- if (!next) return;
9537
+ const next = [...this.enableRetries].filter(([id, retry]) => retry.at <= now && !this.ticked.has(id)).sort((a, b) => a[1].at - b[1].at).at(0);
9538
+ if (!next) return void 0;
9589
9539
  const [workflowId, retry] = next;
9590
9540
  const binding = this.bindings.get(workflowId);
9591
9541
  if (!binding) {
9592
9542
  this.enableRetries.delete(workflowId);
9593
9543
  return;
9594
9544
  }
9545
+ return this.startOnLane(workflowId, () => this.retryEnable(workflowId, retry, binding, now), true);
9546
+ }
9547
+ async retryEnable(workflowId, retry, binding, now) {
9595
9548
  try {
9596
9549
  await this.enable(binding);
9597
9550
  } catch (error) {
@@ -9606,7 +9559,7 @@ var TriggerSupervisor = class {
9606
9559
  logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
9607
9560
  }
9608
9561
  }
9609
- async pollDue() {
9562
+ async startDue(started) {
9610
9563
  const store = await this.store();
9611
9564
  if (!store) {
9612
9565
  if (!this.warnedMissingJournal) {
@@ -9615,37 +9568,35 @@ var TriggerSupervisor = class {
9615
9568
  }
9616
9569
  return;
9617
9570
  }
9618
- const due = await store.listDueTriggerStates(this.now().toISOString());
9619
- for (const row of due) {
9571
+ const ordered = [...await store.listDueTriggerStates(this.now().toISOString())].sort((a, b) => Number(!this.isScheduleRow(a)) - Number(!this.isScheduleRow(b)));
9572
+ for (const row of ordered) {
9620
9573
  this.throwIfStopped();
9621
9574
  const binding = this.bindings.get(row.workflow_id);
9622
9575
  if (!binding) {
9623
- await store.setTriggerStatus(row.workflow_id, "DISABLED");
9576
+ started.push(this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false));
9624
9577
  continue;
9625
9578
  }
9626
- if (isSchedule(binding)) await this.fireSchedule(store, row, binding);
9627
- else await this.poll(store, row, binding);
9579
+ started.push(isSchedule(binding) ? this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.fireSchedule(store, row, binding)), false) : this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.poll(store, row, binding)), true));
9628
9580
  }
9629
9581
  this.throwIfStopped();
9630
- await this.renewDue(store);
9582
+ started.push(...await this.startRenewals(store));
9631
9583
  this.throwIfStopped();
9632
- await this.retryOneEnable();
9584
+ started.push(this.startOneEnableRetry());
9633
9585
  }
9634
- async renewDue(store) {
9635
- const due = await store.listDueTriggerRenewals(this.now().toISOString());
9636
- for (const row of due) {
9637
- this.throwIfStopped();
9586
+ whileBound(binding, task) {
9587
+ return () => this.bindings.get(binding.workflowId) === binding ? task() : Promise.resolve();
9588
+ }
9589
+ isScheduleRow(row) {
9590
+ const binding = this.bindings.get(row.workflow_id);
9591
+ return binding !== void 0 && isSchedule(binding);
9592
+ }
9593
+ async startRenewals(store) {
9594
+ return (await store.listDueTriggerRenewals(this.now().toISOString())).map((row) => {
9638
9595
  const binding = this.bindings.get(row.workflow_id);
9639
- if (!binding) {
9640
- await store.setTriggerStatus(row.workflow_id, "DISABLED");
9641
- continue;
9642
- }
9643
- if (isSchedule(binding)) {
9644
- await store.setTriggerRenewAt(row.workflow_id, null);
9645
- continue;
9646
- }
9647
- await this.renew(store, row, binding);
9648
- }
9596
+ if (!binding) return this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false);
9597
+ if (isSchedule(binding)) return this.startOnLane(row.workflow_id, () => store.setTriggerRenewAt(row.workflow_id, null), false);
9598
+ return this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.renew(store, row, binding)), true);
9599
+ });
9649
9600
  }
9650
9601
  async renew(store, row, binding) {
9651
9602
  const now = this.now();
@@ -9707,8 +9658,11 @@ var TriggerSupervisor = class {
9707
9658
  throw error;
9708
9659
  });
9709
9660
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
9710
- await store.recordPollSuccess(row.workflow_id, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
9711
- await this.fireItems(store, binding, result.output, now, rewind);
9661
+ await this.fireItems(store, binding, result.output, now, rewind, {
9662
+ workflowId: row.workflow_id,
9663
+ storeState: VESTIGIAL_STORE_STATE,
9664
+ nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
9665
+ });
9712
9666
  } catch (error) {
9713
9667
  if (!refused) await rewind();
9714
9668
  if (this.stopped) throw error;
@@ -9719,8 +9673,11 @@ var TriggerSupervisor = class {
9719
9673
  logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
9720
9674
  }
9721
9675
  }
9722
- async fireItems(store, binding, items, now, rewind) {
9723
- if (items.length === 0) return;
9676
+ async fireItems(store, binding, items, now, rewind, success) {
9677
+ if (items.length === 0) {
9678
+ if (success) await store.recordPollSuccess(success.workflowId, success.storeState, now.toISOString(), success.nextPollAtIso);
9679
+ return;
9680
+ }
9724
9681
  const kind = pieceTriggerKind(binding.block);
9725
9682
  const journaled = (await store.journalTriggerItems(items.map((item) => ({
9726
9683
  dedupeKey: extractDedupeKey(item),
@@ -9729,7 +9686,7 @@ var TriggerSupervisor = class {
9729
9686
  triggerKind: kind,
9730
9687
  triggerPayload: item
9731
9688
  }
9732
- })), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
9689
+ })), DEDUPE_TTL_MS, now.toISOString(), success)).flatMap((runId, index) => runId ? [{
9733
9690
  runId,
9734
9691
  item: items[index]
9735
9692
  }] : []);
@@ -9752,6 +9709,25 @@ var TriggerSupervisor = class {
9752
9709
  }
9753
9710
  }
9754
9711
  };
9712
+ var Slots = class {
9713
+ limit;
9714
+ active = 0;
9715
+ waiting = [];
9716
+ constructor(limit) {
9717
+ this.limit = limit;
9718
+ }
9719
+ async run(task) {
9720
+ if (this.active >= Math.max(1, this.limit)) await new Promise((resolve) => this.waiting.push(resolve));
9721
+ else this.active++;
9722
+ try {
9723
+ return await task();
9724
+ } finally {
9725
+ const next = this.waiting.shift();
9726
+ if (next) next();
9727
+ else this.active--;
9728
+ }
9729
+ }
9730
+ };
9755
9731
  //#endregion
9756
9732
  //#region src/reactor/webhook.ts
9757
9733
  const WEBHOOK = "Webhook";
@@ -9979,6 +9955,7 @@ function bindingTarget(binding) {
9979
9955
  }
9980
9956
  const RUNS_SCAN_LIMIT = 1e3;
9981
9957
  const WEBHOOK_TEST_TIMEOUT_MS = 3e5;
9958
+ const PENDING_SWEEP_INTERVAL_MS = 6e4;
9982
9959
  function draftBlocks(state) {
9983
9960
  return [...state.trigger ? [{
9984
9961
  id: state.trigger.id,
@@ -10149,6 +10126,10 @@ function configRecord(config) {
10149
10126
  if (config && typeof config === "object" && !Array.isArray(config)) return config;
10150
10127
  return {};
10151
10128
  }
10129
+ const WORKFLOW_CACHE_MAX = 1e3;
10130
+ function globalRevision(document) {
10131
+ return document.header.revision?.global ?? 0;
10132
+ }
10152
10133
  function runJournalName(stateName, documentName) {
10153
10134
  return stateName?.trim() ? stateName : documentName ?? "";
10154
10135
  }
@@ -10160,6 +10141,9 @@ var WorkflowRuntimeService = class {
10160
10141
  executor;
10161
10142
  pieceWorkers;
10162
10143
  unobservePool;
10144
+ runDispatcher;
10145
+ unobserveDispatcher;
10146
+ eventLoop = new EventLoopMonitor();
10163
10147
  storePromise;
10164
10148
  storeError;
10165
10149
  storeOpening = false;
@@ -10655,10 +10639,35 @@ var WorkflowRuntimeService = class {
10655
10639
  await this.updateRegistration(workflowId, carried, onlyIfChanged);
10656
10640
  return;
10657
10641
  }
10642
+ const generation = this.workflowGeneration(workflowId);
10658
10643
  const document = await this.host.reactorClient.get(workflowId);
10644
+ if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
10659
10645
  await this.updateRegistration(workflowId, document.state.global, onlyIfChanged);
10660
10646
  }
10661
10647
  seenOps = /* @__PURE__ */ new Set();
10648
+ workflowCache = /* @__PURE__ */ new Map();
10649
+ workflowGenerations = /* @__PURE__ */ new Map();
10650
+ workflowGeneration(workflowId) {
10651
+ return this.workflowGenerations.get(workflowId) ?? 0;
10652
+ }
10653
+ cacheLoaded(workflowId, generation, document) {
10654
+ if (this.workflowGeneration(workflowId) !== generation) return;
10655
+ this.cacheWorkflow(workflowId, {
10656
+ state: document.state.global,
10657
+ name: document.header.name,
10658
+ revision: globalRevision(document)
10659
+ });
10660
+ }
10661
+ cacheWorkflow(workflowId, entry) {
10662
+ const held = this.workflowCache.get(workflowId);
10663
+ if (held && held.revision > entry.revision) return;
10664
+ this.workflowCache.delete(workflowId);
10665
+ this.workflowCache.set(workflowId, entry);
10666
+ if (this.workflowCache.size > WORKFLOW_CACHE_MAX) {
10667
+ const oldest = this.workflowCache.keys().next().value;
10668
+ if (oldest !== void 0) this.workflowCache.delete(oldest);
10669
+ }
10670
+ }
10662
10671
  seenOpsQueue = [];
10663
10672
  alreadySeen(key) {
10664
10673
  if (this.seenOps.has(key)) return true;
@@ -10694,12 +10703,22 @@ var WorkflowRuntimeService = class {
10694
10703
  context
10695
10704
  });
10696
10705
  if (this.alreadySeen(opKey)) continue;
10706
+ if (context.documentType === WORKFLOW_DOCUMENT_TYPE) this.workflowGenerations.set(context.documentId, this.workflowGeneration(context.documentId) + 1);
10697
10707
  if (context.scope === DOCUMENT_SCOPE) {
10708
+ this.workflowCache.delete(context.documentId);
10698
10709
  await this.matchDocumentLifecycle(operation, context, hints);
10699
10710
  await this.forgetDeletedWorkflow(operation, context);
10700
10711
  continue;
10701
10712
  }
10702
10713
  if (context.documentType === "powerhouse/workflow") {
10714
+ const cached = this.workflowCache.get(context.documentId);
10715
+ const carried = cached && operation.error === void 0 ? parseWorkflowState(operation.resultingState) : void 0;
10716
+ if (cached && carried) this.cacheWorkflow(context.documentId, {
10717
+ state: carried,
10718
+ name: cached.name,
10719
+ revision: operation.index + 1
10720
+ });
10721
+ else if (cached) this.workflowCache.delete(context.documentId);
10703
10722
  const rechecked = await this.recheckReactorAccess(context.documentId, operation);
10704
10723
  await this.refreshRegistration(context.documentId, operation.resultingState, !rechecked);
10705
10724
  }
@@ -10809,7 +10828,7 @@ var WorkflowRuntimeService = class {
10809
10828
  this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
10810
10829
  }, (error) => {
10811
10830
  if (enqueuedRunId && isShutdownRefusal(error)) {
10812
- this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to rerun`);
10831
+ this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left for the next owner to start`);
10813
10832
  return;
10814
10833
  }
10815
10834
  this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
@@ -10871,6 +10890,7 @@ var WorkflowRuntimeService = class {
10871
10890
  if (context.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
10872
10891
  const workflowId = stringField(inputRecord(operation.action.input), "documentId") ?? context.documentId;
10873
10892
  this.disarmDeleted(workflowId);
10893
+ this.workflowCache.delete(workflowId);
10874
10894
  try {
10875
10895
  await (await this.store())?.deleteDedupe(workflowId);
10876
10896
  } catch (error) {
@@ -10972,6 +10992,8 @@ var WorkflowRuntimeService = class {
10972
10992
  webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
10973
10993
  reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
10974
10994
  models: this.models,
10995
+ worker: this.workers().session("trigger"),
10996
+ hookConcurrency: this.workers().triggerSlots,
10975
10997
  telemetry: this.telemetry,
10976
10998
  cacheDir: bundleCacheDir(),
10977
10999
  resolver: pieceResolver(),
@@ -10991,8 +11013,29 @@ var WorkflowRuntimeService = class {
10991
11013
  }
10992
11014
  startTriggerSupervisor() {
10993
11015
  this.supervisor().start();
11016
+ this.admitAbandonedRuns();
11017
+ this.pendingSweepTimer ??= setInterval(() => {
11018
+ this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
11019
+ }, PENDING_SWEEP_INTERVAL_MS);
11020
+ this.pendingSweepTimer.unref();
11021
+ }
11022
+ pendingSweepTimer;
11023
+ admittingAbandoned = false;
11024
+ async admitAbandonedRuns(minAgeMs) {
11025
+ if (this.admittingAbandoned || this.closed) return;
11026
+ this.admittingAbandoned = true;
11027
+ try {
11028
+ const admitted = await (await this.store())?.admitAbandonedRuns(minAgeMs === void 0 ? {} : { enqueuedBefore: new Date(Date.now() - minAgeMs) });
11029
+ for (const run of admitted ?? []) this.startFiring(run.workflowId, run.triggerPayload, run.triggerKind, run.runId);
11030
+ } catch (error) {
11031
+ this.logger.error("Could not admit the runs a stopped reactor journaled: @error", error);
11032
+ } finally {
11033
+ this.admittingAbandoned = false;
11034
+ }
10994
11035
  }
10995
11036
  stopTriggerSupervisor() {
11037
+ clearInterval(this.pendingSweepTimer);
11038
+ this.pendingSweepTimer = void 0;
10996
11039
  this.triggerSupervisor?.stop();
10997
11040
  }
10998
11041
  closed = false;
@@ -11002,6 +11045,10 @@ var WorkflowRuntimeService = class {
11002
11045
  shutdown() {
11003
11046
  this.closed = true;
11004
11047
  this.runGate.close();
11048
+ this.runDispatcher?.close();
11049
+ this.eventLoop.stop();
11050
+ this.unobserveDispatcher?.();
11051
+ this.unobserveDispatcher = void 0;
11005
11052
  clearInterval(this.retentionTimer);
11006
11053
  for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
11007
11054
  this.resolutionRetries.clear();
@@ -11010,7 +11057,6 @@ var WorkflowRuntimeService = class {
11010
11057
  this.pieceWorkers?.dispose();
11011
11058
  this.unobservePool?.();
11012
11059
  this.unobservePool = void 0;
11013
- this.designWorker?.dispose();
11014
11060
  this.designWorker = void 0;
11015
11061
  }
11016
11062
  async triggerStates(ctx) {
@@ -11297,10 +11343,7 @@ var WorkflowRuntimeService = class {
11297
11343
  descriptors = /* @__PURE__ */ new Map();
11298
11344
  designWorker;
11299
11345
  createDesignWorker() {
11300
- return new PieceWorker({
11301
- models: this.models,
11302
- telemetry: this.telemetry
11303
- });
11346
+ return this.workers().session("design");
11304
11347
  }
11305
11348
  designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
11306
11349
  describing = /* @__PURE__ */ new Map();
@@ -12099,17 +12142,51 @@ var WorkflowRuntimeService = class {
12099
12142
  }
12100
12143
  workers() {
12101
12144
  if (this.pieceWorkers) return this.pieceWorkers;
12145
+ const env = (name) => Number(process.env[name]) || void 0;
12146
+ const count = (name) => {
12147
+ const value = Number(process.env[name] || NaN);
12148
+ return value >= 0 ? value : void 0;
12149
+ };
12102
12150
  const pool = new PieceWorkerPool({
12103
12151
  models: this.models,
12104
- size: Number(process.env.PH_WORKFLOWS_RUN_CONCURRENCY) || void 0,
12105
- maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0,
12152
+ size: env("PH_WORKFLOWS_RUN_CONCURRENCY"),
12153
+ maxQueueDepth: env("PH_WORKFLOWS_RUN_QUEUE_DEPTH"),
12154
+ heapMb: env("PH_WORKFLOWS_WORKER_MEMORY_MB"),
12155
+ maxRequests: env("PH_WORKFLOWS_WORKER_MAX_REQUESTS"),
12156
+ minWarm: count("PH_WORKFLOWS_WORKER_MIN"),
12157
+ triggerSlots: env("PH_WORKFLOWS_TRIGGER_SLOTS"),
12158
+ reactorThreads: this.host.reactorThreads,
12106
12159
  telemetry: this.telemetry
12107
12160
  });
12161
+ if (this.closed) {
12162
+ pool.dispose();
12163
+ return this.pieceWorkers = pool;
12164
+ }
12108
12165
  this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
12109
12166
  return this.pieceWorkers = pool;
12110
12167
  }
12168
+ dispatcher() {
12169
+ if (this.runDispatcher) return this.runDispatcher;
12170
+ const configured = Number(process.env.PH_WORKFLOWS_ADMIT_MAX_ELU);
12171
+ const dispatcher = new RunDispatcher({
12172
+ maxInFlight: Number(process.env.PH_WORKFLOWS_RUNS_IN_FLIGHT) || 8 * this.workers().size,
12173
+ maxElu: configured > 0 && configured <= 1 ? configured : DEFAULT_ADMIT_MAX_ELU,
12174
+ loop: this.eventLoop
12175
+ });
12176
+ this.runDispatcher = dispatcher;
12177
+ if (this.closed) {
12178
+ dispatcher.close();
12179
+ return dispatcher;
12180
+ }
12181
+ this.eventLoop.start();
12182
+ this.unobserveDispatcher = this.telemetry.observeDispatcher(() => ({
12183
+ ...dispatcher.stats(),
12184
+ utilization: this.eventLoop.utilization()
12185
+ }));
12186
+ return dispatcher;
12187
+ }
12111
12188
  async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
12112
- if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
12189
+ if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId, triggerKind);
12113
12190
  const started = performance.now();
12114
12191
  const attributes = {
12115
12192
  "workflow.id": workflowId,
@@ -12144,18 +12221,29 @@ var WorkflowRuntimeService = class {
12144
12221
  let documentName;
12145
12222
  let definition;
12146
12223
  let runUser;
12224
+ let readGeneration;
12147
12225
  try {
12148
- if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
12226
+ if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
12149
12227
  run.refused = true;
12150
12228
  throw error;
12151
12229
  });
12152
- const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
12153
- if (document.header.documentType !== "powerhouse/workflow") {
12154
- run.refused = true;
12155
- throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
12230
+ const cached = triggerKind === "manual" || resume ? void 0 : this.workflowCache.get(workflowId);
12231
+ if (cached) {
12232
+ state = cached.state;
12233
+ documentName = cached.name;
12234
+ readGeneration = this.workflowGeneration(workflowId);
12235
+ } else {
12236
+ const generation = this.workflowGeneration(workflowId);
12237
+ readGeneration = generation;
12238
+ const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
12239
+ if (document.header.documentType !== "powerhouse/workflow") {
12240
+ run.refused = true;
12241
+ throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
12242
+ }
12243
+ state = document.state.global;
12244
+ documentName = document.header.name;
12245
+ this.cacheLoaded(workflowId, generation, document);
12156
12246
  }
12157
- state = document.state.global;
12158
- documentName = document.header.name;
12159
12247
  if (state.status !== "ENABLED") {
12160
12248
  run.refused = true;
12161
12249
  throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
@@ -12170,6 +12258,19 @@ var WorkflowRuntimeService = class {
12170
12258
  if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
12171
12259
  throw error;
12172
12260
  }
12261
+ let journaling;
12262
+ const journalOnQueue = () => {
12263
+ if (triggerKind !== "manual" || enqueuedRunId || resume || journaling) return;
12264
+ if (!store) return;
12265
+ journaling = store.enqueueRun({
12266
+ workflowId,
12267
+ triggerKind,
12268
+ triggerPayload
12269
+ }).then((runId) => {
12270
+ enqueuedRunId = runId;
12271
+ });
12272
+ journaling.catch(() => void 0);
12273
+ };
12173
12274
  const connections = declaredConnectionIds(definition);
12174
12275
  const executor = this.blockExecutor(store);
12175
12276
  const runnable = runnableDefinition(state);
@@ -12190,30 +12291,72 @@ var WorkflowRuntimeService = class {
12190
12291
  throw error;
12191
12292
  }
12192
12293
  if (parked) return skipped(parked, "parked");
12193
- const admission = await this.runGate.admit(workflowId, policy);
12294
+ const admission = await this.runGate.admit(workflowId, policy, journalOnQueue);
12295
+ try {
12296
+ await journaling;
12297
+ } catch (error) {
12298
+ if (admission.admitted) admission.release();
12299
+ throw error;
12300
+ }
12194
12301
  if (this.closed) {
12195
12302
  if (admission.admitted) admission.release();
12196
- return this.refuseClosed(store, workflowId, enqueuedRunId);
12303
+ return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
12197
12304
  }
12198
12305
  if (!admission.admitted) {
12199
- if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
12306
+ if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
12200
12307
  return skipped(admission.reason, admission.refusal);
12201
12308
  }
12202
- if (admission.waited) {
12203
- let stale;
12309
+ const staleAfter = async () => {
12204
12310
  try {
12205
- stale = await this.staleAfterWait(workflowId, triggerKind, runnable.version);
12311
+ return await this.staleAfterWait(workflowId, triggerKind, runnable.version, readGeneration);
12206
12312
  } catch (error) {
12207
- stale = `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
12313
+ return `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
12208
12314
  }
12315
+ };
12316
+ if (admission.waited) {
12317
+ const stale = await staleAfter();
12209
12318
  if (stale) {
12210
12319
  admission.release();
12211
12320
  return skipped(stale, "stale");
12212
12321
  }
12213
12322
  }
12323
+ const expired = () => skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
12214
12324
  if (deadline !== void 0 && Date.now() >= deadline) {
12215
12325
  admission.release();
12216
- return skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
12326
+ return expired();
12327
+ }
12328
+ let dispatched;
12329
+ let dispatcherWaited = false;
12330
+ try {
12331
+ dispatched = await this.dispatcher().admit(workflowId, () => {
12332
+ dispatcherWaited = true;
12333
+ journalOnQueue();
12334
+ });
12335
+ } catch (error) {
12336
+ await journaling?.catch(() => void 0);
12337
+ admission.release();
12338
+ if (this.closed) return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
12339
+ throw error;
12340
+ }
12341
+ try {
12342
+ await journaling;
12343
+ } catch (error) {
12344
+ dispatched();
12345
+ admission.release();
12346
+ throw error;
12347
+ }
12348
+ if (this.closed || deadline !== void 0 && Date.now() >= deadline) {
12349
+ dispatched();
12350
+ admission.release();
12351
+ return this.closed ? this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind) : expired();
12352
+ }
12353
+ if (dispatcherWaited) {
12354
+ const stale = await staleAfter();
12355
+ if (stale) {
12356
+ dispatched();
12357
+ admission.release();
12358
+ return skipped(stale, "stale");
12359
+ }
12217
12360
  }
12218
12361
  let runId = enqueuedRunId ?? null;
12219
12362
  let journalFailed = false;
@@ -12238,6 +12381,7 @@ var WorkflowRuntimeService = class {
12238
12381
  }) ?? null;
12239
12382
  });
12240
12383
  if (runId) run.span.setAttribute("run.id", runId);
12384
+ if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
12241
12385
  session = this.workers().session();
12242
12386
  const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
12243
12387
  const journal = store;
@@ -12300,6 +12444,7 @@ var WorkflowRuntimeService = class {
12300
12444
  } finally {
12301
12445
  session?.close();
12302
12446
  admission.release();
12447
+ dispatched();
12303
12448
  }
12304
12449
  }
12305
12450
  runGate = new WorkflowRunGate();
@@ -12313,7 +12458,8 @@ var WorkflowRuntimeService = class {
12313
12458
  return `Skipped: this workflow is PARKED (${park.reason})`;
12314
12459
  }
12315
12460
  /** Why a firing that waited for its slot may no longer run, if it may not. */
12316
- async staleAfterWait(workflowId, triggerKind, publishedVersion) {
12461
+ async staleAfterWait(workflowId, triggerKind, publishedVersion, readGeneration) {
12462
+ if (this.workflowGeneration(workflowId) === readGeneration) return this.parkedFiring(workflowId, triggerKind, publishedVersion);
12317
12463
  const state = (await this.host.reactorClient.get(workflowId)).state.global;
12318
12464
  if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
12319
12465
  if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
@@ -12345,10 +12491,18 @@ var WorkflowRuntimeService = class {
12345
12491
  * of bug this work package exists to stamp out. An already-enqueued row is
12346
12492
  * closed out in place, so nothing is left PENDING for a sweep to find.
12347
12493
  */
12348
- async refuseClosed(store, workflowId, enqueuedRunId) {
12494
+ async refuseClosed(store, workflowId, enqueuedRunId, triggerKind) {
12349
12495
  const error = new WorkflowRuntimeClosedError(workflowId);
12350
12496
  if (enqueuedRunId) {
12351
12497
  const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
12498
+ if (!redelivered && triggerKind !== "manual") {
12499
+ try {
12500
+ await store?.releaseAdmission(enqueuedRunId);
12501
+ } catch (failure) {
12502
+ this.logger.warn(`Could not give back the boot admission of run ${enqueuedRunId}: @error`, failure);
12503
+ }
12504
+ throw error;
12505
+ }
12352
12506
  try {
12353
12507
  if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
12354
12508
  else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));