@powerhousedao/reactor-workflow 6.2.3-dev.55 → 6.2.3-dev.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- import { $ as isPrivateAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerError, H as PieceWorkerPool, I as SecretDeletedError, J as redact, K as PieceWorkerTimeoutError, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as DEFAULT_EGRESS_POLICY, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerDisposedError, X as rememberSecrets, Y as redactMessage, Z as secretsFor, _ as resolvedBlock, _t as isIndeterminateError, at as pieceModuleRef, b as resolutionOf, c as canonicalJson, ct as UnsupportedPieceFeatureError, d as DynamicPropertiesError, dt as unsupportedTrigger, et as parseAddress, f as checkDynamicProperties, g as isHostBound, gt as setPieceRegistryUrl, h as CompositeBlockExecutor, ht as pieceRegistrySource, i as packagePieces, it as bundleResolver, j as declaredReactorConnectionIds, l as hashOf, lt as authMethodFor, m as ActivepiecesBlockExecutor, mt as npmRegistryBaseUrl, n as installedPiece, nt as FileTooLargeError, o as errorNameOf, ot as sourcedResolver, p as effectiveRetryPolicy, q as containsRedactedMarker, r as installedPieces, rt as maxFileBytes, s as runWorkflow, t as PieceRegistry, tt as extractDedupeKey, u as stepConfigHash, ut as unsupportedAuth, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-CsBrnpGl.js";
1
+ import { $ as rememberSecrets, A as AttachmentCache, B as SecretNotFoundError, C as blockLabel, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as PieceWorkerPool, H as secretRefFromId, I as resolveStepInput, J as PieceWorkerError, K as PieceWorker, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as redactMessage, S as storeHandlers, T as blockPorts, U as stepBlock, V as parseSecretRef, W as triggerBlock, X as containsRedactedMarker, Y as PieceWorkerTimeoutError, Z as redact, _ as effectiveRetryPolicy, _t as npmRegistryBaseUrl, a as PieceRegistry, at as FileTooLargeError, b as isHostBound, bt as isIndeterminateError, c as packagePieces, ct as pieceModuleRef, d as runWorkflow, dt as UnsupportedPieceFeatureError, et as secretsFor, f as canonicalJson, ft as authMethodFor, g as checkDynamicProperties, h as DynamicPropertiesError, i as tracedSteps, it as extractDedupeKey, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as sourcedResolver, m as stepConfigHash, mt as unsupportedTrigger, n as WorkflowTelemetry, nt as isPrivateAddress, o as installedPiece, ot as maxFileBytes, p as hashOf, pt as unsupportedAuth, q as PieceWorkerDisposedError, r as tracedMethods, rt as parseAddress, s as installedPieces, st as bundleResolver, t as TELEMETRY_SCOPE, tt as DEFAULT_EGRESS_POLICY, u as errorNameOf, v as ActivepiecesBlockExecutor, vt as pieceRegistrySource, w as unpinnedNote, x as resolvedBlock, y as CompositeBlockExecutor, yt as setPieceRegistryUrl, z as SecretDeletedError } from "./telemetry-DZYSsZSP.js";
2
2
  import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
3
3
  import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
4
4
  import { open, readdir, rm, stat } from "node:fs/promises";
@@ -18,9 +18,10 @@ import https from "node:https";
18
18
  import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
19
19
  import { Kind, parse } from "graphql";
20
20
  import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
21
- import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
21
+ import { SpanStatusCode, context } from "@opentelemetry/api";
22
22
  import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
23
23
  import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
24
+ import { performance as performance$1 } from "node:perf_hooks";
24
25
  //#region src/pieces/activepieces/piece-versions.ts
25
26
  const LISTED_TTL_MS = 3e5;
26
27
  const ABSENT_TTL_MS = 3e4;
@@ -1951,227 +1952,6 @@ async function buildReactorRunScope(host, request) {
1951
1952
  connection: { access: parsed.config.access ?? "write" }
1952
1953
  };
1953
1954
  }
1954
- //#endregion
1955
- //#region src/telemetry.ts
1956
- const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
1957
- var WorkflowTelemetry = class {
1958
- tracer;
1959
- meter;
1960
- runs;
1961
- runDuration;
1962
- stepDuration;
1963
- phaseDuration;
1964
- poolGauges;
1965
- constructor(options = {}) {
1966
- this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
1967
- this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
1968
- this.runs = this.meter.createCounter("workflow.runs", {
1969
- description: "Runs finished, by status and trigger kind",
1970
- unit: "{run}"
1971
- });
1972
- this.runDuration = this.meter.createHistogram("workflow.run.duration", {
1973
- description: "Run duration from fire to the journal closing it",
1974
- unit: "ms"
1975
- });
1976
- this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
1977
- description: "Step duration, by piece and status",
1978
- unit: "ms"
1979
- });
1980
- this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
1981
- description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
1982
- unit: "ms"
1983
- });
1984
- }
1985
- phase(name, attributes, fn, parent) {
1986
- const started = performance.now();
1987
- return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
1988
- }
1989
- workerTimings(request, timings, forkedAt, ipc) {
1990
- const under = (span) => trace.setSpan(context.active(), span);
1991
- const child = (name, start, end, parent = request, attributes) => {
1992
- const span = this.tracer.startSpan(`workflow.worker.${name}`, {
1993
- startTime: start,
1994
- ...attributes ? { attributes } : {}
1995
- }, under(parent));
1996
- span.end(end);
1997
- this.recordPhase(`worker.${name}`, end - start);
1998
- return span;
1999
- };
2000
- const ready = epoch(timings.ready);
2001
- if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
2002
- const received = epoch(timings.received);
2003
- const sent = epoch(timings.sent);
2004
- const delivered = Math.max(ipc.sent, ready ?? 0);
2005
- if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
2006
- const byName = /* @__PURE__ */ new Map();
2007
- const phases = Array.isArray(timings.phases) ? timings.phases : [];
2008
- for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
2009
- const parent = phase.parent ? byName.get(phase.parent) : void 0;
2010
- byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
2011
- }
2012
- if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
2013
- }
2014
- recordPhase(name, ms) {
2015
- this.phaseDuration.record(ms, { phase: name });
2016
- }
2017
- span(name, attributes, fn, parent = context.active()) {
2018
- return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
2019
- try {
2020
- return await fn(span);
2021
- } catch (error) {
2022
- failSpan(span, error);
2023
- throw error;
2024
- } finally {
2025
- span.end();
2026
- }
2027
- });
2028
- }
2029
- recordRun(ms, status, triggerKind) {
2030
- const attributes = {
2031
- "run.status": status,
2032
- "trigger.kind": metricTriggerKind(triggerKind)
2033
- };
2034
- this.runs.add(1, attributes);
2035
- this.runDuration.record(ms, attributes);
2036
- }
2037
- recordStep(ms, pieceName, status) {
2038
- this.stepDuration.record(ms, {
2039
- "piece.name": pieceName,
2040
- "step.status": status
2041
- });
2042
- }
2043
- observeWorkerPool(stats) {
2044
- this.poolGauges ??= this.createPoolGauges();
2045
- const { size, active, waiting } = this.poolGauges;
2046
- const observables = [
2047
- size,
2048
- active,
2049
- waiting
2050
- ];
2051
- const callback = (result) => {
2052
- const current = stats();
2053
- result.observe(size, current.size);
2054
- result.observe(active, current.active);
2055
- result.observe(waiting, current.waiting);
2056
- };
2057
- this.meter.addBatchObservableCallback(callback, observables);
2058
- return () => this.meter.removeBatchObservableCallback(callback, observables);
2059
- }
2060
- createPoolGauges() {
2061
- const gauge = (name, description) => this.meter.createObservableGauge(name, {
2062
- description,
2063
- unit: "{worker}"
2064
- });
2065
- return {
2066
- size: gauge("workflow.worker.pool.size", "Worker slots"),
2067
- active: gauge("workflow.worker.pool.active", "Slots held by runs"),
2068
- waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
2069
- };
2070
- }
2071
- };
2072
- const WORKER_PHASES = /* @__PURE__ */ new Set([
2073
- "piece.load",
2074
- "reactor.open",
2075
- "models",
2076
- "action"
2077
- ]);
2078
- const WORKER_ATTRIBUTES = {
2079
- "piece.cached": "boolean",
2080
- "document.type": "string"
2081
- };
2082
- const MAX_ATTRIBUTE_LENGTH = 256;
2083
- function epoch(value) {
2084
- return typeof value === "number" && Number.isFinite(value) ? value : void 0;
2085
- }
2086
- function knownPhase(phase) {
2087
- if (typeof phase !== "object" || phase === null) return false;
2088
- const { name, start, end, parent } = phase;
2089
- return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
2090
- }
2091
- function phaseAttributes(attributes) {
2092
- if (typeof attributes !== "object" || attributes === null) return void 0;
2093
- const kept = {};
2094
- for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
2095
- const value = attributes[key];
2096
- if (typeof value !== type) continue;
2097
- kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
2098
- }
2099
- return Object.keys(kept).length > 0 ? kept : void 0;
2100
- }
2101
- function metricTriggerKind(triggerKind) {
2102
- return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
2103
- }
2104
- function metricPieceName(step, resolution) {
2105
- return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
2106
- }
2107
- function tracedSteps(executor, telemetry) {
2108
- return { execute(execution) {
2109
- const { step } = execution;
2110
- const started = performance.now();
2111
- let status = "FAILED";
2112
- let resolution;
2113
- return telemetry.span("workflow.step", {
2114
- "step.id": step.id,
2115
- "step.key": step.key,
2116
- "piece.name": step.pieceName,
2117
- "action.name": step.actionName
2118
- }, async (span) => {
2119
- try {
2120
- const result = await executor.execute(execution);
2121
- resolution = result.resolution;
2122
- status = "SUCCEEDED";
2123
- const version = resolution?.resolved?.version;
2124
- if (version) span.setAttribute("piece.version", version);
2125
- return result;
2126
- } catch (error) {
2127
- resolution = resolutionOf(error);
2128
- throw error;
2129
- }
2130
- }).finally(() => {
2131
- telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
2132
- });
2133
- } };
2134
- }
2135
- function failSpan(span, error) {
2136
- span.recordException(error instanceof Error ? error : String(error));
2137
- span.setStatus({
2138
- code: SpanStatusCode.ERROR,
2139
- message: error instanceof Error ? error.message : String(error)
2140
- });
2141
- }
2142
- function tracedMethods(target, telemetry, prefix, parent) {
2143
- return new Proxy(target, { get(object, property, receiver) {
2144
- const value = Reflect.get(object, property, receiver);
2145
- if (typeof value !== "function" || typeof property !== "string") return value;
2146
- const phase = `${prefix}.${property}`;
2147
- return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
2148
- const started = performance.now();
2149
- const end = (error) => {
2150
- if (error !== void 0) failSpan(span, error);
2151
- span.end();
2152
- telemetry.recordPhase(phase, performance.now() - started);
2153
- };
2154
- let result;
2155
- try {
2156
- result = value.apply(object, args);
2157
- } catch (error) {
2158
- end(error);
2159
- throw error;
2160
- }
2161
- if (!(result instanceof Promise)) {
2162
- end();
2163
- return result;
2164
- }
2165
- return result.then((resolved) => {
2166
- end();
2167
- return resolved;
2168
- }, (error) => {
2169
- end(error);
2170
- throw error;
2171
- });
2172
- });
2173
- } });
2174
- }
2175
1955
  const JOB_NOT_FOUND = "Job not found";
2176
1956
  const MIN_DEADLINE_MARGIN_MS = 250;
2177
1957
  const MAX_DEADLINE_MARGIN_MS = 2e3;
@@ -7575,6 +7355,9 @@ function triggerBlockColumns(block) {
7575
7355
  }
7576
7356
  const FIRED_WITHOUT_RUN_ID = "fired-without-run";
7577
7357
  const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
7358
+ function abandonedAdmissionError(attempts) {
7359
+ return `This run was admitted at ${attempts} boots and the reactor stopped each time before it started, so it is no longer admitted on boot. Fix the cause, then rerun it to fire the workflow with the same payload.`;
7360
+ }
7578
7361
  function abandonedFireError(attempts) {
7579
7362
  return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
7580
7363
  }
@@ -7592,7 +7375,6 @@ const logger$4 = childLogger([
7592
7375
  "store"
7593
7376
  ]);
7594
7377
  const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
7595
- const ABANDONED_PENDING_RUN_ERROR = "Reactor stopped before the matched trigger started its run; rerun it to fire the workflow with the same payload";
7596
7378
  const PENDING_RUN_STATUS = "PENDING";
7597
7379
  function errorMessage(error) {
7598
7380
  return error instanceof Error ? error.message : String(error);
@@ -7676,6 +7458,9 @@ async function up(db) {
7676
7458
  await db.schema.alterTable("run").addColumn("enqueued_at", "text").execute();
7677
7459
  } catch {}
7678
7460
  await db.updateTable("run").set({ enqueued_at: sql.ref("started_at") }).where("enqueued_at", "is", null).execute();
7461
+ try {
7462
+ await db.schema.alterTable("run").addColumn("admit_attempts", "integer", (col) => col.notNull().defaultTo(0)).execute();
7463
+ } catch {}
7679
7464
  await db.schema.dropIndex("run_workflow_started").ifExists().execute();
7680
7465
  await db.schema.dropIndex("run_started").ifExists().execute();
7681
7466
  await db.schema.createIndex("run_workflow_enqueued").ifNotExists().on("run").columns([
@@ -7687,6 +7472,7 @@ async function up(db) {
7687
7472
  await db.schema.createIndex("trigger_dedupe_workflow_created").ifNotExists().on("trigger_dedupe").columns(["workflow_id", "created_at"]).execute();
7688
7473
  await db.schema.createIndex("run_ended").ifNotExists().on("run").column("ended_at").execute();
7689
7474
  await db.schema.createIndex("trigger_dedupe_created").ifNotExists().on("trigger_dedupe").column("created_at").execute();
7475
+ await db.schema.createIndex("run_pending").ifNotExists().on("run").columns(["enqueued_at", "id"]).where(sql.ref("status"), "=", PENDING_RUN_STATUS).execute();
7690
7476
  await db.schema.createIndex("trigger_state_due").ifNotExists().on("trigger_state").column("next_poll_at").where(sql.ref("status"), "=", "ENABLED").execute();
7691
7477
  try {
7692
7478
  await db.schema.dropTable("webhook_endpoint").ifExists().execute();
@@ -7894,6 +7680,10 @@ const LEGACY_MARKER_KEYS = [
7894
7680
  * ambiguous, and nothing can resolve that for a row already written; the
7895
7681
  * window is as narrow as the stored data allows.
7896
7682
  */
7683
+ function unreplayablePayload(payload) {
7684
+ if (isTruncatedStepPayload(payload)) return "truncated";
7685
+ if (containsRedactedMarker(payload)) return "redacted";
7686
+ }
7897
7687
  function isTruncatedStepPayload(value) {
7898
7688
  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
7899
7689
  const record = value;
@@ -7924,6 +7714,9 @@ function cappedPayload(value) {
7924
7714
  };
7925
7715
  return JSON.stringify(marker);
7926
7716
  }
7717
+ function rowHash(values) {
7718
+ return createHash("sha1").update(JSON.stringify(values)).digest("base64");
7719
+ }
7927
7720
  function stepValues(runId, ordinal, step) {
7928
7721
  return {
7929
7722
  run_id: runId,
@@ -8041,6 +7834,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8041
7834
  db;
8042
7835
  unmigrated;
8043
7836
  runsInFlight = /* @__PURE__ */ new Set();
7837
+ journaledSteps = /* @__PURE__ */ new Map();
8044
7838
  constructor(db, unmigrated) {
8045
7839
  this.db = db;
8046
7840
  this.unmigrated = unmigrated;
@@ -8050,7 +7844,6 @@ var WorkflowRunStore = class WorkflowRunStore {
8050
7844
  const unmigrated = await up(db);
8051
7845
  const store = new WorkflowRunStore(db, unmigrated);
8052
7846
  await store.recoverOrphanedRuns();
8053
- await store.recoverAbandonedRuns();
8054
7847
  return store;
8055
7848
  }
8056
7849
  hasUnmigratedTriggerState(workflowId) {
@@ -8070,16 +7863,56 @@ var WorkflowRunStore = class WorkflowRunStore {
8070
7863
  if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
8071
7864
  return recovered;
8072
7865
  }
8073
- async recoverAbandonedRuns() {
8074
- let query = this.db.updateTable("run").set({
8075
- status: "FAILED",
8076
- error: ABANDONED_PENDING_RUN_ERROR,
8077
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8078
- }).where("status", "=", PENDING_RUN_STATUS);
8079
- if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
8080
- const recovered = (await query.returning("id").execute()).length;
8081
- if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
8082
- return recovered;
7866
+ async admitAbandonedRuns(options = {}) {
7867
+ const admitted = await this.db.transaction().execute(async (trx) => {
7868
+ let query = trx.selectFrom("run").select([
7869
+ "id",
7870
+ "workflow_id",
7871
+ "trigger_kind",
7872
+ "trigger_payload",
7873
+ "admit_attempts"
7874
+ ]).where("status", "=", PENDING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc");
7875
+ if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
7876
+ if (options.enqueuedBefore) query = query.where("enqueued_at", "<", options.enqueuedBefore.toISOString());
7877
+ const rows = await query.execute();
7878
+ const runs = [];
7879
+ const now = (/* @__PURE__ */ new Date()).toISOString();
7880
+ for (const row of rows) {
7881
+ if (row.admit_attempts >= 3) {
7882
+ await trx.updateTable("run").set({
7883
+ status: "FAILED",
7884
+ error: abandonedAdmissionError(row.admit_attempts),
7885
+ ended_at: now
7886
+ }).where("id", "=", row.id).execute();
7887
+ continue;
7888
+ }
7889
+ const triggerPayload = row.trigger_payload === null ? void 0 : JSON.parse(row.trigger_payload);
7890
+ const unreplayable = unreplayablePayload(triggerPayload);
7891
+ if (unreplayable) {
7892
+ await trx.updateTable("run").set({
7893
+ status: "FAILED",
7894
+ error: `The trigger payload was ${unreplayable} by the journal and cannot be run after a restart; fire the workflow again`,
7895
+ ended_at: now
7896
+ }).where("id", "=", row.id).execute();
7897
+ continue;
7898
+ }
7899
+ await trx.updateTable("run").set({ admit_attempts: row.admit_attempts + 1 }).where("id", "=", row.id).execute();
7900
+ runs.push({
7901
+ runId: row.id,
7902
+ workflowId: row.workflow_id,
7903
+ triggerKind: row.trigger_kind,
7904
+ triggerPayload
7905
+ });
7906
+ }
7907
+ return runs;
7908
+ });
7909
+ for (const run of admitted) this.runsInFlight.add(run.runId);
7910
+ if (admitted.length > 0) logger$4.info(`Admitting ${admitted.length} workflow run(s) journaled by a stopped reactor`);
7911
+ return admitted;
7912
+ }
7913
+ async releaseAdmission(runId) {
7914
+ this.runsInFlight.delete(runId);
7915
+ await this.db.updateTable("run").set({ admit_attempts: sql`greatest(admit_attempts - 1, 0)` }).where("id", "=", runId).where("status", "=", PENDING_RUN_STATUS).execute();
8083
7916
  }
8084
7917
  /**
8085
7918
  * Closes a run out as CANCELLED, with the reason as its error.
@@ -8091,6 +7924,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8091
7924
  */
8092
7925
  async cancelRun(runId, reason) {
8093
7926
  this.runsInFlight.delete(runId);
7927
+ this.journaledSteps.delete(runId);
8094
7928
  if (erasedRuns.delete(runId)) return;
8095
7929
  await this.db.updateTable("run").set({
8096
7930
  status: "CANCELLED",
@@ -8100,10 +7934,20 @@ var WorkflowRunStore = class WorkflowRunStore {
8100
7934
  }
8101
7935
  async enqueueRun(options) {
8102
7936
  const id = randomUUID();
8103
- await this.insertPendingRun(this.db, id, options);
8104
- this.runsInFlight.add(id);
7937
+ await this.unmarkOnFailure([id], () => this.insertPendingRun(this.db, id, options));
8105
7938
  return id;
8106
7939
  }
7940
+ async unmarkOnFailure(ids, write) {
7941
+ try {
7942
+ return await write();
7943
+ } catch (error) {
7944
+ for (const id of ids) {
7945
+ this.runsInFlight.delete(id);
7946
+ this.journaledSteps.delete(id);
7947
+ }
7948
+ throw error;
7949
+ }
7950
+ }
8107
7951
  /**
8108
7952
  * Claims the dedupe key and journals the PENDING run, bounding crash replays.
8109
7953
  *
@@ -8147,38 +7991,39 @@ var WorkflowRunStore = class WorkflowRunStore {
8147
7991
  attempts: attempt.attempts
8148
7992
  };
8149
7993
  const id = randomUUID();
8150
- if (!await this.db.transaction().execute(async (trx) => {
7994
+ if (!await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
8151
7995
  if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
8152
7996
  await this.insertPendingRun(trx, id, options);
8153
7997
  return true;
8154
- })) return { outcome: "duplicate" };
8155
- this.runsInFlight.add(id);
7998
+ }))) return { outcome: "duplicate" };
8156
7999
  return {
8157
8000
  outcome: "claimed",
8158
8001
  runId: id
8159
8002
  };
8160
8003
  }
8161
- async journalTriggerItems(items, ttlMs, nowIso) {
8162
- const ids = await this.db.transaction().execute(async (trx) => {
8004
+ async journalTriggerItems(items, ttlMs, nowIso, success) {
8005
+ const marked = [];
8006
+ return this.unmarkOnFailure(marked, () => this.db.transaction().execute(async (trx) => {
8007
+ if (success) await pollSucceededIn(trx, success.workflowId, success.storeState, nowIso, success.nextPollAtIso);
8163
8008
  const journaled = [];
8164
8009
  for (const { dedupeKey, options } of items) {
8165
8010
  const id = randomUUID();
8166
8011
  const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
8167
- if (won) await this.insertPendingRun(trx, id, options);
8012
+ if (won) {
8013
+ marked.push(id);
8014
+ await this.insertPendingRun(trx, id, options);
8015
+ }
8168
8016
  journaled.push(won ? id : void 0);
8169
8017
  }
8170
8018
  return journaled;
8171
- });
8172
- for (const id of ids) if (id) this.runsInFlight.add(id);
8173
- return ids;
8019
+ }));
8174
8020
  }
8175
8021
  async recordScheduleFire(nowIso, nextPollAtIso, options) {
8176
8022
  const id = randomUUID();
8177
- await this.db.transaction().execute(async (trx) => {
8023
+ await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
8178
8024
  await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
8179
8025
  await this.insertPendingRun(trx, id, options);
8180
- });
8181
- this.runsInFlight.add(id);
8026
+ }));
8182
8027
  return id;
8183
8028
  }
8184
8029
  /**
@@ -8214,6 +8059,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8214
8059
  });
8215
8060
  for (const runId of redelivered.runIds) {
8216
8061
  this.runsInFlight.delete(runId);
8062
+ this.journaledSteps.delete(runId);
8217
8063
  erasedRuns.delete(runId);
8218
8064
  }
8219
8065
  }
@@ -8270,6 +8116,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8270
8116
  return id;
8271
8117
  }
8272
8118
  async insertPendingRun(db, id, options) {
8119
+ this.runsInFlight.add(id);
8273
8120
  const now = (/* @__PURE__ */ new Date()).toISOString();
8274
8121
  await db.insertInto("run").values({
8275
8122
  id,
@@ -8290,6 +8137,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8290
8137
  }
8291
8138
  async beginRun(runId, details) {
8292
8139
  this.runsInFlight.add(runId);
8140
+ this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
8293
8141
  await this.db.updateTable("run").set({
8294
8142
  status: "RUNNING",
8295
8143
  workflow_name: details.workflowName,
@@ -8317,6 +8165,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8317
8165
  warning_notes: null
8318
8166
  }).execute();
8319
8167
  this.runsInFlight.add(id);
8168
+ this.journaledSteps.set(id, /* @__PURE__ */ new Map());
8320
8169
  return id;
8321
8170
  }
8322
8171
  async recordStep(runId, ordinal, step) {
@@ -8327,11 +8176,26 @@ var WorkflowRunStore = class WorkflowRunStore {
8327
8176
  id: randomUUID(),
8328
8177
  ...values
8329
8178
  }).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
8179
+ this.journaledSteps.get(runId)?.set(step.stepId, {
8180
+ ordinal,
8181
+ hash: rowHash(values)
8182
+ });
8183
+ }
8184
+ sweepIsNoop(runId, result) {
8185
+ const journaled = this.journaledSteps.get(runId);
8186
+ if (!journaled) return false;
8187
+ return result.steps.every((step) => {
8188
+ const row = journaled.get(step.stepId);
8189
+ if (row === void 0) return false;
8190
+ return row.hash === rowHash(stepValues(runId, row.ordinal, step));
8191
+ });
8330
8192
  }
8331
8193
  async finishRun(runId, result, executionOrder) {
8332
8194
  this.runsInFlight.delete(runId);
8195
+ const sweep = !this.sweepIsNoop(runId, result);
8196
+ this.journaledSteps.delete(runId);
8333
8197
  if (erasedRuns.delete(runId)) return;
8334
- if (result.steps.length > 0) try {
8198
+ if (sweep && result.steps.length > 0) try {
8335
8199
  await this.sweepSteps(runId, result, executionOrder);
8336
8200
  } catch (error) {
8337
8201
  logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
@@ -8382,6 +8246,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8382
8246
  }
8383
8247
  async failRun(runId, error, errorName) {
8384
8248
  this.runsInFlight.delete(runId);
8249
+ this.journaledSteps.delete(runId);
8385
8250
  if (erasedRuns.delete(runId)) return;
8386
8251
  await this.db.updateTable("run").set({
8387
8252
  status: "FAILED",
@@ -8764,7 +8629,7 @@ var WorkflowRunGate = class {
8764
8629
  * journals the refusal rather than dropping it silently, because a firing
8765
8630
  * that vanished is indistinguishable from a trigger that never fired.
8766
8631
  */
8767
- async admit(workflowId, policy) {
8632
+ async admit(workflowId, policy, onQueue) {
8768
8633
  if (this.closed) return {
8769
8634
  admitted: false,
8770
8635
  reason: CLOSED_REASON,
@@ -8796,6 +8661,7 @@ var WorkflowRunGate = class {
8796
8661
  refusal: "queue-full",
8797
8662
  reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
8798
8663
  };
8664
+ onQueue?.();
8799
8665
  if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
8800
8666
  admitted: false,
8801
8667
  reason: CLOSED_REASON,
@@ -8831,6 +8697,117 @@ var WorkflowRunGate = class {
8831
8697
  }
8832
8698
  };
8833
8699
  //#endregion
8700
+ //#region src/reactor/shutdown.ts
8701
+ /** A firing refused because this runtime has shut down: retryable elsewhere. */
8702
+ var WorkflowRuntimeClosedError = class extends Error {
8703
+ workflowId;
8704
+ constructor(workflowId) {
8705
+ super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
8706
+ this.workflowId = workflowId;
8707
+ this.name = "WorkflowRuntimeClosedError";
8708
+ }
8709
+ };
8710
+ var TriggerSupervisorStoppedError = class extends Error {
8711
+ constructor(options) {
8712
+ super("The trigger supervisor has stopped", options);
8713
+ this.name = "TriggerSupervisorStoppedError";
8714
+ }
8715
+ };
8716
+ function isShutdownRefusal(error) {
8717
+ return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
8718
+ }
8719
+ //#endregion
8720
+ //#region src/reactor/run-dispatcher.ts
8721
+ const DEFAULT_ADMIT_MAX_ELU = .8;
8722
+ var RunDispatcher = class {
8723
+ maxInFlight;
8724
+ maxElu;
8725
+ loop;
8726
+ waiters = [];
8727
+ inFlight = 0;
8728
+ closed = false;
8729
+ unsubscribe;
8730
+ constructor(options) {
8731
+ this.maxInFlight = Math.max(1, Math.trunc(options.maxInFlight));
8732
+ this.maxElu = options.maxElu ?? .8;
8733
+ this.loop = options.loop;
8734
+ this.unsubscribe = this.loop.onSample(() => this.drain());
8735
+ }
8736
+ admit(workflowId, onQueue) {
8737
+ if (this.closed) return Promise.reject(new WorkflowRuntimeClosedError(workflowId));
8738
+ if (this.waiters.length === 0 && this.room()) return Promise.resolve(this.take());
8739
+ onQueue?.();
8740
+ return new Promise((resolve, reject) => {
8741
+ this.waiters.push({
8742
+ workflowId,
8743
+ resolve,
8744
+ reject
8745
+ });
8746
+ });
8747
+ }
8748
+ stats() {
8749
+ return {
8750
+ inFlight: this.inFlight,
8751
+ waiting: this.waiters.length
8752
+ };
8753
+ }
8754
+ close() {
8755
+ if (this.closed) return;
8756
+ this.closed = true;
8757
+ this.unsubscribe();
8758
+ for (const waiter of this.waiters.splice(0)) waiter.reject(new WorkflowRuntimeClosedError(waiter.workflowId));
8759
+ }
8760
+ room() {
8761
+ if (this.inFlight === 0) return true;
8762
+ return this.inFlight < this.maxInFlight && this.loop.utilization() <= this.maxElu;
8763
+ }
8764
+ take() {
8765
+ this.inFlight++;
8766
+ let released = false;
8767
+ return () => {
8768
+ if (released) return;
8769
+ released = true;
8770
+ this.inFlight--;
8771
+ this.drain();
8772
+ };
8773
+ }
8774
+ drain() {
8775
+ while (this.waiters.length > 0 && !this.closed && this.room()) this.waiters.shift().resolve(this.take());
8776
+ }
8777
+ };
8778
+ //#endregion
8779
+ //#region src/reactor/event-loop.ts
8780
+ const SAMPLE_MS = 1e3;
8781
+ var EventLoopMonitor = class {
8782
+ last = performance$1.eventLoopUtilization();
8783
+ current = 0;
8784
+ timer;
8785
+ listeners = /* @__PURE__ */ new Set();
8786
+ start() {
8787
+ if (this.timer) return;
8788
+ this.last = performance$1.eventLoopUtilization();
8789
+ this.timer = setInterval(() => this.sample(), SAMPLE_MS);
8790
+ this.timer.unref();
8791
+ }
8792
+ stop() {
8793
+ clearInterval(this.timer);
8794
+ this.timer = void 0;
8795
+ }
8796
+ utilization() {
8797
+ return this.current;
8798
+ }
8799
+ onSample(listener) {
8800
+ this.listeners.add(listener);
8801
+ return () => this.listeners.delete(listener);
8802
+ }
8803
+ sample() {
8804
+ const now = performance$1.eventLoopUtilization();
8805
+ this.current = performance$1.eventLoopUtilization(now, this.last).utilization;
8806
+ this.last = now;
8807
+ for (const listener of this.listeners) listener();
8808
+ }
8809
+ };
8810
+ //#endregion
8834
8811
  //#region src/reactor/run-retention.ts
8835
8812
  const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
8836
8813
  const RETENTION_SWEEP_INTERVAL_MS = 36e5;
@@ -8866,26 +8843,6 @@ async function sweepRetention(store, options) {
8866
8843
  dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
8867
8844
  };
8868
8845
  }
8869
- //#endregion
8870
- //#region src/reactor/shutdown.ts
8871
- /** A firing refused because this runtime has shut down: retryable elsewhere. */
8872
- var WorkflowRuntimeClosedError = class extends Error {
8873
- workflowId;
8874
- constructor(workflowId) {
8875
- super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
8876
- this.workflowId = workflowId;
8877
- this.name = "WorkflowRuntimeClosedError";
8878
- }
8879
- };
8880
- var TriggerSupervisorStoppedError = class extends Error {
8881
- constructor(options) {
8882
- super("The trigger supervisor has stopped", options);
8883
- this.name = "TriggerSupervisorStoppedError";
8884
- }
8885
- };
8886
- function isShutdownRefusal(error) {
8887
- return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
8888
- }
8889
8846
  childLogger(["workflow", "trigger-binding"]);
8890
8847
  function pieceTriggerKind(block) {
8891
8848
  return `piece:${block.pieceName}:${block.name}`;
@@ -8976,7 +8933,9 @@ var TriggerSupervisor = class {
8976
8933
  egress;
8977
8934
  parks;
8978
8935
  timer;
8979
- ops = Promise.resolve();
8936
+ lanes = /* @__PURE__ */ new Map();
8937
+ ticked = /* @__PURE__ */ new Set();
8938
+ hookSlots;
8980
8939
  ticking = false;
8981
8940
  warnedMissingJournal = false;
8982
8941
  stopped = false;
@@ -8989,6 +8948,7 @@ var TriggerSupervisor = class {
8989
8948
  this.tickMs = options.tickMs ?? 15e3;
8990
8949
  this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
8991
8950
  this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
8951
+ this.hookSlots = new Slots(options.hookConcurrency ?? 1);
8992
8952
  this.now = options.now ?? (() => /* @__PURE__ */ new Date());
8993
8953
  this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
8994
8954
  this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
@@ -9015,8 +8975,8 @@ var TriggerSupervisor = class {
9015
8975
  if (!this.options.worker) this.worker.dispose();
9016
8976
  logger$2.info("Trigger supervisor stopped");
9017
8977
  }
9018
- enqueue(task) {
9019
- const run = this.ops.then(async () => {
8978
+ enqueue(workflowId, task) {
8979
+ const run = (this.lanes.get(workflowId) ?? Promise.resolve()).then(async () => {
9020
8980
  this.throwIfStopped();
9021
8981
  try {
9022
8982
  return await task();
@@ -9025,7 +8985,11 @@ var TriggerSupervisor = class {
9025
8985
  throw error;
9026
8986
  }
9027
8987
  });
9028
- this.ops = run.catch(() => void 0);
8988
+ const settled = run.catch(() => void 0);
8989
+ this.lanes.set(workflowId, settled);
8990
+ settled.then(() => {
8991
+ if (this.lanes.get(workflowId) === settled) this.lanes.delete(workflowId);
8992
+ });
9029
8993
  return run;
9030
8994
  }
9031
8995
  throwIfStopped() {
@@ -9063,12 +9027,12 @@ var TriggerSupervisor = class {
9063
9027
  this.bindings.set(binding.workflowId, binding);
9064
9028
  const park = this.parks.known(binding.workflowId)?.park;
9065
9029
  if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
9066
- return this.enqueue(() => this.enable(binding, previous));
9030
+ return this.enqueue(binding.workflowId, () => this.enable(binding, previous));
9067
9031
  }
9068
9032
  /** Parks a workflow; true when a trigger row was parked with it. */
9069
9033
  park(workflowId, publishedVersion, reason) {
9070
9034
  this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
9071
- return this.enqueue(async () => {
9035
+ return this.enqueue(workflowId, async () => {
9072
9036
  const store = await this.store();
9073
9037
  if (!store) return false;
9074
9038
  return store.parkWorkflow(workflowId, publishedVersion, reason);
@@ -9077,7 +9041,7 @@ var TriggerSupervisor = class {
9077
9041
  /** Undoes {@link park} for that version only: a later park stands. */
9078
9042
  liftPark(workflowId, publishedVersion, trigger) {
9079
9043
  if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
9080
- return this.enqueue(async () => {
9044
+ return this.enqueue(workflowId, async () => {
9081
9045
  const store = await this.store();
9082
9046
  if (!store) return;
9083
9047
  if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
@@ -9088,7 +9052,7 @@ var TriggerSupervisor = class {
9088
9052
  * row through onDisable (a boot), the row is turned DISABLED here. */
9089
9053
  releasePark(workflowId, flipRow) {
9090
9054
  this.parks.delete(workflowId);
9091
- return this.enqueue(async () => {
9055
+ return this.enqueue(workflowId, async () => {
9092
9056
  const store = await this.store();
9093
9057
  if (!store) return;
9094
9058
  if (flipRow) await store.clearParkOnDisable(workflowId);
@@ -9106,7 +9070,7 @@ var TriggerSupervisor = class {
9106
9070
  if (retry) retry.at = 0;
9107
9071
  const park = this.parks.known(workflowId)?.park;
9108
9072
  if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
9109
- return this.enqueue(async () => {
9073
+ return this.enqueue(workflowId, async () => {
9110
9074
  const store = await this.store();
9111
9075
  if (!store) return;
9112
9076
  const park = await store.getWorkflowPark(workflowId);
@@ -9116,12 +9080,12 @@ var TriggerSupervisor = class {
9116
9080
  }
9117
9081
  remove(workflowId) {
9118
9082
  const binding = this.unbind(workflowId);
9119
- return this.enqueue(() => this.disable(workflowId, binding));
9083
+ return this.enqueue(workflowId, () => this.disable(workflowId, binding));
9120
9084
  }
9121
9085
  forget(workflowId) {
9122
9086
  const binding = this.unbind(workflowId);
9123
9087
  this.parks.delete(workflowId);
9124
- return this.enqueue(async () => {
9088
+ return this.enqueue(workflowId, async () => {
9125
9089
  const store = await this.store();
9126
9090
  if (!store) return;
9127
9091
  await this.disable(workflowId, binding);
@@ -9139,7 +9103,7 @@ var TriggerSupervisor = class {
9139
9103
  }
9140
9104
  reject(workflowId, block, config, message, retryAt) {
9141
9105
  this.unbind(workflowId);
9142
- return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
9106
+ return this.enqueue(workflowId, () => this.recordRejection(workflowId, block, config, message, retryAt));
9143
9107
  }
9144
9108
  async recordRejection(workflowId, block, config, message, retryAt) {
9145
9109
  const store = await this.store();
@@ -9172,7 +9136,7 @@ var TriggerSupervisor = class {
9172
9136
  });
9173
9137
  }
9174
9138
  test(binding, options = {}) {
9175
- return this.enqueue(async () => {
9139
+ return this.enqueue(binding.workflowId, async () => {
9176
9140
  const store = await this.store();
9177
9141
  try {
9178
9142
  return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
@@ -9191,10 +9155,10 @@ var TriggerSupervisor = class {
9191
9155
  }
9192
9156
  }
9193
9157
  handshake(binding, payload) {
9194
- return this.enqueue(() => this.hook(binding, "onHandshake", { payload }));
9158
+ return this.enqueue(binding.workflowId, () => this.hook(binding, "onHandshake", { payload }));
9195
9159
  }
9196
9160
  deliverWebhook(workflowId, payload) {
9197
- return this.enqueue(async () => {
9161
+ return this.enqueue(workflowId, async () => {
9198
9162
  const binding = this.bindings.get(workflowId);
9199
9163
  if (!binding || isSchedule(binding)) {
9200
9164
  logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
@@ -9212,8 +9176,12 @@ var TriggerSupervisor = class {
9212
9176
  throw error;
9213
9177
  });
9214
9178
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
9215
- if (row?.status === "ENABLED") await store.recordPollSuccess(workflowId, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
9216
- await this.fireItems(store, binding, result.output, now, rewind);
9179
+ const success = row?.status === "ENABLED" ? {
9180
+ workflowId,
9181
+ storeState: VESTIGIAL_STORE_STATE,
9182
+ nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
9183
+ } : void 0;
9184
+ await this.fireItems(store, binding, result.output, now, rewind, success);
9217
9185
  } catch (error) {
9218
9186
  if (!refused) await rewind();
9219
9187
  throw error;
@@ -9545,22 +9513,38 @@ var TriggerSupervisor = class {
9545
9513
  async tick() {
9546
9514
  if (this.ticking || this.stopped) return;
9547
9515
  this.ticking = true;
9516
+ const started = [];
9517
+ let listing;
9548
9518
  try {
9549
- await this.enqueue(() => this.pollDue());
9519
+ await this.startDue(started);
9520
+ } catch (error) {
9521
+ listing = error;
9550
9522
  } finally {
9551
9523
  this.ticking = false;
9552
9524
  }
9525
+ const outcomes = await Promise.allSettled(started.filter((run) => run !== void 0));
9526
+ if (listing !== void 0) throw listing instanceof Error ? listing : new Error(String(listing));
9527
+ const failed = outcomes.find((outcome) => outcome.status === "rejected");
9528
+ if (failed) throw failed.reason;
9553
9529
  }
9554
- async retryOneEnable() {
9530
+ startOnLane(workflowId, task, hook) {
9531
+ if (this.ticked.has(workflowId)) return void 0;
9532
+ this.ticked.add(workflowId);
9533
+ return (hook ? this.hookSlots.run(() => this.enqueue(workflowId, task)) : this.enqueue(workflowId, task)).finally(() => this.ticked.delete(workflowId));
9534
+ }
9535
+ startOneEnableRetry() {
9555
9536
  const now = this.now().getTime();
9556
- const next = [...this.enableRetries].filter(([, retry]) => retry.at <= now).sort((a, b) => a[1].at - b[1].at).at(0);
9557
- if (!next) return;
9537
+ const next = [...this.enableRetries].filter(([id, retry]) => retry.at <= now && !this.ticked.has(id)).sort((a, b) => a[1].at - b[1].at).at(0);
9538
+ if (!next) return void 0;
9558
9539
  const [workflowId, retry] = next;
9559
9540
  const binding = this.bindings.get(workflowId);
9560
9541
  if (!binding) {
9561
9542
  this.enableRetries.delete(workflowId);
9562
9543
  return;
9563
9544
  }
9545
+ return this.startOnLane(workflowId, () => this.retryEnable(workflowId, retry, binding, now), true);
9546
+ }
9547
+ async retryEnable(workflowId, retry, binding, now) {
9564
9548
  try {
9565
9549
  await this.enable(binding);
9566
9550
  } catch (error) {
@@ -9575,7 +9559,7 @@ var TriggerSupervisor = class {
9575
9559
  logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
9576
9560
  }
9577
9561
  }
9578
- async pollDue() {
9562
+ async startDue(started) {
9579
9563
  const store = await this.store();
9580
9564
  if (!store) {
9581
9565
  if (!this.warnedMissingJournal) {
@@ -9584,37 +9568,35 @@ var TriggerSupervisor = class {
9584
9568
  }
9585
9569
  return;
9586
9570
  }
9587
- const due = await store.listDueTriggerStates(this.now().toISOString());
9588
- for (const row of due) {
9571
+ const ordered = [...await store.listDueTriggerStates(this.now().toISOString())].sort((a, b) => Number(!this.isScheduleRow(a)) - Number(!this.isScheduleRow(b)));
9572
+ for (const row of ordered) {
9589
9573
  this.throwIfStopped();
9590
9574
  const binding = this.bindings.get(row.workflow_id);
9591
9575
  if (!binding) {
9592
- await store.setTriggerStatus(row.workflow_id, "DISABLED");
9576
+ started.push(this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false));
9593
9577
  continue;
9594
9578
  }
9595
- if (isSchedule(binding)) await this.fireSchedule(store, row, binding);
9596
- else await this.poll(store, row, binding);
9579
+ started.push(isSchedule(binding) ? this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.fireSchedule(store, row, binding)), false) : this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.poll(store, row, binding)), true));
9597
9580
  }
9598
9581
  this.throwIfStopped();
9599
- await this.renewDue(store);
9582
+ started.push(...await this.startRenewals(store));
9600
9583
  this.throwIfStopped();
9601
- await this.retryOneEnable();
9584
+ started.push(this.startOneEnableRetry());
9602
9585
  }
9603
- async renewDue(store) {
9604
- const due = await store.listDueTriggerRenewals(this.now().toISOString());
9605
- for (const row of due) {
9606
- this.throwIfStopped();
9586
+ whileBound(binding, task) {
9587
+ return () => this.bindings.get(binding.workflowId) === binding ? task() : Promise.resolve();
9588
+ }
9589
+ isScheduleRow(row) {
9590
+ const binding = this.bindings.get(row.workflow_id);
9591
+ return binding !== void 0 && isSchedule(binding);
9592
+ }
9593
+ async startRenewals(store) {
9594
+ return (await store.listDueTriggerRenewals(this.now().toISOString())).map((row) => {
9607
9595
  const binding = this.bindings.get(row.workflow_id);
9608
- if (!binding) {
9609
- await store.setTriggerStatus(row.workflow_id, "DISABLED");
9610
- continue;
9611
- }
9612
- if (isSchedule(binding)) {
9613
- await store.setTriggerRenewAt(row.workflow_id, null);
9614
- continue;
9615
- }
9616
- await this.renew(store, row, binding);
9617
- }
9596
+ if (!binding) return this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false);
9597
+ if (isSchedule(binding)) return this.startOnLane(row.workflow_id, () => store.setTriggerRenewAt(row.workflow_id, null), false);
9598
+ return this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.renew(store, row, binding)), true);
9599
+ });
9618
9600
  }
9619
9601
  async renew(store, row, binding) {
9620
9602
  const now = this.now();
@@ -9676,8 +9658,11 @@ var TriggerSupervisor = class {
9676
9658
  throw error;
9677
9659
  });
9678
9660
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
9679
- await store.recordPollSuccess(row.workflow_id, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
9680
- await this.fireItems(store, binding, result.output, now, rewind);
9661
+ await this.fireItems(store, binding, result.output, now, rewind, {
9662
+ workflowId: row.workflow_id,
9663
+ storeState: VESTIGIAL_STORE_STATE,
9664
+ nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
9665
+ });
9681
9666
  } catch (error) {
9682
9667
  if (!refused) await rewind();
9683
9668
  if (this.stopped) throw error;
@@ -9688,8 +9673,11 @@ var TriggerSupervisor = class {
9688
9673
  logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
9689
9674
  }
9690
9675
  }
9691
- async fireItems(store, binding, items, now, rewind) {
9692
- if (items.length === 0) return;
9676
+ async fireItems(store, binding, items, now, rewind, success) {
9677
+ if (items.length === 0) {
9678
+ if (success) await store.recordPollSuccess(success.workflowId, success.storeState, now.toISOString(), success.nextPollAtIso);
9679
+ return;
9680
+ }
9693
9681
  const kind = pieceTriggerKind(binding.block);
9694
9682
  const journaled = (await store.journalTriggerItems(items.map((item) => ({
9695
9683
  dedupeKey: extractDedupeKey(item),
@@ -9698,7 +9686,7 @@ var TriggerSupervisor = class {
9698
9686
  triggerKind: kind,
9699
9687
  triggerPayload: item
9700
9688
  }
9701
- })), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
9689
+ })), DEDUPE_TTL_MS, now.toISOString(), success)).flatMap((runId, index) => runId ? [{
9702
9690
  runId,
9703
9691
  item: items[index]
9704
9692
  }] : []);
@@ -9721,6 +9709,25 @@ var TriggerSupervisor = class {
9721
9709
  }
9722
9710
  }
9723
9711
  };
9712
+ var Slots = class {
9713
+ limit;
9714
+ active = 0;
9715
+ waiting = [];
9716
+ constructor(limit) {
9717
+ this.limit = limit;
9718
+ }
9719
+ async run(task) {
9720
+ if (this.active >= Math.max(1, this.limit)) await new Promise((resolve) => this.waiting.push(resolve));
9721
+ else this.active++;
9722
+ try {
9723
+ return await task();
9724
+ } finally {
9725
+ const next = this.waiting.shift();
9726
+ if (next) next();
9727
+ else this.active--;
9728
+ }
9729
+ }
9730
+ };
9724
9731
  //#endregion
9725
9732
  //#region src/reactor/webhook.ts
9726
9733
  const WEBHOOK = "Webhook";
@@ -9948,6 +9955,7 @@ function bindingTarget(binding) {
9948
9955
  }
9949
9956
  const RUNS_SCAN_LIMIT = 1e3;
9950
9957
  const WEBHOOK_TEST_TIMEOUT_MS = 3e5;
9958
+ const PENDING_SWEEP_INTERVAL_MS = 6e4;
9951
9959
  function draftBlocks(state) {
9952
9960
  return [...state.trigger ? [{
9953
9961
  id: state.trigger.id,
@@ -10118,6 +10126,10 @@ function configRecord(config) {
10118
10126
  if (config && typeof config === "object" && !Array.isArray(config)) return config;
10119
10127
  return {};
10120
10128
  }
10129
+ const WORKFLOW_CACHE_MAX = 1e3;
10130
+ function globalRevision(document) {
10131
+ return document.header.revision?.global ?? 0;
10132
+ }
10121
10133
  function runJournalName(stateName, documentName) {
10122
10134
  return stateName?.trim() ? stateName : documentName ?? "";
10123
10135
  }
@@ -10129,6 +10141,9 @@ var WorkflowRuntimeService = class {
10129
10141
  executor;
10130
10142
  pieceWorkers;
10131
10143
  unobservePool;
10144
+ runDispatcher;
10145
+ unobserveDispatcher;
10146
+ eventLoop = new EventLoopMonitor();
10132
10147
  storePromise;
10133
10148
  storeError;
10134
10149
  storeOpening = false;
@@ -10624,10 +10639,35 @@ var WorkflowRuntimeService = class {
10624
10639
  await this.updateRegistration(workflowId, carried, onlyIfChanged);
10625
10640
  return;
10626
10641
  }
10642
+ const generation = this.workflowGeneration(workflowId);
10627
10643
  const document = await this.host.reactorClient.get(workflowId);
10644
+ if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
10628
10645
  await this.updateRegistration(workflowId, document.state.global, onlyIfChanged);
10629
10646
  }
10630
10647
  seenOps = /* @__PURE__ */ new Set();
10648
+ workflowCache = /* @__PURE__ */ new Map();
10649
+ workflowGenerations = /* @__PURE__ */ new Map();
10650
+ workflowGeneration(workflowId) {
10651
+ return this.workflowGenerations.get(workflowId) ?? 0;
10652
+ }
10653
+ cacheLoaded(workflowId, generation, document) {
10654
+ if (this.workflowGeneration(workflowId) !== generation) return;
10655
+ this.cacheWorkflow(workflowId, {
10656
+ state: document.state.global,
10657
+ name: document.header.name,
10658
+ revision: globalRevision(document)
10659
+ });
10660
+ }
10661
+ cacheWorkflow(workflowId, entry) {
10662
+ const held = this.workflowCache.get(workflowId);
10663
+ if (held && held.revision > entry.revision) return;
10664
+ this.workflowCache.delete(workflowId);
10665
+ this.workflowCache.set(workflowId, entry);
10666
+ if (this.workflowCache.size > WORKFLOW_CACHE_MAX) {
10667
+ const oldest = this.workflowCache.keys().next().value;
10668
+ if (oldest !== void 0) this.workflowCache.delete(oldest);
10669
+ }
10670
+ }
10631
10671
  seenOpsQueue = [];
10632
10672
  alreadySeen(key) {
10633
10673
  if (this.seenOps.has(key)) return true;
@@ -10663,12 +10703,22 @@ var WorkflowRuntimeService = class {
10663
10703
  context
10664
10704
  });
10665
10705
  if (this.alreadySeen(opKey)) continue;
10706
+ if (context.documentType === WORKFLOW_DOCUMENT_TYPE) this.workflowGenerations.set(context.documentId, this.workflowGeneration(context.documentId) + 1);
10666
10707
  if (context.scope === DOCUMENT_SCOPE) {
10708
+ this.workflowCache.delete(context.documentId);
10667
10709
  await this.matchDocumentLifecycle(operation, context, hints);
10668
10710
  await this.forgetDeletedWorkflow(operation, context);
10669
10711
  continue;
10670
10712
  }
10671
10713
  if (context.documentType === "powerhouse/workflow") {
10714
+ const cached = this.workflowCache.get(context.documentId);
10715
+ const carried = cached && operation.error === void 0 ? parseWorkflowState(operation.resultingState) : void 0;
10716
+ if (cached && carried) this.cacheWorkflow(context.documentId, {
10717
+ state: carried,
10718
+ name: cached.name,
10719
+ revision: operation.index + 1
10720
+ });
10721
+ else if (cached) this.workflowCache.delete(context.documentId);
10672
10722
  const rechecked = await this.recheckReactorAccess(context.documentId, operation);
10673
10723
  await this.refreshRegistration(context.documentId, operation.resultingState, !rechecked);
10674
10724
  }
@@ -10778,7 +10828,7 @@ var WorkflowRuntimeService = class {
10778
10828
  this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
10779
10829
  }, (error) => {
10780
10830
  if (enqueuedRunId && isShutdownRefusal(error)) {
10781
- this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to rerun`);
10831
+ this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left for the next owner to start`);
10782
10832
  return;
10783
10833
  }
10784
10834
  this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
@@ -10840,6 +10890,7 @@ var WorkflowRuntimeService = class {
10840
10890
  if (context.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
10841
10891
  const workflowId = stringField(inputRecord(operation.action.input), "documentId") ?? context.documentId;
10842
10892
  this.disarmDeleted(workflowId);
10893
+ this.workflowCache.delete(workflowId);
10843
10894
  try {
10844
10895
  await (await this.store())?.deleteDedupe(workflowId);
10845
10896
  } catch (error) {
@@ -10941,6 +10992,8 @@ var WorkflowRuntimeService = class {
10941
10992
  webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
10942
10993
  reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
10943
10994
  models: this.models,
10995
+ worker: this.workers().session("trigger"),
10996
+ hookConcurrency: this.workers().triggerSlots,
10944
10997
  telemetry: this.telemetry,
10945
10998
  cacheDir: bundleCacheDir(),
10946
10999
  resolver: pieceResolver(),
@@ -10960,8 +11013,29 @@ var WorkflowRuntimeService = class {
10960
11013
  }
10961
11014
  startTriggerSupervisor() {
10962
11015
  this.supervisor().start();
11016
+ this.admitAbandonedRuns();
11017
+ this.pendingSweepTimer ??= setInterval(() => {
11018
+ this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
11019
+ }, PENDING_SWEEP_INTERVAL_MS);
11020
+ this.pendingSweepTimer.unref();
11021
+ }
11022
+ pendingSweepTimer;
11023
+ admittingAbandoned = false;
11024
+ async admitAbandonedRuns(minAgeMs) {
11025
+ if (this.admittingAbandoned || this.closed) return;
11026
+ this.admittingAbandoned = true;
11027
+ try {
11028
+ const admitted = await (await this.store())?.admitAbandonedRuns(minAgeMs === void 0 ? {} : { enqueuedBefore: new Date(Date.now() - minAgeMs) });
11029
+ for (const run of admitted ?? []) this.startFiring(run.workflowId, run.triggerPayload, run.triggerKind, run.runId);
11030
+ } catch (error) {
11031
+ this.logger.error("Could not admit the runs a stopped reactor journaled: @error", error);
11032
+ } finally {
11033
+ this.admittingAbandoned = false;
11034
+ }
10963
11035
  }
10964
11036
  stopTriggerSupervisor() {
11037
+ clearInterval(this.pendingSweepTimer);
11038
+ this.pendingSweepTimer = void 0;
10965
11039
  this.triggerSupervisor?.stop();
10966
11040
  }
10967
11041
  closed = false;
@@ -10971,6 +11045,10 @@ var WorkflowRuntimeService = class {
10971
11045
  shutdown() {
10972
11046
  this.closed = true;
10973
11047
  this.runGate.close();
11048
+ this.runDispatcher?.close();
11049
+ this.eventLoop.stop();
11050
+ this.unobserveDispatcher?.();
11051
+ this.unobserveDispatcher = void 0;
10974
11052
  clearInterval(this.retentionTimer);
10975
11053
  for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
10976
11054
  this.resolutionRetries.clear();
@@ -10979,7 +11057,6 @@ var WorkflowRuntimeService = class {
10979
11057
  this.pieceWorkers?.dispose();
10980
11058
  this.unobservePool?.();
10981
11059
  this.unobservePool = void 0;
10982
- this.designWorker?.dispose();
10983
11060
  this.designWorker = void 0;
10984
11061
  }
10985
11062
  async triggerStates(ctx) {
@@ -11266,10 +11343,7 @@ var WorkflowRuntimeService = class {
11266
11343
  descriptors = /* @__PURE__ */ new Map();
11267
11344
  designWorker;
11268
11345
  createDesignWorker() {
11269
- return new PieceWorker({
11270
- models: this.models,
11271
- telemetry: this.telemetry
11272
- });
11346
+ return this.workers().session("design");
11273
11347
  }
11274
11348
  designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
11275
11349
  describing = /* @__PURE__ */ new Map();
@@ -12068,17 +12142,51 @@ var WorkflowRuntimeService = class {
12068
12142
  }
12069
12143
  workers() {
12070
12144
  if (this.pieceWorkers) return this.pieceWorkers;
12145
+ const env = (name) => Number(process.env[name]) || void 0;
12146
+ const count = (name) => {
12147
+ const value = Number(process.env[name] || NaN);
12148
+ return value >= 0 ? value : void 0;
12149
+ };
12071
12150
  const pool = new PieceWorkerPool({
12072
12151
  models: this.models,
12073
- size: Number(process.env.PH_WORKFLOWS_RUN_CONCURRENCY) || void 0,
12074
- maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0,
12152
+ size: env("PH_WORKFLOWS_RUN_CONCURRENCY"),
12153
+ maxQueueDepth: env("PH_WORKFLOWS_RUN_QUEUE_DEPTH"),
12154
+ heapMb: env("PH_WORKFLOWS_WORKER_MEMORY_MB"),
12155
+ maxRequests: env("PH_WORKFLOWS_WORKER_MAX_REQUESTS"),
12156
+ minWarm: count("PH_WORKFLOWS_WORKER_MIN"),
12157
+ triggerSlots: env("PH_WORKFLOWS_TRIGGER_SLOTS"),
12158
+ reactorThreads: this.host.reactorThreads,
12075
12159
  telemetry: this.telemetry
12076
12160
  });
12161
+ if (this.closed) {
12162
+ pool.dispose();
12163
+ return this.pieceWorkers = pool;
12164
+ }
12077
12165
  this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
12078
12166
  return this.pieceWorkers = pool;
12079
12167
  }
12168
+ dispatcher() {
12169
+ if (this.runDispatcher) return this.runDispatcher;
12170
+ const configured = Number(process.env.PH_WORKFLOWS_ADMIT_MAX_ELU);
12171
+ const dispatcher = new RunDispatcher({
12172
+ maxInFlight: Number(process.env.PH_WORKFLOWS_RUNS_IN_FLIGHT) || 8 * this.workers().size,
12173
+ maxElu: configured > 0 && configured <= 1 ? configured : DEFAULT_ADMIT_MAX_ELU,
12174
+ loop: this.eventLoop
12175
+ });
12176
+ this.runDispatcher = dispatcher;
12177
+ if (this.closed) {
12178
+ dispatcher.close();
12179
+ return dispatcher;
12180
+ }
12181
+ this.eventLoop.start();
12182
+ this.unobserveDispatcher = this.telemetry.observeDispatcher(() => ({
12183
+ ...dispatcher.stats(),
12184
+ utilization: this.eventLoop.utilization()
12185
+ }));
12186
+ return dispatcher;
12187
+ }
12080
12188
  async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
12081
- if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
12189
+ if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId, triggerKind);
12082
12190
  const started = performance.now();
12083
12191
  const attributes = {
12084
12192
  "workflow.id": workflowId,
@@ -12113,18 +12221,29 @@ var WorkflowRuntimeService = class {
12113
12221
  let documentName;
12114
12222
  let definition;
12115
12223
  let runUser;
12224
+ let readGeneration;
12116
12225
  try {
12117
- if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
12226
+ if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
12118
12227
  run.refused = true;
12119
12228
  throw error;
12120
12229
  });
12121
- const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
12122
- if (document.header.documentType !== "powerhouse/workflow") {
12123
- run.refused = true;
12124
- throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
12230
+ const cached = triggerKind === "manual" || resume ? void 0 : this.workflowCache.get(workflowId);
12231
+ if (cached) {
12232
+ state = cached.state;
12233
+ documentName = cached.name;
12234
+ readGeneration = this.workflowGeneration(workflowId);
12235
+ } else {
12236
+ const generation = this.workflowGeneration(workflowId);
12237
+ readGeneration = generation;
12238
+ const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
12239
+ if (document.header.documentType !== "powerhouse/workflow") {
12240
+ run.refused = true;
12241
+ throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
12242
+ }
12243
+ state = document.state.global;
12244
+ documentName = document.header.name;
12245
+ this.cacheLoaded(workflowId, generation, document);
12125
12246
  }
12126
- state = document.state.global;
12127
- documentName = document.header.name;
12128
12247
  if (state.status !== "ENABLED") {
12129
12248
  run.refused = true;
12130
12249
  throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
@@ -12139,6 +12258,19 @@ var WorkflowRuntimeService = class {
12139
12258
  if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
12140
12259
  throw error;
12141
12260
  }
12261
+ let journaling;
12262
+ const journalOnQueue = () => {
12263
+ if (triggerKind !== "manual" || enqueuedRunId || resume || journaling) return;
12264
+ if (!store) return;
12265
+ journaling = store.enqueueRun({
12266
+ workflowId,
12267
+ triggerKind,
12268
+ triggerPayload
12269
+ }).then((runId) => {
12270
+ enqueuedRunId = runId;
12271
+ });
12272
+ journaling.catch(() => void 0);
12273
+ };
12142
12274
  const connections = declaredConnectionIds(definition);
12143
12275
  const executor = this.blockExecutor(store);
12144
12276
  const runnable = runnableDefinition(state);
@@ -12159,30 +12291,72 @@ var WorkflowRuntimeService = class {
12159
12291
  throw error;
12160
12292
  }
12161
12293
  if (parked) return skipped(parked, "parked");
12162
- const admission = await this.runGate.admit(workflowId, policy);
12294
+ const admission = await this.runGate.admit(workflowId, policy, journalOnQueue);
12295
+ try {
12296
+ await journaling;
12297
+ } catch (error) {
12298
+ if (admission.admitted) admission.release();
12299
+ throw error;
12300
+ }
12163
12301
  if (this.closed) {
12164
12302
  if (admission.admitted) admission.release();
12165
- return this.refuseClosed(store, workflowId, enqueuedRunId);
12303
+ return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
12166
12304
  }
12167
12305
  if (!admission.admitted) {
12168
- if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
12306
+ if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
12169
12307
  return skipped(admission.reason, admission.refusal);
12170
12308
  }
12171
- if (admission.waited) {
12172
- let stale;
12309
+ const staleAfter = async () => {
12173
12310
  try {
12174
- stale = await this.staleAfterWait(workflowId, triggerKind, runnable.version);
12311
+ return await this.staleAfterWait(workflowId, triggerKind, runnable.version, readGeneration);
12175
12312
  } catch (error) {
12176
- stale = `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
12313
+ return `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
12177
12314
  }
12315
+ };
12316
+ if (admission.waited) {
12317
+ const stale = await staleAfter();
12178
12318
  if (stale) {
12179
12319
  admission.release();
12180
12320
  return skipped(stale, "stale");
12181
12321
  }
12182
12322
  }
12323
+ const expired = () => skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
12183
12324
  if (deadline !== void 0 && Date.now() >= deadline) {
12184
12325
  admission.release();
12185
- return skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
12326
+ return expired();
12327
+ }
12328
+ let dispatched;
12329
+ let dispatcherWaited = false;
12330
+ try {
12331
+ dispatched = await this.dispatcher().admit(workflowId, () => {
12332
+ dispatcherWaited = true;
12333
+ journalOnQueue();
12334
+ });
12335
+ } catch (error) {
12336
+ await journaling?.catch(() => void 0);
12337
+ admission.release();
12338
+ if (this.closed) return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
12339
+ throw error;
12340
+ }
12341
+ try {
12342
+ await journaling;
12343
+ } catch (error) {
12344
+ dispatched();
12345
+ admission.release();
12346
+ throw error;
12347
+ }
12348
+ if (this.closed || deadline !== void 0 && Date.now() >= deadline) {
12349
+ dispatched();
12350
+ admission.release();
12351
+ return this.closed ? this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind) : expired();
12352
+ }
12353
+ if (dispatcherWaited) {
12354
+ const stale = await staleAfter();
12355
+ if (stale) {
12356
+ dispatched();
12357
+ admission.release();
12358
+ return skipped(stale, "stale");
12359
+ }
12186
12360
  }
12187
12361
  let runId = enqueuedRunId ?? null;
12188
12362
  let journalFailed = false;
@@ -12207,6 +12381,7 @@ var WorkflowRuntimeService = class {
12207
12381
  }) ?? null;
12208
12382
  });
12209
12383
  if (runId) run.span.setAttribute("run.id", runId);
12384
+ if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
12210
12385
  session = this.workers().session();
12211
12386
  const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
12212
12387
  const journal = store;
@@ -12269,6 +12444,7 @@ var WorkflowRuntimeService = class {
12269
12444
  } finally {
12270
12445
  session?.close();
12271
12446
  admission.release();
12447
+ dispatched();
12272
12448
  }
12273
12449
  }
12274
12450
  runGate = new WorkflowRunGate();
@@ -12282,7 +12458,8 @@ var WorkflowRuntimeService = class {
12282
12458
  return `Skipped: this workflow is PARKED (${park.reason})`;
12283
12459
  }
12284
12460
  /** Why a firing that waited for its slot may no longer run, if it may not. */
12285
- async staleAfterWait(workflowId, triggerKind, publishedVersion) {
12461
+ async staleAfterWait(workflowId, triggerKind, publishedVersion, readGeneration) {
12462
+ if (this.workflowGeneration(workflowId) === readGeneration) return this.parkedFiring(workflowId, triggerKind, publishedVersion);
12286
12463
  const state = (await this.host.reactorClient.get(workflowId)).state.global;
12287
12464
  if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
12288
12465
  if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
@@ -12314,10 +12491,18 @@ var WorkflowRuntimeService = class {
12314
12491
  * of bug this work package exists to stamp out. An already-enqueued row is
12315
12492
  * closed out in place, so nothing is left PENDING for a sweep to find.
12316
12493
  */
12317
- async refuseClosed(store, workflowId, enqueuedRunId) {
12494
+ async refuseClosed(store, workflowId, enqueuedRunId, triggerKind) {
12318
12495
  const error = new WorkflowRuntimeClosedError(workflowId);
12319
12496
  if (enqueuedRunId) {
12320
12497
  const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
12498
+ if (!redelivered && triggerKind !== "manual") {
12499
+ try {
12500
+ await store?.releaseAdmission(enqueuedRunId);
12501
+ } catch (failure) {
12502
+ this.logger.warn(`Could not give back the boot admission of run ${enqueuedRunId}: @error`, failure);
12503
+ }
12504
+ throw error;
12505
+ }
12321
12506
  try {
12322
12507
  if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
12323
12508
  else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));