@powerhousedao/reactor-workflow 6.2.3-dev.84 → 6.2.3-dev.85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,9 +1,9 @@
1
- import { $ as redact, A as AttachmentCache, B as SecretDeletedError, C as blockLabel, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as stepBlock, H as SecretOwnerMismatchError, I as resolveStepInput, J as PieceWorker, K as triggerBlock, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as containsRedactedMarker, S as storeHandlers, St as isIndeterminateError, T as blockPorts, U as parseSecretRef, V as SecretNotFoundError, W as secretRefFromId, X as PieceWorkerError, Y as PieceWorkerDisposedError, Z as PieceWorkerTimeoutError, _ as effectiveRetryPolicy, a as PieceRegistry, at as parseAddress, b as isHostBound, bt as pieceRegistrySource, c as packagePieces, ct as maxFileBytes, d as runWorkflow, dt as sourcedResolver, et as redactMessage, f as canonicalJson, g as checkDynamicProperties, gt as unsupportedTrigger, h as DynamicPropertiesError, ht as unsupportedAuth, i as tracedSteps, it as isPrivateAddress, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as bundleResolver, m as stepConfigHash, mt as authMethodFor, n as WorkflowTelemetry, nt as secretsFor, o as installedPiece, ot as extractDedupeKey, p as hashOf, pt as UnsupportedPieceFeatureError, q as PieceWorkerPool, r as tracedMethods, rt as DEFAULT_EGRESS_POLICY, s as installedPieces, st as FileTooLargeError, t as TELEMETRY_SCOPE, tt as rememberSecrets, u as errorNameOf, ut as pieceModuleRef, v as ActivepiecesBlockExecutor, w as unpinnedNote, x as resolvedBlock, xt as setPieceRegistryUrl, y as CompositeBlockExecutor, yt as npmRegistryBaseUrl, z as SecretCreatorMismatchError } from "./telemetry-CSemNycC.js";
1
+ import { $ as containsRedactedMarker, A as AttachmentCache, B as SecretCreatorMismatchError, C as blockLabel, Ct as isIndeterminateError, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as secretRefFromId, H as SecretNotFoundError, I as resolveStepInput, J as PieceWorkerPool, K as stepBlock, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as PieceWorkerTimeoutError, R as unavailableValue, S as storeHandlers, St as setPieceRegistryUrl, T as blockPorts, U as SecretOwnerMismatchError, V as SecretDeletedError, W as parseSecretRef, X as PieceWorkerDisposedError, Y as PieceWorker, Z as PieceWorkerError, _ as effectiveRetryPolicy, _t as unsupportedTrigger, a as PieceRegistry, at as isPrivateAddress, b as isHostBound, bt as npmRegistryBaseUrl, c as packagePieces, ct as FileTooLargeError, d as runWorkflow, dt as pieceModuleRef, et as redact, f as canonicalJson, ft as sourcedResolver, g as checkDynamicProperties, gt as unsupportedAuth, h as DynamicPropertiesError, ht as authMethodFor, i as tracedSteps, it as DEFAULT_EGRESS_POLICY, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as maxFileBytes, m as stepConfigHash, mt as UnsupportedPieceFeatureError, n as WorkflowTelemetry, nt as rememberSecrets, o as installedPiece, ot as parseAddress, p as hashOf, q as triggerBlock, r as tracedMethods, rt as secretsFor, s as installedPieces, st as extractDedupeKey, t as TELEMETRY_SCOPE, tt as redactMessage, u as errorNameOf, ut as bundleResolver, v as ActivepiecesBlockExecutor, w as unpinnedNote, x as resolvedBlock, xt as pieceRegistrySource, y as CompositeBlockExecutor } from "./telemetry-B5OOpfCS.js";
2
2
  import { DENIED_ACTION_INDEXES, ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy, assertActionsApplied } from "@powerhousedao/pieces-framework";
3
3
  import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
4
4
  import { open, readFile, readdir, rm, stat } from "node:fs/promises";
5
5
  import { dirname, join } from "node:path";
6
- import { childLogger, generateId } from "document-model";
6
+ import { childLogger, garbageCollect, generateId, hashDocumentStateForScope, isDenied, sortOperations } from "document-model";
7
7
  import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, coerceManualPayload, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, manualInputTypeName, parseManualInputs, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
8
8
  import { Cron } from "croner";
9
9
  import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
@@ -13,9 +13,9 @@ import { AsyncLocalStorage } from "node:async_hooks";
13
13
  import { hostname } from "node:os";
14
14
  import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
15
15
  import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
16
- import http from "node:http";
16
+ import http, { validateHeaderName, validateHeaderValue } from "node:http";
17
17
  import https from "node:https";
18
- import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
18
+ import { actions as actions$1, reducer, utils } from "@powerhousedao/workflow/document-models/workflow";
19
19
  import { Kind, parse } from "graphql";
20
20
  import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
21
21
  import { SpanStatusCode, context } from "@opentelemetry/api";
@@ -1555,6 +1555,9 @@ function currentStepTest() {
1555
1555
  function currentRunUser() {
1556
1556
  return storage.getStore()?.runUser;
1557
1557
  }
1558
+ function currentWaitpoints() {
1559
+ return storage.getStore()?.waitpoints;
1560
+ }
1558
1561
  //#endregion
1559
1562
  //#region src/reactor/piece-store-port.ts
1560
1563
  const logger$5 = childLogger(["workflow", "piece-store"]);
@@ -1750,6 +1753,7 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
1750
1753
  egress: configuredEgress(),
1751
1754
  worker: currentPieceWorker,
1752
1755
  stepTest: currentStepTest,
1756
+ waitpoints: currentWaitpoints,
1753
1757
  identity: () => ({
1754
1758
  runId: currentRunId(),
1755
1759
  flowId: currentWorkflowId(),
@@ -2691,6 +2695,10 @@ const SCHEDULE_BLOCK = trigger("schedule");
2691
2695
  const WEBHOOK_BLOCK = trigger("webhook");
2692
2696
  const BRANCH_BLOCK = action("branch");
2693
2697
  const ASSERT_BLOCK = action("assert");
2698
+ const DELAY_BLOCK = action("delay");
2699
+ const APPROVAL_BLOCK = action("approval");
2700
+ const STOP_BLOCK = action("stop");
2701
+ const WAIT_FOR_WEBHOOK_BLOCK = action("waitForWebhook");
2694
2702
  function isCoreBlock(block) {
2695
2703
  return block.pieceName === CORE_PIECE_NAME;
2696
2704
  }
@@ -7680,6 +7688,16 @@ const PARKED_TRIGGER_STATUS = "PARKED";
7680
7688
  * not finish and nothing is wrong with the workflow", which is what the
7681
7689
  * document model's RunStatus.CANCELLED means. */
7682
7690
  const CANCELLED_RUN_STATUS = "CANCELLED";
7691
+ /** How long a run may stay suspended; null when the policy sets no bound. */
7692
+ function maxSuspensionMs(runnable) {
7693
+ const policy = runnable.policy;
7694
+ const days = positive(policy?.maxSuspensionDays);
7695
+ return days === null ? null : days * 864e5;
7696
+ }
7697
+ //#endregion
7698
+ //#region src/reactor/waitpoint-types.ts
7699
+ const WAITPOINT_EXPIRED_ERROR_NAME = "WaitpointExpired";
7700
+ const WAITPOINT_NOT_LIVE_ERROR_NAME = "WaitpointNotLive";
7683
7701
  //#endregion
7684
7702
  //#region src/reactor/store.ts
7685
7703
  function triggerRowBlock(row) {
@@ -7711,6 +7729,7 @@ function workflowParkRow(workflowId, publishedVersion, reason) {
7711
7729
  parked_at: (/* @__PURE__ */ new Date()).toISOString()
7712
7730
  };
7713
7731
  }
7732
+ const WAITING_RUN_STATUS = "WAITING";
7714
7733
  const logger$4 = childLogger([
7715
7734
  "workflow",
7716
7735
  "runtime",
@@ -7718,6 +7737,64 @@ const logger$4 = childLogger([
7718
7737
  ]);
7719
7738
  const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
7720
7739
  const PENDING_RUN_STATUS = "PENDING";
7740
+ const LIVE_RUN_STATUSES = [
7741
+ PENDING_RUN_STATUS,
7742
+ "RUNNING",
7743
+ WAITING_RUN_STATUS
7744
+ ];
7745
+ const LIVE_WAITPOINT_STATUSES = ["PENDING", "COMPLETED"];
7746
+ async function closeLiveWaitpoints(db, runIds, status, resolvedAt, keep) {
7747
+ if (runIds.length === 0) return [];
7748
+ return (await db.updateTable("waitpoint").set({
7749
+ status,
7750
+ resolved_at: resolvedAt
7751
+ }).where("run_id", "in", runIds).where("status", "in", LIVE_WAITPOINT_STATUSES).$if(keep !== void 0, (qb) => qb.where((eb) => eb.not(keep(eb)))).returning("id").execute()).map((row) => row.id);
7752
+ }
7753
+ async function failWaitingSteps(db, runIds, reason, errorName, endedAt) {
7754
+ if (runIds.length === 0) return;
7755
+ await db.updateTable("step_execution").set({
7756
+ status: "FAILED",
7757
+ error: redactMessage(reason),
7758
+ error_name: errorName,
7759
+ ended_at: endedAt
7760
+ }).where("run_id", "in", runIds).where("status", "=", WAITING_RUN_STATUS).execute();
7761
+ }
7762
+ function waitpointExpiryReason(waitpoint) {
7763
+ return `The ${waitpoint.kind === "approval" ? "approval" : "wait for a resume"} at step "${waitpoint.step_key}" expired at ${waitpoint.expires_at} without being resolved`;
7764
+ }
7765
+ function strandedWait(eb) {
7766
+ const live = eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES);
7767
+ return eb.and([eb.not(eb.exists(completedOf(eb))), eb.or([eb.and([eb("run.waiting_on", "is not", null), eb.not(eb.exists(live.whereRef("waitpoint.id", "=", "run.waiting_on")))]), eb.and([eb("run.waiting_on", "is", null), eb.not(eb.exists(live.whereRef("waitpoint.run_id", "=", "run.id")))])])]);
7768
+ }
7769
+ function completedOf(eb) {
7770
+ return eb.selectFrom("waitpoint").select("waitpoint.id").whereRef("waitpoint.run_id", "=", "run.id").where("waitpoint.status", "=", "COMPLETED");
7771
+ }
7772
+ async function endWaitingRun(db, runId, end) {
7773
+ const { waitpointId } = end;
7774
+ if ((await db.updateTable("run").set({
7775
+ status: "CANCELLED",
7776
+ error: redactMessage(end.reason),
7777
+ error_name: end.errorName,
7778
+ ended_at: end.endedAt
7779
+ }).where("id", "=", runId).where("status", "=", "WAITING").where((eb) => waitpointId === void 0 ? strandedWait(eb) : eb.and([eb.or([eb("run.waiting_on", "=", waitpointId), eb("run.waiting_on", "is", null)]), eb.not(eb.exists(completedOf(eb)))])).returning("id").execute()).length === 0) return void 0;
7780
+ await failWaitingSteps(db, [runId], end.reason, end.errorName, end.endedAt);
7781
+ const closed = await closeLiveWaitpoints(db, [runId], "EXPIRED", end.endedAt);
7782
+ return {
7783
+ runId,
7784
+ reason: end.reason,
7785
+ closed
7786
+ };
7787
+ }
7788
+ function priorNotes(text) {
7789
+ if (!text) return [];
7790
+ try {
7791
+ const notes = JSON.parse(text);
7792
+ return Array.isArray(notes) ? notes.filter((note) => typeof note === "string") : [];
7793
+ } catch {
7794
+ return [];
7795
+ }
7796
+ }
7797
+ const STRANDED_WAIT_REASON = "The run waited on a waitpoint that is no longer live";
7721
7798
  function errorMessage(error) {
7722
7799
  return error instanceof Error ? error.message : String(error);
7723
7800
  }
@@ -7819,6 +7896,31 @@ async function up(db) {
7819
7896
  try {
7820
7897
  await db.schema.dropTable("webhook_endpoint").ifExists().execute();
7821
7898
  } catch {}
7899
+ for (const [column, type] of [
7900
+ ["definition_hash", "text"],
7901
+ ["definition_op_index", "integer"],
7902
+ ["definition_scope_hash", "text"],
7903
+ ["waiting_on", "text"]
7904
+ ]) try {
7905
+ await db.schema.alterTable("run").addColumn(column, type).execute();
7906
+ } catch {}
7907
+ await db.schema.createTable("waitpoint").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("step_path", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("type", "text", (col) => col.notNull()).addColumn("kind", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("token_hash", "text").addColumn("created_at", "text", (col) => col.notNull()).addColumn("resume_at", "text").addColumn("expires_at", "text").addColumn("resolved_at", "text").addColumn("decided_by", "text").addColumn("approval", "text").addColumn("resume_payload", "text").addColumn("response_to_send", "text").ifNotExists().execute();
7908
+ await db.schema.createIndex("waitpoint_token").ifNotExists().unique().on("waitpoint").column("token_hash").execute();
7909
+ await db.schema.createIndex("waitpoint_status_resume").ifNotExists().on("waitpoint").columns(["status", "resume_at"]).execute();
7910
+ await db.schema.createIndex("waitpoint_status_expires").ifNotExists().on("waitpoint").columns(["status", "expires_at"]).execute();
7911
+ await db.schema.createIndex("waitpoint_run").ifNotExists().on("waitpoint").column("run_id").execute();
7912
+ const liveStep = db.schema.createIndex("waitpoint_live_step").ifNotExists().unique().on("waitpoint").columns(["run_id", "step_path"]).where(sql.ref("status"), "in", LIVE_WAITPOINT_STATUSES);
7913
+ try {
7914
+ await liveStep.execute();
7915
+ } catch {
7916
+ await db.updateTable("waitpoint").set({
7917
+ status: "CANCELLED",
7918
+ resolved_at: (/* @__PURE__ */ new Date()).toISOString()
7919
+ }).where("status", "in", LIVE_WAITPOINT_STATUSES).where((eb) => eb.and([eb.not(eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.waiting_on", "=", "waitpoint.id"))), eb.exists(eb.selectFrom("waitpoint as sibling").select("sibling.id").whereRef("sibling.run_id", "=", "waitpoint.run_id").whereRef("sibling.step_path", "=", "waitpoint.step_path").whereRef("sibling.id", "!=", "waitpoint.id").where("sibling.status", "in", LIVE_WAITPOINT_STATUSES))])).execute();
7920
+ await liveStep.execute();
7921
+ }
7922
+ await db.schema.createIndex("waitpoint_workflow_status").ifNotExists().on("waitpoint").columns(["workflow_id", "status"]).execute();
7923
+ await db.schema.createIndex("run_waiting").ifNotExists().on("run").columns(["workflow_id", "enqueued_at"]).where(sql.ref("status"), "=", WAITING_RUN_STATUS).execute();
7822
7924
  await migrateBlockType(db, "trigger_state", "trigger_name");
7823
7925
  await migrateBlockType(db, "step_execution", "block_name");
7824
7926
  return migrateTriggerStoreState(db);
@@ -8177,6 +8279,8 @@ var WorkflowRunStore = class WorkflowRunStore {
8177
8279
  unmigrated;
8178
8280
  runsInFlight = /* @__PURE__ */ new Set();
8179
8281
  journaledSteps = /* @__PURE__ */ new Map();
8282
+ pausingRuns = /* @__PURE__ */ new Set();
8283
+ resumedLegs = /* @__PURE__ */ new Set();
8180
8284
  constructor(db, unmigrated) {
8181
8285
  this.db = db;
8182
8286
  this.unmigrated = unmigrated;
@@ -8195,13 +8299,20 @@ var WorkflowRunStore = class WorkflowRunStore {
8195
8299
  this.unmigrated.delete(workflowId);
8196
8300
  }
8197
8301
  async recoverOrphanedRuns() {
8198
- let query = this.db.updateTable("run").set({
8199
- status: "FAILED",
8200
- error: ORPHANED_RUN_ERROR,
8201
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8202
- }).where("status", "=", "RUNNING");
8203
- if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
8204
- const recovered = (await query.returning("id").execute()).length;
8302
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
8303
+ const inFlight = [...this.runsInFlight];
8304
+ const recovered = await this.db.transaction().execute(async (trx) => {
8305
+ let query = trx.updateTable("run").set({
8306
+ status: "FAILED",
8307
+ error: ORPHANED_RUN_ERROR,
8308
+ ended_at: endedAt
8309
+ }).where("status", "=", "RUNNING");
8310
+ if (inFlight.length > 0) query = query.where("id", "not in", inFlight);
8311
+ const ids = (await query.returning("id").execute()).map((row) => row.id);
8312
+ await failWaitingSteps(trx, ids, ORPHANED_RUN_ERROR, null, endedAt);
8313
+ await closeLiveWaitpoints(trx, ids, "CANCELLED", endedAt);
8314
+ return ids.length;
8315
+ });
8205
8316
  if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
8206
8317
  return recovered;
8207
8318
  }
@@ -8265,14 +8376,25 @@ var WorkflowRunStore = class WorkflowRunStore {
8265
8376
  * CANCELLED run is not rerunnable and must not read as a workflow defect.
8266
8377
  */
8267
8378
  async cancelRun(runId, reason) {
8379
+ return this.endLiveRun(runId, "CANCELLED", reason, null);
8380
+ }
8381
+ async endLiveRun(runId, status, reason, errorName) {
8268
8382
  this.runsInFlight.delete(runId);
8269
8383
  this.journaledSteps.delete(runId);
8270
- if (erasedRuns.delete(runId)) return;
8271
- await this.db.updateTable("run").set({
8272
- status: "CANCELLED",
8273
- error: redactMessage(reason),
8274
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8275
- }).where("id", "=", runId).execute();
8384
+ this.pausingRuns.delete(runId);
8385
+ this.resumedLegs.delete(runId);
8386
+ if (erasedRuns.delete(runId)) return [];
8387
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
8388
+ return this.db.transaction().execute(async (trx) => {
8389
+ if ((await trx.updateTable("run").set({
8390
+ status,
8391
+ error: redactMessage(reason),
8392
+ error_name: errorName,
8393
+ ended_at: endedAt
8394
+ }).where("id", "=", runId).where("status", "in", LIVE_RUN_STATUSES).returning("id").execute()).length === 0) return [];
8395
+ await failWaitingSteps(trx, [runId], reason, errorName, endedAt);
8396
+ return closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
8397
+ });
8276
8398
  }
8277
8399
  async enqueueRun(options) {
8278
8400
  const id = randomUUID();
@@ -8480,12 +8602,15 @@ var WorkflowRunStore = class WorkflowRunStore {
8480
8602
  async beginRun(runId, details) {
8481
8603
  this.runsInFlight.add(runId);
8482
8604
  this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
8483
- await this.db.updateTable("run").set({
8605
+ if ((await this.db.updateTable("run").set({
8484
8606
  status: "RUNNING",
8485
8607
  workflow_name: details.workflowName,
8486
8608
  workflow_version: details.workflowVersion,
8487
8609
  started_at: (/* @__PURE__ */ new Date()).toISOString()
8488
- }).where("id", "=", runId).execute();
8610
+ }).where("id", "=", runId).where("status", "=", "PENDING").returning("id").execute()).length > 0) return true;
8611
+ this.runsInFlight.delete(runId);
8612
+ this.journaledSteps.delete(runId);
8613
+ return false;
8489
8614
  }
8490
8615
  async startRun(options) {
8491
8616
  const id = randomUUID();
@@ -8514,10 +8639,10 @@ var WorkflowRunStore = class WorkflowRunStore {
8514
8639
  if (erasedRuns.has(runId)) return;
8515
8640
  const values = stepValues(runId, ordinal, step);
8516
8641
  const { run_id: _run, step_id: _step, ...mutable } = values;
8517
- await this.db.insertInto("step_execution").values({
8642
+ if (!await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values({
8518
8643
  id: randomUUID(),
8519
8644
  ...values
8520
- }).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
8645
+ }).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute())) return;
8521
8646
  this.journaledSteps.get(runId)?.set(step.stepId, {
8522
8647
  ordinal,
8523
8648
  hash: rowHash(values)
@@ -8532,25 +8657,61 @@ var WorkflowRunStore = class WorkflowRunStore {
8532
8657
  return row.hash === rowHash(stepValues(runId, row.ordinal, step));
8533
8658
  });
8534
8659
  }
8535
- async finishRun(runId, result, executionOrder) {
8660
+ async finishRun(runId, result, executionOrder, pin) {
8536
8661
  this.runsInFlight.delete(runId);
8537
- const sweep = !this.sweepIsNoop(runId, result);
8662
+ const resumed = this.resumedLegs.delete(runId);
8663
+ const paused = this.pausingRuns.delete(runId) || resumed;
8664
+ const own = result.steps.some((step) => step.carried) ? {
8665
+ ...result,
8666
+ steps: result.steps.filter((step) => !step.carried)
8667
+ } : result;
8668
+ const sweep = !this.sweepIsNoop(runId, own);
8538
8669
  this.journaledSteps.delete(runId);
8539
- if (erasedRuns.delete(runId)) return;
8540
- if (sweep && result.steps.length > 0) try {
8541
- await this.sweepSteps(runId, result, executionOrder);
8670
+ if (erasedRuns.delete(runId)) return {
8671
+ held: true,
8672
+ closed: []
8673
+ };
8674
+ if (sweep && own.steps.length > 0) try {
8675
+ await this.sweepSteps(runId, own, executionOrder);
8542
8676
  } catch (error) {
8543
8677
  logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
8544
8678
  }
8545
- const notes = runWarningNotes(result);
8546
- await this.db.updateTable("run").set({
8547
- status: result.status,
8548
- error: result.error ? redactMessage(result.error) : null,
8549
- error_name: result.errorName ?? null,
8550
- ended_at: (/* @__PURE__ */ new Date()).toISOString(),
8551
- warnings: notes.length,
8552
- warning_notes: notes.length > 0 ? JSON.stringify(notes) : null
8553
- }).where("id", "=", runId).execute();
8679
+ const waiting = result.status === WAITING_RUN_STATUS;
8680
+ const on = result.waiting;
8681
+ const now = (/* @__PURE__ */ new Date()).toISOString();
8682
+ const close = async (trx) => {
8683
+ const prior = resumed ? await trx.selectFrom("run").select("warning_notes").where("id", "=", runId).executeTakeFirst() : void 0;
8684
+ const notes = [.../* @__PURE__ */ new Set([...priorNotes(prior?.warning_notes ?? null), ...runWarningNotes(own)])];
8685
+ const written = await trx.updateTable("run").set({
8686
+ status: result.status,
8687
+ error: result.error ? redactMessage(result.error) : null,
8688
+ error_name: result.errorName ?? null,
8689
+ ended_at: waiting ? null : now,
8690
+ warnings: notes.length,
8691
+ warning_notes: notes.length > 0 ? JSON.stringify(notes) : null,
8692
+ waiting_on: waiting ? on?.waitpointId ?? null : null,
8693
+ ...pin ?? {}
8694
+ }).where("id", "=", runId).$if(waiting, (qb) => qb.where((eb) => eb.exists(eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.id", "=", on?.waitpointId ?? "").where("waitpoint.run_id", "=", runId).where("waitpoint.step_id", "=", on?.stepId ?? "").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES)))).returning("id").execute();
8695
+ if (!waiting) return {
8696
+ held: true,
8697
+ closed: paused ? await closeLiveWaitpoints(trx, [runId], "EXPIRED", now) : []
8698
+ };
8699
+ if (written.length === 0) return {
8700
+ held: false,
8701
+ closed: []
8702
+ };
8703
+ const linkStatuses = (await trx.selectFrom("waitpoint").select("kind").where("id", "=", on.waitpointId).executeTakeFirst())?.kind === "webhook" ? LIVE_WAITPOINT_STATUSES : ["PENDING"];
8704
+ return {
8705
+ held: true,
8706
+ closed: await closeLiveWaitpoints(trx, [runId], "CANCELLED", now, (eb) => eb.or([eb("waitpoint.id", "=", on.waitpointId), eb.and([
8707
+ eb("waitpoint.type", "=", "WEBHOOK"),
8708
+ eb("waitpoint.kind", "=", "webhook"),
8709
+ eb("waitpoint.status", "in", linkStatuses),
8710
+ eb.exists(eb.selectFrom("step_execution").select("step_execution.id").whereRef("step_execution.run_id", "=", "waitpoint.run_id").whereRef("step_execution.step_id", "=", "waitpoint.step_id").where("step_execution.status", "=", "SUCCEEDED"))
8711
+ ])]))
8712
+ };
8713
+ };
8714
+ return waiting || paused ? this.db.transaction().execute(close) : close(this.db);
8554
8715
  }
8555
8716
  async sweepSteps(runId, result, executionOrder) {
8556
8717
  const journaled = await this.db.selectFrom("step_execution").select(["step_id", "ordinal"]).where("run_id", "=", runId).execute();
@@ -8563,7 +8724,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8563
8724
  nextOrdinal = Math.max(nextOrdinal, ran + 1);
8564
8725
  }
8565
8726
  const ordinalFor = (step) => ordinals.get(step.stepId) ?? nextOrdinal++;
8566
- await this.db.insertInto("step_execution").values(result.steps.map((step) => ({
8727
+ await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values(result.steps.map((step) => ({
8567
8728
  id: randomUUID(),
8568
8729
  ...stepValues(runId, ordinalFor(step), step)
8569
8730
  }))).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet((eb) => ({
@@ -8584,18 +8745,15 @@ var WorkflowRunStore = class WorkflowRunStore {
8584
8745
  version_match: eb.ref("excluded.version_match"),
8585
8746
  version_note: eb.ref("excluded.version_note"),
8586
8747
  config_hash: eb.ref("excluded.config_hash")
8587
- }))).execute();
8748
+ }))).execute());
8749
+ }
8750
+ async withRunRow(runId, fn) {
8751
+ return this.db.transaction().execute(async (trx) => {
8752
+ return await trx.selectFrom("run").select("id").where("id", "=", runId).forKeyShare().executeTakeFirst() ? fn(trx) : void 0;
8753
+ });
8588
8754
  }
8589
8755
  async failRun(runId, error, errorName) {
8590
- this.runsInFlight.delete(runId);
8591
- this.journaledSteps.delete(runId);
8592
- if (erasedRuns.delete(runId)) return;
8593
- await this.db.updateTable("run").set({
8594
- status: "FAILED",
8595
- error: redactMessage(error),
8596
- error_name: errorName ?? null,
8597
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8598
- }).where("id", "=", runId).execute();
8756
+ return this.endLiveRun(runId, "FAILED", error, errorName ?? null);
8599
8757
  }
8600
8758
  async listRuns(workflowId, limit = 25, options = {}) {
8601
8759
  if (Array.isArray(workflowId) && workflowId.length === 0) return [];
@@ -8627,10 +8785,10 @@ var WorkflowRunStore = class WorkflowRunStore {
8627
8785
  }
8628
8786
  async recordRunDocuments(runId, documentIds) {
8629
8787
  if (documentIds.length === 0 || erasedRuns.has(runId)) return;
8630
- await this.db.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
8788
+ await this.withRunRow(runId, (trx) => trx.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
8631
8789
  run_id: runId,
8632
8790
  document_id: documentId
8633
- }))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute();
8791
+ }))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute());
8634
8792
  }
8635
8793
  async getRunDocuments(runId) {
8636
8794
  return (await this.db.selectFrom("run_document").select("document_id").where("run_id", "=", runId).execute()).map((row) => row.document_id);
@@ -8775,6 +8933,181 @@ var WorkflowRunStore = class WorkflowRunStore {
8775
8933
  async deleteDedupe(workflowId) {
8776
8934
  await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
8777
8935
  }
8936
+ async createWaitpoint(row) {
8937
+ if (this.runsInFlight.has(row.run_id)) this.pausingRuns.add(row.run_id);
8938
+ for (let attempt = 1;; attempt++) {
8939
+ const inserted = await this.withRunRow(row.run_id, (trx) => trx.insertInto("waitpoint").values(row).onConflict((oc) => oc.doNothing()).returningAll().executeTakeFirst().then((waitpoint) => ({ waitpoint })));
8940
+ if (!inserted) throw new Error(`Run "${row.run_id}" no longer exists`);
8941
+ if (inserted.waitpoint) return {
8942
+ waitpoint: inserted.waitpoint,
8943
+ created: true
8944
+ };
8945
+ const live = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "=", row.run_id).where("step_path", "=", row.step_path).where("status", "in", LIVE_WAITPOINT_STATUSES).executeTakeFirst();
8946
+ if (live) return {
8947
+ waitpoint: live,
8948
+ created: false
8949
+ };
8950
+ if (attempt >= 3) throw new Error(`Could not create a waitpoint for step "${row.step_key}"`);
8951
+ }
8952
+ }
8953
+ async getWaitpoint(id) {
8954
+ return this.db.selectFrom("waitpoint").selectAll().where("id", "=", id).executeTakeFirst();
8955
+ }
8956
+ async getWaitpointByTokenHash(tokenHash) {
8957
+ return this.db.selectFrom("waitpoint").selectAll().where("token_hash", "=", tokenHash).executeTakeFirst();
8958
+ }
8959
+ async completeWaitpoint(id, resolution) {
8960
+ const now = (/* @__PURE__ */ new Date()).toISOString();
8961
+ return (await this.db.updateTable("waitpoint").set({
8962
+ status: "COMPLETED",
8963
+ resume_payload: JSON.stringify(resolution.payload ?? {}),
8964
+ resolved_at: resolution.resolvedAt ?? now,
8965
+ decided_by: resolution.decidedBy ?? null
8966
+ }).where("id", "=", id).where("status", "=", "PENDING").where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", now)])).where((eb) => eb.not(eb.exists(eb.selectFrom("run").innerJoin("waitpoint as awaited", "awaited.id", "run.waiting_on").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").whereRef("awaited.id", "!=", "waitpoint.id").where("awaited.kind", "!=", "webhook").where("awaited.status", "in", LIVE_WAITPOINT_STATUSES)))).where((eb) => eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").where("run.status", "in", ["RUNNING", WAITING_RUN_STATUS]))).returningAll().execute())[0];
8967
+ }
8968
+ async claimDueDelayWaitpoints(nowIso, limit) {
8969
+ return this.db.updateTable("waitpoint").set({
8970
+ status: "COMPLETED",
8971
+ resolved_at: nowIso,
8972
+ resume_payload: "{}"
8973
+ }).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "<=", nowIso).orderBy("resume_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
8974
+ }
8975
+ async expireDueWaitpoints(nowIso, limit) {
8976
+ return this.db.transaction().execute(async (trx) => {
8977
+ const expired = await trx.updateTable("waitpoint").set({
8978
+ status: "EXPIRED",
8979
+ resolved_at: nowIso
8980
+ }).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("expires_at", "<=", nowIso).orderBy("expires_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
8981
+ const ended = [];
8982
+ for (const waitpoint of expired) {
8983
+ const end = await endWaitingRun(trx, waitpoint.run_id, {
8984
+ reason: waitpointExpiryReason(waitpoint),
8985
+ errorName: WAITPOINT_EXPIRED_ERROR_NAME,
8986
+ endedAt: nowIso,
8987
+ waitpointId: waitpoint.id
8988
+ });
8989
+ if (end) ended.push(end);
8990
+ }
8991
+ return {
8992
+ expired,
8993
+ ended
8994
+ };
8995
+ });
8996
+ }
8997
+ async endStrandedWaits(limit) {
8998
+ const runs = await this.db.selectFrom("run").select("id").where("status", "=", WAITING_RUN_STATUS).where((eb) => strandedWait(eb)).limit(limit).execute();
8999
+ const ended = [];
9000
+ for (const run of runs) {
9001
+ const end = await this.db.transaction().execute((trx) => endWaitingRun(trx, run.id, {
9002
+ reason: STRANDED_WAIT_REASON,
9003
+ errorName: WAITPOINT_NOT_LIVE_ERROR_NAME,
9004
+ endedAt: (/* @__PURE__ */ new Date()).toISOString()
9005
+ }));
9006
+ if (end) ended.push(end);
9007
+ }
9008
+ return ended;
9009
+ }
9010
+ async nextExpiry() {
9011
+ return (await this.db.selectFrom("waitpoint").select("expires_at").where("status", "=", "PENDING").where("expires_at", "is not", null).orderBy("expires_at").limit(1).executeTakeFirst())?.expires_at ?? void 0;
9012
+ }
9013
+ async countWaitingRuns() {
9014
+ const row = await this.db.selectFrom("run").select((eb) => eb.fn.countAll().as("count")).where("status", "=", WAITING_RUN_STATUS).executeTakeFirst();
9015
+ return Number(row?.count ?? 0);
9016
+ }
9017
+ async nextDelayDue() {
9018
+ return (await this.db.selectFrom("waitpoint").select("resume_at").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "is not", null).orderBy("resume_at").limit(1).executeTakeFirst())?.resume_at ?? void 0;
9019
+ }
9020
+ async hasCompletedWaitpoint(runId) {
9021
+ return await this.db.selectFrom("waitpoint").select("id").where("run_id", "=", runId).where("status", "=", "COMPLETED").limit(1).executeTakeFirst() !== void 0;
9022
+ }
9023
+ async heldRunIds() {
9024
+ return (await this.db.selectFrom("waitpoint").innerJoin("run", "run.id", "waitpoint.run_id").select("waitpoint.run_id").distinct().where("waitpoint.status", "=", "COMPLETED").where("run.status", "=", WAITING_RUN_STATUS).execute()).map((row) => row.run_id);
9025
+ }
9026
+ isRunInFlight(runId) {
9027
+ return this.runsInFlight.has(runId);
9028
+ }
9029
+ async claimWaitingRun(runId) {
9030
+ if ((await this.db.updateTable("run").set({ status: "RUNNING" }).where("id", "=", runId).where("status", "=", "WAITING").returning("id").execute()).length === 0) return false;
9031
+ this.runsInFlight.add(runId);
9032
+ this.resumedLegs.add(runId);
9033
+ this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
9034
+ return true;
9035
+ }
9036
+ async returnToWaiting(runId) {
9037
+ this.runsInFlight.delete(runId);
9038
+ this.resumedLegs.delete(runId);
9039
+ this.journaledSteps.delete(runId);
9040
+ await this.db.updateTable("run").set({ status: WAITING_RUN_STATUS }).where("id", "=", runId).where("status", "=", "RUNNING").execute();
9041
+ }
9042
+ async consumeCompletedWaitpoint(runId) {
9043
+ return this.db.transaction().execute(async (trx) => {
9044
+ const waitingOn = (await trx.selectFrom("run").select("waiting_on").where("id", "=", runId).executeTakeFirst())?.waiting_on ?? null;
9045
+ const candidates = await trx.selectFrom("waitpoint").selectAll().where("run_id", "=", runId).where("status", "=", "COMPLETED").orderBy("resolved_at").execute();
9046
+ const awaited = waitingOn ? await trx.selectFrom("waitpoint").select("kind").where("id", "=", waitingOn).executeTakeFirst() : void 0;
9047
+ const linksResume = !awaited || awaited.kind === "webhook";
9048
+ const chosen = candidates.find((row) => row.id === waitingOn) ?? (linksResume ? candidates[0] : void 0);
9049
+ if (!chosen) return void 0;
9050
+ const waitpoint = (await trx.updateTable("waitpoint").set({ status: "CONSUMED" }).where("id", "=", chosen.id).where("status", "=", "COMPLETED").returningAll().execute())[0];
9051
+ if (!waitpoint) return void 0;
9052
+ return {
9053
+ waitpoint,
9054
+ closed: waitingOn && waitingOn !== waitpoint.id ? await closeLiveWaitpoints(trx, [runId], "CANCELLED", (/* @__PURE__ */ new Date()).toISOString(), (eb) => eb("waitpoint.id", "!=", waitingOn)) : []
9055
+ };
9056
+ });
9057
+ }
9058
+ async getRunPin(runId) {
9059
+ return this.db.selectFrom("run").select([
9060
+ "definition_hash",
9061
+ "definition_op_index",
9062
+ "definition_scope_hash"
9063
+ ]).where("id", "=", runId).executeTakeFirst();
9064
+ }
9065
+ async cancelIdleRun(runId, reason) {
9066
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
9067
+ if (!await this.db.transaction().execute(async (trx) => {
9068
+ if ((await trx.updateTable("run").set({
9069
+ status: "CANCELLED",
9070
+ error: redactMessage(reason),
9071
+ ended_at: endedAt
9072
+ }).where("id", "=", runId).where("status", "in", ["WAITING", "PENDING"]).returning("id").execute()).length === 0) return false;
9073
+ await failWaitingSteps(trx, [runId], reason, null, endedAt);
9074
+ await closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
9075
+ return true;
9076
+ })) return void 0;
9077
+ this.runsInFlight.delete(runId);
9078
+ return runId;
9079
+ }
9080
+ async endWait(runId, end) {
9081
+ this.runsInFlight.delete(runId);
9082
+ this.journaledSteps.delete(runId);
9083
+ this.pausingRuns.delete(runId);
9084
+ this.resumedLegs.delete(runId);
9085
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
9086
+ return this.db.transaction().execute(async (trx) => {
9087
+ if ((await trx.updateTable("run").set({
9088
+ status: end.status,
9089
+ error: redactMessage(end.reason),
9090
+ error_name: end.errorName,
9091
+ ended_at: endedAt
9092
+ }).where("id", "=", runId).where("status", "=", "RUNNING").returning("id").execute()).length === 0) return [];
9093
+ await failWaitingSteps(trx, [runId], end.reason, end.errorName, endedAt);
9094
+ return closeLiveWaitpoints(trx, [runId], end.waitpoints, endedAt);
9095
+ });
9096
+ }
9097
+ async listWaitpointsForRuns(runIds) {
9098
+ const byRun = new Map(runIds.map((id) => [id, []]));
9099
+ if (runIds.length === 0) return byRun;
9100
+ const rows = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "in", [...new Set(runIds)]).orderBy("created_at").execute();
9101
+ for (const row of rows) byRun.get(row.run_id)?.push(row);
9102
+ return byRun;
9103
+ }
9104
+ async listWaitingRuns(workflowId, limit) {
9105
+ if (Array.isArray(workflowId) && workflowId.length === 0) return [];
9106
+ let query = this.db.selectFrom("run").selectAll().where("status", "=", WAITING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc").limit(Math.min(Math.max(limit, 1), 100));
9107
+ if (Array.isArray(workflowId)) query = query.where("workflow_id", "in", workflowId);
9108
+ else if (workflowId) query = query.where("workflow_id", "=", workflowId);
9109
+ return query.execute();
9110
+ }
8778
9111
  async pruneFinishedRuns(cutoffIso, batchSize = PRUNE_BATCH_SIZE) {
8779
9112
  let pruned = 0;
8780
9113
  for (;;) {
@@ -8782,6 +9115,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8782
9115
  const ids = (await trx.selectFrom("run").select("id").where("ended_at", "is not", null).where("ended_at", "<", cutoffIso).limit(batchSize).execute()).map((row) => row.id);
8783
9116
  if (ids.length === 0) return 0;
8784
9117
  await trx.deleteFrom("step_execution").where("run_id", "in", ids).execute();
9118
+ await trx.deleteFrom("waitpoint").where("run_id", "in", ids).execute();
8785
9119
  await trx.deleteFrom("run_document").where("run_id", "in", ids).execute();
8786
9120
  await trx.deleteFrom("run").where("id", "in", ids).execute();
8787
9121
  return ids.length;
@@ -8813,10 +9147,11 @@ var WorkflowRunStore = class WorkflowRunStore {
8813
9147
  for (const id of frontier) runIds.add(id);
8814
9148
  }
8815
9149
  if (runIds.size === 0) return erased;
8816
- const all = [...runIds];
8817
- const live = await trx.selectFrom("run").select("id").where("id", "in", all).where("status", "in", ["RUNNING", PENDING_RUN_STATUS]).execute();
8818
- for (const row of live) erasedRuns.add(row.id);
9150
+ const all = [...runIds].sort();
9151
+ const locked = await trx.selectFrom("run").select(["id", "status"]).where("id", "in", all).orderBy("id").forUpdate().execute();
9152
+ for (const row of locked) if (row.status === "RUNNING" || row.status === "PENDING") erasedRuns.add(row.id);
8819
9153
  erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
9154
+ await trx.deleteFrom("waitpoint").where("run_id", "in", all).execute();
8820
9155
  erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
8821
9156
  erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
8822
9157
  erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
@@ -8933,6 +9268,118 @@ function decodeRunCursor(cursor) {
8933
9268
  id: parsed[1]
8934
9269
  };
8935
9270
  }
9271
+ const WAITPOINT_POLL_MAX_MS = 3e5;
9272
+ const WAITPOINT_RETRY_MS = 3e4;
9273
+ const SUSPENDED_COUNT_MS = 3e4;
9274
+ var WaitpointPoller = class {
9275
+ deps;
9276
+ running = false;
9277
+ polling = false;
9278
+ again = false;
9279
+ timer;
9280
+ timerAt;
9281
+ nudged;
9282
+ suspended = 0;
9283
+ suspendedAt = 0;
9284
+ counting = false;
9285
+ constructor(deps) {
9286
+ this.deps = deps;
9287
+ }
9288
+ get active() {
9289
+ return this.running;
9290
+ }
9291
+ start() {
9292
+ if (this.running) return;
9293
+ this.running = true;
9294
+ this.poll();
9295
+ }
9296
+ stop() {
9297
+ this.running = false;
9298
+ clearTimeout(this.timer);
9299
+ this.timer = void 0;
9300
+ this.timerAt = void 0;
9301
+ this.nudged = void 0;
9302
+ }
9303
+ nudge(dueIso) {
9304
+ if (!this.running) return;
9305
+ const due = Date.parse(dueIso);
9306
+ if (Number.isNaN(due)) return;
9307
+ if (this.polling) {
9308
+ this.nudged = Math.min(this.nudged ?? Infinity, due);
9309
+ return;
9310
+ }
9311
+ if (this.timerAt !== void 0 && due >= this.timerAt) return;
9312
+ this.arm(due);
9313
+ }
9314
+ suspendedRuns() {
9315
+ if (Date.now() - this.suspendedAt > SUSPENDED_COUNT_MS) this.count();
9316
+ return this.suspended;
9317
+ }
9318
+ async count(store) {
9319
+ if (this.counting) return this.suspended;
9320
+ this.counting = true;
9321
+ try {
9322
+ const journal = store ?? await this.deps.store();
9323
+ if (journal) {
9324
+ this.suspended = await journal.countWaitingRuns();
9325
+ this.suspendedAt = Date.now();
9326
+ }
9327
+ } catch (error) {
9328
+ this.deps.logger.warn("Counting suspended runs failed: @error", error);
9329
+ } finally {
9330
+ this.counting = false;
9331
+ }
9332
+ return this.suspended;
9333
+ }
9334
+ /** One pass: due delays and held resumes deliver, overdue waits expire, the
9335
+ * timer re-arms. */
9336
+ async poll() {
9337
+ if (!this.running) return;
9338
+ if (this.polling) {
9339
+ this.again = true;
9340
+ return;
9341
+ }
9342
+ this.polling = true;
9343
+ let next = Date.now() + WAITPOINT_RETRY_MS;
9344
+ try {
9345
+ const store = await this.deps.store();
9346
+ if (!store) return;
9347
+ const batch = this.deps.batch ?? 100;
9348
+ do {
9349
+ this.again = false;
9350
+ const now = (/* @__PURE__ */ new Date()).toISOString();
9351
+ const due = await store.claimDueDelayWaitpoints(now, batch);
9352
+ for (const runId of new Set(due.map((row) => row.run_id))) await this.deps.resume(runId);
9353
+ const { expired, ended } = await store.expireDueWaitpoints(now, batch);
9354
+ const stranded = await store.endStrandedWaits(batch);
9355
+ for (const end of [...ended, ...stranded]) await this.deps.ended(end);
9356
+ if (due.length === batch || expired.length === batch || stranded.length === batch) this.again = true;
9357
+ } while (this.again && this.active);
9358
+ for (const runId of await store.heldRunIds()) await this.deps.resume(runId);
9359
+ await this.count(store);
9360
+ const [delay, expiry] = await Promise.all([store.nextDelayDue(), store.nextExpiry()]);
9361
+ next = Math.min(delay ? Date.parse(delay) : Infinity, expiry ? Date.parse(expiry) : Infinity);
9362
+ } catch (error) {
9363
+ this.deps.logger.warn("Waitpoint poll failed: @error", error);
9364
+ } finally {
9365
+ this.polling = false;
9366
+ if (this.nudged !== void 0) next = Math.min(next, this.nudged);
9367
+ this.nudged = void 0;
9368
+ if (this.active) this.arm(next);
9369
+ }
9370
+ }
9371
+ arm(due) {
9372
+ clearTimeout(this.timer);
9373
+ const delay = Math.min(Math.max(Number.isFinite(due) ? due - Date.now() : Infinity, 0), WAITPOINT_POLL_MAX_MS);
9374
+ this.timerAt = Date.now() + delay;
9375
+ this.timer = setTimeout(() => {
9376
+ this.timer = void 0;
9377
+ this.timerAt = void 0;
9378
+ this.poll();
9379
+ }, delay);
9380
+ this.timer.unref();
9381
+ }
9382
+ };
8936
9383
  //#endregion
8937
9384
  //#region src/reactor/run-gate.ts
8938
9385
  const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
@@ -8970,8 +9417,11 @@ var WorkflowRunGate = class {
8970
9417
  * - **SINGLETON**: refused outright while a run is active. The caller
8971
9418
  * journals the refusal rather than dropping it silently, because a firing
8972
9419
  * that vanished is indistinguishable from a trigger that never fired.
9420
+ *
9421
+ * A `resume` is a waiting run's next leg, not a new firing: it queues for
9422
+ * the slot under every policy, past the queue depth too.
8973
9423
  */
8974
- async admit(workflowId, policy, onQueue) {
9424
+ async admit(workflowId, policy, onQueue, options = {}) {
8975
9425
  if (this.closed) return {
8976
9426
  admitted: false,
8977
9427
  reason: CLOSED_REASON,
@@ -8990,7 +9440,7 @@ var WorkflowRunGate = class {
8990
9440
  this.lanes.set(workflowId, lane);
8991
9441
  let waited = false;
8992
9442
  if (lane.active >= limit) {
8993
- if (policy.concurrency === "SINGLETON") {
9443
+ if (policy.concurrency === "SINGLETON" && !options.resume) {
8994
9444
  this.forgetIfIdle(workflowId, lane);
8995
9445
  return {
8996
9446
  admitted: false,
@@ -8998,7 +9448,7 @@ var WorkflowRunGate = class {
8998
9448
  reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
8999
9449
  };
9000
9450
  }
9001
- if (lane.waiting.length >= this.maxQueued) return {
9451
+ if (!options.resume && lane.waiting.length >= this.maxQueued) return {
9002
9452
  admitted: false,
9003
9453
  refusal: "queue-full",
9004
9454
  reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
@@ -10653,12 +11103,72 @@ function refusalStatus(refusal) {
10653
11103
  switch (refusal) {
10654
11104
  case "parked":
10655
11105
  case "singleton":
10656
- case "stale": return 409;
11106
+ case "stale":
11107
+ case "cancelled": return 409;
10657
11108
  case "queue-full":
10658
11109
  case "expired": return 429;
10659
11110
  default: return 500;
10660
11111
  }
10661
11112
  }
11113
+ function runResultReply(result, succeededStatus) {
11114
+ return {
11115
+ status: result.status === "SUCCEEDED" ? succeededStatus : result.status === "WAITING" ? 202 : refusalStatus(result.refusal),
11116
+ contentType: JSON_CONTENT_TYPE,
11117
+ body: JSON.stringify({
11118
+ runId: result.runId,
11119
+ status: result.status,
11120
+ error: result.error ?? null
11121
+ })
11122
+ };
11123
+ }
11124
+ const RESPONDED = Symbol("a step responded");
11125
+ const DROPPED_RESPONSE_HEADERS = /* @__PURE__ */ new Set([
11126
+ "connection",
11127
+ "content-length",
11128
+ "keep-alive",
11129
+ "set-cookie",
11130
+ "transfer-encoding",
11131
+ "upgrade"
11132
+ ]);
11133
+ const ORIGIN_SAFE_HEADERS = {
11134
+ "x-content-type-options": "nosniff",
11135
+ "content-security-policy": "sandbox"
11136
+ };
11137
+ function sendableHeader(name, value) {
11138
+ try {
11139
+ validateHeaderName(name);
11140
+ validateHeaderValue(name, value);
11141
+ return true;
11142
+ } catch {
11143
+ return false;
11144
+ }
11145
+ }
11146
+ /** A step's `run.respond` / `run.stop` answer as an HTTP reply. As upstream,
11147
+ * none answers 200 with `{}`, and a string body goes as text. */
11148
+ function runResponseReply(response) {
11149
+ const status = typeof response.status === "number" && Number.isInteger(response.status) && response.status >= 200 && response.status <= 599 ? response.status : 200;
11150
+ const body = response.body ?? {};
11151
+ let contentType = typeof body === "string" ? "text/plain; charset=utf-8" : JSON_CONTENT_TYPE;
11152
+ const headers = {};
11153
+ const given = response.headers && typeof response.headers === "object" ? response.headers : {};
11154
+ for (const [rawName, rawValue] of Object.entries(given)) {
11155
+ const name = rawName.toLowerCase();
11156
+ if (typeof rawValue !== "string" && typeof rawValue !== "number") continue;
11157
+ const value = String(rawValue);
11158
+ if (!sendableHeader(name, value)) continue;
11159
+ if (name === "content-type") contentType = value;
11160
+ else if (!DROPPED_RESPONSE_HEADERS.has(name)) headers[name] = value;
11161
+ }
11162
+ return {
11163
+ status,
11164
+ contentType,
11165
+ body: typeof body === "string" ? body : JSON.stringify(body),
11166
+ headers: {
11167
+ ...headers,
11168
+ ...ORIGIN_SAFE_HEADERS
11169
+ }
11170
+ };
11171
+ }
10662
11172
  const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
10663
11173
  function registrationKey(state) {
10664
11174
  const trigger = runnableDefinition(state).trigger;
@@ -11619,6 +12129,8 @@ var WorkflowRuntimeService = class {
11619
12129
  startTriggerSupervisor() {
11620
12130
  this.supervisor().start();
11621
12131
  this.admitAbandonedRuns();
12132
+ this.deliverAllHeld();
12133
+ this.startWaitpointPoller();
11622
12134
  this.pendingSweepTimer ??= setInterval(() => {
11623
12135
  this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
11624
12136
  }, PENDING_SWEEP_INTERVAL_MS);
@@ -11641,6 +12153,7 @@ var WorkflowRuntimeService = class {
11641
12153
  stopTriggerSupervisor() {
11642
12154
  clearInterval(this.pendingSweepTimer);
11643
12155
  this.pendingSweepTimer = void 0;
12156
+ this.stopWaitpointPoller();
11644
12157
  this.triggerSupervisor?.stop();
11645
12158
  }
11646
12159
  closed = false;
@@ -11848,13 +12361,54 @@ var WorkflowRuntimeService = class {
11848
12361
  }));
11849
12362
  let replied = false;
11850
12363
  const takeReply = () => !replied && (replied = true);
11851
- if (runId) this.heldReplies.set(runId, takeReply);
12364
+ let sent;
12365
+ let responded = () => void 0;
12366
+ const respondedFirst = new Promise((resolve) => {
12367
+ responded = () => resolve(RESPONDED);
12368
+ });
12369
+ const responder = (response) => {
12370
+ if (sent || !takeReply()) return;
12371
+ sent = response;
12372
+ responded();
12373
+ };
12374
+ let ended;
12375
+ let endedEarly = () => void 0;
12376
+ const endedFirst = new Promise((resolve) => {
12377
+ endedEarly = () => resolve({
12378
+ ok: true,
12379
+ result: ended
12380
+ });
12381
+ });
12382
+ const ender = (result) => {
12383
+ if (ended || !takeReply()) return;
12384
+ ended = result;
12385
+ endedEarly();
12386
+ };
12387
+ if (runId) {
12388
+ this.heldReplies.set(runId, takeReply);
12389
+ this.runResponders.set(runId, responder);
12390
+ this.queuedReplies.set(runId, ender);
12391
+ }
11852
12392
  let run;
11853
12393
  try {
11854
- run = await Promise.race([firing, timeout(DELIVERY_TIMEOUT_MS)]);
11855
- if (run === TIMED_OUT && !takeReply()) run = await firing;
12394
+ const raced = await Promise.race([
12395
+ firing,
12396
+ respondedFirst,
12397
+ endedFirst,
12398
+ timeout(DELIVERY_TIMEOUT_MS)
12399
+ ]);
12400
+ if (sent) return runResponseReply(sent);
12401
+ run = raced;
12402
+ if (run === TIMED_OUT && !takeReply()) run = ended ? {
12403
+ ok: true,
12404
+ result: ended
12405
+ } : await firing;
11856
12406
  } finally {
11857
- if (runId) this.heldReplies.delete(runId);
12407
+ if (runId) {
12408
+ this.heldReplies.delete(runId);
12409
+ if (this.runResponders.get(runId) === responder) this.runResponders.delete(runId);
12410
+ if (this.queuedReplies.get(runId) === ender) this.queuedReplies.delete(runId);
12411
+ }
11858
12412
  }
11859
12413
  if (run === TIMED_OUT) {
11860
12414
  this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
@@ -11880,15 +12434,7 @@ var WorkflowRuntimeService = class {
11880
12434
  body: JSON.stringify({ error: message })
11881
12435
  };
11882
12436
  }
11883
- return {
11884
- status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
11885
- contentType: JSON_CONTENT_TYPE,
11886
- body: JSON.stringify({
11887
- runId: run.result.runId,
11888
- status: run.result.status,
11889
- error: run.result.error ?? null
11890
- })
11891
- };
12437
+ return runResultReply(run.result, config.responseStatus);
11892
12438
  }
11893
12439
  async pieceHandshake(binding, request) {
11894
12440
  const handshake = await this.pieceHandshakeConfig(binding);
@@ -12624,6 +13170,61 @@ var WorkflowRuntimeService = class {
12624
13170
  type: "value"
12625
13171
  }]
12626
13172
  };
13173
+ case DELAY_BLOCK: return {
13174
+ source: "static",
13175
+ nodes: [{
13176
+ name: "resumedAt",
13177
+ type: "DateTime!"
13178
+ }]
13179
+ };
13180
+ case APPROVAL_BLOCK: return {
13181
+ source: "static",
13182
+ nodes: [
13183
+ {
13184
+ name: "approved",
13185
+ type: "Boolean!"
13186
+ },
13187
+ {
13188
+ name: "reason",
13189
+ type: "String"
13190
+ },
13191
+ {
13192
+ name: "decidedBy",
13193
+ type: "String",
13194
+ description: "The approver's DID"
13195
+ },
13196
+ {
13197
+ name: "decidedAt",
13198
+ type: "DateTime!"
13199
+ }
13200
+ ]
13201
+ };
13202
+ case STOP_BLOCK: return {
13203
+ source: "static",
13204
+ nodes: [{
13205
+ name: "response",
13206
+ type: "JSONObject",
13207
+ description: "The answer sent, when one was set"
13208
+ }]
13209
+ };
13210
+ case WAIT_FOR_WEBHOOK_BLOCK: return {
13211
+ source: "static",
13212
+ nodes: [
13213
+ {
13214
+ name: "body",
13215
+ type: "Unknown",
13216
+ description: "The resuming request's body"
13217
+ },
13218
+ {
13219
+ name: "headers",
13220
+ type: "JSONObject!"
13221
+ },
13222
+ {
13223
+ name: "queryParams",
13224
+ type: "JSONObject!"
13225
+ }
13226
+ ]
13227
+ };
12627
13228
  case DOCUMENT_CREATED_BLOCK:
12628
13229
  case DOCUMENT_DELETED_BLOCK: return {
12629
13230
  source: "static",
@@ -12945,6 +13546,7 @@ var WorkflowRuntimeService = class {
12945
13546
  let definition;
12946
13547
  let runUser;
12947
13548
  let readGeneration;
13549
+ let revision;
12948
13550
  try {
12949
13551
  if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
12950
13552
  run.refused = true;
@@ -12954,6 +13556,7 @@ var WorkflowRuntimeService = class {
12954
13556
  if (cached) {
12955
13557
  state = cached.state;
12956
13558
  documentName = cached.name;
13559
+ revision = cached.revision;
12957
13560
  readGeneration = this.workflowGeneration(workflowId);
12958
13561
  } else {
12959
13562
  const generation = this.workflowGeneration(workflowId);
@@ -12965,6 +13568,7 @@ var WorkflowRuntimeService = class {
12965
13568
  }
12966
13569
  state = document.state.global;
12967
13570
  documentName = document.header.name;
13571
+ revision = globalRevision(document);
12968
13572
  this.cacheLoaded(workflowId, generation, document);
12969
13573
  }
12970
13574
  if (state.status !== "ENABLED") {
@@ -13093,74 +13697,63 @@ var WorkflowRuntimeService = class {
13093
13697
  }
13094
13698
  }
13095
13699
  let runId = enqueuedRunId ?? null;
13096
- let journalFailed = false;
13097
- const executionOrder = /* @__PURE__ */ new Map();
13098
13700
  let session;
13099
13701
  try {
13100
- runId = await this.telemetry.phase("journal.start", {}, async () => {
13702
+ const journaled = await this.telemetry.phase("journal.start", {}, async () => {
13101
13703
  if (enqueuedRunId) {
13102
- await store?.beginRun(enqueuedRunId, {
13704
+ if (await store?.beginRun(enqueuedRunId, {
13103
13705
  workflowName: runJournalName(state.name, documentName),
13104
13706
  workflowVersion: runnable.version
13105
- });
13106
- return enqueuedRunId;
13707
+ }) === false) return {
13708
+ runId: null,
13709
+ cancelled: true
13710
+ };
13711
+ return { runId: enqueuedRunId };
13107
13712
  }
13108
- return await store?.startRun({
13713
+ return { runId: await store?.startRun({
13109
13714
  workflowId,
13110
13715
  workflowName: runJournalName(state.name, documentName),
13111
13716
  workflowVersion: runnable.version,
13112
13717
  triggerKind,
13113
13718
  triggerPayload,
13114
13719
  rerunOf: resume?.rerunOf
13115
- }) ?? null;
13720
+ }) ?? null };
13116
13721
  });
13722
+ if (journaled.cancelled) {
13723
+ const row = enqueuedRunId ? await store?.getRun(enqueuedRunId) : void 0;
13724
+ return {
13725
+ status: CANCELLED_RUN_STATUS,
13726
+ steps: [],
13727
+ runId: enqueuedRunId ?? null,
13728
+ ...row?.error ? { error: row.error } : {},
13729
+ refusal: "cancelled"
13730
+ };
13731
+ }
13732
+ runId = journaled.runId;
13117
13733
  if (runId) run.span.setAttribute("run.id", runId);
13118
13734
  if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
13119
13735
  session = this.workers().session();
13120
- const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider(), workflowId);
13121
- const journal = store;
13122
- const journaledRunId = runId;
13123
- const handed = /* @__PURE__ */ new Set();
13124
- const result = await withRunScope({
13736
+ const { result, handed } = await this.executeLeg({
13125
13737
  workflowId,
13126
13738
  runId,
13739
+ store,
13740
+ session,
13741
+ definition,
13742
+ runnable,
13743
+ policy,
13744
+ deadline,
13127
13745
  connections,
13128
- pieceWorker: session,
13129
- ...runUser !== void 0 ? { runUser } : {},
13130
- attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
13131
- recordDocuments: async (documentIds) => {
13132
- for (const documentId of documentIds) handed.add(documentId);
13133
- if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
13134
- }
13135
- }, () => runWorkflow({
13136
- definition: {
13137
- ...definition,
13138
- variables
13139
- },
13140
- executor,
13141
- declaredPorts: blockPorts,
13142
- blockDefaults: (block) => this.blockDefaults(block),
13143
- ...secretValues.length > 0 ? { redactValues: secretValues } : {},
13746
+ runUser,
13144
13747
  triggerPayload,
13748
+ executor,
13145
13749
  completedSteps: resume?.completedSteps,
13146
- ...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
13147
- ...deadline !== void 0 ? { deadline } : {},
13148
- onStep: store && journaledRunId ? async (record, ordinal) => {
13149
- executionOrder.set(record.stepId, ordinal);
13150
- try {
13151
- await this.telemetry.phase("journal.step", {}, () => store.recordStep(journaledRunId, ordinal, record));
13152
- } catch (error) {
13153
- if (journalFailed) return;
13154
- journalFailed = true;
13155
- this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
13156
- }
13157
- } : void 0
13158
- }));
13159
- if (store && journaledRunId) try {
13160
- await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
13161
- } catch (error) {
13162
- this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
13163
- }
13750
+ replayed: resume?.completedSteps,
13751
+ pin: () => ({
13752
+ definition_hash: this.definitionHash(workflowId, state),
13753
+ definition_op_index: revision,
13754
+ definition_scope_hash: scopeHash(state)
13755
+ })
13756
+ });
13164
13757
  if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
13165
13758
  const finished = {
13166
13759
  ...result,
@@ -13174,7 +13767,14 @@ var WorkflowRuntimeService = class {
13174
13767
  runId
13175
13768
  };
13176
13769
  } catch (error) {
13177
- if (store && runId) await store.failRun(runId, error instanceof Error ? error.message : String(error), errorNameOf(error));
13770
+ if (store && runId) {
13771
+ const message = error instanceof Error ? error.message : String(error);
13772
+ const closed = await store.failRun(runId, message, errorNameOf(error));
13773
+ this.answerClosed(closed, runId, {
13774
+ status: "FAILED",
13775
+ error: message
13776
+ });
13777
+ }
13178
13778
  throw error;
13179
13779
  } finally {
13180
13780
  session?.close();
@@ -13182,6 +13782,115 @@ var WorkflowRuntimeService = class {
13182
13782
  dispatched();
13183
13783
  }
13184
13784
  }
13785
+ async executeLeg(leg) {
13786
+ const { workflowId, runId, store, policy } = leg;
13787
+ let journalFailed = false;
13788
+ const executionOrder = /* @__PURE__ */ new Map();
13789
+ const { variables, secretValues } = await resolveVariables(leg.runnable.variables, this.secretProvider(), workflowId);
13790
+ const handed = new Set(leg.handed);
13791
+ const attachments = attachmentOrigins(leg.triggerPayload, handed, leg.replayed);
13792
+ for (const ref of leg.written ?? []) attachments.written.add(ref);
13793
+ const waitpoints = store && runId ? this.waitpointPort(store, workflowId, runId, maxSuspensionMs(leg.runnable)) : void 0;
13794
+ const respond = leg.respond ?? (runId && !leg.resume ? (response) => this.runResponders.get(runId)?.(response) : void 0);
13795
+ let result = await withRunScope({
13796
+ workflowId,
13797
+ runId,
13798
+ connections: leg.connections,
13799
+ pieceWorker: leg.session,
13800
+ ...leg.runUser !== void 0 ? { runUser: leg.runUser } : {},
13801
+ attachments,
13802
+ recordDocuments: async (documentIds) => {
13803
+ for (const documentId of documentIds) handed.add(documentId);
13804
+ if (store && runId) await store.recordRunDocuments(runId, documentIds);
13805
+ },
13806
+ ...waitpoints ? { waitpoints: waitpoints.port } : {}
13807
+ }, () => runWorkflow({
13808
+ definition: {
13809
+ ...leg.definition,
13810
+ variables
13811
+ },
13812
+ executor: leg.executor,
13813
+ declaredPorts: blockPorts,
13814
+ blockDefaults: (block) => this.blockDefaults(block),
13815
+ ...secretValues.length > 0 ? { redactValues: secretValues } : {},
13816
+ triggerPayload: leg.triggerPayload,
13817
+ completedSteps: leg.completedSteps,
13818
+ ...leg.carriedSteps ? { carriedSteps: leg.carriedSteps } : {},
13819
+ ...leg.resume ? { resume: leg.resume } : {},
13820
+ ...leg.firstOrdinal !== void 0 ? { firstOrdinal: leg.firstOrdinal } : {},
13821
+ ...respond ? { onRespond: respond } : {},
13822
+ ...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
13823
+ ...leg.deadline !== void 0 ? { deadline: leg.deadline } : {},
13824
+ onStep: store && runId ? async (record, ordinal) => {
13825
+ executionOrder.set(record.stepId, ordinal);
13826
+ try {
13827
+ await this.telemetry.phase("journal.step", {}, () => store.recordStep(runId, ordinal, record));
13828
+ } catch (error) {
13829
+ if (journalFailed) return;
13830
+ journalFailed = true;
13831
+ this.logger.warn(`Run ${runId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
13832
+ }
13833
+ } : void 0
13834
+ }));
13835
+ if (store && runId) {
13836
+ let waiting = result.status === WAITING_RUN_STATUS;
13837
+ try {
13838
+ const { held, closed } = await this.telemetry.phase("journal.finish", {}, () => store.finishRun(runId, result, executionOrder, waiting ? leg.pin?.() : void 0));
13839
+ this.answerClosed(closed, runId, {
13840
+ status: waiting ? CANCELLED_RUN_STATUS : result.status,
13841
+ error: waiting ? "The step that created this waitpoint no longer waits on it" : result.error ?? "The run ended without resuming on it"
13842
+ });
13843
+ if (!held) {
13844
+ result = await this.endUnheldWait(store, runId, result);
13845
+ waiting = false;
13846
+ }
13847
+ } catch (error) {
13848
+ this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
13849
+ }
13850
+ if (waiting) {
13851
+ const toSend = result.waiting && waitpoints?.responseFor(result.waiting.waitpointId);
13852
+ if (toSend) respond?.(toSend);
13853
+ await this.deliverHeld(runId);
13854
+ }
13855
+ }
13856
+ return {
13857
+ result,
13858
+ handed
13859
+ };
13860
+ }
13861
+ async endUnheldWait(store, runId, result) {
13862
+ const on = result.waiting;
13863
+ const step = result.steps.find((record) => record.stepId === on?.stepId);
13864
+ const row = on ? await store.getWaitpoint(on.waitpointId) : void 0;
13865
+ const end = row?.run_id === runId && row.step_id === on?.stepId && row.status === "EXPIRED" ? {
13866
+ status: "CANCELLED",
13867
+ reason: `The wait at step "${row.step_key}" expired at ${row.expires_at} before the run could wait on it`,
13868
+ errorName: WAITPOINT_EXPIRED_ERROR_NAME
13869
+ } : {
13870
+ status: "FAILED",
13871
+ reason: `Step "${step?.key ?? on?.stepId}" waited on waitpoint "${on?.waitpointId}", which is not a live waitpoint it created`,
13872
+ errorName: WAITPOINT_NOT_LIVE_ERROR_NAME
13873
+ };
13874
+ const closed = await store.endWait(runId, {
13875
+ ...end,
13876
+ waitpoints: "EXPIRED"
13877
+ });
13878
+ this.answerClosed(closed, runId, {
13879
+ status: end.status,
13880
+ error: end.reason
13881
+ });
13882
+ return {
13883
+ status: end.status,
13884
+ error: end.reason,
13885
+ errorName: end.errorName,
13886
+ steps: result.steps.map((record) => record === step ? {
13887
+ ...record,
13888
+ status: "FAILED",
13889
+ error: end.reason,
13890
+ errorName: end.errorName
13891
+ } : record)
13892
+ };
13893
+ }
13185
13894
  runGate = new WorkflowRunGate();
13186
13895
  unsettledFirings = /* @__PURE__ */ new Map();
13187
13896
  runLimits = /* @__PURE__ */ new Map();
@@ -13206,6 +13915,9 @@ var WorkflowRuntimeService = class {
13206
13915
  else this.unsettledFirings.delete(workflowId);
13207
13916
  }
13208
13917
  heldReplies = /* @__PURE__ */ new Map();
13918
+ runResponders = /* @__PURE__ */ new Map();
13919
+ queuedReplies = /* @__PURE__ */ new Map();
13920
+ syncResumes = /* @__PURE__ */ new Map();
13209
13921
  /** Why a trigger's firing of a PARKED workflow is refused; undefined when
13210
13922
  * it is not parked, or when an operator started the run. */
13211
13923
  async parkedFiring(workflowId, triggerKind, publishedVersion) {
@@ -13273,12 +13985,23 @@ var WorkflowRuntimeService = class {
13273
13985
  this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
13274
13986
  let runId = enqueuedRunId ?? null;
13275
13987
  if (store) try {
13276
- if (runId) await store.beginRun(runId, details);
13988
+ let adopted = true;
13989
+ if (runId) adopted = await store.beginRun(runId, details);
13277
13990
  else runId = await store.startRun({
13278
13991
  workflowId,
13279
13992
  ...details
13280
13993
  });
13281
- await store.cancelRun(runId, details.reason);
13994
+ if (adopted) await store.cancelRun(runId, details.reason);
13995
+ else {
13996
+ const row = await store.getRun(runId);
13997
+ if (row) return {
13998
+ status: row.status,
13999
+ steps: [],
14000
+ runId,
14001
+ ...row.error ? { error: row.error } : {},
14002
+ refusal: row.status === "CANCELLED" ? "cancelled" : refusal
14003
+ };
14004
+ }
13282
14005
  } catch (error) {
13283
14006
  this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
13284
14007
  }
@@ -13374,6 +14097,677 @@ var WorkflowRuntimeService = class {
13374
14097
  const parsed = resolvedBlock(await this.resolveBlock(stepBlock(step)));
13375
14098
  return parsed ? await this.declaredReactor(parsed) === "read" : false;
13376
14099
  }
14100
+ resuming = /* @__PURE__ */ new Set();
14101
+ definitionHashes = /* @__PURE__ */ new Map();
14102
+ /** Our hash of the runnable definition: computed once per published version. */
14103
+ definitionHash(workflowId, state) {
14104
+ const runnable = runnableDefinition(state);
14105
+ if (!runnable.published) return hashOf(canonicalJson(runnable));
14106
+ const key = `${workflowId}@${runnable.version}`;
14107
+ let hash = this.definitionHashes.get(key);
14108
+ if (hash === void 0) {
14109
+ hash = hashOf(canonicalJson(runnable));
14110
+ if (this.definitionHashes.size >= WORKFLOW_CACHE_MAX) this.definitionHashes.clear();
14111
+ this.definitionHashes.set(key, hash);
14112
+ }
14113
+ return hash;
14114
+ }
14115
+ waitpointPort(store, workflowId, runId, suspendMs) {
14116
+ const responses = /* @__PURE__ */ new Map();
14117
+ const tokens = /* @__PURE__ */ new Map();
14118
+ const configured = this.host.waitpointBaseUrl?.replace(/\/+$/, "");
14119
+ const baseUrl = configured && URL.canParse(configured) ? configured : void 0;
14120
+ return {
14121
+ port: {
14122
+ ...baseUrl ? { resumeBaseUrl: baseUrl } : {},
14123
+ create: async (request) => {
14124
+ if (!baseUrl && request.type === "WEBHOOK" && !request.approval) throw new Error("This host serves no resume URLs: resume URLs need an HTTP host");
14125
+ const createdAt = /* @__PURE__ */ new Date();
14126
+ const resumeAt = request.resumeDateTime ? new Date(Date.parse(request.resumeDateTime)).toISOString() : null;
14127
+ if (resumeAt && Date.parse(resumeAt) > MAX_WAITPOINT_MS) throw new Error(`A delay until ${resumeAt} is past ${MAX_WAITPOINT_ISO}, the latest a run can wait until`);
14128
+ const expiresAt = waitpointExpiry(createdAt, request, suspendMs);
14129
+ if (request.type === "DELAY" && resumeAt && suspendMs !== null) {
14130
+ if (Date.parse(resumeAt) - createdAt.getTime() > suspendMs) throw new Error(`A delay until ${resumeAt} is longer than the workflow's maxSuspensionDays allows`);
14131
+ }
14132
+ const token = baseUrl && !request.approval && request.type !== "DELAY" ? request.token ?? randomBytes(32).toString("base64url") : null;
14133
+ const id = randomUUID();
14134
+ const { waitpoint, created: inserted } = await store.createWaitpoint({
14135
+ id,
14136
+ run_id: runId,
14137
+ workflow_id: workflowId,
14138
+ step_id: request.stepId,
14139
+ step_key: request.stepKey,
14140
+ step_path: request.stepId,
14141
+ piece_name: request.pieceName,
14142
+ block_name: request.blockName,
14143
+ type: request.type,
14144
+ kind: request.approval ? "approval" : request.type === "DELAY" ? "delay" : "webhook",
14145
+ status: "PENDING",
14146
+ token_hash: token ? tokenHash(token) : null,
14147
+ created_at: createdAt.toISOString(),
14148
+ resume_at: resumeAt,
14149
+ expires_at: request.type === "DELAY" ? null : expiresAt,
14150
+ resolved_at: null,
14151
+ decided_by: null,
14152
+ approval: request.approval ? JSON.stringify(request.approval) : null,
14153
+ resume_payload: null,
14154
+ response_to_send: request.responseToSend ? JSON.stringify(request.responseToSend) : null
14155
+ });
14156
+ if (!inserted) {
14157
+ const known = tokens.get(waitpoint.id);
14158
+ if (waitpoint.token_hash !== null && known === void 0) throw new Error(`Step "${request.stepKey}" has a waitpoint from an earlier leg, whose resume URL cannot be issued again`);
14159
+ return {
14160
+ id: waitpoint.id,
14161
+ resumeUrl: known ? `${baseUrl}/${known}` : ""
14162
+ };
14163
+ }
14164
+ if (token) tokens.set(id, token);
14165
+ if (request.responseToSend) responses.set(id, request.responseToSend);
14166
+ const due = request.type === "DELAY" ? resumeAt : expiresAt;
14167
+ if (due) this.waitpointPoller.nudge(due);
14168
+ return {
14169
+ id,
14170
+ resumeUrl: token ? `${baseUrl}/${token}` : ""
14171
+ };
14172
+ }
14173
+ },
14174
+ responseFor: (waitpointId) => responses.get(waitpointId)
14175
+ };
14176
+ }
14177
+ /** Re-enters a WAITING run that holds a COMPLETED waitpoint, through the
14178
+ * dispatcher. Whoever writes second, the resume or the WAITING, delivers. */
14179
+ async deliverHeld(runId) {
14180
+ if (this.closed) return this.answerShutDown(runId);
14181
+ if (this.resuming.has(runId)) return;
14182
+ try {
14183
+ const store = await this.store();
14184
+ if (!store || !await store.hasCompletedWaitpoint(runId)) return;
14185
+ const run = await store.getRun(runId);
14186
+ if (run?.status !== "WAITING") return this.answerNotWaiting(store, runId, run?.status);
14187
+ if (this.resuming.has(runId)) return;
14188
+ this.resuming.add(runId);
14189
+ this.resumeRun(runId, run.workflow_id);
14190
+ } catch (error) {
14191
+ this.logger.warn(`Could not resume run ${runId}: @error`, error);
14192
+ }
14193
+ }
14194
+ answerNotWaiting(store, runId, status) {
14195
+ if (store.isRunInFlight(runId)) return;
14196
+ this.answerUnresumed(runId, {
14197
+ status: status ?? "gone",
14198
+ error: "The run is not waiting to resume in this runtime",
14199
+ code: 409
14200
+ });
14201
+ }
14202
+ waitpointPoller = new WaitpointPoller({
14203
+ store: () => this.store(),
14204
+ resume: (runId) => this.deliverHeld(runId),
14205
+ ended: (end) => Promise.resolve(this.answerClosed(end.closed, end.runId, {
14206
+ status: CANCELLED_RUN_STATUS,
14207
+ error: end.reason
14208
+ })),
14209
+ logger: { warn: (message, ...args) => this.logger.warn(message, ...args) }
14210
+ });
14211
+ unobserveSuspended;
14212
+ /** Test hook: one waitpoint poll pass now; a no-op until the supervisor starts. */
14213
+ pollWaitpointsNow() {
14214
+ return this.waitpointPoller.poll();
14215
+ }
14216
+ startWaitpointPoller() {
14217
+ if (this.closed) return;
14218
+ this.unobserveSuspended ??= this.telemetry.observeSuspended(() => this.waitpointPoller.suspendedRuns());
14219
+ this.waitpointPoller.start();
14220
+ }
14221
+ stopWaitpointPoller() {
14222
+ this.waitpointPoller.stop();
14223
+ this.unobserveSuspended?.();
14224
+ this.unobserveSuspended = void 0;
14225
+ }
14226
+ async answerHeldResumes(store, runId, error) {
14227
+ if (this.syncResumes.size === 0) return;
14228
+ const rows = (await store.listWaitpointsForRuns([runId])).get(runId) ?? [];
14229
+ this.answerClosed(rows.map((row) => row.id), runId, {
14230
+ status: CANCELLED_RUN_STATUS,
14231
+ error
14232
+ });
14233
+ }
14234
+ answerClosed(waitpointIds, runId, end) {
14235
+ if (this.syncResumes.size === 0) return;
14236
+ for (const id of waitpointIds) this.heldResume(id)?.closed(runId, end.status, end.error);
14237
+ }
14238
+ heldResume(waitpointId) {
14239
+ const candidates = [...this.syncResumes.get(waitpointId) ?? []];
14240
+ if (candidates.length === 0) return void 0;
14241
+ const winner = (async () => {
14242
+ for (const candidate of candidates) if (await candidate.won) return candidate.entry;
14243
+ })();
14244
+ return {
14245
+ respond: (response) => void winner.then((e) => e?.respond(response)),
14246
+ ended: (result) => void winner.then((e) => e?.ended(result)),
14247
+ closed: (runId, status, error, code) => void winner.then((e) => e?.closed(runId, status, error, code))
14248
+ };
14249
+ }
14250
+ answerUnresumed(runId, end) {
14251
+ for (const [id, candidates] of this.syncResumes) {
14252
+ if (![...candidates].some((candidate) => candidate.runId === runId)) continue;
14253
+ this.heldResume(id)?.closed(runId, end.status, end.error, end.code);
14254
+ }
14255
+ }
14256
+ async deliverAllHeld() {
14257
+ try {
14258
+ const store = await this.store();
14259
+ for (const runId of await store?.heldRunIds() ?? []) await this.deliverHeld(runId);
14260
+ } catch (error) {
14261
+ this.logger.warn("Could not deliver held resumes: @error", error);
14262
+ }
14263
+ }
14264
+ resumeRun(runId, workflowId) {
14265
+ const started = performance.now();
14266
+ const attributes = {
14267
+ "workflow.id": workflowId,
14268
+ "trigger.kind": "resume",
14269
+ "run.id": runId
14270
+ };
14271
+ return this.telemetry.span("workflow.run", attributes, async (span) => {
14272
+ let status = "REFUSED";
14273
+ try {
14274
+ const result = await this.runResumed(runId, workflowId);
14275
+ if (result) status = result.status;
14276
+ if (status === "FAILED") span.setStatus({
14277
+ code: SpanStatusCode.ERROR,
14278
+ message: status
14279
+ });
14280
+ return result;
14281
+ } finally {
14282
+ span.setAttribute("run.status", status);
14283
+ this.telemetry.recordRun(performance.now() - started, status, "resume");
14284
+ }
14285
+ });
14286
+ }
14287
+ async runResumed(runId, workflowId) {
14288
+ const admission = await this.runGate.admit(workflowId, await this.resumePolicy(workflowId), void 0, { resume: true });
14289
+ if (!admission.admitted) {
14290
+ this.resuming.delete(runId);
14291
+ this.answerShutDown(runId);
14292
+ return;
14293
+ }
14294
+ let dispatched;
14295
+ try {
14296
+ dispatched = await this.dispatcher().admit(workflowId);
14297
+ } catch {
14298
+ admission.release();
14299
+ this.resuming.delete(runId);
14300
+ this.answerShutDown(runId);
14301
+ return;
14302
+ }
14303
+ const store = await this.store();
14304
+ let claimed = false;
14305
+ let redeliver = false;
14306
+ let session;
14307
+ let held;
14308
+ let outcome;
14309
+ try {
14310
+ if (!store || this.closed) {
14311
+ this.answerShutDown(runId);
14312
+ return;
14313
+ }
14314
+ claimed = await store.claimWaitingRun(runId);
14315
+ this.resuming.delete(runId);
14316
+ if (!claimed) {
14317
+ this.answerNotWaiting(store, runId, (await store.getRun(runId))?.status);
14318
+ return;
14319
+ }
14320
+ const row = await store.getRun(runId);
14321
+ if (!row) throw new Error(`Run "${runId}" not found`);
14322
+ const generation = this.workflowGeneration(workflowId);
14323
+ const document = await this.host.reactorClient.get(workflowId);
14324
+ if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
14325
+ const current = document.state.global;
14326
+ const stopped = await this.resumeStopped(workflowId, row.trigger_kind, current);
14327
+ if (stopped) {
14328
+ const closed = await store.endWait(runId, {
14329
+ status: CANCELLED_RUN_STATUS,
14330
+ reason: stopped.reason,
14331
+ errorName: stopped.errorName,
14332
+ waitpoints: "CANCELLED"
14333
+ });
14334
+ claimed = false;
14335
+ this.answerClosed(closed, runId, {
14336
+ status: CANCELLED_RUN_STATUS,
14337
+ error: stopped.reason
14338
+ });
14339
+ return {
14340
+ status: CANCELLED_RUN_STATUS,
14341
+ steps: [],
14342
+ runId,
14343
+ error: stopped.reason,
14344
+ errorName: stopped.errorName,
14345
+ refusal: "cancelled"
14346
+ };
14347
+ }
14348
+ const consumed = await store.consumeCompletedWaitpoint(runId);
14349
+ if (!consumed) {
14350
+ await store.returnToWaiting(runId);
14351
+ claimed = false;
14352
+ redeliver = true;
14353
+ return;
14354
+ }
14355
+ const { waitpoint } = consumed;
14356
+ held = this.heldResume(waitpoint.id);
14357
+ this.answerClosed(consumed.closed, runId, {
14358
+ status: "RUNNING",
14359
+ error: "The run resumed on another of its waitpoints"
14360
+ });
14361
+ const state = await this.pinnedState(workflowId, runId, store, current);
14362
+ const runnable = runnableDefinition(state);
14363
+ const definition = toWorkflowDefinition(state);
14364
+ const policy = effectiveRunPolicy(runnable);
14365
+ const connections = declaredConnectionIds(definition);
14366
+ const reactorConnections = declaredReactorConnectionIds(definition);
14367
+ const runUser = reactorConnections.size === 0 ? void 0 : await assertReactorConnectionsReadable(this.host, workflowId, reactorConnections);
14368
+ const journal = carriedFromJournal(await store.getSteps(runId));
14369
+ if (!journal.waitingStepId) throw new Error(`Run "${runId}" has no waiting step to resume`);
14370
+ session = this.workers().session();
14371
+ const { result } = await this.executeLeg({
14372
+ workflowId,
14373
+ runId,
14374
+ store,
14375
+ session,
14376
+ definition,
14377
+ runnable,
14378
+ policy,
14379
+ deadline: policy.runTimeoutSeconds ? Date.now() + policy.runTimeoutSeconds * 1e3 : void 0,
14380
+ connections,
14381
+ runUser,
14382
+ triggerPayload: journaledTriggerPayload(row.trigger_payload),
14383
+ executor: this.blockExecutor(store),
14384
+ carriedSteps: journal.carried,
14385
+ resume: {
14386
+ stepId: journal.waitingStepId,
14387
+ payload: parseJson(waitpoint.resume_payload) ?? {}
14388
+ },
14389
+ firstOrdinal: journal.nextOrdinal,
14390
+ written: journal.attachmentRefs,
14391
+ handed: await store.getRunDocuments(runId),
14392
+ ...held ? { respond: held.respond } : {}
14393
+ });
14394
+ if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(row.trigger_kind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
14395
+ outcome = {
14396
+ ...result,
14397
+ runId
14398
+ };
14399
+ return outcome;
14400
+ } catch (error) {
14401
+ if (store && claimed) {
14402
+ const message = error instanceof Error ? error.message : String(error);
14403
+ const errorName = errorNameOf(error) ?? "Error";
14404
+ const closed = await store.endWait(runId, {
14405
+ status: "FAILED",
14406
+ reason: message,
14407
+ errorName,
14408
+ waitpoints: "CANCELLED"
14409
+ }).catch(() => []);
14410
+ this.answerClosed(closed, runId, {
14411
+ status: "FAILED",
14412
+ error: message
14413
+ });
14414
+ outcome = {
14415
+ status: "FAILED",
14416
+ steps: [],
14417
+ runId,
14418
+ error: message,
14419
+ errorName
14420
+ };
14421
+ }
14422
+ this.logger.warn(`Resuming run ${runId} failed: @error`, error);
14423
+ return;
14424
+ } finally {
14425
+ this.resuming.delete(runId);
14426
+ session?.close();
14427
+ dispatched();
14428
+ admission.release();
14429
+ held?.ended(outcome);
14430
+ if (redeliver) this.deliverHeld(runId);
14431
+ }
14432
+ }
14433
+ answerShutDown(runId) {
14434
+ if (!this.closed) return;
14435
+ this.answerUnresumed(runId, {
14436
+ status: WAITING_RUN_STATUS,
14437
+ error: "The runtime shut down before the run could resume",
14438
+ code: 503
14439
+ });
14440
+ }
14441
+ async resumePolicy(workflowId) {
14442
+ try {
14443
+ let state = this.workflowCache.get(workflowId)?.state;
14444
+ if (!state) {
14445
+ const generation = this.workflowGeneration(workflowId);
14446
+ const document = await this.host.reactorClient.get(workflowId);
14447
+ if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
14448
+ state = document.state.global;
14449
+ }
14450
+ return effectiveRunPolicy(runnableDefinition(state));
14451
+ } catch {
14452
+ return UNENFORCED_POLICY;
14453
+ }
14454
+ }
14455
+ async resumeStopped(workflowId, triggerKind, current) {
14456
+ if (current.status !== "ENABLED") return {
14457
+ reason: `The workflow is ${current.status}; a run waiting on it does not resume`,
14458
+ errorName: WORKFLOW_DISABLED_ERROR_NAME
14459
+ };
14460
+ const parked = await this.parkedFiring(workflowId, triggerKind, runnableDefinition(current).version);
14461
+ return parked ? {
14462
+ reason: parked,
14463
+ errorName: WORKFLOW_PARKED_ERROR_NAME
14464
+ } : void 0;
14465
+ }
14466
+ async pinnedState(workflowId, runId, store, current) {
14467
+ const pin = await store.getRunPin(runId);
14468
+ if (!pin?.definition_hash) return current;
14469
+ if (this.definitionHash(workflowId, current) === pin.definition_hash) return current;
14470
+ return this.rebuildWorkflowState(workflowId, pin);
14471
+ }
14472
+ async rebuildWorkflowState(workflowId, pin) {
14473
+ const revision = pin.definition_op_index ?? 0;
14474
+ const operations = [];
14475
+ let page = await this.host.reactorClient.getOperations(workflowId, { scopes: ["document", "global"] });
14476
+ operations.push(...page.results);
14477
+ while (page.next) {
14478
+ page = await page.next();
14479
+ operations.push(...page.results);
14480
+ }
14481
+ const applied = (operation) => !operation.error && !isDenied(operation);
14482
+ const past = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index >= revision)).at(0);
14483
+ const atOrBeforePin = (operation) => {
14484
+ const at = operation.action.input.revision?.global;
14485
+ if (at !== void 0) return at <= revision;
14486
+ return !past || operation.timestampUtcMs <= past.timestampUtcMs;
14487
+ };
14488
+ const seed = sortOperations(operations.filter((operation) => operation.action.scope === "document" && operation.action.type === "UPGRADE_DOCUMENT" && applied(operation) && atOrBeforePin(operation))).map((operation) => operation.action.input.initialState).filter((state) => state?.global !== void 0).at(-1);
14489
+ let document = utils.createDocument(seed);
14490
+ const global = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index < revision));
14491
+ for (const operation of garbageCollect(global).filter(applied)) document = reducer(document, operation.action);
14492
+ const state = document.state.global;
14493
+ if (scopeHash(state) !== pin.definition_scope_hash || this.definitionHash(workflowId, state) !== pin.definition_hash) throw new Error(`The workflow was re-published while this run waited, and the definition it started from could not be rebuilt at revision ${revision}`);
14494
+ return state;
14495
+ }
14496
+ /** Resolves a WEBHOOK waitpoint from the editor, as the calling DID. */
14497
+ async resolveWaitpoint(waitpointId, input, ctx) {
14498
+ const store = await this.store();
14499
+ if (!store) throw new Error("Run journal is unavailable");
14500
+ const waitpoint = await store.getWaitpoint(waitpointId);
14501
+ if (!waitpoint) throw new Error(`Waitpoint "${waitpointId}" not found`);
14502
+ await this.assertCanReadDocument(waitpoint.workflow_id, ctx);
14503
+ await this.assertCanWriteDocument(waitpoint.workflow_id, ctx);
14504
+ const owner = await store.getRun(waitpoint.run_id);
14505
+ if (!owner || (await this.servedRuns([owner], ctx)).length === 0) throw new Error(`Waitpoint "${waitpointId}" not found`);
14506
+ if (waitpoint.type !== "WEBHOOK") throw new Error(`Waitpoint "${waitpointId}" is a ${waitpoint.type}`);
14507
+ const decidedBy = this.callerDid(ctx);
14508
+ const approval = parseJson(waitpoint.approval);
14509
+ if (approval && approval.approvers.length > 0) {
14510
+ if (!(decidedBy !== void 0 && approval.approvers.some((approver) => sameApprover(approver, decidedBy)))) throw new Error("You are not one of this approval's approvers");
14511
+ }
14512
+ const reason = input.reason?.trim() || void 0;
14513
+ if ((approval?.reasonRequiredOn === "both" || approval?.reasonRequiredOn === "reject" && !input.approved) && !reason) throw new Error("A reason is required for this decision");
14514
+ const decidedAt = (/* @__PURE__ */ new Date()).toISOString();
14515
+ const payload = {
14516
+ queryParams: {
14517
+ action: input.approved ? "approve" : "disapprove",
14518
+ ...reason ? { reason } : {}
14519
+ },
14520
+ body: {
14521
+ approved: input.approved,
14522
+ reason: reason ?? null,
14523
+ decidedBy: decidedBy ?? null,
14524
+ decidedAt
14525
+ },
14526
+ headers: {}
14527
+ };
14528
+ const completed = await store.completeWaitpoint(waitpointId, {
14529
+ payload,
14530
+ decidedBy: decidedBy ?? null,
14531
+ resolvedAt: decidedAt
14532
+ });
14533
+ if (!completed) {
14534
+ const now = await store.getWaitpoint(waitpointId);
14535
+ const run = await store.getRun(waitpoint.run_id);
14536
+ const live = run?.status === "RUNNING" || run?.status === "WAITING";
14537
+ const status = now?.status !== "PENDING" ? now?.status ?? "gone" : waitpointExpired(now) ? "EXPIRED" : live ? "not the one its run waits on" : `on a ${run?.status ?? "deleted"} run`;
14538
+ throw new Error(`Waitpoint "${waitpointId}" is ${status}; it can no longer be resolved`);
14539
+ }
14540
+ await this.deliverHeld(completed.run_id);
14541
+ return waitpointRecord(completed);
14542
+ }
14543
+ /** A POST to a resume URL. Single-use: only a PENDING waitpoint resumes. */
14544
+ resumeWaitpointByToken(token, payload) {
14545
+ return this.resumeByToken(token, payload);
14546
+ }
14547
+ /** A POST to a resume URL's `/sync`, held like a sync webhook until the
14548
+ * resumed leg responds, waits, ends, or times out. The first answer wins. */
14549
+ async resumeWaitpointByTokenSync(token, payload, options = {}) {
14550
+ let hold;
14551
+ try {
14552
+ const { status } = await this.resumeByToken(token, payload, (id, runId) => {
14553
+ hold = this.holdSyncResume(id, runId);
14554
+ return hold.decided;
14555
+ });
14556
+ if (status !== "resumed" || !hold) return { status };
14557
+ return {
14558
+ status,
14559
+ reply: await hold.reply(options.timeoutMs ?? DELIVERY_TIMEOUT_MS)
14560
+ };
14561
+ } finally {
14562
+ hold?.release();
14563
+ }
14564
+ }
14565
+ holdSyncResume(waitpointId, runId) {
14566
+ let settle = () => void 0;
14567
+ const answer = new Promise((resolve) => {
14568
+ settle = resolve;
14569
+ });
14570
+ let done = false;
14571
+ const finish = (reply) => {
14572
+ if (done) return;
14573
+ done = true;
14574
+ settle(reply);
14575
+ };
14576
+ const entry = {
14577
+ respond: (response) => finish(runResponseReply(response)),
14578
+ ended: (result) => finish(result ? runResultReply(result, 200) : {
14579
+ status: 500,
14580
+ contentType: JSON_CONTENT_TYPE,
14581
+ body: JSON.stringify({ error: "The run could not resume" })
14582
+ }),
14583
+ closed: (runId, status, error, code = 409) => finish({
14584
+ status: code,
14585
+ contentType: JSON_CONTENT_TYPE,
14586
+ body: JSON.stringify({
14587
+ runId,
14588
+ status,
14589
+ error
14590
+ })
14591
+ })
14592
+ };
14593
+ let decided = () => void 0;
14594
+ const candidate = {
14595
+ entry,
14596
+ runId,
14597
+ won: new Promise((resolve) => {
14598
+ decided = resolve;
14599
+ })
14600
+ };
14601
+ const held = this.syncResumes.get(waitpointId) ?? /* @__PURE__ */ new Set();
14602
+ this.syncResumes.set(waitpointId, held);
14603
+ held.add(candidate);
14604
+ return {
14605
+ decided,
14606
+ reply: async (timeoutMs) => {
14607
+ const raced = await Promise.race([answer, timeout(timeoutMs)]);
14608
+ if (raced !== TIMED_OUT) return raced;
14609
+ done = true;
14610
+ return {
14611
+ status: 504,
14612
+ contentType: JSON_CONTENT_TYPE,
14613
+ body: JSON.stringify({
14614
+ status: "RUNNING",
14615
+ error: "The run did not finish in time"
14616
+ })
14617
+ };
14618
+ },
14619
+ release: () => {
14620
+ decided(false);
14621
+ held.delete(candidate);
14622
+ if (held.size === 0 && this.syncResumes.get(waitpointId) === held) this.syncResumes.delete(waitpointId);
14623
+ }
14624
+ };
14625
+ }
14626
+ async resumeByToken(token, payload, beforeComplete) {
14627
+ const store = await this.store();
14628
+ if (!store || !token) return { status: "gone" };
14629
+ const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
14630
+ if (!waitpoint || waitpoint.kind !== "webhook") return { status: "gone" };
14631
+ let completed;
14632
+ if (waitpoint.status === "PENDING") {
14633
+ const decided = beforeComplete?.(waitpoint.id, waitpoint.run_id);
14634
+ try {
14635
+ completed = await store.completeWaitpoint(waitpoint.id, { payload });
14636
+ } finally {
14637
+ decided?.(completed !== void 0);
14638
+ }
14639
+ }
14640
+ if (!completed) {
14641
+ const now = (await store.getWaitpoint(waitpoint.id))?.status;
14642
+ if (now === "CONSUMED") return { status: "already" };
14643
+ const run = now === "COMPLETED" ? await store.getRun(waitpoint.run_id) : void 0;
14644
+ return { status: run?.status === "RUNNING" || run?.status === "WAITING" ? "already" : "gone" };
14645
+ }
14646
+ await this.deliverHeld(completed.run_id);
14647
+ return { status: "resumed" };
14648
+ }
14649
+ /** What a resume URL's confirm page shows. */
14650
+ async describeWaitpointByToken(token) {
14651
+ const store = await this.store();
14652
+ if (!store || !token) return void 0;
14653
+ const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
14654
+ if (!waitpoint || waitpoint.kind !== "webhook") return void 0;
14655
+ const run = await store.getRun(waitpoint.run_id);
14656
+ const status = !(run?.status === "RUNNING" || run?.status === "WAITING") && (waitpoint.status === "PENDING" || waitpoint.status === "COMPLETED") ? "CANCELLED" : waitpoint.status;
14657
+ const state = await this.workflowStateForPage(waitpoint.workflow_id);
14658
+ const step = state?.steps.find((entry) => entry.id === waitpoint.step_id);
14659
+ const action = step ? await this.blockDisplayName({
14660
+ pieceName: step.pieceName,
14661
+ pieceVersion: step.pieceVersion,
14662
+ kind: "action",
14663
+ name: step.actionName
14664
+ }) : void 0;
14665
+ const trigger = state?.trigger ? await this.blockDisplayName({
14666
+ pieceName: state.trigger.pieceName,
14667
+ pieceVersion: state.trigger.pieceVersion,
14668
+ kind: "trigger",
14669
+ name: state.trigger.triggerName
14670
+ }) ?? state.trigger.triggerName : void 0;
14671
+ return {
14672
+ workflowName: run?.workflow_name || state?.name || "",
14673
+ stepName: step?.name || waitpoint.step_key,
14674
+ status,
14675
+ ...state?.description ? { workflowDescription: state.description } : {},
14676
+ ...action ? { actionName: action } : {},
14677
+ waitingSince: waitpoint.created_at,
14678
+ ...waitpoint.expires_at ? { expiresAt: waitpoint.expires_at } : {},
14679
+ ...run ? { startedAt: run.enqueued_at } : {},
14680
+ ...trigger ? { triggerName: trigger } : {}
14681
+ };
14682
+ }
14683
+ async workflowStateForPage(workflowId) {
14684
+ const cached = this.workflowCache.get(workflowId)?.state;
14685
+ if (cached) return cached;
14686
+ try {
14687
+ const generation = this.workflowGeneration(workflowId);
14688
+ const document = await this.host.reactorClient.get(workflowId);
14689
+ if (document.header.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
14690
+ this.cacheLoaded(workflowId, generation, document);
14691
+ return document.state.global;
14692
+ } catch {
14693
+ return;
14694
+ }
14695
+ }
14696
+ async blockDisplayName(block) {
14697
+ try {
14698
+ const parsed = resolvedBlock(await this.resolveBlock(block));
14699
+ if (!parsed) return void 0;
14700
+ const descriptor = await this.pieceDescriptor(targetOf(parsed));
14701
+ return (block.kind === "trigger" ? descriptor.triggers : descriptor.actions).find((entry) => entry.name === parsed.name)?.displayName;
14702
+ } catch {
14703
+ return;
14704
+ }
14705
+ }
14706
+ /** The waitpoints of one run, for a caller who may read the run. */
14707
+ async waitpointsForRun(runId, ctx) {
14708
+ const store = await this.store();
14709
+ if (!store || !await this.run(runId, ctx)) return [];
14710
+ return ((await store.listWaitpointsForRuns([runId])).get(runId) ?? []).map(waitpointRecord);
14711
+ }
14712
+ /** WAITING runs this caller may read, oldest first, with their waitpoints. */
14713
+ async waitingRuns(args, ctx) {
14714
+ const store = await this.store();
14715
+ if (!store || !ctx) return [];
14716
+ let scope;
14717
+ if (args.workflowId) {
14718
+ await this.assertCanReadDocument(args.workflowId, ctx);
14719
+ scope = args.workflowId;
14720
+ } else if (args.driveId) {
14721
+ scope = await this.driveWorkflowIds(args.driveId, ctx);
14722
+ if (scope.length === 0) return [];
14723
+ }
14724
+ const rows = await this.servedRuns(await this.readableRows(await store.listWaitingRuns(scope, args.first ?? 50), (row) => row.workflow_id, ctx), ctx);
14725
+ const ids = rows.map((row) => row.id);
14726
+ const steps = await store.getStepsForRuns(ids, { withData: false });
14727
+ const waitpoints = await store.listWaitpointsForRuns(ids);
14728
+ return rows.map((row) => ({
14729
+ run: {
14730
+ row,
14731
+ steps: steps.get(row.id) ?? []
14732
+ },
14733
+ waitpoints: (waitpoints.get(row.id) ?? []).map(waitpointRecord)
14734
+ }));
14735
+ }
14736
+ /** Cancels a WAITING or PENDING run; its waitpoints stop resuming. */
14737
+ async cancelRun(runId, ctx) {
14738
+ const store = await this.store();
14739
+ if (!store) throw new Error("Run journal is unavailable");
14740
+ const run = await store.getRun(runId);
14741
+ if (!run) throw new Error(`Run "${runId}" not found`);
14742
+ await this.assertCanReadDocument(run.workflow_id, ctx);
14743
+ await this.assertCanWriteDocument(run.workflow_id, ctx);
14744
+ if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
14745
+ const refuse = (status) => status === "RUNNING" ? /* @__PURE__ */ new Error("Cancelling a RUNNING run is not supported yet (#3226)") : /* @__PURE__ */ new Error(`Run is ${status}; only WAITING and PENDING runs can be cancelled`);
14746
+ if (run.status !== "WAITING" && run.status !== "PENDING") throw refuse(run.status);
14747
+ const by = this.callerDid(ctx);
14748
+ const reason = by ? `Cancelled by ${by}` : "Cancelled";
14749
+ if (!await store.cancelIdleRun(runId, reason)) throw refuse((await store.getRun(runId))?.status ?? "gone");
14750
+ await this.answerHeldResumes(store, runId, reason);
14751
+ const row = await store.getRun(runId);
14752
+ this.queuedReplies.get(runId)?.({
14753
+ status: row.status,
14754
+ steps: [],
14755
+ runId,
14756
+ ...row.error ? { error: row.error } : {},
14757
+ refusal: "cancelled"
14758
+ });
14759
+ return {
14760
+ row,
14761
+ steps: await store.getSteps(runId)
14762
+ };
14763
+ }
14764
+ callerDid(ctx) {
14765
+ if (!ctx) return void 0;
14766
+ const subject = this.host.subjectOf?.(ctx);
14767
+ if (subject?.address) return subject.address;
14768
+ const user = ctx.user;
14769
+ return typeof user?.address === "string" ? user.address : void 0;
14770
+ }
13377
14771
  async testStep(workflowId, stepId, ctx, access = {}) {
13378
14772
  await this.assertCanReadWorkflow(workflowId, ctx, access.driveId);
13379
14773
  const document = await this.host.reactorClient.get(workflowId);
@@ -13682,6 +15076,112 @@ var WorkflowRuntimeService = class {
13682
15076
  return (await this.host.reactorClient.get(workflowId)).state.global;
13683
15077
  }
13684
15078
  };
15079
+ function scopeHash(state) {
15080
+ return hashDocumentStateForScope({ state: { global: state } }, "global");
15081
+ }
15082
+ function tokenHash(token) {
15083
+ return createHash("sha256").update(token).digest("hex");
15084
+ }
15085
+ function waitpointExpired(waitpoint) {
15086
+ return waitpoint.expires_at !== null && Date.parse(waitpoint.expires_at) <= Date.now();
15087
+ }
15088
+ function parseJson(value) {
15089
+ return value === null ? null : JSON.parse(value);
15090
+ }
15091
+ const WORKFLOW_DISABLED_ERROR_NAME = "WorkflowDisabled";
15092
+ const WORKFLOW_PARKED_ERROR_NAME = "WorkflowParked";
15093
+ const MAX_WAITPOINT_ISO = "9999-12-31T23:59:59.999Z";
15094
+ const MAX_WAITPOINT_MS = Date.parse(MAX_WAITPOINT_ISO);
15095
+ function waitpointExpiry(createdAt, request, suspendMs) {
15096
+ const candidates = [request.expiresAt ? Date.parse(request.expiresAt) : NaN, suspendMs === null ? NaN : createdAt.getTime() + suspendMs].filter((at) => !Number.isNaN(at));
15097
+ return candidates.length > 0 ? new Date(Math.min(...candidates, MAX_WAITPOINT_MS)).toISOString() : null;
15098
+ }
15099
+ function sameApprover(approver, caller) {
15100
+ const tail = (value) => value.split(":").pop().toLowerCase();
15101
+ return approver.toLowerCase() === caller.toLowerCase() || tail(approver) === tail(caller);
15102
+ }
15103
+ function waitpointRecord(row) {
15104
+ return {
15105
+ id: row.id,
15106
+ runId: row.run_id,
15107
+ workflowId: row.workflow_id,
15108
+ stepId: row.step_id,
15109
+ stepKey: row.step_key,
15110
+ stepPath: row.step_path,
15111
+ pieceName: row.piece_name,
15112
+ blockName: row.block_name,
15113
+ type: row.type,
15114
+ kind: row.kind,
15115
+ status: row.status,
15116
+ createdAt: row.created_at,
15117
+ resumeAt: row.resume_at,
15118
+ expiresAt: row.expires_at,
15119
+ resolvedAt: row.resolved_at,
15120
+ decidedBy: row.decided_by,
15121
+ approval: parseJson(row.approval)
15122
+ };
15123
+ }
15124
+ function journaledValue(raw, what) {
15125
+ const value = raw === null ? void 0 : JSON.parse(raw);
15126
+ if (isTruncatedStepPayload(value)) return unavailableValue(`${what} is unavailable: the run journal truncated it past the payload cap`);
15127
+ return redactedLeavesUnavailable(value, what, []);
15128
+ }
15129
+ function redactedLeavesUnavailable(value, what, path) {
15130
+ if (!containsRedactedMarker(value)) return value;
15131
+ if (typeof value === "string") {
15132
+ const at = path.length > 0 ? ` at "${path.join(".")}"` : "";
15133
+ return unavailableValue(`${what} is unavailable: the run journal holds it redacted${at}`);
15134
+ }
15135
+ if (Array.isArray(value)) return value.map((entry, index) => redactedLeavesUnavailable(entry, what, [...path, index]));
15136
+ return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactedLeavesUnavailable(entry, what, [...path, key])]));
15137
+ }
15138
+ function journaledTriggerPayload(raw) {
15139
+ return journaledValue(raw, "The trigger payload");
15140
+ }
15141
+ const ATTACHMENT_REF_TEXT = /"(attachment:\/\/[^"\\]*)"/g;
15142
+ function carriedFromJournal(rows) {
15143
+ const carried = /* @__PURE__ */ new Map();
15144
+ const attachmentRefs = /* @__PURE__ */ new Set();
15145
+ let waitingStepId;
15146
+ let nextOrdinal = 0;
15147
+ for (const row of rows) {
15148
+ nextOrdinal = Math.max(nextOrdinal, row.ordinal + 1);
15149
+ for (const match of row.output?.matchAll(ATTACHMENT_REF_TEXT) ?? []) attachmentRefs.add(match[1]);
15150
+ const what = `Output of step "${row.step_key}"`;
15151
+ switch (row.status) {
15152
+ case "WAITING":
15153
+ waitingStepId = row.step_id;
15154
+ break;
15155
+ case "SUCCEEDED":
15156
+ case "REPLAYED":
15157
+ carried.set(row.step_id, {
15158
+ status: row.status,
15159
+ port: row.port,
15160
+ output: () => journaledValue(row.output, what)
15161
+ });
15162
+ break;
15163
+ case "FAILED":
15164
+ carried.set(row.step_id, {
15165
+ status: "FAILED",
15166
+ port: "error",
15167
+ output: () => void 0,
15168
+ error: row.error ?? ""
15169
+ });
15170
+ break;
15171
+ case "SKIPPED": if (row.port) carried.set(row.step_id, {
15172
+ status: "SKIPPED",
15173
+ port: row.port,
15174
+ output: () => null
15175
+ });
15176
+ }
15177
+ }
15178
+ return {
15179
+ carried,
15180
+ waitingStepId,
15181
+ nextOrdinal,
15182
+ attachmentRefs
15183
+ };
15184
+ }
13685
15185
  /** The runtime a host composes: one instance, its lifetime the host's. */
13686
15186
  function createWorkflowRuntime(deps) {
13687
15187
  return new WorkflowRuntimeService(deps);
@@ -13974,6 +15474,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
13974
15474
  }
13975
15475
  };
13976
15476
  //#endregion
13977
- export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName };
15477
+ export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WAITPOINT_EXPIRED_ERROR_NAME, WAITPOINT_NOT_LIVE_ERROR_NAME, WORKFLOW_DISABLED_ERROR_NAME, WORKFLOW_PACKAGE_NAME, WORKFLOW_PARKED_ERROR_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName, waitpointRecord };
13978
15478
 
13979
15479
  //# sourceMappingURL=index.js.map