@powerhousedao/reactor-workflow 6.2.3-dev.84 → 6.2.3-dev.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,9 +1,9 @@
1
- import { $ as redact, A as AttachmentCache, B as SecretDeletedError, C as blockLabel, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as stepBlock, H as SecretOwnerMismatchError, I as resolveStepInput, J as PieceWorker, K as triggerBlock, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as containsRedactedMarker, S as storeHandlers, St as isIndeterminateError, T as blockPorts, U as parseSecretRef, V as SecretNotFoundError, W as secretRefFromId, X as PieceWorkerError, Y as PieceWorkerDisposedError, Z as PieceWorkerTimeoutError, _ as effectiveRetryPolicy, a as PieceRegistry, at as parseAddress, b as isHostBound, bt as pieceRegistrySource, c as packagePieces, ct as maxFileBytes, d as runWorkflow, dt as sourcedResolver, et as redactMessage, f as canonicalJson, g as checkDynamicProperties, gt as unsupportedTrigger, h as DynamicPropertiesError, ht as unsupportedAuth, i as tracedSteps, it as isPrivateAddress, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as bundleResolver, m as stepConfigHash, mt as authMethodFor, n as WorkflowTelemetry, nt as secretsFor, o as installedPiece, ot as extractDedupeKey, p as hashOf, pt as UnsupportedPieceFeatureError, q as PieceWorkerPool, r as tracedMethods, rt as DEFAULT_EGRESS_POLICY, s as installedPieces, st as FileTooLargeError, t as TELEMETRY_SCOPE, tt as rememberSecrets, u as errorNameOf, ut as pieceModuleRef, v as ActivepiecesBlockExecutor, w as unpinnedNote, x as resolvedBlock, xt as setPieceRegistryUrl, y as CompositeBlockExecutor, yt as npmRegistryBaseUrl, z as SecretCreatorMismatchError } from "./telemetry-CSemNycC.js";
1
+ import { $ as containsRedactedMarker, A as AttachmentCache, B as SecretCreatorMismatchError, C as blockLabel, Ct as isIndeterminateError, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as secretRefFromId, H as SecretNotFoundError, I as resolveStepInput, J as PieceWorkerPool, K as stepBlock, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as PieceWorkerTimeoutError, R as unavailableValue, S as storeHandlers, St as setPieceRegistryUrl, T as blockPorts, U as SecretOwnerMismatchError, V as SecretDeletedError, W as parseSecretRef, X as PieceWorkerDisposedError, Y as PieceWorker, Z as PieceWorkerError, _ as effectiveRetryPolicy, _t as unsupportedTrigger, a as PieceRegistry, at as isPrivateAddress, b as isHostBound, bt as npmRegistryBaseUrl, c as packagePieces, ct as FileTooLargeError, d as runWorkflow, dt as pieceModuleRef, et as redact, f as canonicalJson, ft as sourcedResolver, g as checkDynamicProperties, gt as unsupportedAuth, h as DynamicPropertiesError, ht as authMethodFor, i as tracedSteps, it as DEFAULT_EGRESS_POLICY, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as maxFileBytes, m as stepConfigHash, mt as UnsupportedPieceFeatureError, n as WorkflowTelemetry, nt as rememberSecrets, o as installedPiece, ot as parseAddress, p as hashOf, q as triggerBlock, r as tracedMethods, rt as secretsFor, s as installedPieces, st as extractDedupeKey, t as TELEMETRY_SCOPE, tt as redactMessage, u as errorNameOf, ut as bundleResolver, v as ActivepiecesBlockExecutor, w as unpinnedNote, x as resolvedBlock, xt as pieceRegistrySource, y as CompositeBlockExecutor } from "./telemetry-GZA1FLlY.js";
2
2
  import { DENIED_ACTION_INDEXES, ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy, assertActionsApplied } from "@powerhousedao/pieces-framework";
3
3
  import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
4
4
  import { open, readFile, readdir, rm, stat } from "node:fs/promises";
5
5
  import { dirname, join } from "node:path";
6
- import { childLogger, generateId } from "document-model";
6
+ import { childLogger, garbageCollect, generateId, hashDocumentStateForScope, isDenied, sortOperations } from "document-model";
7
7
  import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, coerceManualPayload, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, manualInputTypeName, parseManualInputs, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
8
8
  import { Cron } from "croner";
9
9
  import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
@@ -12,14 +12,14 @@ import net from "node:net";
12
12
  import { AsyncLocalStorage } from "node:async_hooks";
13
13
  import { hostname } from "node:os";
14
14
  import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
15
+ import { BaseReadModel, JobStatus, PropagationMode, WriteHandleFactory, createEmptyDocument, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker, jobIdsOf, placeholderJob, registerTelemetryGauge, remoteStagePort } from "@powerhousedao/reactor";
15
16
  import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
16
- import http from "node:http";
17
+ import http, { validateHeaderName, validateHeaderValue } from "node:http";
17
18
  import https from "node:https";
18
- import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
19
+ import { actions as actions$1, reducer, utils } from "@powerhousedao/workflow/document-models/workflow";
19
20
  import { Kind, parse } from "graphql";
20
21
  import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
21
22
  import { SpanStatusCode, context } from "@opentelemetry/api";
22
- import { BaseReadModel, JobStatus, PropagationMode, WriteHandleFactory, createEmptyDocument, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker, jobIdsOf, placeholderJob, remoteStagePort } from "@powerhousedao/reactor";
23
23
  import { isFolderNode } from "@powerhousedao/shared/document-drive";
24
24
  import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
25
25
  import { performance as performance$1 } from "node:perf_hooks";
@@ -439,6 +439,12 @@ function tokenSetFrom(response, context, previous) {
439
439
  }
440
440
  };
441
441
  }
442
+ function relayEgress(egress) {
443
+ return {
444
+ ...egress,
445
+ allowPrivateAddresses: true
446
+ };
447
+ }
442
448
  async function exchangeCode(attempt, code, clientSecret, egress, fromRegistry = false) {
443
449
  const method = authorizationMethod(attempt.authorization_method);
444
450
  const auth = clientAuth$1(method, attempt.client_id, clientSecret);
@@ -448,7 +454,7 @@ async function exchangeCode(attempt, code, clientSecret, egress, fromRegistry =
448
454
  redirect_uri: attempt.redirect_uri,
449
455
  ...attempt.code_verifier ? { code_verifier: attempt.code_verifier } : {},
450
456
  ...auth.form
451
- }, auth.headers, egress), {
457
+ }, auth.headers, clientSecret === null ? relayEgress(egress) : egress), {
452
458
  token_url: attempt.token_url,
453
459
  authorization_method: method,
454
460
  redirect_url: attempt.redirect_uri,
@@ -523,7 +529,7 @@ var StoreTokenRefresher = class {
523
529
  grant_type: "refresh_token",
524
530
  refresh_token: tokens.refresh_token,
525
531
  ...auth.form
526
- }, auth.headers, this.egress);
532
+ }, auth.headers, tokens.relayed ? relayEgress(this.egress) : this.egress);
527
533
  } catch (error) {
528
534
  const detail = error instanceof Error ? error.message : String(error);
529
535
  if (!tokenExpired(tokens, Math.round(Date.now() / 1e3))) {
@@ -1555,6 +1561,9 @@ function currentStepTest() {
1555
1561
  function currentRunUser() {
1556
1562
  return storage.getStore()?.runUser;
1557
1563
  }
1564
+ function currentWaitpoints() {
1565
+ return storage.getStore()?.waitpoints;
1566
+ }
1558
1567
  //#endregion
1559
1568
  //#region src/reactor/piece-store-port.ts
1560
1569
  const logger$5 = childLogger(["workflow", "piece-store"]);
@@ -1750,6 +1759,7 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
1750
1759
  egress: configuredEgress(),
1751
1760
  worker: currentPieceWorker,
1752
1761
  stepTest: currentStepTest,
1762
+ waitpoints: currentWaitpoints,
1753
1763
  identity: () => ({
1754
1764
  runId: currentRunId(),
1755
1765
  flowId: currentWorkflowId(),
@@ -2691,6 +2701,10 @@ const SCHEDULE_BLOCK = trigger("schedule");
2691
2701
  const WEBHOOK_BLOCK = trigger("webhook");
2692
2702
  const BRANCH_BLOCK = action("branch");
2693
2703
  const ASSERT_BLOCK = action("assert");
2704
+ const DELAY_BLOCK = action("delay");
2705
+ const APPROVAL_BLOCK = action("approval");
2706
+ const STOP_BLOCK = action("stop");
2707
+ const WAIT_FOR_WEBHOOK_BLOCK = action("waitForWebhook");
2694
2708
  function isCoreBlock(block) {
2695
2709
  return block.pieceName === CORE_PIECE_NAME;
2696
2710
  }
@@ -7680,6 +7694,16 @@ const PARKED_TRIGGER_STATUS = "PARKED";
7680
7694
  * not finish and nothing is wrong with the workflow", which is what the
7681
7695
  * document model's RunStatus.CANCELLED means. */
7682
7696
  const CANCELLED_RUN_STATUS = "CANCELLED";
7697
+ /** How long a run may stay suspended; null when the policy sets no bound. */
7698
+ function maxSuspensionMs(runnable) {
7699
+ const policy = runnable.policy;
7700
+ const days = positive(policy?.maxSuspensionDays);
7701
+ return days === null ? null : days * 864e5;
7702
+ }
7703
+ //#endregion
7704
+ //#region src/reactor/waitpoint-types.ts
7705
+ const WAITPOINT_EXPIRED_ERROR_NAME = "WaitpointExpired";
7706
+ const WAITPOINT_NOT_LIVE_ERROR_NAME = "WaitpointNotLive";
7683
7707
  //#endregion
7684
7708
  //#region src/reactor/store.ts
7685
7709
  function triggerRowBlock(row) {
@@ -7711,6 +7735,7 @@ function workflowParkRow(workflowId, publishedVersion, reason) {
7711
7735
  parked_at: (/* @__PURE__ */ new Date()).toISOString()
7712
7736
  };
7713
7737
  }
7738
+ const WAITING_RUN_STATUS = "WAITING";
7714
7739
  const logger$4 = childLogger([
7715
7740
  "workflow",
7716
7741
  "runtime",
@@ -7718,6 +7743,64 @@ const logger$4 = childLogger([
7718
7743
  ]);
7719
7744
  const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
7720
7745
  const PENDING_RUN_STATUS = "PENDING";
7746
+ const LIVE_RUN_STATUSES = [
7747
+ PENDING_RUN_STATUS,
7748
+ "RUNNING",
7749
+ WAITING_RUN_STATUS
7750
+ ];
7751
+ const LIVE_WAITPOINT_STATUSES = ["PENDING", "COMPLETED"];
7752
+ async function closeLiveWaitpoints(db, runIds, status, resolvedAt, keep) {
7753
+ if (runIds.length === 0) return [];
7754
+ return (await db.updateTable("waitpoint").set({
7755
+ status,
7756
+ resolved_at: resolvedAt
7757
+ }).where("run_id", "in", runIds).where("status", "in", LIVE_WAITPOINT_STATUSES).$if(keep !== void 0, (qb) => qb.where((eb) => eb.not(keep(eb)))).returning("id").execute()).map((row) => row.id);
7758
+ }
7759
+ async function failWaitingSteps(db, runIds, reason, errorName, endedAt) {
7760
+ if (runIds.length === 0) return;
7761
+ await db.updateTable("step_execution").set({
7762
+ status: "FAILED",
7763
+ error: redactMessage(reason),
7764
+ error_name: errorName,
7765
+ ended_at: endedAt
7766
+ }).where("run_id", "in", runIds).where("status", "=", WAITING_RUN_STATUS).execute();
7767
+ }
7768
+ function waitpointExpiryReason(waitpoint) {
7769
+ return `The ${waitpoint.kind === "approval" ? "approval" : "wait for a resume"} at step "${waitpoint.step_key}" expired at ${waitpoint.expires_at} without being resolved`;
7770
+ }
7771
+ function strandedWait(eb) {
7772
+ const live = eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES);
7773
+ return eb.and([eb.not(eb.exists(completedOf(eb))), eb.or([eb.and([eb("run.waiting_on", "is not", null), eb.not(eb.exists(live.whereRef("waitpoint.id", "=", "run.waiting_on")))]), eb.and([eb("run.waiting_on", "is", null), eb.not(eb.exists(live.whereRef("waitpoint.run_id", "=", "run.id")))])])]);
7774
+ }
7775
+ function completedOf(eb) {
7776
+ return eb.selectFrom("waitpoint").select("waitpoint.id").whereRef("waitpoint.run_id", "=", "run.id").where("waitpoint.status", "=", "COMPLETED");
7777
+ }
7778
+ async function endWaitingRun(db, runId, end) {
7779
+ const { waitpointId } = end;
7780
+ if ((await db.updateTable("run").set({
7781
+ status: "CANCELLED",
7782
+ error: redactMessage(end.reason),
7783
+ error_name: end.errorName,
7784
+ ended_at: end.endedAt
7785
+ }).where("id", "=", runId).where("status", "=", "WAITING").where((eb) => waitpointId === void 0 ? strandedWait(eb) : eb.and([eb.or([eb("run.waiting_on", "=", waitpointId), eb("run.waiting_on", "is", null)]), eb.not(eb.exists(completedOf(eb)))])).returning("id").execute()).length === 0) return void 0;
7786
+ await failWaitingSteps(db, [runId], end.reason, end.errorName, end.endedAt);
7787
+ const closed = await closeLiveWaitpoints(db, [runId], "EXPIRED", end.endedAt);
7788
+ return {
7789
+ runId,
7790
+ reason: end.reason,
7791
+ closed
7792
+ };
7793
+ }
7794
+ function priorNotes(text) {
7795
+ if (!text) return [];
7796
+ try {
7797
+ const notes = JSON.parse(text);
7798
+ return Array.isArray(notes) ? notes.filter((note) => typeof note === "string") : [];
7799
+ } catch {
7800
+ return [];
7801
+ }
7802
+ }
7803
+ const STRANDED_WAIT_REASON = "The run waited on a waitpoint that is no longer live";
7721
7804
  function errorMessage(error) {
7722
7805
  return error instanceof Error ? error.message : String(error);
7723
7806
  }
@@ -7819,6 +7902,31 @@ async function up(db) {
7819
7902
  try {
7820
7903
  await db.schema.dropTable("webhook_endpoint").ifExists().execute();
7821
7904
  } catch {}
7905
+ for (const [column, type] of [
7906
+ ["definition_hash", "text"],
7907
+ ["definition_op_index", "integer"],
7908
+ ["definition_scope_hash", "text"],
7909
+ ["waiting_on", "text"]
7910
+ ]) try {
7911
+ await db.schema.alterTable("run").addColumn(column, type).execute();
7912
+ } catch {}
7913
+ await db.schema.createTable("waitpoint").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("step_path", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("type", "text", (col) => col.notNull()).addColumn("kind", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("token_hash", "text").addColumn("created_at", "text", (col) => col.notNull()).addColumn("resume_at", "text").addColumn("expires_at", "text").addColumn("resolved_at", "text").addColumn("decided_by", "text").addColumn("approval", "text").addColumn("resume_payload", "text").addColumn("response_to_send", "text").ifNotExists().execute();
7914
+ await db.schema.createIndex("waitpoint_token").ifNotExists().unique().on("waitpoint").column("token_hash").execute();
7915
+ await db.schema.createIndex("waitpoint_status_resume").ifNotExists().on("waitpoint").columns(["status", "resume_at"]).execute();
7916
+ await db.schema.createIndex("waitpoint_status_expires").ifNotExists().on("waitpoint").columns(["status", "expires_at"]).execute();
7917
+ await db.schema.createIndex("waitpoint_run").ifNotExists().on("waitpoint").column("run_id").execute();
7918
+ const liveStep = db.schema.createIndex("waitpoint_live_step").ifNotExists().unique().on("waitpoint").columns(["run_id", "step_path"]).where(sql.ref("status"), "in", LIVE_WAITPOINT_STATUSES);
7919
+ try {
7920
+ await liveStep.execute();
7921
+ } catch {
7922
+ await db.updateTable("waitpoint").set({
7923
+ status: "CANCELLED",
7924
+ resolved_at: (/* @__PURE__ */ new Date()).toISOString()
7925
+ }).where("status", "in", LIVE_WAITPOINT_STATUSES).where((eb) => eb.and([eb.not(eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.waiting_on", "=", "waitpoint.id"))), eb.exists(eb.selectFrom("waitpoint as sibling").select("sibling.id").whereRef("sibling.run_id", "=", "waitpoint.run_id").whereRef("sibling.step_path", "=", "waitpoint.step_path").whereRef("sibling.id", "!=", "waitpoint.id").where("sibling.status", "in", LIVE_WAITPOINT_STATUSES))])).execute();
7926
+ await liveStep.execute();
7927
+ }
7928
+ await db.schema.createIndex("waitpoint_workflow_status").ifNotExists().on("waitpoint").columns(["workflow_id", "status"]).execute();
7929
+ await db.schema.createIndex("run_waiting").ifNotExists().on("run").columns(["workflow_id", "enqueued_at"]).where(sql.ref("status"), "=", WAITING_RUN_STATUS).execute();
7822
7930
  await migrateBlockType(db, "trigger_state", "trigger_name");
7823
7931
  await migrateBlockType(db, "step_execution", "block_name");
7824
7932
  return migrateTriggerStoreState(db);
@@ -8177,6 +8285,8 @@ var WorkflowRunStore = class WorkflowRunStore {
8177
8285
  unmigrated;
8178
8286
  runsInFlight = /* @__PURE__ */ new Set();
8179
8287
  journaledSteps = /* @__PURE__ */ new Map();
8288
+ pausingRuns = /* @__PURE__ */ new Set();
8289
+ resumedLegs = /* @__PURE__ */ new Set();
8180
8290
  constructor(db, unmigrated) {
8181
8291
  this.db = db;
8182
8292
  this.unmigrated = unmigrated;
@@ -8195,13 +8305,20 @@ var WorkflowRunStore = class WorkflowRunStore {
8195
8305
  this.unmigrated.delete(workflowId);
8196
8306
  }
8197
8307
  async recoverOrphanedRuns() {
8198
- let query = this.db.updateTable("run").set({
8199
- status: "FAILED",
8200
- error: ORPHANED_RUN_ERROR,
8201
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8202
- }).where("status", "=", "RUNNING");
8203
- if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
8204
- const recovered = (await query.returning("id").execute()).length;
8308
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
8309
+ const inFlight = [...this.runsInFlight];
8310
+ const recovered = await this.db.transaction().execute(async (trx) => {
8311
+ let query = trx.updateTable("run").set({
8312
+ status: "FAILED",
8313
+ error: ORPHANED_RUN_ERROR,
8314
+ ended_at: endedAt
8315
+ }).where("status", "=", "RUNNING");
8316
+ if (inFlight.length > 0) query = query.where("id", "not in", inFlight);
8317
+ const ids = (await query.returning("id").execute()).map((row) => row.id);
8318
+ await failWaitingSteps(trx, ids, ORPHANED_RUN_ERROR, null, endedAt);
8319
+ await closeLiveWaitpoints(trx, ids, "CANCELLED", endedAt);
8320
+ return ids.length;
8321
+ });
8205
8322
  if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
8206
8323
  return recovered;
8207
8324
  }
@@ -8265,14 +8382,25 @@ var WorkflowRunStore = class WorkflowRunStore {
8265
8382
  * CANCELLED run is not rerunnable and must not read as a workflow defect.
8266
8383
  */
8267
8384
  async cancelRun(runId, reason) {
8385
+ return this.endLiveRun(runId, "CANCELLED", reason, null);
8386
+ }
8387
+ async endLiveRun(runId, status, reason, errorName) {
8268
8388
  this.runsInFlight.delete(runId);
8269
8389
  this.journaledSteps.delete(runId);
8270
- if (erasedRuns.delete(runId)) return;
8271
- await this.db.updateTable("run").set({
8272
- status: "CANCELLED",
8273
- error: redactMessage(reason),
8274
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8275
- }).where("id", "=", runId).execute();
8390
+ this.pausingRuns.delete(runId);
8391
+ this.resumedLegs.delete(runId);
8392
+ if (erasedRuns.delete(runId)) return [];
8393
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
8394
+ return this.db.transaction().execute(async (trx) => {
8395
+ if ((await trx.updateTable("run").set({
8396
+ status,
8397
+ error: redactMessage(reason),
8398
+ error_name: errorName,
8399
+ ended_at: endedAt
8400
+ }).where("id", "=", runId).where("status", "in", LIVE_RUN_STATUSES).returning("id").execute()).length === 0) return [];
8401
+ await failWaitingSteps(trx, [runId], reason, errorName, endedAt);
8402
+ return closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
8403
+ });
8276
8404
  }
8277
8405
  async enqueueRun(options) {
8278
8406
  const id = randomUUID();
@@ -8480,12 +8608,15 @@ var WorkflowRunStore = class WorkflowRunStore {
8480
8608
  async beginRun(runId, details) {
8481
8609
  this.runsInFlight.add(runId);
8482
8610
  this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
8483
- await this.db.updateTable("run").set({
8611
+ if ((await this.db.updateTable("run").set({
8484
8612
  status: "RUNNING",
8485
8613
  workflow_name: details.workflowName,
8486
8614
  workflow_version: details.workflowVersion,
8487
8615
  started_at: (/* @__PURE__ */ new Date()).toISOString()
8488
- }).where("id", "=", runId).execute();
8616
+ }).where("id", "=", runId).where("status", "=", "PENDING").returning("id").execute()).length > 0) return true;
8617
+ this.runsInFlight.delete(runId);
8618
+ this.journaledSteps.delete(runId);
8619
+ return false;
8489
8620
  }
8490
8621
  async startRun(options) {
8491
8622
  const id = randomUUID();
@@ -8514,10 +8645,10 @@ var WorkflowRunStore = class WorkflowRunStore {
8514
8645
  if (erasedRuns.has(runId)) return;
8515
8646
  const values = stepValues(runId, ordinal, step);
8516
8647
  const { run_id: _run, step_id: _step, ...mutable } = values;
8517
- await this.db.insertInto("step_execution").values({
8648
+ if (!await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values({
8518
8649
  id: randomUUID(),
8519
8650
  ...values
8520
- }).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
8651
+ }).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute())) return;
8521
8652
  this.journaledSteps.get(runId)?.set(step.stepId, {
8522
8653
  ordinal,
8523
8654
  hash: rowHash(values)
@@ -8532,25 +8663,61 @@ var WorkflowRunStore = class WorkflowRunStore {
8532
8663
  return row.hash === rowHash(stepValues(runId, row.ordinal, step));
8533
8664
  });
8534
8665
  }
8535
- async finishRun(runId, result, executionOrder) {
8666
+ async finishRun(runId, result, executionOrder, pin) {
8536
8667
  this.runsInFlight.delete(runId);
8537
- const sweep = !this.sweepIsNoop(runId, result);
8668
+ const resumed = this.resumedLegs.delete(runId);
8669
+ const paused = this.pausingRuns.delete(runId) || resumed;
8670
+ const own = result.steps.some((step) => step.carried) ? {
8671
+ ...result,
8672
+ steps: result.steps.filter((step) => !step.carried)
8673
+ } : result;
8674
+ const sweep = !this.sweepIsNoop(runId, own);
8538
8675
  this.journaledSteps.delete(runId);
8539
- if (erasedRuns.delete(runId)) return;
8540
- if (sweep && result.steps.length > 0) try {
8541
- await this.sweepSteps(runId, result, executionOrder);
8676
+ if (erasedRuns.delete(runId)) return {
8677
+ held: true,
8678
+ closed: []
8679
+ };
8680
+ if (sweep && own.steps.length > 0) try {
8681
+ await this.sweepSteps(runId, own, executionOrder);
8542
8682
  } catch (error) {
8543
8683
  logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
8544
8684
  }
8545
- const notes = runWarningNotes(result);
8546
- await this.db.updateTable("run").set({
8547
- status: result.status,
8548
- error: result.error ? redactMessage(result.error) : null,
8549
- error_name: result.errorName ?? null,
8550
- ended_at: (/* @__PURE__ */ new Date()).toISOString(),
8551
- warnings: notes.length,
8552
- warning_notes: notes.length > 0 ? JSON.stringify(notes) : null
8553
- }).where("id", "=", runId).execute();
8685
+ const waiting = result.status === WAITING_RUN_STATUS;
8686
+ const on = result.waiting;
8687
+ const now = (/* @__PURE__ */ new Date()).toISOString();
8688
+ const close = async (trx) => {
8689
+ const prior = resumed ? await trx.selectFrom("run").select("warning_notes").where("id", "=", runId).executeTakeFirst() : void 0;
8690
+ const notes = [.../* @__PURE__ */ new Set([...priorNotes(prior?.warning_notes ?? null), ...runWarningNotes(own)])];
8691
+ const written = await trx.updateTable("run").set({
8692
+ status: result.status,
8693
+ error: result.error ? redactMessage(result.error) : null,
8694
+ error_name: result.errorName ?? null,
8695
+ ended_at: waiting ? null : now,
8696
+ warnings: notes.length,
8697
+ warning_notes: notes.length > 0 ? JSON.stringify(notes) : null,
8698
+ waiting_on: waiting ? on?.waitpointId ?? null : null,
8699
+ ...pin ?? {}
8700
+ }).where("id", "=", runId).$if(waiting, (qb) => qb.where((eb) => eb.exists(eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.id", "=", on?.waitpointId ?? "").where("waitpoint.run_id", "=", runId).where("waitpoint.step_id", "=", on?.stepId ?? "").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES)))).returning("id").execute();
8701
+ if (!waiting) return {
8702
+ held: true,
8703
+ closed: paused ? await closeLiveWaitpoints(trx, [runId], "EXPIRED", now) : []
8704
+ };
8705
+ if (written.length === 0) return {
8706
+ held: false,
8707
+ closed: []
8708
+ };
8709
+ const linkStatuses = (await trx.selectFrom("waitpoint").select("kind").where("id", "=", on.waitpointId).executeTakeFirst())?.kind === "webhook" ? LIVE_WAITPOINT_STATUSES : ["PENDING"];
8710
+ return {
8711
+ held: true,
8712
+ closed: await closeLiveWaitpoints(trx, [runId], "CANCELLED", now, (eb) => eb.or([eb("waitpoint.id", "=", on.waitpointId), eb.and([
8713
+ eb("waitpoint.type", "=", "WEBHOOK"),
8714
+ eb("waitpoint.kind", "=", "webhook"),
8715
+ eb("waitpoint.status", "in", linkStatuses),
8716
+ eb.exists(eb.selectFrom("step_execution").select("step_execution.id").whereRef("step_execution.run_id", "=", "waitpoint.run_id").whereRef("step_execution.step_id", "=", "waitpoint.step_id").where("step_execution.status", "=", "SUCCEEDED"))
8717
+ ])]))
8718
+ };
8719
+ };
8720
+ return waiting || paused ? this.db.transaction().execute(close) : close(this.db);
8554
8721
  }
8555
8722
  async sweepSteps(runId, result, executionOrder) {
8556
8723
  const journaled = await this.db.selectFrom("step_execution").select(["step_id", "ordinal"]).where("run_id", "=", runId).execute();
@@ -8563,7 +8730,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8563
8730
  nextOrdinal = Math.max(nextOrdinal, ran + 1);
8564
8731
  }
8565
8732
  const ordinalFor = (step) => ordinals.get(step.stepId) ?? nextOrdinal++;
8566
- await this.db.insertInto("step_execution").values(result.steps.map((step) => ({
8733
+ await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values(result.steps.map((step) => ({
8567
8734
  id: randomUUID(),
8568
8735
  ...stepValues(runId, ordinalFor(step), step)
8569
8736
  }))).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet((eb) => ({
@@ -8584,18 +8751,15 @@ var WorkflowRunStore = class WorkflowRunStore {
8584
8751
  version_match: eb.ref("excluded.version_match"),
8585
8752
  version_note: eb.ref("excluded.version_note"),
8586
8753
  config_hash: eb.ref("excluded.config_hash")
8587
- }))).execute();
8754
+ }))).execute());
8755
+ }
8756
+ async withRunRow(runId, fn) {
8757
+ return this.db.transaction().execute(async (trx) => {
8758
+ return await trx.selectFrom("run").select("id").where("id", "=", runId).forKeyShare().executeTakeFirst() ? fn(trx) : void 0;
8759
+ });
8588
8760
  }
8589
8761
  async failRun(runId, error, errorName) {
8590
- this.runsInFlight.delete(runId);
8591
- this.journaledSteps.delete(runId);
8592
- if (erasedRuns.delete(runId)) return;
8593
- await this.db.updateTable("run").set({
8594
- status: "FAILED",
8595
- error: redactMessage(error),
8596
- error_name: errorName ?? null,
8597
- ended_at: (/* @__PURE__ */ new Date()).toISOString()
8598
- }).where("id", "=", runId).execute();
8762
+ return this.endLiveRun(runId, "FAILED", error, errorName ?? null);
8599
8763
  }
8600
8764
  async listRuns(workflowId, limit = 25, options = {}) {
8601
8765
  if (Array.isArray(workflowId) && workflowId.length === 0) return [];
@@ -8627,10 +8791,10 @@ var WorkflowRunStore = class WorkflowRunStore {
8627
8791
  }
8628
8792
  async recordRunDocuments(runId, documentIds) {
8629
8793
  if (documentIds.length === 0 || erasedRuns.has(runId)) return;
8630
- await this.db.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
8794
+ await this.withRunRow(runId, (trx) => trx.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
8631
8795
  run_id: runId,
8632
8796
  document_id: documentId
8633
- }))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute();
8797
+ }))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute());
8634
8798
  }
8635
8799
  async getRunDocuments(runId) {
8636
8800
  return (await this.db.selectFrom("run_document").select("document_id").where("run_id", "=", runId).execute()).map((row) => row.document_id);
@@ -8775,6 +8939,181 @@ var WorkflowRunStore = class WorkflowRunStore {
8775
8939
  async deleteDedupe(workflowId) {
8776
8940
  await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
8777
8941
  }
8942
+ async createWaitpoint(row) {
8943
+ if (this.runsInFlight.has(row.run_id)) this.pausingRuns.add(row.run_id);
8944
+ for (let attempt = 1;; attempt++) {
8945
+ const inserted = await this.withRunRow(row.run_id, (trx) => trx.insertInto("waitpoint").values(row).onConflict((oc) => oc.doNothing()).returningAll().executeTakeFirst().then((waitpoint) => ({ waitpoint })));
8946
+ if (!inserted) throw new Error(`Run "${row.run_id}" no longer exists`);
8947
+ if (inserted.waitpoint) return {
8948
+ waitpoint: inserted.waitpoint,
8949
+ created: true
8950
+ };
8951
+ const live = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "=", row.run_id).where("step_path", "=", row.step_path).where("status", "in", LIVE_WAITPOINT_STATUSES).executeTakeFirst();
8952
+ if (live) return {
8953
+ waitpoint: live,
8954
+ created: false
8955
+ };
8956
+ if (attempt >= 3) throw new Error(`Could not create a waitpoint for step "${row.step_key}"`);
8957
+ }
8958
+ }
8959
+ async getWaitpoint(id) {
8960
+ return this.db.selectFrom("waitpoint").selectAll().where("id", "=", id).executeTakeFirst();
8961
+ }
8962
+ async getWaitpointByTokenHash(tokenHash) {
8963
+ return this.db.selectFrom("waitpoint").selectAll().where("token_hash", "=", tokenHash).executeTakeFirst();
8964
+ }
8965
+ async completeWaitpoint(id, resolution) {
8966
+ const now = (/* @__PURE__ */ new Date()).toISOString();
8967
+ return (await this.db.updateTable("waitpoint").set({
8968
+ status: "COMPLETED",
8969
+ resume_payload: JSON.stringify(resolution.payload ?? {}),
8970
+ resolved_at: resolution.resolvedAt ?? now,
8971
+ decided_by: resolution.decidedBy ?? null
8972
+ }).where("id", "=", id).where("status", "=", "PENDING").where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", now)])).where((eb) => eb.not(eb.exists(eb.selectFrom("run").innerJoin("waitpoint as awaited", "awaited.id", "run.waiting_on").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").whereRef("awaited.id", "!=", "waitpoint.id").where("awaited.kind", "!=", "webhook").where("awaited.status", "in", LIVE_WAITPOINT_STATUSES)))).where((eb) => eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").where("run.status", "in", ["RUNNING", WAITING_RUN_STATUS]))).returningAll().execute())[0];
8973
+ }
8974
+ async claimDueDelayWaitpoints(nowIso, limit) {
8975
+ return this.db.updateTable("waitpoint").set({
8976
+ status: "COMPLETED",
8977
+ resolved_at: nowIso,
8978
+ resume_payload: "{}"
8979
+ }).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "<=", nowIso).orderBy("resume_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
8980
+ }
8981
+ async expireDueWaitpoints(nowIso, limit) {
8982
+ return this.db.transaction().execute(async (trx) => {
8983
+ const expired = await trx.updateTable("waitpoint").set({
8984
+ status: "EXPIRED",
8985
+ resolved_at: nowIso
8986
+ }).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("expires_at", "<=", nowIso).orderBy("expires_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
8987
+ const ended = [];
8988
+ for (const waitpoint of expired) {
8989
+ const end = await endWaitingRun(trx, waitpoint.run_id, {
8990
+ reason: waitpointExpiryReason(waitpoint),
8991
+ errorName: WAITPOINT_EXPIRED_ERROR_NAME,
8992
+ endedAt: nowIso,
8993
+ waitpointId: waitpoint.id
8994
+ });
8995
+ if (end) ended.push(end);
8996
+ }
8997
+ return {
8998
+ expired,
8999
+ ended
9000
+ };
9001
+ });
9002
+ }
9003
+ async endStrandedWaits(limit) {
9004
+ const runs = await this.db.selectFrom("run").select("id").where("status", "=", WAITING_RUN_STATUS).where((eb) => strandedWait(eb)).limit(limit).execute();
9005
+ const ended = [];
9006
+ for (const run of runs) {
9007
+ const end = await this.db.transaction().execute((trx) => endWaitingRun(trx, run.id, {
9008
+ reason: STRANDED_WAIT_REASON,
9009
+ errorName: WAITPOINT_NOT_LIVE_ERROR_NAME,
9010
+ endedAt: (/* @__PURE__ */ new Date()).toISOString()
9011
+ }));
9012
+ if (end) ended.push(end);
9013
+ }
9014
+ return ended;
9015
+ }
9016
+ async nextExpiry() {
9017
+ return (await this.db.selectFrom("waitpoint").select("expires_at").where("status", "=", "PENDING").where("expires_at", "is not", null).orderBy("expires_at").limit(1).executeTakeFirst())?.expires_at ?? void 0;
9018
+ }
9019
+ async countWaitingRuns() {
9020
+ const row = await this.db.selectFrom("run").select((eb) => eb.fn.countAll().as("count")).where("status", "=", WAITING_RUN_STATUS).executeTakeFirst();
9021
+ return Number(row?.count ?? 0);
9022
+ }
9023
+ async nextDelayDue() {
9024
+ return (await this.db.selectFrom("waitpoint").select("resume_at").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "is not", null).orderBy("resume_at").limit(1).executeTakeFirst())?.resume_at ?? void 0;
9025
+ }
9026
+ async hasCompletedWaitpoint(runId) {
9027
+ return await this.db.selectFrom("waitpoint").select("id").where("run_id", "=", runId).where("status", "=", "COMPLETED").limit(1).executeTakeFirst() !== void 0;
9028
+ }
9029
+ async heldRunIds() {
9030
+ return (await this.db.selectFrom("waitpoint").innerJoin("run", "run.id", "waitpoint.run_id").select("waitpoint.run_id").distinct().where("waitpoint.status", "=", "COMPLETED").where("run.status", "=", WAITING_RUN_STATUS).execute()).map((row) => row.run_id);
9031
+ }
9032
+ isRunInFlight(runId) {
9033
+ return this.runsInFlight.has(runId);
9034
+ }
9035
+ async claimWaitingRun(runId) {
9036
+ if ((await this.db.updateTable("run").set({ status: "RUNNING" }).where("id", "=", runId).where("status", "=", "WAITING").returning("id").execute()).length === 0) return false;
9037
+ this.runsInFlight.add(runId);
9038
+ this.resumedLegs.add(runId);
9039
+ this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
9040
+ return true;
9041
+ }
9042
+ async returnToWaiting(runId) {
9043
+ this.runsInFlight.delete(runId);
9044
+ this.resumedLegs.delete(runId);
9045
+ this.journaledSteps.delete(runId);
9046
+ await this.db.updateTable("run").set({ status: WAITING_RUN_STATUS }).where("id", "=", runId).where("status", "=", "RUNNING").execute();
9047
+ }
9048
+ async consumeCompletedWaitpoint(runId) {
9049
+ return this.db.transaction().execute(async (trx) => {
9050
+ const waitingOn = (await trx.selectFrom("run").select("waiting_on").where("id", "=", runId).executeTakeFirst())?.waiting_on ?? null;
9051
+ const candidates = await trx.selectFrom("waitpoint").selectAll().where("run_id", "=", runId).where("status", "=", "COMPLETED").orderBy("resolved_at").execute();
9052
+ const awaited = waitingOn ? await trx.selectFrom("waitpoint").select("kind").where("id", "=", waitingOn).executeTakeFirst() : void 0;
9053
+ const linksResume = !awaited || awaited.kind === "webhook";
9054
+ const chosen = candidates.find((row) => row.id === waitingOn) ?? (linksResume ? candidates[0] : void 0);
9055
+ if (!chosen) return void 0;
9056
+ const waitpoint = (await trx.updateTable("waitpoint").set({ status: "CONSUMED" }).where("id", "=", chosen.id).where("status", "=", "COMPLETED").returningAll().execute())[0];
9057
+ if (!waitpoint) return void 0;
9058
+ return {
9059
+ waitpoint,
9060
+ closed: waitingOn && waitingOn !== waitpoint.id ? await closeLiveWaitpoints(trx, [runId], "CANCELLED", (/* @__PURE__ */ new Date()).toISOString(), (eb) => eb("waitpoint.id", "!=", waitingOn)) : []
9061
+ };
9062
+ });
9063
+ }
9064
+ async getRunPin(runId) {
9065
+ return this.db.selectFrom("run").select([
9066
+ "definition_hash",
9067
+ "definition_op_index",
9068
+ "definition_scope_hash"
9069
+ ]).where("id", "=", runId).executeTakeFirst();
9070
+ }
9071
+ async cancelIdleRun(runId, reason) {
9072
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
9073
+ if (!await this.db.transaction().execute(async (trx) => {
9074
+ if ((await trx.updateTable("run").set({
9075
+ status: "CANCELLED",
9076
+ error: redactMessage(reason),
9077
+ ended_at: endedAt
9078
+ }).where("id", "=", runId).where("status", "in", ["WAITING", "PENDING"]).returning("id").execute()).length === 0) return false;
9079
+ await failWaitingSteps(trx, [runId], reason, null, endedAt);
9080
+ await closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
9081
+ return true;
9082
+ })) return void 0;
9083
+ this.runsInFlight.delete(runId);
9084
+ return runId;
9085
+ }
9086
+ async endWait(runId, end) {
9087
+ this.runsInFlight.delete(runId);
9088
+ this.journaledSteps.delete(runId);
9089
+ this.pausingRuns.delete(runId);
9090
+ this.resumedLegs.delete(runId);
9091
+ const endedAt = (/* @__PURE__ */ new Date()).toISOString();
9092
+ return this.db.transaction().execute(async (trx) => {
9093
+ if ((await trx.updateTable("run").set({
9094
+ status: end.status,
9095
+ error: redactMessage(end.reason),
9096
+ error_name: end.errorName,
9097
+ ended_at: endedAt
9098
+ }).where("id", "=", runId).where("status", "=", "RUNNING").returning("id").execute()).length === 0) return [];
9099
+ await failWaitingSteps(trx, [runId], end.reason, end.errorName, endedAt);
9100
+ return closeLiveWaitpoints(trx, [runId], end.waitpoints, endedAt);
9101
+ });
9102
+ }
9103
+ async listWaitpointsForRuns(runIds) {
9104
+ const byRun = new Map(runIds.map((id) => [id, []]));
9105
+ if (runIds.length === 0) return byRun;
9106
+ const rows = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "in", [...new Set(runIds)]).orderBy("created_at").execute();
9107
+ for (const row of rows) byRun.get(row.run_id)?.push(row);
9108
+ return byRun;
9109
+ }
9110
+ async listWaitingRuns(workflowId, limit) {
9111
+ if (Array.isArray(workflowId) && workflowId.length === 0) return [];
9112
+ let query = this.db.selectFrom("run").selectAll().where("status", "=", WAITING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc").limit(Math.min(Math.max(limit, 1), 100));
9113
+ if (Array.isArray(workflowId)) query = query.where("workflow_id", "in", workflowId);
9114
+ else if (workflowId) query = query.where("workflow_id", "=", workflowId);
9115
+ return query.execute();
9116
+ }
8778
9117
  async pruneFinishedRuns(cutoffIso, batchSize = PRUNE_BATCH_SIZE) {
8779
9118
  let pruned = 0;
8780
9119
  for (;;) {
@@ -8782,6 +9121,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8782
9121
  const ids = (await trx.selectFrom("run").select("id").where("ended_at", "is not", null).where("ended_at", "<", cutoffIso).limit(batchSize).execute()).map((row) => row.id);
8783
9122
  if (ids.length === 0) return 0;
8784
9123
  await trx.deleteFrom("step_execution").where("run_id", "in", ids).execute();
9124
+ await trx.deleteFrom("waitpoint").where("run_id", "in", ids).execute();
8785
9125
  await trx.deleteFrom("run_document").where("run_id", "in", ids).execute();
8786
9126
  await trx.deleteFrom("run").where("id", "in", ids).execute();
8787
9127
  return ids.length;
@@ -8813,10 +9153,11 @@ var WorkflowRunStore = class WorkflowRunStore {
8813
9153
  for (const id of frontier) runIds.add(id);
8814
9154
  }
8815
9155
  if (runIds.size === 0) return erased;
8816
- const all = [...runIds];
8817
- const live = await trx.selectFrom("run").select("id").where("id", "in", all).where("status", "in", ["RUNNING", PENDING_RUN_STATUS]).execute();
8818
- for (const row of live) erasedRuns.add(row.id);
9156
+ const all = [...runIds].sort();
9157
+ const locked = await trx.selectFrom("run").select(["id", "status"]).where("id", "in", all).orderBy("id").forUpdate().execute();
9158
+ for (const row of locked) if (row.status === "RUNNING" || row.status === "PENDING") erasedRuns.add(row.id);
8819
9159
  erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
9160
+ await trx.deleteFrom("waitpoint").where("run_id", "in", all).execute();
8820
9161
  erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
8821
9162
  erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
8822
9163
  erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
@@ -8933,6 +9274,118 @@ function decodeRunCursor(cursor) {
8933
9274
  id: parsed[1]
8934
9275
  };
8935
9276
  }
9277
+ const WAITPOINT_POLL_MAX_MS = 3e5;
9278
+ const WAITPOINT_RETRY_MS = 3e4;
9279
+ const SUSPENDED_COUNT_MS = 3e4;
9280
+ var WaitpointPoller = class {
9281
+ deps;
9282
+ running = false;
9283
+ polling = false;
9284
+ again = false;
9285
+ timer;
9286
+ timerAt;
9287
+ nudged;
9288
+ suspended = 0;
9289
+ suspendedAt = 0;
9290
+ counting = false;
9291
+ constructor(deps) {
9292
+ this.deps = deps;
9293
+ }
9294
+ get active() {
9295
+ return this.running;
9296
+ }
9297
+ start() {
9298
+ if (this.running) return;
9299
+ this.running = true;
9300
+ this.poll();
9301
+ }
9302
+ stop() {
9303
+ this.running = false;
9304
+ clearTimeout(this.timer);
9305
+ this.timer = void 0;
9306
+ this.timerAt = void 0;
9307
+ this.nudged = void 0;
9308
+ }
9309
+ nudge(dueIso) {
9310
+ if (!this.running) return;
9311
+ const due = Date.parse(dueIso);
9312
+ if (Number.isNaN(due)) return;
9313
+ if (this.polling) {
9314
+ this.nudged = Math.min(this.nudged ?? Infinity, due);
9315
+ return;
9316
+ }
9317
+ if (this.timerAt !== void 0 && due >= this.timerAt) return;
9318
+ this.arm(due);
9319
+ }
9320
+ suspendedRuns() {
9321
+ if (Date.now() - this.suspendedAt > SUSPENDED_COUNT_MS) this.count();
9322
+ return this.suspended;
9323
+ }
9324
+ async count(store) {
9325
+ if (this.counting) return this.suspended;
9326
+ this.counting = true;
9327
+ try {
9328
+ const journal = store ?? await this.deps.store();
9329
+ if (journal) {
9330
+ this.suspended = await journal.countWaitingRuns();
9331
+ this.suspendedAt = Date.now();
9332
+ }
9333
+ } catch (error) {
9334
+ this.deps.logger.warn("Counting suspended runs failed: @error", error);
9335
+ } finally {
9336
+ this.counting = false;
9337
+ }
9338
+ return this.suspended;
9339
+ }
9340
+ /** One pass: due delays and held resumes deliver, overdue waits expire, the
9341
+ * timer re-arms. */
9342
+ async poll() {
9343
+ if (!this.running) return;
9344
+ if (this.polling) {
9345
+ this.again = true;
9346
+ return;
9347
+ }
9348
+ this.polling = true;
9349
+ let next = Date.now() + WAITPOINT_RETRY_MS;
9350
+ try {
9351
+ const store = await this.deps.store();
9352
+ if (!store) return;
9353
+ const batch = this.deps.batch ?? 100;
9354
+ do {
9355
+ this.again = false;
9356
+ const now = (/* @__PURE__ */ new Date()).toISOString();
9357
+ const due = await store.claimDueDelayWaitpoints(now, batch);
9358
+ for (const runId of new Set(due.map((row) => row.run_id))) await this.deps.resume(runId);
9359
+ const { expired, ended } = await store.expireDueWaitpoints(now, batch);
9360
+ const stranded = await store.endStrandedWaits(batch);
9361
+ for (const end of [...ended, ...stranded]) await this.deps.ended(end);
9362
+ if (due.length === batch || expired.length === batch || stranded.length === batch) this.again = true;
9363
+ } while (this.again && this.active);
9364
+ for (const runId of await store.heldRunIds()) await this.deps.resume(runId);
9365
+ await this.count(store);
9366
+ const [delay, expiry] = await Promise.all([store.nextDelayDue(), store.nextExpiry()]);
9367
+ next = Math.min(delay ? Date.parse(delay) : Infinity, expiry ? Date.parse(expiry) : Infinity);
9368
+ } catch (error) {
9369
+ this.deps.logger.warn("Waitpoint poll failed: @error", error);
9370
+ } finally {
9371
+ this.polling = false;
9372
+ if (this.nudged !== void 0) next = Math.min(next, this.nudged);
9373
+ this.nudged = void 0;
9374
+ if (this.active) this.arm(next);
9375
+ }
9376
+ }
9377
+ arm(due) {
9378
+ clearTimeout(this.timer);
9379
+ const delay = Math.min(Math.max(Number.isFinite(due) ? due - Date.now() : Infinity, 0), WAITPOINT_POLL_MAX_MS);
9380
+ this.timerAt = Date.now() + delay;
9381
+ this.timer = setTimeout(() => {
9382
+ this.timer = void 0;
9383
+ this.timerAt = void 0;
9384
+ this.poll();
9385
+ }, delay);
9386
+ this.timer.unref();
9387
+ }
9388
+ };
8936
9389
  //#endregion
8937
9390
  //#region src/reactor/run-gate.ts
8938
9391
  const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
@@ -8970,8 +9423,11 @@ var WorkflowRunGate = class {
8970
9423
  * - **SINGLETON**: refused outright while a run is active. The caller
8971
9424
  * journals the refusal rather than dropping it silently, because a firing
8972
9425
  * that vanished is indistinguishable from a trigger that never fired.
9426
+ *
9427
+ * A `resume` is a waiting run's next leg, not a new firing: it queues for
9428
+ * the slot under every policy, past the queue depth too.
8973
9429
  */
8974
- async admit(workflowId, policy, onQueue) {
9430
+ async admit(workflowId, policy, onQueue, options = {}) {
8975
9431
  if (this.closed) return {
8976
9432
  admitted: false,
8977
9433
  reason: CLOSED_REASON,
@@ -8990,7 +9446,7 @@ var WorkflowRunGate = class {
8990
9446
  this.lanes.set(workflowId, lane);
8991
9447
  let waited = false;
8992
9448
  if (lane.active >= limit) {
8993
- if (policy.concurrency === "SINGLETON") {
9449
+ if (policy.concurrency === "SINGLETON" && !options.resume) {
8994
9450
  this.forgetIfIdle(workflowId, lane);
8995
9451
  return {
8996
9452
  admitted: false,
@@ -8998,7 +9454,7 @@ var WorkflowRunGate = class {
8998
9454
  reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
8999
9455
  };
9000
9456
  }
9001
- if (lane.waiting.length >= this.maxQueued) return {
9457
+ if (!options.resume && lane.waiting.length >= this.maxQueued) return {
9002
9458
  admitted: false,
9003
9459
  refusal: "queue-full",
9004
9460
  reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
@@ -10653,12 +11109,72 @@ function refusalStatus(refusal) {
10653
11109
  switch (refusal) {
10654
11110
  case "parked":
10655
11111
  case "singleton":
10656
- case "stale": return 409;
11112
+ case "stale":
11113
+ case "cancelled": return 409;
10657
11114
  case "queue-full":
10658
11115
  case "expired": return 429;
10659
11116
  default: return 500;
10660
11117
  }
10661
11118
  }
11119
+ function runResultReply(result, succeededStatus) {
11120
+ return {
11121
+ status: result.status === "SUCCEEDED" ? succeededStatus : result.status === "WAITING" ? 202 : refusalStatus(result.refusal),
11122
+ contentType: JSON_CONTENT_TYPE,
11123
+ body: JSON.stringify({
11124
+ runId: result.runId,
11125
+ status: result.status,
11126
+ error: result.error ?? null
11127
+ })
11128
+ };
11129
+ }
11130
+ const RESPONDED = Symbol("a step responded");
11131
+ const DROPPED_RESPONSE_HEADERS = /* @__PURE__ */ new Set([
11132
+ "connection",
11133
+ "content-length",
11134
+ "keep-alive",
11135
+ "set-cookie",
11136
+ "transfer-encoding",
11137
+ "upgrade"
11138
+ ]);
11139
+ const ORIGIN_SAFE_HEADERS = {
11140
+ "x-content-type-options": "nosniff",
11141
+ "content-security-policy": "sandbox"
11142
+ };
11143
+ function sendableHeader(name, value) {
11144
+ try {
11145
+ validateHeaderName(name);
11146
+ validateHeaderValue(name, value);
11147
+ return true;
11148
+ } catch {
11149
+ return false;
11150
+ }
11151
+ }
11152
+ /** A step's `run.respond` / `run.stop` answer as an HTTP reply. As upstream,
11153
+ * none answers 200 with `{}`, and a string body goes as text. */
11154
+ function runResponseReply(response) {
11155
+ const status = typeof response.status === "number" && Number.isInteger(response.status) && response.status >= 200 && response.status <= 599 ? response.status : 200;
11156
+ const body = response.body ?? {};
11157
+ let contentType = typeof body === "string" ? "text/plain; charset=utf-8" : JSON_CONTENT_TYPE;
11158
+ const headers = {};
11159
+ const given = response.headers && typeof response.headers === "object" ? response.headers : {};
11160
+ for (const [rawName, rawValue] of Object.entries(given)) {
11161
+ const name = rawName.toLowerCase();
11162
+ if (typeof rawValue !== "string" && typeof rawValue !== "number") continue;
11163
+ const value = String(rawValue);
11164
+ if (!sendableHeader(name, value)) continue;
11165
+ if (name === "content-type") contentType = value;
11166
+ else if (!DROPPED_RESPONSE_HEADERS.has(name)) headers[name] = value;
11167
+ }
11168
+ return {
11169
+ status,
11170
+ contentType,
11171
+ body: typeof body === "string" ? body : JSON.stringify(body),
11172
+ headers: {
11173
+ ...headers,
11174
+ ...ORIGIN_SAFE_HEADERS
11175
+ }
11176
+ };
11177
+ }
10662
11178
  const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
10663
11179
  function registrationKey(state) {
10664
11180
  const trigger = runnableDefinition(state).trigger;
@@ -11619,6 +12135,8 @@ var WorkflowRuntimeService = class {
11619
12135
  startTriggerSupervisor() {
11620
12136
  this.supervisor().start();
11621
12137
  this.admitAbandonedRuns();
12138
+ this.deliverAllHeld();
12139
+ this.startWaitpointPoller();
11622
12140
  this.pendingSweepTimer ??= setInterval(() => {
11623
12141
  this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
11624
12142
  }, PENDING_SWEEP_INTERVAL_MS);
@@ -11641,6 +12159,7 @@ var WorkflowRuntimeService = class {
11641
12159
  stopTriggerSupervisor() {
11642
12160
  clearInterval(this.pendingSweepTimer);
11643
12161
  this.pendingSweepTimer = void 0;
12162
+ this.stopWaitpointPoller();
11644
12163
  this.triggerSupervisor?.stop();
11645
12164
  }
11646
12165
  closed = false;
@@ -11848,13 +12367,54 @@ var WorkflowRuntimeService = class {
11848
12367
  }));
11849
12368
  let replied = false;
11850
12369
  const takeReply = () => !replied && (replied = true);
11851
- if (runId) this.heldReplies.set(runId, takeReply);
12370
+ let sent;
12371
+ let responded = () => void 0;
12372
+ const respondedFirst = new Promise((resolve) => {
12373
+ responded = () => resolve(RESPONDED);
12374
+ });
12375
+ const responder = (response) => {
12376
+ if (sent || !takeReply()) return;
12377
+ sent = response;
12378
+ responded();
12379
+ };
12380
+ let ended;
12381
+ let endedEarly = () => void 0;
12382
+ const endedFirst = new Promise((resolve) => {
12383
+ endedEarly = () => resolve({
12384
+ ok: true,
12385
+ result: ended
12386
+ });
12387
+ });
12388
+ const ender = (result) => {
12389
+ if (ended || !takeReply()) return;
12390
+ ended = result;
12391
+ endedEarly();
12392
+ };
12393
+ if (runId) {
12394
+ this.heldReplies.set(runId, takeReply);
12395
+ this.runResponders.set(runId, responder);
12396
+ this.queuedReplies.set(runId, ender);
12397
+ }
11852
12398
  let run;
11853
12399
  try {
11854
- run = await Promise.race([firing, timeout(DELIVERY_TIMEOUT_MS)]);
11855
- if (run === TIMED_OUT && !takeReply()) run = await firing;
12400
+ const raced = await Promise.race([
12401
+ firing,
12402
+ respondedFirst,
12403
+ endedFirst,
12404
+ timeout(DELIVERY_TIMEOUT_MS)
12405
+ ]);
12406
+ if (sent) return runResponseReply(sent);
12407
+ run = raced;
12408
+ if (run === TIMED_OUT && !takeReply()) run = ended ? {
12409
+ ok: true,
12410
+ result: ended
12411
+ } : await firing;
11856
12412
  } finally {
11857
- if (runId) this.heldReplies.delete(runId);
12413
+ if (runId) {
12414
+ this.heldReplies.delete(runId);
12415
+ if (this.runResponders.get(runId) === responder) this.runResponders.delete(runId);
12416
+ if (this.queuedReplies.get(runId) === ender) this.queuedReplies.delete(runId);
12417
+ }
11858
12418
  }
11859
12419
  if (run === TIMED_OUT) {
11860
12420
  this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
@@ -11880,15 +12440,7 @@ var WorkflowRuntimeService = class {
11880
12440
  body: JSON.stringify({ error: message })
11881
12441
  };
11882
12442
  }
11883
- return {
11884
- status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
11885
- contentType: JSON_CONTENT_TYPE,
11886
- body: JSON.stringify({
11887
- runId: run.result.runId,
11888
- status: run.result.status,
11889
- error: run.result.error ?? null
11890
- })
11891
- };
12443
+ return runResultReply(run.result, config.responseStatus);
11892
12444
  }
11893
12445
  async pieceHandshake(binding, request) {
11894
12446
  const handshake = await this.pieceHandshakeConfig(binding);
@@ -12624,6 +13176,61 @@ var WorkflowRuntimeService = class {
12624
13176
  type: "value"
12625
13177
  }]
12626
13178
  };
13179
+ case DELAY_BLOCK: return {
13180
+ source: "static",
13181
+ nodes: [{
13182
+ name: "resumedAt",
13183
+ type: "DateTime!"
13184
+ }]
13185
+ };
13186
+ case APPROVAL_BLOCK: return {
13187
+ source: "static",
13188
+ nodes: [
13189
+ {
13190
+ name: "approved",
13191
+ type: "Boolean!"
13192
+ },
13193
+ {
13194
+ name: "reason",
13195
+ type: "String"
13196
+ },
13197
+ {
13198
+ name: "decidedBy",
13199
+ type: "String",
13200
+ description: "The approver's DID"
13201
+ },
13202
+ {
13203
+ name: "decidedAt",
13204
+ type: "DateTime!"
13205
+ }
13206
+ ]
13207
+ };
13208
+ case STOP_BLOCK: return {
13209
+ source: "static",
13210
+ nodes: [{
13211
+ name: "response",
13212
+ type: "JSONObject",
13213
+ description: "The answer sent, when one was set"
13214
+ }]
13215
+ };
13216
+ case WAIT_FOR_WEBHOOK_BLOCK: return {
13217
+ source: "static",
13218
+ nodes: [
13219
+ {
13220
+ name: "body",
13221
+ type: "Unknown",
13222
+ description: "The resuming request's body"
13223
+ },
13224
+ {
13225
+ name: "headers",
13226
+ type: "JSONObject!"
13227
+ },
13228
+ {
13229
+ name: "queryParams",
13230
+ type: "JSONObject!"
13231
+ }
13232
+ ]
13233
+ };
12627
13234
  case DOCUMENT_CREATED_BLOCK:
12628
13235
  case DOCUMENT_DELETED_BLOCK: return {
12629
13236
  source: "static",
@@ -12879,7 +13486,12 @@ var WorkflowRuntimeService = class {
12879
13486
  pool.dispose();
12880
13487
  return this.pieceWorkers = pool;
12881
13488
  }
12882
- this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
13489
+ const unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
13490
+ const unregisterGauge = registerTelemetryGauge("workflow.pool", () => pool.memoryStats());
13491
+ this.unobservePool = () => {
13492
+ unobservePool();
13493
+ unregisterGauge();
13494
+ };
12883
13495
  return this.pieceWorkers = pool;
12884
13496
  }
12885
13497
  dispatcher() {
@@ -12945,6 +13557,7 @@ var WorkflowRuntimeService = class {
12945
13557
  let definition;
12946
13558
  let runUser;
12947
13559
  let readGeneration;
13560
+ let revision;
12948
13561
  try {
12949
13562
  if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
12950
13563
  run.refused = true;
@@ -12954,6 +13567,7 @@ var WorkflowRuntimeService = class {
12954
13567
  if (cached) {
12955
13568
  state = cached.state;
12956
13569
  documentName = cached.name;
13570
+ revision = cached.revision;
12957
13571
  readGeneration = this.workflowGeneration(workflowId);
12958
13572
  } else {
12959
13573
  const generation = this.workflowGeneration(workflowId);
@@ -12965,6 +13579,7 @@ var WorkflowRuntimeService = class {
12965
13579
  }
12966
13580
  state = document.state.global;
12967
13581
  documentName = document.header.name;
13582
+ revision = globalRevision(document);
12968
13583
  this.cacheLoaded(workflowId, generation, document);
12969
13584
  }
12970
13585
  if (state.status !== "ENABLED") {
@@ -13093,74 +13708,63 @@ var WorkflowRuntimeService = class {
13093
13708
  }
13094
13709
  }
13095
13710
  let runId = enqueuedRunId ?? null;
13096
- let journalFailed = false;
13097
- const executionOrder = /* @__PURE__ */ new Map();
13098
13711
  let session;
13099
13712
  try {
13100
- runId = await this.telemetry.phase("journal.start", {}, async () => {
13713
+ const journaled = await this.telemetry.phase("journal.start", {}, async () => {
13101
13714
  if (enqueuedRunId) {
13102
- await store?.beginRun(enqueuedRunId, {
13715
+ if (await store?.beginRun(enqueuedRunId, {
13103
13716
  workflowName: runJournalName(state.name, documentName),
13104
13717
  workflowVersion: runnable.version
13105
- });
13106
- return enqueuedRunId;
13718
+ }) === false) return {
13719
+ runId: null,
13720
+ cancelled: true
13721
+ };
13722
+ return { runId: enqueuedRunId };
13107
13723
  }
13108
- return await store?.startRun({
13724
+ return { runId: await store?.startRun({
13109
13725
  workflowId,
13110
13726
  workflowName: runJournalName(state.name, documentName),
13111
13727
  workflowVersion: runnable.version,
13112
13728
  triggerKind,
13113
13729
  triggerPayload,
13114
13730
  rerunOf: resume?.rerunOf
13115
- }) ?? null;
13731
+ }) ?? null };
13116
13732
  });
13733
+ if (journaled.cancelled) {
13734
+ const row = enqueuedRunId ? await store?.getRun(enqueuedRunId) : void 0;
13735
+ return {
13736
+ status: CANCELLED_RUN_STATUS,
13737
+ steps: [],
13738
+ runId: enqueuedRunId ?? null,
13739
+ ...row?.error ? { error: row.error } : {},
13740
+ refusal: "cancelled"
13741
+ };
13742
+ }
13743
+ runId = journaled.runId;
13117
13744
  if (runId) run.span.setAttribute("run.id", runId);
13118
13745
  if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
13119
13746
  session = this.workers().session();
13120
- const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider(), workflowId);
13121
- const journal = store;
13122
- const journaledRunId = runId;
13123
- const handed = /* @__PURE__ */ new Set();
13124
- const result = await withRunScope({
13747
+ const { result, handed } = await this.executeLeg({
13125
13748
  workflowId,
13126
13749
  runId,
13750
+ store,
13751
+ session,
13752
+ definition,
13753
+ runnable,
13754
+ policy,
13755
+ deadline,
13127
13756
  connections,
13128
- pieceWorker: session,
13129
- ...runUser !== void 0 ? { runUser } : {},
13130
- attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
13131
- recordDocuments: async (documentIds) => {
13132
- for (const documentId of documentIds) handed.add(documentId);
13133
- if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
13134
- }
13135
- }, () => runWorkflow({
13136
- definition: {
13137
- ...definition,
13138
- variables
13139
- },
13140
- executor,
13141
- declaredPorts: blockPorts,
13142
- blockDefaults: (block) => this.blockDefaults(block),
13143
- ...secretValues.length > 0 ? { redactValues: secretValues } : {},
13757
+ runUser,
13144
13758
  triggerPayload,
13759
+ executor,
13145
13760
  completedSteps: resume?.completedSteps,
13146
- ...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
13147
- ...deadline !== void 0 ? { deadline } : {},
13148
- onStep: store && journaledRunId ? async (record, ordinal) => {
13149
- executionOrder.set(record.stepId, ordinal);
13150
- try {
13151
- await this.telemetry.phase("journal.step", {}, () => store.recordStep(journaledRunId, ordinal, record));
13152
- } catch (error) {
13153
- if (journalFailed) return;
13154
- journalFailed = true;
13155
- this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
13156
- }
13157
- } : void 0
13158
- }));
13159
- if (store && journaledRunId) try {
13160
- await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
13161
- } catch (error) {
13162
- this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
13163
- }
13761
+ replayed: resume?.completedSteps,
13762
+ pin: () => ({
13763
+ definition_hash: this.definitionHash(workflowId, state),
13764
+ definition_op_index: revision,
13765
+ definition_scope_hash: scopeHash(state)
13766
+ })
13767
+ });
13164
13768
  if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
13165
13769
  const finished = {
13166
13770
  ...result,
@@ -13174,7 +13778,14 @@ var WorkflowRuntimeService = class {
13174
13778
  runId
13175
13779
  };
13176
13780
  } catch (error) {
13177
- if (store && runId) await store.failRun(runId, error instanceof Error ? error.message : String(error), errorNameOf(error));
13781
+ if (store && runId) {
13782
+ const message = error instanceof Error ? error.message : String(error);
13783
+ const closed = await store.failRun(runId, message, errorNameOf(error));
13784
+ this.answerClosed(closed, runId, {
13785
+ status: "FAILED",
13786
+ error: message
13787
+ });
13788
+ }
13178
13789
  throw error;
13179
13790
  } finally {
13180
13791
  session?.close();
@@ -13182,6 +13793,115 @@ var WorkflowRuntimeService = class {
13182
13793
  dispatched();
13183
13794
  }
13184
13795
  }
13796
+ async executeLeg(leg) {
13797
+ const { workflowId, runId, store, policy } = leg;
13798
+ let journalFailed = false;
13799
+ const executionOrder = /* @__PURE__ */ new Map();
13800
+ const { variables, secretValues } = await resolveVariables(leg.runnable.variables, this.secretProvider(), workflowId);
13801
+ const handed = new Set(leg.handed);
13802
+ const attachments = attachmentOrigins(leg.triggerPayload, handed, leg.replayed);
13803
+ for (const ref of leg.written ?? []) attachments.written.add(ref);
13804
+ const waitpoints = store && runId ? this.waitpointPort(store, workflowId, runId, maxSuspensionMs(leg.runnable)) : void 0;
13805
+ const respond = leg.respond ?? (runId && !leg.resume ? (response) => this.runResponders.get(runId)?.(response) : void 0);
13806
+ let result = await withRunScope({
13807
+ workflowId,
13808
+ runId,
13809
+ connections: leg.connections,
13810
+ pieceWorker: leg.session,
13811
+ ...leg.runUser !== void 0 ? { runUser: leg.runUser } : {},
13812
+ attachments,
13813
+ recordDocuments: async (documentIds) => {
13814
+ for (const documentId of documentIds) handed.add(documentId);
13815
+ if (store && runId) await store.recordRunDocuments(runId, documentIds);
13816
+ },
13817
+ ...waitpoints ? { waitpoints: waitpoints.port } : {}
13818
+ }, () => runWorkflow({
13819
+ definition: {
13820
+ ...leg.definition,
13821
+ variables
13822
+ },
13823
+ executor: leg.executor,
13824
+ declaredPorts: blockPorts,
13825
+ blockDefaults: (block) => this.blockDefaults(block),
13826
+ ...secretValues.length > 0 ? { redactValues: secretValues } : {},
13827
+ triggerPayload: leg.triggerPayload,
13828
+ completedSteps: leg.completedSteps,
13829
+ ...leg.carriedSteps ? { carriedSteps: leg.carriedSteps } : {},
13830
+ ...leg.resume ? { resume: leg.resume } : {},
13831
+ ...leg.firstOrdinal !== void 0 ? { firstOrdinal: leg.firstOrdinal } : {},
13832
+ ...respond ? { onRespond: respond } : {},
13833
+ ...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
13834
+ ...leg.deadline !== void 0 ? { deadline: leg.deadline } : {},
13835
+ onStep: store && runId ? async (record, ordinal) => {
13836
+ executionOrder.set(record.stepId, ordinal);
13837
+ try {
13838
+ await this.telemetry.phase("journal.step", {}, () => store.recordStep(runId, ordinal, record));
13839
+ } catch (error) {
13840
+ if (journalFailed) return;
13841
+ journalFailed = true;
13842
+ this.logger.warn(`Run ${runId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
13843
+ }
13844
+ } : void 0
13845
+ }));
13846
+ if (store && runId) {
13847
+ let waiting = result.status === WAITING_RUN_STATUS;
13848
+ try {
13849
+ const { held, closed } = await this.telemetry.phase("journal.finish", {}, () => store.finishRun(runId, result, executionOrder, waiting ? leg.pin?.() : void 0));
13850
+ this.answerClosed(closed, runId, {
13851
+ status: waiting ? CANCELLED_RUN_STATUS : result.status,
13852
+ error: waiting ? "The step that created this waitpoint no longer waits on it" : result.error ?? "The run ended without resuming on it"
13853
+ });
13854
+ if (!held) {
13855
+ result = await this.endUnheldWait(store, runId, result);
13856
+ waiting = false;
13857
+ }
13858
+ } catch (error) {
13859
+ this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
13860
+ }
13861
+ if (waiting) {
13862
+ const toSend = result.waiting && waitpoints?.responseFor(result.waiting.waitpointId);
13863
+ if (toSend) respond?.(toSend);
13864
+ await this.deliverHeld(runId);
13865
+ }
13866
+ }
13867
+ return {
13868
+ result,
13869
+ handed
13870
+ };
13871
+ }
13872
+ async endUnheldWait(store, runId, result) {
13873
+ const on = result.waiting;
13874
+ const step = result.steps.find((record) => record.stepId === on?.stepId);
13875
+ const row = on ? await store.getWaitpoint(on.waitpointId) : void 0;
13876
+ const end = row?.run_id === runId && row.step_id === on?.stepId && row.status === "EXPIRED" ? {
13877
+ status: "CANCELLED",
13878
+ reason: `The wait at step "${row.step_key}" expired at ${row.expires_at} before the run could wait on it`,
13879
+ errorName: WAITPOINT_EXPIRED_ERROR_NAME
13880
+ } : {
13881
+ status: "FAILED",
13882
+ reason: `Step "${step?.key ?? on?.stepId}" waited on waitpoint "${on?.waitpointId}", which is not a live waitpoint it created`,
13883
+ errorName: WAITPOINT_NOT_LIVE_ERROR_NAME
13884
+ };
13885
+ const closed = await store.endWait(runId, {
13886
+ ...end,
13887
+ waitpoints: "EXPIRED"
13888
+ });
13889
+ this.answerClosed(closed, runId, {
13890
+ status: end.status,
13891
+ error: end.reason
13892
+ });
13893
+ return {
13894
+ status: end.status,
13895
+ error: end.reason,
13896
+ errorName: end.errorName,
13897
+ steps: result.steps.map((record) => record === step ? {
13898
+ ...record,
13899
+ status: "FAILED",
13900
+ error: end.reason,
13901
+ errorName: end.errorName
13902
+ } : record)
13903
+ };
13904
+ }
13185
13905
  runGate = new WorkflowRunGate();
13186
13906
  unsettledFirings = /* @__PURE__ */ new Map();
13187
13907
  runLimits = /* @__PURE__ */ new Map();
@@ -13206,6 +13926,9 @@ var WorkflowRuntimeService = class {
13206
13926
  else this.unsettledFirings.delete(workflowId);
13207
13927
  }
13208
13928
  heldReplies = /* @__PURE__ */ new Map();
13929
+ runResponders = /* @__PURE__ */ new Map();
13930
+ queuedReplies = /* @__PURE__ */ new Map();
13931
+ syncResumes = /* @__PURE__ */ new Map();
13209
13932
  /** Why a trigger's firing of a PARKED workflow is refused; undefined when
13210
13933
  * it is not parked, or when an operator started the run. */
13211
13934
  async parkedFiring(workflowId, triggerKind, publishedVersion) {
@@ -13273,12 +13996,23 @@ var WorkflowRuntimeService = class {
13273
13996
  this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
13274
13997
  let runId = enqueuedRunId ?? null;
13275
13998
  if (store) try {
13276
- if (runId) await store.beginRun(runId, details);
13999
+ let adopted = true;
14000
+ if (runId) adopted = await store.beginRun(runId, details);
13277
14001
  else runId = await store.startRun({
13278
14002
  workflowId,
13279
14003
  ...details
13280
14004
  });
13281
- await store.cancelRun(runId, details.reason);
14005
+ if (adopted) await store.cancelRun(runId, details.reason);
14006
+ else {
14007
+ const row = await store.getRun(runId);
14008
+ if (row) return {
14009
+ status: row.status,
14010
+ steps: [],
14011
+ runId,
14012
+ ...row.error ? { error: row.error } : {},
14013
+ refusal: row.status === "CANCELLED" ? "cancelled" : refusal
14014
+ };
14015
+ }
13282
14016
  } catch (error) {
13283
14017
  this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
13284
14018
  }
@@ -13374,6 +14108,677 @@ var WorkflowRuntimeService = class {
13374
14108
  const parsed = resolvedBlock(await this.resolveBlock(stepBlock(step)));
13375
14109
  return parsed ? await this.declaredReactor(parsed) === "read" : false;
13376
14110
  }
14111
+ resuming = /* @__PURE__ */ new Set();
14112
+ definitionHashes = /* @__PURE__ */ new Map();
14113
+ /** Our hash of the runnable definition: computed once per published version. */
14114
+ definitionHash(workflowId, state) {
14115
+ const runnable = runnableDefinition(state);
14116
+ if (!runnable.published) return hashOf(canonicalJson(runnable));
14117
+ const key = `${workflowId}@${runnable.version}`;
14118
+ let hash = this.definitionHashes.get(key);
14119
+ if (hash === void 0) {
14120
+ hash = hashOf(canonicalJson(runnable));
14121
+ if (this.definitionHashes.size >= WORKFLOW_CACHE_MAX) this.definitionHashes.clear();
14122
+ this.definitionHashes.set(key, hash);
14123
+ }
14124
+ return hash;
14125
+ }
14126
+ waitpointPort(store, workflowId, runId, suspendMs) {
14127
+ const responses = /* @__PURE__ */ new Map();
14128
+ const tokens = /* @__PURE__ */ new Map();
14129
+ const configured = this.host.waitpointBaseUrl?.replace(/\/+$/, "");
14130
+ const baseUrl = configured && URL.canParse(configured) ? configured : void 0;
14131
+ return {
14132
+ port: {
14133
+ ...baseUrl ? { resumeBaseUrl: baseUrl } : {},
14134
+ create: async (request) => {
14135
+ if (!baseUrl && request.type === "WEBHOOK" && !request.approval) throw new Error("This host serves no resume URLs: resume URLs need an HTTP host");
14136
+ const createdAt = /* @__PURE__ */ new Date();
14137
+ const resumeAt = request.resumeDateTime ? new Date(Date.parse(request.resumeDateTime)).toISOString() : null;
14138
+ if (resumeAt && Date.parse(resumeAt) > MAX_WAITPOINT_MS) throw new Error(`A delay until ${resumeAt} is past ${MAX_WAITPOINT_ISO}, the latest a run can wait until`);
14139
+ const expiresAt = waitpointExpiry(createdAt, request, suspendMs);
14140
+ if (request.type === "DELAY" && resumeAt && suspendMs !== null) {
14141
+ if (Date.parse(resumeAt) - createdAt.getTime() > suspendMs) throw new Error(`A delay until ${resumeAt} is longer than the workflow's maxSuspensionDays allows`);
14142
+ }
14143
+ const token = baseUrl && !request.approval && request.type !== "DELAY" ? request.token ?? randomBytes(32).toString("base64url") : null;
14144
+ const id = randomUUID();
14145
+ const { waitpoint, created: inserted } = await store.createWaitpoint({
14146
+ id,
14147
+ run_id: runId,
14148
+ workflow_id: workflowId,
14149
+ step_id: request.stepId,
14150
+ step_key: request.stepKey,
14151
+ step_path: request.stepId,
14152
+ piece_name: request.pieceName,
14153
+ block_name: request.blockName,
14154
+ type: request.type,
14155
+ kind: request.approval ? "approval" : request.type === "DELAY" ? "delay" : "webhook",
14156
+ status: "PENDING",
14157
+ token_hash: token ? tokenHash(token) : null,
14158
+ created_at: createdAt.toISOString(),
14159
+ resume_at: resumeAt,
14160
+ expires_at: request.type === "DELAY" ? null : expiresAt,
14161
+ resolved_at: null,
14162
+ decided_by: null,
14163
+ approval: request.approval ? JSON.stringify(request.approval) : null,
14164
+ resume_payload: null,
14165
+ response_to_send: request.responseToSend ? JSON.stringify(request.responseToSend) : null
14166
+ });
14167
+ if (!inserted) {
14168
+ const known = tokens.get(waitpoint.id);
14169
+ if (waitpoint.token_hash !== null && known === void 0) throw new Error(`Step "${request.stepKey}" has a waitpoint from an earlier leg, whose resume URL cannot be issued again`);
14170
+ return {
14171
+ id: waitpoint.id,
14172
+ resumeUrl: known ? `${baseUrl}/${known}` : ""
14173
+ };
14174
+ }
14175
+ if (token) tokens.set(id, token);
14176
+ if (request.responseToSend) responses.set(id, request.responseToSend);
14177
+ const due = request.type === "DELAY" ? resumeAt : expiresAt;
14178
+ if (due) this.waitpointPoller.nudge(due);
14179
+ return {
14180
+ id,
14181
+ resumeUrl: token ? `${baseUrl}/${token}` : ""
14182
+ };
14183
+ }
14184
+ },
14185
+ responseFor: (waitpointId) => responses.get(waitpointId)
14186
+ };
14187
+ }
14188
+ /** Re-enters a WAITING run that holds a COMPLETED waitpoint, through the
14189
+ * dispatcher. Whoever writes second, the resume or the WAITING, delivers. */
14190
+ async deliverHeld(runId) {
14191
+ if (this.closed) return this.answerShutDown(runId);
14192
+ if (this.resuming.has(runId)) return;
14193
+ try {
14194
+ const store = await this.store();
14195
+ if (!store || !await store.hasCompletedWaitpoint(runId)) return;
14196
+ const run = await store.getRun(runId);
14197
+ if (run?.status !== "WAITING") return this.answerNotWaiting(store, runId, run?.status);
14198
+ if (this.resuming.has(runId)) return;
14199
+ this.resuming.add(runId);
14200
+ this.resumeRun(runId, run.workflow_id);
14201
+ } catch (error) {
14202
+ this.logger.warn(`Could not resume run ${runId}: @error`, error);
14203
+ }
14204
+ }
14205
+ answerNotWaiting(store, runId, status) {
14206
+ if (store.isRunInFlight(runId)) return;
14207
+ this.answerUnresumed(runId, {
14208
+ status: status ?? "gone",
14209
+ error: "The run is not waiting to resume in this runtime",
14210
+ code: 409
14211
+ });
14212
+ }
14213
+ waitpointPoller = new WaitpointPoller({
14214
+ store: () => this.store(),
14215
+ resume: (runId) => this.deliverHeld(runId),
14216
+ ended: (end) => Promise.resolve(this.answerClosed(end.closed, end.runId, {
14217
+ status: CANCELLED_RUN_STATUS,
14218
+ error: end.reason
14219
+ })),
14220
+ logger: { warn: (message, ...args) => this.logger.warn(message, ...args) }
14221
+ });
14222
+ unobserveSuspended;
14223
+ /** Test hook: one waitpoint poll pass now; a no-op until the supervisor starts. */
14224
+ pollWaitpointsNow() {
14225
+ return this.waitpointPoller.poll();
14226
+ }
14227
+ startWaitpointPoller() {
14228
+ if (this.closed) return;
14229
+ this.unobserveSuspended ??= this.telemetry.observeSuspended(() => this.waitpointPoller.suspendedRuns());
14230
+ this.waitpointPoller.start();
14231
+ }
14232
+ stopWaitpointPoller() {
14233
+ this.waitpointPoller.stop();
14234
+ this.unobserveSuspended?.();
14235
+ this.unobserveSuspended = void 0;
14236
+ }
14237
+ async answerHeldResumes(store, runId, error) {
14238
+ if (this.syncResumes.size === 0) return;
14239
+ const rows = (await store.listWaitpointsForRuns([runId])).get(runId) ?? [];
14240
+ this.answerClosed(rows.map((row) => row.id), runId, {
14241
+ status: CANCELLED_RUN_STATUS,
14242
+ error
14243
+ });
14244
+ }
14245
+ answerClosed(waitpointIds, runId, end) {
14246
+ if (this.syncResumes.size === 0) return;
14247
+ for (const id of waitpointIds) this.heldResume(id)?.closed(runId, end.status, end.error);
14248
+ }
14249
+ heldResume(waitpointId) {
14250
+ const candidates = [...this.syncResumes.get(waitpointId) ?? []];
14251
+ if (candidates.length === 0) return void 0;
14252
+ const winner = (async () => {
14253
+ for (const candidate of candidates) if (await candidate.won) return candidate.entry;
14254
+ })();
14255
+ return {
14256
+ respond: (response) => void winner.then((e) => e?.respond(response)),
14257
+ ended: (result) => void winner.then((e) => e?.ended(result)),
14258
+ closed: (runId, status, error, code) => void winner.then((e) => e?.closed(runId, status, error, code))
14259
+ };
14260
+ }
14261
+ answerUnresumed(runId, end) {
14262
+ for (const [id, candidates] of this.syncResumes) {
14263
+ if (![...candidates].some((candidate) => candidate.runId === runId)) continue;
14264
+ this.heldResume(id)?.closed(runId, end.status, end.error, end.code);
14265
+ }
14266
+ }
14267
+ async deliverAllHeld() {
14268
+ try {
14269
+ const store = await this.store();
14270
+ for (const runId of await store?.heldRunIds() ?? []) await this.deliverHeld(runId);
14271
+ } catch (error) {
14272
+ this.logger.warn("Could not deliver held resumes: @error", error);
14273
+ }
14274
+ }
14275
+ resumeRun(runId, workflowId) {
14276
+ const started = performance.now();
14277
+ const attributes = {
14278
+ "workflow.id": workflowId,
14279
+ "trigger.kind": "resume",
14280
+ "run.id": runId
14281
+ };
14282
+ return this.telemetry.span("workflow.run", attributes, async (span) => {
14283
+ let status = "REFUSED";
14284
+ try {
14285
+ const result = await this.runResumed(runId, workflowId);
14286
+ if (result) status = result.status;
14287
+ if (status === "FAILED") span.setStatus({
14288
+ code: SpanStatusCode.ERROR,
14289
+ message: status
14290
+ });
14291
+ return result;
14292
+ } finally {
14293
+ span.setAttribute("run.status", status);
14294
+ this.telemetry.recordRun(performance.now() - started, status, "resume");
14295
+ }
14296
+ });
14297
+ }
14298
+ async runResumed(runId, workflowId) {
14299
+ const admission = await this.runGate.admit(workflowId, await this.resumePolicy(workflowId), void 0, { resume: true });
14300
+ if (!admission.admitted) {
14301
+ this.resuming.delete(runId);
14302
+ this.answerShutDown(runId);
14303
+ return;
14304
+ }
14305
+ let dispatched;
14306
+ try {
14307
+ dispatched = await this.dispatcher().admit(workflowId);
14308
+ } catch {
14309
+ admission.release();
14310
+ this.resuming.delete(runId);
14311
+ this.answerShutDown(runId);
14312
+ return;
14313
+ }
14314
+ const store = await this.store();
14315
+ let claimed = false;
14316
+ let redeliver = false;
14317
+ let session;
14318
+ let held;
14319
+ let outcome;
14320
+ try {
14321
+ if (!store || this.closed) {
14322
+ this.answerShutDown(runId);
14323
+ return;
14324
+ }
14325
+ claimed = await store.claimWaitingRun(runId);
14326
+ this.resuming.delete(runId);
14327
+ if (!claimed) {
14328
+ this.answerNotWaiting(store, runId, (await store.getRun(runId))?.status);
14329
+ return;
14330
+ }
14331
+ const row = await store.getRun(runId);
14332
+ if (!row) throw new Error(`Run "${runId}" not found`);
14333
+ const generation = this.workflowGeneration(workflowId);
14334
+ const document = await this.host.reactorClient.get(workflowId);
14335
+ if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
14336
+ const current = document.state.global;
14337
+ const stopped = await this.resumeStopped(workflowId, row.trigger_kind, current);
14338
+ if (stopped) {
14339
+ const closed = await store.endWait(runId, {
14340
+ status: CANCELLED_RUN_STATUS,
14341
+ reason: stopped.reason,
14342
+ errorName: stopped.errorName,
14343
+ waitpoints: "CANCELLED"
14344
+ });
14345
+ claimed = false;
14346
+ this.answerClosed(closed, runId, {
14347
+ status: CANCELLED_RUN_STATUS,
14348
+ error: stopped.reason
14349
+ });
14350
+ return {
14351
+ status: CANCELLED_RUN_STATUS,
14352
+ steps: [],
14353
+ runId,
14354
+ error: stopped.reason,
14355
+ errorName: stopped.errorName,
14356
+ refusal: "cancelled"
14357
+ };
14358
+ }
14359
+ const consumed = await store.consumeCompletedWaitpoint(runId);
14360
+ if (!consumed) {
14361
+ await store.returnToWaiting(runId);
14362
+ claimed = false;
14363
+ redeliver = true;
14364
+ return;
14365
+ }
14366
+ const { waitpoint } = consumed;
14367
+ held = this.heldResume(waitpoint.id);
14368
+ this.answerClosed(consumed.closed, runId, {
14369
+ status: "RUNNING",
14370
+ error: "The run resumed on another of its waitpoints"
14371
+ });
14372
+ const state = await this.pinnedState(workflowId, runId, store, current);
14373
+ const runnable = runnableDefinition(state);
14374
+ const definition = toWorkflowDefinition(state);
14375
+ const policy = effectiveRunPolicy(runnable);
14376
+ const connections = declaredConnectionIds(definition);
14377
+ const reactorConnections = declaredReactorConnectionIds(definition);
14378
+ const runUser = reactorConnections.size === 0 ? void 0 : await assertReactorConnectionsReadable(this.host, workflowId, reactorConnections);
14379
+ const journal = carriedFromJournal(await store.getSteps(runId));
14380
+ if (!journal.waitingStepId) throw new Error(`Run "${runId}" has no waiting step to resume`);
14381
+ session = this.workers().session();
14382
+ const { result } = await this.executeLeg({
14383
+ workflowId,
14384
+ runId,
14385
+ store,
14386
+ session,
14387
+ definition,
14388
+ runnable,
14389
+ policy,
14390
+ deadline: policy.runTimeoutSeconds ? Date.now() + policy.runTimeoutSeconds * 1e3 : void 0,
14391
+ connections,
14392
+ runUser,
14393
+ triggerPayload: journaledTriggerPayload(row.trigger_payload),
14394
+ executor: this.blockExecutor(store),
14395
+ carriedSteps: journal.carried,
14396
+ resume: {
14397
+ stepId: journal.waitingStepId,
14398
+ payload: parseJson(waitpoint.resume_payload) ?? {}
14399
+ },
14400
+ firstOrdinal: journal.nextOrdinal,
14401
+ written: journal.attachmentRefs,
14402
+ handed: await store.getRunDocuments(runId),
14403
+ ...held ? { respond: held.respond } : {}
14404
+ });
14405
+ if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(row.trigger_kind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
14406
+ outcome = {
14407
+ ...result,
14408
+ runId
14409
+ };
14410
+ return outcome;
14411
+ } catch (error) {
14412
+ if (store && claimed) {
14413
+ const message = error instanceof Error ? error.message : String(error);
14414
+ const errorName = errorNameOf(error) ?? "Error";
14415
+ const closed = await store.endWait(runId, {
14416
+ status: "FAILED",
14417
+ reason: message,
14418
+ errorName,
14419
+ waitpoints: "CANCELLED"
14420
+ }).catch(() => []);
14421
+ this.answerClosed(closed, runId, {
14422
+ status: "FAILED",
14423
+ error: message
14424
+ });
14425
+ outcome = {
14426
+ status: "FAILED",
14427
+ steps: [],
14428
+ runId,
14429
+ error: message,
14430
+ errorName
14431
+ };
14432
+ }
14433
+ this.logger.warn(`Resuming run ${runId} failed: @error`, error);
14434
+ return;
14435
+ } finally {
14436
+ this.resuming.delete(runId);
14437
+ session?.close();
14438
+ dispatched();
14439
+ admission.release();
14440
+ held?.ended(outcome);
14441
+ if (redeliver) this.deliverHeld(runId);
14442
+ }
14443
+ }
14444
+ answerShutDown(runId) {
14445
+ if (!this.closed) return;
14446
+ this.answerUnresumed(runId, {
14447
+ status: WAITING_RUN_STATUS,
14448
+ error: "The runtime shut down before the run could resume",
14449
+ code: 503
14450
+ });
14451
+ }
14452
+ async resumePolicy(workflowId) {
14453
+ try {
14454
+ let state = this.workflowCache.get(workflowId)?.state;
14455
+ if (!state) {
14456
+ const generation = this.workflowGeneration(workflowId);
14457
+ const document = await this.host.reactorClient.get(workflowId);
14458
+ if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
14459
+ state = document.state.global;
14460
+ }
14461
+ return effectiveRunPolicy(runnableDefinition(state));
14462
+ } catch {
14463
+ return UNENFORCED_POLICY;
14464
+ }
14465
+ }
14466
+ async resumeStopped(workflowId, triggerKind, current) {
14467
+ if (current.status !== "ENABLED") return {
14468
+ reason: `The workflow is ${current.status}; a run waiting on it does not resume`,
14469
+ errorName: WORKFLOW_DISABLED_ERROR_NAME
14470
+ };
14471
+ const parked = await this.parkedFiring(workflowId, triggerKind, runnableDefinition(current).version);
14472
+ return parked ? {
14473
+ reason: parked,
14474
+ errorName: WORKFLOW_PARKED_ERROR_NAME
14475
+ } : void 0;
14476
+ }
14477
+ async pinnedState(workflowId, runId, store, current) {
14478
+ const pin = await store.getRunPin(runId);
14479
+ if (!pin?.definition_hash) return current;
14480
+ if (this.definitionHash(workflowId, current) === pin.definition_hash) return current;
14481
+ return this.rebuildWorkflowState(workflowId, pin);
14482
+ }
14483
+ async rebuildWorkflowState(workflowId, pin) {
14484
+ const revision = pin.definition_op_index ?? 0;
14485
+ const operations = [];
14486
+ let page = await this.host.reactorClient.getOperations(workflowId, { scopes: ["document", "global"] });
14487
+ operations.push(...page.results);
14488
+ while (page.next) {
14489
+ page = await page.next();
14490
+ operations.push(...page.results);
14491
+ }
14492
+ const applied = (operation) => !operation.error && !isDenied(operation);
14493
+ const past = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index >= revision)).at(0);
14494
+ const atOrBeforePin = (operation) => {
14495
+ const at = operation.action.input.revision?.global;
14496
+ if (at !== void 0) return at <= revision;
14497
+ return !past || operation.timestampUtcMs <= past.timestampUtcMs;
14498
+ };
14499
+ const seed = sortOperations(operations.filter((operation) => operation.action.scope === "document" && operation.action.type === "UPGRADE_DOCUMENT" && applied(operation) && atOrBeforePin(operation))).map((operation) => operation.action.input.initialState).filter((state) => state?.global !== void 0).at(-1);
14500
+ let document = utils.createDocument(seed);
14501
+ const global = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index < revision));
14502
+ for (const operation of garbageCollect(global).filter(applied)) document = reducer(document, operation.action);
14503
+ const state = document.state.global;
14504
+ if (scopeHash(state) !== pin.definition_scope_hash || this.definitionHash(workflowId, state) !== pin.definition_hash) throw new Error(`The workflow was re-published while this run waited, and the definition it started from could not be rebuilt at revision ${revision}`);
14505
+ return state;
14506
+ }
14507
+ /** Resolves a WEBHOOK waitpoint from the editor, as the calling DID. */
14508
+ async resolveWaitpoint(waitpointId, input, ctx) {
14509
+ const store = await this.store();
14510
+ if (!store) throw new Error("Run journal is unavailable");
14511
+ const waitpoint = await store.getWaitpoint(waitpointId);
14512
+ if (!waitpoint) throw new Error(`Waitpoint "${waitpointId}" not found`);
14513
+ await this.assertCanReadDocument(waitpoint.workflow_id, ctx);
14514
+ await this.assertCanWriteDocument(waitpoint.workflow_id, ctx);
14515
+ const owner = await store.getRun(waitpoint.run_id);
14516
+ if (!owner || (await this.servedRuns([owner], ctx)).length === 0) throw new Error(`Waitpoint "${waitpointId}" not found`);
14517
+ if (waitpoint.type !== "WEBHOOK") throw new Error(`Waitpoint "${waitpointId}" is a ${waitpoint.type}`);
14518
+ const decidedBy = this.callerDid(ctx);
14519
+ const approval = parseJson(waitpoint.approval);
14520
+ if (approval && approval.approvers.length > 0) {
14521
+ if (!(decidedBy !== void 0 && approval.approvers.some((approver) => sameApprover(approver, decidedBy)))) throw new Error("You are not one of this approval's approvers");
14522
+ }
14523
+ const reason = input.reason?.trim() || void 0;
14524
+ if ((approval?.reasonRequiredOn === "both" || approval?.reasonRequiredOn === "reject" && !input.approved) && !reason) throw new Error("A reason is required for this decision");
14525
+ const decidedAt = (/* @__PURE__ */ new Date()).toISOString();
14526
+ const payload = {
14527
+ queryParams: {
14528
+ action: input.approved ? "approve" : "disapprove",
14529
+ ...reason ? { reason } : {}
14530
+ },
14531
+ body: {
14532
+ approved: input.approved,
14533
+ reason: reason ?? null,
14534
+ decidedBy: decidedBy ?? null,
14535
+ decidedAt
14536
+ },
14537
+ headers: {}
14538
+ };
14539
+ const completed = await store.completeWaitpoint(waitpointId, {
14540
+ payload,
14541
+ decidedBy: decidedBy ?? null,
14542
+ resolvedAt: decidedAt
14543
+ });
14544
+ if (!completed) {
14545
+ const now = await store.getWaitpoint(waitpointId);
14546
+ const run = await store.getRun(waitpoint.run_id);
14547
+ const live = run?.status === "RUNNING" || run?.status === "WAITING";
14548
+ const status = now?.status !== "PENDING" ? now?.status ?? "gone" : waitpointExpired(now) ? "EXPIRED" : live ? "not the one its run waits on" : `on a ${run?.status ?? "deleted"} run`;
14549
+ throw new Error(`Waitpoint "${waitpointId}" is ${status}; it can no longer be resolved`);
14550
+ }
14551
+ await this.deliverHeld(completed.run_id);
14552
+ return waitpointRecord(completed);
14553
+ }
14554
+ /** A POST to a resume URL. Single-use: only a PENDING waitpoint resumes. */
14555
+ resumeWaitpointByToken(token, payload) {
14556
+ return this.resumeByToken(token, payload);
14557
+ }
14558
+ /** A POST to a resume URL's `/sync`, held like a sync webhook until the
14559
+ * resumed leg responds, waits, ends, or times out. The first answer wins. */
14560
+ async resumeWaitpointByTokenSync(token, payload, options = {}) {
14561
+ let hold;
14562
+ try {
14563
+ const { status } = await this.resumeByToken(token, payload, (id, runId) => {
14564
+ hold = this.holdSyncResume(id, runId);
14565
+ return hold.decided;
14566
+ });
14567
+ if (status !== "resumed" || !hold) return { status };
14568
+ return {
14569
+ status,
14570
+ reply: await hold.reply(options.timeoutMs ?? DELIVERY_TIMEOUT_MS)
14571
+ };
14572
+ } finally {
14573
+ hold?.release();
14574
+ }
14575
+ }
14576
+ holdSyncResume(waitpointId, runId) {
14577
+ let settle = () => void 0;
14578
+ const answer = new Promise((resolve) => {
14579
+ settle = resolve;
14580
+ });
14581
+ let done = false;
14582
+ const finish = (reply) => {
14583
+ if (done) return;
14584
+ done = true;
14585
+ settle(reply);
14586
+ };
14587
+ const entry = {
14588
+ respond: (response) => finish(runResponseReply(response)),
14589
+ ended: (result) => finish(result ? runResultReply(result, 200) : {
14590
+ status: 500,
14591
+ contentType: JSON_CONTENT_TYPE,
14592
+ body: JSON.stringify({ error: "The run could not resume" })
14593
+ }),
14594
+ closed: (runId, status, error, code = 409) => finish({
14595
+ status: code,
14596
+ contentType: JSON_CONTENT_TYPE,
14597
+ body: JSON.stringify({
14598
+ runId,
14599
+ status,
14600
+ error
14601
+ })
14602
+ })
14603
+ };
14604
+ let decided = () => void 0;
14605
+ const candidate = {
14606
+ entry,
14607
+ runId,
14608
+ won: new Promise((resolve) => {
14609
+ decided = resolve;
14610
+ })
14611
+ };
14612
+ const held = this.syncResumes.get(waitpointId) ?? /* @__PURE__ */ new Set();
14613
+ this.syncResumes.set(waitpointId, held);
14614
+ held.add(candidate);
14615
+ return {
14616
+ decided,
14617
+ reply: async (timeoutMs) => {
14618
+ const raced = await Promise.race([answer, timeout(timeoutMs)]);
14619
+ if (raced !== TIMED_OUT) return raced;
14620
+ done = true;
14621
+ return {
14622
+ status: 504,
14623
+ contentType: JSON_CONTENT_TYPE,
14624
+ body: JSON.stringify({
14625
+ status: "RUNNING",
14626
+ error: "The run did not finish in time"
14627
+ })
14628
+ };
14629
+ },
14630
+ release: () => {
14631
+ decided(false);
14632
+ held.delete(candidate);
14633
+ if (held.size === 0 && this.syncResumes.get(waitpointId) === held) this.syncResumes.delete(waitpointId);
14634
+ }
14635
+ };
14636
+ }
14637
+ async resumeByToken(token, payload, beforeComplete) {
14638
+ const store = await this.store();
14639
+ if (!store || !token) return { status: "gone" };
14640
+ const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
14641
+ if (!waitpoint || waitpoint.kind !== "webhook") return { status: "gone" };
14642
+ let completed;
14643
+ if (waitpoint.status === "PENDING") {
14644
+ const decided = beforeComplete?.(waitpoint.id, waitpoint.run_id);
14645
+ try {
14646
+ completed = await store.completeWaitpoint(waitpoint.id, { payload });
14647
+ } finally {
14648
+ decided?.(completed !== void 0);
14649
+ }
14650
+ }
14651
+ if (!completed) {
14652
+ const now = (await store.getWaitpoint(waitpoint.id))?.status;
14653
+ if (now === "CONSUMED") return { status: "already" };
14654
+ const run = now === "COMPLETED" ? await store.getRun(waitpoint.run_id) : void 0;
14655
+ return { status: run?.status === "RUNNING" || run?.status === "WAITING" ? "already" : "gone" };
14656
+ }
14657
+ await this.deliverHeld(completed.run_id);
14658
+ return { status: "resumed" };
14659
+ }
14660
+ /** What a resume URL's confirm page shows. */
14661
+ async describeWaitpointByToken(token) {
14662
+ const store = await this.store();
14663
+ if (!store || !token) return void 0;
14664
+ const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
14665
+ if (!waitpoint || waitpoint.kind !== "webhook") return void 0;
14666
+ const run = await store.getRun(waitpoint.run_id);
14667
+ const status = !(run?.status === "RUNNING" || run?.status === "WAITING") && (waitpoint.status === "PENDING" || waitpoint.status === "COMPLETED") ? "CANCELLED" : waitpoint.status;
14668
+ const state = await this.workflowStateForPage(waitpoint.workflow_id);
14669
+ const step = state?.steps.find((entry) => entry.id === waitpoint.step_id);
14670
+ const action = step ? await this.blockDisplayName({
14671
+ pieceName: step.pieceName,
14672
+ pieceVersion: step.pieceVersion,
14673
+ kind: "action",
14674
+ name: step.actionName
14675
+ }) : void 0;
14676
+ const trigger = state?.trigger ? await this.blockDisplayName({
14677
+ pieceName: state.trigger.pieceName,
14678
+ pieceVersion: state.trigger.pieceVersion,
14679
+ kind: "trigger",
14680
+ name: state.trigger.triggerName
14681
+ }) ?? state.trigger.triggerName : void 0;
14682
+ return {
14683
+ workflowName: run?.workflow_name || state?.name || "",
14684
+ stepName: step?.name || waitpoint.step_key,
14685
+ status,
14686
+ ...state?.description ? { workflowDescription: state.description } : {},
14687
+ ...action ? { actionName: action } : {},
14688
+ waitingSince: waitpoint.created_at,
14689
+ ...waitpoint.expires_at ? { expiresAt: waitpoint.expires_at } : {},
14690
+ ...run ? { startedAt: run.enqueued_at } : {},
14691
+ ...trigger ? { triggerName: trigger } : {}
14692
+ };
14693
+ }
14694
+ async workflowStateForPage(workflowId) {
14695
+ const cached = this.workflowCache.get(workflowId)?.state;
14696
+ if (cached) return cached;
14697
+ try {
14698
+ const generation = this.workflowGeneration(workflowId);
14699
+ const document = await this.host.reactorClient.get(workflowId);
14700
+ if (document.header.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
14701
+ this.cacheLoaded(workflowId, generation, document);
14702
+ return document.state.global;
14703
+ } catch {
14704
+ return;
14705
+ }
14706
+ }
14707
+ async blockDisplayName(block) {
14708
+ try {
14709
+ const parsed = resolvedBlock(await this.resolveBlock(block));
14710
+ if (!parsed) return void 0;
14711
+ const descriptor = await this.pieceDescriptor(targetOf(parsed));
14712
+ return (block.kind === "trigger" ? descriptor.triggers : descriptor.actions).find((entry) => entry.name === parsed.name)?.displayName;
14713
+ } catch {
14714
+ return;
14715
+ }
14716
+ }
14717
+ /** The waitpoints of one run, for a caller who may read the run. */
14718
+ async waitpointsForRun(runId, ctx) {
14719
+ const store = await this.store();
14720
+ if (!store || !await this.run(runId, ctx)) return [];
14721
+ return ((await store.listWaitpointsForRuns([runId])).get(runId) ?? []).map(waitpointRecord);
14722
+ }
14723
+ /** WAITING runs this caller may read, oldest first, with their waitpoints. */
14724
+ async waitingRuns(args, ctx) {
14725
+ const store = await this.store();
14726
+ if (!store || !ctx) return [];
14727
+ let scope;
14728
+ if (args.workflowId) {
14729
+ await this.assertCanReadDocument(args.workflowId, ctx);
14730
+ scope = args.workflowId;
14731
+ } else if (args.driveId) {
14732
+ scope = await this.driveWorkflowIds(args.driveId, ctx);
14733
+ if (scope.length === 0) return [];
14734
+ }
14735
+ const rows = await this.servedRuns(await this.readableRows(await store.listWaitingRuns(scope, args.first ?? 50), (row) => row.workflow_id, ctx), ctx);
14736
+ const ids = rows.map((row) => row.id);
14737
+ const steps = await store.getStepsForRuns(ids, { withData: false });
14738
+ const waitpoints = await store.listWaitpointsForRuns(ids);
14739
+ return rows.map((row) => ({
14740
+ run: {
14741
+ row,
14742
+ steps: steps.get(row.id) ?? []
14743
+ },
14744
+ waitpoints: (waitpoints.get(row.id) ?? []).map(waitpointRecord)
14745
+ }));
14746
+ }
14747
+ /** Cancels a WAITING or PENDING run; its waitpoints stop resuming. */
14748
+ async cancelRun(runId, ctx) {
14749
+ const store = await this.store();
14750
+ if (!store) throw new Error("Run journal is unavailable");
14751
+ const run = await store.getRun(runId);
14752
+ if (!run) throw new Error(`Run "${runId}" not found`);
14753
+ await this.assertCanReadDocument(run.workflow_id, ctx);
14754
+ await this.assertCanWriteDocument(run.workflow_id, ctx);
14755
+ if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
14756
+ const refuse = (status) => status === "RUNNING" ? /* @__PURE__ */ new Error("Cancelling a RUNNING run is not supported yet (#3226)") : /* @__PURE__ */ new Error(`Run is ${status}; only WAITING and PENDING runs can be cancelled`);
14757
+ if (run.status !== "WAITING" && run.status !== "PENDING") throw refuse(run.status);
14758
+ const by = this.callerDid(ctx);
14759
+ const reason = by ? `Cancelled by ${by}` : "Cancelled";
14760
+ if (!await store.cancelIdleRun(runId, reason)) throw refuse((await store.getRun(runId))?.status ?? "gone");
14761
+ await this.answerHeldResumes(store, runId, reason);
14762
+ const row = await store.getRun(runId);
14763
+ this.queuedReplies.get(runId)?.({
14764
+ status: row.status,
14765
+ steps: [],
14766
+ runId,
14767
+ ...row.error ? { error: row.error } : {},
14768
+ refusal: "cancelled"
14769
+ });
14770
+ return {
14771
+ row,
14772
+ steps: await store.getSteps(runId)
14773
+ };
14774
+ }
14775
+ callerDid(ctx) {
14776
+ if (!ctx) return void 0;
14777
+ const subject = this.host.subjectOf?.(ctx);
14778
+ if (subject?.address) return subject.address;
14779
+ const user = ctx.user;
14780
+ return typeof user?.address === "string" ? user.address : void 0;
14781
+ }
13377
14782
  async testStep(workflowId, stepId, ctx, access = {}) {
13378
14783
  await this.assertCanReadWorkflow(workflowId, ctx, access.driveId);
13379
14784
  const document = await this.host.reactorClient.get(workflowId);
@@ -13682,6 +15087,112 @@ var WorkflowRuntimeService = class {
13682
15087
  return (await this.host.reactorClient.get(workflowId)).state.global;
13683
15088
  }
13684
15089
  };
15090
+ function scopeHash(state) {
15091
+ return hashDocumentStateForScope({ state: { global: state } }, "global");
15092
+ }
15093
+ function tokenHash(token) {
15094
+ return createHash("sha256").update(token).digest("hex");
15095
+ }
15096
+ function waitpointExpired(waitpoint) {
15097
+ return waitpoint.expires_at !== null && Date.parse(waitpoint.expires_at) <= Date.now();
15098
+ }
15099
+ function parseJson(value) {
15100
+ return value === null ? null : JSON.parse(value);
15101
+ }
15102
+ const WORKFLOW_DISABLED_ERROR_NAME = "WorkflowDisabled";
15103
+ const WORKFLOW_PARKED_ERROR_NAME = "WorkflowParked";
15104
+ const MAX_WAITPOINT_ISO = "9999-12-31T23:59:59.999Z";
15105
+ const MAX_WAITPOINT_MS = Date.parse(MAX_WAITPOINT_ISO);
15106
+ function waitpointExpiry(createdAt, request, suspendMs) {
15107
+ const candidates = [request.expiresAt ? Date.parse(request.expiresAt) : NaN, suspendMs === null ? NaN : createdAt.getTime() + suspendMs].filter((at) => !Number.isNaN(at));
15108
+ return candidates.length > 0 ? new Date(Math.min(...candidates, MAX_WAITPOINT_MS)).toISOString() : null;
15109
+ }
15110
+ function sameApprover(approver, caller) {
15111
+ const tail = (value) => value.split(":").pop().toLowerCase();
15112
+ return approver.toLowerCase() === caller.toLowerCase() || tail(approver) === tail(caller);
15113
+ }
15114
+ function waitpointRecord(row) {
15115
+ return {
15116
+ id: row.id,
15117
+ runId: row.run_id,
15118
+ workflowId: row.workflow_id,
15119
+ stepId: row.step_id,
15120
+ stepKey: row.step_key,
15121
+ stepPath: row.step_path,
15122
+ pieceName: row.piece_name,
15123
+ blockName: row.block_name,
15124
+ type: row.type,
15125
+ kind: row.kind,
15126
+ status: row.status,
15127
+ createdAt: row.created_at,
15128
+ resumeAt: row.resume_at,
15129
+ expiresAt: row.expires_at,
15130
+ resolvedAt: row.resolved_at,
15131
+ decidedBy: row.decided_by,
15132
+ approval: parseJson(row.approval)
15133
+ };
15134
+ }
15135
+ function journaledValue(raw, what) {
15136
+ const value = raw === null ? void 0 : JSON.parse(raw);
15137
+ if (isTruncatedStepPayload(value)) return unavailableValue(`${what} is unavailable: the run journal truncated it past the payload cap`);
15138
+ return redactedLeavesUnavailable(value, what, []);
15139
+ }
15140
+ function redactedLeavesUnavailable(value, what, path) {
15141
+ if (!containsRedactedMarker(value)) return value;
15142
+ if (typeof value === "string") {
15143
+ const at = path.length > 0 ? ` at "${path.join(".")}"` : "";
15144
+ return unavailableValue(`${what} is unavailable: the run journal holds it redacted${at}`);
15145
+ }
15146
+ if (Array.isArray(value)) return value.map((entry, index) => redactedLeavesUnavailable(entry, what, [...path, index]));
15147
+ return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactedLeavesUnavailable(entry, what, [...path, key])]));
15148
+ }
15149
+ function journaledTriggerPayload(raw) {
15150
+ return journaledValue(raw, "The trigger payload");
15151
+ }
15152
+ const ATTACHMENT_REF_TEXT = /"(attachment:\/\/[^"\\]*)"/g;
15153
+ function carriedFromJournal(rows) {
15154
+ const carried = /* @__PURE__ */ new Map();
15155
+ const attachmentRefs = /* @__PURE__ */ new Set();
15156
+ let waitingStepId;
15157
+ let nextOrdinal = 0;
15158
+ for (const row of rows) {
15159
+ nextOrdinal = Math.max(nextOrdinal, row.ordinal + 1);
15160
+ for (const match of row.output?.matchAll(ATTACHMENT_REF_TEXT) ?? []) attachmentRefs.add(match[1]);
15161
+ const what = `Output of step "${row.step_key}"`;
15162
+ switch (row.status) {
15163
+ case "WAITING":
15164
+ waitingStepId = row.step_id;
15165
+ break;
15166
+ case "SUCCEEDED":
15167
+ case "REPLAYED":
15168
+ carried.set(row.step_id, {
15169
+ status: row.status,
15170
+ port: row.port,
15171
+ output: () => journaledValue(row.output, what)
15172
+ });
15173
+ break;
15174
+ case "FAILED":
15175
+ carried.set(row.step_id, {
15176
+ status: "FAILED",
15177
+ port: "error",
15178
+ output: () => void 0,
15179
+ error: row.error ?? ""
15180
+ });
15181
+ break;
15182
+ case "SKIPPED": if (row.port) carried.set(row.step_id, {
15183
+ status: "SKIPPED",
15184
+ port: row.port,
15185
+ output: () => null
15186
+ });
15187
+ }
15188
+ }
15189
+ return {
15190
+ carried,
15191
+ waitingStepId,
15192
+ nextOrdinal,
15193
+ attachmentRefs
15194
+ };
15195
+ }
13685
15196
  /** The runtime a host composes: one instance, its lifetime the host's. */
13686
15197
  function createWorkflowRuntime(deps) {
13687
15198
  return new WorkflowRuntimeService(deps);
@@ -13974,6 +15485,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
13974
15485
  }
13975
15486
  };
13976
15487
  //#endregion
13977
- export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName };
15488
+ export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WAITPOINT_EXPIRED_ERROR_NAME, WAITPOINT_NOT_LIVE_ERROR_NAME, WORKFLOW_DISABLED_ERROR_NAME, WORKFLOW_PACKAGE_NAME, WORKFLOW_PARKED_ERROR_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName, waitpointRecord };
13978
15489
 
13979
15490
  //# sourceMappingURL=index.js.map