@powerhousedao/reactor-workflow 6.2.3-dev.84 → 6.2.3-dev.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -37
- package/dist/index.d.ts +217 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1619 -119
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-ByALLpvA.d.ts → piece-registry-DNIk31Vv.d.ts} +82 -3
- package/dist/piece-registry-DNIk31Vv.d.ts.map +1 -0
- package/dist/{telemetry-CSemNycC.js → telemetry-B5OOpfCS.js} +656 -28
- package/dist/telemetry-B5OOpfCS.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1 -1
- package/dist/worker-entry.js +161 -16
- package/dist/worker-entry.js.map +1 -1
- package/package.json +7 -7
- package/dist/piece-registry-ByALLpvA.d.ts.map +0 -1
- package/dist/telemetry-CSemNycC.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as containsRedactedMarker, A as AttachmentCache, B as SecretCreatorMismatchError, C as blockLabel, Ct as isIndeterminateError, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as secretRefFromId, H as SecretNotFoundError, I as resolveStepInput, J as PieceWorkerPool, K as stepBlock, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as PieceWorkerTimeoutError, R as unavailableValue, S as storeHandlers, St as setPieceRegistryUrl, T as blockPorts, U as SecretOwnerMismatchError, V as SecretDeletedError, W as parseSecretRef, X as PieceWorkerDisposedError, Y as PieceWorker, Z as PieceWorkerError, _ as effectiveRetryPolicy, _t as unsupportedTrigger, a as PieceRegistry, at as isPrivateAddress, b as isHostBound, bt as npmRegistryBaseUrl, c as packagePieces, ct as FileTooLargeError, d as runWorkflow, dt as pieceModuleRef, et as redact, f as canonicalJson, ft as sourcedResolver, g as checkDynamicProperties, gt as unsupportedAuth, h as DynamicPropertiesError, ht as authMethodFor, i as tracedSteps, it as DEFAULT_EGRESS_POLICY, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as maxFileBytes, m as stepConfigHash, mt as UnsupportedPieceFeatureError, n as WorkflowTelemetry, nt as rememberSecrets, o as installedPiece, ot as parseAddress, p as hashOf, q as triggerBlock, r as tracedMethods, rt as secretsFor, s as installedPieces, st as extractDedupeKey, t as TELEMETRY_SCOPE, tt as redactMessage, u as errorNameOf, ut as bundleResolver, v as ActivepiecesBlockExecutor, w as unpinnedNote, x as resolvedBlock, xt as pieceRegistrySource, y as CompositeBlockExecutor } from "./telemetry-B5OOpfCS.js";
|
|
2
2
|
import { DENIED_ACTION_INDEXES, ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy, assertActionsApplied } from "@powerhousedao/pieces-framework";
|
|
3
3
|
import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { open, readFile, readdir, rm, stat } from "node:fs/promises";
|
|
5
5
|
import { dirname, join } from "node:path";
|
|
6
|
-
import { childLogger, generateId } from "document-model";
|
|
6
|
+
import { childLogger, garbageCollect, generateId, hashDocumentStateForScope, isDenied, sortOperations } from "document-model";
|
|
7
7
|
import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, coerceManualPayload, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, manualInputTypeName, parseManualInputs, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
|
|
8
8
|
import { Cron } from "croner";
|
|
9
9
|
import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
@@ -13,9 +13,9 @@ import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
13
13
|
import { hostname } from "node:os";
|
|
14
14
|
import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
|
|
15
15
|
import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
|
|
16
|
-
import http from "node:http";
|
|
16
|
+
import http, { validateHeaderName, validateHeaderValue } from "node:http";
|
|
17
17
|
import https from "node:https";
|
|
18
|
-
import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
|
|
18
|
+
import { actions as actions$1, reducer, utils } from "@powerhousedao/workflow/document-models/workflow";
|
|
19
19
|
import { Kind, parse } from "graphql";
|
|
20
20
|
import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
|
|
21
21
|
import { SpanStatusCode, context } from "@opentelemetry/api";
|
|
@@ -1555,6 +1555,9 @@ function currentStepTest() {
|
|
|
1555
1555
|
function currentRunUser() {
|
|
1556
1556
|
return storage.getStore()?.runUser;
|
|
1557
1557
|
}
|
|
1558
|
+
function currentWaitpoints() {
|
|
1559
|
+
return storage.getStore()?.waitpoints;
|
|
1560
|
+
}
|
|
1558
1561
|
//#endregion
|
|
1559
1562
|
//#region src/reactor/piece-store-port.ts
|
|
1560
1563
|
const logger$5 = childLogger(["workflow", "piece-store"]);
|
|
@@ -1750,6 +1753,7 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
|
|
|
1750
1753
|
egress: configuredEgress(),
|
|
1751
1754
|
worker: currentPieceWorker,
|
|
1752
1755
|
stepTest: currentStepTest,
|
|
1756
|
+
waitpoints: currentWaitpoints,
|
|
1753
1757
|
identity: () => ({
|
|
1754
1758
|
runId: currentRunId(),
|
|
1755
1759
|
flowId: currentWorkflowId(),
|
|
@@ -2691,6 +2695,10 @@ const SCHEDULE_BLOCK = trigger("schedule");
|
|
|
2691
2695
|
const WEBHOOK_BLOCK = trigger("webhook");
|
|
2692
2696
|
const BRANCH_BLOCK = action("branch");
|
|
2693
2697
|
const ASSERT_BLOCK = action("assert");
|
|
2698
|
+
const DELAY_BLOCK = action("delay");
|
|
2699
|
+
const APPROVAL_BLOCK = action("approval");
|
|
2700
|
+
const STOP_BLOCK = action("stop");
|
|
2701
|
+
const WAIT_FOR_WEBHOOK_BLOCK = action("waitForWebhook");
|
|
2694
2702
|
function isCoreBlock(block) {
|
|
2695
2703
|
return block.pieceName === CORE_PIECE_NAME;
|
|
2696
2704
|
}
|
|
@@ -7680,6 +7688,16 @@ const PARKED_TRIGGER_STATUS = "PARKED";
|
|
|
7680
7688
|
* not finish and nothing is wrong with the workflow", which is what the
|
|
7681
7689
|
* document model's RunStatus.CANCELLED means. */
|
|
7682
7690
|
const CANCELLED_RUN_STATUS = "CANCELLED";
|
|
7691
|
+
/** How long a run may stay suspended; null when the policy sets no bound. */
|
|
7692
|
+
function maxSuspensionMs(runnable) {
|
|
7693
|
+
const policy = runnable.policy;
|
|
7694
|
+
const days = positive(policy?.maxSuspensionDays);
|
|
7695
|
+
return days === null ? null : days * 864e5;
|
|
7696
|
+
}
|
|
7697
|
+
//#endregion
|
|
7698
|
+
//#region src/reactor/waitpoint-types.ts
|
|
7699
|
+
const WAITPOINT_EXPIRED_ERROR_NAME = "WaitpointExpired";
|
|
7700
|
+
const WAITPOINT_NOT_LIVE_ERROR_NAME = "WaitpointNotLive";
|
|
7683
7701
|
//#endregion
|
|
7684
7702
|
//#region src/reactor/store.ts
|
|
7685
7703
|
function triggerRowBlock(row) {
|
|
@@ -7711,6 +7729,7 @@ function workflowParkRow(workflowId, publishedVersion, reason) {
|
|
|
7711
7729
|
parked_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7712
7730
|
};
|
|
7713
7731
|
}
|
|
7732
|
+
const WAITING_RUN_STATUS = "WAITING";
|
|
7714
7733
|
const logger$4 = childLogger([
|
|
7715
7734
|
"workflow",
|
|
7716
7735
|
"runtime",
|
|
@@ -7718,6 +7737,64 @@ const logger$4 = childLogger([
|
|
|
7718
7737
|
]);
|
|
7719
7738
|
const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
|
|
7720
7739
|
const PENDING_RUN_STATUS = "PENDING";
|
|
7740
|
+
const LIVE_RUN_STATUSES = [
|
|
7741
|
+
PENDING_RUN_STATUS,
|
|
7742
|
+
"RUNNING",
|
|
7743
|
+
WAITING_RUN_STATUS
|
|
7744
|
+
];
|
|
7745
|
+
const LIVE_WAITPOINT_STATUSES = ["PENDING", "COMPLETED"];
|
|
7746
|
+
async function closeLiveWaitpoints(db, runIds, status, resolvedAt, keep) {
|
|
7747
|
+
if (runIds.length === 0) return [];
|
|
7748
|
+
return (await db.updateTable("waitpoint").set({
|
|
7749
|
+
status,
|
|
7750
|
+
resolved_at: resolvedAt
|
|
7751
|
+
}).where("run_id", "in", runIds).where("status", "in", LIVE_WAITPOINT_STATUSES).$if(keep !== void 0, (qb) => qb.where((eb) => eb.not(keep(eb)))).returning("id").execute()).map((row) => row.id);
|
|
7752
|
+
}
|
|
7753
|
+
async function failWaitingSteps(db, runIds, reason, errorName, endedAt) {
|
|
7754
|
+
if (runIds.length === 0) return;
|
|
7755
|
+
await db.updateTable("step_execution").set({
|
|
7756
|
+
status: "FAILED",
|
|
7757
|
+
error: redactMessage(reason),
|
|
7758
|
+
error_name: errorName,
|
|
7759
|
+
ended_at: endedAt
|
|
7760
|
+
}).where("run_id", "in", runIds).where("status", "=", WAITING_RUN_STATUS).execute();
|
|
7761
|
+
}
|
|
7762
|
+
function waitpointExpiryReason(waitpoint) {
|
|
7763
|
+
return `The ${waitpoint.kind === "approval" ? "approval" : "wait for a resume"} at step "${waitpoint.step_key}" expired at ${waitpoint.expires_at} without being resolved`;
|
|
7764
|
+
}
|
|
7765
|
+
function strandedWait(eb) {
|
|
7766
|
+
const live = eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES);
|
|
7767
|
+
return eb.and([eb.not(eb.exists(completedOf(eb))), eb.or([eb.and([eb("run.waiting_on", "is not", null), eb.not(eb.exists(live.whereRef("waitpoint.id", "=", "run.waiting_on")))]), eb.and([eb("run.waiting_on", "is", null), eb.not(eb.exists(live.whereRef("waitpoint.run_id", "=", "run.id")))])])]);
|
|
7768
|
+
}
|
|
7769
|
+
function completedOf(eb) {
|
|
7770
|
+
return eb.selectFrom("waitpoint").select("waitpoint.id").whereRef("waitpoint.run_id", "=", "run.id").where("waitpoint.status", "=", "COMPLETED");
|
|
7771
|
+
}
|
|
7772
|
+
async function endWaitingRun(db, runId, end) {
|
|
7773
|
+
const { waitpointId } = end;
|
|
7774
|
+
if ((await db.updateTable("run").set({
|
|
7775
|
+
status: "CANCELLED",
|
|
7776
|
+
error: redactMessage(end.reason),
|
|
7777
|
+
error_name: end.errorName,
|
|
7778
|
+
ended_at: end.endedAt
|
|
7779
|
+
}).where("id", "=", runId).where("status", "=", "WAITING").where((eb) => waitpointId === void 0 ? strandedWait(eb) : eb.and([eb.or([eb("run.waiting_on", "=", waitpointId), eb("run.waiting_on", "is", null)]), eb.not(eb.exists(completedOf(eb)))])).returning("id").execute()).length === 0) return void 0;
|
|
7780
|
+
await failWaitingSteps(db, [runId], end.reason, end.errorName, end.endedAt);
|
|
7781
|
+
const closed = await closeLiveWaitpoints(db, [runId], "EXPIRED", end.endedAt);
|
|
7782
|
+
return {
|
|
7783
|
+
runId,
|
|
7784
|
+
reason: end.reason,
|
|
7785
|
+
closed
|
|
7786
|
+
};
|
|
7787
|
+
}
|
|
7788
|
+
function priorNotes(text) {
|
|
7789
|
+
if (!text) return [];
|
|
7790
|
+
try {
|
|
7791
|
+
const notes = JSON.parse(text);
|
|
7792
|
+
return Array.isArray(notes) ? notes.filter((note) => typeof note === "string") : [];
|
|
7793
|
+
} catch {
|
|
7794
|
+
return [];
|
|
7795
|
+
}
|
|
7796
|
+
}
|
|
7797
|
+
const STRANDED_WAIT_REASON = "The run waited on a waitpoint that is no longer live";
|
|
7721
7798
|
function errorMessage(error) {
|
|
7722
7799
|
return error instanceof Error ? error.message : String(error);
|
|
7723
7800
|
}
|
|
@@ -7819,6 +7896,31 @@ async function up(db) {
|
|
|
7819
7896
|
try {
|
|
7820
7897
|
await db.schema.dropTable("webhook_endpoint").ifExists().execute();
|
|
7821
7898
|
} catch {}
|
|
7899
|
+
for (const [column, type] of [
|
|
7900
|
+
["definition_hash", "text"],
|
|
7901
|
+
["definition_op_index", "integer"],
|
|
7902
|
+
["definition_scope_hash", "text"],
|
|
7903
|
+
["waiting_on", "text"]
|
|
7904
|
+
]) try {
|
|
7905
|
+
await db.schema.alterTable("run").addColumn(column, type).execute();
|
|
7906
|
+
} catch {}
|
|
7907
|
+
await db.schema.createTable("waitpoint").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("step_path", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("type", "text", (col) => col.notNull()).addColumn("kind", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("token_hash", "text").addColumn("created_at", "text", (col) => col.notNull()).addColumn("resume_at", "text").addColumn("expires_at", "text").addColumn("resolved_at", "text").addColumn("decided_by", "text").addColumn("approval", "text").addColumn("resume_payload", "text").addColumn("response_to_send", "text").ifNotExists().execute();
|
|
7908
|
+
await db.schema.createIndex("waitpoint_token").ifNotExists().unique().on("waitpoint").column("token_hash").execute();
|
|
7909
|
+
await db.schema.createIndex("waitpoint_status_resume").ifNotExists().on("waitpoint").columns(["status", "resume_at"]).execute();
|
|
7910
|
+
await db.schema.createIndex("waitpoint_status_expires").ifNotExists().on("waitpoint").columns(["status", "expires_at"]).execute();
|
|
7911
|
+
await db.schema.createIndex("waitpoint_run").ifNotExists().on("waitpoint").column("run_id").execute();
|
|
7912
|
+
const liveStep = db.schema.createIndex("waitpoint_live_step").ifNotExists().unique().on("waitpoint").columns(["run_id", "step_path"]).where(sql.ref("status"), "in", LIVE_WAITPOINT_STATUSES);
|
|
7913
|
+
try {
|
|
7914
|
+
await liveStep.execute();
|
|
7915
|
+
} catch {
|
|
7916
|
+
await db.updateTable("waitpoint").set({
|
|
7917
|
+
status: "CANCELLED",
|
|
7918
|
+
resolved_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7919
|
+
}).where("status", "in", LIVE_WAITPOINT_STATUSES).where((eb) => eb.and([eb.not(eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.waiting_on", "=", "waitpoint.id"))), eb.exists(eb.selectFrom("waitpoint as sibling").select("sibling.id").whereRef("sibling.run_id", "=", "waitpoint.run_id").whereRef("sibling.step_path", "=", "waitpoint.step_path").whereRef("sibling.id", "!=", "waitpoint.id").where("sibling.status", "in", LIVE_WAITPOINT_STATUSES))])).execute();
|
|
7920
|
+
await liveStep.execute();
|
|
7921
|
+
}
|
|
7922
|
+
await db.schema.createIndex("waitpoint_workflow_status").ifNotExists().on("waitpoint").columns(["workflow_id", "status"]).execute();
|
|
7923
|
+
await db.schema.createIndex("run_waiting").ifNotExists().on("run").columns(["workflow_id", "enqueued_at"]).where(sql.ref("status"), "=", WAITING_RUN_STATUS).execute();
|
|
7822
7924
|
await migrateBlockType(db, "trigger_state", "trigger_name");
|
|
7823
7925
|
await migrateBlockType(db, "step_execution", "block_name");
|
|
7824
7926
|
return migrateTriggerStoreState(db);
|
|
@@ -8177,6 +8279,8 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8177
8279
|
unmigrated;
|
|
8178
8280
|
runsInFlight = /* @__PURE__ */ new Set();
|
|
8179
8281
|
journaledSteps = /* @__PURE__ */ new Map();
|
|
8282
|
+
pausingRuns = /* @__PURE__ */ new Set();
|
|
8283
|
+
resumedLegs = /* @__PURE__ */ new Set();
|
|
8180
8284
|
constructor(db, unmigrated) {
|
|
8181
8285
|
this.db = db;
|
|
8182
8286
|
this.unmigrated = unmigrated;
|
|
@@ -8195,13 +8299,20 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8195
8299
|
this.unmigrated.delete(workflowId);
|
|
8196
8300
|
}
|
|
8197
8301
|
async recoverOrphanedRuns() {
|
|
8198
|
-
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
|
|
8203
|
-
|
|
8204
|
-
|
|
8302
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8303
|
+
const inFlight = [...this.runsInFlight];
|
|
8304
|
+
const recovered = await this.db.transaction().execute(async (trx) => {
|
|
8305
|
+
let query = trx.updateTable("run").set({
|
|
8306
|
+
status: "FAILED",
|
|
8307
|
+
error: ORPHANED_RUN_ERROR,
|
|
8308
|
+
ended_at: endedAt
|
|
8309
|
+
}).where("status", "=", "RUNNING");
|
|
8310
|
+
if (inFlight.length > 0) query = query.where("id", "not in", inFlight);
|
|
8311
|
+
const ids = (await query.returning("id").execute()).map((row) => row.id);
|
|
8312
|
+
await failWaitingSteps(trx, ids, ORPHANED_RUN_ERROR, null, endedAt);
|
|
8313
|
+
await closeLiveWaitpoints(trx, ids, "CANCELLED", endedAt);
|
|
8314
|
+
return ids.length;
|
|
8315
|
+
});
|
|
8205
8316
|
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
|
|
8206
8317
|
return recovered;
|
|
8207
8318
|
}
|
|
@@ -8265,14 +8376,25 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8265
8376
|
* CANCELLED run is not rerunnable and must not read as a workflow defect.
|
|
8266
8377
|
*/
|
|
8267
8378
|
async cancelRun(runId, reason) {
|
|
8379
|
+
return this.endLiveRun(runId, "CANCELLED", reason, null);
|
|
8380
|
+
}
|
|
8381
|
+
async endLiveRun(runId, status, reason, errorName) {
|
|
8268
8382
|
this.runsInFlight.delete(runId);
|
|
8269
8383
|
this.journaledSteps.delete(runId);
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
|
|
8273
|
-
|
|
8274
|
-
|
|
8275
|
-
|
|
8384
|
+
this.pausingRuns.delete(runId);
|
|
8385
|
+
this.resumedLegs.delete(runId);
|
|
8386
|
+
if (erasedRuns.delete(runId)) return [];
|
|
8387
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8388
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8389
|
+
if ((await trx.updateTable("run").set({
|
|
8390
|
+
status,
|
|
8391
|
+
error: redactMessage(reason),
|
|
8392
|
+
error_name: errorName,
|
|
8393
|
+
ended_at: endedAt
|
|
8394
|
+
}).where("id", "=", runId).where("status", "in", LIVE_RUN_STATUSES).returning("id").execute()).length === 0) return [];
|
|
8395
|
+
await failWaitingSteps(trx, [runId], reason, errorName, endedAt);
|
|
8396
|
+
return closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
|
|
8397
|
+
});
|
|
8276
8398
|
}
|
|
8277
8399
|
async enqueueRun(options) {
|
|
8278
8400
|
const id = randomUUID();
|
|
@@ -8480,12 +8602,15 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8480
8602
|
async beginRun(runId, details) {
|
|
8481
8603
|
this.runsInFlight.add(runId);
|
|
8482
8604
|
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
8483
|
-
await this.db.updateTable("run").set({
|
|
8605
|
+
if ((await this.db.updateTable("run").set({
|
|
8484
8606
|
status: "RUNNING",
|
|
8485
8607
|
workflow_name: details.workflowName,
|
|
8486
8608
|
workflow_version: details.workflowVersion,
|
|
8487
8609
|
started_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8488
|
-
}).where("id", "=", runId).execute();
|
|
8610
|
+
}).where("id", "=", runId).where("status", "=", "PENDING").returning("id").execute()).length > 0) return true;
|
|
8611
|
+
this.runsInFlight.delete(runId);
|
|
8612
|
+
this.journaledSteps.delete(runId);
|
|
8613
|
+
return false;
|
|
8489
8614
|
}
|
|
8490
8615
|
async startRun(options) {
|
|
8491
8616
|
const id = randomUUID();
|
|
@@ -8514,10 +8639,10 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8514
8639
|
if (erasedRuns.has(runId)) return;
|
|
8515
8640
|
const values = stepValues(runId, ordinal, step);
|
|
8516
8641
|
const { run_id: _run, step_id: _step, ...mutable } = values;
|
|
8517
|
-
await this.
|
|
8642
|
+
if (!await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values({
|
|
8518
8643
|
id: randomUUID(),
|
|
8519
8644
|
...values
|
|
8520
|
-
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
|
|
8645
|
+
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute())) return;
|
|
8521
8646
|
this.journaledSteps.get(runId)?.set(step.stepId, {
|
|
8522
8647
|
ordinal,
|
|
8523
8648
|
hash: rowHash(values)
|
|
@@ -8532,25 +8657,61 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8532
8657
|
return row.hash === rowHash(stepValues(runId, row.ordinal, step));
|
|
8533
8658
|
});
|
|
8534
8659
|
}
|
|
8535
|
-
async finishRun(runId, result, executionOrder) {
|
|
8660
|
+
async finishRun(runId, result, executionOrder, pin) {
|
|
8536
8661
|
this.runsInFlight.delete(runId);
|
|
8537
|
-
const
|
|
8662
|
+
const resumed = this.resumedLegs.delete(runId);
|
|
8663
|
+
const paused = this.pausingRuns.delete(runId) || resumed;
|
|
8664
|
+
const own = result.steps.some((step) => step.carried) ? {
|
|
8665
|
+
...result,
|
|
8666
|
+
steps: result.steps.filter((step) => !step.carried)
|
|
8667
|
+
} : result;
|
|
8668
|
+
const sweep = !this.sweepIsNoop(runId, own);
|
|
8538
8669
|
this.journaledSteps.delete(runId);
|
|
8539
|
-
if (erasedRuns.delete(runId)) return
|
|
8540
|
-
|
|
8541
|
-
|
|
8670
|
+
if (erasedRuns.delete(runId)) return {
|
|
8671
|
+
held: true,
|
|
8672
|
+
closed: []
|
|
8673
|
+
};
|
|
8674
|
+
if (sweep && own.steps.length > 0) try {
|
|
8675
|
+
await this.sweepSteps(runId, own, executionOrder);
|
|
8542
8676
|
} catch (error) {
|
|
8543
8677
|
logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
|
|
8544
8678
|
}
|
|
8545
|
-
const
|
|
8546
|
-
|
|
8547
|
-
|
|
8548
|
-
|
|
8549
|
-
|
|
8550
|
-
|
|
8551
|
-
|
|
8552
|
-
|
|
8553
|
-
|
|
8679
|
+
const waiting = result.status === WAITING_RUN_STATUS;
|
|
8680
|
+
const on = result.waiting;
|
|
8681
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8682
|
+
const close = async (trx) => {
|
|
8683
|
+
const prior = resumed ? await trx.selectFrom("run").select("warning_notes").where("id", "=", runId).executeTakeFirst() : void 0;
|
|
8684
|
+
const notes = [.../* @__PURE__ */ new Set([...priorNotes(prior?.warning_notes ?? null), ...runWarningNotes(own)])];
|
|
8685
|
+
const written = await trx.updateTable("run").set({
|
|
8686
|
+
status: result.status,
|
|
8687
|
+
error: result.error ? redactMessage(result.error) : null,
|
|
8688
|
+
error_name: result.errorName ?? null,
|
|
8689
|
+
ended_at: waiting ? null : now,
|
|
8690
|
+
warnings: notes.length,
|
|
8691
|
+
warning_notes: notes.length > 0 ? JSON.stringify(notes) : null,
|
|
8692
|
+
waiting_on: waiting ? on?.waitpointId ?? null : null,
|
|
8693
|
+
...pin ?? {}
|
|
8694
|
+
}).where("id", "=", runId).$if(waiting, (qb) => qb.where((eb) => eb.exists(eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.id", "=", on?.waitpointId ?? "").where("waitpoint.run_id", "=", runId).where("waitpoint.step_id", "=", on?.stepId ?? "").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES)))).returning("id").execute();
|
|
8695
|
+
if (!waiting) return {
|
|
8696
|
+
held: true,
|
|
8697
|
+
closed: paused ? await closeLiveWaitpoints(trx, [runId], "EXPIRED", now) : []
|
|
8698
|
+
};
|
|
8699
|
+
if (written.length === 0) return {
|
|
8700
|
+
held: false,
|
|
8701
|
+
closed: []
|
|
8702
|
+
};
|
|
8703
|
+
const linkStatuses = (await trx.selectFrom("waitpoint").select("kind").where("id", "=", on.waitpointId).executeTakeFirst())?.kind === "webhook" ? LIVE_WAITPOINT_STATUSES : ["PENDING"];
|
|
8704
|
+
return {
|
|
8705
|
+
held: true,
|
|
8706
|
+
closed: await closeLiveWaitpoints(trx, [runId], "CANCELLED", now, (eb) => eb.or([eb("waitpoint.id", "=", on.waitpointId), eb.and([
|
|
8707
|
+
eb("waitpoint.type", "=", "WEBHOOK"),
|
|
8708
|
+
eb("waitpoint.kind", "=", "webhook"),
|
|
8709
|
+
eb("waitpoint.status", "in", linkStatuses),
|
|
8710
|
+
eb.exists(eb.selectFrom("step_execution").select("step_execution.id").whereRef("step_execution.run_id", "=", "waitpoint.run_id").whereRef("step_execution.step_id", "=", "waitpoint.step_id").where("step_execution.status", "=", "SUCCEEDED"))
|
|
8711
|
+
])]))
|
|
8712
|
+
};
|
|
8713
|
+
};
|
|
8714
|
+
return waiting || paused ? this.db.transaction().execute(close) : close(this.db);
|
|
8554
8715
|
}
|
|
8555
8716
|
async sweepSteps(runId, result, executionOrder) {
|
|
8556
8717
|
const journaled = await this.db.selectFrom("step_execution").select(["step_id", "ordinal"]).where("run_id", "=", runId).execute();
|
|
@@ -8563,7 +8724,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8563
8724
|
nextOrdinal = Math.max(nextOrdinal, ran + 1);
|
|
8564
8725
|
}
|
|
8565
8726
|
const ordinalFor = (step) => ordinals.get(step.stepId) ?? nextOrdinal++;
|
|
8566
|
-
await this.
|
|
8727
|
+
await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values(result.steps.map((step) => ({
|
|
8567
8728
|
id: randomUUID(),
|
|
8568
8729
|
...stepValues(runId, ordinalFor(step), step)
|
|
8569
8730
|
}))).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet((eb) => ({
|
|
@@ -8584,18 +8745,15 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8584
8745
|
version_match: eb.ref("excluded.version_match"),
|
|
8585
8746
|
version_note: eb.ref("excluded.version_note"),
|
|
8586
8747
|
config_hash: eb.ref("excluded.config_hash")
|
|
8587
|
-
}))).execute();
|
|
8748
|
+
}))).execute());
|
|
8749
|
+
}
|
|
8750
|
+
async withRunRow(runId, fn) {
|
|
8751
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8752
|
+
return await trx.selectFrom("run").select("id").where("id", "=", runId).forKeyShare().executeTakeFirst() ? fn(trx) : void 0;
|
|
8753
|
+
});
|
|
8588
8754
|
}
|
|
8589
8755
|
async failRun(runId, error, errorName) {
|
|
8590
|
-
this.
|
|
8591
|
-
this.journaledSteps.delete(runId);
|
|
8592
|
-
if (erasedRuns.delete(runId)) return;
|
|
8593
|
-
await this.db.updateTable("run").set({
|
|
8594
|
-
status: "FAILED",
|
|
8595
|
-
error: redactMessage(error),
|
|
8596
|
-
error_name: errorName ?? null,
|
|
8597
|
-
ended_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8598
|
-
}).where("id", "=", runId).execute();
|
|
8756
|
+
return this.endLiveRun(runId, "FAILED", error, errorName ?? null);
|
|
8599
8757
|
}
|
|
8600
8758
|
async listRuns(workflowId, limit = 25, options = {}) {
|
|
8601
8759
|
if (Array.isArray(workflowId) && workflowId.length === 0) return [];
|
|
@@ -8627,10 +8785,10 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8627
8785
|
}
|
|
8628
8786
|
async recordRunDocuments(runId, documentIds) {
|
|
8629
8787
|
if (documentIds.length === 0 || erasedRuns.has(runId)) return;
|
|
8630
|
-
await this.
|
|
8788
|
+
await this.withRunRow(runId, (trx) => trx.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
|
|
8631
8789
|
run_id: runId,
|
|
8632
8790
|
document_id: documentId
|
|
8633
|
-
}))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute();
|
|
8791
|
+
}))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute());
|
|
8634
8792
|
}
|
|
8635
8793
|
async getRunDocuments(runId) {
|
|
8636
8794
|
return (await this.db.selectFrom("run_document").select("document_id").where("run_id", "=", runId).execute()).map((row) => row.document_id);
|
|
@@ -8775,6 +8933,181 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8775
8933
|
async deleteDedupe(workflowId) {
|
|
8776
8934
|
await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
|
|
8777
8935
|
}
|
|
8936
|
+
async createWaitpoint(row) {
|
|
8937
|
+
if (this.runsInFlight.has(row.run_id)) this.pausingRuns.add(row.run_id);
|
|
8938
|
+
for (let attempt = 1;; attempt++) {
|
|
8939
|
+
const inserted = await this.withRunRow(row.run_id, (trx) => trx.insertInto("waitpoint").values(row).onConflict((oc) => oc.doNothing()).returningAll().executeTakeFirst().then((waitpoint) => ({ waitpoint })));
|
|
8940
|
+
if (!inserted) throw new Error(`Run "${row.run_id}" no longer exists`);
|
|
8941
|
+
if (inserted.waitpoint) return {
|
|
8942
|
+
waitpoint: inserted.waitpoint,
|
|
8943
|
+
created: true
|
|
8944
|
+
};
|
|
8945
|
+
const live = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "=", row.run_id).where("step_path", "=", row.step_path).where("status", "in", LIVE_WAITPOINT_STATUSES).executeTakeFirst();
|
|
8946
|
+
if (live) return {
|
|
8947
|
+
waitpoint: live,
|
|
8948
|
+
created: false
|
|
8949
|
+
};
|
|
8950
|
+
if (attempt >= 3) throw new Error(`Could not create a waitpoint for step "${row.step_key}"`);
|
|
8951
|
+
}
|
|
8952
|
+
}
|
|
8953
|
+
async getWaitpoint(id) {
|
|
8954
|
+
return this.db.selectFrom("waitpoint").selectAll().where("id", "=", id).executeTakeFirst();
|
|
8955
|
+
}
|
|
8956
|
+
async getWaitpointByTokenHash(tokenHash) {
|
|
8957
|
+
return this.db.selectFrom("waitpoint").selectAll().where("token_hash", "=", tokenHash).executeTakeFirst();
|
|
8958
|
+
}
|
|
8959
|
+
async completeWaitpoint(id, resolution) {
|
|
8960
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8961
|
+
return (await this.db.updateTable("waitpoint").set({
|
|
8962
|
+
status: "COMPLETED",
|
|
8963
|
+
resume_payload: JSON.stringify(resolution.payload ?? {}),
|
|
8964
|
+
resolved_at: resolution.resolvedAt ?? now,
|
|
8965
|
+
decided_by: resolution.decidedBy ?? null
|
|
8966
|
+
}).where("id", "=", id).where("status", "=", "PENDING").where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", now)])).where((eb) => eb.not(eb.exists(eb.selectFrom("run").innerJoin("waitpoint as awaited", "awaited.id", "run.waiting_on").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").whereRef("awaited.id", "!=", "waitpoint.id").where("awaited.kind", "!=", "webhook").where("awaited.status", "in", LIVE_WAITPOINT_STATUSES)))).where((eb) => eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").where("run.status", "in", ["RUNNING", WAITING_RUN_STATUS]))).returningAll().execute())[0];
|
|
8967
|
+
}
|
|
8968
|
+
async claimDueDelayWaitpoints(nowIso, limit) {
|
|
8969
|
+
return this.db.updateTable("waitpoint").set({
|
|
8970
|
+
status: "COMPLETED",
|
|
8971
|
+
resolved_at: nowIso,
|
|
8972
|
+
resume_payload: "{}"
|
|
8973
|
+
}).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "<=", nowIso).orderBy("resume_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
|
|
8974
|
+
}
|
|
8975
|
+
async expireDueWaitpoints(nowIso, limit) {
|
|
8976
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8977
|
+
const expired = await trx.updateTable("waitpoint").set({
|
|
8978
|
+
status: "EXPIRED",
|
|
8979
|
+
resolved_at: nowIso
|
|
8980
|
+
}).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("expires_at", "<=", nowIso).orderBy("expires_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
|
|
8981
|
+
const ended = [];
|
|
8982
|
+
for (const waitpoint of expired) {
|
|
8983
|
+
const end = await endWaitingRun(trx, waitpoint.run_id, {
|
|
8984
|
+
reason: waitpointExpiryReason(waitpoint),
|
|
8985
|
+
errorName: WAITPOINT_EXPIRED_ERROR_NAME,
|
|
8986
|
+
endedAt: nowIso,
|
|
8987
|
+
waitpointId: waitpoint.id
|
|
8988
|
+
});
|
|
8989
|
+
if (end) ended.push(end);
|
|
8990
|
+
}
|
|
8991
|
+
return {
|
|
8992
|
+
expired,
|
|
8993
|
+
ended
|
|
8994
|
+
};
|
|
8995
|
+
});
|
|
8996
|
+
}
|
|
8997
|
+
async endStrandedWaits(limit) {
|
|
8998
|
+
const runs = await this.db.selectFrom("run").select("id").where("status", "=", WAITING_RUN_STATUS).where((eb) => strandedWait(eb)).limit(limit).execute();
|
|
8999
|
+
const ended = [];
|
|
9000
|
+
for (const run of runs) {
|
|
9001
|
+
const end = await this.db.transaction().execute((trx) => endWaitingRun(trx, run.id, {
|
|
9002
|
+
reason: STRANDED_WAIT_REASON,
|
|
9003
|
+
errorName: WAITPOINT_NOT_LIVE_ERROR_NAME,
|
|
9004
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9005
|
+
}));
|
|
9006
|
+
if (end) ended.push(end);
|
|
9007
|
+
}
|
|
9008
|
+
return ended;
|
|
9009
|
+
}
|
|
9010
|
+
async nextExpiry() {
|
|
9011
|
+
return (await this.db.selectFrom("waitpoint").select("expires_at").where("status", "=", "PENDING").where("expires_at", "is not", null).orderBy("expires_at").limit(1).executeTakeFirst())?.expires_at ?? void 0;
|
|
9012
|
+
}
|
|
9013
|
+
async countWaitingRuns() {
|
|
9014
|
+
const row = await this.db.selectFrom("run").select((eb) => eb.fn.countAll().as("count")).where("status", "=", WAITING_RUN_STATUS).executeTakeFirst();
|
|
9015
|
+
return Number(row?.count ?? 0);
|
|
9016
|
+
}
|
|
9017
|
+
async nextDelayDue() {
|
|
9018
|
+
return (await this.db.selectFrom("waitpoint").select("resume_at").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "is not", null).orderBy("resume_at").limit(1).executeTakeFirst())?.resume_at ?? void 0;
|
|
9019
|
+
}
|
|
9020
|
+
async hasCompletedWaitpoint(runId) {
|
|
9021
|
+
return await this.db.selectFrom("waitpoint").select("id").where("run_id", "=", runId).where("status", "=", "COMPLETED").limit(1).executeTakeFirst() !== void 0;
|
|
9022
|
+
}
|
|
9023
|
+
async heldRunIds() {
|
|
9024
|
+
return (await this.db.selectFrom("waitpoint").innerJoin("run", "run.id", "waitpoint.run_id").select("waitpoint.run_id").distinct().where("waitpoint.status", "=", "COMPLETED").where("run.status", "=", WAITING_RUN_STATUS).execute()).map((row) => row.run_id);
|
|
9025
|
+
}
|
|
9026
|
+
isRunInFlight(runId) {
|
|
9027
|
+
return this.runsInFlight.has(runId);
|
|
9028
|
+
}
|
|
9029
|
+
async claimWaitingRun(runId) {
|
|
9030
|
+
if ((await this.db.updateTable("run").set({ status: "RUNNING" }).where("id", "=", runId).where("status", "=", "WAITING").returning("id").execute()).length === 0) return false;
|
|
9031
|
+
this.runsInFlight.add(runId);
|
|
9032
|
+
this.resumedLegs.add(runId);
|
|
9033
|
+
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
9034
|
+
return true;
|
|
9035
|
+
}
|
|
9036
|
+
async returnToWaiting(runId) {
|
|
9037
|
+
this.runsInFlight.delete(runId);
|
|
9038
|
+
this.resumedLegs.delete(runId);
|
|
9039
|
+
this.journaledSteps.delete(runId);
|
|
9040
|
+
await this.db.updateTable("run").set({ status: WAITING_RUN_STATUS }).where("id", "=", runId).where("status", "=", "RUNNING").execute();
|
|
9041
|
+
}
|
|
9042
|
+
async consumeCompletedWaitpoint(runId) {
|
|
9043
|
+
return this.db.transaction().execute(async (trx) => {
|
|
9044
|
+
const waitingOn = (await trx.selectFrom("run").select("waiting_on").where("id", "=", runId).executeTakeFirst())?.waiting_on ?? null;
|
|
9045
|
+
const candidates = await trx.selectFrom("waitpoint").selectAll().where("run_id", "=", runId).where("status", "=", "COMPLETED").orderBy("resolved_at").execute();
|
|
9046
|
+
const awaited = waitingOn ? await trx.selectFrom("waitpoint").select("kind").where("id", "=", waitingOn).executeTakeFirst() : void 0;
|
|
9047
|
+
const linksResume = !awaited || awaited.kind === "webhook";
|
|
9048
|
+
const chosen = candidates.find((row) => row.id === waitingOn) ?? (linksResume ? candidates[0] : void 0);
|
|
9049
|
+
if (!chosen) return void 0;
|
|
9050
|
+
const waitpoint = (await trx.updateTable("waitpoint").set({ status: "CONSUMED" }).where("id", "=", chosen.id).where("status", "=", "COMPLETED").returningAll().execute())[0];
|
|
9051
|
+
if (!waitpoint) return void 0;
|
|
9052
|
+
return {
|
|
9053
|
+
waitpoint,
|
|
9054
|
+
closed: waitingOn && waitingOn !== waitpoint.id ? await closeLiveWaitpoints(trx, [runId], "CANCELLED", (/* @__PURE__ */ new Date()).toISOString(), (eb) => eb("waitpoint.id", "!=", waitingOn)) : []
|
|
9055
|
+
};
|
|
9056
|
+
});
|
|
9057
|
+
}
|
|
9058
|
+
async getRunPin(runId) {
|
|
9059
|
+
return this.db.selectFrom("run").select([
|
|
9060
|
+
"definition_hash",
|
|
9061
|
+
"definition_op_index",
|
|
9062
|
+
"definition_scope_hash"
|
|
9063
|
+
]).where("id", "=", runId).executeTakeFirst();
|
|
9064
|
+
}
|
|
9065
|
+
async cancelIdleRun(runId, reason) {
|
|
9066
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9067
|
+
if (!await this.db.transaction().execute(async (trx) => {
|
|
9068
|
+
if ((await trx.updateTable("run").set({
|
|
9069
|
+
status: "CANCELLED",
|
|
9070
|
+
error: redactMessage(reason),
|
|
9071
|
+
ended_at: endedAt
|
|
9072
|
+
}).where("id", "=", runId).where("status", "in", ["WAITING", "PENDING"]).returning("id").execute()).length === 0) return false;
|
|
9073
|
+
await failWaitingSteps(trx, [runId], reason, null, endedAt);
|
|
9074
|
+
await closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
|
|
9075
|
+
return true;
|
|
9076
|
+
})) return void 0;
|
|
9077
|
+
this.runsInFlight.delete(runId);
|
|
9078
|
+
return runId;
|
|
9079
|
+
}
|
|
9080
|
+
async endWait(runId, end) {
|
|
9081
|
+
this.runsInFlight.delete(runId);
|
|
9082
|
+
this.journaledSteps.delete(runId);
|
|
9083
|
+
this.pausingRuns.delete(runId);
|
|
9084
|
+
this.resumedLegs.delete(runId);
|
|
9085
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9086
|
+
return this.db.transaction().execute(async (trx) => {
|
|
9087
|
+
if ((await trx.updateTable("run").set({
|
|
9088
|
+
status: end.status,
|
|
9089
|
+
error: redactMessage(end.reason),
|
|
9090
|
+
error_name: end.errorName,
|
|
9091
|
+
ended_at: endedAt
|
|
9092
|
+
}).where("id", "=", runId).where("status", "=", "RUNNING").returning("id").execute()).length === 0) return [];
|
|
9093
|
+
await failWaitingSteps(trx, [runId], end.reason, end.errorName, endedAt);
|
|
9094
|
+
return closeLiveWaitpoints(trx, [runId], end.waitpoints, endedAt);
|
|
9095
|
+
});
|
|
9096
|
+
}
|
|
9097
|
+
async listWaitpointsForRuns(runIds) {
|
|
9098
|
+
const byRun = new Map(runIds.map((id) => [id, []]));
|
|
9099
|
+
if (runIds.length === 0) return byRun;
|
|
9100
|
+
const rows = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "in", [...new Set(runIds)]).orderBy("created_at").execute();
|
|
9101
|
+
for (const row of rows) byRun.get(row.run_id)?.push(row);
|
|
9102
|
+
return byRun;
|
|
9103
|
+
}
|
|
9104
|
+
async listWaitingRuns(workflowId, limit) {
|
|
9105
|
+
if (Array.isArray(workflowId) && workflowId.length === 0) return [];
|
|
9106
|
+
let query = this.db.selectFrom("run").selectAll().where("status", "=", WAITING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc").limit(Math.min(Math.max(limit, 1), 100));
|
|
9107
|
+
if (Array.isArray(workflowId)) query = query.where("workflow_id", "in", workflowId);
|
|
9108
|
+
else if (workflowId) query = query.where("workflow_id", "=", workflowId);
|
|
9109
|
+
return query.execute();
|
|
9110
|
+
}
|
|
8778
9111
|
async pruneFinishedRuns(cutoffIso, batchSize = PRUNE_BATCH_SIZE) {
|
|
8779
9112
|
let pruned = 0;
|
|
8780
9113
|
for (;;) {
|
|
@@ -8782,6 +9115,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8782
9115
|
const ids = (await trx.selectFrom("run").select("id").where("ended_at", "is not", null).where("ended_at", "<", cutoffIso).limit(batchSize).execute()).map((row) => row.id);
|
|
8783
9116
|
if (ids.length === 0) return 0;
|
|
8784
9117
|
await trx.deleteFrom("step_execution").where("run_id", "in", ids).execute();
|
|
9118
|
+
await trx.deleteFrom("waitpoint").where("run_id", "in", ids).execute();
|
|
8785
9119
|
await trx.deleteFrom("run_document").where("run_id", "in", ids).execute();
|
|
8786
9120
|
await trx.deleteFrom("run").where("id", "in", ids).execute();
|
|
8787
9121
|
return ids.length;
|
|
@@ -8813,10 +9147,11 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8813
9147
|
for (const id of frontier) runIds.add(id);
|
|
8814
9148
|
}
|
|
8815
9149
|
if (runIds.size === 0) return erased;
|
|
8816
|
-
const all = [...runIds];
|
|
8817
|
-
const
|
|
8818
|
-
for (const row of
|
|
9150
|
+
const all = [...runIds].sort();
|
|
9151
|
+
const locked = await trx.selectFrom("run").select(["id", "status"]).where("id", "in", all).orderBy("id").forUpdate().execute();
|
|
9152
|
+
for (const row of locked) if (row.status === "RUNNING" || row.status === "PENDING") erasedRuns.add(row.id);
|
|
8819
9153
|
erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
|
|
9154
|
+
await trx.deleteFrom("waitpoint").where("run_id", "in", all).execute();
|
|
8820
9155
|
erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
|
|
8821
9156
|
erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
|
|
8822
9157
|
erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
|
|
@@ -8933,6 +9268,118 @@ function decodeRunCursor(cursor) {
|
|
|
8933
9268
|
id: parsed[1]
|
|
8934
9269
|
};
|
|
8935
9270
|
}
|
|
9271
|
+
const WAITPOINT_POLL_MAX_MS = 3e5;
|
|
9272
|
+
const WAITPOINT_RETRY_MS = 3e4;
|
|
9273
|
+
const SUSPENDED_COUNT_MS = 3e4;
|
|
9274
|
+
var WaitpointPoller = class {
|
|
9275
|
+
deps;
|
|
9276
|
+
running = false;
|
|
9277
|
+
polling = false;
|
|
9278
|
+
again = false;
|
|
9279
|
+
timer;
|
|
9280
|
+
timerAt;
|
|
9281
|
+
nudged;
|
|
9282
|
+
suspended = 0;
|
|
9283
|
+
suspendedAt = 0;
|
|
9284
|
+
counting = false;
|
|
9285
|
+
constructor(deps) {
|
|
9286
|
+
this.deps = deps;
|
|
9287
|
+
}
|
|
9288
|
+
get active() {
|
|
9289
|
+
return this.running;
|
|
9290
|
+
}
|
|
9291
|
+
start() {
|
|
9292
|
+
if (this.running) return;
|
|
9293
|
+
this.running = true;
|
|
9294
|
+
this.poll();
|
|
9295
|
+
}
|
|
9296
|
+
stop() {
|
|
9297
|
+
this.running = false;
|
|
9298
|
+
clearTimeout(this.timer);
|
|
9299
|
+
this.timer = void 0;
|
|
9300
|
+
this.timerAt = void 0;
|
|
9301
|
+
this.nudged = void 0;
|
|
9302
|
+
}
|
|
9303
|
+
nudge(dueIso) {
|
|
9304
|
+
if (!this.running) return;
|
|
9305
|
+
const due = Date.parse(dueIso);
|
|
9306
|
+
if (Number.isNaN(due)) return;
|
|
9307
|
+
if (this.polling) {
|
|
9308
|
+
this.nudged = Math.min(this.nudged ?? Infinity, due);
|
|
9309
|
+
return;
|
|
9310
|
+
}
|
|
9311
|
+
if (this.timerAt !== void 0 && due >= this.timerAt) return;
|
|
9312
|
+
this.arm(due);
|
|
9313
|
+
}
|
|
9314
|
+
suspendedRuns() {
|
|
9315
|
+
if (Date.now() - this.suspendedAt > SUSPENDED_COUNT_MS) this.count();
|
|
9316
|
+
return this.suspended;
|
|
9317
|
+
}
|
|
9318
|
+
async count(store) {
|
|
9319
|
+
if (this.counting) return this.suspended;
|
|
9320
|
+
this.counting = true;
|
|
9321
|
+
try {
|
|
9322
|
+
const journal = store ?? await this.deps.store();
|
|
9323
|
+
if (journal) {
|
|
9324
|
+
this.suspended = await journal.countWaitingRuns();
|
|
9325
|
+
this.suspendedAt = Date.now();
|
|
9326
|
+
}
|
|
9327
|
+
} catch (error) {
|
|
9328
|
+
this.deps.logger.warn("Counting suspended runs failed: @error", error);
|
|
9329
|
+
} finally {
|
|
9330
|
+
this.counting = false;
|
|
9331
|
+
}
|
|
9332
|
+
return this.suspended;
|
|
9333
|
+
}
|
|
9334
|
+
/** One pass: due delays and held resumes deliver, overdue waits expire, the
|
|
9335
|
+
* timer re-arms. */
|
|
9336
|
+
async poll() {
|
|
9337
|
+
if (!this.running) return;
|
|
9338
|
+
if (this.polling) {
|
|
9339
|
+
this.again = true;
|
|
9340
|
+
return;
|
|
9341
|
+
}
|
|
9342
|
+
this.polling = true;
|
|
9343
|
+
let next = Date.now() + WAITPOINT_RETRY_MS;
|
|
9344
|
+
try {
|
|
9345
|
+
const store = await this.deps.store();
|
|
9346
|
+
if (!store) return;
|
|
9347
|
+
const batch = this.deps.batch ?? 100;
|
|
9348
|
+
do {
|
|
9349
|
+
this.again = false;
|
|
9350
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9351
|
+
const due = await store.claimDueDelayWaitpoints(now, batch);
|
|
9352
|
+
for (const runId of new Set(due.map((row) => row.run_id))) await this.deps.resume(runId);
|
|
9353
|
+
const { expired, ended } = await store.expireDueWaitpoints(now, batch);
|
|
9354
|
+
const stranded = await store.endStrandedWaits(batch);
|
|
9355
|
+
for (const end of [...ended, ...stranded]) await this.deps.ended(end);
|
|
9356
|
+
if (due.length === batch || expired.length === batch || stranded.length === batch) this.again = true;
|
|
9357
|
+
} while (this.again && this.active);
|
|
9358
|
+
for (const runId of await store.heldRunIds()) await this.deps.resume(runId);
|
|
9359
|
+
await this.count(store);
|
|
9360
|
+
const [delay, expiry] = await Promise.all([store.nextDelayDue(), store.nextExpiry()]);
|
|
9361
|
+
next = Math.min(delay ? Date.parse(delay) : Infinity, expiry ? Date.parse(expiry) : Infinity);
|
|
9362
|
+
} catch (error) {
|
|
9363
|
+
this.deps.logger.warn("Waitpoint poll failed: @error", error);
|
|
9364
|
+
} finally {
|
|
9365
|
+
this.polling = false;
|
|
9366
|
+
if (this.nudged !== void 0) next = Math.min(next, this.nudged);
|
|
9367
|
+
this.nudged = void 0;
|
|
9368
|
+
if (this.active) this.arm(next);
|
|
9369
|
+
}
|
|
9370
|
+
}
|
|
9371
|
+
arm(due) {
|
|
9372
|
+
clearTimeout(this.timer);
|
|
9373
|
+
const delay = Math.min(Math.max(Number.isFinite(due) ? due - Date.now() : Infinity, 0), WAITPOINT_POLL_MAX_MS);
|
|
9374
|
+
this.timerAt = Date.now() + delay;
|
|
9375
|
+
this.timer = setTimeout(() => {
|
|
9376
|
+
this.timer = void 0;
|
|
9377
|
+
this.timerAt = void 0;
|
|
9378
|
+
this.poll();
|
|
9379
|
+
}, delay);
|
|
9380
|
+
this.timer.unref();
|
|
9381
|
+
}
|
|
9382
|
+
};
|
|
8936
9383
|
//#endregion
|
|
8937
9384
|
//#region src/reactor/run-gate.ts
|
|
8938
9385
|
const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
|
|
@@ -8970,8 +9417,11 @@ var WorkflowRunGate = class {
|
|
|
8970
9417
|
* - **SINGLETON**: refused outright while a run is active. The caller
|
|
8971
9418
|
* journals the refusal rather than dropping it silently, because a firing
|
|
8972
9419
|
* that vanished is indistinguishable from a trigger that never fired.
|
|
9420
|
+
*
|
|
9421
|
+
* A `resume` is a waiting run's next leg, not a new firing: it queues for
|
|
9422
|
+
* the slot under every policy, past the queue depth too.
|
|
8973
9423
|
*/
|
|
8974
|
-
async admit(workflowId, policy, onQueue) {
|
|
9424
|
+
async admit(workflowId, policy, onQueue, options = {}) {
|
|
8975
9425
|
if (this.closed) return {
|
|
8976
9426
|
admitted: false,
|
|
8977
9427
|
reason: CLOSED_REASON,
|
|
@@ -8990,7 +9440,7 @@ var WorkflowRunGate = class {
|
|
|
8990
9440
|
this.lanes.set(workflowId, lane);
|
|
8991
9441
|
let waited = false;
|
|
8992
9442
|
if (lane.active >= limit) {
|
|
8993
|
-
if (policy.concurrency === "SINGLETON") {
|
|
9443
|
+
if (policy.concurrency === "SINGLETON" && !options.resume) {
|
|
8994
9444
|
this.forgetIfIdle(workflowId, lane);
|
|
8995
9445
|
return {
|
|
8996
9446
|
admitted: false,
|
|
@@ -8998,7 +9448,7 @@ var WorkflowRunGate = class {
|
|
|
8998
9448
|
reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
|
|
8999
9449
|
};
|
|
9000
9450
|
}
|
|
9001
|
-
if (lane.waiting.length >= this.maxQueued) return {
|
|
9451
|
+
if (!options.resume && lane.waiting.length >= this.maxQueued) return {
|
|
9002
9452
|
admitted: false,
|
|
9003
9453
|
refusal: "queue-full",
|
|
9004
9454
|
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
@@ -10653,12 +11103,72 @@ function refusalStatus(refusal) {
|
|
|
10653
11103
|
switch (refusal) {
|
|
10654
11104
|
case "parked":
|
|
10655
11105
|
case "singleton":
|
|
10656
|
-
case "stale":
|
|
11106
|
+
case "stale":
|
|
11107
|
+
case "cancelled": return 409;
|
|
10657
11108
|
case "queue-full":
|
|
10658
11109
|
case "expired": return 429;
|
|
10659
11110
|
default: return 500;
|
|
10660
11111
|
}
|
|
10661
11112
|
}
|
|
11113
|
+
function runResultReply(result, succeededStatus) {
|
|
11114
|
+
return {
|
|
11115
|
+
status: result.status === "SUCCEEDED" ? succeededStatus : result.status === "WAITING" ? 202 : refusalStatus(result.refusal),
|
|
11116
|
+
contentType: JSON_CONTENT_TYPE,
|
|
11117
|
+
body: JSON.stringify({
|
|
11118
|
+
runId: result.runId,
|
|
11119
|
+
status: result.status,
|
|
11120
|
+
error: result.error ?? null
|
|
11121
|
+
})
|
|
11122
|
+
};
|
|
11123
|
+
}
|
|
11124
|
+
const RESPONDED = Symbol("a step responded");
|
|
11125
|
+
const DROPPED_RESPONSE_HEADERS = /* @__PURE__ */ new Set([
|
|
11126
|
+
"connection",
|
|
11127
|
+
"content-length",
|
|
11128
|
+
"keep-alive",
|
|
11129
|
+
"set-cookie",
|
|
11130
|
+
"transfer-encoding",
|
|
11131
|
+
"upgrade"
|
|
11132
|
+
]);
|
|
11133
|
+
const ORIGIN_SAFE_HEADERS = {
|
|
11134
|
+
"x-content-type-options": "nosniff",
|
|
11135
|
+
"content-security-policy": "sandbox"
|
|
11136
|
+
};
|
|
11137
|
+
function sendableHeader(name, value) {
|
|
11138
|
+
try {
|
|
11139
|
+
validateHeaderName(name);
|
|
11140
|
+
validateHeaderValue(name, value);
|
|
11141
|
+
return true;
|
|
11142
|
+
} catch {
|
|
11143
|
+
return false;
|
|
11144
|
+
}
|
|
11145
|
+
}
|
|
11146
|
+
/** A step's `run.respond` / `run.stop` answer as an HTTP reply. As upstream,
|
|
11147
|
+
* none answers 200 with `{}`, and a string body goes as text. */
|
|
11148
|
+
function runResponseReply(response) {
|
|
11149
|
+
const status = typeof response.status === "number" && Number.isInteger(response.status) && response.status >= 200 && response.status <= 599 ? response.status : 200;
|
|
11150
|
+
const body = response.body ?? {};
|
|
11151
|
+
let contentType = typeof body === "string" ? "text/plain; charset=utf-8" : JSON_CONTENT_TYPE;
|
|
11152
|
+
const headers = {};
|
|
11153
|
+
const given = response.headers && typeof response.headers === "object" ? response.headers : {};
|
|
11154
|
+
for (const [rawName, rawValue] of Object.entries(given)) {
|
|
11155
|
+
const name = rawName.toLowerCase();
|
|
11156
|
+
if (typeof rawValue !== "string" && typeof rawValue !== "number") continue;
|
|
11157
|
+
const value = String(rawValue);
|
|
11158
|
+
if (!sendableHeader(name, value)) continue;
|
|
11159
|
+
if (name === "content-type") contentType = value;
|
|
11160
|
+
else if (!DROPPED_RESPONSE_HEADERS.has(name)) headers[name] = value;
|
|
11161
|
+
}
|
|
11162
|
+
return {
|
|
11163
|
+
status,
|
|
11164
|
+
contentType,
|
|
11165
|
+
body: typeof body === "string" ? body : JSON.stringify(body),
|
|
11166
|
+
headers: {
|
|
11167
|
+
...headers,
|
|
11168
|
+
...ORIGIN_SAFE_HEADERS
|
|
11169
|
+
}
|
|
11170
|
+
};
|
|
11171
|
+
}
|
|
10662
11172
|
const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
|
|
10663
11173
|
function registrationKey(state) {
|
|
10664
11174
|
const trigger = runnableDefinition(state).trigger;
|
|
@@ -11619,6 +12129,8 @@ var WorkflowRuntimeService = class {
|
|
|
11619
12129
|
startTriggerSupervisor() {
|
|
11620
12130
|
this.supervisor().start();
|
|
11621
12131
|
this.admitAbandonedRuns();
|
|
12132
|
+
this.deliverAllHeld();
|
|
12133
|
+
this.startWaitpointPoller();
|
|
11622
12134
|
this.pendingSweepTimer ??= setInterval(() => {
|
|
11623
12135
|
this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
|
|
11624
12136
|
}, PENDING_SWEEP_INTERVAL_MS);
|
|
@@ -11641,6 +12153,7 @@ var WorkflowRuntimeService = class {
|
|
|
11641
12153
|
stopTriggerSupervisor() {
|
|
11642
12154
|
clearInterval(this.pendingSweepTimer);
|
|
11643
12155
|
this.pendingSweepTimer = void 0;
|
|
12156
|
+
this.stopWaitpointPoller();
|
|
11644
12157
|
this.triggerSupervisor?.stop();
|
|
11645
12158
|
}
|
|
11646
12159
|
closed = false;
|
|
@@ -11848,13 +12361,54 @@ var WorkflowRuntimeService = class {
|
|
|
11848
12361
|
}));
|
|
11849
12362
|
let replied = false;
|
|
11850
12363
|
const takeReply = () => !replied && (replied = true);
|
|
11851
|
-
|
|
12364
|
+
let sent;
|
|
12365
|
+
let responded = () => void 0;
|
|
12366
|
+
const respondedFirst = new Promise((resolve) => {
|
|
12367
|
+
responded = () => resolve(RESPONDED);
|
|
12368
|
+
});
|
|
12369
|
+
const responder = (response) => {
|
|
12370
|
+
if (sent || !takeReply()) return;
|
|
12371
|
+
sent = response;
|
|
12372
|
+
responded();
|
|
12373
|
+
};
|
|
12374
|
+
let ended;
|
|
12375
|
+
let endedEarly = () => void 0;
|
|
12376
|
+
const endedFirst = new Promise((resolve) => {
|
|
12377
|
+
endedEarly = () => resolve({
|
|
12378
|
+
ok: true,
|
|
12379
|
+
result: ended
|
|
12380
|
+
});
|
|
12381
|
+
});
|
|
12382
|
+
const ender = (result) => {
|
|
12383
|
+
if (ended || !takeReply()) return;
|
|
12384
|
+
ended = result;
|
|
12385
|
+
endedEarly();
|
|
12386
|
+
};
|
|
12387
|
+
if (runId) {
|
|
12388
|
+
this.heldReplies.set(runId, takeReply);
|
|
12389
|
+
this.runResponders.set(runId, responder);
|
|
12390
|
+
this.queuedReplies.set(runId, ender);
|
|
12391
|
+
}
|
|
11852
12392
|
let run;
|
|
11853
12393
|
try {
|
|
11854
|
-
|
|
11855
|
-
|
|
12394
|
+
const raced = await Promise.race([
|
|
12395
|
+
firing,
|
|
12396
|
+
respondedFirst,
|
|
12397
|
+
endedFirst,
|
|
12398
|
+
timeout(DELIVERY_TIMEOUT_MS)
|
|
12399
|
+
]);
|
|
12400
|
+
if (sent) return runResponseReply(sent);
|
|
12401
|
+
run = raced;
|
|
12402
|
+
if (run === TIMED_OUT && !takeReply()) run = ended ? {
|
|
12403
|
+
ok: true,
|
|
12404
|
+
result: ended
|
|
12405
|
+
} : await firing;
|
|
11856
12406
|
} finally {
|
|
11857
|
-
if (runId)
|
|
12407
|
+
if (runId) {
|
|
12408
|
+
this.heldReplies.delete(runId);
|
|
12409
|
+
if (this.runResponders.get(runId) === responder) this.runResponders.delete(runId);
|
|
12410
|
+
if (this.queuedReplies.get(runId) === ender) this.queuedReplies.delete(runId);
|
|
12411
|
+
}
|
|
11858
12412
|
}
|
|
11859
12413
|
if (run === TIMED_OUT) {
|
|
11860
12414
|
this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
|
|
@@ -11880,15 +12434,7 @@ var WorkflowRuntimeService = class {
|
|
|
11880
12434
|
body: JSON.stringify({ error: message })
|
|
11881
12435
|
};
|
|
11882
12436
|
}
|
|
11883
|
-
return
|
|
11884
|
-
status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
|
|
11885
|
-
contentType: JSON_CONTENT_TYPE,
|
|
11886
|
-
body: JSON.stringify({
|
|
11887
|
-
runId: run.result.runId,
|
|
11888
|
-
status: run.result.status,
|
|
11889
|
-
error: run.result.error ?? null
|
|
11890
|
-
})
|
|
11891
|
-
};
|
|
12437
|
+
return runResultReply(run.result, config.responseStatus);
|
|
11892
12438
|
}
|
|
11893
12439
|
async pieceHandshake(binding, request) {
|
|
11894
12440
|
const handshake = await this.pieceHandshakeConfig(binding);
|
|
@@ -12624,6 +13170,61 @@ var WorkflowRuntimeService = class {
|
|
|
12624
13170
|
type: "value"
|
|
12625
13171
|
}]
|
|
12626
13172
|
};
|
|
13173
|
+
case DELAY_BLOCK: return {
|
|
13174
|
+
source: "static",
|
|
13175
|
+
nodes: [{
|
|
13176
|
+
name: "resumedAt",
|
|
13177
|
+
type: "DateTime!"
|
|
13178
|
+
}]
|
|
13179
|
+
};
|
|
13180
|
+
case APPROVAL_BLOCK: return {
|
|
13181
|
+
source: "static",
|
|
13182
|
+
nodes: [
|
|
13183
|
+
{
|
|
13184
|
+
name: "approved",
|
|
13185
|
+
type: "Boolean!"
|
|
13186
|
+
},
|
|
13187
|
+
{
|
|
13188
|
+
name: "reason",
|
|
13189
|
+
type: "String"
|
|
13190
|
+
},
|
|
13191
|
+
{
|
|
13192
|
+
name: "decidedBy",
|
|
13193
|
+
type: "String",
|
|
13194
|
+
description: "The approver's DID"
|
|
13195
|
+
},
|
|
13196
|
+
{
|
|
13197
|
+
name: "decidedAt",
|
|
13198
|
+
type: "DateTime!"
|
|
13199
|
+
}
|
|
13200
|
+
]
|
|
13201
|
+
};
|
|
13202
|
+
case STOP_BLOCK: return {
|
|
13203
|
+
source: "static",
|
|
13204
|
+
nodes: [{
|
|
13205
|
+
name: "response",
|
|
13206
|
+
type: "JSONObject",
|
|
13207
|
+
description: "The answer sent, when one was set"
|
|
13208
|
+
}]
|
|
13209
|
+
};
|
|
13210
|
+
case WAIT_FOR_WEBHOOK_BLOCK: return {
|
|
13211
|
+
source: "static",
|
|
13212
|
+
nodes: [
|
|
13213
|
+
{
|
|
13214
|
+
name: "body",
|
|
13215
|
+
type: "Unknown",
|
|
13216
|
+
description: "The resuming request's body"
|
|
13217
|
+
},
|
|
13218
|
+
{
|
|
13219
|
+
name: "headers",
|
|
13220
|
+
type: "JSONObject!"
|
|
13221
|
+
},
|
|
13222
|
+
{
|
|
13223
|
+
name: "queryParams",
|
|
13224
|
+
type: "JSONObject!"
|
|
13225
|
+
}
|
|
13226
|
+
]
|
|
13227
|
+
};
|
|
12627
13228
|
case DOCUMENT_CREATED_BLOCK:
|
|
12628
13229
|
case DOCUMENT_DELETED_BLOCK: return {
|
|
12629
13230
|
source: "static",
|
|
@@ -12945,6 +13546,7 @@ var WorkflowRuntimeService = class {
|
|
|
12945
13546
|
let definition;
|
|
12946
13547
|
let runUser;
|
|
12947
13548
|
let readGeneration;
|
|
13549
|
+
let revision;
|
|
12948
13550
|
try {
|
|
12949
13551
|
if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12950
13552
|
run.refused = true;
|
|
@@ -12954,6 +13556,7 @@ var WorkflowRuntimeService = class {
|
|
|
12954
13556
|
if (cached) {
|
|
12955
13557
|
state = cached.state;
|
|
12956
13558
|
documentName = cached.name;
|
|
13559
|
+
revision = cached.revision;
|
|
12957
13560
|
readGeneration = this.workflowGeneration(workflowId);
|
|
12958
13561
|
} else {
|
|
12959
13562
|
const generation = this.workflowGeneration(workflowId);
|
|
@@ -12965,6 +13568,7 @@ var WorkflowRuntimeService = class {
|
|
|
12965
13568
|
}
|
|
12966
13569
|
state = document.state.global;
|
|
12967
13570
|
documentName = document.header.name;
|
|
13571
|
+
revision = globalRevision(document);
|
|
12968
13572
|
this.cacheLoaded(workflowId, generation, document);
|
|
12969
13573
|
}
|
|
12970
13574
|
if (state.status !== "ENABLED") {
|
|
@@ -13093,74 +13697,63 @@ var WorkflowRuntimeService = class {
|
|
|
13093
13697
|
}
|
|
13094
13698
|
}
|
|
13095
13699
|
let runId = enqueuedRunId ?? null;
|
|
13096
|
-
let journalFailed = false;
|
|
13097
|
-
const executionOrder = /* @__PURE__ */ new Map();
|
|
13098
13700
|
let session;
|
|
13099
13701
|
try {
|
|
13100
|
-
|
|
13702
|
+
const journaled = await this.telemetry.phase("journal.start", {}, async () => {
|
|
13101
13703
|
if (enqueuedRunId) {
|
|
13102
|
-
await store?.beginRun(enqueuedRunId, {
|
|
13704
|
+
if (await store?.beginRun(enqueuedRunId, {
|
|
13103
13705
|
workflowName: runJournalName(state.name, documentName),
|
|
13104
13706
|
workflowVersion: runnable.version
|
|
13105
|
-
})
|
|
13106
|
-
|
|
13707
|
+
}) === false) return {
|
|
13708
|
+
runId: null,
|
|
13709
|
+
cancelled: true
|
|
13710
|
+
};
|
|
13711
|
+
return { runId: enqueuedRunId };
|
|
13107
13712
|
}
|
|
13108
|
-
return await store?.startRun({
|
|
13713
|
+
return { runId: await store?.startRun({
|
|
13109
13714
|
workflowId,
|
|
13110
13715
|
workflowName: runJournalName(state.name, documentName),
|
|
13111
13716
|
workflowVersion: runnable.version,
|
|
13112
13717
|
triggerKind,
|
|
13113
13718
|
triggerPayload,
|
|
13114
13719
|
rerunOf: resume?.rerunOf
|
|
13115
|
-
}) ?? null;
|
|
13720
|
+
}) ?? null };
|
|
13116
13721
|
});
|
|
13722
|
+
if (journaled.cancelled) {
|
|
13723
|
+
const row = enqueuedRunId ? await store?.getRun(enqueuedRunId) : void 0;
|
|
13724
|
+
return {
|
|
13725
|
+
status: CANCELLED_RUN_STATUS,
|
|
13726
|
+
steps: [],
|
|
13727
|
+
runId: enqueuedRunId ?? null,
|
|
13728
|
+
...row?.error ? { error: row.error } : {},
|
|
13729
|
+
refusal: "cancelled"
|
|
13730
|
+
};
|
|
13731
|
+
}
|
|
13732
|
+
runId = journaled.runId;
|
|
13117
13733
|
if (runId) run.span.setAttribute("run.id", runId);
|
|
13118
13734
|
if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
|
|
13119
13735
|
session = this.workers().session();
|
|
13120
|
-
const {
|
|
13121
|
-
const journal = store;
|
|
13122
|
-
const journaledRunId = runId;
|
|
13123
|
-
const handed = /* @__PURE__ */ new Set();
|
|
13124
|
-
const result = await withRunScope({
|
|
13736
|
+
const { result, handed } = await this.executeLeg({
|
|
13125
13737
|
workflowId,
|
|
13126
13738
|
runId,
|
|
13739
|
+
store,
|
|
13740
|
+
session,
|
|
13741
|
+
definition,
|
|
13742
|
+
runnable,
|
|
13743
|
+
policy,
|
|
13744
|
+
deadline,
|
|
13127
13745
|
connections,
|
|
13128
|
-
|
|
13129
|
-
...runUser !== void 0 ? { runUser } : {},
|
|
13130
|
-
attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
|
|
13131
|
-
recordDocuments: async (documentIds) => {
|
|
13132
|
-
for (const documentId of documentIds) handed.add(documentId);
|
|
13133
|
-
if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
|
|
13134
|
-
}
|
|
13135
|
-
}, () => runWorkflow({
|
|
13136
|
-
definition: {
|
|
13137
|
-
...definition,
|
|
13138
|
-
variables
|
|
13139
|
-
},
|
|
13140
|
-
executor,
|
|
13141
|
-
declaredPorts: blockPorts,
|
|
13142
|
-
blockDefaults: (block) => this.blockDefaults(block),
|
|
13143
|
-
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
13746
|
+
runUser,
|
|
13144
13747
|
triggerPayload,
|
|
13748
|
+
executor,
|
|
13145
13749
|
completedSteps: resume?.completedSteps,
|
|
13146
|
-
|
|
13147
|
-
|
|
13148
|
-
|
|
13149
|
-
|
|
13150
|
-
|
|
13151
|
-
|
|
13152
|
-
|
|
13153
|
-
if (journalFailed) return;
|
|
13154
|
-
journalFailed = true;
|
|
13155
|
-
this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
13156
|
-
}
|
|
13157
|
-
} : void 0
|
|
13158
|
-
}));
|
|
13159
|
-
if (store && journaledRunId) try {
|
|
13160
|
-
await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
|
|
13161
|
-
} catch (error) {
|
|
13162
|
-
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
13163
|
-
}
|
|
13750
|
+
replayed: resume?.completedSteps,
|
|
13751
|
+
pin: () => ({
|
|
13752
|
+
definition_hash: this.definitionHash(workflowId, state),
|
|
13753
|
+
definition_op_index: revision,
|
|
13754
|
+
definition_scope_hash: scopeHash(state)
|
|
13755
|
+
})
|
|
13756
|
+
});
|
|
13164
13757
|
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
13165
13758
|
const finished = {
|
|
13166
13759
|
...result,
|
|
@@ -13174,7 +13767,14 @@ var WorkflowRuntimeService = class {
|
|
|
13174
13767
|
runId
|
|
13175
13768
|
};
|
|
13176
13769
|
} catch (error) {
|
|
13177
|
-
if (store && runId)
|
|
13770
|
+
if (store && runId) {
|
|
13771
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
13772
|
+
const closed = await store.failRun(runId, message, errorNameOf(error));
|
|
13773
|
+
this.answerClosed(closed, runId, {
|
|
13774
|
+
status: "FAILED",
|
|
13775
|
+
error: message
|
|
13776
|
+
});
|
|
13777
|
+
}
|
|
13178
13778
|
throw error;
|
|
13179
13779
|
} finally {
|
|
13180
13780
|
session?.close();
|
|
@@ -13182,6 +13782,115 @@ var WorkflowRuntimeService = class {
|
|
|
13182
13782
|
dispatched();
|
|
13183
13783
|
}
|
|
13184
13784
|
}
|
|
13785
|
+
async executeLeg(leg) {
|
|
13786
|
+
const { workflowId, runId, store, policy } = leg;
|
|
13787
|
+
let journalFailed = false;
|
|
13788
|
+
const executionOrder = /* @__PURE__ */ new Map();
|
|
13789
|
+
const { variables, secretValues } = await resolveVariables(leg.runnable.variables, this.secretProvider(), workflowId);
|
|
13790
|
+
const handed = new Set(leg.handed);
|
|
13791
|
+
const attachments = attachmentOrigins(leg.triggerPayload, handed, leg.replayed);
|
|
13792
|
+
for (const ref of leg.written ?? []) attachments.written.add(ref);
|
|
13793
|
+
const waitpoints = store && runId ? this.waitpointPort(store, workflowId, runId, maxSuspensionMs(leg.runnable)) : void 0;
|
|
13794
|
+
const respond = leg.respond ?? (runId && !leg.resume ? (response) => this.runResponders.get(runId)?.(response) : void 0);
|
|
13795
|
+
let result = await withRunScope({
|
|
13796
|
+
workflowId,
|
|
13797
|
+
runId,
|
|
13798
|
+
connections: leg.connections,
|
|
13799
|
+
pieceWorker: leg.session,
|
|
13800
|
+
...leg.runUser !== void 0 ? { runUser: leg.runUser } : {},
|
|
13801
|
+
attachments,
|
|
13802
|
+
recordDocuments: async (documentIds) => {
|
|
13803
|
+
for (const documentId of documentIds) handed.add(documentId);
|
|
13804
|
+
if (store && runId) await store.recordRunDocuments(runId, documentIds);
|
|
13805
|
+
},
|
|
13806
|
+
...waitpoints ? { waitpoints: waitpoints.port } : {}
|
|
13807
|
+
}, () => runWorkflow({
|
|
13808
|
+
definition: {
|
|
13809
|
+
...leg.definition,
|
|
13810
|
+
variables
|
|
13811
|
+
},
|
|
13812
|
+
executor: leg.executor,
|
|
13813
|
+
declaredPorts: blockPorts,
|
|
13814
|
+
blockDefaults: (block) => this.blockDefaults(block),
|
|
13815
|
+
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
13816
|
+
triggerPayload: leg.triggerPayload,
|
|
13817
|
+
completedSteps: leg.completedSteps,
|
|
13818
|
+
...leg.carriedSteps ? { carriedSteps: leg.carriedSteps } : {},
|
|
13819
|
+
...leg.resume ? { resume: leg.resume } : {},
|
|
13820
|
+
...leg.firstOrdinal !== void 0 ? { firstOrdinal: leg.firstOrdinal } : {},
|
|
13821
|
+
...respond ? { onRespond: respond } : {},
|
|
13822
|
+
...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
|
|
13823
|
+
...leg.deadline !== void 0 ? { deadline: leg.deadline } : {},
|
|
13824
|
+
onStep: store && runId ? async (record, ordinal) => {
|
|
13825
|
+
executionOrder.set(record.stepId, ordinal);
|
|
13826
|
+
try {
|
|
13827
|
+
await this.telemetry.phase("journal.step", {}, () => store.recordStep(runId, ordinal, record));
|
|
13828
|
+
} catch (error) {
|
|
13829
|
+
if (journalFailed) return;
|
|
13830
|
+
journalFailed = true;
|
|
13831
|
+
this.logger.warn(`Run ${runId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
13832
|
+
}
|
|
13833
|
+
} : void 0
|
|
13834
|
+
}));
|
|
13835
|
+
if (store && runId) {
|
|
13836
|
+
let waiting = result.status === WAITING_RUN_STATUS;
|
|
13837
|
+
try {
|
|
13838
|
+
const { held, closed } = await this.telemetry.phase("journal.finish", {}, () => store.finishRun(runId, result, executionOrder, waiting ? leg.pin?.() : void 0));
|
|
13839
|
+
this.answerClosed(closed, runId, {
|
|
13840
|
+
status: waiting ? CANCELLED_RUN_STATUS : result.status,
|
|
13841
|
+
error: waiting ? "The step that created this waitpoint no longer waits on it" : result.error ?? "The run ended without resuming on it"
|
|
13842
|
+
});
|
|
13843
|
+
if (!held) {
|
|
13844
|
+
result = await this.endUnheldWait(store, runId, result);
|
|
13845
|
+
waiting = false;
|
|
13846
|
+
}
|
|
13847
|
+
} catch (error) {
|
|
13848
|
+
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
13849
|
+
}
|
|
13850
|
+
if (waiting) {
|
|
13851
|
+
const toSend = result.waiting && waitpoints?.responseFor(result.waiting.waitpointId);
|
|
13852
|
+
if (toSend) respond?.(toSend);
|
|
13853
|
+
await this.deliverHeld(runId);
|
|
13854
|
+
}
|
|
13855
|
+
}
|
|
13856
|
+
return {
|
|
13857
|
+
result,
|
|
13858
|
+
handed
|
|
13859
|
+
};
|
|
13860
|
+
}
|
|
13861
|
+
async endUnheldWait(store, runId, result) {
|
|
13862
|
+
const on = result.waiting;
|
|
13863
|
+
const step = result.steps.find((record) => record.stepId === on?.stepId);
|
|
13864
|
+
const row = on ? await store.getWaitpoint(on.waitpointId) : void 0;
|
|
13865
|
+
const end = row?.run_id === runId && row.step_id === on?.stepId && row.status === "EXPIRED" ? {
|
|
13866
|
+
status: "CANCELLED",
|
|
13867
|
+
reason: `The wait at step "${row.step_key}" expired at ${row.expires_at} before the run could wait on it`,
|
|
13868
|
+
errorName: WAITPOINT_EXPIRED_ERROR_NAME
|
|
13869
|
+
} : {
|
|
13870
|
+
status: "FAILED",
|
|
13871
|
+
reason: `Step "${step?.key ?? on?.stepId}" waited on waitpoint "${on?.waitpointId}", which is not a live waitpoint it created`,
|
|
13872
|
+
errorName: WAITPOINT_NOT_LIVE_ERROR_NAME
|
|
13873
|
+
};
|
|
13874
|
+
const closed = await store.endWait(runId, {
|
|
13875
|
+
...end,
|
|
13876
|
+
waitpoints: "EXPIRED"
|
|
13877
|
+
});
|
|
13878
|
+
this.answerClosed(closed, runId, {
|
|
13879
|
+
status: end.status,
|
|
13880
|
+
error: end.reason
|
|
13881
|
+
});
|
|
13882
|
+
return {
|
|
13883
|
+
status: end.status,
|
|
13884
|
+
error: end.reason,
|
|
13885
|
+
errorName: end.errorName,
|
|
13886
|
+
steps: result.steps.map((record) => record === step ? {
|
|
13887
|
+
...record,
|
|
13888
|
+
status: "FAILED",
|
|
13889
|
+
error: end.reason,
|
|
13890
|
+
errorName: end.errorName
|
|
13891
|
+
} : record)
|
|
13892
|
+
};
|
|
13893
|
+
}
|
|
13185
13894
|
runGate = new WorkflowRunGate();
|
|
13186
13895
|
unsettledFirings = /* @__PURE__ */ new Map();
|
|
13187
13896
|
runLimits = /* @__PURE__ */ new Map();
|
|
@@ -13206,6 +13915,9 @@ var WorkflowRuntimeService = class {
|
|
|
13206
13915
|
else this.unsettledFirings.delete(workflowId);
|
|
13207
13916
|
}
|
|
13208
13917
|
heldReplies = /* @__PURE__ */ new Map();
|
|
13918
|
+
runResponders = /* @__PURE__ */ new Map();
|
|
13919
|
+
queuedReplies = /* @__PURE__ */ new Map();
|
|
13920
|
+
syncResumes = /* @__PURE__ */ new Map();
|
|
13209
13921
|
/** Why a trigger's firing of a PARKED workflow is refused; undefined when
|
|
13210
13922
|
* it is not parked, or when an operator started the run. */
|
|
13211
13923
|
async parkedFiring(workflowId, triggerKind, publishedVersion) {
|
|
@@ -13273,12 +13985,23 @@ var WorkflowRuntimeService = class {
|
|
|
13273
13985
|
this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
|
|
13274
13986
|
let runId = enqueuedRunId ?? null;
|
|
13275
13987
|
if (store) try {
|
|
13276
|
-
|
|
13988
|
+
let adopted = true;
|
|
13989
|
+
if (runId) adopted = await store.beginRun(runId, details);
|
|
13277
13990
|
else runId = await store.startRun({
|
|
13278
13991
|
workflowId,
|
|
13279
13992
|
...details
|
|
13280
13993
|
});
|
|
13281
|
-
await store.cancelRun(runId, details.reason);
|
|
13994
|
+
if (adopted) await store.cancelRun(runId, details.reason);
|
|
13995
|
+
else {
|
|
13996
|
+
const row = await store.getRun(runId);
|
|
13997
|
+
if (row) return {
|
|
13998
|
+
status: row.status,
|
|
13999
|
+
steps: [],
|
|
14000
|
+
runId,
|
|
14001
|
+
...row.error ? { error: row.error } : {},
|
|
14002
|
+
refusal: row.status === "CANCELLED" ? "cancelled" : refusal
|
|
14003
|
+
};
|
|
14004
|
+
}
|
|
13282
14005
|
} catch (error) {
|
|
13283
14006
|
this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
|
|
13284
14007
|
}
|
|
@@ -13374,6 +14097,677 @@ var WorkflowRuntimeService = class {
|
|
|
13374
14097
|
const parsed = resolvedBlock(await this.resolveBlock(stepBlock(step)));
|
|
13375
14098
|
return parsed ? await this.declaredReactor(parsed) === "read" : false;
|
|
13376
14099
|
}
|
|
14100
|
+
resuming = /* @__PURE__ */ new Set();
|
|
14101
|
+
definitionHashes = /* @__PURE__ */ new Map();
|
|
14102
|
+
/** Our hash of the runnable definition: computed once per published version. */
|
|
14103
|
+
definitionHash(workflowId, state) {
|
|
14104
|
+
const runnable = runnableDefinition(state);
|
|
14105
|
+
if (!runnable.published) return hashOf(canonicalJson(runnable));
|
|
14106
|
+
const key = `${workflowId}@${runnable.version}`;
|
|
14107
|
+
let hash = this.definitionHashes.get(key);
|
|
14108
|
+
if (hash === void 0) {
|
|
14109
|
+
hash = hashOf(canonicalJson(runnable));
|
|
14110
|
+
if (this.definitionHashes.size >= WORKFLOW_CACHE_MAX) this.definitionHashes.clear();
|
|
14111
|
+
this.definitionHashes.set(key, hash);
|
|
14112
|
+
}
|
|
14113
|
+
return hash;
|
|
14114
|
+
}
|
|
14115
|
+
waitpointPort(store, workflowId, runId, suspendMs) {
|
|
14116
|
+
const responses = /* @__PURE__ */ new Map();
|
|
14117
|
+
const tokens = /* @__PURE__ */ new Map();
|
|
14118
|
+
const configured = this.host.waitpointBaseUrl?.replace(/\/+$/, "");
|
|
14119
|
+
const baseUrl = configured && URL.canParse(configured) ? configured : void 0;
|
|
14120
|
+
return {
|
|
14121
|
+
port: {
|
|
14122
|
+
...baseUrl ? { resumeBaseUrl: baseUrl } : {},
|
|
14123
|
+
create: async (request) => {
|
|
14124
|
+
if (!baseUrl && request.type === "WEBHOOK" && !request.approval) throw new Error("This host serves no resume URLs: resume URLs need an HTTP host");
|
|
14125
|
+
const createdAt = /* @__PURE__ */ new Date();
|
|
14126
|
+
const resumeAt = request.resumeDateTime ? new Date(Date.parse(request.resumeDateTime)).toISOString() : null;
|
|
14127
|
+
if (resumeAt && Date.parse(resumeAt) > MAX_WAITPOINT_MS) throw new Error(`A delay until ${resumeAt} is past ${MAX_WAITPOINT_ISO}, the latest a run can wait until`);
|
|
14128
|
+
const expiresAt = waitpointExpiry(createdAt, request, suspendMs);
|
|
14129
|
+
if (request.type === "DELAY" && resumeAt && suspendMs !== null) {
|
|
14130
|
+
if (Date.parse(resumeAt) - createdAt.getTime() > suspendMs) throw new Error(`A delay until ${resumeAt} is longer than the workflow's maxSuspensionDays allows`);
|
|
14131
|
+
}
|
|
14132
|
+
const token = baseUrl && !request.approval && request.type !== "DELAY" ? request.token ?? randomBytes(32).toString("base64url") : null;
|
|
14133
|
+
const id = randomUUID();
|
|
14134
|
+
const { waitpoint, created: inserted } = await store.createWaitpoint({
|
|
14135
|
+
id,
|
|
14136
|
+
run_id: runId,
|
|
14137
|
+
workflow_id: workflowId,
|
|
14138
|
+
step_id: request.stepId,
|
|
14139
|
+
step_key: request.stepKey,
|
|
14140
|
+
step_path: request.stepId,
|
|
14141
|
+
piece_name: request.pieceName,
|
|
14142
|
+
block_name: request.blockName,
|
|
14143
|
+
type: request.type,
|
|
14144
|
+
kind: request.approval ? "approval" : request.type === "DELAY" ? "delay" : "webhook",
|
|
14145
|
+
status: "PENDING",
|
|
14146
|
+
token_hash: token ? tokenHash(token) : null,
|
|
14147
|
+
created_at: createdAt.toISOString(),
|
|
14148
|
+
resume_at: resumeAt,
|
|
14149
|
+
expires_at: request.type === "DELAY" ? null : expiresAt,
|
|
14150
|
+
resolved_at: null,
|
|
14151
|
+
decided_by: null,
|
|
14152
|
+
approval: request.approval ? JSON.stringify(request.approval) : null,
|
|
14153
|
+
resume_payload: null,
|
|
14154
|
+
response_to_send: request.responseToSend ? JSON.stringify(request.responseToSend) : null
|
|
14155
|
+
});
|
|
14156
|
+
if (!inserted) {
|
|
14157
|
+
const known = tokens.get(waitpoint.id);
|
|
14158
|
+
if (waitpoint.token_hash !== null && known === void 0) throw new Error(`Step "${request.stepKey}" has a waitpoint from an earlier leg, whose resume URL cannot be issued again`);
|
|
14159
|
+
return {
|
|
14160
|
+
id: waitpoint.id,
|
|
14161
|
+
resumeUrl: known ? `${baseUrl}/${known}` : ""
|
|
14162
|
+
};
|
|
14163
|
+
}
|
|
14164
|
+
if (token) tokens.set(id, token);
|
|
14165
|
+
if (request.responseToSend) responses.set(id, request.responseToSend);
|
|
14166
|
+
const due = request.type === "DELAY" ? resumeAt : expiresAt;
|
|
14167
|
+
if (due) this.waitpointPoller.nudge(due);
|
|
14168
|
+
return {
|
|
14169
|
+
id,
|
|
14170
|
+
resumeUrl: token ? `${baseUrl}/${token}` : ""
|
|
14171
|
+
};
|
|
14172
|
+
}
|
|
14173
|
+
},
|
|
14174
|
+
responseFor: (waitpointId) => responses.get(waitpointId)
|
|
14175
|
+
};
|
|
14176
|
+
}
|
|
14177
|
+
/** Re-enters a WAITING run that holds a COMPLETED waitpoint, through the
|
|
14178
|
+
* dispatcher. Whoever writes second, the resume or the WAITING, delivers. */
|
|
14179
|
+
async deliverHeld(runId) {
|
|
14180
|
+
if (this.closed) return this.answerShutDown(runId);
|
|
14181
|
+
if (this.resuming.has(runId)) return;
|
|
14182
|
+
try {
|
|
14183
|
+
const store = await this.store();
|
|
14184
|
+
if (!store || !await store.hasCompletedWaitpoint(runId)) return;
|
|
14185
|
+
const run = await store.getRun(runId);
|
|
14186
|
+
if (run?.status !== "WAITING") return this.answerNotWaiting(store, runId, run?.status);
|
|
14187
|
+
if (this.resuming.has(runId)) return;
|
|
14188
|
+
this.resuming.add(runId);
|
|
14189
|
+
this.resumeRun(runId, run.workflow_id);
|
|
14190
|
+
} catch (error) {
|
|
14191
|
+
this.logger.warn(`Could not resume run ${runId}: @error`, error);
|
|
14192
|
+
}
|
|
14193
|
+
}
|
|
14194
|
+
answerNotWaiting(store, runId, status) {
|
|
14195
|
+
if (store.isRunInFlight(runId)) return;
|
|
14196
|
+
this.answerUnresumed(runId, {
|
|
14197
|
+
status: status ?? "gone",
|
|
14198
|
+
error: "The run is not waiting to resume in this runtime",
|
|
14199
|
+
code: 409
|
|
14200
|
+
});
|
|
14201
|
+
}
|
|
14202
|
+
waitpointPoller = new WaitpointPoller({
|
|
14203
|
+
store: () => this.store(),
|
|
14204
|
+
resume: (runId) => this.deliverHeld(runId),
|
|
14205
|
+
ended: (end) => Promise.resolve(this.answerClosed(end.closed, end.runId, {
|
|
14206
|
+
status: CANCELLED_RUN_STATUS,
|
|
14207
|
+
error: end.reason
|
|
14208
|
+
})),
|
|
14209
|
+
logger: { warn: (message, ...args) => this.logger.warn(message, ...args) }
|
|
14210
|
+
});
|
|
14211
|
+
unobserveSuspended;
|
|
14212
|
+
/** Test hook: one waitpoint poll pass now; a no-op until the supervisor starts. */
|
|
14213
|
+
pollWaitpointsNow() {
|
|
14214
|
+
return this.waitpointPoller.poll();
|
|
14215
|
+
}
|
|
14216
|
+
startWaitpointPoller() {
|
|
14217
|
+
if (this.closed) return;
|
|
14218
|
+
this.unobserveSuspended ??= this.telemetry.observeSuspended(() => this.waitpointPoller.suspendedRuns());
|
|
14219
|
+
this.waitpointPoller.start();
|
|
14220
|
+
}
|
|
14221
|
+
stopWaitpointPoller() {
|
|
14222
|
+
this.waitpointPoller.stop();
|
|
14223
|
+
this.unobserveSuspended?.();
|
|
14224
|
+
this.unobserveSuspended = void 0;
|
|
14225
|
+
}
|
|
14226
|
+
async answerHeldResumes(store, runId, error) {
|
|
14227
|
+
if (this.syncResumes.size === 0) return;
|
|
14228
|
+
const rows = (await store.listWaitpointsForRuns([runId])).get(runId) ?? [];
|
|
14229
|
+
this.answerClosed(rows.map((row) => row.id), runId, {
|
|
14230
|
+
status: CANCELLED_RUN_STATUS,
|
|
14231
|
+
error
|
|
14232
|
+
});
|
|
14233
|
+
}
|
|
14234
|
+
answerClosed(waitpointIds, runId, end) {
|
|
14235
|
+
if (this.syncResumes.size === 0) return;
|
|
14236
|
+
for (const id of waitpointIds) this.heldResume(id)?.closed(runId, end.status, end.error);
|
|
14237
|
+
}
|
|
14238
|
+
heldResume(waitpointId) {
|
|
14239
|
+
const candidates = [...this.syncResumes.get(waitpointId) ?? []];
|
|
14240
|
+
if (candidates.length === 0) return void 0;
|
|
14241
|
+
const winner = (async () => {
|
|
14242
|
+
for (const candidate of candidates) if (await candidate.won) return candidate.entry;
|
|
14243
|
+
})();
|
|
14244
|
+
return {
|
|
14245
|
+
respond: (response) => void winner.then((e) => e?.respond(response)),
|
|
14246
|
+
ended: (result) => void winner.then((e) => e?.ended(result)),
|
|
14247
|
+
closed: (runId, status, error, code) => void winner.then((e) => e?.closed(runId, status, error, code))
|
|
14248
|
+
};
|
|
14249
|
+
}
|
|
14250
|
+
answerUnresumed(runId, end) {
|
|
14251
|
+
for (const [id, candidates] of this.syncResumes) {
|
|
14252
|
+
if (![...candidates].some((candidate) => candidate.runId === runId)) continue;
|
|
14253
|
+
this.heldResume(id)?.closed(runId, end.status, end.error, end.code);
|
|
14254
|
+
}
|
|
14255
|
+
}
|
|
14256
|
+
async deliverAllHeld() {
|
|
14257
|
+
try {
|
|
14258
|
+
const store = await this.store();
|
|
14259
|
+
for (const runId of await store?.heldRunIds() ?? []) await this.deliverHeld(runId);
|
|
14260
|
+
} catch (error) {
|
|
14261
|
+
this.logger.warn("Could not deliver held resumes: @error", error);
|
|
14262
|
+
}
|
|
14263
|
+
}
|
|
14264
|
+
resumeRun(runId, workflowId) {
|
|
14265
|
+
const started = performance.now();
|
|
14266
|
+
const attributes = {
|
|
14267
|
+
"workflow.id": workflowId,
|
|
14268
|
+
"trigger.kind": "resume",
|
|
14269
|
+
"run.id": runId
|
|
14270
|
+
};
|
|
14271
|
+
return this.telemetry.span("workflow.run", attributes, async (span) => {
|
|
14272
|
+
let status = "REFUSED";
|
|
14273
|
+
try {
|
|
14274
|
+
const result = await this.runResumed(runId, workflowId);
|
|
14275
|
+
if (result) status = result.status;
|
|
14276
|
+
if (status === "FAILED") span.setStatus({
|
|
14277
|
+
code: SpanStatusCode.ERROR,
|
|
14278
|
+
message: status
|
|
14279
|
+
});
|
|
14280
|
+
return result;
|
|
14281
|
+
} finally {
|
|
14282
|
+
span.setAttribute("run.status", status);
|
|
14283
|
+
this.telemetry.recordRun(performance.now() - started, status, "resume");
|
|
14284
|
+
}
|
|
14285
|
+
});
|
|
14286
|
+
}
|
|
14287
|
+
async runResumed(runId, workflowId) {
|
|
14288
|
+
const admission = await this.runGate.admit(workflowId, await this.resumePolicy(workflowId), void 0, { resume: true });
|
|
14289
|
+
if (!admission.admitted) {
|
|
14290
|
+
this.resuming.delete(runId);
|
|
14291
|
+
this.answerShutDown(runId);
|
|
14292
|
+
return;
|
|
14293
|
+
}
|
|
14294
|
+
let dispatched;
|
|
14295
|
+
try {
|
|
14296
|
+
dispatched = await this.dispatcher().admit(workflowId);
|
|
14297
|
+
} catch {
|
|
14298
|
+
admission.release();
|
|
14299
|
+
this.resuming.delete(runId);
|
|
14300
|
+
this.answerShutDown(runId);
|
|
14301
|
+
return;
|
|
14302
|
+
}
|
|
14303
|
+
const store = await this.store();
|
|
14304
|
+
let claimed = false;
|
|
14305
|
+
let redeliver = false;
|
|
14306
|
+
let session;
|
|
14307
|
+
let held;
|
|
14308
|
+
let outcome;
|
|
14309
|
+
try {
|
|
14310
|
+
if (!store || this.closed) {
|
|
14311
|
+
this.answerShutDown(runId);
|
|
14312
|
+
return;
|
|
14313
|
+
}
|
|
14314
|
+
claimed = await store.claimWaitingRun(runId);
|
|
14315
|
+
this.resuming.delete(runId);
|
|
14316
|
+
if (!claimed) {
|
|
14317
|
+
this.answerNotWaiting(store, runId, (await store.getRun(runId))?.status);
|
|
14318
|
+
return;
|
|
14319
|
+
}
|
|
14320
|
+
const row = await store.getRun(runId);
|
|
14321
|
+
if (!row) throw new Error(`Run "${runId}" not found`);
|
|
14322
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14323
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14324
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
14325
|
+
const current = document.state.global;
|
|
14326
|
+
const stopped = await this.resumeStopped(workflowId, row.trigger_kind, current);
|
|
14327
|
+
if (stopped) {
|
|
14328
|
+
const closed = await store.endWait(runId, {
|
|
14329
|
+
status: CANCELLED_RUN_STATUS,
|
|
14330
|
+
reason: stopped.reason,
|
|
14331
|
+
errorName: stopped.errorName,
|
|
14332
|
+
waitpoints: "CANCELLED"
|
|
14333
|
+
});
|
|
14334
|
+
claimed = false;
|
|
14335
|
+
this.answerClosed(closed, runId, {
|
|
14336
|
+
status: CANCELLED_RUN_STATUS,
|
|
14337
|
+
error: stopped.reason
|
|
14338
|
+
});
|
|
14339
|
+
return {
|
|
14340
|
+
status: CANCELLED_RUN_STATUS,
|
|
14341
|
+
steps: [],
|
|
14342
|
+
runId,
|
|
14343
|
+
error: stopped.reason,
|
|
14344
|
+
errorName: stopped.errorName,
|
|
14345
|
+
refusal: "cancelled"
|
|
14346
|
+
};
|
|
14347
|
+
}
|
|
14348
|
+
const consumed = await store.consumeCompletedWaitpoint(runId);
|
|
14349
|
+
if (!consumed) {
|
|
14350
|
+
await store.returnToWaiting(runId);
|
|
14351
|
+
claimed = false;
|
|
14352
|
+
redeliver = true;
|
|
14353
|
+
return;
|
|
14354
|
+
}
|
|
14355
|
+
const { waitpoint } = consumed;
|
|
14356
|
+
held = this.heldResume(waitpoint.id);
|
|
14357
|
+
this.answerClosed(consumed.closed, runId, {
|
|
14358
|
+
status: "RUNNING",
|
|
14359
|
+
error: "The run resumed on another of its waitpoints"
|
|
14360
|
+
});
|
|
14361
|
+
const state = await this.pinnedState(workflowId, runId, store, current);
|
|
14362
|
+
const runnable = runnableDefinition(state);
|
|
14363
|
+
const definition = toWorkflowDefinition(state);
|
|
14364
|
+
const policy = effectiveRunPolicy(runnable);
|
|
14365
|
+
const connections = declaredConnectionIds(definition);
|
|
14366
|
+
const reactorConnections = declaredReactorConnectionIds(definition);
|
|
14367
|
+
const runUser = reactorConnections.size === 0 ? void 0 : await assertReactorConnectionsReadable(this.host, workflowId, reactorConnections);
|
|
14368
|
+
const journal = carriedFromJournal(await store.getSteps(runId));
|
|
14369
|
+
if (!journal.waitingStepId) throw new Error(`Run "${runId}" has no waiting step to resume`);
|
|
14370
|
+
session = this.workers().session();
|
|
14371
|
+
const { result } = await this.executeLeg({
|
|
14372
|
+
workflowId,
|
|
14373
|
+
runId,
|
|
14374
|
+
store,
|
|
14375
|
+
session,
|
|
14376
|
+
definition,
|
|
14377
|
+
runnable,
|
|
14378
|
+
policy,
|
|
14379
|
+
deadline: policy.runTimeoutSeconds ? Date.now() + policy.runTimeoutSeconds * 1e3 : void 0,
|
|
14380
|
+
connections,
|
|
14381
|
+
runUser,
|
|
14382
|
+
triggerPayload: journaledTriggerPayload(row.trigger_payload),
|
|
14383
|
+
executor: this.blockExecutor(store),
|
|
14384
|
+
carriedSteps: journal.carried,
|
|
14385
|
+
resume: {
|
|
14386
|
+
stepId: journal.waitingStepId,
|
|
14387
|
+
payload: parseJson(waitpoint.resume_payload) ?? {}
|
|
14388
|
+
},
|
|
14389
|
+
firstOrdinal: journal.nextOrdinal,
|
|
14390
|
+
written: journal.attachmentRefs,
|
|
14391
|
+
handed: await store.getRunDocuments(runId),
|
|
14392
|
+
...held ? { respond: held.respond } : {}
|
|
14393
|
+
});
|
|
14394
|
+
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(row.trigger_kind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
14395
|
+
outcome = {
|
|
14396
|
+
...result,
|
|
14397
|
+
runId
|
|
14398
|
+
};
|
|
14399
|
+
return outcome;
|
|
14400
|
+
} catch (error) {
|
|
14401
|
+
if (store && claimed) {
|
|
14402
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
14403
|
+
const errorName = errorNameOf(error) ?? "Error";
|
|
14404
|
+
const closed = await store.endWait(runId, {
|
|
14405
|
+
status: "FAILED",
|
|
14406
|
+
reason: message,
|
|
14407
|
+
errorName,
|
|
14408
|
+
waitpoints: "CANCELLED"
|
|
14409
|
+
}).catch(() => []);
|
|
14410
|
+
this.answerClosed(closed, runId, {
|
|
14411
|
+
status: "FAILED",
|
|
14412
|
+
error: message
|
|
14413
|
+
});
|
|
14414
|
+
outcome = {
|
|
14415
|
+
status: "FAILED",
|
|
14416
|
+
steps: [],
|
|
14417
|
+
runId,
|
|
14418
|
+
error: message,
|
|
14419
|
+
errorName
|
|
14420
|
+
};
|
|
14421
|
+
}
|
|
14422
|
+
this.logger.warn(`Resuming run ${runId} failed: @error`, error);
|
|
14423
|
+
return;
|
|
14424
|
+
} finally {
|
|
14425
|
+
this.resuming.delete(runId);
|
|
14426
|
+
session?.close();
|
|
14427
|
+
dispatched();
|
|
14428
|
+
admission.release();
|
|
14429
|
+
held?.ended(outcome);
|
|
14430
|
+
if (redeliver) this.deliverHeld(runId);
|
|
14431
|
+
}
|
|
14432
|
+
}
|
|
14433
|
+
answerShutDown(runId) {
|
|
14434
|
+
if (!this.closed) return;
|
|
14435
|
+
this.answerUnresumed(runId, {
|
|
14436
|
+
status: WAITING_RUN_STATUS,
|
|
14437
|
+
error: "The runtime shut down before the run could resume",
|
|
14438
|
+
code: 503
|
|
14439
|
+
});
|
|
14440
|
+
}
|
|
14441
|
+
async resumePolicy(workflowId) {
|
|
14442
|
+
try {
|
|
14443
|
+
let state = this.workflowCache.get(workflowId)?.state;
|
|
14444
|
+
if (!state) {
|
|
14445
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14446
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14447
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
14448
|
+
state = document.state.global;
|
|
14449
|
+
}
|
|
14450
|
+
return effectiveRunPolicy(runnableDefinition(state));
|
|
14451
|
+
} catch {
|
|
14452
|
+
return UNENFORCED_POLICY;
|
|
14453
|
+
}
|
|
14454
|
+
}
|
|
14455
|
+
async resumeStopped(workflowId, triggerKind, current) {
|
|
14456
|
+
if (current.status !== "ENABLED") return {
|
|
14457
|
+
reason: `The workflow is ${current.status}; a run waiting on it does not resume`,
|
|
14458
|
+
errorName: WORKFLOW_DISABLED_ERROR_NAME
|
|
14459
|
+
};
|
|
14460
|
+
const parked = await this.parkedFiring(workflowId, triggerKind, runnableDefinition(current).version);
|
|
14461
|
+
return parked ? {
|
|
14462
|
+
reason: parked,
|
|
14463
|
+
errorName: WORKFLOW_PARKED_ERROR_NAME
|
|
14464
|
+
} : void 0;
|
|
14465
|
+
}
|
|
14466
|
+
async pinnedState(workflowId, runId, store, current) {
|
|
14467
|
+
const pin = await store.getRunPin(runId);
|
|
14468
|
+
if (!pin?.definition_hash) return current;
|
|
14469
|
+
if (this.definitionHash(workflowId, current) === pin.definition_hash) return current;
|
|
14470
|
+
return this.rebuildWorkflowState(workflowId, pin);
|
|
14471
|
+
}
|
|
14472
|
+
async rebuildWorkflowState(workflowId, pin) {
|
|
14473
|
+
const revision = pin.definition_op_index ?? 0;
|
|
14474
|
+
const operations = [];
|
|
14475
|
+
let page = await this.host.reactorClient.getOperations(workflowId, { scopes: ["document", "global"] });
|
|
14476
|
+
operations.push(...page.results);
|
|
14477
|
+
while (page.next) {
|
|
14478
|
+
page = await page.next();
|
|
14479
|
+
operations.push(...page.results);
|
|
14480
|
+
}
|
|
14481
|
+
const applied = (operation) => !operation.error && !isDenied(operation);
|
|
14482
|
+
const past = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index >= revision)).at(0);
|
|
14483
|
+
const atOrBeforePin = (operation) => {
|
|
14484
|
+
const at = operation.action.input.revision?.global;
|
|
14485
|
+
if (at !== void 0) return at <= revision;
|
|
14486
|
+
return !past || operation.timestampUtcMs <= past.timestampUtcMs;
|
|
14487
|
+
};
|
|
14488
|
+
const seed = sortOperations(operations.filter((operation) => operation.action.scope === "document" && operation.action.type === "UPGRADE_DOCUMENT" && applied(operation) && atOrBeforePin(operation))).map((operation) => operation.action.input.initialState).filter((state) => state?.global !== void 0).at(-1);
|
|
14489
|
+
let document = utils.createDocument(seed);
|
|
14490
|
+
const global = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index < revision));
|
|
14491
|
+
for (const operation of garbageCollect(global).filter(applied)) document = reducer(document, operation.action);
|
|
14492
|
+
const state = document.state.global;
|
|
14493
|
+
if (scopeHash(state) !== pin.definition_scope_hash || this.definitionHash(workflowId, state) !== pin.definition_hash) throw new Error(`The workflow was re-published while this run waited, and the definition it started from could not be rebuilt at revision ${revision}`);
|
|
14494
|
+
return state;
|
|
14495
|
+
}
|
|
14496
|
+
/** Resolves a WEBHOOK waitpoint from the editor, as the calling DID. */
|
|
14497
|
+
async resolveWaitpoint(waitpointId, input, ctx) {
|
|
14498
|
+
const store = await this.store();
|
|
14499
|
+
if (!store) throw new Error("Run journal is unavailable");
|
|
14500
|
+
const waitpoint = await store.getWaitpoint(waitpointId);
|
|
14501
|
+
if (!waitpoint) throw new Error(`Waitpoint "${waitpointId}" not found`);
|
|
14502
|
+
await this.assertCanReadDocument(waitpoint.workflow_id, ctx);
|
|
14503
|
+
await this.assertCanWriteDocument(waitpoint.workflow_id, ctx);
|
|
14504
|
+
const owner = await store.getRun(waitpoint.run_id);
|
|
14505
|
+
if (!owner || (await this.servedRuns([owner], ctx)).length === 0) throw new Error(`Waitpoint "${waitpointId}" not found`);
|
|
14506
|
+
if (waitpoint.type !== "WEBHOOK") throw new Error(`Waitpoint "${waitpointId}" is a ${waitpoint.type}`);
|
|
14507
|
+
const decidedBy = this.callerDid(ctx);
|
|
14508
|
+
const approval = parseJson(waitpoint.approval);
|
|
14509
|
+
if (approval && approval.approvers.length > 0) {
|
|
14510
|
+
if (!(decidedBy !== void 0 && approval.approvers.some((approver) => sameApprover(approver, decidedBy)))) throw new Error("You are not one of this approval's approvers");
|
|
14511
|
+
}
|
|
14512
|
+
const reason = input.reason?.trim() || void 0;
|
|
14513
|
+
if ((approval?.reasonRequiredOn === "both" || approval?.reasonRequiredOn === "reject" && !input.approved) && !reason) throw new Error("A reason is required for this decision");
|
|
14514
|
+
const decidedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14515
|
+
const payload = {
|
|
14516
|
+
queryParams: {
|
|
14517
|
+
action: input.approved ? "approve" : "disapprove",
|
|
14518
|
+
...reason ? { reason } : {}
|
|
14519
|
+
},
|
|
14520
|
+
body: {
|
|
14521
|
+
approved: input.approved,
|
|
14522
|
+
reason: reason ?? null,
|
|
14523
|
+
decidedBy: decidedBy ?? null,
|
|
14524
|
+
decidedAt
|
|
14525
|
+
},
|
|
14526
|
+
headers: {}
|
|
14527
|
+
};
|
|
14528
|
+
const completed = await store.completeWaitpoint(waitpointId, {
|
|
14529
|
+
payload,
|
|
14530
|
+
decidedBy: decidedBy ?? null,
|
|
14531
|
+
resolvedAt: decidedAt
|
|
14532
|
+
});
|
|
14533
|
+
if (!completed) {
|
|
14534
|
+
const now = await store.getWaitpoint(waitpointId);
|
|
14535
|
+
const run = await store.getRun(waitpoint.run_id);
|
|
14536
|
+
const live = run?.status === "RUNNING" || run?.status === "WAITING";
|
|
14537
|
+
const status = now?.status !== "PENDING" ? now?.status ?? "gone" : waitpointExpired(now) ? "EXPIRED" : live ? "not the one its run waits on" : `on a ${run?.status ?? "deleted"} run`;
|
|
14538
|
+
throw new Error(`Waitpoint "${waitpointId}" is ${status}; it can no longer be resolved`);
|
|
14539
|
+
}
|
|
14540
|
+
await this.deliverHeld(completed.run_id);
|
|
14541
|
+
return waitpointRecord(completed);
|
|
14542
|
+
}
|
|
14543
|
+
/** A POST to a resume URL. Single-use: only a PENDING waitpoint resumes. */
|
|
14544
|
+
resumeWaitpointByToken(token, payload) {
|
|
14545
|
+
return this.resumeByToken(token, payload);
|
|
14546
|
+
}
|
|
14547
|
+
/** A POST to a resume URL's `/sync`, held like a sync webhook until the
|
|
14548
|
+
* resumed leg responds, waits, ends, or times out. The first answer wins. */
|
|
14549
|
+
async resumeWaitpointByTokenSync(token, payload, options = {}) {
|
|
14550
|
+
let hold;
|
|
14551
|
+
try {
|
|
14552
|
+
const { status } = await this.resumeByToken(token, payload, (id, runId) => {
|
|
14553
|
+
hold = this.holdSyncResume(id, runId);
|
|
14554
|
+
return hold.decided;
|
|
14555
|
+
});
|
|
14556
|
+
if (status !== "resumed" || !hold) return { status };
|
|
14557
|
+
return {
|
|
14558
|
+
status,
|
|
14559
|
+
reply: await hold.reply(options.timeoutMs ?? DELIVERY_TIMEOUT_MS)
|
|
14560
|
+
};
|
|
14561
|
+
} finally {
|
|
14562
|
+
hold?.release();
|
|
14563
|
+
}
|
|
14564
|
+
}
|
|
14565
|
+
holdSyncResume(waitpointId, runId) {
|
|
14566
|
+
let settle = () => void 0;
|
|
14567
|
+
const answer = new Promise((resolve) => {
|
|
14568
|
+
settle = resolve;
|
|
14569
|
+
});
|
|
14570
|
+
let done = false;
|
|
14571
|
+
const finish = (reply) => {
|
|
14572
|
+
if (done) return;
|
|
14573
|
+
done = true;
|
|
14574
|
+
settle(reply);
|
|
14575
|
+
};
|
|
14576
|
+
const entry = {
|
|
14577
|
+
respond: (response) => finish(runResponseReply(response)),
|
|
14578
|
+
ended: (result) => finish(result ? runResultReply(result, 200) : {
|
|
14579
|
+
status: 500,
|
|
14580
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14581
|
+
body: JSON.stringify({ error: "The run could not resume" })
|
|
14582
|
+
}),
|
|
14583
|
+
closed: (runId, status, error, code = 409) => finish({
|
|
14584
|
+
status: code,
|
|
14585
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14586
|
+
body: JSON.stringify({
|
|
14587
|
+
runId,
|
|
14588
|
+
status,
|
|
14589
|
+
error
|
|
14590
|
+
})
|
|
14591
|
+
})
|
|
14592
|
+
};
|
|
14593
|
+
let decided = () => void 0;
|
|
14594
|
+
const candidate = {
|
|
14595
|
+
entry,
|
|
14596
|
+
runId,
|
|
14597
|
+
won: new Promise((resolve) => {
|
|
14598
|
+
decided = resolve;
|
|
14599
|
+
})
|
|
14600
|
+
};
|
|
14601
|
+
const held = this.syncResumes.get(waitpointId) ?? /* @__PURE__ */ new Set();
|
|
14602
|
+
this.syncResumes.set(waitpointId, held);
|
|
14603
|
+
held.add(candidate);
|
|
14604
|
+
return {
|
|
14605
|
+
decided,
|
|
14606
|
+
reply: async (timeoutMs) => {
|
|
14607
|
+
const raced = await Promise.race([answer, timeout(timeoutMs)]);
|
|
14608
|
+
if (raced !== TIMED_OUT) return raced;
|
|
14609
|
+
done = true;
|
|
14610
|
+
return {
|
|
14611
|
+
status: 504,
|
|
14612
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14613
|
+
body: JSON.stringify({
|
|
14614
|
+
status: "RUNNING",
|
|
14615
|
+
error: "The run did not finish in time"
|
|
14616
|
+
})
|
|
14617
|
+
};
|
|
14618
|
+
},
|
|
14619
|
+
release: () => {
|
|
14620
|
+
decided(false);
|
|
14621
|
+
held.delete(candidate);
|
|
14622
|
+
if (held.size === 0 && this.syncResumes.get(waitpointId) === held) this.syncResumes.delete(waitpointId);
|
|
14623
|
+
}
|
|
14624
|
+
};
|
|
14625
|
+
}
|
|
14626
|
+
async resumeByToken(token, payload, beforeComplete) {
|
|
14627
|
+
const store = await this.store();
|
|
14628
|
+
if (!store || !token) return { status: "gone" };
|
|
14629
|
+
const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
|
|
14630
|
+
if (!waitpoint || waitpoint.kind !== "webhook") return { status: "gone" };
|
|
14631
|
+
let completed;
|
|
14632
|
+
if (waitpoint.status === "PENDING") {
|
|
14633
|
+
const decided = beforeComplete?.(waitpoint.id, waitpoint.run_id);
|
|
14634
|
+
try {
|
|
14635
|
+
completed = await store.completeWaitpoint(waitpoint.id, { payload });
|
|
14636
|
+
} finally {
|
|
14637
|
+
decided?.(completed !== void 0);
|
|
14638
|
+
}
|
|
14639
|
+
}
|
|
14640
|
+
if (!completed) {
|
|
14641
|
+
const now = (await store.getWaitpoint(waitpoint.id))?.status;
|
|
14642
|
+
if (now === "CONSUMED") return { status: "already" };
|
|
14643
|
+
const run = now === "COMPLETED" ? await store.getRun(waitpoint.run_id) : void 0;
|
|
14644
|
+
return { status: run?.status === "RUNNING" || run?.status === "WAITING" ? "already" : "gone" };
|
|
14645
|
+
}
|
|
14646
|
+
await this.deliverHeld(completed.run_id);
|
|
14647
|
+
return { status: "resumed" };
|
|
14648
|
+
}
|
|
14649
|
+
/** What a resume URL's confirm page shows. */
|
|
14650
|
+
async describeWaitpointByToken(token) {
|
|
14651
|
+
const store = await this.store();
|
|
14652
|
+
if (!store || !token) return void 0;
|
|
14653
|
+
const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
|
|
14654
|
+
if (!waitpoint || waitpoint.kind !== "webhook") return void 0;
|
|
14655
|
+
const run = await store.getRun(waitpoint.run_id);
|
|
14656
|
+
const status = !(run?.status === "RUNNING" || run?.status === "WAITING") && (waitpoint.status === "PENDING" || waitpoint.status === "COMPLETED") ? "CANCELLED" : waitpoint.status;
|
|
14657
|
+
const state = await this.workflowStateForPage(waitpoint.workflow_id);
|
|
14658
|
+
const step = state?.steps.find((entry) => entry.id === waitpoint.step_id);
|
|
14659
|
+
const action = step ? await this.blockDisplayName({
|
|
14660
|
+
pieceName: step.pieceName,
|
|
14661
|
+
pieceVersion: step.pieceVersion,
|
|
14662
|
+
kind: "action",
|
|
14663
|
+
name: step.actionName
|
|
14664
|
+
}) : void 0;
|
|
14665
|
+
const trigger = state?.trigger ? await this.blockDisplayName({
|
|
14666
|
+
pieceName: state.trigger.pieceName,
|
|
14667
|
+
pieceVersion: state.trigger.pieceVersion,
|
|
14668
|
+
kind: "trigger",
|
|
14669
|
+
name: state.trigger.triggerName
|
|
14670
|
+
}) ?? state.trigger.triggerName : void 0;
|
|
14671
|
+
return {
|
|
14672
|
+
workflowName: run?.workflow_name || state?.name || "",
|
|
14673
|
+
stepName: step?.name || waitpoint.step_key,
|
|
14674
|
+
status,
|
|
14675
|
+
...state?.description ? { workflowDescription: state.description } : {},
|
|
14676
|
+
...action ? { actionName: action } : {},
|
|
14677
|
+
waitingSince: waitpoint.created_at,
|
|
14678
|
+
...waitpoint.expires_at ? { expiresAt: waitpoint.expires_at } : {},
|
|
14679
|
+
...run ? { startedAt: run.enqueued_at } : {},
|
|
14680
|
+
...trigger ? { triggerName: trigger } : {}
|
|
14681
|
+
};
|
|
14682
|
+
}
|
|
14683
|
+
async workflowStateForPage(workflowId) {
|
|
14684
|
+
const cached = this.workflowCache.get(workflowId)?.state;
|
|
14685
|
+
if (cached) return cached;
|
|
14686
|
+
try {
|
|
14687
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14688
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14689
|
+
if (document.header.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
|
|
14690
|
+
this.cacheLoaded(workflowId, generation, document);
|
|
14691
|
+
return document.state.global;
|
|
14692
|
+
} catch {
|
|
14693
|
+
return;
|
|
14694
|
+
}
|
|
14695
|
+
}
|
|
14696
|
+
async blockDisplayName(block) {
|
|
14697
|
+
try {
|
|
14698
|
+
const parsed = resolvedBlock(await this.resolveBlock(block));
|
|
14699
|
+
if (!parsed) return void 0;
|
|
14700
|
+
const descriptor = await this.pieceDescriptor(targetOf(parsed));
|
|
14701
|
+
return (block.kind === "trigger" ? descriptor.triggers : descriptor.actions).find((entry) => entry.name === parsed.name)?.displayName;
|
|
14702
|
+
} catch {
|
|
14703
|
+
return;
|
|
14704
|
+
}
|
|
14705
|
+
}
|
|
14706
|
+
/** The waitpoints of one run, for a caller who may read the run. */
|
|
14707
|
+
async waitpointsForRun(runId, ctx) {
|
|
14708
|
+
const store = await this.store();
|
|
14709
|
+
if (!store || !await this.run(runId, ctx)) return [];
|
|
14710
|
+
return ((await store.listWaitpointsForRuns([runId])).get(runId) ?? []).map(waitpointRecord);
|
|
14711
|
+
}
|
|
14712
|
+
/** WAITING runs this caller may read, oldest first, with their waitpoints. */
|
|
14713
|
+
async waitingRuns(args, ctx) {
|
|
14714
|
+
const store = await this.store();
|
|
14715
|
+
if (!store || !ctx) return [];
|
|
14716
|
+
let scope;
|
|
14717
|
+
if (args.workflowId) {
|
|
14718
|
+
await this.assertCanReadDocument(args.workflowId, ctx);
|
|
14719
|
+
scope = args.workflowId;
|
|
14720
|
+
} else if (args.driveId) {
|
|
14721
|
+
scope = await this.driveWorkflowIds(args.driveId, ctx);
|
|
14722
|
+
if (scope.length === 0) return [];
|
|
14723
|
+
}
|
|
14724
|
+
const rows = await this.servedRuns(await this.readableRows(await store.listWaitingRuns(scope, args.first ?? 50), (row) => row.workflow_id, ctx), ctx);
|
|
14725
|
+
const ids = rows.map((row) => row.id);
|
|
14726
|
+
const steps = await store.getStepsForRuns(ids, { withData: false });
|
|
14727
|
+
const waitpoints = await store.listWaitpointsForRuns(ids);
|
|
14728
|
+
return rows.map((row) => ({
|
|
14729
|
+
run: {
|
|
14730
|
+
row,
|
|
14731
|
+
steps: steps.get(row.id) ?? []
|
|
14732
|
+
},
|
|
14733
|
+
waitpoints: (waitpoints.get(row.id) ?? []).map(waitpointRecord)
|
|
14734
|
+
}));
|
|
14735
|
+
}
|
|
14736
|
+
/** Cancels a WAITING or PENDING run; its waitpoints stop resuming. */
|
|
14737
|
+
async cancelRun(runId, ctx) {
|
|
14738
|
+
const store = await this.store();
|
|
14739
|
+
if (!store) throw new Error("Run journal is unavailable");
|
|
14740
|
+
const run = await store.getRun(runId);
|
|
14741
|
+
if (!run) throw new Error(`Run "${runId}" not found`);
|
|
14742
|
+
await this.assertCanReadDocument(run.workflow_id, ctx);
|
|
14743
|
+
await this.assertCanWriteDocument(run.workflow_id, ctx);
|
|
14744
|
+
if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
|
|
14745
|
+
const refuse = (status) => status === "RUNNING" ? /* @__PURE__ */ new Error("Cancelling a RUNNING run is not supported yet (#3226)") : /* @__PURE__ */ new Error(`Run is ${status}; only WAITING and PENDING runs can be cancelled`);
|
|
14746
|
+
if (run.status !== "WAITING" && run.status !== "PENDING") throw refuse(run.status);
|
|
14747
|
+
const by = this.callerDid(ctx);
|
|
14748
|
+
const reason = by ? `Cancelled by ${by}` : "Cancelled";
|
|
14749
|
+
if (!await store.cancelIdleRun(runId, reason)) throw refuse((await store.getRun(runId))?.status ?? "gone");
|
|
14750
|
+
await this.answerHeldResumes(store, runId, reason);
|
|
14751
|
+
const row = await store.getRun(runId);
|
|
14752
|
+
this.queuedReplies.get(runId)?.({
|
|
14753
|
+
status: row.status,
|
|
14754
|
+
steps: [],
|
|
14755
|
+
runId,
|
|
14756
|
+
...row.error ? { error: row.error } : {},
|
|
14757
|
+
refusal: "cancelled"
|
|
14758
|
+
});
|
|
14759
|
+
return {
|
|
14760
|
+
row,
|
|
14761
|
+
steps: await store.getSteps(runId)
|
|
14762
|
+
};
|
|
14763
|
+
}
|
|
14764
|
+
callerDid(ctx) {
|
|
14765
|
+
if (!ctx) return void 0;
|
|
14766
|
+
const subject = this.host.subjectOf?.(ctx);
|
|
14767
|
+
if (subject?.address) return subject.address;
|
|
14768
|
+
const user = ctx.user;
|
|
14769
|
+
return typeof user?.address === "string" ? user.address : void 0;
|
|
14770
|
+
}
|
|
13377
14771
|
async testStep(workflowId, stepId, ctx, access = {}) {
|
|
13378
14772
|
await this.assertCanReadWorkflow(workflowId, ctx, access.driveId);
|
|
13379
14773
|
const document = await this.host.reactorClient.get(workflowId);
|
|
@@ -13682,6 +15076,112 @@ var WorkflowRuntimeService = class {
|
|
|
13682
15076
|
return (await this.host.reactorClient.get(workflowId)).state.global;
|
|
13683
15077
|
}
|
|
13684
15078
|
};
|
|
15079
|
+
function scopeHash(state) {
|
|
15080
|
+
return hashDocumentStateForScope({ state: { global: state } }, "global");
|
|
15081
|
+
}
|
|
15082
|
+
function tokenHash(token) {
|
|
15083
|
+
return createHash("sha256").update(token).digest("hex");
|
|
15084
|
+
}
|
|
15085
|
+
function waitpointExpired(waitpoint) {
|
|
15086
|
+
return waitpoint.expires_at !== null && Date.parse(waitpoint.expires_at) <= Date.now();
|
|
15087
|
+
}
|
|
15088
|
+
function parseJson(value) {
|
|
15089
|
+
return value === null ? null : JSON.parse(value);
|
|
15090
|
+
}
|
|
15091
|
+
const WORKFLOW_DISABLED_ERROR_NAME = "WorkflowDisabled";
|
|
15092
|
+
const WORKFLOW_PARKED_ERROR_NAME = "WorkflowParked";
|
|
15093
|
+
const MAX_WAITPOINT_ISO = "9999-12-31T23:59:59.999Z";
|
|
15094
|
+
const MAX_WAITPOINT_MS = Date.parse(MAX_WAITPOINT_ISO);
|
|
15095
|
+
function waitpointExpiry(createdAt, request, suspendMs) {
|
|
15096
|
+
const candidates = [request.expiresAt ? Date.parse(request.expiresAt) : NaN, suspendMs === null ? NaN : createdAt.getTime() + suspendMs].filter((at) => !Number.isNaN(at));
|
|
15097
|
+
return candidates.length > 0 ? new Date(Math.min(...candidates, MAX_WAITPOINT_MS)).toISOString() : null;
|
|
15098
|
+
}
|
|
15099
|
+
function sameApprover(approver, caller) {
|
|
15100
|
+
const tail = (value) => value.split(":").pop().toLowerCase();
|
|
15101
|
+
return approver.toLowerCase() === caller.toLowerCase() || tail(approver) === tail(caller);
|
|
15102
|
+
}
|
|
15103
|
+
function waitpointRecord(row) {
|
|
15104
|
+
return {
|
|
15105
|
+
id: row.id,
|
|
15106
|
+
runId: row.run_id,
|
|
15107
|
+
workflowId: row.workflow_id,
|
|
15108
|
+
stepId: row.step_id,
|
|
15109
|
+
stepKey: row.step_key,
|
|
15110
|
+
stepPath: row.step_path,
|
|
15111
|
+
pieceName: row.piece_name,
|
|
15112
|
+
blockName: row.block_name,
|
|
15113
|
+
type: row.type,
|
|
15114
|
+
kind: row.kind,
|
|
15115
|
+
status: row.status,
|
|
15116
|
+
createdAt: row.created_at,
|
|
15117
|
+
resumeAt: row.resume_at,
|
|
15118
|
+
expiresAt: row.expires_at,
|
|
15119
|
+
resolvedAt: row.resolved_at,
|
|
15120
|
+
decidedBy: row.decided_by,
|
|
15121
|
+
approval: parseJson(row.approval)
|
|
15122
|
+
};
|
|
15123
|
+
}
|
|
15124
|
+
function journaledValue(raw, what) {
|
|
15125
|
+
const value = raw === null ? void 0 : JSON.parse(raw);
|
|
15126
|
+
if (isTruncatedStepPayload(value)) return unavailableValue(`${what} is unavailable: the run journal truncated it past the payload cap`);
|
|
15127
|
+
return redactedLeavesUnavailable(value, what, []);
|
|
15128
|
+
}
|
|
15129
|
+
function redactedLeavesUnavailable(value, what, path) {
|
|
15130
|
+
if (!containsRedactedMarker(value)) return value;
|
|
15131
|
+
if (typeof value === "string") {
|
|
15132
|
+
const at = path.length > 0 ? ` at "${path.join(".")}"` : "";
|
|
15133
|
+
return unavailableValue(`${what} is unavailable: the run journal holds it redacted${at}`);
|
|
15134
|
+
}
|
|
15135
|
+
if (Array.isArray(value)) return value.map((entry, index) => redactedLeavesUnavailable(entry, what, [...path, index]));
|
|
15136
|
+
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactedLeavesUnavailable(entry, what, [...path, key])]));
|
|
15137
|
+
}
|
|
15138
|
+
function journaledTriggerPayload(raw) {
|
|
15139
|
+
return journaledValue(raw, "The trigger payload");
|
|
15140
|
+
}
|
|
15141
|
+
const ATTACHMENT_REF_TEXT = /"(attachment:\/\/[^"\\]*)"/g;
|
|
15142
|
+
function carriedFromJournal(rows) {
|
|
15143
|
+
const carried = /* @__PURE__ */ new Map();
|
|
15144
|
+
const attachmentRefs = /* @__PURE__ */ new Set();
|
|
15145
|
+
let waitingStepId;
|
|
15146
|
+
let nextOrdinal = 0;
|
|
15147
|
+
for (const row of rows) {
|
|
15148
|
+
nextOrdinal = Math.max(nextOrdinal, row.ordinal + 1);
|
|
15149
|
+
for (const match of row.output?.matchAll(ATTACHMENT_REF_TEXT) ?? []) attachmentRefs.add(match[1]);
|
|
15150
|
+
const what = `Output of step "${row.step_key}"`;
|
|
15151
|
+
switch (row.status) {
|
|
15152
|
+
case "WAITING":
|
|
15153
|
+
waitingStepId = row.step_id;
|
|
15154
|
+
break;
|
|
15155
|
+
case "SUCCEEDED":
|
|
15156
|
+
case "REPLAYED":
|
|
15157
|
+
carried.set(row.step_id, {
|
|
15158
|
+
status: row.status,
|
|
15159
|
+
port: row.port,
|
|
15160
|
+
output: () => journaledValue(row.output, what)
|
|
15161
|
+
});
|
|
15162
|
+
break;
|
|
15163
|
+
case "FAILED":
|
|
15164
|
+
carried.set(row.step_id, {
|
|
15165
|
+
status: "FAILED",
|
|
15166
|
+
port: "error",
|
|
15167
|
+
output: () => void 0,
|
|
15168
|
+
error: row.error ?? ""
|
|
15169
|
+
});
|
|
15170
|
+
break;
|
|
15171
|
+
case "SKIPPED": if (row.port) carried.set(row.step_id, {
|
|
15172
|
+
status: "SKIPPED",
|
|
15173
|
+
port: row.port,
|
|
15174
|
+
output: () => null
|
|
15175
|
+
});
|
|
15176
|
+
}
|
|
15177
|
+
}
|
|
15178
|
+
return {
|
|
15179
|
+
carried,
|
|
15180
|
+
waitingStepId,
|
|
15181
|
+
nextOrdinal,
|
|
15182
|
+
attachmentRefs
|
|
15183
|
+
};
|
|
15184
|
+
}
|
|
13685
15185
|
/** The runtime a host composes: one instance, its lifetime the host's. */
|
|
13686
15186
|
function createWorkflowRuntime(deps) {
|
|
13687
15187
|
return new WorkflowRuntimeService(deps);
|
|
@@ -13974,6 +15474,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
|
|
|
13974
15474
|
}
|
|
13975
15475
|
};
|
|
13976
15476
|
//#endregion
|
|
13977
|
-
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName };
|
|
15477
|
+
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WAITPOINT_EXPIRED_ERROR_NAME, WAITPOINT_NOT_LIVE_ERROR_NAME, WORKFLOW_DISABLED_ERROR_NAME, WORKFLOW_PACKAGE_NAME, WORKFLOW_PARKED_ERROR_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName, waitpointRecord };
|
|
13978
15478
|
|
|
13979
15479
|
//# sourceMappingURL=index.js.map
|