@powerhousedao/reactor-workflow 6.2.3-dev.84 → 6.2.3-dev.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -37
- package/dist/index.d.ts +218 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1634 -123
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-ByALLpvA.d.ts → piece-registry-C1Pc0hD6.d.ts} +83 -3
- package/dist/piece-registry-C1Pc0hD6.d.ts.map +1 -0
- package/dist/{telemetry-CSemNycC.js → telemetry-GZA1FLlY.js} +674 -28
- package/dist/telemetry-GZA1FLlY.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1 -1
- package/dist/worker-entry.js +161 -16
- package/dist/worker-entry.js.map +1 -1
- package/package.json +7 -7
- package/dist/piece-registry-ByALLpvA.d.ts.map +0 -1
- package/dist/telemetry-CSemNycC.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as containsRedactedMarker, A as AttachmentCache, B as SecretCreatorMismatchError, C as blockLabel, Ct as isIndeterminateError, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as secretRefFromId, H as SecretNotFoundError, I as resolveStepInput, J as PieceWorkerPool, K as stepBlock, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as PieceWorkerTimeoutError, R as unavailableValue, S as storeHandlers, St as setPieceRegistryUrl, T as blockPorts, U as SecretOwnerMismatchError, V as SecretDeletedError, W as parseSecretRef, X as PieceWorkerDisposedError, Y as PieceWorker, Z as PieceWorkerError, _ as effectiveRetryPolicy, _t as unsupportedTrigger, a as PieceRegistry, at as isPrivateAddress, b as isHostBound, bt as npmRegistryBaseUrl, c as packagePieces, ct as FileTooLargeError, d as runWorkflow, dt as pieceModuleRef, et as redact, f as canonicalJson, ft as sourcedResolver, g as checkDynamicProperties, gt as unsupportedAuth, h as DynamicPropertiesError, ht as authMethodFor, i as tracedSteps, it as DEFAULT_EGRESS_POLICY, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as maxFileBytes, m as stepConfigHash, mt as UnsupportedPieceFeatureError, n as WorkflowTelemetry, nt as rememberSecrets, o as installedPiece, ot as parseAddress, p as hashOf, q as triggerBlock, r as tracedMethods, rt as secretsFor, s as installedPieces, st as extractDedupeKey, t as TELEMETRY_SCOPE, tt as redactMessage, u as errorNameOf, ut as bundleResolver, v as ActivepiecesBlockExecutor, w as unpinnedNote, x as resolvedBlock, xt as pieceRegistrySource, y as CompositeBlockExecutor } from "./telemetry-GZA1FLlY.js";
|
|
2
2
|
import { DENIED_ACTION_INDEXES, ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy, assertActionsApplied } from "@powerhousedao/pieces-framework";
|
|
3
3
|
import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { open, readFile, readdir, rm, stat } from "node:fs/promises";
|
|
5
5
|
import { dirname, join } from "node:path";
|
|
6
|
-
import { childLogger, generateId } from "document-model";
|
|
6
|
+
import { childLogger, garbageCollect, generateId, hashDocumentStateForScope, isDenied, sortOperations } from "document-model";
|
|
7
7
|
import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, coerceManualPayload, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, manualInputTypeName, parseManualInputs, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
|
|
8
8
|
import { Cron } from "croner";
|
|
9
9
|
import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
@@ -12,14 +12,14 @@ import net from "node:net";
|
|
|
12
12
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
13
13
|
import { hostname } from "node:os";
|
|
14
14
|
import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
|
|
15
|
+
import { BaseReadModel, JobStatus, PropagationMode, WriteHandleFactory, createEmptyDocument, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker, jobIdsOf, placeholderJob, registerTelemetryGauge, remoteStagePort } from "@powerhousedao/reactor";
|
|
15
16
|
import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
|
|
16
|
-
import http from "node:http";
|
|
17
|
+
import http, { validateHeaderName, validateHeaderValue } from "node:http";
|
|
17
18
|
import https from "node:https";
|
|
18
|
-
import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
|
|
19
|
+
import { actions as actions$1, reducer, utils } from "@powerhousedao/workflow/document-models/workflow";
|
|
19
20
|
import { Kind, parse } from "graphql";
|
|
20
21
|
import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
|
|
21
22
|
import { SpanStatusCode, context } from "@opentelemetry/api";
|
|
22
|
-
import { BaseReadModel, JobStatus, PropagationMode, WriteHandleFactory, createEmptyDocument, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker, jobIdsOf, placeholderJob, remoteStagePort } from "@powerhousedao/reactor";
|
|
23
23
|
import { isFolderNode } from "@powerhousedao/shared/document-drive";
|
|
24
24
|
import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
25
25
|
import { performance as performance$1 } from "node:perf_hooks";
|
|
@@ -439,6 +439,12 @@ function tokenSetFrom(response, context, previous) {
|
|
|
439
439
|
}
|
|
440
440
|
};
|
|
441
441
|
}
|
|
442
|
+
function relayEgress(egress) {
|
|
443
|
+
return {
|
|
444
|
+
...egress,
|
|
445
|
+
allowPrivateAddresses: true
|
|
446
|
+
};
|
|
447
|
+
}
|
|
442
448
|
async function exchangeCode(attempt, code, clientSecret, egress, fromRegistry = false) {
|
|
443
449
|
const method = authorizationMethod(attempt.authorization_method);
|
|
444
450
|
const auth = clientAuth$1(method, attempt.client_id, clientSecret);
|
|
@@ -448,7 +454,7 @@ async function exchangeCode(attempt, code, clientSecret, egress, fromRegistry =
|
|
|
448
454
|
redirect_uri: attempt.redirect_uri,
|
|
449
455
|
...attempt.code_verifier ? { code_verifier: attempt.code_verifier } : {},
|
|
450
456
|
...auth.form
|
|
451
|
-
}, auth.headers, egress), {
|
|
457
|
+
}, auth.headers, clientSecret === null ? relayEgress(egress) : egress), {
|
|
452
458
|
token_url: attempt.token_url,
|
|
453
459
|
authorization_method: method,
|
|
454
460
|
redirect_url: attempt.redirect_uri,
|
|
@@ -523,7 +529,7 @@ var StoreTokenRefresher = class {
|
|
|
523
529
|
grant_type: "refresh_token",
|
|
524
530
|
refresh_token: tokens.refresh_token,
|
|
525
531
|
...auth.form
|
|
526
|
-
}, auth.headers, this.egress);
|
|
532
|
+
}, auth.headers, tokens.relayed ? relayEgress(this.egress) : this.egress);
|
|
527
533
|
} catch (error) {
|
|
528
534
|
const detail = error instanceof Error ? error.message : String(error);
|
|
529
535
|
if (!tokenExpired(tokens, Math.round(Date.now() / 1e3))) {
|
|
@@ -1555,6 +1561,9 @@ function currentStepTest() {
|
|
|
1555
1561
|
function currentRunUser() {
|
|
1556
1562
|
return storage.getStore()?.runUser;
|
|
1557
1563
|
}
|
|
1564
|
+
function currentWaitpoints() {
|
|
1565
|
+
return storage.getStore()?.waitpoints;
|
|
1566
|
+
}
|
|
1558
1567
|
//#endregion
|
|
1559
1568
|
//#region src/reactor/piece-store-port.ts
|
|
1560
1569
|
const logger$5 = childLogger(["workflow", "piece-store"]);
|
|
@@ -1750,6 +1759,7 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
|
|
|
1750
1759
|
egress: configuredEgress(),
|
|
1751
1760
|
worker: currentPieceWorker,
|
|
1752
1761
|
stepTest: currentStepTest,
|
|
1762
|
+
waitpoints: currentWaitpoints,
|
|
1753
1763
|
identity: () => ({
|
|
1754
1764
|
runId: currentRunId(),
|
|
1755
1765
|
flowId: currentWorkflowId(),
|
|
@@ -2691,6 +2701,10 @@ const SCHEDULE_BLOCK = trigger("schedule");
|
|
|
2691
2701
|
const WEBHOOK_BLOCK = trigger("webhook");
|
|
2692
2702
|
const BRANCH_BLOCK = action("branch");
|
|
2693
2703
|
const ASSERT_BLOCK = action("assert");
|
|
2704
|
+
const DELAY_BLOCK = action("delay");
|
|
2705
|
+
const APPROVAL_BLOCK = action("approval");
|
|
2706
|
+
const STOP_BLOCK = action("stop");
|
|
2707
|
+
const WAIT_FOR_WEBHOOK_BLOCK = action("waitForWebhook");
|
|
2694
2708
|
function isCoreBlock(block) {
|
|
2695
2709
|
return block.pieceName === CORE_PIECE_NAME;
|
|
2696
2710
|
}
|
|
@@ -7680,6 +7694,16 @@ const PARKED_TRIGGER_STATUS = "PARKED";
|
|
|
7680
7694
|
* not finish and nothing is wrong with the workflow", which is what the
|
|
7681
7695
|
* document model's RunStatus.CANCELLED means. */
|
|
7682
7696
|
const CANCELLED_RUN_STATUS = "CANCELLED";
|
|
7697
|
+
/** How long a run may stay suspended; null when the policy sets no bound. */
|
|
7698
|
+
function maxSuspensionMs(runnable) {
|
|
7699
|
+
const policy = runnable.policy;
|
|
7700
|
+
const days = positive(policy?.maxSuspensionDays);
|
|
7701
|
+
return days === null ? null : days * 864e5;
|
|
7702
|
+
}
|
|
7703
|
+
//#endregion
|
|
7704
|
+
//#region src/reactor/waitpoint-types.ts
|
|
7705
|
+
const WAITPOINT_EXPIRED_ERROR_NAME = "WaitpointExpired";
|
|
7706
|
+
const WAITPOINT_NOT_LIVE_ERROR_NAME = "WaitpointNotLive";
|
|
7683
7707
|
//#endregion
|
|
7684
7708
|
//#region src/reactor/store.ts
|
|
7685
7709
|
function triggerRowBlock(row) {
|
|
@@ -7711,6 +7735,7 @@ function workflowParkRow(workflowId, publishedVersion, reason) {
|
|
|
7711
7735
|
parked_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7712
7736
|
};
|
|
7713
7737
|
}
|
|
7738
|
+
const WAITING_RUN_STATUS = "WAITING";
|
|
7714
7739
|
const logger$4 = childLogger([
|
|
7715
7740
|
"workflow",
|
|
7716
7741
|
"runtime",
|
|
@@ -7718,6 +7743,64 @@ const logger$4 = childLogger([
|
|
|
7718
7743
|
]);
|
|
7719
7744
|
const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
|
|
7720
7745
|
const PENDING_RUN_STATUS = "PENDING";
|
|
7746
|
+
const LIVE_RUN_STATUSES = [
|
|
7747
|
+
PENDING_RUN_STATUS,
|
|
7748
|
+
"RUNNING",
|
|
7749
|
+
WAITING_RUN_STATUS
|
|
7750
|
+
];
|
|
7751
|
+
const LIVE_WAITPOINT_STATUSES = ["PENDING", "COMPLETED"];
|
|
7752
|
+
async function closeLiveWaitpoints(db, runIds, status, resolvedAt, keep) {
|
|
7753
|
+
if (runIds.length === 0) return [];
|
|
7754
|
+
return (await db.updateTable("waitpoint").set({
|
|
7755
|
+
status,
|
|
7756
|
+
resolved_at: resolvedAt
|
|
7757
|
+
}).where("run_id", "in", runIds).where("status", "in", LIVE_WAITPOINT_STATUSES).$if(keep !== void 0, (qb) => qb.where((eb) => eb.not(keep(eb)))).returning("id").execute()).map((row) => row.id);
|
|
7758
|
+
}
|
|
7759
|
+
async function failWaitingSteps(db, runIds, reason, errorName, endedAt) {
|
|
7760
|
+
if (runIds.length === 0) return;
|
|
7761
|
+
await db.updateTable("step_execution").set({
|
|
7762
|
+
status: "FAILED",
|
|
7763
|
+
error: redactMessage(reason),
|
|
7764
|
+
error_name: errorName,
|
|
7765
|
+
ended_at: endedAt
|
|
7766
|
+
}).where("run_id", "in", runIds).where("status", "=", WAITING_RUN_STATUS).execute();
|
|
7767
|
+
}
|
|
7768
|
+
function waitpointExpiryReason(waitpoint) {
|
|
7769
|
+
return `The ${waitpoint.kind === "approval" ? "approval" : "wait for a resume"} at step "${waitpoint.step_key}" expired at ${waitpoint.expires_at} without being resolved`;
|
|
7770
|
+
}
|
|
7771
|
+
function strandedWait(eb) {
|
|
7772
|
+
const live = eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES);
|
|
7773
|
+
return eb.and([eb.not(eb.exists(completedOf(eb))), eb.or([eb.and([eb("run.waiting_on", "is not", null), eb.not(eb.exists(live.whereRef("waitpoint.id", "=", "run.waiting_on")))]), eb.and([eb("run.waiting_on", "is", null), eb.not(eb.exists(live.whereRef("waitpoint.run_id", "=", "run.id")))])])]);
|
|
7774
|
+
}
|
|
7775
|
+
function completedOf(eb) {
|
|
7776
|
+
return eb.selectFrom("waitpoint").select("waitpoint.id").whereRef("waitpoint.run_id", "=", "run.id").where("waitpoint.status", "=", "COMPLETED");
|
|
7777
|
+
}
|
|
7778
|
+
async function endWaitingRun(db, runId, end) {
|
|
7779
|
+
const { waitpointId } = end;
|
|
7780
|
+
if ((await db.updateTable("run").set({
|
|
7781
|
+
status: "CANCELLED",
|
|
7782
|
+
error: redactMessage(end.reason),
|
|
7783
|
+
error_name: end.errorName,
|
|
7784
|
+
ended_at: end.endedAt
|
|
7785
|
+
}).where("id", "=", runId).where("status", "=", "WAITING").where((eb) => waitpointId === void 0 ? strandedWait(eb) : eb.and([eb.or([eb("run.waiting_on", "=", waitpointId), eb("run.waiting_on", "is", null)]), eb.not(eb.exists(completedOf(eb)))])).returning("id").execute()).length === 0) return void 0;
|
|
7786
|
+
await failWaitingSteps(db, [runId], end.reason, end.errorName, end.endedAt);
|
|
7787
|
+
const closed = await closeLiveWaitpoints(db, [runId], "EXPIRED", end.endedAt);
|
|
7788
|
+
return {
|
|
7789
|
+
runId,
|
|
7790
|
+
reason: end.reason,
|
|
7791
|
+
closed
|
|
7792
|
+
};
|
|
7793
|
+
}
|
|
7794
|
+
function priorNotes(text) {
|
|
7795
|
+
if (!text) return [];
|
|
7796
|
+
try {
|
|
7797
|
+
const notes = JSON.parse(text);
|
|
7798
|
+
return Array.isArray(notes) ? notes.filter((note) => typeof note === "string") : [];
|
|
7799
|
+
} catch {
|
|
7800
|
+
return [];
|
|
7801
|
+
}
|
|
7802
|
+
}
|
|
7803
|
+
const STRANDED_WAIT_REASON = "The run waited on a waitpoint that is no longer live";
|
|
7721
7804
|
function errorMessage(error) {
|
|
7722
7805
|
return error instanceof Error ? error.message : String(error);
|
|
7723
7806
|
}
|
|
@@ -7819,6 +7902,31 @@ async function up(db) {
|
|
|
7819
7902
|
try {
|
|
7820
7903
|
await db.schema.dropTable("webhook_endpoint").ifExists().execute();
|
|
7821
7904
|
} catch {}
|
|
7905
|
+
for (const [column, type] of [
|
|
7906
|
+
["definition_hash", "text"],
|
|
7907
|
+
["definition_op_index", "integer"],
|
|
7908
|
+
["definition_scope_hash", "text"],
|
|
7909
|
+
["waiting_on", "text"]
|
|
7910
|
+
]) try {
|
|
7911
|
+
await db.schema.alterTable("run").addColumn(column, type).execute();
|
|
7912
|
+
} catch {}
|
|
7913
|
+
await db.schema.createTable("waitpoint").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("step_path", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("type", "text", (col) => col.notNull()).addColumn("kind", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("token_hash", "text").addColumn("created_at", "text", (col) => col.notNull()).addColumn("resume_at", "text").addColumn("expires_at", "text").addColumn("resolved_at", "text").addColumn("decided_by", "text").addColumn("approval", "text").addColumn("resume_payload", "text").addColumn("response_to_send", "text").ifNotExists().execute();
|
|
7914
|
+
await db.schema.createIndex("waitpoint_token").ifNotExists().unique().on("waitpoint").column("token_hash").execute();
|
|
7915
|
+
await db.schema.createIndex("waitpoint_status_resume").ifNotExists().on("waitpoint").columns(["status", "resume_at"]).execute();
|
|
7916
|
+
await db.schema.createIndex("waitpoint_status_expires").ifNotExists().on("waitpoint").columns(["status", "expires_at"]).execute();
|
|
7917
|
+
await db.schema.createIndex("waitpoint_run").ifNotExists().on("waitpoint").column("run_id").execute();
|
|
7918
|
+
const liveStep = db.schema.createIndex("waitpoint_live_step").ifNotExists().unique().on("waitpoint").columns(["run_id", "step_path"]).where(sql.ref("status"), "in", LIVE_WAITPOINT_STATUSES);
|
|
7919
|
+
try {
|
|
7920
|
+
await liveStep.execute();
|
|
7921
|
+
} catch {
|
|
7922
|
+
await db.updateTable("waitpoint").set({
|
|
7923
|
+
status: "CANCELLED",
|
|
7924
|
+
resolved_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7925
|
+
}).where("status", "in", LIVE_WAITPOINT_STATUSES).where((eb) => eb.and([eb.not(eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.waiting_on", "=", "waitpoint.id"))), eb.exists(eb.selectFrom("waitpoint as sibling").select("sibling.id").whereRef("sibling.run_id", "=", "waitpoint.run_id").whereRef("sibling.step_path", "=", "waitpoint.step_path").whereRef("sibling.id", "!=", "waitpoint.id").where("sibling.status", "in", LIVE_WAITPOINT_STATUSES))])).execute();
|
|
7926
|
+
await liveStep.execute();
|
|
7927
|
+
}
|
|
7928
|
+
await db.schema.createIndex("waitpoint_workflow_status").ifNotExists().on("waitpoint").columns(["workflow_id", "status"]).execute();
|
|
7929
|
+
await db.schema.createIndex("run_waiting").ifNotExists().on("run").columns(["workflow_id", "enqueued_at"]).where(sql.ref("status"), "=", WAITING_RUN_STATUS).execute();
|
|
7822
7930
|
await migrateBlockType(db, "trigger_state", "trigger_name");
|
|
7823
7931
|
await migrateBlockType(db, "step_execution", "block_name");
|
|
7824
7932
|
return migrateTriggerStoreState(db);
|
|
@@ -8177,6 +8285,8 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8177
8285
|
unmigrated;
|
|
8178
8286
|
runsInFlight = /* @__PURE__ */ new Set();
|
|
8179
8287
|
journaledSteps = /* @__PURE__ */ new Map();
|
|
8288
|
+
pausingRuns = /* @__PURE__ */ new Set();
|
|
8289
|
+
resumedLegs = /* @__PURE__ */ new Set();
|
|
8180
8290
|
constructor(db, unmigrated) {
|
|
8181
8291
|
this.db = db;
|
|
8182
8292
|
this.unmigrated = unmigrated;
|
|
@@ -8195,13 +8305,20 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8195
8305
|
this.unmigrated.delete(workflowId);
|
|
8196
8306
|
}
|
|
8197
8307
|
async recoverOrphanedRuns() {
|
|
8198
|
-
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
|
|
8203
|
-
|
|
8204
|
-
|
|
8308
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8309
|
+
const inFlight = [...this.runsInFlight];
|
|
8310
|
+
const recovered = await this.db.transaction().execute(async (trx) => {
|
|
8311
|
+
let query = trx.updateTable("run").set({
|
|
8312
|
+
status: "FAILED",
|
|
8313
|
+
error: ORPHANED_RUN_ERROR,
|
|
8314
|
+
ended_at: endedAt
|
|
8315
|
+
}).where("status", "=", "RUNNING");
|
|
8316
|
+
if (inFlight.length > 0) query = query.where("id", "not in", inFlight);
|
|
8317
|
+
const ids = (await query.returning("id").execute()).map((row) => row.id);
|
|
8318
|
+
await failWaitingSteps(trx, ids, ORPHANED_RUN_ERROR, null, endedAt);
|
|
8319
|
+
await closeLiveWaitpoints(trx, ids, "CANCELLED", endedAt);
|
|
8320
|
+
return ids.length;
|
|
8321
|
+
});
|
|
8205
8322
|
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
|
|
8206
8323
|
return recovered;
|
|
8207
8324
|
}
|
|
@@ -8265,14 +8382,25 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8265
8382
|
* CANCELLED run is not rerunnable and must not read as a workflow defect.
|
|
8266
8383
|
*/
|
|
8267
8384
|
async cancelRun(runId, reason) {
|
|
8385
|
+
return this.endLiveRun(runId, "CANCELLED", reason, null);
|
|
8386
|
+
}
|
|
8387
|
+
async endLiveRun(runId, status, reason, errorName) {
|
|
8268
8388
|
this.runsInFlight.delete(runId);
|
|
8269
8389
|
this.journaledSteps.delete(runId);
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
|
|
8273
|
-
|
|
8274
|
-
|
|
8275
|
-
|
|
8390
|
+
this.pausingRuns.delete(runId);
|
|
8391
|
+
this.resumedLegs.delete(runId);
|
|
8392
|
+
if (erasedRuns.delete(runId)) return [];
|
|
8393
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8394
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8395
|
+
if ((await trx.updateTable("run").set({
|
|
8396
|
+
status,
|
|
8397
|
+
error: redactMessage(reason),
|
|
8398
|
+
error_name: errorName,
|
|
8399
|
+
ended_at: endedAt
|
|
8400
|
+
}).where("id", "=", runId).where("status", "in", LIVE_RUN_STATUSES).returning("id").execute()).length === 0) return [];
|
|
8401
|
+
await failWaitingSteps(trx, [runId], reason, errorName, endedAt);
|
|
8402
|
+
return closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
|
|
8403
|
+
});
|
|
8276
8404
|
}
|
|
8277
8405
|
async enqueueRun(options) {
|
|
8278
8406
|
const id = randomUUID();
|
|
@@ -8480,12 +8608,15 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8480
8608
|
async beginRun(runId, details) {
|
|
8481
8609
|
this.runsInFlight.add(runId);
|
|
8482
8610
|
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
8483
|
-
await this.db.updateTable("run").set({
|
|
8611
|
+
if ((await this.db.updateTable("run").set({
|
|
8484
8612
|
status: "RUNNING",
|
|
8485
8613
|
workflow_name: details.workflowName,
|
|
8486
8614
|
workflow_version: details.workflowVersion,
|
|
8487
8615
|
started_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8488
|
-
}).where("id", "=", runId).execute();
|
|
8616
|
+
}).where("id", "=", runId).where("status", "=", "PENDING").returning("id").execute()).length > 0) return true;
|
|
8617
|
+
this.runsInFlight.delete(runId);
|
|
8618
|
+
this.journaledSteps.delete(runId);
|
|
8619
|
+
return false;
|
|
8489
8620
|
}
|
|
8490
8621
|
async startRun(options) {
|
|
8491
8622
|
const id = randomUUID();
|
|
@@ -8514,10 +8645,10 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8514
8645
|
if (erasedRuns.has(runId)) return;
|
|
8515
8646
|
const values = stepValues(runId, ordinal, step);
|
|
8516
8647
|
const { run_id: _run, step_id: _step, ...mutable } = values;
|
|
8517
|
-
await this.
|
|
8648
|
+
if (!await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values({
|
|
8518
8649
|
id: randomUUID(),
|
|
8519
8650
|
...values
|
|
8520
|
-
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
|
|
8651
|
+
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute())) return;
|
|
8521
8652
|
this.journaledSteps.get(runId)?.set(step.stepId, {
|
|
8522
8653
|
ordinal,
|
|
8523
8654
|
hash: rowHash(values)
|
|
@@ -8532,25 +8663,61 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8532
8663
|
return row.hash === rowHash(stepValues(runId, row.ordinal, step));
|
|
8533
8664
|
});
|
|
8534
8665
|
}
|
|
8535
|
-
async finishRun(runId, result, executionOrder) {
|
|
8666
|
+
async finishRun(runId, result, executionOrder, pin) {
|
|
8536
8667
|
this.runsInFlight.delete(runId);
|
|
8537
|
-
const
|
|
8668
|
+
const resumed = this.resumedLegs.delete(runId);
|
|
8669
|
+
const paused = this.pausingRuns.delete(runId) || resumed;
|
|
8670
|
+
const own = result.steps.some((step) => step.carried) ? {
|
|
8671
|
+
...result,
|
|
8672
|
+
steps: result.steps.filter((step) => !step.carried)
|
|
8673
|
+
} : result;
|
|
8674
|
+
const sweep = !this.sweepIsNoop(runId, own);
|
|
8538
8675
|
this.journaledSteps.delete(runId);
|
|
8539
|
-
if (erasedRuns.delete(runId)) return
|
|
8540
|
-
|
|
8541
|
-
|
|
8676
|
+
if (erasedRuns.delete(runId)) return {
|
|
8677
|
+
held: true,
|
|
8678
|
+
closed: []
|
|
8679
|
+
};
|
|
8680
|
+
if (sweep && own.steps.length > 0) try {
|
|
8681
|
+
await this.sweepSteps(runId, own, executionOrder);
|
|
8542
8682
|
} catch (error) {
|
|
8543
8683
|
logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
|
|
8544
8684
|
}
|
|
8545
|
-
const
|
|
8546
|
-
|
|
8547
|
-
|
|
8548
|
-
|
|
8549
|
-
|
|
8550
|
-
|
|
8551
|
-
|
|
8552
|
-
|
|
8553
|
-
|
|
8685
|
+
const waiting = result.status === WAITING_RUN_STATUS;
|
|
8686
|
+
const on = result.waiting;
|
|
8687
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8688
|
+
const close = async (trx) => {
|
|
8689
|
+
const prior = resumed ? await trx.selectFrom("run").select("warning_notes").where("id", "=", runId).executeTakeFirst() : void 0;
|
|
8690
|
+
const notes = [.../* @__PURE__ */ new Set([...priorNotes(prior?.warning_notes ?? null), ...runWarningNotes(own)])];
|
|
8691
|
+
const written = await trx.updateTable("run").set({
|
|
8692
|
+
status: result.status,
|
|
8693
|
+
error: result.error ? redactMessage(result.error) : null,
|
|
8694
|
+
error_name: result.errorName ?? null,
|
|
8695
|
+
ended_at: waiting ? null : now,
|
|
8696
|
+
warnings: notes.length,
|
|
8697
|
+
warning_notes: notes.length > 0 ? JSON.stringify(notes) : null,
|
|
8698
|
+
waiting_on: waiting ? on?.waitpointId ?? null : null,
|
|
8699
|
+
...pin ?? {}
|
|
8700
|
+
}).where("id", "=", runId).$if(waiting, (qb) => qb.where((eb) => eb.exists(eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.id", "=", on?.waitpointId ?? "").where("waitpoint.run_id", "=", runId).where("waitpoint.step_id", "=", on?.stepId ?? "").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES)))).returning("id").execute();
|
|
8701
|
+
if (!waiting) return {
|
|
8702
|
+
held: true,
|
|
8703
|
+
closed: paused ? await closeLiveWaitpoints(trx, [runId], "EXPIRED", now) : []
|
|
8704
|
+
};
|
|
8705
|
+
if (written.length === 0) return {
|
|
8706
|
+
held: false,
|
|
8707
|
+
closed: []
|
|
8708
|
+
};
|
|
8709
|
+
const linkStatuses = (await trx.selectFrom("waitpoint").select("kind").where("id", "=", on.waitpointId).executeTakeFirst())?.kind === "webhook" ? LIVE_WAITPOINT_STATUSES : ["PENDING"];
|
|
8710
|
+
return {
|
|
8711
|
+
held: true,
|
|
8712
|
+
closed: await closeLiveWaitpoints(trx, [runId], "CANCELLED", now, (eb) => eb.or([eb("waitpoint.id", "=", on.waitpointId), eb.and([
|
|
8713
|
+
eb("waitpoint.type", "=", "WEBHOOK"),
|
|
8714
|
+
eb("waitpoint.kind", "=", "webhook"),
|
|
8715
|
+
eb("waitpoint.status", "in", linkStatuses),
|
|
8716
|
+
eb.exists(eb.selectFrom("step_execution").select("step_execution.id").whereRef("step_execution.run_id", "=", "waitpoint.run_id").whereRef("step_execution.step_id", "=", "waitpoint.step_id").where("step_execution.status", "=", "SUCCEEDED"))
|
|
8717
|
+
])]))
|
|
8718
|
+
};
|
|
8719
|
+
};
|
|
8720
|
+
return waiting || paused ? this.db.transaction().execute(close) : close(this.db);
|
|
8554
8721
|
}
|
|
8555
8722
|
async sweepSteps(runId, result, executionOrder) {
|
|
8556
8723
|
const journaled = await this.db.selectFrom("step_execution").select(["step_id", "ordinal"]).where("run_id", "=", runId).execute();
|
|
@@ -8563,7 +8730,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8563
8730
|
nextOrdinal = Math.max(nextOrdinal, ran + 1);
|
|
8564
8731
|
}
|
|
8565
8732
|
const ordinalFor = (step) => ordinals.get(step.stepId) ?? nextOrdinal++;
|
|
8566
|
-
await this.
|
|
8733
|
+
await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values(result.steps.map((step) => ({
|
|
8567
8734
|
id: randomUUID(),
|
|
8568
8735
|
...stepValues(runId, ordinalFor(step), step)
|
|
8569
8736
|
}))).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet((eb) => ({
|
|
@@ -8584,18 +8751,15 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8584
8751
|
version_match: eb.ref("excluded.version_match"),
|
|
8585
8752
|
version_note: eb.ref("excluded.version_note"),
|
|
8586
8753
|
config_hash: eb.ref("excluded.config_hash")
|
|
8587
|
-
}))).execute();
|
|
8754
|
+
}))).execute());
|
|
8755
|
+
}
|
|
8756
|
+
async withRunRow(runId, fn) {
|
|
8757
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8758
|
+
return await trx.selectFrom("run").select("id").where("id", "=", runId).forKeyShare().executeTakeFirst() ? fn(trx) : void 0;
|
|
8759
|
+
});
|
|
8588
8760
|
}
|
|
8589
8761
|
async failRun(runId, error, errorName) {
|
|
8590
|
-
this.
|
|
8591
|
-
this.journaledSteps.delete(runId);
|
|
8592
|
-
if (erasedRuns.delete(runId)) return;
|
|
8593
|
-
await this.db.updateTable("run").set({
|
|
8594
|
-
status: "FAILED",
|
|
8595
|
-
error: redactMessage(error),
|
|
8596
|
-
error_name: errorName ?? null,
|
|
8597
|
-
ended_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8598
|
-
}).where("id", "=", runId).execute();
|
|
8762
|
+
return this.endLiveRun(runId, "FAILED", error, errorName ?? null);
|
|
8599
8763
|
}
|
|
8600
8764
|
async listRuns(workflowId, limit = 25, options = {}) {
|
|
8601
8765
|
if (Array.isArray(workflowId) && workflowId.length === 0) return [];
|
|
@@ -8627,10 +8791,10 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8627
8791
|
}
|
|
8628
8792
|
async recordRunDocuments(runId, documentIds) {
|
|
8629
8793
|
if (documentIds.length === 0 || erasedRuns.has(runId)) return;
|
|
8630
|
-
await this.
|
|
8794
|
+
await this.withRunRow(runId, (trx) => trx.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
|
|
8631
8795
|
run_id: runId,
|
|
8632
8796
|
document_id: documentId
|
|
8633
|
-
}))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute();
|
|
8797
|
+
}))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute());
|
|
8634
8798
|
}
|
|
8635
8799
|
async getRunDocuments(runId) {
|
|
8636
8800
|
return (await this.db.selectFrom("run_document").select("document_id").where("run_id", "=", runId).execute()).map((row) => row.document_id);
|
|
@@ -8775,6 +8939,181 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8775
8939
|
async deleteDedupe(workflowId) {
|
|
8776
8940
|
await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
|
|
8777
8941
|
}
|
|
8942
|
+
async createWaitpoint(row) {
|
|
8943
|
+
if (this.runsInFlight.has(row.run_id)) this.pausingRuns.add(row.run_id);
|
|
8944
|
+
for (let attempt = 1;; attempt++) {
|
|
8945
|
+
const inserted = await this.withRunRow(row.run_id, (trx) => trx.insertInto("waitpoint").values(row).onConflict((oc) => oc.doNothing()).returningAll().executeTakeFirst().then((waitpoint) => ({ waitpoint })));
|
|
8946
|
+
if (!inserted) throw new Error(`Run "${row.run_id}" no longer exists`);
|
|
8947
|
+
if (inserted.waitpoint) return {
|
|
8948
|
+
waitpoint: inserted.waitpoint,
|
|
8949
|
+
created: true
|
|
8950
|
+
};
|
|
8951
|
+
const live = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "=", row.run_id).where("step_path", "=", row.step_path).where("status", "in", LIVE_WAITPOINT_STATUSES).executeTakeFirst();
|
|
8952
|
+
if (live) return {
|
|
8953
|
+
waitpoint: live,
|
|
8954
|
+
created: false
|
|
8955
|
+
};
|
|
8956
|
+
if (attempt >= 3) throw new Error(`Could not create a waitpoint for step "${row.step_key}"`);
|
|
8957
|
+
}
|
|
8958
|
+
}
|
|
8959
|
+
async getWaitpoint(id) {
|
|
8960
|
+
return this.db.selectFrom("waitpoint").selectAll().where("id", "=", id).executeTakeFirst();
|
|
8961
|
+
}
|
|
8962
|
+
async getWaitpointByTokenHash(tokenHash) {
|
|
8963
|
+
return this.db.selectFrom("waitpoint").selectAll().where("token_hash", "=", tokenHash).executeTakeFirst();
|
|
8964
|
+
}
|
|
8965
|
+
async completeWaitpoint(id, resolution) {
|
|
8966
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8967
|
+
return (await this.db.updateTable("waitpoint").set({
|
|
8968
|
+
status: "COMPLETED",
|
|
8969
|
+
resume_payload: JSON.stringify(resolution.payload ?? {}),
|
|
8970
|
+
resolved_at: resolution.resolvedAt ?? now,
|
|
8971
|
+
decided_by: resolution.decidedBy ?? null
|
|
8972
|
+
}).where("id", "=", id).where("status", "=", "PENDING").where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", now)])).where((eb) => eb.not(eb.exists(eb.selectFrom("run").innerJoin("waitpoint as awaited", "awaited.id", "run.waiting_on").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").whereRef("awaited.id", "!=", "waitpoint.id").where("awaited.kind", "!=", "webhook").where("awaited.status", "in", LIVE_WAITPOINT_STATUSES)))).where((eb) => eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").where("run.status", "in", ["RUNNING", WAITING_RUN_STATUS]))).returningAll().execute())[0];
|
|
8973
|
+
}
|
|
8974
|
+
async claimDueDelayWaitpoints(nowIso, limit) {
|
|
8975
|
+
return this.db.updateTable("waitpoint").set({
|
|
8976
|
+
status: "COMPLETED",
|
|
8977
|
+
resolved_at: nowIso,
|
|
8978
|
+
resume_payload: "{}"
|
|
8979
|
+
}).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "<=", nowIso).orderBy("resume_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
|
|
8980
|
+
}
|
|
8981
|
+
async expireDueWaitpoints(nowIso, limit) {
|
|
8982
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8983
|
+
const expired = await trx.updateTable("waitpoint").set({
|
|
8984
|
+
status: "EXPIRED",
|
|
8985
|
+
resolved_at: nowIso
|
|
8986
|
+
}).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("expires_at", "<=", nowIso).orderBy("expires_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
|
|
8987
|
+
const ended = [];
|
|
8988
|
+
for (const waitpoint of expired) {
|
|
8989
|
+
const end = await endWaitingRun(trx, waitpoint.run_id, {
|
|
8990
|
+
reason: waitpointExpiryReason(waitpoint),
|
|
8991
|
+
errorName: WAITPOINT_EXPIRED_ERROR_NAME,
|
|
8992
|
+
endedAt: nowIso,
|
|
8993
|
+
waitpointId: waitpoint.id
|
|
8994
|
+
});
|
|
8995
|
+
if (end) ended.push(end);
|
|
8996
|
+
}
|
|
8997
|
+
return {
|
|
8998
|
+
expired,
|
|
8999
|
+
ended
|
|
9000
|
+
};
|
|
9001
|
+
});
|
|
9002
|
+
}
|
|
9003
|
+
async endStrandedWaits(limit) {
|
|
9004
|
+
const runs = await this.db.selectFrom("run").select("id").where("status", "=", WAITING_RUN_STATUS).where((eb) => strandedWait(eb)).limit(limit).execute();
|
|
9005
|
+
const ended = [];
|
|
9006
|
+
for (const run of runs) {
|
|
9007
|
+
const end = await this.db.transaction().execute((trx) => endWaitingRun(trx, run.id, {
|
|
9008
|
+
reason: STRANDED_WAIT_REASON,
|
|
9009
|
+
errorName: WAITPOINT_NOT_LIVE_ERROR_NAME,
|
|
9010
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9011
|
+
}));
|
|
9012
|
+
if (end) ended.push(end);
|
|
9013
|
+
}
|
|
9014
|
+
return ended;
|
|
9015
|
+
}
|
|
9016
|
+
async nextExpiry() {
|
|
9017
|
+
return (await this.db.selectFrom("waitpoint").select("expires_at").where("status", "=", "PENDING").where("expires_at", "is not", null).orderBy("expires_at").limit(1).executeTakeFirst())?.expires_at ?? void 0;
|
|
9018
|
+
}
|
|
9019
|
+
async countWaitingRuns() {
|
|
9020
|
+
const row = await this.db.selectFrom("run").select((eb) => eb.fn.countAll().as("count")).where("status", "=", WAITING_RUN_STATUS).executeTakeFirst();
|
|
9021
|
+
return Number(row?.count ?? 0);
|
|
9022
|
+
}
|
|
9023
|
+
async nextDelayDue() {
|
|
9024
|
+
return (await this.db.selectFrom("waitpoint").select("resume_at").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "is not", null).orderBy("resume_at").limit(1).executeTakeFirst())?.resume_at ?? void 0;
|
|
9025
|
+
}
|
|
9026
|
+
async hasCompletedWaitpoint(runId) {
|
|
9027
|
+
return await this.db.selectFrom("waitpoint").select("id").where("run_id", "=", runId).where("status", "=", "COMPLETED").limit(1).executeTakeFirst() !== void 0;
|
|
9028
|
+
}
|
|
9029
|
+
async heldRunIds() {
|
|
9030
|
+
return (await this.db.selectFrom("waitpoint").innerJoin("run", "run.id", "waitpoint.run_id").select("waitpoint.run_id").distinct().where("waitpoint.status", "=", "COMPLETED").where("run.status", "=", WAITING_RUN_STATUS).execute()).map((row) => row.run_id);
|
|
9031
|
+
}
|
|
9032
|
+
isRunInFlight(runId) {
|
|
9033
|
+
return this.runsInFlight.has(runId);
|
|
9034
|
+
}
|
|
9035
|
+
async claimWaitingRun(runId) {
|
|
9036
|
+
if ((await this.db.updateTable("run").set({ status: "RUNNING" }).where("id", "=", runId).where("status", "=", "WAITING").returning("id").execute()).length === 0) return false;
|
|
9037
|
+
this.runsInFlight.add(runId);
|
|
9038
|
+
this.resumedLegs.add(runId);
|
|
9039
|
+
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
9040
|
+
return true;
|
|
9041
|
+
}
|
|
9042
|
+
async returnToWaiting(runId) {
|
|
9043
|
+
this.runsInFlight.delete(runId);
|
|
9044
|
+
this.resumedLegs.delete(runId);
|
|
9045
|
+
this.journaledSteps.delete(runId);
|
|
9046
|
+
await this.db.updateTable("run").set({ status: WAITING_RUN_STATUS }).where("id", "=", runId).where("status", "=", "RUNNING").execute();
|
|
9047
|
+
}
|
|
9048
|
+
async consumeCompletedWaitpoint(runId) {
|
|
9049
|
+
return this.db.transaction().execute(async (trx) => {
|
|
9050
|
+
const waitingOn = (await trx.selectFrom("run").select("waiting_on").where("id", "=", runId).executeTakeFirst())?.waiting_on ?? null;
|
|
9051
|
+
const candidates = await trx.selectFrom("waitpoint").selectAll().where("run_id", "=", runId).where("status", "=", "COMPLETED").orderBy("resolved_at").execute();
|
|
9052
|
+
const awaited = waitingOn ? await trx.selectFrom("waitpoint").select("kind").where("id", "=", waitingOn).executeTakeFirst() : void 0;
|
|
9053
|
+
const linksResume = !awaited || awaited.kind === "webhook";
|
|
9054
|
+
const chosen = candidates.find((row) => row.id === waitingOn) ?? (linksResume ? candidates[0] : void 0);
|
|
9055
|
+
if (!chosen) return void 0;
|
|
9056
|
+
const waitpoint = (await trx.updateTable("waitpoint").set({ status: "CONSUMED" }).where("id", "=", chosen.id).where("status", "=", "COMPLETED").returningAll().execute())[0];
|
|
9057
|
+
if (!waitpoint) return void 0;
|
|
9058
|
+
return {
|
|
9059
|
+
waitpoint,
|
|
9060
|
+
closed: waitingOn && waitingOn !== waitpoint.id ? await closeLiveWaitpoints(trx, [runId], "CANCELLED", (/* @__PURE__ */ new Date()).toISOString(), (eb) => eb("waitpoint.id", "!=", waitingOn)) : []
|
|
9061
|
+
};
|
|
9062
|
+
});
|
|
9063
|
+
}
|
|
9064
|
+
async getRunPin(runId) {
|
|
9065
|
+
return this.db.selectFrom("run").select([
|
|
9066
|
+
"definition_hash",
|
|
9067
|
+
"definition_op_index",
|
|
9068
|
+
"definition_scope_hash"
|
|
9069
|
+
]).where("id", "=", runId).executeTakeFirst();
|
|
9070
|
+
}
|
|
9071
|
+
async cancelIdleRun(runId, reason) {
|
|
9072
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9073
|
+
if (!await this.db.transaction().execute(async (trx) => {
|
|
9074
|
+
if ((await trx.updateTable("run").set({
|
|
9075
|
+
status: "CANCELLED",
|
|
9076
|
+
error: redactMessage(reason),
|
|
9077
|
+
ended_at: endedAt
|
|
9078
|
+
}).where("id", "=", runId).where("status", "in", ["WAITING", "PENDING"]).returning("id").execute()).length === 0) return false;
|
|
9079
|
+
await failWaitingSteps(trx, [runId], reason, null, endedAt);
|
|
9080
|
+
await closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
|
|
9081
|
+
return true;
|
|
9082
|
+
})) return void 0;
|
|
9083
|
+
this.runsInFlight.delete(runId);
|
|
9084
|
+
return runId;
|
|
9085
|
+
}
|
|
9086
|
+
async endWait(runId, end) {
|
|
9087
|
+
this.runsInFlight.delete(runId);
|
|
9088
|
+
this.journaledSteps.delete(runId);
|
|
9089
|
+
this.pausingRuns.delete(runId);
|
|
9090
|
+
this.resumedLegs.delete(runId);
|
|
9091
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9092
|
+
return this.db.transaction().execute(async (trx) => {
|
|
9093
|
+
if ((await trx.updateTable("run").set({
|
|
9094
|
+
status: end.status,
|
|
9095
|
+
error: redactMessage(end.reason),
|
|
9096
|
+
error_name: end.errorName,
|
|
9097
|
+
ended_at: endedAt
|
|
9098
|
+
}).where("id", "=", runId).where("status", "=", "RUNNING").returning("id").execute()).length === 0) return [];
|
|
9099
|
+
await failWaitingSteps(trx, [runId], end.reason, end.errorName, endedAt);
|
|
9100
|
+
return closeLiveWaitpoints(trx, [runId], end.waitpoints, endedAt);
|
|
9101
|
+
});
|
|
9102
|
+
}
|
|
9103
|
+
async listWaitpointsForRuns(runIds) {
|
|
9104
|
+
const byRun = new Map(runIds.map((id) => [id, []]));
|
|
9105
|
+
if (runIds.length === 0) return byRun;
|
|
9106
|
+
const rows = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "in", [...new Set(runIds)]).orderBy("created_at").execute();
|
|
9107
|
+
for (const row of rows) byRun.get(row.run_id)?.push(row);
|
|
9108
|
+
return byRun;
|
|
9109
|
+
}
|
|
9110
|
+
async listWaitingRuns(workflowId, limit) {
|
|
9111
|
+
if (Array.isArray(workflowId) && workflowId.length === 0) return [];
|
|
9112
|
+
let query = this.db.selectFrom("run").selectAll().where("status", "=", WAITING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc").limit(Math.min(Math.max(limit, 1), 100));
|
|
9113
|
+
if (Array.isArray(workflowId)) query = query.where("workflow_id", "in", workflowId);
|
|
9114
|
+
else if (workflowId) query = query.where("workflow_id", "=", workflowId);
|
|
9115
|
+
return query.execute();
|
|
9116
|
+
}
|
|
8778
9117
|
async pruneFinishedRuns(cutoffIso, batchSize = PRUNE_BATCH_SIZE) {
|
|
8779
9118
|
let pruned = 0;
|
|
8780
9119
|
for (;;) {
|
|
@@ -8782,6 +9121,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8782
9121
|
const ids = (await trx.selectFrom("run").select("id").where("ended_at", "is not", null).where("ended_at", "<", cutoffIso).limit(batchSize).execute()).map((row) => row.id);
|
|
8783
9122
|
if (ids.length === 0) return 0;
|
|
8784
9123
|
await trx.deleteFrom("step_execution").where("run_id", "in", ids).execute();
|
|
9124
|
+
await trx.deleteFrom("waitpoint").where("run_id", "in", ids).execute();
|
|
8785
9125
|
await trx.deleteFrom("run_document").where("run_id", "in", ids).execute();
|
|
8786
9126
|
await trx.deleteFrom("run").where("id", "in", ids).execute();
|
|
8787
9127
|
return ids.length;
|
|
@@ -8813,10 +9153,11 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8813
9153
|
for (const id of frontier) runIds.add(id);
|
|
8814
9154
|
}
|
|
8815
9155
|
if (runIds.size === 0) return erased;
|
|
8816
|
-
const all = [...runIds];
|
|
8817
|
-
const
|
|
8818
|
-
for (const row of
|
|
9156
|
+
const all = [...runIds].sort();
|
|
9157
|
+
const locked = await trx.selectFrom("run").select(["id", "status"]).where("id", "in", all).orderBy("id").forUpdate().execute();
|
|
9158
|
+
for (const row of locked) if (row.status === "RUNNING" || row.status === "PENDING") erasedRuns.add(row.id);
|
|
8819
9159
|
erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
|
|
9160
|
+
await trx.deleteFrom("waitpoint").where("run_id", "in", all).execute();
|
|
8820
9161
|
erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
|
|
8821
9162
|
erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
|
|
8822
9163
|
erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
|
|
@@ -8933,6 +9274,118 @@ function decodeRunCursor(cursor) {
|
|
|
8933
9274
|
id: parsed[1]
|
|
8934
9275
|
};
|
|
8935
9276
|
}
|
|
9277
|
+
const WAITPOINT_POLL_MAX_MS = 3e5;
|
|
9278
|
+
const WAITPOINT_RETRY_MS = 3e4;
|
|
9279
|
+
const SUSPENDED_COUNT_MS = 3e4;
|
|
9280
|
+
var WaitpointPoller = class {
|
|
9281
|
+
deps;
|
|
9282
|
+
running = false;
|
|
9283
|
+
polling = false;
|
|
9284
|
+
again = false;
|
|
9285
|
+
timer;
|
|
9286
|
+
timerAt;
|
|
9287
|
+
nudged;
|
|
9288
|
+
suspended = 0;
|
|
9289
|
+
suspendedAt = 0;
|
|
9290
|
+
counting = false;
|
|
9291
|
+
constructor(deps) {
|
|
9292
|
+
this.deps = deps;
|
|
9293
|
+
}
|
|
9294
|
+
get active() {
|
|
9295
|
+
return this.running;
|
|
9296
|
+
}
|
|
9297
|
+
start() {
|
|
9298
|
+
if (this.running) return;
|
|
9299
|
+
this.running = true;
|
|
9300
|
+
this.poll();
|
|
9301
|
+
}
|
|
9302
|
+
stop() {
|
|
9303
|
+
this.running = false;
|
|
9304
|
+
clearTimeout(this.timer);
|
|
9305
|
+
this.timer = void 0;
|
|
9306
|
+
this.timerAt = void 0;
|
|
9307
|
+
this.nudged = void 0;
|
|
9308
|
+
}
|
|
9309
|
+
nudge(dueIso) {
|
|
9310
|
+
if (!this.running) return;
|
|
9311
|
+
const due = Date.parse(dueIso);
|
|
9312
|
+
if (Number.isNaN(due)) return;
|
|
9313
|
+
if (this.polling) {
|
|
9314
|
+
this.nudged = Math.min(this.nudged ?? Infinity, due);
|
|
9315
|
+
return;
|
|
9316
|
+
}
|
|
9317
|
+
if (this.timerAt !== void 0 && due >= this.timerAt) return;
|
|
9318
|
+
this.arm(due);
|
|
9319
|
+
}
|
|
9320
|
+
suspendedRuns() {
|
|
9321
|
+
if (Date.now() - this.suspendedAt > SUSPENDED_COUNT_MS) this.count();
|
|
9322
|
+
return this.suspended;
|
|
9323
|
+
}
|
|
9324
|
+
async count(store) {
|
|
9325
|
+
if (this.counting) return this.suspended;
|
|
9326
|
+
this.counting = true;
|
|
9327
|
+
try {
|
|
9328
|
+
const journal = store ?? await this.deps.store();
|
|
9329
|
+
if (journal) {
|
|
9330
|
+
this.suspended = await journal.countWaitingRuns();
|
|
9331
|
+
this.suspendedAt = Date.now();
|
|
9332
|
+
}
|
|
9333
|
+
} catch (error) {
|
|
9334
|
+
this.deps.logger.warn("Counting suspended runs failed: @error", error);
|
|
9335
|
+
} finally {
|
|
9336
|
+
this.counting = false;
|
|
9337
|
+
}
|
|
9338
|
+
return this.suspended;
|
|
9339
|
+
}
|
|
9340
|
+
/** One pass: due delays and held resumes deliver, overdue waits expire, the
|
|
9341
|
+
* timer re-arms. */
|
|
9342
|
+
async poll() {
|
|
9343
|
+
if (!this.running) return;
|
|
9344
|
+
if (this.polling) {
|
|
9345
|
+
this.again = true;
|
|
9346
|
+
return;
|
|
9347
|
+
}
|
|
9348
|
+
this.polling = true;
|
|
9349
|
+
let next = Date.now() + WAITPOINT_RETRY_MS;
|
|
9350
|
+
try {
|
|
9351
|
+
const store = await this.deps.store();
|
|
9352
|
+
if (!store) return;
|
|
9353
|
+
const batch = this.deps.batch ?? 100;
|
|
9354
|
+
do {
|
|
9355
|
+
this.again = false;
|
|
9356
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9357
|
+
const due = await store.claimDueDelayWaitpoints(now, batch);
|
|
9358
|
+
for (const runId of new Set(due.map((row) => row.run_id))) await this.deps.resume(runId);
|
|
9359
|
+
const { expired, ended } = await store.expireDueWaitpoints(now, batch);
|
|
9360
|
+
const stranded = await store.endStrandedWaits(batch);
|
|
9361
|
+
for (const end of [...ended, ...stranded]) await this.deps.ended(end);
|
|
9362
|
+
if (due.length === batch || expired.length === batch || stranded.length === batch) this.again = true;
|
|
9363
|
+
} while (this.again && this.active);
|
|
9364
|
+
for (const runId of await store.heldRunIds()) await this.deps.resume(runId);
|
|
9365
|
+
await this.count(store);
|
|
9366
|
+
const [delay, expiry] = await Promise.all([store.nextDelayDue(), store.nextExpiry()]);
|
|
9367
|
+
next = Math.min(delay ? Date.parse(delay) : Infinity, expiry ? Date.parse(expiry) : Infinity);
|
|
9368
|
+
} catch (error) {
|
|
9369
|
+
this.deps.logger.warn("Waitpoint poll failed: @error", error);
|
|
9370
|
+
} finally {
|
|
9371
|
+
this.polling = false;
|
|
9372
|
+
if (this.nudged !== void 0) next = Math.min(next, this.nudged);
|
|
9373
|
+
this.nudged = void 0;
|
|
9374
|
+
if (this.active) this.arm(next);
|
|
9375
|
+
}
|
|
9376
|
+
}
|
|
9377
|
+
arm(due) {
|
|
9378
|
+
clearTimeout(this.timer);
|
|
9379
|
+
const delay = Math.min(Math.max(Number.isFinite(due) ? due - Date.now() : Infinity, 0), WAITPOINT_POLL_MAX_MS);
|
|
9380
|
+
this.timerAt = Date.now() + delay;
|
|
9381
|
+
this.timer = setTimeout(() => {
|
|
9382
|
+
this.timer = void 0;
|
|
9383
|
+
this.timerAt = void 0;
|
|
9384
|
+
this.poll();
|
|
9385
|
+
}, delay);
|
|
9386
|
+
this.timer.unref();
|
|
9387
|
+
}
|
|
9388
|
+
};
|
|
8936
9389
|
//#endregion
|
|
8937
9390
|
//#region src/reactor/run-gate.ts
|
|
8938
9391
|
const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
|
|
@@ -8970,8 +9423,11 @@ var WorkflowRunGate = class {
|
|
|
8970
9423
|
* - **SINGLETON**: refused outright while a run is active. The caller
|
|
8971
9424
|
* journals the refusal rather than dropping it silently, because a firing
|
|
8972
9425
|
* that vanished is indistinguishable from a trigger that never fired.
|
|
9426
|
+
*
|
|
9427
|
+
* A `resume` is a waiting run's next leg, not a new firing: it queues for
|
|
9428
|
+
* the slot under every policy, past the queue depth too.
|
|
8973
9429
|
*/
|
|
8974
|
-
async admit(workflowId, policy, onQueue) {
|
|
9430
|
+
async admit(workflowId, policy, onQueue, options = {}) {
|
|
8975
9431
|
if (this.closed) return {
|
|
8976
9432
|
admitted: false,
|
|
8977
9433
|
reason: CLOSED_REASON,
|
|
@@ -8990,7 +9446,7 @@ var WorkflowRunGate = class {
|
|
|
8990
9446
|
this.lanes.set(workflowId, lane);
|
|
8991
9447
|
let waited = false;
|
|
8992
9448
|
if (lane.active >= limit) {
|
|
8993
|
-
if (policy.concurrency === "SINGLETON") {
|
|
9449
|
+
if (policy.concurrency === "SINGLETON" && !options.resume) {
|
|
8994
9450
|
this.forgetIfIdle(workflowId, lane);
|
|
8995
9451
|
return {
|
|
8996
9452
|
admitted: false,
|
|
@@ -8998,7 +9454,7 @@ var WorkflowRunGate = class {
|
|
|
8998
9454
|
reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
|
|
8999
9455
|
};
|
|
9000
9456
|
}
|
|
9001
|
-
if (lane.waiting.length >= this.maxQueued) return {
|
|
9457
|
+
if (!options.resume && lane.waiting.length >= this.maxQueued) return {
|
|
9002
9458
|
admitted: false,
|
|
9003
9459
|
refusal: "queue-full",
|
|
9004
9460
|
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
@@ -10653,12 +11109,72 @@ function refusalStatus(refusal) {
|
|
|
10653
11109
|
switch (refusal) {
|
|
10654
11110
|
case "parked":
|
|
10655
11111
|
case "singleton":
|
|
10656
|
-
case "stale":
|
|
11112
|
+
case "stale":
|
|
11113
|
+
case "cancelled": return 409;
|
|
10657
11114
|
case "queue-full":
|
|
10658
11115
|
case "expired": return 429;
|
|
10659
11116
|
default: return 500;
|
|
10660
11117
|
}
|
|
10661
11118
|
}
|
|
11119
|
+
function runResultReply(result, succeededStatus) {
|
|
11120
|
+
return {
|
|
11121
|
+
status: result.status === "SUCCEEDED" ? succeededStatus : result.status === "WAITING" ? 202 : refusalStatus(result.refusal),
|
|
11122
|
+
contentType: JSON_CONTENT_TYPE,
|
|
11123
|
+
body: JSON.stringify({
|
|
11124
|
+
runId: result.runId,
|
|
11125
|
+
status: result.status,
|
|
11126
|
+
error: result.error ?? null
|
|
11127
|
+
})
|
|
11128
|
+
};
|
|
11129
|
+
}
|
|
11130
|
+
const RESPONDED = Symbol("a step responded");
|
|
11131
|
+
const DROPPED_RESPONSE_HEADERS = /* @__PURE__ */ new Set([
|
|
11132
|
+
"connection",
|
|
11133
|
+
"content-length",
|
|
11134
|
+
"keep-alive",
|
|
11135
|
+
"set-cookie",
|
|
11136
|
+
"transfer-encoding",
|
|
11137
|
+
"upgrade"
|
|
11138
|
+
]);
|
|
11139
|
+
const ORIGIN_SAFE_HEADERS = {
|
|
11140
|
+
"x-content-type-options": "nosniff",
|
|
11141
|
+
"content-security-policy": "sandbox"
|
|
11142
|
+
};
|
|
11143
|
+
function sendableHeader(name, value) {
|
|
11144
|
+
try {
|
|
11145
|
+
validateHeaderName(name);
|
|
11146
|
+
validateHeaderValue(name, value);
|
|
11147
|
+
return true;
|
|
11148
|
+
} catch {
|
|
11149
|
+
return false;
|
|
11150
|
+
}
|
|
11151
|
+
}
|
|
11152
|
+
/** A step's `run.respond` / `run.stop` answer as an HTTP reply. As upstream,
|
|
11153
|
+
* none answers 200 with `{}`, and a string body goes as text. */
|
|
11154
|
+
function runResponseReply(response) {
|
|
11155
|
+
const status = typeof response.status === "number" && Number.isInteger(response.status) && response.status >= 200 && response.status <= 599 ? response.status : 200;
|
|
11156
|
+
const body = response.body ?? {};
|
|
11157
|
+
let contentType = typeof body === "string" ? "text/plain; charset=utf-8" : JSON_CONTENT_TYPE;
|
|
11158
|
+
const headers = {};
|
|
11159
|
+
const given = response.headers && typeof response.headers === "object" ? response.headers : {};
|
|
11160
|
+
for (const [rawName, rawValue] of Object.entries(given)) {
|
|
11161
|
+
const name = rawName.toLowerCase();
|
|
11162
|
+
if (typeof rawValue !== "string" && typeof rawValue !== "number") continue;
|
|
11163
|
+
const value = String(rawValue);
|
|
11164
|
+
if (!sendableHeader(name, value)) continue;
|
|
11165
|
+
if (name === "content-type") contentType = value;
|
|
11166
|
+
else if (!DROPPED_RESPONSE_HEADERS.has(name)) headers[name] = value;
|
|
11167
|
+
}
|
|
11168
|
+
return {
|
|
11169
|
+
status,
|
|
11170
|
+
contentType,
|
|
11171
|
+
body: typeof body === "string" ? body : JSON.stringify(body),
|
|
11172
|
+
headers: {
|
|
11173
|
+
...headers,
|
|
11174
|
+
...ORIGIN_SAFE_HEADERS
|
|
11175
|
+
}
|
|
11176
|
+
};
|
|
11177
|
+
}
|
|
10662
11178
|
const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
|
|
10663
11179
|
function registrationKey(state) {
|
|
10664
11180
|
const trigger = runnableDefinition(state).trigger;
|
|
@@ -11619,6 +12135,8 @@ var WorkflowRuntimeService = class {
|
|
|
11619
12135
|
startTriggerSupervisor() {
|
|
11620
12136
|
this.supervisor().start();
|
|
11621
12137
|
this.admitAbandonedRuns();
|
|
12138
|
+
this.deliverAllHeld();
|
|
12139
|
+
this.startWaitpointPoller();
|
|
11622
12140
|
this.pendingSweepTimer ??= setInterval(() => {
|
|
11623
12141
|
this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
|
|
11624
12142
|
}, PENDING_SWEEP_INTERVAL_MS);
|
|
@@ -11641,6 +12159,7 @@ var WorkflowRuntimeService = class {
|
|
|
11641
12159
|
stopTriggerSupervisor() {
|
|
11642
12160
|
clearInterval(this.pendingSweepTimer);
|
|
11643
12161
|
this.pendingSweepTimer = void 0;
|
|
12162
|
+
this.stopWaitpointPoller();
|
|
11644
12163
|
this.triggerSupervisor?.stop();
|
|
11645
12164
|
}
|
|
11646
12165
|
closed = false;
|
|
@@ -11848,13 +12367,54 @@ var WorkflowRuntimeService = class {
|
|
|
11848
12367
|
}));
|
|
11849
12368
|
let replied = false;
|
|
11850
12369
|
const takeReply = () => !replied && (replied = true);
|
|
11851
|
-
|
|
12370
|
+
let sent;
|
|
12371
|
+
let responded = () => void 0;
|
|
12372
|
+
const respondedFirst = new Promise((resolve) => {
|
|
12373
|
+
responded = () => resolve(RESPONDED);
|
|
12374
|
+
});
|
|
12375
|
+
const responder = (response) => {
|
|
12376
|
+
if (sent || !takeReply()) return;
|
|
12377
|
+
sent = response;
|
|
12378
|
+
responded();
|
|
12379
|
+
};
|
|
12380
|
+
let ended;
|
|
12381
|
+
let endedEarly = () => void 0;
|
|
12382
|
+
const endedFirst = new Promise((resolve) => {
|
|
12383
|
+
endedEarly = () => resolve({
|
|
12384
|
+
ok: true,
|
|
12385
|
+
result: ended
|
|
12386
|
+
});
|
|
12387
|
+
});
|
|
12388
|
+
const ender = (result) => {
|
|
12389
|
+
if (ended || !takeReply()) return;
|
|
12390
|
+
ended = result;
|
|
12391
|
+
endedEarly();
|
|
12392
|
+
};
|
|
12393
|
+
if (runId) {
|
|
12394
|
+
this.heldReplies.set(runId, takeReply);
|
|
12395
|
+
this.runResponders.set(runId, responder);
|
|
12396
|
+
this.queuedReplies.set(runId, ender);
|
|
12397
|
+
}
|
|
11852
12398
|
let run;
|
|
11853
12399
|
try {
|
|
11854
|
-
|
|
11855
|
-
|
|
12400
|
+
const raced = await Promise.race([
|
|
12401
|
+
firing,
|
|
12402
|
+
respondedFirst,
|
|
12403
|
+
endedFirst,
|
|
12404
|
+
timeout(DELIVERY_TIMEOUT_MS)
|
|
12405
|
+
]);
|
|
12406
|
+
if (sent) return runResponseReply(sent);
|
|
12407
|
+
run = raced;
|
|
12408
|
+
if (run === TIMED_OUT && !takeReply()) run = ended ? {
|
|
12409
|
+
ok: true,
|
|
12410
|
+
result: ended
|
|
12411
|
+
} : await firing;
|
|
11856
12412
|
} finally {
|
|
11857
|
-
if (runId)
|
|
12413
|
+
if (runId) {
|
|
12414
|
+
this.heldReplies.delete(runId);
|
|
12415
|
+
if (this.runResponders.get(runId) === responder) this.runResponders.delete(runId);
|
|
12416
|
+
if (this.queuedReplies.get(runId) === ender) this.queuedReplies.delete(runId);
|
|
12417
|
+
}
|
|
11858
12418
|
}
|
|
11859
12419
|
if (run === TIMED_OUT) {
|
|
11860
12420
|
this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
|
|
@@ -11880,15 +12440,7 @@ var WorkflowRuntimeService = class {
|
|
|
11880
12440
|
body: JSON.stringify({ error: message })
|
|
11881
12441
|
};
|
|
11882
12442
|
}
|
|
11883
|
-
return
|
|
11884
|
-
status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
|
|
11885
|
-
contentType: JSON_CONTENT_TYPE,
|
|
11886
|
-
body: JSON.stringify({
|
|
11887
|
-
runId: run.result.runId,
|
|
11888
|
-
status: run.result.status,
|
|
11889
|
-
error: run.result.error ?? null
|
|
11890
|
-
})
|
|
11891
|
-
};
|
|
12443
|
+
return runResultReply(run.result, config.responseStatus);
|
|
11892
12444
|
}
|
|
11893
12445
|
async pieceHandshake(binding, request) {
|
|
11894
12446
|
const handshake = await this.pieceHandshakeConfig(binding);
|
|
@@ -12624,6 +13176,61 @@ var WorkflowRuntimeService = class {
|
|
|
12624
13176
|
type: "value"
|
|
12625
13177
|
}]
|
|
12626
13178
|
};
|
|
13179
|
+
case DELAY_BLOCK: return {
|
|
13180
|
+
source: "static",
|
|
13181
|
+
nodes: [{
|
|
13182
|
+
name: "resumedAt",
|
|
13183
|
+
type: "DateTime!"
|
|
13184
|
+
}]
|
|
13185
|
+
};
|
|
13186
|
+
case APPROVAL_BLOCK: return {
|
|
13187
|
+
source: "static",
|
|
13188
|
+
nodes: [
|
|
13189
|
+
{
|
|
13190
|
+
name: "approved",
|
|
13191
|
+
type: "Boolean!"
|
|
13192
|
+
},
|
|
13193
|
+
{
|
|
13194
|
+
name: "reason",
|
|
13195
|
+
type: "String"
|
|
13196
|
+
},
|
|
13197
|
+
{
|
|
13198
|
+
name: "decidedBy",
|
|
13199
|
+
type: "String",
|
|
13200
|
+
description: "The approver's DID"
|
|
13201
|
+
},
|
|
13202
|
+
{
|
|
13203
|
+
name: "decidedAt",
|
|
13204
|
+
type: "DateTime!"
|
|
13205
|
+
}
|
|
13206
|
+
]
|
|
13207
|
+
};
|
|
13208
|
+
case STOP_BLOCK: return {
|
|
13209
|
+
source: "static",
|
|
13210
|
+
nodes: [{
|
|
13211
|
+
name: "response",
|
|
13212
|
+
type: "JSONObject",
|
|
13213
|
+
description: "The answer sent, when one was set"
|
|
13214
|
+
}]
|
|
13215
|
+
};
|
|
13216
|
+
case WAIT_FOR_WEBHOOK_BLOCK: return {
|
|
13217
|
+
source: "static",
|
|
13218
|
+
nodes: [
|
|
13219
|
+
{
|
|
13220
|
+
name: "body",
|
|
13221
|
+
type: "Unknown",
|
|
13222
|
+
description: "The resuming request's body"
|
|
13223
|
+
},
|
|
13224
|
+
{
|
|
13225
|
+
name: "headers",
|
|
13226
|
+
type: "JSONObject!"
|
|
13227
|
+
},
|
|
13228
|
+
{
|
|
13229
|
+
name: "queryParams",
|
|
13230
|
+
type: "JSONObject!"
|
|
13231
|
+
}
|
|
13232
|
+
]
|
|
13233
|
+
};
|
|
12627
13234
|
case DOCUMENT_CREATED_BLOCK:
|
|
12628
13235
|
case DOCUMENT_DELETED_BLOCK: return {
|
|
12629
13236
|
source: "static",
|
|
@@ -12879,7 +13486,12 @@ var WorkflowRuntimeService = class {
|
|
|
12879
13486
|
pool.dispose();
|
|
12880
13487
|
return this.pieceWorkers = pool;
|
|
12881
13488
|
}
|
|
12882
|
-
|
|
13489
|
+
const unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
|
|
13490
|
+
const unregisterGauge = registerTelemetryGauge("workflow.pool", () => pool.memoryStats());
|
|
13491
|
+
this.unobservePool = () => {
|
|
13492
|
+
unobservePool();
|
|
13493
|
+
unregisterGauge();
|
|
13494
|
+
};
|
|
12883
13495
|
return this.pieceWorkers = pool;
|
|
12884
13496
|
}
|
|
12885
13497
|
dispatcher() {
|
|
@@ -12945,6 +13557,7 @@ var WorkflowRuntimeService = class {
|
|
|
12945
13557
|
let definition;
|
|
12946
13558
|
let runUser;
|
|
12947
13559
|
let readGeneration;
|
|
13560
|
+
let revision;
|
|
12948
13561
|
try {
|
|
12949
13562
|
if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12950
13563
|
run.refused = true;
|
|
@@ -12954,6 +13567,7 @@ var WorkflowRuntimeService = class {
|
|
|
12954
13567
|
if (cached) {
|
|
12955
13568
|
state = cached.state;
|
|
12956
13569
|
documentName = cached.name;
|
|
13570
|
+
revision = cached.revision;
|
|
12957
13571
|
readGeneration = this.workflowGeneration(workflowId);
|
|
12958
13572
|
} else {
|
|
12959
13573
|
const generation = this.workflowGeneration(workflowId);
|
|
@@ -12965,6 +13579,7 @@ var WorkflowRuntimeService = class {
|
|
|
12965
13579
|
}
|
|
12966
13580
|
state = document.state.global;
|
|
12967
13581
|
documentName = document.header.name;
|
|
13582
|
+
revision = globalRevision(document);
|
|
12968
13583
|
this.cacheLoaded(workflowId, generation, document);
|
|
12969
13584
|
}
|
|
12970
13585
|
if (state.status !== "ENABLED") {
|
|
@@ -13093,74 +13708,63 @@ var WorkflowRuntimeService = class {
|
|
|
13093
13708
|
}
|
|
13094
13709
|
}
|
|
13095
13710
|
let runId = enqueuedRunId ?? null;
|
|
13096
|
-
let journalFailed = false;
|
|
13097
|
-
const executionOrder = /* @__PURE__ */ new Map();
|
|
13098
13711
|
let session;
|
|
13099
13712
|
try {
|
|
13100
|
-
|
|
13713
|
+
const journaled = await this.telemetry.phase("journal.start", {}, async () => {
|
|
13101
13714
|
if (enqueuedRunId) {
|
|
13102
|
-
await store?.beginRun(enqueuedRunId, {
|
|
13715
|
+
if (await store?.beginRun(enqueuedRunId, {
|
|
13103
13716
|
workflowName: runJournalName(state.name, documentName),
|
|
13104
13717
|
workflowVersion: runnable.version
|
|
13105
|
-
})
|
|
13106
|
-
|
|
13718
|
+
}) === false) return {
|
|
13719
|
+
runId: null,
|
|
13720
|
+
cancelled: true
|
|
13721
|
+
};
|
|
13722
|
+
return { runId: enqueuedRunId };
|
|
13107
13723
|
}
|
|
13108
|
-
return await store?.startRun({
|
|
13724
|
+
return { runId: await store?.startRun({
|
|
13109
13725
|
workflowId,
|
|
13110
13726
|
workflowName: runJournalName(state.name, documentName),
|
|
13111
13727
|
workflowVersion: runnable.version,
|
|
13112
13728
|
triggerKind,
|
|
13113
13729
|
triggerPayload,
|
|
13114
13730
|
rerunOf: resume?.rerunOf
|
|
13115
|
-
}) ?? null;
|
|
13731
|
+
}) ?? null };
|
|
13116
13732
|
});
|
|
13733
|
+
if (journaled.cancelled) {
|
|
13734
|
+
const row = enqueuedRunId ? await store?.getRun(enqueuedRunId) : void 0;
|
|
13735
|
+
return {
|
|
13736
|
+
status: CANCELLED_RUN_STATUS,
|
|
13737
|
+
steps: [],
|
|
13738
|
+
runId: enqueuedRunId ?? null,
|
|
13739
|
+
...row?.error ? { error: row.error } : {},
|
|
13740
|
+
refusal: "cancelled"
|
|
13741
|
+
};
|
|
13742
|
+
}
|
|
13743
|
+
runId = journaled.runId;
|
|
13117
13744
|
if (runId) run.span.setAttribute("run.id", runId);
|
|
13118
13745
|
if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
|
|
13119
13746
|
session = this.workers().session();
|
|
13120
|
-
const {
|
|
13121
|
-
const journal = store;
|
|
13122
|
-
const journaledRunId = runId;
|
|
13123
|
-
const handed = /* @__PURE__ */ new Set();
|
|
13124
|
-
const result = await withRunScope({
|
|
13747
|
+
const { result, handed } = await this.executeLeg({
|
|
13125
13748
|
workflowId,
|
|
13126
13749
|
runId,
|
|
13750
|
+
store,
|
|
13751
|
+
session,
|
|
13752
|
+
definition,
|
|
13753
|
+
runnable,
|
|
13754
|
+
policy,
|
|
13755
|
+
deadline,
|
|
13127
13756
|
connections,
|
|
13128
|
-
|
|
13129
|
-
...runUser !== void 0 ? { runUser } : {},
|
|
13130
|
-
attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
|
|
13131
|
-
recordDocuments: async (documentIds) => {
|
|
13132
|
-
for (const documentId of documentIds) handed.add(documentId);
|
|
13133
|
-
if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
|
|
13134
|
-
}
|
|
13135
|
-
}, () => runWorkflow({
|
|
13136
|
-
definition: {
|
|
13137
|
-
...definition,
|
|
13138
|
-
variables
|
|
13139
|
-
},
|
|
13140
|
-
executor,
|
|
13141
|
-
declaredPorts: blockPorts,
|
|
13142
|
-
blockDefaults: (block) => this.blockDefaults(block),
|
|
13143
|
-
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
13757
|
+
runUser,
|
|
13144
13758
|
triggerPayload,
|
|
13759
|
+
executor,
|
|
13145
13760
|
completedSteps: resume?.completedSteps,
|
|
13146
|
-
|
|
13147
|
-
|
|
13148
|
-
|
|
13149
|
-
|
|
13150
|
-
|
|
13151
|
-
|
|
13152
|
-
|
|
13153
|
-
if (journalFailed) return;
|
|
13154
|
-
journalFailed = true;
|
|
13155
|
-
this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
13156
|
-
}
|
|
13157
|
-
} : void 0
|
|
13158
|
-
}));
|
|
13159
|
-
if (store && journaledRunId) try {
|
|
13160
|
-
await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
|
|
13161
|
-
} catch (error) {
|
|
13162
|
-
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
13163
|
-
}
|
|
13761
|
+
replayed: resume?.completedSteps,
|
|
13762
|
+
pin: () => ({
|
|
13763
|
+
definition_hash: this.definitionHash(workflowId, state),
|
|
13764
|
+
definition_op_index: revision,
|
|
13765
|
+
definition_scope_hash: scopeHash(state)
|
|
13766
|
+
})
|
|
13767
|
+
});
|
|
13164
13768
|
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
13165
13769
|
const finished = {
|
|
13166
13770
|
...result,
|
|
@@ -13174,7 +13778,14 @@ var WorkflowRuntimeService = class {
|
|
|
13174
13778
|
runId
|
|
13175
13779
|
};
|
|
13176
13780
|
} catch (error) {
|
|
13177
|
-
if (store && runId)
|
|
13781
|
+
if (store && runId) {
|
|
13782
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
13783
|
+
const closed = await store.failRun(runId, message, errorNameOf(error));
|
|
13784
|
+
this.answerClosed(closed, runId, {
|
|
13785
|
+
status: "FAILED",
|
|
13786
|
+
error: message
|
|
13787
|
+
});
|
|
13788
|
+
}
|
|
13178
13789
|
throw error;
|
|
13179
13790
|
} finally {
|
|
13180
13791
|
session?.close();
|
|
@@ -13182,6 +13793,115 @@ var WorkflowRuntimeService = class {
|
|
|
13182
13793
|
dispatched();
|
|
13183
13794
|
}
|
|
13184
13795
|
}
|
|
13796
|
+
async executeLeg(leg) {
|
|
13797
|
+
const { workflowId, runId, store, policy } = leg;
|
|
13798
|
+
let journalFailed = false;
|
|
13799
|
+
const executionOrder = /* @__PURE__ */ new Map();
|
|
13800
|
+
const { variables, secretValues } = await resolveVariables(leg.runnable.variables, this.secretProvider(), workflowId);
|
|
13801
|
+
const handed = new Set(leg.handed);
|
|
13802
|
+
const attachments = attachmentOrigins(leg.triggerPayload, handed, leg.replayed);
|
|
13803
|
+
for (const ref of leg.written ?? []) attachments.written.add(ref);
|
|
13804
|
+
const waitpoints = store && runId ? this.waitpointPort(store, workflowId, runId, maxSuspensionMs(leg.runnable)) : void 0;
|
|
13805
|
+
const respond = leg.respond ?? (runId && !leg.resume ? (response) => this.runResponders.get(runId)?.(response) : void 0);
|
|
13806
|
+
let result = await withRunScope({
|
|
13807
|
+
workflowId,
|
|
13808
|
+
runId,
|
|
13809
|
+
connections: leg.connections,
|
|
13810
|
+
pieceWorker: leg.session,
|
|
13811
|
+
...leg.runUser !== void 0 ? { runUser: leg.runUser } : {},
|
|
13812
|
+
attachments,
|
|
13813
|
+
recordDocuments: async (documentIds) => {
|
|
13814
|
+
for (const documentId of documentIds) handed.add(documentId);
|
|
13815
|
+
if (store && runId) await store.recordRunDocuments(runId, documentIds);
|
|
13816
|
+
},
|
|
13817
|
+
...waitpoints ? { waitpoints: waitpoints.port } : {}
|
|
13818
|
+
}, () => runWorkflow({
|
|
13819
|
+
definition: {
|
|
13820
|
+
...leg.definition,
|
|
13821
|
+
variables
|
|
13822
|
+
},
|
|
13823
|
+
executor: leg.executor,
|
|
13824
|
+
declaredPorts: blockPorts,
|
|
13825
|
+
blockDefaults: (block) => this.blockDefaults(block),
|
|
13826
|
+
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
13827
|
+
triggerPayload: leg.triggerPayload,
|
|
13828
|
+
completedSteps: leg.completedSteps,
|
|
13829
|
+
...leg.carriedSteps ? { carriedSteps: leg.carriedSteps } : {},
|
|
13830
|
+
...leg.resume ? { resume: leg.resume } : {},
|
|
13831
|
+
...leg.firstOrdinal !== void 0 ? { firstOrdinal: leg.firstOrdinal } : {},
|
|
13832
|
+
...respond ? { onRespond: respond } : {},
|
|
13833
|
+
...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
|
|
13834
|
+
...leg.deadline !== void 0 ? { deadline: leg.deadline } : {},
|
|
13835
|
+
onStep: store && runId ? async (record, ordinal) => {
|
|
13836
|
+
executionOrder.set(record.stepId, ordinal);
|
|
13837
|
+
try {
|
|
13838
|
+
await this.telemetry.phase("journal.step", {}, () => store.recordStep(runId, ordinal, record));
|
|
13839
|
+
} catch (error) {
|
|
13840
|
+
if (journalFailed) return;
|
|
13841
|
+
journalFailed = true;
|
|
13842
|
+
this.logger.warn(`Run ${runId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
13843
|
+
}
|
|
13844
|
+
} : void 0
|
|
13845
|
+
}));
|
|
13846
|
+
if (store && runId) {
|
|
13847
|
+
let waiting = result.status === WAITING_RUN_STATUS;
|
|
13848
|
+
try {
|
|
13849
|
+
const { held, closed } = await this.telemetry.phase("journal.finish", {}, () => store.finishRun(runId, result, executionOrder, waiting ? leg.pin?.() : void 0));
|
|
13850
|
+
this.answerClosed(closed, runId, {
|
|
13851
|
+
status: waiting ? CANCELLED_RUN_STATUS : result.status,
|
|
13852
|
+
error: waiting ? "The step that created this waitpoint no longer waits on it" : result.error ?? "The run ended without resuming on it"
|
|
13853
|
+
});
|
|
13854
|
+
if (!held) {
|
|
13855
|
+
result = await this.endUnheldWait(store, runId, result);
|
|
13856
|
+
waiting = false;
|
|
13857
|
+
}
|
|
13858
|
+
} catch (error) {
|
|
13859
|
+
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
13860
|
+
}
|
|
13861
|
+
if (waiting) {
|
|
13862
|
+
const toSend = result.waiting && waitpoints?.responseFor(result.waiting.waitpointId);
|
|
13863
|
+
if (toSend) respond?.(toSend);
|
|
13864
|
+
await this.deliverHeld(runId);
|
|
13865
|
+
}
|
|
13866
|
+
}
|
|
13867
|
+
return {
|
|
13868
|
+
result,
|
|
13869
|
+
handed
|
|
13870
|
+
};
|
|
13871
|
+
}
|
|
13872
|
+
async endUnheldWait(store, runId, result) {
|
|
13873
|
+
const on = result.waiting;
|
|
13874
|
+
const step = result.steps.find((record) => record.stepId === on?.stepId);
|
|
13875
|
+
const row = on ? await store.getWaitpoint(on.waitpointId) : void 0;
|
|
13876
|
+
const end = row?.run_id === runId && row.step_id === on?.stepId && row.status === "EXPIRED" ? {
|
|
13877
|
+
status: "CANCELLED",
|
|
13878
|
+
reason: `The wait at step "${row.step_key}" expired at ${row.expires_at} before the run could wait on it`,
|
|
13879
|
+
errorName: WAITPOINT_EXPIRED_ERROR_NAME
|
|
13880
|
+
} : {
|
|
13881
|
+
status: "FAILED",
|
|
13882
|
+
reason: `Step "${step?.key ?? on?.stepId}" waited on waitpoint "${on?.waitpointId}", which is not a live waitpoint it created`,
|
|
13883
|
+
errorName: WAITPOINT_NOT_LIVE_ERROR_NAME
|
|
13884
|
+
};
|
|
13885
|
+
const closed = await store.endWait(runId, {
|
|
13886
|
+
...end,
|
|
13887
|
+
waitpoints: "EXPIRED"
|
|
13888
|
+
});
|
|
13889
|
+
this.answerClosed(closed, runId, {
|
|
13890
|
+
status: end.status,
|
|
13891
|
+
error: end.reason
|
|
13892
|
+
});
|
|
13893
|
+
return {
|
|
13894
|
+
status: end.status,
|
|
13895
|
+
error: end.reason,
|
|
13896
|
+
errorName: end.errorName,
|
|
13897
|
+
steps: result.steps.map((record) => record === step ? {
|
|
13898
|
+
...record,
|
|
13899
|
+
status: "FAILED",
|
|
13900
|
+
error: end.reason,
|
|
13901
|
+
errorName: end.errorName
|
|
13902
|
+
} : record)
|
|
13903
|
+
};
|
|
13904
|
+
}
|
|
13185
13905
|
runGate = new WorkflowRunGate();
|
|
13186
13906
|
unsettledFirings = /* @__PURE__ */ new Map();
|
|
13187
13907
|
runLimits = /* @__PURE__ */ new Map();
|
|
@@ -13206,6 +13926,9 @@ var WorkflowRuntimeService = class {
|
|
|
13206
13926
|
else this.unsettledFirings.delete(workflowId);
|
|
13207
13927
|
}
|
|
13208
13928
|
heldReplies = /* @__PURE__ */ new Map();
|
|
13929
|
+
runResponders = /* @__PURE__ */ new Map();
|
|
13930
|
+
queuedReplies = /* @__PURE__ */ new Map();
|
|
13931
|
+
syncResumes = /* @__PURE__ */ new Map();
|
|
13209
13932
|
/** Why a trigger's firing of a PARKED workflow is refused; undefined when
|
|
13210
13933
|
* it is not parked, or when an operator started the run. */
|
|
13211
13934
|
async parkedFiring(workflowId, triggerKind, publishedVersion) {
|
|
@@ -13273,12 +13996,23 @@ var WorkflowRuntimeService = class {
|
|
|
13273
13996
|
this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
|
|
13274
13997
|
let runId = enqueuedRunId ?? null;
|
|
13275
13998
|
if (store) try {
|
|
13276
|
-
|
|
13999
|
+
let adopted = true;
|
|
14000
|
+
if (runId) adopted = await store.beginRun(runId, details);
|
|
13277
14001
|
else runId = await store.startRun({
|
|
13278
14002
|
workflowId,
|
|
13279
14003
|
...details
|
|
13280
14004
|
});
|
|
13281
|
-
await store.cancelRun(runId, details.reason);
|
|
14005
|
+
if (adopted) await store.cancelRun(runId, details.reason);
|
|
14006
|
+
else {
|
|
14007
|
+
const row = await store.getRun(runId);
|
|
14008
|
+
if (row) return {
|
|
14009
|
+
status: row.status,
|
|
14010
|
+
steps: [],
|
|
14011
|
+
runId,
|
|
14012
|
+
...row.error ? { error: row.error } : {},
|
|
14013
|
+
refusal: row.status === "CANCELLED" ? "cancelled" : refusal
|
|
14014
|
+
};
|
|
14015
|
+
}
|
|
13282
14016
|
} catch (error) {
|
|
13283
14017
|
this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
|
|
13284
14018
|
}
|
|
@@ -13374,6 +14108,677 @@ var WorkflowRuntimeService = class {
|
|
|
13374
14108
|
const parsed = resolvedBlock(await this.resolveBlock(stepBlock(step)));
|
|
13375
14109
|
return parsed ? await this.declaredReactor(parsed) === "read" : false;
|
|
13376
14110
|
}
|
|
14111
|
+
resuming = /* @__PURE__ */ new Set();
|
|
14112
|
+
definitionHashes = /* @__PURE__ */ new Map();
|
|
14113
|
+
/** Our hash of the runnable definition: computed once per published version. */
|
|
14114
|
+
definitionHash(workflowId, state) {
|
|
14115
|
+
const runnable = runnableDefinition(state);
|
|
14116
|
+
if (!runnable.published) return hashOf(canonicalJson(runnable));
|
|
14117
|
+
const key = `${workflowId}@${runnable.version}`;
|
|
14118
|
+
let hash = this.definitionHashes.get(key);
|
|
14119
|
+
if (hash === void 0) {
|
|
14120
|
+
hash = hashOf(canonicalJson(runnable));
|
|
14121
|
+
if (this.definitionHashes.size >= WORKFLOW_CACHE_MAX) this.definitionHashes.clear();
|
|
14122
|
+
this.definitionHashes.set(key, hash);
|
|
14123
|
+
}
|
|
14124
|
+
return hash;
|
|
14125
|
+
}
|
|
14126
|
+
waitpointPort(store, workflowId, runId, suspendMs) {
|
|
14127
|
+
const responses = /* @__PURE__ */ new Map();
|
|
14128
|
+
const tokens = /* @__PURE__ */ new Map();
|
|
14129
|
+
const configured = this.host.waitpointBaseUrl?.replace(/\/+$/, "");
|
|
14130
|
+
const baseUrl = configured && URL.canParse(configured) ? configured : void 0;
|
|
14131
|
+
return {
|
|
14132
|
+
port: {
|
|
14133
|
+
...baseUrl ? { resumeBaseUrl: baseUrl } : {},
|
|
14134
|
+
create: async (request) => {
|
|
14135
|
+
if (!baseUrl && request.type === "WEBHOOK" && !request.approval) throw new Error("This host serves no resume URLs: resume URLs need an HTTP host");
|
|
14136
|
+
const createdAt = /* @__PURE__ */ new Date();
|
|
14137
|
+
const resumeAt = request.resumeDateTime ? new Date(Date.parse(request.resumeDateTime)).toISOString() : null;
|
|
14138
|
+
if (resumeAt && Date.parse(resumeAt) > MAX_WAITPOINT_MS) throw new Error(`A delay until ${resumeAt} is past ${MAX_WAITPOINT_ISO}, the latest a run can wait until`);
|
|
14139
|
+
const expiresAt = waitpointExpiry(createdAt, request, suspendMs);
|
|
14140
|
+
if (request.type === "DELAY" && resumeAt && suspendMs !== null) {
|
|
14141
|
+
if (Date.parse(resumeAt) - createdAt.getTime() > suspendMs) throw new Error(`A delay until ${resumeAt} is longer than the workflow's maxSuspensionDays allows`);
|
|
14142
|
+
}
|
|
14143
|
+
const token = baseUrl && !request.approval && request.type !== "DELAY" ? request.token ?? randomBytes(32).toString("base64url") : null;
|
|
14144
|
+
const id = randomUUID();
|
|
14145
|
+
const { waitpoint, created: inserted } = await store.createWaitpoint({
|
|
14146
|
+
id,
|
|
14147
|
+
run_id: runId,
|
|
14148
|
+
workflow_id: workflowId,
|
|
14149
|
+
step_id: request.stepId,
|
|
14150
|
+
step_key: request.stepKey,
|
|
14151
|
+
step_path: request.stepId,
|
|
14152
|
+
piece_name: request.pieceName,
|
|
14153
|
+
block_name: request.blockName,
|
|
14154
|
+
type: request.type,
|
|
14155
|
+
kind: request.approval ? "approval" : request.type === "DELAY" ? "delay" : "webhook",
|
|
14156
|
+
status: "PENDING",
|
|
14157
|
+
token_hash: token ? tokenHash(token) : null,
|
|
14158
|
+
created_at: createdAt.toISOString(),
|
|
14159
|
+
resume_at: resumeAt,
|
|
14160
|
+
expires_at: request.type === "DELAY" ? null : expiresAt,
|
|
14161
|
+
resolved_at: null,
|
|
14162
|
+
decided_by: null,
|
|
14163
|
+
approval: request.approval ? JSON.stringify(request.approval) : null,
|
|
14164
|
+
resume_payload: null,
|
|
14165
|
+
response_to_send: request.responseToSend ? JSON.stringify(request.responseToSend) : null
|
|
14166
|
+
});
|
|
14167
|
+
if (!inserted) {
|
|
14168
|
+
const known = tokens.get(waitpoint.id);
|
|
14169
|
+
if (waitpoint.token_hash !== null && known === void 0) throw new Error(`Step "${request.stepKey}" has a waitpoint from an earlier leg, whose resume URL cannot be issued again`);
|
|
14170
|
+
return {
|
|
14171
|
+
id: waitpoint.id,
|
|
14172
|
+
resumeUrl: known ? `${baseUrl}/${known}` : ""
|
|
14173
|
+
};
|
|
14174
|
+
}
|
|
14175
|
+
if (token) tokens.set(id, token);
|
|
14176
|
+
if (request.responseToSend) responses.set(id, request.responseToSend);
|
|
14177
|
+
const due = request.type === "DELAY" ? resumeAt : expiresAt;
|
|
14178
|
+
if (due) this.waitpointPoller.nudge(due);
|
|
14179
|
+
return {
|
|
14180
|
+
id,
|
|
14181
|
+
resumeUrl: token ? `${baseUrl}/${token}` : ""
|
|
14182
|
+
};
|
|
14183
|
+
}
|
|
14184
|
+
},
|
|
14185
|
+
responseFor: (waitpointId) => responses.get(waitpointId)
|
|
14186
|
+
};
|
|
14187
|
+
}
|
|
14188
|
+
/** Re-enters a WAITING run that holds a COMPLETED waitpoint, through the
|
|
14189
|
+
* dispatcher. Whoever writes second, the resume or the WAITING, delivers. */
|
|
14190
|
+
async deliverHeld(runId) {
|
|
14191
|
+
if (this.closed) return this.answerShutDown(runId);
|
|
14192
|
+
if (this.resuming.has(runId)) return;
|
|
14193
|
+
try {
|
|
14194
|
+
const store = await this.store();
|
|
14195
|
+
if (!store || !await store.hasCompletedWaitpoint(runId)) return;
|
|
14196
|
+
const run = await store.getRun(runId);
|
|
14197
|
+
if (run?.status !== "WAITING") return this.answerNotWaiting(store, runId, run?.status);
|
|
14198
|
+
if (this.resuming.has(runId)) return;
|
|
14199
|
+
this.resuming.add(runId);
|
|
14200
|
+
this.resumeRun(runId, run.workflow_id);
|
|
14201
|
+
} catch (error) {
|
|
14202
|
+
this.logger.warn(`Could not resume run ${runId}: @error`, error);
|
|
14203
|
+
}
|
|
14204
|
+
}
|
|
14205
|
+
answerNotWaiting(store, runId, status) {
|
|
14206
|
+
if (store.isRunInFlight(runId)) return;
|
|
14207
|
+
this.answerUnresumed(runId, {
|
|
14208
|
+
status: status ?? "gone",
|
|
14209
|
+
error: "The run is not waiting to resume in this runtime",
|
|
14210
|
+
code: 409
|
|
14211
|
+
});
|
|
14212
|
+
}
|
|
14213
|
+
waitpointPoller = new WaitpointPoller({
|
|
14214
|
+
store: () => this.store(),
|
|
14215
|
+
resume: (runId) => this.deliverHeld(runId),
|
|
14216
|
+
ended: (end) => Promise.resolve(this.answerClosed(end.closed, end.runId, {
|
|
14217
|
+
status: CANCELLED_RUN_STATUS,
|
|
14218
|
+
error: end.reason
|
|
14219
|
+
})),
|
|
14220
|
+
logger: { warn: (message, ...args) => this.logger.warn(message, ...args) }
|
|
14221
|
+
});
|
|
14222
|
+
unobserveSuspended;
|
|
14223
|
+
/** Test hook: one waitpoint poll pass now; a no-op until the supervisor starts. */
|
|
14224
|
+
pollWaitpointsNow() {
|
|
14225
|
+
return this.waitpointPoller.poll();
|
|
14226
|
+
}
|
|
14227
|
+
startWaitpointPoller() {
|
|
14228
|
+
if (this.closed) return;
|
|
14229
|
+
this.unobserveSuspended ??= this.telemetry.observeSuspended(() => this.waitpointPoller.suspendedRuns());
|
|
14230
|
+
this.waitpointPoller.start();
|
|
14231
|
+
}
|
|
14232
|
+
stopWaitpointPoller() {
|
|
14233
|
+
this.waitpointPoller.stop();
|
|
14234
|
+
this.unobserveSuspended?.();
|
|
14235
|
+
this.unobserveSuspended = void 0;
|
|
14236
|
+
}
|
|
14237
|
+
async answerHeldResumes(store, runId, error) {
|
|
14238
|
+
if (this.syncResumes.size === 0) return;
|
|
14239
|
+
const rows = (await store.listWaitpointsForRuns([runId])).get(runId) ?? [];
|
|
14240
|
+
this.answerClosed(rows.map((row) => row.id), runId, {
|
|
14241
|
+
status: CANCELLED_RUN_STATUS,
|
|
14242
|
+
error
|
|
14243
|
+
});
|
|
14244
|
+
}
|
|
14245
|
+
answerClosed(waitpointIds, runId, end) {
|
|
14246
|
+
if (this.syncResumes.size === 0) return;
|
|
14247
|
+
for (const id of waitpointIds) this.heldResume(id)?.closed(runId, end.status, end.error);
|
|
14248
|
+
}
|
|
14249
|
+
heldResume(waitpointId) {
|
|
14250
|
+
const candidates = [...this.syncResumes.get(waitpointId) ?? []];
|
|
14251
|
+
if (candidates.length === 0) return void 0;
|
|
14252
|
+
const winner = (async () => {
|
|
14253
|
+
for (const candidate of candidates) if (await candidate.won) return candidate.entry;
|
|
14254
|
+
})();
|
|
14255
|
+
return {
|
|
14256
|
+
respond: (response) => void winner.then((e) => e?.respond(response)),
|
|
14257
|
+
ended: (result) => void winner.then((e) => e?.ended(result)),
|
|
14258
|
+
closed: (runId, status, error, code) => void winner.then((e) => e?.closed(runId, status, error, code))
|
|
14259
|
+
};
|
|
14260
|
+
}
|
|
14261
|
+
answerUnresumed(runId, end) {
|
|
14262
|
+
for (const [id, candidates] of this.syncResumes) {
|
|
14263
|
+
if (![...candidates].some((candidate) => candidate.runId === runId)) continue;
|
|
14264
|
+
this.heldResume(id)?.closed(runId, end.status, end.error, end.code);
|
|
14265
|
+
}
|
|
14266
|
+
}
|
|
14267
|
+
async deliverAllHeld() {
|
|
14268
|
+
try {
|
|
14269
|
+
const store = await this.store();
|
|
14270
|
+
for (const runId of await store?.heldRunIds() ?? []) await this.deliverHeld(runId);
|
|
14271
|
+
} catch (error) {
|
|
14272
|
+
this.logger.warn("Could not deliver held resumes: @error", error);
|
|
14273
|
+
}
|
|
14274
|
+
}
|
|
14275
|
+
resumeRun(runId, workflowId) {
|
|
14276
|
+
const started = performance.now();
|
|
14277
|
+
const attributes = {
|
|
14278
|
+
"workflow.id": workflowId,
|
|
14279
|
+
"trigger.kind": "resume",
|
|
14280
|
+
"run.id": runId
|
|
14281
|
+
};
|
|
14282
|
+
return this.telemetry.span("workflow.run", attributes, async (span) => {
|
|
14283
|
+
let status = "REFUSED";
|
|
14284
|
+
try {
|
|
14285
|
+
const result = await this.runResumed(runId, workflowId);
|
|
14286
|
+
if (result) status = result.status;
|
|
14287
|
+
if (status === "FAILED") span.setStatus({
|
|
14288
|
+
code: SpanStatusCode.ERROR,
|
|
14289
|
+
message: status
|
|
14290
|
+
});
|
|
14291
|
+
return result;
|
|
14292
|
+
} finally {
|
|
14293
|
+
span.setAttribute("run.status", status);
|
|
14294
|
+
this.telemetry.recordRun(performance.now() - started, status, "resume");
|
|
14295
|
+
}
|
|
14296
|
+
});
|
|
14297
|
+
}
|
|
14298
|
+
async runResumed(runId, workflowId) {
|
|
14299
|
+
const admission = await this.runGate.admit(workflowId, await this.resumePolicy(workflowId), void 0, { resume: true });
|
|
14300
|
+
if (!admission.admitted) {
|
|
14301
|
+
this.resuming.delete(runId);
|
|
14302
|
+
this.answerShutDown(runId);
|
|
14303
|
+
return;
|
|
14304
|
+
}
|
|
14305
|
+
let dispatched;
|
|
14306
|
+
try {
|
|
14307
|
+
dispatched = await this.dispatcher().admit(workflowId);
|
|
14308
|
+
} catch {
|
|
14309
|
+
admission.release();
|
|
14310
|
+
this.resuming.delete(runId);
|
|
14311
|
+
this.answerShutDown(runId);
|
|
14312
|
+
return;
|
|
14313
|
+
}
|
|
14314
|
+
const store = await this.store();
|
|
14315
|
+
let claimed = false;
|
|
14316
|
+
let redeliver = false;
|
|
14317
|
+
let session;
|
|
14318
|
+
let held;
|
|
14319
|
+
let outcome;
|
|
14320
|
+
try {
|
|
14321
|
+
if (!store || this.closed) {
|
|
14322
|
+
this.answerShutDown(runId);
|
|
14323
|
+
return;
|
|
14324
|
+
}
|
|
14325
|
+
claimed = await store.claimWaitingRun(runId);
|
|
14326
|
+
this.resuming.delete(runId);
|
|
14327
|
+
if (!claimed) {
|
|
14328
|
+
this.answerNotWaiting(store, runId, (await store.getRun(runId))?.status);
|
|
14329
|
+
return;
|
|
14330
|
+
}
|
|
14331
|
+
const row = await store.getRun(runId);
|
|
14332
|
+
if (!row) throw new Error(`Run "${runId}" not found`);
|
|
14333
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14334
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14335
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
14336
|
+
const current = document.state.global;
|
|
14337
|
+
const stopped = await this.resumeStopped(workflowId, row.trigger_kind, current);
|
|
14338
|
+
if (stopped) {
|
|
14339
|
+
const closed = await store.endWait(runId, {
|
|
14340
|
+
status: CANCELLED_RUN_STATUS,
|
|
14341
|
+
reason: stopped.reason,
|
|
14342
|
+
errorName: stopped.errorName,
|
|
14343
|
+
waitpoints: "CANCELLED"
|
|
14344
|
+
});
|
|
14345
|
+
claimed = false;
|
|
14346
|
+
this.answerClosed(closed, runId, {
|
|
14347
|
+
status: CANCELLED_RUN_STATUS,
|
|
14348
|
+
error: stopped.reason
|
|
14349
|
+
});
|
|
14350
|
+
return {
|
|
14351
|
+
status: CANCELLED_RUN_STATUS,
|
|
14352
|
+
steps: [],
|
|
14353
|
+
runId,
|
|
14354
|
+
error: stopped.reason,
|
|
14355
|
+
errorName: stopped.errorName,
|
|
14356
|
+
refusal: "cancelled"
|
|
14357
|
+
};
|
|
14358
|
+
}
|
|
14359
|
+
const consumed = await store.consumeCompletedWaitpoint(runId);
|
|
14360
|
+
if (!consumed) {
|
|
14361
|
+
await store.returnToWaiting(runId);
|
|
14362
|
+
claimed = false;
|
|
14363
|
+
redeliver = true;
|
|
14364
|
+
return;
|
|
14365
|
+
}
|
|
14366
|
+
const { waitpoint } = consumed;
|
|
14367
|
+
held = this.heldResume(waitpoint.id);
|
|
14368
|
+
this.answerClosed(consumed.closed, runId, {
|
|
14369
|
+
status: "RUNNING",
|
|
14370
|
+
error: "The run resumed on another of its waitpoints"
|
|
14371
|
+
});
|
|
14372
|
+
const state = await this.pinnedState(workflowId, runId, store, current);
|
|
14373
|
+
const runnable = runnableDefinition(state);
|
|
14374
|
+
const definition = toWorkflowDefinition(state);
|
|
14375
|
+
const policy = effectiveRunPolicy(runnable);
|
|
14376
|
+
const connections = declaredConnectionIds(definition);
|
|
14377
|
+
const reactorConnections = declaredReactorConnectionIds(definition);
|
|
14378
|
+
const runUser = reactorConnections.size === 0 ? void 0 : await assertReactorConnectionsReadable(this.host, workflowId, reactorConnections);
|
|
14379
|
+
const journal = carriedFromJournal(await store.getSteps(runId));
|
|
14380
|
+
if (!journal.waitingStepId) throw new Error(`Run "${runId}" has no waiting step to resume`);
|
|
14381
|
+
session = this.workers().session();
|
|
14382
|
+
const { result } = await this.executeLeg({
|
|
14383
|
+
workflowId,
|
|
14384
|
+
runId,
|
|
14385
|
+
store,
|
|
14386
|
+
session,
|
|
14387
|
+
definition,
|
|
14388
|
+
runnable,
|
|
14389
|
+
policy,
|
|
14390
|
+
deadline: policy.runTimeoutSeconds ? Date.now() + policy.runTimeoutSeconds * 1e3 : void 0,
|
|
14391
|
+
connections,
|
|
14392
|
+
runUser,
|
|
14393
|
+
triggerPayload: journaledTriggerPayload(row.trigger_payload),
|
|
14394
|
+
executor: this.blockExecutor(store),
|
|
14395
|
+
carriedSteps: journal.carried,
|
|
14396
|
+
resume: {
|
|
14397
|
+
stepId: journal.waitingStepId,
|
|
14398
|
+
payload: parseJson(waitpoint.resume_payload) ?? {}
|
|
14399
|
+
},
|
|
14400
|
+
firstOrdinal: journal.nextOrdinal,
|
|
14401
|
+
written: journal.attachmentRefs,
|
|
14402
|
+
handed: await store.getRunDocuments(runId),
|
|
14403
|
+
...held ? { respond: held.respond } : {}
|
|
14404
|
+
});
|
|
14405
|
+
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(row.trigger_kind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
14406
|
+
outcome = {
|
|
14407
|
+
...result,
|
|
14408
|
+
runId
|
|
14409
|
+
};
|
|
14410
|
+
return outcome;
|
|
14411
|
+
} catch (error) {
|
|
14412
|
+
if (store && claimed) {
|
|
14413
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
14414
|
+
const errorName = errorNameOf(error) ?? "Error";
|
|
14415
|
+
const closed = await store.endWait(runId, {
|
|
14416
|
+
status: "FAILED",
|
|
14417
|
+
reason: message,
|
|
14418
|
+
errorName,
|
|
14419
|
+
waitpoints: "CANCELLED"
|
|
14420
|
+
}).catch(() => []);
|
|
14421
|
+
this.answerClosed(closed, runId, {
|
|
14422
|
+
status: "FAILED",
|
|
14423
|
+
error: message
|
|
14424
|
+
});
|
|
14425
|
+
outcome = {
|
|
14426
|
+
status: "FAILED",
|
|
14427
|
+
steps: [],
|
|
14428
|
+
runId,
|
|
14429
|
+
error: message,
|
|
14430
|
+
errorName
|
|
14431
|
+
};
|
|
14432
|
+
}
|
|
14433
|
+
this.logger.warn(`Resuming run ${runId} failed: @error`, error);
|
|
14434
|
+
return;
|
|
14435
|
+
} finally {
|
|
14436
|
+
this.resuming.delete(runId);
|
|
14437
|
+
session?.close();
|
|
14438
|
+
dispatched();
|
|
14439
|
+
admission.release();
|
|
14440
|
+
held?.ended(outcome);
|
|
14441
|
+
if (redeliver) this.deliverHeld(runId);
|
|
14442
|
+
}
|
|
14443
|
+
}
|
|
14444
|
+
answerShutDown(runId) {
|
|
14445
|
+
if (!this.closed) return;
|
|
14446
|
+
this.answerUnresumed(runId, {
|
|
14447
|
+
status: WAITING_RUN_STATUS,
|
|
14448
|
+
error: "The runtime shut down before the run could resume",
|
|
14449
|
+
code: 503
|
|
14450
|
+
});
|
|
14451
|
+
}
|
|
14452
|
+
async resumePolicy(workflowId) {
|
|
14453
|
+
try {
|
|
14454
|
+
let state = this.workflowCache.get(workflowId)?.state;
|
|
14455
|
+
if (!state) {
|
|
14456
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14457
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14458
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
14459
|
+
state = document.state.global;
|
|
14460
|
+
}
|
|
14461
|
+
return effectiveRunPolicy(runnableDefinition(state));
|
|
14462
|
+
} catch {
|
|
14463
|
+
return UNENFORCED_POLICY;
|
|
14464
|
+
}
|
|
14465
|
+
}
|
|
14466
|
+
async resumeStopped(workflowId, triggerKind, current) {
|
|
14467
|
+
if (current.status !== "ENABLED") return {
|
|
14468
|
+
reason: `The workflow is ${current.status}; a run waiting on it does not resume`,
|
|
14469
|
+
errorName: WORKFLOW_DISABLED_ERROR_NAME
|
|
14470
|
+
};
|
|
14471
|
+
const parked = await this.parkedFiring(workflowId, triggerKind, runnableDefinition(current).version);
|
|
14472
|
+
return parked ? {
|
|
14473
|
+
reason: parked,
|
|
14474
|
+
errorName: WORKFLOW_PARKED_ERROR_NAME
|
|
14475
|
+
} : void 0;
|
|
14476
|
+
}
|
|
14477
|
+
async pinnedState(workflowId, runId, store, current) {
|
|
14478
|
+
const pin = await store.getRunPin(runId);
|
|
14479
|
+
if (!pin?.definition_hash) return current;
|
|
14480
|
+
if (this.definitionHash(workflowId, current) === pin.definition_hash) return current;
|
|
14481
|
+
return this.rebuildWorkflowState(workflowId, pin);
|
|
14482
|
+
}
|
|
14483
|
+
async rebuildWorkflowState(workflowId, pin) {
|
|
14484
|
+
const revision = pin.definition_op_index ?? 0;
|
|
14485
|
+
const operations = [];
|
|
14486
|
+
let page = await this.host.reactorClient.getOperations(workflowId, { scopes: ["document", "global"] });
|
|
14487
|
+
operations.push(...page.results);
|
|
14488
|
+
while (page.next) {
|
|
14489
|
+
page = await page.next();
|
|
14490
|
+
operations.push(...page.results);
|
|
14491
|
+
}
|
|
14492
|
+
const applied = (operation) => !operation.error && !isDenied(operation);
|
|
14493
|
+
const past = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index >= revision)).at(0);
|
|
14494
|
+
const atOrBeforePin = (operation) => {
|
|
14495
|
+
const at = operation.action.input.revision?.global;
|
|
14496
|
+
if (at !== void 0) return at <= revision;
|
|
14497
|
+
return !past || operation.timestampUtcMs <= past.timestampUtcMs;
|
|
14498
|
+
};
|
|
14499
|
+
const seed = sortOperations(operations.filter((operation) => operation.action.scope === "document" && operation.action.type === "UPGRADE_DOCUMENT" && applied(operation) && atOrBeforePin(operation))).map((operation) => operation.action.input.initialState).filter((state) => state?.global !== void 0).at(-1);
|
|
14500
|
+
let document = utils.createDocument(seed);
|
|
14501
|
+
const global = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index < revision));
|
|
14502
|
+
for (const operation of garbageCollect(global).filter(applied)) document = reducer(document, operation.action);
|
|
14503
|
+
const state = document.state.global;
|
|
14504
|
+
if (scopeHash(state) !== pin.definition_scope_hash || this.definitionHash(workflowId, state) !== pin.definition_hash) throw new Error(`The workflow was re-published while this run waited, and the definition it started from could not be rebuilt at revision ${revision}`);
|
|
14505
|
+
return state;
|
|
14506
|
+
}
|
|
14507
|
+
/** Resolves a WEBHOOK waitpoint from the editor, as the calling DID. */
|
|
14508
|
+
async resolveWaitpoint(waitpointId, input, ctx) {
|
|
14509
|
+
const store = await this.store();
|
|
14510
|
+
if (!store) throw new Error("Run journal is unavailable");
|
|
14511
|
+
const waitpoint = await store.getWaitpoint(waitpointId);
|
|
14512
|
+
if (!waitpoint) throw new Error(`Waitpoint "${waitpointId}" not found`);
|
|
14513
|
+
await this.assertCanReadDocument(waitpoint.workflow_id, ctx);
|
|
14514
|
+
await this.assertCanWriteDocument(waitpoint.workflow_id, ctx);
|
|
14515
|
+
const owner = await store.getRun(waitpoint.run_id);
|
|
14516
|
+
if (!owner || (await this.servedRuns([owner], ctx)).length === 0) throw new Error(`Waitpoint "${waitpointId}" not found`);
|
|
14517
|
+
if (waitpoint.type !== "WEBHOOK") throw new Error(`Waitpoint "${waitpointId}" is a ${waitpoint.type}`);
|
|
14518
|
+
const decidedBy = this.callerDid(ctx);
|
|
14519
|
+
const approval = parseJson(waitpoint.approval);
|
|
14520
|
+
if (approval && approval.approvers.length > 0) {
|
|
14521
|
+
if (!(decidedBy !== void 0 && approval.approvers.some((approver) => sameApprover(approver, decidedBy)))) throw new Error("You are not one of this approval's approvers");
|
|
14522
|
+
}
|
|
14523
|
+
const reason = input.reason?.trim() || void 0;
|
|
14524
|
+
if ((approval?.reasonRequiredOn === "both" || approval?.reasonRequiredOn === "reject" && !input.approved) && !reason) throw new Error("A reason is required for this decision");
|
|
14525
|
+
const decidedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14526
|
+
const payload = {
|
|
14527
|
+
queryParams: {
|
|
14528
|
+
action: input.approved ? "approve" : "disapprove",
|
|
14529
|
+
...reason ? { reason } : {}
|
|
14530
|
+
},
|
|
14531
|
+
body: {
|
|
14532
|
+
approved: input.approved,
|
|
14533
|
+
reason: reason ?? null,
|
|
14534
|
+
decidedBy: decidedBy ?? null,
|
|
14535
|
+
decidedAt
|
|
14536
|
+
},
|
|
14537
|
+
headers: {}
|
|
14538
|
+
};
|
|
14539
|
+
const completed = await store.completeWaitpoint(waitpointId, {
|
|
14540
|
+
payload,
|
|
14541
|
+
decidedBy: decidedBy ?? null,
|
|
14542
|
+
resolvedAt: decidedAt
|
|
14543
|
+
});
|
|
14544
|
+
if (!completed) {
|
|
14545
|
+
const now = await store.getWaitpoint(waitpointId);
|
|
14546
|
+
const run = await store.getRun(waitpoint.run_id);
|
|
14547
|
+
const live = run?.status === "RUNNING" || run?.status === "WAITING";
|
|
14548
|
+
const status = now?.status !== "PENDING" ? now?.status ?? "gone" : waitpointExpired(now) ? "EXPIRED" : live ? "not the one its run waits on" : `on a ${run?.status ?? "deleted"} run`;
|
|
14549
|
+
throw new Error(`Waitpoint "${waitpointId}" is ${status}; it can no longer be resolved`);
|
|
14550
|
+
}
|
|
14551
|
+
await this.deliverHeld(completed.run_id);
|
|
14552
|
+
return waitpointRecord(completed);
|
|
14553
|
+
}
|
|
14554
|
+
/** A POST to a resume URL. Single-use: only a PENDING waitpoint resumes. */
|
|
14555
|
+
resumeWaitpointByToken(token, payload) {
|
|
14556
|
+
return this.resumeByToken(token, payload);
|
|
14557
|
+
}
|
|
14558
|
+
/** A POST to a resume URL's `/sync`, held like a sync webhook until the
|
|
14559
|
+
* resumed leg responds, waits, ends, or times out. The first answer wins. */
|
|
14560
|
+
async resumeWaitpointByTokenSync(token, payload, options = {}) {
|
|
14561
|
+
let hold;
|
|
14562
|
+
try {
|
|
14563
|
+
const { status } = await this.resumeByToken(token, payload, (id, runId) => {
|
|
14564
|
+
hold = this.holdSyncResume(id, runId);
|
|
14565
|
+
return hold.decided;
|
|
14566
|
+
});
|
|
14567
|
+
if (status !== "resumed" || !hold) return { status };
|
|
14568
|
+
return {
|
|
14569
|
+
status,
|
|
14570
|
+
reply: await hold.reply(options.timeoutMs ?? DELIVERY_TIMEOUT_MS)
|
|
14571
|
+
};
|
|
14572
|
+
} finally {
|
|
14573
|
+
hold?.release();
|
|
14574
|
+
}
|
|
14575
|
+
}
|
|
14576
|
+
holdSyncResume(waitpointId, runId) {
|
|
14577
|
+
let settle = () => void 0;
|
|
14578
|
+
const answer = new Promise((resolve) => {
|
|
14579
|
+
settle = resolve;
|
|
14580
|
+
});
|
|
14581
|
+
let done = false;
|
|
14582
|
+
const finish = (reply) => {
|
|
14583
|
+
if (done) return;
|
|
14584
|
+
done = true;
|
|
14585
|
+
settle(reply);
|
|
14586
|
+
};
|
|
14587
|
+
const entry = {
|
|
14588
|
+
respond: (response) => finish(runResponseReply(response)),
|
|
14589
|
+
ended: (result) => finish(result ? runResultReply(result, 200) : {
|
|
14590
|
+
status: 500,
|
|
14591
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14592
|
+
body: JSON.stringify({ error: "The run could not resume" })
|
|
14593
|
+
}),
|
|
14594
|
+
closed: (runId, status, error, code = 409) => finish({
|
|
14595
|
+
status: code,
|
|
14596
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14597
|
+
body: JSON.stringify({
|
|
14598
|
+
runId,
|
|
14599
|
+
status,
|
|
14600
|
+
error
|
|
14601
|
+
})
|
|
14602
|
+
})
|
|
14603
|
+
};
|
|
14604
|
+
let decided = () => void 0;
|
|
14605
|
+
const candidate = {
|
|
14606
|
+
entry,
|
|
14607
|
+
runId,
|
|
14608
|
+
won: new Promise((resolve) => {
|
|
14609
|
+
decided = resolve;
|
|
14610
|
+
})
|
|
14611
|
+
};
|
|
14612
|
+
const held = this.syncResumes.get(waitpointId) ?? /* @__PURE__ */ new Set();
|
|
14613
|
+
this.syncResumes.set(waitpointId, held);
|
|
14614
|
+
held.add(candidate);
|
|
14615
|
+
return {
|
|
14616
|
+
decided,
|
|
14617
|
+
reply: async (timeoutMs) => {
|
|
14618
|
+
const raced = await Promise.race([answer, timeout(timeoutMs)]);
|
|
14619
|
+
if (raced !== TIMED_OUT) return raced;
|
|
14620
|
+
done = true;
|
|
14621
|
+
return {
|
|
14622
|
+
status: 504,
|
|
14623
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14624
|
+
body: JSON.stringify({
|
|
14625
|
+
status: "RUNNING",
|
|
14626
|
+
error: "The run did not finish in time"
|
|
14627
|
+
})
|
|
14628
|
+
};
|
|
14629
|
+
},
|
|
14630
|
+
release: () => {
|
|
14631
|
+
decided(false);
|
|
14632
|
+
held.delete(candidate);
|
|
14633
|
+
if (held.size === 0 && this.syncResumes.get(waitpointId) === held) this.syncResumes.delete(waitpointId);
|
|
14634
|
+
}
|
|
14635
|
+
};
|
|
14636
|
+
}
|
|
14637
|
+
async resumeByToken(token, payload, beforeComplete) {
|
|
14638
|
+
const store = await this.store();
|
|
14639
|
+
if (!store || !token) return { status: "gone" };
|
|
14640
|
+
const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
|
|
14641
|
+
if (!waitpoint || waitpoint.kind !== "webhook") return { status: "gone" };
|
|
14642
|
+
let completed;
|
|
14643
|
+
if (waitpoint.status === "PENDING") {
|
|
14644
|
+
const decided = beforeComplete?.(waitpoint.id, waitpoint.run_id);
|
|
14645
|
+
try {
|
|
14646
|
+
completed = await store.completeWaitpoint(waitpoint.id, { payload });
|
|
14647
|
+
} finally {
|
|
14648
|
+
decided?.(completed !== void 0);
|
|
14649
|
+
}
|
|
14650
|
+
}
|
|
14651
|
+
if (!completed) {
|
|
14652
|
+
const now = (await store.getWaitpoint(waitpoint.id))?.status;
|
|
14653
|
+
if (now === "CONSUMED") return { status: "already" };
|
|
14654
|
+
const run = now === "COMPLETED" ? await store.getRun(waitpoint.run_id) : void 0;
|
|
14655
|
+
return { status: run?.status === "RUNNING" || run?.status === "WAITING" ? "already" : "gone" };
|
|
14656
|
+
}
|
|
14657
|
+
await this.deliverHeld(completed.run_id);
|
|
14658
|
+
return { status: "resumed" };
|
|
14659
|
+
}
|
|
14660
|
+
/** What a resume URL's confirm page shows. */
|
|
14661
|
+
async describeWaitpointByToken(token) {
|
|
14662
|
+
const store = await this.store();
|
|
14663
|
+
if (!store || !token) return void 0;
|
|
14664
|
+
const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
|
|
14665
|
+
if (!waitpoint || waitpoint.kind !== "webhook") return void 0;
|
|
14666
|
+
const run = await store.getRun(waitpoint.run_id);
|
|
14667
|
+
const status = !(run?.status === "RUNNING" || run?.status === "WAITING") && (waitpoint.status === "PENDING" || waitpoint.status === "COMPLETED") ? "CANCELLED" : waitpoint.status;
|
|
14668
|
+
const state = await this.workflowStateForPage(waitpoint.workflow_id);
|
|
14669
|
+
const step = state?.steps.find((entry) => entry.id === waitpoint.step_id);
|
|
14670
|
+
const action = step ? await this.blockDisplayName({
|
|
14671
|
+
pieceName: step.pieceName,
|
|
14672
|
+
pieceVersion: step.pieceVersion,
|
|
14673
|
+
kind: "action",
|
|
14674
|
+
name: step.actionName
|
|
14675
|
+
}) : void 0;
|
|
14676
|
+
const trigger = state?.trigger ? await this.blockDisplayName({
|
|
14677
|
+
pieceName: state.trigger.pieceName,
|
|
14678
|
+
pieceVersion: state.trigger.pieceVersion,
|
|
14679
|
+
kind: "trigger",
|
|
14680
|
+
name: state.trigger.triggerName
|
|
14681
|
+
}) ?? state.trigger.triggerName : void 0;
|
|
14682
|
+
return {
|
|
14683
|
+
workflowName: run?.workflow_name || state?.name || "",
|
|
14684
|
+
stepName: step?.name || waitpoint.step_key,
|
|
14685
|
+
status,
|
|
14686
|
+
...state?.description ? { workflowDescription: state.description } : {},
|
|
14687
|
+
...action ? { actionName: action } : {},
|
|
14688
|
+
waitingSince: waitpoint.created_at,
|
|
14689
|
+
...waitpoint.expires_at ? { expiresAt: waitpoint.expires_at } : {},
|
|
14690
|
+
...run ? { startedAt: run.enqueued_at } : {},
|
|
14691
|
+
...trigger ? { triggerName: trigger } : {}
|
|
14692
|
+
};
|
|
14693
|
+
}
|
|
14694
|
+
async workflowStateForPage(workflowId) {
|
|
14695
|
+
const cached = this.workflowCache.get(workflowId)?.state;
|
|
14696
|
+
if (cached) return cached;
|
|
14697
|
+
try {
|
|
14698
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14699
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14700
|
+
if (document.header.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
|
|
14701
|
+
this.cacheLoaded(workflowId, generation, document);
|
|
14702
|
+
return document.state.global;
|
|
14703
|
+
} catch {
|
|
14704
|
+
return;
|
|
14705
|
+
}
|
|
14706
|
+
}
|
|
14707
|
+
async blockDisplayName(block) {
|
|
14708
|
+
try {
|
|
14709
|
+
const parsed = resolvedBlock(await this.resolveBlock(block));
|
|
14710
|
+
if (!parsed) return void 0;
|
|
14711
|
+
const descriptor = await this.pieceDescriptor(targetOf(parsed));
|
|
14712
|
+
return (block.kind === "trigger" ? descriptor.triggers : descriptor.actions).find((entry) => entry.name === parsed.name)?.displayName;
|
|
14713
|
+
} catch {
|
|
14714
|
+
return;
|
|
14715
|
+
}
|
|
14716
|
+
}
|
|
14717
|
+
/** The waitpoints of one run, for a caller who may read the run. */
|
|
14718
|
+
async waitpointsForRun(runId, ctx) {
|
|
14719
|
+
const store = await this.store();
|
|
14720
|
+
if (!store || !await this.run(runId, ctx)) return [];
|
|
14721
|
+
return ((await store.listWaitpointsForRuns([runId])).get(runId) ?? []).map(waitpointRecord);
|
|
14722
|
+
}
|
|
14723
|
+
/** WAITING runs this caller may read, oldest first, with their waitpoints. */
|
|
14724
|
+
async waitingRuns(args, ctx) {
|
|
14725
|
+
const store = await this.store();
|
|
14726
|
+
if (!store || !ctx) return [];
|
|
14727
|
+
let scope;
|
|
14728
|
+
if (args.workflowId) {
|
|
14729
|
+
await this.assertCanReadDocument(args.workflowId, ctx);
|
|
14730
|
+
scope = args.workflowId;
|
|
14731
|
+
} else if (args.driveId) {
|
|
14732
|
+
scope = await this.driveWorkflowIds(args.driveId, ctx);
|
|
14733
|
+
if (scope.length === 0) return [];
|
|
14734
|
+
}
|
|
14735
|
+
const rows = await this.servedRuns(await this.readableRows(await store.listWaitingRuns(scope, args.first ?? 50), (row) => row.workflow_id, ctx), ctx);
|
|
14736
|
+
const ids = rows.map((row) => row.id);
|
|
14737
|
+
const steps = await store.getStepsForRuns(ids, { withData: false });
|
|
14738
|
+
const waitpoints = await store.listWaitpointsForRuns(ids);
|
|
14739
|
+
return rows.map((row) => ({
|
|
14740
|
+
run: {
|
|
14741
|
+
row,
|
|
14742
|
+
steps: steps.get(row.id) ?? []
|
|
14743
|
+
},
|
|
14744
|
+
waitpoints: (waitpoints.get(row.id) ?? []).map(waitpointRecord)
|
|
14745
|
+
}));
|
|
14746
|
+
}
|
|
14747
|
+
/** Cancels a WAITING or PENDING run; its waitpoints stop resuming. */
|
|
14748
|
+
async cancelRun(runId, ctx) {
|
|
14749
|
+
const store = await this.store();
|
|
14750
|
+
if (!store) throw new Error("Run journal is unavailable");
|
|
14751
|
+
const run = await store.getRun(runId);
|
|
14752
|
+
if (!run) throw new Error(`Run "${runId}" not found`);
|
|
14753
|
+
await this.assertCanReadDocument(run.workflow_id, ctx);
|
|
14754
|
+
await this.assertCanWriteDocument(run.workflow_id, ctx);
|
|
14755
|
+
if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
|
|
14756
|
+
const refuse = (status) => status === "RUNNING" ? /* @__PURE__ */ new Error("Cancelling a RUNNING run is not supported yet (#3226)") : /* @__PURE__ */ new Error(`Run is ${status}; only WAITING and PENDING runs can be cancelled`);
|
|
14757
|
+
if (run.status !== "WAITING" && run.status !== "PENDING") throw refuse(run.status);
|
|
14758
|
+
const by = this.callerDid(ctx);
|
|
14759
|
+
const reason = by ? `Cancelled by ${by}` : "Cancelled";
|
|
14760
|
+
if (!await store.cancelIdleRun(runId, reason)) throw refuse((await store.getRun(runId))?.status ?? "gone");
|
|
14761
|
+
await this.answerHeldResumes(store, runId, reason);
|
|
14762
|
+
const row = await store.getRun(runId);
|
|
14763
|
+
this.queuedReplies.get(runId)?.({
|
|
14764
|
+
status: row.status,
|
|
14765
|
+
steps: [],
|
|
14766
|
+
runId,
|
|
14767
|
+
...row.error ? { error: row.error } : {},
|
|
14768
|
+
refusal: "cancelled"
|
|
14769
|
+
});
|
|
14770
|
+
return {
|
|
14771
|
+
row,
|
|
14772
|
+
steps: await store.getSteps(runId)
|
|
14773
|
+
};
|
|
14774
|
+
}
|
|
14775
|
+
callerDid(ctx) {
|
|
14776
|
+
if (!ctx) return void 0;
|
|
14777
|
+
const subject = this.host.subjectOf?.(ctx);
|
|
14778
|
+
if (subject?.address) return subject.address;
|
|
14779
|
+
const user = ctx.user;
|
|
14780
|
+
return typeof user?.address === "string" ? user.address : void 0;
|
|
14781
|
+
}
|
|
13377
14782
|
async testStep(workflowId, stepId, ctx, access = {}) {
|
|
13378
14783
|
await this.assertCanReadWorkflow(workflowId, ctx, access.driveId);
|
|
13379
14784
|
const document = await this.host.reactorClient.get(workflowId);
|
|
@@ -13682,6 +15087,112 @@ var WorkflowRuntimeService = class {
|
|
|
13682
15087
|
return (await this.host.reactorClient.get(workflowId)).state.global;
|
|
13683
15088
|
}
|
|
13684
15089
|
};
|
|
15090
|
+
function scopeHash(state) {
|
|
15091
|
+
return hashDocumentStateForScope({ state: { global: state } }, "global");
|
|
15092
|
+
}
|
|
15093
|
+
function tokenHash(token) {
|
|
15094
|
+
return createHash("sha256").update(token).digest("hex");
|
|
15095
|
+
}
|
|
15096
|
+
function waitpointExpired(waitpoint) {
|
|
15097
|
+
return waitpoint.expires_at !== null && Date.parse(waitpoint.expires_at) <= Date.now();
|
|
15098
|
+
}
|
|
15099
|
+
function parseJson(value) {
|
|
15100
|
+
return value === null ? null : JSON.parse(value);
|
|
15101
|
+
}
|
|
15102
|
+
const WORKFLOW_DISABLED_ERROR_NAME = "WorkflowDisabled";
|
|
15103
|
+
const WORKFLOW_PARKED_ERROR_NAME = "WorkflowParked";
|
|
15104
|
+
const MAX_WAITPOINT_ISO = "9999-12-31T23:59:59.999Z";
|
|
15105
|
+
const MAX_WAITPOINT_MS = Date.parse(MAX_WAITPOINT_ISO);
|
|
15106
|
+
function waitpointExpiry(createdAt, request, suspendMs) {
|
|
15107
|
+
const candidates = [request.expiresAt ? Date.parse(request.expiresAt) : NaN, suspendMs === null ? NaN : createdAt.getTime() + suspendMs].filter((at) => !Number.isNaN(at));
|
|
15108
|
+
return candidates.length > 0 ? new Date(Math.min(...candidates, MAX_WAITPOINT_MS)).toISOString() : null;
|
|
15109
|
+
}
|
|
15110
|
+
function sameApprover(approver, caller) {
|
|
15111
|
+
const tail = (value) => value.split(":").pop().toLowerCase();
|
|
15112
|
+
return approver.toLowerCase() === caller.toLowerCase() || tail(approver) === tail(caller);
|
|
15113
|
+
}
|
|
15114
|
+
function waitpointRecord(row) {
|
|
15115
|
+
return {
|
|
15116
|
+
id: row.id,
|
|
15117
|
+
runId: row.run_id,
|
|
15118
|
+
workflowId: row.workflow_id,
|
|
15119
|
+
stepId: row.step_id,
|
|
15120
|
+
stepKey: row.step_key,
|
|
15121
|
+
stepPath: row.step_path,
|
|
15122
|
+
pieceName: row.piece_name,
|
|
15123
|
+
blockName: row.block_name,
|
|
15124
|
+
type: row.type,
|
|
15125
|
+
kind: row.kind,
|
|
15126
|
+
status: row.status,
|
|
15127
|
+
createdAt: row.created_at,
|
|
15128
|
+
resumeAt: row.resume_at,
|
|
15129
|
+
expiresAt: row.expires_at,
|
|
15130
|
+
resolvedAt: row.resolved_at,
|
|
15131
|
+
decidedBy: row.decided_by,
|
|
15132
|
+
approval: parseJson(row.approval)
|
|
15133
|
+
};
|
|
15134
|
+
}
|
|
15135
|
+
function journaledValue(raw, what) {
|
|
15136
|
+
const value = raw === null ? void 0 : JSON.parse(raw);
|
|
15137
|
+
if (isTruncatedStepPayload(value)) return unavailableValue(`${what} is unavailable: the run journal truncated it past the payload cap`);
|
|
15138
|
+
return redactedLeavesUnavailable(value, what, []);
|
|
15139
|
+
}
|
|
15140
|
+
function redactedLeavesUnavailable(value, what, path) {
|
|
15141
|
+
if (!containsRedactedMarker(value)) return value;
|
|
15142
|
+
if (typeof value === "string") {
|
|
15143
|
+
const at = path.length > 0 ? ` at "${path.join(".")}"` : "";
|
|
15144
|
+
return unavailableValue(`${what} is unavailable: the run journal holds it redacted${at}`);
|
|
15145
|
+
}
|
|
15146
|
+
if (Array.isArray(value)) return value.map((entry, index) => redactedLeavesUnavailable(entry, what, [...path, index]));
|
|
15147
|
+
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactedLeavesUnavailable(entry, what, [...path, key])]));
|
|
15148
|
+
}
|
|
15149
|
+
function journaledTriggerPayload(raw) {
|
|
15150
|
+
return journaledValue(raw, "The trigger payload");
|
|
15151
|
+
}
|
|
15152
|
+
const ATTACHMENT_REF_TEXT = /"(attachment:\/\/[^"\\]*)"/g;
|
|
15153
|
+
function carriedFromJournal(rows) {
|
|
15154
|
+
const carried = /* @__PURE__ */ new Map();
|
|
15155
|
+
const attachmentRefs = /* @__PURE__ */ new Set();
|
|
15156
|
+
let waitingStepId;
|
|
15157
|
+
let nextOrdinal = 0;
|
|
15158
|
+
for (const row of rows) {
|
|
15159
|
+
nextOrdinal = Math.max(nextOrdinal, row.ordinal + 1);
|
|
15160
|
+
for (const match of row.output?.matchAll(ATTACHMENT_REF_TEXT) ?? []) attachmentRefs.add(match[1]);
|
|
15161
|
+
const what = `Output of step "${row.step_key}"`;
|
|
15162
|
+
switch (row.status) {
|
|
15163
|
+
case "WAITING":
|
|
15164
|
+
waitingStepId = row.step_id;
|
|
15165
|
+
break;
|
|
15166
|
+
case "SUCCEEDED":
|
|
15167
|
+
case "REPLAYED":
|
|
15168
|
+
carried.set(row.step_id, {
|
|
15169
|
+
status: row.status,
|
|
15170
|
+
port: row.port,
|
|
15171
|
+
output: () => journaledValue(row.output, what)
|
|
15172
|
+
});
|
|
15173
|
+
break;
|
|
15174
|
+
case "FAILED":
|
|
15175
|
+
carried.set(row.step_id, {
|
|
15176
|
+
status: "FAILED",
|
|
15177
|
+
port: "error",
|
|
15178
|
+
output: () => void 0,
|
|
15179
|
+
error: row.error ?? ""
|
|
15180
|
+
});
|
|
15181
|
+
break;
|
|
15182
|
+
case "SKIPPED": if (row.port) carried.set(row.step_id, {
|
|
15183
|
+
status: "SKIPPED",
|
|
15184
|
+
port: row.port,
|
|
15185
|
+
output: () => null
|
|
15186
|
+
});
|
|
15187
|
+
}
|
|
15188
|
+
}
|
|
15189
|
+
return {
|
|
15190
|
+
carried,
|
|
15191
|
+
waitingStepId,
|
|
15192
|
+
nextOrdinal,
|
|
15193
|
+
attachmentRefs
|
|
15194
|
+
};
|
|
15195
|
+
}
|
|
13685
15196
|
/** The runtime a host composes: one instance, its lifetime the host's. */
|
|
13686
15197
|
function createWorkflowRuntime(deps) {
|
|
13687
15198
|
return new WorkflowRuntimeService(deps);
|
|
@@ -13974,6 +15485,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
|
|
|
13974
15485
|
}
|
|
13975
15486
|
};
|
|
13976
15487
|
//#endregion
|
|
13977
|
-
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName };
|
|
15488
|
+
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WAITPOINT_EXPIRED_ERROR_NAME, WAITPOINT_NOT_LIVE_ERROR_NAME, WORKFLOW_DISABLED_ERROR_NAME, WORKFLOW_PACKAGE_NAME, WORKFLOW_PARKED_ERROR_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName, waitpointRecord };
|
|
13978
15489
|
|
|
13979
15490
|
//# sourceMappingURL=index.js.map
|