@powerhousedao/reactor-workflow 6.2.3-dev.83 → 6.2.3-dev.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +206 -38
- package/dist/index.d.ts +263 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1786 -171
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-ByALLpvA.d.ts → piece-registry-DNIk31Vv.d.ts} +82 -3
- package/dist/piece-registry-DNIk31Vv.d.ts.map +1 -0
- package/dist/{telemetry-5AAX-Pst.js → telemetry-B5OOpfCS.js} +708 -29
- package/dist/telemetry-B5OOpfCS.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1 -1
- package/dist/worker-entry.js +161 -16
- package/dist/worker-entry.js.map +1 -1
- package/package.json +7 -7
- package/dist/piece-registry-ByALLpvA.d.ts.map +0 -1
- package/dist/telemetry-5AAX-Pst.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as containsRedactedMarker, A as AttachmentCache, B as SecretCreatorMismatchError, C as blockLabel, Ct as isIndeterminateError, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as secretRefFromId, H as SecretNotFoundError, I as resolveStepInput, J as PieceWorkerPool, K as stepBlock, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as PieceWorkerTimeoutError, R as unavailableValue, S as storeHandlers, St as setPieceRegistryUrl, T as blockPorts, U as SecretOwnerMismatchError, V as SecretDeletedError, W as parseSecretRef, X as PieceWorkerDisposedError, Y as PieceWorker, Z as PieceWorkerError, _ as effectiveRetryPolicy, _t as unsupportedTrigger, a as PieceRegistry, at as isPrivateAddress, b as isHostBound, bt as npmRegistryBaseUrl, c as packagePieces, ct as FileTooLargeError, d as runWorkflow, dt as pieceModuleRef, et as redact, f as canonicalJson, ft as sourcedResolver, g as checkDynamicProperties, gt as unsupportedAuth, h as DynamicPropertiesError, ht as authMethodFor, i as tracedSteps, it as DEFAULT_EGRESS_POLICY, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as maxFileBytes, m as stepConfigHash, mt as UnsupportedPieceFeatureError, n as WorkflowTelemetry, nt as rememberSecrets, o as installedPiece, ot as parseAddress, p as hashOf, q as triggerBlock, r as tracedMethods, rt as secretsFor, s as installedPieces, st as extractDedupeKey, t as TELEMETRY_SCOPE, tt as redactMessage, u as errorNameOf, ut as bundleResolver, v as ActivepiecesBlockExecutor, w as unpinnedNote, x as resolvedBlock, xt as pieceRegistrySource, y as CompositeBlockExecutor } from "./telemetry-B5OOpfCS.js";
|
|
2
2
|
import { DENIED_ACTION_INDEXES, ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy, assertActionsApplied } from "@powerhousedao/pieces-framework";
|
|
3
3
|
import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { open, readFile, readdir, rm, stat } from "node:fs/promises";
|
|
5
5
|
import { dirname, join } from "node:path";
|
|
6
|
-
import { childLogger, generateId } from "document-model";
|
|
7
|
-
import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
|
|
6
|
+
import { childLogger, garbageCollect, generateId, hashDocumentStateForScope, isDenied, sortOperations } from "document-model";
|
|
7
|
+
import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, coerceManualPayload, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, manualInputTypeName, parseManualInputs, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
|
|
8
8
|
import { Cron } from "croner";
|
|
9
9
|
import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
10
10
|
import dns from "node:dns";
|
|
@@ -13,9 +13,9 @@ import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
13
13
|
import { hostname } from "node:os";
|
|
14
14
|
import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
|
|
15
15
|
import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
|
|
16
|
-
import http from "node:http";
|
|
16
|
+
import http, { validateHeaderName, validateHeaderValue } from "node:http";
|
|
17
17
|
import https from "node:https";
|
|
18
|
-
import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
|
|
18
|
+
import { actions as actions$1, reducer, utils } from "@powerhousedao/workflow/document-models/workflow";
|
|
19
19
|
import { Kind, parse } from "graphql";
|
|
20
20
|
import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
|
|
21
21
|
import { SpanStatusCode, context } from "@opentelemetry/api";
|
|
@@ -205,10 +205,11 @@ var OAuthError = class extends Error {
|
|
|
205
205
|
}
|
|
206
206
|
};
|
|
207
207
|
async function up$2(db) {
|
|
208
|
-
await db.schema.createTable("oauth_attempt").addColumn("state", "text", (col) => col.primaryKey()).addColumn("connection_id", "text", (col) => col.notNull()).addColumn("client_id", "text", (col) => col.notNull()).addColumn("client_secret_ref", "text"
|
|
208
|
+
await db.schema.createTable("oauth_attempt").addColumn("state", "text", (col) => col.primaryKey()).addColumn("connection_id", "text", (col) => col.notNull()).addColumn("client_id", "text", (col) => col.notNull()).addColumn("client_secret_ref", "text").addColumn("code_verifier", "text").addColumn("redirect_uri", "text", (col) => col.notNull()).addColumn("token_url", "text", (col) => col.notNull()).addColumn("authorization_method", "text").addColumn("return_url", "text").addColumn("created_by", "text").addColumn("status", "text", (col) => col.notNull()).addColumn("error", "text").addColumn("created_at", "text", (col) => col.notNull()).addColumn("expires_at", "text", (col) => col.notNull()).ifNotExists().execute();
|
|
209
209
|
try {
|
|
210
210
|
await db.schema.alterTable("oauth_attempt").addColumn("created_by", "text").execute();
|
|
211
211
|
} catch {}
|
|
212
|
+
await db.schema.alterTable("oauth_attempt").alterColumn("client_secret_ref", (col) => col.dropNotNull()).execute();
|
|
212
213
|
}
|
|
213
214
|
function fillOAuthUrl(template, props) {
|
|
214
215
|
const filled = template.replace(/\{(\w+)\}/g, (_match, name) => {
|
|
@@ -241,7 +242,8 @@ var OAuthAttemptStore = class OAuthAttemptStore {
|
|
|
241
242
|
const now = /* @__PURE__ */ new Date();
|
|
242
243
|
await this.db.deleteFrom("oauth_attempt").where("expires_at", "<", (/* @__PURE__ */ new Date(now.getTime() - 864e5)).toISOString()).execute();
|
|
243
244
|
const { method } = input;
|
|
244
|
-
const
|
|
245
|
+
const nonce = base64url(randomBytes(32));
|
|
246
|
+
const state = input.stateSuffix ? `${nonce}.${input.stateSuffix}` : nonce;
|
|
245
247
|
const verifier = method.pkce === true || input.pkce === true ? base64url(randomBytes(48)) : null;
|
|
246
248
|
const authUrl = new URL(fillOAuthUrl(method.authUrl, input.props));
|
|
247
249
|
const params = authUrl.searchParams;
|
|
@@ -268,7 +270,7 @@ var OAuthAttemptStore = class OAuthAttemptStore {
|
|
|
268
270
|
client_secret_ref: input.clientSecretRef,
|
|
269
271
|
code_verifier: verifier,
|
|
270
272
|
redirect_uri: input.redirectUri,
|
|
271
|
-
token_url: fillOAuthUrl(method.tokenUrl, input.props),
|
|
273
|
+
token_url: input.tokenUrl ?? fillOAuthUrl(method.tokenUrl, input.props),
|
|
272
274
|
authorization_method: method.authorizationMethod ?? null,
|
|
273
275
|
return_url: input.returnUrl ?? null,
|
|
274
276
|
created_by: input.createdBy?.toLowerCase() ?? null,
|
|
@@ -395,6 +397,10 @@ async function postTokenForm(url, form, headers, egress) {
|
|
|
395
397
|
return parsed;
|
|
396
398
|
}
|
|
397
399
|
function clientAuth$1(method, clientId, clientSecret) {
|
|
400
|
+
if (clientSecret === null) return {
|
|
401
|
+
form: { client_id: clientId },
|
|
402
|
+
headers: {}
|
|
403
|
+
};
|
|
398
404
|
if (method === "HEADER") return {
|
|
399
405
|
form: {},
|
|
400
406
|
headers: { authorization: `Basic ${Buffer.from(`${clientId}:${clientSecret}`).toString("base64")}` }
|
|
@@ -426,6 +432,7 @@ function tokenSetFrom(response, context, previous) {
|
|
|
426
432
|
redirect_url: context.redirect_url,
|
|
427
433
|
...context.client_id ?? previous?.client_id ? { client_id: context.client_id ?? previous?.client_id } : {},
|
|
428
434
|
...context.client_secret_ref ?? previous?.client_secret_ref ? { client_secret_ref: context.client_secret_ref ?? previous?.client_secret_ref } : {},
|
|
435
|
+
...context.relayed ?? previous?.relayed ? { relayed: true } : {},
|
|
429
436
|
data: {
|
|
430
437
|
...previous?.data,
|
|
431
438
|
...response
|
|
@@ -447,8 +454,9 @@ async function exchangeCode(attempt, code, clientSecret, egress, fromRegistry =
|
|
|
447
454
|
redirect_url: attempt.redirect_uri,
|
|
448
455
|
...fromRegistry ? {
|
|
449
456
|
client_id: attempt.client_id,
|
|
450
|
-
client_secret_ref: attempt.client_secret_ref
|
|
451
|
-
} : {}
|
|
457
|
+
...attempt.client_secret_ref ? { client_secret_ref: attempt.client_secret_ref } : {}
|
|
458
|
+
} : {},
|
|
459
|
+
...clientSecret === null ? { relayed: true } : {}
|
|
452
460
|
});
|
|
453
461
|
}
|
|
454
462
|
function parseTokenSet(value) {
|
|
@@ -499,10 +507,16 @@ var StoreTokenRefresher = class {
|
|
|
499
507
|
const read = (ref) => store.get(ref, owner);
|
|
500
508
|
const tokens = parseTokenSet(await read(tokenRef));
|
|
501
509
|
if (!tokenDue(tokens, Math.round(Date.now() / 1e3))) return;
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
510
|
+
let auth;
|
|
511
|
+
if (tokens.relayed) {
|
|
512
|
+
if (!tokens.client_id) return;
|
|
513
|
+
auth = clientAuth$1(void 0, tokens.client_id, null);
|
|
514
|
+
} else {
|
|
515
|
+
const clientId = own?.clientId ?? tokens.client_id;
|
|
516
|
+
const secretRef = own?.secretRef ?? tokens.client_secret_ref;
|
|
517
|
+
if (!clientId || !secretRef) return;
|
|
518
|
+
auth = clientAuth$1(tokens.authorization_method, clientId, own ? await read(secretRef) : await store.get(secretRef, null));
|
|
519
|
+
}
|
|
506
520
|
let response;
|
|
507
521
|
try {
|
|
508
522
|
response = await postTokenForm(tokens.token_url, {
|
|
@@ -779,6 +793,9 @@ function webhookTriggerTree() {
|
|
|
779
793
|
leaf("body", "Unknown", "Parsed JSON or form fields; text otherwise")
|
|
780
794
|
];
|
|
781
795
|
}
|
|
796
|
+
function manualTriggerTree(inputs) {
|
|
797
|
+
return inputs.map((input) => leaf(input.name, manualInputTypeName(input), input.description));
|
|
798
|
+
}
|
|
782
799
|
function documentEventTree(actionInputChildren) {
|
|
783
800
|
return [
|
|
784
801
|
leaf("documentId", "PHID!"),
|
|
@@ -1538,6 +1555,9 @@ function currentStepTest() {
|
|
|
1538
1555
|
function currentRunUser() {
|
|
1539
1556
|
return storage.getStore()?.runUser;
|
|
1540
1557
|
}
|
|
1558
|
+
function currentWaitpoints() {
|
|
1559
|
+
return storage.getStore()?.waitpoints;
|
|
1560
|
+
}
|
|
1541
1561
|
//#endregion
|
|
1542
1562
|
//#region src/reactor/piece-store-port.ts
|
|
1543
1563
|
const logger$5 = childLogger(["workflow", "piece-store"]);
|
|
@@ -1733,6 +1753,7 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
|
|
|
1733
1753
|
egress: configuredEgress(),
|
|
1734
1754
|
worker: currentPieceWorker,
|
|
1735
1755
|
stepTest: currentStepTest,
|
|
1756
|
+
waitpoints: currentWaitpoints,
|
|
1736
1757
|
identity: () => ({
|
|
1737
1758
|
runId: currentRunId(),
|
|
1738
1759
|
flowId: currentWorkflowId(),
|
|
@@ -2674,6 +2695,10 @@ const SCHEDULE_BLOCK = trigger("schedule");
|
|
|
2674
2695
|
const WEBHOOK_BLOCK = trigger("webhook");
|
|
2675
2696
|
const BRANCH_BLOCK = action("branch");
|
|
2676
2697
|
const ASSERT_BLOCK = action("assert");
|
|
2698
|
+
const DELAY_BLOCK = action("delay");
|
|
2699
|
+
const APPROVAL_BLOCK = action("approval");
|
|
2700
|
+
const STOP_BLOCK = action("stop");
|
|
2701
|
+
const WAIT_FOR_WEBHOOK_BLOCK = action("waitForWebhook");
|
|
2677
2702
|
function isCoreBlock(block) {
|
|
2678
2703
|
return block.pieceName === CORE_PIECE_NAME;
|
|
2679
2704
|
}
|
|
@@ -7663,6 +7688,16 @@ const PARKED_TRIGGER_STATUS = "PARKED";
|
|
|
7663
7688
|
* not finish and nothing is wrong with the workflow", which is what the
|
|
7664
7689
|
* document model's RunStatus.CANCELLED means. */
|
|
7665
7690
|
const CANCELLED_RUN_STATUS = "CANCELLED";
|
|
7691
|
+
/** How long a run may stay suspended; null when the policy sets no bound. */
|
|
7692
|
+
function maxSuspensionMs(runnable) {
|
|
7693
|
+
const policy = runnable.policy;
|
|
7694
|
+
const days = positive(policy?.maxSuspensionDays);
|
|
7695
|
+
return days === null ? null : days * 864e5;
|
|
7696
|
+
}
|
|
7697
|
+
//#endregion
|
|
7698
|
+
//#region src/reactor/waitpoint-types.ts
|
|
7699
|
+
const WAITPOINT_EXPIRED_ERROR_NAME = "WaitpointExpired";
|
|
7700
|
+
const WAITPOINT_NOT_LIVE_ERROR_NAME = "WaitpointNotLive";
|
|
7666
7701
|
//#endregion
|
|
7667
7702
|
//#region src/reactor/store.ts
|
|
7668
7703
|
function triggerRowBlock(row) {
|
|
@@ -7694,6 +7729,7 @@ function workflowParkRow(workflowId, publishedVersion, reason) {
|
|
|
7694
7729
|
parked_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7695
7730
|
};
|
|
7696
7731
|
}
|
|
7732
|
+
const WAITING_RUN_STATUS = "WAITING";
|
|
7697
7733
|
const logger$4 = childLogger([
|
|
7698
7734
|
"workflow",
|
|
7699
7735
|
"runtime",
|
|
@@ -7701,6 +7737,64 @@ const logger$4 = childLogger([
|
|
|
7701
7737
|
]);
|
|
7702
7738
|
const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
|
|
7703
7739
|
const PENDING_RUN_STATUS = "PENDING";
|
|
7740
|
+
const LIVE_RUN_STATUSES = [
|
|
7741
|
+
PENDING_RUN_STATUS,
|
|
7742
|
+
"RUNNING",
|
|
7743
|
+
WAITING_RUN_STATUS
|
|
7744
|
+
];
|
|
7745
|
+
const LIVE_WAITPOINT_STATUSES = ["PENDING", "COMPLETED"];
|
|
7746
|
+
async function closeLiveWaitpoints(db, runIds, status, resolvedAt, keep) {
|
|
7747
|
+
if (runIds.length === 0) return [];
|
|
7748
|
+
return (await db.updateTable("waitpoint").set({
|
|
7749
|
+
status,
|
|
7750
|
+
resolved_at: resolvedAt
|
|
7751
|
+
}).where("run_id", "in", runIds).where("status", "in", LIVE_WAITPOINT_STATUSES).$if(keep !== void 0, (qb) => qb.where((eb) => eb.not(keep(eb)))).returning("id").execute()).map((row) => row.id);
|
|
7752
|
+
}
|
|
7753
|
+
async function failWaitingSteps(db, runIds, reason, errorName, endedAt) {
|
|
7754
|
+
if (runIds.length === 0) return;
|
|
7755
|
+
await db.updateTable("step_execution").set({
|
|
7756
|
+
status: "FAILED",
|
|
7757
|
+
error: redactMessage(reason),
|
|
7758
|
+
error_name: errorName,
|
|
7759
|
+
ended_at: endedAt
|
|
7760
|
+
}).where("run_id", "in", runIds).where("status", "=", WAITING_RUN_STATUS).execute();
|
|
7761
|
+
}
|
|
7762
|
+
function waitpointExpiryReason(waitpoint) {
|
|
7763
|
+
return `The ${waitpoint.kind === "approval" ? "approval" : "wait for a resume"} at step "${waitpoint.step_key}" expired at ${waitpoint.expires_at} without being resolved`;
|
|
7764
|
+
}
|
|
7765
|
+
function strandedWait(eb) {
|
|
7766
|
+
const live = eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES);
|
|
7767
|
+
return eb.and([eb.not(eb.exists(completedOf(eb))), eb.or([eb.and([eb("run.waiting_on", "is not", null), eb.not(eb.exists(live.whereRef("waitpoint.id", "=", "run.waiting_on")))]), eb.and([eb("run.waiting_on", "is", null), eb.not(eb.exists(live.whereRef("waitpoint.run_id", "=", "run.id")))])])]);
|
|
7768
|
+
}
|
|
7769
|
+
function completedOf(eb) {
|
|
7770
|
+
return eb.selectFrom("waitpoint").select("waitpoint.id").whereRef("waitpoint.run_id", "=", "run.id").where("waitpoint.status", "=", "COMPLETED");
|
|
7771
|
+
}
|
|
7772
|
+
async function endWaitingRun(db, runId, end) {
|
|
7773
|
+
const { waitpointId } = end;
|
|
7774
|
+
if ((await db.updateTable("run").set({
|
|
7775
|
+
status: "CANCELLED",
|
|
7776
|
+
error: redactMessage(end.reason),
|
|
7777
|
+
error_name: end.errorName,
|
|
7778
|
+
ended_at: end.endedAt
|
|
7779
|
+
}).where("id", "=", runId).where("status", "=", "WAITING").where((eb) => waitpointId === void 0 ? strandedWait(eb) : eb.and([eb.or([eb("run.waiting_on", "=", waitpointId), eb("run.waiting_on", "is", null)]), eb.not(eb.exists(completedOf(eb)))])).returning("id").execute()).length === 0) return void 0;
|
|
7780
|
+
await failWaitingSteps(db, [runId], end.reason, end.errorName, end.endedAt);
|
|
7781
|
+
const closed = await closeLiveWaitpoints(db, [runId], "EXPIRED", end.endedAt);
|
|
7782
|
+
return {
|
|
7783
|
+
runId,
|
|
7784
|
+
reason: end.reason,
|
|
7785
|
+
closed
|
|
7786
|
+
};
|
|
7787
|
+
}
|
|
7788
|
+
function priorNotes(text) {
|
|
7789
|
+
if (!text) return [];
|
|
7790
|
+
try {
|
|
7791
|
+
const notes = JSON.parse(text);
|
|
7792
|
+
return Array.isArray(notes) ? notes.filter((note) => typeof note === "string") : [];
|
|
7793
|
+
} catch {
|
|
7794
|
+
return [];
|
|
7795
|
+
}
|
|
7796
|
+
}
|
|
7797
|
+
const STRANDED_WAIT_REASON = "The run waited on a waitpoint that is no longer live";
|
|
7704
7798
|
function errorMessage(error) {
|
|
7705
7799
|
return error instanceof Error ? error.message : String(error);
|
|
7706
7800
|
}
|
|
@@ -7802,6 +7896,31 @@ async function up(db) {
|
|
|
7802
7896
|
try {
|
|
7803
7897
|
await db.schema.dropTable("webhook_endpoint").ifExists().execute();
|
|
7804
7898
|
} catch {}
|
|
7899
|
+
for (const [column, type] of [
|
|
7900
|
+
["definition_hash", "text"],
|
|
7901
|
+
["definition_op_index", "integer"],
|
|
7902
|
+
["definition_scope_hash", "text"],
|
|
7903
|
+
["waiting_on", "text"]
|
|
7904
|
+
]) try {
|
|
7905
|
+
await db.schema.alterTable("run").addColumn(column, type).execute();
|
|
7906
|
+
} catch {}
|
|
7907
|
+
await db.schema.createTable("waitpoint").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("step_path", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("type", "text", (col) => col.notNull()).addColumn("kind", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("token_hash", "text").addColumn("created_at", "text", (col) => col.notNull()).addColumn("resume_at", "text").addColumn("expires_at", "text").addColumn("resolved_at", "text").addColumn("decided_by", "text").addColumn("approval", "text").addColumn("resume_payload", "text").addColumn("response_to_send", "text").ifNotExists().execute();
|
|
7908
|
+
await db.schema.createIndex("waitpoint_token").ifNotExists().unique().on("waitpoint").column("token_hash").execute();
|
|
7909
|
+
await db.schema.createIndex("waitpoint_status_resume").ifNotExists().on("waitpoint").columns(["status", "resume_at"]).execute();
|
|
7910
|
+
await db.schema.createIndex("waitpoint_status_expires").ifNotExists().on("waitpoint").columns(["status", "expires_at"]).execute();
|
|
7911
|
+
await db.schema.createIndex("waitpoint_run").ifNotExists().on("waitpoint").column("run_id").execute();
|
|
7912
|
+
const liveStep = db.schema.createIndex("waitpoint_live_step").ifNotExists().unique().on("waitpoint").columns(["run_id", "step_path"]).where(sql.ref("status"), "in", LIVE_WAITPOINT_STATUSES);
|
|
7913
|
+
try {
|
|
7914
|
+
await liveStep.execute();
|
|
7915
|
+
} catch {
|
|
7916
|
+
await db.updateTable("waitpoint").set({
|
|
7917
|
+
status: "CANCELLED",
|
|
7918
|
+
resolved_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7919
|
+
}).where("status", "in", LIVE_WAITPOINT_STATUSES).where((eb) => eb.and([eb.not(eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.waiting_on", "=", "waitpoint.id"))), eb.exists(eb.selectFrom("waitpoint as sibling").select("sibling.id").whereRef("sibling.run_id", "=", "waitpoint.run_id").whereRef("sibling.step_path", "=", "waitpoint.step_path").whereRef("sibling.id", "!=", "waitpoint.id").where("sibling.status", "in", LIVE_WAITPOINT_STATUSES))])).execute();
|
|
7920
|
+
await liveStep.execute();
|
|
7921
|
+
}
|
|
7922
|
+
await db.schema.createIndex("waitpoint_workflow_status").ifNotExists().on("waitpoint").columns(["workflow_id", "status"]).execute();
|
|
7923
|
+
await db.schema.createIndex("run_waiting").ifNotExists().on("run").columns(["workflow_id", "enqueued_at"]).where(sql.ref("status"), "=", WAITING_RUN_STATUS).execute();
|
|
7805
7924
|
await migrateBlockType(db, "trigger_state", "trigger_name");
|
|
7806
7925
|
await migrateBlockType(db, "step_execution", "block_name");
|
|
7807
7926
|
return migrateTriggerStoreState(db);
|
|
@@ -8160,6 +8279,8 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8160
8279
|
unmigrated;
|
|
8161
8280
|
runsInFlight = /* @__PURE__ */ new Set();
|
|
8162
8281
|
journaledSteps = /* @__PURE__ */ new Map();
|
|
8282
|
+
pausingRuns = /* @__PURE__ */ new Set();
|
|
8283
|
+
resumedLegs = /* @__PURE__ */ new Set();
|
|
8163
8284
|
constructor(db, unmigrated) {
|
|
8164
8285
|
this.db = db;
|
|
8165
8286
|
this.unmigrated = unmigrated;
|
|
@@ -8178,13 +8299,20 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8178
8299
|
this.unmigrated.delete(workflowId);
|
|
8179
8300
|
}
|
|
8180
8301
|
async recoverOrphanedRuns() {
|
|
8181
|
-
|
|
8182
|
-
|
|
8183
|
-
|
|
8184
|
-
|
|
8185
|
-
|
|
8186
|
-
|
|
8187
|
-
|
|
8302
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8303
|
+
const inFlight = [...this.runsInFlight];
|
|
8304
|
+
const recovered = await this.db.transaction().execute(async (trx) => {
|
|
8305
|
+
let query = trx.updateTable("run").set({
|
|
8306
|
+
status: "FAILED",
|
|
8307
|
+
error: ORPHANED_RUN_ERROR,
|
|
8308
|
+
ended_at: endedAt
|
|
8309
|
+
}).where("status", "=", "RUNNING");
|
|
8310
|
+
if (inFlight.length > 0) query = query.where("id", "not in", inFlight);
|
|
8311
|
+
const ids = (await query.returning("id").execute()).map((row) => row.id);
|
|
8312
|
+
await failWaitingSteps(trx, ids, ORPHANED_RUN_ERROR, null, endedAt);
|
|
8313
|
+
await closeLiveWaitpoints(trx, ids, "CANCELLED", endedAt);
|
|
8314
|
+
return ids.length;
|
|
8315
|
+
});
|
|
8188
8316
|
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
|
|
8189
8317
|
return recovered;
|
|
8190
8318
|
}
|
|
@@ -8248,14 +8376,25 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8248
8376
|
* CANCELLED run is not rerunnable and must not read as a workflow defect.
|
|
8249
8377
|
*/
|
|
8250
8378
|
async cancelRun(runId, reason) {
|
|
8379
|
+
return this.endLiveRun(runId, "CANCELLED", reason, null);
|
|
8380
|
+
}
|
|
8381
|
+
async endLiveRun(runId, status, reason, errorName) {
|
|
8251
8382
|
this.runsInFlight.delete(runId);
|
|
8252
8383
|
this.journaledSteps.delete(runId);
|
|
8253
|
-
|
|
8254
|
-
|
|
8255
|
-
|
|
8256
|
-
|
|
8257
|
-
|
|
8258
|
-
|
|
8384
|
+
this.pausingRuns.delete(runId);
|
|
8385
|
+
this.resumedLegs.delete(runId);
|
|
8386
|
+
if (erasedRuns.delete(runId)) return [];
|
|
8387
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
8388
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8389
|
+
if ((await trx.updateTable("run").set({
|
|
8390
|
+
status,
|
|
8391
|
+
error: redactMessage(reason),
|
|
8392
|
+
error_name: errorName,
|
|
8393
|
+
ended_at: endedAt
|
|
8394
|
+
}).where("id", "=", runId).where("status", "in", LIVE_RUN_STATUSES).returning("id").execute()).length === 0) return [];
|
|
8395
|
+
await failWaitingSteps(trx, [runId], reason, errorName, endedAt);
|
|
8396
|
+
return closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
|
|
8397
|
+
});
|
|
8259
8398
|
}
|
|
8260
8399
|
async enqueueRun(options) {
|
|
8261
8400
|
const id = randomUUID();
|
|
@@ -8463,12 +8602,15 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8463
8602
|
async beginRun(runId, details) {
|
|
8464
8603
|
this.runsInFlight.add(runId);
|
|
8465
8604
|
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
8466
|
-
await this.db.updateTable("run").set({
|
|
8605
|
+
if ((await this.db.updateTable("run").set({
|
|
8467
8606
|
status: "RUNNING",
|
|
8468
8607
|
workflow_name: details.workflowName,
|
|
8469
8608
|
workflow_version: details.workflowVersion,
|
|
8470
8609
|
started_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8471
|
-
}).where("id", "=", runId).execute();
|
|
8610
|
+
}).where("id", "=", runId).where("status", "=", "PENDING").returning("id").execute()).length > 0) return true;
|
|
8611
|
+
this.runsInFlight.delete(runId);
|
|
8612
|
+
this.journaledSteps.delete(runId);
|
|
8613
|
+
return false;
|
|
8472
8614
|
}
|
|
8473
8615
|
async startRun(options) {
|
|
8474
8616
|
const id = randomUUID();
|
|
@@ -8497,10 +8639,10 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8497
8639
|
if (erasedRuns.has(runId)) return;
|
|
8498
8640
|
const values = stepValues(runId, ordinal, step);
|
|
8499
8641
|
const { run_id: _run, step_id: _step, ...mutable } = values;
|
|
8500
|
-
await this.
|
|
8642
|
+
if (!await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values({
|
|
8501
8643
|
id: randomUUID(),
|
|
8502
8644
|
...values
|
|
8503
|
-
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
|
|
8645
|
+
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute())) return;
|
|
8504
8646
|
this.journaledSteps.get(runId)?.set(step.stepId, {
|
|
8505
8647
|
ordinal,
|
|
8506
8648
|
hash: rowHash(values)
|
|
@@ -8515,25 +8657,61 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8515
8657
|
return row.hash === rowHash(stepValues(runId, row.ordinal, step));
|
|
8516
8658
|
});
|
|
8517
8659
|
}
|
|
8518
|
-
async finishRun(runId, result, executionOrder) {
|
|
8660
|
+
async finishRun(runId, result, executionOrder, pin) {
|
|
8519
8661
|
this.runsInFlight.delete(runId);
|
|
8520
|
-
const
|
|
8662
|
+
const resumed = this.resumedLegs.delete(runId);
|
|
8663
|
+
const paused = this.pausingRuns.delete(runId) || resumed;
|
|
8664
|
+
const own = result.steps.some((step) => step.carried) ? {
|
|
8665
|
+
...result,
|
|
8666
|
+
steps: result.steps.filter((step) => !step.carried)
|
|
8667
|
+
} : result;
|
|
8668
|
+
const sweep = !this.sweepIsNoop(runId, own);
|
|
8521
8669
|
this.journaledSteps.delete(runId);
|
|
8522
|
-
if (erasedRuns.delete(runId)) return
|
|
8523
|
-
|
|
8524
|
-
|
|
8670
|
+
if (erasedRuns.delete(runId)) return {
|
|
8671
|
+
held: true,
|
|
8672
|
+
closed: []
|
|
8673
|
+
};
|
|
8674
|
+
if (sweep && own.steps.length > 0) try {
|
|
8675
|
+
await this.sweepSteps(runId, own, executionOrder);
|
|
8525
8676
|
} catch (error) {
|
|
8526
8677
|
logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
|
|
8527
8678
|
}
|
|
8528
|
-
const
|
|
8529
|
-
|
|
8530
|
-
|
|
8531
|
-
|
|
8532
|
-
|
|
8533
|
-
|
|
8534
|
-
|
|
8535
|
-
|
|
8536
|
-
|
|
8679
|
+
const waiting = result.status === WAITING_RUN_STATUS;
|
|
8680
|
+
const on = result.waiting;
|
|
8681
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8682
|
+
const close = async (trx) => {
|
|
8683
|
+
const prior = resumed ? await trx.selectFrom("run").select("warning_notes").where("id", "=", runId).executeTakeFirst() : void 0;
|
|
8684
|
+
const notes = [.../* @__PURE__ */ new Set([...priorNotes(prior?.warning_notes ?? null), ...runWarningNotes(own)])];
|
|
8685
|
+
const written = await trx.updateTable("run").set({
|
|
8686
|
+
status: result.status,
|
|
8687
|
+
error: result.error ? redactMessage(result.error) : null,
|
|
8688
|
+
error_name: result.errorName ?? null,
|
|
8689
|
+
ended_at: waiting ? null : now,
|
|
8690
|
+
warnings: notes.length,
|
|
8691
|
+
warning_notes: notes.length > 0 ? JSON.stringify(notes) : null,
|
|
8692
|
+
waiting_on: waiting ? on?.waitpointId ?? null : null,
|
|
8693
|
+
...pin ?? {}
|
|
8694
|
+
}).where("id", "=", runId).$if(waiting, (qb) => qb.where((eb) => eb.exists(eb.selectFrom("waitpoint").select("waitpoint.id").where("waitpoint.id", "=", on?.waitpointId ?? "").where("waitpoint.run_id", "=", runId).where("waitpoint.step_id", "=", on?.stepId ?? "").where("waitpoint.status", "in", LIVE_WAITPOINT_STATUSES)))).returning("id").execute();
|
|
8695
|
+
if (!waiting) return {
|
|
8696
|
+
held: true,
|
|
8697
|
+
closed: paused ? await closeLiveWaitpoints(trx, [runId], "EXPIRED", now) : []
|
|
8698
|
+
};
|
|
8699
|
+
if (written.length === 0) return {
|
|
8700
|
+
held: false,
|
|
8701
|
+
closed: []
|
|
8702
|
+
};
|
|
8703
|
+
const linkStatuses = (await trx.selectFrom("waitpoint").select("kind").where("id", "=", on.waitpointId).executeTakeFirst())?.kind === "webhook" ? LIVE_WAITPOINT_STATUSES : ["PENDING"];
|
|
8704
|
+
return {
|
|
8705
|
+
held: true,
|
|
8706
|
+
closed: await closeLiveWaitpoints(trx, [runId], "CANCELLED", now, (eb) => eb.or([eb("waitpoint.id", "=", on.waitpointId), eb.and([
|
|
8707
|
+
eb("waitpoint.type", "=", "WEBHOOK"),
|
|
8708
|
+
eb("waitpoint.kind", "=", "webhook"),
|
|
8709
|
+
eb("waitpoint.status", "in", linkStatuses),
|
|
8710
|
+
eb.exists(eb.selectFrom("step_execution").select("step_execution.id").whereRef("step_execution.run_id", "=", "waitpoint.run_id").whereRef("step_execution.step_id", "=", "waitpoint.step_id").where("step_execution.status", "=", "SUCCEEDED"))
|
|
8711
|
+
])]))
|
|
8712
|
+
};
|
|
8713
|
+
};
|
|
8714
|
+
return waiting || paused ? this.db.transaction().execute(close) : close(this.db);
|
|
8537
8715
|
}
|
|
8538
8716
|
async sweepSteps(runId, result, executionOrder) {
|
|
8539
8717
|
const journaled = await this.db.selectFrom("step_execution").select(["step_id", "ordinal"]).where("run_id", "=", runId).execute();
|
|
@@ -8546,7 +8724,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8546
8724
|
nextOrdinal = Math.max(nextOrdinal, ran + 1);
|
|
8547
8725
|
}
|
|
8548
8726
|
const ordinalFor = (step) => ordinals.get(step.stepId) ?? nextOrdinal++;
|
|
8549
|
-
await this.
|
|
8727
|
+
await this.withRunRow(runId, (trx) => trx.insertInto("step_execution").values(result.steps.map((step) => ({
|
|
8550
8728
|
id: randomUUID(),
|
|
8551
8729
|
...stepValues(runId, ordinalFor(step), step)
|
|
8552
8730
|
}))).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet((eb) => ({
|
|
@@ -8567,18 +8745,15 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8567
8745
|
version_match: eb.ref("excluded.version_match"),
|
|
8568
8746
|
version_note: eb.ref("excluded.version_note"),
|
|
8569
8747
|
config_hash: eb.ref("excluded.config_hash")
|
|
8570
|
-
}))).execute();
|
|
8748
|
+
}))).execute());
|
|
8749
|
+
}
|
|
8750
|
+
async withRunRow(runId, fn) {
|
|
8751
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8752
|
+
return await trx.selectFrom("run").select("id").where("id", "=", runId).forKeyShare().executeTakeFirst() ? fn(trx) : void 0;
|
|
8753
|
+
});
|
|
8571
8754
|
}
|
|
8572
8755
|
async failRun(runId, error, errorName) {
|
|
8573
|
-
this.
|
|
8574
|
-
this.journaledSteps.delete(runId);
|
|
8575
|
-
if (erasedRuns.delete(runId)) return;
|
|
8576
|
-
await this.db.updateTable("run").set({
|
|
8577
|
-
status: "FAILED",
|
|
8578
|
-
error: redactMessage(error),
|
|
8579
|
-
error_name: errorName ?? null,
|
|
8580
|
-
ended_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8581
|
-
}).where("id", "=", runId).execute();
|
|
8756
|
+
return this.endLiveRun(runId, "FAILED", error, errorName ?? null);
|
|
8582
8757
|
}
|
|
8583
8758
|
async listRuns(workflowId, limit = 25, options = {}) {
|
|
8584
8759
|
if (Array.isArray(workflowId) && workflowId.length === 0) return [];
|
|
@@ -8610,10 +8785,10 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8610
8785
|
}
|
|
8611
8786
|
async recordRunDocuments(runId, documentIds) {
|
|
8612
8787
|
if (documentIds.length === 0 || erasedRuns.has(runId)) return;
|
|
8613
|
-
await this.
|
|
8788
|
+
await this.withRunRow(runId, (trx) => trx.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
|
|
8614
8789
|
run_id: runId,
|
|
8615
8790
|
document_id: documentId
|
|
8616
|
-
}))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute();
|
|
8791
|
+
}))).onConflict((oc) => oc.columns(["run_id", "document_id"]).doNothing()).execute());
|
|
8617
8792
|
}
|
|
8618
8793
|
async getRunDocuments(runId) {
|
|
8619
8794
|
return (await this.db.selectFrom("run_document").select("document_id").where("run_id", "=", runId).execute()).map((row) => row.document_id);
|
|
@@ -8758,6 +8933,181 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8758
8933
|
async deleteDedupe(workflowId) {
|
|
8759
8934
|
await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
|
|
8760
8935
|
}
|
|
8936
|
+
async createWaitpoint(row) {
|
|
8937
|
+
if (this.runsInFlight.has(row.run_id)) this.pausingRuns.add(row.run_id);
|
|
8938
|
+
for (let attempt = 1;; attempt++) {
|
|
8939
|
+
const inserted = await this.withRunRow(row.run_id, (trx) => trx.insertInto("waitpoint").values(row).onConflict((oc) => oc.doNothing()).returningAll().executeTakeFirst().then((waitpoint) => ({ waitpoint })));
|
|
8940
|
+
if (!inserted) throw new Error(`Run "${row.run_id}" no longer exists`);
|
|
8941
|
+
if (inserted.waitpoint) return {
|
|
8942
|
+
waitpoint: inserted.waitpoint,
|
|
8943
|
+
created: true
|
|
8944
|
+
};
|
|
8945
|
+
const live = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "=", row.run_id).where("step_path", "=", row.step_path).where("status", "in", LIVE_WAITPOINT_STATUSES).executeTakeFirst();
|
|
8946
|
+
if (live) return {
|
|
8947
|
+
waitpoint: live,
|
|
8948
|
+
created: false
|
|
8949
|
+
};
|
|
8950
|
+
if (attempt >= 3) throw new Error(`Could not create a waitpoint for step "${row.step_key}"`);
|
|
8951
|
+
}
|
|
8952
|
+
}
|
|
8953
|
+
async getWaitpoint(id) {
|
|
8954
|
+
return this.db.selectFrom("waitpoint").selectAll().where("id", "=", id).executeTakeFirst();
|
|
8955
|
+
}
|
|
8956
|
+
async getWaitpointByTokenHash(tokenHash) {
|
|
8957
|
+
return this.db.selectFrom("waitpoint").selectAll().where("token_hash", "=", tokenHash).executeTakeFirst();
|
|
8958
|
+
}
|
|
8959
|
+
async completeWaitpoint(id, resolution) {
|
|
8960
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8961
|
+
return (await this.db.updateTable("waitpoint").set({
|
|
8962
|
+
status: "COMPLETED",
|
|
8963
|
+
resume_payload: JSON.stringify(resolution.payload ?? {}),
|
|
8964
|
+
resolved_at: resolution.resolvedAt ?? now,
|
|
8965
|
+
decided_by: resolution.decidedBy ?? null
|
|
8966
|
+
}).where("id", "=", id).where("status", "=", "PENDING").where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", now)])).where((eb) => eb.not(eb.exists(eb.selectFrom("run").innerJoin("waitpoint as awaited", "awaited.id", "run.waiting_on").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").whereRef("awaited.id", "!=", "waitpoint.id").where("awaited.kind", "!=", "webhook").where("awaited.status", "in", LIVE_WAITPOINT_STATUSES)))).where((eb) => eb.exists(eb.selectFrom("run").select("run.id").whereRef("run.id", "=", "waitpoint.run_id").where("run.status", "in", ["RUNNING", WAITING_RUN_STATUS]))).returningAll().execute())[0];
|
|
8967
|
+
}
|
|
8968
|
+
async claimDueDelayWaitpoints(nowIso, limit) {
|
|
8969
|
+
return this.db.updateTable("waitpoint").set({
|
|
8970
|
+
status: "COMPLETED",
|
|
8971
|
+
resolved_at: nowIso,
|
|
8972
|
+
resume_payload: "{}"
|
|
8973
|
+
}).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "<=", nowIso).orderBy("resume_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
|
|
8974
|
+
}
|
|
8975
|
+
async expireDueWaitpoints(nowIso, limit) {
|
|
8976
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8977
|
+
const expired = await trx.updateTable("waitpoint").set({
|
|
8978
|
+
status: "EXPIRED",
|
|
8979
|
+
resolved_at: nowIso
|
|
8980
|
+
}).where("id", "in", (eb) => eb.selectFrom("waitpoint").select("id").where("status", "=", "PENDING").where("expires_at", "<=", nowIso).orderBy("expires_at").limit(limit)).where("status", "=", "PENDING").returningAll().execute();
|
|
8981
|
+
const ended = [];
|
|
8982
|
+
for (const waitpoint of expired) {
|
|
8983
|
+
const end = await endWaitingRun(trx, waitpoint.run_id, {
|
|
8984
|
+
reason: waitpointExpiryReason(waitpoint),
|
|
8985
|
+
errorName: WAITPOINT_EXPIRED_ERROR_NAME,
|
|
8986
|
+
endedAt: nowIso,
|
|
8987
|
+
waitpointId: waitpoint.id
|
|
8988
|
+
});
|
|
8989
|
+
if (end) ended.push(end);
|
|
8990
|
+
}
|
|
8991
|
+
return {
|
|
8992
|
+
expired,
|
|
8993
|
+
ended
|
|
8994
|
+
};
|
|
8995
|
+
});
|
|
8996
|
+
}
|
|
8997
|
+
async endStrandedWaits(limit) {
|
|
8998
|
+
const runs = await this.db.selectFrom("run").select("id").where("status", "=", WAITING_RUN_STATUS).where((eb) => strandedWait(eb)).limit(limit).execute();
|
|
8999
|
+
const ended = [];
|
|
9000
|
+
for (const run of runs) {
|
|
9001
|
+
const end = await this.db.transaction().execute((trx) => endWaitingRun(trx, run.id, {
|
|
9002
|
+
reason: STRANDED_WAIT_REASON,
|
|
9003
|
+
errorName: WAITPOINT_NOT_LIVE_ERROR_NAME,
|
|
9004
|
+
endedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9005
|
+
}));
|
|
9006
|
+
if (end) ended.push(end);
|
|
9007
|
+
}
|
|
9008
|
+
return ended;
|
|
9009
|
+
}
|
|
9010
|
+
async nextExpiry() {
|
|
9011
|
+
return (await this.db.selectFrom("waitpoint").select("expires_at").where("status", "=", "PENDING").where("expires_at", "is not", null).orderBy("expires_at").limit(1).executeTakeFirst())?.expires_at ?? void 0;
|
|
9012
|
+
}
|
|
9013
|
+
async countWaitingRuns() {
|
|
9014
|
+
const row = await this.db.selectFrom("run").select((eb) => eb.fn.countAll().as("count")).where("status", "=", WAITING_RUN_STATUS).executeTakeFirst();
|
|
9015
|
+
return Number(row?.count ?? 0);
|
|
9016
|
+
}
|
|
9017
|
+
async nextDelayDue() {
|
|
9018
|
+
return (await this.db.selectFrom("waitpoint").select("resume_at").where("status", "=", "PENDING").where("type", "=", "DELAY").where("resume_at", "is not", null).orderBy("resume_at").limit(1).executeTakeFirst())?.resume_at ?? void 0;
|
|
9019
|
+
}
|
|
9020
|
+
async hasCompletedWaitpoint(runId) {
|
|
9021
|
+
return await this.db.selectFrom("waitpoint").select("id").where("run_id", "=", runId).where("status", "=", "COMPLETED").limit(1).executeTakeFirst() !== void 0;
|
|
9022
|
+
}
|
|
9023
|
+
async heldRunIds() {
|
|
9024
|
+
return (await this.db.selectFrom("waitpoint").innerJoin("run", "run.id", "waitpoint.run_id").select("waitpoint.run_id").distinct().where("waitpoint.status", "=", "COMPLETED").where("run.status", "=", WAITING_RUN_STATUS).execute()).map((row) => row.run_id);
|
|
9025
|
+
}
|
|
9026
|
+
isRunInFlight(runId) {
|
|
9027
|
+
return this.runsInFlight.has(runId);
|
|
9028
|
+
}
|
|
9029
|
+
async claimWaitingRun(runId) {
|
|
9030
|
+
if ((await this.db.updateTable("run").set({ status: "RUNNING" }).where("id", "=", runId).where("status", "=", "WAITING").returning("id").execute()).length === 0) return false;
|
|
9031
|
+
this.runsInFlight.add(runId);
|
|
9032
|
+
this.resumedLegs.add(runId);
|
|
9033
|
+
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
9034
|
+
return true;
|
|
9035
|
+
}
|
|
9036
|
+
async returnToWaiting(runId) {
|
|
9037
|
+
this.runsInFlight.delete(runId);
|
|
9038
|
+
this.resumedLegs.delete(runId);
|
|
9039
|
+
this.journaledSteps.delete(runId);
|
|
9040
|
+
await this.db.updateTable("run").set({ status: WAITING_RUN_STATUS }).where("id", "=", runId).where("status", "=", "RUNNING").execute();
|
|
9041
|
+
}
|
|
9042
|
+
async consumeCompletedWaitpoint(runId) {
|
|
9043
|
+
return this.db.transaction().execute(async (trx) => {
|
|
9044
|
+
const waitingOn = (await trx.selectFrom("run").select("waiting_on").where("id", "=", runId).executeTakeFirst())?.waiting_on ?? null;
|
|
9045
|
+
const candidates = await trx.selectFrom("waitpoint").selectAll().where("run_id", "=", runId).where("status", "=", "COMPLETED").orderBy("resolved_at").execute();
|
|
9046
|
+
const awaited = waitingOn ? await trx.selectFrom("waitpoint").select("kind").where("id", "=", waitingOn).executeTakeFirst() : void 0;
|
|
9047
|
+
const linksResume = !awaited || awaited.kind === "webhook";
|
|
9048
|
+
const chosen = candidates.find((row) => row.id === waitingOn) ?? (linksResume ? candidates[0] : void 0);
|
|
9049
|
+
if (!chosen) return void 0;
|
|
9050
|
+
const waitpoint = (await trx.updateTable("waitpoint").set({ status: "CONSUMED" }).where("id", "=", chosen.id).where("status", "=", "COMPLETED").returningAll().execute())[0];
|
|
9051
|
+
if (!waitpoint) return void 0;
|
|
9052
|
+
return {
|
|
9053
|
+
waitpoint,
|
|
9054
|
+
closed: waitingOn && waitingOn !== waitpoint.id ? await closeLiveWaitpoints(trx, [runId], "CANCELLED", (/* @__PURE__ */ new Date()).toISOString(), (eb) => eb("waitpoint.id", "!=", waitingOn)) : []
|
|
9055
|
+
};
|
|
9056
|
+
});
|
|
9057
|
+
}
|
|
9058
|
+
async getRunPin(runId) {
|
|
9059
|
+
return this.db.selectFrom("run").select([
|
|
9060
|
+
"definition_hash",
|
|
9061
|
+
"definition_op_index",
|
|
9062
|
+
"definition_scope_hash"
|
|
9063
|
+
]).where("id", "=", runId).executeTakeFirst();
|
|
9064
|
+
}
|
|
9065
|
+
async cancelIdleRun(runId, reason) {
|
|
9066
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9067
|
+
if (!await this.db.transaction().execute(async (trx) => {
|
|
9068
|
+
if ((await trx.updateTable("run").set({
|
|
9069
|
+
status: "CANCELLED",
|
|
9070
|
+
error: redactMessage(reason),
|
|
9071
|
+
ended_at: endedAt
|
|
9072
|
+
}).where("id", "=", runId).where("status", "in", ["WAITING", "PENDING"]).returning("id").execute()).length === 0) return false;
|
|
9073
|
+
await failWaitingSteps(trx, [runId], reason, null, endedAt);
|
|
9074
|
+
await closeLiveWaitpoints(trx, [runId], "CANCELLED", endedAt);
|
|
9075
|
+
return true;
|
|
9076
|
+
})) return void 0;
|
|
9077
|
+
this.runsInFlight.delete(runId);
|
|
9078
|
+
return runId;
|
|
9079
|
+
}
|
|
9080
|
+
async endWait(runId, end) {
|
|
9081
|
+
this.runsInFlight.delete(runId);
|
|
9082
|
+
this.journaledSteps.delete(runId);
|
|
9083
|
+
this.pausingRuns.delete(runId);
|
|
9084
|
+
this.resumedLegs.delete(runId);
|
|
9085
|
+
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9086
|
+
return this.db.transaction().execute(async (trx) => {
|
|
9087
|
+
if ((await trx.updateTable("run").set({
|
|
9088
|
+
status: end.status,
|
|
9089
|
+
error: redactMessage(end.reason),
|
|
9090
|
+
error_name: end.errorName,
|
|
9091
|
+
ended_at: endedAt
|
|
9092
|
+
}).where("id", "=", runId).where("status", "=", "RUNNING").returning("id").execute()).length === 0) return [];
|
|
9093
|
+
await failWaitingSteps(trx, [runId], end.reason, end.errorName, endedAt);
|
|
9094
|
+
return closeLiveWaitpoints(trx, [runId], end.waitpoints, endedAt);
|
|
9095
|
+
});
|
|
9096
|
+
}
|
|
9097
|
+
async listWaitpointsForRuns(runIds) {
|
|
9098
|
+
const byRun = new Map(runIds.map((id) => [id, []]));
|
|
9099
|
+
if (runIds.length === 0) return byRun;
|
|
9100
|
+
const rows = await this.db.selectFrom("waitpoint").selectAll().where("run_id", "in", [...new Set(runIds)]).orderBy("created_at").execute();
|
|
9101
|
+
for (const row of rows) byRun.get(row.run_id)?.push(row);
|
|
9102
|
+
return byRun;
|
|
9103
|
+
}
|
|
9104
|
+
async listWaitingRuns(workflowId, limit) {
|
|
9105
|
+
if (Array.isArray(workflowId) && workflowId.length === 0) return [];
|
|
9106
|
+
let query = this.db.selectFrom("run").selectAll().where("status", "=", WAITING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc").limit(Math.min(Math.max(limit, 1), 100));
|
|
9107
|
+
if (Array.isArray(workflowId)) query = query.where("workflow_id", "in", workflowId);
|
|
9108
|
+
else if (workflowId) query = query.where("workflow_id", "=", workflowId);
|
|
9109
|
+
return query.execute();
|
|
9110
|
+
}
|
|
8761
9111
|
async pruneFinishedRuns(cutoffIso, batchSize = PRUNE_BATCH_SIZE) {
|
|
8762
9112
|
let pruned = 0;
|
|
8763
9113
|
for (;;) {
|
|
@@ -8765,6 +9115,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8765
9115
|
const ids = (await trx.selectFrom("run").select("id").where("ended_at", "is not", null).where("ended_at", "<", cutoffIso).limit(batchSize).execute()).map((row) => row.id);
|
|
8766
9116
|
if (ids.length === 0) return 0;
|
|
8767
9117
|
await trx.deleteFrom("step_execution").where("run_id", "in", ids).execute();
|
|
9118
|
+
await trx.deleteFrom("waitpoint").where("run_id", "in", ids).execute();
|
|
8768
9119
|
await trx.deleteFrom("run_document").where("run_id", "in", ids).execute();
|
|
8769
9120
|
await trx.deleteFrom("run").where("id", "in", ids).execute();
|
|
8770
9121
|
return ids.length;
|
|
@@ -8796,10 +9147,11 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8796
9147
|
for (const id of frontier) runIds.add(id);
|
|
8797
9148
|
}
|
|
8798
9149
|
if (runIds.size === 0) return erased;
|
|
8799
|
-
const all = [...runIds];
|
|
8800
|
-
const
|
|
8801
|
-
for (const row of
|
|
9150
|
+
const all = [...runIds].sort();
|
|
9151
|
+
const locked = await trx.selectFrom("run").select(["id", "status"]).where("id", "in", all).orderBy("id").forUpdate().execute();
|
|
9152
|
+
for (const row of locked) if (row.status === "RUNNING" || row.status === "PENDING") erasedRuns.add(row.id);
|
|
8802
9153
|
erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
|
|
9154
|
+
await trx.deleteFrom("waitpoint").where("run_id", "in", all).execute();
|
|
8803
9155
|
erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
|
|
8804
9156
|
erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
|
|
8805
9157
|
erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
|
|
@@ -8916,6 +9268,118 @@ function decodeRunCursor(cursor) {
|
|
|
8916
9268
|
id: parsed[1]
|
|
8917
9269
|
};
|
|
8918
9270
|
}
|
|
9271
|
+
const WAITPOINT_POLL_MAX_MS = 3e5;
|
|
9272
|
+
const WAITPOINT_RETRY_MS = 3e4;
|
|
9273
|
+
const SUSPENDED_COUNT_MS = 3e4;
|
|
9274
|
+
var WaitpointPoller = class {
|
|
9275
|
+
deps;
|
|
9276
|
+
running = false;
|
|
9277
|
+
polling = false;
|
|
9278
|
+
again = false;
|
|
9279
|
+
timer;
|
|
9280
|
+
timerAt;
|
|
9281
|
+
nudged;
|
|
9282
|
+
suspended = 0;
|
|
9283
|
+
suspendedAt = 0;
|
|
9284
|
+
counting = false;
|
|
9285
|
+
constructor(deps) {
|
|
9286
|
+
this.deps = deps;
|
|
9287
|
+
}
|
|
9288
|
+
get active() {
|
|
9289
|
+
return this.running;
|
|
9290
|
+
}
|
|
9291
|
+
start() {
|
|
9292
|
+
if (this.running) return;
|
|
9293
|
+
this.running = true;
|
|
9294
|
+
this.poll();
|
|
9295
|
+
}
|
|
9296
|
+
stop() {
|
|
9297
|
+
this.running = false;
|
|
9298
|
+
clearTimeout(this.timer);
|
|
9299
|
+
this.timer = void 0;
|
|
9300
|
+
this.timerAt = void 0;
|
|
9301
|
+
this.nudged = void 0;
|
|
9302
|
+
}
|
|
9303
|
+
nudge(dueIso) {
|
|
9304
|
+
if (!this.running) return;
|
|
9305
|
+
const due = Date.parse(dueIso);
|
|
9306
|
+
if (Number.isNaN(due)) return;
|
|
9307
|
+
if (this.polling) {
|
|
9308
|
+
this.nudged = Math.min(this.nudged ?? Infinity, due);
|
|
9309
|
+
return;
|
|
9310
|
+
}
|
|
9311
|
+
if (this.timerAt !== void 0 && due >= this.timerAt) return;
|
|
9312
|
+
this.arm(due);
|
|
9313
|
+
}
|
|
9314
|
+
suspendedRuns() {
|
|
9315
|
+
if (Date.now() - this.suspendedAt > SUSPENDED_COUNT_MS) this.count();
|
|
9316
|
+
return this.suspended;
|
|
9317
|
+
}
|
|
9318
|
+
async count(store) {
|
|
9319
|
+
if (this.counting) return this.suspended;
|
|
9320
|
+
this.counting = true;
|
|
9321
|
+
try {
|
|
9322
|
+
const journal = store ?? await this.deps.store();
|
|
9323
|
+
if (journal) {
|
|
9324
|
+
this.suspended = await journal.countWaitingRuns();
|
|
9325
|
+
this.suspendedAt = Date.now();
|
|
9326
|
+
}
|
|
9327
|
+
} catch (error) {
|
|
9328
|
+
this.deps.logger.warn("Counting suspended runs failed: @error", error);
|
|
9329
|
+
} finally {
|
|
9330
|
+
this.counting = false;
|
|
9331
|
+
}
|
|
9332
|
+
return this.suspended;
|
|
9333
|
+
}
|
|
9334
|
+
/** One pass: due delays and held resumes deliver, overdue waits expire, the
|
|
9335
|
+
* timer re-arms. */
|
|
9336
|
+
async poll() {
|
|
9337
|
+
if (!this.running) return;
|
|
9338
|
+
if (this.polling) {
|
|
9339
|
+
this.again = true;
|
|
9340
|
+
return;
|
|
9341
|
+
}
|
|
9342
|
+
this.polling = true;
|
|
9343
|
+
let next = Date.now() + WAITPOINT_RETRY_MS;
|
|
9344
|
+
try {
|
|
9345
|
+
const store = await this.deps.store();
|
|
9346
|
+
if (!store) return;
|
|
9347
|
+
const batch = this.deps.batch ?? 100;
|
|
9348
|
+
do {
|
|
9349
|
+
this.again = false;
|
|
9350
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9351
|
+
const due = await store.claimDueDelayWaitpoints(now, batch);
|
|
9352
|
+
for (const runId of new Set(due.map((row) => row.run_id))) await this.deps.resume(runId);
|
|
9353
|
+
const { expired, ended } = await store.expireDueWaitpoints(now, batch);
|
|
9354
|
+
const stranded = await store.endStrandedWaits(batch);
|
|
9355
|
+
for (const end of [...ended, ...stranded]) await this.deps.ended(end);
|
|
9356
|
+
if (due.length === batch || expired.length === batch || stranded.length === batch) this.again = true;
|
|
9357
|
+
} while (this.again && this.active);
|
|
9358
|
+
for (const runId of await store.heldRunIds()) await this.deps.resume(runId);
|
|
9359
|
+
await this.count(store);
|
|
9360
|
+
const [delay, expiry] = await Promise.all([store.nextDelayDue(), store.nextExpiry()]);
|
|
9361
|
+
next = Math.min(delay ? Date.parse(delay) : Infinity, expiry ? Date.parse(expiry) : Infinity);
|
|
9362
|
+
} catch (error) {
|
|
9363
|
+
this.deps.logger.warn("Waitpoint poll failed: @error", error);
|
|
9364
|
+
} finally {
|
|
9365
|
+
this.polling = false;
|
|
9366
|
+
if (this.nudged !== void 0) next = Math.min(next, this.nudged);
|
|
9367
|
+
this.nudged = void 0;
|
|
9368
|
+
if (this.active) this.arm(next);
|
|
9369
|
+
}
|
|
9370
|
+
}
|
|
9371
|
+
arm(due) {
|
|
9372
|
+
clearTimeout(this.timer);
|
|
9373
|
+
const delay = Math.min(Math.max(Number.isFinite(due) ? due - Date.now() : Infinity, 0), WAITPOINT_POLL_MAX_MS);
|
|
9374
|
+
this.timerAt = Date.now() + delay;
|
|
9375
|
+
this.timer = setTimeout(() => {
|
|
9376
|
+
this.timer = void 0;
|
|
9377
|
+
this.timerAt = void 0;
|
|
9378
|
+
this.poll();
|
|
9379
|
+
}, delay);
|
|
9380
|
+
this.timer.unref();
|
|
9381
|
+
}
|
|
9382
|
+
};
|
|
8919
9383
|
//#endregion
|
|
8920
9384
|
//#region src/reactor/run-gate.ts
|
|
8921
9385
|
const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
|
|
@@ -8953,8 +9417,11 @@ var WorkflowRunGate = class {
|
|
|
8953
9417
|
* - **SINGLETON**: refused outright while a run is active. The caller
|
|
8954
9418
|
* journals the refusal rather than dropping it silently, because a firing
|
|
8955
9419
|
* that vanished is indistinguishable from a trigger that never fired.
|
|
9420
|
+
*
|
|
9421
|
+
* A `resume` is a waiting run's next leg, not a new firing: it queues for
|
|
9422
|
+
* the slot under every policy, past the queue depth too.
|
|
8956
9423
|
*/
|
|
8957
|
-
async admit(workflowId, policy, onQueue) {
|
|
9424
|
+
async admit(workflowId, policy, onQueue, options = {}) {
|
|
8958
9425
|
if (this.closed) return {
|
|
8959
9426
|
admitted: false,
|
|
8960
9427
|
reason: CLOSED_REASON,
|
|
@@ -8973,7 +9440,7 @@ var WorkflowRunGate = class {
|
|
|
8973
9440
|
this.lanes.set(workflowId, lane);
|
|
8974
9441
|
let waited = false;
|
|
8975
9442
|
if (lane.active >= limit) {
|
|
8976
|
-
if (policy.concurrency === "SINGLETON") {
|
|
9443
|
+
if (policy.concurrency === "SINGLETON" && !options.resume) {
|
|
8977
9444
|
this.forgetIfIdle(workflowId, lane);
|
|
8978
9445
|
return {
|
|
8979
9446
|
admitted: false,
|
|
@@ -8981,7 +9448,7 @@ var WorkflowRunGate = class {
|
|
|
8981
9448
|
reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
|
|
8982
9449
|
};
|
|
8983
9450
|
}
|
|
8984
|
-
if (lane.waiting.length >= this.maxQueued) return {
|
|
9451
|
+
if (!options.resume && lane.waiting.length >= this.maxQueued) return {
|
|
8985
9452
|
admitted: false,
|
|
8986
9453
|
refusal: "queue-full",
|
|
8987
9454
|
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
@@ -10288,10 +10755,41 @@ const OAUTH_APP = "oauth_app";
|
|
|
10288
10755
|
* for `calendar.readonly` where the piece would ask to edit too.
|
|
10289
10756
|
*/
|
|
10290
10757
|
const OAUTH_SCOPES = "oauth_scopes";
|
|
10758
|
+
function httpsUrl(raw) {
|
|
10759
|
+
if (typeof raw !== "string" || raw === "") return void 0;
|
|
10760
|
+
let url;
|
|
10761
|
+
try {
|
|
10762
|
+
url = new URL(raw);
|
|
10763
|
+
} catch {
|
|
10764
|
+
return;
|
|
10765
|
+
}
|
|
10766
|
+
if (url.protocol !== "https:" || url.username || url.password) return;
|
|
10767
|
+
return url.href;
|
|
10768
|
+
}
|
|
10769
|
+
/**
|
|
10770
|
+
* A relay as configured, or the reason it cannot be used: it needs a client
|
|
10771
|
+
* id and two absolute https URLs without credentials.
|
|
10772
|
+
*/
|
|
10773
|
+
function parseRelay(raw) {
|
|
10774
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return "not an object with clientId, redirectUri and tokenUrl";
|
|
10775
|
+
const { clientId, redirectUri, tokenUrl } = raw;
|
|
10776
|
+
if (typeof clientId !== "string" || clientId === "") return "no clientId";
|
|
10777
|
+
const redirect = httpsUrl(redirectUri);
|
|
10778
|
+
if (!redirect) return "redirectUri is not an absolute https URL";
|
|
10779
|
+
const token = httpsUrl(tokenUrl);
|
|
10780
|
+
if (!token) return "tokenUrl is not an absolute https URL";
|
|
10781
|
+
return {
|
|
10782
|
+
clientId,
|
|
10783
|
+
redirectUri: redirect,
|
|
10784
|
+
tokenUrl: token
|
|
10785
|
+
};
|
|
10786
|
+
}
|
|
10291
10787
|
var StaticOAuthAppRegistry = class {
|
|
10292
10788
|
entries;
|
|
10293
|
-
|
|
10789
|
+
relays;
|
|
10790
|
+
constructor(entries, relays = /* @__PURE__ */ new Map()) {
|
|
10294
10791
|
this.entries = entries;
|
|
10792
|
+
this.relays = relays;
|
|
10295
10793
|
}
|
|
10296
10794
|
client(appId, kind) {
|
|
10297
10795
|
return this.entries.find((e) => e.appId === appId && e.kind === kind);
|
|
@@ -10299,6 +10797,9 @@ var StaticOAuthAppRegistry = class {
|
|
|
10299
10797
|
clients(appId) {
|
|
10300
10798
|
return this.entries.filter((e) => e.appId === appId);
|
|
10301
10799
|
}
|
|
10800
|
+
relay(appId) {
|
|
10801
|
+
return this.relays.get(appId);
|
|
10802
|
+
}
|
|
10302
10803
|
};
|
|
10303
10804
|
const LOOPBACK = /* @__PURE__ */ new Set([
|
|
10304
10805
|
"localhost",
|
|
@@ -10316,6 +10817,22 @@ function clientKindFor(redirectUri) {
|
|
|
10316
10817
|
}
|
|
10317
10818
|
return LOOPBACK.has(host) ? "desktop" : "web";
|
|
10318
10819
|
}
|
|
10820
|
+
function oauthSignInFor(apps, appId, redirectUri) {
|
|
10821
|
+
const kind = clientKindFor(redirectUri);
|
|
10822
|
+
const client = apps.client(appId, kind);
|
|
10823
|
+
if (client) return {
|
|
10824
|
+
mode: kind,
|
|
10825
|
+
client
|
|
10826
|
+
};
|
|
10827
|
+
const relay = kind === "web" ? apps.relay(appId) : void 0;
|
|
10828
|
+
return relay ? {
|
|
10829
|
+
mode: "relay",
|
|
10830
|
+
relay
|
|
10831
|
+
} : {
|
|
10832
|
+
mode: "none",
|
|
10833
|
+
kind
|
|
10834
|
+
};
|
|
10835
|
+
}
|
|
10319
10836
|
/**
|
|
10320
10837
|
* Config key the runtime writes after a sign-in: the scopes the provider
|
|
10321
10838
|
* granted (its token response's `scope`, else the ones asked for). A
|
|
@@ -10364,39 +10881,49 @@ function clientSecretLabel(appId, kind) {
|
|
|
10364
10881
|
* Reads each configured client file and keeps its secret in the store under a
|
|
10365
10882
|
* fixed label: created the first time, rotated when the file changed, left
|
|
10366
10883
|
* alone otherwise. A client whose file cannot be read is skipped with the
|
|
10367
|
-
* reason, so one bad file does not take the others down.
|
|
10884
|
+
* reason, so one bad file does not take the others down. A relay without a
|
|
10885
|
+
* client id or with a URL that is not absolute https is dropped with the
|
|
10886
|
+
* reason.
|
|
10368
10887
|
*/
|
|
10369
10888
|
async function loadOAuthApps(definitions, store, options = {}) {
|
|
10370
10889
|
const read = options.read ?? ((path) => readFile(path, "utf8"));
|
|
10371
10890
|
const existing = new Map((await store.list()).filter((s) => s.status === "ACTIVE" && s.label && s.ownerId === null).map((s) => [s.label, s.ref]));
|
|
10372
10891
|
const entries = [];
|
|
10373
|
-
|
|
10374
|
-
|
|
10375
|
-
if (
|
|
10376
|
-
|
|
10377
|
-
|
|
10378
|
-
|
|
10379
|
-
}
|
|
10380
|
-
|
|
10381
|
-
|
|
10892
|
+
const relays = /* @__PURE__ */ new Map();
|
|
10893
|
+
for (const app of definitions) {
|
|
10894
|
+
if (app.relay !== void 0) {
|
|
10895
|
+
const relay = parseRelay(app.relay);
|
|
10896
|
+
if (typeof relay === "string") options.onRelayDropped?.(app.id, relay);
|
|
10897
|
+
else relays.set(app.id, relay);
|
|
10898
|
+
}
|
|
10899
|
+
for (const kind of ["desktop", "web"]) {
|
|
10900
|
+
const path = app.clients[kind];
|
|
10901
|
+
if (!path) continue;
|
|
10902
|
+
let client;
|
|
10903
|
+
try {
|
|
10904
|
+
client = parseClientFile(await read(path));
|
|
10905
|
+
} catch (error) {
|
|
10906
|
+
options.onSkip?.(app.id, kind, `${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
10907
|
+
continue;
|
|
10908
|
+
}
|
|
10909
|
+
const label = clientSecretLabel(app.id, kind);
|
|
10910
|
+
let ref = existing.get(label);
|
|
10911
|
+
if (ref) {
|
|
10912
|
+
if (await store.get(ref, null) !== client.clientSecret) await store.rotate(ref, client.clientSecret);
|
|
10913
|
+
} else ref = (await store.create({
|
|
10914
|
+
value: client.clientSecret,
|
|
10915
|
+
label,
|
|
10916
|
+
ownerId: null
|
|
10917
|
+
})).ref;
|
|
10918
|
+
entries.push({
|
|
10919
|
+
appId: app.id,
|
|
10920
|
+
kind,
|
|
10921
|
+
clientId: client.clientId,
|
|
10922
|
+
clientSecretRef: ref
|
|
10923
|
+
});
|
|
10382
10924
|
}
|
|
10383
|
-
const label = clientSecretLabel(app.id, kind);
|
|
10384
|
-
let ref = existing.get(label);
|
|
10385
|
-
if (ref) {
|
|
10386
|
-
if (await store.get(ref, null) !== client.clientSecret) await store.rotate(ref, client.clientSecret);
|
|
10387
|
-
} else ref = (await store.create({
|
|
10388
|
-
value: client.clientSecret,
|
|
10389
|
-
label,
|
|
10390
|
-
ownerId: null
|
|
10391
|
-
})).ref;
|
|
10392
|
-
entries.push({
|
|
10393
|
-
appId: app.id,
|
|
10394
|
-
kind,
|
|
10395
|
-
clientId: client.clientId,
|
|
10396
|
-
clientSecretRef: ref
|
|
10397
|
-
});
|
|
10398
10925
|
}
|
|
10399
|
-
return new StaticOAuthAppRegistry(entries);
|
|
10926
|
+
return new StaticOAuthAppRegistry(entries, relays);
|
|
10400
10927
|
}
|
|
10401
10928
|
//#endregion
|
|
10402
10929
|
//#region src/reactor/service.ts
|
|
@@ -10576,12 +11103,72 @@ function refusalStatus(refusal) {
|
|
|
10576
11103
|
switch (refusal) {
|
|
10577
11104
|
case "parked":
|
|
10578
11105
|
case "singleton":
|
|
10579
|
-
case "stale":
|
|
11106
|
+
case "stale":
|
|
11107
|
+
case "cancelled": return 409;
|
|
10580
11108
|
case "queue-full":
|
|
10581
11109
|
case "expired": return 429;
|
|
10582
11110
|
default: return 500;
|
|
10583
11111
|
}
|
|
10584
11112
|
}
|
|
11113
|
+
function runResultReply(result, succeededStatus) {
|
|
11114
|
+
return {
|
|
11115
|
+
status: result.status === "SUCCEEDED" ? succeededStatus : result.status === "WAITING" ? 202 : refusalStatus(result.refusal),
|
|
11116
|
+
contentType: JSON_CONTENT_TYPE,
|
|
11117
|
+
body: JSON.stringify({
|
|
11118
|
+
runId: result.runId,
|
|
11119
|
+
status: result.status,
|
|
11120
|
+
error: result.error ?? null
|
|
11121
|
+
})
|
|
11122
|
+
};
|
|
11123
|
+
}
|
|
11124
|
+
const RESPONDED = Symbol("a step responded");
|
|
11125
|
+
const DROPPED_RESPONSE_HEADERS = /* @__PURE__ */ new Set([
|
|
11126
|
+
"connection",
|
|
11127
|
+
"content-length",
|
|
11128
|
+
"keep-alive",
|
|
11129
|
+
"set-cookie",
|
|
11130
|
+
"transfer-encoding",
|
|
11131
|
+
"upgrade"
|
|
11132
|
+
]);
|
|
11133
|
+
const ORIGIN_SAFE_HEADERS = {
|
|
11134
|
+
"x-content-type-options": "nosniff",
|
|
11135
|
+
"content-security-policy": "sandbox"
|
|
11136
|
+
};
|
|
11137
|
+
function sendableHeader(name, value) {
|
|
11138
|
+
try {
|
|
11139
|
+
validateHeaderName(name);
|
|
11140
|
+
validateHeaderValue(name, value);
|
|
11141
|
+
return true;
|
|
11142
|
+
} catch {
|
|
11143
|
+
return false;
|
|
11144
|
+
}
|
|
11145
|
+
}
|
|
11146
|
+
/** A step's `run.respond` / `run.stop` answer as an HTTP reply. As upstream,
|
|
11147
|
+
* none answers 200 with `{}`, and a string body goes as text. */
|
|
11148
|
+
function runResponseReply(response) {
|
|
11149
|
+
const status = typeof response.status === "number" && Number.isInteger(response.status) && response.status >= 200 && response.status <= 599 ? response.status : 200;
|
|
11150
|
+
const body = response.body ?? {};
|
|
11151
|
+
let contentType = typeof body === "string" ? "text/plain; charset=utf-8" : JSON_CONTENT_TYPE;
|
|
11152
|
+
const headers = {};
|
|
11153
|
+
const given = response.headers && typeof response.headers === "object" ? response.headers : {};
|
|
11154
|
+
for (const [rawName, rawValue] of Object.entries(given)) {
|
|
11155
|
+
const name = rawName.toLowerCase();
|
|
11156
|
+
if (typeof rawValue !== "string" && typeof rawValue !== "number") continue;
|
|
11157
|
+
const value = String(rawValue);
|
|
11158
|
+
if (!sendableHeader(name, value)) continue;
|
|
11159
|
+
if (name === "content-type") contentType = value;
|
|
11160
|
+
else if (!DROPPED_RESPONSE_HEADERS.has(name)) headers[name] = value;
|
|
11161
|
+
}
|
|
11162
|
+
return {
|
|
11163
|
+
status,
|
|
11164
|
+
contentType,
|
|
11165
|
+
body: typeof body === "string" ? body : JSON.stringify(body),
|
|
11166
|
+
headers: {
|
|
11167
|
+
...headers,
|
|
11168
|
+
...ORIGIN_SAFE_HEADERS
|
|
11169
|
+
}
|
|
11170
|
+
};
|
|
11171
|
+
}
|
|
10585
11172
|
const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
|
|
10586
11173
|
function registrationKey(state) {
|
|
10587
11174
|
const trigger = runnableDefinition(state).trigger;
|
|
@@ -10769,7 +11356,10 @@ var WorkflowRuntimeService = class {
|
|
|
10769
11356
|
}
|
|
10770
11357
|
/** The host's OAuth2 apps, their client secrets loaded into the store once. */
|
|
10771
11358
|
oauthApps() {
|
|
10772
|
-
this.oauthAppsPromise ??= this.secrets().then((store) => loadOAuthApps(this.host.oauthApps ?? [], store, {
|
|
11359
|
+
this.oauthAppsPromise ??= this.secrets().then((store) => loadOAuthApps(this.host.oauthApps ?? [], store, {
|
|
11360
|
+
onSkip: (appId, kind, reason) => logger$1.warn(`OAuth2 app "${appId}" has no usable ${kind} client: ${reason}`),
|
|
11361
|
+
onRelayDropped: (appId, reason) => logger$1.warn(`OAuth2 app "${appId}" relay ignored: ${reason}`)
|
|
11362
|
+
}));
|
|
10773
11363
|
return this.oauthAppsPromise;
|
|
10774
11364
|
}
|
|
10775
11365
|
oauthRefresher() {
|
|
@@ -11539,6 +12129,8 @@ var WorkflowRuntimeService = class {
|
|
|
11539
12129
|
startTriggerSupervisor() {
|
|
11540
12130
|
this.supervisor().start();
|
|
11541
12131
|
this.admitAbandonedRuns();
|
|
12132
|
+
this.deliverAllHeld();
|
|
12133
|
+
this.startWaitpointPoller();
|
|
11542
12134
|
this.pendingSweepTimer ??= setInterval(() => {
|
|
11543
12135
|
this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
|
|
11544
12136
|
}, PENDING_SWEEP_INTERVAL_MS);
|
|
@@ -11561,6 +12153,7 @@ var WorkflowRuntimeService = class {
|
|
|
11561
12153
|
stopTriggerSupervisor() {
|
|
11562
12154
|
clearInterval(this.pendingSweepTimer);
|
|
11563
12155
|
this.pendingSweepTimer = void 0;
|
|
12156
|
+
this.stopWaitpointPoller();
|
|
11564
12157
|
this.triggerSupervisor?.stop();
|
|
11565
12158
|
}
|
|
11566
12159
|
closed = false;
|
|
@@ -11768,13 +12361,54 @@ var WorkflowRuntimeService = class {
|
|
|
11768
12361
|
}));
|
|
11769
12362
|
let replied = false;
|
|
11770
12363
|
const takeReply = () => !replied && (replied = true);
|
|
11771
|
-
|
|
12364
|
+
let sent;
|
|
12365
|
+
let responded = () => void 0;
|
|
12366
|
+
const respondedFirst = new Promise((resolve) => {
|
|
12367
|
+
responded = () => resolve(RESPONDED);
|
|
12368
|
+
});
|
|
12369
|
+
const responder = (response) => {
|
|
12370
|
+
if (sent || !takeReply()) return;
|
|
12371
|
+
sent = response;
|
|
12372
|
+
responded();
|
|
12373
|
+
};
|
|
12374
|
+
let ended;
|
|
12375
|
+
let endedEarly = () => void 0;
|
|
12376
|
+
const endedFirst = new Promise((resolve) => {
|
|
12377
|
+
endedEarly = () => resolve({
|
|
12378
|
+
ok: true,
|
|
12379
|
+
result: ended
|
|
12380
|
+
});
|
|
12381
|
+
});
|
|
12382
|
+
const ender = (result) => {
|
|
12383
|
+
if (ended || !takeReply()) return;
|
|
12384
|
+
ended = result;
|
|
12385
|
+
endedEarly();
|
|
12386
|
+
};
|
|
12387
|
+
if (runId) {
|
|
12388
|
+
this.heldReplies.set(runId, takeReply);
|
|
12389
|
+
this.runResponders.set(runId, responder);
|
|
12390
|
+
this.queuedReplies.set(runId, ender);
|
|
12391
|
+
}
|
|
11772
12392
|
let run;
|
|
11773
12393
|
try {
|
|
11774
|
-
|
|
11775
|
-
|
|
12394
|
+
const raced = await Promise.race([
|
|
12395
|
+
firing,
|
|
12396
|
+
respondedFirst,
|
|
12397
|
+
endedFirst,
|
|
12398
|
+
timeout(DELIVERY_TIMEOUT_MS)
|
|
12399
|
+
]);
|
|
12400
|
+
if (sent) return runResponseReply(sent);
|
|
12401
|
+
run = raced;
|
|
12402
|
+
if (run === TIMED_OUT && !takeReply()) run = ended ? {
|
|
12403
|
+
ok: true,
|
|
12404
|
+
result: ended
|
|
12405
|
+
} : await firing;
|
|
11776
12406
|
} finally {
|
|
11777
|
-
if (runId)
|
|
12407
|
+
if (runId) {
|
|
12408
|
+
this.heldReplies.delete(runId);
|
|
12409
|
+
if (this.runResponders.get(runId) === responder) this.runResponders.delete(runId);
|
|
12410
|
+
if (this.queuedReplies.get(runId) === ender) this.queuedReplies.delete(runId);
|
|
12411
|
+
}
|
|
11778
12412
|
}
|
|
11779
12413
|
if (run === TIMED_OUT) {
|
|
11780
12414
|
this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
|
|
@@ -11800,15 +12434,7 @@ var WorkflowRuntimeService = class {
|
|
|
11800
12434
|
body: JSON.stringify({ error: message })
|
|
11801
12435
|
};
|
|
11802
12436
|
}
|
|
11803
|
-
return
|
|
11804
|
-
status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
|
|
11805
|
-
contentType: JSON_CONTENT_TYPE,
|
|
11806
|
-
body: JSON.stringify({
|
|
11807
|
-
runId: run.result.runId,
|
|
11808
|
-
status: run.result.status,
|
|
11809
|
-
error: run.result.error ?? null
|
|
11810
|
-
})
|
|
11811
|
-
};
|
|
12437
|
+
return runResultReply(run.result, config.responseStatus);
|
|
11812
12438
|
}
|
|
11813
12439
|
async pieceHandshake(binding, request) {
|
|
11814
12440
|
const handshake = await this.pieceHandshakeConfig(binding);
|
|
@@ -12163,6 +12789,9 @@ var WorkflowRuntimeService = class {
|
|
|
12163
12789
|
accountLabel: outcome.accountLabel ?? idTokenEmail(shapedAuth) ?? accountLabel
|
|
12164
12790
|
});
|
|
12165
12791
|
}
|
|
12792
|
+
async oauthSignInMode(appId, redirectUri) {
|
|
12793
|
+
return oauthSignInFor(await this.oauthApps(), appId, redirectUri).mode;
|
|
12794
|
+
}
|
|
12166
12795
|
async startOAuth(connectionId, ctx, options) {
|
|
12167
12796
|
await this.assertCanReadDocument(connectionId, ctx);
|
|
12168
12797
|
await this.assertCanWriteDocument(connectionId, ctx);
|
|
@@ -12173,13 +12802,25 @@ var WorkflowRuntimeService = class {
|
|
|
12173
12802
|
let clientId;
|
|
12174
12803
|
let clientSecretRef;
|
|
12175
12804
|
let pkce = false;
|
|
12805
|
+
let redirectUri = options.redirectUri;
|
|
12806
|
+
let tokenUrl;
|
|
12807
|
+
let stateSuffix;
|
|
12176
12808
|
if (typeof appId === "string" && appId !== "") {
|
|
12177
|
-
const
|
|
12178
|
-
|
|
12179
|
-
if (
|
|
12180
|
-
|
|
12181
|
-
|
|
12182
|
-
|
|
12809
|
+
const signIn = oauthSignInFor(await this.oauthApps(), appId, options.redirectUri);
|
|
12810
|
+
if (signIn.mode === "none") throw new OAuthError(`The app "${appId}" has no ${signIn.kind} client on this host; configure it in workflows.oauthApps`);
|
|
12811
|
+
if (signIn.mode === "relay") {
|
|
12812
|
+
const { relay } = signIn;
|
|
12813
|
+
clientId = relay.clientId;
|
|
12814
|
+
clientSecretRef = null;
|
|
12815
|
+
redirectUri = relay.redirectUri;
|
|
12816
|
+
tokenUrl = relay.tokenUrl;
|
|
12817
|
+
stateSuffix = Buffer.from(options.redirectUri, "utf8").toString("base64url");
|
|
12818
|
+
pkce = true;
|
|
12819
|
+
} else {
|
|
12820
|
+
clientId = signIn.client.clientId;
|
|
12821
|
+
clientSecretRef = signIn.client.clientSecretRef;
|
|
12822
|
+
pkce = signIn.mode === "desktop";
|
|
12823
|
+
}
|
|
12183
12824
|
} else {
|
|
12184
12825
|
if (typeof ownClientId !== "string" || ownClientId === "") throw new OAuthError("Set the client ID before connecting");
|
|
12185
12826
|
const ref = state.secretRefs.find((entry) => entry.name === OAUTH_CLIENT_SECRET)?.ref;
|
|
@@ -12204,10 +12845,12 @@ var WorkflowRuntimeService = class {
|
|
|
12204
12845
|
props,
|
|
12205
12846
|
clientId,
|
|
12206
12847
|
clientSecretRef,
|
|
12207
|
-
redirectUri
|
|
12848
|
+
redirectUri,
|
|
12849
|
+
...tokenUrl ? { tokenUrl } : {},
|
|
12208
12850
|
...options.returnUrl ? { returnUrl: options.returnUrl } : {},
|
|
12209
12851
|
...scopes ? { scopes } : {},
|
|
12210
12852
|
...pkce ? { pkce } : {},
|
|
12853
|
+
...stateSuffix ? { stateSuffix } : {},
|
|
12211
12854
|
createdBy: this.callerAddress(ctx)
|
|
12212
12855
|
});
|
|
12213
12856
|
}
|
|
@@ -12239,11 +12882,16 @@ var WorkflowRuntimeService = class {
|
|
|
12239
12882
|
const state = document.state.global;
|
|
12240
12883
|
const config = state.config ?? {};
|
|
12241
12884
|
const appId = config[OAUTH_APP];
|
|
12242
|
-
const
|
|
12243
|
-
|
|
12885
|
+
const relayed = attempt.client_secret_ref === null;
|
|
12886
|
+
let fromRegistry = false;
|
|
12887
|
+
if (typeof appId === "string" && appId !== "") {
|
|
12888
|
+
const apps = await this.oauthApps();
|
|
12889
|
+
fromRegistry = relayed ? apps.relay(appId)?.clientId === attempt.client_id : apps.clients(appId).some((c) => c.clientId === attempt.client_id);
|
|
12890
|
+
}
|
|
12891
|
+
if (state.authType !== "OAUTH2" || !fromRegistry && (relayed || config["client_id"] !== attempt.client_id)) return await fail("The connection changed during sign-in; try again");
|
|
12244
12892
|
const secrets = await this.secrets();
|
|
12245
12893
|
const connectionId = document.header.id;
|
|
12246
|
-
const tokens = await exchangeCode(attempt, callback.code, await secrets.get(attempt.client_secret_ref, fromRegistry ? null : {
|
|
12894
|
+
const tokens = await exchangeCode(attempt, callback.code, attempt.client_secret_ref === null ? null : await secrets.get(attempt.client_secret_ref, fromRegistry ? null : {
|
|
12247
12895
|
documentId: connectionId,
|
|
12248
12896
|
kind: "connection"
|
|
12249
12897
|
}), this.designEgress, fromRegistry);
|
|
@@ -12479,10 +13127,13 @@ var WorkflowRuntimeService = class {
|
|
|
12479
13127
|
async blockOutputTree(block, config) {
|
|
12480
13128
|
const record = config ?? {};
|
|
12481
13129
|
switch (blockKey(block)) {
|
|
12482
|
-
case MANUAL_BLOCK:
|
|
12483
|
-
|
|
12484
|
-
|
|
12485
|
-
|
|
13130
|
+
case MANUAL_BLOCK: {
|
|
13131
|
+
const nodes = manualTriggerTree(parseManualInputs(record));
|
|
13132
|
+
return {
|
|
13133
|
+
source: nodes.length > 0 ? "static" : "none",
|
|
13134
|
+
nodes
|
|
13135
|
+
};
|
|
13136
|
+
}
|
|
12486
13137
|
case SCHEDULE_BLOCK: return {
|
|
12487
13138
|
source: "static",
|
|
12488
13139
|
nodes: scheduleTriggerTree()
|
|
@@ -12519,6 +13170,61 @@ var WorkflowRuntimeService = class {
|
|
|
12519
13170
|
type: "value"
|
|
12520
13171
|
}]
|
|
12521
13172
|
};
|
|
13173
|
+
case DELAY_BLOCK: return {
|
|
13174
|
+
source: "static",
|
|
13175
|
+
nodes: [{
|
|
13176
|
+
name: "resumedAt",
|
|
13177
|
+
type: "DateTime!"
|
|
13178
|
+
}]
|
|
13179
|
+
};
|
|
13180
|
+
case APPROVAL_BLOCK: return {
|
|
13181
|
+
source: "static",
|
|
13182
|
+
nodes: [
|
|
13183
|
+
{
|
|
13184
|
+
name: "approved",
|
|
13185
|
+
type: "Boolean!"
|
|
13186
|
+
},
|
|
13187
|
+
{
|
|
13188
|
+
name: "reason",
|
|
13189
|
+
type: "String"
|
|
13190
|
+
},
|
|
13191
|
+
{
|
|
13192
|
+
name: "decidedBy",
|
|
13193
|
+
type: "String",
|
|
13194
|
+
description: "The approver's DID"
|
|
13195
|
+
},
|
|
13196
|
+
{
|
|
13197
|
+
name: "decidedAt",
|
|
13198
|
+
type: "DateTime!"
|
|
13199
|
+
}
|
|
13200
|
+
]
|
|
13201
|
+
};
|
|
13202
|
+
case STOP_BLOCK: return {
|
|
13203
|
+
source: "static",
|
|
13204
|
+
nodes: [{
|
|
13205
|
+
name: "response",
|
|
13206
|
+
type: "JSONObject",
|
|
13207
|
+
description: "The answer sent, when one was set"
|
|
13208
|
+
}]
|
|
13209
|
+
};
|
|
13210
|
+
case WAIT_FOR_WEBHOOK_BLOCK: return {
|
|
13211
|
+
source: "static",
|
|
13212
|
+
nodes: [
|
|
13213
|
+
{
|
|
13214
|
+
name: "body",
|
|
13215
|
+
type: "Unknown",
|
|
13216
|
+
description: "The resuming request's body"
|
|
13217
|
+
},
|
|
13218
|
+
{
|
|
13219
|
+
name: "headers",
|
|
13220
|
+
type: "JSONObject!"
|
|
13221
|
+
},
|
|
13222
|
+
{
|
|
13223
|
+
name: "queryParams",
|
|
13224
|
+
type: "JSONObject!"
|
|
13225
|
+
}
|
|
13226
|
+
]
|
|
13227
|
+
};
|
|
12522
13228
|
case DOCUMENT_CREATED_BLOCK:
|
|
12523
13229
|
case DOCUMENT_DELETED_BLOCK: return {
|
|
12524
13230
|
source: "static",
|
|
@@ -12840,6 +13546,7 @@ var WorkflowRuntimeService = class {
|
|
|
12840
13546
|
let definition;
|
|
12841
13547
|
let runUser;
|
|
12842
13548
|
let readGeneration;
|
|
13549
|
+
let revision;
|
|
12843
13550
|
try {
|
|
12844
13551
|
if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12845
13552
|
run.refused = true;
|
|
@@ -12849,6 +13556,7 @@ var WorkflowRuntimeService = class {
|
|
|
12849
13556
|
if (cached) {
|
|
12850
13557
|
state = cached.state;
|
|
12851
13558
|
documentName = cached.name;
|
|
13559
|
+
revision = cached.revision;
|
|
12852
13560
|
readGeneration = this.workflowGeneration(workflowId);
|
|
12853
13561
|
} else {
|
|
12854
13562
|
const generation = this.workflowGeneration(workflowId);
|
|
@@ -12860,12 +13568,23 @@ var WorkflowRuntimeService = class {
|
|
|
12860
13568
|
}
|
|
12861
13569
|
state = document.state.global;
|
|
12862
13570
|
documentName = document.header.name;
|
|
13571
|
+
revision = globalRevision(document);
|
|
12863
13572
|
this.cacheLoaded(workflowId, generation, document);
|
|
12864
13573
|
}
|
|
12865
13574
|
if (state.status !== "ENABLED") {
|
|
12866
13575
|
run.refused = true;
|
|
12867
13576
|
throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
|
|
12868
13577
|
}
|
|
13578
|
+
if (triggerKind === "manual" && !resume && !enqueuedRunId) {
|
|
13579
|
+
const fired = runnableDefinition(state).trigger;
|
|
13580
|
+
const inputs = fired && blockKey(triggerBlock(fired)) === MANUAL_BLOCK ? parseManualInputs(configRecord(fired.config)) : [];
|
|
13581
|
+
if (inputs.length > 0) try {
|
|
13582
|
+
triggerPayload = coerceManualPayload(inputs, triggerPayload);
|
|
13583
|
+
} catch (error) {
|
|
13584
|
+
run.refused = true;
|
|
13585
|
+
throw error;
|
|
13586
|
+
}
|
|
13587
|
+
}
|
|
12869
13588
|
definition = toWorkflowDefinition(state);
|
|
12870
13589
|
const reactorConnections = declaredReactorConnectionIds(definition);
|
|
12871
13590
|
runUser = reactorConnections.size === 0 ? void 0 : await this.telemetry.phase("run_user", {}, () => assertReactorConnectionsReadable(this.host, workflowId, reactorConnections)).catch((error) => {
|
|
@@ -12978,74 +13697,63 @@ var WorkflowRuntimeService = class {
|
|
|
12978
13697
|
}
|
|
12979
13698
|
}
|
|
12980
13699
|
let runId = enqueuedRunId ?? null;
|
|
12981
|
-
let journalFailed = false;
|
|
12982
|
-
const executionOrder = /* @__PURE__ */ new Map();
|
|
12983
13700
|
let session;
|
|
12984
13701
|
try {
|
|
12985
|
-
|
|
13702
|
+
const journaled = await this.telemetry.phase("journal.start", {}, async () => {
|
|
12986
13703
|
if (enqueuedRunId) {
|
|
12987
|
-
await store?.beginRun(enqueuedRunId, {
|
|
13704
|
+
if (await store?.beginRun(enqueuedRunId, {
|
|
12988
13705
|
workflowName: runJournalName(state.name, documentName),
|
|
12989
13706
|
workflowVersion: runnable.version
|
|
12990
|
-
})
|
|
12991
|
-
|
|
13707
|
+
}) === false) return {
|
|
13708
|
+
runId: null,
|
|
13709
|
+
cancelled: true
|
|
13710
|
+
};
|
|
13711
|
+
return { runId: enqueuedRunId };
|
|
12992
13712
|
}
|
|
12993
|
-
return await store?.startRun({
|
|
13713
|
+
return { runId: await store?.startRun({
|
|
12994
13714
|
workflowId,
|
|
12995
13715
|
workflowName: runJournalName(state.name, documentName),
|
|
12996
13716
|
workflowVersion: runnable.version,
|
|
12997
13717
|
triggerKind,
|
|
12998
13718
|
triggerPayload,
|
|
12999
13719
|
rerunOf: resume?.rerunOf
|
|
13000
|
-
}) ?? null;
|
|
13720
|
+
}) ?? null };
|
|
13001
13721
|
});
|
|
13722
|
+
if (journaled.cancelled) {
|
|
13723
|
+
const row = enqueuedRunId ? await store?.getRun(enqueuedRunId) : void 0;
|
|
13724
|
+
return {
|
|
13725
|
+
status: CANCELLED_RUN_STATUS,
|
|
13726
|
+
steps: [],
|
|
13727
|
+
runId: enqueuedRunId ?? null,
|
|
13728
|
+
...row?.error ? { error: row.error } : {},
|
|
13729
|
+
refusal: "cancelled"
|
|
13730
|
+
};
|
|
13731
|
+
}
|
|
13732
|
+
runId = journaled.runId;
|
|
13002
13733
|
if (runId) run.span.setAttribute("run.id", runId);
|
|
13003
13734
|
if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
|
|
13004
13735
|
session = this.workers().session();
|
|
13005
|
-
const {
|
|
13006
|
-
const journal = store;
|
|
13007
|
-
const journaledRunId = runId;
|
|
13008
|
-
const handed = /* @__PURE__ */ new Set();
|
|
13009
|
-
const result = await withRunScope({
|
|
13736
|
+
const { result, handed } = await this.executeLeg({
|
|
13010
13737
|
workflowId,
|
|
13011
13738
|
runId,
|
|
13739
|
+
store,
|
|
13740
|
+
session,
|
|
13741
|
+
definition,
|
|
13742
|
+
runnable,
|
|
13743
|
+
policy,
|
|
13744
|
+
deadline,
|
|
13012
13745
|
connections,
|
|
13013
|
-
|
|
13014
|
-
...runUser !== void 0 ? { runUser } : {},
|
|
13015
|
-
attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
|
|
13016
|
-
recordDocuments: async (documentIds) => {
|
|
13017
|
-
for (const documentId of documentIds) handed.add(documentId);
|
|
13018
|
-
if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
|
|
13019
|
-
}
|
|
13020
|
-
}, () => runWorkflow({
|
|
13021
|
-
definition: {
|
|
13022
|
-
...definition,
|
|
13023
|
-
variables
|
|
13024
|
-
},
|
|
13025
|
-
executor,
|
|
13026
|
-
declaredPorts: blockPorts,
|
|
13027
|
-
blockDefaults: (block) => this.blockDefaults(block),
|
|
13028
|
-
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
13746
|
+
runUser,
|
|
13029
13747
|
triggerPayload,
|
|
13748
|
+
executor,
|
|
13030
13749
|
completedSteps: resume?.completedSteps,
|
|
13031
|
-
|
|
13032
|
-
|
|
13033
|
-
|
|
13034
|
-
|
|
13035
|
-
|
|
13036
|
-
|
|
13037
|
-
|
|
13038
|
-
if (journalFailed) return;
|
|
13039
|
-
journalFailed = true;
|
|
13040
|
-
this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
13041
|
-
}
|
|
13042
|
-
} : void 0
|
|
13043
|
-
}));
|
|
13044
|
-
if (store && journaledRunId) try {
|
|
13045
|
-
await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
|
|
13046
|
-
} catch (error) {
|
|
13047
|
-
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
13048
|
-
}
|
|
13750
|
+
replayed: resume?.completedSteps,
|
|
13751
|
+
pin: () => ({
|
|
13752
|
+
definition_hash: this.definitionHash(workflowId, state),
|
|
13753
|
+
definition_op_index: revision,
|
|
13754
|
+
definition_scope_hash: scopeHash(state)
|
|
13755
|
+
})
|
|
13756
|
+
});
|
|
13049
13757
|
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
13050
13758
|
const finished = {
|
|
13051
13759
|
...result,
|
|
@@ -13059,7 +13767,14 @@ var WorkflowRuntimeService = class {
|
|
|
13059
13767
|
runId
|
|
13060
13768
|
};
|
|
13061
13769
|
} catch (error) {
|
|
13062
|
-
if (store && runId)
|
|
13770
|
+
if (store && runId) {
|
|
13771
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
13772
|
+
const closed = await store.failRun(runId, message, errorNameOf(error));
|
|
13773
|
+
this.answerClosed(closed, runId, {
|
|
13774
|
+
status: "FAILED",
|
|
13775
|
+
error: message
|
|
13776
|
+
});
|
|
13777
|
+
}
|
|
13063
13778
|
throw error;
|
|
13064
13779
|
} finally {
|
|
13065
13780
|
session?.close();
|
|
@@ -13067,6 +13782,115 @@ var WorkflowRuntimeService = class {
|
|
|
13067
13782
|
dispatched();
|
|
13068
13783
|
}
|
|
13069
13784
|
}
|
|
13785
|
+
async executeLeg(leg) {
|
|
13786
|
+
const { workflowId, runId, store, policy } = leg;
|
|
13787
|
+
let journalFailed = false;
|
|
13788
|
+
const executionOrder = /* @__PURE__ */ new Map();
|
|
13789
|
+
const { variables, secretValues } = await resolveVariables(leg.runnable.variables, this.secretProvider(), workflowId);
|
|
13790
|
+
const handed = new Set(leg.handed);
|
|
13791
|
+
const attachments = attachmentOrigins(leg.triggerPayload, handed, leg.replayed);
|
|
13792
|
+
for (const ref of leg.written ?? []) attachments.written.add(ref);
|
|
13793
|
+
const waitpoints = store && runId ? this.waitpointPort(store, workflowId, runId, maxSuspensionMs(leg.runnable)) : void 0;
|
|
13794
|
+
const respond = leg.respond ?? (runId && !leg.resume ? (response) => this.runResponders.get(runId)?.(response) : void 0);
|
|
13795
|
+
let result = await withRunScope({
|
|
13796
|
+
workflowId,
|
|
13797
|
+
runId,
|
|
13798
|
+
connections: leg.connections,
|
|
13799
|
+
pieceWorker: leg.session,
|
|
13800
|
+
...leg.runUser !== void 0 ? { runUser: leg.runUser } : {},
|
|
13801
|
+
attachments,
|
|
13802
|
+
recordDocuments: async (documentIds) => {
|
|
13803
|
+
for (const documentId of documentIds) handed.add(documentId);
|
|
13804
|
+
if (store && runId) await store.recordRunDocuments(runId, documentIds);
|
|
13805
|
+
},
|
|
13806
|
+
...waitpoints ? { waitpoints: waitpoints.port } : {}
|
|
13807
|
+
}, () => runWorkflow({
|
|
13808
|
+
definition: {
|
|
13809
|
+
...leg.definition,
|
|
13810
|
+
variables
|
|
13811
|
+
},
|
|
13812
|
+
executor: leg.executor,
|
|
13813
|
+
declaredPorts: blockPorts,
|
|
13814
|
+
blockDefaults: (block) => this.blockDefaults(block),
|
|
13815
|
+
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
13816
|
+
triggerPayload: leg.triggerPayload,
|
|
13817
|
+
completedSteps: leg.completedSteps,
|
|
13818
|
+
...leg.carriedSteps ? { carriedSteps: leg.carriedSteps } : {},
|
|
13819
|
+
...leg.resume ? { resume: leg.resume } : {},
|
|
13820
|
+
...leg.firstOrdinal !== void 0 ? { firstOrdinal: leg.firstOrdinal } : {},
|
|
13821
|
+
...respond ? { onRespond: respond } : {},
|
|
13822
|
+
...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
|
|
13823
|
+
...leg.deadline !== void 0 ? { deadline: leg.deadline } : {},
|
|
13824
|
+
onStep: store && runId ? async (record, ordinal) => {
|
|
13825
|
+
executionOrder.set(record.stepId, ordinal);
|
|
13826
|
+
try {
|
|
13827
|
+
await this.telemetry.phase("journal.step", {}, () => store.recordStep(runId, ordinal, record));
|
|
13828
|
+
} catch (error) {
|
|
13829
|
+
if (journalFailed) return;
|
|
13830
|
+
journalFailed = true;
|
|
13831
|
+
this.logger.warn(`Run ${runId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
13832
|
+
}
|
|
13833
|
+
} : void 0
|
|
13834
|
+
}));
|
|
13835
|
+
if (store && runId) {
|
|
13836
|
+
let waiting = result.status === WAITING_RUN_STATUS;
|
|
13837
|
+
try {
|
|
13838
|
+
const { held, closed } = await this.telemetry.phase("journal.finish", {}, () => store.finishRun(runId, result, executionOrder, waiting ? leg.pin?.() : void 0));
|
|
13839
|
+
this.answerClosed(closed, runId, {
|
|
13840
|
+
status: waiting ? CANCELLED_RUN_STATUS : result.status,
|
|
13841
|
+
error: waiting ? "The step that created this waitpoint no longer waits on it" : result.error ?? "The run ended without resuming on it"
|
|
13842
|
+
});
|
|
13843
|
+
if (!held) {
|
|
13844
|
+
result = await this.endUnheldWait(store, runId, result);
|
|
13845
|
+
waiting = false;
|
|
13846
|
+
}
|
|
13847
|
+
} catch (error) {
|
|
13848
|
+
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
13849
|
+
}
|
|
13850
|
+
if (waiting) {
|
|
13851
|
+
const toSend = result.waiting && waitpoints?.responseFor(result.waiting.waitpointId);
|
|
13852
|
+
if (toSend) respond?.(toSend);
|
|
13853
|
+
await this.deliverHeld(runId);
|
|
13854
|
+
}
|
|
13855
|
+
}
|
|
13856
|
+
return {
|
|
13857
|
+
result,
|
|
13858
|
+
handed
|
|
13859
|
+
};
|
|
13860
|
+
}
|
|
13861
|
+
async endUnheldWait(store, runId, result) {
|
|
13862
|
+
const on = result.waiting;
|
|
13863
|
+
const step = result.steps.find((record) => record.stepId === on?.stepId);
|
|
13864
|
+
const row = on ? await store.getWaitpoint(on.waitpointId) : void 0;
|
|
13865
|
+
const end = row?.run_id === runId && row.step_id === on?.stepId && row.status === "EXPIRED" ? {
|
|
13866
|
+
status: "CANCELLED",
|
|
13867
|
+
reason: `The wait at step "${row.step_key}" expired at ${row.expires_at} before the run could wait on it`,
|
|
13868
|
+
errorName: WAITPOINT_EXPIRED_ERROR_NAME
|
|
13869
|
+
} : {
|
|
13870
|
+
status: "FAILED",
|
|
13871
|
+
reason: `Step "${step?.key ?? on?.stepId}" waited on waitpoint "${on?.waitpointId}", which is not a live waitpoint it created`,
|
|
13872
|
+
errorName: WAITPOINT_NOT_LIVE_ERROR_NAME
|
|
13873
|
+
};
|
|
13874
|
+
const closed = await store.endWait(runId, {
|
|
13875
|
+
...end,
|
|
13876
|
+
waitpoints: "EXPIRED"
|
|
13877
|
+
});
|
|
13878
|
+
this.answerClosed(closed, runId, {
|
|
13879
|
+
status: end.status,
|
|
13880
|
+
error: end.reason
|
|
13881
|
+
});
|
|
13882
|
+
return {
|
|
13883
|
+
status: end.status,
|
|
13884
|
+
error: end.reason,
|
|
13885
|
+
errorName: end.errorName,
|
|
13886
|
+
steps: result.steps.map((record) => record === step ? {
|
|
13887
|
+
...record,
|
|
13888
|
+
status: "FAILED",
|
|
13889
|
+
error: end.reason,
|
|
13890
|
+
errorName: end.errorName
|
|
13891
|
+
} : record)
|
|
13892
|
+
};
|
|
13893
|
+
}
|
|
13070
13894
|
runGate = new WorkflowRunGate();
|
|
13071
13895
|
unsettledFirings = /* @__PURE__ */ new Map();
|
|
13072
13896
|
runLimits = /* @__PURE__ */ new Map();
|
|
@@ -13091,6 +13915,9 @@ var WorkflowRuntimeService = class {
|
|
|
13091
13915
|
else this.unsettledFirings.delete(workflowId);
|
|
13092
13916
|
}
|
|
13093
13917
|
heldReplies = /* @__PURE__ */ new Map();
|
|
13918
|
+
runResponders = /* @__PURE__ */ new Map();
|
|
13919
|
+
queuedReplies = /* @__PURE__ */ new Map();
|
|
13920
|
+
syncResumes = /* @__PURE__ */ new Map();
|
|
13094
13921
|
/** Why a trigger's firing of a PARKED workflow is refused; undefined when
|
|
13095
13922
|
* it is not parked, or when an operator started the run. */
|
|
13096
13923
|
async parkedFiring(workflowId, triggerKind, publishedVersion) {
|
|
@@ -13158,12 +13985,23 @@ var WorkflowRuntimeService = class {
|
|
|
13158
13985
|
this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
|
|
13159
13986
|
let runId = enqueuedRunId ?? null;
|
|
13160
13987
|
if (store) try {
|
|
13161
|
-
|
|
13988
|
+
let adopted = true;
|
|
13989
|
+
if (runId) adopted = await store.beginRun(runId, details);
|
|
13162
13990
|
else runId = await store.startRun({
|
|
13163
13991
|
workflowId,
|
|
13164
13992
|
...details
|
|
13165
13993
|
});
|
|
13166
|
-
await store.cancelRun(runId, details.reason);
|
|
13994
|
+
if (adopted) await store.cancelRun(runId, details.reason);
|
|
13995
|
+
else {
|
|
13996
|
+
const row = await store.getRun(runId);
|
|
13997
|
+
if (row) return {
|
|
13998
|
+
status: row.status,
|
|
13999
|
+
steps: [],
|
|
14000
|
+
runId,
|
|
14001
|
+
...row.error ? { error: row.error } : {},
|
|
14002
|
+
refusal: row.status === "CANCELLED" ? "cancelled" : refusal
|
|
14003
|
+
};
|
|
14004
|
+
}
|
|
13167
14005
|
} catch (error) {
|
|
13168
14006
|
this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
|
|
13169
14007
|
}
|
|
@@ -13259,6 +14097,677 @@ var WorkflowRuntimeService = class {
|
|
|
13259
14097
|
const parsed = resolvedBlock(await this.resolveBlock(stepBlock(step)));
|
|
13260
14098
|
return parsed ? await this.declaredReactor(parsed) === "read" : false;
|
|
13261
14099
|
}
|
|
14100
|
+
resuming = /* @__PURE__ */ new Set();
|
|
14101
|
+
definitionHashes = /* @__PURE__ */ new Map();
|
|
14102
|
+
/** Our hash of the runnable definition: computed once per published version. */
|
|
14103
|
+
definitionHash(workflowId, state) {
|
|
14104
|
+
const runnable = runnableDefinition(state);
|
|
14105
|
+
if (!runnable.published) return hashOf(canonicalJson(runnable));
|
|
14106
|
+
const key = `${workflowId}@${runnable.version}`;
|
|
14107
|
+
let hash = this.definitionHashes.get(key);
|
|
14108
|
+
if (hash === void 0) {
|
|
14109
|
+
hash = hashOf(canonicalJson(runnable));
|
|
14110
|
+
if (this.definitionHashes.size >= WORKFLOW_CACHE_MAX) this.definitionHashes.clear();
|
|
14111
|
+
this.definitionHashes.set(key, hash);
|
|
14112
|
+
}
|
|
14113
|
+
return hash;
|
|
14114
|
+
}
|
|
14115
|
+
waitpointPort(store, workflowId, runId, suspendMs) {
|
|
14116
|
+
const responses = /* @__PURE__ */ new Map();
|
|
14117
|
+
const tokens = /* @__PURE__ */ new Map();
|
|
14118
|
+
const configured = this.host.waitpointBaseUrl?.replace(/\/+$/, "");
|
|
14119
|
+
const baseUrl = configured && URL.canParse(configured) ? configured : void 0;
|
|
14120
|
+
return {
|
|
14121
|
+
port: {
|
|
14122
|
+
...baseUrl ? { resumeBaseUrl: baseUrl } : {},
|
|
14123
|
+
create: async (request) => {
|
|
14124
|
+
if (!baseUrl && request.type === "WEBHOOK" && !request.approval) throw new Error("This host serves no resume URLs: resume URLs need an HTTP host");
|
|
14125
|
+
const createdAt = /* @__PURE__ */ new Date();
|
|
14126
|
+
const resumeAt = request.resumeDateTime ? new Date(Date.parse(request.resumeDateTime)).toISOString() : null;
|
|
14127
|
+
if (resumeAt && Date.parse(resumeAt) > MAX_WAITPOINT_MS) throw new Error(`A delay until ${resumeAt} is past ${MAX_WAITPOINT_ISO}, the latest a run can wait until`);
|
|
14128
|
+
const expiresAt = waitpointExpiry(createdAt, request, suspendMs);
|
|
14129
|
+
if (request.type === "DELAY" && resumeAt && suspendMs !== null) {
|
|
14130
|
+
if (Date.parse(resumeAt) - createdAt.getTime() > suspendMs) throw new Error(`A delay until ${resumeAt} is longer than the workflow's maxSuspensionDays allows`);
|
|
14131
|
+
}
|
|
14132
|
+
const token = baseUrl && !request.approval && request.type !== "DELAY" ? request.token ?? randomBytes(32).toString("base64url") : null;
|
|
14133
|
+
const id = randomUUID();
|
|
14134
|
+
const { waitpoint, created: inserted } = await store.createWaitpoint({
|
|
14135
|
+
id,
|
|
14136
|
+
run_id: runId,
|
|
14137
|
+
workflow_id: workflowId,
|
|
14138
|
+
step_id: request.stepId,
|
|
14139
|
+
step_key: request.stepKey,
|
|
14140
|
+
step_path: request.stepId,
|
|
14141
|
+
piece_name: request.pieceName,
|
|
14142
|
+
block_name: request.blockName,
|
|
14143
|
+
type: request.type,
|
|
14144
|
+
kind: request.approval ? "approval" : request.type === "DELAY" ? "delay" : "webhook",
|
|
14145
|
+
status: "PENDING",
|
|
14146
|
+
token_hash: token ? tokenHash(token) : null,
|
|
14147
|
+
created_at: createdAt.toISOString(),
|
|
14148
|
+
resume_at: resumeAt,
|
|
14149
|
+
expires_at: request.type === "DELAY" ? null : expiresAt,
|
|
14150
|
+
resolved_at: null,
|
|
14151
|
+
decided_by: null,
|
|
14152
|
+
approval: request.approval ? JSON.stringify(request.approval) : null,
|
|
14153
|
+
resume_payload: null,
|
|
14154
|
+
response_to_send: request.responseToSend ? JSON.stringify(request.responseToSend) : null
|
|
14155
|
+
});
|
|
14156
|
+
if (!inserted) {
|
|
14157
|
+
const known = tokens.get(waitpoint.id);
|
|
14158
|
+
if (waitpoint.token_hash !== null && known === void 0) throw new Error(`Step "${request.stepKey}" has a waitpoint from an earlier leg, whose resume URL cannot be issued again`);
|
|
14159
|
+
return {
|
|
14160
|
+
id: waitpoint.id,
|
|
14161
|
+
resumeUrl: known ? `${baseUrl}/${known}` : ""
|
|
14162
|
+
};
|
|
14163
|
+
}
|
|
14164
|
+
if (token) tokens.set(id, token);
|
|
14165
|
+
if (request.responseToSend) responses.set(id, request.responseToSend);
|
|
14166
|
+
const due = request.type === "DELAY" ? resumeAt : expiresAt;
|
|
14167
|
+
if (due) this.waitpointPoller.nudge(due);
|
|
14168
|
+
return {
|
|
14169
|
+
id,
|
|
14170
|
+
resumeUrl: token ? `${baseUrl}/${token}` : ""
|
|
14171
|
+
};
|
|
14172
|
+
}
|
|
14173
|
+
},
|
|
14174
|
+
responseFor: (waitpointId) => responses.get(waitpointId)
|
|
14175
|
+
};
|
|
14176
|
+
}
|
|
14177
|
+
/** Re-enters a WAITING run that holds a COMPLETED waitpoint, through the
|
|
14178
|
+
* dispatcher. Whoever writes second, the resume or the WAITING, delivers. */
|
|
14179
|
+
async deliverHeld(runId) {
|
|
14180
|
+
if (this.closed) return this.answerShutDown(runId);
|
|
14181
|
+
if (this.resuming.has(runId)) return;
|
|
14182
|
+
try {
|
|
14183
|
+
const store = await this.store();
|
|
14184
|
+
if (!store || !await store.hasCompletedWaitpoint(runId)) return;
|
|
14185
|
+
const run = await store.getRun(runId);
|
|
14186
|
+
if (run?.status !== "WAITING") return this.answerNotWaiting(store, runId, run?.status);
|
|
14187
|
+
if (this.resuming.has(runId)) return;
|
|
14188
|
+
this.resuming.add(runId);
|
|
14189
|
+
this.resumeRun(runId, run.workflow_id);
|
|
14190
|
+
} catch (error) {
|
|
14191
|
+
this.logger.warn(`Could not resume run ${runId}: @error`, error);
|
|
14192
|
+
}
|
|
14193
|
+
}
|
|
14194
|
+
answerNotWaiting(store, runId, status) {
|
|
14195
|
+
if (store.isRunInFlight(runId)) return;
|
|
14196
|
+
this.answerUnresumed(runId, {
|
|
14197
|
+
status: status ?? "gone",
|
|
14198
|
+
error: "The run is not waiting to resume in this runtime",
|
|
14199
|
+
code: 409
|
|
14200
|
+
});
|
|
14201
|
+
}
|
|
14202
|
+
waitpointPoller = new WaitpointPoller({
|
|
14203
|
+
store: () => this.store(),
|
|
14204
|
+
resume: (runId) => this.deliverHeld(runId),
|
|
14205
|
+
ended: (end) => Promise.resolve(this.answerClosed(end.closed, end.runId, {
|
|
14206
|
+
status: CANCELLED_RUN_STATUS,
|
|
14207
|
+
error: end.reason
|
|
14208
|
+
})),
|
|
14209
|
+
logger: { warn: (message, ...args) => this.logger.warn(message, ...args) }
|
|
14210
|
+
});
|
|
14211
|
+
unobserveSuspended;
|
|
14212
|
+
/** Test hook: one waitpoint poll pass now; a no-op until the supervisor starts. */
|
|
14213
|
+
pollWaitpointsNow() {
|
|
14214
|
+
return this.waitpointPoller.poll();
|
|
14215
|
+
}
|
|
14216
|
+
startWaitpointPoller() {
|
|
14217
|
+
if (this.closed) return;
|
|
14218
|
+
this.unobserveSuspended ??= this.telemetry.observeSuspended(() => this.waitpointPoller.suspendedRuns());
|
|
14219
|
+
this.waitpointPoller.start();
|
|
14220
|
+
}
|
|
14221
|
+
stopWaitpointPoller() {
|
|
14222
|
+
this.waitpointPoller.stop();
|
|
14223
|
+
this.unobserveSuspended?.();
|
|
14224
|
+
this.unobserveSuspended = void 0;
|
|
14225
|
+
}
|
|
14226
|
+
async answerHeldResumes(store, runId, error) {
|
|
14227
|
+
if (this.syncResumes.size === 0) return;
|
|
14228
|
+
const rows = (await store.listWaitpointsForRuns([runId])).get(runId) ?? [];
|
|
14229
|
+
this.answerClosed(rows.map((row) => row.id), runId, {
|
|
14230
|
+
status: CANCELLED_RUN_STATUS,
|
|
14231
|
+
error
|
|
14232
|
+
});
|
|
14233
|
+
}
|
|
14234
|
+
answerClosed(waitpointIds, runId, end) {
|
|
14235
|
+
if (this.syncResumes.size === 0) return;
|
|
14236
|
+
for (const id of waitpointIds) this.heldResume(id)?.closed(runId, end.status, end.error);
|
|
14237
|
+
}
|
|
14238
|
+
heldResume(waitpointId) {
|
|
14239
|
+
const candidates = [...this.syncResumes.get(waitpointId) ?? []];
|
|
14240
|
+
if (candidates.length === 0) return void 0;
|
|
14241
|
+
const winner = (async () => {
|
|
14242
|
+
for (const candidate of candidates) if (await candidate.won) return candidate.entry;
|
|
14243
|
+
})();
|
|
14244
|
+
return {
|
|
14245
|
+
respond: (response) => void winner.then((e) => e?.respond(response)),
|
|
14246
|
+
ended: (result) => void winner.then((e) => e?.ended(result)),
|
|
14247
|
+
closed: (runId, status, error, code) => void winner.then((e) => e?.closed(runId, status, error, code))
|
|
14248
|
+
};
|
|
14249
|
+
}
|
|
14250
|
+
answerUnresumed(runId, end) {
|
|
14251
|
+
for (const [id, candidates] of this.syncResumes) {
|
|
14252
|
+
if (![...candidates].some((candidate) => candidate.runId === runId)) continue;
|
|
14253
|
+
this.heldResume(id)?.closed(runId, end.status, end.error, end.code);
|
|
14254
|
+
}
|
|
14255
|
+
}
|
|
14256
|
+
async deliverAllHeld() {
|
|
14257
|
+
try {
|
|
14258
|
+
const store = await this.store();
|
|
14259
|
+
for (const runId of await store?.heldRunIds() ?? []) await this.deliverHeld(runId);
|
|
14260
|
+
} catch (error) {
|
|
14261
|
+
this.logger.warn("Could not deliver held resumes: @error", error);
|
|
14262
|
+
}
|
|
14263
|
+
}
|
|
14264
|
+
resumeRun(runId, workflowId) {
|
|
14265
|
+
const started = performance.now();
|
|
14266
|
+
const attributes = {
|
|
14267
|
+
"workflow.id": workflowId,
|
|
14268
|
+
"trigger.kind": "resume",
|
|
14269
|
+
"run.id": runId
|
|
14270
|
+
};
|
|
14271
|
+
return this.telemetry.span("workflow.run", attributes, async (span) => {
|
|
14272
|
+
let status = "REFUSED";
|
|
14273
|
+
try {
|
|
14274
|
+
const result = await this.runResumed(runId, workflowId);
|
|
14275
|
+
if (result) status = result.status;
|
|
14276
|
+
if (status === "FAILED") span.setStatus({
|
|
14277
|
+
code: SpanStatusCode.ERROR,
|
|
14278
|
+
message: status
|
|
14279
|
+
});
|
|
14280
|
+
return result;
|
|
14281
|
+
} finally {
|
|
14282
|
+
span.setAttribute("run.status", status);
|
|
14283
|
+
this.telemetry.recordRun(performance.now() - started, status, "resume");
|
|
14284
|
+
}
|
|
14285
|
+
});
|
|
14286
|
+
}
|
|
14287
|
+
async runResumed(runId, workflowId) {
|
|
14288
|
+
const admission = await this.runGate.admit(workflowId, await this.resumePolicy(workflowId), void 0, { resume: true });
|
|
14289
|
+
if (!admission.admitted) {
|
|
14290
|
+
this.resuming.delete(runId);
|
|
14291
|
+
this.answerShutDown(runId);
|
|
14292
|
+
return;
|
|
14293
|
+
}
|
|
14294
|
+
let dispatched;
|
|
14295
|
+
try {
|
|
14296
|
+
dispatched = await this.dispatcher().admit(workflowId);
|
|
14297
|
+
} catch {
|
|
14298
|
+
admission.release();
|
|
14299
|
+
this.resuming.delete(runId);
|
|
14300
|
+
this.answerShutDown(runId);
|
|
14301
|
+
return;
|
|
14302
|
+
}
|
|
14303
|
+
const store = await this.store();
|
|
14304
|
+
let claimed = false;
|
|
14305
|
+
let redeliver = false;
|
|
14306
|
+
let session;
|
|
14307
|
+
let held;
|
|
14308
|
+
let outcome;
|
|
14309
|
+
try {
|
|
14310
|
+
if (!store || this.closed) {
|
|
14311
|
+
this.answerShutDown(runId);
|
|
14312
|
+
return;
|
|
14313
|
+
}
|
|
14314
|
+
claimed = await store.claimWaitingRun(runId);
|
|
14315
|
+
this.resuming.delete(runId);
|
|
14316
|
+
if (!claimed) {
|
|
14317
|
+
this.answerNotWaiting(store, runId, (await store.getRun(runId))?.status);
|
|
14318
|
+
return;
|
|
14319
|
+
}
|
|
14320
|
+
const row = await store.getRun(runId);
|
|
14321
|
+
if (!row) throw new Error(`Run "${runId}" not found`);
|
|
14322
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14323
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14324
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
14325
|
+
const current = document.state.global;
|
|
14326
|
+
const stopped = await this.resumeStopped(workflowId, row.trigger_kind, current);
|
|
14327
|
+
if (stopped) {
|
|
14328
|
+
const closed = await store.endWait(runId, {
|
|
14329
|
+
status: CANCELLED_RUN_STATUS,
|
|
14330
|
+
reason: stopped.reason,
|
|
14331
|
+
errorName: stopped.errorName,
|
|
14332
|
+
waitpoints: "CANCELLED"
|
|
14333
|
+
});
|
|
14334
|
+
claimed = false;
|
|
14335
|
+
this.answerClosed(closed, runId, {
|
|
14336
|
+
status: CANCELLED_RUN_STATUS,
|
|
14337
|
+
error: stopped.reason
|
|
14338
|
+
});
|
|
14339
|
+
return {
|
|
14340
|
+
status: CANCELLED_RUN_STATUS,
|
|
14341
|
+
steps: [],
|
|
14342
|
+
runId,
|
|
14343
|
+
error: stopped.reason,
|
|
14344
|
+
errorName: stopped.errorName,
|
|
14345
|
+
refusal: "cancelled"
|
|
14346
|
+
};
|
|
14347
|
+
}
|
|
14348
|
+
const consumed = await store.consumeCompletedWaitpoint(runId);
|
|
14349
|
+
if (!consumed) {
|
|
14350
|
+
await store.returnToWaiting(runId);
|
|
14351
|
+
claimed = false;
|
|
14352
|
+
redeliver = true;
|
|
14353
|
+
return;
|
|
14354
|
+
}
|
|
14355
|
+
const { waitpoint } = consumed;
|
|
14356
|
+
held = this.heldResume(waitpoint.id);
|
|
14357
|
+
this.answerClosed(consumed.closed, runId, {
|
|
14358
|
+
status: "RUNNING",
|
|
14359
|
+
error: "The run resumed on another of its waitpoints"
|
|
14360
|
+
});
|
|
14361
|
+
const state = await this.pinnedState(workflowId, runId, store, current);
|
|
14362
|
+
const runnable = runnableDefinition(state);
|
|
14363
|
+
const definition = toWorkflowDefinition(state);
|
|
14364
|
+
const policy = effectiveRunPolicy(runnable);
|
|
14365
|
+
const connections = declaredConnectionIds(definition);
|
|
14366
|
+
const reactorConnections = declaredReactorConnectionIds(definition);
|
|
14367
|
+
const runUser = reactorConnections.size === 0 ? void 0 : await assertReactorConnectionsReadable(this.host, workflowId, reactorConnections);
|
|
14368
|
+
const journal = carriedFromJournal(await store.getSteps(runId));
|
|
14369
|
+
if (!journal.waitingStepId) throw new Error(`Run "${runId}" has no waiting step to resume`);
|
|
14370
|
+
session = this.workers().session();
|
|
14371
|
+
const { result } = await this.executeLeg({
|
|
14372
|
+
workflowId,
|
|
14373
|
+
runId,
|
|
14374
|
+
store,
|
|
14375
|
+
session,
|
|
14376
|
+
definition,
|
|
14377
|
+
runnable,
|
|
14378
|
+
policy,
|
|
14379
|
+
deadline: policy.runTimeoutSeconds ? Date.now() + policy.runTimeoutSeconds * 1e3 : void 0,
|
|
14380
|
+
connections,
|
|
14381
|
+
runUser,
|
|
14382
|
+
triggerPayload: journaledTriggerPayload(row.trigger_payload),
|
|
14383
|
+
executor: this.blockExecutor(store),
|
|
14384
|
+
carriedSteps: journal.carried,
|
|
14385
|
+
resume: {
|
|
14386
|
+
stepId: journal.waitingStepId,
|
|
14387
|
+
payload: parseJson(waitpoint.resume_payload) ?? {}
|
|
14388
|
+
},
|
|
14389
|
+
firstOrdinal: journal.nextOrdinal,
|
|
14390
|
+
written: journal.attachmentRefs,
|
|
14391
|
+
handed: await store.getRunDocuments(runId),
|
|
14392
|
+
...held ? { respond: held.respond } : {}
|
|
14393
|
+
});
|
|
14394
|
+
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(row.trigger_kind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
14395
|
+
outcome = {
|
|
14396
|
+
...result,
|
|
14397
|
+
runId
|
|
14398
|
+
};
|
|
14399
|
+
return outcome;
|
|
14400
|
+
} catch (error) {
|
|
14401
|
+
if (store && claimed) {
|
|
14402
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
14403
|
+
const errorName = errorNameOf(error) ?? "Error";
|
|
14404
|
+
const closed = await store.endWait(runId, {
|
|
14405
|
+
status: "FAILED",
|
|
14406
|
+
reason: message,
|
|
14407
|
+
errorName,
|
|
14408
|
+
waitpoints: "CANCELLED"
|
|
14409
|
+
}).catch(() => []);
|
|
14410
|
+
this.answerClosed(closed, runId, {
|
|
14411
|
+
status: "FAILED",
|
|
14412
|
+
error: message
|
|
14413
|
+
});
|
|
14414
|
+
outcome = {
|
|
14415
|
+
status: "FAILED",
|
|
14416
|
+
steps: [],
|
|
14417
|
+
runId,
|
|
14418
|
+
error: message,
|
|
14419
|
+
errorName
|
|
14420
|
+
};
|
|
14421
|
+
}
|
|
14422
|
+
this.logger.warn(`Resuming run ${runId} failed: @error`, error);
|
|
14423
|
+
return;
|
|
14424
|
+
} finally {
|
|
14425
|
+
this.resuming.delete(runId);
|
|
14426
|
+
session?.close();
|
|
14427
|
+
dispatched();
|
|
14428
|
+
admission.release();
|
|
14429
|
+
held?.ended(outcome);
|
|
14430
|
+
if (redeliver) this.deliverHeld(runId);
|
|
14431
|
+
}
|
|
14432
|
+
}
|
|
14433
|
+
answerShutDown(runId) {
|
|
14434
|
+
if (!this.closed) return;
|
|
14435
|
+
this.answerUnresumed(runId, {
|
|
14436
|
+
status: WAITING_RUN_STATUS,
|
|
14437
|
+
error: "The runtime shut down before the run could resume",
|
|
14438
|
+
code: 503
|
|
14439
|
+
});
|
|
14440
|
+
}
|
|
14441
|
+
async resumePolicy(workflowId) {
|
|
14442
|
+
try {
|
|
14443
|
+
let state = this.workflowCache.get(workflowId)?.state;
|
|
14444
|
+
if (!state) {
|
|
14445
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14446
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14447
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
14448
|
+
state = document.state.global;
|
|
14449
|
+
}
|
|
14450
|
+
return effectiveRunPolicy(runnableDefinition(state));
|
|
14451
|
+
} catch {
|
|
14452
|
+
return UNENFORCED_POLICY;
|
|
14453
|
+
}
|
|
14454
|
+
}
|
|
14455
|
+
async resumeStopped(workflowId, triggerKind, current) {
|
|
14456
|
+
if (current.status !== "ENABLED") return {
|
|
14457
|
+
reason: `The workflow is ${current.status}; a run waiting on it does not resume`,
|
|
14458
|
+
errorName: WORKFLOW_DISABLED_ERROR_NAME
|
|
14459
|
+
};
|
|
14460
|
+
const parked = await this.parkedFiring(workflowId, triggerKind, runnableDefinition(current).version);
|
|
14461
|
+
return parked ? {
|
|
14462
|
+
reason: parked,
|
|
14463
|
+
errorName: WORKFLOW_PARKED_ERROR_NAME
|
|
14464
|
+
} : void 0;
|
|
14465
|
+
}
|
|
14466
|
+
async pinnedState(workflowId, runId, store, current) {
|
|
14467
|
+
const pin = await store.getRunPin(runId);
|
|
14468
|
+
if (!pin?.definition_hash) return current;
|
|
14469
|
+
if (this.definitionHash(workflowId, current) === pin.definition_hash) return current;
|
|
14470
|
+
return this.rebuildWorkflowState(workflowId, pin);
|
|
14471
|
+
}
|
|
14472
|
+
async rebuildWorkflowState(workflowId, pin) {
|
|
14473
|
+
const revision = pin.definition_op_index ?? 0;
|
|
14474
|
+
const operations = [];
|
|
14475
|
+
let page = await this.host.reactorClient.getOperations(workflowId, { scopes: ["document", "global"] });
|
|
14476
|
+
operations.push(...page.results);
|
|
14477
|
+
while (page.next) {
|
|
14478
|
+
page = await page.next();
|
|
14479
|
+
operations.push(...page.results);
|
|
14480
|
+
}
|
|
14481
|
+
const applied = (operation) => !operation.error && !isDenied(operation);
|
|
14482
|
+
const past = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index >= revision)).at(0);
|
|
14483
|
+
const atOrBeforePin = (operation) => {
|
|
14484
|
+
const at = operation.action.input.revision?.global;
|
|
14485
|
+
if (at !== void 0) return at <= revision;
|
|
14486
|
+
return !past || operation.timestampUtcMs <= past.timestampUtcMs;
|
|
14487
|
+
};
|
|
14488
|
+
const seed = sortOperations(operations.filter((operation) => operation.action.scope === "document" && operation.action.type === "UPGRADE_DOCUMENT" && applied(operation) && atOrBeforePin(operation))).map((operation) => operation.action.input.initialState).filter((state) => state?.global !== void 0).at(-1);
|
|
14489
|
+
let document = utils.createDocument(seed);
|
|
14490
|
+
const global = sortOperations(operations.filter((operation) => operation.action.scope === "global" && operation.index < revision));
|
|
14491
|
+
for (const operation of garbageCollect(global).filter(applied)) document = reducer(document, operation.action);
|
|
14492
|
+
const state = document.state.global;
|
|
14493
|
+
if (scopeHash(state) !== pin.definition_scope_hash || this.definitionHash(workflowId, state) !== pin.definition_hash) throw new Error(`The workflow was re-published while this run waited, and the definition it started from could not be rebuilt at revision ${revision}`);
|
|
14494
|
+
return state;
|
|
14495
|
+
}
|
|
14496
|
+
/** Resolves a WEBHOOK waitpoint from the editor, as the calling DID. */
|
|
14497
|
+
async resolveWaitpoint(waitpointId, input, ctx) {
|
|
14498
|
+
const store = await this.store();
|
|
14499
|
+
if (!store) throw new Error("Run journal is unavailable");
|
|
14500
|
+
const waitpoint = await store.getWaitpoint(waitpointId);
|
|
14501
|
+
if (!waitpoint) throw new Error(`Waitpoint "${waitpointId}" not found`);
|
|
14502
|
+
await this.assertCanReadDocument(waitpoint.workflow_id, ctx);
|
|
14503
|
+
await this.assertCanWriteDocument(waitpoint.workflow_id, ctx);
|
|
14504
|
+
const owner = await store.getRun(waitpoint.run_id);
|
|
14505
|
+
if (!owner || (await this.servedRuns([owner], ctx)).length === 0) throw new Error(`Waitpoint "${waitpointId}" not found`);
|
|
14506
|
+
if (waitpoint.type !== "WEBHOOK") throw new Error(`Waitpoint "${waitpointId}" is a ${waitpoint.type}`);
|
|
14507
|
+
const decidedBy = this.callerDid(ctx);
|
|
14508
|
+
const approval = parseJson(waitpoint.approval);
|
|
14509
|
+
if (approval && approval.approvers.length > 0) {
|
|
14510
|
+
if (!(decidedBy !== void 0 && approval.approvers.some((approver) => sameApprover(approver, decidedBy)))) throw new Error("You are not one of this approval's approvers");
|
|
14511
|
+
}
|
|
14512
|
+
const reason = input.reason?.trim() || void 0;
|
|
14513
|
+
if ((approval?.reasonRequiredOn === "both" || approval?.reasonRequiredOn === "reject" && !input.approved) && !reason) throw new Error("A reason is required for this decision");
|
|
14514
|
+
const decidedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14515
|
+
const payload = {
|
|
14516
|
+
queryParams: {
|
|
14517
|
+
action: input.approved ? "approve" : "disapprove",
|
|
14518
|
+
...reason ? { reason } : {}
|
|
14519
|
+
},
|
|
14520
|
+
body: {
|
|
14521
|
+
approved: input.approved,
|
|
14522
|
+
reason: reason ?? null,
|
|
14523
|
+
decidedBy: decidedBy ?? null,
|
|
14524
|
+
decidedAt
|
|
14525
|
+
},
|
|
14526
|
+
headers: {}
|
|
14527
|
+
};
|
|
14528
|
+
const completed = await store.completeWaitpoint(waitpointId, {
|
|
14529
|
+
payload,
|
|
14530
|
+
decidedBy: decidedBy ?? null,
|
|
14531
|
+
resolvedAt: decidedAt
|
|
14532
|
+
});
|
|
14533
|
+
if (!completed) {
|
|
14534
|
+
const now = await store.getWaitpoint(waitpointId);
|
|
14535
|
+
const run = await store.getRun(waitpoint.run_id);
|
|
14536
|
+
const live = run?.status === "RUNNING" || run?.status === "WAITING";
|
|
14537
|
+
const status = now?.status !== "PENDING" ? now?.status ?? "gone" : waitpointExpired(now) ? "EXPIRED" : live ? "not the one its run waits on" : `on a ${run?.status ?? "deleted"} run`;
|
|
14538
|
+
throw new Error(`Waitpoint "${waitpointId}" is ${status}; it can no longer be resolved`);
|
|
14539
|
+
}
|
|
14540
|
+
await this.deliverHeld(completed.run_id);
|
|
14541
|
+
return waitpointRecord(completed);
|
|
14542
|
+
}
|
|
14543
|
+
/** A POST to a resume URL. Single-use: only a PENDING waitpoint resumes. */
|
|
14544
|
+
resumeWaitpointByToken(token, payload) {
|
|
14545
|
+
return this.resumeByToken(token, payload);
|
|
14546
|
+
}
|
|
14547
|
+
/** A POST to a resume URL's `/sync`, held like a sync webhook until the
|
|
14548
|
+
* resumed leg responds, waits, ends, or times out. The first answer wins. */
|
|
14549
|
+
async resumeWaitpointByTokenSync(token, payload, options = {}) {
|
|
14550
|
+
let hold;
|
|
14551
|
+
try {
|
|
14552
|
+
const { status } = await this.resumeByToken(token, payload, (id, runId) => {
|
|
14553
|
+
hold = this.holdSyncResume(id, runId);
|
|
14554
|
+
return hold.decided;
|
|
14555
|
+
});
|
|
14556
|
+
if (status !== "resumed" || !hold) return { status };
|
|
14557
|
+
return {
|
|
14558
|
+
status,
|
|
14559
|
+
reply: await hold.reply(options.timeoutMs ?? DELIVERY_TIMEOUT_MS)
|
|
14560
|
+
};
|
|
14561
|
+
} finally {
|
|
14562
|
+
hold?.release();
|
|
14563
|
+
}
|
|
14564
|
+
}
|
|
14565
|
+
holdSyncResume(waitpointId, runId) {
|
|
14566
|
+
let settle = () => void 0;
|
|
14567
|
+
const answer = new Promise((resolve) => {
|
|
14568
|
+
settle = resolve;
|
|
14569
|
+
});
|
|
14570
|
+
let done = false;
|
|
14571
|
+
const finish = (reply) => {
|
|
14572
|
+
if (done) return;
|
|
14573
|
+
done = true;
|
|
14574
|
+
settle(reply);
|
|
14575
|
+
};
|
|
14576
|
+
const entry = {
|
|
14577
|
+
respond: (response) => finish(runResponseReply(response)),
|
|
14578
|
+
ended: (result) => finish(result ? runResultReply(result, 200) : {
|
|
14579
|
+
status: 500,
|
|
14580
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14581
|
+
body: JSON.stringify({ error: "The run could not resume" })
|
|
14582
|
+
}),
|
|
14583
|
+
closed: (runId, status, error, code = 409) => finish({
|
|
14584
|
+
status: code,
|
|
14585
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14586
|
+
body: JSON.stringify({
|
|
14587
|
+
runId,
|
|
14588
|
+
status,
|
|
14589
|
+
error
|
|
14590
|
+
})
|
|
14591
|
+
})
|
|
14592
|
+
};
|
|
14593
|
+
let decided = () => void 0;
|
|
14594
|
+
const candidate = {
|
|
14595
|
+
entry,
|
|
14596
|
+
runId,
|
|
14597
|
+
won: new Promise((resolve) => {
|
|
14598
|
+
decided = resolve;
|
|
14599
|
+
})
|
|
14600
|
+
};
|
|
14601
|
+
const held = this.syncResumes.get(waitpointId) ?? /* @__PURE__ */ new Set();
|
|
14602
|
+
this.syncResumes.set(waitpointId, held);
|
|
14603
|
+
held.add(candidate);
|
|
14604
|
+
return {
|
|
14605
|
+
decided,
|
|
14606
|
+
reply: async (timeoutMs) => {
|
|
14607
|
+
const raced = await Promise.race([answer, timeout(timeoutMs)]);
|
|
14608
|
+
if (raced !== TIMED_OUT) return raced;
|
|
14609
|
+
done = true;
|
|
14610
|
+
return {
|
|
14611
|
+
status: 504,
|
|
14612
|
+
contentType: JSON_CONTENT_TYPE,
|
|
14613
|
+
body: JSON.stringify({
|
|
14614
|
+
status: "RUNNING",
|
|
14615
|
+
error: "The run did not finish in time"
|
|
14616
|
+
})
|
|
14617
|
+
};
|
|
14618
|
+
},
|
|
14619
|
+
release: () => {
|
|
14620
|
+
decided(false);
|
|
14621
|
+
held.delete(candidate);
|
|
14622
|
+
if (held.size === 0 && this.syncResumes.get(waitpointId) === held) this.syncResumes.delete(waitpointId);
|
|
14623
|
+
}
|
|
14624
|
+
};
|
|
14625
|
+
}
|
|
14626
|
+
async resumeByToken(token, payload, beforeComplete) {
|
|
14627
|
+
const store = await this.store();
|
|
14628
|
+
if (!store || !token) return { status: "gone" };
|
|
14629
|
+
const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
|
|
14630
|
+
if (!waitpoint || waitpoint.kind !== "webhook") return { status: "gone" };
|
|
14631
|
+
let completed;
|
|
14632
|
+
if (waitpoint.status === "PENDING") {
|
|
14633
|
+
const decided = beforeComplete?.(waitpoint.id, waitpoint.run_id);
|
|
14634
|
+
try {
|
|
14635
|
+
completed = await store.completeWaitpoint(waitpoint.id, { payload });
|
|
14636
|
+
} finally {
|
|
14637
|
+
decided?.(completed !== void 0);
|
|
14638
|
+
}
|
|
14639
|
+
}
|
|
14640
|
+
if (!completed) {
|
|
14641
|
+
const now = (await store.getWaitpoint(waitpoint.id))?.status;
|
|
14642
|
+
if (now === "CONSUMED") return { status: "already" };
|
|
14643
|
+
const run = now === "COMPLETED" ? await store.getRun(waitpoint.run_id) : void 0;
|
|
14644
|
+
return { status: run?.status === "RUNNING" || run?.status === "WAITING" ? "already" : "gone" };
|
|
14645
|
+
}
|
|
14646
|
+
await this.deliverHeld(completed.run_id);
|
|
14647
|
+
return { status: "resumed" };
|
|
14648
|
+
}
|
|
14649
|
+
/** What a resume URL's confirm page shows. */
|
|
14650
|
+
async describeWaitpointByToken(token) {
|
|
14651
|
+
const store = await this.store();
|
|
14652
|
+
if (!store || !token) return void 0;
|
|
14653
|
+
const waitpoint = await store.getWaitpointByTokenHash(tokenHash(token));
|
|
14654
|
+
if (!waitpoint || waitpoint.kind !== "webhook") return void 0;
|
|
14655
|
+
const run = await store.getRun(waitpoint.run_id);
|
|
14656
|
+
const status = !(run?.status === "RUNNING" || run?.status === "WAITING") && (waitpoint.status === "PENDING" || waitpoint.status === "COMPLETED") ? "CANCELLED" : waitpoint.status;
|
|
14657
|
+
const state = await this.workflowStateForPage(waitpoint.workflow_id);
|
|
14658
|
+
const step = state?.steps.find((entry) => entry.id === waitpoint.step_id);
|
|
14659
|
+
const action = step ? await this.blockDisplayName({
|
|
14660
|
+
pieceName: step.pieceName,
|
|
14661
|
+
pieceVersion: step.pieceVersion,
|
|
14662
|
+
kind: "action",
|
|
14663
|
+
name: step.actionName
|
|
14664
|
+
}) : void 0;
|
|
14665
|
+
const trigger = state?.trigger ? await this.blockDisplayName({
|
|
14666
|
+
pieceName: state.trigger.pieceName,
|
|
14667
|
+
pieceVersion: state.trigger.pieceVersion,
|
|
14668
|
+
kind: "trigger",
|
|
14669
|
+
name: state.trigger.triggerName
|
|
14670
|
+
}) ?? state.trigger.triggerName : void 0;
|
|
14671
|
+
return {
|
|
14672
|
+
workflowName: run?.workflow_name || state?.name || "",
|
|
14673
|
+
stepName: step?.name || waitpoint.step_key,
|
|
14674
|
+
status,
|
|
14675
|
+
...state?.description ? { workflowDescription: state.description } : {},
|
|
14676
|
+
...action ? { actionName: action } : {},
|
|
14677
|
+
waitingSince: waitpoint.created_at,
|
|
14678
|
+
...waitpoint.expires_at ? { expiresAt: waitpoint.expires_at } : {},
|
|
14679
|
+
...run ? { startedAt: run.enqueued_at } : {},
|
|
14680
|
+
...trigger ? { triggerName: trigger } : {}
|
|
14681
|
+
};
|
|
14682
|
+
}
|
|
14683
|
+
async workflowStateForPage(workflowId) {
|
|
14684
|
+
const cached = this.workflowCache.get(workflowId)?.state;
|
|
14685
|
+
if (cached) return cached;
|
|
14686
|
+
try {
|
|
14687
|
+
const generation = this.workflowGeneration(workflowId);
|
|
14688
|
+
const document = await this.host.reactorClient.get(workflowId);
|
|
14689
|
+
if (document.header.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
|
|
14690
|
+
this.cacheLoaded(workflowId, generation, document);
|
|
14691
|
+
return document.state.global;
|
|
14692
|
+
} catch {
|
|
14693
|
+
return;
|
|
14694
|
+
}
|
|
14695
|
+
}
|
|
14696
|
+
async blockDisplayName(block) {
|
|
14697
|
+
try {
|
|
14698
|
+
const parsed = resolvedBlock(await this.resolveBlock(block));
|
|
14699
|
+
if (!parsed) return void 0;
|
|
14700
|
+
const descriptor = await this.pieceDescriptor(targetOf(parsed));
|
|
14701
|
+
return (block.kind === "trigger" ? descriptor.triggers : descriptor.actions).find((entry) => entry.name === parsed.name)?.displayName;
|
|
14702
|
+
} catch {
|
|
14703
|
+
return;
|
|
14704
|
+
}
|
|
14705
|
+
}
|
|
14706
|
+
/** The waitpoints of one run, for a caller who may read the run. */
|
|
14707
|
+
async waitpointsForRun(runId, ctx) {
|
|
14708
|
+
const store = await this.store();
|
|
14709
|
+
if (!store || !await this.run(runId, ctx)) return [];
|
|
14710
|
+
return ((await store.listWaitpointsForRuns([runId])).get(runId) ?? []).map(waitpointRecord);
|
|
14711
|
+
}
|
|
14712
|
+
/** WAITING runs this caller may read, oldest first, with their waitpoints. */
|
|
14713
|
+
async waitingRuns(args, ctx) {
|
|
14714
|
+
const store = await this.store();
|
|
14715
|
+
if (!store || !ctx) return [];
|
|
14716
|
+
let scope;
|
|
14717
|
+
if (args.workflowId) {
|
|
14718
|
+
await this.assertCanReadDocument(args.workflowId, ctx);
|
|
14719
|
+
scope = args.workflowId;
|
|
14720
|
+
} else if (args.driveId) {
|
|
14721
|
+
scope = await this.driveWorkflowIds(args.driveId, ctx);
|
|
14722
|
+
if (scope.length === 0) return [];
|
|
14723
|
+
}
|
|
14724
|
+
const rows = await this.servedRuns(await this.readableRows(await store.listWaitingRuns(scope, args.first ?? 50), (row) => row.workflow_id, ctx), ctx);
|
|
14725
|
+
const ids = rows.map((row) => row.id);
|
|
14726
|
+
const steps = await store.getStepsForRuns(ids, { withData: false });
|
|
14727
|
+
const waitpoints = await store.listWaitpointsForRuns(ids);
|
|
14728
|
+
return rows.map((row) => ({
|
|
14729
|
+
run: {
|
|
14730
|
+
row,
|
|
14731
|
+
steps: steps.get(row.id) ?? []
|
|
14732
|
+
},
|
|
14733
|
+
waitpoints: (waitpoints.get(row.id) ?? []).map(waitpointRecord)
|
|
14734
|
+
}));
|
|
14735
|
+
}
|
|
14736
|
+
/** Cancels a WAITING or PENDING run; its waitpoints stop resuming. */
|
|
14737
|
+
async cancelRun(runId, ctx) {
|
|
14738
|
+
const store = await this.store();
|
|
14739
|
+
if (!store) throw new Error("Run journal is unavailable");
|
|
14740
|
+
const run = await store.getRun(runId);
|
|
14741
|
+
if (!run) throw new Error(`Run "${runId}" not found`);
|
|
14742
|
+
await this.assertCanReadDocument(run.workflow_id, ctx);
|
|
14743
|
+
await this.assertCanWriteDocument(run.workflow_id, ctx);
|
|
14744
|
+
if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
|
|
14745
|
+
const refuse = (status) => status === "RUNNING" ? /* @__PURE__ */ new Error("Cancelling a RUNNING run is not supported yet (#3226)") : /* @__PURE__ */ new Error(`Run is ${status}; only WAITING and PENDING runs can be cancelled`);
|
|
14746
|
+
if (run.status !== "WAITING" && run.status !== "PENDING") throw refuse(run.status);
|
|
14747
|
+
const by = this.callerDid(ctx);
|
|
14748
|
+
const reason = by ? `Cancelled by ${by}` : "Cancelled";
|
|
14749
|
+
if (!await store.cancelIdleRun(runId, reason)) throw refuse((await store.getRun(runId))?.status ?? "gone");
|
|
14750
|
+
await this.answerHeldResumes(store, runId, reason);
|
|
14751
|
+
const row = await store.getRun(runId);
|
|
14752
|
+
this.queuedReplies.get(runId)?.({
|
|
14753
|
+
status: row.status,
|
|
14754
|
+
steps: [],
|
|
14755
|
+
runId,
|
|
14756
|
+
...row.error ? { error: row.error } : {},
|
|
14757
|
+
refusal: "cancelled"
|
|
14758
|
+
});
|
|
14759
|
+
return {
|
|
14760
|
+
row,
|
|
14761
|
+
steps: await store.getSteps(runId)
|
|
14762
|
+
};
|
|
14763
|
+
}
|
|
14764
|
+
callerDid(ctx) {
|
|
14765
|
+
if (!ctx) return void 0;
|
|
14766
|
+
const subject = this.host.subjectOf?.(ctx);
|
|
14767
|
+
if (subject?.address) return subject.address;
|
|
14768
|
+
const user = ctx.user;
|
|
14769
|
+
return typeof user?.address === "string" ? user.address : void 0;
|
|
14770
|
+
}
|
|
13262
14771
|
async testStep(workflowId, stepId, ctx, access = {}) {
|
|
13263
14772
|
await this.assertCanReadWorkflow(workflowId, ctx, access.driveId);
|
|
13264
14773
|
const document = await this.host.reactorClient.get(workflowId);
|
|
@@ -13567,6 +15076,112 @@ var WorkflowRuntimeService = class {
|
|
|
13567
15076
|
return (await this.host.reactorClient.get(workflowId)).state.global;
|
|
13568
15077
|
}
|
|
13569
15078
|
};
|
|
15079
|
+
function scopeHash(state) {
|
|
15080
|
+
return hashDocumentStateForScope({ state: { global: state } }, "global");
|
|
15081
|
+
}
|
|
15082
|
+
function tokenHash(token) {
|
|
15083
|
+
return createHash("sha256").update(token).digest("hex");
|
|
15084
|
+
}
|
|
15085
|
+
function waitpointExpired(waitpoint) {
|
|
15086
|
+
return waitpoint.expires_at !== null && Date.parse(waitpoint.expires_at) <= Date.now();
|
|
15087
|
+
}
|
|
15088
|
+
function parseJson(value) {
|
|
15089
|
+
return value === null ? null : JSON.parse(value);
|
|
15090
|
+
}
|
|
15091
|
+
const WORKFLOW_DISABLED_ERROR_NAME = "WorkflowDisabled";
|
|
15092
|
+
const WORKFLOW_PARKED_ERROR_NAME = "WorkflowParked";
|
|
15093
|
+
const MAX_WAITPOINT_ISO = "9999-12-31T23:59:59.999Z";
|
|
15094
|
+
const MAX_WAITPOINT_MS = Date.parse(MAX_WAITPOINT_ISO);
|
|
15095
|
+
function waitpointExpiry(createdAt, request, suspendMs) {
|
|
15096
|
+
const candidates = [request.expiresAt ? Date.parse(request.expiresAt) : NaN, suspendMs === null ? NaN : createdAt.getTime() + suspendMs].filter((at) => !Number.isNaN(at));
|
|
15097
|
+
return candidates.length > 0 ? new Date(Math.min(...candidates, MAX_WAITPOINT_MS)).toISOString() : null;
|
|
15098
|
+
}
|
|
15099
|
+
function sameApprover(approver, caller) {
|
|
15100
|
+
const tail = (value) => value.split(":").pop().toLowerCase();
|
|
15101
|
+
return approver.toLowerCase() === caller.toLowerCase() || tail(approver) === tail(caller);
|
|
15102
|
+
}
|
|
15103
|
+
function waitpointRecord(row) {
|
|
15104
|
+
return {
|
|
15105
|
+
id: row.id,
|
|
15106
|
+
runId: row.run_id,
|
|
15107
|
+
workflowId: row.workflow_id,
|
|
15108
|
+
stepId: row.step_id,
|
|
15109
|
+
stepKey: row.step_key,
|
|
15110
|
+
stepPath: row.step_path,
|
|
15111
|
+
pieceName: row.piece_name,
|
|
15112
|
+
blockName: row.block_name,
|
|
15113
|
+
type: row.type,
|
|
15114
|
+
kind: row.kind,
|
|
15115
|
+
status: row.status,
|
|
15116
|
+
createdAt: row.created_at,
|
|
15117
|
+
resumeAt: row.resume_at,
|
|
15118
|
+
expiresAt: row.expires_at,
|
|
15119
|
+
resolvedAt: row.resolved_at,
|
|
15120
|
+
decidedBy: row.decided_by,
|
|
15121
|
+
approval: parseJson(row.approval)
|
|
15122
|
+
};
|
|
15123
|
+
}
|
|
15124
|
+
function journaledValue(raw, what) {
|
|
15125
|
+
const value = raw === null ? void 0 : JSON.parse(raw);
|
|
15126
|
+
if (isTruncatedStepPayload(value)) return unavailableValue(`${what} is unavailable: the run journal truncated it past the payload cap`);
|
|
15127
|
+
return redactedLeavesUnavailable(value, what, []);
|
|
15128
|
+
}
|
|
15129
|
+
function redactedLeavesUnavailable(value, what, path) {
|
|
15130
|
+
if (!containsRedactedMarker(value)) return value;
|
|
15131
|
+
if (typeof value === "string") {
|
|
15132
|
+
const at = path.length > 0 ? ` at "${path.join(".")}"` : "";
|
|
15133
|
+
return unavailableValue(`${what} is unavailable: the run journal holds it redacted${at}`);
|
|
15134
|
+
}
|
|
15135
|
+
if (Array.isArray(value)) return value.map((entry, index) => redactedLeavesUnavailable(entry, what, [...path, index]));
|
|
15136
|
+
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redactedLeavesUnavailable(entry, what, [...path, key])]));
|
|
15137
|
+
}
|
|
15138
|
+
function journaledTriggerPayload(raw) {
|
|
15139
|
+
return journaledValue(raw, "The trigger payload");
|
|
15140
|
+
}
|
|
15141
|
+
const ATTACHMENT_REF_TEXT = /"(attachment:\/\/[^"\\]*)"/g;
|
|
15142
|
+
function carriedFromJournal(rows) {
|
|
15143
|
+
const carried = /* @__PURE__ */ new Map();
|
|
15144
|
+
const attachmentRefs = /* @__PURE__ */ new Set();
|
|
15145
|
+
let waitingStepId;
|
|
15146
|
+
let nextOrdinal = 0;
|
|
15147
|
+
for (const row of rows) {
|
|
15148
|
+
nextOrdinal = Math.max(nextOrdinal, row.ordinal + 1);
|
|
15149
|
+
for (const match of row.output?.matchAll(ATTACHMENT_REF_TEXT) ?? []) attachmentRefs.add(match[1]);
|
|
15150
|
+
const what = `Output of step "${row.step_key}"`;
|
|
15151
|
+
switch (row.status) {
|
|
15152
|
+
case "WAITING":
|
|
15153
|
+
waitingStepId = row.step_id;
|
|
15154
|
+
break;
|
|
15155
|
+
case "SUCCEEDED":
|
|
15156
|
+
case "REPLAYED":
|
|
15157
|
+
carried.set(row.step_id, {
|
|
15158
|
+
status: row.status,
|
|
15159
|
+
port: row.port,
|
|
15160
|
+
output: () => journaledValue(row.output, what)
|
|
15161
|
+
});
|
|
15162
|
+
break;
|
|
15163
|
+
case "FAILED":
|
|
15164
|
+
carried.set(row.step_id, {
|
|
15165
|
+
status: "FAILED",
|
|
15166
|
+
port: "error",
|
|
15167
|
+
output: () => void 0,
|
|
15168
|
+
error: row.error ?? ""
|
|
15169
|
+
});
|
|
15170
|
+
break;
|
|
15171
|
+
case "SKIPPED": if (row.port) carried.set(row.step_id, {
|
|
15172
|
+
status: "SKIPPED",
|
|
15173
|
+
port: row.port,
|
|
15174
|
+
output: () => null
|
|
15175
|
+
});
|
|
15176
|
+
}
|
|
15177
|
+
}
|
|
15178
|
+
return {
|
|
15179
|
+
carried,
|
|
15180
|
+
waitingStepId,
|
|
15181
|
+
nextOrdinal,
|
|
15182
|
+
attachmentRefs
|
|
15183
|
+
};
|
|
15184
|
+
}
|
|
13570
15185
|
/** The runtime a host composes: one instance, its lifetime the host's. */
|
|
13571
15186
|
function createWorkflowRuntime(deps) {
|
|
13572
15187
|
return new WorkflowRuntimeService(deps);
|
|
@@ -13859,6 +15474,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
|
|
|
13859
15474
|
}
|
|
13860
15475
|
};
|
|
13861
15476
|
//#endregion
|
|
13862
|
-
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, packagePieces, parseClientFile, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName };
|
|
15477
|
+
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, OAUTH_APP, OAUTH_GRANTED_SCOPES, OAUTH_SCOPES, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, StaticOAuthAppRegistry, TELEMETRY_SCOPE, WAITPOINT_EXPIRED_ERROR_NAME, WAITPOINT_NOT_LIVE_ERROR_NAME, WORKFLOW_DISABLED_ERROR_NAME, WORKFLOW_PACKAGE_NAME, WORKFLOW_PARKED_ERROR_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, clientKindFor, createAttachmentPort, createWorkflowRuntime, grantedScopes, loadOAuthApps, oauthSignInFor, packagePieces, parseClientFile, parseRelay, publishRunUser, scopesOverride, setPieceRegistryUrl, singletonOwnerName, waitpointRecord };
|
|
13863
15478
|
|
|
13864
15479
|
//# sourceMappingURL=index.js.map
|