@powerhousedao/reactor-workflow 6.2.3-dev.49 → 6.2.3-dev.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +362 -8
- package/dist/index.d.ts +446 -37
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2152 -305
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-wQBhKJUl.d.ts → piece-registry-BdkdVugw.d.ts} +233 -99
- package/dist/piece-registry-BdkdVugw.d.ts.map +1 -0
- package/dist/{piece-registry-CpiVMWNv.js → piece-registry-D0pa4v0T.js} +1226 -102
- package/dist/piece-registry-D0pa4v0T.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +91 -3
- package/dist/testing.js.map +1 -0
- package/dist/worker-entry.js +5685 -285
- package/dist/worker-entry.js.map +1 -1
- package/package.json +16 -8
- package/dist/loader-Gkx6I_Bo.js +0 -91
- package/dist/loader-Gkx6I_Bo.js.map +0 -1
- package/dist/piece-registry-CpiVMWNv.js.map +0 -1
- package/dist/piece-registry-wQBhKJUl.d.ts.map +0 -1
- package/dist/reactor-rpc-BqAjCK1p.js +0 -1134
- package/dist/reactor-rpc-BqAjCK1p.js.map +0 -1
- package/dist/worker-entry.d.ts +0 -2
package/dist/index.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
|
-
import { B as
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { dirname, join } from "node:path";
|
|
1
|
+
import { $ as isPrivateAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerError, H as PieceWorkerPool, I as SecretDeletedError, J as redact, K as PieceWorkerTimeoutError, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as DEFAULT_EGRESS_POLICY, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerDisposedError, X as rememberSecrets, Y as redactMessage, Z as secretsFor, _ as resolvedBlock, _t as isIndeterminateError, at as pieceModuleRef, b as resolutionOf, c as canonicalJson, ct as UnsupportedPieceFeatureError, d as DynamicPropertiesError, dt as unsupportedTrigger, et as parseAddress, f as checkDynamicProperties, g as isHostBound, gt as setPieceRegistryUrl, h as CompositeBlockExecutor, ht as pieceRegistrySource, i as packagePieces, it as bundleResolver, j as declaredReactorConnectionIds, l as hashOf, lt as authMethodFor, m as ActivepiecesBlockExecutor, mt as npmRegistryBaseUrl, n as installedPiece, nt as FileTooLargeError, o as errorNameOf, ot as sourcedResolver, p as effectiveRetryPolicy, q as containsRedactedMarker, r as installedPieces, rt as maxFileBytes, s as runWorkflow, t as PieceRegistry, tt as extractDedupeKey, u as stepConfigHash, ut as unsupportedAuth, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-D0pa4v0T.js";
|
|
2
|
+
import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
|
|
3
|
+
import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { open, readdir, rm, stat } from "node:fs/promises";
|
|
5
|
+
import path, { dirname, join } from "node:path";
|
|
7
6
|
import { childLogger, generateId } from "document-model";
|
|
8
7
|
import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
|
|
9
8
|
import { Cron } from "croner";
|
|
10
9
|
import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
10
|
+
import { Readable } from "node:stream";
|
|
11
11
|
import dns from "node:dns";
|
|
12
12
|
import net from "node:net";
|
|
13
13
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
14
|
+
import { hostname } from "node:os";
|
|
14
15
|
import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
|
|
15
16
|
import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
|
|
16
17
|
import http from "node:http";
|
|
@@ -18,6 +19,7 @@ import https from "node:https";
|
|
|
18
19
|
import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
|
|
19
20
|
import { Kind, parse } from "graphql";
|
|
20
21
|
import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
|
|
22
|
+
import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
|
|
21
23
|
import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
|
|
22
24
|
import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
23
25
|
//#region src/pieces/activepieces/piece-versions.ts
|
|
@@ -98,7 +100,15 @@ function npmVersions(name, options = {}) {
|
|
|
98
100
|
}
|
|
99
101
|
//#endregion
|
|
100
102
|
//#region src/reactor/attachment-port.ts
|
|
101
|
-
const logger$
|
|
103
|
+
const logger$7 = childLogger(["workflow", "attachments"]);
|
|
104
|
+
function fileStream(filePath) {
|
|
105
|
+
return Readable.toWeb(createReadStream(filePath));
|
|
106
|
+
}
|
|
107
|
+
async function sha256File(filePath) {
|
|
108
|
+
const hash = createHash("sha256");
|
|
109
|
+
for await (const chunk of createReadStream(filePath)) hash.update(chunk);
|
|
110
|
+
return hash.digest("hex");
|
|
111
|
+
}
|
|
102
112
|
function refOf(result) {
|
|
103
113
|
const direct = result.ref;
|
|
104
114
|
if (typeof direct === "string") return direct;
|
|
@@ -128,16 +138,24 @@ async function writeCapped(body, destPath, limit) {
|
|
|
128
138
|
}
|
|
129
139
|
await handle.close();
|
|
130
140
|
}
|
|
131
|
-
function createAttachmentPort(client,
|
|
141
|
+
function createAttachmentPort(client, access) {
|
|
142
|
+
const authorize = async (ref) => {
|
|
143
|
+
const documentId = access.documentIdFor();
|
|
144
|
+
if (!documentId) throw new Error(`Cannot resolve ${ref}: no workflow document is in scope to authorize the read`);
|
|
145
|
+
if (!await access.canReadRef(ref)) throw new Error(`Cannot resolve ${ref}: workflow "${documentId}" may not read it`);
|
|
146
|
+
return documentId;
|
|
147
|
+
};
|
|
132
148
|
return {
|
|
133
|
-
async
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
149
|
+
async authorize(ref) {
|
|
150
|
+
await authorize(ref);
|
|
151
|
+
},
|
|
152
|
+
async read(ref, destPath, signal) {
|
|
153
|
+
const documentId = await authorize(ref);
|
|
137
154
|
const limit = maxFileBytes();
|
|
138
155
|
const { header, body } = await client.download({
|
|
139
156
|
documentId,
|
|
140
|
-
ref
|
|
157
|
+
ref,
|
|
158
|
+
...signal ? { signal } : {}
|
|
141
159
|
});
|
|
142
160
|
if (typeof header.sizeBytes === "number" && Number.isFinite(header.sizeBytes) && header.sizeBytes > limit) {
|
|
143
161
|
await body.cancel().catch(() => void 0);
|
|
@@ -150,14 +168,30 @@ function createAttachmentPort(client, documentIdFor, canReadRef) {
|
|
|
150
168
|
contentType
|
|
151
169
|
};
|
|
152
170
|
},
|
|
153
|
-
async write(file) {
|
|
154
|
-
const
|
|
171
|
+
async write(file, signal) {
|
|
172
|
+
const hash = await sha256File(file.path);
|
|
173
|
+
const mimeType = file.contentType ?? "application/octet-stream";
|
|
174
|
+
const extension = path.extname(file.fileName).slice(1);
|
|
175
|
+
const preprocessed = {
|
|
176
|
+
ref: `attachment://v1:${hash}`,
|
|
177
|
+
hash,
|
|
178
|
+
sizeBytes: file.size,
|
|
179
|
+
options: {
|
|
180
|
+
mimeType,
|
|
181
|
+
fileName: file.fileName,
|
|
182
|
+
...extension ? { extension } : {},
|
|
183
|
+
clientHash: hash,
|
|
184
|
+
sizeBytes: file.size
|
|
185
|
+
},
|
|
186
|
+
data: fileStream(file.path),
|
|
187
|
+
stream: () => fileStream(file.path)
|
|
188
|
+
};
|
|
155
189
|
const ref = refOf(await client.upload({
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
mimeType: file.contentType
|
|
190
|
+
preprocessed,
|
|
191
|
+
...signal ? { signal } : {}
|
|
159
192
|
}));
|
|
160
|
-
|
|
193
|
+
access.onWritten?.(ref);
|
|
194
|
+
logger$7.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
|
|
161
195
|
return ref;
|
|
162
196
|
}
|
|
163
197
|
};
|
|
@@ -771,7 +805,7 @@ const CACHE_TTL_MS = 3e5;
|
|
|
771
805
|
const PINNED_TTL_MS = 36e5;
|
|
772
806
|
const REGISTRY_PAGE_SIZE = 50;
|
|
773
807
|
const REGISTRY_MAX_PAGES = 200;
|
|
774
|
-
const logger$
|
|
808
|
+
const logger$6 = childLogger(["workflow", "piece-catalog"]);
|
|
775
809
|
function aiLast(audience) {
|
|
776
810
|
return audience === "ai" ? 1 : 0;
|
|
777
811
|
}
|
|
@@ -817,7 +851,7 @@ async function fetchRegistryCatalog(url, timeoutMs) {
|
|
|
817
851
|
items.push(...page.items);
|
|
818
852
|
if (!page.hasMore || page.items.length === 0) return items;
|
|
819
853
|
}
|
|
820
|
-
logger$
|
|
854
|
+
logger$6.warn(`${url} still had more pieces after ${REGISTRY_MAX_PAGES} pages`);
|
|
821
855
|
return items;
|
|
822
856
|
}
|
|
823
857
|
function asList(value) {
|
|
@@ -831,10 +865,10 @@ async function publishedLists(suggestions, timeoutMs) {
|
|
|
831
865
|
const cloudUrl = suggestions ? `${CATALOG_URL}?suggestionType=ACTION_AND_TRIGGER` : CATALOG_URL;
|
|
832
866
|
const [fromRegistry, fromCloud] = await Promise.allSettled([source ? fetchRegistryCatalog(source.catalogUrl(suggestions), timeoutMs) : Promise.resolve([]), fetchJson(cloudUrl, timeoutMs)]);
|
|
833
867
|
const registry = fromRegistry.status === "fulfilled" ? asList(fromRegistry.value).filter((entry) => typeof entry.name === "string") : [];
|
|
834
|
-
if (fromRegistry.status === "rejected") logger$
|
|
868
|
+
if (fromRegistry.status === "rejected") logger$6.warn("Piece registry @url did not answer: @error", source?.baseUrl ?? "?", String(fromRegistry.reason));
|
|
835
869
|
if (fromCloud.status === "rejected") {
|
|
836
870
|
if (registry.length === 0) throw asError(fromCloud.reason);
|
|
837
|
-
logger$
|
|
871
|
+
logger$6.warn("Serving registry pieces only: @error", String(fromCloud.reason));
|
|
838
872
|
return {
|
|
839
873
|
registry,
|
|
840
874
|
cloud: []
|
|
@@ -849,12 +883,18 @@ function registryFirst(registry, cloud) {
|
|
|
849
883
|
const claimed = new Set(registry.map((entry) => entry.name));
|
|
850
884
|
return [...registry, ...cloud.filter((entry) => !claimed.has(entry.name))];
|
|
851
885
|
}
|
|
886
|
+
function tagged(entries, source) {
|
|
887
|
+
return entries.map((entry) => ({
|
|
888
|
+
...entry,
|
|
889
|
+
source
|
|
890
|
+
}));
|
|
891
|
+
}
|
|
852
892
|
async function fetchCatalogWithSuggestions() {
|
|
853
893
|
const { registry, cloud } = await publishedLists(true, 12e4);
|
|
854
|
-
return registryFirst(registry, cloud);
|
|
894
|
+
return registryFirst(tagged(registry, "registry"), tagged(cloud, "activepieces"));
|
|
855
895
|
}
|
|
856
896
|
let catalogCache;
|
|
857
|
-
function toSummaries(raw) {
|
|
897
|
+
function toSummaries(raw, source) {
|
|
858
898
|
return raw.filter((entry) => typeof entry.name === "string" && typeof entry.version === "string" && !SERVER_ONLY_PIECES.has(entry.name) && (typeof entry.actions === "number" && entry.actions > 0 || typeof entry.triggers === "number" && entry.triggers > 0)).map((entry) => ({
|
|
859
899
|
name: entry.name,
|
|
860
900
|
displayName: entry.displayName ?? entry.name,
|
|
@@ -864,6 +904,7 @@ function toSummaries(raw) {
|
|
|
864
904
|
actionCount: typeof entry.actions === "number" ? entry.actions : 0,
|
|
865
905
|
triggerCount: typeof entry.triggers === "number" ? entry.triggers : 0,
|
|
866
906
|
categories: entry.categories ?? [],
|
|
907
|
+
source,
|
|
867
908
|
auth: entry.auth ?? null,
|
|
868
909
|
...entry.deprecated === true ? { deprecated: true } : {},
|
|
869
910
|
...reasonOf(unsupportedAuth(entry.auth))
|
|
@@ -872,7 +913,7 @@ function toSummaries(raw) {
|
|
|
872
913
|
async function fetchPieceCatalog() {
|
|
873
914
|
if (catalogCache && catalogCache.expiresAt > Date.now()) return catalogCache.value;
|
|
874
915
|
const lists = await publishedLists(false, 3e4);
|
|
875
|
-
const value = [...registryFirst(toSummaries(lists.registry), toSummaries(lists.cloud))].sort((a, b) => a.displayName.localeCompare(b.displayName));
|
|
916
|
+
const value = [...registryFirst(toSummaries(lists.registry, "registry"), toSummaries(lists.cloud, "activepieces"))].sort((a, b) => a.displayName.localeCompare(b.displayName));
|
|
876
917
|
catalogCache = {
|
|
877
918
|
value,
|
|
878
919
|
expiresAt: Date.now() + CACHE_TTL_MS
|
|
@@ -884,8 +925,8 @@ async function fetchPieceJson(packageName, version) {
|
|
|
884
925
|
if (source) try {
|
|
885
926
|
return await fetchJson(atVersion(source.pieceUrl(packageName), version));
|
|
886
927
|
} catch (error) {
|
|
887
|
-
if (error instanceof CatalogStatusError && error.status === 404) logger$
|
|
888
|
-
else logger$
|
|
928
|
+
if (error instanceof CatalogStatusError && error.status === 404) logger$6.debug("@registry does not serve @piece", source.baseUrl, packageName);
|
|
929
|
+
else logger$6.warn("Piece registry @registry did not answer for @piece: @error", source.baseUrl, packageName, String(error));
|
|
889
930
|
}
|
|
890
931
|
return fetchJson(pieceUrl(packageName, version));
|
|
891
932
|
}
|
|
@@ -974,6 +1015,7 @@ function catalogEntry(descriptor, pieceName, version) {
|
|
|
974
1015
|
actionCount: descriptor.actions.length,
|
|
975
1016
|
triggerCount: descriptor.triggers.length,
|
|
976
1017
|
categories: descriptor.categories ?? [],
|
|
1018
|
+
source: "local",
|
|
977
1019
|
auth: clientAuth(descriptor.auth),
|
|
978
1020
|
...reasonOf(descriptor.unsupported)
|
|
979
1021
|
};
|
|
@@ -1008,32 +1050,44 @@ function triggersResult(descriptor, pieceName, version) {
|
|
|
1008
1050
|
auth: clientAuth(descriptor.auth)
|
|
1009
1051
|
};
|
|
1010
1052
|
}
|
|
1011
|
-
function
|
|
1053
|
+
function localSearchPiece(descriptor, pieceName, version) {
|
|
1012
1054
|
const pieceDisplayName = descriptor.displayName || pieceName;
|
|
1013
1055
|
const logoUrl = descriptor.logoUrl ?? "";
|
|
1014
|
-
return
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1056
|
+
return {
|
|
1057
|
+
meta: {
|
|
1058
|
+
pieceName,
|
|
1059
|
+
pieceVersion: version,
|
|
1060
|
+
displayName: pieceDisplayName,
|
|
1061
|
+
description: descriptor.description ?? "",
|
|
1062
|
+
logoUrl,
|
|
1063
|
+
categories: descriptor.categories ?? [],
|
|
1064
|
+
source: "local",
|
|
1065
|
+
...reasonOf(descriptor.unsupported)
|
|
1066
|
+
},
|
|
1067
|
+
blocks: [...descriptor.actions.map((action) => ({
|
|
1068
|
+
pieceName,
|
|
1069
|
+
pieceVersion: version,
|
|
1070
|
+
name: action.name,
|
|
1071
|
+
pieceDisplayName,
|
|
1072
|
+
logoUrl,
|
|
1073
|
+
displayName: action.displayName,
|
|
1074
|
+
description: action.description ?? "",
|
|
1075
|
+
kind: "action",
|
|
1076
|
+
strategy: null,
|
|
1077
|
+
...reasonOf(descriptor.unsupported)
|
|
1078
|
+
})), ...descriptor.triggers.map((trigger) => ({
|
|
1079
|
+
pieceName,
|
|
1080
|
+
pieceVersion: version,
|
|
1081
|
+
name: trigger.name,
|
|
1082
|
+
pieceDisplayName,
|
|
1083
|
+
logoUrl,
|
|
1084
|
+
displayName: trigger.displayName,
|
|
1085
|
+
description: trigger.description ?? "",
|
|
1086
|
+
kind: "trigger",
|
|
1087
|
+
strategy: trigger.strategy,
|
|
1088
|
+
...reasonOf(descriptor.unsupported ?? trigger.unsupported)
|
|
1089
|
+
}))]
|
|
1090
|
+
};
|
|
1037
1091
|
}
|
|
1038
1092
|
function detailResult(descriptor, pieceName, version) {
|
|
1039
1093
|
return {
|
|
@@ -1073,11 +1127,41 @@ function detailResult(descriptor, pieceName, version) {
|
|
|
1073
1127
|
//#endregion
|
|
1074
1128
|
//#region src/reactor/block-search.ts
|
|
1075
1129
|
const INDEX_TTL_MS = 36e5;
|
|
1076
|
-
const DEFAULT_LIMIT =
|
|
1077
|
-
const MAX_LIMIT =
|
|
1130
|
+
const DEFAULT_LIMIT = 20;
|
|
1131
|
+
const MAX_LIMIT = 50;
|
|
1132
|
+
const MAX_BLOCKS_PER_PIECE = 50;
|
|
1133
|
+
function tokenize(text) {
|
|
1134
|
+
return text.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
|
1135
|
+
}
|
|
1136
|
+
function wordsOf(...texts) {
|
|
1137
|
+
const words = texts.flatMap(tokenize);
|
|
1138
|
+
return {
|
|
1139
|
+
words,
|
|
1140
|
+
compact: words.join("")
|
|
1141
|
+
};
|
|
1142
|
+
}
|
|
1143
|
+
function slugOf(pieceName) {
|
|
1144
|
+
return pieceName.replace(/^@[^/]+\//, "").replace(/^piece-/, "");
|
|
1145
|
+
}
|
|
1146
|
+
function indexPiece(piece) {
|
|
1147
|
+
const { meta } = piece;
|
|
1148
|
+
return {
|
|
1149
|
+
meta,
|
|
1150
|
+
name: wordsOf(meta.displayName, slugOf(meta.pieceName)),
|
|
1151
|
+
exactName: tokenize(meta.displayName).join(" "),
|
|
1152
|
+
about: wordsOf(meta.description, ...meta.categories),
|
|
1153
|
+
blocks: piece.blocks.map((hit) => ({
|
|
1154
|
+
hit,
|
|
1155
|
+
name: wordsOf(hit.displayName, hit.name),
|
|
1156
|
+
description: wordsOf(hit.description)
|
|
1157
|
+
}))
|
|
1158
|
+
};
|
|
1159
|
+
}
|
|
1160
|
+
function indexPieces(pieces) {
|
|
1161
|
+
return { pieces: pieces.map(indexPiece) };
|
|
1162
|
+
}
|
|
1078
1163
|
function buildSearchIndex(raw) {
|
|
1079
|
-
const
|
|
1080
|
-
let pieces = 0;
|
|
1164
|
+
const pieces = [];
|
|
1081
1165
|
for (const entry of raw) {
|
|
1082
1166
|
if (typeof entry.name !== "string" || typeof entry.version !== "string" || SERVER_ONLY_PIECES.has(entry.name)) continue;
|
|
1083
1167
|
const pieceName = entry.name;
|
|
@@ -1085,78 +1169,141 @@ function buildSearchIndex(raw) {
|
|
|
1085
1169
|
const pieceDisplayName = entry.displayName ?? pieceName;
|
|
1086
1170
|
const logoUrl = entry.logoUrl ?? "";
|
|
1087
1171
|
const pieceUnsupported = unsupportedAuth(entry.auth);
|
|
1172
|
+
const blocks = [];
|
|
1088
1173
|
const push = (kind, item) => {
|
|
1089
1174
|
if (typeof item.name !== "string" || item.name === "") return;
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
kind,
|
|
1102
|
-
strategy: kind === "trigger" ? item.type ?? null : null,
|
|
1103
|
-
...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
|
|
1104
|
-
},
|
|
1105
|
-
name: `${displayName} ${item.name}`.toLowerCase(),
|
|
1106
|
-
description: description.toLowerCase(),
|
|
1107
|
-
piece: pieceDisplayName.toLowerCase()
|
|
1175
|
+
blocks.push({
|
|
1176
|
+
pieceName,
|
|
1177
|
+
pieceVersion,
|
|
1178
|
+
name: item.name,
|
|
1179
|
+
pieceDisplayName,
|
|
1180
|
+
logoUrl,
|
|
1181
|
+
displayName: item.displayName ?? item.name,
|
|
1182
|
+
description: item.description ?? "",
|
|
1183
|
+
kind,
|
|
1184
|
+
strategy: kind === "trigger" ? item.type ?? null : null,
|
|
1185
|
+
...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
|
|
1108
1186
|
});
|
|
1109
1187
|
};
|
|
1110
1188
|
for (const action of entry.suggestedActions ?? []) push("action", action);
|
|
1111
1189
|
for (const trigger of entry.suggestedTriggers ?? []) push("trigger", trigger);
|
|
1112
|
-
pieces
|
|
1190
|
+
pieces.push({
|
|
1191
|
+
meta: {
|
|
1192
|
+
pieceName,
|
|
1193
|
+
pieceVersion,
|
|
1194
|
+
displayName: pieceDisplayName,
|
|
1195
|
+
description: entry.description ?? "",
|
|
1196
|
+
logoUrl,
|
|
1197
|
+
categories: Array.isArray(entry.categories) ? entry.categories : [],
|
|
1198
|
+
source: entry.source ?? "activepieces",
|
|
1199
|
+
...entry.deprecated === true ? { deprecated: true } : {},
|
|
1200
|
+
...reasonOf(pieceUnsupported)
|
|
1201
|
+
},
|
|
1202
|
+
blocks
|
|
1203
|
+
});
|
|
1113
1204
|
}
|
|
1114
|
-
return
|
|
1115
|
-
entries,
|
|
1116
|
-
pieces
|
|
1117
|
-
};
|
|
1118
|
-
}
|
|
1119
|
-
function indexFromHits(hits) {
|
|
1120
|
-
const pieces = /* @__PURE__ */ new Set();
|
|
1121
|
-
return {
|
|
1122
|
-
entries: hits.map((hit) => {
|
|
1123
|
-
pieces.add(hit.pieceName);
|
|
1124
|
-
return {
|
|
1125
|
-
hit,
|
|
1126
|
-
name: `${hit.displayName} ${hit.name}`.toLowerCase(),
|
|
1127
|
-
description: hit.description.toLowerCase(),
|
|
1128
|
-
piece: hit.pieceDisplayName.toLowerCase()
|
|
1129
|
-
};
|
|
1130
|
-
}),
|
|
1131
|
-
pieces: pieces.size
|
|
1132
|
-
};
|
|
1205
|
+
return indexPieces(pieces);
|
|
1133
1206
|
}
|
|
1134
1207
|
function merge(index, local) {
|
|
1135
|
-
const
|
|
1136
|
-
const localNames = new Set(
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1208
|
+
const localPieces = local?.pieces ?? [];
|
|
1209
|
+
const localNames = new Set(localPieces.map((piece) => piece.meta.pieceName));
|
|
1210
|
+
return { pieces: [...localPieces, ...(index?.pieces ?? []).filter((piece) => !localNames.has(piece.meta.pieceName))] };
|
|
1211
|
+
}
|
|
1212
|
+
const EXACT = 4;
|
|
1213
|
+
const PREFIX = 3;
|
|
1214
|
+
const INNER = 1.5;
|
|
1215
|
+
const FUZZY = 1;
|
|
1216
|
+
const PIECE_NAME = 3;
|
|
1217
|
+
const BLOCK_NAME = 2;
|
|
1218
|
+
const BLOCK_DESCRIPTION = 1;
|
|
1219
|
+
const PIECE_ABOUT = .5;
|
|
1220
|
+
function editDistance(a, b, max) {
|
|
1221
|
+
if (Math.abs(a.length - b.length) > max) return max + 1;
|
|
1222
|
+
let prevPrev = [];
|
|
1223
|
+
let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
|
|
1224
|
+
for (let i = 1; i <= a.length; i++) {
|
|
1225
|
+
const row = [i];
|
|
1226
|
+
let rowMin = i;
|
|
1227
|
+
for (let j = 1; j <= b.length; j++) {
|
|
1228
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
1229
|
+
let value = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost);
|
|
1230
|
+
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) value = Math.min(value, prevPrev[j - 2] + 1);
|
|
1231
|
+
row.push(value);
|
|
1232
|
+
rowMin = Math.min(rowMin, value);
|
|
1233
|
+
}
|
|
1234
|
+
if (rowMin > max) return max + 1;
|
|
1235
|
+
prevPrev = prev;
|
|
1236
|
+
prev = row;
|
|
1237
|
+
}
|
|
1238
|
+
return prev[b.length];
|
|
1239
|
+
}
|
|
1240
|
+
function termScore(token, field, fuzzy) {
|
|
1241
|
+
let best = 0;
|
|
1242
|
+
for (const word of field.words) {
|
|
1243
|
+
if (word === token) return EXACT;
|
|
1244
|
+
if (word.startsWith(token)) best = PREFIX;
|
|
1245
|
+
else if (best < INNER && token.length >= 3 && word.includes(token)) best = INNER;
|
|
1246
|
+
}
|
|
1247
|
+
if (best > 0) return best;
|
|
1248
|
+
if (token.length >= 3 && field.compact.includes(token)) return INNER;
|
|
1249
|
+
if (!fuzzy || token.length < 4) return 0;
|
|
1250
|
+
const max = token.length >= 8 ? 2 : 1;
|
|
1251
|
+
for (const word of field.words) if (editDistance(token, word, max) <= max || word.length > token.length && editDistance(token, word.slice(0, token.length), max) <= max) return FUZZY;
|
|
1252
|
+
return 0;
|
|
1253
|
+
}
|
|
1254
|
+
function allowed(piece, filter) {
|
|
1255
|
+
const { sources, categories } = filter;
|
|
1256
|
+
if (sources?.length && !sources.includes(piece.meta.source)) return false;
|
|
1257
|
+
if (categories?.length && !piece.meta.categories.some((category) => categories.includes(category))) return false;
|
|
1258
|
+
return true;
|
|
1259
|
+
}
|
|
1260
|
+
function searchIndex(index, query, filter) {
|
|
1261
|
+
const tokens = tokenize(query);
|
|
1146
1262
|
if (tokens.length === 0) return [];
|
|
1147
|
-
const
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
if (!
|
|
1151
|
-
const
|
|
1152
|
-
const
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1263
|
+
const exactName = tokens.join(" ");
|
|
1264
|
+
const matches = [];
|
|
1265
|
+
for (const piece of index.pieces) {
|
|
1266
|
+
if (!allowed(piece, filter)) continue;
|
|
1267
|
+
const nameScores = tokens.map((token) => termScore(token, piece.name, true));
|
|
1268
|
+
const ofKind = piece.blocks.filter((block) => block.hit.kind === filter.kind).map((block) => ({
|
|
1269
|
+
hit: block.hit,
|
|
1270
|
+
scores: tokens.map((token) => Math.max(BLOCK_NAME * termScore(token, block.name, true), BLOCK_DESCRIPTION * termScore(token, block.description, false)))
|
|
1271
|
+
}));
|
|
1272
|
+
const matchBlocks = (pieceScores) => {
|
|
1273
|
+
const found = [];
|
|
1274
|
+
ofKind.forEach((block, order) => {
|
|
1275
|
+
let score = 0;
|
|
1276
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
1277
|
+
const best = Math.max(pieceScores[i], block.scores[i]);
|
|
1278
|
+
if (best === 0) return;
|
|
1279
|
+
score += best;
|
|
1280
|
+
}
|
|
1281
|
+
found.push({
|
|
1282
|
+
score,
|
|
1283
|
+
order,
|
|
1284
|
+
hit: block.hit
|
|
1285
|
+
});
|
|
1286
|
+
});
|
|
1287
|
+
return found;
|
|
1288
|
+
};
|
|
1289
|
+
let blocks = matchBlocks(nameScores.map((score) => PIECE_NAME * score));
|
|
1290
|
+
if (blocks.length === 0 && ofKind.length > 0) blocks = matchBlocks(tokens.map((token, i) => Math.max(PIECE_NAME * nameScores[i], PIECE_ABOUT * termScore(token, piece.about, false))));
|
|
1291
|
+
if (blocks.length === 0) continue;
|
|
1292
|
+
blocks.sort((a, b) => b.score - a.score || a.order - b.order);
|
|
1293
|
+
const nameBonus = piece.exactName === exactName ? EXACT : 0;
|
|
1294
|
+
matches.push({
|
|
1295
|
+
score: blocks[0].score + nameBonus,
|
|
1296
|
+
match: {
|
|
1297
|
+
...piece.meta,
|
|
1298
|
+
namedPiece: nameScores.every((score) => score > 0),
|
|
1299
|
+
blocks: blocks.slice(0, MAX_BLOCKS_PER_PIECE).map(({ hit }) => hit)
|
|
1300
|
+
}
|
|
1156
1301
|
});
|
|
1157
1302
|
}
|
|
1158
|
-
|
|
1159
|
-
|
|
1303
|
+
const demoted = (match) => (match.deprecated ? 2 : 0) + (match.unsupported ? 1 : 0);
|
|
1304
|
+
matches.sort((a, b) => b.score - a.score || demoted(a.match) - demoted(b.match) || a.match.displayName.localeCompare(b.match.displayName));
|
|
1305
|
+
const limit = Math.min(Math.max(filter.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT);
|
|
1306
|
+
return matches.slice(0, limit).map(({ match }) => match);
|
|
1160
1307
|
}
|
|
1161
1308
|
let cached;
|
|
1162
1309
|
function ensureIndex() {
|
|
@@ -1173,29 +1320,35 @@ function ensureIndex() {
|
|
|
1173
1320
|
cached = entry;
|
|
1174
1321
|
return entry;
|
|
1175
1322
|
}
|
|
1176
|
-
function
|
|
1323
|
+
function warmSearchIndex() {
|
|
1324
|
+
const entry = ensureIndex();
|
|
1325
|
+
entry.promise.catch(() => {
|
|
1326
|
+
if (cached === entry) cached = void 0;
|
|
1327
|
+
});
|
|
1328
|
+
}
|
|
1329
|
+
function searchPieces(query, filter, local) {
|
|
1177
1330
|
const index = ensureIndex();
|
|
1178
1331
|
if (index.error) {
|
|
1179
1332
|
const message = index.error;
|
|
1180
1333
|
cached = void 0;
|
|
1181
1334
|
return {
|
|
1182
1335
|
status: "error",
|
|
1183
|
-
|
|
1184
|
-
indexedPieces: local?.pieces ?? 0,
|
|
1336
|
+
pieces: searchIndex(merge(void 0, local), query, filter),
|
|
1337
|
+
indexedPieces: local?.pieces.length ?? 0,
|
|
1185
1338
|
error: message
|
|
1186
1339
|
};
|
|
1187
1340
|
}
|
|
1188
1341
|
if (!index.value) return {
|
|
1189
1342
|
status: "indexing",
|
|
1190
|
-
|
|
1191
|
-
indexedPieces: local?.pieces ?? 0,
|
|
1343
|
+
pieces: searchIndex(merge(void 0, local), query, filter),
|
|
1344
|
+
indexedPieces: local?.pieces.length ?? 0,
|
|
1192
1345
|
error: null
|
|
1193
1346
|
};
|
|
1194
1347
|
const merged = merge(index.value, local);
|
|
1195
1348
|
return {
|
|
1196
1349
|
status: "ready",
|
|
1197
|
-
|
|
1198
|
-
indexedPieces: merged.pieces,
|
|
1350
|
+
pieces: searchIndex(merged, query, filter),
|
|
1351
|
+
indexedPieces: merged.pieces.length,
|
|
1199
1352
|
error: null
|
|
1200
1353
|
};
|
|
1201
1354
|
}
|
|
@@ -1363,6 +1516,9 @@ function currentDocumentRecorder() {
|
|
|
1363
1516
|
function currentPieceWorker() {
|
|
1364
1517
|
return storage.getStore()?.pieceWorker;
|
|
1365
1518
|
}
|
|
1519
|
+
function currentAttachmentOrigins() {
|
|
1520
|
+
return storage.getStore()?.attachments;
|
|
1521
|
+
}
|
|
1366
1522
|
function currentStepTest() {
|
|
1367
1523
|
return storage.getStore()?.stepTest ?? false;
|
|
1368
1524
|
}
|
|
@@ -1371,7 +1527,7 @@ function currentRunUser() {
|
|
|
1371
1527
|
}
|
|
1372
1528
|
//#endregion
|
|
1373
1529
|
//#region src/reactor/piece-store-port.ts
|
|
1374
|
-
const logger$
|
|
1530
|
+
const logger$5 = childLogger(["workflow", "piece-store"]);
|
|
1375
1531
|
const PROJECT_SCOPE_KEY = "reactor";
|
|
1376
1532
|
const TEST_PARTITION_SUFFIX = "#test";
|
|
1377
1533
|
function testPartitionKey(scope, workflowId) {
|
|
@@ -1405,7 +1561,7 @@ function createPieceStorePort(store, workflowIdFor, sample = false, clock = Date
|
|
|
1405
1561
|
if (!isCursorRef(key) || isPlausibleCursor(value, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
|
|
1406
1562
|
const kept = await store.getPieceStoreValue(scope, partition(scope), key);
|
|
1407
1563
|
if (!isCursorShaped(value) && !isPlausibleCursor(kept, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
|
|
1408
|
-
logger$
|
|
1564
|
+
logger$5.warn(`Rejected @key=@value from workflow ${flowKey()}; keeping @kept`, key, showCursor(value), isPlausibleCursor(kept, now) ? String(kept) : "no cursor");
|
|
1409
1565
|
if (!isPlausibleCursor(kept, now)) await store.deletePieceStoreValue(scope, partition(scope), key);
|
|
1410
1566
|
},
|
|
1411
1567
|
delete: async (key, scope) => store.deletePieceStoreValue(scope, partition(scope), key)
|
|
@@ -1440,6 +1596,20 @@ function runnableDefinition(state) {
|
|
|
1440
1596
|
//#endregion
|
|
1441
1597
|
//#region src/reactor/lib.ts
|
|
1442
1598
|
const pieceLogger = childLogger(["workflow", "piece"]);
|
|
1599
|
+
/**
|
|
1600
|
+
* The ceiling on one line this engine writes to a log.
|
|
1601
|
+
*
|
|
1602
|
+
* Nothing here is worth more than this to a reader, and an unbounded write is
|
|
1603
|
+
* a liability: a piece error carrying an HTML error page, or a piece logging a
|
|
1604
|
+
* whole document, becomes a multi-megabyte write to a pipe that may be
|
|
1605
|
+
* blocked — and an EPIPE on the fatal path is what produced the boot loop in
|
|
1606
|
+
* backlog item 5. Bounding the write is cheaper than handling the throw.
|
|
1607
|
+
*/
|
|
1608
|
+
const MAX_LOG_LINE_CHARS = 8192;
|
|
1609
|
+
function truncateForLog(text) {
|
|
1610
|
+
if (text.length <= 8192) return text;
|
|
1611
|
+
return `${text.slice(0, MAX_LOG_LINE_CHARS)}… [truncated, ${text.length} chars]`;
|
|
1612
|
+
}
|
|
1443
1613
|
const connectionLogger = childLogger(["workflow", "connection"]);
|
|
1444
1614
|
function assertConnectorMatches(state, request) {
|
|
1445
1615
|
const wanted = request?.piecePackage;
|
|
@@ -1502,6 +1672,29 @@ function bundleCacheDir() {
|
|
|
1502
1672
|
return bundleCache;
|
|
1503
1673
|
}
|
|
1504
1674
|
const ATTACHMENT_STAGING_DIR = join(process.cwd(), ".ph", "ap-attachment-staging");
|
|
1675
|
+
const ATTACHMENT_CACHE_DIR = join(process.cwd(), ".ph", "ap-attachment-cache");
|
|
1676
|
+
const DEFAULT_ATTACHMENT_CACHE_BYTES = 1073741824;
|
|
1677
|
+
function attachmentCacheBytes() {
|
|
1678
|
+
const raw = process.env.PH_WORKFLOWS_ATTACHMENT_CACHE_BYTES;
|
|
1679
|
+
const parsed = raw === void 0 ? NaN : Number(raw);
|
|
1680
|
+
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : DEFAULT_ATTACHMENT_CACHE_BYTES;
|
|
1681
|
+
}
|
|
1682
|
+
async function sweepAttachmentStaging(root = ATTACHMENT_STAGING_DIR, olderThanMs = 36e5) {
|
|
1683
|
+
const names = await readdir(root).catch(() => []);
|
|
1684
|
+
let swept = 0;
|
|
1685
|
+
for (const name of names) {
|
|
1686
|
+
const entry = join(root, name);
|
|
1687
|
+
const info = await stat(entry).catch(() => void 0);
|
|
1688
|
+
if (!info || Date.now() - info.mtimeMs < olderThanMs) continue;
|
|
1689
|
+
await rm(entry, {
|
|
1690
|
+
recursive: true,
|
|
1691
|
+
force: true
|
|
1692
|
+
});
|
|
1693
|
+
swept += 1;
|
|
1694
|
+
}
|
|
1695
|
+
return swept;
|
|
1696
|
+
}
|
|
1697
|
+
let stagingSwept = false;
|
|
1505
1698
|
let resolver;
|
|
1506
1699
|
function pieceResolver() {
|
|
1507
1700
|
return resolver ??= sourcedResolver({
|
|
@@ -1514,7 +1707,14 @@ function boundConnections(inner) {
|
|
|
1514
1707
|
connectionLogger.warn(`Step "${request?.stepKey ?? "?"}" of workflow "${currentWorkflowId() ?? "?"}" asked for connection "${connectionId}", which its definition does not declare`);
|
|
1515
1708
|
});
|
|
1516
1709
|
}
|
|
1517
|
-
function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess) {
|
|
1710
|
+
function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess, telemetry) {
|
|
1711
|
+
if (attachments && !stagingSwept) {
|
|
1712
|
+
stagingSwept = true;
|
|
1713
|
+
sweepAttachmentStaging().then((swept) => {
|
|
1714
|
+
if (swept > 0) pieceLogger.info(`Removed ${swept} leftover attachment staging dir(s)`);
|
|
1715
|
+
}, () => void 0);
|
|
1716
|
+
}
|
|
1717
|
+
const cacheBytes = attachmentCacheBytes();
|
|
1518
1718
|
return new CompositeBlockExecutor(new ActivepiecesBlockExecutor({
|
|
1519
1719
|
cacheDir: bundleCacheDir(),
|
|
1520
1720
|
egress: configuredEgress(),
|
|
@@ -1528,10 +1728,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
|
|
|
1528
1728
|
resolver: pieceResolver(),
|
|
1529
1729
|
...resolveBlock ? { resolveBlock } : {},
|
|
1530
1730
|
...reactorAccess ? { reactorAccess } : {},
|
|
1731
|
+
...telemetry ? { telemetry } : {},
|
|
1531
1732
|
connections: boundConnections(new DocumentConnectionResolver(host, secrets, oauth)),
|
|
1532
1733
|
...pieceStore ? { pieceStore } : {},
|
|
1533
1734
|
onPieceLog: (entry, execution) => {
|
|
1534
|
-
const line = `[${execution.step.key}] ${entry.message}
|
|
1735
|
+
const line = truncateForLog(`[${execution.step.key}] ${entry.message}`);
|
|
1535
1736
|
if (entry.level === "error") pieceLogger.error(line);
|
|
1536
1737
|
else if (entry.level === "warn") pieceLogger.warn(line);
|
|
1537
1738
|
else if (entry.level === "debug") pieceLogger.debug(line);
|
|
@@ -1539,7 +1740,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
|
|
|
1539
1740
|
},
|
|
1540
1741
|
...attachments ? {
|
|
1541
1742
|
attachments,
|
|
1542
|
-
stagingRoot: ATTACHMENT_STAGING_DIR
|
|
1743
|
+
stagingRoot: ATTACHMENT_STAGING_DIR,
|
|
1744
|
+
...cacheBytes > 0 ? { attachmentCache: new AttachmentCache({
|
|
1745
|
+
dir: ATTACHMENT_CACHE_DIR,
|
|
1746
|
+
maxBytes: cacheBytes
|
|
1747
|
+
}) } : {}
|
|
1543
1748
|
} : {}
|
|
1544
1749
|
}));
|
|
1545
1750
|
}
|
|
@@ -1563,7 +1768,8 @@ function stepDefinition(step) {
|
|
|
1563
1768
|
config: step.config,
|
|
1564
1769
|
timeoutSeconds: step.timeoutSeconds,
|
|
1565
1770
|
propertySettings: propertySettings(step.propertySettings),
|
|
1566
|
-
skip: step.skip
|
|
1771
|
+
skip: step.skip,
|
|
1772
|
+
retry: step.retry
|
|
1567
1773
|
};
|
|
1568
1774
|
}
|
|
1569
1775
|
function toWorkflowDefinition(state) {
|
|
@@ -1776,6 +1982,227 @@ async function buildReactorRunScope(host, request) {
|
|
|
1776
1982
|
connection: { access: parsed.config.access ?? "write" }
|
|
1777
1983
|
};
|
|
1778
1984
|
}
|
|
1985
|
+
//#endregion
|
|
1986
|
+
//#region src/telemetry.ts
|
|
1987
|
+
const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
|
|
1988
|
+
var WorkflowTelemetry = class {
|
|
1989
|
+
tracer;
|
|
1990
|
+
meter;
|
|
1991
|
+
runs;
|
|
1992
|
+
runDuration;
|
|
1993
|
+
stepDuration;
|
|
1994
|
+
phaseDuration;
|
|
1995
|
+
poolGauges;
|
|
1996
|
+
constructor(options = {}) {
|
|
1997
|
+
this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
|
|
1998
|
+
this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
|
|
1999
|
+
this.runs = this.meter.createCounter("workflow.runs", {
|
|
2000
|
+
description: "Runs finished, by status and trigger kind",
|
|
2001
|
+
unit: "{run}"
|
|
2002
|
+
});
|
|
2003
|
+
this.runDuration = this.meter.createHistogram("workflow.run.duration", {
|
|
2004
|
+
description: "Run duration from fire to the journal closing it",
|
|
2005
|
+
unit: "ms"
|
|
2006
|
+
});
|
|
2007
|
+
this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
|
|
2008
|
+
description: "Step duration, by piece and status",
|
|
2009
|
+
unit: "ms"
|
|
2010
|
+
});
|
|
2011
|
+
this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
|
|
2012
|
+
description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
|
|
2013
|
+
unit: "ms"
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
2016
|
+
phase(name, attributes, fn, parent) {
|
|
2017
|
+
const started = performance.now();
|
|
2018
|
+
return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
|
|
2019
|
+
}
|
|
2020
|
+
workerTimings(request, timings, forkedAt, ipc) {
|
|
2021
|
+
const under = (span) => trace.setSpan(context.active(), span);
|
|
2022
|
+
const child = (name, start, end, parent = request, attributes) => {
|
|
2023
|
+
const span = this.tracer.startSpan(`workflow.worker.${name}`, {
|
|
2024
|
+
startTime: start,
|
|
2025
|
+
...attributes ? { attributes } : {}
|
|
2026
|
+
}, under(parent));
|
|
2027
|
+
span.end(end);
|
|
2028
|
+
this.recordPhase(`worker.${name}`, end - start);
|
|
2029
|
+
return span;
|
|
2030
|
+
};
|
|
2031
|
+
const ready = epoch(timings.ready);
|
|
2032
|
+
if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
|
|
2033
|
+
const received = epoch(timings.received);
|
|
2034
|
+
const sent = epoch(timings.sent);
|
|
2035
|
+
const delivered = Math.max(ipc.sent, ready ?? 0);
|
|
2036
|
+
if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
|
|
2037
|
+
const byName = /* @__PURE__ */ new Map();
|
|
2038
|
+
const phases = Array.isArray(timings.phases) ? timings.phases : [];
|
|
2039
|
+
for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
|
|
2040
|
+
const parent = phase.parent ? byName.get(phase.parent) : void 0;
|
|
2041
|
+
byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
|
|
2042
|
+
}
|
|
2043
|
+
if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
|
|
2044
|
+
}
|
|
2045
|
+
recordPhase(name, ms) {
|
|
2046
|
+
this.phaseDuration.record(ms, { phase: name });
|
|
2047
|
+
}
|
|
2048
|
+
span(name, attributes, fn, parent = context.active()) {
|
|
2049
|
+
return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
|
|
2050
|
+
try {
|
|
2051
|
+
return await fn(span);
|
|
2052
|
+
} catch (error) {
|
|
2053
|
+
failSpan(span, error);
|
|
2054
|
+
throw error;
|
|
2055
|
+
} finally {
|
|
2056
|
+
span.end();
|
|
2057
|
+
}
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
recordRun(ms, status, triggerKind) {
|
|
2061
|
+
const attributes = {
|
|
2062
|
+
"run.status": status,
|
|
2063
|
+
"trigger.kind": metricTriggerKind(triggerKind)
|
|
2064
|
+
};
|
|
2065
|
+
this.runs.add(1, attributes);
|
|
2066
|
+
this.runDuration.record(ms, attributes);
|
|
2067
|
+
}
|
|
2068
|
+
recordStep(ms, pieceName, status) {
|
|
2069
|
+
this.stepDuration.record(ms, {
|
|
2070
|
+
"piece.name": pieceName,
|
|
2071
|
+
"step.status": status
|
|
2072
|
+
});
|
|
2073
|
+
}
|
|
2074
|
+
observeWorkerPool(stats) {
|
|
2075
|
+
this.poolGauges ??= this.createPoolGauges();
|
|
2076
|
+
const { size, active, waiting } = this.poolGauges;
|
|
2077
|
+
const observables = [
|
|
2078
|
+
size,
|
|
2079
|
+
active,
|
|
2080
|
+
waiting
|
|
2081
|
+
];
|
|
2082
|
+
const callback = (result) => {
|
|
2083
|
+
const current = stats();
|
|
2084
|
+
result.observe(size, current.size);
|
|
2085
|
+
result.observe(active, current.active);
|
|
2086
|
+
result.observe(waiting, current.waiting);
|
|
2087
|
+
};
|
|
2088
|
+
this.meter.addBatchObservableCallback(callback, observables);
|
|
2089
|
+
return () => this.meter.removeBatchObservableCallback(callback, observables);
|
|
2090
|
+
}
|
|
2091
|
+
createPoolGauges() {
|
|
2092
|
+
const gauge = (name, description) => this.meter.createObservableGauge(name, {
|
|
2093
|
+
description,
|
|
2094
|
+
unit: "{worker}"
|
|
2095
|
+
});
|
|
2096
|
+
return {
|
|
2097
|
+
size: gauge("workflow.worker.pool.size", "Worker slots"),
|
|
2098
|
+
active: gauge("workflow.worker.pool.active", "Slots held by runs"),
|
|
2099
|
+
waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
|
|
2100
|
+
};
|
|
2101
|
+
}
|
|
2102
|
+
};
|
|
2103
|
+
const WORKER_PHASES = /* @__PURE__ */ new Set([
|
|
2104
|
+
"piece.load",
|
|
2105
|
+
"reactor.open",
|
|
2106
|
+
"models",
|
|
2107
|
+
"action"
|
|
2108
|
+
]);
|
|
2109
|
+
const WORKER_ATTRIBUTES = {
|
|
2110
|
+
"piece.cached": "boolean",
|
|
2111
|
+
"document.type": "string"
|
|
2112
|
+
};
|
|
2113
|
+
const MAX_ATTRIBUTE_LENGTH = 256;
|
|
2114
|
+
function epoch(value) {
|
|
2115
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
2116
|
+
}
|
|
2117
|
+
function knownPhase(phase) {
|
|
2118
|
+
if (typeof phase !== "object" || phase === null) return false;
|
|
2119
|
+
const { name, start, end, parent } = phase;
|
|
2120
|
+
return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
|
|
2121
|
+
}
|
|
2122
|
+
function phaseAttributes(attributes) {
|
|
2123
|
+
if (typeof attributes !== "object" || attributes === null) return void 0;
|
|
2124
|
+
const kept = {};
|
|
2125
|
+
for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
|
|
2126
|
+
const value = attributes[key];
|
|
2127
|
+
if (typeof value !== type) continue;
|
|
2128
|
+
kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
|
|
2129
|
+
}
|
|
2130
|
+
return Object.keys(kept).length > 0 ? kept : void 0;
|
|
2131
|
+
}
|
|
2132
|
+
function metricTriggerKind(triggerKind) {
|
|
2133
|
+
return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
|
|
2134
|
+
}
|
|
2135
|
+
function metricPieceName(step, resolution) {
|
|
2136
|
+
return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
|
|
2137
|
+
}
|
|
2138
|
+
function tracedSteps(executor, telemetry) {
|
|
2139
|
+
return { execute(execution) {
|
|
2140
|
+
const { step } = execution;
|
|
2141
|
+
const started = performance.now();
|
|
2142
|
+
let status = "FAILED";
|
|
2143
|
+
let resolution;
|
|
2144
|
+
return telemetry.span("workflow.step", {
|
|
2145
|
+
"step.id": step.id,
|
|
2146
|
+
"step.key": step.key,
|
|
2147
|
+
"piece.name": step.pieceName,
|
|
2148
|
+
"action.name": step.actionName
|
|
2149
|
+
}, async (span) => {
|
|
2150
|
+
try {
|
|
2151
|
+
const result = await executor.execute(execution);
|
|
2152
|
+
resolution = result.resolution;
|
|
2153
|
+
status = "SUCCEEDED";
|
|
2154
|
+
const version = resolution?.resolved?.version;
|
|
2155
|
+
if (version) span.setAttribute("piece.version", version);
|
|
2156
|
+
return result;
|
|
2157
|
+
} catch (error) {
|
|
2158
|
+
resolution = resolutionOf(error);
|
|
2159
|
+
throw error;
|
|
2160
|
+
}
|
|
2161
|
+
}).finally(() => {
|
|
2162
|
+
telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
|
|
2163
|
+
});
|
|
2164
|
+
} };
|
|
2165
|
+
}
|
|
2166
|
+
function failSpan(span, error) {
|
|
2167
|
+
span.recordException(error instanceof Error ? error : String(error));
|
|
2168
|
+
span.setStatus({
|
|
2169
|
+
code: SpanStatusCode.ERROR,
|
|
2170
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2171
|
+
});
|
|
2172
|
+
}
|
|
2173
|
+
function tracedMethods(target, telemetry, prefix, parent) {
|
|
2174
|
+
return new Proxy(target, { get(object, property, receiver) {
|
|
2175
|
+
const value = Reflect.get(object, property, receiver);
|
|
2176
|
+
if (typeof value !== "function" || typeof property !== "string") return value;
|
|
2177
|
+
const phase = `${prefix}.${property}`;
|
|
2178
|
+
return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
|
|
2179
|
+
const started = performance.now();
|
|
2180
|
+
const end = (error) => {
|
|
2181
|
+
if (error !== void 0) failSpan(span, error);
|
|
2182
|
+
span.end();
|
|
2183
|
+
telemetry.recordPhase(phase, performance.now() - started);
|
|
2184
|
+
};
|
|
2185
|
+
let result;
|
|
2186
|
+
try {
|
|
2187
|
+
result = value.apply(object, args);
|
|
2188
|
+
} catch (error) {
|
|
2189
|
+
end(error);
|
|
2190
|
+
throw error;
|
|
2191
|
+
}
|
|
2192
|
+
if (!(result instanceof Promise)) {
|
|
2193
|
+
end();
|
|
2194
|
+
return result;
|
|
2195
|
+
}
|
|
2196
|
+
return result.then((resolved) => {
|
|
2197
|
+
end();
|
|
2198
|
+
return resolved;
|
|
2199
|
+
}, (error) => {
|
|
2200
|
+
end(error);
|
|
2201
|
+
throw error;
|
|
2202
|
+
});
|
|
2203
|
+
});
|
|
2204
|
+
} });
|
|
2205
|
+
}
|
|
1779
2206
|
const JOB_NOT_FOUND = "Job not found";
|
|
1780
2207
|
const MIN_DEADLINE_MARGIN_MS = 250;
|
|
1781
2208
|
const MAX_DEADLINE_MARGIN_MS = 2e3;
|
|
@@ -1792,7 +2219,7 @@ const AUTHORIZATION_DENIED = "AuthorizationDeniedError";
|
|
|
1792
2219
|
function jobFailed(job) {
|
|
1793
2220
|
const error = job.error;
|
|
1794
2221
|
if (error?.name === AUTHORIZATION_DENIED) return accessDenied(`The Switchboard has no grant for this operation: ${error.message}`);
|
|
1795
|
-
return new ReactorError(ReactorJobFailedError
|
|
2222
|
+
return new ReactorError(ReactorJobFailedError, `Reactor job ${job.id} failed: ${job.error?.message ?? "unknown error"}`);
|
|
1796
2223
|
}
|
|
1797
2224
|
function assertActionsApplied(job, actions) {
|
|
1798
2225
|
const outcomes = job.result?.actions ?? [];
|
|
@@ -1810,7 +2237,7 @@ function assertActionsApplied(job, actions) {
|
|
|
1810
2237
|
if (outcome.kind === "denied") return [`Action ${label} was denied: ${outcome.reason}`];
|
|
1811
2238
|
return [];
|
|
1812
2239
|
});
|
|
1813
|
-
if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError
|
|
2240
|
+
if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError, failed.join("; "));
|
|
1814
2241
|
}
|
|
1815
2242
|
var RunScopedReactorClient = class {
|
|
1816
2243
|
inner;
|
|
@@ -2141,7 +2568,7 @@ const NO_JOURNAL = {
|
|
|
2141
2568
|
recordDocuments: () => Promise.resolve(),
|
|
2142
2569
|
recordJob: () => Promise.resolve()
|
|
2143
2570
|
};
|
|
2144
|
-
function reactorTap(host, base, journal, options = {}) {
|
|
2571
|
+
function reactorTap(host, base, journal, options = {}, telemetry) {
|
|
2145
2572
|
const clientOptions = {
|
|
2146
2573
|
...host.hostPrincipal ? { hostPrincipal: host.hostPrincipal } : {},
|
|
2147
2574
|
...options
|
|
@@ -2149,12 +2576,13 @@ function reactorTap(host, base, journal, options = {}) {
|
|
|
2149
2576
|
return {
|
|
2150
2577
|
requireReactor: base.requireReactor,
|
|
2151
2578
|
open(transport, { deadline }) {
|
|
2152
|
-
const
|
|
2579
|
+
const served = servedClient(new RunScopedReactorClient(host.reactorClient, {
|
|
2153
2580
|
...base,
|
|
2154
2581
|
deadline,
|
|
2155
2582
|
journal
|
|
2156
|
-
}, host, clientOptions);
|
|
2157
|
-
const
|
|
2583
|
+
}, host, clientOptions));
|
|
2584
|
+
const parent = context.active();
|
|
2585
|
+
const server = new ReactorHostServer(telemetry ? tracedMethods(served, telemetry, "reactor", parent) : served, transport);
|
|
2158
2586
|
server.start();
|
|
2159
2587
|
return () => server.stop();
|
|
2160
2588
|
}
|
|
@@ -2451,6 +2879,19 @@ async function resolveVariables(variables, secrets) {
|
|
|
2451
2879
|
}
|
|
2452
2880
|
//#endregion
|
|
2453
2881
|
//#region src/reactor/step-test.ts
|
|
2882
|
+
/**
|
|
2883
|
+
* A step record's status as a test reports it.
|
|
2884
|
+
*
|
|
2885
|
+
* Collapsing this to `FAILED : SUCCEEDED` is how an INDETERMINATE test came
|
|
2886
|
+
* to read green — the one status that must never be mistaken for a confirmed
|
|
2887
|
+
* one, since the whole point of it is that nobody knows whether the write
|
|
2888
|
+
* landed.
|
|
2889
|
+
*/
|
|
2890
|
+
function testStatusOf(status) {
|
|
2891
|
+
if (status === "FAILED") return "FAILED";
|
|
2892
|
+
if (status === "INDETERMINATE") return "INDETERMINATE";
|
|
2893
|
+
return "SUCCEEDED";
|
|
2894
|
+
}
|
|
2454
2895
|
function collectStrings(value, into) {
|
|
2455
2896
|
if (typeof value === "string") into.push(value);
|
|
2456
2897
|
else if (Array.isArray(value)) for (const item of value) collectStrings(item, into);
|
|
@@ -2534,6 +2975,7 @@ function untestedError(label, sample) {
|
|
|
2534
2975
|
case "hidden": return `Test ${label} first: its last test is not visible to you`;
|
|
2535
2976
|
case "failed": return `Test ${label} first: its last test failed`;
|
|
2536
2977
|
case "truncated": return `Test ${label} again: the journal kept only a truncated copy of its last output`;
|
|
2978
|
+
case "indeterminate": return `Test ${label} again: its last test is INDETERMINATE, so it has no confirmed output`;
|
|
2537
2979
|
default: return `Test ${label} first`;
|
|
2538
2980
|
}
|
|
2539
2981
|
}
|
|
@@ -7095,6 +7537,59 @@ function parseParameter(param) {
|
|
|
7095
7537
|
return parseValueExpression(param);
|
|
7096
7538
|
}
|
|
7097
7539
|
//#endregion
|
|
7540
|
+
//#region src/reactor/policy.ts
|
|
7541
|
+
/** The pre-enforcement behaviour, for a definition that declares no policy. */
|
|
7542
|
+
const UNENFORCED_POLICY = Object.freeze({
|
|
7543
|
+
concurrency: "PARALLEL",
|
|
7544
|
+
maxParallelRuns: null,
|
|
7545
|
+
runTimeoutSeconds: null,
|
|
7546
|
+
onFailure: "IGNORE",
|
|
7547
|
+
defaultRetry: null
|
|
7548
|
+
});
|
|
7549
|
+
const CONCURRENCY_MODES = [
|
|
7550
|
+
"SINGLETON",
|
|
7551
|
+
"QUEUE",
|
|
7552
|
+
"PARALLEL"
|
|
7553
|
+
];
|
|
7554
|
+
const FAILURE_MODES = [
|
|
7555
|
+
"PARK",
|
|
7556
|
+
"NOTIFY",
|
|
7557
|
+
"IGNORE"
|
|
7558
|
+
];
|
|
7559
|
+
function positive(value) {
|
|
7560
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
7561
|
+
}
|
|
7562
|
+
/** What the runtime enforces for a run of this definition. */
|
|
7563
|
+
function effectiveRunPolicy(runnable) {
|
|
7564
|
+
const policy = runnable.policy;
|
|
7565
|
+
if (!policy) return UNENFORCED_POLICY;
|
|
7566
|
+
const concurrency = CONCURRENCY_MODES.includes(policy.concurrency) ? policy.concurrency : "PARALLEL";
|
|
7567
|
+
return {
|
|
7568
|
+
concurrency,
|
|
7569
|
+
maxParallelRuns: concurrency === "PARALLEL" ? positive(policy.maxParallelRuns) : null,
|
|
7570
|
+
runTimeoutSeconds: positive(policy.runTimeoutSeconds),
|
|
7571
|
+
onFailure: FAILURE_MODES.includes(policy.onFailure) ? policy.onFailure : "IGNORE",
|
|
7572
|
+
defaultRetry: effectiveRetryPolicy(policy.defaultRetry)
|
|
7573
|
+
};
|
|
7574
|
+
}
|
|
7575
|
+
/** How many runs of one workflow may execute at once; 1 for both serialising
|
|
7576
|
+
* modes, so the gate holds one number rather than three branches. Null is
|
|
7577
|
+
* unbounded, i.e. no gate at all. */
|
|
7578
|
+
function concurrencyLimit(policy) {
|
|
7579
|
+
if (policy.concurrency === "PARALLEL") return policy.maxParallelRuns;
|
|
7580
|
+
return 1;
|
|
7581
|
+
}
|
|
7582
|
+
/** The trigger status PARK leaves on a supervised trigger's row. Not ENABLED,
|
|
7583
|
+
* so the supervisor's due-trigger query does not return it and the schedule
|
|
7584
|
+
* stops refiring; a re-publish or re-enable arms it again. The park itself, for
|
|
7585
|
+
* every trigger kind, is the store's `workflow_park` row. */
|
|
7586
|
+
const PARKED_TRIGGER_STATUS = "PARKED";
|
|
7587
|
+
/** The run status a firing skipped by SINGLETON is journaled under, and the
|
|
7588
|
+
* one a run that passed `runTimeoutSeconds` ends in. Both are "this run did
|
|
7589
|
+
* not finish and nothing is wrong with the workflow", which is what the
|
|
7590
|
+
* document model's RunStatus.CANCELLED means. */
|
|
7591
|
+
const CANCELLED_RUN_STATUS = "CANCELLED";
|
|
7592
|
+
//#endregion
|
|
7098
7593
|
//#region src/reactor/store.ts
|
|
7099
7594
|
function triggerRowBlock(row) {
|
|
7100
7595
|
return {
|
|
@@ -7109,7 +7604,20 @@ function triggerBlockColumns(block) {
|
|
|
7109
7604
|
trigger_name: block.name
|
|
7110
7605
|
};
|
|
7111
7606
|
}
|
|
7112
|
-
const
|
|
7607
|
+
const FIRED_WITHOUT_RUN_ID = "fired-without-run";
|
|
7608
|
+
const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
|
|
7609
|
+
function abandonedFireError(attempts) {
|
|
7610
|
+
return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
|
|
7611
|
+
}
|
|
7612
|
+
function workflowParkRow(workflowId, publishedVersion, reason) {
|
|
7613
|
+
return {
|
|
7614
|
+
workflow_id: workflowId,
|
|
7615
|
+
published_version: publishedVersion,
|
|
7616
|
+
reason: redactMessage(reason),
|
|
7617
|
+
parked_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7618
|
+
};
|
|
7619
|
+
}
|
|
7620
|
+
const logger$4 = childLogger([
|
|
7113
7621
|
"workflow",
|
|
7114
7622
|
"runtime",
|
|
7115
7623
|
"store"
|
|
@@ -7136,6 +7644,7 @@ async function up(db) {
|
|
|
7136
7644
|
} catch {}
|
|
7137
7645
|
await db.schema.createTable("trigger_state").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("trigger_name", "text", (col) => col.notNull()).addColumn("config_hash", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("store_state", "text", (col) => col.notNull()).addColumn("interval_ms", "integer", (col) => col.notNull()).addColumn("next_poll_at", "text").addColumn("last_poll_at", "text").addColumn("last_error", "text").addColumn("consecutive_failures", "integer", (col) => col.notNull()).addColumn("lease_owner", "text").addColumn("lease_expires_at", "text").addColumn("updated_at", "text", (col) => col.notNull()).ifNotExists().execute();
|
|
7138
7646
|
await db.schema.createTable("trigger_dedupe").addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("dedupe_key", "text", (col) => col.notNull()).addColumn("run_id", "text").addColumn("created_at", "text", (col) => col.notNull()).addPrimaryKeyConstraint("trigger_dedupe_pk", ["workflow_id", "dedupe_key"]).ifNotExists().execute();
|
|
7647
|
+
await db.schema.createTable("workflow_park").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("published_version", "integer", (col) => col.notNull()).addColumn("reason", "text", (col) => col.notNull()).addColumn("parked_at", "text", (col) => col.notNull()).ifNotExists().execute();
|
|
7139
7648
|
await db.schema.createTable("step_execution").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("ordinal", "integer", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("input", "text").addColumn("output", "text").addColumn("port", "text").addColumn("error", "text").addColumn("started_at", "text").addColumn("ended_at", "text").addUniqueConstraint("step_execution_run_step", ["run_id", "step_id"]).ifNotExists().execute();
|
|
7140
7649
|
for (const column of ["started_at", "ended_at"]) try {
|
|
7141
7650
|
await db.schema.alterTable("step_execution").addColumn(column, "text").execute();
|
|
@@ -7154,6 +7663,21 @@ async function up(db) {
|
|
|
7154
7663
|
]) try {
|
|
7155
7664
|
await db.schema.alterTable(table).addColumn(column, "text").execute();
|
|
7156
7665
|
} catch {}
|
|
7666
|
+
try {
|
|
7667
|
+
await db.schema.alterTable("trigger_dedupe").addColumn("attempts", "integer", (col) => col.notNull().defaultTo(1)).execute();
|
|
7668
|
+
} catch {}
|
|
7669
|
+
await db.schema.createTable("migration_mark").addColumn("name", "text", (col) => col.primaryKey()).addColumn("applied_at", "text", (col) => col.notNull()).ifNotExists().execute();
|
|
7670
|
+
await db.transaction().execute(async (trx) => {
|
|
7671
|
+
if (await trx.selectFrom("migration_mark").select("name").where("name", "=", DEDUPE_CLAIM_MIGRATION).executeTakeFirst()) return;
|
|
7672
|
+
await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "is", null).execute();
|
|
7673
|
+
await trx.insertInto("migration_mark").values({
|
|
7674
|
+
name: DEDUPE_CLAIM_MIGRATION,
|
|
7675
|
+
applied_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7676
|
+
}).onConflict((oc) => oc.column("name").doNothing()).execute();
|
|
7677
|
+
});
|
|
7678
|
+
try {
|
|
7679
|
+
await db.schema.alterTable("step_execution").addColumn("attempts", "integer").execute();
|
|
7680
|
+
} catch {}
|
|
7157
7681
|
try {
|
|
7158
7682
|
await db.schema.alterTable("trigger_state").addColumn("next_renew_at", "text").execute();
|
|
7159
7683
|
} catch {}
|
|
@@ -7245,7 +7769,7 @@ async function migrateTriggerStoreState(db) {
|
|
|
7245
7769
|
try {
|
|
7246
7770
|
rows = await db.selectFrom("trigger_state").select(["workflow_id", "store_state"]).orderBy("updated_at", "asc").orderBy("workflow_id", "asc").execute();
|
|
7247
7771
|
} catch (error) {
|
|
7248
|
-
logger$
|
|
7772
|
+
logger$4.error("Could not read trigger_state to migrate it: @error", error);
|
|
7249
7773
|
return unmigrated;
|
|
7250
7774
|
}
|
|
7251
7775
|
const pending = [];
|
|
@@ -7261,7 +7785,7 @@ async function migrateTriggerStoreState(db) {
|
|
|
7261
7785
|
await migrateOneRow(db, row, projectWinner);
|
|
7262
7786
|
} catch (error) {
|
|
7263
7787
|
unmigrated.add(row.workflowId);
|
|
7264
|
-
logger$
|
|
7788
|
+
logger$4.error(`Could not migrate trigger store state for ${row.workflowId}`, error);
|
|
7265
7789
|
}
|
|
7266
7790
|
return unmigrated;
|
|
7267
7791
|
}
|
|
@@ -7271,7 +7795,7 @@ function parseLegacyBlob(row, unmigrated) {
|
|
|
7271
7795
|
try {
|
|
7272
7796
|
state = JSON.parse(row.store_state);
|
|
7273
7797
|
} catch {
|
|
7274
|
-
logger$
|
|
7798
|
+
logger$4.warn(`Leaving unparseable store_state for ${row.workflow_id} in place`);
|
|
7275
7799
|
unmigrated.add(row.workflow_id);
|
|
7276
7800
|
return null;
|
|
7277
7801
|
}
|
|
@@ -7292,7 +7816,7 @@ function parseLegacyBlob(row, unmigrated) {
|
|
|
7292
7816
|
});
|
|
7293
7817
|
continue;
|
|
7294
7818
|
}
|
|
7295
|
-
if (key.startsWith("test")) logger$
|
|
7819
|
+
if (key.startsWith("test")) logger$4.info(`Migrating "${key}" for ${row.workflow_id} as a project key; it may be a test leftover`);
|
|
7296
7820
|
entries.push({
|
|
7297
7821
|
scope: "PROJECT",
|
|
7298
7822
|
scopeKey: PROJECT_SCOPE_KEY,
|
|
@@ -7314,7 +7838,7 @@ function resolveProjectCollisions(pending) {
|
|
|
7314
7838
|
}
|
|
7315
7839
|
winner.set(entry.key, row.workflowId);
|
|
7316
7840
|
}
|
|
7317
|
-
for (const [key, workflows] of contested) logger$
|
|
7841
|
+
for (const [key, workflows] of contested) logger$4.warn(`Project store key "@key" was written by ${workflows.join(", ")}; keeping the value last updated, from ${winner.get(key)}, and discarding the rest`, key);
|
|
7318
7842
|
return winner;
|
|
7319
7843
|
}
|
|
7320
7844
|
async function migrateOneRow(db, row, projectWinner) {
|
|
@@ -7343,9 +7867,9 @@ async function insertPieceStoreIfAbsent(db, scope, scopeKey, key, value) {
|
|
|
7343
7867
|
}
|
|
7344
7868
|
function warnIfOverPieceStoreLimits(scope, scopeKey, key, encoded) {
|
|
7345
7869
|
const at = `${scope}/${scopeKey}/${key}`;
|
|
7346
|
-
if (key.length > 128) logger$
|
|
7870
|
+
if (key.length > 128) logger$4.warn(`Migrated store key ${at} is ${key.length} chars, over the 128 limit; writes to it will fail`);
|
|
7347
7871
|
const size = Buffer.byteLength(encoded, "utf8");
|
|
7348
|
-
if (size > 524288) logger$
|
|
7872
|
+
if (size > 524288) logger$4.warn(`Migrated store value ${at} is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} limit; writes to it will fail`);
|
|
7349
7873
|
}
|
|
7350
7874
|
const PIECE_STORE_MAX_VALUE_BYTES = 524288;
|
|
7351
7875
|
var PieceStoreLimitError = class extends Error {
|
|
@@ -7362,10 +7886,50 @@ function assertPieceStoreEntry(key, value) {
|
|
|
7362
7886
|
if (size > 524288) throw new PieceStoreLimitError(`Store value for "${key}" is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} byte limit`);
|
|
7363
7887
|
}
|
|
7364
7888
|
const STEP_PAYLOAD_PREFIX_CHARS = 32768;
|
|
7889
|
+
/**
|
|
7890
|
+
* The reserved key a truncation marker is recognised by, and its value.
|
|
7891
|
+
*
|
|
7892
|
+
* A marker has to be distinguishable from a legitimate payload that merely
|
|
7893
|
+
* LOOKS like one, because a run whose output is mistaken for a marker is a
|
|
7894
|
+
* step that silently re-executes on rerun. The original predicate duck-typed
|
|
7895
|
+
* `{truncated: true, bytes: number, prefix: string}` (review backlog item 15),
|
|
7896
|
+
* which any piece returning a truncation report of its own would match — and
|
|
7897
|
+
* "did we truncate this?" is exactly the shape such a report takes.
|
|
7898
|
+
*
|
|
7899
|
+
* So the marker is keyed on a namespaced sentinel whose VALUE is a versioned
|
|
7900
|
+
* magic string. A piece would have to emit that exact pair to collide, which
|
|
7901
|
+
* is not something a payload produces by accident.
|
|
7902
|
+
*/
|
|
7903
|
+
const TRUNCATED_PAYLOAD_KEY = "@powerhousedao/reactor-workflow:runJournal";
|
|
7904
|
+
const TRUNCATED_PAYLOAD_SENTINEL = "truncated-payload/v1";
|
|
7905
|
+
const LEGACY_MARKER_KEYS = [
|
|
7906
|
+
"truncated",
|
|
7907
|
+
"bytes",
|
|
7908
|
+
"prefix",
|
|
7909
|
+
"documentId",
|
|
7910
|
+
"driveId"
|
|
7911
|
+
];
|
|
7912
|
+
/**
|
|
7913
|
+
* True for a journaled value this store truncated.
|
|
7914
|
+
*
|
|
7915
|
+
* Two shapes are accepted and only one is written. New rows carry the reserved
|
|
7916
|
+
* key, which a payload cannot produce by accident. Rows journaled before it
|
|
7917
|
+
* carry the old `{truncated: true, bytes, prefix}`, and those still have to be
|
|
7918
|
+
* recognised: reading one as data would re-run the side-effectful step that
|
|
7919
|
+
* produced it, which is the whole bug.
|
|
7920
|
+
*
|
|
7921
|
+
* So the legacy branch is matched by its EXACT key set, not by duck-typing.
|
|
7922
|
+
* `{truncated, bytes, prefix, source}` — a truncation report a piece might
|
|
7923
|
+
* legitimately return, and the collision the review found — is no longer a
|
|
7924
|
+
* marker. A payload whose keys are exactly the legacy marker's is still
|
|
7925
|
+
* ambiguous, and nothing can resolve that for a row already written; the
|
|
7926
|
+
* window is as narrow as the stored data allows.
|
|
7927
|
+
*/
|
|
7365
7928
|
function isTruncatedStepPayload(value) {
|
|
7366
7929
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
7367
7930
|
const record = value;
|
|
7368
|
-
|
|
7931
|
+
if (record["@powerhousedao/reactor-workflow:runJournal"] === "truncated-payload/v1") return true;
|
|
7932
|
+
return record.truncated === true && typeof record.bytes === "number" && typeof record.prefix === "string" && Object.keys(record).every((key) => LEGACY_MARKER_KEYS.includes(key));
|
|
7369
7933
|
}
|
|
7370
7934
|
function topLevelDocumentIds(value) {
|
|
7371
7935
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return {};
|
|
@@ -7384,7 +7948,7 @@ function cappedPayload(value) {
|
|
|
7384
7948
|
const last = prefix.charCodeAt(prefix.length - 1);
|
|
7385
7949
|
if (last >= 55296 && last <= 56319) prefix = prefix.slice(0, -1);
|
|
7386
7950
|
const marker = {
|
|
7387
|
-
|
|
7951
|
+
[TRUNCATED_PAYLOAD_KEY]: TRUNCATED_PAYLOAD_SENTINEL,
|
|
7388
7952
|
bytes,
|
|
7389
7953
|
prefix,
|
|
7390
7954
|
...topLevelDocumentIds(value)
|
|
@@ -7401,7 +7965,7 @@ function stepValues(runId, ordinal, step) {
|
|
|
7401
7965
|
block_name: step.blockName,
|
|
7402
7966
|
status: step.status,
|
|
7403
7967
|
input: cappedPayload(redact(step.input)),
|
|
7404
|
-
output: cappedPayload(redact(referenceDocuments(step.output))),
|
|
7968
|
+
output: cappedPayload(redact(referenceDocuments(step.journaledOutput !== void 0 ? step.journaledOutput : step.output))),
|
|
7405
7969
|
port: step.port ?? null,
|
|
7406
7970
|
error: step.error ? redactMessage(step.error) : null,
|
|
7407
7971
|
error_name: step.errorName ?? null,
|
|
@@ -7411,7 +7975,8 @@ function stepValues(runId, ordinal, step) {
|
|
|
7411
7975
|
piece_source: step.piece?.source ?? null,
|
|
7412
7976
|
version_match: step.piece?.match ?? null,
|
|
7413
7977
|
version_note: step.piece?.note ?? null,
|
|
7414
|
-
config_hash: step.configHash ?? null
|
|
7978
|
+
config_hash: step.configHash ?? null,
|
|
7979
|
+
attempts: step.attempts ?? null
|
|
7415
7980
|
};
|
|
7416
7981
|
}
|
|
7417
7982
|
function runWarningNotes(result) {
|
|
@@ -7460,15 +8025,26 @@ function journaledPayloadNames(json, ids) {
|
|
|
7460
8025
|
});
|
|
7461
8026
|
}
|
|
7462
8027
|
const erasedRuns = /* @__PURE__ */ new Set();
|
|
7463
|
-
async function
|
|
8028
|
+
async function pollSucceededIn(db, workflowId, storeState, nowIso, nextPollAtIso) {
|
|
8029
|
+
await db.updateTable("trigger_state").set({
|
|
8030
|
+
store_state: storeState,
|
|
8031
|
+
last_poll_at: nowIso,
|
|
8032
|
+
next_poll_at: nextPollAtIso,
|
|
8033
|
+
last_error: null,
|
|
8034
|
+
consecutive_failures: 0,
|
|
8035
|
+
updated_at: nowIso
|
|
8036
|
+
}).where("workflow_id", "=", workflowId).execute();
|
|
8037
|
+
}
|
|
8038
|
+
async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso, runId = FIRED_WITHOUT_RUN_ID) {
|
|
7464
8039
|
const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
|
|
7465
8040
|
await db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
|
|
7466
8041
|
return await db.insertInto("trigger_dedupe").values({
|
|
7467
8042
|
workflow_id: workflowId,
|
|
7468
8043
|
dedupe_key: dedupeKey,
|
|
7469
8044
|
run_id: runId,
|
|
7470
|
-
created_at: nowIso
|
|
7471
|
-
|
|
8045
|
+
created_at: nowIso,
|
|
8046
|
+
attempts: 1
|
|
8047
|
+
}).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id: runId }).where("trigger_dedupe.run_id", "is", null)).returning("dedupe_key").executeTakeFirst() !== void 0;
|
|
7472
8048
|
}
|
|
7473
8049
|
const STEP_COLUMNS_WITHOUT_DATA = [
|
|
7474
8050
|
"id",
|
|
@@ -7488,7 +8064,8 @@ const STEP_COLUMNS_WITHOUT_DATA = [
|
|
|
7488
8064
|
"piece_source",
|
|
7489
8065
|
"version_match",
|
|
7490
8066
|
"version_note",
|
|
7491
|
-
"config_hash"
|
|
8067
|
+
"config_hash",
|
|
8068
|
+
"attempts"
|
|
7492
8069
|
];
|
|
7493
8070
|
const PRUNE_BATCH_SIZE = 500;
|
|
7494
8071
|
var WorkflowRunStore = class WorkflowRunStore {
|
|
@@ -7521,7 +8098,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7521
8098
|
}).where("status", "=", "RUNNING");
|
|
7522
8099
|
if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
|
|
7523
8100
|
const recovered = (await query.returning("id").execute()).length;
|
|
7524
|
-
if (recovered > 0) logger$
|
|
8101
|
+
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
|
|
7525
8102
|
return recovered;
|
|
7526
8103
|
}
|
|
7527
8104
|
async recoverAbandonedRuns() {
|
|
@@ -7532,25 +8109,197 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7532
8109
|
}).where("status", "=", PENDING_RUN_STATUS);
|
|
7533
8110
|
if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
|
|
7534
8111
|
const recovered = (await query.returning("id").execute()).length;
|
|
7535
|
-
if (recovered > 0) logger$
|
|
8112
|
+
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
|
|
7536
8113
|
return recovered;
|
|
7537
8114
|
}
|
|
8115
|
+
/**
|
|
8116
|
+
* Closes a run out as CANCELLED, with the reason as its error.
|
|
8117
|
+
*
|
|
8118
|
+
* Two callers, one meaning — "this run did not finish, and the workflow is
|
|
8119
|
+
* not at fault": a firing SINGLETON concurrency refused, and a run that
|
|
8120
|
+
* passed its `runTimeoutSeconds`. Separate from `failRun` because a
|
|
8121
|
+
* CANCELLED run is not rerunnable and must not read as a workflow defect.
|
|
8122
|
+
*/
|
|
8123
|
+
async cancelRun(runId, reason) {
|
|
8124
|
+
this.runsInFlight.delete(runId);
|
|
8125
|
+
if (erasedRuns.delete(runId)) return;
|
|
8126
|
+
await this.db.updateTable("run").set({
|
|
8127
|
+
status: "CANCELLED",
|
|
8128
|
+
error: redactMessage(reason),
|
|
8129
|
+
ended_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8130
|
+
}).where("id", "=", runId).execute();
|
|
8131
|
+
}
|
|
7538
8132
|
async enqueueRun(options) {
|
|
7539
8133
|
const id = randomUUID();
|
|
7540
8134
|
await this.insertPendingRun(this.db, id, options);
|
|
7541
8135
|
this.runsInFlight.add(id);
|
|
7542
8136
|
return id;
|
|
7543
8137
|
}
|
|
8138
|
+
/**
|
|
8139
|
+
* Claims the dedupe key and journals the PENDING run, bounding crash replays.
|
|
8140
|
+
*
|
|
8141
|
+
* Three outcomes, as {@link FireClaim}:
|
|
8142
|
+
*
|
|
8143
|
+
* - **`"claimed"`** with a run id: this fire is ours to run.
|
|
8144
|
+
* - **`"duplicate"`**: an earlier delivery of this operation already
|
|
8145
|
+
* fired: it journaled a run, or ran without one
|
|
8146
|
+
* ({@link FIRED_WITHOUT_RUN_ID}).
|
|
8147
|
+
* - **`"abandoned"`**: the operation has been delivered
|
|
8148
|
+
* {@link FIRE_CRASH_BUDGET} times and has never once got as far as
|
|
8149
|
+
* journaling a run — i.e. it is taking the process down before the write
|
|
8150
|
+
* lands, and the read-model cursor replays it on every boot. That is the
|
|
8151
|
+
* EPIPE boot loop (backlog item 5), and a budget is what turns it from an
|
|
8152
|
+
* unbootable reactor into one FAILED run naming the loop.
|
|
8153
|
+
*
|
|
8154
|
+
* Two writes, and which one is which matters:
|
|
8155
|
+
*
|
|
8156
|
+
* - The **attempt count** is bumped in its own committed transaction,
|
|
8157
|
+
* BEFORE the risky work. That is the only way a delivery that leaves no
|
|
8158
|
+
* run behind can be counted at all, which is what the budget needs. The
|
|
8159
|
+
* counter is never a claim signal — it only decides the budget.
|
|
8160
|
+
* - The **claim** is `run_id`, and it is set in the SAME transaction as the
|
|
8161
|
+
* run row, under a `WHERE run_id IS NULL` guard. The guard takes the
|
|
8162
|
+
* row's lock, so of two concurrent deliveries of one operation the second
|
|
8163
|
+
* blocks until the first commits and then matches no row: exactly one
|
|
8164
|
+
* delivery wins, and the run it journaled is committed with the claim. A
|
|
8165
|
+
* crash in between rolls BOTH back, so a claim that holds a run id always
|
|
8166
|
+
* holds a run — and a claim without one is retried, up to the budget.
|
|
8167
|
+
*
|
|
8168
|
+
* Counting and claiming are deliberately not the same write: if the bump
|
|
8169
|
+
* rode along inside the claim transaction, a run insert that takes the
|
|
8170
|
+
* process down would roll the count back with it and the budget could never
|
|
8171
|
+
* reach its limit — the boot loop it exists to bound.
|
|
8172
|
+
*/
|
|
7544
8173
|
async claimAndEnqueueRun(dedupeKey, ttlMs, nowIso, options) {
|
|
8174
|
+
const attempt = await this.recordFireAttempt(options.workflowId, dedupeKey, ttlMs, nowIso);
|
|
8175
|
+
if (attempt.runId !== null) return { outcome: "duplicate" };
|
|
8176
|
+
if (attempt.attempts > 3) return {
|
|
8177
|
+
outcome: "abandoned",
|
|
8178
|
+
attempts: attempt.attempts
|
|
8179
|
+
};
|
|
7545
8180
|
const id = randomUUID();
|
|
7546
8181
|
if (!await this.db.transaction().execute(async (trx) => {
|
|
7547
|
-
if (
|
|
8182
|
+
if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
|
|
7548
8183
|
await this.insertPendingRun(trx, id, options);
|
|
7549
8184
|
return true;
|
|
7550
|
-
})) return
|
|
8185
|
+
})) return { outcome: "duplicate" };
|
|
8186
|
+
this.runsInFlight.add(id);
|
|
8187
|
+
return {
|
|
8188
|
+
outcome: "claimed",
|
|
8189
|
+
runId: id
|
|
8190
|
+
};
|
|
8191
|
+
}
|
|
8192
|
+
async journalTriggerItems(items, ttlMs, nowIso) {
|
|
8193
|
+
const ids = await this.db.transaction().execute(async (trx) => {
|
|
8194
|
+
const journaled = [];
|
|
8195
|
+
for (const { dedupeKey, options } of items) {
|
|
8196
|
+
const id = randomUUID();
|
|
8197
|
+
const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
|
|
8198
|
+
if (won) await this.insertPendingRun(trx, id, options);
|
|
8199
|
+
journaled.push(won ? id : void 0);
|
|
8200
|
+
}
|
|
8201
|
+
return journaled;
|
|
8202
|
+
});
|
|
8203
|
+
for (const id of ids) if (id) this.runsInFlight.add(id);
|
|
8204
|
+
return ids;
|
|
8205
|
+
}
|
|
8206
|
+
async recordScheduleFire(nowIso, nextPollAtIso, options) {
|
|
8207
|
+
const id = randomUUID();
|
|
8208
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8209
|
+
await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
|
|
8210
|
+
await this.insertPendingRun(trx, id, options);
|
|
8211
|
+
});
|
|
7551
8212
|
this.runsInFlight.add(id);
|
|
7552
8213
|
return id;
|
|
7553
8214
|
}
|
|
8215
|
+
/**
|
|
8216
|
+
* Puts a piece_store partition back to `entries` and, in the same write,
|
|
8217
|
+
* cancels the runs the restored cursor delivers again and releases their
|
|
8218
|
+
* claims, so no reader sees the old cursor while the claims still hold.
|
|
8219
|
+
*/
|
|
8220
|
+
async rewindPieceStore(scope, scopeKey, entries, redelivered = {
|
|
8221
|
+
runIds: [],
|
|
8222
|
+
reason: ""
|
|
8223
|
+
}) {
|
|
8224
|
+
const live = redelivered.runIds.filter((runId) => !erasedRuns.has(runId));
|
|
8225
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
8226
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8227
|
+
await trx.deleteFrom("piece_store").where("scope", "=", scope).where("scope_key", "=", scopeKey).execute();
|
|
8228
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
8229
|
+
assertPieceStoreEntry(key, value);
|
|
8230
|
+
await trx.insertInto("piece_store").values({
|
|
8231
|
+
scope,
|
|
8232
|
+
scope_key: scopeKey,
|
|
8233
|
+
key,
|
|
8234
|
+
value: JSON.stringify(value),
|
|
8235
|
+
updated_at: nowIso
|
|
8236
|
+
}).execute();
|
|
8237
|
+
}
|
|
8238
|
+
if (live.length === 0) return;
|
|
8239
|
+
await trx.updateTable("run").set({
|
|
8240
|
+
status: "CANCELLED",
|
|
8241
|
+
error: redactMessage(redelivered.reason),
|
|
8242
|
+
ended_at: nowIso
|
|
8243
|
+
}).where("id", "in", live).execute();
|
|
8244
|
+
await trx.deleteFrom("trigger_dedupe").where("run_id", "in", live).execute();
|
|
8245
|
+
});
|
|
8246
|
+
for (const runId of redelivered.runIds) {
|
|
8247
|
+
this.runsInFlight.delete(runId);
|
|
8248
|
+
erasedRuns.delete(runId);
|
|
8249
|
+
}
|
|
8250
|
+
}
|
|
8251
|
+
/**
|
|
8252
|
+
* Counts one delivery of an operation, and reports whether a run was ever
|
|
8253
|
+
* journaled for it.
|
|
8254
|
+
*
|
|
8255
|
+
* Committed on its own: the point is to leave a trace even when whatever
|
|
8256
|
+
* comes next takes the process down. It is NOT a claim — the row it writes
|
|
8257
|
+
* holds no run id, and {@link claimAndEnqueueRun} is what decides, under a
|
|
8258
|
+
* lock, which delivery gets to fill it in.
|
|
8259
|
+
*/
|
|
8260
|
+
async recordFireAttempt(workflowId, dedupeKey, ttlMs, nowIso) {
|
|
8261
|
+
const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
|
|
8262
|
+
await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
|
|
8263
|
+
const row = await this.db.insertInto("trigger_dedupe").values({
|
|
8264
|
+
workflow_id: workflowId,
|
|
8265
|
+
dedupe_key: dedupeKey,
|
|
8266
|
+
run_id: null,
|
|
8267
|
+
created_at: nowIso,
|
|
8268
|
+
attempts: 1
|
|
8269
|
+
}).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ attempts: sql`${sql.table("trigger_dedupe")}.attempts + 1` })).returning(["attempts", "run_id"]).executeTakeFirstOrThrow();
|
|
8270
|
+
return {
|
|
8271
|
+
attempts: row.attempts ?? 1,
|
|
8272
|
+
runId: row.run_id
|
|
8273
|
+
};
|
|
8274
|
+
}
|
|
8275
|
+
/**
|
|
8276
|
+
* The abandonment of an over-budget fire, as a visible FAILED run, linked to
|
|
8277
|
+
* its dedupe key so a later delivery is a duplicate rather than another one.
|
|
8278
|
+
*/
|
|
8279
|
+
async journalAbandonedFire(dedupeKey, options) {
|
|
8280
|
+
const id = randomUUID();
|
|
8281
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8282
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8283
|
+
await trx.insertInto("run").values({
|
|
8284
|
+
id,
|
|
8285
|
+
workflow_id: options.workflowId,
|
|
8286
|
+
workflow_name: "",
|
|
8287
|
+
workflow_version: 0,
|
|
8288
|
+
trigger_kind: options.triggerKind,
|
|
8289
|
+
trigger_payload: cappedPayload(redact(options.triggerPayload)),
|
|
8290
|
+
status: "FAILED",
|
|
8291
|
+
error: redactMessage(abandonedFireError(options.attempts)),
|
|
8292
|
+
enqueued_at: now,
|
|
8293
|
+
started_at: now,
|
|
8294
|
+
ended_at: now,
|
|
8295
|
+
rerun_of: null,
|
|
8296
|
+
warnings: 0,
|
|
8297
|
+
warning_notes: null
|
|
8298
|
+
}).execute();
|
|
8299
|
+
await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).execute();
|
|
8300
|
+
});
|
|
8301
|
+
return id;
|
|
8302
|
+
}
|
|
7554
8303
|
async insertPendingRun(db, id, options) {
|
|
7555
8304
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7556
8305
|
await db.insertInto("run").values({
|
|
@@ -7616,7 +8365,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7616
8365
|
if (result.steps.length > 0) try {
|
|
7617
8366
|
await this.sweepSteps(runId, result, executionOrder);
|
|
7618
8367
|
} catch (error) {
|
|
7619
|
-
logger$
|
|
8368
|
+
logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
|
|
7620
8369
|
}
|
|
7621
8370
|
const notes = runWarningNotes(result);
|
|
7622
8371
|
await this.db.updateTable("run").set({
|
|
@@ -7717,6 +8466,58 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7717
8466
|
for (const row of rows) byRun.get(row.run_id)?.push(row.document_id);
|
|
7718
8467
|
return byRun;
|
|
7719
8468
|
}
|
|
8469
|
+
/**
|
|
8470
|
+
* Parks a workflow: its park row, and an ENABLED trigger_state row turned
|
|
8471
|
+
* PARKED, in one transaction. True when a trigger_state row was parked.
|
|
8472
|
+
*/
|
|
8473
|
+
async parkWorkflow(workflowId, publishedVersion, reason) {
|
|
8474
|
+
const { workflow_id: _, ...row } = workflowParkRow(workflowId, publishedVersion, reason);
|
|
8475
|
+
const now = row.parked_at;
|
|
8476
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8477
|
+
await trx.insertInto("workflow_park").values({
|
|
8478
|
+
workflow_id: workflowId,
|
|
8479
|
+
...row
|
|
8480
|
+
}).onConflict((oc) => oc.column("workflow_id").doUpdateSet(row)).execute();
|
|
8481
|
+
return (await trx.updateTable("trigger_state").set({
|
|
8482
|
+
status: PARKED_TRIGGER_STATUS,
|
|
8483
|
+
last_error: row.reason,
|
|
8484
|
+
updated_at: now
|
|
8485
|
+
}).where("workflow_id", "=", workflowId).where("status", "=", "ENABLED").returning("workflow_id").execute()).length > 0;
|
|
8486
|
+
});
|
|
8487
|
+
}
|
|
8488
|
+
/** A disable clears a park: the park row, and a PARKED trigger row turned
|
|
8489
|
+
* DISABLED, so re-enabling arms it whether or not the supervisor held it. */
|
|
8490
|
+
async clearParkOnDisable(workflowId) {
|
|
8491
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8492
|
+
await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
|
|
8493
|
+
await trx.updateTable("trigger_state").set({
|
|
8494
|
+
status: "DISABLED",
|
|
8495
|
+
...RENEW_CLEARED,
|
|
8496
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8497
|
+
}).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
|
|
8498
|
+
});
|
|
8499
|
+
}
|
|
8500
|
+
/** Undoes {@link parkWorkflow}; `trigger` says whether it parked a row. */
|
|
8501
|
+
async liftPark(workflowId, trigger) {
|
|
8502
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8503
|
+
await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
|
|
8504
|
+
if (!trigger) return;
|
|
8505
|
+
await trx.updateTable("trigger_state").set({
|
|
8506
|
+
status: "ENABLED",
|
|
8507
|
+
last_error: null,
|
|
8508
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8509
|
+
}).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
|
|
8510
|
+
});
|
|
8511
|
+
}
|
|
8512
|
+
async getWorkflowPark(workflowId) {
|
|
8513
|
+
return this.db.selectFrom("workflow_park").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
|
|
8514
|
+
}
|
|
8515
|
+
async listWorkflowParks() {
|
|
8516
|
+
return this.db.selectFrom("workflow_park").selectAll().execute();
|
|
8517
|
+
}
|
|
8518
|
+
async clearWorkflowPark(workflowId) {
|
|
8519
|
+
await this.db.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
|
|
8520
|
+
}
|
|
7720
8521
|
async getTriggerState(workflowId) {
|
|
7721
8522
|
return this.db.selectFrom("trigger_state").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
|
|
7722
8523
|
}
|
|
@@ -7781,14 +8582,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7781
8582
|
return this.db.selectFrom("trigger_state").selectAll().orderBy("workflow_id", "asc").execute();
|
|
7782
8583
|
}
|
|
7783
8584
|
async recordPollSuccess(workflowId, storeState, nowIso, nextPollAtIso) {
|
|
7784
|
-
await this.db
|
|
7785
|
-
store_state: storeState,
|
|
7786
|
-
last_poll_at: nowIso,
|
|
7787
|
-
next_poll_at: nextPollAtIso,
|
|
7788
|
-
last_error: null,
|
|
7789
|
-
consecutive_failures: 0,
|
|
7790
|
-
updated_at: nowIso
|
|
7791
|
-
}).where("workflow_id", "=", workflowId).execute();
|
|
8585
|
+
await pollSucceededIn(this.db, workflowId, storeState, nowIso, nextPollAtIso);
|
|
7792
8586
|
}
|
|
7793
8587
|
async recordPollFailure(workflowId, error, nowIso, nextPollAtIso, consecutiveFailures) {
|
|
7794
8588
|
await this.db.updateTable("trigger_state").set({
|
|
@@ -7800,7 +8594,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7800
8594
|
}).where("workflow_id", "=", workflowId).execute();
|
|
7801
8595
|
}
|
|
7802
8596
|
async claimDedupe(workflowId, dedupeKey, ttlMs, nowIso) {
|
|
7803
|
-
return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso
|
|
8597
|
+
return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso);
|
|
7804
8598
|
}
|
|
7805
8599
|
async deleteDedupe(workflowId) {
|
|
7806
8600
|
await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
|
|
@@ -7848,7 +8642,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7848
8642
|
for (const row of live) erasedRuns.add(row.id);
|
|
7849
8643
|
erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
|
|
7850
8644
|
erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
|
|
7851
|
-
erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id:
|
|
8645
|
+
erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
|
|
7852
8646
|
erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
|
|
7853
8647
|
return erased;
|
|
7854
8648
|
});
|
|
@@ -7900,6 +8694,46 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7900
8694
|
}
|
|
7901
8695
|
};
|
|
7902
8696
|
//#endregion
|
|
8697
|
+
//#region src/reactor/park-state.ts
|
|
8698
|
+
var ParkState = class {
|
|
8699
|
+
load;
|
|
8700
|
+
parks = /* @__PURE__ */ new Map();
|
|
8701
|
+
written = /* @__PURE__ */ new Set();
|
|
8702
|
+
seeded = false;
|
|
8703
|
+
seeding;
|
|
8704
|
+
constructor(load) {
|
|
8705
|
+
this.load = load;
|
|
8706
|
+
}
|
|
8707
|
+
async get(workflowId) {
|
|
8708
|
+
if (!this.seeded) {
|
|
8709
|
+
this.seeding ??= this.seed().finally(() => {
|
|
8710
|
+
this.seeding = void 0;
|
|
8711
|
+
});
|
|
8712
|
+
await this.seeding;
|
|
8713
|
+
}
|
|
8714
|
+
return this.parks.get(workflowId);
|
|
8715
|
+
}
|
|
8716
|
+
known(workflowId) {
|
|
8717
|
+
if (!this.seeded && !this.written.has(workflowId)) return void 0;
|
|
8718
|
+
return { park: this.parks.get(workflowId) };
|
|
8719
|
+
}
|
|
8720
|
+
set(park) {
|
|
8721
|
+
this.parks.set(park.workflow_id, park);
|
|
8722
|
+
if (!this.seeded) this.written.add(park.workflow_id);
|
|
8723
|
+
}
|
|
8724
|
+
delete(workflowId) {
|
|
8725
|
+
this.parks.delete(workflowId);
|
|
8726
|
+
if (!this.seeded) this.written.add(workflowId);
|
|
8727
|
+
}
|
|
8728
|
+
async seed() {
|
|
8729
|
+
const rows = await this.load();
|
|
8730
|
+
if (!rows) return;
|
|
8731
|
+
for (const row of rows) if (!this.written.has(row.workflow_id)) this.parks.set(row.workflow_id, row);
|
|
8732
|
+
this.written.clear();
|
|
8733
|
+
this.seeded = true;
|
|
8734
|
+
}
|
|
8735
|
+
};
|
|
8736
|
+
//#endregion
|
|
7903
8737
|
//#region src/reactor/run-cursor.ts
|
|
7904
8738
|
var InvalidRunCursorError = class extends Error {
|
|
7905
8739
|
constructor() {
|
|
@@ -7924,13 +8758,137 @@ function decodeRunCursor(cursor) {
|
|
|
7924
8758
|
};
|
|
7925
8759
|
}
|
|
7926
8760
|
//#endregion
|
|
8761
|
+
//#region src/reactor/run-gate.ts
|
|
8762
|
+
const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
|
|
8763
|
+
const QUEUE_DEPTH_ENV = "PH_WORKFLOWS_MAX_QUEUED_FIRINGS";
|
|
8764
|
+
function maxQueuedFirings(env = process.env) {
|
|
8765
|
+
const raw = Number(env[QUEUE_DEPTH_ENV]);
|
|
8766
|
+
return Number.isFinite(raw) && raw > 0 ? raw : 100;
|
|
8767
|
+
}
|
|
8768
|
+
var WorkflowRunGate = class {
|
|
8769
|
+
lanes = /* @__PURE__ */ new Map();
|
|
8770
|
+
closed = false;
|
|
8771
|
+
maxQueued;
|
|
8772
|
+
constructor(options = {}) {
|
|
8773
|
+
this.maxQueued = options.maxQueued ?? maxQueuedFirings();
|
|
8774
|
+
}
|
|
8775
|
+
/** Runs of this workflow executing right now. */
|
|
8776
|
+
active(workflowId) {
|
|
8777
|
+
return this.lanes.get(workflowId)?.active ?? 0;
|
|
8778
|
+
}
|
|
8779
|
+
/** Firings of this workflow waiting for a slot. */
|
|
8780
|
+
waiting(workflowId) {
|
|
8781
|
+
return this.lanes.get(workflowId)?.waiting.length ?? 0;
|
|
8782
|
+
}
|
|
8783
|
+
/**
|
|
8784
|
+
* Takes a slot for one run.
|
|
8785
|
+
*
|
|
8786
|
+
* - **PARALLEL** with no `maxParallelRuns`, or no policy at all: no gate,
|
|
8787
|
+
* so an unbounded workflow's hot path costs nothing. Not the default: a
|
|
8788
|
+
* workflow document starts as QUEUE.
|
|
8789
|
+
* - **PARALLEL** with a bound, and **QUEUE**: waits, first come first
|
|
8790
|
+
* served. Latency, not failure — which is what QUEUE means, up to
|
|
8791
|
+
* {@link DEFAULT_MAX_QUEUED_FIRINGS} waiters. Past that the queue is full
|
|
8792
|
+
* and the firing is refused, so the caller journals it CANCELLED rather
|
|
8793
|
+
* than growing a backlog nothing bounds.
|
|
8794
|
+
* - **SINGLETON**: refused outright while a run is active. The caller
|
|
8795
|
+
* journals the refusal rather than dropping it silently, because a firing
|
|
8796
|
+
* that vanished is indistinguishable from a trigger that never fired.
|
|
8797
|
+
*/
|
|
8798
|
+
async admit(workflowId, policy) {
|
|
8799
|
+
if (this.closed) return {
|
|
8800
|
+
admitted: false,
|
|
8801
|
+
reason: CLOSED_REASON,
|
|
8802
|
+
refusal: "closed"
|
|
8803
|
+
};
|
|
8804
|
+
const limit = concurrencyLimit(policy);
|
|
8805
|
+
if (limit === null) return {
|
|
8806
|
+
admitted: true,
|
|
8807
|
+
waited: false,
|
|
8808
|
+
release: () => void 0
|
|
8809
|
+
};
|
|
8810
|
+
const lane = this.lanes.get(workflowId) ?? {
|
|
8811
|
+
active: 0,
|
|
8812
|
+
waiting: []
|
|
8813
|
+
};
|
|
8814
|
+
this.lanes.set(workflowId, lane);
|
|
8815
|
+
let waited = false;
|
|
8816
|
+
if (lane.active >= limit) {
|
|
8817
|
+
if (policy.concurrency === "SINGLETON") {
|
|
8818
|
+
this.forgetIfIdle(workflowId, lane);
|
|
8819
|
+
return {
|
|
8820
|
+
admitted: false,
|
|
8821
|
+
refusal: "singleton",
|
|
8822
|
+
reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
|
|
8823
|
+
};
|
|
8824
|
+
}
|
|
8825
|
+
if (lane.waiting.length >= this.maxQueued) return {
|
|
8826
|
+
admitted: false,
|
|
8827
|
+
refusal: "queue-full",
|
|
8828
|
+
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
8829
|
+
};
|
|
8830
|
+
if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
|
|
8831
|
+
admitted: false,
|
|
8832
|
+
reason: CLOSED_REASON,
|
|
8833
|
+
refusal: "closed"
|
|
8834
|
+
};
|
|
8835
|
+
waited = true;
|
|
8836
|
+
} else lane.active += 1;
|
|
8837
|
+
let released = false;
|
|
8838
|
+
return {
|
|
8839
|
+
admitted: true,
|
|
8840
|
+
waited,
|
|
8841
|
+
release: () => {
|
|
8842
|
+
if (released) return;
|
|
8843
|
+
released = true;
|
|
8844
|
+
lane.active -= 1;
|
|
8845
|
+
const next = lane.waiting.shift();
|
|
8846
|
+
if (next) {
|
|
8847
|
+
lane.active += 1;
|
|
8848
|
+
next(true);
|
|
8849
|
+
return;
|
|
8850
|
+
}
|
|
8851
|
+
this.forgetIfIdle(workflowId, lane);
|
|
8852
|
+
}
|
|
8853
|
+
};
|
|
8854
|
+
}
|
|
8855
|
+
/** Refuses every waiting and later firing: a shut-down runtime runs none. */
|
|
8856
|
+
close() {
|
|
8857
|
+
this.closed = true;
|
|
8858
|
+
for (const lane of this.lanes.values()) for (const waiter of lane.waiting.splice(0)) waiter(false);
|
|
8859
|
+
}
|
|
8860
|
+
forgetIfIdle(workflowId, lane) {
|
|
8861
|
+
if (lane.active === 0 && lane.waiting.length === 0) this.lanes.delete(workflowId);
|
|
8862
|
+
}
|
|
8863
|
+
};
|
|
8864
|
+
//#endregion
|
|
7927
8865
|
//#region src/reactor/run-retention.ts
|
|
7928
8866
|
const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
|
|
7929
8867
|
const RETENTION_SWEEP_INTERVAL_MS = 36e5;
|
|
7930
8868
|
const DAY_MS = 864e5;
|
|
8869
|
+
/** The spellings that turn retention OFF, compared case-insensitively after
|
|
8870
|
+
* trimming. Wider than `0` because "off" and "never" are what an operator
|
|
8871
|
+
* reaches for, and a variable that silently keeps pruning because they wrote
|
|
8872
|
+
* one of those is the worst outcome available. */
|
|
8873
|
+
const RETENTION_OFF_SPELLINGS = [
|
|
8874
|
+
"0",
|
|
8875
|
+
"off",
|
|
8876
|
+
"never",
|
|
8877
|
+
"false",
|
|
8878
|
+
"none"
|
|
8879
|
+
];
|
|
8880
|
+
/**
|
|
8881
|
+
* The retention window, or undefined when an operator turned it off.
|
|
8882
|
+
*
|
|
8883
|
+
* Unset means the default. An unparseable value also means the default rather
|
|
8884
|
+
* than silently disabling: the variable's purpose is a bound, and a typo must
|
|
8885
|
+
* not remove one.
|
|
8886
|
+
*/
|
|
7931
8887
|
function runRetentionMs(env = process.env) {
|
|
7932
|
-
const
|
|
7933
|
-
|
|
8888
|
+
const raw = env[RUN_RETENTION_ENV]?.trim();
|
|
8889
|
+
if (raw !== void 0 && RETENTION_OFF_SPELLINGS.includes(raw.toLowerCase())) return;
|
|
8890
|
+
const days = Number(raw);
|
|
8891
|
+
return (days > 0 ? days : 30) * DAY_MS;
|
|
7934
8892
|
}
|
|
7935
8893
|
async function sweepRetention(store, options) {
|
|
7936
8894
|
const now = (options.now ?? /* @__PURE__ */ new Date()).getTime();
|
|
@@ -7939,13 +8897,33 @@ async function sweepRetention(store, options) {
|
|
|
7939
8897
|
dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
|
|
7940
8898
|
};
|
|
7941
8899
|
}
|
|
8900
|
+
//#endregion
|
|
8901
|
+
//#region src/reactor/shutdown.ts
|
|
8902
|
+
/** A firing refused because this runtime has shut down: retryable elsewhere. */
|
|
8903
|
+
var WorkflowRuntimeClosedError = class extends Error {
|
|
8904
|
+
workflowId;
|
|
8905
|
+
constructor(workflowId) {
|
|
8906
|
+
super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
|
|
8907
|
+
this.workflowId = workflowId;
|
|
8908
|
+
this.name = "WorkflowRuntimeClosedError";
|
|
8909
|
+
}
|
|
8910
|
+
};
|
|
8911
|
+
var TriggerSupervisorStoppedError = class extends Error {
|
|
8912
|
+
constructor(options) {
|
|
8913
|
+
super("The trigger supervisor has stopped", options);
|
|
8914
|
+
this.name = "TriggerSupervisorStoppedError";
|
|
8915
|
+
}
|
|
8916
|
+
};
|
|
8917
|
+
function isShutdownRefusal(error) {
|
|
8918
|
+
return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
|
|
8919
|
+
}
|
|
7942
8920
|
childLogger(["workflow", "trigger-binding"]);
|
|
7943
8921
|
function pieceTriggerKind(block) {
|
|
7944
8922
|
return `piece:${block.pieceName}:${block.name}`;
|
|
7945
8923
|
}
|
|
7946
8924
|
//#endregion
|
|
7947
8925
|
//#region src/reactor/trigger-supervisor.ts
|
|
7948
|
-
const logger$
|
|
8926
|
+
const logger$2 = childLogger(["workflow", "trigger-supervisor"]);
|
|
7949
8927
|
function targetOf$1(binding) {
|
|
7950
8928
|
return {
|
|
7951
8929
|
name: binding.packageName,
|
|
@@ -7965,6 +8943,7 @@ const SCHEDULE_TRIGGER_KIND = "schedule";
|
|
|
7965
8943
|
const MIN_INTERVAL_MS = MIN_SCHEDULE_INTERVAL_MS;
|
|
7966
8944
|
const MAX_BACKOFF_MS = 18e5;
|
|
7967
8945
|
const DEDUPE_TTL_MS = 3e4;
|
|
8946
|
+
const REDELIVERED_REASON = "Redelivered: the trigger stopped before firing this item and put its cursor back, so the next poll delivers it again";
|
|
7968
8947
|
const DEFAULT_RECONCILE_INTERVAL_MS = 9e5;
|
|
7969
8948
|
const RENEW_RETRY_BASE_MS = 6e4;
|
|
7970
8949
|
function isSchedule(binding) {
|
|
@@ -7980,7 +8959,7 @@ function intervalFromSchedules(schedules, defaultMs) {
|
|
|
7980
8959
|
const cron = schedule.cronExpression;
|
|
7981
8960
|
const intervalMs = cronIntervalMs(cron);
|
|
7982
8961
|
if (intervalMs === void 0) {
|
|
7983
|
-
logger$
|
|
8962
|
+
logger$2.warn("Unsupported setSchedule cron \"@cron\"; using the default", cron);
|
|
7984
8963
|
return Math.max(defaultMs, MIN_INTERVAL_MS);
|
|
7985
8964
|
}
|
|
7986
8965
|
return intervalMs;
|
|
@@ -8026,40 +9005,80 @@ var TriggerSupervisor = class {
|
|
|
8026
9005
|
hookTimeoutMs;
|
|
8027
9006
|
now;
|
|
8028
9007
|
egress;
|
|
9008
|
+
parks;
|
|
8029
9009
|
timer;
|
|
8030
9010
|
ops = Promise.resolve();
|
|
8031
9011
|
ticking = false;
|
|
8032
9012
|
warnedMissingJournal = false;
|
|
9013
|
+
stopped = false;
|
|
8033
9014
|
constructor(options) {
|
|
8034
9015
|
this.options = options;
|
|
8035
|
-
this.worker = options.worker ?? new PieceWorker({
|
|
9016
|
+
this.worker = options.worker ?? new PieceWorker({
|
|
9017
|
+
models: options.models,
|
|
9018
|
+
telemetry: options.telemetry
|
|
9019
|
+
});
|
|
8036
9020
|
this.tickMs = options.tickMs ?? 15e3;
|
|
8037
9021
|
this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
|
|
8038
9022
|
this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
|
|
8039
9023
|
this.now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
8040
9024
|
this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
|
|
9025
|
+
this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
|
|
8041
9026
|
}
|
|
8042
9027
|
start() {
|
|
8043
|
-
if (this.timer) return;
|
|
9028
|
+
if (this.timer || this.stopped) return;
|
|
8044
9029
|
this.timer = setInterval(() => {
|
|
8045
9030
|
this.tick().catch((error) => {
|
|
8046
|
-
|
|
9031
|
+
if (isShutdownRefusal(error)) {
|
|
9032
|
+
logger$2.debug("Trigger tick stopped: @error", error);
|
|
9033
|
+
return;
|
|
9034
|
+
}
|
|
9035
|
+
logger$2.error("Trigger tick failed: @error", error);
|
|
8047
9036
|
});
|
|
8048
9037
|
}, this.tickMs);
|
|
8049
9038
|
this.timer.unref();
|
|
8050
|
-
logger$
|
|
9039
|
+
logger$2.info(`Trigger supervisor started (tick ${this.tickMs}ms)`);
|
|
8051
9040
|
}
|
|
8052
9041
|
stop() {
|
|
9042
|
+
if (this.stopped) return;
|
|
9043
|
+
this.stopped = true;
|
|
8053
9044
|
if (this.timer) clearInterval(this.timer);
|
|
8054
9045
|
this.timer = void 0;
|
|
8055
9046
|
if (!this.options.worker) this.worker.dispose();
|
|
8056
|
-
logger$
|
|
9047
|
+
logger$2.info("Trigger supervisor stopped");
|
|
8057
9048
|
}
|
|
8058
9049
|
enqueue(task) {
|
|
8059
|
-
const run = this.ops.then(
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
|
|
9050
|
+
const run = this.ops.then(async () => {
|
|
9051
|
+
this.throwIfStopped();
|
|
9052
|
+
try {
|
|
9053
|
+
return await task();
|
|
9054
|
+
} catch (error) {
|
|
9055
|
+
if (this.stopped && !(error instanceof TriggerSupervisorStoppedError)) throw new TriggerSupervisorStoppedError({ cause: error });
|
|
9056
|
+
throw error;
|
|
9057
|
+
}
|
|
9058
|
+
});
|
|
9059
|
+
this.ops = run.catch(() => void 0);
|
|
9060
|
+
return run;
|
|
9061
|
+
}
|
|
9062
|
+
throwIfStopped() {
|
|
9063
|
+
if (this.stopped) throw new TriggerSupervisorStoppedError();
|
|
9064
|
+
}
|
|
9065
|
+
async store() {
|
|
9066
|
+
this.throwIfStopped();
|
|
9067
|
+
const store = await this.options.store();
|
|
9068
|
+
if (!store) return void 0;
|
|
9069
|
+
return new Proxy(store, { get: (target, property) => {
|
|
9070
|
+
const value = Reflect.get(target, property, target);
|
|
9071
|
+
if (typeof value !== "function") return value;
|
|
9072
|
+
return (...args) => {
|
|
9073
|
+
this.throwIfStopped();
|
|
9074
|
+
return value.apply(target, args);
|
|
9075
|
+
};
|
|
9076
|
+
} });
|
|
9077
|
+
}
|
|
9078
|
+
fire(workflowId, payload, kind, runId) {
|
|
9079
|
+
this.throwIfStopped();
|
|
9080
|
+
this.options.fire(workflowId, payload, kind, runId);
|
|
9081
|
+
}
|
|
8063
9082
|
enabledOk = /* @__PURE__ */ new Set();
|
|
8064
9083
|
descriptors = /* @__PURE__ */ new Map();
|
|
8065
9084
|
own;
|
|
@@ -8067,28 +9086,84 @@ var TriggerSupervisor = class {
|
|
|
8067
9086
|
return this.options.resolver ?? (this.own ??= bundleResolver({ cacheDir: this.options.cacheDir }));
|
|
8068
9087
|
}
|
|
8069
9088
|
enableRetries = /* @__PURE__ */ new Map();
|
|
8070
|
-
|
|
9089
|
+
versions = /* @__PURE__ */ new Map();
|
|
9090
|
+
upsert(binding, publishedVersion) {
|
|
9091
|
+
if (publishedVersion !== void 0) this.versions.set(binding.workflowId, publishedVersion);
|
|
8071
9092
|
const previous = this.bindings.get(binding.workflowId);
|
|
8072
9093
|
if (previous && this.enabledOk.has(binding.workflowId) && JSON.stringify(previous) === JSON.stringify(binding)) return Promise.resolve();
|
|
8073
9094
|
this.bindings.set(binding.workflowId, binding);
|
|
9095
|
+
const park = this.parks.known(binding.workflowId)?.park;
|
|
9096
|
+
if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
|
|
8074
9097
|
return this.enqueue(() => this.enable(binding, previous));
|
|
8075
9098
|
}
|
|
9099
|
+
/** Parks a workflow; true when a trigger row was parked with it. */
|
|
9100
|
+
park(workflowId, publishedVersion, reason) {
|
|
9101
|
+
this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
|
|
9102
|
+
return this.enqueue(async () => {
|
|
9103
|
+
const store = await this.store();
|
|
9104
|
+
if (!store) return false;
|
|
9105
|
+
return store.parkWorkflow(workflowId, publishedVersion, reason);
|
|
9106
|
+
});
|
|
9107
|
+
}
|
|
9108
|
+
/** Undoes {@link park} for that version only: a later park stands. */
|
|
9109
|
+
liftPark(workflowId, publishedVersion, trigger) {
|
|
9110
|
+
if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
|
|
9111
|
+
return this.enqueue(async () => {
|
|
9112
|
+
const store = await this.store();
|
|
9113
|
+
if (!store) return;
|
|
9114
|
+
if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
|
|
9115
|
+
await store.liftPark(workflowId, trigger);
|
|
9116
|
+
});
|
|
9117
|
+
}
|
|
9118
|
+
/** A disable clears the park. With no binding in memory to release a PARKED
|
|
9119
|
+
* row through onDisable (a boot), the row is turned DISABLED here. */
|
|
9120
|
+
releasePark(workflowId, flipRow) {
|
|
9121
|
+
this.parks.delete(workflowId);
|
|
9122
|
+
return this.enqueue(async () => {
|
|
9123
|
+
const store = await this.store();
|
|
9124
|
+
if (!store) return;
|
|
9125
|
+
if (flipRow) await store.clearParkOnDisable(workflowId);
|
|
9126
|
+
else await store.clearWorkflowPark(workflowId);
|
|
9127
|
+
});
|
|
9128
|
+
}
|
|
9129
|
+
/** A re-publish past the parked version: on this lane, a park older than
|
|
9130
|
+
* `publishedVersion` goes and a row it parked is ENABLED again, as it was
|
|
9131
|
+
* before (only an ENABLED row is ever parked); a park as new stands. The
|
|
9132
|
+
* next upsert of the same binding then arms rather than skipping as
|
|
9133
|
+
* unchanged, and a retry held back by the park is due now. */
|
|
9134
|
+
unpark(workflowId, publishedVersion) {
|
|
9135
|
+
this.enabledOk.delete(workflowId);
|
|
9136
|
+
const retry = this.enableRetries.get(workflowId);
|
|
9137
|
+
if (retry) retry.at = 0;
|
|
9138
|
+
const park = this.parks.known(workflowId)?.park;
|
|
9139
|
+
if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
|
|
9140
|
+
return this.enqueue(async () => {
|
|
9141
|
+
const store = await this.store();
|
|
9142
|
+
if (!store) return;
|
|
9143
|
+
const park = await store.getWorkflowPark(workflowId);
|
|
9144
|
+
if (!park || park.published_version >= publishedVersion) return;
|
|
9145
|
+
await store.liftPark(workflowId, true);
|
|
9146
|
+
});
|
|
9147
|
+
}
|
|
8076
9148
|
remove(workflowId) {
|
|
8077
9149
|
const binding = this.unbind(workflowId);
|
|
8078
9150
|
return this.enqueue(() => this.disable(workflowId, binding));
|
|
8079
9151
|
}
|
|
8080
9152
|
forget(workflowId) {
|
|
8081
9153
|
const binding = this.unbind(workflowId);
|
|
9154
|
+
this.parks.delete(workflowId);
|
|
8082
9155
|
return this.enqueue(async () => {
|
|
8083
|
-
const store = await this.
|
|
9156
|
+
const store = await this.store();
|
|
8084
9157
|
if (!store) return;
|
|
8085
9158
|
await this.disable(workflowId, binding);
|
|
8086
9159
|
await store.deleteTriggerState(workflowId);
|
|
9160
|
+
await store.clearWorkflowPark(workflowId);
|
|
8087
9161
|
});
|
|
8088
9162
|
}
|
|
8089
9163
|
unbind(workflowId) {
|
|
8090
9164
|
const binding = this.bindings.get(workflowId);
|
|
8091
9165
|
this.bindings.delete(workflowId);
|
|
9166
|
+
this.versions.delete(workflowId);
|
|
8092
9167
|
this.enabledOk.delete(workflowId);
|
|
8093
9168
|
this.enableRetries.delete(workflowId);
|
|
8094
9169
|
return binding;
|
|
@@ -8098,12 +9173,16 @@ var TriggerSupervisor = class {
|
|
|
8098
9173
|
return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
|
|
8099
9174
|
}
|
|
8100
9175
|
async recordRejection(workflowId, block, config, message, retryAt) {
|
|
8101
|
-
const store = await this.
|
|
9176
|
+
const store = await this.store();
|
|
8102
9177
|
if (!store) return;
|
|
8103
9178
|
const hash = configHash(block, config);
|
|
8104
9179
|
const existing = await store.getTriggerState(workflowId);
|
|
8105
9180
|
if (existing?.status === "ENABLED" && existing.config_hash === hash) {
|
|
8106
|
-
logger$
|
|
9181
|
+
logger$2.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
|
|
9182
|
+
return;
|
|
9183
|
+
}
|
|
9184
|
+
if (existing?.status === "PARKED" && existing.config_hash === hash) {
|
|
9185
|
+
logger$2.warn(`Workflow ${workflowId} is PARKED and could not be resolved; the park stands. @error`, message);
|
|
8107
9186
|
return;
|
|
8108
9187
|
}
|
|
8109
9188
|
await store.upsertTriggerState({
|
|
@@ -8125,7 +9204,7 @@ var TriggerSupervisor = class {
|
|
|
8125
9204
|
}
|
|
8126
9205
|
test(binding, options = {}) {
|
|
8127
9206
|
return this.enqueue(async () => {
|
|
8128
|
-
const store = await this.
|
|
9207
|
+
const store = await this.store();
|
|
8129
9208
|
try {
|
|
8130
9209
|
return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
|
|
8131
9210
|
} finally {
|
|
@@ -8139,7 +9218,7 @@ var TriggerSupervisor = class {
|
|
|
8139
9218
|
await store.deletePieceStore("FLOW", testPartitionKey("FLOW", workflowId));
|
|
8140
9219
|
await store.deletePieceStore("PROJECT", testPartitionKey("PROJECT", workflowId));
|
|
8141
9220
|
} catch (error) {
|
|
8142
|
-
logger$
|
|
9221
|
+
logger$2.warn(`Could not clear test store for ${workflowId}`, error);
|
|
8143
9222
|
}
|
|
8144
9223
|
}
|
|
8145
9224
|
handshake(binding, payload) {
|
|
@@ -8149,45 +9228,58 @@ var TriggerSupervisor = class {
|
|
|
8149
9228
|
return this.enqueue(async () => {
|
|
8150
9229
|
const binding = this.bindings.get(workflowId);
|
|
8151
9230
|
if (!binding || isSchedule(binding)) {
|
|
8152
|
-
logger$
|
|
9231
|
+
logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
|
|
8153
9232
|
return;
|
|
8154
9233
|
}
|
|
8155
|
-
const store = await this.
|
|
9234
|
+
const store = await this.store();
|
|
8156
9235
|
if (!store) throw new MissingJournalError("Webhook delivery");
|
|
8157
9236
|
const row = await store.getTriggerState(workflowId);
|
|
8158
9237
|
const now = this.now();
|
|
8159
9238
|
const rewind = await this.cursorRewind(store, workflowId);
|
|
9239
|
+
let refused = false;
|
|
8160
9240
|
try {
|
|
8161
|
-
const result = await this.hook(binding, "run", { payload })
|
|
9241
|
+
const result = await this.hook(binding, "run", { payload }).catch((error) => {
|
|
9242
|
+
refused = isShutdownRefusal(error);
|
|
9243
|
+
throw error;
|
|
9244
|
+
});
|
|
8162
9245
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
8163
9246
|
if (row?.status === "ENABLED") await store.recordPollSuccess(workflowId, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
|
|
8164
|
-
|
|
9247
|
+
await this.fireItems(store, binding, result.output, now, rewind);
|
|
8165
9248
|
} catch (error) {
|
|
8166
|
-
await rewind();
|
|
9249
|
+
if (!refused) await rewind();
|
|
8167
9250
|
throw error;
|
|
8168
9251
|
}
|
|
8169
9252
|
});
|
|
8170
9253
|
}
|
|
8171
9254
|
async cursorRewind(store, workflowId) {
|
|
8172
9255
|
const before = await store.listPieceStore("FLOW", workflowId);
|
|
8173
|
-
|
|
9256
|
+
let rewound;
|
|
9257
|
+
const rewind = async (redelivered) => {
|
|
8174
9258
|
try {
|
|
8175
|
-
await store
|
|
8176
|
-
|
|
9259
|
+
const raw = await this.options.store();
|
|
9260
|
+
if (!raw) return false;
|
|
9261
|
+
await raw.rewindPieceStore("FLOW", workflowId, before, {
|
|
9262
|
+
runIds: redelivered,
|
|
9263
|
+
reason: REDELIVERED_REASON
|
|
9264
|
+
});
|
|
9265
|
+
return true;
|
|
8177
9266
|
} catch (error) {
|
|
8178
|
-
logger$
|
|
9267
|
+
logger$2.warn(`Could not rewind the cursor for ${workflowId}`, error);
|
|
9268
|
+
return false;
|
|
8179
9269
|
}
|
|
8180
9270
|
};
|
|
9271
|
+
return (redelivered = []) => rewound ??= rewind(redelivered);
|
|
8181
9272
|
}
|
|
8182
9273
|
async hook(binding, hook, options = {}) {
|
|
8183
9274
|
if (hook !== "onDisable") checkDynamicProperties(binding.config, binding.propertySettings);
|
|
8184
|
-
const store = await this.
|
|
9275
|
+
const store = await this.store();
|
|
8185
9276
|
if (!store && hook !== "test") throw new MissingJournalError(`Trigger hook "${hook}"`);
|
|
8186
9277
|
const pieceStore = store ? createPieceStorePort(store, () => binding.workflowId, hook === "test") : void 0;
|
|
8187
9278
|
const piece = await this.resolver().resolve(targetOf$1(binding));
|
|
8188
9279
|
const auth = await this.options.resolveAuth(binding.connectionId, { piecePackage: binding.packageName });
|
|
8189
9280
|
const redactValues = secretsFor(auth);
|
|
8190
9281
|
const reactor = await this.reactorFor(binding, hook, options.runUser);
|
|
9282
|
+
this.throwIfStopped();
|
|
8191
9283
|
return this.worker.runTriggerHook({
|
|
8192
9284
|
...pieceModuleRef(piece),
|
|
8193
9285
|
triggerName: binding.triggerName,
|
|
@@ -8255,11 +9347,17 @@ var TriggerSupervisor = class {
|
|
|
8255
9347
|
};
|
|
8256
9348
|
}
|
|
8257
9349
|
async enable(binding, superseded) {
|
|
8258
|
-
const store = await this.
|
|
9350
|
+
const store = await this.store();
|
|
8259
9351
|
if (!store) throw new MissingJournalError("Enabling a trigger");
|
|
8260
9352
|
const hash = configHash(binding.block, binding.config);
|
|
8261
|
-
const existing = await store.getTriggerState(binding.workflowId);
|
|
8262
9353
|
const now = this.now();
|
|
9354
|
+
const park = await store.getWorkflowPark(binding.workflowId);
|
|
9355
|
+
if (park && await this.blockingPark(store, binding.workflowId)) {
|
|
9356
|
+
this.holdParked(binding, superseded, park);
|
|
9357
|
+
return;
|
|
9358
|
+
}
|
|
9359
|
+
if (park) await store.liftPark(binding.workflowId, true);
|
|
9360
|
+
const existing = await store.getTriggerState(binding.workflowId);
|
|
8263
9361
|
const isRepublish = existing?.config_hash === hash && existing.status === "ENABLED" && !store.hasUnmigratedTriggerState(binding.workflowId);
|
|
8264
9362
|
if (existing && !isRepublish && existing.status === "ENABLED") await this.disableRow(binding.workflowId, existing, superseded);
|
|
8265
9363
|
const pending = this.enableRetries.get(binding.workflowId);
|
|
@@ -8290,7 +9388,7 @@ var TriggerSupervisor = class {
|
|
|
8290
9388
|
workflow_id: binding.workflowId,
|
|
8291
9389
|
...triggerBlockColumns(binding.block),
|
|
8292
9390
|
config_hash: hash,
|
|
8293
|
-
status:
|
|
9391
|
+
status: await this.armedStatus(store, binding.workflowId),
|
|
8294
9392
|
store_state: VESTIGIAL_STORE_STATE,
|
|
8295
9393
|
interval_ms: intervalMs,
|
|
8296
9394
|
next_poll_at: new Date(now.getTime() + intervalMs).toISOString(),
|
|
@@ -8308,8 +9406,9 @@ var TriggerSupervisor = class {
|
|
|
8308
9406
|
this.enabledOk.add(binding.workflowId);
|
|
8309
9407
|
this.enableRetries.delete(binding.workflowId);
|
|
8310
9408
|
store.clearUnmigratedTriggerState(binding.workflowId);
|
|
8311
|
-
logger$
|
|
9409
|
+
logger$2.info(`Enabled @block for workflow ${binding.workflowId} (every ${intervalMs}ms)`, blockLabel(binding.block));
|
|
8312
9410
|
} catch (error) {
|
|
9411
|
+
if (this.stopped) throw error;
|
|
8313
9412
|
this.enabledOk.delete(binding.workflowId);
|
|
8314
9413
|
const message = error instanceof Error ? error.message : String(error);
|
|
8315
9414
|
const failures = (existing?.consecutive_failures ?? 0) + 1;
|
|
@@ -8337,7 +9436,7 @@ var TriggerSupervisor = class {
|
|
|
8337
9436
|
updated_at: now.toISOString(),
|
|
8338
9437
|
...pieceColumns(binding)
|
|
8339
9438
|
});
|
|
8340
|
-
logger$
|
|
9439
|
+
logger$2.error(`onEnable failed for workflow ${binding.workflowId} (${failures}x): @error` + (retryAt ? `; retrying at ${retryAt.toISOString()}` : "; not retrying"), message);
|
|
8341
9440
|
}
|
|
8342
9441
|
}
|
|
8343
9442
|
renewAtAfterEnable(renew, existing, isRepublish, now) {
|
|
@@ -8352,6 +9451,43 @@ var TriggerSupervisor = class {
|
|
|
8352
9451
|
if (!url) throw new Error("The reactor's webhook endpoint is not available yet for this workflow");
|
|
8353
9452
|
return url;
|
|
8354
9453
|
}
|
|
9454
|
+
/**
|
|
9455
|
+
* The park, when it holds this registration back.
|
|
9456
|
+
*
|
|
9457
|
+
* `policy.onFailure = PARK` is a **runtime override of the document's
|
|
9458
|
+
* enabled-ness**: the document still says ENABLED, and re-arming from it is
|
|
9459
|
+
* what a restart does for every workflow it finds. So the park row is the
|
|
9460
|
+
* authority, read on this lane, and it blocks every registration of the
|
|
9461
|
+
* version that failed or an older one — a boot re-arm, an ERROR row's
|
|
9462
|
+
* retry. A newer version is what a re-publish is: it is never blocked, and
|
|
9463
|
+
* {@link enable} lifts the park it outlived. A re-enable lifts it through
|
|
9464
|
+
* the disable before it.
|
|
9465
|
+
*/
|
|
9466
|
+
async blockingPark(store, workflowId) {
|
|
9467
|
+
const park = await store.getWorkflowPark(workflowId);
|
|
9468
|
+
if (!park) return void 0;
|
|
9469
|
+
const version = this.versions.get(workflowId);
|
|
9470
|
+
return version === void 0 || version <= park.published_version ? park : void 0;
|
|
9471
|
+
}
|
|
9472
|
+
/**
|
|
9473
|
+
* An enable a park held back. The row is left as it stands, and a pending
|
|
9474
|
+
* retry keeps its release flag, so the enable after the park lifts releases
|
|
9475
|
+
* a failed attempt and arms afresh; the retry is only pushed back, so it
|
|
9476
|
+
* does not take every tick's slot. A binding this replaced is restored, so
|
|
9477
|
+
* nothing is released later with one that was never armed.
|
|
9478
|
+
*/
|
|
9479
|
+
holdParked(binding, superseded, park) {
|
|
9480
|
+
const { workflowId } = binding;
|
|
9481
|
+
const retry = this.enableRetries.get(workflowId);
|
|
9482
|
+
if (retry) retry.at = this.now().getTime() + this.defaultIntervalMs;
|
|
9483
|
+
if (superseded && this.bindings.get(workflowId) === binding) this.bindings.set(workflowId, superseded);
|
|
9484
|
+
logger$2.warn(`Workflow ${workflowId} is PARKED at version ${park.published_version}; its trigger is not armed until the workflow is re-published or re-enabled. @error`, park.reason);
|
|
9485
|
+
}
|
|
9486
|
+
/** The status an enable that succeeded writes: never ENABLED while a park
|
|
9487
|
+
* stands, whatever ran meanwhile. The registration stays recorded. */
|
|
9488
|
+
async armedStatus(store, workflowId) {
|
|
9489
|
+
return await this.blockingPark(store, workflowId) ? PARKED_TRIGGER_STATUS : "ENABLED";
|
|
9490
|
+
}
|
|
8355
9491
|
deferToStoredRetry(binding, hash, existing, now) {
|
|
8356
9492
|
if (existing?.status !== "ERROR" || existing.config_hash !== hash) return false;
|
|
8357
9493
|
const pending = this.enableRetries.get(binding.workflowId);
|
|
@@ -8364,7 +9500,7 @@ var TriggerSupervisor = class {
|
|
|
8364
9500
|
failures: existing.consecutive_failures,
|
|
8365
9501
|
release: true
|
|
8366
9502
|
});
|
|
8367
|
-
logger$
|
|
9503
|
+
logger$2.info(`Enable for workflow ${binding.workflowId} still backing off until ${existing.next_poll_at}`);
|
|
8368
9504
|
}
|
|
8369
9505
|
return true;
|
|
8370
9506
|
}
|
|
@@ -8372,7 +9508,7 @@ var TriggerSupervisor = class {
|
|
|
8372
9508
|
try {
|
|
8373
9509
|
await this.hook(binding, "onDisable");
|
|
8374
9510
|
} catch (error) {
|
|
8375
|
-
logger$
|
|
9511
|
+
logger$2.warn(`onDisable before retrying workflow ${binding.workflowId} failed`, error);
|
|
8376
9512
|
}
|
|
8377
9513
|
}
|
|
8378
9514
|
async enableSchedule(store, binding, hash, existing) {
|
|
@@ -8394,14 +9530,14 @@ var TriggerSupervisor = class {
|
|
|
8394
9530
|
const nextAt = carried ? new Date(carried) : nextFireAt(schedule, now);
|
|
8395
9531
|
await store.upsertTriggerState({
|
|
8396
9532
|
...base,
|
|
8397
|
-
status:
|
|
9533
|
+
status: await this.armedStatus(store, binding.workflowId),
|
|
8398
9534
|
interval_ms: schedule.mode === "interval" ? schedule.everyMs : MIN_SCHEDULE_INTERVAL_MS,
|
|
8399
9535
|
next_poll_at: nextAt.toISOString(),
|
|
8400
9536
|
last_error: null,
|
|
8401
9537
|
consecutive_failures: 0
|
|
8402
9538
|
});
|
|
8403
9539
|
this.enabledOk.add(binding.workflowId);
|
|
8404
|
-
logger$
|
|
9540
|
+
logger$2.info(`Scheduled workflow ${binding.workflowId}: next fire ${nextAt.toISOString()}` + (carried ? " (carried over)" : ""));
|
|
8405
9541
|
} catch (error) {
|
|
8406
9542
|
this.enabledOk.delete(binding.workflowId);
|
|
8407
9543
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -8413,24 +9549,24 @@ var TriggerSupervisor = class {
|
|
|
8413
9549
|
last_error: message,
|
|
8414
9550
|
consecutive_failures: (existing?.consecutive_failures ?? 0) + 1
|
|
8415
9551
|
});
|
|
8416
|
-
logger$
|
|
9552
|
+
logger$2.error(`Invalid schedule for workflow ${binding.workflowId}: @error`, message);
|
|
8417
9553
|
}
|
|
8418
9554
|
}
|
|
8419
9555
|
async disable(workflowId, binding) {
|
|
8420
|
-
const store = await this.
|
|
9556
|
+
const store = await this.store();
|
|
8421
9557
|
if (!store) throw new MissingJournalError("Disabling a trigger");
|
|
8422
9558
|
const row = await store.getTriggerState(workflowId);
|
|
8423
9559
|
if (!row || row.status === "DISABLED") return;
|
|
8424
9560
|
await this.disableRow(workflowId, row, binding);
|
|
8425
9561
|
}
|
|
8426
9562
|
async disableRow(workflowId, row, binding) {
|
|
8427
|
-
const store = await this.
|
|
9563
|
+
const store = await this.store();
|
|
8428
9564
|
if (!store) return;
|
|
8429
9565
|
const target = binding ?? this.bindingFromRow(row);
|
|
8430
9566
|
if (target && !isSchedule(target) && blockKey(triggerRowBlock(row)) !== SCHEDULE_BLOCK) try {
|
|
8431
9567
|
await this.hook(target, "onDisable");
|
|
8432
9568
|
} catch (error) {
|
|
8433
|
-
logger$
|
|
9569
|
+
logger$2.warn(`onDisable failed for workflow ${workflowId}`, error);
|
|
8434
9570
|
}
|
|
8435
9571
|
await store.setTriggerStatus(workflowId, "DISABLED");
|
|
8436
9572
|
}
|
|
@@ -8438,7 +9574,7 @@ var TriggerSupervisor = class {
|
|
|
8438
9574
|
return this.bindings.get(row.workflow_id);
|
|
8439
9575
|
}
|
|
8440
9576
|
async tick() {
|
|
8441
|
-
if (this.ticking) return;
|
|
9577
|
+
if (this.ticking || this.stopped) return;
|
|
8442
9578
|
this.ticking = true;
|
|
8443
9579
|
try {
|
|
8444
9580
|
await this.enqueue(() => this.pollDue());
|
|
@@ -8459,6 +9595,7 @@ var TriggerSupervisor = class {
|
|
|
8459
9595
|
try {
|
|
8460
9596
|
await this.enable(binding);
|
|
8461
9597
|
} catch (error) {
|
|
9598
|
+
if (this.stopped) throw error;
|
|
8462
9599
|
const failures = retry.failures + 1;
|
|
8463
9600
|
const intervalMs = isSchedule(binding) ? MIN_INTERVAL_MS : pollIntervalFor(binding, void 0, this.defaultIntervalMs);
|
|
8464
9601
|
this.enableRetries.set(workflowId, {
|
|
@@ -8466,20 +9603,21 @@ var TriggerSupervisor = class {
|
|
|
8466
9603
|
failures,
|
|
8467
9604
|
release: retry.release
|
|
8468
9605
|
});
|
|
8469
|
-
logger$
|
|
9606
|
+
logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
|
|
8470
9607
|
}
|
|
8471
9608
|
}
|
|
8472
9609
|
async pollDue() {
|
|
8473
|
-
const store = await this.
|
|
9610
|
+
const store = await this.store();
|
|
8474
9611
|
if (!store) {
|
|
8475
9612
|
if (!this.warnedMissingJournal) {
|
|
8476
9613
|
this.warnedMissingJournal = true;
|
|
8477
|
-
logger$
|
|
9614
|
+
logger$2.error("Trigger polling is off: @error", new MissingJournalError("Polling"));
|
|
8478
9615
|
}
|
|
8479
9616
|
return;
|
|
8480
9617
|
}
|
|
8481
9618
|
const due = await store.listDueTriggerStates(this.now().toISOString());
|
|
8482
9619
|
for (const row of due) {
|
|
9620
|
+
this.throwIfStopped();
|
|
8483
9621
|
const binding = this.bindings.get(row.workflow_id);
|
|
8484
9622
|
if (!binding) {
|
|
8485
9623
|
await store.setTriggerStatus(row.workflow_id, "DISABLED");
|
|
@@ -8488,12 +9626,15 @@ var TriggerSupervisor = class {
|
|
|
8488
9626
|
if (isSchedule(binding)) await this.fireSchedule(store, row, binding);
|
|
8489
9627
|
else await this.poll(store, row, binding);
|
|
8490
9628
|
}
|
|
9629
|
+
this.throwIfStopped();
|
|
8491
9630
|
await this.renewDue(store);
|
|
9631
|
+
this.throwIfStopped();
|
|
8492
9632
|
await this.retryOneEnable();
|
|
8493
9633
|
}
|
|
8494
9634
|
async renewDue(store) {
|
|
8495
9635
|
const due = await store.listDueTriggerRenewals(this.now().toISOString());
|
|
8496
9636
|
for (const row of due) {
|
|
9637
|
+
this.throwIfStopped();
|
|
8497
9638
|
const binding = this.bindings.get(row.workflow_id);
|
|
8498
9639
|
if (!binding) {
|
|
8499
9640
|
await store.setTriggerStatus(row.workflow_id, "DISABLED");
|
|
@@ -8519,7 +9660,7 @@ var TriggerSupervisor = class {
|
|
|
8519
9660
|
await this.hook(binding, "onRenew", { webhookUrl });
|
|
8520
9661
|
const nextAt = nextRenewAt(renew, now);
|
|
8521
9662
|
await store.recordRenewSuccess(row.workflow_id, now.toISOString(), nextAt.toISOString());
|
|
8522
|
-
logger$
|
|
9663
|
+
logger$2.info(`Renewed trigger of workflow ${row.workflow_id}; next ${nextAt.toISOString()}`);
|
|
8523
9664
|
} catch (error) {
|
|
8524
9665
|
const message = error instanceof Error ? error.message : String(error);
|
|
8525
9666
|
const failures = row.renew_failures + 1;
|
|
@@ -8527,7 +9668,7 @@ var TriggerSupervisor = class {
|
|
|
8527
9668
|
const slot = renew ? nextRenewAt(renew, now).getTime() : backoff;
|
|
8528
9669
|
const retryAt = new Date(Math.min(backoff, slot));
|
|
8529
9670
|
await store.recordRenewFailure(row.workflow_id, message, now.toISOString(), retryAt.toISOString(), failures);
|
|
8530
|
-
logger$
|
|
9671
|
+
logger$2.warn(`onRenew failed for workflow ${row.workflow_id} (${failures}x): @error; retrying at ${retryAt.toISOString()}`, message);
|
|
8531
9672
|
}
|
|
8532
9673
|
}
|
|
8533
9674
|
async fireSchedule(store, row, binding) {
|
|
@@ -8536,38 +9677,79 @@ var TriggerSupervisor = class {
|
|
|
8536
9677
|
const schedule = parseScheduleConfig$1(binding.config);
|
|
8537
9678
|
const scheduledFor = row.next_poll_at ? new Date(row.next_poll_at) : now;
|
|
8538
9679
|
const nextAt = rescheduleAfterFire(schedule, scheduledFor, now);
|
|
8539
|
-
|
|
8540
|
-
|
|
9680
|
+
const payload = schedulePayload(schedule, scheduledFor, now);
|
|
9681
|
+
const runId = await store.recordScheduleFire(now.toISOString(), nextAt.toISOString(), {
|
|
9682
|
+
workflowId: binding.workflowId,
|
|
9683
|
+
triggerKind: SCHEDULE_TRIGGER_KIND,
|
|
9684
|
+
triggerPayload: payload
|
|
9685
|
+
});
|
|
9686
|
+
try {
|
|
9687
|
+
this.fire(binding.workflowId, payload, SCHEDULE_TRIGGER_KIND, runId);
|
|
9688
|
+
} catch (error) {
|
|
9689
|
+
await this.failRefused([runId], error);
|
|
9690
|
+
throw error;
|
|
9691
|
+
}
|
|
8541
9692
|
} catch (error) {
|
|
9693
|
+
if (this.stopped) throw error;
|
|
8542
9694
|
const message = error instanceof Error ? error.message : String(error);
|
|
8543
9695
|
this.enabledOk.delete(binding.workflowId);
|
|
8544
9696
|
await store.setTriggerStatus(row.workflow_id, "ERROR", message);
|
|
8545
|
-
logger$
|
|
9697
|
+
logger$2.error(`Schedule fire failed for workflow ${row.workflow_id}: @error`, message);
|
|
8546
9698
|
}
|
|
8547
9699
|
}
|
|
8548
9700
|
async poll(store, row, binding) {
|
|
8549
9701
|
const now = this.now();
|
|
8550
9702
|
const rewind = await this.cursorRewind(store, row.workflow_id);
|
|
9703
|
+
let refused = false;
|
|
8551
9704
|
try {
|
|
8552
|
-
const result = await this.hook(binding, "run")
|
|
9705
|
+
const result = await this.hook(binding, "run").catch((error) => {
|
|
9706
|
+
refused = isShutdownRefusal(error);
|
|
9707
|
+
throw error;
|
|
9708
|
+
});
|
|
8553
9709
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
8554
9710
|
await store.recordPollSuccess(row.workflow_id, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
|
|
8555
|
-
|
|
9711
|
+
await this.fireItems(store, binding, result.output, now, rewind);
|
|
8556
9712
|
} catch (error) {
|
|
8557
|
-
await rewind();
|
|
9713
|
+
if (!refused) await rewind();
|
|
9714
|
+
if (this.stopped) throw error;
|
|
8558
9715
|
const message = error instanceof Error ? error.message : String(error);
|
|
8559
9716
|
const failures = row.consecutive_failures + 1;
|
|
8560
9717
|
const backoff = backoffMs(row.interval_ms, failures);
|
|
8561
9718
|
await store.recordPollFailure(row.workflow_id, message, now.toISOString(), new Date(now.getTime() + backoff).toISOString(), failures);
|
|
8562
|
-
logger$
|
|
9719
|
+
logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
|
|
8563
9720
|
}
|
|
8564
9721
|
}
|
|
8565
|
-
async
|
|
8566
|
-
|
|
8567
|
-
|
|
8568
|
-
|
|
9722
|
+
async fireItems(store, binding, items, now, rewind) {
|
|
9723
|
+
if (items.length === 0) return;
|
|
9724
|
+
const kind = pieceTriggerKind(binding.block);
|
|
9725
|
+
const journaled = (await store.journalTriggerItems(items.map((item) => ({
|
|
9726
|
+
dedupeKey: extractDedupeKey(item),
|
|
9727
|
+
options: {
|
|
9728
|
+
workflowId: binding.workflowId,
|
|
9729
|
+
triggerKind: kind,
|
|
9730
|
+
triggerPayload: item
|
|
9731
|
+
}
|
|
9732
|
+
})), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
|
|
9733
|
+
runId,
|
|
9734
|
+
item: items[index]
|
|
9735
|
+
}] : []);
|
|
9736
|
+
if (this.stopped) {
|
|
9737
|
+
const error = new TriggerSupervisorStoppedError();
|
|
9738
|
+
await this.redeliver(journaled.map(({ runId }) => runId), rewind, error);
|
|
9739
|
+
throw error;
|
|
9740
|
+
}
|
|
9741
|
+
for (const { runId, item } of journaled) this.fire(binding.workflowId, item, kind, runId);
|
|
9742
|
+
}
|
|
9743
|
+
async redeliver(runIds, rewind, error) {
|
|
9744
|
+
if (!await rewind(runIds)) await this.failRefused(runIds, error);
|
|
9745
|
+
}
|
|
9746
|
+
async failRefused(runIds, error) {
|
|
9747
|
+
try {
|
|
9748
|
+
const raw = await this.options.store();
|
|
9749
|
+
for (const runId of runIds) await raw?.failRun(runId, error instanceof Error ? error.message : String(error), error instanceof Error ? error.name : void 0);
|
|
9750
|
+
} catch (failure) {
|
|
9751
|
+
logger$2.warn(`Could not fail the refused runs ${runIds.join(", ")}`, failure);
|
|
8569
9752
|
}
|
|
8570
|
-
this.options.fire(binding.workflowId, item, pieceTriggerKind(binding.block));
|
|
8571
9753
|
}
|
|
8572
9754
|
};
|
|
8573
9755
|
//#endregion
|
|
@@ -8811,7 +9993,7 @@ const DESCRIBE_TIMEOUT_MS = 3e4;
|
|
|
8811
9993
|
const SEED_ATTEMPTS = 3;
|
|
8812
9994
|
const SEED_RETRY_BASE_MS = 250;
|
|
8813
9995
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref());
|
|
8814
|
-
const logger = childLogger(["workflow", "runtime"]);
|
|
9996
|
+
const logger$1 = childLogger(["workflow", "runtime"]);
|
|
8815
9997
|
const DRIVE_DOCUMENT_TYPE = "powerhouse/document-drive";
|
|
8816
9998
|
const WORKFLOW_DOCUMENT_TYPE = "powerhouse/workflow";
|
|
8817
9999
|
const DOCUMENT_SCOPE = "document";
|
|
@@ -8904,6 +10086,7 @@ const UNAUTHORIZED = { status: 401 };
|
|
|
8904
10086
|
const JSON_CONTENT_TYPE = "application/json; charset=utf-8";
|
|
8905
10087
|
const DELIVERY_TIMEOUT_MS = Number(process.env.PH_WORKFLOWS_WEBHOOK_TIMEOUT_MS) || 3e4;
|
|
8906
10088
|
const TIMED_OUT = Symbol("webhook delivery timed out");
|
|
10089
|
+
const WEBHOOK_REDELIVERED_REASON = "Redelivered: the runtime shut down before this delivery ran, and the sender was told to retry";
|
|
8907
10090
|
/** Resolves to TIMED_OUT, and never keeps the process alive waiting to. */
|
|
8908
10091
|
function timeout(ms) {
|
|
8909
10092
|
return new Promise((resolve) => {
|
|
@@ -8927,7 +10110,7 @@ function splitPollInterval(config) {
|
|
|
8927
10110
|
const { [POLL_INTERVAL_CONFIG_KEY]: raw, ...rest } = config;
|
|
8928
10111
|
const seconds = typeof raw === "string" ? Number(raw) : raw;
|
|
8929
10112
|
if (typeof seconds !== "number" || !Number.isFinite(seconds) || seconds <= 0) {
|
|
8930
|
-
logger.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
|
|
10113
|
+
logger$1.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
|
|
8931
10114
|
return { config: rest };
|
|
8932
10115
|
}
|
|
8933
10116
|
return {
|
|
@@ -8943,6 +10126,17 @@ function parseWorkflowState(resultingState) {
|
|
|
8943
10126
|
return;
|
|
8944
10127
|
}
|
|
8945
10128
|
}
|
|
10129
|
+
function refusalStatus(refusal) {
|
|
10130
|
+
switch (refusal) {
|
|
10131
|
+
case "parked":
|
|
10132
|
+
case "singleton":
|
|
10133
|
+
case "stale": return 409;
|
|
10134
|
+
case "queue-full":
|
|
10135
|
+
case "expired": return 429;
|
|
10136
|
+
default: return 500;
|
|
10137
|
+
}
|
|
10138
|
+
}
|
|
10139
|
+
const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
|
|
8946
10140
|
function registrationKey(state) {
|
|
8947
10141
|
const trigger = runnableDefinition(state).trigger;
|
|
8948
10142
|
const { lastTest: _lastTest, ...armed } = trigger ?? {};
|
|
@@ -8961,9 +10155,11 @@ function runJournalName(stateName, documentName) {
|
|
|
8961
10155
|
var WorkflowRuntimeService = class {
|
|
8962
10156
|
host;
|
|
8963
10157
|
logger;
|
|
10158
|
+
telemetry;
|
|
8964
10159
|
attachments;
|
|
8965
10160
|
executor;
|
|
8966
10161
|
pieceWorkers;
|
|
10162
|
+
unobservePool;
|
|
8967
10163
|
storePromise;
|
|
8968
10164
|
storeError;
|
|
8969
10165
|
storeOpening = false;
|
|
@@ -8978,13 +10174,31 @@ var WorkflowRuntimeService = class {
|
|
|
8978
10174
|
seedError;
|
|
8979
10175
|
constructor(host) {
|
|
8980
10176
|
this.host = host;
|
|
10177
|
+
this.telemetry = new WorkflowTelemetry(host.telemetry);
|
|
8981
10178
|
this.blockResolver = new BlockResolver({
|
|
8982
10179
|
local: installedPiece,
|
|
8983
10180
|
timeoutMs: host.pieceVersionLookupMs ?? PIECE_VERSION_LOOKUP_TIMEOUT_MS,
|
|
8984
10181
|
hasBlock: (ref, version, source) => this.versionHasBlock(ref, version, source)
|
|
8985
10182
|
});
|
|
8986
|
-
this.logger = host.logger ?? logger;
|
|
8987
|
-
this.attachments = host.attachments ? createAttachmentPort(host.attachments,
|
|
10183
|
+
this.logger = host.logger ?? logger$1;
|
|
10184
|
+
this.attachments = host.attachments ? createAttachmentPort(host.attachments, {
|
|
10185
|
+
documentIdFor: () => currentWorkflowId(),
|
|
10186
|
+
canReadRef: async (ref) => {
|
|
10187
|
+
const workflowId = currentWorkflowId();
|
|
10188
|
+
const origins = currentAttachmentOrigins();
|
|
10189
|
+
if (!workflowId) return false;
|
|
10190
|
+
if (origins?.written.has(ref)) return true;
|
|
10191
|
+
if (!host.canReadAttachmentRef) return false;
|
|
10192
|
+
const runUser = currentRunUser();
|
|
10193
|
+
return host.canReadAttachmentRef({
|
|
10194
|
+
workflowId,
|
|
10195
|
+
ref,
|
|
10196
|
+
documentIds: [...new Set(origins?.documentIds() ?? [])],
|
|
10197
|
+
...runUser !== void 0 ? { runUser } : {}
|
|
10198
|
+
});
|
|
10199
|
+
},
|
|
10200
|
+
onWritten: (ref) => currentAttachmentOrigins()?.written.add(ref)
|
|
10201
|
+
}) : void 0;
|
|
8988
10202
|
this.storePromise = this.openStore();
|
|
8989
10203
|
this.seedPromise = this.seedWithRetries();
|
|
8990
10204
|
this.startRetention();
|
|
@@ -9014,7 +10228,7 @@ var WorkflowRuntimeService = class {
|
|
|
9014
10228
|
}
|
|
9015
10229
|
}
|
|
9016
10230
|
async store() {
|
|
9017
|
-
if (this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
|
|
10231
|
+
if (!this.closed && this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
|
|
9018
10232
|
try {
|
|
9019
10233
|
return await this.storePromise;
|
|
9020
10234
|
} catch {
|
|
@@ -9093,9 +10307,22 @@ var WorkflowRuntimeService = class {
|
|
|
9093
10307
|
return await this.endpointCount() > 0;
|
|
9094
10308
|
}
|
|
9095
10309
|
registeredAs = /* @__PURE__ */ new Map();
|
|
9096
|
-
|
|
10310
|
+
registrationChains = /* @__PURE__ */ new Map();
|
|
10311
|
+
inRegistrationOrder(workflowId, task) {
|
|
10312
|
+
const run = (this.registrationChains.get(workflowId) ?? Promise.resolve()).then(task);
|
|
10313
|
+
const tail = run.then(() => void 0, () => void 0);
|
|
10314
|
+
this.registrationChains.set(workflowId, tail);
|
|
10315
|
+
tail.then(() => {
|
|
10316
|
+
if (this.registrationChains.get(workflowId) === tail) this.registrationChains.delete(workflowId);
|
|
10317
|
+
});
|
|
10318
|
+
return run;
|
|
10319
|
+
}
|
|
10320
|
+
updateRegistration(workflowId, state, onlyIfChanged = false) {
|
|
10321
|
+
return this.inRegistrationOrder(workflowId, () => this.registerNow(workflowId, state, onlyIfChanged));
|
|
10322
|
+
}
|
|
10323
|
+
async registerNow(workflowId, state, onlyIfChanged) {
|
|
9097
10324
|
const key = registrationKey(state);
|
|
9098
|
-
if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key) return;
|
|
10325
|
+
if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key && !await this.outdatedPark(workflowId, state)) return;
|
|
9099
10326
|
this.registeredAs.set(workflowId, key);
|
|
9100
10327
|
try {
|
|
9101
10328
|
await this.applyRegistration(workflowId, state);
|
|
@@ -9114,6 +10341,23 @@ var WorkflowRuntimeService = class {
|
|
|
9114
10341
|
}
|
|
9115
10342
|
const trigger = state.status === "ENABLED" ? runnableDefinition(state).trigger : void 0;
|
|
9116
10343
|
const block = trigger ? triggerBlock(trigger) : void 0;
|
|
10344
|
+
const version = runnableDefinition(state).version;
|
|
10345
|
+
if (state.status !== "ENABLED") {
|
|
10346
|
+
const held = this.registry.get(workflowId);
|
|
10347
|
+
this.supervisor().releasePark(workflowId, !(held && SUPERVISED_KINDS.has(held.kind))).catch((error) => {
|
|
10348
|
+
this.laneFailed(`Could not clear the park of disabled workflow ${workflowId}`, error);
|
|
10349
|
+
});
|
|
10350
|
+
} else if (await this.outdatedPark(workflowId, state)) this.supervisor().unpark(workflowId, version).catch((error) => {
|
|
10351
|
+
this.laneFailed(`Could not lift the outdated park of workflow ${workflowId}`, error);
|
|
10352
|
+
});
|
|
10353
|
+
const park = block ? await this.parks.get(workflowId) : void 0;
|
|
10354
|
+
if (block && (blockKey(block) === WEBHOOK_BLOCK || triggerKindOf(block)) && park && park.published_version >= version) {
|
|
10355
|
+
const had = this.registry.get(workflowId);
|
|
10356
|
+
this.registry.delete(workflowId);
|
|
10357
|
+
if (had && SUPERVISED_KINDS.has(had.kind)) this.dropSupervised(workflowId);
|
|
10358
|
+
this.logger.warn(`Workflow ${workflowId} is PARKED; its trigger stays unarmed until the workflow is re-published or re-enabled`);
|
|
10359
|
+
return;
|
|
10360
|
+
}
|
|
9117
10361
|
if (block && blockKey(block) === WEBHOOK_BLOCK) {
|
|
9118
10362
|
await this.registerWebhook(workflowId, block, trigger.config);
|
|
9119
10363
|
return;
|
|
@@ -9136,14 +10380,14 @@ var WorkflowRuntimeService = class {
|
|
|
9136
10380
|
workflowId,
|
|
9137
10381
|
kind: "schedule"
|
|
9138
10382
|
});
|
|
9139
|
-
this.enableSupervised(workflowId, supervised);
|
|
10383
|
+
this.enableSupervised(workflowId, supervised, version);
|
|
9140
10384
|
return;
|
|
9141
10385
|
}
|
|
9142
10386
|
this.registry.set(workflowId, {
|
|
9143
10387
|
workflowId,
|
|
9144
10388
|
kind: "piece"
|
|
9145
10389
|
});
|
|
9146
|
-
await this.registerPieceTrigger(workflowId, supervised);
|
|
10390
|
+
await this.registerPieceTrigger(workflowId, supervised, version);
|
|
9147
10391
|
return;
|
|
9148
10392
|
}
|
|
9149
10393
|
const had = this.registry.get(workflowId);
|
|
@@ -9159,12 +10403,12 @@ var WorkflowRuntimeService = class {
|
|
|
9159
10403
|
filter: parseLifecycleFilter(config)
|
|
9160
10404
|
});
|
|
9161
10405
|
}
|
|
9162
|
-
enableSupervised(workflowId, binding) {
|
|
9163
|
-
this.supervisor().upsert(binding).catch((error) => {
|
|
9164
|
-
this.
|
|
10406
|
+
enableSupervised(workflowId, binding, publishedVersion) {
|
|
10407
|
+
this.supervisor().upsert(binding, publishedVersion).catch((error) => {
|
|
10408
|
+
this.laneFailed(`Trigger enable failed for ${workflowId}`, error);
|
|
9165
10409
|
});
|
|
9166
10410
|
}
|
|
9167
|
-
async registerPieceTrigger(workflowId, binding) {
|
|
10411
|
+
async registerPieceTrigger(workflowId, binding, publishedVersion) {
|
|
9168
10412
|
const delivery = await this.pieceDelivery(binding);
|
|
9169
10413
|
if (typeof delivery !== "string") {
|
|
9170
10414
|
this.refusePieceTrigger(workflowId, binding, delivery);
|
|
@@ -9182,7 +10426,7 @@ var WorkflowRuntimeService = class {
|
|
|
9182
10426
|
});
|
|
9183
10427
|
await (await this.endpoints())?.endpointFor(workflowId);
|
|
9184
10428
|
}
|
|
9185
|
-
this.enableSupervised(workflowId, resolved);
|
|
10429
|
+
this.enableSupervised(workflowId, resolved, publishedVersion);
|
|
9186
10430
|
}
|
|
9187
10431
|
async pieceDelivery(binding) {
|
|
9188
10432
|
let strategy;
|
|
@@ -9218,7 +10462,7 @@ var WorkflowRuntimeService = class {
|
|
|
9218
10462
|
const reason = `The trigger ${blockLabel(binding.block)} is not armed: ${refusal.reason}`;
|
|
9219
10463
|
this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
|
|
9220
10464
|
this.supervisor().reject(workflowId, binding.block, binding.config, reason, retryAt).catch((error) => {
|
|
9221
|
-
this.
|
|
10465
|
+
this.laneFailed(`Could not record the refused trigger for workflow ${workflowId}`, error);
|
|
9222
10466
|
});
|
|
9223
10467
|
}
|
|
9224
10468
|
async registerWebhook(workflowId, block, rawConfig) {
|
|
@@ -9233,7 +10477,7 @@ var WorkflowRuntimeService = class {
|
|
|
9233
10477
|
this.logger.error(`Webhook trigger rejected for ${workflowId}: @error`, message);
|
|
9234
10478
|
this.unarmed.add(workflowId);
|
|
9235
10479
|
this.supervisor().reject(workflowId, block, configRecord(rawConfig), message).catch((recordError) => {
|
|
9236
|
-
this.
|
|
10480
|
+
this.laneFailed(`Could not record the rejected webhook trigger for workflow ${workflowId}`, recordError);
|
|
9237
10481
|
});
|
|
9238
10482
|
return;
|
|
9239
10483
|
}
|
|
@@ -9359,7 +10603,7 @@ var WorkflowRuntimeService = class {
|
|
|
9359
10603
|
const trigger = runnableDefinition(state).trigger;
|
|
9360
10604
|
if (!trigger) return;
|
|
9361
10605
|
this.supervisor().reject(workflowId, triggerBlock(trigger), configRecord(trigger.config), reason).catch((error) => {
|
|
9362
|
-
this.
|
|
10606
|
+
this.laneFailed(`Could not record the reactor access denial for workflow ${workflowId}`, error);
|
|
9363
10607
|
});
|
|
9364
10608
|
}
|
|
9365
10609
|
unarmed = /* @__PURE__ */ new Set();
|
|
@@ -9373,7 +10617,7 @@ var WorkflowRuntimeService = class {
|
|
|
9373
10617
|
this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
|
|
9374
10618
|
this.unarmed.add(workflowId);
|
|
9375
10619
|
this.supervisor().reject(workflowId, block, configRecord(trigger.config), reason, retryAt).catch((error) => {
|
|
9376
|
-
this.
|
|
10620
|
+
this.laneFailed(`Could not record the unresolved trigger for workflow ${workflowId}`, error);
|
|
9377
10621
|
});
|
|
9378
10622
|
}
|
|
9379
10623
|
resolutionRetries = /* @__PURE__ */ new Map();
|
|
@@ -9402,7 +10646,7 @@ var WorkflowRuntimeService = class {
|
|
|
9402
10646
|
}
|
|
9403
10647
|
dropSupervised(workflowId) {
|
|
9404
10648
|
this.supervisor().remove(workflowId).catch((error) => {
|
|
9405
|
-
this.
|
|
10649
|
+
this.laneFailed(`Trigger disable failed for ${workflowId}`, error);
|
|
9406
10650
|
});
|
|
9407
10651
|
}
|
|
9408
10652
|
async refreshRegistration(workflowId, resultingState, onlyIfChanged = false) {
|
|
@@ -9427,6 +10671,7 @@ var WorkflowRuntimeService = class {
|
|
|
9427
10671
|
return false;
|
|
9428
10672
|
}
|
|
9429
10673
|
async onDocumentsPurged(markers) {
|
|
10674
|
+
this.throwIfClosed();
|
|
9430
10675
|
const documentIds = [...new Set(markers.map((marker) => marker.context.documentId))];
|
|
9431
10676
|
const store = await this.store();
|
|
9432
10677
|
if (!store) {
|
|
@@ -9434,13 +10679,15 @@ var WorkflowRuntimeService = class {
|
|
|
9434
10679
|
throw new Error("Erasing purged documents needs the run journal", { cause: this.storeError });
|
|
9435
10680
|
}
|
|
9436
10681
|
for (const workflowId of purgedWorkflowIds(markers)) await this.erasePurgedWorkflow(store, workflowId);
|
|
9437
|
-
const erased = await store.eraseRunsForDocuments(documentIds);
|
|
10682
|
+
const erased = await this.telemetry.phase("journal.purge", { "documents.count": documentIds.length }, () => store.eraseRunsForDocuments(documentIds));
|
|
9438
10683
|
if (erased.runs > 0) this.logger.info("Erased @runs workflow run(s) of purged documents @ids", erased.runs, documentIds);
|
|
9439
10684
|
return erased;
|
|
9440
10685
|
}
|
|
9441
10686
|
async onOperations(operations) {
|
|
10687
|
+
this.throwIfClosed();
|
|
9442
10688
|
const hints = collectLifecycleParentHints(operations);
|
|
9443
10689
|
for (const { operation, context } of operations) {
|
|
10690
|
+
this.throwIfClosed();
|
|
9444
10691
|
if (context.scope !== DOCUMENT_SCOPE && context.scope !== "global") continue;
|
|
9445
10692
|
const opKey = operationKey({
|
|
9446
10693
|
operation,
|
|
@@ -9490,15 +10737,16 @@ var WorkflowRuntimeService = class {
|
|
|
9490
10737
|
this.unjournaledFires.delete(oldest);
|
|
9491
10738
|
this.logger.warn("Too many unjournaled workflow fires held; a redelivery of the oldest fires again");
|
|
9492
10739
|
}
|
|
9493
|
-
this.
|
|
10740
|
+
this.startFiring(workflowId, payload, kind);
|
|
9494
10741
|
}
|
|
9495
10742
|
async enqueueFire(workflowId, payload, kind, dedupeKey) {
|
|
9496
10743
|
const fireKey = JSON.stringify([workflowId, dedupeKey]);
|
|
9497
10744
|
const store = await this.store();
|
|
10745
|
+
this.throwIfClosed();
|
|
9498
10746
|
if (this.unjournaledFires.has(fireKey)) {
|
|
9499
10747
|
if (!store) return;
|
|
9500
10748
|
try {
|
|
9501
|
-
await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString());
|
|
10749
|
+
if (!await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString())) this.logger.warn(`Workflow ${workflowId}: a ${kind} fire this process ran unjournaled had its key already marked fired; it may have run twice`);
|
|
9502
10750
|
this.unjournaledFires.delete(fireKey);
|
|
9503
10751
|
} catch (error) {
|
|
9504
10752
|
this.logger.warn(`Could not record the unjournaled ${kind} fire for workflow ${workflowId}: @error`, error);
|
|
@@ -9509,26 +10757,62 @@ var WorkflowRuntimeService = class {
|
|
|
9509
10757
|
this.fireUnjournaled(fireKey, workflowId, payload, kind);
|
|
9510
10758
|
return;
|
|
9511
10759
|
}
|
|
9512
|
-
|
|
10760
|
+
const enqueue = {
|
|
10761
|
+
workflowId,
|
|
10762
|
+
triggerKind: kind,
|
|
10763
|
+
triggerPayload: payload
|
|
10764
|
+
};
|
|
10765
|
+
let claim;
|
|
10766
|
+
try {
|
|
10767
|
+
claim = await store.claimAndEnqueueRun(dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString(), enqueue);
|
|
10768
|
+
} catch (error) {
|
|
10769
|
+
this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
|
|
10770
|
+
this.fireUnjournaled(fireKey, workflowId, payload, kind);
|
|
10771
|
+
return;
|
|
10772
|
+
}
|
|
10773
|
+
if (claim.outcome === "duplicate") return;
|
|
10774
|
+
if (claim.outcome === "abandoned") {
|
|
10775
|
+
this.logger.error(`Workflow ${workflowId}: a ${kind} fire has been delivered ${claim.attempts} times without ever journaling a run; abandoning it rather than replaying it on every boot`);
|
|
10776
|
+
try {
|
|
10777
|
+
await store.journalAbandonedFire(dedupeKey, {
|
|
10778
|
+
...enqueue,
|
|
10779
|
+
attempts: claim.attempts
|
|
10780
|
+
});
|
|
10781
|
+
} catch (error) {
|
|
10782
|
+
this.logger.warn(`Could not journal the abandoned ${kind} fire for workflow ${workflowId}: @error`, error);
|
|
10783
|
+
}
|
|
10784
|
+
return;
|
|
10785
|
+
}
|
|
10786
|
+
this.startFiring(workflowId, payload, kind, claim.runId);
|
|
10787
|
+
}
|
|
10788
|
+
async fireFromTrigger(workflowId, payload, kind) {
|
|
10789
|
+
const runId = await this.journalFiring(workflowId, payload, kind);
|
|
10790
|
+
this.startFiring(workflowId, payload, kind, runId);
|
|
10791
|
+
}
|
|
10792
|
+
async journalFiring(workflowId, payload, kind) {
|
|
10793
|
+
const store = await this.store();
|
|
10794
|
+
if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
|
|
10795
|
+
if (!store) return void 0;
|
|
9513
10796
|
try {
|
|
9514
|
-
|
|
10797
|
+
return await store.enqueueRun({
|
|
9515
10798
|
workflowId,
|
|
9516
10799
|
triggerKind: kind,
|
|
9517
10800
|
triggerPayload: payload
|
|
9518
10801
|
});
|
|
9519
10802
|
} catch (error) {
|
|
9520
10803
|
this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
|
|
9521
|
-
this.fireUnjournaled(fireKey, workflowId, payload, kind);
|
|
9522
10804
|
return;
|
|
9523
10805
|
}
|
|
9524
|
-
if (runId === null) return;
|
|
9525
|
-
this.fireFromTrigger(workflowId, payload, kind, runId);
|
|
9526
10806
|
}
|
|
9527
|
-
|
|
10807
|
+
startFiring(workflowId, payload, kind, enqueuedRunId) {
|
|
9528
10808
|
this.fire(workflowId, payload, kind, void 0, void 0, enqueuedRunId).then((run) => {
|
|
9529
10809
|
this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
|
|
9530
10810
|
}, (error) => {
|
|
9531
|
-
|
|
10811
|
+
if (enqueuedRunId && isShutdownRefusal(error)) {
|
|
10812
|
+
this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to rerun`);
|
|
10813
|
+
return;
|
|
10814
|
+
}
|
|
10815
|
+
this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
|
|
9532
10816
|
});
|
|
9533
10817
|
}
|
|
9534
10818
|
firedLifecycle = /* @__PURE__ */ new Set();
|
|
@@ -9607,8 +10891,14 @@ var WorkflowRuntimeService = class {
|
|
|
9607
10891
|
this.cancelTriggerTest(workflowId, "stopped: the workflow was deleted");
|
|
9608
10892
|
}
|
|
9609
10893
|
async releaseDeleted(workflowId) {
|
|
9610
|
-
|
|
10894
|
+
let failure;
|
|
10895
|
+
try {
|
|
10896
|
+
await this.supervisor().forget(workflowId);
|
|
10897
|
+
} catch (error) {
|
|
10898
|
+
failure = { error };
|
|
10899
|
+
}
|
|
9611
10900
|
await (await this.endpoints())?.revoke(workflowId);
|
|
10901
|
+
if (failure) throw failure.error;
|
|
9612
10902
|
}
|
|
9613
10903
|
async erasePurgedWorkflow(store, workflowId) {
|
|
9614
10904
|
this.dropDeleted(workflowId);
|
|
@@ -9666,41 +10956,60 @@ var WorkflowRuntimeService = class {
|
|
|
9666
10956
|
});
|
|
9667
10957
|
}
|
|
9668
10958
|
triggerSupervisor;
|
|
10959
|
+
parks = new ParkState(async () => (await this.store())?.listWorkflowParks());
|
|
9669
10960
|
supervisor() {
|
|
9670
10961
|
this.triggerSupervisor ??= new TriggerSupervisor({
|
|
9671
10962
|
store: () => this.store(),
|
|
10963
|
+
parks: this.parks,
|
|
9672
10964
|
resolveAuth: async (connectionId, request) => {
|
|
9673
10965
|
if (!connectionId) return void 0;
|
|
9674
10966
|
const resolved = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolveWithSecrets(connectionId, request);
|
|
9675
10967
|
return rememberSecrets(resolved.auth, resolved.secretValues);
|
|
9676
10968
|
},
|
|
9677
|
-
fire: (workflowId, payload, kind) => {
|
|
9678
|
-
this.
|
|
10969
|
+
fire: (workflowId, payload, kind, runId) => {
|
|
10970
|
+
this.startFiring(workflowId, payload, kind, runId);
|
|
9679
10971
|
},
|
|
9680
10972
|
webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
|
|
9681
10973
|
reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
|
|
9682
10974
|
models: this.models,
|
|
10975
|
+
telemetry: this.telemetry,
|
|
9683
10976
|
cacheDir: bundleCacheDir(),
|
|
9684
10977
|
resolver: pieceResolver(),
|
|
9685
10978
|
egress: configuredEgress(),
|
|
9686
10979
|
defaultIntervalMs: Number(process.env.PH_WORKFLOWS_POLL_INTERVAL_MS) || void 0,
|
|
9687
10980
|
reconcileIntervalMs: Number(process.env.PH_WORKFLOWS_WEBHOOK_RECONCILE_MS) || void 0
|
|
9688
10981
|
});
|
|
10982
|
+
if (this.closed) this.triggerSupervisor.stop();
|
|
9689
10983
|
return this.triggerSupervisor;
|
|
9690
10984
|
}
|
|
10985
|
+
laneFailed(message, error) {
|
|
10986
|
+
if (isShutdownRefusal(error)) {
|
|
10987
|
+
this.logger.debug(message, error);
|
|
10988
|
+
return;
|
|
10989
|
+
}
|
|
10990
|
+
this.logger.error(message, error);
|
|
10991
|
+
}
|
|
9691
10992
|
startTriggerSupervisor() {
|
|
9692
10993
|
this.supervisor().start();
|
|
9693
10994
|
}
|
|
9694
10995
|
stopTriggerSupervisor() {
|
|
9695
10996
|
this.triggerSupervisor?.stop();
|
|
9696
10997
|
}
|
|
10998
|
+
closed = false;
|
|
10999
|
+
throwIfClosed() {
|
|
11000
|
+
if (this.closed) throw new Error("This workflow runtime has shut down; the operations are left for the next owner");
|
|
11001
|
+
}
|
|
9697
11002
|
shutdown() {
|
|
11003
|
+
this.closed = true;
|
|
11004
|
+
this.runGate.close();
|
|
9698
11005
|
clearInterval(this.retentionTimer);
|
|
9699
11006
|
for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
|
|
9700
11007
|
this.resolutionRetries.clear();
|
|
9701
11008
|
for (const workflowId of [...this.webhookTests.keys()]) this.cancelTriggerTest(workflowId, "stopped: the runtime shut down");
|
|
9702
11009
|
this.stopTriggerSupervisor();
|
|
9703
11010
|
this.pieceWorkers?.dispose();
|
|
11011
|
+
this.unobservePool?.();
|
|
11012
|
+
this.unobservePool = void 0;
|
|
9704
11013
|
this.designWorker?.dispose();
|
|
9705
11014
|
this.designWorker = void 0;
|
|
9706
11015
|
}
|
|
@@ -9802,6 +11111,10 @@ var WorkflowRuntimeService = class {
|
|
|
9802
11111
|
/** A delivery the service has already rate-limited, verified, de-duplicated and
|
|
9803
11112
|
* answered any challenge for; all that is left is deciding what it means. */
|
|
9804
11113
|
async deliverWebhook(request) {
|
|
11114
|
+
if (this.closed) return {
|
|
11115
|
+
status: 503,
|
|
11116
|
+
unprocessed: true
|
|
11117
|
+
};
|
|
9805
11118
|
const workflowId = request.key;
|
|
9806
11119
|
const registration = this.liveWebhook(workflowId);
|
|
9807
11120
|
const test = this.webhookTests.get(workflowId);
|
|
@@ -9816,16 +11129,44 @@ var WorkflowRuntimeService = class {
|
|
|
9816
11129
|
const { config } = registration;
|
|
9817
11130
|
const payload = webhookPayload(request);
|
|
9818
11131
|
if (config.responseMode === "async") {
|
|
9819
|
-
|
|
11132
|
+
try {
|
|
11133
|
+
await this.fireFromTrigger(workflowId, payload, WEBHOOK_TRIGGER_KIND);
|
|
11134
|
+
} catch (error) {
|
|
11135
|
+
if (error instanceof WorkflowRuntimeClosedError) return {
|
|
11136
|
+
status: 503,
|
|
11137
|
+
unprocessed: true
|
|
11138
|
+
};
|
|
11139
|
+
throw error;
|
|
11140
|
+
}
|
|
9820
11141
|
return { status: config.responseStatus };
|
|
9821
11142
|
}
|
|
9822
|
-
|
|
11143
|
+
let runId;
|
|
11144
|
+
try {
|
|
11145
|
+
runId = await this.journalFiring(workflowId, payload, WEBHOOK_TRIGGER_KIND);
|
|
11146
|
+
} catch (error) {
|
|
11147
|
+
if (error instanceof WorkflowRuntimeClosedError) return {
|
|
11148
|
+
status: 503,
|
|
11149
|
+
unprocessed: true
|
|
11150
|
+
};
|
|
11151
|
+
throw error;
|
|
11152
|
+
}
|
|
11153
|
+
const firing = this.fire(workflowId, payload, WEBHOOK_TRIGGER_KIND, void 0, void 0, runId).then((result) => ({
|
|
9823
11154
|
ok: true,
|
|
9824
11155
|
result
|
|
9825
11156
|
}), (error) => ({
|
|
9826
11157
|
ok: false,
|
|
9827
11158
|
error
|
|
9828
|
-
}))
|
|
11159
|
+
}));
|
|
11160
|
+
let replied = false;
|
|
11161
|
+
const takeReply = () => !replied && (replied = true);
|
|
11162
|
+
if (runId) this.heldReplies.set(runId, takeReply);
|
|
11163
|
+
let run;
|
|
11164
|
+
try {
|
|
11165
|
+
run = await Promise.race([firing, timeout(DELIVERY_TIMEOUT_MS)]);
|
|
11166
|
+
if (run === TIMED_OUT && !takeReply()) run = await firing;
|
|
11167
|
+
} finally {
|
|
11168
|
+
if (runId) this.heldReplies.delete(runId);
|
|
11169
|
+
}
|
|
9829
11170
|
if (run === TIMED_OUT) {
|
|
9830
11171
|
this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
|
|
9831
11172
|
return {
|
|
@@ -9838,6 +11179,10 @@ var WorkflowRuntimeService = class {
|
|
|
9838
11179
|
};
|
|
9839
11180
|
}
|
|
9840
11181
|
if (!run.ok) {
|
|
11182
|
+
if (run.error instanceof WorkflowRuntimeClosedError) return {
|
|
11183
|
+
status: 503,
|
|
11184
|
+
unprocessed: true
|
|
11185
|
+
};
|
|
9841
11186
|
const message = run.error instanceof Error ? run.error.message : String(run.error);
|
|
9842
11187
|
this.logger.error(`Webhook run failed for ${workflowId}: @error`, message);
|
|
9843
11188
|
return {
|
|
@@ -9847,7 +11192,7 @@ var WorkflowRuntimeService = class {
|
|
|
9847
11192
|
};
|
|
9848
11193
|
}
|
|
9849
11194
|
return {
|
|
9850
|
-
status: run.result.status === "SUCCEEDED" ? config.responseStatus :
|
|
11195
|
+
status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
|
|
9851
11196
|
contentType: JSON_CONTENT_TYPE,
|
|
9852
11197
|
body: JSON.stringify({
|
|
9853
11198
|
runId: run.result.runId,
|
|
@@ -9862,7 +11207,7 @@ var WorkflowRuntimeService = class {
|
|
|
9862
11207
|
try {
|
|
9863
11208
|
return handshakeReply((await this.supervisor().handshake(binding, webhookPayload(request))).output);
|
|
9864
11209
|
} catch (error) {
|
|
9865
|
-
this.logger
|
|
11210
|
+
this.logger[isShutdownRefusal(error) ? "debug" : "error"]("Handshake failed for @block on workflow @workflow", blockLabel(binding.block), binding.workflowId, error);
|
|
9866
11211
|
return { status: 500 };
|
|
9867
11212
|
}
|
|
9868
11213
|
}
|
|
@@ -9881,7 +11226,7 @@ var WorkflowRuntimeService = class {
|
|
|
9881
11226
|
this.supervisor().deliverWebhook(binding.workflowId, payload).then(() => {
|
|
9882
11227
|
this.logger.info("Webhook delivered to @block for workflow @workflow", blockLabel(binding.block), binding.workflowId);
|
|
9883
11228
|
}, (error) => {
|
|
9884
|
-
this.
|
|
11229
|
+
this.laneFailed(`Webhook delivery failed for workflow ${binding.workflowId}`, error);
|
|
9885
11230
|
});
|
|
9886
11231
|
return { status: 200 };
|
|
9887
11232
|
}
|
|
@@ -9933,14 +11278,14 @@ var WorkflowRuntimeService = class {
|
|
|
9933
11278
|
async stepReactorAccess(request) {
|
|
9934
11279
|
const declared = await this.declaredReactor(request.block);
|
|
9935
11280
|
if (!declared) return void 0;
|
|
9936
|
-
const base = await this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null);
|
|
9937
|
-
return reactorTap(this.host, base, runJournal());
|
|
11281
|
+
const base = await this.telemetry.phase("reactor.scope", {}, () => this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null));
|
|
11282
|
+
return reactorTap(this.host, base, runJournal(), {}, this.telemetry);
|
|
9938
11283
|
}
|
|
9939
11284
|
async triggerReactorAccess(binding, requireReactor, runUser) {
|
|
9940
11285
|
const bound = binding.reactorConnectionId;
|
|
9941
11286
|
const user = runUser !== void 0 ? runUser : await assertReactorConnectionsReadable(this.host, binding.workflowId, new Set(bound ? [bound] : [])) ?? null;
|
|
9942
11287
|
const base = await this.reactorScope(bound, requireReactor, user);
|
|
9943
|
-
return reactorTap(this.host, base, NO_JOURNAL);
|
|
11288
|
+
return reactorTap(this.host, base, NO_JOURNAL, {}, this.telemetry);
|
|
9944
11289
|
}
|
|
9945
11290
|
async designReactorAccess(block, ctx, reactorConnectionId) {
|
|
9946
11291
|
if (!await this.declaredReactor(block)) return void 0;
|
|
@@ -9951,6 +11296,12 @@ var WorkflowRuntimeService = class {
|
|
|
9951
11296
|
}
|
|
9952
11297
|
descriptors = /* @__PURE__ */ new Map();
|
|
9953
11298
|
designWorker;
|
|
11299
|
+
createDesignWorker() {
|
|
11300
|
+
return new PieceWorker({
|
|
11301
|
+
models: this.models,
|
|
11302
|
+
telemetry: this.telemetry
|
|
11303
|
+
});
|
|
11304
|
+
}
|
|
9954
11305
|
designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
|
|
9955
11306
|
describing = /* @__PURE__ */ new Map();
|
|
9956
11307
|
pieceDescriptor(target) {
|
|
@@ -9971,7 +11322,7 @@ var WorkflowRuntimeService = class {
|
|
|
9971
11322
|
let descriptor = this.descriptors.get(cacheKey);
|
|
9972
11323
|
if (!descriptor) {
|
|
9973
11324
|
const piece = await pieceResolver().resolve(target);
|
|
9974
|
-
this.designWorker ??=
|
|
11325
|
+
this.designWorker ??= this.createDesignWorker();
|
|
9975
11326
|
let output;
|
|
9976
11327
|
try {
|
|
9977
11328
|
output = (await this.designWorker.describePiece({
|
|
@@ -10002,7 +11353,14 @@ var WorkflowRuntimeService = class {
|
|
|
10002
11353
|
async runs(args, ctx) {
|
|
10003
11354
|
return (await this.runsPage(args, ctx)).records;
|
|
10004
11355
|
}
|
|
10005
|
-
|
|
11356
|
+
runsPage(args, ctx) {
|
|
11357
|
+
return this.telemetry.phase("runs.page", { "runs.scope": args.workflowId ? "workflow" : args.driveId ? "drive" : "all" }, async (span) => {
|
|
11358
|
+
const page = await this.listRunsPage(args, ctx);
|
|
11359
|
+
span.setAttribute("runs.returned", page.records.length);
|
|
11360
|
+
return page;
|
|
11361
|
+
});
|
|
11362
|
+
}
|
|
11363
|
+
async listRunsPage(args, ctx) {
|
|
10006
11364
|
const empty = {
|
|
10007
11365
|
records: [],
|
|
10008
11366
|
hasNextPage: false,
|
|
@@ -10158,7 +11516,7 @@ var WorkflowRuntimeService = class {
|
|
|
10158
11516
|
}
|
|
10159
11517
|
let outcome;
|
|
10160
11518
|
try {
|
|
10161
|
-
this.designWorker ??=
|
|
11519
|
+
this.designWorker ??= this.createDesignWorker();
|
|
10162
11520
|
outcome = (await this.designWorker.checkConnection({
|
|
10163
11521
|
...moduleRef,
|
|
10164
11522
|
auth: shapedAuth,
|
|
@@ -10335,14 +11693,17 @@ var WorkflowRuntimeService = class {
|
|
|
10335
11693
|
const local = await this.localPieceAt(packageName, version);
|
|
10336
11694
|
return local ? triggersResult(local.descriptor, local.piece.name, local.piece.version) : fetchPieceTriggers(packageName, version);
|
|
10337
11695
|
}
|
|
10338
|
-
|
|
11696
|
+
warmPieceSearch() {
|
|
11697
|
+
warmSearchIndex();
|
|
11698
|
+
}
|
|
11699
|
+
async searchPieces(query, filter) {
|
|
10339
11700
|
let local;
|
|
10340
11701
|
try {
|
|
10341
|
-
local =
|
|
11702
|
+
local = indexPieces((await this.localPieces()).map(({ piece, descriptor }) => localSearchPiece(descriptor, piece.name, piece.version)));
|
|
10342
11703
|
} catch (error) {
|
|
10343
11704
|
this.logger.warn("Could not index the package pieces: @error", String(error));
|
|
10344
11705
|
}
|
|
10345
|
-
return
|
|
11706
|
+
return searchPieces(query, filter, local);
|
|
10346
11707
|
}
|
|
10347
11708
|
async pieceDetail(packageName, version) {
|
|
10348
11709
|
const local = await this.localPieceAt(packageName, version);
|
|
@@ -10447,7 +11808,7 @@ var WorkflowRuntimeService = class {
|
|
|
10447
11808
|
if (connectionId) auth = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolve(connectionId, { piecePackage: parsed.packageName });
|
|
10448
11809
|
const piece = await pieceResolver().resolve(targetOf(parsed));
|
|
10449
11810
|
const reactor = await this.designReactorAccess(parsed, ctx, reactorConnectionId);
|
|
10450
|
-
this.designWorker ??=
|
|
11811
|
+
this.designWorker ??= this.createDesignWorker();
|
|
10451
11812
|
return (await this.designWorker.resolveOptions({
|
|
10452
11813
|
...pieceModuleRef(piece),
|
|
10453
11814
|
actionName: parsed.name,
|
|
@@ -10618,7 +11979,7 @@ var WorkflowRuntimeService = class {
|
|
|
10618
11979
|
} catch (error) {
|
|
10619
11980
|
const errorName = errorNameOf(error);
|
|
10620
11981
|
await recordTest({
|
|
10621
|
-
status: "FAILED",
|
|
11982
|
+
status: isIndeterminateError(error) ? "INDETERMINATE" : "FAILED",
|
|
10622
11983
|
error: pieceFailureDetail(error, "Trigger test timed out"),
|
|
10623
11984
|
...errorName ? { errorName } : {}
|
|
10624
11985
|
});
|
|
@@ -10709,10 +12070,11 @@ var WorkflowRuntimeService = class {
|
|
|
10709
12070
|
triggerKind: TEST_TRIGGER_KIND
|
|
10710
12071
|
});
|
|
10711
12072
|
await store.recordStep(runId, 0, record);
|
|
12073
|
+
const error = record.status === "INDETERMINATE" ? `This test is INDETERMINATE: ${record.error ?? "a host call it made timed out"}` : record.error;
|
|
10712
12074
|
await store.finishRun(runId, {
|
|
10713
|
-
status: record.status === "
|
|
12075
|
+
status: record.status === "SUCCEEDED" ? "SUCCEEDED" : "FAILED",
|
|
10714
12076
|
steps: [record],
|
|
10715
|
-
...
|
|
12077
|
+
...error ? { error } : {},
|
|
10716
12078
|
...record.errorName ? { errorName: record.errorName } : {}
|
|
10717
12079
|
});
|
|
10718
12080
|
} catch (error) {
|
|
@@ -10733,30 +12095,77 @@ var WorkflowRuntimeService = class {
|
|
|
10733
12095
|
}
|
|
10734
12096
|
}
|
|
10735
12097
|
blockExecutor(store) {
|
|
10736
|
-
return this.executor ??= createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request));
|
|
12098
|
+
return this.executor ??= tracedSteps(createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request), this.telemetry), this.telemetry);
|
|
10737
12099
|
}
|
|
10738
12100
|
workers() {
|
|
10739
|
-
|
|
12101
|
+
if (this.pieceWorkers) return this.pieceWorkers;
|
|
12102
|
+
const pool = new PieceWorkerPool({
|
|
10740
12103
|
models: this.models,
|
|
10741
12104
|
size: Number(process.env.PH_WORKFLOWS_RUN_CONCURRENCY) || void 0,
|
|
10742
|
-
maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0
|
|
12105
|
+
maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0,
|
|
12106
|
+
telemetry: this.telemetry
|
|
10743
12107
|
});
|
|
12108
|
+
this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
|
|
12109
|
+
return this.pieceWorkers = pool;
|
|
10744
12110
|
}
|
|
10745
12111
|
async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
|
|
12112
|
+
if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
|
|
12113
|
+
const started = performance.now();
|
|
12114
|
+
const attributes = {
|
|
12115
|
+
"workflow.id": workflowId,
|
|
12116
|
+
"trigger.kind": triggerKind,
|
|
12117
|
+
...resume ? { "run.rerun_of": resume.rerunOf } : {}
|
|
12118
|
+
};
|
|
12119
|
+
return this.telemetry.span("workflow.run", attributes, async (span) => {
|
|
12120
|
+
const run = {
|
|
12121
|
+
span,
|
|
12122
|
+
refused: false
|
|
12123
|
+
};
|
|
12124
|
+
let status = "FAILED";
|
|
12125
|
+
try {
|
|
12126
|
+
const result = await this.runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId);
|
|
12127
|
+
status = result.status;
|
|
12128
|
+
if (status === "FAILED") span.setStatus({
|
|
12129
|
+
code: SpanStatusCode.ERROR,
|
|
12130
|
+
message: status
|
|
12131
|
+
});
|
|
12132
|
+
return result;
|
|
12133
|
+
} finally {
|
|
12134
|
+
if (run.refused) status = "REFUSED";
|
|
12135
|
+
span.setAttribute("run.status", status);
|
|
12136
|
+
this.telemetry.recordRun(performance.now() - started, status, triggerKind);
|
|
12137
|
+
}
|
|
12138
|
+
});
|
|
12139
|
+
}
|
|
12140
|
+
async runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId) {
|
|
12141
|
+
const firedAt = Date.now();
|
|
10746
12142
|
const store = await this.store();
|
|
10747
12143
|
let state;
|
|
10748
12144
|
let documentName;
|
|
10749
12145
|
let definition;
|
|
10750
12146
|
let runUser;
|
|
10751
12147
|
try {
|
|
10752
|
-
if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx)
|
|
10753
|
-
|
|
10754
|
-
|
|
12148
|
+
if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12149
|
+
run.refused = true;
|
|
12150
|
+
throw error;
|
|
12151
|
+
});
|
|
12152
|
+
const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
|
|
12153
|
+
if (document.header.documentType !== "powerhouse/workflow") {
|
|
12154
|
+
run.refused = true;
|
|
12155
|
+
throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
|
|
12156
|
+
}
|
|
10755
12157
|
state = document.state.global;
|
|
10756
12158
|
documentName = document.header.name;
|
|
10757
|
-
if (state.status !== "ENABLED")
|
|
12159
|
+
if (state.status !== "ENABLED") {
|
|
12160
|
+
run.refused = true;
|
|
12161
|
+
throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
|
|
12162
|
+
}
|
|
10758
12163
|
definition = toWorkflowDefinition(state);
|
|
10759
|
-
|
|
12164
|
+
const reactorConnections = declaredReactorConnectionIds(definition);
|
|
12165
|
+
runUser = reactorConnections.size === 0 ? void 0 : await this.telemetry.phase("run_user", {}, () => assertReactorConnectionsReadable(this.host, workflowId, reactorConnections)).catch((error) => {
|
|
12166
|
+
run.refused = isReactorError(error, ReactorAccessDeniedError$1);
|
|
12167
|
+
throw error;
|
|
12168
|
+
});
|
|
10760
12169
|
} catch (error) {
|
|
10761
12170
|
if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
|
|
10762
12171
|
throw error;
|
|
@@ -10764,23 +12173,71 @@ var WorkflowRuntimeService = class {
|
|
|
10764
12173
|
const connections = declaredConnectionIds(definition);
|
|
10765
12174
|
const executor = this.blockExecutor(store);
|
|
10766
12175
|
const runnable = runnableDefinition(state);
|
|
10767
|
-
|
|
10768
|
-
|
|
10769
|
-
|
|
10770
|
-
|
|
10771
|
-
});
|
|
10772
|
-
else runId = await store?.startRun({
|
|
10773
|
-
workflowId,
|
|
10774
|
-
workflowName: runJournalName(state.name, documentName),
|
|
10775
|
-
workflowVersion: runnable.version,
|
|
12176
|
+
const policy = effectiveRunPolicy(runnable);
|
|
12177
|
+
const deadline = policy.runTimeoutSeconds ? firedAt + policy.runTimeoutSeconds * 1e3 : void 0;
|
|
12178
|
+
const skipped = (reason, refusal) => this.skipFiring(store, workflowId, enqueuedRunId, {
|
|
12179
|
+
reason,
|
|
10776
12180
|
triggerKind,
|
|
10777
12181
|
triggerPayload,
|
|
10778
|
-
|
|
10779
|
-
|
|
12182
|
+
workflowName: runJournalName(state.name, documentName),
|
|
12183
|
+
workflowVersion: runnable.version
|
|
12184
|
+
}, refusal);
|
|
12185
|
+
let parked;
|
|
12186
|
+
try {
|
|
12187
|
+
parked = await this.parkedFiring(workflowId, triggerKind, runnable.version);
|
|
12188
|
+
} catch (error) {
|
|
12189
|
+
if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
|
|
12190
|
+
throw error;
|
|
12191
|
+
}
|
|
12192
|
+
if (parked) return skipped(parked, "parked");
|
|
12193
|
+
const admission = await this.runGate.admit(workflowId, policy);
|
|
12194
|
+
if (this.closed) {
|
|
12195
|
+
if (admission.admitted) admission.release();
|
|
12196
|
+
return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12197
|
+
}
|
|
12198
|
+
if (!admission.admitted) {
|
|
12199
|
+
if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12200
|
+
return skipped(admission.reason, admission.refusal);
|
|
12201
|
+
}
|
|
12202
|
+
if (admission.waited) {
|
|
12203
|
+
let stale;
|
|
12204
|
+
try {
|
|
12205
|
+
stale = await this.staleAfterWait(workflowId, triggerKind, runnable.version);
|
|
12206
|
+
} catch (error) {
|
|
12207
|
+
stale = `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
|
|
12208
|
+
}
|
|
12209
|
+
if (stale) {
|
|
12210
|
+
admission.release();
|
|
12211
|
+
return skipped(stale, "stale");
|
|
12212
|
+
}
|
|
12213
|
+
}
|
|
12214
|
+
if (deadline !== void 0 && Date.now() >= deadline) {
|
|
12215
|
+
admission.release();
|
|
12216
|
+
return skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
|
|
12217
|
+
}
|
|
12218
|
+
let runId = enqueuedRunId ?? null;
|
|
10780
12219
|
let journalFailed = false;
|
|
10781
12220
|
const executionOrder = /* @__PURE__ */ new Map();
|
|
10782
12221
|
let session;
|
|
10783
12222
|
try {
|
|
12223
|
+
runId = await this.telemetry.phase("journal.start", {}, async () => {
|
|
12224
|
+
if (enqueuedRunId) {
|
|
12225
|
+
await store?.beginRun(enqueuedRunId, {
|
|
12226
|
+
workflowName: runJournalName(state.name, documentName),
|
|
12227
|
+
workflowVersion: runnable.version
|
|
12228
|
+
});
|
|
12229
|
+
return enqueuedRunId;
|
|
12230
|
+
}
|
|
12231
|
+
return await store?.startRun({
|
|
12232
|
+
workflowId,
|
|
12233
|
+
workflowName: runJournalName(state.name, documentName),
|
|
12234
|
+
workflowVersion: runnable.version,
|
|
12235
|
+
triggerKind,
|
|
12236
|
+
triggerPayload,
|
|
12237
|
+
rerunOf: resume?.rerunOf
|
|
12238
|
+
}) ?? null;
|
|
12239
|
+
});
|
|
12240
|
+
if (runId) run.span.setAttribute("run.id", runId);
|
|
10784
12241
|
session = this.workers().session();
|
|
10785
12242
|
const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
|
|
10786
12243
|
const journal = store;
|
|
@@ -10792,6 +12249,7 @@ var WorkflowRuntimeService = class {
|
|
|
10792
12249
|
connections,
|
|
10793
12250
|
pieceWorker: session,
|
|
10794
12251
|
...runUser !== void 0 ? { runUser } : {},
|
|
12252
|
+
attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
|
|
10795
12253
|
recordDocuments: async (documentIds) => {
|
|
10796
12254
|
for (const documentId of documentIds) handed.add(documentId);
|
|
10797
12255
|
if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
|
|
@@ -10806,22 +12264,25 @@ var WorkflowRuntimeService = class {
|
|
|
10806
12264
|
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
10807
12265
|
triggerPayload,
|
|
10808
12266
|
completedSteps: resume?.completedSteps,
|
|
10809
|
-
|
|
12267
|
+
...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
|
|
12268
|
+
...deadline !== void 0 ? { deadline } : {},
|
|
12269
|
+
onStep: store && journaledRunId ? async (record, ordinal) => {
|
|
10810
12270
|
executionOrder.set(record.stepId, ordinal);
|
|
10811
12271
|
try {
|
|
10812
|
-
await store.recordStep(
|
|
12272
|
+
await this.telemetry.phase("journal.step", {}, () => store.recordStep(journaledRunId, ordinal, record));
|
|
10813
12273
|
} catch (error) {
|
|
10814
12274
|
if (journalFailed) return;
|
|
10815
12275
|
journalFailed = true;
|
|
10816
|
-
this.logger.warn(`Run ${
|
|
12276
|
+
this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
10817
12277
|
}
|
|
10818
12278
|
} : void 0
|
|
10819
12279
|
}));
|
|
10820
|
-
if (store &&
|
|
10821
|
-
await store.finishRun(
|
|
12280
|
+
if (store && journaledRunId) try {
|
|
12281
|
+
await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
|
|
10822
12282
|
} catch (error) {
|
|
10823
12283
|
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
10824
12284
|
}
|
|
12285
|
+
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
10825
12286
|
const finished = {
|
|
10826
12287
|
...result,
|
|
10827
12288
|
runId
|
|
@@ -10838,6 +12299,123 @@ var WorkflowRuntimeService = class {
|
|
|
10838
12299
|
throw error;
|
|
10839
12300
|
} finally {
|
|
10840
12301
|
session?.close();
|
|
12302
|
+
admission.release();
|
|
12303
|
+
}
|
|
12304
|
+
}
|
|
12305
|
+
runGate = new WorkflowRunGate();
|
|
12306
|
+
heldReplies = /* @__PURE__ */ new Map();
|
|
12307
|
+
/** Why a trigger's firing of a PARKED workflow is refused; undefined when
|
|
12308
|
+
* it is not parked, or when an operator started the run. */
|
|
12309
|
+
async parkedFiring(workflowId, triggerKind, publishedVersion) {
|
|
12310
|
+
if (OPERATOR_RUN_KINDS.has(triggerKind)) return void 0;
|
|
12311
|
+
const park = await this.parks.get(workflowId);
|
|
12312
|
+
if (!park || park.published_version < publishedVersion) return void 0;
|
|
12313
|
+
return `Skipped: this workflow is PARKED (${park.reason})`;
|
|
12314
|
+
}
|
|
12315
|
+
/** Why a firing that waited for its slot may no longer run, if it may not. */
|
|
12316
|
+
async staleAfterWait(workflowId, triggerKind, publishedVersion) {
|
|
12317
|
+
const state = (await this.host.reactorClient.get(workflowId)).state.global;
|
|
12318
|
+
if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
|
|
12319
|
+
if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
|
|
12320
|
+
return this.parkedFiring(workflowId, triggerKind, publishedVersion);
|
|
12321
|
+
}
|
|
12322
|
+
/** Whether the workflow is still ENABLED at this published version. An
|
|
12323
|
+
* unreadable document counts as unchanged, so a failure still parks. */
|
|
12324
|
+
async stillRunsVersion(workflowId, publishedVersion) {
|
|
12325
|
+
let state;
|
|
12326
|
+
try {
|
|
12327
|
+
state = (await this.host.reactorClient.get(workflowId)).state.global;
|
|
12328
|
+
} catch (error) {
|
|
12329
|
+
this.logger.warn(`Could not re-read workflow ${workflowId} before parking it: @error`, error);
|
|
12330
|
+
return true;
|
|
12331
|
+
}
|
|
12332
|
+
return state.status === "ENABLED" && runnableDefinition(state).version === publishedVersion;
|
|
12333
|
+
}
|
|
12334
|
+
/** A park recorded against an earlier published version than this state's. */
|
|
12335
|
+
async outdatedPark(workflowId, state) {
|
|
12336
|
+
if (state.status !== "ENABLED") return false;
|
|
12337
|
+
const park = await this.parks.get(workflowId);
|
|
12338
|
+
return park !== void 0 && runnableDefinition(state).version > park.published_version;
|
|
12339
|
+
}
|
|
12340
|
+
/**
|
|
12341
|
+
* A firing SINGLETON refused.
|
|
12342
|
+
*
|
|
12343
|
+
* Journaled as a CANCELLED run rather than dropped: a firing that vanished
|
|
12344
|
+
* is indistinguishable from a trigger that never fired, which is the class
|
|
12345
|
+
* of bug this work package exists to stamp out. An already-enqueued row is
|
|
12346
|
+
* closed out in place, so nothing is left PENDING for a sweep to find.
|
|
12347
|
+
*/
|
|
12348
|
+
async refuseClosed(store, workflowId, enqueuedRunId) {
|
|
12349
|
+
const error = new WorkflowRuntimeClosedError(workflowId);
|
|
12350
|
+
if (enqueuedRunId) {
|
|
12351
|
+
const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
|
|
12352
|
+
try {
|
|
12353
|
+
if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
|
|
12354
|
+
else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));
|
|
12355
|
+
} catch (failure) {
|
|
12356
|
+
this.logger.warn(`Could not close out run ${enqueuedRunId} after shutdown: @error`, failure);
|
|
12357
|
+
}
|
|
12358
|
+
}
|
|
12359
|
+
throw error;
|
|
12360
|
+
}
|
|
12361
|
+
async skipFiring(store, workflowId, enqueuedRunId, details, refusal) {
|
|
12362
|
+
this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
|
|
12363
|
+
let runId = enqueuedRunId ?? null;
|
|
12364
|
+
if (store) try {
|
|
12365
|
+
if (runId) await store.beginRun(runId, details);
|
|
12366
|
+
else runId = await store.startRun({
|
|
12367
|
+
workflowId,
|
|
12368
|
+
...details
|
|
12369
|
+
});
|
|
12370
|
+
await store.cancelRun(runId, details.reason);
|
|
12371
|
+
} catch (error) {
|
|
12372
|
+
this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
|
|
12373
|
+
}
|
|
12374
|
+
return {
|
|
12375
|
+
status: CANCELLED_RUN_STATUS,
|
|
12376
|
+
steps: [],
|
|
12377
|
+
runId,
|
|
12378
|
+
refusal
|
|
12379
|
+
};
|
|
12380
|
+
}
|
|
12381
|
+
/**
|
|
12382
|
+
* `policy.onFailure` for a run that failed terminally.
|
|
12383
|
+
*
|
|
12384
|
+
* PARK is the document model's own default, so this is where enforcing the
|
|
12385
|
+
* knob becomes visible: a terminal failure parks the workflow, whatever its
|
|
12386
|
+
* trigger kind, and nothing fires it until it is re-published or
|
|
12387
|
+
* re-enabled. That is what PARK means, and
|
|
12388
|
+
* leaving a broken workflow firing every minute is what it meant before.
|
|
12389
|
+
*
|
|
12390
|
+
* NOTIFY logs at error level, which is the only notification channel this
|
|
12391
|
+
* engine has; it is marked as such rather than pretending to page anyone.
|
|
12392
|
+
*/
|
|
12393
|
+
async applyFailureMode(policy, workflowId, publishedVersion, runId, error) {
|
|
12394
|
+
if (policy.onFailure === "IGNORE" || this.closed) return;
|
|
12395
|
+
const detail = error ?? "the run failed";
|
|
12396
|
+
if (policy.onFailure === "NOTIFY") {
|
|
12397
|
+
this.logger.error(`Workflow ${workflowId} run ${runId ?? "(unjournaled)"} failed and its policy is NOTIFY: @error`, detail);
|
|
12398
|
+
return;
|
|
12399
|
+
}
|
|
12400
|
+
if (!await this.store()) return;
|
|
12401
|
+
const reason = `Parked after a failed run (policy.onFailure = PARK): ${detail}`;
|
|
12402
|
+
try {
|
|
12403
|
+
if (!await this.stillRunsVersion(workflowId, publishedVersion)) return;
|
|
12404
|
+
if (this.closed) return;
|
|
12405
|
+
const trigger = await this.supervisor().park(workflowId, publishedVersion, reason);
|
|
12406
|
+
if (!await this.stillRunsVersion(workflowId, publishedVersion)) {
|
|
12407
|
+
await this.supervisor().liftPark(workflowId, publishedVersion, trigger);
|
|
12408
|
+
return;
|
|
12409
|
+
}
|
|
12410
|
+
if (!await this.inRegistrationOrder(workflowId, async () => {
|
|
12411
|
+
const current = (await this.parks.get(workflowId))?.published_version === publishedVersion;
|
|
12412
|
+
const registered = this.registry.get(workflowId);
|
|
12413
|
+
if (current && registered && !SUPERVISED_KINDS.has(registered.kind)) this.registry.delete(workflowId);
|
|
12414
|
+
return current;
|
|
12415
|
+
})) return;
|
|
12416
|
+
this.logger.error(`Workflow ${workflowId} is PARKED after run ${runId ?? "(unjournaled)"} failed; its trigger will not fire again until the workflow is re-published or re-enabled`);
|
|
12417
|
+
} catch (parkError) {
|
|
12418
|
+
this.logger.warn(`Applying onFailure = PARK to workflow ${workflowId} after a failed run failed; whether it is parked is unknown: @error`, parkError);
|
|
10841
12419
|
}
|
|
10842
12420
|
}
|
|
10843
12421
|
async rerun(runId, ctx) {
|
|
@@ -10848,7 +12426,8 @@ var WorkflowRuntimeService = class {
|
|
|
10848
12426
|
await this.assertCanReadDocument(run.workflow_id, ctx);
|
|
10849
12427
|
if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
|
|
10850
12428
|
if (run.trigger_kind === "test") throw new Error(`Run "${runId}" tested a single step or trigger; test it again instead of rerunning it`);
|
|
10851
|
-
|
|
12429
|
+
const deadlineCancelled = run.status === "CANCELLED" && run.error_name === "RunDeadlineExceeded";
|
|
12430
|
+
if (run.status !== "FAILED" && !deadlineCancelled) throw new Error(`Only FAILED runs, and runs their deadline cancelled, can be rerun; run is ${run.status}`);
|
|
10852
12431
|
const triggerPayload = run.trigger_payload === null ? void 0 : JSON.parse(run.trigger_payload);
|
|
10853
12432
|
if (isTruncatedStepPayload(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was truncated by the journal and cannot be replayed; fire the workflow again instead of rerunning it`);
|
|
10854
12433
|
if (containsRedactedMarker(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was redacted and cannot be replayed; fire the workflow again instead of rerunning it`);
|
|
@@ -10860,7 +12439,15 @@ var WorkflowRuntimeService = class {
|
|
|
10860
12439
|
const current = currentSteps.get(row.step_id);
|
|
10861
12440
|
if (!current || current.key !== row.step_key || row.config_hash === null || stepConfigHash(current) !== row.config_hash) continue;
|
|
10862
12441
|
const output = row.output === null ? void 0 : JSON.parse(row.output);
|
|
10863
|
-
if (isTruncatedStepPayload(output))
|
|
12442
|
+
if (isTruncatedStepPayload(output)) {
|
|
12443
|
+
if (await this.rereads(current)) continue;
|
|
12444
|
+
completedSteps.set(row.step_id, {
|
|
12445
|
+
port: row.port,
|
|
12446
|
+
outputTruncated: true,
|
|
12447
|
+
truncatedOutput: output
|
|
12448
|
+
});
|
|
12449
|
+
continue;
|
|
12450
|
+
}
|
|
10864
12451
|
if (containsDocumentRef(output) && await this.rereads(current)) continue;
|
|
10865
12452
|
completedSteps.set(row.step_id, {
|
|
10866
12453
|
output,
|
|
@@ -10947,6 +12534,7 @@ var WorkflowRuntimeService = class {
|
|
|
10947
12534
|
pieceWorker: session,
|
|
10948
12535
|
stepTest: true,
|
|
10949
12536
|
runUser: this.callerRunUser(ctx),
|
|
12537
|
+
attachments: attachmentOrigins(upstream.triggerPayload, handed, upstream.priorSteps),
|
|
10950
12538
|
recordDocuments: async (documentIds) => {
|
|
10951
12539
|
for (const documentId of documentIds) handed.add(documentId);
|
|
10952
12540
|
if (store && journaledRunId) await store.recordRunDocuments(journaledRunId, documentIds);
|
|
@@ -10986,7 +12574,7 @@ var WorkflowRuntimeService = class {
|
|
|
10986
12574
|
const served = ctx ? await this.servesDocuments(ids, ctx) : false;
|
|
10987
12575
|
return {
|
|
10988
12576
|
runId,
|
|
10989
|
-
status: record.status
|
|
12577
|
+
status: testStatusOf(record.status),
|
|
10990
12578
|
...served && record.output !== void 0 ? { output: record.output } : {},
|
|
10991
12579
|
...record.error ? { error: record.error } : {},
|
|
10992
12580
|
...record.errorName ? { errorName: record.errorName } : {},
|
|
@@ -11114,6 +12702,7 @@ var WorkflowRuntimeService = class {
|
|
|
11114
12702
|
testedAt,
|
|
11115
12703
|
error: row.error ?? ""
|
|
11116
12704
|
};
|
|
12705
|
+
if (row.status === "INDETERMINATE") return { kind: "indeterminate" };
|
|
11117
12706
|
const output = row.output === null ? null : JSON.parse(row.output);
|
|
11118
12707
|
if (isTruncatedStepPayload(output)) return { kind: "truncated" };
|
|
11119
12708
|
return {
|
|
@@ -11182,6 +12771,264 @@ var WorkflowRuntimeService = class {
|
|
|
11182
12771
|
function createWorkflowRuntime(deps) {
|
|
11183
12772
|
return new WorkflowRuntimeService(deps);
|
|
11184
12773
|
}
|
|
12774
|
+
function attachmentOrigins(triggerPayload, handed, replayed) {
|
|
12775
|
+
const written = /* @__PURE__ */ new Set();
|
|
12776
|
+
collectAttachmentRefs(replayed, written);
|
|
12777
|
+
return {
|
|
12778
|
+
documentIds: () => [...triggerDocumentIds(triggerPayload), ...handed],
|
|
12779
|
+
written
|
|
12780
|
+
};
|
|
12781
|
+
}
|
|
12782
|
+
function collectAttachmentRefs(value, found) {
|
|
12783
|
+
if (typeof value === "string") {
|
|
12784
|
+
if (value.startsWith("attachment://")) found.add(value);
|
|
12785
|
+
} else if (Array.isArray(value)) for (const entry of value) collectAttachmentRefs(entry, found);
|
|
12786
|
+
else if (value instanceof Map) for (const entry of value.values()) collectAttachmentRefs(entry, found);
|
|
12787
|
+
else if (typeof value === "object" && value !== null) for (const entry of Object.values(value)) collectAttachmentRefs(entry, found);
|
|
12788
|
+
}
|
|
12789
|
+
//#endregion
|
|
12790
|
+
//#region src/reactor/singleton-lease.ts
|
|
12791
|
+
const logger = childLogger([
|
|
12792
|
+
"workflow",
|
|
12793
|
+
"runtime",
|
|
12794
|
+
"singleton"
|
|
12795
|
+
]);
|
|
12796
|
+
/**
|
|
12797
|
+
* Names this process as the workflow singleton's owner.
|
|
12798
|
+
*
|
|
12799
|
+
* Set it to a STABLE name per deployment slot (`switchboard-0`, the
|
|
12800
|
+
* StatefulSet pod name, the Render service id). The owner name is the
|
|
12801
|
+
* operator's contract: a restart under the same name re-claims its own lease
|
|
12802
|
+
* once the old heartbeat is stale instead of waiting out
|
|
12803
|
+
* {@link SINGLETON_LEASE_TTL_MS}.
|
|
12804
|
+
*
|
|
12805
|
+
* Unset, the owner is derived from a STABLE identity — the hostname and a
|
|
12806
|
+
* fingerprint of the journal's own storage location ({@link
|
|
12807
|
+
* AcquireSingletonOptions.storageId}) — not from the pid and a random suffix.
|
|
12808
|
+
* A random per-process owner meant every unclean kill locked the next boot out
|
|
12809
|
+
* for the whole {@link SINGLETON_LEASE_TTL_MS}: the dead process's lease was
|
|
12810
|
+
* nobody's to re-claim, and `release()` never ran. One deployment slot
|
|
12811
|
+
* restarting is the common case and it has to be quick; a genuine second
|
|
12812
|
+
* replica still differs, by hostname or by the journal it points at.
|
|
12813
|
+
*/
|
|
12814
|
+
const WORKFLOW_SINGLETON_OWNER_ENV = "PH_WORKFLOWS_SINGLETON_OWNER";
|
|
12815
|
+
/** How long a claim stays valid without a heartbeat. */
|
|
12816
|
+
const SINGLETON_LEASE_TTL_MS = 6e4;
|
|
12817
|
+
/** How often the holder renews it; a third of the TTL, so two renewals may be
|
|
12818
|
+
* lost (a GC pause, a slow database) before anyone may take over. */
|
|
12819
|
+
const SINGLETON_HEARTBEAT_MS = SINGLETON_LEASE_TTL_MS / 3;
|
|
12820
|
+
/**
|
|
12821
|
+
* How long the holder's heartbeat must have been silent before a claim under
|
|
12822
|
+
* the SAME owner name takes the lease over, short of its expiry. Two renewal
|
|
12823
|
+
* periods: a live holder renews well inside it, and its self-fence fires
|
|
12824
|
+
* before it passes.
|
|
12825
|
+
*/
|
|
12826
|
+
const SINGLETON_STALE_HEARTBEATS = 2;
|
|
12827
|
+
/** The one row. The table holds a single lease, named rather than keyed by a
|
|
12828
|
+
* magic empty string, so a `SELECT *` reads legibly in an operator's shell. */
|
|
12829
|
+
const LEASE_ID = "workflow-runtime";
|
|
12830
|
+
const dbNow = () => sql`now()`;
|
|
12831
|
+
const dbNowPlus = (ms) => sql`now() + ${ms} * interval '1 millisecond'`;
|
|
12832
|
+
/** Another live process holds the singleton. The refusal is the point: a host
|
|
12833
|
+
* that cannot own workflow execution must not compose the engine. */
|
|
12834
|
+
var WorkflowSingletonConflictError = class extends Error {
|
|
12835
|
+
owner;
|
|
12836
|
+
expiresAt;
|
|
12837
|
+
wouldBe;
|
|
12838
|
+
constructor(owner, expiresAt, wouldBe) {
|
|
12839
|
+
super(`Workflow execution is a singleton and "${owner}" holds it until ${expiresAt}; this process ("${wouldBe}") must not compose the workflow runtime. Two replicas over one run journal fail each other's live runs and arm every trigger twice. Run workflows on exactly one reactor, or set ${WORKFLOW_SINGLETON_OWNER_ENV} to the same stable owner name on the slot that owns them so a restart re-claims its own lease.`);
|
|
12840
|
+
this.owner = owner;
|
|
12841
|
+
this.expiresAt = expiresAt;
|
|
12842
|
+
this.wouldBe = wouldBe;
|
|
12843
|
+
this.name = "WorkflowSingletonConflictError";
|
|
12844
|
+
}
|
|
12845
|
+
};
|
|
12846
|
+
/**
|
|
12847
|
+
* The owner name this process claims under.
|
|
12848
|
+
*
|
|
12849
|
+
* `<hostname>/<storage fingerprint>`, so it is the same name on every boot of
|
|
12850
|
+
* one deployment slot — a restart re-claims its OWN lease once the dead
|
|
12851
|
+
* process's heartbeat is stale, rather than waiting out the TTL — and a different
|
|
12852
|
+
* name on any host or journal that is genuinely somebody else.
|
|
12853
|
+
*
|
|
12854
|
+
* The storage location is hashed, never printed: it can be a Postgres URL with
|
|
12855
|
+
* credentials in it, and the owner name goes into a database row an operator
|
|
12856
|
+
* reads and into every log line about the lease.
|
|
12857
|
+
*
|
|
12858
|
+
* The residual case a stable name cannot tell apart is two processes on ONE
|
|
12859
|
+
* host over ONE journal, which is a misconfiguration those two already share
|
|
12860
|
+
* (one read-model directory, one run journal). It is not silent: the loser's
|
|
12861
|
+
* heartbeat finds the lease taken and says so by name.
|
|
12862
|
+
*/
|
|
12863
|
+
function singletonOwnerName(env = process.env, storageId) {
|
|
12864
|
+
const configured = env[WORKFLOW_SINGLETON_OWNER_ENV]?.trim();
|
|
12865
|
+
if (configured) return configured;
|
|
12866
|
+
return `${hostname()}/${storageFingerprint(storageId)}`;
|
|
12867
|
+
}
|
|
12868
|
+
function storageFingerprint(storageId) {
|
|
12869
|
+
const source = storageId?.trim();
|
|
12870
|
+
if (!source) return "default";
|
|
12871
|
+
return createHash("sha256").update(source).digest("hex").slice(0, 12);
|
|
12872
|
+
}
|
|
12873
|
+
async function ensureTable(db) {
|
|
12874
|
+
await db.schema.createTable("workflow_singleton").addColumn("id", "text", (col) => col.primaryKey()).addColumn("owner", "text", (col) => col.notNull()).addColumn("instance", "text", (col) => col.notNull()).addColumn("acquired_at", "timestamptz", (col) => col.notNull()).addColumn("heartbeat_at", "timestamptz", (col) => col.notNull()).addColumn("expires_at", "timestamptz", (col) => col.notNull()).ifNotExists().execute();
|
|
12875
|
+
}
|
|
12876
|
+
const DDL_ATTEMPTS = 3;
|
|
12877
|
+
const CONCURRENT_DDL_CODES = /* @__PURE__ */ new Set([
|
|
12878
|
+
"23505",
|
|
12879
|
+
"42710",
|
|
12880
|
+
"42P06",
|
|
12881
|
+
"42P07"
|
|
12882
|
+
]);
|
|
12883
|
+
function isConcurrentDdl(error) {
|
|
12884
|
+
const code = error?.code;
|
|
12885
|
+
return typeof code === "string" && CONCURRENT_DDL_CODES.has(code);
|
|
12886
|
+
}
|
|
12887
|
+
async function openLeaseTable(relationalDb) {
|
|
12888
|
+
const open = async () => {
|
|
12889
|
+
const db = await relationalDb.createNamespace("workflow_runtime");
|
|
12890
|
+
await ensureTable(db);
|
|
12891
|
+
return db;
|
|
12892
|
+
};
|
|
12893
|
+
for (let attempt = 1;; attempt++) try {
|
|
12894
|
+
return await open();
|
|
12895
|
+
} catch (error) {
|
|
12896
|
+
if (attempt >= DDL_ATTEMPTS || !isConcurrentDdl(error)) throw error;
|
|
12897
|
+
}
|
|
12898
|
+
}
|
|
12899
|
+
/**
|
|
12900
|
+
* Claims the workflow singleton, or refuses.
|
|
12901
|
+
*
|
|
12902
|
+
* Three steps, each one a single statement whose own outcome is the claim — a
|
|
12903
|
+
* prior `SELECT` would be a race:
|
|
12904
|
+
*
|
|
12905
|
+
* 1. `INSERT … ON CONFLICT DO NOTHING RETURNING` — the first ever boot.
|
|
12906
|
+
* 2. `UPDATE … WHERE expires_at <= now OR (owner = me AND heartbeat stale)
|
|
12907
|
+
* RETURNING` — a dead lease taken over once it has expired, or our own
|
|
12908
|
+
* slot's lease once its holder stopped renewing (a restart under a stable
|
|
12909
|
+
* owner name), never a live holder's. Either way the row gets this claim's
|
|
12910
|
+
* instance, so the previous holder's heartbeat and release no longer
|
|
12911
|
+
* match.
|
|
12912
|
+
* 3. Neither matched: someone live holds it, and we refuse by name.
|
|
12913
|
+
*/
|
|
12914
|
+
async function acquireWorkflowSingletonLease(options) {
|
|
12915
|
+
const log = options.logger ?? logger;
|
|
12916
|
+
const ttlMs = options.ttlMs ?? 6e4;
|
|
12917
|
+
const heartbeatMs = options.heartbeatMs ?? 2e4;
|
|
12918
|
+
const staleMs = heartbeatMs * 2;
|
|
12919
|
+
const owner = options.owner ?? singletonOwnerName(options.env, options.storageId);
|
|
12920
|
+
const instance = randomUUID();
|
|
12921
|
+
const db = await openLeaseTable(options.relationalDb);
|
|
12922
|
+
const stamps = () => ({
|
|
12923
|
+
heartbeat_at: dbNow(),
|
|
12924
|
+
expires_at: dbNowPlus(ttlMs)
|
|
12925
|
+
});
|
|
12926
|
+
const claim = async () => {
|
|
12927
|
+
if (await db.insertInto("workflow_singleton").values({
|
|
12928
|
+
id: LEASE_ID,
|
|
12929
|
+
owner,
|
|
12930
|
+
instance,
|
|
12931
|
+
acquired_at: dbNow(),
|
|
12932
|
+
...stamps()
|
|
12933
|
+
}).onConflict((oc) => oc.column("id").doNothing()).returning("owner").executeTakeFirst() !== void 0) return true;
|
|
12934
|
+
return await db.updateTable("workflow_singleton").set({
|
|
12935
|
+
owner,
|
|
12936
|
+
instance,
|
|
12937
|
+
acquired_at: dbNow(),
|
|
12938
|
+
...stamps()
|
|
12939
|
+
}).where("id", "=", LEASE_ID).where((eb) => eb.or([eb("expires_at", "<=", dbNow()), eb.and([eb("owner", "=", owner), eb("heartbeat_at", "<=", dbNowPlus(-staleMs))])])).returning("owner").executeTakeFirst() !== void 0;
|
|
12940
|
+
};
|
|
12941
|
+
const claimSentAt = performance.now();
|
|
12942
|
+
if (!await claim()) {
|
|
12943
|
+
const held = await db.selectFrom("workflow_singleton").selectAll().where("id", "=", LEASE_ID).executeTakeFirst();
|
|
12944
|
+
throw new WorkflowSingletonConflictError(held?.owner ?? "unknown", held ? new Date(held.expires_at).toISOString() : "unknown", owner);
|
|
12945
|
+
}
|
|
12946
|
+
log.info(`Workflow singleton claimed by "${owner}" (lease ${ttlMs}ms, renewed every ${heartbeatMs}ms)`);
|
|
12947
|
+
const localNow = () => performance.now();
|
|
12948
|
+
const tickMs = heartbeatMs / 4;
|
|
12949
|
+
const fenceMs = Math.min(staleMs, ttlMs) - Math.min(heartbeatMs, ttlMs) / 2;
|
|
12950
|
+
const selfFence = options.selfFence ?? true;
|
|
12951
|
+
let renewedAt = claimSentAt;
|
|
12952
|
+
let renewFailed = false;
|
|
12953
|
+
let inFlight;
|
|
12954
|
+
let timer;
|
|
12955
|
+
let lost = false;
|
|
12956
|
+
let takenOver = false;
|
|
12957
|
+
let releasing = false;
|
|
12958
|
+
const stop = () => {
|
|
12959
|
+
if (timer) clearInterval(timer);
|
|
12960
|
+
timer = void 0;
|
|
12961
|
+
};
|
|
12962
|
+
const markLost = (loss) => {
|
|
12963
|
+
if (lost) return;
|
|
12964
|
+
lost = true;
|
|
12965
|
+
stop();
|
|
12966
|
+
log.error(loss.reason === "taken" ? `Workflow singleton lease of "${owner}" was taken over by "${loss.heldBy ?? "nobody"}"; this process no longer owns workflow execution and stops running it.` : `Workflow singleton lease of "${owner}" could not be renewed for ${Math.round(loss.silentMs)}ms, long enough that another process could take it over (none was seen to); this process stops running workflows.`);
|
|
12967
|
+
options.onLost?.(loss);
|
|
12968
|
+
};
|
|
12969
|
+
const renew = async () => {
|
|
12970
|
+
const sentAt = localNow();
|
|
12971
|
+
if (await db.updateTable("workflow_singleton").set(stamps()).where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).returning("owner").executeTakeFirst() !== void 0) {
|
|
12972
|
+
renewedAt = sentAt;
|
|
12973
|
+
return true;
|
|
12974
|
+
}
|
|
12975
|
+
if (releasing || lost) return false;
|
|
12976
|
+
let heldBy;
|
|
12977
|
+
try {
|
|
12978
|
+
heldBy = (await db.selectFrom("workflow_singleton").select("owner").where("id", "=", LEASE_ID).executeTakeFirst())?.owner;
|
|
12979
|
+
} catch {
|
|
12980
|
+
heldBy = void 0;
|
|
12981
|
+
}
|
|
12982
|
+
takenOver = true;
|
|
12983
|
+
markLost({
|
|
12984
|
+
reason: "taken",
|
|
12985
|
+
heldBy
|
|
12986
|
+
});
|
|
12987
|
+
return false;
|
|
12988
|
+
};
|
|
12989
|
+
const heartbeat = () => {
|
|
12990
|
+
if (lost) return Promise.resolve(false);
|
|
12991
|
+
inFlight ??= renew().finally(() => {
|
|
12992
|
+
inFlight = void 0;
|
|
12993
|
+
});
|
|
12994
|
+
return inFlight;
|
|
12995
|
+
};
|
|
12996
|
+
timer = setInterval(() => {
|
|
12997
|
+
if (lost || releasing) return;
|
|
12998
|
+
const silentMs = localNow() - renewedAt;
|
|
12999
|
+
if (selfFence && silentMs >= fenceMs) {
|
|
13000
|
+
markLost({
|
|
13001
|
+
reason: "unrenewable",
|
|
13002
|
+
silentMs
|
|
13003
|
+
});
|
|
13004
|
+
return;
|
|
13005
|
+
}
|
|
13006
|
+
if (inFlight) return;
|
|
13007
|
+
if (!renewFailed && silentMs < heartbeatMs - tickMs / 2) return;
|
|
13008
|
+
heartbeat().then(() => {
|
|
13009
|
+
renewFailed = false;
|
|
13010
|
+
}, (error) => {
|
|
13011
|
+
renewFailed = true;
|
|
13012
|
+
log.warn("Workflow singleton heartbeat failed: @error", error);
|
|
13013
|
+
});
|
|
13014
|
+
}, tickMs);
|
|
13015
|
+
timer.unref();
|
|
13016
|
+
return {
|
|
13017
|
+
owner,
|
|
13018
|
+
heartbeat,
|
|
13019
|
+
async release() {
|
|
13020
|
+
releasing = true;
|
|
13021
|
+
stop();
|
|
13022
|
+
if (takenOver) return;
|
|
13023
|
+
if (inFlight) await Promise.race([inFlight.catch(() => false), new Promise((resolve) => setTimeout(resolve, tickMs).unref())]);
|
|
13024
|
+
try {
|
|
13025
|
+
await db.deleteFrom("workflow_singleton").where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).execute();
|
|
13026
|
+
} catch (error) {
|
|
13027
|
+
log.warn("Could not release the workflow singleton lease: @error", error);
|
|
13028
|
+
}
|
|
13029
|
+
}
|
|
13030
|
+
};
|
|
13031
|
+
}
|
|
11185
13032
|
//#endregion
|
|
11186
13033
|
//#region src/reactor/workflow-triggers-read-model.ts
|
|
11187
13034
|
const WORKFLOW_TRIGGERS_READ_MODEL = "workflow-triggers";
|
|
@@ -11209,6 +13056,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
|
|
|
11209
13056
|
}
|
|
11210
13057
|
};
|
|
11211
13058
|
//#endregion
|
|
11212
|
-
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
|
|
13059
|
+
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl, singletonOwnerName };
|
|
11213
13060
|
|
|
11214
13061
|
//# sourceMappingURL=index.js.map
|