@powerhousedao/reactor-workflow 6.2.3-dev.50 → 6.2.3-dev.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -1
- package/dist/index.d.ts +185 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +878 -217
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-Ds8WkewF.d.ts → piece-registry-BdkdVugw.d.ts} +2 -1
- package/dist/{piece-registry-Ds8WkewF.d.ts.map → piece-registry-BdkdVugw.d.ts.map} +1 -1
- package/dist/{piece-registry-BKRLK7Kv.js → piece-registry-D0pa4v0T.js} +37 -15
- package/dist/piece-registry-D0pa4v0T.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +1 -1
- package/package.json +6 -6
- package/dist/piece-registry-BKRLK7Kv.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as isPrivateAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerError, H as PieceWorkerPool, I as SecretDeletedError, J as redact, K as PieceWorkerTimeoutError, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as DEFAULT_EGRESS_POLICY, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerDisposedError, X as rememberSecrets, Y as redactMessage, Z as secretsFor, _ as resolvedBlock, _t as isIndeterminateError, at as pieceModuleRef, b as resolutionOf, c as canonicalJson, ct as UnsupportedPieceFeatureError, d as DynamicPropertiesError, dt as unsupportedTrigger, et as parseAddress, f as checkDynamicProperties, g as isHostBound, gt as setPieceRegistryUrl, h as CompositeBlockExecutor, ht as pieceRegistrySource, i as packagePieces, it as bundleResolver, j as declaredReactorConnectionIds, l as hashOf, lt as authMethodFor, m as ActivepiecesBlockExecutor, mt as npmRegistryBaseUrl, n as installedPiece, nt as FileTooLargeError, o as errorNameOf, ot as sourcedResolver, p as effectiveRetryPolicy, q as containsRedactedMarker, r as installedPieces, rt as maxFileBytes, s as runWorkflow, t as PieceRegistry, tt as extractDedupeKey, u as stepConfigHash, ut as unsupportedAuth, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-D0pa4v0T.js";
|
|
2
2
|
import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
|
|
3
3
|
import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { open, readdir, rm, stat } from "node:fs/promises";
|
|
@@ -11,6 +11,7 @@ import { Readable } from "node:stream";
|
|
|
11
11
|
import dns from "node:dns";
|
|
12
12
|
import net from "node:net";
|
|
13
13
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
14
|
+
import { hostname } from "node:os";
|
|
14
15
|
import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
|
|
15
16
|
import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
|
|
16
17
|
import http from "node:http";
|
|
@@ -99,7 +100,7 @@ function npmVersions(name, options = {}) {
|
|
|
99
100
|
}
|
|
100
101
|
//#endregion
|
|
101
102
|
//#region src/reactor/attachment-port.ts
|
|
102
|
-
const logger$
|
|
103
|
+
const logger$7 = childLogger(["workflow", "attachments"]);
|
|
103
104
|
function fileStream(filePath) {
|
|
104
105
|
return Readable.toWeb(createReadStream(filePath));
|
|
105
106
|
}
|
|
@@ -190,7 +191,7 @@ function createAttachmentPort(client, access) {
|
|
|
190
191
|
...signal ? { signal } : {}
|
|
191
192
|
}));
|
|
192
193
|
access.onWritten?.(ref);
|
|
193
|
-
logger$
|
|
194
|
+
logger$7.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
|
|
194
195
|
return ref;
|
|
195
196
|
}
|
|
196
197
|
};
|
|
@@ -804,7 +805,7 @@ const CACHE_TTL_MS = 3e5;
|
|
|
804
805
|
const PINNED_TTL_MS = 36e5;
|
|
805
806
|
const REGISTRY_PAGE_SIZE = 50;
|
|
806
807
|
const REGISTRY_MAX_PAGES = 200;
|
|
807
|
-
const logger$
|
|
808
|
+
const logger$6 = childLogger(["workflow", "piece-catalog"]);
|
|
808
809
|
function aiLast(audience) {
|
|
809
810
|
return audience === "ai" ? 1 : 0;
|
|
810
811
|
}
|
|
@@ -850,7 +851,7 @@ async function fetchRegistryCatalog(url, timeoutMs) {
|
|
|
850
851
|
items.push(...page.items);
|
|
851
852
|
if (!page.hasMore || page.items.length === 0) return items;
|
|
852
853
|
}
|
|
853
|
-
logger$
|
|
854
|
+
logger$6.warn(`${url} still had more pieces after ${REGISTRY_MAX_PAGES} pages`);
|
|
854
855
|
return items;
|
|
855
856
|
}
|
|
856
857
|
function asList(value) {
|
|
@@ -864,10 +865,10 @@ async function publishedLists(suggestions, timeoutMs) {
|
|
|
864
865
|
const cloudUrl = suggestions ? `${CATALOG_URL}?suggestionType=ACTION_AND_TRIGGER` : CATALOG_URL;
|
|
865
866
|
const [fromRegistry, fromCloud] = await Promise.allSettled([source ? fetchRegistryCatalog(source.catalogUrl(suggestions), timeoutMs) : Promise.resolve([]), fetchJson(cloudUrl, timeoutMs)]);
|
|
866
867
|
const registry = fromRegistry.status === "fulfilled" ? asList(fromRegistry.value).filter((entry) => typeof entry.name === "string") : [];
|
|
867
|
-
if (fromRegistry.status === "rejected") logger$
|
|
868
|
+
if (fromRegistry.status === "rejected") logger$6.warn("Piece registry @url did not answer: @error", source?.baseUrl ?? "?", String(fromRegistry.reason));
|
|
868
869
|
if (fromCloud.status === "rejected") {
|
|
869
870
|
if (registry.length === 0) throw asError(fromCloud.reason);
|
|
870
|
-
logger$
|
|
871
|
+
logger$6.warn("Serving registry pieces only: @error", String(fromCloud.reason));
|
|
871
872
|
return {
|
|
872
873
|
registry,
|
|
873
874
|
cloud: []
|
|
@@ -882,12 +883,18 @@ function registryFirst(registry, cloud) {
|
|
|
882
883
|
const claimed = new Set(registry.map((entry) => entry.name));
|
|
883
884
|
return [...registry, ...cloud.filter((entry) => !claimed.has(entry.name))];
|
|
884
885
|
}
|
|
886
|
+
function tagged(entries, source) {
|
|
887
|
+
return entries.map((entry) => ({
|
|
888
|
+
...entry,
|
|
889
|
+
source
|
|
890
|
+
}));
|
|
891
|
+
}
|
|
885
892
|
async function fetchCatalogWithSuggestions() {
|
|
886
893
|
const { registry, cloud } = await publishedLists(true, 12e4);
|
|
887
|
-
return registryFirst(registry, cloud);
|
|
894
|
+
return registryFirst(tagged(registry, "registry"), tagged(cloud, "activepieces"));
|
|
888
895
|
}
|
|
889
896
|
let catalogCache;
|
|
890
|
-
function toSummaries(raw) {
|
|
897
|
+
function toSummaries(raw, source) {
|
|
891
898
|
return raw.filter((entry) => typeof entry.name === "string" && typeof entry.version === "string" && !SERVER_ONLY_PIECES.has(entry.name) && (typeof entry.actions === "number" && entry.actions > 0 || typeof entry.triggers === "number" && entry.triggers > 0)).map((entry) => ({
|
|
892
899
|
name: entry.name,
|
|
893
900
|
displayName: entry.displayName ?? entry.name,
|
|
@@ -897,6 +904,7 @@ function toSummaries(raw) {
|
|
|
897
904
|
actionCount: typeof entry.actions === "number" ? entry.actions : 0,
|
|
898
905
|
triggerCount: typeof entry.triggers === "number" ? entry.triggers : 0,
|
|
899
906
|
categories: entry.categories ?? [],
|
|
907
|
+
source,
|
|
900
908
|
auth: entry.auth ?? null,
|
|
901
909
|
...entry.deprecated === true ? { deprecated: true } : {},
|
|
902
910
|
...reasonOf(unsupportedAuth(entry.auth))
|
|
@@ -905,7 +913,7 @@ function toSummaries(raw) {
|
|
|
905
913
|
async function fetchPieceCatalog() {
|
|
906
914
|
if (catalogCache && catalogCache.expiresAt > Date.now()) return catalogCache.value;
|
|
907
915
|
const lists = await publishedLists(false, 3e4);
|
|
908
|
-
const value = [...registryFirst(toSummaries(lists.registry), toSummaries(lists.cloud))].sort((a, b) => a.displayName.localeCompare(b.displayName));
|
|
916
|
+
const value = [...registryFirst(toSummaries(lists.registry, "registry"), toSummaries(lists.cloud, "activepieces"))].sort((a, b) => a.displayName.localeCompare(b.displayName));
|
|
909
917
|
catalogCache = {
|
|
910
918
|
value,
|
|
911
919
|
expiresAt: Date.now() + CACHE_TTL_MS
|
|
@@ -917,8 +925,8 @@ async function fetchPieceJson(packageName, version) {
|
|
|
917
925
|
if (source) try {
|
|
918
926
|
return await fetchJson(atVersion(source.pieceUrl(packageName), version));
|
|
919
927
|
} catch (error) {
|
|
920
|
-
if (error instanceof CatalogStatusError && error.status === 404) logger$
|
|
921
|
-
else logger$
|
|
928
|
+
if (error instanceof CatalogStatusError && error.status === 404) logger$6.debug("@registry does not serve @piece", source.baseUrl, packageName);
|
|
929
|
+
else logger$6.warn("Piece registry @registry did not answer for @piece: @error", source.baseUrl, packageName, String(error));
|
|
922
930
|
}
|
|
923
931
|
return fetchJson(pieceUrl(packageName, version));
|
|
924
932
|
}
|
|
@@ -1007,6 +1015,7 @@ function catalogEntry(descriptor, pieceName, version) {
|
|
|
1007
1015
|
actionCount: descriptor.actions.length,
|
|
1008
1016
|
triggerCount: descriptor.triggers.length,
|
|
1009
1017
|
categories: descriptor.categories ?? [],
|
|
1018
|
+
source: "local",
|
|
1010
1019
|
auth: clientAuth(descriptor.auth),
|
|
1011
1020
|
...reasonOf(descriptor.unsupported)
|
|
1012
1021
|
};
|
|
@@ -1041,32 +1050,44 @@ function triggersResult(descriptor, pieceName, version) {
|
|
|
1041
1050
|
auth: clientAuth(descriptor.auth)
|
|
1042
1051
|
};
|
|
1043
1052
|
}
|
|
1044
|
-
function
|
|
1053
|
+
function localSearchPiece(descriptor, pieceName, version) {
|
|
1045
1054
|
const pieceDisplayName = descriptor.displayName || pieceName;
|
|
1046
1055
|
const logoUrl = descriptor.logoUrl ?? "";
|
|
1047
|
-
return
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1056
|
+
return {
|
|
1057
|
+
meta: {
|
|
1058
|
+
pieceName,
|
|
1059
|
+
pieceVersion: version,
|
|
1060
|
+
displayName: pieceDisplayName,
|
|
1061
|
+
description: descriptor.description ?? "",
|
|
1062
|
+
logoUrl,
|
|
1063
|
+
categories: descriptor.categories ?? [],
|
|
1064
|
+
source: "local",
|
|
1065
|
+
...reasonOf(descriptor.unsupported)
|
|
1066
|
+
},
|
|
1067
|
+
blocks: [...descriptor.actions.map((action) => ({
|
|
1068
|
+
pieceName,
|
|
1069
|
+
pieceVersion: version,
|
|
1070
|
+
name: action.name,
|
|
1071
|
+
pieceDisplayName,
|
|
1072
|
+
logoUrl,
|
|
1073
|
+
displayName: action.displayName,
|
|
1074
|
+
description: action.description ?? "",
|
|
1075
|
+
kind: "action",
|
|
1076
|
+
strategy: null,
|
|
1077
|
+
...reasonOf(descriptor.unsupported)
|
|
1078
|
+
})), ...descriptor.triggers.map((trigger) => ({
|
|
1079
|
+
pieceName,
|
|
1080
|
+
pieceVersion: version,
|
|
1081
|
+
name: trigger.name,
|
|
1082
|
+
pieceDisplayName,
|
|
1083
|
+
logoUrl,
|
|
1084
|
+
displayName: trigger.displayName,
|
|
1085
|
+
description: trigger.description ?? "",
|
|
1086
|
+
kind: "trigger",
|
|
1087
|
+
strategy: trigger.strategy,
|
|
1088
|
+
...reasonOf(descriptor.unsupported ?? trigger.unsupported)
|
|
1089
|
+
}))]
|
|
1090
|
+
};
|
|
1070
1091
|
}
|
|
1071
1092
|
function detailResult(descriptor, pieceName, version) {
|
|
1072
1093
|
return {
|
|
@@ -1106,11 +1127,41 @@ function detailResult(descriptor, pieceName, version) {
|
|
|
1106
1127
|
//#endregion
|
|
1107
1128
|
//#region src/reactor/block-search.ts
|
|
1108
1129
|
const INDEX_TTL_MS = 36e5;
|
|
1109
|
-
const DEFAULT_LIMIT =
|
|
1110
|
-
const MAX_LIMIT =
|
|
1130
|
+
const DEFAULT_LIMIT = 20;
|
|
1131
|
+
const MAX_LIMIT = 50;
|
|
1132
|
+
const MAX_BLOCKS_PER_PIECE = 50;
|
|
1133
|
+
function tokenize(text) {
|
|
1134
|
+
return text.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
|
1135
|
+
}
|
|
1136
|
+
function wordsOf(...texts) {
|
|
1137
|
+
const words = texts.flatMap(tokenize);
|
|
1138
|
+
return {
|
|
1139
|
+
words,
|
|
1140
|
+
compact: words.join("")
|
|
1141
|
+
};
|
|
1142
|
+
}
|
|
1143
|
+
function slugOf(pieceName) {
|
|
1144
|
+
return pieceName.replace(/^@[^/]+\//, "").replace(/^piece-/, "");
|
|
1145
|
+
}
|
|
1146
|
+
function indexPiece(piece) {
|
|
1147
|
+
const { meta } = piece;
|
|
1148
|
+
return {
|
|
1149
|
+
meta,
|
|
1150
|
+
name: wordsOf(meta.displayName, slugOf(meta.pieceName)),
|
|
1151
|
+
exactName: tokenize(meta.displayName).join(" "),
|
|
1152
|
+
about: wordsOf(meta.description, ...meta.categories),
|
|
1153
|
+
blocks: piece.blocks.map((hit) => ({
|
|
1154
|
+
hit,
|
|
1155
|
+
name: wordsOf(hit.displayName, hit.name),
|
|
1156
|
+
description: wordsOf(hit.description)
|
|
1157
|
+
}))
|
|
1158
|
+
};
|
|
1159
|
+
}
|
|
1160
|
+
function indexPieces(pieces) {
|
|
1161
|
+
return { pieces: pieces.map(indexPiece) };
|
|
1162
|
+
}
|
|
1111
1163
|
function buildSearchIndex(raw) {
|
|
1112
|
-
const
|
|
1113
|
-
let pieces = 0;
|
|
1164
|
+
const pieces = [];
|
|
1114
1165
|
for (const entry of raw) {
|
|
1115
1166
|
if (typeof entry.name !== "string" || typeof entry.version !== "string" || SERVER_ONLY_PIECES.has(entry.name)) continue;
|
|
1116
1167
|
const pieceName = entry.name;
|
|
@@ -1118,78 +1169,141 @@ function buildSearchIndex(raw) {
|
|
|
1118
1169
|
const pieceDisplayName = entry.displayName ?? pieceName;
|
|
1119
1170
|
const logoUrl = entry.logoUrl ?? "";
|
|
1120
1171
|
const pieceUnsupported = unsupportedAuth(entry.auth);
|
|
1172
|
+
const blocks = [];
|
|
1121
1173
|
const push = (kind, item) => {
|
|
1122
1174
|
if (typeof item.name !== "string" || item.name === "") return;
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
kind,
|
|
1135
|
-
strategy: kind === "trigger" ? item.type ?? null : null,
|
|
1136
|
-
...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
|
|
1137
|
-
},
|
|
1138
|
-
name: `${displayName} ${item.name}`.toLowerCase(),
|
|
1139
|
-
description: description.toLowerCase(),
|
|
1140
|
-
piece: pieceDisplayName.toLowerCase()
|
|
1175
|
+
blocks.push({
|
|
1176
|
+
pieceName,
|
|
1177
|
+
pieceVersion,
|
|
1178
|
+
name: item.name,
|
|
1179
|
+
pieceDisplayName,
|
|
1180
|
+
logoUrl,
|
|
1181
|
+
displayName: item.displayName ?? item.name,
|
|
1182
|
+
description: item.description ?? "",
|
|
1183
|
+
kind,
|
|
1184
|
+
strategy: kind === "trigger" ? item.type ?? null : null,
|
|
1185
|
+
...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
|
|
1141
1186
|
});
|
|
1142
1187
|
};
|
|
1143
1188
|
for (const action of entry.suggestedActions ?? []) push("action", action);
|
|
1144
1189
|
for (const trigger of entry.suggestedTriggers ?? []) push("trigger", trigger);
|
|
1145
|
-
pieces
|
|
1190
|
+
pieces.push({
|
|
1191
|
+
meta: {
|
|
1192
|
+
pieceName,
|
|
1193
|
+
pieceVersion,
|
|
1194
|
+
displayName: pieceDisplayName,
|
|
1195
|
+
description: entry.description ?? "",
|
|
1196
|
+
logoUrl,
|
|
1197
|
+
categories: Array.isArray(entry.categories) ? entry.categories : [],
|
|
1198
|
+
source: entry.source ?? "activepieces",
|
|
1199
|
+
...entry.deprecated === true ? { deprecated: true } : {},
|
|
1200
|
+
...reasonOf(pieceUnsupported)
|
|
1201
|
+
},
|
|
1202
|
+
blocks
|
|
1203
|
+
});
|
|
1146
1204
|
}
|
|
1147
|
-
return
|
|
1148
|
-
entries,
|
|
1149
|
-
pieces
|
|
1150
|
-
};
|
|
1151
|
-
}
|
|
1152
|
-
function indexFromHits(hits) {
|
|
1153
|
-
const pieces = /* @__PURE__ */ new Set();
|
|
1154
|
-
return {
|
|
1155
|
-
entries: hits.map((hit) => {
|
|
1156
|
-
pieces.add(hit.pieceName);
|
|
1157
|
-
return {
|
|
1158
|
-
hit,
|
|
1159
|
-
name: `${hit.displayName} ${hit.name}`.toLowerCase(),
|
|
1160
|
-
description: hit.description.toLowerCase(),
|
|
1161
|
-
piece: hit.pieceDisplayName.toLowerCase()
|
|
1162
|
-
};
|
|
1163
|
-
}),
|
|
1164
|
-
pieces: pieces.size
|
|
1165
|
-
};
|
|
1205
|
+
return indexPieces(pieces);
|
|
1166
1206
|
}
|
|
1167
1207
|
function merge(index, local) {
|
|
1168
|
-
const
|
|
1169
|
-
const localNames = new Set(
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1208
|
+
const localPieces = local?.pieces ?? [];
|
|
1209
|
+
const localNames = new Set(localPieces.map((piece) => piece.meta.pieceName));
|
|
1210
|
+
return { pieces: [...localPieces, ...(index?.pieces ?? []).filter((piece) => !localNames.has(piece.meta.pieceName))] };
|
|
1211
|
+
}
|
|
1212
|
+
const EXACT = 4;
|
|
1213
|
+
const PREFIX = 3;
|
|
1214
|
+
const INNER = 1.5;
|
|
1215
|
+
const FUZZY = 1;
|
|
1216
|
+
const PIECE_NAME = 3;
|
|
1217
|
+
const BLOCK_NAME = 2;
|
|
1218
|
+
const BLOCK_DESCRIPTION = 1;
|
|
1219
|
+
const PIECE_ABOUT = .5;
|
|
1220
|
+
function editDistance(a, b, max) {
|
|
1221
|
+
if (Math.abs(a.length - b.length) > max) return max + 1;
|
|
1222
|
+
let prevPrev = [];
|
|
1223
|
+
let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
|
|
1224
|
+
for (let i = 1; i <= a.length; i++) {
|
|
1225
|
+
const row = [i];
|
|
1226
|
+
let rowMin = i;
|
|
1227
|
+
for (let j = 1; j <= b.length; j++) {
|
|
1228
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
1229
|
+
let value = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost);
|
|
1230
|
+
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) value = Math.min(value, prevPrev[j - 2] + 1);
|
|
1231
|
+
row.push(value);
|
|
1232
|
+
rowMin = Math.min(rowMin, value);
|
|
1233
|
+
}
|
|
1234
|
+
if (rowMin > max) return max + 1;
|
|
1235
|
+
prevPrev = prev;
|
|
1236
|
+
prev = row;
|
|
1237
|
+
}
|
|
1238
|
+
return prev[b.length];
|
|
1239
|
+
}
|
|
1240
|
+
function termScore(token, field, fuzzy) {
|
|
1241
|
+
let best = 0;
|
|
1242
|
+
for (const word of field.words) {
|
|
1243
|
+
if (word === token) return EXACT;
|
|
1244
|
+
if (word.startsWith(token)) best = PREFIX;
|
|
1245
|
+
else if (best < INNER && token.length >= 3 && word.includes(token)) best = INNER;
|
|
1246
|
+
}
|
|
1247
|
+
if (best > 0) return best;
|
|
1248
|
+
if (token.length >= 3 && field.compact.includes(token)) return INNER;
|
|
1249
|
+
if (!fuzzy || token.length < 4) return 0;
|
|
1250
|
+
const max = token.length >= 8 ? 2 : 1;
|
|
1251
|
+
for (const word of field.words) if (editDistance(token, word, max) <= max || word.length > token.length && editDistance(token, word.slice(0, token.length), max) <= max) return FUZZY;
|
|
1252
|
+
return 0;
|
|
1253
|
+
}
|
|
1254
|
+
function allowed(piece, filter) {
|
|
1255
|
+
const { sources, categories } = filter;
|
|
1256
|
+
if (sources?.length && !sources.includes(piece.meta.source)) return false;
|
|
1257
|
+
if (categories?.length && !piece.meta.categories.some((category) => categories.includes(category))) return false;
|
|
1258
|
+
return true;
|
|
1259
|
+
}
|
|
1260
|
+
function searchIndex(index, query, filter) {
|
|
1261
|
+
const tokens = tokenize(query);
|
|
1179
1262
|
if (tokens.length === 0) return [];
|
|
1180
|
-
const
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
if (!
|
|
1184
|
-
const
|
|
1185
|
-
const
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1263
|
+
const exactName = tokens.join(" ");
|
|
1264
|
+
const matches = [];
|
|
1265
|
+
for (const piece of index.pieces) {
|
|
1266
|
+
if (!allowed(piece, filter)) continue;
|
|
1267
|
+
const nameScores = tokens.map((token) => termScore(token, piece.name, true));
|
|
1268
|
+
const ofKind = piece.blocks.filter((block) => block.hit.kind === filter.kind).map((block) => ({
|
|
1269
|
+
hit: block.hit,
|
|
1270
|
+
scores: tokens.map((token) => Math.max(BLOCK_NAME * termScore(token, block.name, true), BLOCK_DESCRIPTION * termScore(token, block.description, false)))
|
|
1271
|
+
}));
|
|
1272
|
+
const matchBlocks = (pieceScores) => {
|
|
1273
|
+
const found = [];
|
|
1274
|
+
ofKind.forEach((block, order) => {
|
|
1275
|
+
let score = 0;
|
|
1276
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
1277
|
+
const best = Math.max(pieceScores[i], block.scores[i]);
|
|
1278
|
+
if (best === 0) return;
|
|
1279
|
+
score += best;
|
|
1280
|
+
}
|
|
1281
|
+
found.push({
|
|
1282
|
+
score,
|
|
1283
|
+
order,
|
|
1284
|
+
hit: block.hit
|
|
1285
|
+
});
|
|
1286
|
+
});
|
|
1287
|
+
return found;
|
|
1288
|
+
};
|
|
1289
|
+
let blocks = matchBlocks(nameScores.map((score) => PIECE_NAME * score));
|
|
1290
|
+
if (blocks.length === 0 && ofKind.length > 0) blocks = matchBlocks(tokens.map((token, i) => Math.max(PIECE_NAME * nameScores[i], PIECE_ABOUT * termScore(token, piece.about, false))));
|
|
1291
|
+
if (blocks.length === 0) continue;
|
|
1292
|
+
blocks.sort((a, b) => b.score - a.score || a.order - b.order);
|
|
1293
|
+
const nameBonus = piece.exactName === exactName ? EXACT : 0;
|
|
1294
|
+
matches.push({
|
|
1295
|
+
score: blocks[0].score + nameBonus,
|
|
1296
|
+
match: {
|
|
1297
|
+
...piece.meta,
|
|
1298
|
+
namedPiece: nameScores.every((score) => score > 0),
|
|
1299
|
+
blocks: blocks.slice(0, MAX_BLOCKS_PER_PIECE).map(({ hit }) => hit)
|
|
1300
|
+
}
|
|
1189
1301
|
});
|
|
1190
1302
|
}
|
|
1191
|
-
|
|
1192
|
-
|
|
1303
|
+
const demoted = (match) => (match.deprecated ? 2 : 0) + (match.unsupported ? 1 : 0);
|
|
1304
|
+
matches.sort((a, b) => b.score - a.score || demoted(a.match) - demoted(b.match) || a.match.displayName.localeCompare(b.match.displayName));
|
|
1305
|
+
const limit = Math.min(Math.max(filter.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT);
|
|
1306
|
+
return matches.slice(0, limit).map(({ match }) => match);
|
|
1193
1307
|
}
|
|
1194
1308
|
let cached;
|
|
1195
1309
|
function ensureIndex() {
|
|
@@ -1206,29 +1320,35 @@ function ensureIndex() {
|
|
|
1206
1320
|
cached = entry;
|
|
1207
1321
|
return entry;
|
|
1208
1322
|
}
|
|
1209
|
-
function
|
|
1323
|
+
function warmSearchIndex() {
|
|
1324
|
+
const entry = ensureIndex();
|
|
1325
|
+
entry.promise.catch(() => {
|
|
1326
|
+
if (cached === entry) cached = void 0;
|
|
1327
|
+
});
|
|
1328
|
+
}
|
|
1329
|
+
function searchPieces(query, filter, local) {
|
|
1210
1330
|
const index = ensureIndex();
|
|
1211
1331
|
if (index.error) {
|
|
1212
1332
|
const message = index.error;
|
|
1213
1333
|
cached = void 0;
|
|
1214
1334
|
return {
|
|
1215
1335
|
status: "error",
|
|
1216
|
-
|
|
1217
|
-
indexedPieces: local?.pieces ?? 0,
|
|
1336
|
+
pieces: searchIndex(merge(void 0, local), query, filter),
|
|
1337
|
+
indexedPieces: local?.pieces.length ?? 0,
|
|
1218
1338
|
error: message
|
|
1219
1339
|
};
|
|
1220
1340
|
}
|
|
1221
1341
|
if (!index.value) return {
|
|
1222
1342
|
status: "indexing",
|
|
1223
|
-
|
|
1224
|
-
indexedPieces: local?.pieces ?? 0,
|
|
1343
|
+
pieces: searchIndex(merge(void 0, local), query, filter),
|
|
1344
|
+
indexedPieces: local?.pieces.length ?? 0,
|
|
1225
1345
|
error: null
|
|
1226
1346
|
};
|
|
1227
1347
|
const merged = merge(index.value, local);
|
|
1228
1348
|
return {
|
|
1229
1349
|
status: "ready",
|
|
1230
|
-
|
|
1231
|
-
indexedPieces: merged.pieces,
|
|
1350
|
+
pieces: searchIndex(merged, query, filter),
|
|
1351
|
+
indexedPieces: merged.pieces.length,
|
|
1232
1352
|
error: null
|
|
1233
1353
|
};
|
|
1234
1354
|
}
|
|
@@ -1407,7 +1527,7 @@ function currentRunUser() {
|
|
|
1407
1527
|
}
|
|
1408
1528
|
//#endregion
|
|
1409
1529
|
//#region src/reactor/piece-store-port.ts
|
|
1410
|
-
const logger$
|
|
1530
|
+
const logger$5 = childLogger(["workflow", "piece-store"]);
|
|
1411
1531
|
const PROJECT_SCOPE_KEY = "reactor";
|
|
1412
1532
|
const TEST_PARTITION_SUFFIX = "#test";
|
|
1413
1533
|
function testPartitionKey(scope, workflowId) {
|
|
@@ -1441,7 +1561,7 @@ function createPieceStorePort(store, workflowIdFor, sample = false, clock = Date
|
|
|
1441
1561
|
if (!isCursorRef(key) || isPlausibleCursor(value, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
|
|
1442
1562
|
const kept = await store.getPieceStoreValue(scope, partition(scope), key);
|
|
1443
1563
|
if (!isCursorShaped(value) && !isPlausibleCursor(kept, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
|
|
1444
|
-
logger$
|
|
1564
|
+
logger$5.warn(`Rejected @key=@value from workflow ${flowKey()}; keeping @kept`, key, showCursor(value), isPlausibleCursor(kept, now) ? String(kept) : "no cursor");
|
|
1445
1565
|
if (!isPlausibleCursor(kept, now)) await store.deletePieceStoreValue(scope, partition(scope), key);
|
|
1446
1566
|
},
|
|
1447
1567
|
delete: async (key, scope) => store.deletePieceStoreValue(scope, partition(scope), key)
|
|
@@ -7497,7 +7617,7 @@ function workflowParkRow(workflowId, publishedVersion, reason) {
|
|
|
7497
7617
|
parked_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7498
7618
|
};
|
|
7499
7619
|
}
|
|
7500
|
-
const logger$
|
|
7620
|
+
const logger$4 = childLogger([
|
|
7501
7621
|
"workflow",
|
|
7502
7622
|
"runtime",
|
|
7503
7623
|
"store"
|
|
@@ -7649,7 +7769,7 @@ async function migrateTriggerStoreState(db) {
|
|
|
7649
7769
|
try {
|
|
7650
7770
|
rows = await db.selectFrom("trigger_state").select(["workflow_id", "store_state"]).orderBy("updated_at", "asc").orderBy("workflow_id", "asc").execute();
|
|
7651
7771
|
} catch (error) {
|
|
7652
|
-
logger$
|
|
7772
|
+
logger$4.error("Could not read trigger_state to migrate it: @error", error);
|
|
7653
7773
|
return unmigrated;
|
|
7654
7774
|
}
|
|
7655
7775
|
const pending = [];
|
|
@@ -7665,7 +7785,7 @@ async function migrateTriggerStoreState(db) {
|
|
|
7665
7785
|
await migrateOneRow(db, row, projectWinner);
|
|
7666
7786
|
} catch (error) {
|
|
7667
7787
|
unmigrated.add(row.workflowId);
|
|
7668
|
-
logger$
|
|
7788
|
+
logger$4.error(`Could not migrate trigger store state for ${row.workflowId}`, error);
|
|
7669
7789
|
}
|
|
7670
7790
|
return unmigrated;
|
|
7671
7791
|
}
|
|
@@ -7675,7 +7795,7 @@ function parseLegacyBlob(row, unmigrated) {
|
|
|
7675
7795
|
try {
|
|
7676
7796
|
state = JSON.parse(row.store_state);
|
|
7677
7797
|
} catch {
|
|
7678
|
-
logger$
|
|
7798
|
+
logger$4.warn(`Leaving unparseable store_state for ${row.workflow_id} in place`);
|
|
7679
7799
|
unmigrated.add(row.workflow_id);
|
|
7680
7800
|
return null;
|
|
7681
7801
|
}
|
|
@@ -7696,7 +7816,7 @@ function parseLegacyBlob(row, unmigrated) {
|
|
|
7696
7816
|
});
|
|
7697
7817
|
continue;
|
|
7698
7818
|
}
|
|
7699
|
-
if (key.startsWith("test")) logger$
|
|
7819
|
+
if (key.startsWith("test")) logger$4.info(`Migrating "${key}" for ${row.workflow_id} as a project key; it may be a test leftover`);
|
|
7700
7820
|
entries.push({
|
|
7701
7821
|
scope: "PROJECT",
|
|
7702
7822
|
scopeKey: PROJECT_SCOPE_KEY,
|
|
@@ -7718,7 +7838,7 @@ function resolveProjectCollisions(pending) {
|
|
|
7718
7838
|
}
|
|
7719
7839
|
winner.set(entry.key, row.workflowId);
|
|
7720
7840
|
}
|
|
7721
|
-
for (const [key, workflows] of contested) logger$
|
|
7841
|
+
for (const [key, workflows] of contested) logger$4.warn(`Project store key "@key" was written by ${workflows.join(", ")}; keeping the value last updated, from ${winner.get(key)}, and discarding the rest`, key);
|
|
7722
7842
|
return winner;
|
|
7723
7843
|
}
|
|
7724
7844
|
async function migrateOneRow(db, row, projectWinner) {
|
|
@@ -7747,9 +7867,9 @@ async function insertPieceStoreIfAbsent(db, scope, scopeKey, key, value) {
|
|
|
7747
7867
|
}
|
|
7748
7868
|
function warnIfOverPieceStoreLimits(scope, scopeKey, key, encoded) {
|
|
7749
7869
|
const at = `${scope}/${scopeKey}/${key}`;
|
|
7750
|
-
if (key.length > 128) logger$
|
|
7870
|
+
if (key.length > 128) logger$4.warn(`Migrated store key ${at} is ${key.length} chars, over the 128 limit; writes to it will fail`);
|
|
7751
7871
|
const size = Buffer.byteLength(encoded, "utf8");
|
|
7752
|
-
if (size > 524288) logger$
|
|
7872
|
+
if (size > 524288) logger$4.warn(`Migrated store value ${at} is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} limit; writes to it will fail`);
|
|
7753
7873
|
}
|
|
7754
7874
|
const PIECE_STORE_MAX_VALUE_BYTES = 524288;
|
|
7755
7875
|
var PieceStoreLimitError = class extends Error {
|
|
@@ -7905,16 +8025,26 @@ function journaledPayloadNames(json, ids) {
|
|
|
7905
8025
|
});
|
|
7906
8026
|
}
|
|
7907
8027
|
const erasedRuns = /* @__PURE__ */ new Set();
|
|
7908
|
-
async function
|
|
8028
|
+
async function pollSucceededIn(db, workflowId, storeState, nowIso, nextPollAtIso) {
|
|
8029
|
+
await db.updateTable("trigger_state").set({
|
|
8030
|
+
store_state: storeState,
|
|
8031
|
+
last_poll_at: nowIso,
|
|
8032
|
+
next_poll_at: nextPollAtIso,
|
|
8033
|
+
last_error: null,
|
|
8034
|
+
consecutive_failures: 0,
|
|
8035
|
+
updated_at: nowIso
|
|
8036
|
+
}).where("workflow_id", "=", workflowId).execute();
|
|
8037
|
+
}
|
|
8038
|
+
async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso, runId = FIRED_WITHOUT_RUN_ID) {
|
|
7909
8039
|
const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
|
|
7910
8040
|
await db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
|
|
7911
8041
|
return await db.insertInto("trigger_dedupe").values({
|
|
7912
8042
|
workflow_id: workflowId,
|
|
7913
8043
|
dedupe_key: dedupeKey,
|
|
7914
|
-
run_id:
|
|
8044
|
+
run_id: runId,
|
|
7915
8045
|
created_at: nowIso,
|
|
7916
8046
|
attempts: 1
|
|
7917
|
-
}).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id:
|
|
8047
|
+
}).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id: runId }).where("trigger_dedupe.run_id", "is", null)).returning("dedupe_key").executeTakeFirst() !== void 0;
|
|
7918
8048
|
}
|
|
7919
8049
|
const STEP_COLUMNS_WITHOUT_DATA = [
|
|
7920
8050
|
"id",
|
|
@@ -7968,7 +8098,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7968
8098
|
}).where("status", "=", "RUNNING");
|
|
7969
8099
|
if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
|
|
7970
8100
|
const recovered = (await query.returning("id").execute()).length;
|
|
7971
|
-
if (recovered > 0) logger$
|
|
8101
|
+
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
|
|
7972
8102
|
return recovered;
|
|
7973
8103
|
}
|
|
7974
8104
|
async recoverAbandonedRuns() {
|
|
@@ -7979,7 +8109,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7979
8109
|
}).where("status", "=", PENDING_RUN_STATUS);
|
|
7980
8110
|
if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
|
|
7981
8111
|
const recovered = (await query.returning("id").execute()).length;
|
|
7982
|
-
if (recovered > 0) logger$
|
|
8112
|
+
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
|
|
7983
8113
|
return recovered;
|
|
7984
8114
|
}
|
|
7985
8115
|
/**
|
|
@@ -8059,6 +8189,65 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8059
8189
|
runId: id
|
|
8060
8190
|
};
|
|
8061
8191
|
}
|
|
8192
|
+
async journalTriggerItems(items, ttlMs, nowIso) {
|
|
8193
|
+
const ids = await this.db.transaction().execute(async (trx) => {
|
|
8194
|
+
const journaled = [];
|
|
8195
|
+
for (const { dedupeKey, options } of items) {
|
|
8196
|
+
const id = randomUUID();
|
|
8197
|
+
const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
|
|
8198
|
+
if (won) await this.insertPendingRun(trx, id, options);
|
|
8199
|
+
journaled.push(won ? id : void 0);
|
|
8200
|
+
}
|
|
8201
|
+
return journaled;
|
|
8202
|
+
});
|
|
8203
|
+
for (const id of ids) if (id) this.runsInFlight.add(id);
|
|
8204
|
+
return ids;
|
|
8205
|
+
}
|
|
8206
|
+
async recordScheduleFire(nowIso, nextPollAtIso, options) {
|
|
8207
|
+
const id = randomUUID();
|
|
8208
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8209
|
+
await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
|
|
8210
|
+
await this.insertPendingRun(trx, id, options);
|
|
8211
|
+
});
|
|
8212
|
+
this.runsInFlight.add(id);
|
|
8213
|
+
return id;
|
|
8214
|
+
}
|
|
8215
|
+
/**
|
|
8216
|
+
* Puts a piece_store partition back to `entries` and, in the same write,
|
|
8217
|
+
* cancels the runs the restored cursor delivers again and releases their
|
|
8218
|
+
* claims, so no reader sees the old cursor while the claims still hold.
|
|
8219
|
+
*/
|
|
8220
|
+
async rewindPieceStore(scope, scopeKey, entries, redelivered = {
|
|
8221
|
+
runIds: [],
|
|
8222
|
+
reason: ""
|
|
8223
|
+
}) {
|
|
8224
|
+
const live = redelivered.runIds.filter((runId) => !erasedRuns.has(runId));
|
|
8225
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
8226
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8227
|
+
await trx.deleteFrom("piece_store").where("scope", "=", scope).where("scope_key", "=", scopeKey).execute();
|
|
8228
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
8229
|
+
assertPieceStoreEntry(key, value);
|
|
8230
|
+
await trx.insertInto("piece_store").values({
|
|
8231
|
+
scope,
|
|
8232
|
+
scope_key: scopeKey,
|
|
8233
|
+
key,
|
|
8234
|
+
value: JSON.stringify(value),
|
|
8235
|
+
updated_at: nowIso
|
|
8236
|
+
}).execute();
|
|
8237
|
+
}
|
|
8238
|
+
if (live.length === 0) return;
|
|
8239
|
+
await trx.updateTable("run").set({
|
|
8240
|
+
status: "CANCELLED",
|
|
8241
|
+
error: redactMessage(redelivered.reason),
|
|
8242
|
+
ended_at: nowIso
|
|
8243
|
+
}).where("id", "in", live).execute();
|
|
8244
|
+
await trx.deleteFrom("trigger_dedupe").where("run_id", "in", live).execute();
|
|
8245
|
+
});
|
|
8246
|
+
for (const runId of redelivered.runIds) {
|
|
8247
|
+
this.runsInFlight.delete(runId);
|
|
8248
|
+
erasedRuns.delete(runId);
|
|
8249
|
+
}
|
|
8250
|
+
}
|
|
8062
8251
|
/**
|
|
8063
8252
|
* Counts one delivery of an operation, and reports whether a run was ever
|
|
8064
8253
|
* journaled for it.
|
|
@@ -8176,7 +8365,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8176
8365
|
if (result.steps.length > 0) try {
|
|
8177
8366
|
await this.sweepSteps(runId, result, executionOrder);
|
|
8178
8367
|
} catch (error) {
|
|
8179
|
-
logger$
|
|
8368
|
+
logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
|
|
8180
8369
|
}
|
|
8181
8370
|
const notes = runWarningNotes(result);
|
|
8182
8371
|
await this.db.updateTable("run").set({
|
|
@@ -8393,14 +8582,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8393
8582
|
return this.db.selectFrom("trigger_state").selectAll().orderBy("workflow_id", "asc").execute();
|
|
8394
8583
|
}
|
|
8395
8584
|
async recordPollSuccess(workflowId, storeState, nowIso, nextPollAtIso) {
|
|
8396
|
-
await this.db
|
|
8397
|
-
store_state: storeState,
|
|
8398
|
-
last_poll_at: nowIso,
|
|
8399
|
-
next_poll_at: nextPollAtIso,
|
|
8400
|
-
last_error: null,
|
|
8401
|
-
consecutive_failures: 0,
|
|
8402
|
-
updated_at: nowIso
|
|
8403
|
-
}).where("workflow_id", "=", workflowId).execute();
|
|
8585
|
+
await pollSucceededIn(this.db, workflowId, storeState, nowIso, nextPollAtIso);
|
|
8404
8586
|
}
|
|
8405
8587
|
async recordPollFailure(workflowId, error, nowIso, nextPollAtIso, consecutiveFailures) {
|
|
8406
8588
|
await this.db.updateTable("trigger_state").set({
|
|
@@ -8577,6 +8759,7 @@ function decodeRunCursor(cursor) {
|
|
|
8577
8759
|
}
|
|
8578
8760
|
//#endregion
|
|
8579
8761
|
//#region src/reactor/run-gate.ts
|
|
8762
|
+
const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
|
|
8580
8763
|
const QUEUE_DEPTH_ENV = "PH_WORKFLOWS_MAX_QUEUED_FIRINGS";
|
|
8581
8764
|
function maxQueuedFirings(env = process.env) {
|
|
8582
8765
|
const raw = Number(env[QUEUE_DEPTH_ENV]);
|
|
@@ -8584,6 +8767,7 @@ function maxQueuedFirings(env = process.env) {
|
|
|
8584
8767
|
}
|
|
8585
8768
|
var WorkflowRunGate = class {
|
|
8586
8769
|
lanes = /* @__PURE__ */ new Map();
|
|
8770
|
+
closed = false;
|
|
8587
8771
|
maxQueued;
|
|
8588
8772
|
constructor(options = {}) {
|
|
8589
8773
|
this.maxQueued = options.maxQueued ?? maxQueuedFirings();
|
|
@@ -8612,6 +8796,11 @@ var WorkflowRunGate = class {
|
|
|
8612
8796
|
* that vanished is indistinguishable from a trigger that never fired.
|
|
8613
8797
|
*/
|
|
8614
8798
|
async admit(workflowId, policy) {
|
|
8799
|
+
if (this.closed) return {
|
|
8800
|
+
admitted: false,
|
|
8801
|
+
reason: CLOSED_REASON,
|
|
8802
|
+
refusal: "closed"
|
|
8803
|
+
};
|
|
8615
8804
|
const limit = concurrencyLimit(policy);
|
|
8616
8805
|
if (limit === null) return {
|
|
8617
8806
|
admitted: true,
|
|
@@ -8638,7 +8827,11 @@ var WorkflowRunGate = class {
|
|
|
8638
8827
|
refusal: "queue-full",
|
|
8639
8828
|
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
8640
8829
|
};
|
|
8641
|
-
await new Promise((resolve) => lane.waiting.push(resolve))
|
|
8830
|
+
if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
|
|
8831
|
+
admitted: false,
|
|
8832
|
+
reason: CLOSED_REASON,
|
|
8833
|
+
refusal: "closed"
|
|
8834
|
+
};
|
|
8642
8835
|
waited = true;
|
|
8643
8836
|
} else lane.active += 1;
|
|
8644
8837
|
let released = false;
|
|
@@ -8652,13 +8845,18 @@ var WorkflowRunGate = class {
|
|
|
8652
8845
|
const next = lane.waiting.shift();
|
|
8653
8846
|
if (next) {
|
|
8654
8847
|
lane.active += 1;
|
|
8655
|
-
next();
|
|
8848
|
+
next(true);
|
|
8656
8849
|
return;
|
|
8657
8850
|
}
|
|
8658
8851
|
this.forgetIfIdle(workflowId, lane);
|
|
8659
8852
|
}
|
|
8660
8853
|
};
|
|
8661
8854
|
}
|
|
8855
|
+
/** Refuses every waiting and later firing: a shut-down runtime runs none. */
|
|
8856
|
+
close() {
|
|
8857
|
+
this.closed = true;
|
|
8858
|
+
for (const lane of this.lanes.values()) for (const waiter of lane.waiting.splice(0)) waiter(false);
|
|
8859
|
+
}
|
|
8662
8860
|
forgetIfIdle(workflowId, lane) {
|
|
8663
8861
|
if (lane.active === 0 && lane.waiting.length === 0) this.lanes.delete(workflowId);
|
|
8664
8862
|
}
|
|
@@ -8699,13 +8897,33 @@ async function sweepRetention(store, options) {
|
|
|
8699
8897
|
dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
|
|
8700
8898
|
};
|
|
8701
8899
|
}
|
|
8900
|
+
//#endregion
|
|
8901
|
+
//#region src/reactor/shutdown.ts
|
|
8902
|
+
/** A firing refused because this runtime has shut down: retryable elsewhere. */
|
|
8903
|
+
var WorkflowRuntimeClosedError = class extends Error {
|
|
8904
|
+
workflowId;
|
|
8905
|
+
constructor(workflowId) {
|
|
8906
|
+
super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
|
|
8907
|
+
this.workflowId = workflowId;
|
|
8908
|
+
this.name = "WorkflowRuntimeClosedError";
|
|
8909
|
+
}
|
|
8910
|
+
};
|
|
8911
|
+
var TriggerSupervisorStoppedError = class extends Error {
|
|
8912
|
+
constructor(options) {
|
|
8913
|
+
super("The trigger supervisor has stopped", options);
|
|
8914
|
+
this.name = "TriggerSupervisorStoppedError";
|
|
8915
|
+
}
|
|
8916
|
+
};
|
|
8917
|
+
function isShutdownRefusal(error) {
|
|
8918
|
+
return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
|
|
8919
|
+
}
|
|
8702
8920
|
childLogger(["workflow", "trigger-binding"]);
|
|
8703
8921
|
function pieceTriggerKind(block) {
|
|
8704
8922
|
return `piece:${block.pieceName}:${block.name}`;
|
|
8705
8923
|
}
|
|
8706
8924
|
//#endregion
|
|
8707
8925
|
//#region src/reactor/trigger-supervisor.ts
|
|
8708
|
-
const logger$
|
|
8926
|
+
const logger$2 = childLogger(["workflow", "trigger-supervisor"]);
|
|
8709
8927
|
function targetOf$1(binding) {
|
|
8710
8928
|
return {
|
|
8711
8929
|
name: binding.packageName,
|
|
@@ -8725,6 +8943,7 @@ const SCHEDULE_TRIGGER_KIND = "schedule";
|
|
|
8725
8943
|
const MIN_INTERVAL_MS = MIN_SCHEDULE_INTERVAL_MS;
|
|
8726
8944
|
const MAX_BACKOFF_MS = 18e5;
|
|
8727
8945
|
const DEDUPE_TTL_MS = 3e4;
|
|
8946
|
+
const REDELIVERED_REASON = "Redelivered: the trigger stopped before firing this item and put its cursor back, so the next poll delivers it again";
|
|
8728
8947
|
const DEFAULT_RECONCILE_INTERVAL_MS = 9e5;
|
|
8729
8948
|
const RENEW_RETRY_BASE_MS = 6e4;
|
|
8730
8949
|
function isSchedule(binding) {
|
|
@@ -8740,7 +8959,7 @@ function intervalFromSchedules(schedules, defaultMs) {
|
|
|
8740
8959
|
const cron = schedule.cronExpression;
|
|
8741
8960
|
const intervalMs = cronIntervalMs(cron);
|
|
8742
8961
|
if (intervalMs === void 0) {
|
|
8743
|
-
logger$
|
|
8962
|
+
logger$2.warn("Unsupported setSchedule cron \"@cron\"; using the default", cron);
|
|
8744
8963
|
return Math.max(defaultMs, MIN_INTERVAL_MS);
|
|
8745
8964
|
}
|
|
8746
8965
|
return intervalMs;
|
|
@@ -8791,6 +9010,7 @@ var TriggerSupervisor = class {
|
|
|
8791
9010
|
ops = Promise.resolve();
|
|
8792
9011
|
ticking = false;
|
|
8793
9012
|
warnedMissingJournal = false;
|
|
9013
|
+
stopped = false;
|
|
8794
9014
|
constructor(options) {
|
|
8795
9015
|
this.options = options;
|
|
8796
9016
|
this.worker = options.worker ?? new PieceWorker({
|
|
@@ -8805,26 +9025,60 @@ var TriggerSupervisor = class {
|
|
|
8805
9025
|
this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
|
|
8806
9026
|
}
|
|
8807
9027
|
start() {
|
|
8808
|
-
if (this.timer) return;
|
|
9028
|
+
if (this.timer || this.stopped) return;
|
|
8809
9029
|
this.timer = setInterval(() => {
|
|
8810
9030
|
this.tick().catch((error) => {
|
|
8811
|
-
|
|
9031
|
+
if (isShutdownRefusal(error)) {
|
|
9032
|
+
logger$2.debug("Trigger tick stopped: @error", error);
|
|
9033
|
+
return;
|
|
9034
|
+
}
|
|
9035
|
+
logger$2.error("Trigger tick failed: @error", error);
|
|
8812
9036
|
});
|
|
8813
9037
|
}, this.tickMs);
|
|
8814
9038
|
this.timer.unref();
|
|
8815
|
-
logger$
|
|
9039
|
+
logger$2.info(`Trigger supervisor started (tick ${this.tickMs}ms)`);
|
|
8816
9040
|
}
|
|
8817
9041
|
stop() {
|
|
9042
|
+
if (this.stopped) return;
|
|
9043
|
+
this.stopped = true;
|
|
8818
9044
|
if (this.timer) clearInterval(this.timer);
|
|
8819
9045
|
this.timer = void 0;
|
|
8820
9046
|
if (!this.options.worker) this.worker.dispose();
|
|
8821
|
-
logger$
|
|
9047
|
+
logger$2.info("Trigger supervisor stopped");
|
|
8822
9048
|
}
|
|
8823
9049
|
enqueue(task) {
|
|
8824
|
-
const run = this.ops.then(
|
|
9050
|
+
const run = this.ops.then(async () => {
|
|
9051
|
+
this.throwIfStopped();
|
|
9052
|
+
try {
|
|
9053
|
+
return await task();
|
|
9054
|
+
} catch (error) {
|
|
9055
|
+
if (this.stopped && !(error instanceof TriggerSupervisorStoppedError)) throw new TriggerSupervisorStoppedError({ cause: error });
|
|
9056
|
+
throw error;
|
|
9057
|
+
}
|
|
9058
|
+
});
|
|
8825
9059
|
this.ops = run.catch(() => void 0);
|
|
8826
9060
|
return run;
|
|
8827
9061
|
}
|
|
9062
|
+
throwIfStopped() {
|
|
9063
|
+
if (this.stopped) throw new TriggerSupervisorStoppedError();
|
|
9064
|
+
}
|
|
9065
|
+
async store() {
|
|
9066
|
+
this.throwIfStopped();
|
|
9067
|
+
const store = await this.options.store();
|
|
9068
|
+
if (!store) return void 0;
|
|
9069
|
+
return new Proxy(store, { get: (target, property) => {
|
|
9070
|
+
const value = Reflect.get(target, property, target);
|
|
9071
|
+
if (typeof value !== "function") return value;
|
|
9072
|
+
return (...args) => {
|
|
9073
|
+
this.throwIfStopped();
|
|
9074
|
+
return value.apply(target, args);
|
|
9075
|
+
};
|
|
9076
|
+
} });
|
|
9077
|
+
}
|
|
9078
|
+
fire(workflowId, payload, kind, runId) {
|
|
9079
|
+
this.throwIfStopped();
|
|
9080
|
+
this.options.fire(workflowId, payload, kind, runId);
|
|
9081
|
+
}
|
|
8828
9082
|
enabledOk = /* @__PURE__ */ new Set();
|
|
8829
9083
|
descriptors = /* @__PURE__ */ new Map();
|
|
8830
9084
|
own;
|
|
@@ -8846,7 +9100,7 @@ var TriggerSupervisor = class {
|
|
|
8846
9100
|
park(workflowId, publishedVersion, reason) {
|
|
8847
9101
|
this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
|
|
8848
9102
|
return this.enqueue(async () => {
|
|
8849
|
-
const store = await this.
|
|
9103
|
+
const store = await this.store();
|
|
8850
9104
|
if (!store) return false;
|
|
8851
9105
|
return store.parkWorkflow(workflowId, publishedVersion, reason);
|
|
8852
9106
|
});
|
|
@@ -8855,7 +9109,7 @@ var TriggerSupervisor = class {
|
|
|
8855
9109
|
liftPark(workflowId, publishedVersion, trigger) {
|
|
8856
9110
|
if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
|
|
8857
9111
|
return this.enqueue(async () => {
|
|
8858
|
-
const store = await this.
|
|
9112
|
+
const store = await this.store();
|
|
8859
9113
|
if (!store) return;
|
|
8860
9114
|
if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
|
|
8861
9115
|
await store.liftPark(workflowId, trigger);
|
|
@@ -8866,7 +9120,7 @@ var TriggerSupervisor = class {
|
|
|
8866
9120
|
releasePark(workflowId, flipRow) {
|
|
8867
9121
|
this.parks.delete(workflowId);
|
|
8868
9122
|
return this.enqueue(async () => {
|
|
8869
|
-
const store = await this.
|
|
9123
|
+
const store = await this.store();
|
|
8870
9124
|
if (!store) return;
|
|
8871
9125
|
if (flipRow) await store.clearParkOnDisable(workflowId);
|
|
8872
9126
|
else await store.clearWorkflowPark(workflowId);
|
|
@@ -8884,7 +9138,7 @@ var TriggerSupervisor = class {
|
|
|
8884
9138
|
const park = this.parks.known(workflowId)?.park;
|
|
8885
9139
|
if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
|
|
8886
9140
|
return this.enqueue(async () => {
|
|
8887
|
-
const store = await this.
|
|
9141
|
+
const store = await this.store();
|
|
8888
9142
|
if (!store) return;
|
|
8889
9143
|
const park = await store.getWorkflowPark(workflowId);
|
|
8890
9144
|
if (!park || park.published_version >= publishedVersion) return;
|
|
@@ -8899,7 +9153,7 @@ var TriggerSupervisor = class {
|
|
|
8899
9153
|
const binding = this.unbind(workflowId);
|
|
8900
9154
|
this.parks.delete(workflowId);
|
|
8901
9155
|
return this.enqueue(async () => {
|
|
8902
|
-
const store = await this.
|
|
9156
|
+
const store = await this.store();
|
|
8903
9157
|
if (!store) return;
|
|
8904
9158
|
await this.disable(workflowId, binding);
|
|
8905
9159
|
await store.deleteTriggerState(workflowId);
|
|
@@ -8919,16 +9173,16 @@ var TriggerSupervisor = class {
|
|
|
8919
9173
|
return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
|
|
8920
9174
|
}
|
|
8921
9175
|
async recordRejection(workflowId, block, config, message, retryAt) {
|
|
8922
|
-
const store = await this.
|
|
9176
|
+
const store = await this.store();
|
|
8923
9177
|
if (!store) return;
|
|
8924
9178
|
const hash = configHash(block, config);
|
|
8925
9179
|
const existing = await store.getTriggerState(workflowId);
|
|
8926
9180
|
if (existing?.status === "ENABLED" && existing.config_hash === hash) {
|
|
8927
|
-
logger$
|
|
9181
|
+
logger$2.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
|
|
8928
9182
|
return;
|
|
8929
9183
|
}
|
|
8930
9184
|
if (existing?.status === "PARKED" && existing.config_hash === hash) {
|
|
8931
|
-
logger$
|
|
9185
|
+
logger$2.warn(`Workflow ${workflowId} is PARKED and could not be resolved; the park stands. @error`, message);
|
|
8932
9186
|
return;
|
|
8933
9187
|
}
|
|
8934
9188
|
await store.upsertTriggerState({
|
|
@@ -8950,7 +9204,7 @@ var TriggerSupervisor = class {
|
|
|
8950
9204
|
}
|
|
8951
9205
|
test(binding, options = {}) {
|
|
8952
9206
|
return this.enqueue(async () => {
|
|
8953
|
-
const store = await this.
|
|
9207
|
+
const store = await this.store();
|
|
8954
9208
|
try {
|
|
8955
9209
|
return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
|
|
8956
9210
|
} finally {
|
|
@@ -8964,7 +9218,7 @@ var TriggerSupervisor = class {
|
|
|
8964
9218
|
await store.deletePieceStore("FLOW", testPartitionKey("FLOW", workflowId));
|
|
8965
9219
|
await store.deletePieceStore("PROJECT", testPartitionKey("PROJECT", workflowId));
|
|
8966
9220
|
} catch (error) {
|
|
8967
|
-
logger$
|
|
9221
|
+
logger$2.warn(`Could not clear test store for ${workflowId}`, error);
|
|
8968
9222
|
}
|
|
8969
9223
|
}
|
|
8970
9224
|
handshake(binding, payload) {
|
|
@@ -8974,45 +9228,58 @@ var TriggerSupervisor = class {
|
|
|
8974
9228
|
return this.enqueue(async () => {
|
|
8975
9229
|
const binding = this.bindings.get(workflowId);
|
|
8976
9230
|
if (!binding || isSchedule(binding)) {
|
|
8977
|
-
logger$
|
|
9231
|
+
logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
|
|
8978
9232
|
return;
|
|
8979
9233
|
}
|
|
8980
|
-
const store = await this.
|
|
9234
|
+
const store = await this.store();
|
|
8981
9235
|
if (!store) throw new MissingJournalError("Webhook delivery");
|
|
8982
9236
|
const row = await store.getTriggerState(workflowId);
|
|
8983
9237
|
const now = this.now();
|
|
8984
9238
|
const rewind = await this.cursorRewind(store, workflowId);
|
|
9239
|
+
let refused = false;
|
|
8985
9240
|
try {
|
|
8986
|
-
const result = await this.hook(binding, "run", { payload })
|
|
9241
|
+
const result = await this.hook(binding, "run", { payload }).catch((error) => {
|
|
9242
|
+
refused = isShutdownRefusal(error);
|
|
9243
|
+
throw error;
|
|
9244
|
+
});
|
|
8987
9245
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
8988
9246
|
if (row?.status === "ENABLED") await store.recordPollSuccess(workflowId, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
|
|
8989
|
-
|
|
9247
|
+
await this.fireItems(store, binding, result.output, now, rewind);
|
|
8990
9248
|
} catch (error) {
|
|
8991
|
-
await rewind();
|
|
9249
|
+
if (!refused) await rewind();
|
|
8992
9250
|
throw error;
|
|
8993
9251
|
}
|
|
8994
9252
|
});
|
|
8995
9253
|
}
|
|
8996
9254
|
async cursorRewind(store, workflowId) {
|
|
8997
9255
|
const before = await store.listPieceStore("FLOW", workflowId);
|
|
8998
|
-
|
|
9256
|
+
let rewound;
|
|
9257
|
+
const rewind = async (redelivered) => {
|
|
8999
9258
|
try {
|
|
9000
|
-
await store
|
|
9001
|
-
|
|
9259
|
+
const raw = await this.options.store();
|
|
9260
|
+
if (!raw) return false;
|
|
9261
|
+
await raw.rewindPieceStore("FLOW", workflowId, before, {
|
|
9262
|
+
runIds: redelivered,
|
|
9263
|
+
reason: REDELIVERED_REASON
|
|
9264
|
+
});
|
|
9265
|
+
return true;
|
|
9002
9266
|
} catch (error) {
|
|
9003
|
-
logger$
|
|
9267
|
+
logger$2.warn(`Could not rewind the cursor for ${workflowId}`, error);
|
|
9268
|
+
return false;
|
|
9004
9269
|
}
|
|
9005
9270
|
};
|
|
9271
|
+
return (redelivered = []) => rewound ??= rewind(redelivered);
|
|
9006
9272
|
}
|
|
9007
9273
|
async hook(binding, hook, options = {}) {
|
|
9008
9274
|
if (hook !== "onDisable") checkDynamicProperties(binding.config, binding.propertySettings);
|
|
9009
|
-
const store = await this.
|
|
9275
|
+
const store = await this.store();
|
|
9010
9276
|
if (!store && hook !== "test") throw new MissingJournalError(`Trigger hook "${hook}"`);
|
|
9011
9277
|
const pieceStore = store ? createPieceStorePort(store, () => binding.workflowId, hook === "test") : void 0;
|
|
9012
9278
|
const piece = await this.resolver().resolve(targetOf$1(binding));
|
|
9013
9279
|
const auth = await this.options.resolveAuth(binding.connectionId, { piecePackage: binding.packageName });
|
|
9014
9280
|
const redactValues = secretsFor(auth);
|
|
9015
9281
|
const reactor = await this.reactorFor(binding, hook, options.runUser);
|
|
9282
|
+
this.throwIfStopped();
|
|
9016
9283
|
return this.worker.runTriggerHook({
|
|
9017
9284
|
...pieceModuleRef(piece),
|
|
9018
9285
|
triggerName: binding.triggerName,
|
|
@@ -9080,7 +9347,7 @@ var TriggerSupervisor = class {
|
|
|
9080
9347
|
};
|
|
9081
9348
|
}
|
|
9082
9349
|
async enable(binding, superseded) {
|
|
9083
|
-
const store = await this.
|
|
9350
|
+
const store = await this.store();
|
|
9084
9351
|
if (!store) throw new MissingJournalError("Enabling a trigger");
|
|
9085
9352
|
const hash = configHash(binding.block, binding.config);
|
|
9086
9353
|
const now = this.now();
|
|
@@ -9139,8 +9406,9 @@ var TriggerSupervisor = class {
|
|
|
9139
9406
|
this.enabledOk.add(binding.workflowId);
|
|
9140
9407
|
this.enableRetries.delete(binding.workflowId);
|
|
9141
9408
|
store.clearUnmigratedTriggerState(binding.workflowId);
|
|
9142
|
-
logger$
|
|
9409
|
+
logger$2.info(`Enabled @block for workflow ${binding.workflowId} (every ${intervalMs}ms)`, blockLabel(binding.block));
|
|
9143
9410
|
} catch (error) {
|
|
9411
|
+
if (this.stopped) throw error;
|
|
9144
9412
|
this.enabledOk.delete(binding.workflowId);
|
|
9145
9413
|
const message = error instanceof Error ? error.message : String(error);
|
|
9146
9414
|
const failures = (existing?.consecutive_failures ?? 0) + 1;
|
|
@@ -9168,7 +9436,7 @@ var TriggerSupervisor = class {
|
|
|
9168
9436
|
updated_at: now.toISOString(),
|
|
9169
9437
|
...pieceColumns(binding)
|
|
9170
9438
|
});
|
|
9171
|
-
logger$
|
|
9439
|
+
logger$2.error(`onEnable failed for workflow ${binding.workflowId} (${failures}x): @error` + (retryAt ? `; retrying at ${retryAt.toISOString()}` : "; not retrying"), message);
|
|
9172
9440
|
}
|
|
9173
9441
|
}
|
|
9174
9442
|
renewAtAfterEnable(renew, existing, isRepublish, now) {
|
|
@@ -9213,7 +9481,7 @@ var TriggerSupervisor = class {
|
|
|
9213
9481
|
const retry = this.enableRetries.get(workflowId);
|
|
9214
9482
|
if (retry) retry.at = this.now().getTime() + this.defaultIntervalMs;
|
|
9215
9483
|
if (superseded && this.bindings.get(workflowId) === binding) this.bindings.set(workflowId, superseded);
|
|
9216
|
-
logger$
|
|
9484
|
+
logger$2.warn(`Workflow ${workflowId} is PARKED at version ${park.published_version}; its trigger is not armed until the workflow is re-published or re-enabled. @error`, park.reason);
|
|
9217
9485
|
}
|
|
9218
9486
|
/** The status an enable that succeeded writes: never ENABLED while a park
|
|
9219
9487
|
* stands, whatever ran meanwhile. The registration stays recorded. */
|
|
@@ -9232,7 +9500,7 @@ var TriggerSupervisor = class {
|
|
|
9232
9500
|
failures: existing.consecutive_failures,
|
|
9233
9501
|
release: true
|
|
9234
9502
|
});
|
|
9235
|
-
logger$
|
|
9503
|
+
logger$2.info(`Enable for workflow ${binding.workflowId} still backing off until ${existing.next_poll_at}`);
|
|
9236
9504
|
}
|
|
9237
9505
|
return true;
|
|
9238
9506
|
}
|
|
@@ -9240,7 +9508,7 @@ var TriggerSupervisor = class {
|
|
|
9240
9508
|
try {
|
|
9241
9509
|
await this.hook(binding, "onDisable");
|
|
9242
9510
|
} catch (error) {
|
|
9243
|
-
logger$
|
|
9511
|
+
logger$2.warn(`onDisable before retrying workflow ${binding.workflowId} failed`, error);
|
|
9244
9512
|
}
|
|
9245
9513
|
}
|
|
9246
9514
|
async enableSchedule(store, binding, hash, existing) {
|
|
@@ -9269,7 +9537,7 @@ var TriggerSupervisor = class {
|
|
|
9269
9537
|
consecutive_failures: 0
|
|
9270
9538
|
});
|
|
9271
9539
|
this.enabledOk.add(binding.workflowId);
|
|
9272
|
-
logger$
|
|
9540
|
+
logger$2.info(`Scheduled workflow ${binding.workflowId}: next fire ${nextAt.toISOString()}` + (carried ? " (carried over)" : ""));
|
|
9273
9541
|
} catch (error) {
|
|
9274
9542
|
this.enabledOk.delete(binding.workflowId);
|
|
9275
9543
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -9281,24 +9549,24 @@ var TriggerSupervisor = class {
|
|
|
9281
9549
|
last_error: message,
|
|
9282
9550
|
consecutive_failures: (existing?.consecutive_failures ?? 0) + 1
|
|
9283
9551
|
});
|
|
9284
|
-
logger$
|
|
9552
|
+
logger$2.error(`Invalid schedule for workflow ${binding.workflowId}: @error`, message);
|
|
9285
9553
|
}
|
|
9286
9554
|
}
|
|
9287
9555
|
async disable(workflowId, binding) {
|
|
9288
|
-
const store = await this.
|
|
9556
|
+
const store = await this.store();
|
|
9289
9557
|
if (!store) throw new MissingJournalError("Disabling a trigger");
|
|
9290
9558
|
const row = await store.getTriggerState(workflowId);
|
|
9291
9559
|
if (!row || row.status === "DISABLED") return;
|
|
9292
9560
|
await this.disableRow(workflowId, row, binding);
|
|
9293
9561
|
}
|
|
9294
9562
|
async disableRow(workflowId, row, binding) {
|
|
9295
|
-
const store = await this.
|
|
9563
|
+
const store = await this.store();
|
|
9296
9564
|
if (!store) return;
|
|
9297
9565
|
const target = binding ?? this.bindingFromRow(row);
|
|
9298
9566
|
if (target && !isSchedule(target) && blockKey(triggerRowBlock(row)) !== SCHEDULE_BLOCK) try {
|
|
9299
9567
|
await this.hook(target, "onDisable");
|
|
9300
9568
|
} catch (error) {
|
|
9301
|
-
logger$
|
|
9569
|
+
logger$2.warn(`onDisable failed for workflow ${workflowId}`, error);
|
|
9302
9570
|
}
|
|
9303
9571
|
await store.setTriggerStatus(workflowId, "DISABLED");
|
|
9304
9572
|
}
|
|
@@ -9306,7 +9574,7 @@ var TriggerSupervisor = class {
|
|
|
9306
9574
|
return this.bindings.get(row.workflow_id);
|
|
9307
9575
|
}
|
|
9308
9576
|
async tick() {
|
|
9309
|
-
if (this.ticking) return;
|
|
9577
|
+
if (this.ticking || this.stopped) return;
|
|
9310
9578
|
this.ticking = true;
|
|
9311
9579
|
try {
|
|
9312
9580
|
await this.enqueue(() => this.pollDue());
|
|
@@ -9327,6 +9595,7 @@ var TriggerSupervisor = class {
|
|
|
9327
9595
|
try {
|
|
9328
9596
|
await this.enable(binding);
|
|
9329
9597
|
} catch (error) {
|
|
9598
|
+
if (this.stopped) throw error;
|
|
9330
9599
|
const failures = retry.failures + 1;
|
|
9331
9600
|
const intervalMs = isSchedule(binding) ? MIN_INTERVAL_MS : pollIntervalFor(binding, void 0, this.defaultIntervalMs);
|
|
9332
9601
|
this.enableRetries.set(workflowId, {
|
|
@@ -9334,20 +9603,21 @@ var TriggerSupervisor = class {
|
|
|
9334
9603
|
failures,
|
|
9335
9604
|
release: retry.release
|
|
9336
9605
|
});
|
|
9337
|
-
logger$
|
|
9606
|
+
logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
|
|
9338
9607
|
}
|
|
9339
9608
|
}
|
|
9340
9609
|
async pollDue() {
|
|
9341
|
-
const store = await this.
|
|
9610
|
+
const store = await this.store();
|
|
9342
9611
|
if (!store) {
|
|
9343
9612
|
if (!this.warnedMissingJournal) {
|
|
9344
9613
|
this.warnedMissingJournal = true;
|
|
9345
|
-
logger$
|
|
9614
|
+
logger$2.error("Trigger polling is off: @error", new MissingJournalError("Polling"));
|
|
9346
9615
|
}
|
|
9347
9616
|
return;
|
|
9348
9617
|
}
|
|
9349
9618
|
const due = await store.listDueTriggerStates(this.now().toISOString());
|
|
9350
9619
|
for (const row of due) {
|
|
9620
|
+
this.throwIfStopped();
|
|
9351
9621
|
const binding = this.bindings.get(row.workflow_id);
|
|
9352
9622
|
if (!binding) {
|
|
9353
9623
|
await store.setTriggerStatus(row.workflow_id, "DISABLED");
|
|
@@ -9356,12 +9626,15 @@ var TriggerSupervisor = class {
|
|
|
9356
9626
|
if (isSchedule(binding)) await this.fireSchedule(store, row, binding);
|
|
9357
9627
|
else await this.poll(store, row, binding);
|
|
9358
9628
|
}
|
|
9629
|
+
this.throwIfStopped();
|
|
9359
9630
|
await this.renewDue(store);
|
|
9631
|
+
this.throwIfStopped();
|
|
9360
9632
|
await this.retryOneEnable();
|
|
9361
9633
|
}
|
|
9362
9634
|
async renewDue(store) {
|
|
9363
9635
|
const due = await store.listDueTriggerRenewals(this.now().toISOString());
|
|
9364
9636
|
for (const row of due) {
|
|
9637
|
+
this.throwIfStopped();
|
|
9365
9638
|
const binding = this.bindings.get(row.workflow_id);
|
|
9366
9639
|
if (!binding) {
|
|
9367
9640
|
await store.setTriggerStatus(row.workflow_id, "DISABLED");
|
|
@@ -9387,7 +9660,7 @@ var TriggerSupervisor = class {
|
|
|
9387
9660
|
await this.hook(binding, "onRenew", { webhookUrl });
|
|
9388
9661
|
const nextAt = nextRenewAt(renew, now);
|
|
9389
9662
|
await store.recordRenewSuccess(row.workflow_id, now.toISOString(), nextAt.toISOString());
|
|
9390
|
-
logger$
|
|
9663
|
+
logger$2.info(`Renewed trigger of workflow ${row.workflow_id}; next ${nextAt.toISOString()}`);
|
|
9391
9664
|
} catch (error) {
|
|
9392
9665
|
const message = error instanceof Error ? error.message : String(error);
|
|
9393
9666
|
const failures = row.renew_failures + 1;
|
|
@@ -9395,7 +9668,7 @@ var TriggerSupervisor = class {
|
|
|
9395
9668
|
const slot = renew ? nextRenewAt(renew, now).getTime() : backoff;
|
|
9396
9669
|
const retryAt = new Date(Math.min(backoff, slot));
|
|
9397
9670
|
await store.recordRenewFailure(row.workflow_id, message, now.toISOString(), retryAt.toISOString(), failures);
|
|
9398
|
-
logger$
|
|
9671
|
+
logger$2.warn(`onRenew failed for workflow ${row.workflow_id} (${failures}x): @error; retrying at ${retryAt.toISOString()}`, message);
|
|
9399
9672
|
}
|
|
9400
9673
|
}
|
|
9401
9674
|
async fireSchedule(store, row, binding) {
|
|
@@ -9404,38 +9677,79 @@ var TriggerSupervisor = class {
|
|
|
9404
9677
|
const schedule = parseScheduleConfig$1(binding.config);
|
|
9405
9678
|
const scheduledFor = row.next_poll_at ? new Date(row.next_poll_at) : now;
|
|
9406
9679
|
const nextAt = rescheduleAfterFire(schedule, scheduledFor, now);
|
|
9407
|
-
|
|
9408
|
-
|
|
9680
|
+
const payload = schedulePayload(schedule, scheduledFor, now);
|
|
9681
|
+
const runId = await store.recordScheduleFire(now.toISOString(), nextAt.toISOString(), {
|
|
9682
|
+
workflowId: binding.workflowId,
|
|
9683
|
+
triggerKind: SCHEDULE_TRIGGER_KIND,
|
|
9684
|
+
triggerPayload: payload
|
|
9685
|
+
});
|
|
9686
|
+
try {
|
|
9687
|
+
this.fire(binding.workflowId, payload, SCHEDULE_TRIGGER_KIND, runId);
|
|
9688
|
+
} catch (error) {
|
|
9689
|
+
await this.failRefused([runId], error);
|
|
9690
|
+
throw error;
|
|
9691
|
+
}
|
|
9409
9692
|
} catch (error) {
|
|
9693
|
+
if (this.stopped) throw error;
|
|
9410
9694
|
const message = error instanceof Error ? error.message : String(error);
|
|
9411
9695
|
this.enabledOk.delete(binding.workflowId);
|
|
9412
9696
|
await store.setTriggerStatus(row.workflow_id, "ERROR", message);
|
|
9413
|
-
logger$
|
|
9697
|
+
logger$2.error(`Schedule fire failed for workflow ${row.workflow_id}: @error`, message);
|
|
9414
9698
|
}
|
|
9415
9699
|
}
|
|
9416
9700
|
async poll(store, row, binding) {
|
|
9417
9701
|
const now = this.now();
|
|
9418
9702
|
const rewind = await this.cursorRewind(store, row.workflow_id);
|
|
9703
|
+
let refused = false;
|
|
9419
9704
|
try {
|
|
9420
|
-
const result = await this.hook(binding, "run")
|
|
9705
|
+
const result = await this.hook(binding, "run").catch((error) => {
|
|
9706
|
+
refused = isShutdownRefusal(error);
|
|
9707
|
+
throw error;
|
|
9708
|
+
});
|
|
9421
9709
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
9422
9710
|
await store.recordPollSuccess(row.workflow_id, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
|
|
9423
|
-
|
|
9711
|
+
await this.fireItems(store, binding, result.output, now, rewind);
|
|
9424
9712
|
} catch (error) {
|
|
9425
|
-
await rewind();
|
|
9713
|
+
if (!refused) await rewind();
|
|
9714
|
+
if (this.stopped) throw error;
|
|
9426
9715
|
const message = error instanceof Error ? error.message : String(error);
|
|
9427
9716
|
const failures = row.consecutive_failures + 1;
|
|
9428
9717
|
const backoff = backoffMs(row.interval_ms, failures);
|
|
9429
9718
|
await store.recordPollFailure(row.workflow_id, message, now.toISOString(), new Date(now.getTime() + backoff).toISOString(), failures);
|
|
9430
|
-
logger$
|
|
9719
|
+
logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
|
|
9720
|
+
}
|
|
9721
|
+
}
|
|
9722
|
+
async fireItems(store, binding, items, now, rewind) {
|
|
9723
|
+
if (items.length === 0) return;
|
|
9724
|
+
const kind = pieceTriggerKind(binding.block);
|
|
9725
|
+
const journaled = (await store.journalTriggerItems(items.map((item) => ({
|
|
9726
|
+
dedupeKey: extractDedupeKey(item),
|
|
9727
|
+
options: {
|
|
9728
|
+
workflowId: binding.workflowId,
|
|
9729
|
+
triggerKind: kind,
|
|
9730
|
+
triggerPayload: item
|
|
9731
|
+
}
|
|
9732
|
+
})), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
|
|
9733
|
+
runId,
|
|
9734
|
+
item: items[index]
|
|
9735
|
+
}] : []);
|
|
9736
|
+
if (this.stopped) {
|
|
9737
|
+
const error = new TriggerSupervisorStoppedError();
|
|
9738
|
+
await this.redeliver(journaled.map(({ runId }) => runId), rewind, error);
|
|
9739
|
+
throw error;
|
|
9431
9740
|
}
|
|
9741
|
+
for (const { runId, item } of journaled) this.fire(binding.workflowId, item, kind, runId);
|
|
9742
|
+
}
|
|
9743
|
+
async redeliver(runIds, rewind, error) {
|
|
9744
|
+
if (!await rewind(runIds)) await this.failRefused(runIds, error);
|
|
9432
9745
|
}
|
|
9433
|
-
async
|
|
9434
|
-
|
|
9435
|
-
|
|
9436
|
-
|
|
9746
|
+
async failRefused(runIds, error) {
|
|
9747
|
+
try {
|
|
9748
|
+
const raw = await this.options.store();
|
|
9749
|
+
for (const runId of runIds) await raw?.failRun(runId, error instanceof Error ? error.message : String(error), error instanceof Error ? error.name : void 0);
|
|
9750
|
+
} catch (failure) {
|
|
9751
|
+
logger$2.warn(`Could not fail the refused runs ${runIds.join(", ")}`, failure);
|
|
9437
9752
|
}
|
|
9438
|
-
this.options.fire(binding.workflowId, item, pieceTriggerKind(binding.block));
|
|
9439
9753
|
}
|
|
9440
9754
|
};
|
|
9441
9755
|
//#endregion
|
|
@@ -9679,7 +9993,7 @@ const DESCRIBE_TIMEOUT_MS = 3e4;
|
|
|
9679
9993
|
const SEED_ATTEMPTS = 3;
|
|
9680
9994
|
const SEED_RETRY_BASE_MS = 250;
|
|
9681
9995
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref());
|
|
9682
|
-
const logger = childLogger(["workflow", "runtime"]);
|
|
9996
|
+
const logger$1 = childLogger(["workflow", "runtime"]);
|
|
9683
9997
|
const DRIVE_DOCUMENT_TYPE = "powerhouse/document-drive";
|
|
9684
9998
|
const WORKFLOW_DOCUMENT_TYPE = "powerhouse/workflow";
|
|
9685
9999
|
const DOCUMENT_SCOPE = "document";
|
|
@@ -9772,6 +10086,7 @@ const UNAUTHORIZED = { status: 401 };
|
|
|
9772
10086
|
const JSON_CONTENT_TYPE = "application/json; charset=utf-8";
|
|
9773
10087
|
const DELIVERY_TIMEOUT_MS = Number(process.env.PH_WORKFLOWS_WEBHOOK_TIMEOUT_MS) || 3e4;
|
|
9774
10088
|
const TIMED_OUT = Symbol("webhook delivery timed out");
|
|
10089
|
+
const WEBHOOK_REDELIVERED_REASON = "Redelivered: the runtime shut down before this delivery ran, and the sender was told to retry";
|
|
9775
10090
|
/** Resolves to TIMED_OUT, and never keeps the process alive waiting to. */
|
|
9776
10091
|
function timeout(ms) {
|
|
9777
10092
|
return new Promise((resolve) => {
|
|
@@ -9795,7 +10110,7 @@ function splitPollInterval(config) {
|
|
|
9795
10110
|
const { [POLL_INTERVAL_CONFIG_KEY]: raw, ...rest } = config;
|
|
9796
10111
|
const seconds = typeof raw === "string" ? Number(raw) : raw;
|
|
9797
10112
|
if (typeof seconds !== "number" || !Number.isFinite(seconds) || seconds <= 0) {
|
|
9798
|
-
logger.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
|
|
10113
|
+
logger$1.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
|
|
9799
10114
|
return { config: rest };
|
|
9800
10115
|
}
|
|
9801
10116
|
return {
|
|
@@ -9865,7 +10180,7 @@ var WorkflowRuntimeService = class {
|
|
|
9865
10180
|
timeoutMs: host.pieceVersionLookupMs ?? PIECE_VERSION_LOOKUP_TIMEOUT_MS,
|
|
9866
10181
|
hasBlock: (ref, version, source) => this.versionHasBlock(ref, version, source)
|
|
9867
10182
|
});
|
|
9868
|
-
this.logger = host.logger ?? logger;
|
|
10183
|
+
this.logger = host.logger ?? logger$1;
|
|
9869
10184
|
this.attachments = host.attachments ? createAttachmentPort(host.attachments, {
|
|
9870
10185
|
documentIdFor: () => currentWorkflowId(),
|
|
9871
10186
|
canReadRef: async (ref) => {
|
|
@@ -9913,7 +10228,7 @@ var WorkflowRuntimeService = class {
|
|
|
9913
10228
|
}
|
|
9914
10229
|
}
|
|
9915
10230
|
async store() {
|
|
9916
|
-
if (this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
|
|
10231
|
+
if (!this.closed && this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
|
|
9917
10232
|
try {
|
|
9918
10233
|
return await this.storePromise;
|
|
9919
10234
|
} catch {
|
|
@@ -10030,10 +10345,10 @@ var WorkflowRuntimeService = class {
|
|
|
10030
10345
|
if (state.status !== "ENABLED") {
|
|
10031
10346
|
const held = this.registry.get(workflowId);
|
|
10032
10347
|
this.supervisor().releasePark(workflowId, !(held && SUPERVISED_KINDS.has(held.kind))).catch((error) => {
|
|
10033
|
-
this.
|
|
10348
|
+
this.laneFailed(`Could not clear the park of disabled workflow ${workflowId}`, error);
|
|
10034
10349
|
});
|
|
10035
10350
|
} else if (await this.outdatedPark(workflowId, state)) this.supervisor().unpark(workflowId, version).catch((error) => {
|
|
10036
|
-
this.
|
|
10351
|
+
this.laneFailed(`Could not lift the outdated park of workflow ${workflowId}`, error);
|
|
10037
10352
|
});
|
|
10038
10353
|
const park = block ? await this.parks.get(workflowId) : void 0;
|
|
10039
10354
|
if (block && (blockKey(block) === WEBHOOK_BLOCK || triggerKindOf(block)) && park && park.published_version >= version) {
|
|
@@ -10090,7 +10405,7 @@ var WorkflowRuntimeService = class {
|
|
|
10090
10405
|
}
|
|
10091
10406
|
enableSupervised(workflowId, binding, publishedVersion) {
|
|
10092
10407
|
this.supervisor().upsert(binding, publishedVersion).catch((error) => {
|
|
10093
|
-
this.
|
|
10408
|
+
this.laneFailed(`Trigger enable failed for ${workflowId}`, error);
|
|
10094
10409
|
});
|
|
10095
10410
|
}
|
|
10096
10411
|
async registerPieceTrigger(workflowId, binding, publishedVersion) {
|
|
@@ -10147,7 +10462,7 @@ var WorkflowRuntimeService = class {
|
|
|
10147
10462
|
const reason = `The trigger ${blockLabel(binding.block)} is not armed: ${refusal.reason}`;
|
|
10148
10463
|
this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
|
|
10149
10464
|
this.supervisor().reject(workflowId, binding.block, binding.config, reason, retryAt).catch((error) => {
|
|
10150
|
-
this.
|
|
10465
|
+
this.laneFailed(`Could not record the refused trigger for workflow ${workflowId}`, error);
|
|
10151
10466
|
});
|
|
10152
10467
|
}
|
|
10153
10468
|
async registerWebhook(workflowId, block, rawConfig) {
|
|
@@ -10162,7 +10477,7 @@ var WorkflowRuntimeService = class {
|
|
|
10162
10477
|
this.logger.error(`Webhook trigger rejected for ${workflowId}: @error`, message);
|
|
10163
10478
|
this.unarmed.add(workflowId);
|
|
10164
10479
|
this.supervisor().reject(workflowId, block, configRecord(rawConfig), message).catch((recordError) => {
|
|
10165
|
-
this.
|
|
10480
|
+
this.laneFailed(`Could not record the rejected webhook trigger for workflow ${workflowId}`, recordError);
|
|
10166
10481
|
});
|
|
10167
10482
|
return;
|
|
10168
10483
|
}
|
|
@@ -10288,7 +10603,7 @@ var WorkflowRuntimeService = class {
|
|
|
10288
10603
|
const trigger = runnableDefinition(state).trigger;
|
|
10289
10604
|
if (!trigger) return;
|
|
10290
10605
|
this.supervisor().reject(workflowId, triggerBlock(trigger), configRecord(trigger.config), reason).catch((error) => {
|
|
10291
|
-
this.
|
|
10606
|
+
this.laneFailed(`Could not record the reactor access denial for workflow ${workflowId}`, error);
|
|
10292
10607
|
});
|
|
10293
10608
|
}
|
|
10294
10609
|
unarmed = /* @__PURE__ */ new Set();
|
|
@@ -10302,7 +10617,7 @@ var WorkflowRuntimeService = class {
|
|
|
10302
10617
|
this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
|
|
10303
10618
|
this.unarmed.add(workflowId);
|
|
10304
10619
|
this.supervisor().reject(workflowId, block, configRecord(trigger.config), reason, retryAt).catch((error) => {
|
|
10305
|
-
this.
|
|
10620
|
+
this.laneFailed(`Could not record the unresolved trigger for workflow ${workflowId}`, error);
|
|
10306
10621
|
});
|
|
10307
10622
|
}
|
|
10308
10623
|
resolutionRetries = /* @__PURE__ */ new Map();
|
|
@@ -10331,7 +10646,7 @@ var WorkflowRuntimeService = class {
|
|
|
10331
10646
|
}
|
|
10332
10647
|
dropSupervised(workflowId) {
|
|
10333
10648
|
this.supervisor().remove(workflowId).catch((error) => {
|
|
10334
|
-
this.
|
|
10649
|
+
this.laneFailed(`Trigger disable failed for ${workflowId}`, error);
|
|
10335
10650
|
});
|
|
10336
10651
|
}
|
|
10337
10652
|
async refreshRegistration(workflowId, resultingState, onlyIfChanged = false) {
|
|
@@ -10356,6 +10671,7 @@ var WorkflowRuntimeService = class {
|
|
|
10356
10671
|
return false;
|
|
10357
10672
|
}
|
|
10358
10673
|
async onDocumentsPurged(markers) {
|
|
10674
|
+
this.throwIfClosed();
|
|
10359
10675
|
const documentIds = [...new Set(markers.map((marker) => marker.context.documentId))];
|
|
10360
10676
|
const store = await this.store();
|
|
10361
10677
|
if (!store) {
|
|
@@ -10368,8 +10684,10 @@ var WorkflowRuntimeService = class {
|
|
|
10368
10684
|
return erased;
|
|
10369
10685
|
}
|
|
10370
10686
|
async onOperations(operations) {
|
|
10687
|
+
this.throwIfClosed();
|
|
10371
10688
|
const hints = collectLifecycleParentHints(operations);
|
|
10372
10689
|
for (const { operation, context } of operations) {
|
|
10690
|
+
this.throwIfClosed();
|
|
10373
10691
|
if (context.scope !== DOCUMENT_SCOPE && context.scope !== "global") continue;
|
|
10374
10692
|
const opKey = operationKey({
|
|
10375
10693
|
operation,
|
|
@@ -10419,11 +10737,12 @@ var WorkflowRuntimeService = class {
|
|
|
10419
10737
|
this.unjournaledFires.delete(oldest);
|
|
10420
10738
|
this.logger.warn("Too many unjournaled workflow fires held; a redelivery of the oldest fires again");
|
|
10421
10739
|
}
|
|
10422
|
-
this.
|
|
10740
|
+
this.startFiring(workflowId, payload, kind);
|
|
10423
10741
|
}
|
|
10424
10742
|
async enqueueFire(workflowId, payload, kind, dedupeKey) {
|
|
10425
10743
|
const fireKey = JSON.stringify([workflowId, dedupeKey]);
|
|
10426
10744
|
const store = await this.store();
|
|
10745
|
+
this.throwIfClosed();
|
|
10427
10746
|
if (this.unjournaledFires.has(fireKey)) {
|
|
10428
10747
|
if (!store) return;
|
|
10429
10748
|
try {
|
|
@@ -10464,12 +10783,35 @@ var WorkflowRuntimeService = class {
|
|
|
10464
10783
|
}
|
|
10465
10784
|
return;
|
|
10466
10785
|
}
|
|
10467
|
-
this.
|
|
10786
|
+
this.startFiring(workflowId, payload, kind, claim.runId);
|
|
10787
|
+
}
|
|
10788
|
+
async fireFromTrigger(workflowId, payload, kind) {
|
|
10789
|
+
const runId = await this.journalFiring(workflowId, payload, kind);
|
|
10790
|
+
this.startFiring(workflowId, payload, kind, runId);
|
|
10791
|
+
}
|
|
10792
|
+
async journalFiring(workflowId, payload, kind) {
|
|
10793
|
+
const store = await this.store();
|
|
10794
|
+
if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
|
|
10795
|
+
if (!store) return void 0;
|
|
10796
|
+
try {
|
|
10797
|
+
return await store.enqueueRun({
|
|
10798
|
+
workflowId,
|
|
10799
|
+
triggerKind: kind,
|
|
10800
|
+
triggerPayload: payload
|
|
10801
|
+
});
|
|
10802
|
+
} catch (error) {
|
|
10803
|
+
this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
|
|
10804
|
+
return;
|
|
10805
|
+
}
|
|
10468
10806
|
}
|
|
10469
|
-
|
|
10807
|
+
startFiring(workflowId, payload, kind, enqueuedRunId) {
|
|
10470
10808
|
this.fire(workflowId, payload, kind, void 0, void 0, enqueuedRunId).then((run) => {
|
|
10471
10809
|
this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
|
|
10472
10810
|
}, (error) => {
|
|
10811
|
+
if (enqueuedRunId && isShutdownRefusal(error)) {
|
|
10812
|
+
this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to rerun`);
|
|
10813
|
+
return;
|
|
10814
|
+
}
|
|
10473
10815
|
this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
|
|
10474
10816
|
});
|
|
10475
10817
|
}
|
|
@@ -10624,8 +10966,8 @@ var WorkflowRuntimeService = class {
|
|
|
10624
10966
|
const resolved = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolveWithSecrets(connectionId, request);
|
|
10625
10967
|
return rememberSecrets(resolved.auth, resolved.secretValues);
|
|
10626
10968
|
},
|
|
10627
|
-
fire: (workflowId, payload, kind) => {
|
|
10628
|
-
this.
|
|
10969
|
+
fire: (workflowId, payload, kind, runId) => {
|
|
10970
|
+
this.startFiring(workflowId, payload, kind, runId);
|
|
10629
10971
|
},
|
|
10630
10972
|
webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
|
|
10631
10973
|
reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
|
|
@@ -10637,15 +10979,29 @@ var WorkflowRuntimeService = class {
|
|
|
10637
10979
|
defaultIntervalMs: Number(process.env.PH_WORKFLOWS_POLL_INTERVAL_MS) || void 0,
|
|
10638
10980
|
reconcileIntervalMs: Number(process.env.PH_WORKFLOWS_WEBHOOK_RECONCILE_MS) || void 0
|
|
10639
10981
|
});
|
|
10982
|
+
if (this.closed) this.triggerSupervisor.stop();
|
|
10640
10983
|
return this.triggerSupervisor;
|
|
10641
10984
|
}
|
|
10985
|
+
laneFailed(message, error) {
|
|
10986
|
+
if (isShutdownRefusal(error)) {
|
|
10987
|
+
this.logger.debug(message, error);
|
|
10988
|
+
return;
|
|
10989
|
+
}
|
|
10990
|
+
this.logger.error(message, error);
|
|
10991
|
+
}
|
|
10642
10992
|
startTriggerSupervisor() {
|
|
10643
10993
|
this.supervisor().start();
|
|
10644
10994
|
}
|
|
10645
10995
|
stopTriggerSupervisor() {
|
|
10646
10996
|
this.triggerSupervisor?.stop();
|
|
10647
10997
|
}
|
|
10998
|
+
closed = false;
|
|
10999
|
+
throwIfClosed() {
|
|
11000
|
+
if (this.closed) throw new Error("This workflow runtime has shut down; the operations are left for the next owner");
|
|
11001
|
+
}
|
|
10648
11002
|
shutdown() {
|
|
11003
|
+
this.closed = true;
|
|
11004
|
+
this.runGate.close();
|
|
10649
11005
|
clearInterval(this.retentionTimer);
|
|
10650
11006
|
for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
|
|
10651
11007
|
this.resolutionRetries.clear();
|
|
@@ -10755,6 +11111,10 @@ var WorkflowRuntimeService = class {
|
|
|
10755
11111
|
/** A delivery the service has already rate-limited, verified, de-duplicated and
|
|
10756
11112
|
* answered any challenge for; all that is left is deciding what it means. */
|
|
10757
11113
|
async deliverWebhook(request) {
|
|
11114
|
+
if (this.closed) return {
|
|
11115
|
+
status: 503,
|
|
11116
|
+
unprocessed: true
|
|
11117
|
+
};
|
|
10758
11118
|
const workflowId = request.key;
|
|
10759
11119
|
const registration = this.liveWebhook(workflowId);
|
|
10760
11120
|
const test = this.webhookTests.get(workflowId);
|
|
@@ -10769,16 +11129,44 @@ var WorkflowRuntimeService = class {
|
|
|
10769
11129
|
const { config } = registration;
|
|
10770
11130
|
const payload = webhookPayload(request);
|
|
10771
11131
|
if (config.responseMode === "async") {
|
|
10772
|
-
|
|
11132
|
+
try {
|
|
11133
|
+
await this.fireFromTrigger(workflowId, payload, WEBHOOK_TRIGGER_KIND);
|
|
11134
|
+
} catch (error) {
|
|
11135
|
+
if (error instanceof WorkflowRuntimeClosedError) return {
|
|
11136
|
+
status: 503,
|
|
11137
|
+
unprocessed: true
|
|
11138
|
+
};
|
|
11139
|
+
throw error;
|
|
11140
|
+
}
|
|
10773
11141
|
return { status: config.responseStatus };
|
|
10774
11142
|
}
|
|
10775
|
-
|
|
11143
|
+
let runId;
|
|
11144
|
+
try {
|
|
11145
|
+
runId = await this.journalFiring(workflowId, payload, WEBHOOK_TRIGGER_KIND);
|
|
11146
|
+
} catch (error) {
|
|
11147
|
+
if (error instanceof WorkflowRuntimeClosedError) return {
|
|
11148
|
+
status: 503,
|
|
11149
|
+
unprocessed: true
|
|
11150
|
+
};
|
|
11151
|
+
throw error;
|
|
11152
|
+
}
|
|
11153
|
+
const firing = this.fire(workflowId, payload, WEBHOOK_TRIGGER_KIND, void 0, void 0, runId).then((result) => ({
|
|
10776
11154
|
ok: true,
|
|
10777
11155
|
result
|
|
10778
11156
|
}), (error) => ({
|
|
10779
11157
|
ok: false,
|
|
10780
11158
|
error
|
|
10781
|
-
}))
|
|
11159
|
+
}));
|
|
11160
|
+
let replied = false;
|
|
11161
|
+
const takeReply = () => !replied && (replied = true);
|
|
11162
|
+
if (runId) this.heldReplies.set(runId, takeReply);
|
|
11163
|
+
let run;
|
|
11164
|
+
try {
|
|
11165
|
+
run = await Promise.race([firing, timeout(DELIVERY_TIMEOUT_MS)]);
|
|
11166
|
+
if (run === TIMED_OUT && !takeReply()) run = await firing;
|
|
11167
|
+
} finally {
|
|
11168
|
+
if (runId) this.heldReplies.delete(runId);
|
|
11169
|
+
}
|
|
10782
11170
|
if (run === TIMED_OUT) {
|
|
10783
11171
|
this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
|
|
10784
11172
|
return {
|
|
@@ -10791,6 +11179,10 @@ var WorkflowRuntimeService = class {
|
|
|
10791
11179
|
};
|
|
10792
11180
|
}
|
|
10793
11181
|
if (!run.ok) {
|
|
11182
|
+
if (run.error instanceof WorkflowRuntimeClosedError) return {
|
|
11183
|
+
status: 503,
|
|
11184
|
+
unprocessed: true
|
|
11185
|
+
};
|
|
10794
11186
|
const message = run.error instanceof Error ? run.error.message : String(run.error);
|
|
10795
11187
|
this.logger.error(`Webhook run failed for ${workflowId}: @error`, message);
|
|
10796
11188
|
return {
|
|
@@ -10815,7 +11207,7 @@ var WorkflowRuntimeService = class {
|
|
|
10815
11207
|
try {
|
|
10816
11208
|
return handshakeReply((await this.supervisor().handshake(binding, webhookPayload(request))).output);
|
|
10817
11209
|
} catch (error) {
|
|
10818
|
-
this.logger
|
|
11210
|
+
this.logger[isShutdownRefusal(error) ? "debug" : "error"]("Handshake failed for @block on workflow @workflow", blockLabel(binding.block), binding.workflowId, error);
|
|
10819
11211
|
return { status: 500 };
|
|
10820
11212
|
}
|
|
10821
11213
|
}
|
|
@@ -10834,7 +11226,7 @@ var WorkflowRuntimeService = class {
|
|
|
10834
11226
|
this.supervisor().deliverWebhook(binding.workflowId, payload).then(() => {
|
|
10835
11227
|
this.logger.info("Webhook delivered to @block for workflow @workflow", blockLabel(binding.block), binding.workflowId);
|
|
10836
11228
|
}, (error) => {
|
|
10837
|
-
this.
|
|
11229
|
+
this.laneFailed(`Webhook delivery failed for workflow ${binding.workflowId}`, error);
|
|
10838
11230
|
});
|
|
10839
11231
|
return { status: 200 };
|
|
10840
11232
|
}
|
|
@@ -11301,14 +11693,17 @@ var WorkflowRuntimeService = class {
|
|
|
11301
11693
|
const local = await this.localPieceAt(packageName, version);
|
|
11302
11694
|
return local ? triggersResult(local.descriptor, local.piece.name, local.piece.version) : fetchPieceTriggers(packageName, version);
|
|
11303
11695
|
}
|
|
11304
|
-
|
|
11696
|
+
warmPieceSearch() {
|
|
11697
|
+
warmSearchIndex();
|
|
11698
|
+
}
|
|
11699
|
+
async searchPieces(query, filter) {
|
|
11305
11700
|
let local;
|
|
11306
11701
|
try {
|
|
11307
|
-
local =
|
|
11702
|
+
local = indexPieces((await this.localPieces()).map(({ piece, descriptor }) => localSearchPiece(descriptor, piece.name, piece.version)));
|
|
11308
11703
|
} catch (error) {
|
|
11309
11704
|
this.logger.warn("Could not index the package pieces: @error", String(error));
|
|
11310
11705
|
}
|
|
11311
|
-
return
|
|
11706
|
+
return searchPieces(query, filter, local);
|
|
11312
11707
|
}
|
|
11313
11708
|
async pieceDetail(packageName, version) {
|
|
11314
11709
|
const local = await this.localPieceAt(packageName, version);
|
|
@@ -11714,6 +12109,7 @@ var WorkflowRuntimeService = class {
|
|
|
11714
12109
|
return this.pieceWorkers = pool;
|
|
11715
12110
|
}
|
|
11716
12111
|
async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
|
|
12112
|
+
if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
|
|
11717
12113
|
const started = performance.now();
|
|
11718
12114
|
const attributes = {
|
|
11719
12115
|
"workflow.id": workflowId,
|
|
@@ -11795,7 +12191,14 @@ var WorkflowRuntimeService = class {
|
|
|
11795
12191
|
}
|
|
11796
12192
|
if (parked) return skipped(parked, "parked");
|
|
11797
12193
|
const admission = await this.runGate.admit(workflowId, policy);
|
|
11798
|
-
if (
|
|
12194
|
+
if (this.closed) {
|
|
12195
|
+
if (admission.admitted) admission.release();
|
|
12196
|
+
return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12197
|
+
}
|
|
12198
|
+
if (!admission.admitted) {
|
|
12199
|
+
if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12200
|
+
return skipped(admission.reason, admission.refusal);
|
|
12201
|
+
}
|
|
11799
12202
|
if (admission.waited) {
|
|
11800
12203
|
let stale;
|
|
11801
12204
|
try {
|
|
@@ -11900,6 +12303,7 @@ var WorkflowRuntimeService = class {
|
|
|
11900
12303
|
}
|
|
11901
12304
|
}
|
|
11902
12305
|
runGate = new WorkflowRunGate();
|
|
12306
|
+
heldReplies = /* @__PURE__ */ new Map();
|
|
11903
12307
|
/** Why a trigger's firing of a PARKED workflow is refused; undefined when
|
|
11904
12308
|
* it is not parked, or when an operator started the run. */
|
|
11905
12309
|
async parkedFiring(workflowId, triggerKind, publishedVersion) {
|
|
@@ -11941,6 +12345,19 @@ var WorkflowRuntimeService = class {
|
|
|
11941
12345
|
* of bug this work package exists to stamp out. An already-enqueued row is
|
|
11942
12346
|
* closed out in place, so nothing is left PENDING for a sweep to find.
|
|
11943
12347
|
*/
|
|
12348
|
+
async refuseClosed(store, workflowId, enqueuedRunId) {
|
|
12349
|
+
const error = new WorkflowRuntimeClosedError(workflowId);
|
|
12350
|
+
if (enqueuedRunId) {
|
|
12351
|
+
const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
|
|
12352
|
+
try {
|
|
12353
|
+
if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
|
|
12354
|
+
else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));
|
|
12355
|
+
} catch (failure) {
|
|
12356
|
+
this.logger.warn(`Could not close out run ${enqueuedRunId} after shutdown: @error`, failure);
|
|
12357
|
+
}
|
|
12358
|
+
}
|
|
12359
|
+
throw error;
|
|
12360
|
+
}
|
|
11944
12361
|
async skipFiring(store, workflowId, enqueuedRunId, details, refusal) {
|
|
11945
12362
|
this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
|
|
11946
12363
|
let runId = enqueuedRunId ?? null;
|
|
@@ -11974,7 +12391,7 @@ var WorkflowRuntimeService = class {
|
|
|
11974
12391
|
* engine has; it is marked as such rather than pretending to page anyone.
|
|
11975
12392
|
*/
|
|
11976
12393
|
async applyFailureMode(policy, workflowId, publishedVersion, runId, error) {
|
|
11977
|
-
if (policy.onFailure === "IGNORE") return;
|
|
12394
|
+
if (policy.onFailure === "IGNORE" || this.closed) return;
|
|
11978
12395
|
const detail = error ?? "the run failed";
|
|
11979
12396
|
if (policy.onFailure === "NOTIFY") {
|
|
11980
12397
|
this.logger.error(`Workflow ${workflowId} run ${runId ?? "(unjournaled)"} failed and its policy is NOTIFY: @error`, detail);
|
|
@@ -11984,6 +12401,7 @@ var WorkflowRuntimeService = class {
|
|
|
11984
12401
|
const reason = `Parked after a failed run (policy.onFailure = PARK): ${detail}`;
|
|
11985
12402
|
try {
|
|
11986
12403
|
if (!await this.stillRunsVersion(workflowId, publishedVersion)) return;
|
|
12404
|
+
if (this.closed) return;
|
|
11987
12405
|
const trigger = await this.supervisor().park(workflowId, publishedVersion, reason);
|
|
11988
12406
|
if (!await this.stillRunsVersion(workflowId, publishedVersion)) {
|
|
11989
12407
|
await this.supervisor().liftPark(workflowId, publishedVersion, trigger);
|
|
@@ -12369,6 +12787,249 @@ function collectAttachmentRefs(value, found) {
|
|
|
12369
12787
|
else if (typeof value === "object" && value !== null) for (const entry of Object.values(value)) collectAttachmentRefs(entry, found);
|
|
12370
12788
|
}
|
|
12371
12789
|
//#endregion
|
|
12790
|
+
//#region src/reactor/singleton-lease.ts
|
|
12791
|
+
const logger = childLogger([
|
|
12792
|
+
"workflow",
|
|
12793
|
+
"runtime",
|
|
12794
|
+
"singleton"
|
|
12795
|
+
]);
|
|
12796
|
+
/**
|
|
12797
|
+
* Names this process as the workflow singleton's owner.
|
|
12798
|
+
*
|
|
12799
|
+
* Set it to a STABLE name per deployment slot (`switchboard-0`, the
|
|
12800
|
+
* StatefulSet pod name, the Render service id). The owner name is the
|
|
12801
|
+
* operator's contract: a restart under the same name re-claims its own lease
|
|
12802
|
+
* once the old heartbeat is stale instead of waiting out
|
|
12803
|
+
* {@link SINGLETON_LEASE_TTL_MS}.
|
|
12804
|
+
*
|
|
12805
|
+
* Unset, the owner is derived from a STABLE identity — the hostname and a
|
|
12806
|
+
* fingerprint of the journal's own storage location ({@link
|
|
12807
|
+
* AcquireSingletonOptions.storageId}) — not from the pid and a random suffix.
|
|
12808
|
+
* A random per-process owner meant every unclean kill locked the next boot out
|
|
12809
|
+
* for the whole {@link SINGLETON_LEASE_TTL_MS}: the dead process's lease was
|
|
12810
|
+
* nobody's to re-claim, and `release()` never ran. One deployment slot
|
|
12811
|
+
* restarting is the common case and it has to be quick; a genuine second
|
|
12812
|
+
* replica still differs, by hostname or by the journal it points at.
|
|
12813
|
+
*/
|
|
12814
|
+
const WORKFLOW_SINGLETON_OWNER_ENV = "PH_WORKFLOWS_SINGLETON_OWNER";
|
|
12815
|
+
/** How long a claim stays valid without a heartbeat. */
|
|
12816
|
+
const SINGLETON_LEASE_TTL_MS = 6e4;
|
|
12817
|
+
/** How often the holder renews it; a third of the TTL, so two renewals may be
|
|
12818
|
+
* lost (a GC pause, a slow database) before anyone may take over. */
|
|
12819
|
+
const SINGLETON_HEARTBEAT_MS = SINGLETON_LEASE_TTL_MS / 3;
|
|
12820
|
+
/**
|
|
12821
|
+
* How long the holder's heartbeat must have been silent before a claim under
|
|
12822
|
+
* the SAME owner name takes the lease over, short of its expiry. Two renewal
|
|
12823
|
+
* periods: a live holder renews well inside it, and its self-fence fires
|
|
12824
|
+
* before it passes.
|
|
12825
|
+
*/
|
|
12826
|
+
const SINGLETON_STALE_HEARTBEATS = 2;
|
|
12827
|
+
/** The one row. The table holds a single lease, named rather than keyed by a
|
|
12828
|
+
* magic empty string, so a `SELECT *` reads legibly in an operator's shell. */
|
|
12829
|
+
const LEASE_ID = "workflow-runtime";
|
|
12830
|
+
const dbNow = () => sql`now()`;
|
|
12831
|
+
const dbNowPlus = (ms) => sql`now() + ${ms} * interval '1 millisecond'`;
|
|
12832
|
+
/** Another live process holds the singleton. The refusal is the point: a host
|
|
12833
|
+
* that cannot own workflow execution must not compose the engine. */
|
|
12834
|
+
var WorkflowSingletonConflictError = class extends Error {
|
|
12835
|
+
owner;
|
|
12836
|
+
expiresAt;
|
|
12837
|
+
wouldBe;
|
|
12838
|
+
constructor(owner, expiresAt, wouldBe) {
|
|
12839
|
+
super(`Workflow execution is a singleton and "${owner}" holds it until ${expiresAt}; this process ("${wouldBe}") must not compose the workflow runtime. Two replicas over one run journal fail each other's live runs and arm every trigger twice. Run workflows on exactly one reactor, or set ${WORKFLOW_SINGLETON_OWNER_ENV} to the same stable owner name on the slot that owns them so a restart re-claims its own lease.`);
|
|
12840
|
+
this.owner = owner;
|
|
12841
|
+
this.expiresAt = expiresAt;
|
|
12842
|
+
this.wouldBe = wouldBe;
|
|
12843
|
+
this.name = "WorkflowSingletonConflictError";
|
|
12844
|
+
}
|
|
12845
|
+
};
|
|
12846
|
+
/**
|
|
12847
|
+
* The owner name this process claims under.
|
|
12848
|
+
*
|
|
12849
|
+
* `<hostname>/<storage fingerprint>`, so it is the same name on every boot of
|
|
12850
|
+
* one deployment slot — a restart re-claims its OWN lease once the dead
|
|
12851
|
+
* process's heartbeat is stale, rather than waiting out the TTL — and a different
|
|
12852
|
+
* name on any host or journal that is genuinely somebody else.
|
|
12853
|
+
*
|
|
12854
|
+
* The storage location is hashed, never printed: it can be a Postgres URL with
|
|
12855
|
+
* credentials in it, and the owner name goes into a database row an operator
|
|
12856
|
+
* reads and into every log line about the lease.
|
|
12857
|
+
*
|
|
12858
|
+
* The residual case a stable name cannot tell apart is two processes on ONE
|
|
12859
|
+
* host over ONE journal, which is a misconfiguration those two already share
|
|
12860
|
+
* (one read-model directory, one run journal). It is not silent: the loser's
|
|
12861
|
+
* heartbeat finds the lease taken and says so by name.
|
|
12862
|
+
*/
|
|
12863
|
+
function singletonOwnerName(env = process.env, storageId) {
|
|
12864
|
+
const configured = env[WORKFLOW_SINGLETON_OWNER_ENV]?.trim();
|
|
12865
|
+
if (configured) return configured;
|
|
12866
|
+
return `${hostname()}/${storageFingerprint(storageId)}`;
|
|
12867
|
+
}
|
|
12868
|
+
function storageFingerprint(storageId) {
|
|
12869
|
+
const source = storageId?.trim();
|
|
12870
|
+
if (!source) return "default";
|
|
12871
|
+
return createHash("sha256").update(source).digest("hex").slice(0, 12);
|
|
12872
|
+
}
|
|
12873
|
+
async function ensureTable(db) {
|
|
12874
|
+
await db.schema.createTable("workflow_singleton").addColumn("id", "text", (col) => col.primaryKey()).addColumn("owner", "text", (col) => col.notNull()).addColumn("instance", "text", (col) => col.notNull()).addColumn("acquired_at", "timestamptz", (col) => col.notNull()).addColumn("heartbeat_at", "timestamptz", (col) => col.notNull()).addColumn("expires_at", "timestamptz", (col) => col.notNull()).ifNotExists().execute();
|
|
12875
|
+
}
|
|
12876
|
+
const DDL_ATTEMPTS = 3;
|
|
12877
|
+
const CONCURRENT_DDL_CODES = /* @__PURE__ */ new Set([
|
|
12878
|
+
"23505",
|
|
12879
|
+
"42710",
|
|
12880
|
+
"42P06",
|
|
12881
|
+
"42P07"
|
|
12882
|
+
]);
|
|
12883
|
+
function isConcurrentDdl(error) {
|
|
12884
|
+
const code = error?.code;
|
|
12885
|
+
return typeof code === "string" && CONCURRENT_DDL_CODES.has(code);
|
|
12886
|
+
}
|
|
12887
|
+
async function openLeaseTable(relationalDb) {
|
|
12888
|
+
const open = async () => {
|
|
12889
|
+
const db = await relationalDb.createNamespace("workflow_runtime");
|
|
12890
|
+
await ensureTable(db);
|
|
12891
|
+
return db;
|
|
12892
|
+
};
|
|
12893
|
+
for (let attempt = 1;; attempt++) try {
|
|
12894
|
+
return await open();
|
|
12895
|
+
} catch (error) {
|
|
12896
|
+
if (attempt >= DDL_ATTEMPTS || !isConcurrentDdl(error)) throw error;
|
|
12897
|
+
}
|
|
12898
|
+
}
|
|
12899
|
+
/**
|
|
12900
|
+
* Claims the workflow singleton, or refuses.
|
|
12901
|
+
*
|
|
12902
|
+
* Three steps, each one a single statement whose own outcome is the claim — a
|
|
12903
|
+
* prior `SELECT` would be a race:
|
|
12904
|
+
*
|
|
12905
|
+
* 1. `INSERT … ON CONFLICT DO NOTHING RETURNING` — the first ever boot.
|
|
12906
|
+
* 2. `UPDATE … WHERE expires_at <= now OR (owner = me AND heartbeat stale)
|
|
12907
|
+
* RETURNING` — a dead lease taken over once it has expired, or our own
|
|
12908
|
+
* slot's lease once its holder stopped renewing (a restart under a stable
|
|
12909
|
+
* owner name), never a live holder's. Either way the row gets this claim's
|
|
12910
|
+
* instance, so the previous holder's heartbeat and release no longer
|
|
12911
|
+
* match.
|
|
12912
|
+
* 3. Neither matched: someone live holds it, and we refuse by name.
|
|
12913
|
+
*/
|
|
12914
|
+
async function acquireWorkflowSingletonLease(options) {
|
|
12915
|
+
const log = options.logger ?? logger;
|
|
12916
|
+
const ttlMs = options.ttlMs ?? 6e4;
|
|
12917
|
+
const heartbeatMs = options.heartbeatMs ?? 2e4;
|
|
12918
|
+
const staleMs = heartbeatMs * 2;
|
|
12919
|
+
const owner = options.owner ?? singletonOwnerName(options.env, options.storageId);
|
|
12920
|
+
const instance = randomUUID();
|
|
12921
|
+
const db = await openLeaseTable(options.relationalDb);
|
|
12922
|
+
const stamps = () => ({
|
|
12923
|
+
heartbeat_at: dbNow(),
|
|
12924
|
+
expires_at: dbNowPlus(ttlMs)
|
|
12925
|
+
});
|
|
12926
|
+
const claim = async () => {
|
|
12927
|
+
if (await db.insertInto("workflow_singleton").values({
|
|
12928
|
+
id: LEASE_ID,
|
|
12929
|
+
owner,
|
|
12930
|
+
instance,
|
|
12931
|
+
acquired_at: dbNow(),
|
|
12932
|
+
...stamps()
|
|
12933
|
+
}).onConflict((oc) => oc.column("id").doNothing()).returning("owner").executeTakeFirst() !== void 0) return true;
|
|
12934
|
+
return await db.updateTable("workflow_singleton").set({
|
|
12935
|
+
owner,
|
|
12936
|
+
instance,
|
|
12937
|
+
acquired_at: dbNow(),
|
|
12938
|
+
...stamps()
|
|
12939
|
+
}).where("id", "=", LEASE_ID).where((eb) => eb.or([eb("expires_at", "<=", dbNow()), eb.and([eb("owner", "=", owner), eb("heartbeat_at", "<=", dbNowPlus(-staleMs))])])).returning("owner").executeTakeFirst() !== void 0;
|
|
12940
|
+
};
|
|
12941
|
+
const claimSentAt = performance.now();
|
|
12942
|
+
if (!await claim()) {
|
|
12943
|
+
const held = await db.selectFrom("workflow_singleton").selectAll().where("id", "=", LEASE_ID).executeTakeFirst();
|
|
12944
|
+
throw new WorkflowSingletonConflictError(held?.owner ?? "unknown", held ? new Date(held.expires_at).toISOString() : "unknown", owner);
|
|
12945
|
+
}
|
|
12946
|
+
log.info(`Workflow singleton claimed by "${owner}" (lease ${ttlMs}ms, renewed every ${heartbeatMs}ms)`);
|
|
12947
|
+
const localNow = () => performance.now();
|
|
12948
|
+
const tickMs = heartbeatMs / 4;
|
|
12949
|
+
const fenceMs = Math.min(staleMs, ttlMs) - Math.min(heartbeatMs, ttlMs) / 2;
|
|
12950
|
+
const selfFence = options.selfFence ?? true;
|
|
12951
|
+
let renewedAt = claimSentAt;
|
|
12952
|
+
let renewFailed = false;
|
|
12953
|
+
let inFlight;
|
|
12954
|
+
let timer;
|
|
12955
|
+
let lost = false;
|
|
12956
|
+
let takenOver = false;
|
|
12957
|
+
let releasing = false;
|
|
12958
|
+
const stop = () => {
|
|
12959
|
+
if (timer) clearInterval(timer);
|
|
12960
|
+
timer = void 0;
|
|
12961
|
+
};
|
|
12962
|
+
const markLost = (loss) => {
|
|
12963
|
+
if (lost) return;
|
|
12964
|
+
lost = true;
|
|
12965
|
+
stop();
|
|
12966
|
+
log.error(loss.reason === "taken" ? `Workflow singleton lease of "${owner}" was taken over by "${loss.heldBy ?? "nobody"}"; this process no longer owns workflow execution and stops running it.` : `Workflow singleton lease of "${owner}" could not be renewed for ${Math.round(loss.silentMs)}ms, long enough that another process could take it over (none was seen to); this process stops running workflows.`);
|
|
12967
|
+
options.onLost?.(loss);
|
|
12968
|
+
};
|
|
12969
|
+
const renew = async () => {
|
|
12970
|
+
const sentAt = localNow();
|
|
12971
|
+
if (await db.updateTable("workflow_singleton").set(stamps()).where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).returning("owner").executeTakeFirst() !== void 0) {
|
|
12972
|
+
renewedAt = sentAt;
|
|
12973
|
+
return true;
|
|
12974
|
+
}
|
|
12975
|
+
if (releasing || lost) return false;
|
|
12976
|
+
let heldBy;
|
|
12977
|
+
try {
|
|
12978
|
+
heldBy = (await db.selectFrom("workflow_singleton").select("owner").where("id", "=", LEASE_ID).executeTakeFirst())?.owner;
|
|
12979
|
+
} catch {
|
|
12980
|
+
heldBy = void 0;
|
|
12981
|
+
}
|
|
12982
|
+
takenOver = true;
|
|
12983
|
+
markLost({
|
|
12984
|
+
reason: "taken",
|
|
12985
|
+
heldBy
|
|
12986
|
+
});
|
|
12987
|
+
return false;
|
|
12988
|
+
};
|
|
12989
|
+
const heartbeat = () => {
|
|
12990
|
+
if (lost) return Promise.resolve(false);
|
|
12991
|
+
inFlight ??= renew().finally(() => {
|
|
12992
|
+
inFlight = void 0;
|
|
12993
|
+
});
|
|
12994
|
+
return inFlight;
|
|
12995
|
+
};
|
|
12996
|
+
timer = setInterval(() => {
|
|
12997
|
+
if (lost || releasing) return;
|
|
12998
|
+
const silentMs = localNow() - renewedAt;
|
|
12999
|
+
if (selfFence && silentMs >= fenceMs) {
|
|
13000
|
+
markLost({
|
|
13001
|
+
reason: "unrenewable",
|
|
13002
|
+
silentMs
|
|
13003
|
+
});
|
|
13004
|
+
return;
|
|
13005
|
+
}
|
|
13006
|
+
if (inFlight) return;
|
|
13007
|
+
if (!renewFailed && silentMs < heartbeatMs - tickMs / 2) return;
|
|
13008
|
+
heartbeat().then(() => {
|
|
13009
|
+
renewFailed = false;
|
|
13010
|
+
}, (error) => {
|
|
13011
|
+
renewFailed = true;
|
|
13012
|
+
log.warn("Workflow singleton heartbeat failed: @error", error);
|
|
13013
|
+
});
|
|
13014
|
+
}, tickMs);
|
|
13015
|
+
timer.unref();
|
|
13016
|
+
return {
|
|
13017
|
+
owner,
|
|
13018
|
+
heartbeat,
|
|
13019
|
+
async release() {
|
|
13020
|
+
releasing = true;
|
|
13021
|
+
stop();
|
|
13022
|
+
if (takenOver) return;
|
|
13023
|
+
if (inFlight) await Promise.race([inFlight.catch(() => false), new Promise((resolve) => setTimeout(resolve, tickMs).unref())]);
|
|
13024
|
+
try {
|
|
13025
|
+
await db.deleteFrom("workflow_singleton").where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).execute();
|
|
13026
|
+
} catch (error) {
|
|
13027
|
+
log.warn("Could not release the workflow singleton lease: @error", error);
|
|
13028
|
+
}
|
|
13029
|
+
}
|
|
13030
|
+
};
|
|
13031
|
+
}
|
|
13032
|
+
//#endregion
|
|
12372
13033
|
//#region src/reactor/workflow-triggers-read-model.ts
|
|
12373
13034
|
const WORKFLOW_TRIGGERS_READ_MODEL = "workflow-triggers";
|
|
12374
13035
|
const WORKFLOW_TRIGGERS_READ_MODEL_STAGE = "post_ready";
|
|
@@ -12395,6 +13056,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
|
|
|
12395
13056
|
}
|
|
12396
13057
|
};
|
|
12397
13058
|
//#endregion
|
|
12398
|
-
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
|
|
13059
|
+
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl, singletonOwnerName };
|
|
12399
13060
|
|
|
12400
13061
|
//# sourceMappingURL=index.js.map
|