@powerhousedao/reactor-workflow 6.2.3-dev.50 → 6.2.3-dev.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- import { $ as parseAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerTimeoutError, H as PieceWorkerPool, I as SecretDeletedError, J as redactMessage, K as containsRedactedMarker, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as isPrivateAddress, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerError, X as secretsFor, Y as rememberSecrets, Z as DEFAULT_EGRESS_POLICY, _ as resolvedBlock, at as sourcedResolver, b as resolutionOf, c as canonicalJson, ct as authMethodFor, d as DynamicPropertiesError, et as extractDedupeKey, f as checkDynamicProperties, g as isHostBound, gt as isIndeterminateError, h as CompositeBlockExecutor, ht as setPieceRegistryUrl, i as packagePieces, it as pieceModuleRef, j as declaredReactorConnectionIds, l as hashOf, lt as unsupportedAuth, m as ActivepiecesBlockExecutor, mt as pieceRegistrySource, n as installedPiece, nt as maxFileBytes, o as errorNameOf, p as effectiveRetryPolicy, pt as npmRegistryBaseUrl, q as redact, r as installedPieces, rt as bundleResolver, s as runWorkflow, st as UnsupportedPieceFeatureError, t as PieceRegistry, tt as FileTooLargeError, u as stepConfigHash, ut as unsupportedTrigger, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-BKRLK7Kv.js";
1
+ import { $ as isPrivateAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerError, H as PieceWorkerPool, I as SecretDeletedError, J as redact, K as PieceWorkerTimeoutError, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as DEFAULT_EGRESS_POLICY, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerDisposedError, X as rememberSecrets, Y as redactMessage, Z as secretsFor, _ as resolvedBlock, _t as isIndeterminateError, at as pieceModuleRef, b as resolutionOf, c as canonicalJson, ct as UnsupportedPieceFeatureError, d as DynamicPropertiesError, dt as unsupportedTrigger, et as parseAddress, f as checkDynamicProperties, g as isHostBound, gt as setPieceRegistryUrl, h as CompositeBlockExecutor, ht as pieceRegistrySource, i as packagePieces, it as bundleResolver, j as declaredReactorConnectionIds, l as hashOf, lt as authMethodFor, m as ActivepiecesBlockExecutor, mt as npmRegistryBaseUrl, n as installedPiece, nt as FileTooLargeError, o as errorNameOf, ot as sourcedResolver, p as effectiveRetryPolicy, q as containsRedactedMarker, r as installedPieces, rt as maxFileBytes, s as runWorkflow, t as PieceRegistry, tt as extractDedupeKey, u as stepConfigHash, ut as unsupportedAuth, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-D0pa4v0T.js";
2
2
  import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
3
3
  import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
4
4
  import { open, readdir, rm, stat } from "node:fs/promises";
@@ -11,6 +11,7 @@ import { Readable } from "node:stream";
11
11
  import dns from "node:dns";
12
12
  import net from "node:net";
13
13
  import { AsyncLocalStorage } from "node:async_hooks";
14
+ import { hostname } from "node:os";
14
15
  import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
15
16
  import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
16
17
  import http from "node:http";
@@ -99,7 +100,7 @@ function npmVersions(name, options = {}) {
99
100
  }
100
101
  //#endregion
101
102
  //#region src/reactor/attachment-port.ts
102
- const logger$6 = childLogger(["workflow", "attachments"]);
103
+ const logger$7 = childLogger(["workflow", "attachments"]);
103
104
  function fileStream(filePath) {
104
105
  return Readable.toWeb(createReadStream(filePath));
105
106
  }
@@ -190,7 +191,7 @@ function createAttachmentPort(client, access) {
190
191
  ...signal ? { signal } : {}
191
192
  }));
192
193
  access.onWritten?.(ref);
193
- logger$6.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
194
+ logger$7.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
194
195
  return ref;
195
196
  }
196
197
  };
@@ -804,7 +805,7 @@ const CACHE_TTL_MS = 3e5;
804
805
  const PINNED_TTL_MS = 36e5;
805
806
  const REGISTRY_PAGE_SIZE = 50;
806
807
  const REGISTRY_MAX_PAGES = 200;
807
- const logger$5 = childLogger(["workflow", "piece-catalog"]);
808
+ const logger$6 = childLogger(["workflow", "piece-catalog"]);
808
809
  function aiLast(audience) {
809
810
  return audience === "ai" ? 1 : 0;
810
811
  }
@@ -850,7 +851,7 @@ async function fetchRegistryCatalog(url, timeoutMs) {
850
851
  items.push(...page.items);
851
852
  if (!page.hasMore || page.items.length === 0) return items;
852
853
  }
853
- logger$5.warn(`${url} still had more pieces after ${REGISTRY_MAX_PAGES} pages`);
854
+ logger$6.warn(`${url} still had more pieces after ${REGISTRY_MAX_PAGES} pages`);
854
855
  return items;
855
856
  }
856
857
  function asList(value) {
@@ -864,10 +865,10 @@ async function publishedLists(suggestions, timeoutMs) {
864
865
  const cloudUrl = suggestions ? `${CATALOG_URL}?suggestionType=ACTION_AND_TRIGGER` : CATALOG_URL;
865
866
  const [fromRegistry, fromCloud] = await Promise.allSettled([source ? fetchRegistryCatalog(source.catalogUrl(suggestions), timeoutMs) : Promise.resolve([]), fetchJson(cloudUrl, timeoutMs)]);
866
867
  const registry = fromRegistry.status === "fulfilled" ? asList(fromRegistry.value).filter((entry) => typeof entry.name === "string") : [];
867
- if (fromRegistry.status === "rejected") logger$5.warn("Piece registry @url did not answer: @error", source?.baseUrl ?? "?", String(fromRegistry.reason));
868
+ if (fromRegistry.status === "rejected") logger$6.warn("Piece registry @url did not answer: @error", source?.baseUrl ?? "?", String(fromRegistry.reason));
868
869
  if (fromCloud.status === "rejected") {
869
870
  if (registry.length === 0) throw asError(fromCloud.reason);
870
- logger$5.warn("Serving registry pieces only: @error", String(fromCloud.reason));
871
+ logger$6.warn("Serving registry pieces only: @error", String(fromCloud.reason));
871
872
  return {
872
873
  registry,
873
874
  cloud: []
@@ -882,12 +883,18 @@ function registryFirst(registry, cloud) {
882
883
  const claimed = new Set(registry.map((entry) => entry.name));
883
884
  return [...registry, ...cloud.filter((entry) => !claimed.has(entry.name))];
884
885
  }
886
+ function tagged(entries, source) {
887
+ return entries.map((entry) => ({
888
+ ...entry,
889
+ source
890
+ }));
891
+ }
885
892
  async function fetchCatalogWithSuggestions() {
886
893
  const { registry, cloud } = await publishedLists(true, 12e4);
887
- return registryFirst(registry, cloud);
894
+ return registryFirst(tagged(registry, "registry"), tagged(cloud, "activepieces"));
888
895
  }
889
896
  let catalogCache;
890
- function toSummaries(raw) {
897
+ function toSummaries(raw, source) {
891
898
  return raw.filter((entry) => typeof entry.name === "string" && typeof entry.version === "string" && !SERVER_ONLY_PIECES.has(entry.name) && (typeof entry.actions === "number" && entry.actions > 0 || typeof entry.triggers === "number" && entry.triggers > 0)).map((entry) => ({
892
899
  name: entry.name,
893
900
  displayName: entry.displayName ?? entry.name,
@@ -897,6 +904,7 @@ function toSummaries(raw) {
897
904
  actionCount: typeof entry.actions === "number" ? entry.actions : 0,
898
905
  triggerCount: typeof entry.triggers === "number" ? entry.triggers : 0,
899
906
  categories: entry.categories ?? [],
907
+ source,
900
908
  auth: entry.auth ?? null,
901
909
  ...entry.deprecated === true ? { deprecated: true } : {},
902
910
  ...reasonOf(unsupportedAuth(entry.auth))
@@ -905,7 +913,7 @@ function toSummaries(raw) {
905
913
  async function fetchPieceCatalog() {
906
914
  if (catalogCache && catalogCache.expiresAt > Date.now()) return catalogCache.value;
907
915
  const lists = await publishedLists(false, 3e4);
908
- const value = [...registryFirst(toSummaries(lists.registry), toSummaries(lists.cloud))].sort((a, b) => a.displayName.localeCompare(b.displayName));
916
+ const value = [...registryFirst(toSummaries(lists.registry, "registry"), toSummaries(lists.cloud, "activepieces"))].sort((a, b) => a.displayName.localeCompare(b.displayName));
909
917
  catalogCache = {
910
918
  value,
911
919
  expiresAt: Date.now() + CACHE_TTL_MS
@@ -917,8 +925,8 @@ async function fetchPieceJson(packageName, version) {
917
925
  if (source) try {
918
926
  return await fetchJson(atVersion(source.pieceUrl(packageName), version));
919
927
  } catch (error) {
920
- if (error instanceof CatalogStatusError && error.status === 404) logger$5.debug("@registry does not serve @piece", source.baseUrl, packageName);
921
- else logger$5.warn("Piece registry @registry did not answer for @piece: @error", source.baseUrl, packageName, String(error));
928
+ if (error instanceof CatalogStatusError && error.status === 404) logger$6.debug("@registry does not serve @piece", source.baseUrl, packageName);
929
+ else logger$6.warn("Piece registry @registry did not answer for @piece: @error", source.baseUrl, packageName, String(error));
922
930
  }
923
931
  return fetchJson(pieceUrl(packageName, version));
924
932
  }
@@ -1007,6 +1015,7 @@ function catalogEntry(descriptor, pieceName, version) {
1007
1015
  actionCount: descriptor.actions.length,
1008
1016
  triggerCount: descriptor.triggers.length,
1009
1017
  categories: descriptor.categories ?? [],
1018
+ source: "local",
1010
1019
  auth: clientAuth(descriptor.auth),
1011
1020
  ...reasonOf(descriptor.unsupported)
1012
1021
  };
@@ -1041,32 +1050,44 @@ function triggersResult(descriptor, pieceName, version) {
1041
1050
  auth: clientAuth(descriptor.auth)
1042
1051
  };
1043
1052
  }
1044
- function localSearchHits(descriptor, pieceName, version) {
1053
+ function localSearchPiece(descriptor, pieceName, version) {
1045
1054
  const pieceDisplayName = descriptor.displayName || pieceName;
1046
1055
  const logoUrl = descriptor.logoUrl ?? "";
1047
- return [...descriptor.actions.map((action) => ({
1048
- pieceName,
1049
- pieceVersion: version,
1050
- name: action.name,
1051
- pieceDisplayName,
1052
- logoUrl,
1053
- displayName: action.displayName,
1054
- description: action.description ?? "",
1055
- kind: "action",
1056
- strategy: null,
1057
- ...reasonOf(descriptor.unsupported)
1058
- })), ...descriptor.triggers.map((trigger) => ({
1059
- pieceName,
1060
- pieceVersion: version,
1061
- name: trigger.name,
1062
- pieceDisplayName,
1063
- logoUrl,
1064
- displayName: trigger.displayName,
1065
- description: trigger.description ?? "",
1066
- kind: "trigger",
1067
- strategy: trigger.strategy,
1068
- ...reasonOf(descriptor.unsupported ?? trigger.unsupported)
1069
- }))];
1056
+ return {
1057
+ meta: {
1058
+ pieceName,
1059
+ pieceVersion: version,
1060
+ displayName: pieceDisplayName,
1061
+ description: descriptor.description ?? "",
1062
+ logoUrl,
1063
+ categories: descriptor.categories ?? [],
1064
+ source: "local",
1065
+ ...reasonOf(descriptor.unsupported)
1066
+ },
1067
+ blocks: [...descriptor.actions.map((action) => ({
1068
+ pieceName,
1069
+ pieceVersion: version,
1070
+ name: action.name,
1071
+ pieceDisplayName,
1072
+ logoUrl,
1073
+ displayName: action.displayName,
1074
+ description: action.description ?? "",
1075
+ kind: "action",
1076
+ strategy: null,
1077
+ ...reasonOf(descriptor.unsupported)
1078
+ })), ...descriptor.triggers.map((trigger) => ({
1079
+ pieceName,
1080
+ pieceVersion: version,
1081
+ name: trigger.name,
1082
+ pieceDisplayName,
1083
+ logoUrl,
1084
+ displayName: trigger.displayName,
1085
+ description: trigger.description ?? "",
1086
+ kind: "trigger",
1087
+ strategy: trigger.strategy,
1088
+ ...reasonOf(descriptor.unsupported ?? trigger.unsupported)
1089
+ }))]
1090
+ };
1070
1091
  }
1071
1092
  function detailResult(descriptor, pieceName, version) {
1072
1093
  return {
@@ -1106,11 +1127,41 @@ function detailResult(descriptor, pieceName, version) {
1106
1127
  //#endregion
1107
1128
  //#region src/reactor/block-search.ts
1108
1129
  const INDEX_TTL_MS = 36e5;
1109
- const DEFAULT_LIMIT = 30;
1110
- const MAX_LIMIT = 100;
1130
+ const DEFAULT_LIMIT = 20;
1131
+ const MAX_LIMIT = 50;
1132
+ const MAX_BLOCKS_PER_PIECE = 50;
1133
+ function tokenize(text) {
1134
+ return text.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
1135
+ }
1136
+ function wordsOf(...texts) {
1137
+ const words = texts.flatMap(tokenize);
1138
+ return {
1139
+ words,
1140
+ compact: words.join("")
1141
+ };
1142
+ }
1143
+ function slugOf(pieceName) {
1144
+ return pieceName.replace(/^@[^/]+\//, "").replace(/^piece-/, "");
1145
+ }
1146
+ function indexPiece(piece) {
1147
+ const { meta } = piece;
1148
+ return {
1149
+ meta,
1150
+ name: wordsOf(meta.displayName, slugOf(meta.pieceName)),
1151
+ exactName: tokenize(meta.displayName).join(" "),
1152
+ about: wordsOf(meta.description, ...meta.categories),
1153
+ blocks: piece.blocks.map((hit) => ({
1154
+ hit,
1155
+ name: wordsOf(hit.displayName, hit.name),
1156
+ description: wordsOf(hit.description)
1157
+ }))
1158
+ };
1159
+ }
1160
+ function indexPieces(pieces) {
1161
+ return { pieces: pieces.map(indexPiece) };
1162
+ }
1111
1163
  function buildSearchIndex(raw) {
1112
- const entries = [];
1113
- let pieces = 0;
1164
+ const pieces = [];
1114
1165
  for (const entry of raw) {
1115
1166
  if (typeof entry.name !== "string" || typeof entry.version !== "string" || SERVER_ONLY_PIECES.has(entry.name)) continue;
1116
1167
  const pieceName = entry.name;
@@ -1118,78 +1169,141 @@ function buildSearchIndex(raw) {
1118
1169
  const pieceDisplayName = entry.displayName ?? pieceName;
1119
1170
  const logoUrl = entry.logoUrl ?? "";
1120
1171
  const pieceUnsupported = unsupportedAuth(entry.auth);
1172
+ const blocks = [];
1121
1173
  const push = (kind, item) => {
1122
1174
  if (typeof item.name !== "string" || item.name === "") return;
1123
- const displayName = item.displayName ?? item.name;
1124
- const description = item.description ?? "";
1125
- entries.push({
1126
- hit: {
1127
- pieceName,
1128
- pieceVersion,
1129
- name: item.name,
1130
- pieceDisplayName,
1131
- logoUrl,
1132
- displayName,
1133
- description,
1134
- kind,
1135
- strategy: kind === "trigger" ? item.type ?? null : null,
1136
- ...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
1137
- },
1138
- name: `${displayName} ${item.name}`.toLowerCase(),
1139
- description: description.toLowerCase(),
1140
- piece: pieceDisplayName.toLowerCase()
1175
+ blocks.push({
1176
+ pieceName,
1177
+ pieceVersion,
1178
+ name: item.name,
1179
+ pieceDisplayName,
1180
+ logoUrl,
1181
+ displayName: item.displayName ?? item.name,
1182
+ description: item.description ?? "",
1183
+ kind,
1184
+ strategy: kind === "trigger" ? item.type ?? null : null,
1185
+ ...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
1141
1186
  });
1142
1187
  };
1143
1188
  for (const action of entry.suggestedActions ?? []) push("action", action);
1144
1189
  for (const trigger of entry.suggestedTriggers ?? []) push("trigger", trigger);
1145
- pieces += 1;
1190
+ pieces.push({
1191
+ meta: {
1192
+ pieceName,
1193
+ pieceVersion,
1194
+ displayName: pieceDisplayName,
1195
+ description: entry.description ?? "",
1196
+ logoUrl,
1197
+ categories: Array.isArray(entry.categories) ? entry.categories : [],
1198
+ source: entry.source ?? "activepieces",
1199
+ ...entry.deprecated === true ? { deprecated: true } : {},
1200
+ ...reasonOf(pieceUnsupported)
1201
+ },
1202
+ blocks
1203
+ });
1146
1204
  }
1147
- return {
1148
- entries,
1149
- pieces
1150
- };
1151
- }
1152
- function indexFromHits(hits) {
1153
- const pieces = /* @__PURE__ */ new Set();
1154
- return {
1155
- entries: hits.map((hit) => {
1156
- pieces.add(hit.pieceName);
1157
- return {
1158
- hit,
1159
- name: `${hit.displayName} ${hit.name}`.toLowerCase(),
1160
- description: hit.description.toLowerCase(),
1161
- piece: hit.pieceDisplayName.toLowerCase()
1162
- };
1163
- }),
1164
- pieces: pieces.size
1165
- };
1205
+ return indexPieces(pieces);
1166
1206
  }
1167
1207
  function merge(index, local) {
1168
- const localEntries = local?.entries ?? [];
1169
- const localNames = new Set(localEntries.map((entry) => entry.hit.pieceName));
1170
- const published = (index?.entries ?? []).filter((entry) => !localNames.has(entry.hit.pieceName));
1171
- const shadowed = new Set((index?.entries ?? []).map((entry) => entry.hit.pieceName).filter((name) => localNames.has(name)));
1172
- return {
1173
- entries: [...localEntries, ...published],
1174
- pieces: (local?.pieces ?? 0) + (index?.pieces ?? 0) - shadowed.size
1175
- };
1176
- }
1177
- function searchIndex(index, query, limit = DEFAULT_LIMIT) {
1178
- const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
1208
+ const localPieces = local?.pieces ?? [];
1209
+ const localNames = new Set(localPieces.map((piece) => piece.meta.pieceName));
1210
+ return { pieces: [...localPieces, ...(index?.pieces ?? []).filter((piece) => !localNames.has(piece.meta.pieceName))] };
1211
+ }
1212
+ const EXACT = 4;
1213
+ const PREFIX = 3;
1214
+ const INNER = 1.5;
1215
+ const FUZZY = 1;
1216
+ const PIECE_NAME = 3;
1217
+ const BLOCK_NAME = 2;
1218
+ const BLOCK_DESCRIPTION = 1;
1219
+ const PIECE_ABOUT = .5;
1220
+ function editDistance(a, b, max) {
1221
+ if (Math.abs(a.length - b.length) > max) return max + 1;
1222
+ let prevPrev = [];
1223
+ let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
1224
+ for (let i = 1; i <= a.length; i++) {
1225
+ const row = [i];
1226
+ let rowMin = i;
1227
+ for (let j = 1; j <= b.length; j++) {
1228
+ const cost = a[i - 1] === b[j - 1] ? 0 : 1;
1229
+ let value = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost);
1230
+ if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) value = Math.min(value, prevPrev[j - 2] + 1);
1231
+ row.push(value);
1232
+ rowMin = Math.min(rowMin, value);
1233
+ }
1234
+ if (rowMin > max) return max + 1;
1235
+ prevPrev = prev;
1236
+ prev = row;
1237
+ }
1238
+ return prev[b.length];
1239
+ }
1240
+ function termScore(token, field, fuzzy) {
1241
+ let best = 0;
1242
+ for (const word of field.words) {
1243
+ if (word === token) return EXACT;
1244
+ if (word.startsWith(token)) best = PREFIX;
1245
+ else if (best < INNER && token.length >= 3 && word.includes(token)) best = INNER;
1246
+ }
1247
+ if (best > 0) return best;
1248
+ if (token.length >= 3 && field.compact.includes(token)) return INNER;
1249
+ if (!fuzzy || token.length < 4) return 0;
1250
+ const max = token.length >= 8 ? 2 : 1;
1251
+ for (const word of field.words) if (editDistance(token, word, max) <= max || word.length > token.length && editDistance(token, word.slice(0, token.length), max) <= max) return FUZZY;
1252
+ return 0;
1253
+ }
1254
+ function allowed(piece, filter) {
1255
+ const { sources, categories } = filter;
1256
+ if (sources?.length && !sources.includes(piece.meta.source)) return false;
1257
+ if (categories?.length && !piece.meta.categories.some((category) => categories.includes(category))) return false;
1258
+ return true;
1259
+ }
1260
+ function searchIndex(index, query, filter) {
1261
+ const tokens = tokenize(query);
1179
1262
  if (tokens.length === 0) return [];
1180
- const scored = [];
1181
- for (const entry of index.entries) {
1182
- const haystack = `${entry.name} ${entry.piece} ${entry.description}`;
1183
- if (!tokens.every((token) => haystack.includes(token))) continue;
1184
- const first = tokens[0];
1185
- const score = entry.name.startsWith(first) ? 0 : entry.name.includes(first) ? 1 : entry.piece.includes(first) ? 2 : 3;
1186
- scored.push({
1187
- score,
1188
- entry
1263
+ const exactName = tokens.join(" ");
1264
+ const matches = [];
1265
+ for (const piece of index.pieces) {
1266
+ if (!allowed(piece, filter)) continue;
1267
+ const nameScores = tokens.map((token) => termScore(token, piece.name, true));
1268
+ const ofKind = piece.blocks.filter((block) => block.hit.kind === filter.kind).map((block) => ({
1269
+ hit: block.hit,
1270
+ scores: tokens.map((token) => Math.max(BLOCK_NAME * termScore(token, block.name, true), BLOCK_DESCRIPTION * termScore(token, block.description, false)))
1271
+ }));
1272
+ const matchBlocks = (pieceScores) => {
1273
+ const found = [];
1274
+ ofKind.forEach((block, order) => {
1275
+ let score = 0;
1276
+ for (let i = 0; i < tokens.length; i++) {
1277
+ const best = Math.max(pieceScores[i], block.scores[i]);
1278
+ if (best === 0) return;
1279
+ score += best;
1280
+ }
1281
+ found.push({
1282
+ score,
1283
+ order,
1284
+ hit: block.hit
1285
+ });
1286
+ });
1287
+ return found;
1288
+ };
1289
+ let blocks = matchBlocks(nameScores.map((score) => PIECE_NAME * score));
1290
+ if (blocks.length === 0 && ofKind.length > 0) blocks = matchBlocks(tokens.map((token, i) => Math.max(PIECE_NAME * nameScores[i], PIECE_ABOUT * termScore(token, piece.about, false))));
1291
+ if (blocks.length === 0) continue;
1292
+ blocks.sort((a, b) => b.score - a.score || a.order - b.order);
1293
+ const nameBonus = piece.exactName === exactName ? EXACT : 0;
1294
+ matches.push({
1295
+ score: blocks[0].score + nameBonus,
1296
+ match: {
1297
+ ...piece.meta,
1298
+ namedPiece: nameScores.every((score) => score > 0),
1299
+ blocks: blocks.slice(0, MAX_BLOCKS_PER_PIECE).map(({ hit }) => hit)
1300
+ }
1189
1301
  });
1190
1302
  }
1191
- scored.sort((a, b) => a.score - b.score || a.entry.hit.displayName.localeCompare(b.entry.hit.displayName));
1192
- return scored.slice(0, Math.min(Math.max(limit, 1), MAX_LIMIT)).map(({ entry }) => entry.hit);
1303
+ const demoted = (match) => (match.deprecated ? 2 : 0) + (match.unsupported ? 1 : 0);
1304
+ matches.sort((a, b) => b.score - a.score || demoted(a.match) - demoted(b.match) || a.match.displayName.localeCompare(b.match.displayName));
1305
+ const limit = Math.min(Math.max(filter.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT);
1306
+ return matches.slice(0, limit).map(({ match }) => match);
1193
1307
  }
1194
1308
  let cached;
1195
1309
  function ensureIndex() {
@@ -1206,29 +1320,35 @@ function ensureIndex() {
1206
1320
  cached = entry;
1207
1321
  return entry;
1208
1322
  }
1209
- function searchBlocks(query, limit, local) {
1323
+ function warmSearchIndex() {
1324
+ const entry = ensureIndex();
1325
+ entry.promise.catch(() => {
1326
+ if (cached === entry) cached = void 0;
1327
+ });
1328
+ }
1329
+ function searchPieces(query, filter, local) {
1210
1330
  const index = ensureIndex();
1211
1331
  if (index.error) {
1212
1332
  const message = index.error;
1213
1333
  cached = void 0;
1214
1334
  return {
1215
1335
  status: "error",
1216
- hits: searchIndex(merge(void 0, local), query, limit),
1217
- indexedPieces: local?.pieces ?? 0,
1336
+ pieces: searchIndex(merge(void 0, local), query, filter),
1337
+ indexedPieces: local?.pieces.length ?? 0,
1218
1338
  error: message
1219
1339
  };
1220
1340
  }
1221
1341
  if (!index.value) return {
1222
1342
  status: "indexing",
1223
- hits: searchIndex(merge(void 0, local), query, limit),
1224
- indexedPieces: local?.pieces ?? 0,
1343
+ pieces: searchIndex(merge(void 0, local), query, filter),
1344
+ indexedPieces: local?.pieces.length ?? 0,
1225
1345
  error: null
1226
1346
  };
1227
1347
  const merged = merge(index.value, local);
1228
1348
  return {
1229
1349
  status: "ready",
1230
- hits: searchIndex(merged, query, limit),
1231
- indexedPieces: merged.pieces,
1350
+ pieces: searchIndex(merged, query, filter),
1351
+ indexedPieces: merged.pieces.length,
1232
1352
  error: null
1233
1353
  };
1234
1354
  }
@@ -1407,7 +1527,7 @@ function currentRunUser() {
1407
1527
  }
1408
1528
  //#endregion
1409
1529
  //#region src/reactor/piece-store-port.ts
1410
- const logger$4 = childLogger(["workflow", "piece-store"]);
1530
+ const logger$5 = childLogger(["workflow", "piece-store"]);
1411
1531
  const PROJECT_SCOPE_KEY = "reactor";
1412
1532
  const TEST_PARTITION_SUFFIX = "#test";
1413
1533
  function testPartitionKey(scope, workflowId) {
@@ -1441,7 +1561,7 @@ function createPieceStorePort(store, workflowIdFor, sample = false, clock = Date
1441
1561
  if (!isCursorRef(key) || isPlausibleCursor(value, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
1442
1562
  const kept = await store.getPieceStoreValue(scope, partition(scope), key);
1443
1563
  if (!isCursorShaped(value) && !isPlausibleCursor(kept, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
1444
- logger$4.warn(`Rejected @key=@value from workflow ${flowKey()}; keeping @kept`, key, showCursor(value), isPlausibleCursor(kept, now) ? String(kept) : "no cursor");
1564
+ logger$5.warn(`Rejected @key=@value from workflow ${flowKey()}; keeping @kept`, key, showCursor(value), isPlausibleCursor(kept, now) ? String(kept) : "no cursor");
1445
1565
  if (!isPlausibleCursor(kept, now)) await store.deletePieceStoreValue(scope, partition(scope), key);
1446
1566
  },
1447
1567
  delete: async (key, scope) => store.deletePieceStoreValue(scope, partition(scope), key)
@@ -7497,7 +7617,7 @@ function workflowParkRow(workflowId, publishedVersion, reason) {
7497
7617
  parked_at: (/* @__PURE__ */ new Date()).toISOString()
7498
7618
  };
7499
7619
  }
7500
- const logger$3 = childLogger([
7620
+ const logger$4 = childLogger([
7501
7621
  "workflow",
7502
7622
  "runtime",
7503
7623
  "store"
@@ -7649,7 +7769,7 @@ async function migrateTriggerStoreState(db) {
7649
7769
  try {
7650
7770
  rows = await db.selectFrom("trigger_state").select(["workflow_id", "store_state"]).orderBy("updated_at", "asc").orderBy("workflow_id", "asc").execute();
7651
7771
  } catch (error) {
7652
- logger$3.error("Could not read trigger_state to migrate it: @error", error);
7772
+ logger$4.error("Could not read trigger_state to migrate it: @error", error);
7653
7773
  return unmigrated;
7654
7774
  }
7655
7775
  const pending = [];
@@ -7665,7 +7785,7 @@ async function migrateTriggerStoreState(db) {
7665
7785
  await migrateOneRow(db, row, projectWinner);
7666
7786
  } catch (error) {
7667
7787
  unmigrated.add(row.workflowId);
7668
- logger$3.error(`Could not migrate trigger store state for ${row.workflowId}`, error);
7788
+ logger$4.error(`Could not migrate trigger store state for ${row.workflowId}`, error);
7669
7789
  }
7670
7790
  return unmigrated;
7671
7791
  }
@@ -7675,7 +7795,7 @@ function parseLegacyBlob(row, unmigrated) {
7675
7795
  try {
7676
7796
  state = JSON.parse(row.store_state);
7677
7797
  } catch {
7678
- logger$3.warn(`Leaving unparseable store_state for ${row.workflow_id} in place`);
7798
+ logger$4.warn(`Leaving unparseable store_state for ${row.workflow_id} in place`);
7679
7799
  unmigrated.add(row.workflow_id);
7680
7800
  return null;
7681
7801
  }
@@ -7696,7 +7816,7 @@ function parseLegacyBlob(row, unmigrated) {
7696
7816
  });
7697
7817
  continue;
7698
7818
  }
7699
- if (key.startsWith("test")) logger$3.info(`Migrating "${key}" for ${row.workflow_id} as a project key; it may be a test leftover`);
7819
+ if (key.startsWith("test")) logger$4.info(`Migrating "${key}" for ${row.workflow_id} as a project key; it may be a test leftover`);
7700
7820
  entries.push({
7701
7821
  scope: "PROJECT",
7702
7822
  scopeKey: PROJECT_SCOPE_KEY,
@@ -7718,7 +7838,7 @@ function resolveProjectCollisions(pending) {
7718
7838
  }
7719
7839
  winner.set(entry.key, row.workflowId);
7720
7840
  }
7721
- for (const [key, workflows] of contested) logger$3.warn(`Project store key "@key" was written by ${workflows.join(", ")}; keeping the value last updated, from ${winner.get(key)}, and discarding the rest`, key);
7841
+ for (const [key, workflows] of contested) logger$4.warn(`Project store key "@key" was written by ${workflows.join(", ")}; keeping the value last updated, from ${winner.get(key)}, and discarding the rest`, key);
7722
7842
  return winner;
7723
7843
  }
7724
7844
  async function migrateOneRow(db, row, projectWinner) {
@@ -7747,9 +7867,9 @@ async function insertPieceStoreIfAbsent(db, scope, scopeKey, key, value) {
7747
7867
  }
7748
7868
  function warnIfOverPieceStoreLimits(scope, scopeKey, key, encoded) {
7749
7869
  const at = `${scope}/${scopeKey}/${key}`;
7750
- if (key.length > 128) logger$3.warn(`Migrated store key ${at} is ${key.length} chars, over the 128 limit; writes to it will fail`);
7870
+ if (key.length > 128) logger$4.warn(`Migrated store key ${at} is ${key.length} chars, over the 128 limit; writes to it will fail`);
7751
7871
  const size = Buffer.byteLength(encoded, "utf8");
7752
- if (size > 524288) logger$3.warn(`Migrated store value ${at} is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} limit; writes to it will fail`);
7872
+ if (size > 524288) logger$4.warn(`Migrated store value ${at} is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} limit; writes to it will fail`);
7753
7873
  }
7754
7874
  const PIECE_STORE_MAX_VALUE_BYTES = 524288;
7755
7875
  var PieceStoreLimitError = class extends Error {
@@ -7905,16 +8025,26 @@ function journaledPayloadNames(json, ids) {
7905
8025
  });
7906
8026
  }
7907
8027
  const erasedRuns = /* @__PURE__ */ new Set();
7908
- async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso) {
8028
+ async function pollSucceededIn(db, workflowId, storeState, nowIso, nextPollAtIso) {
8029
+ await db.updateTable("trigger_state").set({
8030
+ store_state: storeState,
8031
+ last_poll_at: nowIso,
8032
+ next_poll_at: nextPollAtIso,
8033
+ last_error: null,
8034
+ consecutive_failures: 0,
8035
+ updated_at: nowIso
8036
+ }).where("workflow_id", "=", workflowId).execute();
8037
+ }
8038
+ async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso, runId = FIRED_WITHOUT_RUN_ID) {
7909
8039
  const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
7910
8040
  await db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
7911
8041
  return await db.insertInto("trigger_dedupe").values({
7912
8042
  workflow_id: workflowId,
7913
8043
  dedupe_key: dedupeKey,
7914
- run_id: FIRED_WITHOUT_RUN_ID,
8044
+ run_id: runId,
7915
8045
  created_at: nowIso,
7916
8046
  attempts: 1
7917
- }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id: FIRED_WITHOUT_RUN_ID }).where("trigger_dedupe.run_id", "is", null)).returning("dedupe_key").executeTakeFirst() !== void 0;
8047
+ }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id: runId }).where("trigger_dedupe.run_id", "is", null)).returning("dedupe_key").executeTakeFirst() !== void 0;
7918
8048
  }
7919
8049
  const STEP_COLUMNS_WITHOUT_DATA = [
7920
8050
  "id",
@@ -7968,7 +8098,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7968
8098
  }).where("status", "=", "RUNNING");
7969
8099
  if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
7970
8100
  const recovered = (await query.returning("id").execute()).length;
7971
- if (recovered > 0) logger$3.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
8101
+ if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
7972
8102
  return recovered;
7973
8103
  }
7974
8104
  async recoverAbandonedRuns() {
@@ -7979,7 +8109,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7979
8109
  }).where("status", "=", PENDING_RUN_STATUS);
7980
8110
  if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
7981
8111
  const recovered = (await query.returning("id").execute()).length;
7982
- if (recovered > 0) logger$3.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
8112
+ if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
7983
8113
  return recovered;
7984
8114
  }
7985
8115
  /**
@@ -8059,6 +8189,65 @@ var WorkflowRunStore = class WorkflowRunStore {
8059
8189
  runId: id
8060
8190
  };
8061
8191
  }
8192
+ async journalTriggerItems(items, ttlMs, nowIso) {
8193
+ const ids = await this.db.transaction().execute(async (trx) => {
8194
+ const journaled = [];
8195
+ for (const { dedupeKey, options } of items) {
8196
+ const id = randomUUID();
8197
+ const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
8198
+ if (won) await this.insertPendingRun(trx, id, options);
8199
+ journaled.push(won ? id : void 0);
8200
+ }
8201
+ return journaled;
8202
+ });
8203
+ for (const id of ids) if (id) this.runsInFlight.add(id);
8204
+ return ids;
8205
+ }
8206
+ async recordScheduleFire(nowIso, nextPollAtIso, options) {
8207
+ const id = randomUUID();
8208
+ await this.db.transaction().execute(async (trx) => {
8209
+ await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
8210
+ await this.insertPendingRun(trx, id, options);
8211
+ });
8212
+ this.runsInFlight.add(id);
8213
+ return id;
8214
+ }
8215
+ /**
8216
+ * Puts a piece_store partition back to `entries` and, in the same write,
8217
+ * cancels the runs the restored cursor delivers again and releases their
8218
+ * claims, so no reader sees the old cursor while the claims still hold.
8219
+ */
8220
+ async rewindPieceStore(scope, scopeKey, entries, redelivered = {
8221
+ runIds: [],
8222
+ reason: ""
8223
+ }) {
8224
+ const live = redelivered.runIds.filter((runId) => !erasedRuns.has(runId));
8225
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
8226
+ await this.db.transaction().execute(async (trx) => {
8227
+ await trx.deleteFrom("piece_store").where("scope", "=", scope).where("scope_key", "=", scopeKey).execute();
8228
+ for (const [key, value] of Object.entries(entries)) {
8229
+ assertPieceStoreEntry(key, value);
8230
+ await trx.insertInto("piece_store").values({
8231
+ scope,
8232
+ scope_key: scopeKey,
8233
+ key,
8234
+ value: JSON.stringify(value),
8235
+ updated_at: nowIso
8236
+ }).execute();
8237
+ }
8238
+ if (live.length === 0) return;
8239
+ await trx.updateTable("run").set({
8240
+ status: "CANCELLED",
8241
+ error: redactMessage(redelivered.reason),
8242
+ ended_at: nowIso
8243
+ }).where("id", "in", live).execute();
8244
+ await trx.deleteFrom("trigger_dedupe").where("run_id", "in", live).execute();
8245
+ });
8246
+ for (const runId of redelivered.runIds) {
8247
+ this.runsInFlight.delete(runId);
8248
+ erasedRuns.delete(runId);
8249
+ }
8250
+ }
8062
8251
  /**
8063
8252
  * Counts one delivery of an operation, and reports whether a run was ever
8064
8253
  * journaled for it.
@@ -8176,7 +8365,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8176
8365
  if (result.steps.length > 0) try {
8177
8366
  await this.sweepSteps(runId, result, executionOrder);
8178
8367
  } catch (error) {
8179
- logger$3.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
8368
+ logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
8180
8369
  }
8181
8370
  const notes = runWarningNotes(result);
8182
8371
  await this.db.updateTable("run").set({
@@ -8393,14 +8582,7 @@ var WorkflowRunStore = class WorkflowRunStore {
8393
8582
  return this.db.selectFrom("trigger_state").selectAll().orderBy("workflow_id", "asc").execute();
8394
8583
  }
8395
8584
  async recordPollSuccess(workflowId, storeState, nowIso, nextPollAtIso) {
8396
- await this.db.updateTable("trigger_state").set({
8397
- store_state: storeState,
8398
- last_poll_at: nowIso,
8399
- next_poll_at: nextPollAtIso,
8400
- last_error: null,
8401
- consecutive_failures: 0,
8402
- updated_at: nowIso
8403
- }).where("workflow_id", "=", workflowId).execute();
8585
+ await pollSucceededIn(this.db, workflowId, storeState, nowIso, nextPollAtIso);
8404
8586
  }
8405
8587
  async recordPollFailure(workflowId, error, nowIso, nextPollAtIso, consecutiveFailures) {
8406
8588
  await this.db.updateTable("trigger_state").set({
@@ -8577,6 +8759,7 @@ function decodeRunCursor(cursor) {
8577
8759
  }
8578
8760
  //#endregion
8579
8761
  //#region src/reactor/run-gate.ts
8762
+ const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
8580
8763
  const QUEUE_DEPTH_ENV = "PH_WORKFLOWS_MAX_QUEUED_FIRINGS";
8581
8764
  function maxQueuedFirings(env = process.env) {
8582
8765
  const raw = Number(env[QUEUE_DEPTH_ENV]);
@@ -8584,6 +8767,7 @@ function maxQueuedFirings(env = process.env) {
8584
8767
  }
8585
8768
  var WorkflowRunGate = class {
8586
8769
  lanes = /* @__PURE__ */ new Map();
8770
+ closed = false;
8587
8771
  maxQueued;
8588
8772
  constructor(options = {}) {
8589
8773
  this.maxQueued = options.maxQueued ?? maxQueuedFirings();
@@ -8612,6 +8796,11 @@ var WorkflowRunGate = class {
8612
8796
  * that vanished is indistinguishable from a trigger that never fired.
8613
8797
  */
8614
8798
  async admit(workflowId, policy) {
8799
+ if (this.closed) return {
8800
+ admitted: false,
8801
+ reason: CLOSED_REASON,
8802
+ refusal: "closed"
8803
+ };
8615
8804
  const limit = concurrencyLimit(policy);
8616
8805
  if (limit === null) return {
8617
8806
  admitted: true,
@@ -8638,7 +8827,11 @@ var WorkflowRunGate = class {
8638
8827
  refusal: "queue-full",
8639
8828
  reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
8640
8829
  };
8641
- await new Promise((resolve) => lane.waiting.push(resolve));
8830
+ if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
8831
+ admitted: false,
8832
+ reason: CLOSED_REASON,
8833
+ refusal: "closed"
8834
+ };
8642
8835
  waited = true;
8643
8836
  } else lane.active += 1;
8644
8837
  let released = false;
@@ -8652,13 +8845,18 @@ var WorkflowRunGate = class {
8652
8845
  const next = lane.waiting.shift();
8653
8846
  if (next) {
8654
8847
  lane.active += 1;
8655
- next();
8848
+ next(true);
8656
8849
  return;
8657
8850
  }
8658
8851
  this.forgetIfIdle(workflowId, lane);
8659
8852
  }
8660
8853
  };
8661
8854
  }
8855
+ /** Refuses every waiting and later firing: a shut-down runtime runs none. */
8856
+ close() {
8857
+ this.closed = true;
8858
+ for (const lane of this.lanes.values()) for (const waiter of lane.waiting.splice(0)) waiter(false);
8859
+ }
8662
8860
  forgetIfIdle(workflowId, lane) {
8663
8861
  if (lane.active === 0 && lane.waiting.length === 0) this.lanes.delete(workflowId);
8664
8862
  }
@@ -8699,13 +8897,33 @@ async function sweepRetention(store, options) {
8699
8897
  dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
8700
8898
  };
8701
8899
  }
8900
+ //#endregion
8901
+ //#region src/reactor/shutdown.ts
8902
+ /** A firing refused because this runtime has shut down: retryable elsewhere. */
8903
+ var WorkflowRuntimeClosedError = class extends Error {
8904
+ workflowId;
8905
+ constructor(workflowId) {
8906
+ super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
8907
+ this.workflowId = workflowId;
8908
+ this.name = "WorkflowRuntimeClosedError";
8909
+ }
8910
+ };
8911
+ var TriggerSupervisorStoppedError = class extends Error {
8912
+ constructor(options) {
8913
+ super("The trigger supervisor has stopped", options);
8914
+ this.name = "TriggerSupervisorStoppedError";
8915
+ }
8916
+ };
8917
+ function isShutdownRefusal(error) {
8918
+ return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
8919
+ }
8702
8920
  childLogger(["workflow", "trigger-binding"]);
8703
8921
  function pieceTriggerKind(block) {
8704
8922
  return `piece:${block.pieceName}:${block.name}`;
8705
8923
  }
8706
8924
  //#endregion
8707
8925
  //#region src/reactor/trigger-supervisor.ts
8708
- const logger$1 = childLogger(["workflow", "trigger-supervisor"]);
8926
+ const logger$2 = childLogger(["workflow", "trigger-supervisor"]);
8709
8927
  function targetOf$1(binding) {
8710
8928
  return {
8711
8929
  name: binding.packageName,
@@ -8725,6 +8943,7 @@ const SCHEDULE_TRIGGER_KIND = "schedule";
8725
8943
  const MIN_INTERVAL_MS = MIN_SCHEDULE_INTERVAL_MS;
8726
8944
  const MAX_BACKOFF_MS = 18e5;
8727
8945
  const DEDUPE_TTL_MS = 3e4;
8946
+ const REDELIVERED_REASON = "Redelivered: the trigger stopped before firing this item and put its cursor back, so the next poll delivers it again";
8728
8947
  const DEFAULT_RECONCILE_INTERVAL_MS = 9e5;
8729
8948
  const RENEW_RETRY_BASE_MS = 6e4;
8730
8949
  function isSchedule(binding) {
@@ -8740,7 +8959,7 @@ function intervalFromSchedules(schedules, defaultMs) {
8740
8959
  const cron = schedule.cronExpression;
8741
8960
  const intervalMs = cronIntervalMs(cron);
8742
8961
  if (intervalMs === void 0) {
8743
- logger$1.warn("Unsupported setSchedule cron \"@cron\"; using the default", cron);
8962
+ logger$2.warn("Unsupported setSchedule cron \"@cron\"; using the default", cron);
8744
8963
  return Math.max(defaultMs, MIN_INTERVAL_MS);
8745
8964
  }
8746
8965
  return intervalMs;
@@ -8791,6 +9010,7 @@ var TriggerSupervisor = class {
8791
9010
  ops = Promise.resolve();
8792
9011
  ticking = false;
8793
9012
  warnedMissingJournal = false;
9013
+ stopped = false;
8794
9014
  constructor(options) {
8795
9015
  this.options = options;
8796
9016
  this.worker = options.worker ?? new PieceWorker({
@@ -8805,26 +9025,60 @@ var TriggerSupervisor = class {
8805
9025
  this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
8806
9026
  }
8807
9027
  start() {
8808
- if (this.timer) return;
9028
+ if (this.timer || this.stopped) return;
8809
9029
  this.timer = setInterval(() => {
8810
9030
  this.tick().catch((error) => {
8811
- logger$1.error("Trigger tick failed: @error", error);
9031
+ if (isShutdownRefusal(error)) {
9032
+ logger$2.debug("Trigger tick stopped: @error", error);
9033
+ return;
9034
+ }
9035
+ logger$2.error("Trigger tick failed: @error", error);
8812
9036
  });
8813
9037
  }, this.tickMs);
8814
9038
  this.timer.unref();
8815
- logger$1.info(`Trigger supervisor started (tick ${this.tickMs}ms)`);
9039
+ logger$2.info(`Trigger supervisor started (tick ${this.tickMs}ms)`);
8816
9040
  }
8817
9041
  stop() {
9042
+ if (this.stopped) return;
9043
+ this.stopped = true;
8818
9044
  if (this.timer) clearInterval(this.timer);
8819
9045
  this.timer = void 0;
8820
9046
  if (!this.options.worker) this.worker.dispose();
8821
- logger$1.info("Trigger supervisor stopped");
9047
+ logger$2.info("Trigger supervisor stopped");
8822
9048
  }
8823
9049
  enqueue(task) {
8824
- const run = this.ops.then(task);
9050
+ const run = this.ops.then(async () => {
9051
+ this.throwIfStopped();
9052
+ try {
9053
+ return await task();
9054
+ } catch (error) {
9055
+ if (this.stopped && !(error instanceof TriggerSupervisorStoppedError)) throw new TriggerSupervisorStoppedError({ cause: error });
9056
+ throw error;
9057
+ }
9058
+ });
8825
9059
  this.ops = run.catch(() => void 0);
8826
9060
  return run;
8827
9061
  }
9062
+ throwIfStopped() {
9063
+ if (this.stopped) throw new TriggerSupervisorStoppedError();
9064
+ }
9065
+ async store() {
9066
+ this.throwIfStopped();
9067
+ const store = await this.options.store();
9068
+ if (!store) return void 0;
9069
+ return new Proxy(store, { get: (target, property) => {
9070
+ const value = Reflect.get(target, property, target);
9071
+ if (typeof value !== "function") return value;
9072
+ return (...args) => {
9073
+ this.throwIfStopped();
9074
+ return value.apply(target, args);
9075
+ };
9076
+ } });
9077
+ }
9078
+ fire(workflowId, payload, kind, runId) {
9079
+ this.throwIfStopped();
9080
+ this.options.fire(workflowId, payload, kind, runId);
9081
+ }
8828
9082
  enabledOk = /* @__PURE__ */ new Set();
8829
9083
  descriptors = /* @__PURE__ */ new Map();
8830
9084
  own;
@@ -8846,7 +9100,7 @@ var TriggerSupervisor = class {
8846
9100
  park(workflowId, publishedVersion, reason) {
8847
9101
  this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
8848
9102
  return this.enqueue(async () => {
8849
- const store = await this.options.store();
9103
+ const store = await this.store();
8850
9104
  if (!store) return false;
8851
9105
  return store.parkWorkflow(workflowId, publishedVersion, reason);
8852
9106
  });
@@ -8855,7 +9109,7 @@ var TriggerSupervisor = class {
8855
9109
  liftPark(workflowId, publishedVersion, trigger) {
8856
9110
  if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
8857
9111
  return this.enqueue(async () => {
8858
- const store = await this.options.store();
9112
+ const store = await this.store();
8859
9113
  if (!store) return;
8860
9114
  if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
8861
9115
  await store.liftPark(workflowId, trigger);
@@ -8866,7 +9120,7 @@ var TriggerSupervisor = class {
8866
9120
  releasePark(workflowId, flipRow) {
8867
9121
  this.parks.delete(workflowId);
8868
9122
  return this.enqueue(async () => {
8869
- const store = await this.options.store();
9123
+ const store = await this.store();
8870
9124
  if (!store) return;
8871
9125
  if (flipRow) await store.clearParkOnDisable(workflowId);
8872
9126
  else await store.clearWorkflowPark(workflowId);
@@ -8884,7 +9138,7 @@ var TriggerSupervisor = class {
8884
9138
  const park = this.parks.known(workflowId)?.park;
8885
9139
  if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
8886
9140
  return this.enqueue(async () => {
8887
- const store = await this.options.store();
9141
+ const store = await this.store();
8888
9142
  if (!store) return;
8889
9143
  const park = await store.getWorkflowPark(workflowId);
8890
9144
  if (!park || park.published_version >= publishedVersion) return;
@@ -8899,7 +9153,7 @@ var TriggerSupervisor = class {
8899
9153
  const binding = this.unbind(workflowId);
8900
9154
  this.parks.delete(workflowId);
8901
9155
  return this.enqueue(async () => {
8902
- const store = await this.options.store();
9156
+ const store = await this.store();
8903
9157
  if (!store) return;
8904
9158
  await this.disable(workflowId, binding);
8905
9159
  await store.deleteTriggerState(workflowId);
@@ -8919,16 +9173,16 @@ var TriggerSupervisor = class {
8919
9173
  return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
8920
9174
  }
8921
9175
  async recordRejection(workflowId, block, config, message, retryAt) {
8922
- const store = await this.options.store();
9176
+ const store = await this.store();
8923
9177
  if (!store) return;
8924
9178
  const hash = configHash(block, config);
8925
9179
  const existing = await store.getTriggerState(workflowId);
8926
9180
  if (existing?.status === "ENABLED" && existing.config_hash === hash) {
8927
- logger$1.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
9181
+ logger$2.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
8928
9182
  return;
8929
9183
  }
8930
9184
  if (existing?.status === "PARKED" && existing.config_hash === hash) {
8931
- logger$1.warn(`Workflow ${workflowId} is PARKED and could not be resolved; the park stands. @error`, message);
9185
+ logger$2.warn(`Workflow ${workflowId} is PARKED and could not be resolved; the park stands. @error`, message);
8932
9186
  return;
8933
9187
  }
8934
9188
  await store.upsertTriggerState({
@@ -8950,7 +9204,7 @@ var TriggerSupervisor = class {
8950
9204
  }
8951
9205
  test(binding, options = {}) {
8952
9206
  return this.enqueue(async () => {
8953
- const store = await this.options.store();
9207
+ const store = await this.store();
8954
9208
  try {
8955
9209
  return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
8956
9210
  } finally {
@@ -8964,7 +9218,7 @@ var TriggerSupervisor = class {
8964
9218
  await store.deletePieceStore("FLOW", testPartitionKey("FLOW", workflowId));
8965
9219
  await store.deletePieceStore("PROJECT", testPartitionKey("PROJECT", workflowId));
8966
9220
  } catch (error) {
8967
- logger$1.warn(`Could not clear test store for ${workflowId}`, error);
9221
+ logger$2.warn(`Could not clear test store for ${workflowId}`, error);
8968
9222
  }
8969
9223
  }
8970
9224
  handshake(binding, payload) {
@@ -8974,45 +9228,58 @@ var TriggerSupervisor = class {
8974
9228
  return this.enqueue(async () => {
8975
9229
  const binding = this.bindings.get(workflowId);
8976
9230
  if (!binding || isSchedule(binding)) {
8977
- logger$1.warn(`Webhook delivery for unknown workflow ${workflowId}`);
9231
+ logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
8978
9232
  return;
8979
9233
  }
8980
- const store = await this.options.store();
9234
+ const store = await this.store();
8981
9235
  if (!store) throw new MissingJournalError("Webhook delivery");
8982
9236
  const row = await store.getTriggerState(workflowId);
8983
9237
  const now = this.now();
8984
9238
  const rewind = await this.cursorRewind(store, workflowId);
9239
+ let refused = false;
8985
9240
  try {
8986
- const result = await this.hook(binding, "run", { payload });
9241
+ const result = await this.hook(binding, "run", { payload }).catch((error) => {
9242
+ refused = isShutdownRefusal(error);
9243
+ throw error;
9244
+ });
8987
9245
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
8988
9246
  if (row?.status === "ENABLED") await store.recordPollSuccess(workflowId, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
8989
- for (const item of result.output) await this.fireItem(store, binding, item, now);
9247
+ await this.fireItems(store, binding, result.output, now, rewind);
8990
9248
  } catch (error) {
8991
- await rewind();
9249
+ if (!refused) await rewind();
8992
9250
  throw error;
8993
9251
  }
8994
9252
  });
8995
9253
  }
8996
9254
  async cursorRewind(store, workflowId) {
8997
9255
  const before = await store.listPieceStore("FLOW", workflowId);
8998
- return async () => {
9256
+ let rewound;
9257
+ const rewind = async (redelivered) => {
8999
9258
  try {
9000
- await store.deletePieceStore("FLOW", workflowId);
9001
- for (const [key, value] of Object.entries(before)) await store.setPieceStoreValue("FLOW", workflowId, key, value);
9259
+ const raw = await this.options.store();
9260
+ if (!raw) return false;
9261
+ await raw.rewindPieceStore("FLOW", workflowId, before, {
9262
+ runIds: redelivered,
9263
+ reason: REDELIVERED_REASON
9264
+ });
9265
+ return true;
9002
9266
  } catch (error) {
9003
- logger$1.warn(`Could not rewind the cursor for ${workflowId}`, error);
9267
+ logger$2.warn(`Could not rewind the cursor for ${workflowId}`, error);
9268
+ return false;
9004
9269
  }
9005
9270
  };
9271
+ return (redelivered = []) => rewound ??= rewind(redelivered);
9006
9272
  }
9007
9273
  async hook(binding, hook, options = {}) {
9008
9274
  if (hook !== "onDisable") checkDynamicProperties(binding.config, binding.propertySettings);
9009
- const store = await this.options.store();
9275
+ const store = await this.store();
9010
9276
  if (!store && hook !== "test") throw new MissingJournalError(`Trigger hook "${hook}"`);
9011
9277
  const pieceStore = store ? createPieceStorePort(store, () => binding.workflowId, hook === "test") : void 0;
9012
9278
  const piece = await this.resolver().resolve(targetOf$1(binding));
9013
9279
  const auth = await this.options.resolveAuth(binding.connectionId, { piecePackage: binding.packageName });
9014
9280
  const redactValues = secretsFor(auth);
9015
9281
  const reactor = await this.reactorFor(binding, hook, options.runUser);
9282
+ this.throwIfStopped();
9016
9283
  return this.worker.runTriggerHook({
9017
9284
  ...pieceModuleRef(piece),
9018
9285
  triggerName: binding.triggerName,
@@ -9080,7 +9347,7 @@ var TriggerSupervisor = class {
9080
9347
  };
9081
9348
  }
9082
9349
  async enable(binding, superseded) {
9083
- const store = await this.options.store();
9350
+ const store = await this.store();
9084
9351
  if (!store) throw new MissingJournalError("Enabling a trigger");
9085
9352
  const hash = configHash(binding.block, binding.config);
9086
9353
  const now = this.now();
@@ -9139,8 +9406,9 @@ var TriggerSupervisor = class {
9139
9406
  this.enabledOk.add(binding.workflowId);
9140
9407
  this.enableRetries.delete(binding.workflowId);
9141
9408
  store.clearUnmigratedTriggerState(binding.workflowId);
9142
- logger$1.info(`Enabled @block for workflow ${binding.workflowId} (every ${intervalMs}ms)`, blockLabel(binding.block));
9409
+ logger$2.info(`Enabled @block for workflow ${binding.workflowId} (every ${intervalMs}ms)`, blockLabel(binding.block));
9143
9410
  } catch (error) {
9411
+ if (this.stopped) throw error;
9144
9412
  this.enabledOk.delete(binding.workflowId);
9145
9413
  const message = error instanceof Error ? error.message : String(error);
9146
9414
  const failures = (existing?.consecutive_failures ?? 0) + 1;
@@ -9168,7 +9436,7 @@ var TriggerSupervisor = class {
9168
9436
  updated_at: now.toISOString(),
9169
9437
  ...pieceColumns(binding)
9170
9438
  });
9171
- logger$1.error(`onEnable failed for workflow ${binding.workflowId} (${failures}x): @error` + (retryAt ? `; retrying at ${retryAt.toISOString()}` : "; not retrying"), message);
9439
+ logger$2.error(`onEnable failed for workflow ${binding.workflowId} (${failures}x): @error` + (retryAt ? `; retrying at ${retryAt.toISOString()}` : "; not retrying"), message);
9172
9440
  }
9173
9441
  }
9174
9442
  renewAtAfterEnable(renew, existing, isRepublish, now) {
@@ -9213,7 +9481,7 @@ var TriggerSupervisor = class {
9213
9481
  const retry = this.enableRetries.get(workflowId);
9214
9482
  if (retry) retry.at = this.now().getTime() + this.defaultIntervalMs;
9215
9483
  if (superseded && this.bindings.get(workflowId) === binding) this.bindings.set(workflowId, superseded);
9216
- logger$1.warn(`Workflow ${workflowId} is PARKED at version ${park.published_version}; its trigger is not armed until the workflow is re-published or re-enabled. @error`, park.reason);
9484
+ logger$2.warn(`Workflow ${workflowId} is PARKED at version ${park.published_version}; its trigger is not armed until the workflow is re-published or re-enabled. @error`, park.reason);
9217
9485
  }
9218
9486
  /** The status an enable that succeeded writes: never ENABLED while a park
9219
9487
  * stands, whatever ran meanwhile. The registration stays recorded. */
@@ -9232,7 +9500,7 @@ var TriggerSupervisor = class {
9232
9500
  failures: existing.consecutive_failures,
9233
9501
  release: true
9234
9502
  });
9235
- logger$1.info(`Enable for workflow ${binding.workflowId} still backing off until ${existing.next_poll_at}`);
9503
+ logger$2.info(`Enable for workflow ${binding.workflowId} still backing off until ${existing.next_poll_at}`);
9236
9504
  }
9237
9505
  return true;
9238
9506
  }
@@ -9240,7 +9508,7 @@ var TriggerSupervisor = class {
9240
9508
  try {
9241
9509
  await this.hook(binding, "onDisable");
9242
9510
  } catch (error) {
9243
- logger$1.warn(`onDisable before retrying workflow ${binding.workflowId} failed`, error);
9511
+ logger$2.warn(`onDisable before retrying workflow ${binding.workflowId} failed`, error);
9244
9512
  }
9245
9513
  }
9246
9514
  async enableSchedule(store, binding, hash, existing) {
@@ -9269,7 +9537,7 @@ var TriggerSupervisor = class {
9269
9537
  consecutive_failures: 0
9270
9538
  });
9271
9539
  this.enabledOk.add(binding.workflowId);
9272
- logger$1.info(`Scheduled workflow ${binding.workflowId}: next fire ${nextAt.toISOString()}` + (carried ? " (carried over)" : ""));
9540
+ logger$2.info(`Scheduled workflow ${binding.workflowId}: next fire ${nextAt.toISOString()}` + (carried ? " (carried over)" : ""));
9273
9541
  } catch (error) {
9274
9542
  this.enabledOk.delete(binding.workflowId);
9275
9543
  const message = error instanceof Error ? error.message : String(error);
@@ -9281,24 +9549,24 @@ var TriggerSupervisor = class {
9281
9549
  last_error: message,
9282
9550
  consecutive_failures: (existing?.consecutive_failures ?? 0) + 1
9283
9551
  });
9284
- logger$1.error(`Invalid schedule for workflow ${binding.workflowId}: @error`, message);
9552
+ logger$2.error(`Invalid schedule for workflow ${binding.workflowId}: @error`, message);
9285
9553
  }
9286
9554
  }
9287
9555
  async disable(workflowId, binding) {
9288
- const store = await this.options.store();
9556
+ const store = await this.store();
9289
9557
  if (!store) throw new MissingJournalError("Disabling a trigger");
9290
9558
  const row = await store.getTriggerState(workflowId);
9291
9559
  if (!row || row.status === "DISABLED") return;
9292
9560
  await this.disableRow(workflowId, row, binding);
9293
9561
  }
9294
9562
  async disableRow(workflowId, row, binding) {
9295
- const store = await this.options.store();
9563
+ const store = await this.store();
9296
9564
  if (!store) return;
9297
9565
  const target = binding ?? this.bindingFromRow(row);
9298
9566
  if (target && !isSchedule(target) && blockKey(triggerRowBlock(row)) !== SCHEDULE_BLOCK) try {
9299
9567
  await this.hook(target, "onDisable");
9300
9568
  } catch (error) {
9301
- logger$1.warn(`onDisable failed for workflow ${workflowId}`, error);
9569
+ logger$2.warn(`onDisable failed for workflow ${workflowId}`, error);
9302
9570
  }
9303
9571
  await store.setTriggerStatus(workflowId, "DISABLED");
9304
9572
  }
@@ -9306,7 +9574,7 @@ var TriggerSupervisor = class {
9306
9574
  return this.bindings.get(row.workflow_id);
9307
9575
  }
9308
9576
  async tick() {
9309
- if (this.ticking) return;
9577
+ if (this.ticking || this.stopped) return;
9310
9578
  this.ticking = true;
9311
9579
  try {
9312
9580
  await this.enqueue(() => this.pollDue());
@@ -9327,6 +9595,7 @@ var TriggerSupervisor = class {
9327
9595
  try {
9328
9596
  await this.enable(binding);
9329
9597
  } catch (error) {
9598
+ if (this.stopped) throw error;
9330
9599
  const failures = retry.failures + 1;
9331
9600
  const intervalMs = isSchedule(binding) ? MIN_INTERVAL_MS : pollIntervalFor(binding, void 0, this.defaultIntervalMs);
9332
9601
  this.enableRetries.set(workflowId, {
@@ -9334,20 +9603,21 @@ var TriggerSupervisor = class {
9334
9603
  failures,
9335
9604
  release: retry.release
9336
9605
  });
9337
- logger$1.error(`Enable retry for workflow ${workflowId} threw`, error);
9606
+ logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
9338
9607
  }
9339
9608
  }
9340
9609
  async pollDue() {
9341
- const store = await this.options.store();
9610
+ const store = await this.store();
9342
9611
  if (!store) {
9343
9612
  if (!this.warnedMissingJournal) {
9344
9613
  this.warnedMissingJournal = true;
9345
- logger$1.error("Trigger polling is off: @error", new MissingJournalError("Polling"));
9614
+ logger$2.error("Trigger polling is off: @error", new MissingJournalError("Polling"));
9346
9615
  }
9347
9616
  return;
9348
9617
  }
9349
9618
  const due = await store.listDueTriggerStates(this.now().toISOString());
9350
9619
  for (const row of due) {
9620
+ this.throwIfStopped();
9351
9621
  const binding = this.bindings.get(row.workflow_id);
9352
9622
  if (!binding) {
9353
9623
  await store.setTriggerStatus(row.workflow_id, "DISABLED");
@@ -9356,12 +9626,15 @@ var TriggerSupervisor = class {
9356
9626
  if (isSchedule(binding)) await this.fireSchedule(store, row, binding);
9357
9627
  else await this.poll(store, row, binding);
9358
9628
  }
9629
+ this.throwIfStopped();
9359
9630
  await this.renewDue(store);
9631
+ this.throwIfStopped();
9360
9632
  await this.retryOneEnable();
9361
9633
  }
9362
9634
  async renewDue(store) {
9363
9635
  const due = await store.listDueTriggerRenewals(this.now().toISOString());
9364
9636
  for (const row of due) {
9637
+ this.throwIfStopped();
9365
9638
  const binding = this.bindings.get(row.workflow_id);
9366
9639
  if (!binding) {
9367
9640
  await store.setTriggerStatus(row.workflow_id, "DISABLED");
@@ -9387,7 +9660,7 @@ var TriggerSupervisor = class {
9387
9660
  await this.hook(binding, "onRenew", { webhookUrl });
9388
9661
  const nextAt = nextRenewAt(renew, now);
9389
9662
  await store.recordRenewSuccess(row.workflow_id, now.toISOString(), nextAt.toISOString());
9390
- logger$1.info(`Renewed trigger of workflow ${row.workflow_id}; next ${nextAt.toISOString()}`);
9663
+ logger$2.info(`Renewed trigger of workflow ${row.workflow_id}; next ${nextAt.toISOString()}`);
9391
9664
  } catch (error) {
9392
9665
  const message = error instanceof Error ? error.message : String(error);
9393
9666
  const failures = row.renew_failures + 1;
@@ -9395,7 +9668,7 @@ var TriggerSupervisor = class {
9395
9668
  const slot = renew ? nextRenewAt(renew, now).getTime() : backoff;
9396
9669
  const retryAt = new Date(Math.min(backoff, slot));
9397
9670
  await store.recordRenewFailure(row.workflow_id, message, now.toISOString(), retryAt.toISOString(), failures);
9398
- logger$1.warn(`onRenew failed for workflow ${row.workflow_id} (${failures}x): @error; retrying at ${retryAt.toISOString()}`, message);
9671
+ logger$2.warn(`onRenew failed for workflow ${row.workflow_id} (${failures}x): @error; retrying at ${retryAt.toISOString()}`, message);
9399
9672
  }
9400
9673
  }
9401
9674
  async fireSchedule(store, row, binding) {
@@ -9404,38 +9677,79 @@ var TriggerSupervisor = class {
9404
9677
  const schedule = parseScheduleConfig$1(binding.config);
9405
9678
  const scheduledFor = row.next_poll_at ? new Date(row.next_poll_at) : now;
9406
9679
  const nextAt = rescheduleAfterFire(schedule, scheduledFor, now);
9407
- await store.recordPollSuccess(row.workflow_id, "{}", now.toISOString(), nextAt.toISOString());
9408
- this.options.fire(binding.workflowId, schedulePayload(schedule, scheduledFor, now), SCHEDULE_TRIGGER_KIND);
9680
+ const payload = schedulePayload(schedule, scheduledFor, now);
9681
+ const runId = await store.recordScheduleFire(now.toISOString(), nextAt.toISOString(), {
9682
+ workflowId: binding.workflowId,
9683
+ triggerKind: SCHEDULE_TRIGGER_KIND,
9684
+ triggerPayload: payload
9685
+ });
9686
+ try {
9687
+ this.fire(binding.workflowId, payload, SCHEDULE_TRIGGER_KIND, runId);
9688
+ } catch (error) {
9689
+ await this.failRefused([runId], error);
9690
+ throw error;
9691
+ }
9409
9692
  } catch (error) {
9693
+ if (this.stopped) throw error;
9410
9694
  const message = error instanceof Error ? error.message : String(error);
9411
9695
  this.enabledOk.delete(binding.workflowId);
9412
9696
  await store.setTriggerStatus(row.workflow_id, "ERROR", message);
9413
- logger$1.error(`Schedule fire failed for workflow ${row.workflow_id}: @error`, message);
9697
+ logger$2.error(`Schedule fire failed for workflow ${row.workflow_id}: @error`, message);
9414
9698
  }
9415
9699
  }
9416
9700
  async poll(store, row, binding) {
9417
9701
  const now = this.now();
9418
9702
  const rewind = await this.cursorRewind(store, row.workflow_id);
9703
+ let refused = false;
9419
9704
  try {
9420
- const result = await this.hook(binding, "run");
9705
+ const result = await this.hook(binding, "run").catch((error) => {
9706
+ refused = isShutdownRefusal(error);
9707
+ throw error;
9708
+ });
9421
9709
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
9422
9710
  await store.recordPollSuccess(row.workflow_id, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
9423
- for (const item of result.output) await this.fireItem(store, binding, item, now);
9711
+ await this.fireItems(store, binding, result.output, now, rewind);
9424
9712
  } catch (error) {
9425
- await rewind();
9713
+ if (!refused) await rewind();
9714
+ if (this.stopped) throw error;
9426
9715
  const message = error instanceof Error ? error.message : String(error);
9427
9716
  const failures = row.consecutive_failures + 1;
9428
9717
  const backoff = backoffMs(row.interval_ms, failures);
9429
9718
  await store.recordPollFailure(row.workflow_id, message, now.toISOString(), new Date(now.getTime() + backoff).toISOString(), failures);
9430
- logger$1.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
9719
+ logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
9720
+ }
9721
+ }
9722
+ async fireItems(store, binding, items, now, rewind) {
9723
+ if (items.length === 0) return;
9724
+ const kind = pieceTriggerKind(binding.block);
9725
+ const journaled = (await store.journalTriggerItems(items.map((item) => ({
9726
+ dedupeKey: extractDedupeKey(item),
9727
+ options: {
9728
+ workflowId: binding.workflowId,
9729
+ triggerKind: kind,
9730
+ triggerPayload: item
9731
+ }
9732
+ })), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
9733
+ runId,
9734
+ item: items[index]
9735
+ }] : []);
9736
+ if (this.stopped) {
9737
+ const error = new TriggerSupervisorStoppedError();
9738
+ await this.redeliver(journaled.map(({ runId }) => runId), rewind, error);
9739
+ throw error;
9431
9740
  }
9741
+ for (const { runId, item } of journaled) this.fire(binding.workflowId, item, kind, runId);
9742
+ }
9743
+ async redeliver(runIds, rewind, error) {
9744
+ if (!await rewind(runIds)) await this.failRefused(runIds, error);
9432
9745
  }
9433
- async fireItem(store, binding, item, now) {
9434
- const dedupeKey = extractDedupeKey(item);
9435
- if (dedupeKey) {
9436
- if (!await store.claimDedupe(binding.workflowId, dedupeKey, DEDUPE_TTL_MS, now.toISOString())) return;
9746
+ async failRefused(runIds, error) {
9747
+ try {
9748
+ const raw = await this.options.store();
9749
+ for (const runId of runIds) await raw?.failRun(runId, error instanceof Error ? error.message : String(error), error instanceof Error ? error.name : void 0);
9750
+ } catch (failure) {
9751
+ logger$2.warn(`Could not fail the refused runs ${runIds.join(", ")}`, failure);
9437
9752
  }
9438
- this.options.fire(binding.workflowId, item, pieceTriggerKind(binding.block));
9439
9753
  }
9440
9754
  };
9441
9755
  //#endregion
@@ -9679,7 +9993,7 @@ const DESCRIBE_TIMEOUT_MS = 3e4;
9679
9993
  const SEED_ATTEMPTS = 3;
9680
9994
  const SEED_RETRY_BASE_MS = 250;
9681
9995
  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref());
9682
- const logger = childLogger(["workflow", "runtime"]);
9996
+ const logger$1 = childLogger(["workflow", "runtime"]);
9683
9997
  const DRIVE_DOCUMENT_TYPE = "powerhouse/document-drive";
9684
9998
  const WORKFLOW_DOCUMENT_TYPE = "powerhouse/workflow";
9685
9999
  const DOCUMENT_SCOPE = "document";
@@ -9772,6 +10086,7 @@ const UNAUTHORIZED = { status: 401 };
9772
10086
  const JSON_CONTENT_TYPE = "application/json; charset=utf-8";
9773
10087
  const DELIVERY_TIMEOUT_MS = Number(process.env.PH_WORKFLOWS_WEBHOOK_TIMEOUT_MS) || 3e4;
9774
10088
  const TIMED_OUT = Symbol("webhook delivery timed out");
10089
+ const WEBHOOK_REDELIVERED_REASON = "Redelivered: the runtime shut down before this delivery ran, and the sender was told to retry";
9775
10090
  /** Resolves to TIMED_OUT, and never keeps the process alive waiting to. */
9776
10091
  function timeout(ms) {
9777
10092
  return new Promise((resolve) => {
@@ -9795,7 +10110,7 @@ function splitPollInterval(config) {
9795
10110
  const { [POLL_INTERVAL_CONFIG_KEY]: raw, ...rest } = config;
9796
10111
  const seconds = typeof raw === "string" ? Number(raw) : raw;
9797
10112
  if (typeof seconds !== "number" || !Number.isFinite(seconds) || seconds <= 0) {
9798
- logger.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
10113
+ logger$1.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
9799
10114
  return { config: rest };
9800
10115
  }
9801
10116
  return {
@@ -9865,7 +10180,7 @@ var WorkflowRuntimeService = class {
9865
10180
  timeoutMs: host.pieceVersionLookupMs ?? PIECE_VERSION_LOOKUP_TIMEOUT_MS,
9866
10181
  hasBlock: (ref, version, source) => this.versionHasBlock(ref, version, source)
9867
10182
  });
9868
- this.logger = host.logger ?? logger;
10183
+ this.logger = host.logger ?? logger$1;
9869
10184
  this.attachments = host.attachments ? createAttachmentPort(host.attachments, {
9870
10185
  documentIdFor: () => currentWorkflowId(),
9871
10186
  canReadRef: async (ref) => {
@@ -9913,7 +10228,7 @@ var WorkflowRuntimeService = class {
9913
10228
  }
9914
10229
  }
9915
10230
  async store() {
9916
- if (this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
10231
+ if (!this.closed && this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
9917
10232
  try {
9918
10233
  return await this.storePromise;
9919
10234
  } catch {
@@ -10030,10 +10345,10 @@ var WorkflowRuntimeService = class {
10030
10345
  if (state.status !== "ENABLED") {
10031
10346
  const held = this.registry.get(workflowId);
10032
10347
  this.supervisor().releasePark(workflowId, !(held && SUPERVISED_KINDS.has(held.kind))).catch((error) => {
10033
- this.logger.error(`Could not clear the park of disabled workflow ${workflowId}`, error);
10348
+ this.laneFailed(`Could not clear the park of disabled workflow ${workflowId}`, error);
10034
10349
  });
10035
10350
  } else if (await this.outdatedPark(workflowId, state)) this.supervisor().unpark(workflowId, version).catch((error) => {
10036
- this.logger.error(`Could not lift the outdated park of workflow ${workflowId}`, error);
10351
+ this.laneFailed(`Could not lift the outdated park of workflow ${workflowId}`, error);
10037
10352
  });
10038
10353
  const park = block ? await this.parks.get(workflowId) : void 0;
10039
10354
  if (block && (blockKey(block) === WEBHOOK_BLOCK || triggerKindOf(block)) && park && park.published_version >= version) {
@@ -10090,7 +10405,7 @@ var WorkflowRuntimeService = class {
10090
10405
  }
10091
10406
  enableSupervised(workflowId, binding, publishedVersion) {
10092
10407
  this.supervisor().upsert(binding, publishedVersion).catch((error) => {
10093
- this.logger.error(`Trigger enable failed for ${workflowId}`, error);
10408
+ this.laneFailed(`Trigger enable failed for ${workflowId}`, error);
10094
10409
  });
10095
10410
  }
10096
10411
  async registerPieceTrigger(workflowId, binding, publishedVersion) {
@@ -10147,7 +10462,7 @@ var WorkflowRuntimeService = class {
10147
10462
  const reason = `The trigger ${blockLabel(binding.block)} is not armed: ${refusal.reason}`;
10148
10463
  this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
10149
10464
  this.supervisor().reject(workflowId, binding.block, binding.config, reason, retryAt).catch((error) => {
10150
- this.logger.error(`Could not record the refused trigger for workflow ${workflowId}`, error);
10465
+ this.laneFailed(`Could not record the refused trigger for workflow ${workflowId}`, error);
10151
10466
  });
10152
10467
  }
10153
10468
  async registerWebhook(workflowId, block, rawConfig) {
@@ -10162,7 +10477,7 @@ var WorkflowRuntimeService = class {
10162
10477
  this.logger.error(`Webhook trigger rejected for ${workflowId}: @error`, message);
10163
10478
  this.unarmed.add(workflowId);
10164
10479
  this.supervisor().reject(workflowId, block, configRecord(rawConfig), message).catch((recordError) => {
10165
- this.logger.error(`Could not record the rejected webhook trigger for workflow ${workflowId}`, recordError);
10480
+ this.laneFailed(`Could not record the rejected webhook trigger for workflow ${workflowId}`, recordError);
10166
10481
  });
10167
10482
  return;
10168
10483
  }
@@ -10288,7 +10603,7 @@ var WorkflowRuntimeService = class {
10288
10603
  const trigger = runnableDefinition(state).trigger;
10289
10604
  if (!trigger) return;
10290
10605
  this.supervisor().reject(workflowId, triggerBlock(trigger), configRecord(trigger.config), reason).catch((error) => {
10291
- this.logger.error(`Could not record the reactor access denial for workflow ${workflowId}`, error);
10606
+ this.laneFailed(`Could not record the reactor access denial for workflow ${workflowId}`, error);
10292
10607
  });
10293
10608
  }
10294
10609
  unarmed = /* @__PURE__ */ new Set();
@@ -10302,7 +10617,7 @@ var WorkflowRuntimeService = class {
10302
10617
  this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
10303
10618
  this.unarmed.add(workflowId);
10304
10619
  this.supervisor().reject(workflowId, block, configRecord(trigger.config), reason, retryAt).catch((error) => {
10305
- this.logger.error(`Could not record the unresolved trigger for workflow ${workflowId}`, error);
10620
+ this.laneFailed(`Could not record the unresolved trigger for workflow ${workflowId}`, error);
10306
10621
  });
10307
10622
  }
10308
10623
  resolutionRetries = /* @__PURE__ */ new Map();
@@ -10331,7 +10646,7 @@ var WorkflowRuntimeService = class {
10331
10646
  }
10332
10647
  dropSupervised(workflowId) {
10333
10648
  this.supervisor().remove(workflowId).catch((error) => {
10334
- this.logger.error(`Trigger disable failed for ${workflowId}`, error);
10649
+ this.laneFailed(`Trigger disable failed for ${workflowId}`, error);
10335
10650
  });
10336
10651
  }
10337
10652
  async refreshRegistration(workflowId, resultingState, onlyIfChanged = false) {
@@ -10356,6 +10671,7 @@ var WorkflowRuntimeService = class {
10356
10671
  return false;
10357
10672
  }
10358
10673
  async onDocumentsPurged(markers) {
10674
+ this.throwIfClosed();
10359
10675
  const documentIds = [...new Set(markers.map((marker) => marker.context.documentId))];
10360
10676
  const store = await this.store();
10361
10677
  if (!store) {
@@ -10368,8 +10684,10 @@ var WorkflowRuntimeService = class {
10368
10684
  return erased;
10369
10685
  }
10370
10686
  async onOperations(operations) {
10687
+ this.throwIfClosed();
10371
10688
  const hints = collectLifecycleParentHints(operations);
10372
10689
  for (const { operation, context } of operations) {
10690
+ this.throwIfClosed();
10373
10691
  if (context.scope !== DOCUMENT_SCOPE && context.scope !== "global") continue;
10374
10692
  const opKey = operationKey({
10375
10693
  operation,
@@ -10419,11 +10737,12 @@ var WorkflowRuntimeService = class {
10419
10737
  this.unjournaledFires.delete(oldest);
10420
10738
  this.logger.warn("Too many unjournaled workflow fires held; a redelivery of the oldest fires again");
10421
10739
  }
10422
- this.fireFromTrigger(workflowId, payload, kind);
10740
+ this.startFiring(workflowId, payload, kind);
10423
10741
  }
10424
10742
  async enqueueFire(workflowId, payload, kind, dedupeKey) {
10425
10743
  const fireKey = JSON.stringify([workflowId, dedupeKey]);
10426
10744
  const store = await this.store();
10745
+ this.throwIfClosed();
10427
10746
  if (this.unjournaledFires.has(fireKey)) {
10428
10747
  if (!store) return;
10429
10748
  try {
@@ -10464,12 +10783,35 @@ var WorkflowRuntimeService = class {
10464
10783
  }
10465
10784
  return;
10466
10785
  }
10467
- this.fireFromTrigger(workflowId, payload, kind, claim.runId);
10786
+ this.startFiring(workflowId, payload, kind, claim.runId);
10787
+ }
10788
+ async fireFromTrigger(workflowId, payload, kind) {
10789
+ const runId = await this.journalFiring(workflowId, payload, kind);
10790
+ this.startFiring(workflowId, payload, kind, runId);
10791
+ }
10792
+ async journalFiring(workflowId, payload, kind) {
10793
+ const store = await this.store();
10794
+ if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
10795
+ if (!store) return void 0;
10796
+ try {
10797
+ return await store.enqueueRun({
10798
+ workflowId,
10799
+ triggerKind: kind,
10800
+ triggerPayload: payload
10801
+ });
10802
+ } catch (error) {
10803
+ this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
10804
+ return;
10805
+ }
10468
10806
  }
10469
- fireFromTrigger(workflowId, payload, kind, enqueuedRunId) {
10807
+ startFiring(workflowId, payload, kind, enqueuedRunId) {
10470
10808
  this.fire(workflowId, payload, kind, void 0, void 0, enqueuedRunId).then((run) => {
10471
10809
  this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
10472
10810
  }, (error) => {
10811
+ if (enqueuedRunId && isShutdownRefusal(error)) {
10812
+ this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to rerun`);
10813
+ return;
10814
+ }
10473
10815
  this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
10474
10816
  });
10475
10817
  }
@@ -10624,8 +10966,8 @@ var WorkflowRuntimeService = class {
10624
10966
  const resolved = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolveWithSecrets(connectionId, request);
10625
10967
  return rememberSecrets(resolved.auth, resolved.secretValues);
10626
10968
  },
10627
- fire: (workflowId, payload, kind) => {
10628
- this.fireFromTrigger(workflowId, payload, kind);
10969
+ fire: (workflowId, payload, kind, runId) => {
10970
+ this.startFiring(workflowId, payload, kind, runId);
10629
10971
  },
10630
10972
  webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
10631
10973
  reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
@@ -10637,15 +10979,29 @@ var WorkflowRuntimeService = class {
10637
10979
  defaultIntervalMs: Number(process.env.PH_WORKFLOWS_POLL_INTERVAL_MS) || void 0,
10638
10980
  reconcileIntervalMs: Number(process.env.PH_WORKFLOWS_WEBHOOK_RECONCILE_MS) || void 0
10639
10981
  });
10982
+ if (this.closed) this.triggerSupervisor.stop();
10640
10983
  return this.triggerSupervisor;
10641
10984
  }
10985
+ laneFailed(message, error) {
10986
+ if (isShutdownRefusal(error)) {
10987
+ this.logger.debug(message, error);
10988
+ return;
10989
+ }
10990
+ this.logger.error(message, error);
10991
+ }
10642
10992
  startTriggerSupervisor() {
10643
10993
  this.supervisor().start();
10644
10994
  }
10645
10995
  stopTriggerSupervisor() {
10646
10996
  this.triggerSupervisor?.stop();
10647
10997
  }
10998
+ closed = false;
10999
+ throwIfClosed() {
11000
+ if (this.closed) throw new Error("This workflow runtime has shut down; the operations are left for the next owner");
11001
+ }
10648
11002
  shutdown() {
11003
+ this.closed = true;
11004
+ this.runGate.close();
10649
11005
  clearInterval(this.retentionTimer);
10650
11006
  for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
10651
11007
  this.resolutionRetries.clear();
@@ -10755,6 +11111,10 @@ var WorkflowRuntimeService = class {
10755
11111
  /** A delivery the service has already rate-limited, verified, de-duplicated and
10756
11112
  * answered any challenge for; all that is left is deciding what it means. */
10757
11113
  async deliverWebhook(request) {
11114
+ if (this.closed) return {
11115
+ status: 503,
11116
+ unprocessed: true
11117
+ };
10758
11118
  const workflowId = request.key;
10759
11119
  const registration = this.liveWebhook(workflowId);
10760
11120
  const test = this.webhookTests.get(workflowId);
@@ -10769,16 +11129,44 @@ var WorkflowRuntimeService = class {
10769
11129
  const { config } = registration;
10770
11130
  const payload = webhookPayload(request);
10771
11131
  if (config.responseMode === "async") {
10772
- this.fireFromTrigger(workflowId, payload, WEBHOOK_TRIGGER_KIND);
11132
+ try {
11133
+ await this.fireFromTrigger(workflowId, payload, WEBHOOK_TRIGGER_KIND);
11134
+ } catch (error) {
11135
+ if (error instanceof WorkflowRuntimeClosedError) return {
11136
+ status: 503,
11137
+ unprocessed: true
11138
+ };
11139
+ throw error;
11140
+ }
10773
11141
  return { status: config.responseStatus };
10774
11142
  }
10775
- const run = await Promise.race([this.fire(workflowId, payload, WEBHOOK_TRIGGER_KIND).then((result) => ({
11143
+ let runId;
11144
+ try {
11145
+ runId = await this.journalFiring(workflowId, payload, WEBHOOK_TRIGGER_KIND);
11146
+ } catch (error) {
11147
+ if (error instanceof WorkflowRuntimeClosedError) return {
11148
+ status: 503,
11149
+ unprocessed: true
11150
+ };
11151
+ throw error;
11152
+ }
11153
+ const firing = this.fire(workflowId, payload, WEBHOOK_TRIGGER_KIND, void 0, void 0, runId).then((result) => ({
10776
11154
  ok: true,
10777
11155
  result
10778
11156
  }), (error) => ({
10779
11157
  ok: false,
10780
11158
  error
10781
- })), timeout(DELIVERY_TIMEOUT_MS)]);
11159
+ }));
11160
+ let replied = false;
11161
+ const takeReply = () => !replied && (replied = true);
11162
+ if (runId) this.heldReplies.set(runId, takeReply);
11163
+ let run;
11164
+ try {
11165
+ run = await Promise.race([firing, timeout(DELIVERY_TIMEOUT_MS)]);
11166
+ if (run === TIMED_OUT && !takeReply()) run = await firing;
11167
+ } finally {
11168
+ if (runId) this.heldReplies.delete(runId);
11169
+ }
10782
11170
  if (run === TIMED_OUT) {
10783
11171
  this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
10784
11172
  return {
@@ -10791,6 +11179,10 @@ var WorkflowRuntimeService = class {
10791
11179
  };
10792
11180
  }
10793
11181
  if (!run.ok) {
11182
+ if (run.error instanceof WorkflowRuntimeClosedError) return {
11183
+ status: 503,
11184
+ unprocessed: true
11185
+ };
10794
11186
  const message = run.error instanceof Error ? run.error.message : String(run.error);
10795
11187
  this.logger.error(`Webhook run failed for ${workflowId}: @error`, message);
10796
11188
  return {
@@ -10815,7 +11207,7 @@ var WorkflowRuntimeService = class {
10815
11207
  try {
10816
11208
  return handshakeReply((await this.supervisor().handshake(binding, webhookPayload(request))).output);
10817
11209
  } catch (error) {
10818
- this.logger.error("Handshake failed for @block on workflow @workflow", blockLabel(binding.block), binding.workflowId, error);
11210
+ this.logger[isShutdownRefusal(error) ? "debug" : "error"]("Handshake failed for @block on workflow @workflow", blockLabel(binding.block), binding.workflowId, error);
10819
11211
  return { status: 500 };
10820
11212
  }
10821
11213
  }
@@ -10834,7 +11226,7 @@ var WorkflowRuntimeService = class {
10834
11226
  this.supervisor().deliverWebhook(binding.workflowId, payload).then(() => {
10835
11227
  this.logger.info("Webhook delivered to @block for workflow @workflow", blockLabel(binding.block), binding.workflowId);
10836
11228
  }, (error) => {
10837
- this.logger.error(`Webhook delivery failed for workflow ${binding.workflowId}`, error);
11229
+ this.laneFailed(`Webhook delivery failed for workflow ${binding.workflowId}`, error);
10838
11230
  });
10839
11231
  return { status: 200 };
10840
11232
  }
@@ -11301,14 +11693,17 @@ var WorkflowRuntimeService = class {
11301
11693
  const local = await this.localPieceAt(packageName, version);
11302
11694
  return local ? triggersResult(local.descriptor, local.piece.name, local.piece.version) : fetchPieceTriggers(packageName, version);
11303
11695
  }
11304
- async searchBlocks(query, limit) {
11696
+ warmPieceSearch() {
11697
+ warmSearchIndex();
11698
+ }
11699
+ async searchPieces(query, filter) {
11305
11700
  let local;
11306
11701
  try {
11307
- local = indexFromHits((await this.localPieces()).flatMap(({ piece, descriptor }) => localSearchHits(descriptor, piece.name, piece.version)));
11702
+ local = indexPieces((await this.localPieces()).map(({ piece, descriptor }) => localSearchPiece(descriptor, piece.name, piece.version)));
11308
11703
  } catch (error) {
11309
11704
  this.logger.warn("Could not index the package pieces: @error", String(error));
11310
11705
  }
11311
- return searchBlocks(query, limit, local);
11706
+ return searchPieces(query, filter, local);
11312
11707
  }
11313
11708
  async pieceDetail(packageName, version) {
11314
11709
  const local = await this.localPieceAt(packageName, version);
@@ -11714,6 +12109,7 @@ var WorkflowRuntimeService = class {
11714
12109
  return this.pieceWorkers = pool;
11715
12110
  }
11716
12111
  async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
12112
+ if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
11717
12113
  const started = performance.now();
11718
12114
  const attributes = {
11719
12115
  "workflow.id": workflowId,
@@ -11795,7 +12191,14 @@ var WorkflowRuntimeService = class {
11795
12191
  }
11796
12192
  if (parked) return skipped(parked, "parked");
11797
12193
  const admission = await this.runGate.admit(workflowId, policy);
11798
- if (!admission.admitted) return skipped(admission.reason, admission.refusal);
12194
+ if (this.closed) {
12195
+ if (admission.admitted) admission.release();
12196
+ return this.refuseClosed(store, workflowId, enqueuedRunId);
12197
+ }
12198
+ if (!admission.admitted) {
12199
+ if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
12200
+ return skipped(admission.reason, admission.refusal);
12201
+ }
11799
12202
  if (admission.waited) {
11800
12203
  let stale;
11801
12204
  try {
@@ -11900,6 +12303,7 @@ var WorkflowRuntimeService = class {
11900
12303
  }
11901
12304
  }
11902
12305
  runGate = new WorkflowRunGate();
12306
+ heldReplies = /* @__PURE__ */ new Map();
11903
12307
  /** Why a trigger's firing of a PARKED workflow is refused; undefined when
11904
12308
  * it is not parked, or when an operator started the run. */
11905
12309
  async parkedFiring(workflowId, triggerKind, publishedVersion) {
@@ -11941,6 +12345,19 @@ var WorkflowRuntimeService = class {
11941
12345
  * of bug this work package exists to stamp out. An already-enqueued row is
11942
12346
  * closed out in place, so nothing is left PENDING for a sweep to find.
11943
12347
  */
12348
+ async refuseClosed(store, workflowId, enqueuedRunId) {
12349
+ const error = new WorkflowRuntimeClosedError(workflowId);
12350
+ if (enqueuedRunId) {
12351
+ const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
12352
+ try {
12353
+ if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
12354
+ else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));
12355
+ } catch (failure) {
12356
+ this.logger.warn(`Could not close out run ${enqueuedRunId} after shutdown: @error`, failure);
12357
+ }
12358
+ }
12359
+ throw error;
12360
+ }
11944
12361
  async skipFiring(store, workflowId, enqueuedRunId, details, refusal) {
11945
12362
  this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
11946
12363
  let runId = enqueuedRunId ?? null;
@@ -11974,7 +12391,7 @@ var WorkflowRuntimeService = class {
11974
12391
  * engine has; it is marked as such rather than pretending to page anyone.
11975
12392
  */
11976
12393
  async applyFailureMode(policy, workflowId, publishedVersion, runId, error) {
11977
- if (policy.onFailure === "IGNORE") return;
12394
+ if (policy.onFailure === "IGNORE" || this.closed) return;
11978
12395
  const detail = error ?? "the run failed";
11979
12396
  if (policy.onFailure === "NOTIFY") {
11980
12397
  this.logger.error(`Workflow ${workflowId} run ${runId ?? "(unjournaled)"} failed and its policy is NOTIFY: @error`, detail);
@@ -11984,6 +12401,7 @@ var WorkflowRuntimeService = class {
11984
12401
  const reason = `Parked after a failed run (policy.onFailure = PARK): ${detail}`;
11985
12402
  try {
11986
12403
  if (!await this.stillRunsVersion(workflowId, publishedVersion)) return;
12404
+ if (this.closed) return;
11987
12405
  const trigger = await this.supervisor().park(workflowId, publishedVersion, reason);
11988
12406
  if (!await this.stillRunsVersion(workflowId, publishedVersion)) {
11989
12407
  await this.supervisor().liftPark(workflowId, publishedVersion, trigger);
@@ -12369,6 +12787,249 @@ function collectAttachmentRefs(value, found) {
12369
12787
  else if (typeof value === "object" && value !== null) for (const entry of Object.values(value)) collectAttachmentRefs(entry, found);
12370
12788
  }
12371
12789
  //#endregion
12790
+ //#region src/reactor/singleton-lease.ts
12791
+ const logger = childLogger([
12792
+ "workflow",
12793
+ "runtime",
12794
+ "singleton"
12795
+ ]);
12796
+ /**
12797
+ * Names this process as the workflow singleton's owner.
12798
+ *
12799
+ * Set it to a STABLE name per deployment slot (`switchboard-0`, the
12800
+ * StatefulSet pod name, the Render service id). The owner name is the
12801
+ * operator's contract: a restart under the same name re-claims its own lease
12802
+ * once the old heartbeat is stale instead of waiting out
12803
+ * {@link SINGLETON_LEASE_TTL_MS}.
12804
+ *
12805
+ * Unset, the owner is derived from a STABLE identity — the hostname and a
12806
+ * fingerprint of the journal's own storage location ({@link
12807
+ * AcquireSingletonOptions.storageId}) — not from the pid and a random suffix.
12808
+ * A random per-process owner meant every unclean kill locked the next boot out
12809
+ * for the whole {@link SINGLETON_LEASE_TTL_MS}: the dead process's lease was
12810
+ * nobody's to re-claim, and `release()` never ran. One deployment slot
12811
+ * restarting is the common case and it has to be quick; a genuine second
12812
+ * replica still differs, by hostname or by the journal it points at.
12813
+ */
12814
+ const WORKFLOW_SINGLETON_OWNER_ENV = "PH_WORKFLOWS_SINGLETON_OWNER";
12815
+ /** How long a claim stays valid without a heartbeat. */
12816
+ const SINGLETON_LEASE_TTL_MS = 6e4;
12817
+ /** How often the holder renews it; a third of the TTL, so two renewals may be
12818
+ * lost (a GC pause, a slow database) before anyone may take over. */
12819
+ const SINGLETON_HEARTBEAT_MS = SINGLETON_LEASE_TTL_MS / 3;
12820
+ /**
12821
+ * How long the holder's heartbeat must have been silent before a claim under
12822
+ * the SAME owner name takes the lease over, short of its expiry. Two renewal
12823
+ * periods: a live holder renews well inside it, and its self-fence fires
12824
+ * before it passes.
12825
+ */
12826
+ const SINGLETON_STALE_HEARTBEATS = 2;
12827
+ /** The one row. The table holds a single lease, named rather than keyed by a
12828
+ * magic empty string, so a `SELECT *` reads legibly in an operator's shell. */
12829
+ const LEASE_ID = "workflow-runtime";
12830
+ const dbNow = () => sql`now()`;
12831
+ const dbNowPlus = (ms) => sql`now() + ${ms} * interval '1 millisecond'`;
12832
+ /** Another live process holds the singleton. The refusal is the point: a host
12833
+ * that cannot own workflow execution must not compose the engine. */
12834
+ var WorkflowSingletonConflictError = class extends Error {
12835
+ owner;
12836
+ expiresAt;
12837
+ wouldBe;
12838
+ constructor(owner, expiresAt, wouldBe) {
12839
+ super(`Workflow execution is a singleton and "${owner}" holds it until ${expiresAt}; this process ("${wouldBe}") must not compose the workflow runtime. Two replicas over one run journal fail each other's live runs and arm every trigger twice. Run workflows on exactly one reactor, or set ${WORKFLOW_SINGLETON_OWNER_ENV} to the same stable owner name on the slot that owns them so a restart re-claims its own lease.`);
12840
+ this.owner = owner;
12841
+ this.expiresAt = expiresAt;
12842
+ this.wouldBe = wouldBe;
12843
+ this.name = "WorkflowSingletonConflictError";
12844
+ }
12845
+ };
12846
+ /**
12847
+ * The owner name this process claims under.
12848
+ *
12849
+ * `<hostname>/<storage fingerprint>`, so it is the same name on every boot of
12850
+ * one deployment slot — a restart re-claims its OWN lease once the dead
12851
+ * process's heartbeat is stale, rather than waiting out the TTL — and a different
12852
+ * name on any host or journal that is genuinely somebody else.
12853
+ *
12854
+ * The storage location is hashed, never printed: it can be a Postgres URL with
12855
+ * credentials in it, and the owner name goes into a database row an operator
12856
+ * reads and into every log line about the lease.
12857
+ *
12858
+ * The residual case a stable name cannot tell apart is two processes on ONE
12859
+ * host over ONE journal, which is a misconfiguration those two already share
12860
+ * (one read-model directory, one run journal). It is not silent: the loser's
12861
+ * heartbeat finds the lease taken and says so by name.
12862
+ */
12863
+ function singletonOwnerName(env = process.env, storageId) {
12864
+ const configured = env[WORKFLOW_SINGLETON_OWNER_ENV]?.trim();
12865
+ if (configured) return configured;
12866
+ return `${hostname()}/${storageFingerprint(storageId)}`;
12867
+ }
12868
+ function storageFingerprint(storageId) {
12869
+ const source = storageId?.trim();
12870
+ if (!source) return "default";
12871
+ return createHash("sha256").update(source).digest("hex").slice(0, 12);
12872
+ }
12873
+ async function ensureTable(db) {
12874
+ await db.schema.createTable("workflow_singleton").addColumn("id", "text", (col) => col.primaryKey()).addColumn("owner", "text", (col) => col.notNull()).addColumn("instance", "text", (col) => col.notNull()).addColumn("acquired_at", "timestamptz", (col) => col.notNull()).addColumn("heartbeat_at", "timestamptz", (col) => col.notNull()).addColumn("expires_at", "timestamptz", (col) => col.notNull()).ifNotExists().execute();
12875
+ }
12876
+ const DDL_ATTEMPTS = 3;
12877
+ const CONCURRENT_DDL_CODES = /* @__PURE__ */ new Set([
12878
+ "23505",
12879
+ "42710",
12880
+ "42P06",
12881
+ "42P07"
12882
+ ]);
12883
+ function isConcurrentDdl(error) {
12884
+ const code = error?.code;
12885
+ return typeof code === "string" && CONCURRENT_DDL_CODES.has(code);
12886
+ }
12887
+ async function openLeaseTable(relationalDb) {
12888
+ const open = async () => {
12889
+ const db = await relationalDb.createNamespace("workflow_runtime");
12890
+ await ensureTable(db);
12891
+ return db;
12892
+ };
12893
+ for (let attempt = 1;; attempt++) try {
12894
+ return await open();
12895
+ } catch (error) {
12896
+ if (attempt >= DDL_ATTEMPTS || !isConcurrentDdl(error)) throw error;
12897
+ }
12898
+ }
12899
+ /**
12900
+ * Claims the workflow singleton, or refuses.
12901
+ *
12902
+ * Three steps, each one a single statement whose own outcome is the claim — a
12903
+ * prior `SELECT` would be a race:
12904
+ *
12905
+ * 1. `INSERT … ON CONFLICT DO NOTHING RETURNING` — the first ever boot.
12906
+ * 2. `UPDATE … WHERE expires_at <= now OR (owner = me AND heartbeat stale)
12907
+ * RETURNING` — a dead lease taken over once it has expired, or our own
12908
+ * slot's lease once its holder stopped renewing (a restart under a stable
12909
+ * owner name), never a live holder's. Either way the row gets this claim's
12910
+ * instance, so the previous holder's heartbeat and release no longer
12911
+ * match.
12912
+ * 3. Neither matched: someone live holds it, and we refuse by name.
12913
+ */
12914
+ async function acquireWorkflowSingletonLease(options) {
12915
+ const log = options.logger ?? logger;
12916
+ const ttlMs = options.ttlMs ?? 6e4;
12917
+ const heartbeatMs = options.heartbeatMs ?? 2e4;
12918
+ const staleMs = heartbeatMs * 2;
12919
+ const owner = options.owner ?? singletonOwnerName(options.env, options.storageId);
12920
+ const instance = randomUUID();
12921
+ const db = await openLeaseTable(options.relationalDb);
12922
+ const stamps = () => ({
12923
+ heartbeat_at: dbNow(),
12924
+ expires_at: dbNowPlus(ttlMs)
12925
+ });
12926
+ const claim = async () => {
12927
+ if (await db.insertInto("workflow_singleton").values({
12928
+ id: LEASE_ID,
12929
+ owner,
12930
+ instance,
12931
+ acquired_at: dbNow(),
12932
+ ...stamps()
12933
+ }).onConflict((oc) => oc.column("id").doNothing()).returning("owner").executeTakeFirst() !== void 0) return true;
12934
+ return await db.updateTable("workflow_singleton").set({
12935
+ owner,
12936
+ instance,
12937
+ acquired_at: dbNow(),
12938
+ ...stamps()
12939
+ }).where("id", "=", LEASE_ID).where((eb) => eb.or([eb("expires_at", "<=", dbNow()), eb.and([eb("owner", "=", owner), eb("heartbeat_at", "<=", dbNowPlus(-staleMs))])])).returning("owner").executeTakeFirst() !== void 0;
12940
+ };
12941
+ const claimSentAt = performance.now();
12942
+ if (!await claim()) {
12943
+ const held = await db.selectFrom("workflow_singleton").selectAll().where("id", "=", LEASE_ID).executeTakeFirst();
12944
+ throw new WorkflowSingletonConflictError(held?.owner ?? "unknown", held ? new Date(held.expires_at).toISOString() : "unknown", owner);
12945
+ }
12946
+ log.info(`Workflow singleton claimed by "${owner}" (lease ${ttlMs}ms, renewed every ${heartbeatMs}ms)`);
12947
+ const localNow = () => performance.now();
12948
+ const tickMs = heartbeatMs / 4;
12949
+ const fenceMs = Math.min(staleMs, ttlMs) - Math.min(heartbeatMs, ttlMs) / 2;
12950
+ const selfFence = options.selfFence ?? true;
12951
+ let renewedAt = claimSentAt;
12952
+ let renewFailed = false;
12953
+ let inFlight;
12954
+ let timer;
12955
+ let lost = false;
12956
+ let takenOver = false;
12957
+ let releasing = false;
12958
+ const stop = () => {
12959
+ if (timer) clearInterval(timer);
12960
+ timer = void 0;
12961
+ };
12962
+ const markLost = (loss) => {
12963
+ if (lost) return;
12964
+ lost = true;
12965
+ stop();
12966
+ log.error(loss.reason === "taken" ? `Workflow singleton lease of "${owner}" was taken over by "${loss.heldBy ?? "nobody"}"; this process no longer owns workflow execution and stops running it.` : `Workflow singleton lease of "${owner}" could not be renewed for ${Math.round(loss.silentMs)}ms, long enough that another process could take it over (none was seen to); this process stops running workflows.`);
12967
+ options.onLost?.(loss);
12968
+ };
12969
+ const renew = async () => {
12970
+ const sentAt = localNow();
12971
+ if (await db.updateTable("workflow_singleton").set(stamps()).where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).returning("owner").executeTakeFirst() !== void 0) {
12972
+ renewedAt = sentAt;
12973
+ return true;
12974
+ }
12975
+ if (releasing || lost) return false;
12976
+ let heldBy;
12977
+ try {
12978
+ heldBy = (await db.selectFrom("workflow_singleton").select("owner").where("id", "=", LEASE_ID).executeTakeFirst())?.owner;
12979
+ } catch {
12980
+ heldBy = void 0;
12981
+ }
12982
+ takenOver = true;
12983
+ markLost({
12984
+ reason: "taken",
12985
+ heldBy
12986
+ });
12987
+ return false;
12988
+ };
12989
+ const heartbeat = () => {
12990
+ if (lost) return Promise.resolve(false);
12991
+ inFlight ??= renew().finally(() => {
12992
+ inFlight = void 0;
12993
+ });
12994
+ return inFlight;
12995
+ };
12996
+ timer = setInterval(() => {
12997
+ if (lost || releasing) return;
12998
+ const silentMs = localNow() - renewedAt;
12999
+ if (selfFence && silentMs >= fenceMs) {
13000
+ markLost({
13001
+ reason: "unrenewable",
13002
+ silentMs
13003
+ });
13004
+ return;
13005
+ }
13006
+ if (inFlight) return;
13007
+ if (!renewFailed && silentMs < heartbeatMs - tickMs / 2) return;
13008
+ heartbeat().then(() => {
13009
+ renewFailed = false;
13010
+ }, (error) => {
13011
+ renewFailed = true;
13012
+ log.warn("Workflow singleton heartbeat failed: @error", error);
13013
+ });
13014
+ }, tickMs);
13015
+ timer.unref();
13016
+ return {
13017
+ owner,
13018
+ heartbeat,
13019
+ async release() {
13020
+ releasing = true;
13021
+ stop();
13022
+ if (takenOver) return;
13023
+ if (inFlight) await Promise.race([inFlight.catch(() => false), new Promise((resolve) => setTimeout(resolve, tickMs).unref())]);
13024
+ try {
13025
+ await db.deleteFrom("workflow_singleton").where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).execute();
13026
+ } catch (error) {
13027
+ log.warn("Could not release the workflow singleton lease: @error", error);
13028
+ }
13029
+ }
13030
+ };
13031
+ }
13032
+ //#endregion
12372
13033
  //#region src/reactor/workflow-triggers-read-model.ts
12373
13034
  const WORKFLOW_TRIGGERS_READ_MODEL = "workflow-triggers";
12374
13035
  const WORKFLOW_TRIGGERS_READ_MODEL_STAGE = "post_ready";
@@ -12395,6 +13056,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
12395
13056
  }
12396
13057
  };
12397
13058
  //#endregion
12398
- export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
13059
+ export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl, singletonOwnerName };
12399
13060
 
12400
13061
  //# sourceMappingURL=index.js.map