@powerhousedao/reactor-workflow 6.2.3-dev.49 → 6.2.3-dev.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,16 +1,17 @@
1
- import { B as UnsupportedPieceFeatureError, H as unsupportedAuth, M as maxFileBytes, U as unsupportedTrigger, V as authMethodFor, a as redact, c as rememberSecrets, f as isPrivateAddress, i as containsRedactedMarker, k as FileTooLargeError, l as secretsFor, p as parseAddress, s as redactMessage, u as DEFAULT_EGRESS_POLICY, w as extractDedupeKey } from "./reactor-rpc-BqAjCK1p.js";
2
- import { A as expressionsIn, B as PieceWorkerError, C as CORE_PIECE_VERSION, D as declaredReactorConnectionIds, E as declaredConnectionIds, F as secretRefFromId, H as bundleResolver, I as stepBlock, J as pieceRegistrySource, L as triggerBlock, M as SecretDeletedError, N as SecretNotFoundError, O as shapeConnection, P as parseSecretRef, R as PieceWorkerPool, S as CORE_PIECE_NAME, U as pieceModuleRef, V as PieceWorkerTimeoutError, W as sourcedResolver, Y as setPieceRegistryUrl, _ as blockLabel, a as errorNameOf, b as builtinPiece, c as hashOf, d as checkDynamicProperties, f as ActivepiecesBlockExecutor, g as storeHandlers, h as resolvedBlock, i as packagePieces, k as resolveStepInput, l as stepConfigHash, m as isHostBound, n as installedPiece, o as runWorkflow, p as CompositeBlockExecutor, q as npmRegistryBaseUrl, r as installedPieces, s as canonicalJson, t as PieceRegistry, u as DynamicPropertiesError, v as unpinnedNote, w as BoundConnectionResolver, x as describeBuiltinPiece, y as blockPorts, z as PieceWorker } from "./piece-registry-CpiVMWNv.js";
3
- import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError as ReactorActionsFailedError$1, ReactorJobFailedError as ReactorJobFailedError$1, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
4
- import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
5
- import { open, readFile, rm } from "node:fs/promises";
6
- import { dirname, join } from "node:path";
1
+ import { $ as isPrivateAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerError, H as PieceWorkerPool, I as SecretDeletedError, J as redact, K as PieceWorkerTimeoutError, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as DEFAULT_EGRESS_POLICY, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerDisposedError, X as rememberSecrets, Y as redactMessage, Z as secretsFor, _ as resolvedBlock, _t as isIndeterminateError, at as pieceModuleRef, b as resolutionOf, c as canonicalJson, ct as UnsupportedPieceFeatureError, d as DynamicPropertiesError, dt as unsupportedTrigger, et as parseAddress, f as checkDynamicProperties, g as isHostBound, gt as setPieceRegistryUrl, h as CompositeBlockExecutor, ht as pieceRegistrySource, i as packagePieces, it as bundleResolver, j as declaredReactorConnectionIds, l as hashOf, lt as authMethodFor, m as ActivepiecesBlockExecutor, mt as npmRegistryBaseUrl, n as installedPiece, nt as FileTooLargeError, o as errorNameOf, ot as sourcedResolver, p as effectiveRetryPolicy, q as containsRedactedMarker, r as installedPieces, rt as maxFileBytes, s as runWorkflow, t as PieceRegistry, tt as extractDedupeKey, u as stepConfigHash, ut as unsupportedAuth, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-D0pa4v0T.js";
2
+ import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
3
+ import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
4
+ import { open, readdir, rm, stat } from "node:fs/promises";
5
+ import path, { dirname, join } from "node:path";
7
6
  import { childLogger, generateId } from "document-model";
8
7
  import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
9
8
  import { Cron } from "croner";
10
9
  import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
10
+ import { Readable } from "node:stream";
11
11
  import dns from "node:dns";
12
12
  import net from "node:net";
13
13
  import { AsyncLocalStorage } from "node:async_hooks";
14
+ import { hostname } from "node:os";
14
15
  import { blockKey, compareVersions, isExactVersion, rankClosestVersions } from "@powerhousedao/pieces-framework/block-type";
15
16
  import { actions, isReactorConnectorId, parseReactorConnectionConfig } from "@powerhousedao/workflow/document-models/connection";
16
17
  import http from "node:http";
@@ -18,6 +19,7 @@ import https from "node:https";
18
19
  import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
19
20
  import { Kind, parse } from "graphql";
20
21
  import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
22
+ import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
21
23
  import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
22
24
  import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
23
25
  //#region src/pieces/activepieces/piece-versions.ts
@@ -98,7 +100,15 @@ function npmVersions(name, options = {}) {
98
100
  }
99
101
  //#endregion
100
102
  //#region src/reactor/attachment-port.ts
101
- const logger$6 = childLogger(["workflow", "attachments"]);
103
+ const logger$7 = childLogger(["workflow", "attachments"]);
104
+ function fileStream(filePath) {
105
+ return Readable.toWeb(createReadStream(filePath));
106
+ }
107
+ async function sha256File(filePath) {
108
+ const hash = createHash("sha256");
109
+ for await (const chunk of createReadStream(filePath)) hash.update(chunk);
110
+ return hash.digest("hex");
111
+ }
102
112
  function refOf(result) {
103
113
  const direct = result.ref;
104
114
  if (typeof direct === "string") return direct;
@@ -128,16 +138,24 @@ async function writeCapped(body, destPath, limit) {
128
138
  }
129
139
  await handle.close();
130
140
  }
131
- function createAttachmentPort(client, documentIdFor, canReadRef) {
141
+ function createAttachmentPort(client, access) {
142
+ const authorize = async (ref) => {
143
+ const documentId = access.documentIdFor();
144
+ if (!documentId) throw new Error(`Cannot resolve ${ref}: no workflow document is in scope to authorize the read`);
145
+ if (!await access.canReadRef(ref)) throw new Error(`Cannot resolve ${ref}: workflow "${documentId}" may not read it`);
146
+ return documentId;
147
+ };
132
148
  return {
133
- async read(ref, destPath) {
134
- const documentId = documentIdFor();
135
- if (!documentId) throw new Error(`Cannot resolve ${ref}: no workflow document is in scope to authorize the read`);
136
- if (!await canReadRef(documentId, ref)) throw new Error(`Cannot resolve ${ref}: workflow "${documentId}" may not read it`);
149
+ async authorize(ref) {
150
+ await authorize(ref);
151
+ },
152
+ async read(ref, destPath, signal) {
153
+ const documentId = await authorize(ref);
137
154
  const limit = maxFileBytes();
138
155
  const { header, body } = await client.download({
139
156
  documentId,
140
- ref
157
+ ref,
158
+ ...signal ? { signal } : {}
141
159
  });
142
160
  if (typeof header.sizeBytes === "number" && Number.isFinite(header.sizeBytes) && header.sizeBytes > limit) {
143
161
  await body.cancel().catch(() => void 0);
@@ -150,14 +168,30 @@ function createAttachmentPort(client, documentIdFor, canReadRef) {
150
168
  contentType
151
169
  };
152
170
  },
153
- async write(file) {
154
- const bytes = await readFile(file.path);
171
+ async write(file, signal) {
172
+ const hash = await sha256File(file.path);
173
+ const mimeType = file.contentType ?? "application/octet-stream";
174
+ const extension = path.extname(file.fileName).slice(1);
175
+ const preprocessed = {
176
+ ref: `attachment://v1:${hash}`,
177
+ hash,
178
+ sizeBytes: file.size,
179
+ options: {
180
+ mimeType,
181
+ fileName: file.fileName,
182
+ ...extension ? { extension } : {},
183
+ clientHash: hash,
184
+ sizeBytes: file.size
185
+ },
186
+ data: fileStream(file.path),
187
+ stream: () => fileStream(file.path)
188
+ };
155
189
  const ref = refOf(await client.upload({
156
- file: new Blob([new Uint8Array(bytes)], { type: file.contentType ?? "application/octet-stream" }),
157
- fileName: file.fileName,
158
- mimeType: file.contentType
190
+ preprocessed,
191
+ ...signal ? { signal } : {}
159
192
  }));
160
- logger$6.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
193
+ access.onWritten?.(ref);
194
+ logger$7.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
161
195
  return ref;
162
196
  }
163
197
  };
@@ -771,7 +805,7 @@ const CACHE_TTL_MS = 3e5;
771
805
  const PINNED_TTL_MS = 36e5;
772
806
  const REGISTRY_PAGE_SIZE = 50;
773
807
  const REGISTRY_MAX_PAGES = 200;
774
- const logger$5 = childLogger(["workflow", "piece-catalog"]);
808
+ const logger$6 = childLogger(["workflow", "piece-catalog"]);
775
809
  function aiLast(audience) {
776
810
  return audience === "ai" ? 1 : 0;
777
811
  }
@@ -817,7 +851,7 @@ async function fetchRegistryCatalog(url, timeoutMs) {
817
851
  items.push(...page.items);
818
852
  if (!page.hasMore || page.items.length === 0) return items;
819
853
  }
820
- logger$5.warn(`${url} still had more pieces after ${REGISTRY_MAX_PAGES} pages`);
854
+ logger$6.warn(`${url} still had more pieces after ${REGISTRY_MAX_PAGES} pages`);
821
855
  return items;
822
856
  }
823
857
  function asList(value) {
@@ -831,10 +865,10 @@ async function publishedLists(suggestions, timeoutMs) {
831
865
  const cloudUrl = suggestions ? `${CATALOG_URL}?suggestionType=ACTION_AND_TRIGGER` : CATALOG_URL;
832
866
  const [fromRegistry, fromCloud] = await Promise.allSettled([source ? fetchRegistryCatalog(source.catalogUrl(suggestions), timeoutMs) : Promise.resolve([]), fetchJson(cloudUrl, timeoutMs)]);
833
867
  const registry = fromRegistry.status === "fulfilled" ? asList(fromRegistry.value).filter((entry) => typeof entry.name === "string") : [];
834
- if (fromRegistry.status === "rejected") logger$5.warn("Piece registry @url did not answer: @error", source?.baseUrl ?? "?", String(fromRegistry.reason));
868
+ if (fromRegistry.status === "rejected") logger$6.warn("Piece registry @url did not answer: @error", source?.baseUrl ?? "?", String(fromRegistry.reason));
835
869
  if (fromCloud.status === "rejected") {
836
870
  if (registry.length === 0) throw asError(fromCloud.reason);
837
- logger$5.warn("Serving registry pieces only: @error", String(fromCloud.reason));
871
+ logger$6.warn("Serving registry pieces only: @error", String(fromCloud.reason));
838
872
  return {
839
873
  registry,
840
874
  cloud: []
@@ -849,12 +883,18 @@ function registryFirst(registry, cloud) {
849
883
  const claimed = new Set(registry.map((entry) => entry.name));
850
884
  return [...registry, ...cloud.filter((entry) => !claimed.has(entry.name))];
851
885
  }
886
+ function tagged(entries, source) {
887
+ return entries.map((entry) => ({
888
+ ...entry,
889
+ source
890
+ }));
891
+ }
852
892
  async function fetchCatalogWithSuggestions() {
853
893
  const { registry, cloud } = await publishedLists(true, 12e4);
854
- return registryFirst(registry, cloud);
894
+ return registryFirst(tagged(registry, "registry"), tagged(cloud, "activepieces"));
855
895
  }
856
896
  let catalogCache;
857
- function toSummaries(raw) {
897
+ function toSummaries(raw, source) {
858
898
  return raw.filter((entry) => typeof entry.name === "string" && typeof entry.version === "string" && !SERVER_ONLY_PIECES.has(entry.name) && (typeof entry.actions === "number" && entry.actions > 0 || typeof entry.triggers === "number" && entry.triggers > 0)).map((entry) => ({
859
899
  name: entry.name,
860
900
  displayName: entry.displayName ?? entry.name,
@@ -864,6 +904,7 @@ function toSummaries(raw) {
864
904
  actionCount: typeof entry.actions === "number" ? entry.actions : 0,
865
905
  triggerCount: typeof entry.triggers === "number" ? entry.triggers : 0,
866
906
  categories: entry.categories ?? [],
907
+ source,
867
908
  auth: entry.auth ?? null,
868
909
  ...entry.deprecated === true ? { deprecated: true } : {},
869
910
  ...reasonOf(unsupportedAuth(entry.auth))
@@ -872,7 +913,7 @@ function toSummaries(raw) {
872
913
  async function fetchPieceCatalog() {
873
914
  if (catalogCache && catalogCache.expiresAt > Date.now()) return catalogCache.value;
874
915
  const lists = await publishedLists(false, 3e4);
875
- const value = [...registryFirst(toSummaries(lists.registry), toSummaries(lists.cloud))].sort((a, b) => a.displayName.localeCompare(b.displayName));
916
+ const value = [...registryFirst(toSummaries(lists.registry, "registry"), toSummaries(lists.cloud, "activepieces"))].sort((a, b) => a.displayName.localeCompare(b.displayName));
876
917
  catalogCache = {
877
918
  value,
878
919
  expiresAt: Date.now() + CACHE_TTL_MS
@@ -884,8 +925,8 @@ async function fetchPieceJson(packageName, version) {
884
925
  if (source) try {
885
926
  return await fetchJson(atVersion(source.pieceUrl(packageName), version));
886
927
  } catch (error) {
887
- if (error instanceof CatalogStatusError && error.status === 404) logger$5.debug("@registry does not serve @piece", source.baseUrl, packageName);
888
- else logger$5.warn("Piece registry @registry did not answer for @piece: @error", source.baseUrl, packageName, String(error));
928
+ if (error instanceof CatalogStatusError && error.status === 404) logger$6.debug("@registry does not serve @piece", source.baseUrl, packageName);
929
+ else logger$6.warn("Piece registry @registry did not answer for @piece: @error", source.baseUrl, packageName, String(error));
889
930
  }
890
931
  return fetchJson(pieceUrl(packageName, version));
891
932
  }
@@ -974,6 +1015,7 @@ function catalogEntry(descriptor, pieceName, version) {
974
1015
  actionCount: descriptor.actions.length,
975
1016
  triggerCount: descriptor.triggers.length,
976
1017
  categories: descriptor.categories ?? [],
1018
+ source: "local",
977
1019
  auth: clientAuth(descriptor.auth),
978
1020
  ...reasonOf(descriptor.unsupported)
979
1021
  };
@@ -1008,32 +1050,44 @@ function triggersResult(descriptor, pieceName, version) {
1008
1050
  auth: clientAuth(descriptor.auth)
1009
1051
  };
1010
1052
  }
1011
- function localSearchHits(descriptor, pieceName, version) {
1053
+ function localSearchPiece(descriptor, pieceName, version) {
1012
1054
  const pieceDisplayName = descriptor.displayName || pieceName;
1013
1055
  const logoUrl = descriptor.logoUrl ?? "";
1014
- return [...descriptor.actions.map((action) => ({
1015
- pieceName,
1016
- pieceVersion: version,
1017
- name: action.name,
1018
- pieceDisplayName,
1019
- logoUrl,
1020
- displayName: action.displayName,
1021
- description: action.description ?? "",
1022
- kind: "action",
1023
- strategy: null,
1024
- ...reasonOf(descriptor.unsupported)
1025
- })), ...descriptor.triggers.map((trigger) => ({
1026
- pieceName,
1027
- pieceVersion: version,
1028
- name: trigger.name,
1029
- pieceDisplayName,
1030
- logoUrl,
1031
- displayName: trigger.displayName,
1032
- description: trigger.description ?? "",
1033
- kind: "trigger",
1034
- strategy: trigger.strategy,
1035
- ...reasonOf(descriptor.unsupported ?? trigger.unsupported)
1036
- }))];
1056
+ return {
1057
+ meta: {
1058
+ pieceName,
1059
+ pieceVersion: version,
1060
+ displayName: pieceDisplayName,
1061
+ description: descriptor.description ?? "",
1062
+ logoUrl,
1063
+ categories: descriptor.categories ?? [],
1064
+ source: "local",
1065
+ ...reasonOf(descriptor.unsupported)
1066
+ },
1067
+ blocks: [...descriptor.actions.map((action) => ({
1068
+ pieceName,
1069
+ pieceVersion: version,
1070
+ name: action.name,
1071
+ pieceDisplayName,
1072
+ logoUrl,
1073
+ displayName: action.displayName,
1074
+ description: action.description ?? "",
1075
+ kind: "action",
1076
+ strategy: null,
1077
+ ...reasonOf(descriptor.unsupported)
1078
+ })), ...descriptor.triggers.map((trigger) => ({
1079
+ pieceName,
1080
+ pieceVersion: version,
1081
+ name: trigger.name,
1082
+ pieceDisplayName,
1083
+ logoUrl,
1084
+ displayName: trigger.displayName,
1085
+ description: trigger.description ?? "",
1086
+ kind: "trigger",
1087
+ strategy: trigger.strategy,
1088
+ ...reasonOf(descriptor.unsupported ?? trigger.unsupported)
1089
+ }))]
1090
+ };
1037
1091
  }
1038
1092
  function detailResult(descriptor, pieceName, version) {
1039
1093
  return {
@@ -1073,11 +1127,41 @@ function detailResult(descriptor, pieceName, version) {
1073
1127
  //#endregion
1074
1128
  //#region src/reactor/block-search.ts
1075
1129
  const INDEX_TTL_MS = 36e5;
1076
- const DEFAULT_LIMIT = 30;
1077
- const MAX_LIMIT = 100;
1130
+ const DEFAULT_LIMIT = 20;
1131
+ const MAX_LIMIT = 50;
1132
+ const MAX_BLOCKS_PER_PIECE = 50;
1133
+ function tokenize(text) {
1134
+ return text.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
1135
+ }
1136
+ function wordsOf(...texts) {
1137
+ const words = texts.flatMap(tokenize);
1138
+ return {
1139
+ words,
1140
+ compact: words.join("")
1141
+ };
1142
+ }
1143
+ function slugOf(pieceName) {
1144
+ return pieceName.replace(/^@[^/]+\//, "").replace(/^piece-/, "");
1145
+ }
1146
+ function indexPiece(piece) {
1147
+ const { meta } = piece;
1148
+ return {
1149
+ meta,
1150
+ name: wordsOf(meta.displayName, slugOf(meta.pieceName)),
1151
+ exactName: tokenize(meta.displayName).join(" "),
1152
+ about: wordsOf(meta.description, ...meta.categories),
1153
+ blocks: piece.blocks.map((hit) => ({
1154
+ hit,
1155
+ name: wordsOf(hit.displayName, hit.name),
1156
+ description: wordsOf(hit.description)
1157
+ }))
1158
+ };
1159
+ }
1160
+ function indexPieces(pieces) {
1161
+ return { pieces: pieces.map(indexPiece) };
1162
+ }
1078
1163
  function buildSearchIndex(raw) {
1079
- const entries = [];
1080
- let pieces = 0;
1164
+ const pieces = [];
1081
1165
  for (const entry of raw) {
1082
1166
  if (typeof entry.name !== "string" || typeof entry.version !== "string" || SERVER_ONLY_PIECES.has(entry.name)) continue;
1083
1167
  const pieceName = entry.name;
@@ -1085,78 +1169,141 @@ function buildSearchIndex(raw) {
1085
1169
  const pieceDisplayName = entry.displayName ?? pieceName;
1086
1170
  const logoUrl = entry.logoUrl ?? "";
1087
1171
  const pieceUnsupported = unsupportedAuth(entry.auth);
1172
+ const blocks = [];
1088
1173
  const push = (kind, item) => {
1089
1174
  if (typeof item.name !== "string" || item.name === "") return;
1090
- const displayName = item.displayName ?? item.name;
1091
- const description = item.description ?? "";
1092
- entries.push({
1093
- hit: {
1094
- pieceName,
1095
- pieceVersion,
1096
- name: item.name,
1097
- pieceDisplayName,
1098
- logoUrl,
1099
- displayName,
1100
- description,
1101
- kind,
1102
- strategy: kind === "trigger" ? item.type ?? null : null,
1103
- ...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
1104
- },
1105
- name: `${displayName} ${item.name}`.toLowerCase(),
1106
- description: description.toLowerCase(),
1107
- piece: pieceDisplayName.toLowerCase()
1175
+ blocks.push({
1176
+ pieceName,
1177
+ pieceVersion,
1178
+ name: item.name,
1179
+ pieceDisplayName,
1180
+ logoUrl,
1181
+ displayName: item.displayName ?? item.name,
1182
+ description: item.description ?? "",
1183
+ kind,
1184
+ strategy: kind === "trigger" ? item.type ?? null : null,
1185
+ ...reasonOf(pieceUnsupported ?? (kind === "trigger" ? unsupportedTrigger(item) : void 0))
1108
1186
  });
1109
1187
  };
1110
1188
  for (const action of entry.suggestedActions ?? []) push("action", action);
1111
1189
  for (const trigger of entry.suggestedTriggers ?? []) push("trigger", trigger);
1112
- pieces += 1;
1190
+ pieces.push({
1191
+ meta: {
1192
+ pieceName,
1193
+ pieceVersion,
1194
+ displayName: pieceDisplayName,
1195
+ description: entry.description ?? "",
1196
+ logoUrl,
1197
+ categories: Array.isArray(entry.categories) ? entry.categories : [],
1198
+ source: entry.source ?? "activepieces",
1199
+ ...entry.deprecated === true ? { deprecated: true } : {},
1200
+ ...reasonOf(pieceUnsupported)
1201
+ },
1202
+ blocks
1203
+ });
1113
1204
  }
1114
- return {
1115
- entries,
1116
- pieces
1117
- };
1118
- }
1119
- function indexFromHits(hits) {
1120
- const pieces = /* @__PURE__ */ new Set();
1121
- return {
1122
- entries: hits.map((hit) => {
1123
- pieces.add(hit.pieceName);
1124
- return {
1125
- hit,
1126
- name: `${hit.displayName} ${hit.name}`.toLowerCase(),
1127
- description: hit.description.toLowerCase(),
1128
- piece: hit.pieceDisplayName.toLowerCase()
1129
- };
1130
- }),
1131
- pieces: pieces.size
1132
- };
1205
+ return indexPieces(pieces);
1133
1206
  }
1134
1207
  function merge(index, local) {
1135
- const localEntries = local?.entries ?? [];
1136
- const localNames = new Set(localEntries.map((entry) => entry.hit.pieceName));
1137
- const published = (index?.entries ?? []).filter((entry) => !localNames.has(entry.hit.pieceName));
1138
- const shadowed = new Set((index?.entries ?? []).map((entry) => entry.hit.pieceName).filter((name) => localNames.has(name)));
1139
- return {
1140
- entries: [...localEntries, ...published],
1141
- pieces: (local?.pieces ?? 0) + (index?.pieces ?? 0) - shadowed.size
1142
- };
1143
- }
1144
- function searchIndex(index, query, limit = DEFAULT_LIMIT) {
1145
- const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
1208
+ const localPieces = local?.pieces ?? [];
1209
+ const localNames = new Set(localPieces.map((piece) => piece.meta.pieceName));
1210
+ return { pieces: [...localPieces, ...(index?.pieces ?? []).filter((piece) => !localNames.has(piece.meta.pieceName))] };
1211
+ }
1212
+ const EXACT = 4;
1213
+ const PREFIX = 3;
1214
+ const INNER = 1.5;
1215
+ const FUZZY = 1;
1216
+ const PIECE_NAME = 3;
1217
+ const BLOCK_NAME = 2;
1218
+ const BLOCK_DESCRIPTION = 1;
1219
+ const PIECE_ABOUT = .5;
1220
+ function editDistance(a, b, max) {
1221
+ if (Math.abs(a.length - b.length) > max) return max + 1;
1222
+ let prevPrev = [];
1223
+ let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
1224
+ for (let i = 1; i <= a.length; i++) {
1225
+ const row = [i];
1226
+ let rowMin = i;
1227
+ for (let j = 1; j <= b.length; j++) {
1228
+ const cost = a[i - 1] === b[j - 1] ? 0 : 1;
1229
+ let value = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost);
1230
+ if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) value = Math.min(value, prevPrev[j - 2] + 1);
1231
+ row.push(value);
1232
+ rowMin = Math.min(rowMin, value);
1233
+ }
1234
+ if (rowMin > max) return max + 1;
1235
+ prevPrev = prev;
1236
+ prev = row;
1237
+ }
1238
+ return prev[b.length];
1239
+ }
1240
+ function termScore(token, field, fuzzy) {
1241
+ let best = 0;
1242
+ for (const word of field.words) {
1243
+ if (word === token) return EXACT;
1244
+ if (word.startsWith(token)) best = PREFIX;
1245
+ else if (best < INNER && token.length >= 3 && word.includes(token)) best = INNER;
1246
+ }
1247
+ if (best > 0) return best;
1248
+ if (token.length >= 3 && field.compact.includes(token)) return INNER;
1249
+ if (!fuzzy || token.length < 4) return 0;
1250
+ const max = token.length >= 8 ? 2 : 1;
1251
+ for (const word of field.words) if (editDistance(token, word, max) <= max || word.length > token.length && editDistance(token, word.slice(0, token.length), max) <= max) return FUZZY;
1252
+ return 0;
1253
+ }
1254
+ function allowed(piece, filter) {
1255
+ const { sources, categories } = filter;
1256
+ if (sources?.length && !sources.includes(piece.meta.source)) return false;
1257
+ if (categories?.length && !piece.meta.categories.some((category) => categories.includes(category))) return false;
1258
+ return true;
1259
+ }
1260
+ function searchIndex(index, query, filter) {
1261
+ const tokens = tokenize(query);
1146
1262
  if (tokens.length === 0) return [];
1147
- const scored = [];
1148
- for (const entry of index.entries) {
1149
- const haystack = `${entry.name} ${entry.piece} ${entry.description}`;
1150
- if (!tokens.every((token) => haystack.includes(token))) continue;
1151
- const first = tokens[0];
1152
- const score = entry.name.startsWith(first) ? 0 : entry.name.includes(first) ? 1 : entry.piece.includes(first) ? 2 : 3;
1153
- scored.push({
1154
- score,
1155
- entry
1263
+ const exactName = tokens.join(" ");
1264
+ const matches = [];
1265
+ for (const piece of index.pieces) {
1266
+ if (!allowed(piece, filter)) continue;
1267
+ const nameScores = tokens.map((token) => termScore(token, piece.name, true));
1268
+ const ofKind = piece.blocks.filter((block) => block.hit.kind === filter.kind).map((block) => ({
1269
+ hit: block.hit,
1270
+ scores: tokens.map((token) => Math.max(BLOCK_NAME * termScore(token, block.name, true), BLOCK_DESCRIPTION * termScore(token, block.description, false)))
1271
+ }));
1272
+ const matchBlocks = (pieceScores) => {
1273
+ const found = [];
1274
+ ofKind.forEach((block, order) => {
1275
+ let score = 0;
1276
+ for (let i = 0; i < tokens.length; i++) {
1277
+ const best = Math.max(pieceScores[i], block.scores[i]);
1278
+ if (best === 0) return;
1279
+ score += best;
1280
+ }
1281
+ found.push({
1282
+ score,
1283
+ order,
1284
+ hit: block.hit
1285
+ });
1286
+ });
1287
+ return found;
1288
+ };
1289
+ let blocks = matchBlocks(nameScores.map((score) => PIECE_NAME * score));
1290
+ if (blocks.length === 0 && ofKind.length > 0) blocks = matchBlocks(tokens.map((token, i) => Math.max(PIECE_NAME * nameScores[i], PIECE_ABOUT * termScore(token, piece.about, false))));
1291
+ if (blocks.length === 0) continue;
1292
+ blocks.sort((a, b) => b.score - a.score || a.order - b.order);
1293
+ const nameBonus = piece.exactName === exactName ? EXACT : 0;
1294
+ matches.push({
1295
+ score: blocks[0].score + nameBonus,
1296
+ match: {
1297
+ ...piece.meta,
1298
+ namedPiece: nameScores.every((score) => score > 0),
1299
+ blocks: blocks.slice(0, MAX_BLOCKS_PER_PIECE).map(({ hit }) => hit)
1300
+ }
1156
1301
  });
1157
1302
  }
1158
- scored.sort((a, b) => a.score - b.score || a.entry.hit.displayName.localeCompare(b.entry.hit.displayName));
1159
- return scored.slice(0, Math.min(Math.max(limit, 1), MAX_LIMIT)).map(({ entry }) => entry.hit);
1303
+ const demoted = (match) => (match.deprecated ? 2 : 0) + (match.unsupported ? 1 : 0);
1304
+ matches.sort((a, b) => b.score - a.score || demoted(a.match) - demoted(b.match) || a.match.displayName.localeCompare(b.match.displayName));
1305
+ const limit = Math.min(Math.max(filter.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT);
1306
+ return matches.slice(0, limit).map(({ match }) => match);
1160
1307
  }
1161
1308
  let cached;
1162
1309
  function ensureIndex() {
@@ -1173,29 +1320,35 @@ function ensureIndex() {
1173
1320
  cached = entry;
1174
1321
  return entry;
1175
1322
  }
1176
- function searchBlocks(query, limit, local) {
1323
+ function warmSearchIndex() {
1324
+ const entry = ensureIndex();
1325
+ entry.promise.catch(() => {
1326
+ if (cached === entry) cached = void 0;
1327
+ });
1328
+ }
1329
+ function searchPieces(query, filter, local) {
1177
1330
  const index = ensureIndex();
1178
1331
  if (index.error) {
1179
1332
  const message = index.error;
1180
1333
  cached = void 0;
1181
1334
  return {
1182
1335
  status: "error",
1183
- hits: searchIndex(merge(void 0, local), query, limit),
1184
- indexedPieces: local?.pieces ?? 0,
1336
+ pieces: searchIndex(merge(void 0, local), query, filter),
1337
+ indexedPieces: local?.pieces.length ?? 0,
1185
1338
  error: message
1186
1339
  };
1187
1340
  }
1188
1341
  if (!index.value) return {
1189
1342
  status: "indexing",
1190
- hits: searchIndex(merge(void 0, local), query, limit),
1191
- indexedPieces: local?.pieces ?? 0,
1343
+ pieces: searchIndex(merge(void 0, local), query, filter),
1344
+ indexedPieces: local?.pieces.length ?? 0,
1192
1345
  error: null
1193
1346
  };
1194
1347
  const merged = merge(index.value, local);
1195
1348
  return {
1196
1349
  status: "ready",
1197
- hits: searchIndex(merged, query, limit),
1198
- indexedPieces: merged.pieces,
1350
+ pieces: searchIndex(merged, query, filter),
1351
+ indexedPieces: merged.pieces.length,
1199
1352
  error: null
1200
1353
  };
1201
1354
  }
@@ -1363,6 +1516,9 @@ function currentDocumentRecorder() {
1363
1516
  function currentPieceWorker() {
1364
1517
  return storage.getStore()?.pieceWorker;
1365
1518
  }
1519
+ function currentAttachmentOrigins() {
1520
+ return storage.getStore()?.attachments;
1521
+ }
1366
1522
  function currentStepTest() {
1367
1523
  return storage.getStore()?.stepTest ?? false;
1368
1524
  }
@@ -1371,7 +1527,7 @@ function currentRunUser() {
1371
1527
  }
1372
1528
  //#endregion
1373
1529
  //#region src/reactor/piece-store-port.ts
1374
- const logger$4 = childLogger(["workflow", "piece-store"]);
1530
+ const logger$5 = childLogger(["workflow", "piece-store"]);
1375
1531
  const PROJECT_SCOPE_KEY = "reactor";
1376
1532
  const TEST_PARTITION_SUFFIX = "#test";
1377
1533
  function testPartitionKey(scope, workflowId) {
@@ -1405,7 +1561,7 @@ function createPieceStorePort(store, workflowIdFor, sample = false, clock = Date
1405
1561
  if (!isCursorRef(key) || isPlausibleCursor(value, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
1406
1562
  const kept = await store.getPieceStoreValue(scope, partition(scope), key);
1407
1563
  if (!isCursorShaped(value) && !isPlausibleCursor(kept, now)) return store.setPieceStoreValue(scope, partition(scope), key, value);
1408
- logger$4.warn(`Rejected @key=@value from workflow ${flowKey()}; keeping @kept`, key, showCursor(value), isPlausibleCursor(kept, now) ? String(kept) : "no cursor");
1564
+ logger$5.warn(`Rejected @key=@value from workflow ${flowKey()}; keeping @kept`, key, showCursor(value), isPlausibleCursor(kept, now) ? String(kept) : "no cursor");
1409
1565
  if (!isPlausibleCursor(kept, now)) await store.deletePieceStoreValue(scope, partition(scope), key);
1410
1566
  },
1411
1567
  delete: async (key, scope) => store.deletePieceStoreValue(scope, partition(scope), key)
@@ -1440,6 +1596,20 @@ function runnableDefinition(state) {
1440
1596
  //#endregion
1441
1597
  //#region src/reactor/lib.ts
1442
1598
  const pieceLogger = childLogger(["workflow", "piece"]);
1599
+ /**
1600
+ * The ceiling on one line this engine writes to a log.
1601
+ *
1602
+ * Nothing here is worth more than this to a reader, and an unbounded write is
1603
+ * a liability: a piece error carrying an HTML error page, or a piece logging a
1604
+ * whole document, becomes a multi-megabyte write to a pipe that may be
1605
+ * blocked — and an EPIPE on the fatal path is what produced the boot loop in
1606
+ * backlog item 5. Bounding the write is cheaper than handling the throw.
1607
+ */
1608
+ const MAX_LOG_LINE_CHARS = 8192;
1609
+ function truncateForLog(text) {
1610
+ if (text.length <= 8192) return text;
1611
+ return `${text.slice(0, MAX_LOG_LINE_CHARS)}… [truncated, ${text.length} chars]`;
1612
+ }
1443
1613
  const connectionLogger = childLogger(["workflow", "connection"]);
1444
1614
  function assertConnectorMatches(state, request) {
1445
1615
  const wanted = request?.piecePackage;
@@ -1502,6 +1672,29 @@ function bundleCacheDir() {
1502
1672
  return bundleCache;
1503
1673
  }
1504
1674
  const ATTACHMENT_STAGING_DIR = join(process.cwd(), ".ph", "ap-attachment-staging");
1675
+ const ATTACHMENT_CACHE_DIR = join(process.cwd(), ".ph", "ap-attachment-cache");
1676
+ const DEFAULT_ATTACHMENT_CACHE_BYTES = 1073741824;
1677
+ function attachmentCacheBytes() {
1678
+ const raw = process.env.PH_WORKFLOWS_ATTACHMENT_CACHE_BYTES;
1679
+ const parsed = raw === void 0 ? NaN : Number(raw);
1680
+ return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : DEFAULT_ATTACHMENT_CACHE_BYTES;
1681
+ }
1682
+ async function sweepAttachmentStaging(root = ATTACHMENT_STAGING_DIR, olderThanMs = 36e5) {
1683
+ const names = await readdir(root).catch(() => []);
1684
+ let swept = 0;
1685
+ for (const name of names) {
1686
+ const entry = join(root, name);
1687
+ const info = await stat(entry).catch(() => void 0);
1688
+ if (!info || Date.now() - info.mtimeMs < olderThanMs) continue;
1689
+ await rm(entry, {
1690
+ recursive: true,
1691
+ force: true
1692
+ });
1693
+ swept += 1;
1694
+ }
1695
+ return swept;
1696
+ }
1697
+ let stagingSwept = false;
1505
1698
  let resolver;
1506
1699
  function pieceResolver() {
1507
1700
  return resolver ??= sourcedResolver({
@@ -1514,7 +1707,14 @@ function boundConnections(inner) {
1514
1707
  connectionLogger.warn(`Step "${request?.stepKey ?? "?"}" of workflow "${currentWorkflowId() ?? "?"}" asked for connection "${connectionId}", which its definition does not declare`);
1515
1708
  });
1516
1709
  }
1517
- function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess) {
1710
+ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess, telemetry) {
1711
+ if (attachments && !stagingSwept) {
1712
+ stagingSwept = true;
1713
+ sweepAttachmentStaging().then((swept) => {
1714
+ if (swept > 0) pieceLogger.info(`Removed ${swept} leftover attachment staging dir(s)`);
1715
+ }, () => void 0);
1716
+ }
1717
+ const cacheBytes = attachmentCacheBytes();
1518
1718
  return new CompositeBlockExecutor(new ActivepiecesBlockExecutor({
1519
1719
  cacheDir: bundleCacheDir(),
1520
1720
  egress: configuredEgress(),
@@ -1528,10 +1728,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
1528
1728
  resolver: pieceResolver(),
1529
1729
  ...resolveBlock ? { resolveBlock } : {},
1530
1730
  ...reactorAccess ? { reactorAccess } : {},
1731
+ ...telemetry ? { telemetry } : {},
1531
1732
  connections: boundConnections(new DocumentConnectionResolver(host, secrets, oauth)),
1532
1733
  ...pieceStore ? { pieceStore } : {},
1533
1734
  onPieceLog: (entry, execution) => {
1534
- const line = `[${execution.step.key}] ${entry.message}`;
1735
+ const line = truncateForLog(`[${execution.step.key}] ${entry.message}`);
1535
1736
  if (entry.level === "error") pieceLogger.error(line);
1536
1737
  else if (entry.level === "warn") pieceLogger.warn(line);
1537
1738
  else if (entry.level === "debug") pieceLogger.debug(line);
@@ -1539,7 +1740,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
1539
1740
  },
1540
1741
  ...attachments ? {
1541
1742
  attachments,
1542
- stagingRoot: ATTACHMENT_STAGING_DIR
1743
+ stagingRoot: ATTACHMENT_STAGING_DIR,
1744
+ ...cacheBytes > 0 ? { attachmentCache: new AttachmentCache({
1745
+ dir: ATTACHMENT_CACHE_DIR,
1746
+ maxBytes: cacheBytes
1747
+ }) } : {}
1543
1748
  } : {}
1544
1749
  }));
1545
1750
  }
@@ -1563,7 +1768,8 @@ function stepDefinition(step) {
1563
1768
  config: step.config,
1564
1769
  timeoutSeconds: step.timeoutSeconds,
1565
1770
  propertySettings: propertySettings(step.propertySettings),
1566
- skip: step.skip
1771
+ skip: step.skip,
1772
+ retry: step.retry
1567
1773
  };
1568
1774
  }
1569
1775
  function toWorkflowDefinition(state) {
@@ -1776,6 +1982,227 @@ async function buildReactorRunScope(host, request) {
1776
1982
  connection: { access: parsed.config.access ?? "write" }
1777
1983
  };
1778
1984
  }
1985
+ //#endregion
1986
+ //#region src/telemetry.ts
1987
+ const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
1988
+ var WorkflowTelemetry = class {
1989
+ tracer;
1990
+ meter;
1991
+ runs;
1992
+ runDuration;
1993
+ stepDuration;
1994
+ phaseDuration;
1995
+ poolGauges;
1996
+ constructor(options = {}) {
1997
+ this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
1998
+ this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
1999
+ this.runs = this.meter.createCounter("workflow.runs", {
2000
+ description: "Runs finished, by status and trigger kind",
2001
+ unit: "{run}"
2002
+ });
2003
+ this.runDuration = this.meter.createHistogram("workflow.run.duration", {
2004
+ description: "Run duration from fire to the journal closing it",
2005
+ unit: "ms"
2006
+ });
2007
+ this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
2008
+ description: "Step duration, by piece and status",
2009
+ unit: "ms"
2010
+ });
2011
+ this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
2012
+ description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
2013
+ unit: "ms"
2014
+ });
2015
+ }
2016
+ phase(name, attributes, fn, parent) {
2017
+ const started = performance.now();
2018
+ return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
2019
+ }
2020
+ workerTimings(request, timings, forkedAt, ipc) {
2021
+ const under = (span) => trace.setSpan(context.active(), span);
2022
+ const child = (name, start, end, parent = request, attributes) => {
2023
+ const span = this.tracer.startSpan(`workflow.worker.${name}`, {
2024
+ startTime: start,
2025
+ ...attributes ? { attributes } : {}
2026
+ }, under(parent));
2027
+ span.end(end);
2028
+ this.recordPhase(`worker.${name}`, end - start);
2029
+ return span;
2030
+ };
2031
+ const ready = epoch(timings.ready);
2032
+ if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
2033
+ const received = epoch(timings.received);
2034
+ const sent = epoch(timings.sent);
2035
+ const delivered = Math.max(ipc.sent, ready ?? 0);
2036
+ if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
2037
+ const byName = /* @__PURE__ */ new Map();
2038
+ const phases = Array.isArray(timings.phases) ? timings.phases : [];
2039
+ for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
2040
+ const parent = phase.parent ? byName.get(phase.parent) : void 0;
2041
+ byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
2042
+ }
2043
+ if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
2044
+ }
2045
+ recordPhase(name, ms) {
2046
+ this.phaseDuration.record(ms, { phase: name });
2047
+ }
2048
+ span(name, attributes, fn, parent = context.active()) {
2049
+ return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
2050
+ try {
2051
+ return await fn(span);
2052
+ } catch (error) {
2053
+ failSpan(span, error);
2054
+ throw error;
2055
+ } finally {
2056
+ span.end();
2057
+ }
2058
+ });
2059
+ }
2060
+ recordRun(ms, status, triggerKind) {
2061
+ const attributes = {
2062
+ "run.status": status,
2063
+ "trigger.kind": metricTriggerKind(triggerKind)
2064
+ };
2065
+ this.runs.add(1, attributes);
2066
+ this.runDuration.record(ms, attributes);
2067
+ }
2068
+ recordStep(ms, pieceName, status) {
2069
+ this.stepDuration.record(ms, {
2070
+ "piece.name": pieceName,
2071
+ "step.status": status
2072
+ });
2073
+ }
2074
+ observeWorkerPool(stats) {
2075
+ this.poolGauges ??= this.createPoolGauges();
2076
+ const { size, active, waiting } = this.poolGauges;
2077
+ const observables = [
2078
+ size,
2079
+ active,
2080
+ waiting
2081
+ ];
2082
+ const callback = (result) => {
2083
+ const current = stats();
2084
+ result.observe(size, current.size);
2085
+ result.observe(active, current.active);
2086
+ result.observe(waiting, current.waiting);
2087
+ };
2088
+ this.meter.addBatchObservableCallback(callback, observables);
2089
+ return () => this.meter.removeBatchObservableCallback(callback, observables);
2090
+ }
2091
+ createPoolGauges() {
2092
+ const gauge = (name, description) => this.meter.createObservableGauge(name, {
2093
+ description,
2094
+ unit: "{worker}"
2095
+ });
2096
+ return {
2097
+ size: gauge("workflow.worker.pool.size", "Worker slots"),
2098
+ active: gauge("workflow.worker.pool.active", "Slots held by runs"),
2099
+ waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
2100
+ };
2101
+ }
2102
+ };
2103
+ const WORKER_PHASES = /* @__PURE__ */ new Set([
2104
+ "piece.load",
2105
+ "reactor.open",
2106
+ "models",
2107
+ "action"
2108
+ ]);
2109
+ const WORKER_ATTRIBUTES = {
2110
+ "piece.cached": "boolean",
2111
+ "document.type": "string"
2112
+ };
2113
+ const MAX_ATTRIBUTE_LENGTH = 256;
2114
+ function epoch(value) {
2115
+ return typeof value === "number" && Number.isFinite(value) ? value : void 0;
2116
+ }
2117
+ function knownPhase(phase) {
2118
+ if (typeof phase !== "object" || phase === null) return false;
2119
+ const { name, start, end, parent } = phase;
2120
+ return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
2121
+ }
2122
+ function phaseAttributes(attributes) {
2123
+ if (typeof attributes !== "object" || attributes === null) return void 0;
2124
+ const kept = {};
2125
+ for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
2126
+ const value = attributes[key];
2127
+ if (typeof value !== type) continue;
2128
+ kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
2129
+ }
2130
+ return Object.keys(kept).length > 0 ? kept : void 0;
2131
+ }
2132
+ function metricTriggerKind(triggerKind) {
2133
+ return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
2134
+ }
2135
+ function metricPieceName(step, resolution) {
2136
+ return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
2137
+ }
2138
+ function tracedSteps(executor, telemetry) {
2139
+ return { execute(execution) {
2140
+ const { step } = execution;
2141
+ const started = performance.now();
2142
+ let status = "FAILED";
2143
+ let resolution;
2144
+ return telemetry.span("workflow.step", {
2145
+ "step.id": step.id,
2146
+ "step.key": step.key,
2147
+ "piece.name": step.pieceName,
2148
+ "action.name": step.actionName
2149
+ }, async (span) => {
2150
+ try {
2151
+ const result = await executor.execute(execution);
2152
+ resolution = result.resolution;
2153
+ status = "SUCCEEDED";
2154
+ const version = resolution?.resolved?.version;
2155
+ if (version) span.setAttribute("piece.version", version);
2156
+ return result;
2157
+ } catch (error) {
2158
+ resolution = resolutionOf(error);
2159
+ throw error;
2160
+ }
2161
+ }).finally(() => {
2162
+ telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
2163
+ });
2164
+ } };
2165
+ }
2166
+ function failSpan(span, error) {
2167
+ span.recordException(error instanceof Error ? error : String(error));
2168
+ span.setStatus({
2169
+ code: SpanStatusCode.ERROR,
2170
+ message: error instanceof Error ? error.message : String(error)
2171
+ });
2172
+ }
2173
+ function tracedMethods(target, telemetry, prefix, parent) {
2174
+ return new Proxy(target, { get(object, property, receiver) {
2175
+ const value = Reflect.get(object, property, receiver);
2176
+ if (typeof value !== "function" || typeof property !== "string") return value;
2177
+ const phase = `${prefix}.${property}`;
2178
+ return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
2179
+ const started = performance.now();
2180
+ const end = (error) => {
2181
+ if (error !== void 0) failSpan(span, error);
2182
+ span.end();
2183
+ telemetry.recordPhase(phase, performance.now() - started);
2184
+ };
2185
+ let result;
2186
+ try {
2187
+ result = value.apply(object, args);
2188
+ } catch (error) {
2189
+ end(error);
2190
+ throw error;
2191
+ }
2192
+ if (!(result instanceof Promise)) {
2193
+ end();
2194
+ return result;
2195
+ }
2196
+ return result.then((resolved) => {
2197
+ end();
2198
+ return resolved;
2199
+ }, (error) => {
2200
+ end(error);
2201
+ throw error;
2202
+ });
2203
+ });
2204
+ } });
2205
+ }
1779
2206
  const JOB_NOT_FOUND = "Job not found";
1780
2207
  const MIN_DEADLINE_MARGIN_MS = 250;
1781
2208
  const MAX_DEADLINE_MARGIN_MS = 2e3;
@@ -1792,7 +2219,7 @@ const AUTHORIZATION_DENIED = "AuthorizationDeniedError";
1792
2219
  function jobFailed(job) {
1793
2220
  const error = job.error;
1794
2221
  if (error?.name === AUTHORIZATION_DENIED) return accessDenied(`The Switchboard has no grant for this operation: ${error.message}`);
1795
- return new ReactorError(ReactorJobFailedError$1, `Reactor job ${job.id} failed: ${job.error?.message ?? "unknown error"}`);
2222
+ return new ReactorError(ReactorJobFailedError, `Reactor job ${job.id} failed: ${job.error?.message ?? "unknown error"}`);
1796
2223
  }
1797
2224
  function assertActionsApplied(job, actions) {
1798
2225
  const outcomes = job.result?.actions ?? [];
@@ -1810,7 +2237,7 @@ function assertActionsApplied(job, actions) {
1810
2237
  if (outcome.kind === "denied") return [`Action ${label} was denied: ${outcome.reason}`];
1811
2238
  return [];
1812
2239
  });
1813
- if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError$1, failed.join("; "));
2240
+ if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError, failed.join("; "));
1814
2241
  }
1815
2242
  var RunScopedReactorClient = class {
1816
2243
  inner;
@@ -2141,7 +2568,7 @@ const NO_JOURNAL = {
2141
2568
  recordDocuments: () => Promise.resolve(),
2142
2569
  recordJob: () => Promise.resolve()
2143
2570
  };
2144
- function reactorTap(host, base, journal, options = {}) {
2571
+ function reactorTap(host, base, journal, options = {}, telemetry) {
2145
2572
  const clientOptions = {
2146
2573
  ...host.hostPrincipal ? { hostPrincipal: host.hostPrincipal } : {},
2147
2574
  ...options
@@ -2149,12 +2576,13 @@ function reactorTap(host, base, journal, options = {}) {
2149
2576
  return {
2150
2577
  requireReactor: base.requireReactor,
2151
2578
  open(transport, { deadline }) {
2152
- const client = new RunScopedReactorClient(host.reactorClient, {
2579
+ const served = servedClient(new RunScopedReactorClient(host.reactorClient, {
2153
2580
  ...base,
2154
2581
  deadline,
2155
2582
  journal
2156
- }, host, clientOptions);
2157
- const server = new ReactorHostServer(servedClient(client), transport);
2583
+ }, host, clientOptions));
2584
+ const parent = context.active();
2585
+ const server = new ReactorHostServer(telemetry ? tracedMethods(served, telemetry, "reactor", parent) : served, transport);
2158
2586
  server.start();
2159
2587
  return () => server.stop();
2160
2588
  }
@@ -2451,6 +2879,19 @@ async function resolveVariables(variables, secrets) {
2451
2879
  }
2452
2880
  //#endregion
2453
2881
  //#region src/reactor/step-test.ts
2882
+ /**
2883
+ * A step record's status as a test reports it.
2884
+ *
2885
+ * Collapsing this to `FAILED : SUCCEEDED` is how an INDETERMINATE test came
2886
+ * to read green — the one status that must never be mistaken for a confirmed
2887
+ * one, since the whole point of it is that nobody knows whether the write
2888
+ * landed.
2889
+ */
2890
+ function testStatusOf(status) {
2891
+ if (status === "FAILED") return "FAILED";
2892
+ if (status === "INDETERMINATE") return "INDETERMINATE";
2893
+ return "SUCCEEDED";
2894
+ }
2454
2895
  function collectStrings(value, into) {
2455
2896
  if (typeof value === "string") into.push(value);
2456
2897
  else if (Array.isArray(value)) for (const item of value) collectStrings(item, into);
@@ -2534,6 +2975,7 @@ function untestedError(label, sample) {
2534
2975
  case "hidden": return `Test ${label} first: its last test is not visible to you`;
2535
2976
  case "failed": return `Test ${label} first: its last test failed`;
2536
2977
  case "truncated": return `Test ${label} again: the journal kept only a truncated copy of its last output`;
2978
+ case "indeterminate": return `Test ${label} again: its last test is INDETERMINATE, so it has no confirmed output`;
2537
2979
  default: return `Test ${label} first`;
2538
2980
  }
2539
2981
  }
@@ -7095,6 +7537,59 @@ function parseParameter(param) {
7095
7537
  return parseValueExpression(param);
7096
7538
  }
7097
7539
  //#endregion
7540
+ //#region src/reactor/policy.ts
7541
+ /** The pre-enforcement behaviour, for a definition that declares no policy. */
7542
+ const UNENFORCED_POLICY = Object.freeze({
7543
+ concurrency: "PARALLEL",
7544
+ maxParallelRuns: null,
7545
+ runTimeoutSeconds: null,
7546
+ onFailure: "IGNORE",
7547
+ defaultRetry: null
7548
+ });
7549
+ const CONCURRENCY_MODES = [
7550
+ "SINGLETON",
7551
+ "QUEUE",
7552
+ "PARALLEL"
7553
+ ];
7554
+ const FAILURE_MODES = [
7555
+ "PARK",
7556
+ "NOTIFY",
7557
+ "IGNORE"
7558
+ ];
7559
+ function positive(value) {
7560
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
7561
+ }
7562
+ /** What the runtime enforces for a run of this definition. */
7563
+ function effectiveRunPolicy(runnable) {
7564
+ const policy = runnable.policy;
7565
+ if (!policy) return UNENFORCED_POLICY;
7566
+ const concurrency = CONCURRENCY_MODES.includes(policy.concurrency) ? policy.concurrency : "PARALLEL";
7567
+ return {
7568
+ concurrency,
7569
+ maxParallelRuns: concurrency === "PARALLEL" ? positive(policy.maxParallelRuns) : null,
7570
+ runTimeoutSeconds: positive(policy.runTimeoutSeconds),
7571
+ onFailure: FAILURE_MODES.includes(policy.onFailure) ? policy.onFailure : "IGNORE",
7572
+ defaultRetry: effectiveRetryPolicy(policy.defaultRetry)
7573
+ };
7574
+ }
7575
+ /** How many runs of one workflow may execute at once; 1 for both serialising
7576
+ * modes, so the gate holds one number rather than three branches. Null is
7577
+ * unbounded, i.e. no gate at all. */
7578
+ function concurrencyLimit(policy) {
7579
+ if (policy.concurrency === "PARALLEL") return policy.maxParallelRuns;
7580
+ return 1;
7581
+ }
7582
+ /** The trigger status PARK leaves on a supervised trigger's row. Not ENABLED,
7583
+ * so the supervisor's due-trigger query does not return it and the schedule
7584
+ * stops refiring; a re-publish or re-enable arms it again. The park itself, for
7585
+ * every trigger kind, is the store's `workflow_park` row. */
7586
+ const PARKED_TRIGGER_STATUS = "PARKED";
7587
+ /** The run status a firing skipped by SINGLETON is journaled under, and the
7588
+ * one a run that passed `runTimeoutSeconds` ends in. Both are "this run did
7589
+ * not finish and nothing is wrong with the workflow", which is what the
7590
+ * document model's RunStatus.CANCELLED means. */
7591
+ const CANCELLED_RUN_STATUS = "CANCELLED";
7592
+ //#endregion
7098
7593
  //#region src/reactor/store.ts
7099
7594
  function triggerRowBlock(row) {
7100
7595
  return {
@@ -7109,7 +7604,20 @@ function triggerBlockColumns(block) {
7109
7604
  trigger_name: block.name
7110
7605
  };
7111
7606
  }
7112
- const logger$3 = childLogger([
7607
+ const FIRED_WITHOUT_RUN_ID = "fired-without-run";
7608
+ const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
7609
+ function abandonedFireError(attempts) {
7610
+ return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
7611
+ }
7612
+ function workflowParkRow(workflowId, publishedVersion, reason) {
7613
+ return {
7614
+ workflow_id: workflowId,
7615
+ published_version: publishedVersion,
7616
+ reason: redactMessage(reason),
7617
+ parked_at: (/* @__PURE__ */ new Date()).toISOString()
7618
+ };
7619
+ }
7620
+ const logger$4 = childLogger([
7113
7621
  "workflow",
7114
7622
  "runtime",
7115
7623
  "store"
@@ -7136,6 +7644,7 @@ async function up(db) {
7136
7644
  } catch {}
7137
7645
  await db.schema.createTable("trigger_state").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("trigger_name", "text", (col) => col.notNull()).addColumn("config_hash", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("store_state", "text", (col) => col.notNull()).addColumn("interval_ms", "integer", (col) => col.notNull()).addColumn("next_poll_at", "text").addColumn("last_poll_at", "text").addColumn("last_error", "text").addColumn("consecutive_failures", "integer", (col) => col.notNull()).addColumn("lease_owner", "text").addColumn("lease_expires_at", "text").addColumn("updated_at", "text", (col) => col.notNull()).ifNotExists().execute();
7138
7646
  await db.schema.createTable("trigger_dedupe").addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("dedupe_key", "text", (col) => col.notNull()).addColumn("run_id", "text").addColumn("created_at", "text", (col) => col.notNull()).addPrimaryKeyConstraint("trigger_dedupe_pk", ["workflow_id", "dedupe_key"]).ifNotExists().execute();
7647
+ await db.schema.createTable("workflow_park").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("published_version", "integer", (col) => col.notNull()).addColumn("reason", "text", (col) => col.notNull()).addColumn("parked_at", "text", (col) => col.notNull()).ifNotExists().execute();
7139
7648
  await db.schema.createTable("step_execution").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("ordinal", "integer", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("input", "text").addColumn("output", "text").addColumn("port", "text").addColumn("error", "text").addColumn("started_at", "text").addColumn("ended_at", "text").addUniqueConstraint("step_execution_run_step", ["run_id", "step_id"]).ifNotExists().execute();
7140
7649
  for (const column of ["started_at", "ended_at"]) try {
7141
7650
  await db.schema.alterTable("step_execution").addColumn(column, "text").execute();
@@ -7154,6 +7663,21 @@ async function up(db) {
7154
7663
  ]) try {
7155
7664
  await db.schema.alterTable(table).addColumn(column, "text").execute();
7156
7665
  } catch {}
7666
+ try {
7667
+ await db.schema.alterTable("trigger_dedupe").addColumn("attempts", "integer", (col) => col.notNull().defaultTo(1)).execute();
7668
+ } catch {}
7669
+ await db.schema.createTable("migration_mark").addColumn("name", "text", (col) => col.primaryKey()).addColumn("applied_at", "text", (col) => col.notNull()).ifNotExists().execute();
7670
+ await db.transaction().execute(async (trx) => {
7671
+ if (await trx.selectFrom("migration_mark").select("name").where("name", "=", DEDUPE_CLAIM_MIGRATION).executeTakeFirst()) return;
7672
+ await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "is", null).execute();
7673
+ await trx.insertInto("migration_mark").values({
7674
+ name: DEDUPE_CLAIM_MIGRATION,
7675
+ applied_at: (/* @__PURE__ */ new Date()).toISOString()
7676
+ }).onConflict((oc) => oc.column("name").doNothing()).execute();
7677
+ });
7678
+ try {
7679
+ await db.schema.alterTable("step_execution").addColumn("attempts", "integer").execute();
7680
+ } catch {}
7157
7681
  try {
7158
7682
  await db.schema.alterTable("trigger_state").addColumn("next_renew_at", "text").execute();
7159
7683
  } catch {}
@@ -7245,7 +7769,7 @@ async function migrateTriggerStoreState(db) {
7245
7769
  try {
7246
7770
  rows = await db.selectFrom("trigger_state").select(["workflow_id", "store_state"]).orderBy("updated_at", "asc").orderBy("workflow_id", "asc").execute();
7247
7771
  } catch (error) {
7248
- logger$3.error("Could not read trigger_state to migrate it: @error", error);
7772
+ logger$4.error("Could not read trigger_state to migrate it: @error", error);
7249
7773
  return unmigrated;
7250
7774
  }
7251
7775
  const pending = [];
@@ -7261,7 +7785,7 @@ async function migrateTriggerStoreState(db) {
7261
7785
  await migrateOneRow(db, row, projectWinner);
7262
7786
  } catch (error) {
7263
7787
  unmigrated.add(row.workflowId);
7264
- logger$3.error(`Could not migrate trigger store state for ${row.workflowId}`, error);
7788
+ logger$4.error(`Could not migrate trigger store state for ${row.workflowId}`, error);
7265
7789
  }
7266
7790
  return unmigrated;
7267
7791
  }
@@ -7271,7 +7795,7 @@ function parseLegacyBlob(row, unmigrated) {
7271
7795
  try {
7272
7796
  state = JSON.parse(row.store_state);
7273
7797
  } catch {
7274
- logger$3.warn(`Leaving unparseable store_state for ${row.workflow_id} in place`);
7798
+ logger$4.warn(`Leaving unparseable store_state for ${row.workflow_id} in place`);
7275
7799
  unmigrated.add(row.workflow_id);
7276
7800
  return null;
7277
7801
  }
@@ -7292,7 +7816,7 @@ function parseLegacyBlob(row, unmigrated) {
7292
7816
  });
7293
7817
  continue;
7294
7818
  }
7295
- if (key.startsWith("test")) logger$3.info(`Migrating "${key}" for ${row.workflow_id} as a project key; it may be a test leftover`);
7819
+ if (key.startsWith("test")) logger$4.info(`Migrating "${key}" for ${row.workflow_id} as a project key; it may be a test leftover`);
7296
7820
  entries.push({
7297
7821
  scope: "PROJECT",
7298
7822
  scopeKey: PROJECT_SCOPE_KEY,
@@ -7314,7 +7838,7 @@ function resolveProjectCollisions(pending) {
7314
7838
  }
7315
7839
  winner.set(entry.key, row.workflowId);
7316
7840
  }
7317
- for (const [key, workflows] of contested) logger$3.warn(`Project store key "@key" was written by ${workflows.join(", ")}; keeping the value last updated, from ${winner.get(key)}, and discarding the rest`, key);
7841
+ for (const [key, workflows] of contested) logger$4.warn(`Project store key "@key" was written by ${workflows.join(", ")}; keeping the value last updated, from ${winner.get(key)}, and discarding the rest`, key);
7318
7842
  return winner;
7319
7843
  }
7320
7844
  async function migrateOneRow(db, row, projectWinner) {
@@ -7343,9 +7867,9 @@ async function insertPieceStoreIfAbsent(db, scope, scopeKey, key, value) {
7343
7867
  }
7344
7868
  function warnIfOverPieceStoreLimits(scope, scopeKey, key, encoded) {
7345
7869
  const at = `${scope}/${scopeKey}/${key}`;
7346
- if (key.length > 128) logger$3.warn(`Migrated store key ${at} is ${key.length} chars, over the 128 limit; writes to it will fail`);
7870
+ if (key.length > 128) logger$4.warn(`Migrated store key ${at} is ${key.length} chars, over the 128 limit; writes to it will fail`);
7347
7871
  const size = Buffer.byteLength(encoded, "utf8");
7348
- if (size > 524288) logger$3.warn(`Migrated store value ${at} is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} limit; writes to it will fail`);
7872
+ if (size > 524288) logger$4.warn(`Migrated store value ${at} is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} limit; writes to it will fail`);
7349
7873
  }
7350
7874
  const PIECE_STORE_MAX_VALUE_BYTES = 524288;
7351
7875
  var PieceStoreLimitError = class extends Error {
@@ -7362,10 +7886,50 @@ function assertPieceStoreEntry(key, value) {
7362
7886
  if (size > 524288) throw new PieceStoreLimitError(`Store value for "${key}" is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} byte limit`);
7363
7887
  }
7364
7888
  const STEP_PAYLOAD_PREFIX_CHARS = 32768;
7889
+ /**
7890
+ * The reserved key a truncation marker is recognised by, and its value.
7891
+ *
7892
+ * A marker has to be distinguishable from a legitimate payload that merely
7893
+ * LOOKS like one, because a run whose output is mistaken for a marker is a
7894
+ * step that silently re-executes on rerun. The original predicate duck-typed
7895
+ * `{truncated: true, bytes: number, prefix: string}` (review backlog item 15),
7896
+ * which any piece returning a truncation report of its own would match — and
7897
+ * "did we truncate this?" is exactly the shape such a report takes.
7898
+ *
7899
+ * So the marker is keyed on a namespaced sentinel whose VALUE is a versioned
7900
+ * magic string. A piece would have to emit that exact pair to collide, which
7901
+ * is not something a payload produces by accident.
7902
+ */
7903
+ const TRUNCATED_PAYLOAD_KEY = "@powerhousedao/reactor-workflow:runJournal";
7904
+ const TRUNCATED_PAYLOAD_SENTINEL = "truncated-payload/v1";
7905
+ const LEGACY_MARKER_KEYS = [
7906
+ "truncated",
7907
+ "bytes",
7908
+ "prefix",
7909
+ "documentId",
7910
+ "driveId"
7911
+ ];
7912
+ /**
7913
+ * True for a journaled value this store truncated.
7914
+ *
7915
+ * Two shapes are accepted and only one is written. New rows carry the reserved
7916
+ * key, which a payload cannot produce by accident. Rows journaled before it
7917
+ * carry the old `{truncated: true, bytes, prefix}`, and those still have to be
7918
+ * recognised: reading one as data would re-run the side-effectful step that
7919
+ * produced it, which is the whole bug.
7920
+ *
7921
+ * So the legacy branch is matched by its EXACT key set, not by duck-typing.
7922
+ * `{truncated, bytes, prefix, source}` — a truncation report a piece might
7923
+ * legitimately return, and the collision the review found — is no longer a
7924
+ * marker. A payload whose keys are exactly the legacy marker's is still
7925
+ * ambiguous, and nothing can resolve that for a row already written; the
7926
+ * window is as narrow as the stored data allows.
7927
+ */
7365
7928
  function isTruncatedStepPayload(value) {
7366
7929
  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
7367
7930
  const record = value;
7368
- return record.truncated === true && typeof record.bytes === "number" && typeof record.prefix === "string";
7931
+ if (record["@powerhousedao/reactor-workflow:runJournal"] === "truncated-payload/v1") return true;
7932
+ return record.truncated === true && typeof record.bytes === "number" && typeof record.prefix === "string" && Object.keys(record).every((key) => LEGACY_MARKER_KEYS.includes(key));
7369
7933
  }
7370
7934
  function topLevelDocumentIds(value) {
7371
7935
  if (typeof value !== "object" || value === null || Array.isArray(value)) return {};
@@ -7384,7 +7948,7 @@ function cappedPayload(value) {
7384
7948
  const last = prefix.charCodeAt(prefix.length - 1);
7385
7949
  if (last >= 55296 && last <= 56319) prefix = prefix.slice(0, -1);
7386
7950
  const marker = {
7387
- truncated: true,
7951
+ [TRUNCATED_PAYLOAD_KEY]: TRUNCATED_PAYLOAD_SENTINEL,
7388
7952
  bytes,
7389
7953
  prefix,
7390
7954
  ...topLevelDocumentIds(value)
@@ -7401,7 +7965,7 @@ function stepValues(runId, ordinal, step) {
7401
7965
  block_name: step.blockName,
7402
7966
  status: step.status,
7403
7967
  input: cappedPayload(redact(step.input)),
7404
- output: cappedPayload(redact(referenceDocuments(step.output))),
7968
+ output: cappedPayload(redact(referenceDocuments(step.journaledOutput !== void 0 ? step.journaledOutput : step.output))),
7405
7969
  port: step.port ?? null,
7406
7970
  error: step.error ? redactMessage(step.error) : null,
7407
7971
  error_name: step.errorName ?? null,
@@ -7411,7 +7975,8 @@ function stepValues(runId, ordinal, step) {
7411
7975
  piece_source: step.piece?.source ?? null,
7412
7976
  version_match: step.piece?.match ?? null,
7413
7977
  version_note: step.piece?.note ?? null,
7414
- config_hash: step.configHash ?? null
7978
+ config_hash: step.configHash ?? null,
7979
+ attempts: step.attempts ?? null
7415
7980
  };
7416
7981
  }
7417
7982
  function runWarningNotes(result) {
@@ -7460,15 +8025,26 @@ function journaledPayloadNames(json, ids) {
7460
8025
  });
7461
8026
  }
7462
8027
  const erasedRuns = /* @__PURE__ */ new Set();
7463
- async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso, runId) {
8028
+ async function pollSucceededIn(db, workflowId, storeState, nowIso, nextPollAtIso) {
8029
+ await db.updateTable("trigger_state").set({
8030
+ store_state: storeState,
8031
+ last_poll_at: nowIso,
8032
+ next_poll_at: nextPollAtIso,
8033
+ last_error: null,
8034
+ consecutive_failures: 0,
8035
+ updated_at: nowIso
8036
+ }).where("workflow_id", "=", workflowId).execute();
8037
+ }
8038
+ async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso, runId = FIRED_WITHOUT_RUN_ID) {
7464
8039
  const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
7465
8040
  await db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
7466
8041
  return await db.insertInto("trigger_dedupe").values({
7467
8042
  workflow_id: workflowId,
7468
8043
  dedupe_key: dedupeKey,
7469
8044
  run_id: runId,
7470
- created_at: nowIso
7471
- }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doNothing()).returning("dedupe_key").executeTakeFirst() !== void 0;
8045
+ created_at: nowIso,
8046
+ attempts: 1
8047
+ }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id: runId }).where("trigger_dedupe.run_id", "is", null)).returning("dedupe_key").executeTakeFirst() !== void 0;
7472
8048
  }
7473
8049
  const STEP_COLUMNS_WITHOUT_DATA = [
7474
8050
  "id",
@@ -7488,7 +8064,8 @@ const STEP_COLUMNS_WITHOUT_DATA = [
7488
8064
  "piece_source",
7489
8065
  "version_match",
7490
8066
  "version_note",
7491
- "config_hash"
8067
+ "config_hash",
8068
+ "attempts"
7492
8069
  ];
7493
8070
  const PRUNE_BATCH_SIZE = 500;
7494
8071
  var WorkflowRunStore = class WorkflowRunStore {
@@ -7521,7 +8098,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7521
8098
  }).where("status", "=", "RUNNING");
7522
8099
  if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
7523
8100
  const recovered = (await query.returning("id").execute()).length;
7524
- if (recovered > 0) logger$3.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
8101
+ if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
7525
8102
  return recovered;
7526
8103
  }
7527
8104
  async recoverAbandonedRuns() {
@@ -7532,25 +8109,197 @@ var WorkflowRunStore = class WorkflowRunStore {
7532
8109
  }).where("status", "=", PENDING_RUN_STATUS);
7533
8110
  if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
7534
8111
  const recovered = (await query.returning("id").execute()).length;
7535
- if (recovered > 0) logger$3.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
8112
+ if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
7536
8113
  return recovered;
7537
8114
  }
8115
+ /**
8116
+ * Closes a run out as CANCELLED, with the reason as its error.
8117
+ *
8118
+ * Two callers, one meaning — "this run did not finish, and the workflow is
8119
+ * not at fault": a firing SINGLETON concurrency refused, and a run that
8120
+ * passed its `runTimeoutSeconds`. Separate from `failRun` because a
8121
+ * CANCELLED run is not rerunnable and must not read as a workflow defect.
8122
+ */
8123
+ async cancelRun(runId, reason) {
8124
+ this.runsInFlight.delete(runId);
8125
+ if (erasedRuns.delete(runId)) return;
8126
+ await this.db.updateTable("run").set({
8127
+ status: "CANCELLED",
8128
+ error: redactMessage(reason),
8129
+ ended_at: (/* @__PURE__ */ new Date()).toISOString()
8130
+ }).where("id", "=", runId).execute();
8131
+ }
7538
8132
  async enqueueRun(options) {
7539
8133
  const id = randomUUID();
7540
8134
  await this.insertPendingRun(this.db, id, options);
7541
8135
  this.runsInFlight.add(id);
7542
8136
  return id;
7543
8137
  }
8138
+ /**
8139
+ * Claims the dedupe key and journals the PENDING run, bounding crash replays.
8140
+ *
8141
+ * Three outcomes, as {@link FireClaim}:
8142
+ *
8143
+ * - **`"claimed"`** with a run id: this fire is ours to run.
8144
+ * - **`"duplicate"`**: an earlier delivery of this operation already
8145
+ * fired: it journaled a run, or ran without one
8146
+ * ({@link FIRED_WITHOUT_RUN_ID}).
8147
+ * - **`"abandoned"`**: the operation has been delivered
8148
+ * {@link FIRE_CRASH_BUDGET} times and has never once got as far as
8149
+ * journaling a run — i.e. it is taking the process down before the write
8150
+ * lands, and the read-model cursor replays it on every boot. That is the
8151
+ * EPIPE boot loop (backlog item 5), and a budget is what turns it from an
8152
+ * unbootable reactor into one FAILED run naming the loop.
8153
+ *
8154
+ * Two writes, and which one is which matters:
8155
+ *
8156
+ * - The **attempt count** is bumped in its own committed transaction,
8157
+ * BEFORE the risky work. That is the only way a delivery that leaves no
8158
+ * run behind can be counted at all, which is what the budget needs. The
8159
+ * counter is never a claim signal — it only decides the budget.
8160
+ * - The **claim** is `run_id`, and it is set in the SAME transaction as the
8161
+ * run row, under a `WHERE run_id IS NULL` guard. The guard takes the
8162
+ * row's lock, so of two concurrent deliveries of one operation the second
8163
+ * blocks until the first commits and then matches no row: exactly one
8164
+ * delivery wins, and the run it journaled is committed with the claim. A
8165
+ * crash in between rolls BOTH back, so a claim that holds a run id always
8166
+ * holds a run — and a claim without one is retried, up to the budget.
8167
+ *
8168
+ * Counting and claiming are deliberately not the same write: if the bump
8169
+ * rode along inside the claim transaction, a run insert that takes the
8170
+ * process down would roll the count back with it and the budget could never
8171
+ * reach its limit — the boot loop it exists to bound.
8172
+ */
7544
8173
  async claimAndEnqueueRun(dedupeKey, ttlMs, nowIso, options) {
8174
+ const attempt = await this.recordFireAttempt(options.workflowId, dedupeKey, ttlMs, nowIso);
8175
+ if (attempt.runId !== null) return { outcome: "duplicate" };
8176
+ if (attempt.attempts > 3) return {
8177
+ outcome: "abandoned",
8178
+ attempts: attempt.attempts
8179
+ };
7545
8180
  const id = randomUUID();
7546
8181
  if (!await this.db.transaction().execute(async (trx) => {
7547
- if (!await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id)) return false;
8182
+ if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
7548
8183
  await this.insertPendingRun(trx, id, options);
7549
8184
  return true;
7550
- })) return null;
8185
+ })) return { outcome: "duplicate" };
8186
+ this.runsInFlight.add(id);
8187
+ return {
8188
+ outcome: "claimed",
8189
+ runId: id
8190
+ };
8191
+ }
8192
+ async journalTriggerItems(items, ttlMs, nowIso) {
8193
+ const ids = await this.db.transaction().execute(async (trx) => {
8194
+ const journaled = [];
8195
+ for (const { dedupeKey, options } of items) {
8196
+ const id = randomUUID();
8197
+ const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
8198
+ if (won) await this.insertPendingRun(trx, id, options);
8199
+ journaled.push(won ? id : void 0);
8200
+ }
8201
+ return journaled;
8202
+ });
8203
+ for (const id of ids) if (id) this.runsInFlight.add(id);
8204
+ return ids;
8205
+ }
8206
+ async recordScheduleFire(nowIso, nextPollAtIso, options) {
8207
+ const id = randomUUID();
8208
+ await this.db.transaction().execute(async (trx) => {
8209
+ await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
8210
+ await this.insertPendingRun(trx, id, options);
8211
+ });
7551
8212
  this.runsInFlight.add(id);
7552
8213
  return id;
7553
8214
  }
8215
+ /**
8216
+ * Puts a piece_store partition back to `entries` and, in the same write,
8217
+ * cancels the runs the restored cursor delivers again and releases their
8218
+ * claims, so no reader sees the old cursor while the claims still hold.
8219
+ */
8220
+ async rewindPieceStore(scope, scopeKey, entries, redelivered = {
8221
+ runIds: [],
8222
+ reason: ""
8223
+ }) {
8224
+ const live = redelivered.runIds.filter((runId) => !erasedRuns.has(runId));
8225
+ const nowIso = (/* @__PURE__ */ new Date()).toISOString();
8226
+ await this.db.transaction().execute(async (trx) => {
8227
+ await trx.deleteFrom("piece_store").where("scope", "=", scope).where("scope_key", "=", scopeKey).execute();
8228
+ for (const [key, value] of Object.entries(entries)) {
8229
+ assertPieceStoreEntry(key, value);
8230
+ await trx.insertInto("piece_store").values({
8231
+ scope,
8232
+ scope_key: scopeKey,
8233
+ key,
8234
+ value: JSON.stringify(value),
8235
+ updated_at: nowIso
8236
+ }).execute();
8237
+ }
8238
+ if (live.length === 0) return;
8239
+ await trx.updateTable("run").set({
8240
+ status: "CANCELLED",
8241
+ error: redactMessage(redelivered.reason),
8242
+ ended_at: nowIso
8243
+ }).where("id", "in", live).execute();
8244
+ await trx.deleteFrom("trigger_dedupe").where("run_id", "in", live).execute();
8245
+ });
8246
+ for (const runId of redelivered.runIds) {
8247
+ this.runsInFlight.delete(runId);
8248
+ erasedRuns.delete(runId);
8249
+ }
8250
+ }
8251
+ /**
8252
+ * Counts one delivery of an operation, and reports whether a run was ever
8253
+ * journaled for it.
8254
+ *
8255
+ * Committed on its own: the point is to leave a trace even when whatever
8256
+ * comes next takes the process down. It is NOT a claim — the row it writes
8257
+ * holds no run id, and {@link claimAndEnqueueRun} is what decides, under a
8258
+ * lock, which delivery gets to fill it in.
8259
+ */
8260
+ async recordFireAttempt(workflowId, dedupeKey, ttlMs, nowIso) {
8261
+ const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
8262
+ await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
8263
+ const row = await this.db.insertInto("trigger_dedupe").values({
8264
+ workflow_id: workflowId,
8265
+ dedupe_key: dedupeKey,
8266
+ run_id: null,
8267
+ created_at: nowIso,
8268
+ attempts: 1
8269
+ }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ attempts: sql`${sql.table("trigger_dedupe")}.attempts + 1` })).returning(["attempts", "run_id"]).executeTakeFirstOrThrow();
8270
+ return {
8271
+ attempts: row.attempts ?? 1,
8272
+ runId: row.run_id
8273
+ };
8274
+ }
8275
+ /**
8276
+ * The abandonment of an over-budget fire, as a visible FAILED run, linked to
8277
+ * its dedupe key so a later delivery is a duplicate rather than another one.
8278
+ */
8279
+ async journalAbandonedFire(dedupeKey, options) {
8280
+ const id = randomUUID();
8281
+ const now = (/* @__PURE__ */ new Date()).toISOString();
8282
+ await this.db.transaction().execute(async (trx) => {
8283
+ await trx.insertInto("run").values({
8284
+ id,
8285
+ workflow_id: options.workflowId,
8286
+ workflow_name: "",
8287
+ workflow_version: 0,
8288
+ trigger_kind: options.triggerKind,
8289
+ trigger_payload: cappedPayload(redact(options.triggerPayload)),
8290
+ status: "FAILED",
8291
+ error: redactMessage(abandonedFireError(options.attempts)),
8292
+ enqueued_at: now,
8293
+ started_at: now,
8294
+ ended_at: now,
8295
+ rerun_of: null,
8296
+ warnings: 0,
8297
+ warning_notes: null
8298
+ }).execute();
8299
+ await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).execute();
8300
+ });
8301
+ return id;
8302
+ }
7554
8303
  async insertPendingRun(db, id, options) {
7555
8304
  const now = (/* @__PURE__ */ new Date()).toISOString();
7556
8305
  await db.insertInto("run").values({
@@ -7616,7 +8365,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7616
8365
  if (result.steps.length > 0) try {
7617
8366
  await this.sweepSteps(runId, result, executionOrder);
7618
8367
  } catch (error) {
7619
- logger$3.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
8368
+ logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
7620
8369
  }
7621
8370
  const notes = runWarningNotes(result);
7622
8371
  await this.db.updateTable("run").set({
@@ -7717,6 +8466,58 @@ var WorkflowRunStore = class WorkflowRunStore {
7717
8466
  for (const row of rows) byRun.get(row.run_id)?.push(row.document_id);
7718
8467
  return byRun;
7719
8468
  }
8469
+ /**
8470
+ * Parks a workflow: its park row, and an ENABLED trigger_state row turned
8471
+ * PARKED, in one transaction. True when a trigger_state row was parked.
8472
+ */
8473
+ async parkWorkflow(workflowId, publishedVersion, reason) {
8474
+ const { workflow_id: _, ...row } = workflowParkRow(workflowId, publishedVersion, reason);
8475
+ const now = row.parked_at;
8476
+ return this.db.transaction().execute(async (trx) => {
8477
+ await trx.insertInto("workflow_park").values({
8478
+ workflow_id: workflowId,
8479
+ ...row
8480
+ }).onConflict((oc) => oc.column("workflow_id").doUpdateSet(row)).execute();
8481
+ return (await trx.updateTable("trigger_state").set({
8482
+ status: PARKED_TRIGGER_STATUS,
8483
+ last_error: row.reason,
8484
+ updated_at: now
8485
+ }).where("workflow_id", "=", workflowId).where("status", "=", "ENABLED").returning("workflow_id").execute()).length > 0;
8486
+ });
8487
+ }
8488
+ /** A disable clears a park: the park row, and a PARKED trigger row turned
8489
+ * DISABLED, so re-enabling arms it whether or not the supervisor held it. */
8490
+ async clearParkOnDisable(workflowId) {
8491
+ await this.db.transaction().execute(async (trx) => {
8492
+ await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
8493
+ await trx.updateTable("trigger_state").set({
8494
+ status: "DISABLED",
8495
+ ...RENEW_CLEARED,
8496
+ updated_at: (/* @__PURE__ */ new Date()).toISOString()
8497
+ }).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
8498
+ });
8499
+ }
8500
+ /** Undoes {@link parkWorkflow}; `trigger` says whether it parked a row. */
8501
+ async liftPark(workflowId, trigger) {
8502
+ await this.db.transaction().execute(async (trx) => {
8503
+ await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
8504
+ if (!trigger) return;
8505
+ await trx.updateTable("trigger_state").set({
8506
+ status: "ENABLED",
8507
+ last_error: null,
8508
+ updated_at: (/* @__PURE__ */ new Date()).toISOString()
8509
+ }).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
8510
+ });
8511
+ }
8512
+ async getWorkflowPark(workflowId) {
8513
+ return this.db.selectFrom("workflow_park").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
8514
+ }
8515
+ async listWorkflowParks() {
8516
+ return this.db.selectFrom("workflow_park").selectAll().execute();
8517
+ }
8518
+ async clearWorkflowPark(workflowId) {
8519
+ await this.db.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
8520
+ }
7720
8521
  async getTriggerState(workflowId) {
7721
8522
  return this.db.selectFrom("trigger_state").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
7722
8523
  }
@@ -7781,14 +8582,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7781
8582
  return this.db.selectFrom("trigger_state").selectAll().orderBy("workflow_id", "asc").execute();
7782
8583
  }
7783
8584
  async recordPollSuccess(workflowId, storeState, nowIso, nextPollAtIso) {
7784
- await this.db.updateTable("trigger_state").set({
7785
- store_state: storeState,
7786
- last_poll_at: nowIso,
7787
- next_poll_at: nextPollAtIso,
7788
- last_error: null,
7789
- consecutive_failures: 0,
7790
- updated_at: nowIso
7791
- }).where("workflow_id", "=", workflowId).execute();
8585
+ await pollSucceededIn(this.db, workflowId, storeState, nowIso, nextPollAtIso);
7792
8586
  }
7793
8587
  async recordPollFailure(workflowId, error, nowIso, nextPollAtIso, consecutiveFailures) {
7794
8588
  await this.db.updateTable("trigger_state").set({
@@ -7800,7 +8594,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7800
8594
  }).where("workflow_id", "=", workflowId).execute();
7801
8595
  }
7802
8596
  async claimDedupe(workflowId, dedupeKey, ttlMs, nowIso) {
7803
- return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso, null);
8597
+ return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso);
7804
8598
  }
7805
8599
  async deleteDedupe(workflowId) {
7806
8600
  await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
@@ -7848,7 +8642,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7848
8642
  for (const row of live) erasedRuns.add(row.id);
7849
8643
  erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
7850
8644
  erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
7851
- erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: null }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
8645
+ erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
7852
8646
  erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
7853
8647
  return erased;
7854
8648
  });
@@ -7900,6 +8694,46 @@ var WorkflowRunStore = class WorkflowRunStore {
7900
8694
  }
7901
8695
  };
7902
8696
  //#endregion
8697
+ //#region src/reactor/park-state.ts
8698
+ var ParkState = class {
8699
+ load;
8700
+ parks = /* @__PURE__ */ new Map();
8701
+ written = /* @__PURE__ */ new Set();
8702
+ seeded = false;
8703
+ seeding;
8704
+ constructor(load) {
8705
+ this.load = load;
8706
+ }
8707
+ async get(workflowId) {
8708
+ if (!this.seeded) {
8709
+ this.seeding ??= this.seed().finally(() => {
8710
+ this.seeding = void 0;
8711
+ });
8712
+ await this.seeding;
8713
+ }
8714
+ return this.parks.get(workflowId);
8715
+ }
8716
+ known(workflowId) {
8717
+ if (!this.seeded && !this.written.has(workflowId)) return void 0;
8718
+ return { park: this.parks.get(workflowId) };
8719
+ }
8720
+ set(park) {
8721
+ this.parks.set(park.workflow_id, park);
8722
+ if (!this.seeded) this.written.add(park.workflow_id);
8723
+ }
8724
+ delete(workflowId) {
8725
+ this.parks.delete(workflowId);
8726
+ if (!this.seeded) this.written.add(workflowId);
8727
+ }
8728
+ async seed() {
8729
+ const rows = await this.load();
8730
+ if (!rows) return;
8731
+ for (const row of rows) if (!this.written.has(row.workflow_id)) this.parks.set(row.workflow_id, row);
8732
+ this.written.clear();
8733
+ this.seeded = true;
8734
+ }
8735
+ };
8736
+ //#endregion
7903
8737
  //#region src/reactor/run-cursor.ts
7904
8738
  var InvalidRunCursorError = class extends Error {
7905
8739
  constructor() {
@@ -7924,13 +8758,137 @@ function decodeRunCursor(cursor) {
7924
8758
  };
7925
8759
  }
7926
8760
  //#endregion
8761
+ //#region src/reactor/run-gate.ts
8762
+ const CLOSED_REASON = "Skipped: the workflow runtime shut down while this firing waited";
8763
+ const QUEUE_DEPTH_ENV = "PH_WORKFLOWS_MAX_QUEUED_FIRINGS";
8764
+ function maxQueuedFirings(env = process.env) {
8765
+ const raw = Number(env[QUEUE_DEPTH_ENV]);
8766
+ return Number.isFinite(raw) && raw > 0 ? raw : 100;
8767
+ }
8768
+ var WorkflowRunGate = class {
8769
+ lanes = /* @__PURE__ */ new Map();
8770
+ closed = false;
8771
+ maxQueued;
8772
+ constructor(options = {}) {
8773
+ this.maxQueued = options.maxQueued ?? maxQueuedFirings();
8774
+ }
8775
+ /** Runs of this workflow executing right now. */
8776
+ active(workflowId) {
8777
+ return this.lanes.get(workflowId)?.active ?? 0;
8778
+ }
8779
+ /** Firings of this workflow waiting for a slot. */
8780
+ waiting(workflowId) {
8781
+ return this.lanes.get(workflowId)?.waiting.length ?? 0;
8782
+ }
8783
+ /**
8784
+ * Takes a slot for one run.
8785
+ *
8786
+ * - **PARALLEL** with no `maxParallelRuns`, or no policy at all: no gate,
8787
+ * so an unbounded workflow's hot path costs nothing. Not the default: a
8788
+ * workflow document starts as QUEUE.
8789
+ * - **PARALLEL** with a bound, and **QUEUE**: waits, first come first
8790
+ * served. Latency, not failure — which is what QUEUE means, up to
8791
+ * {@link DEFAULT_MAX_QUEUED_FIRINGS} waiters. Past that the queue is full
8792
+ * and the firing is refused, so the caller journals it CANCELLED rather
8793
+ * than growing a backlog nothing bounds.
8794
+ * - **SINGLETON**: refused outright while a run is active. The caller
8795
+ * journals the refusal rather than dropping it silently, because a firing
8796
+ * that vanished is indistinguishable from a trigger that never fired.
8797
+ */
8798
+ async admit(workflowId, policy) {
8799
+ if (this.closed) return {
8800
+ admitted: false,
8801
+ reason: CLOSED_REASON,
8802
+ refusal: "closed"
8803
+ };
8804
+ const limit = concurrencyLimit(policy);
8805
+ if (limit === null) return {
8806
+ admitted: true,
8807
+ waited: false,
8808
+ release: () => void 0
8809
+ };
8810
+ const lane = this.lanes.get(workflowId) ?? {
8811
+ active: 0,
8812
+ waiting: []
8813
+ };
8814
+ this.lanes.set(workflowId, lane);
8815
+ let waited = false;
8816
+ if (lane.active >= limit) {
8817
+ if (policy.concurrency === "SINGLETON") {
8818
+ this.forgetIfIdle(workflowId, lane);
8819
+ return {
8820
+ admitted: false,
8821
+ refusal: "singleton",
8822
+ reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
8823
+ };
8824
+ }
8825
+ if (lane.waiting.length >= this.maxQueued) return {
8826
+ admitted: false,
8827
+ refusal: "queue-full",
8828
+ reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
8829
+ };
8830
+ if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
8831
+ admitted: false,
8832
+ reason: CLOSED_REASON,
8833
+ refusal: "closed"
8834
+ };
8835
+ waited = true;
8836
+ } else lane.active += 1;
8837
+ let released = false;
8838
+ return {
8839
+ admitted: true,
8840
+ waited,
8841
+ release: () => {
8842
+ if (released) return;
8843
+ released = true;
8844
+ lane.active -= 1;
8845
+ const next = lane.waiting.shift();
8846
+ if (next) {
8847
+ lane.active += 1;
8848
+ next(true);
8849
+ return;
8850
+ }
8851
+ this.forgetIfIdle(workflowId, lane);
8852
+ }
8853
+ };
8854
+ }
8855
+ /** Refuses every waiting and later firing: a shut-down runtime runs none. */
8856
+ close() {
8857
+ this.closed = true;
8858
+ for (const lane of this.lanes.values()) for (const waiter of lane.waiting.splice(0)) waiter(false);
8859
+ }
8860
+ forgetIfIdle(workflowId, lane) {
8861
+ if (lane.active === 0 && lane.waiting.length === 0) this.lanes.delete(workflowId);
8862
+ }
8863
+ };
8864
+ //#endregion
7927
8865
  //#region src/reactor/run-retention.ts
7928
8866
  const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
7929
8867
  const RETENTION_SWEEP_INTERVAL_MS = 36e5;
7930
8868
  const DAY_MS = 864e5;
8869
+ /** The spellings that turn retention OFF, compared case-insensitively after
8870
+ * trimming. Wider than `0` because "off" and "never" are what an operator
8871
+ * reaches for, and a variable that silently keeps pruning because they wrote
8872
+ * one of those is the worst outcome available. */
8873
+ const RETENTION_OFF_SPELLINGS = [
8874
+ "0",
8875
+ "off",
8876
+ "never",
8877
+ "false",
8878
+ "none"
8879
+ ];
8880
+ /**
8881
+ * The retention window, or undefined when an operator turned it off.
8882
+ *
8883
+ * Unset means the default. An unparseable value also means the default rather
8884
+ * than silently disabling: the variable's purpose is a bound, and a typo must
8885
+ * not remove one.
8886
+ */
7931
8887
  function runRetentionMs(env = process.env) {
7932
- const days = Number(env[RUN_RETENTION_ENV]);
7933
- return days > 0 ? days * DAY_MS : void 0;
8888
+ const raw = env[RUN_RETENTION_ENV]?.trim();
8889
+ if (raw !== void 0 && RETENTION_OFF_SPELLINGS.includes(raw.toLowerCase())) return;
8890
+ const days = Number(raw);
8891
+ return (days > 0 ? days : 30) * DAY_MS;
7934
8892
  }
7935
8893
  async function sweepRetention(store, options) {
7936
8894
  const now = (options.now ?? /* @__PURE__ */ new Date()).getTime();
@@ -7939,13 +8897,33 @@ async function sweepRetention(store, options) {
7939
8897
  dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
7940
8898
  };
7941
8899
  }
8900
+ //#endregion
8901
+ //#region src/reactor/shutdown.ts
8902
+ /** A firing refused because this runtime has shut down: retryable elsewhere. */
8903
+ var WorkflowRuntimeClosedError = class extends Error {
8904
+ workflowId;
8905
+ constructor(workflowId) {
8906
+ super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
8907
+ this.workflowId = workflowId;
8908
+ this.name = "WorkflowRuntimeClosedError";
8909
+ }
8910
+ };
8911
+ var TriggerSupervisorStoppedError = class extends Error {
8912
+ constructor(options) {
8913
+ super("The trigger supervisor has stopped", options);
8914
+ this.name = "TriggerSupervisorStoppedError";
8915
+ }
8916
+ };
8917
+ function isShutdownRefusal(error) {
8918
+ return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
8919
+ }
7942
8920
  childLogger(["workflow", "trigger-binding"]);
7943
8921
  function pieceTriggerKind(block) {
7944
8922
  return `piece:${block.pieceName}:${block.name}`;
7945
8923
  }
7946
8924
  //#endregion
7947
8925
  //#region src/reactor/trigger-supervisor.ts
7948
- const logger$1 = childLogger(["workflow", "trigger-supervisor"]);
8926
+ const logger$2 = childLogger(["workflow", "trigger-supervisor"]);
7949
8927
  function targetOf$1(binding) {
7950
8928
  return {
7951
8929
  name: binding.packageName,
@@ -7965,6 +8943,7 @@ const SCHEDULE_TRIGGER_KIND = "schedule";
7965
8943
  const MIN_INTERVAL_MS = MIN_SCHEDULE_INTERVAL_MS;
7966
8944
  const MAX_BACKOFF_MS = 18e5;
7967
8945
  const DEDUPE_TTL_MS = 3e4;
8946
+ const REDELIVERED_REASON = "Redelivered: the trigger stopped before firing this item and put its cursor back, so the next poll delivers it again";
7968
8947
  const DEFAULT_RECONCILE_INTERVAL_MS = 9e5;
7969
8948
  const RENEW_RETRY_BASE_MS = 6e4;
7970
8949
  function isSchedule(binding) {
@@ -7980,7 +8959,7 @@ function intervalFromSchedules(schedules, defaultMs) {
7980
8959
  const cron = schedule.cronExpression;
7981
8960
  const intervalMs = cronIntervalMs(cron);
7982
8961
  if (intervalMs === void 0) {
7983
- logger$1.warn("Unsupported setSchedule cron \"@cron\"; using the default", cron);
8962
+ logger$2.warn("Unsupported setSchedule cron \"@cron\"; using the default", cron);
7984
8963
  return Math.max(defaultMs, MIN_INTERVAL_MS);
7985
8964
  }
7986
8965
  return intervalMs;
@@ -8026,40 +9005,80 @@ var TriggerSupervisor = class {
8026
9005
  hookTimeoutMs;
8027
9006
  now;
8028
9007
  egress;
9008
+ parks;
8029
9009
  timer;
8030
9010
  ops = Promise.resolve();
8031
9011
  ticking = false;
8032
9012
  warnedMissingJournal = false;
9013
+ stopped = false;
8033
9014
  constructor(options) {
8034
9015
  this.options = options;
8035
- this.worker = options.worker ?? new PieceWorker({ models: options.models });
9016
+ this.worker = options.worker ?? new PieceWorker({
9017
+ models: options.models,
9018
+ telemetry: options.telemetry
9019
+ });
8036
9020
  this.tickMs = options.tickMs ?? 15e3;
8037
9021
  this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
8038
9022
  this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
8039
9023
  this.now = options.now ?? (() => /* @__PURE__ */ new Date());
8040
9024
  this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
9025
+ this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
8041
9026
  }
8042
9027
  start() {
8043
- if (this.timer) return;
9028
+ if (this.timer || this.stopped) return;
8044
9029
  this.timer = setInterval(() => {
8045
9030
  this.tick().catch((error) => {
8046
- logger$1.error("Trigger tick failed: @error", error);
9031
+ if (isShutdownRefusal(error)) {
9032
+ logger$2.debug("Trigger tick stopped: @error", error);
9033
+ return;
9034
+ }
9035
+ logger$2.error("Trigger tick failed: @error", error);
8047
9036
  });
8048
9037
  }, this.tickMs);
8049
9038
  this.timer.unref();
8050
- logger$1.info(`Trigger supervisor started (tick ${this.tickMs}ms)`);
9039
+ logger$2.info(`Trigger supervisor started (tick ${this.tickMs}ms)`);
8051
9040
  }
8052
9041
  stop() {
9042
+ if (this.stopped) return;
9043
+ this.stopped = true;
8053
9044
  if (this.timer) clearInterval(this.timer);
8054
9045
  this.timer = void 0;
8055
9046
  if (!this.options.worker) this.worker.dispose();
8056
- logger$1.info("Trigger supervisor stopped");
9047
+ logger$2.info("Trigger supervisor stopped");
8057
9048
  }
8058
9049
  enqueue(task) {
8059
- const run = this.ops.then(task);
8060
- this.ops = run.catch(() => void 0);
8061
- return run;
8062
- }
9050
+ const run = this.ops.then(async () => {
9051
+ this.throwIfStopped();
9052
+ try {
9053
+ return await task();
9054
+ } catch (error) {
9055
+ if (this.stopped && !(error instanceof TriggerSupervisorStoppedError)) throw new TriggerSupervisorStoppedError({ cause: error });
9056
+ throw error;
9057
+ }
9058
+ });
9059
+ this.ops = run.catch(() => void 0);
9060
+ return run;
9061
+ }
9062
+ throwIfStopped() {
9063
+ if (this.stopped) throw new TriggerSupervisorStoppedError();
9064
+ }
9065
+ async store() {
9066
+ this.throwIfStopped();
9067
+ const store = await this.options.store();
9068
+ if (!store) return void 0;
9069
+ return new Proxy(store, { get: (target, property) => {
9070
+ const value = Reflect.get(target, property, target);
9071
+ if (typeof value !== "function") return value;
9072
+ return (...args) => {
9073
+ this.throwIfStopped();
9074
+ return value.apply(target, args);
9075
+ };
9076
+ } });
9077
+ }
9078
+ fire(workflowId, payload, kind, runId) {
9079
+ this.throwIfStopped();
9080
+ this.options.fire(workflowId, payload, kind, runId);
9081
+ }
8063
9082
  enabledOk = /* @__PURE__ */ new Set();
8064
9083
  descriptors = /* @__PURE__ */ new Map();
8065
9084
  own;
@@ -8067,28 +9086,84 @@ var TriggerSupervisor = class {
8067
9086
  return this.options.resolver ?? (this.own ??= bundleResolver({ cacheDir: this.options.cacheDir }));
8068
9087
  }
8069
9088
  enableRetries = /* @__PURE__ */ new Map();
8070
- upsert(binding) {
9089
+ versions = /* @__PURE__ */ new Map();
9090
+ upsert(binding, publishedVersion) {
9091
+ if (publishedVersion !== void 0) this.versions.set(binding.workflowId, publishedVersion);
8071
9092
  const previous = this.bindings.get(binding.workflowId);
8072
9093
  if (previous && this.enabledOk.has(binding.workflowId) && JSON.stringify(previous) === JSON.stringify(binding)) return Promise.resolve();
8073
9094
  this.bindings.set(binding.workflowId, binding);
9095
+ const park = this.parks.known(binding.workflowId)?.park;
9096
+ if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
8074
9097
  return this.enqueue(() => this.enable(binding, previous));
8075
9098
  }
9099
+ /** Parks a workflow; true when a trigger row was parked with it. */
9100
+ park(workflowId, publishedVersion, reason) {
9101
+ this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
9102
+ return this.enqueue(async () => {
9103
+ const store = await this.store();
9104
+ if (!store) return false;
9105
+ return store.parkWorkflow(workflowId, publishedVersion, reason);
9106
+ });
9107
+ }
9108
+ /** Undoes {@link park} for that version only: a later park stands. */
9109
+ liftPark(workflowId, publishedVersion, trigger) {
9110
+ if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
9111
+ return this.enqueue(async () => {
9112
+ const store = await this.store();
9113
+ if (!store) return;
9114
+ if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
9115
+ await store.liftPark(workflowId, trigger);
9116
+ });
9117
+ }
9118
+ /** A disable clears the park. With no binding in memory to release a PARKED
9119
+ * row through onDisable (a boot), the row is turned DISABLED here. */
9120
+ releasePark(workflowId, flipRow) {
9121
+ this.parks.delete(workflowId);
9122
+ return this.enqueue(async () => {
9123
+ const store = await this.store();
9124
+ if (!store) return;
9125
+ if (flipRow) await store.clearParkOnDisable(workflowId);
9126
+ else await store.clearWorkflowPark(workflowId);
9127
+ });
9128
+ }
9129
+ /** A re-publish past the parked version: on this lane, a park older than
9130
+ * `publishedVersion` goes and a row it parked is ENABLED again, as it was
9131
+ * before (only an ENABLED row is ever parked); a park as new stands. The
9132
+ * next upsert of the same binding then arms rather than skipping as
9133
+ * unchanged, and a retry held back by the park is due now. */
9134
+ unpark(workflowId, publishedVersion) {
9135
+ this.enabledOk.delete(workflowId);
9136
+ const retry = this.enableRetries.get(workflowId);
9137
+ if (retry) retry.at = 0;
9138
+ const park = this.parks.known(workflowId)?.park;
9139
+ if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
9140
+ return this.enqueue(async () => {
9141
+ const store = await this.store();
9142
+ if (!store) return;
9143
+ const park = await store.getWorkflowPark(workflowId);
9144
+ if (!park || park.published_version >= publishedVersion) return;
9145
+ await store.liftPark(workflowId, true);
9146
+ });
9147
+ }
8076
9148
  remove(workflowId) {
8077
9149
  const binding = this.unbind(workflowId);
8078
9150
  return this.enqueue(() => this.disable(workflowId, binding));
8079
9151
  }
8080
9152
  forget(workflowId) {
8081
9153
  const binding = this.unbind(workflowId);
9154
+ this.parks.delete(workflowId);
8082
9155
  return this.enqueue(async () => {
8083
- const store = await this.options.store();
9156
+ const store = await this.store();
8084
9157
  if (!store) return;
8085
9158
  await this.disable(workflowId, binding);
8086
9159
  await store.deleteTriggerState(workflowId);
9160
+ await store.clearWorkflowPark(workflowId);
8087
9161
  });
8088
9162
  }
8089
9163
  unbind(workflowId) {
8090
9164
  const binding = this.bindings.get(workflowId);
8091
9165
  this.bindings.delete(workflowId);
9166
+ this.versions.delete(workflowId);
8092
9167
  this.enabledOk.delete(workflowId);
8093
9168
  this.enableRetries.delete(workflowId);
8094
9169
  return binding;
@@ -8098,12 +9173,16 @@ var TriggerSupervisor = class {
8098
9173
  return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
8099
9174
  }
8100
9175
  async recordRejection(workflowId, block, config, message, retryAt) {
8101
- const store = await this.options.store();
9176
+ const store = await this.store();
8102
9177
  if (!store) return;
8103
9178
  const hash = configHash(block, config);
8104
9179
  const existing = await store.getTriggerState(workflowId);
8105
9180
  if (existing?.status === "ENABLED" && existing.config_hash === hash) {
8106
- logger$1.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
9181
+ logger$2.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
9182
+ return;
9183
+ }
9184
+ if (existing?.status === "PARKED" && existing.config_hash === hash) {
9185
+ logger$2.warn(`Workflow ${workflowId} is PARKED and could not be resolved; the park stands. @error`, message);
8107
9186
  return;
8108
9187
  }
8109
9188
  await store.upsertTriggerState({
@@ -8125,7 +9204,7 @@ var TriggerSupervisor = class {
8125
9204
  }
8126
9205
  test(binding, options = {}) {
8127
9206
  return this.enqueue(async () => {
8128
- const store = await this.options.store();
9207
+ const store = await this.store();
8129
9208
  try {
8130
9209
  return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
8131
9210
  } finally {
@@ -8139,7 +9218,7 @@ var TriggerSupervisor = class {
8139
9218
  await store.deletePieceStore("FLOW", testPartitionKey("FLOW", workflowId));
8140
9219
  await store.deletePieceStore("PROJECT", testPartitionKey("PROJECT", workflowId));
8141
9220
  } catch (error) {
8142
- logger$1.warn(`Could not clear test store for ${workflowId}`, error);
9221
+ logger$2.warn(`Could not clear test store for ${workflowId}`, error);
8143
9222
  }
8144
9223
  }
8145
9224
  handshake(binding, payload) {
@@ -8149,45 +9228,58 @@ var TriggerSupervisor = class {
8149
9228
  return this.enqueue(async () => {
8150
9229
  const binding = this.bindings.get(workflowId);
8151
9230
  if (!binding || isSchedule(binding)) {
8152
- logger$1.warn(`Webhook delivery for unknown workflow ${workflowId}`);
9231
+ logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
8153
9232
  return;
8154
9233
  }
8155
- const store = await this.options.store();
9234
+ const store = await this.store();
8156
9235
  if (!store) throw new MissingJournalError("Webhook delivery");
8157
9236
  const row = await store.getTriggerState(workflowId);
8158
9237
  const now = this.now();
8159
9238
  const rewind = await this.cursorRewind(store, workflowId);
9239
+ let refused = false;
8160
9240
  try {
8161
- const result = await this.hook(binding, "run", { payload });
9241
+ const result = await this.hook(binding, "run", { payload }).catch((error) => {
9242
+ refused = isShutdownRefusal(error);
9243
+ throw error;
9244
+ });
8162
9245
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
8163
9246
  if (row?.status === "ENABLED") await store.recordPollSuccess(workflowId, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
8164
- for (const item of result.output) await this.fireItem(store, binding, item, now);
9247
+ await this.fireItems(store, binding, result.output, now, rewind);
8165
9248
  } catch (error) {
8166
- await rewind();
9249
+ if (!refused) await rewind();
8167
9250
  throw error;
8168
9251
  }
8169
9252
  });
8170
9253
  }
8171
9254
  async cursorRewind(store, workflowId) {
8172
9255
  const before = await store.listPieceStore("FLOW", workflowId);
8173
- return async () => {
9256
+ let rewound;
9257
+ const rewind = async (redelivered) => {
8174
9258
  try {
8175
- await store.deletePieceStore("FLOW", workflowId);
8176
- for (const [key, value] of Object.entries(before)) await store.setPieceStoreValue("FLOW", workflowId, key, value);
9259
+ const raw = await this.options.store();
9260
+ if (!raw) return false;
9261
+ await raw.rewindPieceStore("FLOW", workflowId, before, {
9262
+ runIds: redelivered,
9263
+ reason: REDELIVERED_REASON
9264
+ });
9265
+ return true;
8177
9266
  } catch (error) {
8178
- logger$1.warn(`Could not rewind the cursor for ${workflowId}`, error);
9267
+ logger$2.warn(`Could not rewind the cursor for ${workflowId}`, error);
9268
+ return false;
8179
9269
  }
8180
9270
  };
9271
+ return (redelivered = []) => rewound ??= rewind(redelivered);
8181
9272
  }
8182
9273
  async hook(binding, hook, options = {}) {
8183
9274
  if (hook !== "onDisable") checkDynamicProperties(binding.config, binding.propertySettings);
8184
- const store = await this.options.store();
9275
+ const store = await this.store();
8185
9276
  if (!store && hook !== "test") throw new MissingJournalError(`Trigger hook "${hook}"`);
8186
9277
  const pieceStore = store ? createPieceStorePort(store, () => binding.workflowId, hook === "test") : void 0;
8187
9278
  const piece = await this.resolver().resolve(targetOf$1(binding));
8188
9279
  const auth = await this.options.resolveAuth(binding.connectionId, { piecePackage: binding.packageName });
8189
9280
  const redactValues = secretsFor(auth);
8190
9281
  const reactor = await this.reactorFor(binding, hook, options.runUser);
9282
+ this.throwIfStopped();
8191
9283
  return this.worker.runTriggerHook({
8192
9284
  ...pieceModuleRef(piece),
8193
9285
  triggerName: binding.triggerName,
@@ -8255,11 +9347,17 @@ var TriggerSupervisor = class {
8255
9347
  };
8256
9348
  }
8257
9349
  async enable(binding, superseded) {
8258
- const store = await this.options.store();
9350
+ const store = await this.store();
8259
9351
  if (!store) throw new MissingJournalError("Enabling a trigger");
8260
9352
  const hash = configHash(binding.block, binding.config);
8261
- const existing = await store.getTriggerState(binding.workflowId);
8262
9353
  const now = this.now();
9354
+ const park = await store.getWorkflowPark(binding.workflowId);
9355
+ if (park && await this.blockingPark(store, binding.workflowId)) {
9356
+ this.holdParked(binding, superseded, park);
9357
+ return;
9358
+ }
9359
+ if (park) await store.liftPark(binding.workflowId, true);
9360
+ const existing = await store.getTriggerState(binding.workflowId);
8263
9361
  const isRepublish = existing?.config_hash === hash && existing.status === "ENABLED" && !store.hasUnmigratedTriggerState(binding.workflowId);
8264
9362
  if (existing && !isRepublish && existing.status === "ENABLED") await this.disableRow(binding.workflowId, existing, superseded);
8265
9363
  const pending = this.enableRetries.get(binding.workflowId);
@@ -8290,7 +9388,7 @@ var TriggerSupervisor = class {
8290
9388
  workflow_id: binding.workflowId,
8291
9389
  ...triggerBlockColumns(binding.block),
8292
9390
  config_hash: hash,
8293
- status: "ENABLED",
9391
+ status: await this.armedStatus(store, binding.workflowId),
8294
9392
  store_state: VESTIGIAL_STORE_STATE,
8295
9393
  interval_ms: intervalMs,
8296
9394
  next_poll_at: new Date(now.getTime() + intervalMs).toISOString(),
@@ -8308,8 +9406,9 @@ var TriggerSupervisor = class {
8308
9406
  this.enabledOk.add(binding.workflowId);
8309
9407
  this.enableRetries.delete(binding.workflowId);
8310
9408
  store.clearUnmigratedTriggerState(binding.workflowId);
8311
- logger$1.info(`Enabled @block for workflow ${binding.workflowId} (every ${intervalMs}ms)`, blockLabel(binding.block));
9409
+ logger$2.info(`Enabled @block for workflow ${binding.workflowId} (every ${intervalMs}ms)`, blockLabel(binding.block));
8312
9410
  } catch (error) {
9411
+ if (this.stopped) throw error;
8313
9412
  this.enabledOk.delete(binding.workflowId);
8314
9413
  const message = error instanceof Error ? error.message : String(error);
8315
9414
  const failures = (existing?.consecutive_failures ?? 0) + 1;
@@ -8337,7 +9436,7 @@ var TriggerSupervisor = class {
8337
9436
  updated_at: now.toISOString(),
8338
9437
  ...pieceColumns(binding)
8339
9438
  });
8340
- logger$1.error(`onEnable failed for workflow ${binding.workflowId} (${failures}x): @error` + (retryAt ? `; retrying at ${retryAt.toISOString()}` : "; not retrying"), message);
9439
+ logger$2.error(`onEnable failed for workflow ${binding.workflowId} (${failures}x): @error` + (retryAt ? `; retrying at ${retryAt.toISOString()}` : "; not retrying"), message);
8341
9440
  }
8342
9441
  }
8343
9442
  renewAtAfterEnable(renew, existing, isRepublish, now) {
@@ -8352,6 +9451,43 @@ var TriggerSupervisor = class {
8352
9451
  if (!url) throw new Error("The reactor's webhook endpoint is not available yet for this workflow");
8353
9452
  return url;
8354
9453
  }
9454
+ /**
9455
+ * The park, when it holds this registration back.
9456
+ *
9457
+ * `policy.onFailure = PARK` is a **runtime override of the document's
9458
+ * enabled-ness**: the document still says ENABLED, and re-arming from it is
9459
+ * what a restart does for every workflow it finds. So the park row is the
9460
+ * authority, read on this lane, and it blocks every registration of the
9461
+ * version that failed or an older one — a boot re-arm, an ERROR row's
9462
+ * retry. A newer version is what a re-publish is: it is never blocked, and
9463
+ * {@link enable} lifts the park it outlived. A re-enable lifts it through
9464
+ * the disable before it.
9465
+ */
9466
+ async blockingPark(store, workflowId) {
9467
+ const park = await store.getWorkflowPark(workflowId);
9468
+ if (!park) return void 0;
9469
+ const version = this.versions.get(workflowId);
9470
+ return version === void 0 || version <= park.published_version ? park : void 0;
9471
+ }
9472
+ /**
9473
+ * An enable a park held back. The row is left as it stands, and a pending
9474
+ * retry keeps its release flag, so the enable after the park lifts releases
9475
+ * a failed attempt and arms afresh; the retry is only pushed back, so it
9476
+ * does not take every tick's slot. A binding this replaced is restored, so
9477
+ * nothing is released later with one that was never armed.
9478
+ */
9479
+ holdParked(binding, superseded, park) {
9480
+ const { workflowId } = binding;
9481
+ const retry = this.enableRetries.get(workflowId);
9482
+ if (retry) retry.at = this.now().getTime() + this.defaultIntervalMs;
9483
+ if (superseded && this.bindings.get(workflowId) === binding) this.bindings.set(workflowId, superseded);
9484
+ logger$2.warn(`Workflow ${workflowId} is PARKED at version ${park.published_version}; its trigger is not armed until the workflow is re-published or re-enabled. @error`, park.reason);
9485
+ }
9486
+ /** The status an enable that succeeded writes: never ENABLED while a park
9487
+ * stands, whatever ran meanwhile. The registration stays recorded. */
9488
+ async armedStatus(store, workflowId) {
9489
+ return await this.blockingPark(store, workflowId) ? PARKED_TRIGGER_STATUS : "ENABLED";
9490
+ }
8355
9491
  deferToStoredRetry(binding, hash, existing, now) {
8356
9492
  if (existing?.status !== "ERROR" || existing.config_hash !== hash) return false;
8357
9493
  const pending = this.enableRetries.get(binding.workflowId);
@@ -8364,7 +9500,7 @@ var TriggerSupervisor = class {
8364
9500
  failures: existing.consecutive_failures,
8365
9501
  release: true
8366
9502
  });
8367
- logger$1.info(`Enable for workflow ${binding.workflowId} still backing off until ${existing.next_poll_at}`);
9503
+ logger$2.info(`Enable for workflow ${binding.workflowId} still backing off until ${existing.next_poll_at}`);
8368
9504
  }
8369
9505
  return true;
8370
9506
  }
@@ -8372,7 +9508,7 @@ var TriggerSupervisor = class {
8372
9508
  try {
8373
9509
  await this.hook(binding, "onDisable");
8374
9510
  } catch (error) {
8375
- logger$1.warn(`onDisable before retrying workflow ${binding.workflowId} failed`, error);
9511
+ logger$2.warn(`onDisable before retrying workflow ${binding.workflowId} failed`, error);
8376
9512
  }
8377
9513
  }
8378
9514
  async enableSchedule(store, binding, hash, existing) {
@@ -8394,14 +9530,14 @@ var TriggerSupervisor = class {
8394
9530
  const nextAt = carried ? new Date(carried) : nextFireAt(schedule, now);
8395
9531
  await store.upsertTriggerState({
8396
9532
  ...base,
8397
- status: "ENABLED",
9533
+ status: await this.armedStatus(store, binding.workflowId),
8398
9534
  interval_ms: schedule.mode === "interval" ? schedule.everyMs : MIN_SCHEDULE_INTERVAL_MS,
8399
9535
  next_poll_at: nextAt.toISOString(),
8400
9536
  last_error: null,
8401
9537
  consecutive_failures: 0
8402
9538
  });
8403
9539
  this.enabledOk.add(binding.workflowId);
8404
- logger$1.info(`Scheduled workflow ${binding.workflowId}: next fire ${nextAt.toISOString()}` + (carried ? " (carried over)" : ""));
9540
+ logger$2.info(`Scheduled workflow ${binding.workflowId}: next fire ${nextAt.toISOString()}` + (carried ? " (carried over)" : ""));
8405
9541
  } catch (error) {
8406
9542
  this.enabledOk.delete(binding.workflowId);
8407
9543
  const message = error instanceof Error ? error.message : String(error);
@@ -8413,24 +9549,24 @@ var TriggerSupervisor = class {
8413
9549
  last_error: message,
8414
9550
  consecutive_failures: (existing?.consecutive_failures ?? 0) + 1
8415
9551
  });
8416
- logger$1.error(`Invalid schedule for workflow ${binding.workflowId}: @error`, message);
9552
+ logger$2.error(`Invalid schedule for workflow ${binding.workflowId}: @error`, message);
8417
9553
  }
8418
9554
  }
8419
9555
  async disable(workflowId, binding) {
8420
- const store = await this.options.store();
9556
+ const store = await this.store();
8421
9557
  if (!store) throw new MissingJournalError("Disabling a trigger");
8422
9558
  const row = await store.getTriggerState(workflowId);
8423
9559
  if (!row || row.status === "DISABLED") return;
8424
9560
  await this.disableRow(workflowId, row, binding);
8425
9561
  }
8426
9562
  async disableRow(workflowId, row, binding) {
8427
- const store = await this.options.store();
9563
+ const store = await this.store();
8428
9564
  if (!store) return;
8429
9565
  const target = binding ?? this.bindingFromRow(row);
8430
9566
  if (target && !isSchedule(target) && blockKey(triggerRowBlock(row)) !== SCHEDULE_BLOCK) try {
8431
9567
  await this.hook(target, "onDisable");
8432
9568
  } catch (error) {
8433
- logger$1.warn(`onDisable failed for workflow ${workflowId}`, error);
9569
+ logger$2.warn(`onDisable failed for workflow ${workflowId}`, error);
8434
9570
  }
8435
9571
  await store.setTriggerStatus(workflowId, "DISABLED");
8436
9572
  }
@@ -8438,7 +9574,7 @@ var TriggerSupervisor = class {
8438
9574
  return this.bindings.get(row.workflow_id);
8439
9575
  }
8440
9576
  async tick() {
8441
- if (this.ticking) return;
9577
+ if (this.ticking || this.stopped) return;
8442
9578
  this.ticking = true;
8443
9579
  try {
8444
9580
  await this.enqueue(() => this.pollDue());
@@ -8459,6 +9595,7 @@ var TriggerSupervisor = class {
8459
9595
  try {
8460
9596
  await this.enable(binding);
8461
9597
  } catch (error) {
9598
+ if (this.stopped) throw error;
8462
9599
  const failures = retry.failures + 1;
8463
9600
  const intervalMs = isSchedule(binding) ? MIN_INTERVAL_MS : pollIntervalFor(binding, void 0, this.defaultIntervalMs);
8464
9601
  this.enableRetries.set(workflowId, {
@@ -8466,20 +9603,21 @@ var TriggerSupervisor = class {
8466
9603
  failures,
8467
9604
  release: retry.release
8468
9605
  });
8469
- logger$1.error(`Enable retry for workflow ${workflowId} threw`, error);
9606
+ logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
8470
9607
  }
8471
9608
  }
8472
9609
  async pollDue() {
8473
- const store = await this.options.store();
9610
+ const store = await this.store();
8474
9611
  if (!store) {
8475
9612
  if (!this.warnedMissingJournal) {
8476
9613
  this.warnedMissingJournal = true;
8477
- logger$1.error("Trigger polling is off: @error", new MissingJournalError("Polling"));
9614
+ logger$2.error("Trigger polling is off: @error", new MissingJournalError("Polling"));
8478
9615
  }
8479
9616
  return;
8480
9617
  }
8481
9618
  const due = await store.listDueTriggerStates(this.now().toISOString());
8482
9619
  for (const row of due) {
9620
+ this.throwIfStopped();
8483
9621
  const binding = this.bindings.get(row.workflow_id);
8484
9622
  if (!binding) {
8485
9623
  await store.setTriggerStatus(row.workflow_id, "DISABLED");
@@ -8488,12 +9626,15 @@ var TriggerSupervisor = class {
8488
9626
  if (isSchedule(binding)) await this.fireSchedule(store, row, binding);
8489
9627
  else await this.poll(store, row, binding);
8490
9628
  }
9629
+ this.throwIfStopped();
8491
9630
  await this.renewDue(store);
9631
+ this.throwIfStopped();
8492
9632
  await this.retryOneEnable();
8493
9633
  }
8494
9634
  async renewDue(store) {
8495
9635
  const due = await store.listDueTriggerRenewals(this.now().toISOString());
8496
9636
  for (const row of due) {
9637
+ this.throwIfStopped();
8497
9638
  const binding = this.bindings.get(row.workflow_id);
8498
9639
  if (!binding) {
8499
9640
  await store.setTriggerStatus(row.workflow_id, "DISABLED");
@@ -8519,7 +9660,7 @@ var TriggerSupervisor = class {
8519
9660
  await this.hook(binding, "onRenew", { webhookUrl });
8520
9661
  const nextAt = nextRenewAt(renew, now);
8521
9662
  await store.recordRenewSuccess(row.workflow_id, now.toISOString(), nextAt.toISOString());
8522
- logger$1.info(`Renewed trigger of workflow ${row.workflow_id}; next ${nextAt.toISOString()}`);
9663
+ logger$2.info(`Renewed trigger of workflow ${row.workflow_id}; next ${nextAt.toISOString()}`);
8523
9664
  } catch (error) {
8524
9665
  const message = error instanceof Error ? error.message : String(error);
8525
9666
  const failures = row.renew_failures + 1;
@@ -8527,7 +9668,7 @@ var TriggerSupervisor = class {
8527
9668
  const slot = renew ? nextRenewAt(renew, now).getTime() : backoff;
8528
9669
  const retryAt = new Date(Math.min(backoff, slot));
8529
9670
  await store.recordRenewFailure(row.workflow_id, message, now.toISOString(), retryAt.toISOString(), failures);
8530
- logger$1.warn(`onRenew failed for workflow ${row.workflow_id} (${failures}x): @error; retrying at ${retryAt.toISOString()}`, message);
9671
+ logger$2.warn(`onRenew failed for workflow ${row.workflow_id} (${failures}x): @error; retrying at ${retryAt.toISOString()}`, message);
8531
9672
  }
8532
9673
  }
8533
9674
  async fireSchedule(store, row, binding) {
@@ -8536,38 +9677,79 @@ var TriggerSupervisor = class {
8536
9677
  const schedule = parseScheduleConfig$1(binding.config);
8537
9678
  const scheduledFor = row.next_poll_at ? new Date(row.next_poll_at) : now;
8538
9679
  const nextAt = rescheduleAfterFire(schedule, scheduledFor, now);
8539
- await store.recordPollSuccess(row.workflow_id, "{}", now.toISOString(), nextAt.toISOString());
8540
- this.options.fire(binding.workflowId, schedulePayload(schedule, scheduledFor, now), SCHEDULE_TRIGGER_KIND);
9680
+ const payload = schedulePayload(schedule, scheduledFor, now);
9681
+ const runId = await store.recordScheduleFire(now.toISOString(), nextAt.toISOString(), {
9682
+ workflowId: binding.workflowId,
9683
+ triggerKind: SCHEDULE_TRIGGER_KIND,
9684
+ triggerPayload: payload
9685
+ });
9686
+ try {
9687
+ this.fire(binding.workflowId, payload, SCHEDULE_TRIGGER_KIND, runId);
9688
+ } catch (error) {
9689
+ await this.failRefused([runId], error);
9690
+ throw error;
9691
+ }
8541
9692
  } catch (error) {
9693
+ if (this.stopped) throw error;
8542
9694
  const message = error instanceof Error ? error.message : String(error);
8543
9695
  this.enabledOk.delete(binding.workflowId);
8544
9696
  await store.setTriggerStatus(row.workflow_id, "ERROR", message);
8545
- logger$1.error(`Schedule fire failed for workflow ${row.workflow_id}: @error`, message);
9697
+ logger$2.error(`Schedule fire failed for workflow ${row.workflow_id}: @error`, message);
8546
9698
  }
8547
9699
  }
8548
9700
  async poll(store, row, binding) {
8549
9701
  const now = this.now();
8550
9702
  const rewind = await this.cursorRewind(store, row.workflow_id);
9703
+ let refused = false;
8551
9704
  try {
8552
- const result = await this.hook(binding, "run");
9705
+ const result = await this.hook(binding, "run").catch((error) => {
9706
+ refused = isShutdownRefusal(error);
9707
+ throw error;
9708
+ });
8553
9709
  if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
8554
9710
  await store.recordPollSuccess(row.workflow_id, VESTIGIAL_STORE_STATE, now.toISOString(), new Date(now.getTime() + row.interval_ms).toISOString());
8555
- for (const item of result.output) await this.fireItem(store, binding, item, now);
9711
+ await this.fireItems(store, binding, result.output, now, rewind);
8556
9712
  } catch (error) {
8557
- await rewind();
9713
+ if (!refused) await rewind();
9714
+ if (this.stopped) throw error;
8558
9715
  const message = error instanceof Error ? error.message : String(error);
8559
9716
  const failures = row.consecutive_failures + 1;
8560
9717
  const backoff = backoffMs(row.interval_ms, failures);
8561
9718
  await store.recordPollFailure(row.workflow_id, message, now.toISOString(), new Date(now.getTime() + backoff).toISOString(), failures);
8562
- logger$1.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
9719
+ logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
8563
9720
  }
8564
9721
  }
8565
- async fireItem(store, binding, item, now) {
8566
- const dedupeKey = extractDedupeKey(item);
8567
- if (dedupeKey) {
8568
- if (!await store.claimDedupe(binding.workflowId, dedupeKey, DEDUPE_TTL_MS, now.toISOString())) return;
9722
+ async fireItems(store, binding, items, now, rewind) {
9723
+ if (items.length === 0) return;
9724
+ const kind = pieceTriggerKind(binding.block);
9725
+ const journaled = (await store.journalTriggerItems(items.map((item) => ({
9726
+ dedupeKey: extractDedupeKey(item),
9727
+ options: {
9728
+ workflowId: binding.workflowId,
9729
+ triggerKind: kind,
9730
+ triggerPayload: item
9731
+ }
9732
+ })), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
9733
+ runId,
9734
+ item: items[index]
9735
+ }] : []);
9736
+ if (this.stopped) {
9737
+ const error = new TriggerSupervisorStoppedError();
9738
+ await this.redeliver(journaled.map(({ runId }) => runId), rewind, error);
9739
+ throw error;
9740
+ }
9741
+ for (const { runId, item } of journaled) this.fire(binding.workflowId, item, kind, runId);
9742
+ }
9743
+ async redeliver(runIds, rewind, error) {
9744
+ if (!await rewind(runIds)) await this.failRefused(runIds, error);
9745
+ }
9746
+ async failRefused(runIds, error) {
9747
+ try {
9748
+ const raw = await this.options.store();
9749
+ for (const runId of runIds) await raw?.failRun(runId, error instanceof Error ? error.message : String(error), error instanceof Error ? error.name : void 0);
9750
+ } catch (failure) {
9751
+ logger$2.warn(`Could not fail the refused runs ${runIds.join(", ")}`, failure);
8569
9752
  }
8570
- this.options.fire(binding.workflowId, item, pieceTriggerKind(binding.block));
8571
9753
  }
8572
9754
  };
8573
9755
  //#endregion
@@ -8811,7 +9993,7 @@ const DESCRIBE_TIMEOUT_MS = 3e4;
8811
9993
  const SEED_ATTEMPTS = 3;
8812
9994
  const SEED_RETRY_BASE_MS = 250;
8813
9995
  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref());
8814
- const logger = childLogger(["workflow", "runtime"]);
9996
+ const logger$1 = childLogger(["workflow", "runtime"]);
8815
9997
  const DRIVE_DOCUMENT_TYPE = "powerhouse/document-drive";
8816
9998
  const WORKFLOW_DOCUMENT_TYPE = "powerhouse/workflow";
8817
9999
  const DOCUMENT_SCOPE = "document";
@@ -8904,6 +10086,7 @@ const UNAUTHORIZED = { status: 401 };
8904
10086
  const JSON_CONTENT_TYPE = "application/json; charset=utf-8";
8905
10087
  const DELIVERY_TIMEOUT_MS = Number(process.env.PH_WORKFLOWS_WEBHOOK_TIMEOUT_MS) || 3e4;
8906
10088
  const TIMED_OUT = Symbol("webhook delivery timed out");
10089
+ const WEBHOOK_REDELIVERED_REASON = "Redelivered: the runtime shut down before this delivery ran, and the sender was told to retry";
8907
10090
  /** Resolves to TIMED_OUT, and never keeps the process alive waiting to. */
8908
10091
  function timeout(ms) {
8909
10092
  return new Promise((resolve) => {
@@ -8927,7 +10110,7 @@ function splitPollInterval(config) {
8927
10110
  const { [POLL_INTERVAL_CONFIG_KEY]: raw, ...rest } = config;
8928
10111
  const seconds = typeof raw === "string" ? Number(raw) : raw;
8929
10112
  if (typeof seconds !== "number" || !Number.isFinite(seconds) || seconds <= 0) {
8930
- logger.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
10113
+ logger$1.warn(`Ignoring ${POLL_INTERVAL_CONFIG_KEY}=${JSON.stringify(raw)}: expected a positive number of seconds`);
8931
10114
  return { config: rest };
8932
10115
  }
8933
10116
  return {
@@ -8943,6 +10126,17 @@ function parseWorkflowState(resultingState) {
8943
10126
  return;
8944
10127
  }
8945
10128
  }
10129
+ function refusalStatus(refusal) {
10130
+ switch (refusal) {
10131
+ case "parked":
10132
+ case "singleton":
10133
+ case "stale": return 409;
10134
+ case "queue-full":
10135
+ case "expired": return 429;
10136
+ default: return 500;
10137
+ }
10138
+ }
10139
+ const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
8946
10140
  function registrationKey(state) {
8947
10141
  const trigger = runnableDefinition(state).trigger;
8948
10142
  const { lastTest: _lastTest, ...armed } = trigger ?? {};
@@ -8961,9 +10155,11 @@ function runJournalName(stateName, documentName) {
8961
10155
  var WorkflowRuntimeService = class {
8962
10156
  host;
8963
10157
  logger;
10158
+ telemetry;
8964
10159
  attachments;
8965
10160
  executor;
8966
10161
  pieceWorkers;
10162
+ unobservePool;
8967
10163
  storePromise;
8968
10164
  storeError;
8969
10165
  storeOpening = false;
@@ -8978,13 +10174,31 @@ var WorkflowRuntimeService = class {
8978
10174
  seedError;
8979
10175
  constructor(host) {
8980
10176
  this.host = host;
10177
+ this.telemetry = new WorkflowTelemetry(host.telemetry);
8981
10178
  this.blockResolver = new BlockResolver({
8982
10179
  local: installedPiece,
8983
10180
  timeoutMs: host.pieceVersionLookupMs ?? PIECE_VERSION_LOOKUP_TIMEOUT_MS,
8984
10181
  hasBlock: (ref, version, source) => this.versionHasBlock(ref, version, source)
8985
10182
  });
8986
- this.logger = host.logger ?? logger;
8987
- this.attachments = host.attachments ? createAttachmentPort(host.attachments, () => currentWorkflowId(), (documentId, ref) => host.canReadAttachmentRef?.(documentId, ref) ?? Promise.resolve(false)) : void 0;
10183
+ this.logger = host.logger ?? logger$1;
10184
+ this.attachments = host.attachments ? createAttachmentPort(host.attachments, {
10185
+ documentIdFor: () => currentWorkflowId(),
10186
+ canReadRef: async (ref) => {
10187
+ const workflowId = currentWorkflowId();
10188
+ const origins = currentAttachmentOrigins();
10189
+ if (!workflowId) return false;
10190
+ if (origins?.written.has(ref)) return true;
10191
+ if (!host.canReadAttachmentRef) return false;
10192
+ const runUser = currentRunUser();
10193
+ return host.canReadAttachmentRef({
10194
+ workflowId,
10195
+ ref,
10196
+ documentIds: [...new Set(origins?.documentIds() ?? [])],
10197
+ ...runUser !== void 0 ? { runUser } : {}
10198
+ });
10199
+ },
10200
+ onWritten: (ref) => currentAttachmentOrigins()?.written.add(ref)
10201
+ }) : void 0;
8988
10202
  this.storePromise = this.openStore();
8989
10203
  this.seedPromise = this.seedWithRetries();
8990
10204
  this.startRetention();
@@ -9014,7 +10228,7 @@ var WorkflowRuntimeService = class {
9014
10228
  }
9015
10229
  }
9016
10230
  async store() {
9017
- if (this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
10231
+ if (!this.closed && this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
9018
10232
  try {
9019
10233
  return await this.storePromise;
9020
10234
  } catch {
@@ -9093,9 +10307,22 @@ var WorkflowRuntimeService = class {
9093
10307
  return await this.endpointCount() > 0;
9094
10308
  }
9095
10309
  registeredAs = /* @__PURE__ */ new Map();
9096
- async updateRegistration(workflowId, state, onlyIfChanged = false) {
10310
+ registrationChains = /* @__PURE__ */ new Map();
10311
+ inRegistrationOrder(workflowId, task) {
10312
+ const run = (this.registrationChains.get(workflowId) ?? Promise.resolve()).then(task);
10313
+ const tail = run.then(() => void 0, () => void 0);
10314
+ this.registrationChains.set(workflowId, tail);
10315
+ tail.then(() => {
10316
+ if (this.registrationChains.get(workflowId) === tail) this.registrationChains.delete(workflowId);
10317
+ });
10318
+ return run;
10319
+ }
10320
+ updateRegistration(workflowId, state, onlyIfChanged = false) {
10321
+ return this.inRegistrationOrder(workflowId, () => this.registerNow(workflowId, state, onlyIfChanged));
10322
+ }
10323
+ async registerNow(workflowId, state, onlyIfChanged) {
9097
10324
  const key = registrationKey(state);
9098
- if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key) return;
10325
+ if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key && !await this.outdatedPark(workflowId, state)) return;
9099
10326
  this.registeredAs.set(workflowId, key);
9100
10327
  try {
9101
10328
  await this.applyRegistration(workflowId, state);
@@ -9114,6 +10341,23 @@ var WorkflowRuntimeService = class {
9114
10341
  }
9115
10342
  const trigger = state.status === "ENABLED" ? runnableDefinition(state).trigger : void 0;
9116
10343
  const block = trigger ? triggerBlock(trigger) : void 0;
10344
+ const version = runnableDefinition(state).version;
10345
+ if (state.status !== "ENABLED") {
10346
+ const held = this.registry.get(workflowId);
10347
+ this.supervisor().releasePark(workflowId, !(held && SUPERVISED_KINDS.has(held.kind))).catch((error) => {
10348
+ this.laneFailed(`Could not clear the park of disabled workflow ${workflowId}`, error);
10349
+ });
10350
+ } else if (await this.outdatedPark(workflowId, state)) this.supervisor().unpark(workflowId, version).catch((error) => {
10351
+ this.laneFailed(`Could not lift the outdated park of workflow ${workflowId}`, error);
10352
+ });
10353
+ const park = block ? await this.parks.get(workflowId) : void 0;
10354
+ if (block && (blockKey(block) === WEBHOOK_BLOCK || triggerKindOf(block)) && park && park.published_version >= version) {
10355
+ const had = this.registry.get(workflowId);
10356
+ this.registry.delete(workflowId);
10357
+ if (had && SUPERVISED_KINDS.has(had.kind)) this.dropSupervised(workflowId);
10358
+ this.logger.warn(`Workflow ${workflowId} is PARKED; its trigger stays unarmed until the workflow is re-published or re-enabled`);
10359
+ return;
10360
+ }
9117
10361
  if (block && blockKey(block) === WEBHOOK_BLOCK) {
9118
10362
  await this.registerWebhook(workflowId, block, trigger.config);
9119
10363
  return;
@@ -9136,14 +10380,14 @@ var WorkflowRuntimeService = class {
9136
10380
  workflowId,
9137
10381
  kind: "schedule"
9138
10382
  });
9139
- this.enableSupervised(workflowId, supervised);
10383
+ this.enableSupervised(workflowId, supervised, version);
9140
10384
  return;
9141
10385
  }
9142
10386
  this.registry.set(workflowId, {
9143
10387
  workflowId,
9144
10388
  kind: "piece"
9145
10389
  });
9146
- await this.registerPieceTrigger(workflowId, supervised);
10390
+ await this.registerPieceTrigger(workflowId, supervised, version);
9147
10391
  return;
9148
10392
  }
9149
10393
  const had = this.registry.get(workflowId);
@@ -9159,12 +10403,12 @@ var WorkflowRuntimeService = class {
9159
10403
  filter: parseLifecycleFilter(config)
9160
10404
  });
9161
10405
  }
9162
- enableSupervised(workflowId, binding) {
9163
- this.supervisor().upsert(binding).catch((error) => {
9164
- this.logger.error(`Trigger enable failed for ${workflowId}`, error);
10406
+ enableSupervised(workflowId, binding, publishedVersion) {
10407
+ this.supervisor().upsert(binding, publishedVersion).catch((error) => {
10408
+ this.laneFailed(`Trigger enable failed for ${workflowId}`, error);
9165
10409
  });
9166
10410
  }
9167
- async registerPieceTrigger(workflowId, binding) {
10411
+ async registerPieceTrigger(workflowId, binding, publishedVersion) {
9168
10412
  const delivery = await this.pieceDelivery(binding);
9169
10413
  if (typeof delivery !== "string") {
9170
10414
  this.refusePieceTrigger(workflowId, binding, delivery);
@@ -9182,7 +10426,7 @@ var WorkflowRuntimeService = class {
9182
10426
  });
9183
10427
  await (await this.endpoints())?.endpointFor(workflowId);
9184
10428
  }
9185
- this.enableSupervised(workflowId, resolved);
10429
+ this.enableSupervised(workflowId, resolved, publishedVersion);
9186
10430
  }
9187
10431
  async pieceDelivery(binding) {
9188
10432
  let strategy;
@@ -9218,7 +10462,7 @@ var WorkflowRuntimeService = class {
9218
10462
  const reason = `The trigger ${blockLabel(binding.block)} is not armed: ${refusal.reason}`;
9219
10463
  this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
9220
10464
  this.supervisor().reject(workflowId, binding.block, binding.config, reason, retryAt).catch((error) => {
9221
- this.logger.error(`Could not record the refused trigger for workflow ${workflowId}`, error);
10465
+ this.laneFailed(`Could not record the refused trigger for workflow ${workflowId}`, error);
9222
10466
  });
9223
10467
  }
9224
10468
  async registerWebhook(workflowId, block, rawConfig) {
@@ -9233,7 +10477,7 @@ var WorkflowRuntimeService = class {
9233
10477
  this.logger.error(`Webhook trigger rejected for ${workflowId}: @error`, message);
9234
10478
  this.unarmed.add(workflowId);
9235
10479
  this.supervisor().reject(workflowId, block, configRecord(rawConfig), message).catch((recordError) => {
9236
- this.logger.error(`Could not record the rejected webhook trigger for workflow ${workflowId}`, recordError);
10480
+ this.laneFailed(`Could not record the rejected webhook trigger for workflow ${workflowId}`, recordError);
9237
10481
  });
9238
10482
  return;
9239
10483
  }
@@ -9359,7 +10603,7 @@ var WorkflowRuntimeService = class {
9359
10603
  const trigger = runnableDefinition(state).trigger;
9360
10604
  if (!trigger) return;
9361
10605
  this.supervisor().reject(workflowId, triggerBlock(trigger), configRecord(trigger.config), reason).catch((error) => {
9362
- this.logger.error(`Could not record the reactor access denial for workflow ${workflowId}`, error);
10606
+ this.laneFailed(`Could not record the reactor access denial for workflow ${workflowId}`, error);
9363
10607
  });
9364
10608
  }
9365
10609
  unarmed = /* @__PURE__ */ new Set();
@@ -9373,7 +10617,7 @@ var WorkflowRuntimeService = class {
9373
10617
  this.logger.warn("Workflow @workflow: @reason", workflowId, reason);
9374
10618
  this.unarmed.add(workflowId);
9375
10619
  this.supervisor().reject(workflowId, block, configRecord(trigger.config), reason, retryAt).catch((error) => {
9376
- this.logger.error(`Could not record the unresolved trigger for workflow ${workflowId}`, error);
10620
+ this.laneFailed(`Could not record the unresolved trigger for workflow ${workflowId}`, error);
9377
10621
  });
9378
10622
  }
9379
10623
  resolutionRetries = /* @__PURE__ */ new Map();
@@ -9402,7 +10646,7 @@ var WorkflowRuntimeService = class {
9402
10646
  }
9403
10647
  dropSupervised(workflowId) {
9404
10648
  this.supervisor().remove(workflowId).catch((error) => {
9405
- this.logger.error(`Trigger disable failed for ${workflowId}`, error);
10649
+ this.laneFailed(`Trigger disable failed for ${workflowId}`, error);
9406
10650
  });
9407
10651
  }
9408
10652
  async refreshRegistration(workflowId, resultingState, onlyIfChanged = false) {
@@ -9427,6 +10671,7 @@ var WorkflowRuntimeService = class {
9427
10671
  return false;
9428
10672
  }
9429
10673
  async onDocumentsPurged(markers) {
10674
+ this.throwIfClosed();
9430
10675
  const documentIds = [...new Set(markers.map((marker) => marker.context.documentId))];
9431
10676
  const store = await this.store();
9432
10677
  if (!store) {
@@ -9434,13 +10679,15 @@ var WorkflowRuntimeService = class {
9434
10679
  throw new Error("Erasing purged documents needs the run journal", { cause: this.storeError });
9435
10680
  }
9436
10681
  for (const workflowId of purgedWorkflowIds(markers)) await this.erasePurgedWorkflow(store, workflowId);
9437
- const erased = await store.eraseRunsForDocuments(documentIds);
10682
+ const erased = await this.telemetry.phase("journal.purge", { "documents.count": documentIds.length }, () => store.eraseRunsForDocuments(documentIds));
9438
10683
  if (erased.runs > 0) this.logger.info("Erased @runs workflow run(s) of purged documents @ids", erased.runs, documentIds);
9439
10684
  return erased;
9440
10685
  }
9441
10686
  async onOperations(operations) {
10687
+ this.throwIfClosed();
9442
10688
  const hints = collectLifecycleParentHints(operations);
9443
10689
  for (const { operation, context } of operations) {
10690
+ this.throwIfClosed();
9444
10691
  if (context.scope !== DOCUMENT_SCOPE && context.scope !== "global") continue;
9445
10692
  const opKey = operationKey({
9446
10693
  operation,
@@ -9490,15 +10737,16 @@ var WorkflowRuntimeService = class {
9490
10737
  this.unjournaledFires.delete(oldest);
9491
10738
  this.logger.warn("Too many unjournaled workflow fires held; a redelivery of the oldest fires again");
9492
10739
  }
9493
- this.fireFromTrigger(workflowId, payload, kind);
10740
+ this.startFiring(workflowId, payload, kind);
9494
10741
  }
9495
10742
  async enqueueFire(workflowId, payload, kind, dedupeKey) {
9496
10743
  const fireKey = JSON.stringify([workflowId, dedupeKey]);
9497
10744
  const store = await this.store();
10745
+ this.throwIfClosed();
9498
10746
  if (this.unjournaledFires.has(fireKey)) {
9499
10747
  if (!store) return;
9500
10748
  try {
9501
- await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString());
10749
+ if (!await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString())) this.logger.warn(`Workflow ${workflowId}: a ${kind} fire this process ran unjournaled had its key already marked fired; it may have run twice`);
9502
10750
  this.unjournaledFires.delete(fireKey);
9503
10751
  } catch (error) {
9504
10752
  this.logger.warn(`Could not record the unjournaled ${kind} fire for workflow ${workflowId}: @error`, error);
@@ -9509,26 +10757,62 @@ var WorkflowRuntimeService = class {
9509
10757
  this.fireUnjournaled(fireKey, workflowId, payload, kind);
9510
10758
  return;
9511
10759
  }
9512
- let runId;
10760
+ const enqueue = {
10761
+ workflowId,
10762
+ triggerKind: kind,
10763
+ triggerPayload: payload
10764
+ };
10765
+ let claim;
10766
+ try {
10767
+ claim = await store.claimAndEnqueueRun(dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString(), enqueue);
10768
+ } catch (error) {
10769
+ this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
10770
+ this.fireUnjournaled(fireKey, workflowId, payload, kind);
10771
+ return;
10772
+ }
10773
+ if (claim.outcome === "duplicate") return;
10774
+ if (claim.outcome === "abandoned") {
10775
+ this.logger.error(`Workflow ${workflowId}: a ${kind} fire has been delivered ${claim.attempts} times without ever journaling a run; abandoning it rather than replaying it on every boot`);
10776
+ try {
10777
+ await store.journalAbandonedFire(dedupeKey, {
10778
+ ...enqueue,
10779
+ attempts: claim.attempts
10780
+ });
10781
+ } catch (error) {
10782
+ this.logger.warn(`Could not journal the abandoned ${kind} fire for workflow ${workflowId}: @error`, error);
10783
+ }
10784
+ return;
10785
+ }
10786
+ this.startFiring(workflowId, payload, kind, claim.runId);
10787
+ }
10788
+ async fireFromTrigger(workflowId, payload, kind) {
10789
+ const runId = await this.journalFiring(workflowId, payload, kind);
10790
+ this.startFiring(workflowId, payload, kind, runId);
10791
+ }
10792
+ async journalFiring(workflowId, payload, kind) {
10793
+ const store = await this.store();
10794
+ if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
10795
+ if (!store) return void 0;
9513
10796
  try {
9514
- runId = await store.claimAndEnqueueRun(dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString(), {
10797
+ return await store.enqueueRun({
9515
10798
  workflowId,
9516
10799
  triggerKind: kind,
9517
10800
  triggerPayload: payload
9518
10801
  });
9519
10802
  } catch (error) {
9520
10803
  this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
9521
- this.fireUnjournaled(fireKey, workflowId, payload, kind);
9522
10804
  return;
9523
10805
  }
9524
- if (runId === null) return;
9525
- this.fireFromTrigger(workflowId, payload, kind, runId);
9526
10806
  }
9527
- fireFromTrigger(workflowId, payload, kind, enqueuedRunId) {
10807
+ startFiring(workflowId, payload, kind, enqueuedRunId) {
9528
10808
  this.fire(workflowId, payload, kind, void 0, void 0, enqueuedRunId).then((run) => {
9529
10809
  this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
9530
10810
  }, (error) => {
9531
- this.logger.error(`${kind} run failed for workflow ${workflowId}`, error);
10811
+ if (enqueuedRunId && isShutdownRefusal(error)) {
10812
+ this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to rerun`);
10813
+ return;
10814
+ }
10815
+ this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
9532
10816
  });
9533
10817
  }
9534
10818
  firedLifecycle = /* @__PURE__ */ new Set();
@@ -9607,8 +10891,14 @@ var WorkflowRuntimeService = class {
9607
10891
  this.cancelTriggerTest(workflowId, "stopped: the workflow was deleted");
9608
10892
  }
9609
10893
  async releaseDeleted(workflowId) {
9610
- await this.supervisor().forget(workflowId);
10894
+ let failure;
10895
+ try {
10896
+ await this.supervisor().forget(workflowId);
10897
+ } catch (error) {
10898
+ failure = { error };
10899
+ }
9611
10900
  await (await this.endpoints())?.revoke(workflowId);
10901
+ if (failure) throw failure.error;
9612
10902
  }
9613
10903
  async erasePurgedWorkflow(store, workflowId) {
9614
10904
  this.dropDeleted(workflowId);
@@ -9666,41 +10956,60 @@ var WorkflowRuntimeService = class {
9666
10956
  });
9667
10957
  }
9668
10958
  triggerSupervisor;
10959
+ parks = new ParkState(async () => (await this.store())?.listWorkflowParks());
9669
10960
  supervisor() {
9670
10961
  this.triggerSupervisor ??= new TriggerSupervisor({
9671
10962
  store: () => this.store(),
10963
+ parks: this.parks,
9672
10964
  resolveAuth: async (connectionId, request) => {
9673
10965
  if (!connectionId) return void 0;
9674
10966
  const resolved = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolveWithSecrets(connectionId, request);
9675
10967
  return rememberSecrets(resolved.auth, resolved.secretValues);
9676
10968
  },
9677
- fire: (workflowId, payload, kind) => {
9678
- this.fireFromTrigger(workflowId, payload, kind);
10969
+ fire: (workflowId, payload, kind, runId) => {
10970
+ this.startFiring(workflowId, payload, kind, runId);
9679
10971
  },
9680
10972
  webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
9681
10973
  reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
9682
10974
  models: this.models,
10975
+ telemetry: this.telemetry,
9683
10976
  cacheDir: bundleCacheDir(),
9684
10977
  resolver: pieceResolver(),
9685
10978
  egress: configuredEgress(),
9686
10979
  defaultIntervalMs: Number(process.env.PH_WORKFLOWS_POLL_INTERVAL_MS) || void 0,
9687
10980
  reconcileIntervalMs: Number(process.env.PH_WORKFLOWS_WEBHOOK_RECONCILE_MS) || void 0
9688
10981
  });
10982
+ if (this.closed) this.triggerSupervisor.stop();
9689
10983
  return this.triggerSupervisor;
9690
10984
  }
10985
+ laneFailed(message, error) {
10986
+ if (isShutdownRefusal(error)) {
10987
+ this.logger.debug(message, error);
10988
+ return;
10989
+ }
10990
+ this.logger.error(message, error);
10991
+ }
9691
10992
  startTriggerSupervisor() {
9692
10993
  this.supervisor().start();
9693
10994
  }
9694
10995
  stopTriggerSupervisor() {
9695
10996
  this.triggerSupervisor?.stop();
9696
10997
  }
10998
+ closed = false;
10999
+ throwIfClosed() {
11000
+ if (this.closed) throw new Error("This workflow runtime has shut down; the operations are left for the next owner");
11001
+ }
9697
11002
  shutdown() {
11003
+ this.closed = true;
11004
+ this.runGate.close();
9698
11005
  clearInterval(this.retentionTimer);
9699
11006
  for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
9700
11007
  this.resolutionRetries.clear();
9701
11008
  for (const workflowId of [...this.webhookTests.keys()]) this.cancelTriggerTest(workflowId, "stopped: the runtime shut down");
9702
11009
  this.stopTriggerSupervisor();
9703
11010
  this.pieceWorkers?.dispose();
11011
+ this.unobservePool?.();
11012
+ this.unobservePool = void 0;
9704
11013
  this.designWorker?.dispose();
9705
11014
  this.designWorker = void 0;
9706
11015
  }
@@ -9802,6 +11111,10 @@ var WorkflowRuntimeService = class {
9802
11111
  /** A delivery the service has already rate-limited, verified, de-duplicated and
9803
11112
  * answered any challenge for; all that is left is deciding what it means. */
9804
11113
  async deliverWebhook(request) {
11114
+ if (this.closed) return {
11115
+ status: 503,
11116
+ unprocessed: true
11117
+ };
9805
11118
  const workflowId = request.key;
9806
11119
  const registration = this.liveWebhook(workflowId);
9807
11120
  const test = this.webhookTests.get(workflowId);
@@ -9816,16 +11129,44 @@ var WorkflowRuntimeService = class {
9816
11129
  const { config } = registration;
9817
11130
  const payload = webhookPayload(request);
9818
11131
  if (config.responseMode === "async") {
9819
- this.fireFromTrigger(workflowId, payload, WEBHOOK_TRIGGER_KIND);
11132
+ try {
11133
+ await this.fireFromTrigger(workflowId, payload, WEBHOOK_TRIGGER_KIND);
11134
+ } catch (error) {
11135
+ if (error instanceof WorkflowRuntimeClosedError) return {
11136
+ status: 503,
11137
+ unprocessed: true
11138
+ };
11139
+ throw error;
11140
+ }
9820
11141
  return { status: config.responseStatus };
9821
11142
  }
9822
- const run = await Promise.race([this.fire(workflowId, payload, WEBHOOK_TRIGGER_KIND).then((result) => ({
11143
+ let runId;
11144
+ try {
11145
+ runId = await this.journalFiring(workflowId, payload, WEBHOOK_TRIGGER_KIND);
11146
+ } catch (error) {
11147
+ if (error instanceof WorkflowRuntimeClosedError) return {
11148
+ status: 503,
11149
+ unprocessed: true
11150
+ };
11151
+ throw error;
11152
+ }
11153
+ const firing = this.fire(workflowId, payload, WEBHOOK_TRIGGER_KIND, void 0, void 0, runId).then((result) => ({
9823
11154
  ok: true,
9824
11155
  result
9825
11156
  }), (error) => ({
9826
11157
  ok: false,
9827
11158
  error
9828
- })), timeout(DELIVERY_TIMEOUT_MS)]);
11159
+ }));
11160
+ let replied = false;
11161
+ const takeReply = () => !replied && (replied = true);
11162
+ if (runId) this.heldReplies.set(runId, takeReply);
11163
+ let run;
11164
+ try {
11165
+ run = await Promise.race([firing, timeout(DELIVERY_TIMEOUT_MS)]);
11166
+ if (run === TIMED_OUT && !takeReply()) run = await firing;
11167
+ } finally {
11168
+ if (runId) this.heldReplies.delete(runId);
11169
+ }
9829
11170
  if (run === TIMED_OUT) {
9830
11171
  this.logger.warn(`Webhook run for ${workflowId} exceeded ${DELIVERY_TIMEOUT_MS}ms; answering 504 while it continues`);
9831
11172
  return {
@@ -9838,6 +11179,10 @@ var WorkflowRuntimeService = class {
9838
11179
  };
9839
11180
  }
9840
11181
  if (!run.ok) {
11182
+ if (run.error instanceof WorkflowRuntimeClosedError) return {
11183
+ status: 503,
11184
+ unprocessed: true
11185
+ };
9841
11186
  const message = run.error instanceof Error ? run.error.message : String(run.error);
9842
11187
  this.logger.error(`Webhook run failed for ${workflowId}: @error`, message);
9843
11188
  return {
@@ -9847,7 +11192,7 @@ var WorkflowRuntimeService = class {
9847
11192
  };
9848
11193
  }
9849
11194
  return {
9850
- status: run.result.status === "SUCCEEDED" ? config.responseStatus : 500,
11195
+ status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
9851
11196
  contentType: JSON_CONTENT_TYPE,
9852
11197
  body: JSON.stringify({
9853
11198
  runId: run.result.runId,
@@ -9862,7 +11207,7 @@ var WorkflowRuntimeService = class {
9862
11207
  try {
9863
11208
  return handshakeReply((await this.supervisor().handshake(binding, webhookPayload(request))).output);
9864
11209
  } catch (error) {
9865
- this.logger.error("Handshake failed for @block on workflow @workflow", blockLabel(binding.block), binding.workflowId, error);
11210
+ this.logger[isShutdownRefusal(error) ? "debug" : "error"]("Handshake failed for @block on workflow @workflow", blockLabel(binding.block), binding.workflowId, error);
9866
11211
  return { status: 500 };
9867
11212
  }
9868
11213
  }
@@ -9881,7 +11226,7 @@ var WorkflowRuntimeService = class {
9881
11226
  this.supervisor().deliverWebhook(binding.workflowId, payload).then(() => {
9882
11227
  this.logger.info("Webhook delivered to @block for workflow @workflow", blockLabel(binding.block), binding.workflowId);
9883
11228
  }, (error) => {
9884
- this.logger.error(`Webhook delivery failed for workflow ${binding.workflowId}`, error);
11229
+ this.laneFailed(`Webhook delivery failed for workflow ${binding.workflowId}`, error);
9885
11230
  });
9886
11231
  return { status: 200 };
9887
11232
  }
@@ -9933,14 +11278,14 @@ var WorkflowRuntimeService = class {
9933
11278
  async stepReactorAccess(request) {
9934
11279
  const declared = await this.declaredReactor(request.block);
9935
11280
  if (!declared) return void 0;
9936
- const base = await this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null);
9937
- return reactorTap(this.host, base, runJournal());
11281
+ const base = await this.telemetry.phase("reactor.scope", {}, () => this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null));
11282
+ return reactorTap(this.host, base, runJournal(), {}, this.telemetry);
9938
11283
  }
9939
11284
  async triggerReactorAccess(binding, requireReactor, runUser) {
9940
11285
  const bound = binding.reactorConnectionId;
9941
11286
  const user = runUser !== void 0 ? runUser : await assertReactorConnectionsReadable(this.host, binding.workflowId, new Set(bound ? [bound] : [])) ?? null;
9942
11287
  const base = await this.reactorScope(bound, requireReactor, user);
9943
- return reactorTap(this.host, base, NO_JOURNAL);
11288
+ return reactorTap(this.host, base, NO_JOURNAL, {}, this.telemetry);
9944
11289
  }
9945
11290
  async designReactorAccess(block, ctx, reactorConnectionId) {
9946
11291
  if (!await this.declaredReactor(block)) return void 0;
@@ -9951,6 +11296,12 @@ var WorkflowRuntimeService = class {
9951
11296
  }
9952
11297
  descriptors = /* @__PURE__ */ new Map();
9953
11298
  designWorker;
11299
+ createDesignWorker() {
11300
+ return new PieceWorker({
11301
+ models: this.models,
11302
+ telemetry: this.telemetry
11303
+ });
11304
+ }
9954
11305
  designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
9955
11306
  describing = /* @__PURE__ */ new Map();
9956
11307
  pieceDescriptor(target) {
@@ -9971,7 +11322,7 @@ var WorkflowRuntimeService = class {
9971
11322
  let descriptor = this.descriptors.get(cacheKey);
9972
11323
  if (!descriptor) {
9973
11324
  const piece = await pieceResolver().resolve(target);
9974
- this.designWorker ??= new PieceWorker({ models: this.models });
11325
+ this.designWorker ??= this.createDesignWorker();
9975
11326
  let output;
9976
11327
  try {
9977
11328
  output = (await this.designWorker.describePiece({
@@ -10002,7 +11353,14 @@ var WorkflowRuntimeService = class {
10002
11353
  async runs(args, ctx) {
10003
11354
  return (await this.runsPage(args, ctx)).records;
10004
11355
  }
10005
- async runsPage(args, ctx) {
11356
+ runsPage(args, ctx) {
11357
+ return this.telemetry.phase("runs.page", { "runs.scope": args.workflowId ? "workflow" : args.driveId ? "drive" : "all" }, async (span) => {
11358
+ const page = await this.listRunsPage(args, ctx);
11359
+ span.setAttribute("runs.returned", page.records.length);
11360
+ return page;
11361
+ });
11362
+ }
11363
+ async listRunsPage(args, ctx) {
10006
11364
  const empty = {
10007
11365
  records: [],
10008
11366
  hasNextPage: false,
@@ -10158,7 +11516,7 @@ var WorkflowRuntimeService = class {
10158
11516
  }
10159
11517
  let outcome;
10160
11518
  try {
10161
- this.designWorker ??= new PieceWorker({ models: this.models });
11519
+ this.designWorker ??= this.createDesignWorker();
10162
11520
  outcome = (await this.designWorker.checkConnection({
10163
11521
  ...moduleRef,
10164
11522
  auth: shapedAuth,
@@ -10335,14 +11693,17 @@ var WorkflowRuntimeService = class {
10335
11693
  const local = await this.localPieceAt(packageName, version);
10336
11694
  return local ? triggersResult(local.descriptor, local.piece.name, local.piece.version) : fetchPieceTriggers(packageName, version);
10337
11695
  }
10338
- async searchBlocks(query, limit) {
11696
+ warmPieceSearch() {
11697
+ warmSearchIndex();
11698
+ }
11699
+ async searchPieces(query, filter) {
10339
11700
  let local;
10340
11701
  try {
10341
- local = indexFromHits((await this.localPieces()).flatMap(({ piece, descriptor }) => localSearchHits(descriptor, piece.name, piece.version)));
11702
+ local = indexPieces((await this.localPieces()).map(({ piece, descriptor }) => localSearchPiece(descriptor, piece.name, piece.version)));
10342
11703
  } catch (error) {
10343
11704
  this.logger.warn("Could not index the package pieces: @error", String(error));
10344
11705
  }
10345
- return searchBlocks(query, limit, local);
11706
+ return searchPieces(query, filter, local);
10346
11707
  }
10347
11708
  async pieceDetail(packageName, version) {
10348
11709
  const local = await this.localPieceAt(packageName, version);
@@ -10447,7 +11808,7 @@ var WorkflowRuntimeService = class {
10447
11808
  if (connectionId) auth = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolve(connectionId, { piecePackage: parsed.packageName });
10448
11809
  const piece = await pieceResolver().resolve(targetOf(parsed));
10449
11810
  const reactor = await this.designReactorAccess(parsed, ctx, reactorConnectionId);
10450
- this.designWorker ??= new PieceWorker({ models: this.models });
11811
+ this.designWorker ??= this.createDesignWorker();
10451
11812
  return (await this.designWorker.resolveOptions({
10452
11813
  ...pieceModuleRef(piece),
10453
11814
  actionName: parsed.name,
@@ -10618,7 +11979,7 @@ var WorkflowRuntimeService = class {
10618
11979
  } catch (error) {
10619
11980
  const errorName = errorNameOf(error);
10620
11981
  await recordTest({
10621
- status: "FAILED",
11982
+ status: isIndeterminateError(error) ? "INDETERMINATE" : "FAILED",
10622
11983
  error: pieceFailureDetail(error, "Trigger test timed out"),
10623
11984
  ...errorName ? { errorName } : {}
10624
11985
  });
@@ -10709,10 +12070,11 @@ var WorkflowRuntimeService = class {
10709
12070
  triggerKind: TEST_TRIGGER_KIND
10710
12071
  });
10711
12072
  await store.recordStep(runId, 0, record);
12073
+ const error = record.status === "INDETERMINATE" ? `This test is INDETERMINATE: ${record.error ?? "a host call it made timed out"}` : record.error;
10712
12074
  await store.finishRun(runId, {
10713
- status: record.status === "FAILED" ? "FAILED" : "SUCCEEDED",
12075
+ status: record.status === "SUCCEEDED" ? "SUCCEEDED" : "FAILED",
10714
12076
  steps: [record],
10715
- ...record.error ? { error: record.error } : {},
12077
+ ...error ? { error } : {},
10716
12078
  ...record.errorName ? { errorName: record.errorName } : {}
10717
12079
  });
10718
12080
  } catch (error) {
@@ -10733,30 +12095,77 @@ var WorkflowRuntimeService = class {
10733
12095
  }
10734
12096
  }
10735
12097
  blockExecutor(store) {
10736
- return this.executor ??= createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request));
12098
+ return this.executor ??= tracedSteps(createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request), this.telemetry), this.telemetry);
10737
12099
  }
10738
12100
  workers() {
10739
- return this.pieceWorkers ??= new PieceWorkerPool({
12101
+ if (this.pieceWorkers) return this.pieceWorkers;
12102
+ const pool = new PieceWorkerPool({
10740
12103
  models: this.models,
10741
12104
  size: Number(process.env.PH_WORKFLOWS_RUN_CONCURRENCY) || void 0,
10742
- maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0
12105
+ maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0,
12106
+ telemetry: this.telemetry
10743
12107
  });
12108
+ this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
12109
+ return this.pieceWorkers = pool;
10744
12110
  }
10745
12111
  async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
12112
+ if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
12113
+ const started = performance.now();
12114
+ const attributes = {
12115
+ "workflow.id": workflowId,
12116
+ "trigger.kind": triggerKind,
12117
+ ...resume ? { "run.rerun_of": resume.rerunOf } : {}
12118
+ };
12119
+ return this.telemetry.span("workflow.run", attributes, async (span) => {
12120
+ const run = {
12121
+ span,
12122
+ refused: false
12123
+ };
12124
+ let status = "FAILED";
12125
+ try {
12126
+ const result = await this.runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId);
12127
+ status = result.status;
12128
+ if (status === "FAILED") span.setStatus({
12129
+ code: SpanStatusCode.ERROR,
12130
+ message: status
12131
+ });
12132
+ return result;
12133
+ } finally {
12134
+ if (run.refused) status = "REFUSED";
12135
+ span.setAttribute("run.status", status);
12136
+ this.telemetry.recordRun(performance.now() - started, status, triggerKind);
12137
+ }
12138
+ });
12139
+ }
12140
+ async runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId) {
12141
+ const firedAt = Date.now();
10746
12142
  const store = await this.store();
10747
12143
  let state;
10748
12144
  let documentName;
10749
12145
  let definition;
10750
12146
  let runUser;
10751
12147
  try {
10752
- if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx);
10753
- const document = await this.host.reactorClient.get(workflowId);
10754
- if (document.header.documentType !== "powerhouse/workflow") throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
12148
+ if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
12149
+ run.refused = true;
12150
+ throw error;
12151
+ });
12152
+ const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
12153
+ if (document.header.documentType !== "powerhouse/workflow") {
12154
+ run.refused = true;
12155
+ throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
12156
+ }
10755
12157
  state = document.state.global;
10756
12158
  documentName = document.header.name;
10757
- if (state.status !== "ENABLED") throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
12159
+ if (state.status !== "ENABLED") {
12160
+ run.refused = true;
12161
+ throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
12162
+ }
10758
12163
  definition = toWorkflowDefinition(state);
10759
- runUser = await assertReactorConnectionsReadable(this.host, workflowId, declaredReactorConnectionIds(definition));
12164
+ const reactorConnections = declaredReactorConnectionIds(definition);
12165
+ runUser = reactorConnections.size === 0 ? void 0 : await this.telemetry.phase("run_user", {}, () => assertReactorConnectionsReadable(this.host, workflowId, reactorConnections)).catch((error) => {
12166
+ run.refused = isReactorError(error, ReactorAccessDeniedError$1);
12167
+ throw error;
12168
+ });
10760
12169
  } catch (error) {
10761
12170
  if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
10762
12171
  throw error;
@@ -10764,23 +12173,71 @@ var WorkflowRuntimeService = class {
10764
12173
  const connections = declaredConnectionIds(definition);
10765
12174
  const executor = this.blockExecutor(store);
10766
12175
  const runnable = runnableDefinition(state);
10767
- let runId = enqueuedRunId ?? null;
10768
- if (enqueuedRunId) await store?.beginRun(enqueuedRunId, {
10769
- workflowName: runJournalName(state.name, documentName),
10770
- workflowVersion: runnable.version
10771
- });
10772
- else runId = await store?.startRun({
10773
- workflowId,
10774
- workflowName: runJournalName(state.name, documentName),
10775
- workflowVersion: runnable.version,
12176
+ const policy = effectiveRunPolicy(runnable);
12177
+ const deadline = policy.runTimeoutSeconds ? firedAt + policy.runTimeoutSeconds * 1e3 : void 0;
12178
+ const skipped = (reason, refusal) => this.skipFiring(store, workflowId, enqueuedRunId, {
12179
+ reason,
10776
12180
  triggerKind,
10777
12181
  triggerPayload,
10778
- rerunOf: resume?.rerunOf
10779
- }) ?? null;
12182
+ workflowName: runJournalName(state.name, documentName),
12183
+ workflowVersion: runnable.version
12184
+ }, refusal);
12185
+ let parked;
12186
+ try {
12187
+ parked = await this.parkedFiring(workflowId, triggerKind, runnable.version);
12188
+ } catch (error) {
12189
+ if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
12190
+ throw error;
12191
+ }
12192
+ if (parked) return skipped(parked, "parked");
12193
+ const admission = await this.runGate.admit(workflowId, policy);
12194
+ if (this.closed) {
12195
+ if (admission.admitted) admission.release();
12196
+ return this.refuseClosed(store, workflowId, enqueuedRunId);
12197
+ }
12198
+ if (!admission.admitted) {
12199
+ if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
12200
+ return skipped(admission.reason, admission.refusal);
12201
+ }
12202
+ if (admission.waited) {
12203
+ let stale;
12204
+ try {
12205
+ stale = await this.staleAfterWait(workflowId, triggerKind, runnable.version);
12206
+ } catch (error) {
12207
+ stale = `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
12208
+ }
12209
+ if (stale) {
12210
+ admission.release();
12211
+ return skipped(stale, "stale");
12212
+ }
12213
+ }
12214
+ if (deadline !== void 0 && Date.now() >= deadline) {
12215
+ admission.release();
12216
+ return skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
12217
+ }
12218
+ let runId = enqueuedRunId ?? null;
10780
12219
  let journalFailed = false;
10781
12220
  const executionOrder = /* @__PURE__ */ new Map();
10782
12221
  let session;
10783
12222
  try {
12223
+ runId = await this.telemetry.phase("journal.start", {}, async () => {
12224
+ if (enqueuedRunId) {
12225
+ await store?.beginRun(enqueuedRunId, {
12226
+ workflowName: runJournalName(state.name, documentName),
12227
+ workflowVersion: runnable.version
12228
+ });
12229
+ return enqueuedRunId;
12230
+ }
12231
+ return await store?.startRun({
12232
+ workflowId,
12233
+ workflowName: runJournalName(state.name, documentName),
12234
+ workflowVersion: runnable.version,
12235
+ triggerKind,
12236
+ triggerPayload,
12237
+ rerunOf: resume?.rerunOf
12238
+ }) ?? null;
12239
+ });
12240
+ if (runId) run.span.setAttribute("run.id", runId);
10784
12241
  session = this.workers().session();
10785
12242
  const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
10786
12243
  const journal = store;
@@ -10792,6 +12249,7 @@ var WorkflowRuntimeService = class {
10792
12249
  connections,
10793
12250
  pieceWorker: session,
10794
12251
  ...runUser !== void 0 ? { runUser } : {},
12252
+ attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
10795
12253
  recordDocuments: async (documentIds) => {
10796
12254
  for (const documentId of documentIds) handed.add(documentId);
10797
12255
  if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
@@ -10806,22 +12264,25 @@ var WorkflowRuntimeService = class {
10806
12264
  ...secretValues.length > 0 ? { redactValues: secretValues } : {},
10807
12265
  triggerPayload,
10808
12266
  completedSteps: resume?.completedSteps,
10809
- onStep: store && runId ? async (record, ordinal) => {
12267
+ ...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
12268
+ ...deadline !== void 0 ? { deadline } : {},
12269
+ onStep: store && journaledRunId ? async (record, ordinal) => {
10810
12270
  executionOrder.set(record.stepId, ordinal);
10811
12271
  try {
10812
- await store.recordStep(runId, ordinal, record);
12272
+ await this.telemetry.phase("journal.step", {}, () => store.recordStep(journaledRunId, ordinal, record));
10813
12273
  } catch (error) {
10814
12274
  if (journalFailed) return;
10815
12275
  journalFailed = true;
10816
- this.logger.warn(`Run ${runId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
12276
+ this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
10817
12277
  }
10818
12278
  } : void 0
10819
12279
  }));
10820
- if (store && runId) try {
10821
- await store.finishRun(runId, result, executionOrder);
12280
+ if (store && journaledRunId) try {
12281
+ await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
10822
12282
  } catch (error) {
10823
12283
  this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
10824
12284
  }
12285
+ if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
10825
12286
  const finished = {
10826
12287
  ...result,
10827
12288
  runId
@@ -10838,6 +12299,123 @@ var WorkflowRuntimeService = class {
10838
12299
  throw error;
10839
12300
  } finally {
10840
12301
  session?.close();
12302
+ admission.release();
12303
+ }
12304
+ }
12305
+ runGate = new WorkflowRunGate();
12306
+ heldReplies = /* @__PURE__ */ new Map();
12307
+ /** Why a trigger's firing of a PARKED workflow is refused; undefined when
12308
+ * it is not parked, or when an operator started the run. */
12309
+ async parkedFiring(workflowId, triggerKind, publishedVersion) {
12310
+ if (OPERATOR_RUN_KINDS.has(triggerKind)) return void 0;
12311
+ const park = await this.parks.get(workflowId);
12312
+ if (!park || park.published_version < publishedVersion) return void 0;
12313
+ return `Skipped: this workflow is PARKED (${park.reason})`;
12314
+ }
12315
+ /** Why a firing that waited for its slot may no longer run, if it may not. */
12316
+ async staleAfterWait(workflowId, triggerKind, publishedVersion) {
12317
+ const state = (await this.host.reactorClient.get(workflowId)).state.global;
12318
+ if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
12319
+ if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
12320
+ return this.parkedFiring(workflowId, triggerKind, publishedVersion);
12321
+ }
12322
+ /** Whether the workflow is still ENABLED at this published version. An
12323
+ * unreadable document counts as unchanged, so a failure still parks. */
12324
+ async stillRunsVersion(workflowId, publishedVersion) {
12325
+ let state;
12326
+ try {
12327
+ state = (await this.host.reactorClient.get(workflowId)).state.global;
12328
+ } catch (error) {
12329
+ this.logger.warn(`Could not re-read workflow ${workflowId} before parking it: @error`, error);
12330
+ return true;
12331
+ }
12332
+ return state.status === "ENABLED" && runnableDefinition(state).version === publishedVersion;
12333
+ }
12334
+ /** A park recorded against an earlier published version than this state's. */
12335
+ async outdatedPark(workflowId, state) {
12336
+ if (state.status !== "ENABLED") return false;
12337
+ const park = await this.parks.get(workflowId);
12338
+ return park !== void 0 && runnableDefinition(state).version > park.published_version;
12339
+ }
12340
+ /**
12341
+ * A firing SINGLETON refused.
12342
+ *
12343
+ * Journaled as a CANCELLED run rather than dropped: a firing that vanished
12344
+ * is indistinguishable from a trigger that never fired, which is the class
12345
+ * of bug this work package exists to stamp out. An already-enqueued row is
12346
+ * closed out in place, so nothing is left PENDING for a sweep to find.
12347
+ */
12348
+ async refuseClosed(store, workflowId, enqueuedRunId) {
12349
+ const error = new WorkflowRuntimeClosedError(workflowId);
12350
+ if (enqueuedRunId) {
12351
+ const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
12352
+ try {
12353
+ if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
12354
+ else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));
12355
+ } catch (failure) {
12356
+ this.logger.warn(`Could not close out run ${enqueuedRunId} after shutdown: @error`, failure);
12357
+ }
12358
+ }
12359
+ throw error;
12360
+ }
12361
+ async skipFiring(store, workflowId, enqueuedRunId, details, refusal) {
12362
+ this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
12363
+ let runId = enqueuedRunId ?? null;
12364
+ if (store) try {
12365
+ if (runId) await store.beginRun(runId, details);
12366
+ else runId = await store.startRun({
12367
+ workflowId,
12368
+ ...details
12369
+ });
12370
+ await store.cancelRun(runId, details.reason);
12371
+ } catch (error) {
12372
+ this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
12373
+ }
12374
+ return {
12375
+ status: CANCELLED_RUN_STATUS,
12376
+ steps: [],
12377
+ runId,
12378
+ refusal
12379
+ };
12380
+ }
12381
+ /**
12382
+ * `policy.onFailure` for a run that failed terminally.
12383
+ *
12384
+ * PARK is the document model's own default, so this is where enforcing the
12385
+ * knob becomes visible: a terminal failure parks the workflow, whatever its
12386
+ * trigger kind, and nothing fires it until it is re-published or
12387
+ * re-enabled. That is what PARK means, and
12388
+ * leaving a broken workflow firing every minute is what it meant before.
12389
+ *
12390
+ * NOTIFY logs at error level, which is the only notification channel this
12391
+ * engine has; it is marked as such rather than pretending to page anyone.
12392
+ */
12393
+ async applyFailureMode(policy, workflowId, publishedVersion, runId, error) {
12394
+ if (policy.onFailure === "IGNORE" || this.closed) return;
12395
+ const detail = error ?? "the run failed";
12396
+ if (policy.onFailure === "NOTIFY") {
12397
+ this.logger.error(`Workflow ${workflowId} run ${runId ?? "(unjournaled)"} failed and its policy is NOTIFY: @error`, detail);
12398
+ return;
12399
+ }
12400
+ if (!await this.store()) return;
12401
+ const reason = `Parked after a failed run (policy.onFailure = PARK): ${detail}`;
12402
+ try {
12403
+ if (!await this.stillRunsVersion(workflowId, publishedVersion)) return;
12404
+ if (this.closed) return;
12405
+ const trigger = await this.supervisor().park(workflowId, publishedVersion, reason);
12406
+ if (!await this.stillRunsVersion(workflowId, publishedVersion)) {
12407
+ await this.supervisor().liftPark(workflowId, publishedVersion, trigger);
12408
+ return;
12409
+ }
12410
+ if (!await this.inRegistrationOrder(workflowId, async () => {
12411
+ const current = (await this.parks.get(workflowId))?.published_version === publishedVersion;
12412
+ const registered = this.registry.get(workflowId);
12413
+ if (current && registered && !SUPERVISED_KINDS.has(registered.kind)) this.registry.delete(workflowId);
12414
+ return current;
12415
+ })) return;
12416
+ this.logger.error(`Workflow ${workflowId} is PARKED after run ${runId ?? "(unjournaled)"} failed; its trigger will not fire again until the workflow is re-published or re-enabled`);
12417
+ } catch (parkError) {
12418
+ this.logger.warn(`Applying onFailure = PARK to workflow ${workflowId} after a failed run failed; whether it is parked is unknown: @error`, parkError);
10841
12419
  }
10842
12420
  }
10843
12421
  async rerun(runId, ctx) {
@@ -10848,7 +12426,8 @@ var WorkflowRuntimeService = class {
10848
12426
  await this.assertCanReadDocument(run.workflow_id, ctx);
10849
12427
  if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
10850
12428
  if (run.trigger_kind === "test") throw new Error(`Run "${runId}" tested a single step or trigger; test it again instead of rerunning it`);
10851
- if (run.status !== "FAILED") throw new Error(`Only FAILED runs can be rerun; run is ${run.status}`);
12429
+ const deadlineCancelled = run.status === "CANCELLED" && run.error_name === "RunDeadlineExceeded";
12430
+ if (run.status !== "FAILED" && !deadlineCancelled) throw new Error(`Only FAILED runs, and runs their deadline cancelled, can be rerun; run is ${run.status}`);
10852
12431
  const triggerPayload = run.trigger_payload === null ? void 0 : JSON.parse(run.trigger_payload);
10853
12432
  if (isTruncatedStepPayload(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was truncated by the journal and cannot be replayed; fire the workflow again instead of rerunning it`);
10854
12433
  if (containsRedactedMarker(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was redacted and cannot be replayed; fire the workflow again instead of rerunning it`);
@@ -10860,7 +12439,15 @@ var WorkflowRuntimeService = class {
10860
12439
  const current = currentSteps.get(row.step_id);
10861
12440
  if (!current || current.key !== row.step_key || row.config_hash === null || stepConfigHash(current) !== row.config_hash) continue;
10862
12441
  const output = row.output === null ? void 0 : JSON.parse(row.output);
10863
- if (isTruncatedStepPayload(output)) continue;
12442
+ if (isTruncatedStepPayload(output)) {
12443
+ if (await this.rereads(current)) continue;
12444
+ completedSteps.set(row.step_id, {
12445
+ port: row.port,
12446
+ outputTruncated: true,
12447
+ truncatedOutput: output
12448
+ });
12449
+ continue;
12450
+ }
10864
12451
  if (containsDocumentRef(output) && await this.rereads(current)) continue;
10865
12452
  completedSteps.set(row.step_id, {
10866
12453
  output,
@@ -10947,6 +12534,7 @@ var WorkflowRuntimeService = class {
10947
12534
  pieceWorker: session,
10948
12535
  stepTest: true,
10949
12536
  runUser: this.callerRunUser(ctx),
12537
+ attachments: attachmentOrigins(upstream.triggerPayload, handed, upstream.priorSteps),
10950
12538
  recordDocuments: async (documentIds) => {
10951
12539
  for (const documentId of documentIds) handed.add(documentId);
10952
12540
  if (store && journaledRunId) await store.recordRunDocuments(journaledRunId, documentIds);
@@ -10986,7 +12574,7 @@ var WorkflowRuntimeService = class {
10986
12574
  const served = ctx ? await this.servesDocuments(ids, ctx) : false;
10987
12575
  return {
10988
12576
  runId,
10989
- status: record.status === "FAILED" ? "FAILED" : "SUCCEEDED",
12577
+ status: testStatusOf(record.status),
10990
12578
  ...served && record.output !== void 0 ? { output: record.output } : {},
10991
12579
  ...record.error ? { error: record.error } : {},
10992
12580
  ...record.errorName ? { errorName: record.errorName } : {},
@@ -11114,6 +12702,7 @@ var WorkflowRuntimeService = class {
11114
12702
  testedAt,
11115
12703
  error: row.error ?? ""
11116
12704
  };
12705
+ if (row.status === "INDETERMINATE") return { kind: "indeterminate" };
11117
12706
  const output = row.output === null ? null : JSON.parse(row.output);
11118
12707
  if (isTruncatedStepPayload(output)) return { kind: "truncated" };
11119
12708
  return {
@@ -11182,6 +12771,264 @@ var WorkflowRuntimeService = class {
11182
12771
  function createWorkflowRuntime(deps) {
11183
12772
  return new WorkflowRuntimeService(deps);
11184
12773
  }
12774
+ function attachmentOrigins(triggerPayload, handed, replayed) {
12775
+ const written = /* @__PURE__ */ new Set();
12776
+ collectAttachmentRefs(replayed, written);
12777
+ return {
12778
+ documentIds: () => [...triggerDocumentIds(triggerPayload), ...handed],
12779
+ written
12780
+ };
12781
+ }
12782
+ function collectAttachmentRefs(value, found) {
12783
+ if (typeof value === "string") {
12784
+ if (value.startsWith("attachment://")) found.add(value);
12785
+ } else if (Array.isArray(value)) for (const entry of value) collectAttachmentRefs(entry, found);
12786
+ else if (value instanceof Map) for (const entry of value.values()) collectAttachmentRefs(entry, found);
12787
+ else if (typeof value === "object" && value !== null) for (const entry of Object.values(value)) collectAttachmentRefs(entry, found);
12788
+ }
12789
+ //#endregion
12790
+ //#region src/reactor/singleton-lease.ts
12791
+ const logger = childLogger([
12792
+ "workflow",
12793
+ "runtime",
12794
+ "singleton"
12795
+ ]);
12796
+ /**
12797
+ * Names this process as the workflow singleton's owner.
12798
+ *
12799
+ * Set it to a STABLE name per deployment slot (`switchboard-0`, the
12800
+ * StatefulSet pod name, the Render service id). The owner name is the
12801
+ * operator's contract: a restart under the same name re-claims its own lease
12802
+ * once the old heartbeat is stale instead of waiting out
12803
+ * {@link SINGLETON_LEASE_TTL_MS}.
12804
+ *
12805
+ * Unset, the owner is derived from a STABLE identity — the hostname and a
12806
+ * fingerprint of the journal's own storage location ({@link
12807
+ * AcquireSingletonOptions.storageId}) — not from the pid and a random suffix.
12808
+ * A random per-process owner meant every unclean kill locked the next boot out
12809
+ * for the whole {@link SINGLETON_LEASE_TTL_MS}: the dead process's lease was
12810
+ * nobody's to re-claim, and `release()` never ran. One deployment slot
12811
+ * restarting is the common case and it has to be quick; a genuine second
12812
+ * replica still differs, by hostname or by the journal it points at.
12813
+ */
12814
+ const WORKFLOW_SINGLETON_OWNER_ENV = "PH_WORKFLOWS_SINGLETON_OWNER";
12815
+ /** How long a claim stays valid without a heartbeat. */
12816
+ const SINGLETON_LEASE_TTL_MS = 6e4;
12817
+ /** How often the holder renews it; a third of the TTL, so two renewals may be
12818
+ * lost (a GC pause, a slow database) before anyone may take over. */
12819
+ const SINGLETON_HEARTBEAT_MS = SINGLETON_LEASE_TTL_MS / 3;
12820
+ /**
12821
+ * How long the holder's heartbeat must have been silent before a claim under
12822
+ * the SAME owner name takes the lease over, short of its expiry. Two renewal
12823
+ * periods: a live holder renews well inside it, and its self-fence fires
12824
+ * before it passes.
12825
+ */
12826
+ const SINGLETON_STALE_HEARTBEATS = 2;
12827
+ /** The one row. The table holds a single lease, named rather than keyed by a
12828
+ * magic empty string, so a `SELECT *` reads legibly in an operator's shell. */
12829
+ const LEASE_ID = "workflow-runtime";
12830
+ const dbNow = () => sql`now()`;
12831
+ const dbNowPlus = (ms) => sql`now() + ${ms} * interval '1 millisecond'`;
12832
+ /** Another live process holds the singleton. The refusal is the point: a host
12833
+ * that cannot own workflow execution must not compose the engine. */
12834
+ var WorkflowSingletonConflictError = class extends Error {
12835
+ owner;
12836
+ expiresAt;
12837
+ wouldBe;
12838
+ constructor(owner, expiresAt, wouldBe) {
12839
+ super(`Workflow execution is a singleton and "${owner}" holds it until ${expiresAt}; this process ("${wouldBe}") must not compose the workflow runtime. Two replicas over one run journal fail each other's live runs and arm every trigger twice. Run workflows on exactly one reactor, or set ${WORKFLOW_SINGLETON_OWNER_ENV} to the same stable owner name on the slot that owns them so a restart re-claims its own lease.`);
12840
+ this.owner = owner;
12841
+ this.expiresAt = expiresAt;
12842
+ this.wouldBe = wouldBe;
12843
+ this.name = "WorkflowSingletonConflictError";
12844
+ }
12845
+ };
12846
+ /**
12847
+ * The owner name this process claims under.
12848
+ *
12849
+ * `<hostname>/<storage fingerprint>`, so it is the same name on every boot of
12850
+ * one deployment slot — a restart re-claims its OWN lease once the dead
12851
+ * process's heartbeat is stale, rather than waiting out the TTL — and a different
12852
+ * name on any host or journal that is genuinely somebody else.
12853
+ *
12854
+ * The storage location is hashed, never printed: it can be a Postgres URL with
12855
+ * credentials in it, and the owner name goes into a database row an operator
12856
+ * reads and into every log line about the lease.
12857
+ *
12858
+ * The residual case a stable name cannot tell apart is two processes on ONE
12859
+ * host over ONE journal, which is a misconfiguration those two already share
12860
+ * (one read-model directory, one run journal). It is not silent: the loser's
12861
+ * heartbeat finds the lease taken and says so by name.
12862
+ */
12863
+ function singletonOwnerName(env = process.env, storageId) {
12864
+ const configured = env[WORKFLOW_SINGLETON_OWNER_ENV]?.trim();
12865
+ if (configured) return configured;
12866
+ return `${hostname()}/${storageFingerprint(storageId)}`;
12867
+ }
12868
+ function storageFingerprint(storageId) {
12869
+ const source = storageId?.trim();
12870
+ if (!source) return "default";
12871
+ return createHash("sha256").update(source).digest("hex").slice(0, 12);
12872
+ }
12873
+ async function ensureTable(db) {
12874
+ await db.schema.createTable("workflow_singleton").addColumn("id", "text", (col) => col.primaryKey()).addColumn("owner", "text", (col) => col.notNull()).addColumn("instance", "text", (col) => col.notNull()).addColumn("acquired_at", "timestamptz", (col) => col.notNull()).addColumn("heartbeat_at", "timestamptz", (col) => col.notNull()).addColumn("expires_at", "timestamptz", (col) => col.notNull()).ifNotExists().execute();
12875
+ }
12876
+ const DDL_ATTEMPTS = 3;
12877
+ const CONCURRENT_DDL_CODES = /* @__PURE__ */ new Set([
12878
+ "23505",
12879
+ "42710",
12880
+ "42P06",
12881
+ "42P07"
12882
+ ]);
12883
+ function isConcurrentDdl(error) {
12884
+ const code = error?.code;
12885
+ return typeof code === "string" && CONCURRENT_DDL_CODES.has(code);
12886
+ }
12887
+ async function openLeaseTable(relationalDb) {
12888
+ const open = async () => {
12889
+ const db = await relationalDb.createNamespace("workflow_runtime");
12890
+ await ensureTable(db);
12891
+ return db;
12892
+ };
12893
+ for (let attempt = 1;; attempt++) try {
12894
+ return await open();
12895
+ } catch (error) {
12896
+ if (attempt >= DDL_ATTEMPTS || !isConcurrentDdl(error)) throw error;
12897
+ }
12898
+ }
12899
+ /**
12900
+ * Claims the workflow singleton, or refuses.
12901
+ *
12902
+ * Three steps, each one a single statement whose own outcome is the claim — a
12903
+ * prior `SELECT` would be a race:
12904
+ *
12905
+ * 1. `INSERT … ON CONFLICT DO NOTHING RETURNING` — the first ever boot.
12906
+ * 2. `UPDATE … WHERE expires_at <= now OR (owner = me AND heartbeat stale)
12907
+ * RETURNING` — a dead lease taken over once it has expired, or our own
12908
+ * slot's lease once its holder stopped renewing (a restart under a stable
12909
+ * owner name), never a live holder's. Either way the row gets this claim's
12910
+ * instance, so the previous holder's heartbeat and release no longer
12911
+ * match.
12912
+ * 3. Neither matched: someone live holds it, and we refuse by name.
12913
+ */
12914
+ async function acquireWorkflowSingletonLease(options) {
12915
+ const log = options.logger ?? logger;
12916
+ const ttlMs = options.ttlMs ?? 6e4;
12917
+ const heartbeatMs = options.heartbeatMs ?? 2e4;
12918
+ const staleMs = heartbeatMs * 2;
12919
+ const owner = options.owner ?? singletonOwnerName(options.env, options.storageId);
12920
+ const instance = randomUUID();
12921
+ const db = await openLeaseTable(options.relationalDb);
12922
+ const stamps = () => ({
12923
+ heartbeat_at: dbNow(),
12924
+ expires_at: dbNowPlus(ttlMs)
12925
+ });
12926
+ const claim = async () => {
12927
+ if (await db.insertInto("workflow_singleton").values({
12928
+ id: LEASE_ID,
12929
+ owner,
12930
+ instance,
12931
+ acquired_at: dbNow(),
12932
+ ...stamps()
12933
+ }).onConflict((oc) => oc.column("id").doNothing()).returning("owner").executeTakeFirst() !== void 0) return true;
12934
+ return await db.updateTable("workflow_singleton").set({
12935
+ owner,
12936
+ instance,
12937
+ acquired_at: dbNow(),
12938
+ ...stamps()
12939
+ }).where("id", "=", LEASE_ID).where((eb) => eb.or([eb("expires_at", "<=", dbNow()), eb.and([eb("owner", "=", owner), eb("heartbeat_at", "<=", dbNowPlus(-staleMs))])])).returning("owner").executeTakeFirst() !== void 0;
12940
+ };
12941
+ const claimSentAt = performance.now();
12942
+ if (!await claim()) {
12943
+ const held = await db.selectFrom("workflow_singleton").selectAll().where("id", "=", LEASE_ID).executeTakeFirst();
12944
+ throw new WorkflowSingletonConflictError(held?.owner ?? "unknown", held ? new Date(held.expires_at).toISOString() : "unknown", owner);
12945
+ }
12946
+ log.info(`Workflow singleton claimed by "${owner}" (lease ${ttlMs}ms, renewed every ${heartbeatMs}ms)`);
12947
+ const localNow = () => performance.now();
12948
+ const tickMs = heartbeatMs / 4;
12949
+ const fenceMs = Math.min(staleMs, ttlMs) - Math.min(heartbeatMs, ttlMs) / 2;
12950
+ const selfFence = options.selfFence ?? true;
12951
+ let renewedAt = claimSentAt;
12952
+ let renewFailed = false;
12953
+ let inFlight;
12954
+ let timer;
12955
+ let lost = false;
12956
+ let takenOver = false;
12957
+ let releasing = false;
12958
+ const stop = () => {
12959
+ if (timer) clearInterval(timer);
12960
+ timer = void 0;
12961
+ };
12962
+ const markLost = (loss) => {
12963
+ if (lost) return;
12964
+ lost = true;
12965
+ stop();
12966
+ log.error(loss.reason === "taken" ? `Workflow singleton lease of "${owner}" was taken over by "${loss.heldBy ?? "nobody"}"; this process no longer owns workflow execution and stops running it.` : `Workflow singleton lease of "${owner}" could not be renewed for ${Math.round(loss.silentMs)}ms, long enough that another process could take it over (none was seen to); this process stops running workflows.`);
12967
+ options.onLost?.(loss);
12968
+ };
12969
+ const renew = async () => {
12970
+ const sentAt = localNow();
12971
+ if (await db.updateTable("workflow_singleton").set(stamps()).where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).returning("owner").executeTakeFirst() !== void 0) {
12972
+ renewedAt = sentAt;
12973
+ return true;
12974
+ }
12975
+ if (releasing || lost) return false;
12976
+ let heldBy;
12977
+ try {
12978
+ heldBy = (await db.selectFrom("workflow_singleton").select("owner").where("id", "=", LEASE_ID).executeTakeFirst())?.owner;
12979
+ } catch {
12980
+ heldBy = void 0;
12981
+ }
12982
+ takenOver = true;
12983
+ markLost({
12984
+ reason: "taken",
12985
+ heldBy
12986
+ });
12987
+ return false;
12988
+ };
12989
+ const heartbeat = () => {
12990
+ if (lost) return Promise.resolve(false);
12991
+ inFlight ??= renew().finally(() => {
12992
+ inFlight = void 0;
12993
+ });
12994
+ return inFlight;
12995
+ };
12996
+ timer = setInterval(() => {
12997
+ if (lost || releasing) return;
12998
+ const silentMs = localNow() - renewedAt;
12999
+ if (selfFence && silentMs >= fenceMs) {
13000
+ markLost({
13001
+ reason: "unrenewable",
13002
+ silentMs
13003
+ });
13004
+ return;
13005
+ }
13006
+ if (inFlight) return;
13007
+ if (!renewFailed && silentMs < heartbeatMs - tickMs / 2) return;
13008
+ heartbeat().then(() => {
13009
+ renewFailed = false;
13010
+ }, (error) => {
13011
+ renewFailed = true;
13012
+ log.warn("Workflow singleton heartbeat failed: @error", error);
13013
+ });
13014
+ }, tickMs);
13015
+ timer.unref();
13016
+ return {
13017
+ owner,
13018
+ heartbeat,
13019
+ async release() {
13020
+ releasing = true;
13021
+ stop();
13022
+ if (takenOver) return;
13023
+ if (inFlight) await Promise.race([inFlight.catch(() => false), new Promise((resolve) => setTimeout(resolve, tickMs).unref())]);
13024
+ try {
13025
+ await db.deleteFrom("workflow_singleton").where("id", "=", LEASE_ID).where("owner", "=", owner).where("instance", "=", instance).execute();
13026
+ } catch (error) {
13027
+ log.warn("Could not release the workflow singleton lease: @error", error);
13028
+ }
13029
+ }
13030
+ };
13031
+ }
11185
13032
  //#endregion
11186
13033
  //#region src/reactor/workflow-triggers-read-model.ts
11187
13034
  const WORKFLOW_TRIGGERS_READ_MODEL = "workflow-triggers";
@@ -11209,6 +13056,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
11209
13056
  }
11210
13057
  };
11211
13058
  //#endregion
11212
- export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
13059
+ export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, SINGLETON_HEARTBEAT_MS, SINGLETON_LEASE_TTL_MS, SINGLETON_STALE_HEARTBEATS, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SINGLETON_OWNER_ENV, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSingletonConflictError, WorkflowSyncingError, WorkflowTriggersReadModel, acquireWorkflowSingletonLease, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl, singletonOwnerName };
11213
13060
 
11214
13061
  //# sourceMappingURL=index.js.map