@powerhousedao/reactor-workflow 6.2.3-dev.49 → 6.2.3-dev.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,13 +1,13 @@
1
- import { B as UnsupportedPieceFeatureError, H as unsupportedAuth, M as maxFileBytes, U as unsupportedTrigger, V as authMethodFor, a as redact, c as rememberSecrets, f as isPrivateAddress, i as containsRedactedMarker, k as FileTooLargeError, l as secretsFor, p as parseAddress, s as redactMessage, u as DEFAULT_EGRESS_POLICY, w as extractDedupeKey } from "./reactor-rpc-BqAjCK1p.js";
2
- import { A as expressionsIn, B as PieceWorkerError, C as CORE_PIECE_VERSION, D as declaredReactorConnectionIds, E as declaredConnectionIds, F as secretRefFromId, H as bundleResolver, I as stepBlock, J as pieceRegistrySource, L as triggerBlock, M as SecretDeletedError, N as SecretNotFoundError, O as shapeConnection, P as parseSecretRef, R as PieceWorkerPool, S as CORE_PIECE_NAME, U as pieceModuleRef, V as PieceWorkerTimeoutError, W as sourcedResolver, Y as setPieceRegistryUrl, _ as blockLabel, a as errorNameOf, b as builtinPiece, c as hashOf, d as checkDynamicProperties, f as ActivepiecesBlockExecutor, g as storeHandlers, h as resolvedBlock, i as packagePieces, k as resolveStepInput, l as stepConfigHash, m as isHostBound, n as installedPiece, o as runWorkflow, p as CompositeBlockExecutor, q as npmRegistryBaseUrl, r as installedPieces, s as canonicalJson, t as PieceRegistry, u as DynamicPropertiesError, v as unpinnedNote, w as BoundConnectionResolver, x as describeBuiltinPiece, y as blockPorts, z as PieceWorker } from "./piece-registry-CpiVMWNv.js";
3
- import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError as ReactorActionsFailedError$1, ReactorJobFailedError as ReactorJobFailedError$1, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
4
- import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
5
- import { open, readFile, rm } from "node:fs/promises";
6
- import { dirname, join } from "node:path";
1
+ import { $ as parseAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerTimeoutError, H as PieceWorkerPool, I as SecretDeletedError, J as redactMessage, K as containsRedactedMarker, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as isPrivateAddress, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerError, X as secretsFor, Y as rememberSecrets, Z as DEFAULT_EGRESS_POLICY, _ as resolvedBlock, at as sourcedResolver, b as resolutionOf, c as canonicalJson, ct as authMethodFor, d as DynamicPropertiesError, et as extractDedupeKey, f as checkDynamicProperties, g as isHostBound, gt as isIndeterminateError, h as CompositeBlockExecutor, ht as setPieceRegistryUrl, i as packagePieces, it as pieceModuleRef, j as declaredReactorConnectionIds, l as hashOf, lt as unsupportedAuth, m as ActivepiecesBlockExecutor, mt as pieceRegistrySource, n as installedPiece, nt as maxFileBytes, o as errorNameOf, p as effectiveRetryPolicy, pt as npmRegistryBaseUrl, q as redact, r as installedPieces, rt as bundleResolver, s as runWorkflow, st as UnsupportedPieceFeatureError, t as PieceRegistry, tt as FileTooLargeError, u as stepConfigHash, ut as unsupportedTrigger, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-BKRLK7Kv.js";
2
+ import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
3
+ import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
4
+ import { open, readdir, rm, stat } from "node:fs/promises";
5
+ import path, { dirname, join } from "node:path";
7
6
  import { childLogger, generateId } from "document-model";
8
7
  import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
9
8
  import { Cron } from "croner";
10
9
  import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
10
+ import { Readable } from "node:stream";
11
11
  import dns from "node:dns";
12
12
  import net from "node:net";
13
13
  import { AsyncLocalStorage } from "node:async_hooks";
@@ -18,6 +18,7 @@ import https from "node:https";
18
18
  import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
19
19
  import { Kind, parse } from "graphql";
20
20
  import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
21
+ import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
21
22
  import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
22
23
  import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
23
24
  //#region src/pieces/activepieces/piece-versions.ts
@@ -99,6 +100,14 @@ function npmVersions(name, options = {}) {
99
100
  //#endregion
100
101
  //#region src/reactor/attachment-port.ts
101
102
  const logger$6 = childLogger(["workflow", "attachments"]);
103
+ function fileStream(filePath) {
104
+ return Readable.toWeb(createReadStream(filePath));
105
+ }
106
+ async function sha256File(filePath) {
107
+ const hash = createHash("sha256");
108
+ for await (const chunk of createReadStream(filePath)) hash.update(chunk);
109
+ return hash.digest("hex");
110
+ }
102
111
  function refOf(result) {
103
112
  const direct = result.ref;
104
113
  if (typeof direct === "string") return direct;
@@ -128,16 +137,24 @@ async function writeCapped(body, destPath, limit) {
128
137
  }
129
138
  await handle.close();
130
139
  }
131
- function createAttachmentPort(client, documentIdFor, canReadRef) {
140
+ function createAttachmentPort(client, access) {
141
+ const authorize = async (ref) => {
142
+ const documentId = access.documentIdFor();
143
+ if (!documentId) throw new Error(`Cannot resolve ${ref}: no workflow document is in scope to authorize the read`);
144
+ if (!await access.canReadRef(ref)) throw new Error(`Cannot resolve ${ref}: workflow "${documentId}" may not read it`);
145
+ return documentId;
146
+ };
132
147
  return {
133
- async read(ref, destPath) {
134
- const documentId = documentIdFor();
135
- if (!documentId) throw new Error(`Cannot resolve ${ref}: no workflow document is in scope to authorize the read`);
136
- if (!await canReadRef(documentId, ref)) throw new Error(`Cannot resolve ${ref}: workflow "${documentId}" may not read it`);
148
+ async authorize(ref) {
149
+ await authorize(ref);
150
+ },
151
+ async read(ref, destPath, signal) {
152
+ const documentId = await authorize(ref);
137
153
  const limit = maxFileBytes();
138
154
  const { header, body } = await client.download({
139
155
  documentId,
140
- ref
156
+ ref,
157
+ ...signal ? { signal } : {}
141
158
  });
142
159
  if (typeof header.sizeBytes === "number" && Number.isFinite(header.sizeBytes) && header.sizeBytes > limit) {
143
160
  await body.cancel().catch(() => void 0);
@@ -150,13 +167,29 @@ function createAttachmentPort(client, documentIdFor, canReadRef) {
150
167
  contentType
151
168
  };
152
169
  },
153
- async write(file) {
154
- const bytes = await readFile(file.path);
170
+ async write(file, signal) {
171
+ const hash = await sha256File(file.path);
172
+ const mimeType = file.contentType ?? "application/octet-stream";
173
+ const extension = path.extname(file.fileName).slice(1);
174
+ const preprocessed = {
175
+ ref: `attachment://v1:${hash}`,
176
+ hash,
177
+ sizeBytes: file.size,
178
+ options: {
179
+ mimeType,
180
+ fileName: file.fileName,
181
+ ...extension ? { extension } : {},
182
+ clientHash: hash,
183
+ sizeBytes: file.size
184
+ },
185
+ data: fileStream(file.path),
186
+ stream: () => fileStream(file.path)
187
+ };
155
188
  const ref = refOf(await client.upload({
156
- file: new Blob([new Uint8Array(bytes)], { type: file.contentType ?? "application/octet-stream" }),
157
- fileName: file.fileName,
158
- mimeType: file.contentType
189
+ preprocessed,
190
+ ...signal ? { signal } : {}
159
191
  }));
192
+ access.onWritten?.(ref);
160
193
  logger$6.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
161
194
  return ref;
162
195
  }
@@ -1363,6 +1396,9 @@ function currentDocumentRecorder() {
1363
1396
  function currentPieceWorker() {
1364
1397
  return storage.getStore()?.pieceWorker;
1365
1398
  }
1399
+ function currentAttachmentOrigins() {
1400
+ return storage.getStore()?.attachments;
1401
+ }
1366
1402
  function currentStepTest() {
1367
1403
  return storage.getStore()?.stepTest ?? false;
1368
1404
  }
@@ -1440,6 +1476,20 @@ function runnableDefinition(state) {
1440
1476
  //#endregion
1441
1477
  //#region src/reactor/lib.ts
1442
1478
  const pieceLogger = childLogger(["workflow", "piece"]);
1479
+ /**
1480
+ * The ceiling on one line this engine writes to a log.
1481
+ *
1482
+ * Nothing here is worth more than this to a reader, and an unbounded write is
1483
+ * a liability: a piece error carrying an HTML error page, or a piece logging a
1484
+ * whole document, becomes a multi-megabyte write to a pipe that may be
1485
+ * blocked — and an EPIPE on the fatal path is what produced the boot loop in
1486
+ * backlog item 5. Bounding the write is cheaper than handling the throw.
1487
+ */
1488
+ const MAX_LOG_LINE_CHARS = 8192;
1489
+ function truncateForLog(text) {
1490
+ if (text.length <= 8192) return text;
1491
+ return `${text.slice(0, MAX_LOG_LINE_CHARS)}… [truncated, ${text.length} chars]`;
1492
+ }
1443
1493
  const connectionLogger = childLogger(["workflow", "connection"]);
1444
1494
  function assertConnectorMatches(state, request) {
1445
1495
  const wanted = request?.piecePackage;
@@ -1502,6 +1552,29 @@ function bundleCacheDir() {
1502
1552
  return bundleCache;
1503
1553
  }
1504
1554
  const ATTACHMENT_STAGING_DIR = join(process.cwd(), ".ph", "ap-attachment-staging");
1555
+ const ATTACHMENT_CACHE_DIR = join(process.cwd(), ".ph", "ap-attachment-cache");
1556
+ const DEFAULT_ATTACHMENT_CACHE_BYTES = 1073741824;
1557
+ function attachmentCacheBytes() {
1558
+ const raw = process.env.PH_WORKFLOWS_ATTACHMENT_CACHE_BYTES;
1559
+ const parsed = raw === void 0 ? NaN : Number(raw);
1560
+ return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : DEFAULT_ATTACHMENT_CACHE_BYTES;
1561
+ }
1562
+ async function sweepAttachmentStaging(root = ATTACHMENT_STAGING_DIR, olderThanMs = 36e5) {
1563
+ const names = await readdir(root).catch(() => []);
1564
+ let swept = 0;
1565
+ for (const name of names) {
1566
+ const entry = join(root, name);
1567
+ const info = await stat(entry).catch(() => void 0);
1568
+ if (!info || Date.now() - info.mtimeMs < olderThanMs) continue;
1569
+ await rm(entry, {
1570
+ recursive: true,
1571
+ force: true
1572
+ });
1573
+ swept += 1;
1574
+ }
1575
+ return swept;
1576
+ }
1577
+ let stagingSwept = false;
1505
1578
  let resolver;
1506
1579
  function pieceResolver() {
1507
1580
  return resolver ??= sourcedResolver({
@@ -1514,7 +1587,14 @@ function boundConnections(inner) {
1514
1587
  connectionLogger.warn(`Step "${request?.stepKey ?? "?"}" of workflow "${currentWorkflowId() ?? "?"}" asked for connection "${connectionId}", which its definition does not declare`);
1515
1588
  });
1516
1589
  }
1517
- function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess) {
1590
+ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess, telemetry) {
1591
+ if (attachments && !stagingSwept) {
1592
+ stagingSwept = true;
1593
+ sweepAttachmentStaging().then((swept) => {
1594
+ if (swept > 0) pieceLogger.info(`Removed ${swept} leftover attachment staging dir(s)`);
1595
+ }, () => void 0);
1596
+ }
1597
+ const cacheBytes = attachmentCacheBytes();
1518
1598
  return new CompositeBlockExecutor(new ActivepiecesBlockExecutor({
1519
1599
  cacheDir: bundleCacheDir(),
1520
1600
  egress: configuredEgress(),
@@ -1528,10 +1608,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
1528
1608
  resolver: pieceResolver(),
1529
1609
  ...resolveBlock ? { resolveBlock } : {},
1530
1610
  ...reactorAccess ? { reactorAccess } : {},
1611
+ ...telemetry ? { telemetry } : {},
1531
1612
  connections: boundConnections(new DocumentConnectionResolver(host, secrets, oauth)),
1532
1613
  ...pieceStore ? { pieceStore } : {},
1533
1614
  onPieceLog: (entry, execution) => {
1534
- const line = `[${execution.step.key}] ${entry.message}`;
1615
+ const line = truncateForLog(`[${execution.step.key}] ${entry.message}`);
1535
1616
  if (entry.level === "error") pieceLogger.error(line);
1536
1617
  else if (entry.level === "warn") pieceLogger.warn(line);
1537
1618
  else if (entry.level === "debug") pieceLogger.debug(line);
@@ -1539,7 +1620,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
1539
1620
  },
1540
1621
  ...attachments ? {
1541
1622
  attachments,
1542
- stagingRoot: ATTACHMENT_STAGING_DIR
1623
+ stagingRoot: ATTACHMENT_STAGING_DIR,
1624
+ ...cacheBytes > 0 ? { attachmentCache: new AttachmentCache({
1625
+ dir: ATTACHMENT_CACHE_DIR,
1626
+ maxBytes: cacheBytes
1627
+ }) } : {}
1543
1628
  } : {}
1544
1629
  }));
1545
1630
  }
@@ -1563,7 +1648,8 @@ function stepDefinition(step) {
1563
1648
  config: step.config,
1564
1649
  timeoutSeconds: step.timeoutSeconds,
1565
1650
  propertySettings: propertySettings(step.propertySettings),
1566
- skip: step.skip
1651
+ skip: step.skip,
1652
+ retry: step.retry
1567
1653
  };
1568
1654
  }
1569
1655
  function toWorkflowDefinition(state) {
@@ -1776,6 +1862,227 @@ async function buildReactorRunScope(host, request) {
1776
1862
  connection: { access: parsed.config.access ?? "write" }
1777
1863
  };
1778
1864
  }
1865
+ //#endregion
1866
+ //#region src/telemetry.ts
1867
+ const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
1868
+ var WorkflowTelemetry = class {
1869
+ tracer;
1870
+ meter;
1871
+ runs;
1872
+ runDuration;
1873
+ stepDuration;
1874
+ phaseDuration;
1875
+ poolGauges;
1876
+ constructor(options = {}) {
1877
+ this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
1878
+ this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
1879
+ this.runs = this.meter.createCounter("workflow.runs", {
1880
+ description: "Runs finished, by status and trigger kind",
1881
+ unit: "{run}"
1882
+ });
1883
+ this.runDuration = this.meter.createHistogram("workflow.run.duration", {
1884
+ description: "Run duration from fire to the journal closing it",
1885
+ unit: "ms"
1886
+ });
1887
+ this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
1888
+ description: "Step duration, by piece and status",
1889
+ unit: "ms"
1890
+ });
1891
+ this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
1892
+ description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
1893
+ unit: "ms"
1894
+ });
1895
+ }
1896
+ phase(name, attributes, fn, parent) {
1897
+ const started = performance.now();
1898
+ return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
1899
+ }
1900
+ workerTimings(request, timings, forkedAt, ipc) {
1901
+ const under = (span) => trace.setSpan(context.active(), span);
1902
+ const child = (name, start, end, parent = request, attributes) => {
1903
+ const span = this.tracer.startSpan(`workflow.worker.${name}`, {
1904
+ startTime: start,
1905
+ ...attributes ? { attributes } : {}
1906
+ }, under(parent));
1907
+ span.end(end);
1908
+ this.recordPhase(`worker.${name}`, end - start);
1909
+ return span;
1910
+ };
1911
+ const ready = epoch(timings.ready);
1912
+ if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
1913
+ const received = epoch(timings.received);
1914
+ const sent = epoch(timings.sent);
1915
+ const delivered = Math.max(ipc.sent, ready ?? 0);
1916
+ if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
1917
+ const byName = /* @__PURE__ */ new Map();
1918
+ const phases = Array.isArray(timings.phases) ? timings.phases : [];
1919
+ for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
1920
+ const parent = phase.parent ? byName.get(phase.parent) : void 0;
1921
+ byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
1922
+ }
1923
+ if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
1924
+ }
1925
+ recordPhase(name, ms) {
1926
+ this.phaseDuration.record(ms, { phase: name });
1927
+ }
1928
+ span(name, attributes, fn, parent = context.active()) {
1929
+ return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
1930
+ try {
1931
+ return await fn(span);
1932
+ } catch (error) {
1933
+ failSpan(span, error);
1934
+ throw error;
1935
+ } finally {
1936
+ span.end();
1937
+ }
1938
+ });
1939
+ }
1940
+ recordRun(ms, status, triggerKind) {
1941
+ const attributes = {
1942
+ "run.status": status,
1943
+ "trigger.kind": metricTriggerKind(triggerKind)
1944
+ };
1945
+ this.runs.add(1, attributes);
1946
+ this.runDuration.record(ms, attributes);
1947
+ }
1948
+ recordStep(ms, pieceName, status) {
1949
+ this.stepDuration.record(ms, {
1950
+ "piece.name": pieceName,
1951
+ "step.status": status
1952
+ });
1953
+ }
1954
+ observeWorkerPool(stats) {
1955
+ this.poolGauges ??= this.createPoolGauges();
1956
+ const { size, active, waiting } = this.poolGauges;
1957
+ const observables = [
1958
+ size,
1959
+ active,
1960
+ waiting
1961
+ ];
1962
+ const callback = (result) => {
1963
+ const current = stats();
1964
+ result.observe(size, current.size);
1965
+ result.observe(active, current.active);
1966
+ result.observe(waiting, current.waiting);
1967
+ };
1968
+ this.meter.addBatchObservableCallback(callback, observables);
1969
+ return () => this.meter.removeBatchObservableCallback(callback, observables);
1970
+ }
1971
+ createPoolGauges() {
1972
+ const gauge = (name, description) => this.meter.createObservableGauge(name, {
1973
+ description,
1974
+ unit: "{worker}"
1975
+ });
1976
+ return {
1977
+ size: gauge("workflow.worker.pool.size", "Worker slots"),
1978
+ active: gauge("workflow.worker.pool.active", "Slots held by runs"),
1979
+ waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
1980
+ };
1981
+ }
1982
+ };
1983
+ const WORKER_PHASES = /* @__PURE__ */ new Set([
1984
+ "piece.load",
1985
+ "reactor.open",
1986
+ "models",
1987
+ "action"
1988
+ ]);
1989
+ const WORKER_ATTRIBUTES = {
1990
+ "piece.cached": "boolean",
1991
+ "document.type": "string"
1992
+ };
1993
+ const MAX_ATTRIBUTE_LENGTH = 256;
1994
+ function epoch(value) {
1995
+ return typeof value === "number" && Number.isFinite(value) ? value : void 0;
1996
+ }
1997
+ function knownPhase(phase) {
1998
+ if (typeof phase !== "object" || phase === null) return false;
1999
+ const { name, start, end, parent } = phase;
2000
+ return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
2001
+ }
2002
+ function phaseAttributes(attributes) {
2003
+ if (typeof attributes !== "object" || attributes === null) return void 0;
2004
+ const kept = {};
2005
+ for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
2006
+ const value = attributes[key];
2007
+ if (typeof value !== type) continue;
2008
+ kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
2009
+ }
2010
+ return Object.keys(kept).length > 0 ? kept : void 0;
2011
+ }
2012
+ function metricTriggerKind(triggerKind) {
2013
+ return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
2014
+ }
2015
+ function metricPieceName(step, resolution) {
2016
+ return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
2017
+ }
2018
+ function tracedSteps(executor, telemetry) {
2019
+ return { execute(execution) {
2020
+ const { step } = execution;
2021
+ const started = performance.now();
2022
+ let status = "FAILED";
2023
+ let resolution;
2024
+ return telemetry.span("workflow.step", {
2025
+ "step.id": step.id,
2026
+ "step.key": step.key,
2027
+ "piece.name": step.pieceName,
2028
+ "action.name": step.actionName
2029
+ }, async (span) => {
2030
+ try {
2031
+ const result = await executor.execute(execution);
2032
+ resolution = result.resolution;
2033
+ status = "SUCCEEDED";
2034
+ const version = resolution?.resolved?.version;
2035
+ if (version) span.setAttribute("piece.version", version);
2036
+ return result;
2037
+ } catch (error) {
2038
+ resolution = resolutionOf(error);
2039
+ throw error;
2040
+ }
2041
+ }).finally(() => {
2042
+ telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
2043
+ });
2044
+ } };
2045
+ }
2046
+ function failSpan(span, error) {
2047
+ span.recordException(error instanceof Error ? error : String(error));
2048
+ span.setStatus({
2049
+ code: SpanStatusCode.ERROR,
2050
+ message: error instanceof Error ? error.message : String(error)
2051
+ });
2052
+ }
2053
+ function tracedMethods(target, telemetry, prefix, parent) {
2054
+ return new Proxy(target, { get(object, property, receiver) {
2055
+ const value = Reflect.get(object, property, receiver);
2056
+ if (typeof value !== "function" || typeof property !== "string") return value;
2057
+ const phase = `${prefix}.${property}`;
2058
+ return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
2059
+ const started = performance.now();
2060
+ const end = (error) => {
2061
+ if (error !== void 0) failSpan(span, error);
2062
+ span.end();
2063
+ telemetry.recordPhase(phase, performance.now() - started);
2064
+ };
2065
+ let result;
2066
+ try {
2067
+ result = value.apply(object, args);
2068
+ } catch (error) {
2069
+ end(error);
2070
+ throw error;
2071
+ }
2072
+ if (!(result instanceof Promise)) {
2073
+ end();
2074
+ return result;
2075
+ }
2076
+ return result.then((resolved) => {
2077
+ end();
2078
+ return resolved;
2079
+ }, (error) => {
2080
+ end(error);
2081
+ throw error;
2082
+ });
2083
+ });
2084
+ } });
2085
+ }
1779
2086
  const JOB_NOT_FOUND = "Job not found";
1780
2087
  const MIN_DEADLINE_MARGIN_MS = 250;
1781
2088
  const MAX_DEADLINE_MARGIN_MS = 2e3;
@@ -1792,7 +2099,7 @@ const AUTHORIZATION_DENIED = "AuthorizationDeniedError";
1792
2099
  function jobFailed(job) {
1793
2100
  const error = job.error;
1794
2101
  if (error?.name === AUTHORIZATION_DENIED) return accessDenied(`The Switchboard has no grant for this operation: ${error.message}`);
1795
- return new ReactorError(ReactorJobFailedError$1, `Reactor job ${job.id} failed: ${job.error?.message ?? "unknown error"}`);
2102
+ return new ReactorError(ReactorJobFailedError, `Reactor job ${job.id} failed: ${job.error?.message ?? "unknown error"}`);
1796
2103
  }
1797
2104
  function assertActionsApplied(job, actions) {
1798
2105
  const outcomes = job.result?.actions ?? [];
@@ -1810,7 +2117,7 @@ function assertActionsApplied(job, actions) {
1810
2117
  if (outcome.kind === "denied") return [`Action ${label} was denied: ${outcome.reason}`];
1811
2118
  return [];
1812
2119
  });
1813
- if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError$1, failed.join("; "));
2120
+ if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError, failed.join("; "));
1814
2121
  }
1815
2122
  var RunScopedReactorClient = class {
1816
2123
  inner;
@@ -2141,7 +2448,7 @@ const NO_JOURNAL = {
2141
2448
  recordDocuments: () => Promise.resolve(),
2142
2449
  recordJob: () => Promise.resolve()
2143
2450
  };
2144
- function reactorTap(host, base, journal, options = {}) {
2451
+ function reactorTap(host, base, journal, options = {}, telemetry) {
2145
2452
  const clientOptions = {
2146
2453
  ...host.hostPrincipal ? { hostPrincipal: host.hostPrincipal } : {},
2147
2454
  ...options
@@ -2149,12 +2456,13 @@ function reactorTap(host, base, journal, options = {}) {
2149
2456
  return {
2150
2457
  requireReactor: base.requireReactor,
2151
2458
  open(transport, { deadline }) {
2152
- const client = new RunScopedReactorClient(host.reactorClient, {
2459
+ const served = servedClient(new RunScopedReactorClient(host.reactorClient, {
2153
2460
  ...base,
2154
2461
  deadline,
2155
2462
  journal
2156
- }, host, clientOptions);
2157
- const server = new ReactorHostServer(servedClient(client), transport);
2463
+ }, host, clientOptions));
2464
+ const parent = context.active();
2465
+ const server = new ReactorHostServer(telemetry ? tracedMethods(served, telemetry, "reactor", parent) : served, transport);
2158
2466
  server.start();
2159
2467
  return () => server.stop();
2160
2468
  }
@@ -2451,6 +2759,19 @@ async function resolveVariables(variables, secrets) {
2451
2759
  }
2452
2760
  //#endregion
2453
2761
  //#region src/reactor/step-test.ts
2762
+ /**
2763
+ * A step record's status as a test reports it.
2764
+ *
2765
+ * Collapsing this to `FAILED : SUCCEEDED` is how an INDETERMINATE test came
2766
+ * to read green — the one status that must never be mistaken for a confirmed
2767
+ * one, since the whole point of it is that nobody knows whether the write
2768
+ * landed.
2769
+ */
2770
+ function testStatusOf(status) {
2771
+ if (status === "FAILED") return "FAILED";
2772
+ if (status === "INDETERMINATE") return "INDETERMINATE";
2773
+ return "SUCCEEDED";
2774
+ }
2454
2775
  function collectStrings(value, into) {
2455
2776
  if (typeof value === "string") into.push(value);
2456
2777
  else if (Array.isArray(value)) for (const item of value) collectStrings(item, into);
@@ -2534,6 +2855,7 @@ function untestedError(label, sample) {
2534
2855
  case "hidden": return `Test ${label} first: its last test is not visible to you`;
2535
2856
  case "failed": return `Test ${label} first: its last test failed`;
2536
2857
  case "truncated": return `Test ${label} again: the journal kept only a truncated copy of its last output`;
2858
+ case "indeterminate": return `Test ${label} again: its last test is INDETERMINATE, so it has no confirmed output`;
2537
2859
  default: return `Test ${label} first`;
2538
2860
  }
2539
2861
  }
@@ -7095,6 +7417,59 @@ function parseParameter(param) {
7095
7417
  return parseValueExpression(param);
7096
7418
  }
7097
7419
  //#endregion
7420
+ //#region src/reactor/policy.ts
7421
+ /** The pre-enforcement behaviour, for a definition that declares no policy. */
7422
+ const UNENFORCED_POLICY = Object.freeze({
7423
+ concurrency: "PARALLEL",
7424
+ maxParallelRuns: null,
7425
+ runTimeoutSeconds: null,
7426
+ onFailure: "IGNORE",
7427
+ defaultRetry: null
7428
+ });
7429
+ const CONCURRENCY_MODES = [
7430
+ "SINGLETON",
7431
+ "QUEUE",
7432
+ "PARALLEL"
7433
+ ];
7434
+ const FAILURE_MODES = [
7435
+ "PARK",
7436
+ "NOTIFY",
7437
+ "IGNORE"
7438
+ ];
7439
+ function positive(value) {
7440
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
7441
+ }
7442
+ /** What the runtime enforces for a run of this definition. */
7443
+ function effectiveRunPolicy(runnable) {
7444
+ const policy = runnable.policy;
7445
+ if (!policy) return UNENFORCED_POLICY;
7446
+ const concurrency = CONCURRENCY_MODES.includes(policy.concurrency) ? policy.concurrency : "PARALLEL";
7447
+ return {
7448
+ concurrency,
7449
+ maxParallelRuns: concurrency === "PARALLEL" ? positive(policy.maxParallelRuns) : null,
7450
+ runTimeoutSeconds: positive(policy.runTimeoutSeconds),
7451
+ onFailure: FAILURE_MODES.includes(policy.onFailure) ? policy.onFailure : "IGNORE",
7452
+ defaultRetry: effectiveRetryPolicy(policy.defaultRetry)
7453
+ };
7454
+ }
7455
+ /** How many runs of one workflow may execute at once; 1 for both serialising
7456
+ * modes, so the gate holds one number rather than three branches. Null is
7457
+ * unbounded, i.e. no gate at all. */
7458
+ function concurrencyLimit(policy) {
7459
+ if (policy.concurrency === "PARALLEL") return policy.maxParallelRuns;
7460
+ return 1;
7461
+ }
7462
+ /** The trigger status PARK leaves on a supervised trigger's row. Not ENABLED,
7463
+ * so the supervisor's due-trigger query does not return it and the schedule
7464
+ * stops refiring; a re-publish or re-enable arms it again. The park itself, for
7465
+ * every trigger kind, is the store's `workflow_park` row. */
7466
+ const PARKED_TRIGGER_STATUS = "PARKED";
7467
+ /** The run status a firing skipped by SINGLETON is journaled under, and the
7468
+ * one a run that passed `runTimeoutSeconds` ends in. Both are "this run did
7469
+ * not finish and nothing is wrong with the workflow", which is what the
7470
+ * document model's RunStatus.CANCELLED means. */
7471
+ const CANCELLED_RUN_STATUS = "CANCELLED";
7472
+ //#endregion
7098
7473
  //#region src/reactor/store.ts
7099
7474
  function triggerRowBlock(row) {
7100
7475
  return {
@@ -7109,6 +7484,19 @@ function triggerBlockColumns(block) {
7109
7484
  trigger_name: block.name
7110
7485
  };
7111
7486
  }
7487
+ const FIRED_WITHOUT_RUN_ID = "fired-without-run";
7488
+ const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
7489
+ function abandonedFireError(attempts) {
7490
+ return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
7491
+ }
7492
+ function workflowParkRow(workflowId, publishedVersion, reason) {
7493
+ return {
7494
+ workflow_id: workflowId,
7495
+ published_version: publishedVersion,
7496
+ reason: redactMessage(reason),
7497
+ parked_at: (/* @__PURE__ */ new Date()).toISOString()
7498
+ };
7499
+ }
7112
7500
  const logger$3 = childLogger([
7113
7501
  "workflow",
7114
7502
  "runtime",
@@ -7136,6 +7524,7 @@ async function up(db) {
7136
7524
  } catch {}
7137
7525
  await db.schema.createTable("trigger_state").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("trigger_name", "text", (col) => col.notNull()).addColumn("config_hash", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("store_state", "text", (col) => col.notNull()).addColumn("interval_ms", "integer", (col) => col.notNull()).addColumn("next_poll_at", "text").addColumn("last_poll_at", "text").addColumn("last_error", "text").addColumn("consecutive_failures", "integer", (col) => col.notNull()).addColumn("lease_owner", "text").addColumn("lease_expires_at", "text").addColumn("updated_at", "text", (col) => col.notNull()).ifNotExists().execute();
7138
7526
  await db.schema.createTable("trigger_dedupe").addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("dedupe_key", "text", (col) => col.notNull()).addColumn("run_id", "text").addColumn("created_at", "text", (col) => col.notNull()).addPrimaryKeyConstraint("trigger_dedupe_pk", ["workflow_id", "dedupe_key"]).ifNotExists().execute();
7527
+ await db.schema.createTable("workflow_park").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("published_version", "integer", (col) => col.notNull()).addColumn("reason", "text", (col) => col.notNull()).addColumn("parked_at", "text", (col) => col.notNull()).ifNotExists().execute();
7139
7528
  await db.schema.createTable("step_execution").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("ordinal", "integer", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("input", "text").addColumn("output", "text").addColumn("port", "text").addColumn("error", "text").addColumn("started_at", "text").addColumn("ended_at", "text").addUniqueConstraint("step_execution_run_step", ["run_id", "step_id"]).ifNotExists().execute();
7140
7529
  for (const column of ["started_at", "ended_at"]) try {
7141
7530
  await db.schema.alterTable("step_execution").addColumn(column, "text").execute();
@@ -7154,6 +7543,21 @@ async function up(db) {
7154
7543
  ]) try {
7155
7544
  await db.schema.alterTable(table).addColumn(column, "text").execute();
7156
7545
  } catch {}
7546
+ try {
7547
+ await db.schema.alterTable("trigger_dedupe").addColumn("attempts", "integer", (col) => col.notNull().defaultTo(1)).execute();
7548
+ } catch {}
7549
+ await db.schema.createTable("migration_mark").addColumn("name", "text", (col) => col.primaryKey()).addColumn("applied_at", "text", (col) => col.notNull()).ifNotExists().execute();
7550
+ await db.transaction().execute(async (trx) => {
7551
+ if (await trx.selectFrom("migration_mark").select("name").where("name", "=", DEDUPE_CLAIM_MIGRATION).executeTakeFirst()) return;
7552
+ await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "is", null).execute();
7553
+ await trx.insertInto("migration_mark").values({
7554
+ name: DEDUPE_CLAIM_MIGRATION,
7555
+ applied_at: (/* @__PURE__ */ new Date()).toISOString()
7556
+ }).onConflict((oc) => oc.column("name").doNothing()).execute();
7557
+ });
7558
+ try {
7559
+ await db.schema.alterTable("step_execution").addColumn("attempts", "integer").execute();
7560
+ } catch {}
7157
7561
  try {
7158
7562
  await db.schema.alterTable("trigger_state").addColumn("next_renew_at", "text").execute();
7159
7563
  } catch {}
@@ -7362,10 +7766,50 @@ function assertPieceStoreEntry(key, value) {
7362
7766
  if (size > 524288) throw new PieceStoreLimitError(`Store value for "${key}" is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} byte limit`);
7363
7767
  }
7364
7768
  const STEP_PAYLOAD_PREFIX_CHARS = 32768;
7769
+ /**
7770
+ * The reserved key a truncation marker is recognised by, and its value.
7771
+ *
7772
+ * A marker has to be distinguishable from a legitimate payload that merely
7773
+ * LOOKS like one, because a run whose output is mistaken for a marker is a
7774
+ * step that silently re-executes on rerun. The original predicate duck-typed
7775
+ * `{truncated: true, bytes: number, prefix: string}` (review backlog item 15),
7776
+ * which any piece returning a truncation report of its own would match — and
7777
+ * "did we truncate this?" is exactly the shape such a report takes.
7778
+ *
7779
+ * So the marker is keyed on a namespaced sentinel whose VALUE is a versioned
7780
+ * magic string. A piece would have to emit that exact pair to collide, which
7781
+ * is not something a payload produces by accident.
7782
+ */
7783
+ const TRUNCATED_PAYLOAD_KEY = "@powerhousedao/reactor-workflow:runJournal";
7784
+ const TRUNCATED_PAYLOAD_SENTINEL = "truncated-payload/v1";
7785
+ const LEGACY_MARKER_KEYS = [
7786
+ "truncated",
7787
+ "bytes",
7788
+ "prefix",
7789
+ "documentId",
7790
+ "driveId"
7791
+ ];
7792
+ /**
7793
+ * True for a journaled value this store truncated.
7794
+ *
7795
+ * Two shapes are accepted and only one is written. New rows carry the reserved
7796
+ * key, which a payload cannot produce by accident. Rows journaled before it
7797
+ * carry the old `{truncated: true, bytes, prefix}`, and those still have to be
7798
+ * recognised: reading one as data would re-run the side-effectful step that
7799
+ * produced it, which is the whole bug.
7800
+ *
7801
+ * So the legacy branch is matched by its EXACT key set, not by duck-typing.
7802
+ * `{truncated, bytes, prefix, source}` — a truncation report a piece might
7803
+ * legitimately return, and the collision the review found — is no longer a
7804
+ * marker. A payload whose keys are exactly the legacy marker's is still
7805
+ * ambiguous, and nothing can resolve that for a row already written; the
7806
+ * window is as narrow as the stored data allows.
7807
+ */
7365
7808
  function isTruncatedStepPayload(value) {
7366
7809
  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
7367
7810
  const record = value;
7368
- return record.truncated === true && typeof record.bytes === "number" && typeof record.prefix === "string";
7811
+ if (record["@powerhousedao/reactor-workflow:runJournal"] === "truncated-payload/v1") return true;
7812
+ return record.truncated === true && typeof record.bytes === "number" && typeof record.prefix === "string" && Object.keys(record).every((key) => LEGACY_MARKER_KEYS.includes(key));
7369
7813
  }
7370
7814
  function topLevelDocumentIds(value) {
7371
7815
  if (typeof value !== "object" || value === null || Array.isArray(value)) return {};
@@ -7384,7 +7828,7 @@ function cappedPayload(value) {
7384
7828
  const last = prefix.charCodeAt(prefix.length - 1);
7385
7829
  if (last >= 55296 && last <= 56319) prefix = prefix.slice(0, -1);
7386
7830
  const marker = {
7387
- truncated: true,
7831
+ [TRUNCATED_PAYLOAD_KEY]: TRUNCATED_PAYLOAD_SENTINEL,
7388
7832
  bytes,
7389
7833
  prefix,
7390
7834
  ...topLevelDocumentIds(value)
@@ -7401,7 +7845,7 @@ function stepValues(runId, ordinal, step) {
7401
7845
  block_name: step.blockName,
7402
7846
  status: step.status,
7403
7847
  input: cappedPayload(redact(step.input)),
7404
- output: cappedPayload(redact(referenceDocuments(step.output))),
7848
+ output: cappedPayload(redact(referenceDocuments(step.journaledOutput !== void 0 ? step.journaledOutput : step.output))),
7405
7849
  port: step.port ?? null,
7406
7850
  error: step.error ? redactMessage(step.error) : null,
7407
7851
  error_name: step.errorName ?? null,
@@ -7411,7 +7855,8 @@ function stepValues(runId, ordinal, step) {
7411
7855
  piece_source: step.piece?.source ?? null,
7412
7856
  version_match: step.piece?.match ?? null,
7413
7857
  version_note: step.piece?.note ?? null,
7414
- config_hash: step.configHash ?? null
7858
+ config_hash: step.configHash ?? null,
7859
+ attempts: step.attempts ?? null
7415
7860
  };
7416
7861
  }
7417
7862
  function runWarningNotes(result) {
@@ -7460,15 +7905,16 @@ function journaledPayloadNames(json, ids) {
7460
7905
  });
7461
7906
  }
7462
7907
  const erasedRuns = /* @__PURE__ */ new Set();
7463
- async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso, runId) {
7908
+ async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso) {
7464
7909
  const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
7465
7910
  await db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
7466
7911
  return await db.insertInto("trigger_dedupe").values({
7467
7912
  workflow_id: workflowId,
7468
7913
  dedupe_key: dedupeKey,
7469
- run_id: runId,
7470
- created_at: nowIso
7471
- }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doNothing()).returning("dedupe_key").executeTakeFirst() !== void 0;
7914
+ run_id: FIRED_WITHOUT_RUN_ID,
7915
+ created_at: nowIso,
7916
+ attempts: 1
7917
+ }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id: FIRED_WITHOUT_RUN_ID }).where("trigger_dedupe.run_id", "is", null)).returning("dedupe_key").executeTakeFirst() !== void 0;
7472
7918
  }
7473
7919
  const STEP_COLUMNS_WITHOUT_DATA = [
7474
7920
  "id",
@@ -7488,7 +7934,8 @@ const STEP_COLUMNS_WITHOUT_DATA = [
7488
7934
  "piece_source",
7489
7935
  "version_match",
7490
7936
  "version_note",
7491
- "config_hash"
7937
+ "config_hash",
7938
+ "attempts"
7492
7939
  ];
7493
7940
  const PRUNE_BATCH_SIZE = 500;
7494
7941
  var WorkflowRunStore = class WorkflowRunStore {
@@ -7535,20 +7982,133 @@ var WorkflowRunStore = class WorkflowRunStore {
7535
7982
  if (recovered > 0) logger$3.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
7536
7983
  return recovered;
7537
7984
  }
7985
+ /**
7986
+ * Closes a run out as CANCELLED, with the reason as its error.
7987
+ *
7988
+ * Two callers, one meaning — "this run did not finish, and the workflow is
7989
+ * not at fault": a firing SINGLETON concurrency refused, and a run that
7990
+ * passed its `runTimeoutSeconds`. Separate from `failRun` because a
7991
+ * CANCELLED run is not rerunnable and must not read as a workflow defect.
7992
+ */
7993
+ async cancelRun(runId, reason) {
7994
+ this.runsInFlight.delete(runId);
7995
+ if (erasedRuns.delete(runId)) return;
7996
+ await this.db.updateTable("run").set({
7997
+ status: "CANCELLED",
7998
+ error: redactMessage(reason),
7999
+ ended_at: (/* @__PURE__ */ new Date()).toISOString()
8000
+ }).where("id", "=", runId).execute();
8001
+ }
7538
8002
  async enqueueRun(options) {
7539
8003
  const id = randomUUID();
7540
8004
  await this.insertPendingRun(this.db, id, options);
7541
8005
  this.runsInFlight.add(id);
7542
8006
  return id;
7543
8007
  }
8008
+ /**
8009
+ * Claims the dedupe key and journals the PENDING run, bounding crash replays.
8010
+ *
8011
+ * Three outcomes, as {@link FireClaim}:
8012
+ *
8013
+ * - **`"claimed"`** with a run id: this fire is ours to run.
8014
+ * - **`"duplicate"`**: an earlier delivery of this operation already
8015
+ * fired: it journaled a run, or ran without one
8016
+ * ({@link FIRED_WITHOUT_RUN_ID}).
8017
+ * - **`"abandoned"`**: the operation has been delivered
8018
+ * {@link FIRE_CRASH_BUDGET} times and has never once got as far as
8019
+ * journaling a run — i.e. it is taking the process down before the write
8020
+ * lands, and the read-model cursor replays it on every boot. That is the
8021
+ * EPIPE boot loop (backlog item 5), and a budget is what turns it from an
8022
+ * unbootable reactor into one FAILED run naming the loop.
8023
+ *
8024
+ * Two writes, and which one is which matters:
8025
+ *
8026
+ * - The **attempt count** is bumped in its own committed transaction,
8027
+ * BEFORE the risky work. That is the only way a delivery that leaves no
8028
+ * run behind can be counted at all, which is what the budget needs. The
8029
+ * counter is never a claim signal — it only decides the budget.
8030
+ * - The **claim** is `run_id`, and it is set in the SAME transaction as the
8031
+ * run row, under a `WHERE run_id IS NULL` guard. The guard takes the
8032
+ * row's lock, so of two concurrent deliveries of one operation the second
8033
+ * blocks until the first commits and then matches no row: exactly one
8034
+ * delivery wins, and the run it journaled is committed with the claim. A
8035
+ * crash in between rolls BOTH back, so a claim that holds a run id always
8036
+ * holds a run — and a claim without one is retried, up to the budget.
8037
+ *
8038
+ * Counting and claiming are deliberately not the same write: if the bump
8039
+ * rode along inside the claim transaction, a run insert that takes the
8040
+ * process down would roll the count back with it and the budget could never
8041
+ * reach its limit — the boot loop it exists to bound.
8042
+ */
7544
8043
  async claimAndEnqueueRun(dedupeKey, ttlMs, nowIso, options) {
8044
+ const attempt = await this.recordFireAttempt(options.workflowId, dedupeKey, ttlMs, nowIso);
8045
+ if (attempt.runId !== null) return { outcome: "duplicate" };
8046
+ if (attempt.attempts > 3) return {
8047
+ outcome: "abandoned",
8048
+ attempts: attempt.attempts
8049
+ };
7545
8050
  const id = randomUUID();
7546
8051
  if (!await this.db.transaction().execute(async (trx) => {
7547
- if (!await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id)) return false;
8052
+ if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
7548
8053
  await this.insertPendingRun(trx, id, options);
7549
8054
  return true;
7550
- })) return null;
8055
+ })) return { outcome: "duplicate" };
7551
8056
  this.runsInFlight.add(id);
8057
+ return {
8058
+ outcome: "claimed",
8059
+ runId: id
8060
+ };
8061
+ }
8062
+ /**
8063
+ * Counts one delivery of an operation, and reports whether a run was ever
8064
+ * journaled for it.
8065
+ *
8066
+ * Committed on its own: the point is to leave a trace even when whatever
8067
+ * comes next takes the process down. It is NOT a claim — the row it writes
8068
+ * holds no run id, and {@link claimAndEnqueueRun} is what decides, under a
8069
+ * lock, which delivery gets to fill it in.
8070
+ */
8071
+ async recordFireAttempt(workflowId, dedupeKey, ttlMs, nowIso) {
8072
+ const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
8073
+ await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
8074
+ const row = await this.db.insertInto("trigger_dedupe").values({
8075
+ workflow_id: workflowId,
8076
+ dedupe_key: dedupeKey,
8077
+ run_id: null,
8078
+ created_at: nowIso,
8079
+ attempts: 1
8080
+ }).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ attempts: sql`${sql.table("trigger_dedupe")}.attempts + 1` })).returning(["attempts", "run_id"]).executeTakeFirstOrThrow();
8081
+ return {
8082
+ attempts: row.attempts ?? 1,
8083
+ runId: row.run_id
8084
+ };
8085
+ }
8086
+ /**
8087
+ * The abandonment of an over-budget fire, as a visible FAILED run, linked to
8088
+ * its dedupe key so a later delivery is a duplicate rather than another one.
8089
+ */
8090
+ async journalAbandonedFire(dedupeKey, options) {
8091
+ const id = randomUUID();
8092
+ const now = (/* @__PURE__ */ new Date()).toISOString();
8093
+ await this.db.transaction().execute(async (trx) => {
8094
+ await trx.insertInto("run").values({
8095
+ id,
8096
+ workflow_id: options.workflowId,
8097
+ workflow_name: "",
8098
+ workflow_version: 0,
8099
+ trigger_kind: options.triggerKind,
8100
+ trigger_payload: cappedPayload(redact(options.triggerPayload)),
8101
+ status: "FAILED",
8102
+ error: redactMessage(abandonedFireError(options.attempts)),
8103
+ enqueued_at: now,
8104
+ started_at: now,
8105
+ ended_at: now,
8106
+ rerun_of: null,
8107
+ warnings: 0,
8108
+ warning_notes: null
8109
+ }).execute();
8110
+ await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).execute();
8111
+ });
7552
8112
  return id;
7553
8113
  }
7554
8114
  async insertPendingRun(db, id, options) {
@@ -7717,6 +8277,58 @@ var WorkflowRunStore = class WorkflowRunStore {
7717
8277
  for (const row of rows) byRun.get(row.run_id)?.push(row.document_id);
7718
8278
  return byRun;
7719
8279
  }
8280
+ /**
8281
+ * Parks a workflow: its park row, and an ENABLED trigger_state row turned
8282
+ * PARKED, in one transaction. True when a trigger_state row was parked.
8283
+ */
8284
+ async parkWorkflow(workflowId, publishedVersion, reason) {
8285
+ const { workflow_id: _, ...row } = workflowParkRow(workflowId, publishedVersion, reason);
8286
+ const now = row.parked_at;
8287
+ return this.db.transaction().execute(async (trx) => {
8288
+ await trx.insertInto("workflow_park").values({
8289
+ workflow_id: workflowId,
8290
+ ...row
8291
+ }).onConflict((oc) => oc.column("workflow_id").doUpdateSet(row)).execute();
8292
+ return (await trx.updateTable("trigger_state").set({
8293
+ status: PARKED_TRIGGER_STATUS,
8294
+ last_error: row.reason,
8295
+ updated_at: now
8296
+ }).where("workflow_id", "=", workflowId).where("status", "=", "ENABLED").returning("workflow_id").execute()).length > 0;
8297
+ });
8298
+ }
8299
+ /** A disable clears a park: the park row, and a PARKED trigger row turned
8300
+ * DISABLED, so re-enabling arms it whether or not the supervisor held it. */
8301
+ async clearParkOnDisable(workflowId) {
8302
+ await this.db.transaction().execute(async (trx) => {
8303
+ await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
8304
+ await trx.updateTable("trigger_state").set({
8305
+ status: "DISABLED",
8306
+ ...RENEW_CLEARED,
8307
+ updated_at: (/* @__PURE__ */ new Date()).toISOString()
8308
+ }).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
8309
+ });
8310
+ }
8311
+ /** Undoes {@link parkWorkflow}; `trigger` says whether it parked a row. */
8312
+ async liftPark(workflowId, trigger) {
8313
+ await this.db.transaction().execute(async (trx) => {
8314
+ await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
8315
+ if (!trigger) return;
8316
+ await trx.updateTable("trigger_state").set({
8317
+ status: "ENABLED",
8318
+ last_error: null,
8319
+ updated_at: (/* @__PURE__ */ new Date()).toISOString()
8320
+ }).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
8321
+ });
8322
+ }
8323
+ async getWorkflowPark(workflowId) {
8324
+ return this.db.selectFrom("workflow_park").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
8325
+ }
8326
+ async listWorkflowParks() {
8327
+ return this.db.selectFrom("workflow_park").selectAll().execute();
8328
+ }
8329
+ async clearWorkflowPark(workflowId) {
8330
+ await this.db.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
8331
+ }
7720
8332
  async getTriggerState(workflowId) {
7721
8333
  return this.db.selectFrom("trigger_state").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
7722
8334
  }
@@ -7800,7 +8412,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7800
8412
  }).where("workflow_id", "=", workflowId).execute();
7801
8413
  }
7802
8414
  async claimDedupe(workflowId, dedupeKey, ttlMs, nowIso) {
7803
- return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso, null);
8415
+ return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso);
7804
8416
  }
7805
8417
  async deleteDedupe(workflowId) {
7806
8418
  await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
@@ -7848,7 +8460,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7848
8460
  for (const row of live) erasedRuns.add(row.id);
7849
8461
  erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
7850
8462
  erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
7851
- erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: null }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
8463
+ erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
7852
8464
  erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
7853
8465
  return erased;
7854
8466
  });
@@ -7900,6 +8512,46 @@ var WorkflowRunStore = class WorkflowRunStore {
7900
8512
  }
7901
8513
  };
7902
8514
  //#endregion
8515
+ //#region src/reactor/park-state.ts
8516
+ var ParkState = class {
8517
+ load;
8518
+ parks = /* @__PURE__ */ new Map();
8519
+ written = /* @__PURE__ */ new Set();
8520
+ seeded = false;
8521
+ seeding;
8522
+ constructor(load) {
8523
+ this.load = load;
8524
+ }
8525
+ async get(workflowId) {
8526
+ if (!this.seeded) {
8527
+ this.seeding ??= this.seed().finally(() => {
8528
+ this.seeding = void 0;
8529
+ });
8530
+ await this.seeding;
8531
+ }
8532
+ return this.parks.get(workflowId);
8533
+ }
8534
+ known(workflowId) {
8535
+ if (!this.seeded && !this.written.has(workflowId)) return void 0;
8536
+ return { park: this.parks.get(workflowId) };
8537
+ }
8538
+ set(park) {
8539
+ this.parks.set(park.workflow_id, park);
8540
+ if (!this.seeded) this.written.add(park.workflow_id);
8541
+ }
8542
+ delete(workflowId) {
8543
+ this.parks.delete(workflowId);
8544
+ if (!this.seeded) this.written.add(workflowId);
8545
+ }
8546
+ async seed() {
8547
+ const rows = await this.load();
8548
+ if (!rows) return;
8549
+ for (const row of rows) if (!this.written.has(row.workflow_id)) this.parks.set(row.workflow_id, row);
8550
+ this.written.clear();
8551
+ this.seeded = true;
8552
+ }
8553
+ };
8554
+ //#endregion
7903
8555
  //#region src/reactor/run-cursor.ts
7904
8556
  var InvalidRunCursorError = class extends Error {
7905
8557
  constructor() {
@@ -7924,13 +8576,121 @@ function decodeRunCursor(cursor) {
7924
8576
  };
7925
8577
  }
7926
8578
  //#endregion
8579
+ //#region src/reactor/run-gate.ts
8580
+ const QUEUE_DEPTH_ENV = "PH_WORKFLOWS_MAX_QUEUED_FIRINGS";
8581
+ function maxQueuedFirings(env = process.env) {
8582
+ const raw = Number(env[QUEUE_DEPTH_ENV]);
8583
+ return Number.isFinite(raw) && raw > 0 ? raw : 100;
8584
+ }
8585
+ var WorkflowRunGate = class {
8586
+ lanes = /* @__PURE__ */ new Map();
8587
+ maxQueued;
8588
+ constructor(options = {}) {
8589
+ this.maxQueued = options.maxQueued ?? maxQueuedFirings();
8590
+ }
8591
+ /** Runs of this workflow executing right now. */
8592
+ active(workflowId) {
8593
+ return this.lanes.get(workflowId)?.active ?? 0;
8594
+ }
8595
+ /** Firings of this workflow waiting for a slot. */
8596
+ waiting(workflowId) {
8597
+ return this.lanes.get(workflowId)?.waiting.length ?? 0;
8598
+ }
8599
+ /**
8600
+ * Takes a slot for one run.
8601
+ *
8602
+ * - **PARALLEL** with no `maxParallelRuns`, or no policy at all: no gate,
8603
+ * so an unbounded workflow's hot path costs nothing. Not the default: a
8604
+ * workflow document starts as QUEUE.
8605
+ * - **PARALLEL** with a bound, and **QUEUE**: waits, first come first
8606
+ * served. Latency, not failure — which is what QUEUE means, up to
8607
+ * {@link DEFAULT_MAX_QUEUED_FIRINGS} waiters. Past that the queue is full
8608
+ * and the firing is refused, so the caller journals it CANCELLED rather
8609
+ * than growing a backlog nothing bounds.
8610
+ * - **SINGLETON**: refused outright while a run is active. The caller
8611
+ * journals the refusal rather than dropping it silently, because a firing
8612
+ * that vanished is indistinguishable from a trigger that never fired.
8613
+ */
8614
+ async admit(workflowId, policy) {
8615
+ const limit = concurrencyLimit(policy);
8616
+ if (limit === null) return {
8617
+ admitted: true,
8618
+ waited: false,
8619
+ release: () => void 0
8620
+ };
8621
+ const lane = this.lanes.get(workflowId) ?? {
8622
+ active: 0,
8623
+ waiting: []
8624
+ };
8625
+ this.lanes.set(workflowId, lane);
8626
+ let waited = false;
8627
+ if (lane.active >= limit) {
8628
+ if (policy.concurrency === "SINGLETON") {
8629
+ this.forgetIfIdle(workflowId, lane);
8630
+ return {
8631
+ admitted: false,
8632
+ refusal: "singleton",
8633
+ reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
8634
+ };
8635
+ }
8636
+ if (lane.waiting.length >= this.maxQueued) return {
8637
+ admitted: false,
8638
+ refusal: "queue-full",
8639
+ reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
8640
+ };
8641
+ await new Promise((resolve) => lane.waiting.push(resolve));
8642
+ waited = true;
8643
+ } else lane.active += 1;
8644
+ let released = false;
8645
+ return {
8646
+ admitted: true,
8647
+ waited,
8648
+ release: () => {
8649
+ if (released) return;
8650
+ released = true;
8651
+ lane.active -= 1;
8652
+ const next = lane.waiting.shift();
8653
+ if (next) {
8654
+ lane.active += 1;
8655
+ next();
8656
+ return;
8657
+ }
8658
+ this.forgetIfIdle(workflowId, lane);
8659
+ }
8660
+ };
8661
+ }
8662
+ forgetIfIdle(workflowId, lane) {
8663
+ if (lane.active === 0 && lane.waiting.length === 0) this.lanes.delete(workflowId);
8664
+ }
8665
+ };
8666
+ //#endregion
7927
8667
  //#region src/reactor/run-retention.ts
7928
8668
  const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
7929
8669
  const RETENTION_SWEEP_INTERVAL_MS = 36e5;
7930
8670
  const DAY_MS = 864e5;
8671
+ /** The spellings that turn retention OFF, compared case-insensitively after
8672
+ * trimming. Wider than `0` because "off" and "never" are what an operator
8673
+ * reaches for, and a variable that silently keeps pruning because they wrote
8674
+ * one of those is the worst outcome available. */
8675
+ const RETENTION_OFF_SPELLINGS = [
8676
+ "0",
8677
+ "off",
8678
+ "never",
8679
+ "false",
8680
+ "none"
8681
+ ];
8682
+ /**
8683
+ * The retention window, or undefined when an operator turned it off.
8684
+ *
8685
+ * Unset means the default. An unparseable value also means the default rather
8686
+ * than silently disabling: the variable's purpose is a bound, and a typo must
8687
+ * not remove one.
8688
+ */
7931
8689
  function runRetentionMs(env = process.env) {
7932
- const days = Number(env[RUN_RETENTION_ENV]);
7933
- return days > 0 ? days * DAY_MS : void 0;
8690
+ const raw = env[RUN_RETENTION_ENV]?.trim();
8691
+ if (raw !== void 0 && RETENTION_OFF_SPELLINGS.includes(raw.toLowerCase())) return;
8692
+ const days = Number(raw);
8693
+ return (days > 0 ? days : 30) * DAY_MS;
7934
8694
  }
7935
8695
  async function sweepRetention(store, options) {
7936
8696
  const now = (options.now ?? /* @__PURE__ */ new Date()).getTime();
@@ -8026,18 +8786,23 @@ var TriggerSupervisor = class {
8026
8786
  hookTimeoutMs;
8027
8787
  now;
8028
8788
  egress;
8789
+ parks;
8029
8790
  timer;
8030
8791
  ops = Promise.resolve();
8031
8792
  ticking = false;
8032
8793
  warnedMissingJournal = false;
8033
8794
  constructor(options) {
8034
8795
  this.options = options;
8035
- this.worker = options.worker ?? new PieceWorker({ models: options.models });
8796
+ this.worker = options.worker ?? new PieceWorker({
8797
+ models: options.models,
8798
+ telemetry: options.telemetry
8799
+ });
8036
8800
  this.tickMs = options.tickMs ?? 15e3;
8037
8801
  this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
8038
8802
  this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
8039
8803
  this.now = options.now ?? (() => /* @__PURE__ */ new Date());
8040
8804
  this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
8805
+ this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
8041
8806
  }
8042
8807
  start() {
8043
8808
  if (this.timer) return;
@@ -8067,28 +8832,84 @@ var TriggerSupervisor = class {
8067
8832
  return this.options.resolver ?? (this.own ??= bundleResolver({ cacheDir: this.options.cacheDir }));
8068
8833
  }
8069
8834
  enableRetries = /* @__PURE__ */ new Map();
8070
- upsert(binding) {
8835
+ versions = /* @__PURE__ */ new Map();
8836
+ upsert(binding, publishedVersion) {
8837
+ if (publishedVersion !== void 0) this.versions.set(binding.workflowId, publishedVersion);
8071
8838
  const previous = this.bindings.get(binding.workflowId);
8072
8839
  if (previous && this.enabledOk.has(binding.workflowId) && JSON.stringify(previous) === JSON.stringify(binding)) return Promise.resolve();
8073
8840
  this.bindings.set(binding.workflowId, binding);
8841
+ const park = this.parks.known(binding.workflowId)?.park;
8842
+ if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
8074
8843
  return this.enqueue(() => this.enable(binding, previous));
8075
8844
  }
8845
+ /** Parks a workflow; true when a trigger row was parked with it. */
8846
+ park(workflowId, publishedVersion, reason) {
8847
+ this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
8848
+ return this.enqueue(async () => {
8849
+ const store = await this.options.store();
8850
+ if (!store) return false;
8851
+ return store.parkWorkflow(workflowId, publishedVersion, reason);
8852
+ });
8853
+ }
8854
+ /** Undoes {@link park} for that version only: a later park stands. */
8855
+ liftPark(workflowId, publishedVersion, trigger) {
8856
+ if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
8857
+ return this.enqueue(async () => {
8858
+ const store = await this.options.store();
8859
+ if (!store) return;
8860
+ if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
8861
+ await store.liftPark(workflowId, trigger);
8862
+ });
8863
+ }
8864
+ /** A disable clears the park. With no binding in memory to release a PARKED
8865
+ * row through onDisable (a boot), the row is turned DISABLED here. */
8866
+ releasePark(workflowId, flipRow) {
8867
+ this.parks.delete(workflowId);
8868
+ return this.enqueue(async () => {
8869
+ const store = await this.options.store();
8870
+ if (!store) return;
8871
+ if (flipRow) await store.clearParkOnDisable(workflowId);
8872
+ else await store.clearWorkflowPark(workflowId);
8873
+ });
8874
+ }
8875
+ /** A re-publish past the parked version: on this lane, a park older than
8876
+ * `publishedVersion` goes and a row it parked is ENABLED again, as it was
8877
+ * before (only an ENABLED row is ever parked); a park as new stands. The
8878
+ * next upsert of the same binding then arms rather than skipping as
8879
+ * unchanged, and a retry held back by the park is due now. */
8880
+ unpark(workflowId, publishedVersion) {
8881
+ this.enabledOk.delete(workflowId);
8882
+ const retry = this.enableRetries.get(workflowId);
8883
+ if (retry) retry.at = 0;
8884
+ const park = this.parks.known(workflowId)?.park;
8885
+ if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
8886
+ return this.enqueue(async () => {
8887
+ const store = await this.options.store();
8888
+ if (!store) return;
8889
+ const park = await store.getWorkflowPark(workflowId);
8890
+ if (!park || park.published_version >= publishedVersion) return;
8891
+ await store.liftPark(workflowId, true);
8892
+ });
8893
+ }
8076
8894
  remove(workflowId) {
8077
8895
  const binding = this.unbind(workflowId);
8078
8896
  return this.enqueue(() => this.disable(workflowId, binding));
8079
8897
  }
8080
8898
  forget(workflowId) {
8081
8899
  const binding = this.unbind(workflowId);
8900
+ this.parks.delete(workflowId);
8082
8901
  return this.enqueue(async () => {
8083
8902
  const store = await this.options.store();
8084
8903
  if (!store) return;
8085
8904
  await this.disable(workflowId, binding);
8086
8905
  await store.deleteTriggerState(workflowId);
8906
+ await store.clearWorkflowPark(workflowId);
8087
8907
  });
8088
8908
  }
8089
8909
  unbind(workflowId) {
8090
8910
  const binding = this.bindings.get(workflowId);
8091
8911
  this.bindings.delete(workflowId);
8912
+ this.versions.delete(workflowId);
8092
8913
  this.enabledOk.delete(workflowId);
8093
8914
  this.enableRetries.delete(workflowId);
8094
8915
  return binding;
@@ -8106,6 +8927,10 @@ var TriggerSupervisor = class {
8106
8927
  logger$1.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
8107
8928
  return;
8108
8929
  }
8930
+ if (existing?.status === "PARKED" && existing.config_hash === hash) {
8931
+ logger$1.warn(`Workflow ${workflowId} is PARKED and could not be resolved; the park stands. @error`, message);
8932
+ return;
8933
+ }
8109
8934
  await store.upsertTriggerState({
8110
8935
  workflow_id: workflowId,
8111
8936
  ...triggerBlockColumns(block),
@@ -8258,8 +9083,14 @@ var TriggerSupervisor = class {
8258
9083
  const store = await this.options.store();
8259
9084
  if (!store) throw new MissingJournalError("Enabling a trigger");
8260
9085
  const hash = configHash(binding.block, binding.config);
8261
- const existing = await store.getTriggerState(binding.workflowId);
8262
9086
  const now = this.now();
9087
+ const park = await store.getWorkflowPark(binding.workflowId);
9088
+ if (park && await this.blockingPark(store, binding.workflowId)) {
9089
+ this.holdParked(binding, superseded, park);
9090
+ return;
9091
+ }
9092
+ if (park) await store.liftPark(binding.workflowId, true);
9093
+ const existing = await store.getTriggerState(binding.workflowId);
8263
9094
  const isRepublish = existing?.config_hash === hash && existing.status === "ENABLED" && !store.hasUnmigratedTriggerState(binding.workflowId);
8264
9095
  if (existing && !isRepublish && existing.status === "ENABLED") await this.disableRow(binding.workflowId, existing, superseded);
8265
9096
  const pending = this.enableRetries.get(binding.workflowId);
@@ -8290,7 +9121,7 @@ var TriggerSupervisor = class {
8290
9121
  workflow_id: binding.workflowId,
8291
9122
  ...triggerBlockColumns(binding.block),
8292
9123
  config_hash: hash,
8293
- status: "ENABLED",
9124
+ status: await this.armedStatus(store, binding.workflowId),
8294
9125
  store_state: VESTIGIAL_STORE_STATE,
8295
9126
  interval_ms: intervalMs,
8296
9127
  next_poll_at: new Date(now.getTime() + intervalMs).toISOString(),
@@ -8352,6 +9183,43 @@ var TriggerSupervisor = class {
8352
9183
  if (!url) throw new Error("The reactor's webhook endpoint is not available yet for this workflow");
8353
9184
  return url;
8354
9185
  }
9186
+ /**
9187
+ * The park, when it holds this registration back.
9188
+ *
9189
+ * `policy.onFailure = PARK` is a **runtime override of the document's
9190
+ * enabled-ness**: the document still says ENABLED, and re-arming from it is
9191
+ * what a restart does for every workflow it finds. So the park row is the
9192
+ * authority, read on this lane, and it blocks every registration of the
9193
+ * version that failed or an older one — a boot re-arm, an ERROR row's
9194
+ * retry. A newer version is what a re-publish is: it is never blocked, and
9195
+ * {@link enable} lifts the park it outlived. A re-enable lifts it through
9196
+ * the disable before it.
9197
+ */
9198
+ async blockingPark(store, workflowId) {
9199
+ const park = await store.getWorkflowPark(workflowId);
9200
+ if (!park) return void 0;
9201
+ const version = this.versions.get(workflowId);
9202
+ return version === void 0 || version <= park.published_version ? park : void 0;
9203
+ }
9204
+ /**
9205
+ * An enable a park held back. The row is left as it stands, and a pending
9206
+ * retry keeps its release flag, so the enable after the park lifts releases
9207
+ * a failed attempt and arms afresh; the retry is only pushed back, so it
9208
+ * does not take every tick's slot. A binding this replaced is restored, so
9209
+ * nothing is released later with one that was never armed.
9210
+ */
9211
+ holdParked(binding, superseded, park) {
9212
+ const { workflowId } = binding;
9213
+ const retry = this.enableRetries.get(workflowId);
9214
+ if (retry) retry.at = this.now().getTime() + this.defaultIntervalMs;
9215
+ if (superseded && this.bindings.get(workflowId) === binding) this.bindings.set(workflowId, superseded);
9216
+ logger$1.warn(`Workflow ${workflowId} is PARKED at version ${park.published_version}; its trigger is not armed until the workflow is re-published or re-enabled. @error`, park.reason);
9217
+ }
9218
+ /** The status an enable that succeeded writes: never ENABLED while a park
9219
+ * stands, whatever ran meanwhile. The registration stays recorded. */
9220
+ async armedStatus(store, workflowId) {
9221
+ return await this.blockingPark(store, workflowId) ? PARKED_TRIGGER_STATUS : "ENABLED";
9222
+ }
8355
9223
  deferToStoredRetry(binding, hash, existing, now) {
8356
9224
  if (existing?.status !== "ERROR" || existing.config_hash !== hash) return false;
8357
9225
  const pending = this.enableRetries.get(binding.workflowId);
@@ -8394,7 +9262,7 @@ var TriggerSupervisor = class {
8394
9262
  const nextAt = carried ? new Date(carried) : nextFireAt(schedule, now);
8395
9263
  await store.upsertTriggerState({
8396
9264
  ...base,
8397
- status: "ENABLED",
9265
+ status: await this.armedStatus(store, binding.workflowId),
8398
9266
  interval_ms: schedule.mode === "interval" ? schedule.everyMs : MIN_SCHEDULE_INTERVAL_MS,
8399
9267
  next_poll_at: nextAt.toISOString(),
8400
9268
  last_error: null,
@@ -8943,6 +9811,17 @@ function parseWorkflowState(resultingState) {
8943
9811
  return;
8944
9812
  }
8945
9813
  }
9814
+ function refusalStatus(refusal) {
9815
+ switch (refusal) {
9816
+ case "parked":
9817
+ case "singleton":
9818
+ case "stale": return 409;
9819
+ case "queue-full":
9820
+ case "expired": return 429;
9821
+ default: return 500;
9822
+ }
9823
+ }
9824
+ const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
8946
9825
  function registrationKey(state) {
8947
9826
  const trigger = runnableDefinition(state).trigger;
8948
9827
  const { lastTest: _lastTest, ...armed } = trigger ?? {};
@@ -8961,9 +9840,11 @@ function runJournalName(stateName, documentName) {
8961
9840
  var WorkflowRuntimeService = class {
8962
9841
  host;
8963
9842
  logger;
9843
+ telemetry;
8964
9844
  attachments;
8965
9845
  executor;
8966
9846
  pieceWorkers;
9847
+ unobservePool;
8967
9848
  storePromise;
8968
9849
  storeError;
8969
9850
  storeOpening = false;
@@ -8978,13 +9859,31 @@ var WorkflowRuntimeService = class {
8978
9859
  seedError;
8979
9860
  constructor(host) {
8980
9861
  this.host = host;
9862
+ this.telemetry = new WorkflowTelemetry(host.telemetry);
8981
9863
  this.blockResolver = new BlockResolver({
8982
9864
  local: installedPiece,
8983
9865
  timeoutMs: host.pieceVersionLookupMs ?? PIECE_VERSION_LOOKUP_TIMEOUT_MS,
8984
9866
  hasBlock: (ref, version, source) => this.versionHasBlock(ref, version, source)
8985
9867
  });
8986
9868
  this.logger = host.logger ?? logger;
8987
- this.attachments = host.attachments ? createAttachmentPort(host.attachments, () => currentWorkflowId(), (documentId, ref) => host.canReadAttachmentRef?.(documentId, ref) ?? Promise.resolve(false)) : void 0;
9869
+ this.attachments = host.attachments ? createAttachmentPort(host.attachments, {
9870
+ documentIdFor: () => currentWorkflowId(),
9871
+ canReadRef: async (ref) => {
9872
+ const workflowId = currentWorkflowId();
9873
+ const origins = currentAttachmentOrigins();
9874
+ if (!workflowId) return false;
9875
+ if (origins?.written.has(ref)) return true;
9876
+ if (!host.canReadAttachmentRef) return false;
9877
+ const runUser = currentRunUser();
9878
+ return host.canReadAttachmentRef({
9879
+ workflowId,
9880
+ ref,
9881
+ documentIds: [...new Set(origins?.documentIds() ?? [])],
9882
+ ...runUser !== void 0 ? { runUser } : {}
9883
+ });
9884
+ },
9885
+ onWritten: (ref) => currentAttachmentOrigins()?.written.add(ref)
9886
+ }) : void 0;
8988
9887
  this.storePromise = this.openStore();
8989
9888
  this.seedPromise = this.seedWithRetries();
8990
9889
  this.startRetention();
@@ -9093,9 +9992,22 @@ var WorkflowRuntimeService = class {
9093
9992
  return await this.endpointCount() > 0;
9094
9993
  }
9095
9994
  registeredAs = /* @__PURE__ */ new Map();
9096
- async updateRegistration(workflowId, state, onlyIfChanged = false) {
9995
+ registrationChains = /* @__PURE__ */ new Map();
9996
+ inRegistrationOrder(workflowId, task) {
9997
+ const run = (this.registrationChains.get(workflowId) ?? Promise.resolve()).then(task);
9998
+ const tail = run.then(() => void 0, () => void 0);
9999
+ this.registrationChains.set(workflowId, tail);
10000
+ tail.then(() => {
10001
+ if (this.registrationChains.get(workflowId) === tail) this.registrationChains.delete(workflowId);
10002
+ });
10003
+ return run;
10004
+ }
10005
+ updateRegistration(workflowId, state, onlyIfChanged = false) {
10006
+ return this.inRegistrationOrder(workflowId, () => this.registerNow(workflowId, state, onlyIfChanged));
10007
+ }
10008
+ async registerNow(workflowId, state, onlyIfChanged) {
9097
10009
  const key = registrationKey(state);
9098
- if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key) return;
10010
+ if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key && !await this.outdatedPark(workflowId, state)) return;
9099
10011
  this.registeredAs.set(workflowId, key);
9100
10012
  try {
9101
10013
  await this.applyRegistration(workflowId, state);
@@ -9114,6 +10026,23 @@ var WorkflowRuntimeService = class {
9114
10026
  }
9115
10027
  const trigger = state.status === "ENABLED" ? runnableDefinition(state).trigger : void 0;
9116
10028
  const block = trigger ? triggerBlock(trigger) : void 0;
10029
+ const version = runnableDefinition(state).version;
10030
+ if (state.status !== "ENABLED") {
10031
+ const held = this.registry.get(workflowId);
10032
+ this.supervisor().releasePark(workflowId, !(held && SUPERVISED_KINDS.has(held.kind))).catch((error) => {
10033
+ this.logger.error(`Could not clear the park of disabled workflow ${workflowId}`, error);
10034
+ });
10035
+ } else if (await this.outdatedPark(workflowId, state)) this.supervisor().unpark(workflowId, version).catch((error) => {
10036
+ this.logger.error(`Could not lift the outdated park of workflow ${workflowId}`, error);
10037
+ });
10038
+ const park = block ? await this.parks.get(workflowId) : void 0;
10039
+ if (block && (blockKey(block) === WEBHOOK_BLOCK || triggerKindOf(block)) && park && park.published_version >= version) {
10040
+ const had = this.registry.get(workflowId);
10041
+ this.registry.delete(workflowId);
10042
+ if (had && SUPERVISED_KINDS.has(had.kind)) this.dropSupervised(workflowId);
10043
+ this.logger.warn(`Workflow ${workflowId} is PARKED; its trigger stays unarmed until the workflow is re-published or re-enabled`);
10044
+ return;
10045
+ }
9117
10046
  if (block && blockKey(block) === WEBHOOK_BLOCK) {
9118
10047
  await this.registerWebhook(workflowId, block, trigger.config);
9119
10048
  return;
@@ -9136,14 +10065,14 @@ var WorkflowRuntimeService = class {
9136
10065
  workflowId,
9137
10066
  kind: "schedule"
9138
10067
  });
9139
- this.enableSupervised(workflowId, supervised);
10068
+ this.enableSupervised(workflowId, supervised, version);
9140
10069
  return;
9141
10070
  }
9142
10071
  this.registry.set(workflowId, {
9143
10072
  workflowId,
9144
10073
  kind: "piece"
9145
10074
  });
9146
- await this.registerPieceTrigger(workflowId, supervised);
10075
+ await this.registerPieceTrigger(workflowId, supervised, version);
9147
10076
  return;
9148
10077
  }
9149
10078
  const had = this.registry.get(workflowId);
@@ -9159,12 +10088,12 @@ var WorkflowRuntimeService = class {
9159
10088
  filter: parseLifecycleFilter(config)
9160
10089
  });
9161
10090
  }
9162
- enableSupervised(workflowId, binding) {
9163
- this.supervisor().upsert(binding).catch((error) => {
10091
+ enableSupervised(workflowId, binding, publishedVersion) {
10092
+ this.supervisor().upsert(binding, publishedVersion).catch((error) => {
9164
10093
  this.logger.error(`Trigger enable failed for ${workflowId}`, error);
9165
10094
  });
9166
10095
  }
9167
- async registerPieceTrigger(workflowId, binding) {
10096
+ async registerPieceTrigger(workflowId, binding, publishedVersion) {
9168
10097
  const delivery = await this.pieceDelivery(binding);
9169
10098
  if (typeof delivery !== "string") {
9170
10099
  this.refusePieceTrigger(workflowId, binding, delivery);
@@ -9182,7 +10111,7 @@ var WorkflowRuntimeService = class {
9182
10111
  });
9183
10112
  await (await this.endpoints())?.endpointFor(workflowId);
9184
10113
  }
9185
- this.enableSupervised(workflowId, resolved);
10114
+ this.enableSupervised(workflowId, resolved, publishedVersion);
9186
10115
  }
9187
10116
  async pieceDelivery(binding) {
9188
10117
  let strategy;
@@ -9434,7 +10363,7 @@ var WorkflowRuntimeService = class {
9434
10363
  throw new Error("Erasing purged documents needs the run journal", { cause: this.storeError });
9435
10364
  }
9436
10365
  for (const workflowId of purgedWorkflowIds(markers)) await this.erasePurgedWorkflow(store, workflowId);
9437
- const erased = await store.eraseRunsForDocuments(documentIds);
10366
+ const erased = await this.telemetry.phase("journal.purge", { "documents.count": documentIds.length }, () => store.eraseRunsForDocuments(documentIds));
9438
10367
  if (erased.runs > 0) this.logger.info("Erased @runs workflow run(s) of purged documents @ids", erased.runs, documentIds);
9439
10368
  return erased;
9440
10369
  }
@@ -9498,7 +10427,7 @@ var WorkflowRuntimeService = class {
9498
10427
  if (this.unjournaledFires.has(fireKey)) {
9499
10428
  if (!store) return;
9500
10429
  try {
9501
- await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString());
10430
+ if (!await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString())) this.logger.warn(`Workflow ${workflowId}: a ${kind} fire this process ran unjournaled had its key already marked fired; it may have run twice`);
9502
10431
  this.unjournaledFires.delete(fireKey);
9503
10432
  } catch (error) {
9504
10433
  this.logger.warn(`Could not record the unjournaled ${kind} fire for workflow ${workflowId}: @error`, error);
@@ -9509,26 +10438,39 @@ var WorkflowRuntimeService = class {
9509
10438
  this.fireUnjournaled(fireKey, workflowId, payload, kind);
9510
10439
  return;
9511
10440
  }
9512
- let runId;
10441
+ const enqueue = {
10442
+ workflowId,
10443
+ triggerKind: kind,
10444
+ triggerPayload: payload
10445
+ };
10446
+ let claim;
9513
10447
  try {
9514
- runId = await store.claimAndEnqueueRun(dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString(), {
9515
- workflowId,
9516
- triggerKind: kind,
9517
- triggerPayload: payload
9518
- });
10448
+ claim = await store.claimAndEnqueueRun(dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString(), enqueue);
9519
10449
  } catch (error) {
9520
10450
  this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
9521
10451
  this.fireUnjournaled(fireKey, workflowId, payload, kind);
9522
10452
  return;
9523
10453
  }
9524
- if (runId === null) return;
9525
- this.fireFromTrigger(workflowId, payload, kind, runId);
10454
+ if (claim.outcome === "duplicate") return;
10455
+ if (claim.outcome === "abandoned") {
10456
+ this.logger.error(`Workflow ${workflowId}: a ${kind} fire has been delivered ${claim.attempts} times without ever journaling a run; abandoning it rather than replaying it on every boot`);
10457
+ try {
10458
+ await store.journalAbandonedFire(dedupeKey, {
10459
+ ...enqueue,
10460
+ attempts: claim.attempts
10461
+ });
10462
+ } catch (error) {
10463
+ this.logger.warn(`Could not journal the abandoned ${kind} fire for workflow ${workflowId}: @error`, error);
10464
+ }
10465
+ return;
10466
+ }
10467
+ this.fireFromTrigger(workflowId, payload, kind, claim.runId);
9526
10468
  }
9527
10469
  fireFromTrigger(workflowId, payload, kind, enqueuedRunId) {
9528
10470
  this.fire(workflowId, payload, kind, void 0, void 0, enqueuedRunId).then((run) => {
9529
10471
  this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
9530
10472
  }, (error) => {
9531
- this.logger.error(`${kind} run failed for workflow ${workflowId}`, error);
10473
+ this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
9532
10474
  });
9533
10475
  }
9534
10476
  firedLifecycle = /* @__PURE__ */ new Set();
@@ -9607,8 +10549,14 @@ var WorkflowRuntimeService = class {
9607
10549
  this.cancelTriggerTest(workflowId, "stopped: the workflow was deleted");
9608
10550
  }
9609
10551
  async releaseDeleted(workflowId) {
9610
- await this.supervisor().forget(workflowId);
10552
+ let failure;
10553
+ try {
10554
+ await this.supervisor().forget(workflowId);
10555
+ } catch (error) {
10556
+ failure = { error };
10557
+ }
9611
10558
  await (await this.endpoints())?.revoke(workflowId);
10559
+ if (failure) throw failure.error;
9612
10560
  }
9613
10561
  async erasePurgedWorkflow(store, workflowId) {
9614
10562
  this.dropDeleted(workflowId);
@@ -9666,9 +10614,11 @@ var WorkflowRuntimeService = class {
9666
10614
  });
9667
10615
  }
9668
10616
  triggerSupervisor;
10617
+ parks = new ParkState(async () => (await this.store())?.listWorkflowParks());
9669
10618
  supervisor() {
9670
10619
  this.triggerSupervisor ??= new TriggerSupervisor({
9671
10620
  store: () => this.store(),
10621
+ parks: this.parks,
9672
10622
  resolveAuth: async (connectionId, request) => {
9673
10623
  if (!connectionId) return void 0;
9674
10624
  const resolved = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolveWithSecrets(connectionId, request);
@@ -9680,6 +10630,7 @@ var WorkflowRuntimeService = class {
9680
10630
  webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
9681
10631
  reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
9682
10632
  models: this.models,
10633
+ telemetry: this.telemetry,
9683
10634
  cacheDir: bundleCacheDir(),
9684
10635
  resolver: pieceResolver(),
9685
10636
  egress: configuredEgress(),
@@ -9701,6 +10652,8 @@ var WorkflowRuntimeService = class {
9701
10652
  for (const workflowId of [...this.webhookTests.keys()]) this.cancelTriggerTest(workflowId, "stopped: the runtime shut down");
9702
10653
  this.stopTriggerSupervisor();
9703
10654
  this.pieceWorkers?.dispose();
10655
+ this.unobservePool?.();
10656
+ this.unobservePool = void 0;
9704
10657
  this.designWorker?.dispose();
9705
10658
  this.designWorker = void 0;
9706
10659
  }
@@ -9847,7 +10800,7 @@ var WorkflowRuntimeService = class {
9847
10800
  };
9848
10801
  }
9849
10802
  return {
9850
- status: run.result.status === "SUCCEEDED" ? config.responseStatus : 500,
10803
+ status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
9851
10804
  contentType: JSON_CONTENT_TYPE,
9852
10805
  body: JSON.stringify({
9853
10806
  runId: run.result.runId,
@@ -9933,14 +10886,14 @@ var WorkflowRuntimeService = class {
9933
10886
  async stepReactorAccess(request) {
9934
10887
  const declared = await this.declaredReactor(request.block);
9935
10888
  if (!declared) return void 0;
9936
- const base = await this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null);
9937
- return reactorTap(this.host, base, runJournal());
10889
+ const base = await this.telemetry.phase("reactor.scope", {}, () => this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null));
10890
+ return reactorTap(this.host, base, runJournal(), {}, this.telemetry);
9938
10891
  }
9939
10892
  async triggerReactorAccess(binding, requireReactor, runUser) {
9940
10893
  const bound = binding.reactorConnectionId;
9941
10894
  const user = runUser !== void 0 ? runUser : await assertReactorConnectionsReadable(this.host, binding.workflowId, new Set(bound ? [bound] : [])) ?? null;
9942
10895
  const base = await this.reactorScope(bound, requireReactor, user);
9943
- return reactorTap(this.host, base, NO_JOURNAL);
10896
+ return reactorTap(this.host, base, NO_JOURNAL, {}, this.telemetry);
9944
10897
  }
9945
10898
  async designReactorAccess(block, ctx, reactorConnectionId) {
9946
10899
  if (!await this.declaredReactor(block)) return void 0;
@@ -9951,6 +10904,12 @@ var WorkflowRuntimeService = class {
9951
10904
  }
9952
10905
  descriptors = /* @__PURE__ */ new Map();
9953
10906
  designWorker;
10907
+ createDesignWorker() {
10908
+ return new PieceWorker({
10909
+ models: this.models,
10910
+ telemetry: this.telemetry
10911
+ });
10912
+ }
9954
10913
  designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
9955
10914
  describing = /* @__PURE__ */ new Map();
9956
10915
  pieceDescriptor(target) {
@@ -9971,7 +10930,7 @@ var WorkflowRuntimeService = class {
9971
10930
  let descriptor = this.descriptors.get(cacheKey);
9972
10931
  if (!descriptor) {
9973
10932
  const piece = await pieceResolver().resolve(target);
9974
- this.designWorker ??= new PieceWorker({ models: this.models });
10933
+ this.designWorker ??= this.createDesignWorker();
9975
10934
  let output;
9976
10935
  try {
9977
10936
  output = (await this.designWorker.describePiece({
@@ -10002,7 +10961,14 @@ var WorkflowRuntimeService = class {
10002
10961
  async runs(args, ctx) {
10003
10962
  return (await this.runsPage(args, ctx)).records;
10004
10963
  }
10005
- async runsPage(args, ctx) {
10964
+ runsPage(args, ctx) {
10965
+ return this.telemetry.phase("runs.page", { "runs.scope": args.workflowId ? "workflow" : args.driveId ? "drive" : "all" }, async (span) => {
10966
+ const page = await this.listRunsPage(args, ctx);
10967
+ span.setAttribute("runs.returned", page.records.length);
10968
+ return page;
10969
+ });
10970
+ }
10971
+ async listRunsPage(args, ctx) {
10006
10972
  const empty = {
10007
10973
  records: [],
10008
10974
  hasNextPage: false,
@@ -10158,7 +11124,7 @@ var WorkflowRuntimeService = class {
10158
11124
  }
10159
11125
  let outcome;
10160
11126
  try {
10161
- this.designWorker ??= new PieceWorker({ models: this.models });
11127
+ this.designWorker ??= this.createDesignWorker();
10162
11128
  outcome = (await this.designWorker.checkConnection({
10163
11129
  ...moduleRef,
10164
11130
  auth: shapedAuth,
@@ -10447,7 +11413,7 @@ var WorkflowRuntimeService = class {
10447
11413
  if (connectionId) auth = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolve(connectionId, { piecePackage: parsed.packageName });
10448
11414
  const piece = await pieceResolver().resolve(targetOf(parsed));
10449
11415
  const reactor = await this.designReactorAccess(parsed, ctx, reactorConnectionId);
10450
- this.designWorker ??= new PieceWorker({ models: this.models });
11416
+ this.designWorker ??= this.createDesignWorker();
10451
11417
  return (await this.designWorker.resolveOptions({
10452
11418
  ...pieceModuleRef(piece),
10453
11419
  actionName: parsed.name,
@@ -10618,7 +11584,7 @@ var WorkflowRuntimeService = class {
10618
11584
  } catch (error) {
10619
11585
  const errorName = errorNameOf(error);
10620
11586
  await recordTest({
10621
- status: "FAILED",
11587
+ status: isIndeterminateError(error) ? "INDETERMINATE" : "FAILED",
10622
11588
  error: pieceFailureDetail(error, "Trigger test timed out"),
10623
11589
  ...errorName ? { errorName } : {}
10624
11590
  });
@@ -10709,10 +11675,11 @@ var WorkflowRuntimeService = class {
10709
11675
  triggerKind: TEST_TRIGGER_KIND
10710
11676
  });
10711
11677
  await store.recordStep(runId, 0, record);
11678
+ const error = record.status === "INDETERMINATE" ? `This test is INDETERMINATE: ${record.error ?? "a host call it made timed out"}` : record.error;
10712
11679
  await store.finishRun(runId, {
10713
- status: record.status === "FAILED" ? "FAILED" : "SUCCEEDED",
11680
+ status: record.status === "SUCCEEDED" ? "SUCCEEDED" : "FAILED",
10714
11681
  steps: [record],
10715
- ...record.error ? { error: record.error } : {},
11682
+ ...error ? { error } : {},
10716
11683
  ...record.errorName ? { errorName: record.errorName } : {}
10717
11684
  });
10718
11685
  } catch (error) {
@@ -10733,30 +11700,76 @@ var WorkflowRuntimeService = class {
10733
11700
  }
10734
11701
  }
10735
11702
  blockExecutor(store) {
10736
- return this.executor ??= createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request));
11703
+ return this.executor ??= tracedSteps(createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request), this.telemetry), this.telemetry);
10737
11704
  }
10738
11705
  workers() {
10739
- return this.pieceWorkers ??= new PieceWorkerPool({
11706
+ if (this.pieceWorkers) return this.pieceWorkers;
11707
+ const pool = new PieceWorkerPool({
10740
11708
  models: this.models,
10741
11709
  size: Number(process.env.PH_WORKFLOWS_RUN_CONCURRENCY) || void 0,
10742
- maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0
11710
+ maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0,
11711
+ telemetry: this.telemetry
10743
11712
  });
11713
+ this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
11714
+ return this.pieceWorkers = pool;
10744
11715
  }
10745
11716
  async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
11717
+ const started = performance.now();
11718
+ const attributes = {
11719
+ "workflow.id": workflowId,
11720
+ "trigger.kind": triggerKind,
11721
+ ...resume ? { "run.rerun_of": resume.rerunOf } : {}
11722
+ };
11723
+ return this.telemetry.span("workflow.run", attributes, async (span) => {
11724
+ const run = {
11725
+ span,
11726
+ refused: false
11727
+ };
11728
+ let status = "FAILED";
11729
+ try {
11730
+ const result = await this.runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId);
11731
+ status = result.status;
11732
+ if (status === "FAILED") span.setStatus({
11733
+ code: SpanStatusCode.ERROR,
11734
+ message: status
11735
+ });
11736
+ return result;
11737
+ } finally {
11738
+ if (run.refused) status = "REFUSED";
11739
+ span.setAttribute("run.status", status);
11740
+ this.telemetry.recordRun(performance.now() - started, status, triggerKind);
11741
+ }
11742
+ });
11743
+ }
11744
+ async runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId) {
11745
+ const firedAt = Date.now();
10746
11746
  const store = await this.store();
10747
11747
  let state;
10748
11748
  let documentName;
10749
11749
  let definition;
10750
11750
  let runUser;
10751
11751
  try {
10752
- if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx);
10753
- const document = await this.host.reactorClient.get(workflowId);
10754
- if (document.header.documentType !== "powerhouse/workflow") throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
11752
+ if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
11753
+ run.refused = true;
11754
+ throw error;
11755
+ });
11756
+ const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
11757
+ if (document.header.documentType !== "powerhouse/workflow") {
11758
+ run.refused = true;
11759
+ throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
11760
+ }
10755
11761
  state = document.state.global;
10756
11762
  documentName = document.header.name;
10757
- if (state.status !== "ENABLED") throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
11763
+ if (state.status !== "ENABLED") {
11764
+ run.refused = true;
11765
+ throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
11766
+ }
10758
11767
  definition = toWorkflowDefinition(state);
10759
- runUser = await assertReactorConnectionsReadable(this.host, workflowId, declaredReactorConnectionIds(definition));
11768
+ const reactorConnections = declaredReactorConnectionIds(definition);
11769
+ runUser = reactorConnections.size === 0 ? void 0 : await this.telemetry.phase("run_user", {}, () => assertReactorConnectionsReadable(this.host, workflowId, reactorConnections)).catch((error) => {
11770
+ run.refused = isReactorError(error, ReactorAccessDeniedError$1);
11771
+ throw error;
11772
+ });
10760
11773
  } catch (error) {
10761
11774
  if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
10762
11775
  throw error;
@@ -10764,23 +11777,64 @@ var WorkflowRuntimeService = class {
10764
11777
  const connections = declaredConnectionIds(definition);
10765
11778
  const executor = this.blockExecutor(store);
10766
11779
  const runnable = runnableDefinition(state);
10767
- let runId = enqueuedRunId ?? null;
10768
- if (enqueuedRunId) await store?.beginRun(enqueuedRunId, {
10769
- workflowName: runJournalName(state.name, documentName),
10770
- workflowVersion: runnable.version
10771
- });
10772
- else runId = await store?.startRun({
10773
- workflowId,
10774
- workflowName: runJournalName(state.name, documentName),
10775
- workflowVersion: runnable.version,
11780
+ const policy = effectiveRunPolicy(runnable);
11781
+ const deadline = policy.runTimeoutSeconds ? firedAt + policy.runTimeoutSeconds * 1e3 : void 0;
11782
+ const skipped = (reason, refusal) => this.skipFiring(store, workflowId, enqueuedRunId, {
11783
+ reason,
10776
11784
  triggerKind,
10777
11785
  triggerPayload,
10778
- rerunOf: resume?.rerunOf
10779
- }) ?? null;
11786
+ workflowName: runJournalName(state.name, documentName),
11787
+ workflowVersion: runnable.version
11788
+ }, refusal);
11789
+ let parked;
11790
+ try {
11791
+ parked = await this.parkedFiring(workflowId, triggerKind, runnable.version);
11792
+ } catch (error) {
11793
+ if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
11794
+ throw error;
11795
+ }
11796
+ if (parked) return skipped(parked, "parked");
11797
+ const admission = await this.runGate.admit(workflowId, policy);
11798
+ if (!admission.admitted) return skipped(admission.reason, admission.refusal);
11799
+ if (admission.waited) {
11800
+ let stale;
11801
+ try {
11802
+ stale = await this.staleAfterWait(workflowId, triggerKind, runnable.version);
11803
+ } catch (error) {
11804
+ stale = `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
11805
+ }
11806
+ if (stale) {
11807
+ admission.release();
11808
+ return skipped(stale, "stale");
11809
+ }
11810
+ }
11811
+ if (deadline !== void 0 && Date.now() >= deadline) {
11812
+ admission.release();
11813
+ return skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
11814
+ }
11815
+ let runId = enqueuedRunId ?? null;
10780
11816
  let journalFailed = false;
10781
11817
  const executionOrder = /* @__PURE__ */ new Map();
10782
11818
  let session;
10783
11819
  try {
11820
+ runId = await this.telemetry.phase("journal.start", {}, async () => {
11821
+ if (enqueuedRunId) {
11822
+ await store?.beginRun(enqueuedRunId, {
11823
+ workflowName: runJournalName(state.name, documentName),
11824
+ workflowVersion: runnable.version
11825
+ });
11826
+ return enqueuedRunId;
11827
+ }
11828
+ return await store?.startRun({
11829
+ workflowId,
11830
+ workflowName: runJournalName(state.name, documentName),
11831
+ workflowVersion: runnable.version,
11832
+ triggerKind,
11833
+ triggerPayload,
11834
+ rerunOf: resume?.rerunOf
11835
+ }) ?? null;
11836
+ });
11837
+ if (runId) run.span.setAttribute("run.id", runId);
10784
11838
  session = this.workers().session();
10785
11839
  const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
10786
11840
  const journal = store;
@@ -10792,6 +11846,7 @@ var WorkflowRuntimeService = class {
10792
11846
  connections,
10793
11847
  pieceWorker: session,
10794
11848
  ...runUser !== void 0 ? { runUser } : {},
11849
+ attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
10795
11850
  recordDocuments: async (documentIds) => {
10796
11851
  for (const documentId of documentIds) handed.add(documentId);
10797
11852
  if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
@@ -10806,22 +11861,25 @@ var WorkflowRuntimeService = class {
10806
11861
  ...secretValues.length > 0 ? { redactValues: secretValues } : {},
10807
11862
  triggerPayload,
10808
11863
  completedSteps: resume?.completedSteps,
10809
- onStep: store && runId ? async (record, ordinal) => {
11864
+ ...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
11865
+ ...deadline !== void 0 ? { deadline } : {},
11866
+ onStep: store && journaledRunId ? async (record, ordinal) => {
10810
11867
  executionOrder.set(record.stepId, ordinal);
10811
11868
  try {
10812
- await store.recordStep(runId, ordinal, record);
11869
+ await this.telemetry.phase("journal.step", {}, () => store.recordStep(journaledRunId, ordinal, record));
10813
11870
  } catch (error) {
10814
11871
  if (journalFailed) return;
10815
11872
  journalFailed = true;
10816
- this.logger.warn(`Run ${runId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
11873
+ this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
10817
11874
  }
10818
11875
  } : void 0
10819
11876
  }));
10820
- if (store && runId) try {
10821
- await store.finishRun(runId, result, executionOrder);
11877
+ if (store && journaledRunId) try {
11878
+ await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
10822
11879
  } catch (error) {
10823
11880
  this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
10824
11881
  }
11882
+ if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
10825
11883
  const finished = {
10826
11884
  ...result,
10827
11885
  runId
@@ -10838,6 +11896,108 @@ var WorkflowRuntimeService = class {
10838
11896
  throw error;
10839
11897
  } finally {
10840
11898
  session?.close();
11899
+ admission.release();
11900
+ }
11901
+ }
11902
+ runGate = new WorkflowRunGate();
11903
+ /** Why a trigger's firing of a PARKED workflow is refused; undefined when
11904
+ * it is not parked, or when an operator started the run. */
11905
+ async parkedFiring(workflowId, triggerKind, publishedVersion) {
11906
+ if (OPERATOR_RUN_KINDS.has(triggerKind)) return void 0;
11907
+ const park = await this.parks.get(workflowId);
11908
+ if (!park || park.published_version < publishedVersion) return void 0;
11909
+ return `Skipped: this workflow is PARKED (${park.reason})`;
11910
+ }
11911
+ /** Why a firing that waited for its slot may no longer run, if it may not. */
11912
+ async staleAfterWait(workflowId, triggerKind, publishedVersion) {
11913
+ const state = (await this.host.reactorClient.get(workflowId)).state.global;
11914
+ if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
11915
+ if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
11916
+ return this.parkedFiring(workflowId, triggerKind, publishedVersion);
11917
+ }
11918
+ /** Whether the workflow is still ENABLED at this published version. An
11919
+ * unreadable document counts as unchanged, so a failure still parks. */
11920
+ async stillRunsVersion(workflowId, publishedVersion) {
11921
+ let state;
11922
+ try {
11923
+ state = (await this.host.reactorClient.get(workflowId)).state.global;
11924
+ } catch (error) {
11925
+ this.logger.warn(`Could not re-read workflow ${workflowId} before parking it: @error`, error);
11926
+ return true;
11927
+ }
11928
+ return state.status === "ENABLED" && runnableDefinition(state).version === publishedVersion;
11929
+ }
11930
+ /** A park recorded against an earlier published version than this state's. */
11931
+ async outdatedPark(workflowId, state) {
11932
+ if (state.status !== "ENABLED") return false;
11933
+ const park = await this.parks.get(workflowId);
11934
+ return park !== void 0 && runnableDefinition(state).version > park.published_version;
11935
+ }
11936
+ /**
11937
+ * A firing SINGLETON refused.
11938
+ *
11939
+ * Journaled as a CANCELLED run rather than dropped: a firing that vanished
11940
+ * is indistinguishable from a trigger that never fired, which is the class
11941
+ * of bug this work package exists to stamp out. An already-enqueued row is
11942
+ * closed out in place, so nothing is left PENDING for a sweep to find.
11943
+ */
11944
+ async skipFiring(store, workflowId, enqueuedRunId, details, refusal) {
11945
+ this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
11946
+ let runId = enqueuedRunId ?? null;
11947
+ if (store) try {
11948
+ if (runId) await store.beginRun(runId, details);
11949
+ else runId = await store.startRun({
11950
+ workflowId,
11951
+ ...details
11952
+ });
11953
+ await store.cancelRun(runId, details.reason);
11954
+ } catch (error) {
11955
+ this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
11956
+ }
11957
+ return {
11958
+ status: CANCELLED_RUN_STATUS,
11959
+ steps: [],
11960
+ runId,
11961
+ refusal
11962
+ };
11963
+ }
11964
+ /**
11965
+ * `policy.onFailure` for a run that failed terminally.
11966
+ *
11967
+ * PARK is the document model's own default, so this is where enforcing the
11968
+ * knob becomes visible: a terminal failure parks the workflow, whatever its
11969
+ * trigger kind, and nothing fires it until it is re-published or
11970
+ * re-enabled. That is what PARK means, and
11971
+ * leaving a broken workflow firing every minute is what it meant before.
11972
+ *
11973
+ * NOTIFY logs at error level, which is the only notification channel this
11974
+ * engine has; it is marked as such rather than pretending to page anyone.
11975
+ */
11976
+ async applyFailureMode(policy, workflowId, publishedVersion, runId, error) {
11977
+ if (policy.onFailure === "IGNORE") return;
11978
+ const detail = error ?? "the run failed";
11979
+ if (policy.onFailure === "NOTIFY") {
11980
+ this.logger.error(`Workflow ${workflowId} run ${runId ?? "(unjournaled)"} failed and its policy is NOTIFY: @error`, detail);
11981
+ return;
11982
+ }
11983
+ if (!await this.store()) return;
11984
+ const reason = `Parked after a failed run (policy.onFailure = PARK): ${detail}`;
11985
+ try {
11986
+ if (!await this.stillRunsVersion(workflowId, publishedVersion)) return;
11987
+ const trigger = await this.supervisor().park(workflowId, publishedVersion, reason);
11988
+ if (!await this.stillRunsVersion(workflowId, publishedVersion)) {
11989
+ await this.supervisor().liftPark(workflowId, publishedVersion, trigger);
11990
+ return;
11991
+ }
11992
+ if (!await this.inRegistrationOrder(workflowId, async () => {
11993
+ const current = (await this.parks.get(workflowId))?.published_version === publishedVersion;
11994
+ const registered = this.registry.get(workflowId);
11995
+ if (current && registered && !SUPERVISED_KINDS.has(registered.kind)) this.registry.delete(workflowId);
11996
+ return current;
11997
+ })) return;
11998
+ this.logger.error(`Workflow ${workflowId} is PARKED after run ${runId ?? "(unjournaled)"} failed; its trigger will not fire again until the workflow is re-published or re-enabled`);
11999
+ } catch (parkError) {
12000
+ this.logger.warn(`Applying onFailure = PARK to workflow ${workflowId} after a failed run failed; whether it is parked is unknown: @error`, parkError);
10841
12001
  }
10842
12002
  }
10843
12003
  async rerun(runId, ctx) {
@@ -10848,7 +12008,8 @@ var WorkflowRuntimeService = class {
10848
12008
  await this.assertCanReadDocument(run.workflow_id, ctx);
10849
12009
  if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
10850
12010
  if (run.trigger_kind === "test") throw new Error(`Run "${runId}" tested a single step or trigger; test it again instead of rerunning it`);
10851
- if (run.status !== "FAILED") throw new Error(`Only FAILED runs can be rerun; run is ${run.status}`);
12011
+ const deadlineCancelled = run.status === "CANCELLED" && run.error_name === "RunDeadlineExceeded";
12012
+ if (run.status !== "FAILED" && !deadlineCancelled) throw new Error(`Only FAILED runs, and runs their deadline cancelled, can be rerun; run is ${run.status}`);
10852
12013
  const triggerPayload = run.trigger_payload === null ? void 0 : JSON.parse(run.trigger_payload);
10853
12014
  if (isTruncatedStepPayload(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was truncated by the journal and cannot be replayed; fire the workflow again instead of rerunning it`);
10854
12015
  if (containsRedactedMarker(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was redacted and cannot be replayed; fire the workflow again instead of rerunning it`);
@@ -10860,7 +12021,15 @@ var WorkflowRuntimeService = class {
10860
12021
  const current = currentSteps.get(row.step_id);
10861
12022
  if (!current || current.key !== row.step_key || row.config_hash === null || stepConfigHash(current) !== row.config_hash) continue;
10862
12023
  const output = row.output === null ? void 0 : JSON.parse(row.output);
10863
- if (isTruncatedStepPayload(output)) continue;
12024
+ if (isTruncatedStepPayload(output)) {
12025
+ if (await this.rereads(current)) continue;
12026
+ completedSteps.set(row.step_id, {
12027
+ port: row.port,
12028
+ outputTruncated: true,
12029
+ truncatedOutput: output
12030
+ });
12031
+ continue;
12032
+ }
10864
12033
  if (containsDocumentRef(output) && await this.rereads(current)) continue;
10865
12034
  completedSteps.set(row.step_id, {
10866
12035
  output,
@@ -10947,6 +12116,7 @@ var WorkflowRuntimeService = class {
10947
12116
  pieceWorker: session,
10948
12117
  stepTest: true,
10949
12118
  runUser: this.callerRunUser(ctx),
12119
+ attachments: attachmentOrigins(upstream.triggerPayload, handed, upstream.priorSteps),
10950
12120
  recordDocuments: async (documentIds) => {
10951
12121
  for (const documentId of documentIds) handed.add(documentId);
10952
12122
  if (store && journaledRunId) await store.recordRunDocuments(journaledRunId, documentIds);
@@ -10986,7 +12156,7 @@ var WorkflowRuntimeService = class {
10986
12156
  const served = ctx ? await this.servesDocuments(ids, ctx) : false;
10987
12157
  return {
10988
12158
  runId,
10989
- status: record.status === "FAILED" ? "FAILED" : "SUCCEEDED",
12159
+ status: testStatusOf(record.status),
10990
12160
  ...served && record.output !== void 0 ? { output: record.output } : {},
10991
12161
  ...record.error ? { error: record.error } : {},
10992
12162
  ...record.errorName ? { errorName: record.errorName } : {},
@@ -11114,6 +12284,7 @@ var WorkflowRuntimeService = class {
11114
12284
  testedAt,
11115
12285
  error: row.error ?? ""
11116
12286
  };
12287
+ if (row.status === "INDETERMINATE") return { kind: "indeterminate" };
11117
12288
  const output = row.output === null ? null : JSON.parse(row.output);
11118
12289
  if (isTruncatedStepPayload(output)) return { kind: "truncated" };
11119
12290
  return {
@@ -11182,6 +12353,21 @@ var WorkflowRuntimeService = class {
11182
12353
  function createWorkflowRuntime(deps) {
11183
12354
  return new WorkflowRuntimeService(deps);
11184
12355
  }
12356
+ function attachmentOrigins(triggerPayload, handed, replayed) {
12357
+ const written = /* @__PURE__ */ new Set();
12358
+ collectAttachmentRefs(replayed, written);
12359
+ return {
12360
+ documentIds: () => [...triggerDocumentIds(triggerPayload), ...handed],
12361
+ written
12362
+ };
12363
+ }
12364
+ function collectAttachmentRefs(value, found) {
12365
+ if (typeof value === "string") {
12366
+ if (value.startsWith("attachment://")) found.add(value);
12367
+ } else if (Array.isArray(value)) for (const entry of value) collectAttachmentRefs(entry, found);
12368
+ else if (value instanceof Map) for (const entry of value.values()) collectAttachmentRefs(entry, found);
12369
+ else if (typeof value === "object" && value !== null) for (const entry of Object.values(value)) collectAttachmentRefs(entry, found);
12370
+ }
11185
12371
  //#endregion
11186
12372
  //#region src/reactor/workflow-triggers-read-model.ts
11187
12373
  const WORKFLOW_TRIGGERS_READ_MODEL = "workflow-triggers";
@@ -11209,6 +12395,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
11209
12395
  }
11210
12396
  };
11211
12397
  //#endregion
11212
- export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
12398
+ export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
11213
12399
 
11214
12400
  //# sourceMappingURL=index.js.map