@powerhousedao/reactor-workflow 6.2.3-dev.48 → 6.2.3-dev.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +274 -8
- package/dist/index.d.ts +263 -32
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1293 -107
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-e-vFPyTd.js → piece-registry-BKRLK7Kv.js} +1192 -90
- package/dist/piece-registry-BKRLK7Kv.js.map +1 -0
- package/dist/{piece-registry-wQBhKJUl.d.ts → piece-registry-Ds8WkewF.d.ts} +232 -99
- package/dist/piece-registry-Ds8WkewF.d.ts.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +91 -3
- package/dist/testing.js.map +1 -0
- package/dist/worker-entry.js +5685 -285
- package/dist/worker-entry.js.map +1 -1
- package/package.json +16 -8
- package/dist/loader-Gkx6I_Bo.js +0 -91
- package/dist/loader-Gkx6I_Bo.js.map +0 -1
- package/dist/piece-registry-e-vFPyTd.js.map +0 -1
- package/dist/piece-registry-wQBhKJUl.d.ts.map +0 -1
- package/dist/reactor-rpc-BqAjCK1p.js +0 -1134
- package/dist/reactor-rpc-BqAjCK1p.js.map +0 -1
- package/dist/worker-entry.d.ts +0 -2
package/dist/index.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { B as
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { dirname, join } from "node:path";
|
|
1
|
+
import { $ as parseAddress, A as declaredConnectionIds, B as stepBlock, C as builtinPiece, D as AttachmentCache, E as CORE_PIECE_VERSION, G as PieceWorkerTimeoutError, H as PieceWorkerPool, I as SecretDeletedError, J as redactMessage, K as containsRedactedMarker, L as SecretNotFoundError, M as shapeConnection, N as resolveStepInput, O as BoundConnectionResolver, P as expressionsIn, Q as isPrivateAddress, R as parseSecretRef, S as blockPorts, T as CORE_PIECE_NAME, U as PieceWorker, V as triggerBlock, W as PieceWorkerError, X as secretsFor, Y as rememberSecrets, Z as DEFAULT_EGRESS_POLICY, _ as resolvedBlock, at as sourcedResolver, b as resolutionOf, c as canonicalJson, ct as authMethodFor, d as DynamicPropertiesError, et as extractDedupeKey, f as checkDynamicProperties, g as isHostBound, gt as isIndeterminateError, h as CompositeBlockExecutor, ht as setPieceRegistryUrl, i as packagePieces, it as pieceModuleRef, j as declaredReactorConnectionIds, l as hashOf, lt as unsupportedAuth, m as ActivepiecesBlockExecutor, mt as pieceRegistrySource, n as installedPiece, nt as maxFileBytes, o as errorNameOf, p as effectiveRetryPolicy, pt as npmRegistryBaseUrl, q as redact, r as installedPieces, rt as bundleResolver, s as runWorkflow, st as UnsupportedPieceFeatureError, t as PieceRegistry, tt as FileTooLargeError, u as stepConfigHash, ut as unsupportedTrigger, v as storeHandlers, w as describeBuiltinPiece, x as unpinnedNote, y as blockLabel, z as secretRefFromId } from "./piece-registry-BKRLK7Kv.js";
|
|
2
|
+
import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
|
|
3
|
+
import { createReadStream, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { open, readdir, rm, stat } from "node:fs/promises";
|
|
5
|
+
import path, { dirname, join } from "node:path";
|
|
7
6
|
import { childLogger, generateId } from "document-model";
|
|
8
7
|
import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
|
|
9
8
|
import { Cron } from "croner";
|
|
10
9
|
import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
10
|
+
import { Readable } from "node:stream";
|
|
11
11
|
import dns from "node:dns";
|
|
12
12
|
import net from "node:net";
|
|
13
13
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
@@ -18,6 +18,7 @@ import https from "node:https";
|
|
|
18
18
|
import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
|
|
19
19
|
import { Kind, parse } from "graphql";
|
|
20
20
|
import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
|
|
21
|
+
import { SpanStatusCode, context, metrics, trace } from "@opentelemetry/api";
|
|
21
22
|
import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
|
|
22
23
|
import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
23
24
|
//#region src/pieces/activepieces/piece-versions.ts
|
|
@@ -99,6 +100,14 @@ function npmVersions(name, options = {}) {
|
|
|
99
100
|
//#endregion
|
|
100
101
|
//#region src/reactor/attachment-port.ts
|
|
101
102
|
const logger$6 = childLogger(["workflow", "attachments"]);
|
|
103
|
+
function fileStream(filePath) {
|
|
104
|
+
return Readable.toWeb(createReadStream(filePath));
|
|
105
|
+
}
|
|
106
|
+
async function sha256File(filePath) {
|
|
107
|
+
const hash = createHash("sha256");
|
|
108
|
+
for await (const chunk of createReadStream(filePath)) hash.update(chunk);
|
|
109
|
+
return hash.digest("hex");
|
|
110
|
+
}
|
|
102
111
|
function refOf(result) {
|
|
103
112
|
const direct = result.ref;
|
|
104
113
|
if (typeof direct === "string") return direct;
|
|
@@ -128,16 +137,24 @@ async function writeCapped(body, destPath, limit) {
|
|
|
128
137
|
}
|
|
129
138
|
await handle.close();
|
|
130
139
|
}
|
|
131
|
-
function createAttachmentPort(client,
|
|
140
|
+
function createAttachmentPort(client, access) {
|
|
141
|
+
const authorize = async (ref) => {
|
|
142
|
+
const documentId = access.documentIdFor();
|
|
143
|
+
if (!documentId) throw new Error(`Cannot resolve ${ref}: no workflow document is in scope to authorize the read`);
|
|
144
|
+
if (!await access.canReadRef(ref)) throw new Error(`Cannot resolve ${ref}: workflow "${documentId}" may not read it`);
|
|
145
|
+
return documentId;
|
|
146
|
+
};
|
|
132
147
|
return {
|
|
133
|
-
async
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
148
|
+
async authorize(ref) {
|
|
149
|
+
await authorize(ref);
|
|
150
|
+
},
|
|
151
|
+
async read(ref, destPath, signal) {
|
|
152
|
+
const documentId = await authorize(ref);
|
|
137
153
|
const limit = maxFileBytes();
|
|
138
154
|
const { header, body } = await client.download({
|
|
139
155
|
documentId,
|
|
140
|
-
ref
|
|
156
|
+
ref,
|
|
157
|
+
...signal ? { signal } : {}
|
|
141
158
|
});
|
|
142
159
|
if (typeof header.sizeBytes === "number" && Number.isFinite(header.sizeBytes) && header.sizeBytes > limit) {
|
|
143
160
|
await body.cancel().catch(() => void 0);
|
|
@@ -150,13 +167,29 @@ function createAttachmentPort(client, documentIdFor, canReadRef) {
|
|
|
150
167
|
contentType
|
|
151
168
|
};
|
|
152
169
|
},
|
|
153
|
-
async write(file) {
|
|
154
|
-
const
|
|
170
|
+
async write(file, signal) {
|
|
171
|
+
const hash = await sha256File(file.path);
|
|
172
|
+
const mimeType = file.contentType ?? "application/octet-stream";
|
|
173
|
+
const extension = path.extname(file.fileName).slice(1);
|
|
174
|
+
const preprocessed = {
|
|
175
|
+
ref: `attachment://v1:${hash}`,
|
|
176
|
+
hash,
|
|
177
|
+
sizeBytes: file.size,
|
|
178
|
+
options: {
|
|
179
|
+
mimeType,
|
|
180
|
+
fileName: file.fileName,
|
|
181
|
+
...extension ? { extension } : {},
|
|
182
|
+
clientHash: hash,
|
|
183
|
+
sizeBytes: file.size
|
|
184
|
+
},
|
|
185
|
+
data: fileStream(file.path),
|
|
186
|
+
stream: () => fileStream(file.path)
|
|
187
|
+
};
|
|
155
188
|
const ref = refOf(await client.upload({
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
mimeType: file.contentType
|
|
189
|
+
preprocessed,
|
|
190
|
+
...signal ? { signal } : {}
|
|
159
191
|
}));
|
|
192
|
+
access.onWritten?.(ref);
|
|
160
193
|
logger$6.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
|
|
161
194
|
return ref;
|
|
162
195
|
}
|
|
@@ -1363,6 +1396,9 @@ function currentDocumentRecorder() {
|
|
|
1363
1396
|
function currentPieceWorker() {
|
|
1364
1397
|
return storage.getStore()?.pieceWorker;
|
|
1365
1398
|
}
|
|
1399
|
+
function currentAttachmentOrigins() {
|
|
1400
|
+
return storage.getStore()?.attachments;
|
|
1401
|
+
}
|
|
1366
1402
|
function currentStepTest() {
|
|
1367
1403
|
return storage.getStore()?.stepTest ?? false;
|
|
1368
1404
|
}
|
|
@@ -1440,6 +1476,20 @@ function runnableDefinition(state) {
|
|
|
1440
1476
|
//#endregion
|
|
1441
1477
|
//#region src/reactor/lib.ts
|
|
1442
1478
|
const pieceLogger = childLogger(["workflow", "piece"]);
|
|
1479
|
+
/**
|
|
1480
|
+
* The ceiling on one line this engine writes to a log.
|
|
1481
|
+
*
|
|
1482
|
+
* Nothing here is worth more than this to a reader, and an unbounded write is
|
|
1483
|
+
* a liability: a piece error carrying an HTML error page, or a piece logging a
|
|
1484
|
+
* whole document, becomes a multi-megabyte write to a pipe that may be
|
|
1485
|
+
* blocked — and an EPIPE on the fatal path is what produced the boot loop in
|
|
1486
|
+
* backlog item 5. Bounding the write is cheaper than handling the throw.
|
|
1487
|
+
*/
|
|
1488
|
+
const MAX_LOG_LINE_CHARS = 8192;
|
|
1489
|
+
function truncateForLog(text) {
|
|
1490
|
+
if (text.length <= 8192) return text;
|
|
1491
|
+
return `${text.slice(0, MAX_LOG_LINE_CHARS)}… [truncated, ${text.length} chars]`;
|
|
1492
|
+
}
|
|
1443
1493
|
const connectionLogger = childLogger(["workflow", "connection"]);
|
|
1444
1494
|
function assertConnectorMatches(state, request) {
|
|
1445
1495
|
const wanted = request?.piecePackage;
|
|
@@ -1502,6 +1552,29 @@ function bundleCacheDir() {
|
|
|
1502
1552
|
return bundleCache;
|
|
1503
1553
|
}
|
|
1504
1554
|
const ATTACHMENT_STAGING_DIR = join(process.cwd(), ".ph", "ap-attachment-staging");
|
|
1555
|
+
const ATTACHMENT_CACHE_DIR = join(process.cwd(), ".ph", "ap-attachment-cache");
|
|
1556
|
+
const DEFAULT_ATTACHMENT_CACHE_BYTES = 1073741824;
|
|
1557
|
+
function attachmentCacheBytes() {
|
|
1558
|
+
const raw = process.env.PH_WORKFLOWS_ATTACHMENT_CACHE_BYTES;
|
|
1559
|
+
const parsed = raw === void 0 ? NaN : Number(raw);
|
|
1560
|
+
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : DEFAULT_ATTACHMENT_CACHE_BYTES;
|
|
1561
|
+
}
|
|
1562
|
+
async function sweepAttachmentStaging(root = ATTACHMENT_STAGING_DIR, olderThanMs = 36e5) {
|
|
1563
|
+
const names = await readdir(root).catch(() => []);
|
|
1564
|
+
let swept = 0;
|
|
1565
|
+
for (const name of names) {
|
|
1566
|
+
const entry = join(root, name);
|
|
1567
|
+
const info = await stat(entry).catch(() => void 0);
|
|
1568
|
+
if (!info || Date.now() - info.mtimeMs < olderThanMs) continue;
|
|
1569
|
+
await rm(entry, {
|
|
1570
|
+
recursive: true,
|
|
1571
|
+
force: true
|
|
1572
|
+
});
|
|
1573
|
+
swept += 1;
|
|
1574
|
+
}
|
|
1575
|
+
return swept;
|
|
1576
|
+
}
|
|
1577
|
+
let stagingSwept = false;
|
|
1505
1578
|
let resolver;
|
|
1506
1579
|
function pieceResolver() {
|
|
1507
1580
|
return resolver ??= sourcedResolver({
|
|
@@ -1514,7 +1587,14 @@ function boundConnections(inner) {
|
|
|
1514
1587
|
connectionLogger.warn(`Step "${request?.stepKey ?? "?"}" of workflow "${currentWorkflowId() ?? "?"}" asked for connection "${connectionId}", which its definition does not declare`);
|
|
1515
1588
|
});
|
|
1516
1589
|
}
|
|
1517
|
-
function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess) {
|
|
1590
|
+
function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBlock, oauth, reactorAccess, telemetry) {
|
|
1591
|
+
if (attachments && !stagingSwept) {
|
|
1592
|
+
stagingSwept = true;
|
|
1593
|
+
sweepAttachmentStaging().then((swept) => {
|
|
1594
|
+
if (swept > 0) pieceLogger.info(`Removed ${swept} leftover attachment staging dir(s)`);
|
|
1595
|
+
}, () => void 0);
|
|
1596
|
+
}
|
|
1597
|
+
const cacheBytes = attachmentCacheBytes();
|
|
1518
1598
|
return new CompositeBlockExecutor(new ActivepiecesBlockExecutor({
|
|
1519
1599
|
cacheDir: bundleCacheDir(),
|
|
1520
1600
|
egress: configuredEgress(),
|
|
@@ -1528,10 +1608,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
|
|
|
1528
1608
|
resolver: pieceResolver(),
|
|
1529
1609
|
...resolveBlock ? { resolveBlock } : {},
|
|
1530
1610
|
...reactorAccess ? { reactorAccess } : {},
|
|
1611
|
+
...telemetry ? { telemetry } : {},
|
|
1531
1612
|
connections: boundConnections(new DocumentConnectionResolver(host, secrets, oauth)),
|
|
1532
1613
|
...pieceStore ? { pieceStore } : {},
|
|
1533
1614
|
onPieceLog: (entry, execution) => {
|
|
1534
|
-
const line = `[${execution.step.key}] ${entry.message}
|
|
1615
|
+
const line = truncateForLog(`[${execution.step.key}] ${entry.message}`);
|
|
1535
1616
|
if (entry.level === "error") pieceLogger.error(line);
|
|
1536
1617
|
else if (entry.level === "warn") pieceLogger.warn(line);
|
|
1537
1618
|
else if (entry.level === "debug") pieceLogger.debug(line);
|
|
@@ -1539,7 +1620,11 @@ function createBlockExecutor(host, secrets, attachments, pieceStore, resolveBloc
|
|
|
1539
1620
|
},
|
|
1540
1621
|
...attachments ? {
|
|
1541
1622
|
attachments,
|
|
1542
|
-
stagingRoot: ATTACHMENT_STAGING_DIR
|
|
1623
|
+
stagingRoot: ATTACHMENT_STAGING_DIR,
|
|
1624
|
+
...cacheBytes > 0 ? { attachmentCache: new AttachmentCache({
|
|
1625
|
+
dir: ATTACHMENT_CACHE_DIR,
|
|
1626
|
+
maxBytes: cacheBytes
|
|
1627
|
+
}) } : {}
|
|
1543
1628
|
} : {}
|
|
1544
1629
|
}));
|
|
1545
1630
|
}
|
|
@@ -1563,7 +1648,8 @@ function stepDefinition(step) {
|
|
|
1563
1648
|
config: step.config,
|
|
1564
1649
|
timeoutSeconds: step.timeoutSeconds,
|
|
1565
1650
|
propertySettings: propertySettings(step.propertySettings),
|
|
1566
|
-
skip: step.skip
|
|
1651
|
+
skip: step.skip,
|
|
1652
|
+
retry: step.retry
|
|
1567
1653
|
};
|
|
1568
1654
|
}
|
|
1569
1655
|
function toWorkflowDefinition(state) {
|
|
@@ -1776,6 +1862,227 @@ async function buildReactorRunScope(host, request) {
|
|
|
1776
1862
|
connection: { access: parsed.config.access ?? "write" }
|
|
1777
1863
|
};
|
|
1778
1864
|
}
|
|
1865
|
+
//#endregion
|
|
1866
|
+
//#region src/telemetry.ts
|
|
1867
|
+
const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
|
|
1868
|
+
var WorkflowTelemetry = class {
|
|
1869
|
+
tracer;
|
|
1870
|
+
meter;
|
|
1871
|
+
runs;
|
|
1872
|
+
runDuration;
|
|
1873
|
+
stepDuration;
|
|
1874
|
+
phaseDuration;
|
|
1875
|
+
poolGauges;
|
|
1876
|
+
constructor(options = {}) {
|
|
1877
|
+
this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
|
|
1878
|
+
this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
|
|
1879
|
+
this.runs = this.meter.createCounter("workflow.runs", {
|
|
1880
|
+
description: "Runs finished, by status and trigger kind",
|
|
1881
|
+
unit: "{run}"
|
|
1882
|
+
});
|
|
1883
|
+
this.runDuration = this.meter.createHistogram("workflow.run.duration", {
|
|
1884
|
+
description: "Run duration from fire to the journal closing it",
|
|
1885
|
+
unit: "ms"
|
|
1886
|
+
});
|
|
1887
|
+
this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
|
|
1888
|
+
description: "Step duration, by piece and status",
|
|
1889
|
+
unit: "ms"
|
|
1890
|
+
});
|
|
1891
|
+
this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
|
|
1892
|
+
description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
|
|
1893
|
+
unit: "ms"
|
|
1894
|
+
});
|
|
1895
|
+
}
|
|
1896
|
+
phase(name, attributes, fn, parent) {
|
|
1897
|
+
const started = performance.now();
|
|
1898
|
+
return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
|
|
1899
|
+
}
|
|
1900
|
+
workerTimings(request, timings, forkedAt, ipc) {
|
|
1901
|
+
const under = (span) => trace.setSpan(context.active(), span);
|
|
1902
|
+
const child = (name, start, end, parent = request, attributes) => {
|
|
1903
|
+
const span = this.tracer.startSpan(`workflow.worker.${name}`, {
|
|
1904
|
+
startTime: start,
|
|
1905
|
+
...attributes ? { attributes } : {}
|
|
1906
|
+
}, under(parent));
|
|
1907
|
+
span.end(end);
|
|
1908
|
+
this.recordPhase(`worker.${name}`, end - start);
|
|
1909
|
+
return span;
|
|
1910
|
+
};
|
|
1911
|
+
const ready = epoch(timings.ready);
|
|
1912
|
+
if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
|
|
1913
|
+
const received = epoch(timings.received);
|
|
1914
|
+
const sent = epoch(timings.sent);
|
|
1915
|
+
const delivered = Math.max(ipc.sent, ready ?? 0);
|
|
1916
|
+
if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
|
|
1917
|
+
const byName = /* @__PURE__ */ new Map();
|
|
1918
|
+
const phases = Array.isArray(timings.phases) ? timings.phases : [];
|
|
1919
|
+
for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
|
|
1920
|
+
const parent = phase.parent ? byName.get(phase.parent) : void 0;
|
|
1921
|
+
byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
|
|
1922
|
+
}
|
|
1923
|
+
if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
|
|
1924
|
+
}
|
|
1925
|
+
recordPhase(name, ms) {
|
|
1926
|
+
this.phaseDuration.record(ms, { phase: name });
|
|
1927
|
+
}
|
|
1928
|
+
span(name, attributes, fn, parent = context.active()) {
|
|
1929
|
+
return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
|
|
1930
|
+
try {
|
|
1931
|
+
return await fn(span);
|
|
1932
|
+
} catch (error) {
|
|
1933
|
+
failSpan(span, error);
|
|
1934
|
+
throw error;
|
|
1935
|
+
} finally {
|
|
1936
|
+
span.end();
|
|
1937
|
+
}
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
recordRun(ms, status, triggerKind) {
|
|
1941
|
+
const attributes = {
|
|
1942
|
+
"run.status": status,
|
|
1943
|
+
"trigger.kind": metricTriggerKind(triggerKind)
|
|
1944
|
+
};
|
|
1945
|
+
this.runs.add(1, attributes);
|
|
1946
|
+
this.runDuration.record(ms, attributes);
|
|
1947
|
+
}
|
|
1948
|
+
recordStep(ms, pieceName, status) {
|
|
1949
|
+
this.stepDuration.record(ms, {
|
|
1950
|
+
"piece.name": pieceName,
|
|
1951
|
+
"step.status": status
|
|
1952
|
+
});
|
|
1953
|
+
}
|
|
1954
|
+
observeWorkerPool(stats) {
|
|
1955
|
+
this.poolGauges ??= this.createPoolGauges();
|
|
1956
|
+
const { size, active, waiting } = this.poolGauges;
|
|
1957
|
+
const observables = [
|
|
1958
|
+
size,
|
|
1959
|
+
active,
|
|
1960
|
+
waiting
|
|
1961
|
+
];
|
|
1962
|
+
const callback = (result) => {
|
|
1963
|
+
const current = stats();
|
|
1964
|
+
result.observe(size, current.size);
|
|
1965
|
+
result.observe(active, current.active);
|
|
1966
|
+
result.observe(waiting, current.waiting);
|
|
1967
|
+
};
|
|
1968
|
+
this.meter.addBatchObservableCallback(callback, observables);
|
|
1969
|
+
return () => this.meter.removeBatchObservableCallback(callback, observables);
|
|
1970
|
+
}
|
|
1971
|
+
createPoolGauges() {
|
|
1972
|
+
const gauge = (name, description) => this.meter.createObservableGauge(name, {
|
|
1973
|
+
description,
|
|
1974
|
+
unit: "{worker}"
|
|
1975
|
+
});
|
|
1976
|
+
return {
|
|
1977
|
+
size: gauge("workflow.worker.pool.size", "Worker slots"),
|
|
1978
|
+
active: gauge("workflow.worker.pool.active", "Slots held by runs"),
|
|
1979
|
+
waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
|
|
1980
|
+
};
|
|
1981
|
+
}
|
|
1982
|
+
};
|
|
1983
|
+
const WORKER_PHASES = /* @__PURE__ */ new Set([
|
|
1984
|
+
"piece.load",
|
|
1985
|
+
"reactor.open",
|
|
1986
|
+
"models",
|
|
1987
|
+
"action"
|
|
1988
|
+
]);
|
|
1989
|
+
const WORKER_ATTRIBUTES = {
|
|
1990
|
+
"piece.cached": "boolean",
|
|
1991
|
+
"document.type": "string"
|
|
1992
|
+
};
|
|
1993
|
+
const MAX_ATTRIBUTE_LENGTH = 256;
|
|
1994
|
+
function epoch(value) {
|
|
1995
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
1996
|
+
}
|
|
1997
|
+
function knownPhase(phase) {
|
|
1998
|
+
if (typeof phase !== "object" || phase === null) return false;
|
|
1999
|
+
const { name, start, end, parent } = phase;
|
|
2000
|
+
return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
|
|
2001
|
+
}
|
|
2002
|
+
function phaseAttributes(attributes) {
|
|
2003
|
+
if (typeof attributes !== "object" || attributes === null) return void 0;
|
|
2004
|
+
const kept = {};
|
|
2005
|
+
for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
|
|
2006
|
+
const value = attributes[key];
|
|
2007
|
+
if (typeof value !== type) continue;
|
|
2008
|
+
kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
|
|
2009
|
+
}
|
|
2010
|
+
return Object.keys(kept).length > 0 ? kept : void 0;
|
|
2011
|
+
}
|
|
2012
|
+
function metricTriggerKind(triggerKind) {
|
|
2013
|
+
return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
|
|
2014
|
+
}
|
|
2015
|
+
function metricPieceName(step, resolution) {
|
|
2016
|
+
return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
|
|
2017
|
+
}
|
|
2018
|
+
function tracedSteps(executor, telemetry) {
|
|
2019
|
+
return { execute(execution) {
|
|
2020
|
+
const { step } = execution;
|
|
2021
|
+
const started = performance.now();
|
|
2022
|
+
let status = "FAILED";
|
|
2023
|
+
let resolution;
|
|
2024
|
+
return telemetry.span("workflow.step", {
|
|
2025
|
+
"step.id": step.id,
|
|
2026
|
+
"step.key": step.key,
|
|
2027
|
+
"piece.name": step.pieceName,
|
|
2028
|
+
"action.name": step.actionName
|
|
2029
|
+
}, async (span) => {
|
|
2030
|
+
try {
|
|
2031
|
+
const result = await executor.execute(execution);
|
|
2032
|
+
resolution = result.resolution;
|
|
2033
|
+
status = "SUCCEEDED";
|
|
2034
|
+
const version = resolution?.resolved?.version;
|
|
2035
|
+
if (version) span.setAttribute("piece.version", version);
|
|
2036
|
+
return result;
|
|
2037
|
+
} catch (error) {
|
|
2038
|
+
resolution = resolutionOf(error);
|
|
2039
|
+
throw error;
|
|
2040
|
+
}
|
|
2041
|
+
}).finally(() => {
|
|
2042
|
+
telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
|
|
2043
|
+
});
|
|
2044
|
+
} };
|
|
2045
|
+
}
|
|
2046
|
+
function failSpan(span, error) {
|
|
2047
|
+
span.recordException(error instanceof Error ? error : String(error));
|
|
2048
|
+
span.setStatus({
|
|
2049
|
+
code: SpanStatusCode.ERROR,
|
|
2050
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2051
|
+
});
|
|
2052
|
+
}
|
|
2053
|
+
function tracedMethods(target, telemetry, prefix, parent) {
|
|
2054
|
+
return new Proxy(target, { get(object, property, receiver) {
|
|
2055
|
+
const value = Reflect.get(object, property, receiver);
|
|
2056
|
+
if (typeof value !== "function" || typeof property !== "string") return value;
|
|
2057
|
+
const phase = `${prefix}.${property}`;
|
|
2058
|
+
return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
|
|
2059
|
+
const started = performance.now();
|
|
2060
|
+
const end = (error) => {
|
|
2061
|
+
if (error !== void 0) failSpan(span, error);
|
|
2062
|
+
span.end();
|
|
2063
|
+
telemetry.recordPhase(phase, performance.now() - started);
|
|
2064
|
+
};
|
|
2065
|
+
let result;
|
|
2066
|
+
try {
|
|
2067
|
+
result = value.apply(object, args);
|
|
2068
|
+
} catch (error) {
|
|
2069
|
+
end(error);
|
|
2070
|
+
throw error;
|
|
2071
|
+
}
|
|
2072
|
+
if (!(result instanceof Promise)) {
|
|
2073
|
+
end();
|
|
2074
|
+
return result;
|
|
2075
|
+
}
|
|
2076
|
+
return result.then((resolved) => {
|
|
2077
|
+
end();
|
|
2078
|
+
return resolved;
|
|
2079
|
+
}, (error) => {
|
|
2080
|
+
end(error);
|
|
2081
|
+
throw error;
|
|
2082
|
+
});
|
|
2083
|
+
});
|
|
2084
|
+
} });
|
|
2085
|
+
}
|
|
1779
2086
|
const JOB_NOT_FOUND = "Job not found";
|
|
1780
2087
|
const MIN_DEADLINE_MARGIN_MS = 250;
|
|
1781
2088
|
const MAX_DEADLINE_MARGIN_MS = 2e3;
|
|
@@ -1792,7 +2099,7 @@ const AUTHORIZATION_DENIED = "AuthorizationDeniedError";
|
|
|
1792
2099
|
function jobFailed(job) {
|
|
1793
2100
|
const error = job.error;
|
|
1794
2101
|
if (error?.name === AUTHORIZATION_DENIED) return accessDenied(`The Switchboard has no grant for this operation: ${error.message}`);
|
|
1795
|
-
return new ReactorError(ReactorJobFailedError
|
|
2102
|
+
return new ReactorError(ReactorJobFailedError, `Reactor job ${job.id} failed: ${job.error?.message ?? "unknown error"}`);
|
|
1796
2103
|
}
|
|
1797
2104
|
function assertActionsApplied(job, actions) {
|
|
1798
2105
|
const outcomes = job.result?.actions ?? [];
|
|
@@ -1810,7 +2117,7 @@ function assertActionsApplied(job, actions) {
|
|
|
1810
2117
|
if (outcome.kind === "denied") return [`Action ${label} was denied: ${outcome.reason}`];
|
|
1811
2118
|
return [];
|
|
1812
2119
|
});
|
|
1813
|
-
if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError
|
|
2120
|
+
if (failed.length > 0) throw new ReactorError(ReactorActionsFailedError, failed.join("; "));
|
|
1814
2121
|
}
|
|
1815
2122
|
var RunScopedReactorClient = class {
|
|
1816
2123
|
inner;
|
|
@@ -2141,7 +2448,7 @@ const NO_JOURNAL = {
|
|
|
2141
2448
|
recordDocuments: () => Promise.resolve(),
|
|
2142
2449
|
recordJob: () => Promise.resolve()
|
|
2143
2450
|
};
|
|
2144
|
-
function reactorTap(host, base, journal, options = {}) {
|
|
2451
|
+
function reactorTap(host, base, journal, options = {}, telemetry) {
|
|
2145
2452
|
const clientOptions = {
|
|
2146
2453
|
...host.hostPrincipal ? { hostPrincipal: host.hostPrincipal } : {},
|
|
2147
2454
|
...options
|
|
@@ -2149,12 +2456,13 @@ function reactorTap(host, base, journal, options = {}) {
|
|
|
2149
2456
|
return {
|
|
2150
2457
|
requireReactor: base.requireReactor,
|
|
2151
2458
|
open(transport, { deadline }) {
|
|
2152
|
-
const
|
|
2459
|
+
const served = servedClient(new RunScopedReactorClient(host.reactorClient, {
|
|
2153
2460
|
...base,
|
|
2154
2461
|
deadline,
|
|
2155
2462
|
journal
|
|
2156
|
-
}, host, clientOptions);
|
|
2157
|
-
const
|
|
2463
|
+
}, host, clientOptions));
|
|
2464
|
+
const parent = context.active();
|
|
2465
|
+
const server = new ReactorHostServer(telemetry ? tracedMethods(served, telemetry, "reactor", parent) : served, transport);
|
|
2158
2466
|
server.start();
|
|
2159
2467
|
return () => server.stop();
|
|
2160
2468
|
}
|
|
@@ -2451,6 +2759,19 @@ async function resolveVariables(variables, secrets) {
|
|
|
2451
2759
|
}
|
|
2452
2760
|
//#endregion
|
|
2453
2761
|
//#region src/reactor/step-test.ts
|
|
2762
|
+
/**
|
|
2763
|
+
* A step record's status as a test reports it.
|
|
2764
|
+
*
|
|
2765
|
+
* Collapsing this to `FAILED : SUCCEEDED` is how an INDETERMINATE test came
|
|
2766
|
+
* to read green — the one status that must never be mistaken for a confirmed
|
|
2767
|
+
* one, since the whole point of it is that nobody knows whether the write
|
|
2768
|
+
* landed.
|
|
2769
|
+
*/
|
|
2770
|
+
function testStatusOf(status) {
|
|
2771
|
+
if (status === "FAILED") return "FAILED";
|
|
2772
|
+
if (status === "INDETERMINATE") return "INDETERMINATE";
|
|
2773
|
+
return "SUCCEEDED";
|
|
2774
|
+
}
|
|
2454
2775
|
function collectStrings(value, into) {
|
|
2455
2776
|
if (typeof value === "string") into.push(value);
|
|
2456
2777
|
else if (Array.isArray(value)) for (const item of value) collectStrings(item, into);
|
|
@@ -2534,6 +2855,7 @@ function untestedError(label, sample) {
|
|
|
2534
2855
|
case "hidden": return `Test ${label} first: its last test is not visible to you`;
|
|
2535
2856
|
case "failed": return `Test ${label} first: its last test failed`;
|
|
2536
2857
|
case "truncated": return `Test ${label} again: the journal kept only a truncated copy of its last output`;
|
|
2858
|
+
case "indeterminate": return `Test ${label} again: its last test is INDETERMINATE, so it has no confirmed output`;
|
|
2537
2859
|
default: return `Test ${label} first`;
|
|
2538
2860
|
}
|
|
2539
2861
|
}
|
|
@@ -7095,6 +7417,59 @@ function parseParameter(param) {
|
|
|
7095
7417
|
return parseValueExpression(param);
|
|
7096
7418
|
}
|
|
7097
7419
|
//#endregion
|
|
7420
|
+
//#region src/reactor/policy.ts
|
|
7421
|
+
/** The pre-enforcement behaviour, for a definition that declares no policy. */
|
|
7422
|
+
const UNENFORCED_POLICY = Object.freeze({
|
|
7423
|
+
concurrency: "PARALLEL",
|
|
7424
|
+
maxParallelRuns: null,
|
|
7425
|
+
runTimeoutSeconds: null,
|
|
7426
|
+
onFailure: "IGNORE",
|
|
7427
|
+
defaultRetry: null
|
|
7428
|
+
});
|
|
7429
|
+
const CONCURRENCY_MODES = [
|
|
7430
|
+
"SINGLETON",
|
|
7431
|
+
"QUEUE",
|
|
7432
|
+
"PARALLEL"
|
|
7433
|
+
];
|
|
7434
|
+
const FAILURE_MODES = [
|
|
7435
|
+
"PARK",
|
|
7436
|
+
"NOTIFY",
|
|
7437
|
+
"IGNORE"
|
|
7438
|
+
];
|
|
7439
|
+
function positive(value) {
|
|
7440
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
7441
|
+
}
|
|
7442
|
+
/** What the runtime enforces for a run of this definition. */
|
|
7443
|
+
function effectiveRunPolicy(runnable) {
|
|
7444
|
+
const policy = runnable.policy;
|
|
7445
|
+
if (!policy) return UNENFORCED_POLICY;
|
|
7446
|
+
const concurrency = CONCURRENCY_MODES.includes(policy.concurrency) ? policy.concurrency : "PARALLEL";
|
|
7447
|
+
return {
|
|
7448
|
+
concurrency,
|
|
7449
|
+
maxParallelRuns: concurrency === "PARALLEL" ? positive(policy.maxParallelRuns) : null,
|
|
7450
|
+
runTimeoutSeconds: positive(policy.runTimeoutSeconds),
|
|
7451
|
+
onFailure: FAILURE_MODES.includes(policy.onFailure) ? policy.onFailure : "IGNORE",
|
|
7452
|
+
defaultRetry: effectiveRetryPolicy(policy.defaultRetry)
|
|
7453
|
+
};
|
|
7454
|
+
}
|
|
7455
|
+
/** How many runs of one workflow may execute at once; 1 for both serialising
|
|
7456
|
+
* modes, so the gate holds one number rather than three branches. Null is
|
|
7457
|
+
* unbounded, i.e. no gate at all. */
|
|
7458
|
+
function concurrencyLimit(policy) {
|
|
7459
|
+
if (policy.concurrency === "PARALLEL") return policy.maxParallelRuns;
|
|
7460
|
+
return 1;
|
|
7461
|
+
}
|
|
7462
|
+
/** The trigger status PARK leaves on a supervised trigger's row. Not ENABLED,
|
|
7463
|
+
* so the supervisor's due-trigger query does not return it and the schedule
|
|
7464
|
+
* stops refiring; a re-publish or re-enable arms it again. The park itself, for
|
|
7465
|
+
* every trigger kind, is the store's `workflow_park` row. */
|
|
7466
|
+
const PARKED_TRIGGER_STATUS = "PARKED";
|
|
7467
|
+
/** The run status a firing skipped by SINGLETON is journaled under, and the
|
|
7468
|
+
* one a run that passed `runTimeoutSeconds` ends in. Both are "this run did
|
|
7469
|
+
* not finish and nothing is wrong with the workflow", which is what the
|
|
7470
|
+
* document model's RunStatus.CANCELLED means. */
|
|
7471
|
+
const CANCELLED_RUN_STATUS = "CANCELLED";
|
|
7472
|
+
//#endregion
|
|
7098
7473
|
//#region src/reactor/store.ts
|
|
7099
7474
|
function triggerRowBlock(row) {
|
|
7100
7475
|
return {
|
|
@@ -7109,6 +7484,19 @@ function triggerBlockColumns(block) {
|
|
|
7109
7484
|
trigger_name: block.name
|
|
7110
7485
|
};
|
|
7111
7486
|
}
|
|
7487
|
+
const FIRED_WITHOUT_RUN_ID = "fired-without-run";
|
|
7488
|
+
const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
|
|
7489
|
+
function abandonedFireError(attempts) {
|
|
7490
|
+
return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
|
|
7491
|
+
}
|
|
7492
|
+
function workflowParkRow(workflowId, publishedVersion, reason) {
|
|
7493
|
+
return {
|
|
7494
|
+
workflow_id: workflowId,
|
|
7495
|
+
published_version: publishedVersion,
|
|
7496
|
+
reason: redactMessage(reason),
|
|
7497
|
+
parked_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7498
|
+
};
|
|
7499
|
+
}
|
|
7112
7500
|
const logger$3 = childLogger([
|
|
7113
7501
|
"workflow",
|
|
7114
7502
|
"runtime",
|
|
@@ -7136,6 +7524,7 @@ async function up(db) {
|
|
|
7136
7524
|
} catch {}
|
|
7137
7525
|
await db.schema.createTable("trigger_state").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("trigger_name", "text", (col) => col.notNull()).addColumn("config_hash", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("store_state", "text", (col) => col.notNull()).addColumn("interval_ms", "integer", (col) => col.notNull()).addColumn("next_poll_at", "text").addColumn("last_poll_at", "text").addColumn("last_error", "text").addColumn("consecutive_failures", "integer", (col) => col.notNull()).addColumn("lease_owner", "text").addColumn("lease_expires_at", "text").addColumn("updated_at", "text", (col) => col.notNull()).ifNotExists().execute();
|
|
7138
7526
|
await db.schema.createTable("trigger_dedupe").addColumn("workflow_id", "text", (col) => col.notNull()).addColumn("dedupe_key", "text", (col) => col.notNull()).addColumn("run_id", "text").addColumn("created_at", "text", (col) => col.notNull()).addPrimaryKeyConstraint("trigger_dedupe_pk", ["workflow_id", "dedupe_key"]).ifNotExists().execute();
|
|
7527
|
+
await db.schema.createTable("workflow_park").addColumn("workflow_id", "text", (col) => col.primaryKey()).addColumn("published_version", "integer", (col) => col.notNull()).addColumn("reason", "text", (col) => col.notNull()).addColumn("parked_at", "text", (col) => col.notNull()).ifNotExists().execute();
|
|
7139
7528
|
await db.schema.createTable("step_execution").addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("ordinal", "integer", (col) => col.notNull()).addColumn("step_id", "text", (col) => col.notNull()).addColumn("step_key", "text", (col) => col.notNull()).addColumn("piece_name", "text", (col) => col.notNull()).addColumn("block_name", "text", (col) => col.notNull()).addColumn("status", "text", (col) => col.notNull()).addColumn("input", "text").addColumn("output", "text").addColumn("port", "text").addColumn("error", "text").addColumn("started_at", "text").addColumn("ended_at", "text").addUniqueConstraint("step_execution_run_step", ["run_id", "step_id"]).ifNotExists().execute();
|
|
7140
7529
|
for (const column of ["started_at", "ended_at"]) try {
|
|
7141
7530
|
await db.schema.alterTable("step_execution").addColumn(column, "text").execute();
|
|
@@ -7154,6 +7543,21 @@ async function up(db) {
|
|
|
7154
7543
|
]) try {
|
|
7155
7544
|
await db.schema.alterTable(table).addColumn(column, "text").execute();
|
|
7156
7545
|
} catch {}
|
|
7546
|
+
try {
|
|
7547
|
+
await db.schema.alterTable("trigger_dedupe").addColumn("attempts", "integer", (col) => col.notNull().defaultTo(1)).execute();
|
|
7548
|
+
} catch {}
|
|
7549
|
+
await db.schema.createTable("migration_mark").addColumn("name", "text", (col) => col.primaryKey()).addColumn("applied_at", "text", (col) => col.notNull()).ifNotExists().execute();
|
|
7550
|
+
await db.transaction().execute(async (trx) => {
|
|
7551
|
+
if (await trx.selectFrom("migration_mark").select("name").where("name", "=", DEDUPE_CLAIM_MIGRATION).executeTakeFirst()) return;
|
|
7552
|
+
await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "is", null).execute();
|
|
7553
|
+
await trx.insertInto("migration_mark").values({
|
|
7554
|
+
name: DEDUPE_CLAIM_MIGRATION,
|
|
7555
|
+
applied_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
7556
|
+
}).onConflict((oc) => oc.column("name").doNothing()).execute();
|
|
7557
|
+
});
|
|
7558
|
+
try {
|
|
7559
|
+
await db.schema.alterTable("step_execution").addColumn("attempts", "integer").execute();
|
|
7560
|
+
} catch {}
|
|
7157
7561
|
try {
|
|
7158
7562
|
await db.schema.alterTable("trigger_state").addColumn("next_renew_at", "text").execute();
|
|
7159
7563
|
} catch {}
|
|
@@ -7362,10 +7766,50 @@ function assertPieceStoreEntry(key, value) {
|
|
|
7362
7766
|
if (size > 524288) throw new PieceStoreLimitError(`Store value for "${key}" is ${size} bytes, over the ${PIECE_STORE_MAX_VALUE_BYTES} byte limit`);
|
|
7363
7767
|
}
|
|
7364
7768
|
const STEP_PAYLOAD_PREFIX_CHARS = 32768;
|
|
7769
|
+
/**
|
|
7770
|
+
* The reserved key a truncation marker is recognised by, and its value.
|
|
7771
|
+
*
|
|
7772
|
+
* A marker has to be distinguishable from a legitimate payload that merely
|
|
7773
|
+
* LOOKS like one, because a run whose output is mistaken for a marker is a
|
|
7774
|
+
* step that silently re-executes on rerun. The original predicate duck-typed
|
|
7775
|
+
* `{truncated: true, bytes: number, prefix: string}` (review backlog item 15),
|
|
7776
|
+
* which any piece returning a truncation report of its own would match — and
|
|
7777
|
+
* "did we truncate this?" is exactly the shape such a report takes.
|
|
7778
|
+
*
|
|
7779
|
+
* So the marker is keyed on a namespaced sentinel whose VALUE is a versioned
|
|
7780
|
+
* magic string. A piece would have to emit that exact pair to collide, which
|
|
7781
|
+
* is not something a payload produces by accident.
|
|
7782
|
+
*/
|
|
7783
|
+
const TRUNCATED_PAYLOAD_KEY = "@powerhousedao/reactor-workflow:runJournal";
|
|
7784
|
+
const TRUNCATED_PAYLOAD_SENTINEL = "truncated-payload/v1";
|
|
7785
|
+
const LEGACY_MARKER_KEYS = [
|
|
7786
|
+
"truncated",
|
|
7787
|
+
"bytes",
|
|
7788
|
+
"prefix",
|
|
7789
|
+
"documentId",
|
|
7790
|
+
"driveId"
|
|
7791
|
+
];
|
|
7792
|
+
/**
|
|
7793
|
+
* True for a journaled value this store truncated.
|
|
7794
|
+
*
|
|
7795
|
+
* Two shapes are accepted and only one is written. New rows carry the reserved
|
|
7796
|
+
* key, which a payload cannot produce by accident. Rows journaled before it
|
|
7797
|
+
* carry the old `{truncated: true, bytes, prefix}`, and those still have to be
|
|
7798
|
+
* recognised: reading one as data would re-run the side-effectful step that
|
|
7799
|
+
* produced it, which is the whole bug.
|
|
7800
|
+
*
|
|
7801
|
+
* So the legacy branch is matched by its EXACT key set, not by duck-typing.
|
|
7802
|
+
* `{truncated, bytes, prefix, source}` — a truncation report a piece might
|
|
7803
|
+
* legitimately return, and the collision the review found — is no longer a
|
|
7804
|
+
* marker. A payload whose keys are exactly the legacy marker's is still
|
|
7805
|
+
* ambiguous, and nothing can resolve that for a row already written; the
|
|
7806
|
+
* window is as narrow as the stored data allows.
|
|
7807
|
+
*/
|
|
7365
7808
|
function isTruncatedStepPayload(value) {
|
|
7366
7809
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
7367
7810
|
const record = value;
|
|
7368
|
-
|
|
7811
|
+
if (record["@powerhousedao/reactor-workflow:runJournal"] === "truncated-payload/v1") return true;
|
|
7812
|
+
return record.truncated === true && typeof record.bytes === "number" && typeof record.prefix === "string" && Object.keys(record).every((key) => LEGACY_MARKER_KEYS.includes(key));
|
|
7369
7813
|
}
|
|
7370
7814
|
function topLevelDocumentIds(value) {
|
|
7371
7815
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return {};
|
|
@@ -7384,7 +7828,7 @@ function cappedPayload(value) {
|
|
|
7384
7828
|
const last = prefix.charCodeAt(prefix.length - 1);
|
|
7385
7829
|
if (last >= 55296 && last <= 56319) prefix = prefix.slice(0, -1);
|
|
7386
7830
|
const marker = {
|
|
7387
|
-
|
|
7831
|
+
[TRUNCATED_PAYLOAD_KEY]: TRUNCATED_PAYLOAD_SENTINEL,
|
|
7388
7832
|
bytes,
|
|
7389
7833
|
prefix,
|
|
7390
7834
|
...topLevelDocumentIds(value)
|
|
@@ -7401,7 +7845,7 @@ function stepValues(runId, ordinal, step) {
|
|
|
7401
7845
|
block_name: step.blockName,
|
|
7402
7846
|
status: step.status,
|
|
7403
7847
|
input: cappedPayload(redact(step.input)),
|
|
7404
|
-
output: cappedPayload(redact(referenceDocuments(step.output))),
|
|
7848
|
+
output: cappedPayload(redact(referenceDocuments(step.journaledOutput !== void 0 ? step.journaledOutput : step.output))),
|
|
7405
7849
|
port: step.port ?? null,
|
|
7406
7850
|
error: step.error ? redactMessage(step.error) : null,
|
|
7407
7851
|
error_name: step.errorName ?? null,
|
|
@@ -7411,7 +7855,8 @@ function stepValues(runId, ordinal, step) {
|
|
|
7411
7855
|
piece_source: step.piece?.source ?? null,
|
|
7412
7856
|
version_match: step.piece?.match ?? null,
|
|
7413
7857
|
version_note: step.piece?.note ?? null,
|
|
7414
|
-
config_hash: step.configHash ?? null
|
|
7858
|
+
config_hash: step.configHash ?? null,
|
|
7859
|
+
attempts: step.attempts ?? null
|
|
7415
7860
|
};
|
|
7416
7861
|
}
|
|
7417
7862
|
function runWarningNotes(result) {
|
|
@@ -7460,15 +7905,16 @@ function journaledPayloadNames(json, ids) {
|
|
|
7460
7905
|
});
|
|
7461
7906
|
}
|
|
7462
7907
|
const erasedRuns = /* @__PURE__ */ new Set();
|
|
7463
|
-
async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso
|
|
7908
|
+
async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso) {
|
|
7464
7909
|
const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
|
|
7465
7910
|
await db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
|
|
7466
7911
|
return await db.insertInto("trigger_dedupe").values({
|
|
7467
7912
|
workflow_id: workflowId,
|
|
7468
7913
|
dedupe_key: dedupeKey,
|
|
7469
|
-
run_id:
|
|
7470
|
-
created_at: nowIso
|
|
7471
|
-
|
|
7914
|
+
run_id: FIRED_WITHOUT_RUN_ID,
|
|
7915
|
+
created_at: nowIso,
|
|
7916
|
+
attempts: 1
|
|
7917
|
+
}).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ run_id: FIRED_WITHOUT_RUN_ID }).where("trigger_dedupe.run_id", "is", null)).returning("dedupe_key").executeTakeFirst() !== void 0;
|
|
7472
7918
|
}
|
|
7473
7919
|
const STEP_COLUMNS_WITHOUT_DATA = [
|
|
7474
7920
|
"id",
|
|
@@ -7488,7 +7934,8 @@ const STEP_COLUMNS_WITHOUT_DATA = [
|
|
|
7488
7934
|
"piece_source",
|
|
7489
7935
|
"version_match",
|
|
7490
7936
|
"version_note",
|
|
7491
|
-
"config_hash"
|
|
7937
|
+
"config_hash",
|
|
7938
|
+
"attempts"
|
|
7492
7939
|
];
|
|
7493
7940
|
const PRUNE_BATCH_SIZE = 500;
|
|
7494
7941
|
var WorkflowRunStore = class WorkflowRunStore {
|
|
@@ -7535,20 +7982,133 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7535
7982
|
if (recovered > 0) logger$3.warn(`Recovered ${recovered} workflow run(s) journaled by a stopped reactor but never started; they are now FAILED and rerunnable`);
|
|
7536
7983
|
return recovered;
|
|
7537
7984
|
}
|
|
7985
|
+
/**
|
|
7986
|
+
* Closes a run out as CANCELLED, with the reason as its error.
|
|
7987
|
+
*
|
|
7988
|
+
* Two callers, one meaning — "this run did not finish, and the workflow is
|
|
7989
|
+
* not at fault": a firing SINGLETON concurrency refused, and a run that
|
|
7990
|
+
* passed its `runTimeoutSeconds`. Separate from `failRun` because a
|
|
7991
|
+
* CANCELLED run is not rerunnable and must not read as a workflow defect.
|
|
7992
|
+
*/
|
|
7993
|
+
async cancelRun(runId, reason) {
|
|
7994
|
+
this.runsInFlight.delete(runId);
|
|
7995
|
+
if (erasedRuns.delete(runId)) return;
|
|
7996
|
+
await this.db.updateTable("run").set({
|
|
7997
|
+
status: "CANCELLED",
|
|
7998
|
+
error: redactMessage(reason),
|
|
7999
|
+
ended_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8000
|
+
}).where("id", "=", runId).execute();
|
|
8001
|
+
}
|
|
7538
8002
|
async enqueueRun(options) {
|
|
7539
8003
|
const id = randomUUID();
|
|
7540
8004
|
await this.insertPendingRun(this.db, id, options);
|
|
7541
8005
|
this.runsInFlight.add(id);
|
|
7542
8006
|
return id;
|
|
7543
8007
|
}
|
|
8008
|
+
/**
|
|
8009
|
+
* Claims the dedupe key and journals the PENDING run, bounding crash replays.
|
|
8010
|
+
*
|
|
8011
|
+
* Three outcomes, as {@link FireClaim}:
|
|
8012
|
+
*
|
|
8013
|
+
* - **`"claimed"`** with a run id: this fire is ours to run.
|
|
8014
|
+
* - **`"duplicate"`**: an earlier delivery of this operation already
|
|
8015
|
+
* fired: it journaled a run, or ran without one
|
|
8016
|
+
* ({@link FIRED_WITHOUT_RUN_ID}).
|
|
8017
|
+
* - **`"abandoned"`**: the operation has been delivered
|
|
8018
|
+
* {@link FIRE_CRASH_BUDGET} times and has never once got as far as
|
|
8019
|
+
* journaling a run — i.e. it is taking the process down before the write
|
|
8020
|
+
* lands, and the read-model cursor replays it on every boot. That is the
|
|
8021
|
+
* EPIPE boot loop (backlog item 5), and a budget is what turns it from an
|
|
8022
|
+
* unbootable reactor into one FAILED run naming the loop.
|
|
8023
|
+
*
|
|
8024
|
+
* Two writes, and which one is which matters:
|
|
8025
|
+
*
|
|
8026
|
+
* - The **attempt count** is bumped in its own committed transaction,
|
|
8027
|
+
* BEFORE the risky work. That is the only way a delivery that leaves no
|
|
8028
|
+
* run behind can be counted at all, which is what the budget needs. The
|
|
8029
|
+
* counter is never a claim signal — it only decides the budget.
|
|
8030
|
+
* - The **claim** is `run_id`, and it is set in the SAME transaction as the
|
|
8031
|
+
* run row, under a `WHERE run_id IS NULL` guard. The guard takes the
|
|
8032
|
+
* row's lock, so of two concurrent deliveries of one operation the second
|
|
8033
|
+
* blocks until the first commits and then matches no row: exactly one
|
|
8034
|
+
* delivery wins, and the run it journaled is committed with the claim. A
|
|
8035
|
+
* crash in between rolls BOTH back, so a claim that holds a run id always
|
|
8036
|
+
* holds a run — and a claim without one is retried, up to the budget.
|
|
8037
|
+
*
|
|
8038
|
+
* Counting and claiming are deliberately not the same write: if the bump
|
|
8039
|
+
* rode along inside the claim transaction, a run insert that takes the
|
|
8040
|
+
* process down would roll the count back with it and the budget could never
|
|
8041
|
+
* reach its limit — the boot loop it exists to bound.
|
|
8042
|
+
*/
|
|
7544
8043
|
async claimAndEnqueueRun(dedupeKey, ttlMs, nowIso, options) {
|
|
8044
|
+
const attempt = await this.recordFireAttempt(options.workflowId, dedupeKey, ttlMs, nowIso);
|
|
8045
|
+
if (attempt.runId !== null) return { outcome: "duplicate" };
|
|
8046
|
+
if (attempt.attempts > 3) return {
|
|
8047
|
+
outcome: "abandoned",
|
|
8048
|
+
attempts: attempt.attempts
|
|
8049
|
+
};
|
|
7545
8050
|
const id = randomUUID();
|
|
7546
8051
|
if (!await this.db.transaction().execute(async (trx) => {
|
|
7547
|
-
if (
|
|
8052
|
+
if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
|
|
7548
8053
|
await this.insertPendingRun(trx, id, options);
|
|
7549
8054
|
return true;
|
|
7550
|
-
})) return
|
|
8055
|
+
})) return { outcome: "duplicate" };
|
|
7551
8056
|
this.runsInFlight.add(id);
|
|
8057
|
+
return {
|
|
8058
|
+
outcome: "claimed",
|
|
8059
|
+
runId: id
|
|
8060
|
+
};
|
|
8061
|
+
}
|
|
8062
|
+
/**
|
|
8063
|
+
* Counts one delivery of an operation, and reports whether a run was ever
|
|
8064
|
+
* journaled for it.
|
|
8065
|
+
*
|
|
8066
|
+
* Committed on its own: the point is to leave a trace even when whatever
|
|
8067
|
+
* comes next takes the process down. It is NOT a claim — the row it writes
|
|
8068
|
+
* holds no run id, and {@link claimAndEnqueueRun} is what decides, under a
|
|
8069
|
+
* lock, which delivery gets to fill it in.
|
|
8070
|
+
*/
|
|
8071
|
+
async recordFireAttempt(workflowId, dedupeKey, ttlMs, nowIso) {
|
|
8072
|
+
const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
|
|
8073
|
+
await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
|
|
8074
|
+
const row = await this.db.insertInto("trigger_dedupe").values({
|
|
8075
|
+
workflow_id: workflowId,
|
|
8076
|
+
dedupe_key: dedupeKey,
|
|
8077
|
+
run_id: null,
|
|
8078
|
+
created_at: nowIso,
|
|
8079
|
+
attempts: 1
|
|
8080
|
+
}).onConflict((oc) => oc.columns(["workflow_id", "dedupe_key"]).doUpdateSet({ attempts: sql`${sql.table("trigger_dedupe")}.attempts + 1` })).returning(["attempts", "run_id"]).executeTakeFirstOrThrow();
|
|
8081
|
+
return {
|
|
8082
|
+
attempts: row.attempts ?? 1,
|
|
8083
|
+
runId: row.run_id
|
|
8084
|
+
};
|
|
8085
|
+
}
|
|
8086
|
+
/**
|
|
8087
|
+
* The abandonment of an over-budget fire, as a visible FAILED run, linked to
|
|
8088
|
+
* its dedupe key so a later delivery is a duplicate rather than another one.
|
|
8089
|
+
*/
|
|
8090
|
+
async journalAbandonedFire(dedupeKey, options) {
|
|
8091
|
+
const id = randomUUID();
|
|
8092
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8093
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8094
|
+
await trx.insertInto("run").values({
|
|
8095
|
+
id,
|
|
8096
|
+
workflow_id: options.workflowId,
|
|
8097
|
+
workflow_name: "",
|
|
8098
|
+
workflow_version: 0,
|
|
8099
|
+
trigger_kind: options.triggerKind,
|
|
8100
|
+
trigger_payload: cappedPayload(redact(options.triggerPayload)),
|
|
8101
|
+
status: "FAILED",
|
|
8102
|
+
error: redactMessage(abandonedFireError(options.attempts)),
|
|
8103
|
+
enqueued_at: now,
|
|
8104
|
+
started_at: now,
|
|
8105
|
+
ended_at: now,
|
|
8106
|
+
rerun_of: null,
|
|
8107
|
+
warnings: 0,
|
|
8108
|
+
warning_notes: null
|
|
8109
|
+
}).execute();
|
|
8110
|
+
await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).execute();
|
|
8111
|
+
});
|
|
7552
8112
|
return id;
|
|
7553
8113
|
}
|
|
7554
8114
|
async insertPendingRun(db, id, options) {
|
|
@@ -7717,6 +8277,58 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7717
8277
|
for (const row of rows) byRun.get(row.run_id)?.push(row.document_id);
|
|
7718
8278
|
return byRun;
|
|
7719
8279
|
}
|
|
8280
|
+
/**
|
|
8281
|
+
* Parks a workflow: its park row, and an ENABLED trigger_state row turned
|
|
8282
|
+
* PARKED, in one transaction. True when a trigger_state row was parked.
|
|
8283
|
+
*/
|
|
8284
|
+
async parkWorkflow(workflowId, publishedVersion, reason) {
|
|
8285
|
+
const { workflow_id: _, ...row } = workflowParkRow(workflowId, publishedVersion, reason);
|
|
8286
|
+
const now = row.parked_at;
|
|
8287
|
+
return this.db.transaction().execute(async (trx) => {
|
|
8288
|
+
await trx.insertInto("workflow_park").values({
|
|
8289
|
+
workflow_id: workflowId,
|
|
8290
|
+
...row
|
|
8291
|
+
}).onConflict((oc) => oc.column("workflow_id").doUpdateSet(row)).execute();
|
|
8292
|
+
return (await trx.updateTable("trigger_state").set({
|
|
8293
|
+
status: PARKED_TRIGGER_STATUS,
|
|
8294
|
+
last_error: row.reason,
|
|
8295
|
+
updated_at: now
|
|
8296
|
+
}).where("workflow_id", "=", workflowId).where("status", "=", "ENABLED").returning("workflow_id").execute()).length > 0;
|
|
8297
|
+
});
|
|
8298
|
+
}
|
|
8299
|
+
/** A disable clears a park: the park row, and a PARKED trigger row turned
|
|
8300
|
+
* DISABLED, so re-enabling arms it whether or not the supervisor held it. */
|
|
8301
|
+
async clearParkOnDisable(workflowId) {
|
|
8302
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8303
|
+
await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
|
|
8304
|
+
await trx.updateTable("trigger_state").set({
|
|
8305
|
+
status: "DISABLED",
|
|
8306
|
+
...RENEW_CLEARED,
|
|
8307
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8308
|
+
}).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
|
|
8309
|
+
});
|
|
8310
|
+
}
|
|
8311
|
+
/** Undoes {@link parkWorkflow}; `trigger` says whether it parked a row. */
|
|
8312
|
+
async liftPark(workflowId, trigger) {
|
|
8313
|
+
await this.db.transaction().execute(async (trx) => {
|
|
8314
|
+
await trx.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
|
|
8315
|
+
if (!trigger) return;
|
|
8316
|
+
await trx.updateTable("trigger_state").set({
|
|
8317
|
+
status: "ENABLED",
|
|
8318
|
+
last_error: null,
|
|
8319
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
8320
|
+
}).where("workflow_id", "=", workflowId).where("status", "=", PARKED_TRIGGER_STATUS).execute();
|
|
8321
|
+
});
|
|
8322
|
+
}
|
|
8323
|
+
async getWorkflowPark(workflowId) {
|
|
8324
|
+
return this.db.selectFrom("workflow_park").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
|
|
8325
|
+
}
|
|
8326
|
+
async listWorkflowParks() {
|
|
8327
|
+
return this.db.selectFrom("workflow_park").selectAll().execute();
|
|
8328
|
+
}
|
|
8329
|
+
async clearWorkflowPark(workflowId) {
|
|
8330
|
+
await this.db.deleteFrom("workflow_park").where("workflow_id", "=", workflowId).execute();
|
|
8331
|
+
}
|
|
7720
8332
|
async getTriggerState(workflowId) {
|
|
7721
8333
|
return this.db.selectFrom("trigger_state").selectAll().where("workflow_id", "=", workflowId).executeTakeFirst();
|
|
7722
8334
|
}
|
|
@@ -7800,7 +8412,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7800
8412
|
}).where("workflow_id", "=", workflowId).execute();
|
|
7801
8413
|
}
|
|
7802
8414
|
async claimDedupe(workflowId, dedupeKey, ttlMs, nowIso) {
|
|
7803
|
-
return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso
|
|
8415
|
+
return claimDedupeIn(this.db, workflowId, dedupeKey, ttlMs, nowIso);
|
|
7804
8416
|
}
|
|
7805
8417
|
async deleteDedupe(workflowId) {
|
|
7806
8418
|
await this.db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).execute();
|
|
@@ -7848,7 +8460,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7848
8460
|
for (const row of live) erasedRuns.add(row.id);
|
|
7849
8461
|
erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
|
|
7850
8462
|
erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
|
|
7851
|
-
erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id:
|
|
8463
|
+
erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: FIRED_WITHOUT_RUN_ID }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
|
|
7852
8464
|
erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
|
|
7853
8465
|
return erased;
|
|
7854
8466
|
});
|
|
@@ -7900,6 +8512,46 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
7900
8512
|
}
|
|
7901
8513
|
};
|
|
7902
8514
|
//#endregion
|
|
8515
|
+
//#region src/reactor/park-state.ts
|
|
8516
|
+
var ParkState = class {
|
|
8517
|
+
load;
|
|
8518
|
+
parks = /* @__PURE__ */ new Map();
|
|
8519
|
+
written = /* @__PURE__ */ new Set();
|
|
8520
|
+
seeded = false;
|
|
8521
|
+
seeding;
|
|
8522
|
+
constructor(load) {
|
|
8523
|
+
this.load = load;
|
|
8524
|
+
}
|
|
8525
|
+
async get(workflowId) {
|
|
8526
|
+
if (!this.seeded) {
|
|
8527
|
+
this.seeding ??= this.seed().finally(() => {
|
|
8528
|
+
this.seeding = void 0;
|
|
8529
|
+
});
|
|
8530
|
+
await this.seeding;
|
|
8531
|
+
}
|
|
8532
|
+
return this.parks.get(workflowId);
|
|
8533
|
+
}
|
|
8534
|
+
known(workflowId) {
|
|
8535
|
+
if (!this.seeded && !this.written.has(workflowId)) return void 0;
|
|
8536
|
+
return { park: this.parks.get(workflowId) };
|
|
8537
|
+
}
|
|
8538
|
+
set(park) {
|
|
8539
|
+
this.parks.set(park.workflow_id, park);
|
|
8540
|
+
if (!this.seeded) this.written.add(park.workflow_id);
|
|
8541
|
+
}
|
|
8542
|
+
delete(workflowId) {
|
|
8543
|
+
this.parks.delete(workflowId);
|
|
8544
|
+
if (!this.seeded) this.written.add(workflowId);
|
|
8545
|
+
}
|
|
8546
|
+
async seed() {
|
|
8547
|
+
const rows = await this.load();
|
|
8548
|
+
if (!rows) return;
|
|
8549
|
+
for (const row of rows) if (!this.written.has(row.workflow_id)) this.parks.set(row.workflow_id, row);
|
|
8550
|
+
this.written.clear();
|
|
8551
|
+
this.seeded = true;
|
|
8552
|
+
}
|
|
8553
|
+
};
|
|
8554
|
+
//#endregion
|
|
7903
8555
|
//#region src/reactor/run-cursor.ts
|
|
7904
8556
|
var InvalidRunCursorError = class extends Error {
|
|
7905
8557
|
constructor() {
|
|
@@ -7924,13 +8576,121 @@ function decodeRunCursor(cursor) {
|
|
|
7924
8576
|
};
|
|
7925
8577
|
}
|
|
7926
8578
|
//#endregion
|
|
8579
|
+
//#region src/reactor/run-gate.ts
|
|
8580
|
+
const QUEUE_DEPTH_ENV = "PH_WORKFLOWS_MAX_QUEUED_FIRINGS";
|
|
8581
|
+
function maxQueuedFirings(env = process.env) {
|
|
8582
|
+
const raw = Number(env[QUEUE_DEPTH_ENV]);
|
|
8583
|
+
return Number.isFinite(raw) && raw > 0 ? raw : 100;
|
|
8584
|
+
}
|
|
8585
|
+
var WorkflowRunGate = class {
|
|
8586
|
+
lanes = /* @__PURE__ */ new Map();
|
|
8587
|
+
maxQueued;
|
|
8588
|
+
constructor(options = {}) {
|
|
8589
|
+
this.maxQueued = options.maxQueued ?? maxQueuedFirings();
|
|
8590
|
+
}
|
|
8591
|
+
/** Runs of this workflow executing right now. */
|
|
8592
|
+
active(workflowId) {
|
|
8593
|
+
return this.lanes.get(workflowId)?.active ?? 0;
|
|
8594
|
+
}
|
|
8595
|
+
/** Firings of this workflow waiting for a slot. */
|
|
8596
|
+
waiting(workflowId) {
|
|
8597
|
+
return this.lanes.get(workflowId)?.waiting.length ?? 0;
|
|
8598
|
+
}
|
|
8599
|
+
/**
|
|
8600
|
+
* Takes a slot for one run.
|
|
8601
|
+
*
|
|
8602
|
+
* - **PARALLEL** with no `maxParallelRuns`, or no policy at all: no gate,
|
|
8603
|
+
* so an unbounded workflow's hot path costs nothing. Not the default: a
|
|
8604
|
+
* workflow document starts as QUEUE.
|
|
8605
|
+
* - **PARALLEL** with a bound, and **QUEUE**: waits, first come first
|
|
8606
|
+
* served. Latency, not failure — which is what QUEUE means, up to
|
|
8607
|
+
* {@link DEFAULT_MAX_QUEUED_FIRINGS} waiters. Past that the queue is full
|
|
8608
|
+
* and the firing is refused, so the caller journals it CANCELLED rather
|
|
8609
|
+
* than growing a backlog nothing bounds.
|
|
8610
|
+
* - **SINGLETON**: refused outright while a run is active. The caller
|
|
8611
|
+
* journals the refusal rather than dropping it silently, because a firing
|
|
8612
|
+
* that vanished is indistinguishable from a trigger that never fired.
|
|
8613
|
+
*/
|
|
8614
|
+
async admit(workflowId, policy) {
|
|
8615
|
+
const limit = concurrencyLimit(policy);
|
|
8616
|
+
if (limit === null) return {
|
|
8617
|
+
admitted: true,
|
|
8618
|
+
waited: false,
|
|
8619
|
+
release: () => void 0
|
|
8620
|
+
};
|
|
8621
|
+
const lane = this.lanes.get(workflowId) ?? {
|
|
8622
|
+
active: 0,
|
|
8623
|
+
waiting: []
|
|
8624
|
+
};
|
|
8625
|
+
this.lanes.set(workflowId, lane);
|
|
8626
|
+
let waited = false;
|
|
8627
|
+
if (lane.active >= limit) {
|
|
8628
|
+
if (policy.concurrency === "SINGLETON") {
|
|
8629
|
+
this.forgetIfIdle(workflowId, lane);
|
|
8630
|
+
return {
|
|
8631
|
+
admitted: false,
|
|
8632
|
+
refusal: "singleton",
|
|
8633
|
+
reason: "Skipped: this workflow's concurrency is SINGLETON and a run was already executing"
|
|
8634
|
+
};
|
|
8635
|
+
}
|
|
8636
|
+
if (lane.waiting.length >= this.maxQueued) return {
|
|
8637
|
+
admitted: false,
|
|
8638
|
+
refusal: "queue-full",
|
|
8639
|
+
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
8640
|
+
};
|
|
8641
|
+
await new Promise((resolve) => lane.waiting.push(resolve));
|
|
8642
|
+
waited = true;
|
|
8643
|
+
} else lane.active += 1;
|
|
8644
|
+
let released = false;
|
|
8645
|
+
return {
|
|
8646
|
+
admitted: true,
|
|
8647
|
+
waited,
|
|
8648
|
+
release: () => {
|
|
8649
|
+
if (released) return;
|
|
8650
|
+
released = true;
|
|
8651
|
+
lane.active -= 1;
|
|
8652
|
+
const next = lane.waiting.shift();
|
|
8653
|
+
if (next) {
|
|
8654
|
+
lane.active += 1;
|
|
8655
|
+
next();
|
|
8656
|
+
return;
|
|
8657
|
+
}
|
|
8658
|
+
this.forgetIfIdle(workflowId, lane);
|
|
8659
|
+
}
|
|
8660
|
+
};
|
|
8661
|
+
}
|
|
8662
|
+
forgetIfIdle(workflowId, lane) {
|
|
8663
|
+
if (lane.active === 0 && lane.waiting.length === 0) this.lanes.delete(workflowId);
|
|
8664
|
+
}
|
|
8665
|
+
};
|
|
8666
|
+
//#endregion
|
|
7927
8667
|
//#region src/reactor/run-retention.ts
|
|
7928
8668
|
const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
|
|
7929
8669
|
const RETENTION_SWEEP_INTERVAL_MS = 36e5;
|
|
7930
8670
|
const DAY_MS = 864e5;
|
|
8671
|
+
/** The spellings that turn retention OFF, compared case-insensitively after
|
|
8672
|
+
* trimming. Wider than `0` because "off" and "never" are what an operator
|
|
8673
|
+
* reaches for, and a variable that silently keeps pruning because they wrote
|
|
8674
|
+
* one of those is the worst outcome available. */
|
|
8675
|
+
const RETENTION_OFF_SPELLINGS = [
|
|
8676
|
+
"0",
|
|
8677
|
+
"off",
|
|
8678
|
+
"never",
|
|
8679
|
+
"false",
|
|
8680
|
+
"none"
|
|
8681
|
+
];
|
|
8682
|
+
/**
|
|
8683
|
+
* The retention window, or undefined when an operator turned it off.
|
|
8684
|
+
*
|
|
8685
|
+
* Unset means the default. An unparseable value also means the default rather
|
|
8686
|
+
* than silently disabling: the variable's purpose is a bound, and a typo must
|
|
8687
|
+
* not remove one.
|
|
8688
|
+
*/
|
|
7931
8689
|
function runRetentionMs(env = process.env) {
|
|
7932
|
-
const
|
|
7933
|
-
|
|
8690
|
+
const raw = env[RUN_RETENTION_ENV]?.trim();
|
|
8691
|
+
if (raw !== void 0 && RETENTION_OFF_SPELLINGS.includes(raw.toLowerCase())) return;
|
|
8692
|
+
const days = Number(raw);
|
|
8693
|
+
return (days > 0 ? days : 30) * DAY_MS;
|
|
7934
8694
|
}
|
|
7935
8695
|
async function sweepRetention(store, options) {
|
|
7936
8696
|
const now = (options.now ?? /* @__PURE__ */ new Date()).getTime();
|
|
@@ -8026,18 +8786,23 @@ var TriggerSupervisor = class {
|
|
|
8026
8786
|
hookTimeoutMs;
|
|
8027
8787
|
now;
|
|
8028
8788
|
egress;
|
|
8789
|
+
parks;
|
|
8029
8790
|
timer;
|
|
8030
8791
|
ops = Promise.resolve();
|
|
8031
8792
|
ticking = false;
|
|
8032
8793
|
warnedMissingJournal = false;
|
|
8033
8794
|
constructor(options) {
|
|
8034
8795
|
this.options = options;
|
|
8035
|
-
this.worker = options.worker ?? new PieceWorker({
|
|
8796
|
+
this.worker = options.worker ?? new PieceWorker({
|
|
8797
|
+
models: options.models,
|
|
8798
|
+
telemetry: options.telemetry
|
|
8799
|
+
});
|
|
8036
8800
|
this.tickMs = options.tickMs ?? 15e3;
|
|
8037
8801
|
this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
|
|
8038
8802
|
this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
|
|
8039
8803
|
this.now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
8040
8804
|
this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
|
|
8805
|
+
this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
|
|
8041
8806
|
}
|
|
8042
8807
|
start() {
|
|
8043
8808
|
if (this.timer) return;
|
|
@@ -8067,28 +8832,84 @@ var TriggerSupervisor = class {
|
|
|
8067
8832
|
return this.options.resolver ?? (this.own ??= bundleResolver({ cacheDir: this.options.cacheDir }));
|
|
8068
8833
|
}
|
|
8069
8834
|
enableRetries = /* @__PURE__ */ new Map();
|
|
8070
|
-
|
|
8835
|
+
versions = /* @__PURE__ */ new Map();
|
|
8836
|
+
upsert(binding, publishedVersion) {
|
|
8837
|
+
if (publishedVersion !== void 0) this.versions.set(binding.workflowId, publishedVersion);
|
|
8071
8838
|
const previous = this.bindings.get(binding.workflowId);
|
|
8072
8839
|
if (previous && this.enabledOk.has(binding.workflowId) && JSON.stringify(previous) === JSON.stringify(binding)) return Promise.resolve();
|
|
8073
8840
|
this.bindings.set(binding.workflowId, binding);
|
|
8841
|
+
const park = this.parks.known(binding.workflowId)?.park;
|
|
8842
|
+
if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
|
|
8074
8843
|
return this.enqueue(() => this.enable(binding, previous));
|
|
8075
8844
|
}
|
|
8845
|
+
/** Parks a workflow; true when a trigger row was parked with it. */
|
|
8846
|
+
park(workflowId, publishedVersion, reason) {
|
|
8847
|
+
this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
|
|
8848
|
+
return this.enqueue(async () => {
|
|
8849
|
+
const store = await this.options.store();
|
|
8850
|
+
if (!store) return false;
|
|
8851
|
+
return store.parkWorkflow(workflowId, publishedVersion, reason);
|
|
8852
|
+
});
|
|
8853
|
+
}
|
|
8854
|
+
/** Undoes {@link park} for that version only: a later park stands. */
|
|
8855
|
+
liftPark(workflowId, publishedVersion, trigger) {
|
|
8856
|
+
if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
|
|
8857
|
+
return this.enqueue(async () => {
|
|
8858
|
+
const store = await this.options.store();
|
|
8859
|
+
if (!store) return;
|
|
8860
|
+
if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
|
|
8861
|
+
await store.liftPark(workflowId, trigger);
|
|
8862
|
+
});
|
|
8863
|
+
}
|
|
8864
|
+
/** A disable clears the park. With no binding in memory to release a PARKED
|
|
8865
|
+
* row through onDisable (a boot), the row is turned DISABLED here. */
|
|
8866
|
+
releasePark(workflowId, flipRow) {
|
|
8867
|
+
this.parks.delete(workflowId);
|
|
8868
|
+
return this.enqueue(async () => {
|
|
8869
|
+
const store = await this.options.store();
|
|
8870
|
+
if (!store) return;
|
|
8871
|
+
if (flipRow) await store.clearParkOnDisable(workflowId);
|
|
8872
|
+
else await store.clearWorkflowPark(workflowId);
|
|
8873
|
+
});
|
|
8874
|
+
}
|
|
8875
|
+
/** A re-publish past the parked version: on this lane, a park older than
|
|
8876
|
+
* `publishedVersion` goes and a row it parked is ENABLED again, as it was
|
|
8877
|
+
* before (only an ENABLED row is ever parked); a park as new stands. The
|
|
8878
|
+
* next upsert of the same binding then arms rather than skipping as
|
|
8879
|
+
* unchanged, and a retry held back by the park is due now. */
|
|
8880
|
+
unpark(workflowId, publishedVersion) {
|
|
8881
|
+
this.enabledOk.delete(workflowId);
|
|
8882
|
+
const retry = this.enableRetries.get(workflowId);
|
|
8883
|
+
if (retry) retry.at = 0;
|
|
8884
|
+
const park = this.parks.known(workflowId)?.park;
|
|
8885
|
+
if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
|
|
8886
|
+
return this.enqueue(async () => {
|
|
8887
|
+
const store = await this.options.store();
|
|
8888
|
+
if (!store) return;
|
|
8889
|
+
const park = await store.getWorkflowPark(workflowId);
|
|
8890
|
+
if (!park || park.published_version >= publishedVersion) return;
|
|
8891
|
+
await store.liftPark(workflowId, true);
|
|
8892
|
+
});
|
|
8893
|
+
}
|
|
8076
8894
|
remove(workflowId) {
|
|
8077
8895
|
const binding = this.unbind(workflowId);
|
|
8078
8896
|
return this.enqueue(() => this.disable(workflowId, binding));
|
|
8079
8897
|
}
|
|
8080
8898
|
forget(workflowId) {
|
|
8081
8899
|
const binding = this.unbind(workflowId);
|
|
8900
|
+
this.parks.delete(workflowId);
|
|
8082
8901
|
return this.enqueue(async () => {
|
|
8083
8902
|
const store = await this.options.store();
|
|
8084
8903
|
if (!store) return;
|
|
8085
8904
|
await this.disable(workflowId, binding);
|
|
8086
8905
|
await store.deleteTriggerState(workflowId);
|
|
8906
|
+
await store.clearWorkflowPark(workflowId);
|
|
8087
8907
|
});
|
|
8088
8908
|
}
|
|
8089
8909
|
unbind(workflowId) {
|
|
8090
8910
|
const binding = this.bindings.get(workflowId);
|
|
8091
8911
|
this.bindings.delete(workflowId);
|
|
8912
|
+
this.versions.delete(workflowId);
|
|
8092
8913
|
this.enabledOk.delete(workflowId);
|
|
8093
8914
|
this.enableRetries.delete(workflowId);
|
|
8094
8915
|
return binding;
|
|
@@ -8106,6 +8927,10 @@ var TriggerSupervisor = class {
|
|
|
8106
8927
|
logger$1.warn(`Workflow ${workflowId} could not be resolved, and its trigger row is left as it stands: a registration from before this reactor started is presumed live, and releasing it needs the binding that would not resolve. @error`, message);
|
|
8107
8928
|
return;
|
|
8108
8929
|
}
|
|
8930
|
+
if (existing?.status === "PARKED" && existing.config_hash === hash) {
|
|
8931
|
+
logger$1.warn(`Workflow ${workflowId} is PARKED and could not be resolved; the park stands. @error`, message);
|
|
8932
|
+
return;
|
|
8933
|
+
}
|
|
8109
8934
|
await store.upsertTriggerState({
|
|
8110
8935
|
workflow_id: workflowId,
|
|
8111
8936
|
...triggerBlockColumns(block),
|
|
@@ -8258,8 +9083,14 @@ var TriggerSupervisor = class {
|
|
|
8258
9083
|
const store = await this.options.store();
|
|
8259
9084
|
if (!store) throw new MissingJournalError("Enabling a trigger");
|
|
8260
9085
|
const hash = configHash(binding.block, binding.config);
|
|
8261
|
-
const existing = await store.getTriggerState(binding.workflowId);
|
|
8262
9086
|
const now = this.now();
|
|
9087
|
+
const park = await store.getWorkflowPark(binding.workflowId);
|
|
9088
|
+
if (park && await this.blockingPark(store, binding.workflowId)) {
|
|
9089
|
+
this.holdParked(binding, superseded, park);
|
|
9090
|
+
return;
|
|
9091
|
+
}
|
|
9092
|
+
if (park) await store.liftPark(binding.workflowId, true);
|
|
9093
|
+
const existing = await store.getTriggerState(binding.workflowId);
|
|
8263
9094
|
const isRepublish = existing?.config_hash === hash && existing.status === "ENABLED" && !store.hasUnmigratedTriggerState(binding.workflowId);
|
|
8264
9095
|
if (existing && !isRepublish && existing.status === "ENABLED") await this.disableRow(binding.workflowId, existing, superseded);
|
|
8265
9096
|
const pending = this.enableRetries.get(binding.workflowId);
|
|
@@ -8290,7 +9121,7 @@ var TriggerSupervisor = class {
|
|
|
8290
9121
|
workflow_id: binding.workflowId,
|
|
8291
9122
|
...triggerBlockColumns(binding.block),
|
|
8292
9123
|
config_hash: hash,
|
|
8293
|
-
status:
|
|
9124
|
+
status: await this.armedStatus(store, binding.workflowId),
|
|
8294
9125
|
store_state: VESTIGIAL_STORE_STATE,
|
|
8295
9126
|
interval_ms: intervalMs,
|
|
8296
9127
|
next_poll_at: new Date(now.getTime() + intervalMs).toISOString(),
|
|
@@ -8352,6 +9183,43 @@ var TriggerSupervisor = class {
|
|
|
8352
9183
|
if (!url) throw new Error("The reactor's webhook endpoint is not available yet for this workflow");
|
|
8353
9184
|
return url;
|
|
8354
9185
|
}
|
|
9186
|
+
/**
|
|
9187
|
+
* The park, when it holds this registration back.
|
|
9188
|
+
*
|
|
9189
|
+
* `policy.onFailure = PARK` is a **runtime override of the document's
|
|
9190
|
+
* enabled-ness**: the document still says ENABLED, and re-arming from it is
|
|
9191
|
+
* what a restart does for every workflow it finds. So the park row is the
|
|
9192
|
+
* authority, read on this lane, and it blocks every registration of the
|
|
9193
|
+
* version that failed or an older one — a boot re-arm, an ERROR row's
|
|
9194
|
+
* retry. A newer version is what a re-publish is: it is never blocked, and
|
|
9195
|
+
* {@link enable} lifts the park it outlived. A re-enable lifts it through
|
|
9196
|
+
* the disable before it.
|
|
9197
|
+
*/
|
|
9198
|
+
async blockingPark(store, workflowId) {
|
|
9199
|
+
const park = await store.getWorkflowPark(workflowId);
|
|
9200
|
+
if (!park) return void 0;
|
|
9201
|
+
const version = this.versions.get(workflowId);
|
|
9202
|
+
return version === void 0 || version <= park.published_version ? park : void 0;
|
|
9203
|
+
}
|
|
9204
|
+
/**
|
|
9205
|
+
* An enable a park held back. The row is left as it stands, and a pending
|
|
9206
|
+
* retry keeps its release flag, so the enable after the park lifts releases
|
|
9207
|
+
* a failed attempt and arms afresh; the retry is only pushed back, so it
|
|
9208
|
+
* does not take every tick's slot. A binding this replaced is restored, so
|
|
9209
|
+
* nothing is released later with one that was never armed.
|
|
9210
|
+
*/
|
|
9211
|
+
holdParked(binding, superseded, park) {
|
|
9212
|
+
const { workflowId } = binding;
|
|
9213
|
+
const retry = this.enableRetries.get(workflowId);
|
|
9214
|
+
if (retry) retry.at = this.now().getTime() + this.defaultIntervalMs;
|
|
9215
|
+
if (superseded && this.bindings.get(workflowId) === binding) this.bindings.set(workflowId, superseded);
|
|
9216
|
+
logger$1.warn(`Workflow ${workflowId} is PARKED at version ${park.published_version}; its trigger is not armed until the workflow is re-published or re-enabled. @error`, park.reason);
|
|
9217
|
+
}
|
|
9218
|
+
/** The status an enable that succeeded writes: never ENABLED while a park
|
|
9219
|
+
* stands, whatever ran meanwhile. The registration stays recorded. */
|
|
9220
|
+
async armedStatus(store, workflowId) {
|
|
9221
|
+
return await this.blockingPark(store, workflowId) ? PARKED_TRIGGER_STATUS : "ENABLED";
|
|
9222
|
+
}
|
|
8355
9223
|
deferToStoredRetry(binding, hash, existing, now) {
|
|
8356
9224
|
if (existing?.status !== "ERROR" || existing.config_hash !== hash) return false;
|
|
8357
9225
|
const pending = this.enableRetries.get(binding.workflowId);
|
|
@@ -8394,7 +9262,7 @@ var TriggerSupervisor = class {
|
|
|
8394
9262
|
const nextAt = carried ? new Date(carried) : nextFireAt(schedule, now);
|
|
8395
9263
|
await store.upsertTriggerState({
|
|
8396
9264
|
...base,
|
|
8397
|
-
status:
|
|
9265
|
+
status: await this.armedStatus(store, binding.workflowId),
|
|
8398
9266
|
interval_ms: schedule.mode === "interval" ? schedule.everyMs : MIN_SCHEDULE_INTERVAL_MS,
|
|
8399
9267
|
next_poll_at: nextAt.toISOString(),
|
|
8400
9268
|
last_error: null,
|
|
@@ -8943,6 +9811,17 @@ function parseWorkflowState(resultingState) {
|
|
|
8943
9811
|
return;
|
|
8944
9812
|
}
|
|
8945
9813
|
}
|
|
9814
|
+
function refusalStatus(refusal) {
|
|
9815
|
+
switch (refusal) {
|
|
9816
|
+
case "parked":
|
|
9817
|
+
case "singleton":
|
|
9818
|
+
case "stale": return 409;
|
|
9819
|
+
case "queue-full":
|
|
9820
|
+
case "expired": return 429;
|
|
9821
|
+
default: return 500;
|
|
9822
|
+
}
|
|
9823
|
+
}
|
|
9824
|
+
const OPERATOR_RUN_KINDS = /* @__PURE__ */ new Set(["manual", "rerun"]);
|
|
8946
9825
|
function registrationKey(state) {
|
|
8947
9826
|
const trigger = runnableDefinition(state).trigger;
|
|
8948
9827
|
const { lastTest: _lastTest, ...armed } = trigger ?? {};
|
|
@@ -8961,9 +9840,11 @@ function runJournalName(stateName, documentName) {
|
|
|
8961
9840
|
var WorkflowRuntimeService = class {
|
|
8962
9841
|
host;
|
|
8963
9842
|
logger;
|
|
9843
|
+
telemetry;
|
|
8964
9844
|
attachments;
|
|
8965
9845
|
executor;
|
|
8966
9846
|
pieceWorkers;
|
|
9847
|
+
unobservePool;
|
|
8967
9848
|
storePromise;
|
|
8968
9849
|
storeError;
|
|
8969
9850
|
storeOpening = false;
|
|
@@ -8978,13 +9859,31 @@ var WorkflowRuntimeService = class {
|
|
|
8978
9859
|
seedError;
|
|
8979
9860
|
constructor(host) {
|
|
8980
9861
|
this.host = host;
|
|
9862
|
+
this.telemetry = new WorkflowTelemetry(host.telemetry);
|
|
8981
9863
|
this.blockResolver = new BlockResolver({
|
|
8982
9864
|
local: installedPiece,
|
|
8983
9865
|
timeoutMs: host.pieceVersionLookupMs ?? PIECE_VERSION_LOOKUP_TIMEOUT_MS,
|
|
8984
9866
|
hasBlock: (ref, version, source) => this.versionHasBlock(ref, version, source)
|
|
8985
9867
|
});
|
|
8986
9868
|
this.logger = host.logger ?? logger;
|
|
8987
|
-
this.attachments = host.attachments ? createAttachmentPort(host.attachments,
|
|
9869
|
+
this.attachments = host.attachments ? createAttachmentPort(host.attachments, {
|
|
9870
|
+
documentIdFor: () => currentWorkflowId(),
|
|
9871
|
+
canReadRef: async (ref) => {
|
|
9872
|
+
const workflowId = currentWorkflowId();
|
|
9873
|
+
const origins = currentAttachmentOrigins();
|
|
9874
|
+
if (!workflowId) return false;
|
|
9875
|
+
if (origins?.written.has(ref)) return true;
|
|
9876
|
+
if (!host.canReadAttachmentRef) return false;
|
|
9877
|
+
const runUser = currentRunUser();
|
|
9878
|
+
return host.canReadAttachmentRef({
|
|
9879
|
+
workflowId,
|
|
9880
|
+
ref,
|
|
9881
|
+
documentIds: [...new Set(origins?.documentIds() ?? [])],
|
|
9882
|
+
...runUser !== void 0 ? { runUser } : {}
|
|
9883
|
+
});
|
|
9884
|
+
},
|
|
9885
|
+
onWritten: (ref) => currentAttachmentOrigins()?.written.add(ref)
|
|
9886
|
+
}) : void 0;
|
|
8988
9887
|
this.storePromise = this.openStore();
|
|
8989
9888
|
this.seedPromise = this.seedWithRetries();
|
|
8990
9889
|
this.startRetention();
|
|
@@ -9093,9 +9992,22 @@ var WorkflowRuntimeService = class {
|
|
|
9093
9992
|
return await this.endpointCount() > 0;
|
|
9094
9993
|
}
|
|
9095
9994
|
registeredAs = /* @__PURE__ */ new Map();
|
|
9096
|
-
|
|
9995
|
+
registrationChains = /* @__PURE__ */ new Map();
|
|
9996
|
+
inRegistrationOrder(workflowId, task) {
|
|
9997
|
+
const run = (this.registrationChains.get(workflowId) ?? Promise.resolve()).then(task);
|
|
9998
|
+
const tail = run.then(() => void 0, () => void 0);
|
|
9999
|
+
this.registrationChains.set(workflowId, tail);
|
|
10000
|
+
tail.then(() => {
|
|
10001
|
+
if (this.registrationChains.get(workflowId) === tail) this.registrationChains.delete(workflowId);
|
|
10002
|
+
});
|
|
10003
|
+
return run;
|
|
10004
|
+
}
|
|
10005
|
+
updateRegistration(workflowId, state, onlyIfChanged = false) {
|
|
10006
|
+
return this.inRegistrationOrder(workflowId, () => this.registerNow(workflowId, state, onlyIfChanged));
|
|
10007
|
+
}
|
|
10008
|
+
async registerNow(workflowId, state, onlyIfChanged) {
|
|
9097
10009
|
const key = registrationKey(state);
|
|
9098
|
-
if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key) return;
|
|
10010
|
+
if (onlyIfChanged && !this.unarmed.has(workflowId) && this.registeredAs.get(workflowId) === key && !await this.outdatedPark(workflowId, state)) return;
|
|
9099
10011
|
this.registeredAs.set(workflowId, key);
|
|
9100
10012
|
try {
|
|
9101
10013
|
await this.applyRegistration(workflowId, state);
|
|
@@ -9114,6 +10026,23 @@ var WorkflowRuntimeService = class {
|
|
|
9114
10026
|
}
|
|
9115
10027
|
const trigger = state.status === "ENABLED" ? runnableDefinition(state).trigger : void 0;
|
|
9116
10028
|
const block = trigger ? triggerBlock(trigger) : void 0;
|
|
10029
|
+
const version = runnableDefinition(state).version;
|
|
10030
|
+
if (state.status !== "ENABLED") {
|
|
10031
|
+
const held = this.registry.get(workflowId);
|
|
10032
|
+
this.supervisor().releasePark(workflowId, !(held && SUPERVISED_KINDS.has(held.kind))).catch((error) => {
|
|
10033
|
+
this.logger.error(`Could not clear the park of disabled workflow ${workflowId}`, error);
|
|
10034
|
+
});
|
|
10035
|
+
} else if (await this.outdatedPark(workflowId, state)) this.supervisor().unpark(workflowId, version).catch((error) => {
|
|
10036
|
+
this.logger.error(`Could not lift the outdated park of workflow ${workflowId}`, error);
|
|
10037
|
+
});
|
|
10038
|
+
const park = block ? await this.parks.get(workflowId) : void 0;
|
|
10039
|
+
if (block && (blockKey(block) === WEBHOOK_BLOCK || triggerKindOf(block)) && park && park.published_version >= version) {
|
|
10040
|
+
const had = this.registry.get(workflowId);
|
|
10041
|
+
this.registry.delete(workflowId);
|
|
10042
|
+
if (had && SUPERVISED_KINDS.has(had.kind)) this.dropSupervised(workflowId);
|
|
10043
|
+
this.logger.warn(`Workflow ${workflowId} is PARKED; its trigger stays unarmed until the workflow is re-published or re-enabled`);
|
|
10044
|
+
return;
|
|
10045
|
+
}
|
|
9117
10046
|
if (block && blockKey(block) === WEBHOOK_BLOCK) {
|
|
9118
10047
|
await this.registerWebhook(workflowId, block, trigger.config);
|
|
9119
10048
|
return;
|
|
@@ -9136,14 +10065,14 @@ var WorkflowRuntimeService = class {
|
|
|
9136
10065
|
workflowId,
|
|
9137
10066
|
kind: "schedule"
|
|
9138
10067
|
});
|
|
9139
|
-
this.enableSupervised(workflowId, supervised);
|
|
10068
|
+
this.enableSupervised(workflowId, supervised, version);
|
|
9140
10069
|
return;
|
|
9141
10070
|
}
|
|
9142
10071
|
this.registry.set(workflowId, {
|
|
9143
10072
|
workflowId,
|
|
9144
10073
|
kind: "piece"
|
|
9145
10074
|
});
|
|
9146
|
-
await this.registerPieceTrigger(workflowId, supervised);
|
|
10075
|
+
await this.registerPieceTrigger(workflowId, supervised, version);
|
|
9147
10076
|
return;
|
|
9148
10077
|
}
|
|
9149
10078
|
const had = this.registry.get(workflowId);
|
|
@@ -9159,12 +10088,12 @@ var WorkflowRuntimeService = class {
|
|
|
9159
10088
|
filter: parseLifecycleFilter(config)
|
|
9160
10089
|
});
|
|
9161
10090
|
}
|
|
9162
|
-
enableSupervised(workflowId, binding) {
|
|
9163
|
-
this.supervisor().upsert(binding).catch((error) => {
|
|
10091
|
+
enableSupervised(workflowId, binding, publishedVersion) {
|
|
10092
|
+
this.supervisor().upsert(binding, publishedVersion).catch((error) => {
|
|
9164
10093
|
this.logger.error(`Trigger enable failed for ${workflowId}`, error);
|
|
9165
10094
|
});
|
|
9166
10095
|
}
|
|
9167
|
-
async registerPieceTrigger(workflowId, binding) {
|
|
10096
|
+
async registerPieceTrigger(workflowId, binding, publishedVersion) {
|
|
9168
10097
|
const delivery = await this.pieceDelivery(binding);
|
|
9169
10098
|
if (typeof delivery !== "string") {
|
|
9170
10099
|
this.refusePieceTrigger(workflowId, binding, delivery);
|
|
@@ -9182,7 +10111,7 @@ var WorkflowRuntimeService = class {
|
|
|
9182
10111
|
});
|
|
9183
10112
|
await (await this.endpoints())?.endpointFor(workflowId);
|
|
9184
10113
|
}
|
|
9185
|
-
this.enableSupervised(workflowId, resolved);
|
|
10114
|
+
this.enableSupervised(workflowId, resolved, publishedVersion);
|
|
9186
10115
|
}
|
|
9187
10116
|
async pieceDelivery(binding) {
|
|
9188
10117
|
let strategy;
|
|
@@ -9434,7 +10363,7 @@ var WorkflowRuntimeService = class {
|
|
|
9434
10363
|
throw new Error("Erasing purged documents needs the run journal", { cause: this.storeError });
|
|
9435
10364
|
}
|
|
9436
10365
|
for (const workflowId of purgedWorkflowIds(markers)) await this.erasePurgedWorkflow(store, workflowId);
|
|
9437
|
-
const erased = await store.eraseRunsForDocuments(documentIds);
|
|
10366
|
+
const erased = await this.telemetry.phase("journal.purge", { "documents.count": documentIds.length }, () => store.eraseRunsForDocuments(documentIds));
|
|
9438
10367
|
if (erased.runs > 0) this.logger.info("Erased @runs workflow run(s) of purged documents @ids", erased.runs, documentIds);
|
|
9439
10368
|
return erased;
|
|
9440
10369
|
}
|
|
@@ -9498,7 +10427,7 @@ var WorkflowRuntimeService = class {
|
|
|
9498
10427
|
if (this.unjournaledFires.has(fireKey)) {
|
|
9499
10428
|
if (!store) return;
|
|
9500
10429
|
try {
|
|
9501
|
-
await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString());
|
|
10430
|
+
if (!await store.claimDedupe(workflowId, dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString())) this.logger.warn(`Workflow ${workflowId}: a ${kind} fire this process ran unjournaled had its key already marked fired; it may have run twice`);
|
|
9502
10431
|
this.unjournaledFires.delete(fireKey);
|
|
9503
10432
|
} catch (error) {
|
|
9504
10433
|
this.logger.warn(`Could not record the unjournaled ${kind} fire for workflow ${workflowId}: @error`, error);
|
|
@@ -9509,26 +10438,39 @@ var WorkflowRuntimeService = class {
|
|
|
9509
10438
|
this.fireUnjournaled(fireKey, workflowId, payload, kind);
|
|
9510
10439
|
return;
|
|
9511
10440
|
}
|
|
9512
|
-
|
|
10441
|
+
const enqueue = {
|
|
10442
|
+
workflowId,
|
|
10443
|
+
triggerKind: kind,
|
|
10444
|
+
triggerPayload: payload
|
|
10445
|
+
};
|
|
10446
|
+
let claim;
|
|
9513
10447
|
try {
|
|
9514
|
-
|
|
9515
|
-
workflowId,
|
|
9516
|
-
triggerKind: kind,
|
|
9517
|
-
triggerPayload: payload
|
|
9518
|
-
});
|
|
10448
|
+
claim = await store.claimAndEnqueueRun(dedupeKey, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString(), enqueue);
|
|
9519
10449
|
} catch (error) {
|
|
9520
10450
|
this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
|
|
9521
10451
|
this.fireUnjournaled(fireKey, workflowId, payload, kind);
|
|
9522
10452
|
return;
|
|
9523
10453
|
}
|
|
9524
|
-
if (
|
|
9525
|
-
|
|
10454
|
+
if (claim.outcome === "duplicate") return;
|
|
10455
|
+
if (claim.outcome === "abandoned") {
|
|
10456
|
+
this.logger.error(`Workflow ${workflowId}: a ${kind} fire has been delivered ${claim.attempts} times without ever journaling a run; abandoning it rather than replaying it on every boot`);
|
|
10457
|
+
try {
|
|
10458
|
+
await store.journalAbandonedFire(dedupeKey, {
|
|
10459
|
+
...enqueue,
|
|
10460
|
+
attempts: claim.attempts
|
|
10461
|
+
});
|
|
10462
|
+
} catch (error) {
|
|
10463
|
+
this.logger.warn(`Could not journal the abandoned ${kind} fire for workflow ${workflowId}: @error`, error);
|
|
10464
|
+
}
|
|
10465
|
+
return;
|
|
10466
|
+
}
|
|
10467
|
+
this.fireFromTrigger(workflowId, payload, kind, claim.runId);
|
|
9526
10468
|
}
|
|
9527
10469
|
fireFromTrigger(workflowId, payload, kind, enqueuedRunId) {
|
|
9528
10470
|
this.fire(workflowId, payload, kind, void 0, void 0, enqueuedRunId).then((run) => {
|
|
9529
10471
|
this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
|
|
9530
10472
|
}, (error) => {
|
|
9531
|
-
this.logger.error(`${kind} run failed for workflow ${workflowId}`, error);
|
|
10473
|
+
this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
|
|
9532
10474
|
});
|
|
9533
10475
|
}
|
|
9534
10476
|
firedLifecycle = /* @__PURE__ */ new Set();
|
|
@@ -9607,8 +10549,14 @@ var WorkflowRuntimeService = class {
|
|
|
9607
10549
|
this.cancelTriggerTest(workflowId, "stopped: the workflow was deleted");
|
|
9608
10550
|
}
|
|
9609
10551
|
async releaseDeleted(workflowId) {
|
|
9610
|
-
|
|
10552
|
+
let failure;
|
|
10553
|
+
try {
|
|
10554
|
+
await this.supervisor().forget(workflowId);
|
|
10555
|
+
} catch (error) {
|
|
10556
|
+
failure = { error };
|
|
10557
|
+
}
|
|
9611
10558
|
await (await this.endpoints())?.revoke(workflowId);
|
|
10559
|
+
if (failure) throw failure.error;
|
|
9612
10560
|
}
|
|
9613
10561
|
async erasePurgedWorkflow(store, workflowId) {
|
|
9614
10562
|
this.dropDeleted(workflowId);
|
|
@@ -9666,9 +10614,11 @@ var WorkflowRuntimeService = class {
|
|
|
9666
10614
|
});
|
|
9667
10615
|
}
|
|
9668
10616
|
triggerSupervisor;
|
|
10617
|
+
parks = new ParkState(async () => (await this.store())?.listWorkflowParks());
|
|
9669
10618
|
supervisor() {
|
|
9670
10619
|
this.triggerSupervisor ??= new TriggerSupervisor({
|
|
9671
10620
|
store: () => this.store(),
|
|
10621
|
+
parks: this.parks,
|
|
9672
10622
|
resolveAuth: async (connectionId, request) => {
|
|
9673
10623
|
if (!connectionId) return void 0;
|
|
9674
10624
|
const resolved = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolveWithSecrets(connectionId, request);
|
|
@@ -9680,6 +10630,7 @@ var WorkflowRuntimeService = class {
|
|
|
9680
10630
|
webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
|
|
9681
10631
|
reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
|
|
9682
10632
|
models: this.models,
|
|
10633
|
+
telemetry: this.telemetry,
|
|
9683
10634
|
cacheDir: bundleCacheDir(),
|
|
9684
10635
|
resolver: pieceResolver(),
|
|
9685
10636
|
egress: configuredEgress(),
|
|
@@ -9701,6 +10652,8 @@ var WorkflowRuntimeService = class {
|
|
|
9701
10652
|
for (const workflowId of [...this.webhookTests.keys()]) this.cancelTriggerTest(workflowId, "stopped: the runtime shut down");
|
|
9702
10653
|
this.stopTriggerSupervisor();
|
|
9703
10654
|
this.pieceWorkers?.dispose();
|
|
10655
|
+
this.unobservePool?.();
|
|
10656
|
+
this.unobservePool = void 0;
|
|
9704
10657
|
this.designWorker?.dispose();
|
|
9705
10658
|
this.designWorker = void 0;
|
|
9706
10659
|
}
|
|
@@ -9847,7 +10800,7 @@ var WorkflowRuntimeService = class {
|
|
|
9847
10800
|
};
|
|
9848
10801
|
}
|
|
9849
10802
|
return {
|
|
9850
|
-
status: run.result.status === "SUCCEEDED" ? config.responseStatus :
|
|
10803
|
+
status: run.result.status === "SUCCEEDED" ? config.responseStatus : refusalStatus(run.result.refusal),
|
|
9851
10804
|
contentType: JSON_CONTENT_TYPE,
|
|
9852
10805
|
body: JSON.stringify({
|
|
9853
10806
|
runId: run.result.runId,
|
|
@@ -9933,14 +10886,14 @@ var WorkflowRuntimeService = class {
|
|
|
9933
10886
|
async stepReactorAccess(request) {
|
|
9934
10887
|
const declared = await this.declaredReactor(request.block);
|
|
9935
10888
|
if (!declared) return void 0;
|
|
9936
|
-
const base = await this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null);
|
|
9937
|
-
return reactorTap(this.host, base, runJournal());
|
|
10889
|
+
const base = await this.telemetry.phase("reactor.scope", {}, () => this.reactorScope(request.reactorConnectionId, declared, currentRunUser() ?? null));
|
|
10890
|
+
return reactorTap(this.host, base, runJournal(), {}, this.telemetry);
|
|
9938
10891
|
}
|
|
9939
10892
|
async triggerReactorAccess(binding, requireReactor, runUser) {
|
|
9940
10893
|
const bound = binding.reactorConnectionId;
|
|
9941
10894
|
const user = runUser !== void 0 ? runUser : await assertReactorConnectionsReadable(this.host, binding.workflowId, new Set(bound ? [bound] : [])) ?? null;
|
|
9942
10895
|
const base = await this.reactorScope(bound, requireReactor, user);
|
|
9943
|
-
return reactorTap(this.host, base, NO_JOURNAL);
|
|
10896
|
+
return reactorTap(this.host, base, NO_JOURNAL, {}, this.telemetry);
|
|
9944
10897
|
}
|
|
9945
10898
|
async designReactorAccess(block, ctx, reactorConnectionId) {
|
|
9946
10899
|
if (!await this.declaredReactor(block)) return void 0;
|
|
@@ -9951,6 +10904,12 @@ var WorkflowRuntimeService = class {
|
|
|
9951
10904
|
}
|
|
9952
10905
|
descriptors = /* @__PURE__ */ new Map();
|
|
9953
10906
|
designWorker;
|
|
10907
|
+
createDesignWorker() {
|
|
10908
|
+
return new PieceWorker({
|
|
10909
|
+
models: this.models,
|
|
10910
|
+
telemetry: this.telemetry
|
|
10911
|
+
});
|
|
10912
|
+
}
|
|
9954
10913
|
designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
|
|
9955
10914
|
describing = /* @__PURE__ */ new Map();
|
|
9956
10915
|
pieceDescriptor(target) {
|
|
@@ -9971,7 +10930,7 @@ var WorkflowRuntimeService = class {
|
|
|
9971
10930
|
let descriptor = this.descriptors.get(cacheKey);
|
|
9972
10931
|
if (!descriptor) {
|
|
9973
10932
|
const piece = await pieceResolver().resolve(target);
|
|
9974
|
-
this.designWorker ??=
|
|
10933
|
+
this.designWorker ??= this.createDesignWorker();
|
|
9975
10934
|
let output;
|
|
9976
10935
|
try {
|
|
9977
10936
|
output = (await this.designWorker.describePiece({
|
|
@@ -10002,7 +10961,14 @@ var WorkflowRuntimeService = class {
|
|
|
10002
10961
|
async runs(args, ctx) {
|
|
10003
10962
|
return (await this.runsPage(args, ctx)).records;
|
|
10004
10963
|
}
|
|
10005
|
-
|
|
10964
|
+
runsPage(args, ctx) {
|
|
10965
|
+
return this.telemetry.phase("runs.page", { "runs.scope": args.workflowId ? "workflow" : args.driveId ? "drive" : "all" }, async (span) => {
|
|
10966
|
+
const page = await this.listRunsPage(args, ctx);
|
|
10967
|
+
span.setAttribute("runs.returned", page.records.length);
|
|
10968
|
+
return page;
|
|
10969
|
+
});
|
|
10970
|
+
}
|
|
10971
|
+
async listRunsPage(args, ctx) {
|
|
10006
10972
|
const empty = {
|
|
10007
10973
|
records: [],
|
|
10008
10974
|
hasNextPage: false,
|
|
@@ -10158,7 +11124,7 @@ var WorkflowRuntimeService = class {
|
|
|
10158
11124
|
}
|
|
10159
11125
|
let outcome;
|
|
10160
11126
|
try {
|
|
10161
|
-
this.designWorker ??=
|
|
11127
|
+
this.designWorker ??= this.createDesignWorker();
|
|
10162
11128
|
outcome = (await this.designWorker.checkConnection({
|
|
10163
11129
|
...moduleRef,
|
|
10164
11130
|
auth: shapedAuth,
|
|
@@ -10447,7 +11413,7 @@ var WorkflowRuntimeService = class {
|
|
|
10447
11413
|
if (connectionId) auth = await new DocumentConnectionResolver(this.host, this.secretProvider(), this.oauthRefresher()).resolve(connectionId, { piecePackage: parsed.packageName });
|
|
10448
11414
|
const piece = await pieceResolver().resolve(targetOf(parsed));
|
|
10449
11415
|
const reactor = await this.designReactorAccess(parsed, ctx, reactorConnectionId);
|
|
10450
|
-
this.designWorker ??=
|
|
11416
|
+
this.designWorker ??= this.createDesignWorker();
|
|
10451
11417
|
return (await this.designWorker.resolveOptions({
|
|
10452
11418
|
...pieceModuleRef(piece),
|
|
10453
11419
|
actionName: parsed.name,
|
|
@@ -10618,7 +11584,7 @@ var WorkflowRuntimeService = class {
|
|
|
10618
11584
|
} catch (error) {
|
|
10619
11585
|
const errorName = errorNameOf(error);
|
|
10620
11586
|
await recordTest({
|
|
10621
|
-
status: "FAILED",
|
|
11587
|
+
status: isIndeterminateError(error) ? "INDETERMINATE" : "FAILED",
|
|
10622
11588
|
error: pieceFailureDetail(error, "Trigger test timed out"),
|
|
10623
11589
|
...errorName ? { errorName } : {}
|
|
10624
11590
|
});
|
|
@@ -10709,10 +11675,11 @@ var WorkflowRuntimeService = class {
|
|
|
10709
11675
|
triggerKind: TEST_TRIGGER_KIND
|
|
10710
11676
|
});
|
|
10711
11677
|
await store.recordStep(runId, 0, record);
|
|
11678
|
+
const error = record.status === "INDETERMINATE" ? `This test is INDETERMINATE: ${record.error ?? "a host call it made timed out"}` : record.error;
|
|
10712
11679
|
await store.finishRun(runId, {
|
|
10713
|
-
status: record.status === "
|
|
11680
|
+
status: record.status === "SUCCEEDED" ? "SUCCEEDED" : "FAILED",
|
|
10714
11681
|
steps: [record],
|
|
10715
|
-
...
|
|
11682
|
+
...error ? { error } : {},
|
|
10716
11683
|
...record.errorName ? { errorName: record.errorName } : {}
|
|
10717
11684
|
});
|
|
10718
11685
|
} catch (error) {
|
|
@@ -10733,30 +11700,76 @@ var WorkflowRuntimeService = class {
|
|
|
10733
11700
|
}
|
|
10734
11701
|
}
|
|
10735
11702
|
blockExecutor(store) {
|
|
10736
|
-
return this.executor ??= createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request));
|
|
11703
|
+
return this.executor ??= tracedSteps(createBlockExecutor(this.host, this.secretProvider(), this.attachments, store ? createPieceStorePort(store, currentWorkflowId) : void 0, (block) => this.resolveBlock(block), this.oauthRefresher(), (request) => this.stepReactorAccess(request), this.telemetry), this.telemetry);
|
|
10737
11704
|
}
|
|
10738
11705
|
workers() {
|
|
10739
|
-
|
|
11706
|
+
if (this.pieceWorkers) return this.pieceWorkers;
|
|
11707
|
+
const pool = new PieceWorkerPool({
|
|
10740
11708
|
models: this.models,
|
|
10741
11709
|
size: Number(process.env.PH_WORKFLOWS_RUN_CONCURRENCY) || void 0,
|
|
10742
|
-
maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0
|
|
11710
|
+
maxQueueDepth: Number(process.env.PH_WORKFLOWS_RUN_QUEUE_DEPTH) || void 0,
|
|
11711
|
+
telemetry: this.telemetry
|
|
10743
11712
|
});
|
|
11713
|
+
this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
|
|
11714
|
+
return this.pieceWorkers = pool;
|
|
10744
11715
|
}
|
|
10745
11716
|
async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
|
|
11717
|
+
const started = performance.now();
|
|
11718
|
+
const attributes = {
|
|
11719
|
+
"workflow.id": workflowId,
|
|
11720
|
+
"trigger.kind": triggerKind,
|
|
11721
|
+
...resume ? { "run.rerun_of": resume.rerunOf } : {}
|
|
11722
|
+
};
|
|
11723
|
+
return this.telemetry.span("workflow.run", attributes, async (span) => {
|
|
11724
|
+
const run = {
|
|
11725
|
+
span,
|
|
11726
|
+
refused: false
|
|
11727
|
+
};
|
|
11728
|
+
let status = "FAILED";
|
|
11729
|
+
try {
|
|
11730
|
+
const result = await this.runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId);
|
|
11731
|
+
status = result.status;
|
|
11732
|
+
if (status === "FAILED") span.setStatus({
|
|
11733
|
+
code: SpanStatusCode.ERROR,
|
|
11734
|
+
message: status
|
|
11735
|
+
});
|
|
11736
|
+
return result;
|
|
11737
|
+
} finally {
|
|
11738
|
+
if (run.refused) status = "REFUSED";
|
|
11739
|
+
span.setAttribute("run.status", status);
|
|
11740
|
+
this.telemetry.recordRun(performance.now() - started, status, triggerKind);
|
|
11741
|
+
}
|
|
11742
|
+
});
|
|
11743
|
+
}
|
|
11744
|
+
async runFired(run, workflowId, triggerPayload, triggerKind, resume, ctx, enqueuedRunId) {
|
|
11745
|
+
const firedAt = Date.now();
|
|
10746
11746
|
const store = await this.store();
|
|
10747
11747
|
let state;
|
|
10748
11748
|
let documentName;
|
|
10749
11749
|
let definition;
|
|
10750
11750
|
let runUser;
|
|
10751
11751
|
try {
|
|
10752
|
-
if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx)
|
|
10753
|
-
|
|
10754
|
-
|
|
11752
|
+
if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
11753
|
+
run.refused = true;
|
|
11754
|
+
throw error;
|
|
11755
|
+
});
|
|
11756
|
+
const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
|
|
11757
|
+
if (document.header.documentType !== "powerhouse/workflow") {
|
|
11758
|
+
run.refused = true;
|
|
11759
|
+
throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
|
|
11760
|
+
}
|
|
10755
11761
|
state = document.state.global;
|
|
10756
11762
|
documentName = document.header.name;
|
|
10757
|
-
if (state.status !== "ENABLED")
|
|
11763
|
+
if (state.status !== "ENABLED") {
|
|
11764
|
+
run.refused = true;
|
|
11765
|
+
throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
|
|
11766
|
+
}
|
|
10758
11767
|
definition = toWorkflowDefinition(state);
|
|
10759
|
-
|
|
11768
|
+
const reactorConnections = declaredReactorConnectionIds(definition);
|
|
11769
|
+
runUser = reactorConnections.size === 0 ? void 0 : await this.telemetry.phase("run_user", {}, () => assertReactorConnectionsReadable(this.host, workflowId, reactorConnections)).catch((error) => {
|
|
11770
|
+
run.refused = isReactorError(error, ReactorAccessDeniedError$1);
|
|
11771
|
+
throw error;
|
|
11772
|
+
});
|
|
10760
11773
|
} catch (error) {
|
|
10761
11774
|
if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
|
|
10762
11775
|
throw error;
|
|
@@ -10764,23 +11777,64 @@ var WorkflowRuntimeService = class {
|
|
|
10764
11777
|
const connections = declaredConnectionIds(definition);
|
|
10765
11778
|
const executor = this.blockExecutor(store);
|
|
10766
11779
|
const runnable = runnableDefinition(state);
|
|
10767
|
-
|
|
10768
|
-
|
|
10769
|
-
|
|
10770
|
-
|
|
10771
|
-
});
|
|
10772
|
-
else runId = await store?.startRun({
|
|
10773
|
-
workflowId,
|
|
10774
|
-
workflowName: runJournalName(state.name, documentName),
|
|
10775
|
-
workflowVersion: runnable.version,
|
|
11780
|
+
const policy = effectiveRunPolicy(runnable);
|
|
11781
|
+
const deadline = policy.runTimeoutSeconds ? firedAt + policy.runTimeoutSeconds * 1e3 : void 0;
|
|
11782
|
+
const skipped = (reason, refusal) => this.skipFiring(store, workflowId, enqueuedRunId, {
|
|
11783
|
+
reason,
|
|
10776
11784
|
triggerKind,
|
|
10777
11785
|
triggerPayload,
|
|
10778
|
-
|
|
10779
|
-
|
|
11786
|
+
workflowName: runJournalName(state.name, documentName),
|
|
11787
|
+
workflowVersion: runnable.version
|
|
11788
|
+
}, refusal);
|
|
11789
|
+
let parked;
|
|
11790
|
+
try {
|
|
11791
|
+
parked = await this.parkedFiring(workflowId, triggerKind, runnable.version);
|
|
11792
|
+
} catch (error) {
|
|
11793
|
+
if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
|
|
11794
|
+
throw error;
|
|
11795
|
+
}
|
|
11796
|
+
if (parked) return skipped(parked, "parked");
|
|
11797
|
+
const admission = await this.runGate.admit(workflowId, policy);
|
|
11798
|
+
if (!admission.admitted) return skipped(admission.reason, admission.refusal);
|
|
11799
|
+
if (admission.waited) {
|
|
11800
|
+
let stale;
|
|
11801
|
+
try {
|
|
11802
|
+
stale = await this.staleAfterWait(workflowId, triggerKind, runnable.version);
|
|
11803
|
+
} catch (error) {
|
|
11804
|
+
stale = `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
|
|
11805
|
+
}
|
|
11806
|
+
if (stale) {
|
|
11807
|
+
admission.release();
|
|
11808
|
+
return skipped(stale, "stale");
|
|
11809
|
+
}
|
|
11810
|
+
}
|
|
11811
|
+
if (deadline !== void 0 && Date.now() >= deadline) {
|
|
11812
|
+
admission.release();
|
|
11813
|
+
return skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
|
|
11814
|
+
}
|
|
11815
|
+
let runId = enqueuedRunId ?? null;
|
|
10780
11816
|
let journalFailed = false;
|
|
10781
11817
|
const executionOrder = /* @__PURE__ */ new Map();
|
|
10782
11818
|
let session;
|
|
10783
11819
|
try {
|
|
11820
|
+
runId = await this.telemetry.phase("journal.start", {}, async () => {
|
|
11821
|
+
if (enqueuedRunId) {
|
|
11822
|
+
await store?.beginRun(enqueuedRunId, {
|
|
11823
|
+
workflowName: runJournalName(state.name, documentName),
|
|
11824
|
+
workflowVersion: runnable.version
|
|
11825
|
+
});
|
|
11826
|
+
return enqueuedRunId;
|
|
11827
|
+
}
|
|
11828
|
+
return await store?.startRun({
|
|
11829
|
+
workflowId,
|
|
11830
|
+
workflowName: runJournalName(state.name, documentName),
|
|
11831
|
+
workflowVersion: runnable.version,
|
|
11832
|
+
triggerKind,
|
|
11833
|
+
triggerPayload,
|
|
11834
|
+
rerunOf: resume?.rerunOf
|
|
11835
|
+
}) ?? null;
|
|
11836
|
+
});
|
|
11837
|
+
if (runId) run.span.setAttribute("run.id", runId);
|
|
10784
11838
|
session = this.workers().session();
|
|
10785
11839
|
const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
|
|
10786
11840
|
const journal = store;
|
|
@@ -10792,6 +11846,7 @@ var WorkflowRuntimeService = class {
|
|
|
10792
11846
|
connections,
|
|
10793
11847
|
pieceWorker: session,
|
|
10794
11848
|
...runUser !== void 0 ? { runUser } : {},
|
|
11849
|
+
attachments: attachmentOrigins(triggerPayload, handed, resume?.completedSteps),
|
|
10795
11850
|
recordDocuments: async (documentIds) => {
|
|
10796
11851
|
for (const documentId of documentIds) handed.add(documentId);
|
|
10797
11852
|
if (journal && journaledRunId) await journal.recordRunDocuments(journaledRunId, documentIds);
|
|
@@ -10806,22 +11861,25 @@ var WorkflowRuntimeService = class {
|
|
|
10806
11861
|
...secretValues.length > 0 ? { redactValues: secretValues } : {},
|
|
10807
11862
|
triggerPayload,
|
|
10808
11863
|
completedSteps: resume?.completedSteps,
|
|
10809
|
-
|
|
11864
|
+
...policy.defaultRetry ? { defaultRetry: policy.defaultRetry } : {},
|
|
11865
|
+
...deadline !== void 0 ? { deadline } : {},
|
|
11866
|
+
onStep: store && journaledRunId ? async (record, ordinal) => {
|
|
10810
11867
|
executionOrder.set(record.stepId, ordinal);
|
|
10811
11868
|
try {
|
|
10812
|
-
await store.recordStep(
|
|
11869
|
+
await this.telemetry.phase("journal.step", {}, () => store.recordStep(journaledRunId, ordinal, record));
|
|
10813
11870
|
} catch (error) {
|
|
10814
11871
|
if (journalFailed) return;
|
|
10815
11872
|
journalFailed = true;
|
|
10816
|
-
this.logger.warn(`Run ${
|
|
11873
|
+
this.logger.warn(`Run ${journaledRunId}: journaling step "@step" failed; the run continues without per-step durability: @error`, record.key, error);
|
|
10817
11874
|
}
|
|
10818
11875
|
} : void 0
|
|
10819
11876
|
}));
|
|
10820
|
-
if (store &&
|
|
10821
|
-
await store.finishRun(
|
|
11877
|
+
if (store && journaledRunId) try {
|
|
11878
|
+
await this.telemetry.phase("journal.finish", {}, () => store.finishRun(journaledRunId, result, executionOrder));
|
|
10822
11879
|
} catch (error) {
|
|
10823
11880
|
this.logger.warn(`Run ${runId}: closing the run journal out failed; the run's outcome stands`, error);
|
|
10824
11881
|
}
|
|
11882
|
+
if (result.status === "FAILED" && !OPERATOR_RUN_KINDS.has(triggerKind)) await this.applyFailureMode(policy, workflowId, runnable.version, runId, result.error);
|
|
10825
11883
|
const finished = {
|
|
10826
11884
|
...result,
|
|
10827
11885
|
runId
|
|
@@ -10838,6 +11896,108 @@ var WorkflowRuntimeService = class {
|
|
|
10838
11896
|
throw error;
|
|
10839
11897
|
} finally {
|
|
10840
11898
|
session?.close();
|
|
11899
|
+
admission.release();
|
|
11900
|
+
}
|
|
11901
|
+
}
|
|
11902
|
+
runGate = new WorkflowRunGate();
|
|
11903
|
+
/** Why a trigger's firing of a PARKED workflow is refused; undefined when
|
|
11904
|
+
* it is not parked, or when an operator started the run. */
|
|
11905
|
+
async parkedFiring(workflowId, triggerKind, publishedVersion) {
|
|
11906
|
+
if (OPERATOR_RUN_KINDS.has(triggerKind)) return void 0;
|
|
11907
|
+
const park = await this.parks.get(workflowId);
|
|
11908
|
+
if (!park || park.published_version < publishedVersion) return void 0;
|
|
11909
|
+
return `Skipped: this workflow is PARKED (${park.reason})`;
|
|
11910
|
+
}
|
|
11911
|
+
/** Why a firing that waited for its slot may no longer run, if it may not. */
|
|
11912
|
+
async staleAfterWait(workflowId, triggerKind, publishedVersion) {
|
|
11913
|
+
const state = (await this.host.reactorClient.get(workflowId)).state.global;
|
|
11914
|
+
if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
|
|
11915
|
+
if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
|
|
11916
|
+
return this.parkedFiring(workflowId, triggerKind, publishedVersion);
|
|
11917
|
+
}
|
|
11918
|
+
/** Whether the workflow is still ENABLED at this published version. An
|
|
11919
|
+
* unreadable document counts as unchanged, so a failure still parks. */
|
|
11920
|
+
async stillRunsVersion(workflowId, publishedVersion) {
|
|
11921
|
+
let state;
|
|
11922
|
+
try {
|
|
11923
|
+
state = (await this.host.reactorClient.get(workflowId)).state.global;
|
|
11924
|
+
} catch (error) {
|
|
11925
|
+
this.logger.warn(`Could not re-read workflow ${workflowId} before parking it: @error`, error);
|
|
11926
|
+
return true;
|
|
11927
|
+
}
|
|
11928
|
+
return state.status === "ENABLED" && runnableDefinition(state).version === publishedVersion;
|
|
11929
|
+
}
|
|
11930
|
+
/** A park recorded against an earlier published version than this state's. */
|
|
11931
|
+
async outdatedPark(workflowId, state) {
|
|
11932
|
+
if (state.status !== "ENABLED") return false;
|
|
11933
|
+
const park = await this.parks.get(workflowId);
|
|
11934
|
+
return park !== void 0 && runnableDefinition(state).version > park.published_version;
|
|
11935
|
+
}
|
|
11936
|
+
/**
|
|
11937
|
+
* A firing SINGLETON refused.
|
|
11938
|
+
*
|
|
11939
|
+
* Journaled as a CANCELLED run rather than dropped: a firing that vanished
|
|
11940
|
+
* is indistinguishable from a trigger that never fired, which is the class
|
|
11941
|
+
* of bug this work package exists to stamp out. An already-enqueued row is
|
|
11942
|
+
* closed out in place, so nothing is left PENDING for a sweep to find.
|
|
11943
|
+
*/
|
|
11944
|
+
async skipFiring(store, workflowId, enqueuedRunId, details, refusal) {
|
|
11945
|
+
this.logger.info(`Workflow ${workflowId}: ${details.reason}`);
|
|
11946
|
+
let runId = enqueuedRunId ?? null;
|
|
11947
|
+
if (store) try {
|
|
11948
|
+
if (runId) await store.beginRun(runId, details);
|
|
11949
|
+
else runId = await store.startRun({
|
|
11950
|
+
workflowId,
|
|
11951
|
+
...details
|
|
11952
|
+
});
|
|
11953
|
+
await store.cancelRun(runId, details.reason);
|
|
11954
|
+
} catch (error) {
|
|
11955
|
+
this.logger.warn(`Could not journal the skipped firing of workflow ${workflowId}: @error`, error);
|
|
11956
|
+
}
|
|
11957
|
+
return {
|
|
11958
|
+
status: CANCELLED_RUN_STATUS,
|
|
11959
|
+
steps: [],
|
|
11960
|
+
runId,
|
|
11961
|
+
refusal
|
|
11962
|
+
};
|
|
11963
|
+
}
|
|
11964
|
+
/**
|
|
11965
|
+
* `policy.onFailure` for a run that failed terminally.
|
|
11966
|
+
*
|
|
11967
|
+
* PARK is the document model's own default, so this is where enforcing the
|
|
11968
|
+
* knob becomes visible: a terminal failure parks the workflow, whatever its
|
|
11969
|
+
* trigger kind, and nothing fires it until it is re-published or
|
|
11970
|
+
* re-enabled. That is what PARK means, and
|
|
11971
|
+
* leaving a broken workflow firing every minute is what it meant before.
|
|
11972
|
+
*
|
|
11973
|
+
* NOTIFY logs at error level, which is the only notification channel this
|
|
11974
|
+
* engine has; it is marked as such rather than pretending to page anyone.
|
|
11975
|
+
*/
|
|
11976
|
+
async applyFailureMode(policy, workflowId, publishedVersion, runId, error) {
|
|
11977
|
+
if (policy.onFailure === "IGNORE") return;
|
|
11978
|
+
const detail = error ?? "the run failed";
|
|
11979
|
+
if (policy.onFailure === "NOTIFY") {
|
|
11980
|
+
this.logger.error(`Workflow ${workflowId} run ${runId ?? "(unjournaled)"} failed and its policy is NOTIFY: @error`, detail);
|
|
11981
|
+
return;
|
|
11982
|
+
}
|
|
11983
|
+
if (!await this.store()) return;
|
|
11984
|
+
const reason = `Parked after a failed run (policy.onFailure = PARK): ${detail}`;
|
|
11985
|
+
try {
|
|
11986
|
+
if (!await this.stillRunsVersion(workflowId, publishedVersion)) return;
|
|
11987
|
+
const trigger = await this.supervisor().park(workflowId, publishedVersion, reason);
|
|
11988
|
+
if (!await this.stillRunsVersion(workflowId, publishedVersion)) {
|
|
11989
|
+
await this.supervisor().liftPark(workflowId, publishedVersion, trigger);
|
|
11990
|
+
return;
|
|
11991
|
+
}
|
|
11992
|
+
if (!await this.inRegistrationOrder(workflowId, async () => {
|
|
11993
|
+
const current = (await this.parks.get(workflowId))?.published_version === publishedVersion;
|
|
11994
|
+
const registered = this.registry.get(workflowId);
|
|
11995
|
+
if (current && registered && !SUPERVISED_KINDS.has(registered.kind)) this.registry.delete(workflowId);
|
|
11996
|
+
return current;
|
|
11997
|
+
})) return;
|
|
11998
|
+
this.logger.error(`Workflow ${workflowId} is PARKED after run ${runId ?? "(unjournaled)"} failed; its trigger will not fire again until the workflow is re-published or re-enabled`);
|
|
11999
|
+
} catch (parkError) {
|
|
12000
|
+
this.logger.warn(`Applying onFailure = PARK to workflow ${workflowId} after a failed run failed; whether it is parked is unknown: @error`, parkError);
|
|
10841
12001
|
}
|
|
10842
12002
|
}
|
|
10843
12003
|
async rerun(runId, ctx) {
|
|
@@ -10848,7 +12008,8 @@ var WorkflowRuntimeService = class {
|
|
|
10848
12008
|
await this.assertCanReadDocument(run.workflow_id, ctx);
|
|
10849
12009
|
if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
|
|
10850
12010
|
if (run.trigger_kind === "test") throw new Error(`Run "${runId}" tested a single step or trigger; test it again instead of rerunning it`);
|
|
10851
|
-
|
|
12011
|
+
const deadlineCancelled = run.status === "CANCELLED" && run.error_name === "RunDeadlineExceeded";
|
|
12012
|
+
if (run.status !== "FAILED" && !deadlineCancelled) throw new Error(`Only FAILED runs, and runs their deadline cancelled, can be rerun; run is ${run.status}`);
|
|
10852
12013
|
const triggerPayload = run.trigger_payload === null ? void 0 : JSON.parse(run.trigger_payload);
|
|
10853
12014
|
if (isTruncatedStepPayload(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was truncated by the journal and cannot be replayed; fire the workflow again instead of rerunning it`);
|
|
10854
12015
|
if (containsRedactedMarker(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was redacted and cannot be replayed; fire the workflow again instead of rerunning it`);
|
|
@@ -10860,7 +12021,15 @@ var WorkflowRuntimeService = class {
|
|
|
10860
12021
|
const current = currentSteps.get(row.step_id);
|
|
10861
12022
|
if (!current || current.key !== row.step_key || row.config_hash === null || stepConfigHash(current) !== row.config_hash) continue;
|
|
10862
12023
|
const output = row.output === null ? void 0 : JSON.parse(row.output);
|
|
10863
|
-
if (isTruncatedStepPayload(output))
|
|
12024
|
+
if (isTruncatedStepPayload(output)) {
|
|
12025
|
+
if (await this.rereads(current)) continue;
|
|
12026
|
+
completedSteps.set(row.step_id, {
|
|
12027
|
+
port: row.port,
|
|
12028
|
+
outputTruncated: true,
|
|
12029
|
+
truncatedOutput: output
|
|
12030
|
+
});
|
|
12031
|
+
continue;
|
|
12032
|
+
}
|
|
10864
12033
|
if (containsDocumentRef(output) && await this.rereads(current)) continue;
|
|
10865
12034
|
completedSteps.set(row.step_id, {
|
|
10866
12035
|
output,
|
|
@@ -10947,6 +12116,7 @@ var WorkflowRuntimeService = class {
|
|
|
10947
12116
|
pieceWorker: session,
|
|
10948
12117
|
stepTest: true,
|
|
10949
12118
|
runUser: this.callerRunUser(ctx),
|
|
12119
|
+
attachments: attachmentOrigins(upstream.triggerPayload, handed, upstream.priorSteps),
|
|
10950
12120
|
recordDocuments: async (documentIds) => {
|
|
10951
12121
|
for (const documentId of documentIds) handed.add(documentId);
|
|
10952
12122
|
if (store && journaledRunId) await store.recordRunDocuments(journaledRunId, documentIds);
|
|
@@ -10986,7 +12156,7 @@ var WorkflowRuntimeService = class {
|
|
|
10986
12156
|
const served = ctx ? await this.servesDocuments(ids, ctx) : false;
|
|
10987
12157
|
return {
|
|
10988
12158
|
runId,
|
|
10989
|
-
status: record.status
|
|
12159
|
+
status: testStatusOf(record.status),
|
|
10990
12160
|
...served && record.output !== void 0 ? { output: record.output } : {},
|
|
10991
12161
|
...record.error ? { error: record.error } : {},
|
|
10992
12162
|
...record.errorName ? { errorName: record.errorName } : {},
|
|
@@ -11114,6 +12284,7 @@ var WorkflowRuntimeService = class {
|
|
|
11114
12284
|
testedAt,
|
|
11115
12285
|
error: row.error ?? ""
|
|
11116
12286
|
};
|
|
12287
|
+
if (row.status === "INDETERMINATE") return { kind: "indeterminate" };
|
|
11117
12288
|
const output = row.output === null ? null : JSON.parse(row.output);
|
|
11118
12289
|
if (isTruncatedStepPayload(output)) return { kind: "truncated" };
|
|
11119
12290
|
return {
|
|
@@ -11182,6 +12353,21 @@ var WorkflowRuntimeService = class {
|
|
|
11182
12353
|
function createWorkflowRuntime(deps) {
|
|
11183
12354
|
return new WorkflowRuntimeService(deps);
|
|
11184
12355
|
}
|
|
12356
|
+
function attachmentOrigins(triggerPayload, handed, replayed) {
|
|
12357
|
+
const written = /* @__PURE__ */ new Set();
|
|
12358
|
+
collectAttachmentRefs(replayed, written);
|
|
12359
|
+
return {
|
|
12360
|
+
documentIds: () => [...triggerDocumentIds(triggerPayload), ...handed],
|
|
12361
|
+
written
|
|
12362
|
+
};
|
|
12363
|
+
}
|
|
12364
|
+
function collectAttachmentRefs(value, found) {
|
|
12365
|
+
if (typeof value === "string") {
|
|
12366
|
+
if (value.startsWith("attachment://")) found.add(value);
|
|
12367
|
+
} else if (Array.isArray(value)) for (const entry of value) collectAttachmentRefs(entry, found);
|
|
12368
|
+
else if (value instanceof Map) for (const entry of value.values()) collectAttachmentRefs(entry, found);
|
|
12369
|
+
else if (typeof value === "object" && value !== null) for (const entry of Object.values(value)) collectAttachmentRefs(entry, found);
|
|
12370
|
+
}
|
|
11185
12371
|
//#endregion
|
|
11186
12372
|
//#region src/reactor/workflow-triggers-read-model.ts
|
|
11187
12373
|
const WORKFLOW_TRIGGERS_READ_MODEL = "workflow-triggers";
|
|
@@ -11209,6 +12395,6 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
|
|
|
11209
12395
|
}
|
|
11210
12396
|
};
|
|
11211
12397
|
//#endregion
|
|
11212
|
-
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
|
|
12398
|
+
export { CORE_PIECE_NAME, CORE_PIECE_VERSION, InvalidRunCursorError, PieceRegistry, TELEMETRY_SCOPE, WORKFLOW_PACKAGE_NAME, WORKFLOW_SYNCING_MESSAGE, WORKFLOW_TRIGGERS_READ_MODEL, WORKFLOW_TRIGGERS_READ_MODEL_STAGE, WorkflowRuntimeService, WorkflowSyncingError, WorkflowTriggersReadModel, createAttachmentPort, createWorkflowRuntime, packagePieces, publishRunUser, setPieceRegistryUrl };
|
|
11213
12399
|
|
|
11214
12400
|
//# sourceMappingURL=index.js.map
|