@powerhousedao/reactor-workflow 6.2.3-dev.54 → 6.2.3-dev.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -10
- package/dist/index.d.ts +3327 -3304
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +538 -384
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-BdkdVugw.d.ts → piece-registry-BbeuE8TB.d.ts} +84 -31
- package/dist/piece-registry-BbeuE8TB.d.ts.map +1 -0
- package/dist/{piece-registry-0pdOQQdk.js → telemetry-CM82x7pC.js} +614 -102
- package/dist/telemetry-CM82x7pC.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +2 -2
- package/dist/worker-entry.js +33 -6
- package/dist/worker-entry.js.map +1 -1
- package/package.json +7 -6
- package/dist/piece-registry-0pdOQQdk.js.map +0 -1
- package/dist/piece-registry-BdkdVugw.d.ts.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as rememberSecrets, A as AttachmentCache, B as SecretNotFoundError, C as blockLabel, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as PieceWorkerPool, H as secretRefFromId, I as resolveStepInput, J as PieceWorkerError, K as PieceWorker, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as redactMessage, S as storeHandlers, T as blockPorts, U as stepBlock, V as parseSecretRef, W as triggerBlock, X as containsRedactedMarker, Y as PieceWorkerTimeoutError, Z as redact, _ as effectiveRetryPolicy, _t as npmRegistryBaseUrl, a as PieceRegistry, at as FileTooLargeError, b as isHostBound, bt as isIndeterminateError, c as packagePieces, ct as pieceModuleRef, d as runWorkflow, dt as UnsupportedPieceFeatureError, et as secretsFor, f as canonicalJson, ft as authMethodFor, g as checkDynamicProperties, h as DynamicPropertiesError, i as tracedSteps, it as extractDedupeKey, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as sourcedResolver, m as stepConfigHash, mt as unsupportedTrigger, n as WorkflowTelemetry, nt as isPrivateAddress, o as installedPiece, ot as maxFileBytes, p as hashOf, pt as unsupportedAuth, q as PieceWorkerDisposedError, r as tracedMethods, rt as parseAddress, s as installedPieces, st as bundleResolver, t as TELEMETRY_SCOPE, tt as DEFAULT_EGRESS_POLICY, u as errorNameOf, v as ActivepiecesBlockExecutor, vt as pieceRegistrySource, w as unpinnedNote, x as resolvedBlock, y as CompositeBlockExecutor, yt as setPieceRegistryUrl, z as SecretDeletedError } from "./telemetry-CM82x7pC.js";
|
|
2
2
|
import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
|
|
3
|
-
import {
|
|
3
|
+
import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { open, readdir, rm, stat } from "node:fs/promises";
|
|
5
|
-
import
|
|
5
|
+
import { dirname, join } from "node:path";
|
|
6
6
|
import { childLogger, generateId } from "document-model";
|
|
7
7
|
import { DEFAULT_TIMEZONE, DOCUMENT_REF_KEY, ScheduleConfigError, checkTriggerStrategy, containsDocumentRef, documentRefsIn, expandDocumentRefs, isDocumentRefMarker, parseScheduleConfig, referenceDocuments } from "@powerhousedao/pieces-framework/workflow";
|
|
8
8
|
import { Cron } from "croner";
|
|
9
9
|
import { createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
10
|
-
import { Readable } from "node:stream";
|
|
11
10
|
import dns from "node:dns";
|
|
12
11
|
import net from "node:net";
|
|
13
12
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
@@ -19,9 +18,10 @@ import https from "node:https";
|
|
|
19
18
|
import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
|
|
20
19
|
import { Kind, parse } from "graphql";
|
|
21
20
|
import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
|
|
22
|
-
import { SpanStatusCode, context
|
|
21
|
+
import { SpanStatusCode, context } from "@opentelemetry/api";
|
|
23
22
|
import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
|
|
24
23
|
import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
24
|
+
import { performance as performance$1 } from "node:perf_hooks";
|
|
25
25
|
//#region src/pieces/activepieces/piece-versions.ts
|
|
26
26
|
const LISTED_TTL_MS = 3e5;
|
|
27
27
|
const ABSENT_TTL_MS = 3e4;
|
|
@@ -101,21 +101,6 @@ function npmVersions(name, options = {}) {
|
|
|
101
101
|
//#endregion
|
|
102
102
|
//#region src/reactor/attachment-port.ts
|
|
103
103
|
const logger$7 = childLogger(["workflow", "attachments"]);
|
|
104
|
-
function fileStream(filePath) {
|
|
105
|
-
return Readable.toWeb(createReadStream(filePath));
|
|
106
|
-
}
|
|
107
|
-
async function sha256File(filePath) {
|
|
108
|
-
const hash = createHash("sha256");
|
|
109
|
-
for await (const chunk of createReadStream(filePath)) hash.update(chunk);
|
|
110
|
-
return hash.digest("hex");
|
|
111
|
-
}
|
|
112
|
-
function refOf(result) {
|
|
113
|
-
const direct = result.ref;
|
|
114
|
-
if (typeof direct === "string") return direct;
|
|
115
|
-
const nested = result.reservation?.ref;
|
|
116
|
-
if (typeof nested === "string") return nested;
|
|
117
|
-
throw new Error("The attachment store returned no reference for the upload");
|
|
118
|
-
}
|
|
119
104
|
async function writeCapped(body, destPath, limit) {
|
|
120
105
|
const reader = body.getReader();
|
|
121
106
|
const handle = await open(destPath, "w");
|
|
@@ -162,34 +147,19 @@ function createAttachmentPort(client, access) {
|
|
|
162
147
|
throw new FileTooLargeError(header.sizeBytes, limit);
|
|
163
148
|
}
|
|
164
149
|
await writeCapped(body, destPath, limit);
|
|
165
|
-
const contentType = header.mimeType !==
|
|
150
|
+
const contentType = header.mimeType !== "" ? header.mimeType : void 0;
|
|
166
151
|
return {
|
|
167
152
|
fileName: header.fileName,
|
|
168
153
|
contentType
|
|
169
154
|
};
|
|
170
155
|
},
|
|
171
156
|
async write(file, signal) {
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
ref: `attachment://v1:${hash}`,
|
|
177
|
-
hash,
|
|
178
|
-
sizeBytes: file.size,
|
|
179
|
-
options: {
|
|
180
|
-
mimeType,
|
|
181
|
-
fileName: file.fileName,
|
|
182
|
-
...extension ? { extension } : {},
|
|
183
|
-
clientHash: hash,
|
|
184
|
-
sizeBytes: file.size
|
|
185
|
-
},
|
|
186
|
-
data: fileStream(file.path),
|
|
187
|
-
stream: () => fileStream(file.path)
|
|
188
|
-
};
|
|
189
|
-
const ref = refOf(await client.upload({
|
|
190
|
-
preprocessed,
|
|
157
|
+
const { ref } = await client.upload({
|
|
158
|
+
file: await openAsBlob(file.path),
|
|
159
|
+
fileName: file.fileName,
|
|
160
|
+
mimeType: file.contentType ?? "application/octet-stream",
|
|
191
161
|
...signal ? { signal } : {}
|
|
192
|
-
})
|
|
162
|
+
});
|
|
193
163
|
access.onWritten?.(ref);
|
|
194
164
|
logger$7.debug(`Ingested ${file.fileName} (${file.size} bytes) as ${ref}`);
|
|
195
165
|
return ref;
|
|
@@ -1982,227 +1952,6 @@ async function buildReactorRunScope(host, request) {
|
|
|
1982
1952
|
connection: { access: parsed.config.access ?? "write" }
|
|
1983
1953
|
};
|
|
1984
1954
|
}
|
|
1985
|
-
//#endregion
|
|
1986
|
-
//#region src/telemetry.ts
|
|
1987
|
-
const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
|
|
1988
|
-
var WorkflowTelemetry = class {
|
|
1989
|
-
tracer;
|
|
1990
|
-
meter;
|
|
1991
|
-
runs;
|
|
1992
|
-
runDuration;
|
|
1993
|
-
stepDuration;
|
|
1994
|
-
phaseDuration;
|
|
1995
|
-
poolGauges;
|
|
1996
|
-
constructor(options = {}) {
|
|
1997
|
-
this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
|
|
1998
|
-
this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
|
|
1999
|
-
this.runs = this.meter.createCounter("workflow.runs", {
|
|
2000
|
-
description: "Runs finished, by status and trigger kind",
|
|
2001
|
-
unit: "{run}"
|
|
2002
|
-
});
|
|
2003
|
-
this.runDuration = this.meter.createHistogram("workflow.run.duration", {
|
|
2004
|
-
description: "Run duration from fire to the journal closing it",
|
|
2005
|
-
unit: "ms"
|
|
2006
|
-
});
|
|
2007
|
-
this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
|
|
2008
|
-
description: "Step duration, by piece and status",
|
|
2009
|
-
unit: "ms"
|
|
2010
|
-
});
|
|
2011
|
-
this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
|
|
2012
|
-
description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
|
|
2013
|
-
unit: "ms"
|
|
2014
|
-
});
|
|
2015
|
-
}
|
|
2016
|
-
phase(name, attributes, fn, parent) {
|
|
2017
|
-
const started = performance.now();
|
|
2018
|
-
return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
|
|
2019
|
-
}
|
|
2020
|
-
workerTimings(request, timings, forkedAt, ipc) {
|
|
2021
|
-
const under = (span) => trace.setSpan(context.active(), span);
|
|
2022
|
-
const child = (name, start, end, parent = request, attributes) => {
|
|
2023
|
-
const span = this.tracer.startSpan(`workflow.worker.${name}`, {
|
|
2024
|
-
startTime: start,
|
|
2025
|
-
...attributes ? { attributes } : {}
|
|
2026
|
-
}, under(parent));
|
|
2027
|
-
span.end(end);
|
|
2028
|
-
this.recordPhase(`worker.${name}`, end - start);
|
|
2029
|
-
return span;
|
|
2030
|
-
};
|
|
2031
|
-
const ready = epoch(timings.ready);
|
|
2032
|
-
if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
|
|
2033
|
-
const received = epoch(timings.received);
|
|
2034
|
-
const sent = epoch(timings.sent);
|
|
2035
|
-
const delivered = Math.max(ipc.sent, ready ?? 0);
|
|
2036
|
-
if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
|
|
2037
|
-
const byName = /* @__PURE__ */ new Map();
|
|
2038
|
-
const phases = Array.isArray(timings.phases) ? timings.phases : [];
|
|
2039
|
-
for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
|
|
2040
|
-
const parent = phase.parent ? byName.get(phase.parent) : void 0;
|
|
2041
|
-
byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
|
|
2042
|
-
}
|
|
2043
|
-
if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
|
|
2044
|
-
}
|
|
2045
|
-
recordPhase(name, ms) {
|
|
2046
|
-
this.phaseDuration.record(ms, { phase: name });
|
|
2047
|
-
}
|
|
2048
|
-
span(name, attributes, fn, parent = context.active()) {
|
|
2049
|
-
return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
|
|
2050
|
-
try {
|
|
2051
|
-
return await fn(span);
|
|
2052
|
-
} catch (error) {
|
|
2053
|
-
failSpan(span, error);
|
|
2054
|
-
throw error;
|
|
2055
|
-
} finally {
|
|
2056
|
-
span.end();
|
|
2057
|
-
}
|
|
2058
|
-
});
|
|
2059
|
-
}
|
|
2060
|
-
recordRun(ms, status, triggerKind) {
|
|
2061
|
-
const attributes = {
|
|
2062
|
-
"run.status": status,
|
|
2063
|
-
"trigger.kind": metricTriggerKind(triggerKind)
|
|
2064
|
-
};
|
|
2065
|
-
this.runs.add(1, attributes);
|
|
2066
|
-
this.runDuration.record(ms, attributes);
|
|
2067
|
-
}
|
|
2068
|
-
recordStep(ms, pieceName, status) {
|
|
2069
|
-
this.stepDuration.record(ms, {
|
|
2070
|
-
"piece.name": pieceName,
|
|
2071
|
-
"step.status": status
|
|
2072
|
-
});
|
|
2073
|
-
}
|
|
2074
|
-
observeWorkerPool(stats) {
|
|
2075
|
-
this.poolGauges ??= this.createPoolGauges();
|
|
2076
|
-
const { size, active, waiting } = this.poolGauges;
|
|
2077
|
-
const observables = [
|
|
2078
|
-
size,
|
|
2079
|
-
active,
|
|
2080
|
-
waiting
|
|
2081
|
-
];
|
|
2082
|
-
const callback = (result) => {
|
|
2083
|
-
const current = stats();
|
|
2084
|
-
result.observe(size, current.size);
|
|
2085
|
-
result.observe(active, current.active);
|
|
2086
|
-
result.observe(waiting, current.waiting);
|
|
2087
|
-
};
|
|
2088
|
-
this.meter.addBatchObservableCallback(callback, observables);
|
|
2089
|
-
return () => this.meter.removeBatchObservableCallback(callback, observables);
|
|
2090
|
-
}
|
|
2091
|
-
createPoolGauges() {
|
|
2092
|
-
const gauge = (name, description) => this.meter.createObservableGauge(name, {
|
|
2093
|
-
description,
|
|
2094
|
-
unit: "{worker}"
|
|
2095
|
-
});
|
|
2096
|
-
return {
|
|
2097
|
-
size: gauge("workflow.worker.pool.size", "Worker slots"),
|
|
2098
|
-
active: gauge("workflow.worker.pool.active", "Slots held by runs"),
|
|
2099
|
-
waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
|
|
2100
|
-
};
|
|
2101
|
-
}
|
|
2102
|
-
};
|
|
2103
|
-
const WORKER_PHASES = /* @__PURE__ */ new Set([
|
|
2104
|
-
"piece.load",
|
|
2105
|
-
"reactor.open",
|
|
2106
|
-
"models",
|
|
2107
|
-
"action"
|
|
2108
|
-
]);
|
|
2109
|
-
const WORKER_ATTRIBUTES = {
|
|
2110
|
-
"piece.cached": "boolean",
|
|
2111
|
-
"document.type": "string"
|
|
2112
|
-
};
|
|
2113
|
-
const MAX_ATTRIBUTE_LENGTH = 256;
|
|
2114
|
-
function epoch(value) {
|
|
2115
|
-
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
2116
|
-
}
|
|
2117
|
-
function knownPhase(phase) {
|
|
2118
|
-
if (typeof phase !== "object" || phase === null) return false;
|
|
2119
|
-
const { name, start, end, parent } = phase;
|
|
2120
|
-
return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
|
|
2121
|
-
}
|
|
2122
|
-
function phaseAttributes(attributes) {
|
|
2123
|
-
if (typeof attributes !== "object" || attributes === null) return void 0;
|
|
2124
|
-
const kept = {};
|
|
2125
|
-
for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
|
|
2126
|
-
const value = attributes[key];
|
|
2127
|
-
if (typeof value !== type) continue;
|
|
2128
|
-
kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
|
|
2129
|
-
}
|
|
2130
|
-
return Object.keys(kept).length > 0 ? kept : void 0;
|
|
2131
|
-
}
|
|
2132
|
-
function metricTriggerKind(triggerKind) {
|
|
2133
|
-
return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
|
|
2134
|
-
}
|
|
2135
|
-
function metricPieceName(step, resolution) {
|
|
2136
|
-
return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
|
|
2137
|
-
}
|
|
2138
|
-
function tracedSteps(executor, telemetry) {
|
|
2139
|
-
return { execute(execution) {
|
|
2140
|
-
const { step } = execution;
|
|
2141
|
-
const started = performance.now();
|
|
2142
|
-
let status = "FAILED";
|
|
2143
|
-
let resolution;
|
|
2144
|
-
return telemetry.span("workflow.step", {
|
|
2145
|
-
"step.id": step.id,
|
|
2146
|
-
"step.key": step.key,
|
|
2147
|
-
"piece.name": step.pieceName,
|
|
2148
|
-
"action.name": step.actionName
|
|
2149
|
-
}, async (span) => {
|
|
2150
|
-
try {
|
|
2151
|
-
const result = await executor.execute(execution);
|
|
2152
|
-
resolution = result.resolution;
|
|
2153
|
-
status = "SUCCEEDED";
|
|
2154
|
-
const version = resolution?.resolved?.version;
|
|
2155
|
-
if (version) span.setAttribute("piece.version", version);
|
|
2156
|
-
return result;
|
|
2157
|
-
} catch (error) {
|
|
2158
|
-
resolution = resolutionOf(error);
|
|
2159
|
-
throw error;
|
|
2160
|
-
}
|
|
2161
|
-
}).finally(() => {
|
|
2162
|
-
telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
|
|
2163
|
-
});
|
|
2164
|
-
} };
|
|
2165
|
-
}
|
|
2166
|
-
function failSpan(span, error) {
|
|
2167
|
-
span.recordException(error instanceof Error ? error : String(error));
|
|
2168
|
-
span.setStatus({
|
|
2169
|
-
code: SpanStatusCode.ERROR,
|
|
2170
|
-
message: error instanceof Error ? error.message : String(error)
|
|
2171
|
-
});
|
|
2172
|
-
}
|
|
2173
|
-
function tracedMethods(target, telemetry, prefix, parent) {
|
|
2174
|
-
return new Proxy(target, { get(object, property, receiver) {
|
|
2175
|
-
const value = Reflect.get(object, property, receiver);
|
|
2176
|
-
if (typeof value !== "function" || typeof property !== "string") return value;
|
|
2177
|
-
const phase = `${prefix}.${property}`;
|
|
2178
|
-
return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
|
|
2179
|
-
const started = performance.now();
|
|
2180
|
-
const end = (error) => {
|
|
2181
|
-
if (error !== void 0) failSpan(span, error);
|
|
2182
|
-
span.end();
|
|
2183
|
-
telemetry.recordPhase(phase, performance.now() - started);
|
|
2184
|
-
};
|
|
2185
|
-
let result;
|
|
2186
|
-
try {
|
|
2187
|
-
result = value.apply(object, args);
|
|
2188
|
-
} catch (error) {
|
|
2189
|
-
end(error);
|
|
2190
|
-
throw error;
|
|
2191
|
-
}
|
|
2192
|
-
if (!(result instanceof Promise)) {
|
|
2193
|
-
end();
|
|
2194
|
-
return result;
|
|
2195
|
-
}
|
|
2196
|
-
return result.then((resolved) => {
|
|
2197
|
-
end();
|
|
2198
|
-
return resolved;
|
|
2199
|
-
}, (error) => {
|
|
2200
|
-
end(error);
|
|
2201
|
-
throw error;
|
|
2202
|
-
});
|
|
2203
|
-
});
|
|
2204
|
-
} });
|
|
2205
|
-
}
|
|
2206
1955
|
const JOB_NOT_FOUND = "Job not found";
|
|
2207
1956
|
const MIN_DEADLINE_MARGIN_MS = 250;
|
|
2208
1957
|
const MAX_DEADLINE_MARGIN_MS = 2e3;
|
|
@@ -7606,6 +7355,9 @@ function triggerBlockColumns(block) {
|
|
|
7606
7355
|
}
|
|
7607
7356
|
const FIRED_WITHOUT_RUN_ID = "fired-without-run";
|
|
7608
7357
|
const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
|
|
7358
|
+
function abandonedAdmissionError(attempts) {
|
|
7359
|
+
return `This run was admitted at ${attempts} boots and the reactor stopped each time before it started, so it is no longer admitted on boot. Fix the cause, then rerun it to fire the workflow with the same payload.`;
|
|
7360
|
+
}
|
|
7609
7361
|
function abandonedFireError(attempts) {
|
|
7610
7362
|
return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
|
|
7611
7363
|
}
|
|
@@ -7623,7 +7375,6 @@ const logger$4 = childLogger([
|
|
|
7623
7375
|
"store"
|
|
7624
7376
|
]);
|
|
7625
7377
|
const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
|
|
7626
|
-
const ABANDONED_PENDING_RUN_ERROR = "Reactor stopped before the matched trigger started its run; rerun it to fire the workflow with the same payload";
|
|
7627
7378
|
const PENDING_RUN_STATUS = "PENDING";
|
|
7628
7379
|
function errorMessage(error) {
|
|
7629
7380
|
return error instanceof Error ? error.message : String(error);
|
|
@@ -7707,6 +7458,9 @@ async function up(db) {
|
|
|
7707
7458
|
await db.schema.alterTable("run").addColumn("enqueued_at", "text").execute();
|
|
7708
7459
|
} catch {}
|
|
7709
7460
|
await db.updateTable("run").set({ enqueued_at: sql.ref("started_at") }).where("enqueued_at", "is", null).execute();
|
|
7461
|
+
try {
|
|
7462
|
+
await db.schema.alterTable("run").addColumn("admit_attempts", "integer", (col) => col.notNull().defaultTo(0)).execute();
|
|
7463
|
+
} catch {}
|
|
7710
7464
|
await db.schema.dropIndex("run_workflow_started").ifExists().execute();
|
|
7711
7465
|
await db.schema.dropIndex("run_started").ifExists().execute();
|
|
7712
7466
|
await db.schema.createIndex("run_workflow_enqueued").ifNotExists().on("run").columns([
|
|
@@ -7718,6 +7472,7 @@ async function up(db) {
|
|
|
7718
7472
|
await db.schema.createIndex("trigger_dedupe_workflow_created").ifNotExists().on("trigger_dedupe").columns(["workflow_id", "created_at"]).execute();
|
|
7719
7473
|
await db.schema.createIndex("run_ended").ifNotExists().on("run").column("ended_at").execute();
|
|
7720
7474
|
await db.schema.createIndex("trigger_dedupe_created").ifNotExists().on("trigger_dedupe").column("created_at").execute();
|
|
7475
|
+
await db.schema.createIndex("run_pending").ifNotExists().on("run").columns(["enqueued_at", "id"]).where(sql.ref("status"), "=", PENDING_RUN_STATUS).execute();
|
|
7721
7476
|
await db.schema.createIndex("trigger_state_due").ifNotExists().on("trigger_state").column("next_poll_at").where(sql.ref("status"), "=", "ENABLED").execute();
|
|
7722
7477
|
try {
|
|
7723
7478
|
await db.schema.dropTable("webhook_endpoint").ifExists().execute();
|
|
@@ -7925,6 +7680,10 @@ const LEGACY_MARKER_KEYS = [
|
|
|
7925
7680
|
* ambiguous, and nothing can resolve that for a row already written; the
|
|
7926
7681
|
* window is as narrow as the stored data allows.
|
|
7927
7682
|
*/
|
|
7683
|
+
function unreplayablePayload(payload) {
|
|
7684
|
+
if (isTruncatedStepPayload(payload)) return "truncated";
|
|
7685
|
+
if (containsRedactedMarker(payload)) return "redacted";
|
|
7686
|
+
}
|
|
7928
7687
|
function isTruncatedStepPayload(value) {
|
|
7929
7688
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
7930
7689
|
const record = value;
|
|
@@ -7955,6 +7714,9 @@ function cappedPayload(value) {
|
|
|
7955
7714
|
};
|
|
7956
7715
|
return JSON.stringify(marker);
|
|
7957
7716
|
}
|
|
7717
|
+
function rowHash(values) {
|
|
7718
|
+
return createHash("sha1").update(JSON.stringify(values)).digest("base64");
|
|
7719
|
+
}
|
|
7958
7720
|
function stepValues(runId, ordinal, step) {
|
|
7959
7721
|
return {
|
|
7960
7722
|
run_id: runId,
|
|
@@ -8072,6 +7834,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8072
7834
|
db;
|
|
8073
7835
|
unmigrated;
|
|
8074
7836
|
runsInFlight = /* @__PURE__ */ new Set();
|
|
7837
|
+
journaledSteps = /* @__PURE__ */ new Map();
|
|
8075
7838
|
constructor(db, unmigrated) {
|
|
8076
7839
|
this.db = db;
|
|
8077
7840
|
this.unmigrated = unmigrated;
|
|
@@ -8081,7 +7844,6 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8081
7844
|
const unmigrated = await up(db);
|
|
8082
7845
|
const store = new WorkflowRunStore(db, unmigrated);
|
|
8083
7846
|
await store.recoverOrphanedRuns();
|
|
8084
|
-
await store.recoverAbandonedRuns();
|
|
8085
7847
|
return store;
|
|
8086
7848
|
}
|
|
8087
7849
|
hasUnmigratedTriggerState(workflowId) {
|
|
@@ -8101,16 +7863,56 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8101
7863
|
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
|
|
8102
7864
|
return recovered;
|
|
8103
7865
|
}
|
|
8104
|
-
async
|
|
8105
|
-
|
|
8106
|
-
|
|
8107
|
-
|
|
8108
|
-
|
|
8109
|
-
|
|
8110
|
-
|
|
8111
|
-
|
|
8112
|
-
|
|
8113
|
-
|
|
7866
|
+
async admitAbandonedRuns(options = {}) {
|
|
7867
|
+
const admitted = await this.db.transaction().execute(async (trx) => {
|
|
7868
|
+
let query = trx.selectFrom("run").select([
|
|
7869
|
+
"id",
|
|
7870
|
+
"workflow_id",
|
|
7871
|
+
"trigger_kind",
|
|
7872
|
+
"trigger_payload",
|
|
7873
|
+
"admit_attempts"
|
|
7874
|
+
]).where("status", "=", PENDING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc");
|
|
7875
|
+
if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
|
|
7876
|
+
if (options.enqueuedBefore) query = query.where("enqueued_at", "<", options.enqueuedBefore.toISOString());
|
|
7877
|
+
const rows = await query.execute();
|
|
7878
|
+
const runs = [];
|
|
7879
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7880
|
+
for (const row of rows) {
|
|
7881
|
+
if (row.admit_attempts >= 3) {
|
|
7882
|
+
await trx.updateTable("run").set({
|
|
7883
|
+
status: "FAILED",
|
|
7884
|
+
error: abandonedAdmissionError(row.admit_attempts),
|
|
7885
|
+
ended_at: now
|
|
7886
|
+
}).where("id", "=", row.id).execute();
|
|
7887
|
+
continue;
|
|
7888
|
+
}
|
|
7889
|
+
const triggerPayload = row.trigger_payload === null ? void 0 : JSON.parse(row.trigger_payload);
|
|
7890
|
+
const unreplayable = unreplayablePayload(triggerPayload);
|
|
7891
|
+
if (unreplayable) {
|
|
7892
|
+
await trx.updateTable("run").set({
|
|
7893
|
+
status: "FAILED",
|
|
7894
|
+
error: `The trigger payload was ${unreplayable} by the journal and cannot be run after a restart; fire the workflow again`,
|
|
7895
|
+
ended_at: now
|
|
7896
|
+
}).where("id", "=", row.id).execute();
|
|
7897
|
+
continue;
|
|
7898
|
+
}
|
|
7899
|
+
await trx.updateTable("run").set({ admit_attempts: row.admit_attempts + 1 }).where("id", "=", row.id).execute();
|
|
7900
|
+
runs.push({
|
|
7901
|
+
runId: row.id,
|
|
7902
|
+
workflowId: row.workflow_id,
|
|
7903
|
+
triggerKind: row.trigger_kind,
|
|
7904
|
+
triggerPayload
|
|
7905
|
+
});
|
|
7906
|
+
}
|
|
7907
|
+
return runs;
|
|
7908
|
+
});
|
|
7909
|
+
for (const run of admitted) this.runsInFlight.add(run.runId);
|
|
7910
|
+
if (admitted.length > 0) logger$4.info(`Admitting ${admitted.length} workflow run(s) journaled by a stopped reactor`);
|
|
7911
|
+
return admitted;
|
|
7912
|
+
}
|
|
7913
|
+
async releaseAdmission(runId) {
|
|
7914
|
+
this.runsInFlight.delete(runId);
|
|
7915
|
+
await this.db.updateTable("run").set({ admit_attempts: sql`greatest(admit_attempts - 1, 0)` }).where("id", "=", runId).where("status", "=", PENDING_RUN_STATUS).execute();
|
|
8114
7916
|
}
|
|
8115
7917
|
/**
|
|
8116
7918
|
* Closes a run out as CANCELLED, with the reason as its error.
|
|
@@ -8122,6 +7924,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8122
7924
|
*/
|
|
8123
7925
|
async cancelRun(runId, reason) {
|
|
8124
7926
|
this.runsInFlight.delete(runId);
|
|
7927
|
+
this.journaledSteps.delete(runId);
|
|
8125
7928
|
if (erasedRuns.delete(runId)) return;
|
|
8126
7929
|
await this.db.updateTable("run").set({
|
|
8127
7930
|
status: "CANCELLED",
|
|
@@ -8131,10 +7934,20 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8131
7934
|
}
|
|
8132
7935
|
async enqueueRun(options) {
|
|
8133
7936
|
const id = randomUUID();
|
|
8134
|
-
await this.insertPendingRun(this.db, id, options);
|
|
8135
|
-
this.runsInFlight.add(id);
|
|
7937
|
+
await this.unmarkOnFailure([id], () => this.insertPendingRun(this.db, id, options));
|
|
8136
7938
|
return id;
|
|
8137
7939
|
}
|
|
7940
|
+
async unmarkOnFailure(ids, write) {
|
|
7941
|
+
try {
|
|
7942
|
+
return await write();
|
|
7943
|
+
} catch (error) {
|
|
7944
|
+
for (const id of ids) {
|
|
7945
|
+
this.runsInFlight.delete(id);
|
|
7946
|
+
this.journaledSteps.delete(id);
|
|
7947
|
+
}
|
|
7948
|
+
throw error;
|
|
7949
|
+
}
|
|
7950
|
+
}
|
|
8138
7951
|
/**
|
|
8139
7952
|
* Claims the dedupe key and journals the PENDING run, bounding crash replays.
|
|
8140
7953
|
*
|
|
@@ -8178,38 +7991,39 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8178
7991
|
attempts: attempt.attempts
|
|
8179
7992
|
};
|
|
8180
7993
|
const id = randomUUID();
|
|
8181
|
-
if (!await this.db.transaction().execute(async (trx) => {
|
|
7994
|
+
if (!await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
|
|
8182
7995
|
if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
|
|
8183
7996
|
await this.insertPendingRun(trx, id, options);
|
|
8184
7997
|
return true;
|
|
8185
|
-
})) return { outcome: "duplicate" };
|
|
8186
|
-
this.runsInFlight.add(id);
|
|
7998
|
+
}))) return { outcome: "duplicate" };
|
|
8187
7999
|
return {
|
|
8188
8000
|
outcome: "claimed",
|
|
8189
8001
|
runId: id
|
|
8190
8002
|
};
|
|
8191
8003
|
}
|
|
8192
|
-
async journalTriggerItems(items, ttlMs, nowIso) {
|
|
8193
|
-
const
|
|
8004
|
+
async journalTriggerItems(items, ttlMs, nowIso, success) {
|
|
8005
|
+
const marked = [];
|
|
8006
|
+
return this.unmarkOnFailure(marked, () => this.db.transaction().execute(async (trx) => {
|
|
8007
|
+
if (success) await pollSucceededIn(trx, success.workflowId, success.storeState, nowIso, success.nextPollAtIso);
|
|
8194
8008
|
const journaled = [];
|
|
8195
8009
|
for (const { dedupeKey, options } of items) {
|
|
8196
8010
|
const id = randomUUID();
|
|
8197
8011
|
const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
|
|
8198
|
-
if (won)
|
|
8012
|
+
if (won) {
|
|
8013
|
+
marked.push(id);
|
|
8014
|
+
await this.insertPendingRun(trx, id, options);
|
|
8015
|
+
}
|
|
8199
8016
|
journaled.push(won ? id : void 0);
|
|
8200
8017
|
}
|
|
8201
8018
|
return journaled;
|
|
8202
|
-
});
|
|
8203
|
-
for (const id of ids) if (id) this.runsInFlight.add(id);
|
|
8204
|
-
return ids;
|
|
8019
|
+
}));
|
|
8205
8020
|
}
|
|
8206
8021
|
async recordScheduleFire(nowIso, nextPollAtIso, options) {
|
|
8207
8022
|
const id = randomUUID();
|
|
8208
|
-
await this.db.transaction().execute(async (trx) => {
|
|
8023
|
+
await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
|
|
8209
8024
|
await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
|
|
8210
8025
|
await this.insertPendingRun(trx, id, options);
|
|
8211
|
-
});
|
|
8212
|
-
this.runsInFlight.add(id);
|
|
8026
|
+
}));
|
|
8213
8027
|
return id;
|
|
8214
8028
|
}
|
|
8215
8029
|
/**
|
|
@@ -8245,6 +8059,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8245
8059
|
});
|
|
8246
8060
|
for (const runId of redelivered.runIds) {
|
|
8247
8061
|
this.runsInFlight.delete(runId);
|
|
8062
|
+
this.journaledSteps.delete(runId);
|
|
8248
8063
|
erasedRuns.delete(runId);
|
|
8249
8064
|
}
|
|
8250
8065
|
}
|
|
@@ -8301,6 +8116,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8301
8116
|
return id;
|
|
8302
8117
|
}
|
|
8303
8118
|
async insertPendingRun(db, id, options) {
|
|
8119
|
+
this.runsInFlight.add(id);
|
|
8304
8120
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8305
8121
|
await db.insertInto("run").values({
|
|
8306
8122
|
id,
|
|
@@ -8321,6 +8137,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8321
8137
|
}
|
|
8322
8138
|
async beginRun(runId, details) {
|
|
8323
8139
|
this.runsInFlight.add(runId);
|
|
8140
|
+
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
8324
8141
|
await this.db.updateTable("run").set({
|
|
8325
8142
|
status: "RUNNING",
|
|
8326
8143
|
workflow_name: details.workflowName,
|
|
@@ -8348,6 +8165,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8348
8165
|
warning_notes: null
|
|
8349
8166
|
}).execute();
|
|
8350
8167
|
this.runsInFlight.add(id);
|
|
8168
|
+
this.journaledSteps.set(id, /* @__PURE__ */ new Map());
|
|
8351
8169
|
return id;
|
|
8352
8170
|
}
|
|
8353
8171
|
async recordStep(runId, ordinal, step) {
|
|
@@ -8358,11 +8176,26 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8358
8176
|
id: randomUUID(),
|
|
8359
8177
|
...values
|
|
8360
8178
|
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
|
|
8179
|
+
this.journaledSteps.get(runId)?.set(step.stepId, {
|
|
8180
|
+
ordinal,
|
|
8181
|
+
hash: rowHash(values)
|
|
8182
|
+
});
|
|
8183
|
+
}
|
|
8184
|
+
sweepIsNoop(runId, result) {
|
|
8185
|
+
const journaled = this.journaledSteps.get(runId);
|
|
8186
|
+
if (!journaled) return false;
|
|
8187
|
+
return result.steps.every((step) => {
|
|
8188
|
+
const row = journaled.get(step.stepId);
|
|
8189
|
+
if (row === void 0) return false;
|
|
8190
|
+
return row.hash === rowHash(stepValues(runId, row.ordinal, step));
|
|
8191
|
+
});
|
|
8361
8192
|
}
|
|
8362
8193
|
async finishRun(runId, result, executionOrder) {
|
|
8363
8194
|
this.runsInFlight.delete(runId);
|
|
8195
|
+
const sweep = !this.sweepIsNoop(runId, result);
|
|
8196
|
+
this.journaledSteps.delete(runId);
|
|
8364
8197
|
if (erasedRuns.delete(runId)) return;
|
|
8365
|
-
if (result.steps.length > 0) try {
|
|
8198
|
+
if (sweep && result.steps.length > 0) try {
|
|
8366
8199
|
await this.sweepSteps(runId, result, executionOrder);
|
|
8367
8200
|
} catch (error) {
|
|
8368
8201
|
logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
|
|
@@ -8413,6 +8246,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8413
8246
|
}
|
|
8414
8247
|
async failRun(runId, error, errorName) {
|
|
8415
8248
|
this.runsInFlight.delete(runId);
|
|
8249
|
+
this.journaledSteps.delete(runId);
|
|
8416
8250
|
if (erasedRuns.delete(runId)) return;
|
|
8417
8251
|
await this.db.updateTable("run").set({
|
|
8418
8252
|
status: "FAILED",
|
|
@@ -8795,7 +8629,7 @@ var WorkflowRunGate = class {
|
|
|
8795
8629
|
* journals the refusal rather than dropping it silently, because a firing
|
|
8796
8630
|
* that vanished is indistinguishable from a trigger that never fired.
|
|
8797
8631
|
*/
|
|
8798
|
-
async admit(workflowId, policy) {
|
|
8632
|
+
async admit(workflowId, policy, onQueue) {
|
|
8799
8633
|
if (this.closed) return {
|
|
8800
8634
|
admitted: false,
|
|
8801
8635
|
reason: CLOSED_REASON,
|
|
@@ -8827,6 +8661,7 @@ var WorkflowRunGate = class {
|
|
|
8827
8661
|
refusal: "queue-full",
|
|
8828
8662
|
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
8829
8663
|
};
|
|
8664
|
+
onQueue?.();
|
|
8830
8665
|
if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
|
|
8831
8666
|
admitted: false,
|
|
8832
8667
|
reason: CLOSED_REASON,
|
|
@@ -8862,6 +8697,117 @@ var WorkflowRunGate = class {
|
|
|
8862
8697
|
}
|
|
8863
8698
|
};
|
|
8864
8699
|
//#endregion
|
|
8700
|
+
//#region src/reactor/shutdown.ts
|
|
8701
|
+
/** A firing refused because this runtime has shut down: retryable elsewhere. */
|
|
8702
|
+
var WorkflowRuntimeClosedError = class extends Error {
|
|
8703
|
+
workflowId;
|
|
8704
|
+
constructor(workflowId) {
|
|
8705
|
+
super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
|
|
8706
|
+
this.workflowId = workflowId;
|
|
8707
|
+
this.name = "WorkflowRuntimeClosedError";
|
|
8708
|
+
}
|
|
8709
|
+
};
|
|
8710
|
+
var TriggerSupervisorStoppedError = class extends Error {
|
|
8711
|
+
constructor(options) {
|
|
8712
|
+
super("The trigger supervisor has stopped", options);
|
|
8713
|
+
this.name = "TriggerSupervisorStoppedError";
|
|
8714
|
+
}
|
|
8715
|
+
};
|
|
8716
|
+
function isShutdownRefusal(error) {
|
|
8717
|
+
return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
|
|
8718
|
+
}
|
|
8719
|
+
//#endregion
|
|
8720
|
+
//#region src/reactor/run-dispatcher.ts
|
|
8721
|
+
const DEFAULT_ADMIT_MAX_ELU = .8;
|
|
8722
|
+
var RunDispatcher = class {
|
|
8723
|
+
maxInFlight;
|
|
8724
|
+
maxElu;
|
|
8725
|
+
loop;
|
|
8726
|
+
waiters = [];
|
|
8727
|
+
inFlight = 0;
|
|
8728
|
+
closed = false;
|
|
8729
|
+
unsubscribe;
|
|
8730
|
+
constructor(options) {
|
|
8731
|
+
this.maxInFlight = Math.max(1, Math.trunc(options.maxInFlight));
|
|
8732
|
+
this.maxElu = options.maxElu ?? .8;
|
|
8733
|
+
this.loop = options.loop;
|
|
8734
|
+
this.unsubscribe = this.loop.onSample(() => this.drain());
|
|
8735
|
+
}
|
|
8736
|
+
admit(workflowId, onQueue) {
|
|
8737
|
+
if (this.closed) return Promise.reject(new WorkflowRuntimeClosedError(workflowId));
|
|
8738
|
+
if (this.waiters.length === 0 && this.room()) return Promise.resolve(this.take());
|
|
8739
|
+
onQueue?.();
|
|
8740
|
+
return new Promise((resolve, reject) => {
|
|
8741
|
+
this.waiters.push({
|
|
8742
|
+
workflowId,
|
|
8743
|
+
resolve,
|
|
8744
|
+
reject
|
|
8745
|
+
});
|
|
8746
|
+
});
|
|
8747
|
+
}
|
|
8748
|
+
stats() {
|
|
8749
|
+
return {
|
|
8750
|
+
inFlight: this.inFlight,
|
|
8751
|
+
waiting: this.waiters.length
|
|
8752
|
+
};
|
|
8753
|
+
}
|
|
8754
|
+
close() {
|
|
8755
|
+
if (this.closed) return;
|
|
8756
|
+
this.closed = true;
|
|
8757
|
+
this.unsubscribe();
|
|
8758
|
+
for (const waiter of this.waiters.splice(0)) waiter.reject(new WorkflowRuntimeClosedError(waiter.workflowId));
|
|
8759
|
+
}
|
|
8760
|
+
room() {
|
|
8761
|
+
if (this.inFlight === 0) return true;
|
|
8762
|
+
return this.inFlight < this.maxInFlight && this.loop.utilization() <= this.maxElu;
|
|
8763
|
+
}
|
|
8764
|
+
take() {
|
|
8765
|
+
this.inFlight++;
|
|
8766
|
+
let released = false;
|
|
8767
|
+
return () => {
|
|
8768
|
+
if (released) return;
|
|
8769
|
+
released = true;
|
|
8770
|
+
this.inFlight--;
|
|
8771
|
+
this.drain();
|
|
8772
|
+
};
|
|
8773
|
+
}
|
|
8774
|
+
drain() {
|
|
8775
|
+
while (this.waiters.length > 0 && !this.closed && this.room()) this.waiters.shift().resolve(this.take());
|
|
8776
|
+
}
|
|
8777
|
+
};
|
|
8778
|
+
//#endregion
|
|
8779
|
+
//#region src/reactor/event-loop.ts
|
|
8780
|
+
const SAMPLE_MS = 1e3;
|
|
8781
|
+
var EventLoopMonitor = class {
|
|
8782
|
+
last = performance$1.eventLoopUtilization();
|
|
8783
|
+
current = 0;
|
|
8784
|
+
timer;
|
|
8785
|
+
listeners = /* @__PURE__ */ new Set();
|
|
8786
|
+
start() {
|
|
8787
|
+
if (this.timer) return;
|
|
8788
|
+
this.last = performance$1.eventLoopUtilization();
|
|
8789
|
+
this.timer = setInterval(() => this.sample(), SAMPLE_MS);
|
|
8790
|
+
this.timer.unref();
|
|
8791
|
+
}
|
|
8792
|
+
stop() {
|
|
8793
|
+
clearInterval(this.timer);
|
|
8794
|
+
this.timer = void 0;
|
|
8795
|
+
}
|
|
8796
|
+
utilization() {
|
|
8797
|
+
return this.current;
|
|
8798
|
+
}
|
|
8799
|
+
onSample(listener) {
|
|
8800
|
+
this.listeners.add(listener);
|
|
8801
|
+
return () => this.listeners.delete(listener);
|
|
8802
|
+
}
|
|
8803
|
+
sample() {
|
|
8804
|
+
const now = performance$1.eventLoopUtilization();
|
|
8805
|
+
this.current = performance$1.eventLoopUtilization(now, this.last).utilization;
|
|
8806
|
+
this.last = now;
|
|
8807
|
+
for (const listener of this.listeners) listener();
|
|
8808
|
+
}
|
|
8809
|
+
};
|
|
8810
|
+
//#endregion
|
|
8865
8811
|
//#region src/reactor/run-retention.ts
|
|
8866
8812
|
const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
|
|
8867
8813
|
const RETENTION_SWEEP_INTERVAL_MS = 36e5;
|
|
@@ -8897,26 +8843,6 @@ async function sweepRetention(store, options) {
|
|
|
8897
8843
|
dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
|
|
8898
8844
|
};
|
|
8899
8845
|
}
|
|
8900
|
-
//#endregion
|
|
8901
|
-
//#region src/reactor/shutdown.ts
|
|
8902
|
-
/** A firing refused because this runtime has shut down: retryable elsewhere. */
|
|
8903
|
-
var WorkflowRuntimeClosedError = class extends Error {
|
|
8904
|
-
workflowId;
|
|
8905
|
-
constructor(workflowId) {
|
|
8906
|
-
super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
|
|
8907
|
-
this.workflowId = workflowId;
|
|
8908
|
-
this.name = "WorkflowRuntimeClosedError";
|
|
8909
|
-
}
|
|
8910
|
-
};
|
|
8911
|
-
var TriggerSupervisorStoppedError = class extends Error {
|
|
8912
|
-
constructor(options) {
|
|
8913
|
-
super("The trigger supervisor has stopped", options);
|
|
8914
|
-
this.name = "TriggerSupervisorStoppedError";
|
|
8915
|
-
}
|
|
8916
|
-
};
|
|
8917
|
-
function isShutdownRefusal(error) {
|
|
8918
|
-
return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
|
|
8919
|
-
}
|
|
8920
8846
|
childLogger(["workflow", "trigger-binding"]);
|
|
8921
8847
|
function pieceTriggerKind(block) {
|
|
8922
8848
|
return `piece:${block.pieceName}:${block.name}`;
|
|
@@ -9007,7 +8933,9 @@ var TriggerSupervisor = class {
|
|
|
9007
8933
|
egress;
|
|
9008
8934
|
parks;
|
|
9009
8935
|
timer;
|
|
9010
|
-
|
|
8936
|
+
lanes = /* @__PURE__ */ new Map();
|
|
8937
|
+
ticked = /* @__PURE__ */ new Set();
|
|
8938
|
+
hookSlots;
|
|
9011
8939
|
ticking = false;
|
|
9012
8940
|
warnedMissingJournal = false;
|
|
9013
8941
|
stopped = false;
|
|
@@ -9020,6 +8948,7 @@ var TriggerSupervisor = class {
|
|
|
9020
8948
|
this.tickMs = options.tickMs ?? 15e3;
|
|
9021
8949
|
this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
|
|
9022
8950
|
this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
|
|
8951
|
+
this.hookSlots = new Slots(options.hookConcurrency ?? 1);
|
|
9023
8952
|
this.now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
9024
8953
|
this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
|
|
9025
8954
|
this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
|
|
@@ -9046,8 +8975,8 @@ var TriggerSupervisor = class {
|
|
|
9046
8975
|
if (!this.options.worker) this.worker.dispose();
|
|
9047
8976
|
logger$2.info("Trigger supervisor stopped");
|
|
9048
8977
|
}
|
|
9049
|
-
enqueue(task) {
|
|
9050
|
-
const run = this.
|
|
8978
|
+
enqueue(workflowId, task) {
|
|
8979
|
+
const run = (this.lanes.get(workflowId) ?? Promise.resolve()).then(async () => {
|
|
9051
8980
|
this.throwIfStopped();
|
|
9052
8981
|
try {
|
|
9053
8982
|
return await task();
|
|
@@ -9056,7 +8985,11 @@ var TriggerSupervisor = class {
|
|
|
9056
8985
|
throw error;
|
|
9057
8986
|
}
|
|
9058
8987
|
});
|
|
9059
|
-
|
|
8988
|
+
const settled = run.catch(() => void 0);
|
|
8989
|
+
this.lanes.set(workflowId, settled);
|
|
8990
|
+
settled.then(() => {
|
|
8991
|
+
if (this.lanes.get(workflowId) === settled) this.lanes.delete(workflowId);
|
|
8992
|
+
});
|
|
9060
8993
|
return run;
|
|
9061
8994
|
}
|
|
9062
8995
|
throwIfStopped() {
|
|
@@ -9094,12 +9027,12 @@ var TriggerSupervisor = class {
|
|
|
9094
9027
|
this.bindings.set(binding.workflowId, binding);
|
|
9095
9028
|
const park = this.parks.known(binding.workflowId)?.park;
|
|
9096
9029
|
if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
|
|
9097
|
-
return this.enqueue(() => this.enable(binding, previous));
|
|
9030
|
+
return this.enqueue(binding.workflowId, () => this.enable(binding, previous));
|
|
9098
9031
|
}
|
|
9099
9032
|
/** Parks a workflow; true when a trigger row was parked with it. */
|
|
9100
9033
|
park(workflowId, publishedVersion, reason) {
|
|
9101
9034
|
this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
|
|
9102
|
-
return this.enqueue(async () => {
|
|
9035
|
+
return this.enqueue(workflowId, async () => {
|
|
9103
9036
|
const store = await this.store();
|
|
9104
9037
|
if (!store) return false;
|
|
9105
9038
|
return store.parkWorkflow(workflowId, publishedVersion, reason);
|
|
@@ -9108,7 +9041,7 @@ var TriggerSupervisor = class {
|
|
|
9108
9041
|
/** Undoes {@link park} for that version only: a later park stands. */
|
|
9109
9042
|
liftPark(workflowId, publishedVersion, trigger) {
|
|
9110
9043
|
if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
|
|
9111
|
-
return this.enqueue(async () => {
|
|
9044
|
+
return this.enqueue(workflowId, async () => {
|
|
9112
9045
|
const store = await this.store();
|
|
9113
9046
|
if (!store) return;
|
|
9114
9047
|
if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
|
|
@@ -9119,7 +9052,7 @@ var TriggerSupervisor = class {
|
|
|
9119
9052
|
* row through onDisable (a boot), the row is turned DISABLED here. */
|
|
9120
9053
|
releasePark(workflowId, flipRow) {
|
|
9121
9054
|
this.parks.delete(workflowId);
|
|
9122
|
-
return this.enqueue(async () => {
|
|
9055
|
+
return this.enqueue(workflowId, async () => {
|
|
9123
9056
|
const store = await this.store();
|
|
9124
9057
|
if (!store) return;
|
|
9125
9058
|
if (flipRow) await store.clearParkOnDisable(workflowId);
|
|
@@ -9137,7 +9070,7 @@ var TriggerSupervisor = class {
|
|
|
9137
9070
|
if (retry) retry.at = 0;
|
|
9138
9071
|
const park = this.parks.known(workflowId)?.park;
|
|
9139
9072
|
if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
|
|
9140
|
-
return this.enqueue(async () => {
|
|
9073
|
+
return this.enqueue(workflowId, async () => {
|
|
9141
9074
|
const store = await this.store();
|
|
9142
9075
|
if (!store) return;
|
|
9143
9076
|
const park = await store.getWorkflowPark(workflowId);
|
|
@@ -9147,12 +9080,12 @@ var TriggerSupervisor = class {
|
|
|
9147
9080
|
}
|
|
9148
9081
|
remove(workflowId) {
|
|
9149
9082
|
const binding = this.unbind(workflowId);
|
|
9150
|
-
return this.enqueue(() => this.disable(workflowId, binding));
|
|
9083
|
+
return this.enqueue(workflowId, () => this.disable(workflowId, binding));
|
|
9151
9084
|
}
|
|
9152
9085
|
forget(workflowId) {
|
|
9153
9086
|
const binding = this.unbind(workflowId);
|
|
9154
9087
|
this.parks.delete(workflowId);
|
|
9155
|
-
return this.enqueue(async () => {
|
|
9088
|
+
return this.enqueue(workflowId, async () => {
|
|
9156
9089
|
const store = await this.store();
|
|
9157
9090
|
if (!store) return;
|
|
9158
9091
|
await this.disable(workflowId, binding);
|
|
@@ -9170,7 +9103,7 @@ var TriggerSupervisor = class {
|
|
|
9170
9103
|
}
|
|
9171
9104
|
reject(workflowId, block, config, message, retryAt) {
|
|
9172
9105
|
this.unbind(workflowId);
|
|
9173
|
-
return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
|
|
9106
|
+
return this.enqueue(workflowId, () => this.recordRejection(workflowId, block, config, message, retryAt));
|
|
9174
9107
|
}
|
|
9175
9108
|
async recordRejection(workflowId, block, config, message, retryAt) {
|
|
9176
9109
|
const store = await this.store();
|
|
@@ -9203,7 +9136,7 @@ var TriggerSupervisor = class {
|
|
|
9203
9136
|
});
|
|
9204
9137
|
}
|
|
9205
9138
|
test(binding, options = {}) {
|
|
9206
|
-
return this.enqueue(async () => {
|
|
9139
|
+
return this.enqueue(binding.workflowId, async () => {
|
|
9207
9140
|
const store = await this.store();
|
|
9208
9141
|
try {
|
|
9209
9142
|
return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
|
|
@@ -9222,10 +9155,10 @@ var TriggerSupervisor = class {
|
|
|
9222
9155
|
}
|
|
9223
9156
|
}
|
|
9224
9157
|
handshake(binding, payload) {
|
|
9225
|
-
return this.enqueue(() => this.hook(binding, "onHandshake", { payload }));
|
|
9158
|
+
return this.enqueue(binding.workflowId, () => this.hook(binding, "onHandshake", { payload }));
|
|
9226
9159
|
}
|
|
9227
9160
|
deliverWebhook(workflowId, payload) {
|
|
9228
|
-
return this.enqueue(async () => {
|
|
9161
|
+
return this.enqueue(workflowId, async () => {
|
|
9229
9162
|
const binding = this.bindings.get(workflowId);
|
|
9230
9163
|
if (!binding || isSchedule(binding)) {
|
|
9231
9164
|
logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
|
|
@@ -9243,8 +9176,12 @@ var TriggerSupervisor = class {
|
|
|
9243
9176
|
throw error;
|
|
9244
9177
|
});
|
|
9245
9178
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
9246
|
-
|
|
9247
|
-
|
|
9179
|
+
const success = row?.status === "ENABLED" ? {
|
|
9180
|
+
workflowId,
|
|
9181
|
+
storeState: VESTIGIAL_STORE_STATE,
|
|
9182
|
+
nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
|
|
9183
|
+
} : void 0;
|
|
9184
|
+
await this.fireItems(store, binding, result.output, now, rewind, success);
|
|
9248
9185
|
} catch (error) {
|
|
9249
9186
|
if (!refused) await rewind();
|
|
9250
9187
|
throw error;
|
|
@@ -9576,22 +9513,38 @@ var TriggerSupervisor = class {
|
|
|
9576
9513
|
async tick() {
|
|
9577
9514
|
if (this.ticking || this.stopped) return;
|
|
9578
9515
|
this.ticking = true;
|
|
9516
|
+
const started = [];
|
|
9517
|
+
let listing;
|
|
9579
9518
|
try {
|
|
9580
|
-
await this.
|
|
9519
|
+
await this.startDue(started);
|
|
9520
|
+
} catch (error) {
|
|
9521
|
+
listing = error;
|
|
9581
9522
|
} finally {
|
|
9582
9523
|
this.ticking = false;
|
|
9583
9524
|
}
|
|
9525
|
+
const outcomes = await Promise.allSettled(started.filter((run) => run !== void 0));
|
|
9526
|
+
if (listing !== void 0) throw listing instanceof Error ? listing : new Error(String(listing));
|
|
9527
|
+
const failed = outcomes.find((outcome) => outcome.status === "rejected");
|
|
9528
|
+
if (failed) throw failed.reason;
|
|
9584
9529
|
}
|
|
9585
|
-
|
|
9530
|
+
startOnLane(workflowId, task, hook) {
|
|
9531
|
+
if (this.ticked.has(workflowId)) return void 0;
|
|
9532
|
+
this.ticked.add(workflowId);
|
|
9533
|
+
return (hook ? this.hookSlots.run(() => this.enqueue(workflowId, task)) : this.enqueue(workflowId, task)).finally(() => this.ticked.delete(workflowId));
|
|
9534
|
+
}
|
|
9535
|
+
startOneEnableRetry() {
|
|
9586
9536
|
const now = this.now().getTime();
|
|
9587
|
-
const next = [...this.enableRetries].filter(([, retry]) => retry.at <= now).sort((a, b) => a[1].at - b[1].at).at(0);
|
|
9588
|
-
if (!next) return;
|
|
9537
|
+
const next = [...this.enableRetries].filter(([id, retry]) => retry.at <= now && !this.ticked.has(id)).sort((a, b) => a[1].at - b[1].at).at(0);
|
|
9538
|
+
if (!next) return void 0;
|
|
9589
9539
|
const [workflowId, retry] = next;
|
|
9590
9540
|
const binding = this.bindings.get(workflowId);
|
|
9591
9541
|
if (!binding) {
|
|
9592
9542
|
this.enableRetries.delete(workflowId);
|
|
9593
9543
|
return;
|
|
9594
9544
|
}
|
|
9545
|
+
return this.startOnLane(workflowId, () => this.retryEnable(workflowId, retry, binding, now), true);
|
|
9546
|
+
}
|
|
9547
|
+
async retryEnable(workflowId, retry, binding, now) {
|
|
9595
9548
|
try {
|
|
9596
9549
|
await this.enable(binding);
|
|
9597
9550
|
} catch (error) {
|
|
@@ -9606,7 +9559,7 @@ var TriggerSupervisor = class {
|
|
|
9606
9559
|
logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
|
|
9607
9560
|
}
|
|
9608
9561
|
}
|
|
9609
|
-
async
|
|
9562
|
+
async startDue(started) {
|
|
9610
9563
|
const store = await this.store();
|
|
9611
9564
|
if (!store) {
|
|
9612
9565
|
if (!this.warnedMissingJournal) {
|
|
@@ -9615,37 +9568,35 @@ var TriggerSupervisor = class {
|
|
|
9615
9568
|
}
|
|
9616
9569
|
return;
|
|
9617
9570
|
}
|
|
9618
|
-
const
|
|
9619
|
-
for (const row of
|
|
9571
|
+
const ordered = [...await store.listDueTriggerStates(this.now().toISOString())].sort((a, b) => Number(!this.isScheduleRow(a)) - Number(!this.isScheduleRow(b)));
|
|
9572
|
+
for (const row of ordered) {
|
|
9620
9573
|
this.throwIfStopped();
|
|
9621
9574
|
const binding = this.bindings.get(row.workflow_id);
|
|
9622
9575
|
if (!binding) {
|
|
9623
|
-
|
|
9576
|
+
started.push(this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false));
|
|
9624
9577
|
continue;
|
|
9625
9578
|
}
|
|
9626
|
-
|
|
9627
|
-
else await this.poll(store, row, binding);
|
|
9579
|
+
started.push(isSchedule(binding) ? this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.fireSchedule(store, row, binding)), false) : this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.poll(store, row, binding)), true));
|
|
9628
9580
|
}
|
|
9629
9581
|
this.throwIfStopped();
|
|
9630
|
-
await this.
|
|
9582
|
+
started.push(...await this.startRenewals(store));
|
|
9631
9583
|
this.throwIfStopped();
|
|
9632
|
-
|
|
9584
|
+
started.push(this.startOneEnableRetry());
|
|
9633
9585
|
}
|
|
9634
|
-
|
|
9635
|
-
|
|
9636
|
-
|
|
9637
|
-
|
|
9586
|
+
whileBound(binding, task) {
|
|
9587
|
+
return () => this.bindings.get(binding.workflowId) === binding ? task() : Promise.resolve();
|
|
9588
|
+
}
|
|
9589
|
+
isScheduleRow(row) {
|
|
9590
|
+
const binding = this.bindings.get(row.workflow_id);
|
|
9591
|
+
return binding !== void 0 && isSchedule(binding);
|
|
9592
|
+
}
|
|
9593
|
+
async startRenewals(store) {
|
|
9594
|
+
return (await store.listDueTriggerRenewals(this.now().toISOString())).map((row) => {
|
|
9638
9595
|
const binding = this.bindings.get(row.workflow_id);
|
|
9639
|
-
if (!binding)
|
|
9640
|
-
|
|
9641
|
-
|
|
9642
|
-
|
|
9643
|
-
if (isSchedule(binding)) {
|
|
9644
|
-
await store.setTriggerRenewAt(row.workflow_id, null);
|
|
9645
|
-
continue;
|
|
9646
|
-
}
|
|
9647
|
-
await this.renew(store, row, binding);
|
|
9648
|
-
}
|
|
9596
|
+
if (!binding) return this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false);
|
|
9597
|
+
if (isSchedule(binding)) return this.startOnLane(row.workflow_id, () => store.setTriggerRenewAt(row.workflow_id, null), false);
|
|
9598
|
+
return this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.renew(store, row, binding)), true);
|
|
9599
|
+
});
|
|
9649
9600
|
}
|
|
9650
9601
|
async renew(store, row, binding) {
|
|
9651
9602
|
const now = this.now();
|
|
@@ -9707,8 +9658,11 @@ var TriggerSupervisor = class {
|
|
|
9707
9658
|
throw error;
|
|
9708
9659
|
});
|
|
9709
9660
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
9710
|
-
await
|
|
9711
|
-
|
|
9661
|
+
await this.fireItems(store, binding, result.output, now, rewind, {
|
|
9662
|
+
workflowId: row.workflow_id,
|
|
9663
|
+
storeState: VESTIGIAL_STORE_STATE,
|
|
9664
|
+
nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
|
|
9665
|
+
});
|
|
9712
9666
|
} catch (error) {
|
|
9713
9667
|
if (!refused) await rewind();
|
|
9714
9668
|
if (this.stopped) throw error;
|
|
@@ -9719,8 +9673,11 @@ var TriggerSupervisor = class {
|
|
|
9719
9673
|
logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
|
|
9720
9674
|
}
|
|
9721
9675
|
}
|
|
9722
|
-
async fireItems(store, binding, items, now, rewind) {
|
|
9723
|
-
if (items.length === 0)
|
|
9676
|
+
async fireItems(store, binding, items, now, rewind, success) {
|
|
9677
|
+
if (items.length === 0) {
|
|
9678
|
+
if (success) await store.recordPollSuccess(success.workflowId, success.storeState, now.toISOString(), success.nextPollAtIso);
|
|
9679
|
+
return;
|
|
9680
|
+
}
|
|
9724
9681
|
const kind = pieceTriggerKind(binding.block);
|
|
9725
9682
|
const journaled = (await store.journalTriggerItems(items.map((item) => ({
|
|
9726
9683
|
dedupeKey: extractDedupeKey(item),
|
|
@@ -9729,7 +9686,7 @@ var TriggerSupervisor = class {
|
|
|
9729
9686
|
triggerKind: kind,
|
|
9730
9687
|
triggerPayload: item
|
|
9731
9688
|
}
|
|
9732
|
-
})), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
|
|
9689
|
+
})), DEDUPE_TTL_MS, now.toISOString(), success)).flatMap((runId, index) => runId ? [{
|
|
9733
9690
|
runId,
|
|
9734
9691
|
item: items[index]
|
|
9735
9692
|
}] : []);
|
|
@@ -9752,6 +9709,25 @@ var TriggerSupervisor = class {
|
|
|
9752
9709
|
}
|
|
9753
9710
|
}
|
|
9754
9711
|
};
|
|
9712
|
+
var Slots = class {
|
|
9713
|
+
limit;
|
|
9714
|
+
active = 0;
|
|
9715
|
+
waiting = [];
|
|
9716
|
+
constructor(limit) {
|
|
9717
|
+
this.limit = limit;
|
|
9718
|
+
}
|
|
9719
|
+
async run(task) {
|
|
9720
|
+
if (this.active >= Math.max(1, this.limit)) await new Promise((resolve) => this.waiting.push(resolve));
|
|
9721
|
+
else this.active++;
|
|
9722
|
+
try {
|
|
9723
|
+
return await task();
|
|
9724
|
+
} finally {
|
|
9725
|
+
const next = this.waiting.shift();
|
|
9726
|
+
if (next) next();
|
|
9727
|
+
else this.active--;
|
|
9728
|
+
}
|
|
9729
|
+
}
|
|
9730
|
+
};
|
|
9755
9731
|
//#endregion
|
|
9756
9732
|
//#region src/reactor/webhook.ts
|
|
9757
9733
|
const WEBHOOK = "Webhook";
|
|
@@ -9979,6 +9955,7 @@ function bindingTarget(binding) {
|
|
|
9979
9955
|
}
|
|
9980
9956
|
const RUNS_SCAN_LIMIT = 1e3;
|
|
9981
9957
|
const WEBHOOK_TEST_TIMEOUT_MS = 3e5;
|
|
9958
|
+
const PENDING_SWEEP_INTERVAL_MS = 6e4;
|
|
9982
9959
|
function draftBlocks(state) {
|
|
9983
9960
|
return [...state.trigger ? [{
|
|
9984
9961
|
id: state.trigger.id,
|
|
@@ -10149,6 +10126,10 @@ function configRecord(config) {
|
|
|
10149
10126
|
if (config && typeof config === "object" && !Array.isArray(config)) return config;
|
|
10150
10127
|
return {};
|
|
10151
10128
|
}
|
|
10129
|
+
const WORKFLOW_CACHE_MAX = 1e3;
|
|
10130
|
+
function globalRevision(document) {
|
|
10131
|
+
return document.header.revision?.global ?? 0;
|
|
10132
|
+
}
|
|
10152
10133
|
function runJournalName(stateName, documentName) {
|
|
10153
10134
|
return stateName?.trim() ? stateName : documentName ?? "";
|
|
10154
10135
|
}
|
|
@@ -10160,6 +10141,9 @@ var WorkflowRuntimeService = class {
|
|
|
10160
10141
|
executor;
|
|
10161
10142
|
pieceWorkers;
|
|
10162
10143
|
unobservePool;
|
|
10144
|
+
runDispatcher;
|
|
10145
|
+
unobserveDispatcher;
|
|
10146
|
+
eventLoop = new EventLoopMonitor();
|
|
10163
10147
|
storePromise;
|
|
10164
10148
|
storeError;
|
|
10165
10149
|
storeOpening = false;
|
|
@@ -10655,10 +10639,35 @@ var WorkflowRuntimeService = class {
|
|
|
10655
10639
|
await this.updateRegistration(workflowId, carried, onlyIfChanged);
|
|
10656
10640
|
return;
|
|
10657
10641
|
}
|
|
10642
|
+
const generation = this.workflowGeneration(workflowId);
|
|
10658
10643
|
const document = await this.host.reactorClient.get(workflowId);
|
|
10644
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
10659
10645
|
await this.updateRegistration(workflowId, document.state.global, onlyIfChanged);
|
|
10660
10646
|
}
|
|
10661
10647
|
seenOps = /* @__PURE__ */ new Set();
|
|
10648
|
+
workflowCache = /* @__PURE__ */ new Map();
|
|
10649
|
+
workflowGenerations = /* @__PURE__ */ new Map();
|
|
10650
|
+
workflowGeneration(workflowId) {
|
|
10651
|
+
return this.workflowGenerations.get(workflowId) ?? 0;
|
|
10652
|
+
}
|
|
10653
|
+
cacheLoaded(workflowId, generation, document) {
|
|
10654
|
+
if (this.workflowGeneration(workflowId) !== generation) return;
|
|
10655
|
+
this.cacheWorkflow(workflowId, {
|
|
10656
|
+
state: document.state.global,
|
|
10657
|
+
name: document.header.name,
|
|
10658
|
+
revision: globalRevision(document)
|
|
10659
|
+
});
|
|
10660
|
+
}
|
|
10661
|
+
cacheWorkflow(workflowId, entry) {
|
|
10662
|
+
const held = this.workflowCache.get(workflowId);
|
|
10663
|
+
if (held && held.revision > entry.revision) return;
|
|
10664
|
+
this.workflowCache.delete(workflowId);
|
|
10665
|
+
this.workflowCache.set(workflowId, entry);
|
|
10666
|
+
if (this.workflowCache.size > WORKFLOW_CACHE_MAX) {
|
|
10667
|
+
const oldest = this.workflowCache.keys().next().value;
|
|
10668
|
+
if (oldest !== void 0) this.workflowCache.delete(oldest);
|
|
10669
|
+
}
|
|
10670
|
+
}
|
|
10662
10671
|
seenOpsQueue = [];
|
|
10663
10672
|
alreadySeen(key) {
|
|
10664
10673
|
if (this.seenOps.has(key)) return true;
|
|
@@ -10694,12 +10703,22 @@ var WorkflowRuntimeService = class {
|
|
|
10694
10703
|
context
|
|
10695
10704
|
});
|
|
10696
10705
|
if (this.alreadySeen(opKey)) continue;
|
|
10706
|
+
if (context.documentType === WORKFLOW_DOCUMENT_TYPE) this.workflowGenerations.set(context.documentId, this.workflowGeneration(context.documentId) + 1);
|
|
10697
10707
|
if (context.scope === DOCUMENT_SCOPE) {
|
|
10708
|
+
this.workflowCache.delete(context.documentId);
|
|
10698
10709
|
await this.matchDocumentLifecycle(operation, context, hints);
|
|
10699
10710
|
await this.forgetDeletedWorkflow(operation, context);
|
|
10700
10711
|
continue;
|
|
10701
10712
|
}
|
|
10702
10713
|
if (context.documentType === "powerhouse/workflow") {
|
|
10714
|
+
const cached = this.workflowCache.get(context.documentId);
|
|
10715
|
+
const carried = cached && operation.error === void 0 ? parseWorkflowState(operation.resultingState) : void 0;
|
|
10716
|
+
if (cached && carried) this.cacheWorkflow(context.documentId, {
|
|
10717
|
+
state: carried,
|
|
10718
|
+
name: cached.name,
|
|
10719
|
+
revision: operation.index + 1
|
|
10720
|
+
});
|
|
10721
|
+
else if (cached) this.workflowCache.delete(context.documentId);
|
|
10703
10722
|
const rechecked = await this.recheckReactorAccess(context.documentId, operation);
|
|
10704
10723
|
await this.refreshRegistration(context.documentId, operation.resultingState, !rechecked);
|
|
10705
10724
|
}
|
|
@@ -10809,7 +10828,7 @@ var WorkflowRuntimeService = class {
|
|
|
10809
10828
|
this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
|
|
10810
10829
|
}, (error) => {
|
|
10811
10830
|
if (enqueuedRunId && isShutdownRefusal(error)) {
|
|
10812
|
-
this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to
|
|
10831
|
+
this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left for the next owner to start`);
|
|
10813
10832
|
return;
|
|
10814
10833
|
}
|
|
10815
10834
|
this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
|
|
@@ -10871,6 +10890,7 @@ var WorkflowRuntimeService = class {
|
|
|
10871
10890
|
if (context.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
|
|
10872
10891
|
const workflowId = stringField(inputRecord(operation.action.input), "documentId") ?? context.documentId;
|
|
10873
10892
|
this.disarmDeleted(workflowId);
|
|
10893
|
+
this.workflowCache.delete(workflowId);
|
|
10874
10894
|
try {
|
|
10875
10895
|
await (await this.store())?.deleteDedupe(workflowId);
|
|
10876
10896
|
} catch (error) {
|
|
@@ -10972,6 +10992,8 @@ var WorkflowRuntimeService = class {
|
|
|
10972
10992
|
webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
|
|
10973
10993
|
reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
|
|
10974
10994
|
models: this.models,
|
|
10995
|
+
worker: this.workers().session("trigger"),
|
|
10996
|
+
hookConcurrency: this.workers().triggerSlots,
|
|
10975
10997
|
telemetry: this.telemetry,
|
|
10976
10998
|
cacheDir: bundleCacheDir(),
|
|
10977
10999
|
resolver: pieceResolver(),
|
|
@@ -10991,8 +11013,29 @@ var WorkflowRuntimeService = class {
|
|
|
10991
11013
|
}
|
|
10992
11014
|
startTriggerSupervisor() {
|
|
10993
11015
|
this.supervisor().start();
|
|
11016
|
+
this.admitAbandonedRuns();
|
|
11017
|
+
this.pendingSweepTimer ??= setInterval(() => {
|
|
11018
|
+
this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
|
|
11019
|
+
}, PENDING_SWEEP_INTERVAL_MS);
|
|
11020
|
+
this.pendingSweepTimer.unref();
|
|
11021
|
+
}
|
|
11022
|
+
pendingSweepTimer;
|
|
11023
|
+
admittingAbandoned = false;
|
|
11024
|
+
async admitAbandonedRuns(minAgeMs) {
|
|
11025
|
+
if (this.admittingAbandoned || this.closed) return;
|
|
11026
|
+
this.admittingAbandoned = true;
|
|
11027
|
+
try {
|
|
11028
|
+
const admitted = await (await this.store())?.admitAbandonedRuns(minAgeMs === void 0 ? {} : { enqueuedBefore: new Date(Date.now() - minAgeMs) });
|
|
11029
|
+
for (const run of admitted ?? []) this.startFiring(run.workflowId, run.triggerPayload, run.triggerKind, run.runId);
|
|
11030
|
+
} catch (error) {
|
|
11031
|
+
this.logger.error("Could not admit the runs a stopped reactor journaled: @error", error);
|
|
11032
|
+
} finally {
|
|
11033
|
+
this.admittingAbandoned = false;
|
|
11034
|
+
}
|
|
10994
11035
|
}
|
|
10995
11036
|
stopTriggerSupervisor() {
|
|
11037
|
+
clearInterval(this.pendingSweepTimer);
|
|
11038
|
+
this.pendingSweepTimer = void 0;
|
|
10996
11039
|
this.triggerSupervisor?.stop();
|
|
10997
11040
|
}
|
|
10998
11041
|
closed = false;
|
|
@@ -11002,6 +11045,10 @@ var WorkflowRuntimeService = class {
|
|
|
11002
11045
|
shutdown() {
|
|
11003
11046
|
this.closed = true;
|
|
11004
11047
|
this.runGate.close();
|
|
11048
|
+
this.runDispatcher?.close();
|
|
11049
|
+
this.eventLoop.stop();
|
|
11050
|
+
this.unobserveDispatcher?.();
|
|
11051
|
+
this.unobserveDispatcher = void 0;
|
|
11005
11052
|
clearInterval(this.retentionTimer);
|
|
11006
11053
|
for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
|
|
11007
11054
|
this.resolutionRetries.clear();
|
|
@@ -11010,7 +11057,6 @@ var WorkflowRuntimeService = class {
|
|
|
11010
11057
|
this.pieceWorkers?.dispose();
|
|
11011
11058
|
this.unobservePool?.();
|
|
11012
11059
|
this.unobservePool = void 0;
|
|
11013
|
-
this.designWorker?.dispose();
|
|
11014
11060
|
this.designWorker = void 0;
|
|
11015
11061
|
}
|
|
11016
11062
|
async triggerStates(ctx) {
|
|
@@ -11297,10 +11343,7 @@ var WorkflowRuntimeService = class {
|
|
|
11297
11343
|
descriptors = /* @__PURE__ */ new Map();
|
|
11298
11344
|
designWorker;
|
|
11299
11345
|
createDesignWorker() {
|
|
11300
|
-
return
|
|
11301
|
-
models: this.models,
|
|
11302
|
-
telemetry: this.telemetry
|
|
11303
|
-
});
|
|
11346
|
+
return this.workers().session("design");
|
|
11304
11347
|
}
|
|
11305
11348
|
designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
|
|
11306
11349
|
describing = /* @__PURE__ */ new Map();
|
|
@@ -12099,17 +12142,51 @@ var WorkflowRuntimeService = class {
|
|
|
12099
12142
|
}
|
|
12100
12143
|
workers() {
|
|
12101
12144
|
if (this.pieceWorkers) return this.pieceWorkers;
|
|
12145
|
+
const env = (name) => Number(process.env[name]) || void 0;
|
|
12146
|
+
const count = (name) => {
|
|
12147
|
+
const value = Number(process.env[name] || NaN);
|
|
12148
|
+
return value >= 0 ? value : void 0;
|
|
12149
|
+
};
|
|
12102
12150
|
const pool = new PieceWorkerPool({
|
|
12103
12151
|
models: this.models,
|
|
12104
|
-
size:
|
|
12105
|
-
maxQueueDepth:
|
|
12152
|
+
size: env("PH_WORKFLOWS_RUN_CONCURRENCY"),
|
|
12153
|
+
maxQueueDepth: env("PH_WORKFLOWS_RUN_QUEUE_DEPTH"),
|
|
12154
|
+
heapMb: env("PH_WORKFLOWS_WORKER_MEMORY_MB"),
|
|
12155
|
+
maxRequests: env("PH_WORKFLOWS_WORKER_MAX_REQUESTS"),
|
|
12156
|
+
minWarm: count("PH_WORKFLOWS_WORKER_MIN"),
|
|
12157
|
+
triggerSlots: env("PH_WORKFLOWS_TRIGGER_SLOTS"),
|
|
12158
|
+
reactorThreads: this.host.reactorThreads,
|
|
12106
12159
|
telemetry: this.telemetry
|
|
12107
12160
|
});
|
|
12161
|
+
if (this.closed) {
|
|
12162
|
+
pool.dispose();
|
|
12163
|
+
return this.pieceWorkers = pool;
|
|
12164
|
+
}
|
|
12108
12165
|
this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
|
|
12109
12166
|
return this.pieceWorkers = pool;
|
|
12110
12167
|
}
|
|
12168
|
+
dispatcher() {
|
|
12169
|
+
if (this.runDispatcher) return this.runDispatcher;
|
|
12170
|
+
const configured = Number(process.env.PH_WORKFLOWS_ADMIT_MAX_ELU);
|
|
12171
|
+
const dispatcher = new RunDispatcher({
|
|
12172
|
+
maxInFlight: Number(process.env.PH_WORKFLOWS_RUNS_IN_FLIGHT) || 8 * this.workers().size,
|
|
12173
|
+
maxElu: configured > 0 && configured <= 1 ? configured : DEFAULT_ADMIT_MAX_ELU,
|
|
12174
|
+
loop: this.eventLoop
|
|
12175
|
+
});
|
|
12176
|
+
this.runDispatcher = dispatcher;
|
|
12177
|
+
if (this.closed) {
|
|
12178
|
+
dispatcher.close();
|
|
12179
|
+
return dispatcher;
|
|
12180
|
+
}
|
|
12181
|
+
this.eventLoop.start();
|
|
12182
|
+
this.unobserveDispatcher = this.telemetry.observeDispatcher(() => ({
|
|
12183
|
+
...dispatcher.stats(),
|
|
12184
|
+
utilization: this.eventLoop.utilization()
|
|
12185
|
+
}));
|
|
12186
|
+
return dispatcher;
|
|
12187
|
+
}
|
|
12111
12188
|
async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
|
|
12112
|
-
if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
|
|
12189
|
+
if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId, triggerKind);
|
|
12113
12190
|
const started = performance.now();
|
|
12114
12191
|
const attributes = {
|
|
12115
12192
|
"workflow.id": workflowId,
|
|
@@ -12144,18 +12221,29 @@ var WorkflowRuntimeService = class {
|
|
|
12144
12221
|
let documentName;
|
|
12145
12222
|
let definition;
|
|
12146
12223
|
let runUser;
|
|
12224
|
+
let readGeneration;
|
|
12147
12225
|
try {
|
|
12148
|
-
if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12226
|
+
if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12149
12227
|
run.refused = true;
|
|
12150
12228
|
throw error;
|
|
12151
12229
|
});
|
|
12152
|
-
const
|
|
12153
|
-
if (
|
|
12154
|
-
|
|
12155
|
-
|
|
12230
|
+
const cached = triggerKind === "manual" || resume ? void 0 : this.workflowCache.get(workflowId);
|
|
12231
|
+
if (cached) {
|
|
12232
|
+
state = cached.state;
|
|
12233
|
+
documentName = cached.name;
|
|
12234
|
+
readGeneration = this.workflowGeneration(workflowId);
|
|
12235
|
+
} else {
|
|
12236
|
+
const generation = this.workflowGeneration(workflowId);
|
|
12237
|
+
readGeneration = generation;
|
|
12238
|
+
const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
|
|
12239
|
+
if (document.header.documentType !== "powerhouse/workflow") {
|
|
12240
|
+
run.refused = true;
|
|
12241
|
+
throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
|
|
12242
|
+
}
|
|
12243
|
+
state = document.state.global;
|
|
12244
|
+
documentName = document.header.name;
|
|
12245
|
+
this.cacheLoaded(workflowId, generation, document);
|
|
12156
12246
|
}
|
|
12157
|
-
state = document.state.global;
|
|
12158
|
-
documentName = document.header.name;
|
|
12159
12247
|
if (state.status !== "ENABLED") {
|
|
12160
12248
|
run.refused = true;
|
|
12161
12249
|
throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
|
|
@@ -12170,6 +12258,19 @@ var WorkflowRuntimeService = class {
|
|
|
12170
12258
|
if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
|
|
12171
12259
|
throw error;
|
|
12172
12260
|
}
|
|
12261
|
+
let journaling;
|
|
12262
|
+
const journalOnQueue = () => {
|
|
12263
|
+
if (triggerKind !== "manual" || enqueuedRunId || resume || journaling) return;
|
|
12264
|
+
if (!store) return;
|
|
12265
|
+
journaling = store.enqueueRun({
|
|
12266
|
+
workflowId,
|
|
12267
|
+
triggerKind,
|
|
12268
|
+
triggerPayload
|
|
12269
|
+
}).then((runId) => {
|
|
12270
|
+
enqueuedRunId = runId;
|
|
12271
|
+
});
|
|
12272
|
+
journaling.catch(() => void 0);
|
|
12273
|
+
};
|
|
12173
12274
|
const connections = declaredConnectionIds(definition);
|
|
12174
12275
|
const executor = this.blockExecutor(store);
|
|
12175
12276
|
const runnable = runnableDefinition(state);
|
|
@@ -12190,30 +12291,72 @@ var WorkflowRuntimeService = class {
|
|
|
12190
12291
|
throw error;
|
|
12191
12292
|
}
|
|
12192
12293
|
if (parked) return skipped(parked, "parked");
|
|
12193
|
-
const admission = await this.runGate.admit(workflowId, policy);
|
|
12294
|
+
const admission = await this.runGate.admit(workflowId, policy, journalOnQueue);
|
|
12295
|
+
try {
|
|
12296
|
+
await journaling;
|
|
12297
|
+
} catch (error) {
|
|
12298
|
+
if (admission.admitted) admission.release();
|
|
12299
|
+
throw error;
|
|
12300
|
+
}
|
|
12194
12301
|
if (this.closed) {
|
|
12195
12302
|
if (admission.admitted) admission.release();
|
|
12196
|
-
return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12303
|
+
return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
|
|
12197
12304
|
}
|
|
12198
12305
|
if (!admission.admitted) {
|
|
12199
|
-
if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12306
|
+
if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
|
|
12200
12307
|
return skipped(admission.reason, admission.refusal);
|
|
12201
12308
|
}
|
|
12202
|
-
|
|
12203
|
-
let stale;
|
|
12309
|
+
const staleAfter = async () => {
|
|
12204
12310
|
try {
|
|
12205
|
-
|
|
12311
|
+
return await this.staleAfterWait(workflowId, triggerKind, runnable.version, readGeneration);
|
|
12206
12312
|
} catch (error) {
|
|
12207
|
-
|
|
12313
|
+
return `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
|
|
12208
12314
|
}
|
|
12315
|
+
};
|
|
12316
|
+
if (admission.waited) {
|
|
12317
|
+
const stale = await staleAfter();
|
|
12209
12318
|
if (stale) {
|
|
12210
12319
|
admission.release();
|
|
12211
12320
|
return skipped(stale, "stale");
|
|
12212
12321
|
}
|
|
12213
12322
|
}
|
|
12323
|
+
const expired = () => skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
|
|
12214
12324
|
if (deadline !== void 0 && Date.now() >= deadline) {
|
|
12215
12325
|
admission.release();
|
|
12216
|
-
return
|
|
12326
|
+
return expired();
|
|
12327
|
+
}
|
|
12328
|
+
let dispatched;
|
|
12329
|
+
let dispatcherWaited = false;
|
|
12330
|
+
try {
|
|
12331
|
+
dispatched = await this.dispatcher().admit(workflowId, () => {
|
|
12332
|
+
dispatcherWaited = true;
|
|
12333
|
+
journalOnQueue();
|
|
12334
|
+
});
|
|
12335
|
+
} catch (error) {
|
|
12336
|
+
await journaling?.catch(() => void 0);
|
|
12337
|
+
admission.release();
|
|
12338
|
+
if (this.closed) return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
|
|
12339
|
+
throw error;
|
|
12340
|
+
}
|
|
12341
|
+
try {
|
|
12342
|
+
await journaling;
|
|
12343
|
+
} catch (error) {
|
|
12344
|
+
dispatched();
|
|
12345
|
+
admission.release();
|
|
12346
|
+
throw error;
|
|
12347
|
+
}
|
|
12348
|
+
if (this.closed || deadline !== void 0 && Date.now() >= deadline) {
|
|
12349
|
+
dispatched();
|
|
12350
|
+
admission.release();
|
|
12351
|
+
return this.closed ? this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind) : expired();
|
|
12352
|
+
}
|
|
12353
|
+
if (dispatcherWaited) {
|
|
12354
|
+
const stale = await staleAfter();
|
|
12355
|
+
if (stale) {
|
|
12356
|
+
dispatched();
|
|
12357
|
+
admission.release();
|
|
12358
|
+
return skipped(stale, "stale");
|
|
12359
|
+
}
|
|
12217
12360
|
}
|
|
12218
12361
|
let runId = enqueuedRunId ?? null;
|
|
12219
12362
|
let journalFailed = false;
|
|
@@ -12238,6 +12381,7 @@ var WorkflowRuntimeService = class {
|
|
|
12238
12381
|
}) ?? null;
|
|
12239
12382
|
});
|
|
12240
12383
|
if (runId) run.span.setAttribute("run.id", runId);
|
|
12384
|
+
if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
|
|
12241
12385
|
session = this.workers().session();
|
|
12242
12386
|
const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
|
|
12243
12387
|
const journal = store;
|
|
@@ -12300,6 +12444,7 @@ var WorkflowRuntimeService = class {
|
|
|
12300
12444
|
} finally {
|
|
12301
12445
|
session?.close();
|
|
12302
12446
|
admission.release();
|
|
12447
|
+
dispatched();
|
|
12303
12448
|
}
|
|
12304
12449
|
}
|
|
12305
12450
|
runGate = new WorkflowRunGate();
|
|
@@ -12313,7 +12458,8 @@ var WorkflowRuntimeService = class {
|
|
|
12313
12458
|
return `Skipped: this workflow is PARKED (${park.reason})`;
|
|
12314
12459
|
}
|
|
12315
12460
|
/** Why a firing that waited for its slot may no longer run, if it may not. */
|
|
12316
|
-
async staleAfterWait(workflowId, triggerKind, publishedVersion) {
|
|
12461
|
+
async staleAfterWait(workflowId, triggerKind, publishedVersion, readGeneration) {
|
|
12462
|
+
if (this.workflowGeneration(workflowId) === readGeneration) return this.parkedFiring(workflowId, triggerKind, publishedVersion);
|
|
12317
12463
|
const state = (await this.host.reactorClient.get(workflowId)).state.global;
|
|
12318
12464
|
if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
|
|
12319
12465
|
if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
|
|
@@ -12345,10 +12491,18 @@ var WorkflowRuntimeService = class {
|
|
|
12345
12491
|
* of bug this work package exists to stamp out. An already-enqueued row is
|
|
12346
12492
|
* closed out in place, so nothing is left PENDING for a sweep to find.
|
|
12347
12493
|
*/
|
|
12348
|
-
async refuseClosed(store, workflowId, enqueuedRunId) {
|
|
12494
|
+
async refuseClosed(store, workflowId, enqueuedRunId, triggerKind) {
|
|
12349
12495
|
const error = new WorkflowRuntimeClosedError(workflowId);
|
|
12350
12496
|
if (enqueuedRunId) {
|
|
12351
12497
|
const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
|
|
12498
|
+
if (!redelivered && triggerKind !== "manual") {
|
|
12499
|
+
try {
|
|
12500
|
+
await store?.releaseAdmission(enqueuedRunId);
|
|
12501
|
+
} catch (failure) {
|
|
12502
|
+
this.logger.warn(`Could not give back the boot admission of run ${enqueuedRunId}: @error`, failure);
|
|
12503
|
+
}
|
|
12504
|
+
throw error;
|
|
12505
|
+
}
|
|
12352
12506
|
try {
|
|
12353
12507
|
if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
|
|
12354
12508
|
else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));
|