@powerhousedao/reactor-workflow 6.2.3-dev.55 → 6.2.3-dev.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -10
- package/dist/index.d.ts +1580 -1541
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +530 -345
- package/dist/index.js.map +1 -1
- package/dist/{piece-registry-iFVZUR7s.d.ts → piece-registry-BbeuE8TB.d.ts} +84 -30
- package/dist/piece-registry-BbeuE8TB.d.ts.map +1 -0
- package/dist/{piece-registry-CsBrnpGl.js → telemetry-DZYSsZSP.js} +614 -101
- package/dist/telemetry-DZYSsZSP.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +2 -2
- package/dist/worker-entry.js +22 -2
- package/dist/worker-entry.js.map +1 -1
- package/package.json +7 -7
- package/dist/piece-registry-CsBrnpGl.js.map +0 -1
- package/dist/piece-registry-iFVZUR7s.d.ts.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as rememberSecrets, A as AttachmentCache, B as SecretNotFoundError, C as blockLabel, D as describeBuiltinPiece, E as builtinPiece, F as shapeConnection, G as PieceWorkerPool, H as secretRefFromId, I as resolveStepInput, J as PieceWorkerError, K as PieceWorker, L as expressionsIn, N as declaredConnectionIds, O as CORE_PIECE_NAME, P as declaredReactorConnectionIds, Q as redactMessage, S as storeHandlers, T as blockPorts, U as stepBlock, V as parseSecretRef, W as triggerBlock, X as containsRedactedMarker, Y as PieceWorkerTimeoutError, Z as redact, _ as effectiveRetryPolicy, _t as npmRegistryBaseUrl, a as PieceRegistry, at as FileTooLargeError, b as isHostBound, bt as isIndeterminateError, c as packagePieces, ct as pieceModuleRef, d as runWorkflow, dt as UnsupportedPieceFeatureError, et as secretsFor, f as canonicalJson, ft as authMethodFor, g as checkDynamicProperties, h as DynamicPropertiesError, i as tracedSteps, it as extractDedupeKey, j as BoundConnectionResolver, k as CORE_PIECE_VERSION, lt as sourcedResolver, m as stepConfigHash, mt as unsupportedTrigger, n as WorkflowTelemetry, nt as isPrivateAddress, o as installedPiece, ot as maxFileBytes, p as hashOf, pt as unsupportedAuth, q as PieceWorkerDisposedError, r as tracedMethods, rt as parseAddress, s as installedPieces, st as bundleResolver, t as TELEMETRY_SCOPE, tt as DEFAULT_EGRESS_POLICY, u as errorNameOf, v as ActivepiecesBlockExecutor, vt as pieceRegistrySource, w as unpinnedNote, x as resolvedBlock, y as CompositeBlockExecutor, yt as setPieceRegistryUrl, z as SecretDeletedError } from "./telemetry-DZYSsZSP.js";
|
|
2
2
|
import { ReactorAccessDeniedError, ReactorAccessDeniedError as ReactorAccessDeniedError$1, ReactorActionsFailedError, ReactorJobFailedError, ReactorJobPendingError as ReactorJobPendingError$1, WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
|
|
3
3
|
import { mkdirSync, openAsBlob, readFileSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { open, readdir, rm, stat } from "node:fs/promises";
|
|
@@ -18,9 +18,10 @@ import https from "node:https";
|
|
|
18
18
|
import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
|
|
19
19
|
import { Kind, parse } from "graphql";
|
|
20
20
|
import { ReactorHostServer } from "@powerhousedao/reactor/rpc";
|
|
21
|
-
import { SpanStatusCode, context
|
|
21
|
+
import { SpanStatusCode, context } from "@opentelemetry/api";
|
|
22
22
|
import { BaseReadModel, JobStatus, PropagationMode, defaultReadModelIndexingConfig, deleteDocumentAction, isPurgeMarker } from "@powerhousedao/reactor";
|
|
23
23
|
import { DOCUMENT_SCOPE_ACTION_TYPES, initializeAuth, normalizeDocumentModelVersion, withSignaturePolicy } from "@powerhousedao/shared/document-model";
|
|
24
|
+
import { performance as performance$1 } from "node:perf_hooks";
|
|
24
25
|
//#region src/pieces/activepieces/piece-versions.ts
|
|
25
26
|
const LISTED_TTL_MS = 3e5;
|
|
26
27
|
const ABSENT_TTL_MS = 3e4;
|
|
@@ -1951,227 +1952,6 @@ async function buildReactorRunScope(host, request) {
|
|
|
1951
1952
|
connection: { access: parsed.config.access ?? "write" }
|
|
1952
1953
|
};
|
|
1953
1954
|
}
|
|
1954
|
-
//#endregion
|
|
1955
|
-
//#region src/telemetry.ts
|
|
1956
|
-
const TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
|
|
1957
|
-
var WorkflowTelemetry = class {
|
|
1958
|
-
tracer;
|
|
1959
|
-
meter;
|
|
1960
|
-
runs;
|
|
1961
|
-
runDuration;
|
|
1962
|
-
stepDuration;
|
|
1963
|
-
phaseDuration;
|
|
1964
|
-
poolGauges;
|
|
1965
|
-
constructor(options = {}) {
|
|
1966
|
-
this.tracer = options.tracer ?? trace.getTracer("@powerhousedao/reactor-workflow");
|
|
1967
|
-
this.meter = options.meter ?? metrics.getMeter("@powerhousedao/reactor-workflow");
|
|
1968
|
-
this.runs = this.meter.createCounter("workflow.runs", {
|
|
1969
|
-
description: "Runs finished, by status and trigger kind",
|
|
1970
|
-
unit: "{run}"
|
|
1971
|
-
});
|
|
1972
|
-
this.runDuration = this.meter.createHistogram("workflow.run.duration", {
|
|
1973
|
-
description: "Run duration from fire to the journal closing it",
|
|
1974
|
-
unit: "ms"
|
|
1975
|
-
});
|
|
1976
|
-
this.stepDuration = this.meter.createHistogram("workflow.step.duration", {
|
|
1977
|
-
description: "Step duration, by piece and status",
|
|
1978
|
-
unit: "ms"
|
|
1979
|
-
});
|
|
1980
|
-
this.phaseDuration = this.meter.createHistogram("workflow.phase.duration", {
|
|
1981
|
-
description: "Duration of one part of a run: load, journal writes, worker, reactor calls",
|
|
1982
|
-
unit: "ms"
|
|
1983
|
-
});
|
|
1984
|
-
}
|
|
1985
|
-
phase(name, attributes, fn, parent) {
|
|
1986
|
-
const started = performance.now();
|
|
1987
|
-
return this.span(`workflow.${name}`, attributes, fn, parent).finally(() => this.recordPhase(name, performance.now() - started));
|
|
1988
|
-
}
|
|
1989
|
-
workerTimings(request, timings, forkedAt, ipc) {
|
|
1990
|
-
const under = (span) => trace.setSpan(context.active(), span);
|
|
1991
|
-
const child = (name, start, end, parent = request, attributes) => {
|
|
1992
|
-
const span = this.tracer.startSpan(`workflow.worker.${name}`, {
|
|
1993
|
-
startTime: start,
|
|
1994
|
-
...attributes ? { attributes } : {}
|
|
1995
|
-
}, under(parent));
|
|
1996
|
-
span.end(end);
|
|
1997
|
-
this.recordPhase(`worker.${name}`, end - start);
|
|
1998
|
-
return span;
|
|
1999
|
-
};
|
|
2000
|
-
const ready = epoch(timings.ready);
|
|
2001
|
-
if (ready !== void 0 && forkedAt > 0) child("boot", forkedAt, Math.max(forkedAt, ready));
|
|
2002
|
-
const received = epoch(timings.received);
|
|
2003
|
-
const sent = epoch(timings.sent);
|
|
2004
|
-
const delivered = Math.max(ipc.sent, ready ?? 0);
|
|
2005
|
-
if (received !== void 0) child("ipc.in", delivered, Math.max(delivered, received));
|
|
2006
|
-
const byName = /* @__PURE__ */ new Map();
|
|
2007
|
-
const phases = Array.isArray(timings.phases) ? timings.phases : [];
|
|
2008
|
-
for (const phase of phases.filter(knownPhase).sort((a, b) => a.start - b.start)) {
|
|
2009
|
-
const parent = phase.parent ? byName.get(phase.parent) : void 0;
|
|
2010
|
-
byName.set(phase.name, child(phase.name, phase.start, phase.end, parent, phaseAttributes(phase.attributes)));
|
|
2011
|
-
}
|
|
2012
|
-
if (sent !== void 0) child("ipc.out", sent, Math.max(sent, ipc.received));
|
|
2013
|
-
}
|
|
2014
|
-
recordPhase(name, ms) {
|
|
2015
|
-
this.phaseDuration.record(ms, { phase: name });
|
|
2016
|
-
}
|
|
2017
|
-
span(name, attributes, fn, parent = context.active()) {
|
|
2018
|
-
return this.tracer.startActiveSpan(name, { attributes }, parent, async (span) => {
|
|
2019
|
-
try {
|
|
2020
|
-
return await fn(span);
|
|
2021
|
-
} catch (error) {
|
|
2022
|
-
failSpan(span, error);
|
|
2023
|
-
throw error;
|
|
2024
|
-
} finally {
|
|
2025
|
-
span.end();
|
|
2026
|
-
}
|
|
2027
|
-
});
|
|
2028
|
-
}
|
|
2029
|
-
recordRun(ms, status, triggerKind) {
|
|
2030
|
-
const attributes = {
|
|
2031
|
-
"run.status": status,
|
|
2032
|
-
"trigger.kind": metricTriggerKind(triggerKind)
|
|
2033
|
-
};
|
|
2034
|
-
this.runs.add(1, attributes);
|
|
2035
|
-
this.runDuration.record(ms, attributes);
|
|
2036
|
-
}
|
|
2037
|
-
recordStep(ms, pieceName, status) {
|
|
2038
|
-
this.stepDuration.record(ms, {
|
|
2039
|
-
"piece.name": pieceName,
|
|
2040
|
-
"step.status": status
|
|
2041
|
-
});
|
|
2042
|
-
}
|
|
2043
|
-
observeWorkerPool(stats) {
|
|
2044
|
-
this.poolGauges ??= this.createPoolGauges();
|
|
2045
|
-
const { size, active, waiting } = this.poolGauges;
|
|
2046
|
-
const observables = [
|
|
2047
|
-
size,
|
|
2048
|
-
active,
|
|
2049
|
-
waiting
|
|
2050
|
-
];
|
|
2051
|
-
const callback = (result) => {
|
|
2052
|
-
const current = stats();
|
|
2053
|
-
result.observe(size, current.size);
|
|
2054
|
-
result.observe(active, current.active);
|
|
2055
|
-
result.observe(waiting, current.waiting);
|
|
2056
|
-
};
|
|
2057
|
-
this.meter.addBatchObservableCallback(callback, observables);
|
|
2058
|
-
return () => this.meter.removeBatchObservableCallback(callback, observables);
|
|
2059
|
-
}
|
|
2060
|
-
createPoolGauges() {
|
|
2061
|
-
const gauge = (name, description) => this.meter.createObservableGauge(name, {
|
|
2062
|
-
description,
|
|
2063
|
-
unit: "{worker}"
|
|
2064
|
-
});
|
|
2065
|
-
return {
|
|
2066
|
-
size: gauge("workflow.worker.pool.size", "Worker slots"),
|
|
2067
|
-
active: gauge("workflow.worker.pool.active", "Slots held by runs"),
|
|
2068
|
-
waiting: gauge("workflow.worker.pool.waiting", "Runs waiting for a slot")
|
|
2069
|
-
};
|
|
2070
|
-
}
|
|
2071
|
-
};
|
|
2072
|
-
const WORKER_PHASES = /* @__PURE__ */ new Set([
|
|
2073
|
-
"piece.load",
|
|
2074
|
-
"reactor.open",
|
|
2075
|
-
"models",
|
|
2076
|
-
"action"
|
|
2077
|
-
]);
|
|
2078
|
-
const WORKER_ATTRIBUTES = {
|
|
2079
|
-
"piece.cached": "boolean",
|
|
2080
|
-
"document.type": "string"
|
|
2081
|
-
};
|
|
2082
|
-
const MAX_ATTRIBUTE_LENGTH = 256;
|
|
2083
|
-
function epoch(value) {
|
|
2084
|
-
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
2085
|
-
}
|
|
2086
|
-
function knownPhase(phase) {
|
|
2087
|
-
if (typeof phase !== "object" || phase === null) return false;
|
|
2088
|
-
const { name, start, end, parent } = phase;
|
|
2089
|
-
return typeof name === "string" && WORKER_PHASES.has(name) && (parent === void 0 || WORKER_PHASES.has(parent)) && epoch(start) !== void 0 && epoch(end) !== void 0 && end >= start;
|
|
2090
|
-
}
|
|
2091
|
-
function phaseAttributes(attributes) {
|
|
2092
|
-
if (typeof attributes !== "object" || attributes === null) return void 0;
|
|
2093
|
-
const kept = {};
|
|
2094
|
-
for (const [key, type] of Object.entries(WORKER_ATTRIBUTES)) {
|
|
2095
|
-
const value = attributes[key];
|
|
2096
|
-
if (typeof value !== type) continue;
|
|
2097
|
-
kept[key] = typeof value === "string" ? value.slice(0, MAX_ATTRIBUTE_LENGTH) : value;
|
|
2098
|
-
}
|
|
2099
|
-
return Object.keys(kept).length > 0 ? kept : void 0;
|
|
2100
|
-
}
|
|
2101
|
-
function metricTriggerKind(triggerKind) {
|
|
2102
|
-
return triggerKind.startsWith("piece:") ? "piece" : triggerKind;
|
|
2103
|
-
}
|
|
2104
|
-
function metricPieceName(step, resolution) {
|
|
2105
|
-
return resolution?.resolved && resolution.match !== "missing" ? step.pieceName : "unresolved";
|
|
2106
|
-
}
|
|
2107
|
-
function tracedSteps(executor, telemetry) {
|
|
2108
|
-
return { execute(execution) {
|
|
2109
|
-
const { step } = execution;
|
|
2110
|
-
const started = performance.now();
|
|
2111
|
-
let status = "FAILED";
|
|
2112
|
-
let resolution;
|
|
2113
|
-
return telemetry.span("workflow.step", {
|
|
2114
|
-
"step.id": step.id,
|
|
2115
|
-
"step.key": step.key,
|
|
2116
|
-
"piece.name": step.pieceName,
|
|
2117
|
-
"action.name": step.actionName
|
|
2118
|
-
}, async (span) => {
|
|
2119
|
-
try {
|
|
2120
|
-
const result = await executor.execute(execution);
|
|
2121
|
-
resolution = result.resolution;
|
|
2122
|
-
status = "SUCCEEDED";
|
|
2123
|
-
const version = resolution?.resolved?.version;
|
|
2124
|
-
if (version) span.setAttribute("piece.version", version);
|
|
2125
|
-
return result;
|
|
2126
|
-
} catch (error) {
|
|
2127
|
-
resolution = resolutionOf(error);
|
|
2128
|
-
throw error;
|
|
2129
|
-
}
|
|
2130
|
-
}).finally(() => {
|
|
2131
|
-
telemetry.recordStep(performance.now() - started, metricPieceName(step, resolution), status);
|
|
2132
|
-
});
|
|
2133
|
-
} };
|
|
2134
|
-
}
|
|
2135
|
-
function failSpan(span, error) {
|
|
2136
|
-
span.recordException(error instanceof Error ? error : String(error));
|
|
2137
|
-
span.setStatus({
|
|
2138
|
-
code: SpanStatusCode.ERROR,
|
|
2139
|
-
message: error instanceof Error ? error.message : String(error)
|
|
2140
|
-
});
|
|
2141
|
-
}
|
|
2142
|
-
function tracedMethods(target, telemetry, prefix, parent) {
|
|
2143
|
-
return new Proxy(target, { get(object, property, receiver) {
|
|
2144
|
-
const value = Reflect.get(object, property, receiver);
|
|
2145
|
-
if (typeof value !== "function" || typeof property !== "string") return value;
|
|
2146
|
-
const phase = `${prefix}.${property}`;
|
|
2147
|
-
return (...args) => telemetry.tracer.startActiveSpan(`workflow.${phase}`, {}, parent, (span) => {
|
|
2148
|
-
const started = performance.now();
|
|
2149
|
-
const end = (error) => {
|
|
2150
|
-
if (error !== void 0) failSpan(span, error);
|
|
2151
|
-
span.end();
|
|
2152
|
-
telemetry.recordPhase(phase, performance.now() - started);
|
|
2153
|
-
};
|
|
2154
|
-
let result;
|
|
2155
|
-
try {
|
|
2156
|
-
result = value.apply(object, args);
|
|
2157
|
-
} catch (error) {
|
|
2158
|
-
end(error);
|
|
2159
|
-
throw error;
|
|
2160
|
-
}
|
|
2161
|
-
if (!(result instanceof Promise)) {
|
|
2162
|
-
end();
|
|
2163
|
-
return result;
|
|
2164
|
-
}
|
|
2165
|
-
return result.then((resolved) => {
|
|
2166
|
-
end();
|
|
2167
|
-
return resolved;
|
|
2168
|
-
}, (error) => {
|
|
2169
|
-
end(error);
|
|
2170
|
-
throw error;
|
|
2171
|
-
});
|
|
2172
|
-
});
|
|
2173
|
-
} });
|
|
2174
|
-
}
|
|
2175
1955
|
const JOB_NOT_FOUND = "Job not found";
|
|
2176
1956
|
const MIN_DEADLINE_MARGIN_MS = 250;
|
|
2177
1957
|
const MAX_DEADLINE_MARGIN_MS = 2e3;
|
|
@@ -7575,6 +7355,9 @@ function triggerBlockColumns(block) {
|
|
|
7575
7355
|
}
|
|
7576
7356
|
const FIRED_WITHOUT_RUN_ID = "fired-without-run";
|
|
7577
7357
|
const DEDUPE_CLAIM_MIGRATION = "trigger_dedupe.run_id-null-is-unclaimed";
|
|
7358
|
+
function abandonedAdmissionError(attempts) {
|
|
7359
|
+
return `This run was admitted at ${attempts} boots and the reactor stopped each time before it started, so it is no longer admitted on boot. Fix the cause, then rerun it to fire the workflow with the same payload.`;
|
|
7360
|
+
}
|
|
7578
7361
|
function abandonedFireError(attempts) {
|
|
7579
7362
|
return `This operation was delivered ${attempts} times and never once got as far as journaling a run, so the reactor was taken down before the write landed each time. It is abandoned rather than replayed on every boot. Fix the cause, then rerun this run to fire the workflow with the same payload.`;
|
|
7580
7363
|
}
|
|
@@ -7592,7 +7375,6 @@ const logger$4 = childLogger([
|
|
|
7592
7375
|
"store"
|
|
7593
7376
|
]);
|
|
7594
7377
|
const ORPHANED_RUN_ERROR = "Reactor stopped before the run finished; steps completed before then were journaled";
|
|
7595
|
-
const ABANDONED_PENDING_RUN_ERROR = "Reactor stopped before the matched trigger started its run; rerun it to fire the workflow with the same payload";
|
|
7596
7378
|
const PENDING_RUN_STATUS = "PENDING";
|
|
7597
7379
|
function errorMessage(error) {
|
|
7598
7380
|
return error instanceof Error ? error.message : String(error);
|
|
@@ -7676,6 +7458,9 @@ async function up(db) {
|
|
|
7676
7458
|
await db.schema.alterTable("run").addColumn("enqueued_at", "text").execute();
|
|
7677
7459
|
} catch {}
|
|
7678
7460
|
await db.updateTable("run").set({ enqueued_at: sql.ref("started_at") }).where("enqueued_at", "is", null).execute();
|
|
7461
|
+
try {
|
|
7462
|
+
await db.schema.alterTable("run").addColumn("admit_attempts", "integer", (col) => col.notNull().defaultTo(0)).execute();
|
|
7463
|
+
} catch {}
|
|
7679
7464
|
await db.schema.dropIndex("run_workflow_started").ifExists().execute();
|
|
7680
7465
|
await db.schema.dropIndex("run_started").ifExists().execute();
|
|
7681
7466
|
await db.schema.createIndex("run_workflow_enqueued").ifNotExists().on("run").columns([
|
|
@@ -7687,6 +7472,7 @@ async function up(db) {
|
|
|
7687
7472
|
await db.schema.createIndex("trigger_dedupe_workflow_created").ifNotExists().on("trigger_dedupe").columns(["workflow_id", "created_at"]).execute();
|
|
7688
7473
|
await db.schema.createIndex("run_ended").ifNotExists().on("run").column("ended_at").execute();
|
|
7689
7474
|
await db.schema.createIndex("trigger_dedupe_created").ifNotExists().on("trigger_dedupe").column("created_at").execute();
|
|
7475
|
+
await db.schema.createIndex("run_pending").ifNotExists().on("run").columns(["enqueued_at", "id"]).where(sql.ref("status"), "=", PENDING_RUN_STATUS).execute();
|
|
7690
7476
|
await db.schema.createIndex("trigger_state_due").ifNotExists().on("trigger_state").column("next_poll_at").where(sql.ref("status"), "=", "ENABLED").execute();
|
|
7691
7477
|
try {
|
|
7692
7478
|
await db.schema.dropTable("webhook_endpoint").ifExists().execute();
|
|
@@ -7894,6 +7680,10 @@ const LEGACY_MARKER_KEYS = [
|
|
|
7894
7680
|
* ambiguous, and nothing can resolve that for a row already written; the
|
|
7895
7681
|
* window is as narrow as the stored data allows.
|
|
7896
7682
|
*/
|
|
7683
|
+
function unreplayablePayload(payload) {
|
|
7684
|
+
if (isTruncatedStepPayload(payload)) return "truncated";
|
|
7685
|
+
if (containsRedactedMarker(payload)) return "redacted";
|
|
7686
|
+
}
|
|
7897
7687
|
function isTruncatedStepPayload(value) {
|
|
7898
7688
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
7899
7689
|
const record = value;
|
|
@@ -7924,6 +7714,9 @@ function cappedPayload(value) {
|
|
|
7924
7714
|
};
|
|
7925
7715
|
return JSON.stringify(marker);
|
|
7926
7716
|
}
|
|
7717
|
+
function rowHash(values) {
|
|
7718
|
+
return createHash("sha1").update(JSON.stringify(values)).digest("base64");
|
|
7719
|
+
}
|
|
7927
7720
|
function stepValues(runId, ordinal, step) {
|
|
7928
7721
|
return {
|
|
7929
7722
|
run_id: runId,
|
|
@@ -8041,6 +7834,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8041
7834
|
db;
|
|
8042
7835
|
unmigrated;
|
|
8043
7836
|
runsInFlight = /* @__PURE__ */ new Set();
|
|
7837
|
+
journaledSteps = /* @__PURE__ */ new Map();
|
|
8044
7838
|
constructor(db, unmigrated) {
|
|
8045
7839
|
this.db = db;
|
|
8046
7840
|
this.unmigrated = unmigrated;
|
|
@@ -8050,7 +7844,6 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8050
7844
|
const unmigrated = await up(db);
|
|
8051
7845
|
const store = new WorkflowRunStore(db, unmigrated);
|
|
8052
7846
|
await store.recoverOrphanedRuns();
|
|
8053
|
-
await store.recoverAbandonedRuns();
|
|
8054
7847
|
return store;
|
|
8055
7848
|
}
|
|
8056
7849
|
hasUnmigratedTriggerState(workflowId) {
|
|
@@ -8070,16 +7863,56 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8070
7863
|
if (recovered > 0) logger$4.warn(`Recovered ${recovered} workflow run(s) left RUNNING by a stopped reactor; they are now FAILED and rerunnable`);
|
|
8071
7864
|
return recovered;
|
|
8072
7865
|
}
|
|
8073
|
-
async
|
|
8074
|
-
|
|
8075
|
-
|
|
8076
|
-
|
|
8077
|
-
|
|
8078
|
-
|
|
8079
|
-
|
|
8080
|
-
|
|
8081
|
-
|
|
8082
|
-
|
|
7866
|
+
async admitAbandonedRuns(options = {}) {
|
|
7867
|
+
const admitted = await this.db.transaction().execute(async (trx) => {
|
|
7868
|
+
let query = trx.selectFrom("run").select([
|
|
7869
|
+
"id",
|
|
7870
|
+
"workflow_id",
|
|
7871
|
+
"trigger_kind",
|
|
7872
|
+
"trigger_payload",
|
|
7873
|
+
"admit_attempts"
|
|
7874
|
+
]).where("status", "=", PENDING_RUN_STATUS).orderBy("enqueued_at", "asc").orderBy("id", "asc");
|
|
7875
|
+
if (this.runsInFlight.size > 0) query = query.where("id", "not in", [...this.runsInFlight]);
|
|
7876
|
+
if (options.enqueuedBefore) query = query.where("enqueued_at", "<", options.enqueuedBefore.toISOString());
|
|
7877
|
+
const rows = await query.execute();
|
|
7878
|
+
const runs = [];
|
|
7879
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7880
|
+
for (const row of rows) {
|
|
7881
|
+
if (row.admit_attempts >= 3) {
|
|
7882
|
+
await trx.updateTable("run").set({
|
|
7883
|
+
status: "FAILED",
|
|
7884
|
+
error: abandonedAdmissionError(row.admit_attempts),
|
|
7885
|
+
ended_at: now
|
|
7886
|
+
}).where("id", "=", row.id).execute();
|
|
7887
|
+
continue;
|
|
7888
|
+
}
|
|
7889
|
+
const triggerPayload = row.trigger_payload === null ? void 0 : JSON.parse(row.trigger_payload);
|
|
7890
|
+
const unreplayable = unreplayablePayload(triggerPayload);
|
|
7891
|
+
if (unreplayable) {
|
|
7892
|
+
await trx.updateTable("run").set({
|
|
7893
|
+
status: "FAILED",
|
|
7894
|
+
error: `The trigger payload was ${unreplayable} by the journal and cannot be run after a restart; fire the workflow again`,
|
|
7895
|
+
ended_at: now
|
|
7896
|
+
}).where("id", "=", row.id).execute();
|
|
7897
|
+
continue;
|
|
7898
|
+
}
|
|
7899
|
+
await trx.updateTable("run").set({ admit_attempts: row.admit_attempts + 1 }).where("id", "=", row.id).execute();
|
|
7900
|
+
runs.push({
|
|
7901
|
+
runId: row.id,
|
|
7902
|
+
workflowId: row.workflow_id,
|
|
7903
|
+
triggerKind: row.trigger_kind,
|
|
7904
|
+
triggerPayload
|
|
7905
|
+
});
|
|
7906
|
+
}
|
|
7907
|
+
return runs;
|
|
7908
|
+
});
|
|
7909
|
+
for (const run of admitted) this.runsInFlight.add(run.runId);
|
|
7910
|
+
if (admitted.length > 0) logger$4.info(`Admitting ${admitted.length} workflow run(s) journaled by a stopped reactor`);
|
|
7911
|
+
return admitted;
|
|
7912
|
+
}
|
|
7913
|
+
async releaseAdmission(runId) {
|
|
7914
|
+
this.runsInFlight.delete(runId);
|
|
7915
|
+
await this.db.updateTable("run").set({ admit_attempts: sql`greatest(admit_attempts - 1, 0)` }).where("id", "=", runId).where("status", "=", PENDING_RUN_STATUS).execute();
|
|
8083
7916
|
}
|
|
8084
7917
|
/**
|
|
8085
7918
|
* Closes a run out as CANCELLED, with the reason as its error.
|
|
@@ -8091,6 +7924,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8091
7924
|
*/
|
|
8092
7925
|
async cancelRun(runId, reason) {
|
|
8093
7926
|
this.runsInFlight.delete(runId);
|
|
7927
|
+
this.journaledSteps.delete(runId);
|
|
8094
7928
|
if (erasedRuns.delete(runId)) return;
|
|
8095
7929
|
await this.db.updateTable("run").set({
|
|
8096
7930
|
status: "CANCELLED",
|
|
@@ -8100,10 +7934,20 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8100
7934
|
}
|
|
8101
7935
|
async enqueueRun(options) {
|
|
8102
7936
|
const id = randomUUID();
|
|
8103
|
-
await this.insertPendingRun(this.db, id, options);
|
|
8104
|
-
this.runsInFlight.add(id);
|
|
7937
|
+
await this.unmarkOnFailure([id], () => this.insertPendingRun(this.db, id, options));
|
|
8105
7938
|
return id;
|
|
8106
7939
|
}
|
|
7940
|
+
async unmarkOnFailure(ids, write) {
|
|
7941
|
+
try {
|
|
7942
|
+
return await write();
|
|
7943
|
+
} catch (error) {
|
|
7944
|
+
for (const id of ids) {
|
|
7945
|
+
this.runsInFlight.delete(id);
|
|
7946
|
+
this.journaledSteps.delete(id);
|
|
7947
|
+
}
|
|
7948
|
+
throw error;
|
|
7949
|
+
}
|
|
7950
|
+
}
|
|
8107
7951
|
/**
|
|
8108
7952
|
* Claims the dedupe key and journals the PENDING run, bounding crash replays.
|
|
8109
7953
|
*
|
|
@@ -8147,38 +7991,39 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8147
7991
|
attempts: attempt.attempts
|
|
8148
7992
|
};
|
|
8149
7993
|
const id = randomUUID();
|
|
8150
|
-
if (!await this.db.transaction().execute(async (trx) => {
|
|
7994
|
+
if (!await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
|
|
8151
7995
|
if (await trx.updateTable("trigger_dedupe").set({ run_id: id }).where("workflow_id", "=", options.workflowId).where("dedupe_key", "=", dedupeKey).where("run_id", "is", null).returning("dedupe_key").executeTakeFirst() === void 0) return false;
|
|
8152
7996
|
await this.insertPendingRun(trx, id, options);
|
|
8153
7997
|
return true;
|
|
8154
|
-
})) return { outcome: "duplicate" };
|
|
8155
|
-
this.runsInFlight.add(id);
|
|
7998
|
+
}))) return { outcome: "duplicate" };
|
|
8156
7999
|
return {
|
|
8157
8000
|
outcome: "claimed",
|
|
8158
8001
|
runId: id
|
|
8159
8002
|
};
|
|
8160
8003
|
}
|
|
8161
|
-
async journalTriggerItems(items, ttlMs, nowIso) {
|
|
8162
|
-
const
|
|
8004
|
+
async journalTriggerItems(items, ttlMs, nowIso, success) {
|
|
8005
|
+
const marked = [];
|
|
8006
|
+
return this.unmarkOnFailure(marked, () => this.db.transaction().execute(async (trx) => {
|
|
8007
|
+
if (success) await pollSucceededIn(trx, success.workflowId, success.storeState, nowIso, success.nextPollAtIso);
|
|
8163
8008
|
const journaled = [];
|
|
8164
8009
|
for (const { dedupeKey, options } of items) {
|
|
8165
8010
|
const id = randomUUID();
|
|
8166
8011
|
const won = dedupeKey === void 0 || await claimDedupeIn(trx, options.workflowId, dedupeKey, ttlMs, nowIso, id);
|
|
8167
|
-
if (won)
|
|
8012
|
+
if (won) {
|
|
8013
|
+
marked.push(id);
|
|
8014
|
+
await this.insertPendingRun(trx, id, options);
|
|
8015
|
+
}
|
|
8168
8016
|
journaled.push(won ? id : void 0);
|
|
8169
8017
|
}
|
|
8170
8018
|
return journaled;
|
|
8171
|
-
});
|
|
8172
|
-
for (const id of ids) if (id) this.runsInFlight.add(id);
|
|
8173
|
-
return ids;
|
|
8019
|
+
}));
|
|
8174
8020
|
}
|
|
8175
8021
|
async recordScheduleFire(nowIso, nextPollAtIso, options) {
|
|
8176
8022
|
const id = randomUUID();
|
|
8177
|
-
await this.db.transaction().execute(async (trx) => {
|
|
8023
|
+
await this.unmarkOnFailure([id], () => this.db.transaction().execute(async (trx) => {
|
|
8178
8024
|
await pollSucceededIn(trx, options.workflowId, "{}", nowIso, nextPollAtIso);
|
|
8179
8025
|
await this.insertPendingRun(trx, id, options);
|
|
8180
|
-
});
|
|
8181
|
-
this.runsInFlight.add(id);
|
|
8026
|
+
}));
|
|
8182
8027
|
return id;
|
|
8183
8028
|
}
|
|
8184
8029
|
/**
|
|
@@ -8214,6 +8059,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8214
8059
|
});
|
|
8215
8060
|
for (const runId of redelivered.runIds) {
|
|
8216
8061
|
this.runsInFlight.delete(runId);
|
|
8062
|
+
this.journaledSteps.delete(runId);
|
|
8217
8063
|
erasedRuns.delete(runId);
|
|
8218
8064
|
}
|
|
8219
8065
|
}
|
|
@@ -8270,6 +8116,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8270
8116
|
return id;
|
|
8271
8117
|
}
|
|
8272
8118
|
async insertPendingRun(db, id, options) {
|
|
8119
|
+
this.runsInFlight.add(id);
|
|
8273
8120
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8274
8121
|
await db.insertInto("run").values({
|
|
8275
8122
|
id,
|
|
@@ -8290,6 +8137,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8290
8137
|
}
|
|
8291
8138
|
async beginRun(runId, details) {
|
|
8292
8139
|
this.runsInFlight.add(runId);
|
|
8140
|
+
this.journaledSteps.set(runId, /* @__PURE__ */ new Map());
|
|
8293
8141
|
await this.db.updateTable("run").set({
|
|
8294
8142
|
status: "RUNNING",
|
|
8295
8143
|
workflow_name: details.workflowName,
|
|
@@ -8317,6 +8165,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8317
8165
|
warning_notes: null
|
|
8318
8166
|
}).execute();
|
|
8319
8167
|
this.runsInFlight.add(id);
|
|
8168
|
+
this.journaledSteps.set(id, /* @__PURE__ */ new Map());
|
|
8320
8169
|
return id;
|
|
8321
8170
|
}
|
|
8322
8171
|
async recordStep(runId, ordinal, step) {
|
|
@@ -8327,11 +8176,26 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8327
8176
|
id: randomUUID(),
|
|
8328
8177
|
...values
|
|
8329
8178
|
}).onConflict((oc) => oc.columns(["run_id", "step_id"]).doUpdateSet(mutable)).execute();
|
|
8179
|
+
this.journaledSteps.get(runId)?.set(step.stepId, {
|
|
8180
|
+
ordinal,
|
|
8181
|
+
hash: rowHash(values)
|
|
8182
|
+
});
|
|
8183
|
+
}
|
|
8184
|
+
sweepIsNoop(runId, result) {
|
|
8185
|
+
const journaled = this.journaledSteps.get(runId);
|
|
8186
|
+
if (!journaled) return false;
|
|
8187
|
+
return result.steps.every((step) => {
|
|
8188
|
+
const row = journaled.get(step.stepId);
|
|
8189
|
+
if (row === void 0) return false;
|
|
8190
|
+
return row.hash === rowHash(stepValues(runId, row.ordinal, step));
|
|
8191
|
+
});
|
|
8330
8192
|
}
|
|
8331
8193
|
async finishRun(runId, result, executionOrder) {
|
|
8332
8194
|
this.runsInFlight.delete(runId);
|
|
8195
|
+
const sweep = !this.sweepIsNoop(runId, result);
|
|
8196
|
+
this.journaledSteps.delete(runId);
|
|
8333
8197
|
if (erasedRuns.delete(runId)) return;
|
|
8334
|
-
if (result.steps.length > 0) try {
|
|
8198
|
+
if (sweep && result.steps.length > 0) try {
|
|
8335
8199
|
await this.sweepSteps(runId, result, executionOrder);
|
|
8336
8200
|
} catch (error) {
|
|
8337
8201
|
logger$4.warn(`Run ${runId}: writing the closing step journal failed; the run is closed out without it`, error);
|
|
@@ -8382,6 +8246,7 @@ var WorkflowRunStore = class WorkflowRunStore {
|
|
|
8382
8246
|
}
|
|
8383
8247
|
async failRun(runId, error, errorName) {
|
|
8384
8248
|
this.runsInFlight.delete(runId);
|
|
8249
|
+
this.journaledSteps.delete(runId);
|
|
8385
8250
|
if (erasedRuns.delete(runId)) return;
|
|
8386
8251
|
await this.db.updateTable("run").set({
|
|
8387
8252
|
status: "FAILED",
|
|
@@ -8764,7 +8629,7 @@ var WorkflowRunGate = class {
|
|
|
8764
8629
|
* journals the refusal rather than dropping it silently, because a firing
|
|
8765
8630
|
* that vanished is indistinguishable from a trigger that never fired.
|
|
8766
8631
|
*/
|
|
8767
|
-
async admit(workflowId, policy) {
|
|
8632
|
+
async admit(workflowId, policy, onQueue) {
|
|
8768
8633
|
if (this.closed) return {
|
|
8769
8634
|
admitted: false,
|
|
8770
8635
|
reason: CLOSED_REASON,
|
|
@@ -8796,6 +8661,7 @@ var WorkflowRunGate = class {
|
|
|
8796
8661
|
refusal: "queue-full",
|
|
8797
8662
|
reason: `Skipped: ${lane.waiting.length} firings of this workflow are already waiting for a slot, which is its queue depth (${this.maxQueued}; raise ${QUEUE_DEPTH_ENV} to allow more)`
|
|
8798
8663
|
};
|
|
8664
|
+
onQueue?.();
|
|
8799
8665
|
if (!await new Promise((resolve) => lane.waiting.push(resolve))) return {
|
|
8800
8666
|
admitted: false,
|
|
8801
8667
|
reason: CLOSED_REASON,
|
|
@@ -8831,6 +8697,117 @@ var WorkflowRunGate = class {
|
|
|
8831
8697
|
}
|
|
8832
8698
|
};
|
|
8833
8699
|
//#endregion
|
|
8700
|
+
//#region src/reactor/shutdown.ts
|
|
8701
|
+
/** A firing refused because this runtime has shut down: retryable elsewhere. */
|
|
8702
|
+
var WorkflowRuntimeClosedError = class extends Error {
|
|
8703
|
+
workflowId;
|
|
8704
|
+
constructor(workflowId) {
|
|
8705
|
+
super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
|
|
8706
|
+
this.workflowId = workflowId;
|
|
8707
|
+
this.name = "WorkflowRuntimeClosedError";
|
|
8708
|
+
}
|
|
8709
|
+
};
|
|
8710
|
+
var TriggerSupervisorStoppedError = class extends Error {
|
|
8711
|
+
constructor(options) {
|
|
8712
|
+
super("The trigger supervisor has stopped", options);
|
|
8713
|
+
this.name = "TriggerSupervisorStoppedError";
|
|
8714
|
+
}
|
|
8715
|
+
};
|
|
8716
|
+
function isShutdownRefusal(error) {
|
|
8717
|
+
return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
|
|
8718
|
+
}
|
|
8719
|
+
//#endregion
|
|
8720
|
+
//#region src/reactor/run-dispatcher.ts
|
|
8721
|
+
const DEFAULT_ADMIT_MAX_ELU = .8;
|
|
8722
|
+
var RunDispatcher = class {
|
|
8723
|
+
maxInFlight;
|
|
8724
|
+
maxElu;
|
|
8725
|
+
loop;
|
|
8726
|
+
waiters = [];
|
|
8727
|
+
inFlight = 0;
|
|
8728
|
+
closed = false;
|
|
8729
|
+
unsubscribe;
|
|
8730
|
+
constructor(options) {
|
|
8731
|
+
this.maxInFlight = Math.max(1, Math.trunc(options.maxInFlight));
|
|
8732
|
+
this.maxElu = options.maxElu ?? .8;
|
|
8733
|
+
this.loop = options.loop;
|
|
8734
|
+
this.unsubscribe = this.loop.onSample(() => this.drain());
|
|
8735
|
+
}
|
|
8736
|
+
admit(workflowId, onQueue) {
|
|
8737
|
+
if (this.closed) return Promise.reject(new WorkflowRuntimeClosedError(workflowId));
|
|
8738
|
+
if (this.waiters.length === 0 && this.room()) return Promise.resolve(this.take());
|
|
8739
|
+
onQueue?.();
|
|
8740
|
+
return new Promise((resolve, reject) => {
|
|
8741
|
+
this.waiters.push({
|
|
8742
|
+
workflowId,
|
|
8743
|
+
resolve,
|
|
8744
|
+
reject
|
|
8745
|
+
});
|
|
8746
|
+
});
|
|
8747
|
+
}
|
|
8748
|
+
stats() {
|
|
8749
|
+
return {
|
|
8750
|
+
inFlight: this.inFlight,
|
|
8751
|
+
waiting: this.waiters.length
|
|
8752
|
+
};
|
|
8753
|
+
}
|
|
8754
|
+
close() {
|
|
8755
|
+
if (this.closed) return;
|
|
8756
|
+
this.closed = true;
|
|
8757
|
+
this.unsubscribe();
|
|
8758
|
+
for (const waiter of this.waiters.splice(0)) waiter.reject(new WorkflowRuntimeClosedError(waiter.workflowId));
|
|
8759
|
+
}
|
|
8760
|
+
room() {
|
|
8761
|
+
if (this.inFlight === 0) return true;
|
|
8762
|
+
return this.inFlight < this.maxInFlight && this.loop.utilization() <= this.maxElu;
|
|
8763
|
+
}
|
|
8764
|
+
take() {
|
|
8765
|
+
this.inFlight++;
|
|
8766
|
+
let released = false;
|
|
8767
|
+
return () => {
|
|
8768
|
+
if (released) return;
|
|
8769
|
+
released = true;
|
|
8770
|
+
this.inFlight--;
|
|
8771
|
+
this.drain();
|
|
8772
|
+
};
|
|
8773
|
+
}
|
|
8774
|
+
drain() {
|
|
8775
|
+
while (this.waiters.length > 0 && !this.closed && this.room()) this.waiters.shift().resolve(this.take());
|
|
8776
|
+
}
|
|
8777
|
+
};
|
|
8778
|
+
//#endregion
|
|
8779
|
+
//#region src/reactor/event-loop.ts
|
|
8780
|
+
const SAMPLE_MS = 1e3;
|
|
8781
|
+
var EventLoopMonitor = class {
|
|
8782
|
+
last = performance$1.eventLoopUtilization();
|
|
8783
|
+
current = 0;
|
|
8784
|
+
timer;
|
|
8785
|
+
listeners = /* @__PURE__ */ new Set();
|
|
8786
|
+
start() {
|
|
8787
|
+
if (this.timer) return;
|
|
8788
|
+
this.last = performance$1.eventLoopUtilization();
|
|
8789
|
+
this.timer = setInterval(() => this.sample(), SAMPLE_MS);
|
|
8790
|
+
this.timer.unref();
|
|
8791
|
+
}
|
|
8792
|
+
stop() {
|
|
8793
|
+
clearInterval(this.timer);
|
|
8794
|
+
this.timer = void 0;
|
|
8795
|
+
}
|
|
8796
|
+
utilization() {
|
|
8797
|
+
return this.current;
|
|
8798
|
+
}
|
|
8799
|
+
onSample(listener) {
|
|
8800
|
+
this.listeners.add(listener);
|
|
8801
|
+
return () => this.listeners.delete(listener);
|
|
8802
|
+
}
|
|
8803
|
+
sample() {
|
|
8804
|
+
const now = performance$1.eventLoopUtilization();
|
|
8805
|
+
this.current = performance$1.eventLoopUtilization(now, this.last).utilization;
|
|
8806
|
+
this.last = now;
|
|
8807
|
+
for (const listener of this.listeners) listener();
|
|
8808
|
+
}
|
|
8809
|
+
};
|
|
8810
|
+
//#endregion
|
|
8834
8811
|
//#region src/reactor/run-retention.ts
|
|
8835
8812
|
const RUN_RETENTION_ENV = "PH_WORKFLOWS_RUN_RETENTION_DAYS";
|
|
8836
8813
|
const RETENTION_SWEEP_INTERVAL_MS = 36e5;
|
|
@@ -8866,26 +8843,6 @@ async function sweepRetention(store, options) {
|
|
|
8866
8843
|
dedupeKeys: await store.pruneDedupe(new Date(now - options.dedupeTtlMs).toISOString())
|
|
8867
8844
|
};
|
|
8868
8845
|
}
|
|
8869
|
-
//#endregion
|
|
8870
|
-
//#region src/reactor/shutdown.ts
|
|
8871
|
-
/** A firing refused because this runtime has shut down: retryable elsewhere. */
|
|
8872
|
-
var WorkflowRuntimeClosedError = class extends Error {
|
|
8873
|
-
workflowId;
|
|
8874
|
-
constructor(workflowId) {
|
|
8875
|
-
super(`Workflow ${workflowId} was not run: this workflow runtime has shut down`);
|
|
8876
|
-
this.workflowId = workflowId;
|
|
8877
|
-
this.name = "WorkflowRuntimeClosedError";
|
|
8878
|
-
}
|
|
8879
|
-
};
|
|
8880
|
-
var TriggerSupervisorStoppedError = class extends Error {
|
|
8881
|
-
constructor(options) {
|
|
8882
|
-
super("The trigger supervisor has stopped", options);
|
|
8883
|
-
this.name = "TriggerSupervisorStoppedError";
|
|
8884
|
-
}
|
|
8885
|
-
};
|
|
8886
|
-
function isShutdownRefusal(error) {
|
|
8887
|
-
return error instanceof TriggerSupervisorStoppedError || error instanceof WorkflowRuntimeClosedError || error instanceof PieceWorkerDisposedError;
|
|
8888
|
-
}
|
|
8889
8846
|
childLogger(["workflow", "trigger-binding"]);
|
|
8890
8847
|
function pieceTriggerKind(block) {
|
|
8891
8848
|
return `piece:${block.pieceName}:${block.name}`;
|
|
@@ -8976,7 +8933,9 @@ var TriggerSupervisor = class {
|
|
|
8976
8933
|
egress;
|
|
8977
8934
|
parks;
|
|
8978
8935
|
timer;
|
|
8979
|
-
|
|
8936
|
+
lanes = /* @__PURE__ */ new Map();
|
|
8937
|
+
ticked = /* @__PURE__ */ new Set();
|
|
8938
|
+
hookSlots;
|
|
8980
8939
|
ticking = false;
|
|
8981
8940
|
warnedMissingJournal = false;
|
|
8982
8941
|
stopped = false;
|
|
@@ -8989,6 +8948,7 @@ var TriggerSupervisor = class {
|
|
|
8989
8948
|
this.tickMs = options.tickMs ?? 15e3;
|
|
8990
8949
|
this.defaultIntervalMs = options.defaultIntervalMs ?? 6e4;
|
|
8991
8950
|
this.hookTimeoutMs = options.hookTimeoutMs ?? 6e4;
|
|
8951
|
+
this.hookSlots = new Slots(options.hookConcurrency ?? 1);
|
|
8992
8952
|
this.now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
8993
8953
|
this.egress = options.egress === void 0 ? DEFAULT_EGRESS_POLICY : options.egress ?? void 0;
|
|
8994
8954
|
this.parks = options.parks ?? new ParkState(async () => (await options.store())?.listWorkflowParks());
|
|
@@ -9015,8 +8975,8 @@ var TriggerSupervisor = class {
|
|
|
9015
8975
|
if (!this.options.worker) this.worker.dispose();
|
|
9016
8976
|
logger$2.info("Trigger supervisor stopped");
|
|
9017
8977
|
}
|
|
9018
|
-
enqueue(task) {
|
|
9019
|
-
const run = this.
|
|
8978
|
+
enqueue(workflowId, task) {
|
|
8979
|
+
const run = (this.lanes.get(workflowId) ?? Promise.resolve()).then(async () => {
|
|
9020
8980
|
this.throwIfStopped();
|
|
9021
8981
|
try {
|
|
9022
8982
|
return await task();
|
|
@@ -9025,7 +8985,11 @@ var TriggerSupervisor = class {
|
|
|
9025
8985
|
throw error;
|
|
9026
8986
|
}
|
|
9027
8987
|
});
|
|
9028
|
-
|
|
8988
|
+
const settled = run.catch(() => void 0);
|
|
8989
|
+
this.lanes.set(workflowId, settled);
|
|
8990
|
+
settled.then(() => {
|
|
8991
|
+
if (this.lanes.get(workflowId) === settled) this.lanes.delete(workflowId);
|
|
8992
|
+
});
|
|
9029
8993
|
return run;
|
|
9030
8994
|
}
|
|
9031
8995
|
throwIfStopped() {
|
|
@@ -9063,12 +9027,12 @@ var TriggerSupervisor = class {
|
|
|
9063
9027
|
this.bindings.set(binding.workflowId, binding);
|
|
9064
9028
|
const park = this.parks.known(binding.workflowId)?.park;
|
|
9065
9029
|
if (park && publishedVersion !== void 0 && publishedVersion > park.published_version) this.parks.delete(binding.workflowId);
|
|
9066
|
-
return this.enqueue(() => this.enable(binding, previous));
|
|
9030
|
+
return this.enqueue(binding.workflowId, () => this.enable(binding, previous));
|
|
9067
9031
|
}
|
|
9068
9032
|
/** Parks a workflow; true when a trigger row was parked with it. */
|
|
9069
9033
|
park(workflowId, publishedVersion, reason) {
|
|
9070
9034
|
this.parks.set(workflowParkRow(workflowId, publishedVersion, reason));
|
|
9071
|
-
return this.enqueue(async () => {
|
|
9035
|
+
return this.enqueue(workflowId, async () => {
|
|
9072
9036
|
const store = await this.store();
|
|
9073
9037
|
if (!store) return false;
|
|
9074
9038
|
return store.parkWorkflow(workflowId, publishedVersion, reason);
|
|
@@ -9077,7 +9041,7 @@ var TriggerSupervisor = class {
|
|
|
9077
9041
|
/** Undoes {@link park} for that version only: a later park stands. */
|
|
9078
9042
|
liftPark(workflowId, publishedVersion, trigger) {
|
|
9079
9043
|
if (this.parks.known(workflowId)?.park?.published_version === publishedVersion) this.parks.delete(workflowId);
|
|
9080
|
-
return this.enqueue(async () => {
|
|
9044
|
+
return this.enqueue(workflowId, async () => {
|
|
9081
9045
|
const store = await this.store();
|
|
9082
9046
|
if (!store) return;
|
|
9083
9047
|
if ((await store.getWorkflowPark(workflowId))?.published_version !== publishedVersion) return;
|
|
@@ -9088,7 +9052,7 @@ var TriggerSupervisor = class {
|
|
|
9088
9052
|
* row through onDisable (a boot), the row is turned DISABLED here. */
|
|
9089
9053
|
releasePark(workflowId, flipRow) {
|
|
9090
9054
|
this.parks.delete(workflowId);
|
|
9091
|
-
return this.enqueue(async () => {
|
|
9055
|
+
return this.enqueue(workflowId, async () => {
|
|
9092
9056
|
const store = await this.store();
|
|
9093
9057
|
if (!store) return;
|
|
9094
9058
|
if (flipRow) await store.clearParkOnDisable(workflowId);
|
|
@@ -9106,7 +9070,7 @@ var TriggerSupervisor = class {
|
|
|
9106
9070
|
if (retry) retry.at = 0;
|
|
9107
9071
|
const park = this.parks.known(workflowId)?.park;
|
|
9108
9072
|
if (park && park.published_version < publishedVersion) this.parks.delete(workflowId);
|
|
9109
|
-
return this.enqueue(async () => {
|
|
9073
|
+
return this.enqueue(workflowId, async () => {
|
|
9110
9074
|
const store = await this.store();
|
|
9111
9075
|
if (!store) return;
|
|
9112
9076
|
const park = await store.getWorkflowPark(workflowId);
|
|
@@ -9116,12 +9080,12 @@ var TriggerSupervisor = class {
|
|
|
9116
9080
|
}
|
|
9117
9081
|
remove(workflowId) {
|
|
9118
9082
|
const binding = this.unbind(workflowId);
|
|
9119
|
-
return this.enqueue(() => this.disable(workflowId, binding));
|
|
9083
|
+
return this.enqueue(workflowId, () => this.disable(workflowId, binding));
|
|
9120
9084
|
}
|
|
9121
9085
|
forget(workflowId) {
|
|
9122
9086
|
const binding = this.unbind(workflowId);
|
|
9123
9087
|
this.parks.delete(workflowId);
|
|
9124
|
-
return this.enqueue(async () => {
|
|
9088
|
+
return this.enqueue(workflowId, async () => {
|
|
9125
9089
|
const store = await this.store();
|
|
9126
9090
|
if (!store) return;
|
|
9127
9091
|
await this.disable(workflowId, binding);
|
|
@@ -9139,7 +9103,7 @@ var TriggerSupervisor = class {
|
|
|
9139
9103
|
}
|
|
9140
9104
|
reject(workflowId, block, config, message, retryAt) {
|
|
9141
9105
|
this.unbind(workflowId);
|
|
9142
|
-
return this.enqueue(() => this.recordRejection(workflowId, block, config, message, retryAt));
|
|
9106
|
+
return this.enqueue(workflowId, () => this.recordRejection(workflowId, block, config, message, retryAt));
|
|
9143
9107
|
}
|
|
9144
9108
|
async recordRejection(workflowId, block, config, message, retryAt) {
|
|
9145
9109
|
const store = await this.store();
|
|
@@ -9172,7 +9136,7 @@ var TriggerSupervisor = class {
|
|
|
9172
9136
|
});
|
|
9173
9137
|
}
|
|
9174
9138
|
test(binding, options = {}) {
|
|
9175
|
-
return this.enqueue(async () => {
|
|
9139
|
+
return this.enqueue(binding.workflowId, async () => {
|
|
9176
9140
|
const store = await this.store();
|
|
9177
9141
|
try {
|
|
9178
9142
|
return (await this.hook(binding, "test", { ...options.runUser !== void 0 ? { runUser: options.runUser } : {} })).output;
|
|
@@ -9191,10 +9155,10 @@ var TriggerSupervisor = class {
|
|
|
9191
9155
|
}
|
|
9192
9156
|
}
|
|
9193
9157
|
handshake(binding, payload) {
|
|
9194
|
-
return this.enqueue(() => this.hook(binding, "onHandshake", { payload }));
|
|
9158
|
+
return this.enqueue(binding.workflowId, () => this.hook(binding, "onHandshake", { payload }));
|
|
9195
9159
|
}
|
|
9196
9160
|
deliverWebhook(workflowId, payload) {
|
|
9197
|
-
return this.enqueue(async () => {
|
|
9161
|
+
return this.enqueue(workflowId, async () => {
|
|
9198
9162
|
const binding = this.bindings.get(workflowId);
|
|
9199
9163
|
if (!binding || isSchedule(binding)) {
|
|
9200
9164
|
logger$2.warn(`Webhook delivery for unknown workflow ${workflowId}`);
|
|
@@ -9212,8 +9176,12 @@ var TriggerSupervisor = class {
|
|
|
9212
9176
|
throw error;
|
|
9213
9177
|
});
|
|
9214
9178
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
9215
|
-
|
|
9216
|
-
|
|
9179
|
+
const success = row?.status === "ENABLED" ? {
|
|
9180
|
+
workflowId,
|
|
9181
|
+
storeState: VESTIGIAL_STORE_STATE,
|
|
9182
|
+
nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
|
|
9183
|
+
} : void 0;
|
|
9184
|
+
await this.fireItems(store, binding, result.output, now, rewind, success);
|
|
9217
9185
|
} catch (error) {
|
|
9218
9186
|
if (!refused) await rewind();
|
|
9219
9187
|
throw error;
|
|
@@ -9545,22 +9513,38 @@ var TriggerSupervisor = class {
|
|
|
9545
9513
|
async tick() {
|
|
9546
9514
|
if (this.ticking || this.stopped) return;
|
|
9547
9515
|
this.ticking = true;
|
|
9516
|
+
const started = [];
|
|
9517
|
+
let listing;
|
|
9548
9518
|
try {
|
|
9549
|
-
await this.
|
|
9519
|
+
await this.startDue(started);
|
|
9520
|
+
} catch (error) {
|
|
9521
|
+
listing = error;
|
|
9550
9522
|
} finally {
|
|
9551
9523
|
this.ticking = false;
|
|
9552
9524
|
}
|
|
9525
|
+
const outcomes = await Promise.allSettled(started.filter((run) => run !== void 0));
|
|
9526
|
+
if (listing !== void 0) throw listing instanceof Error ? listing : new Error(String(listing));
|
|
9527
|
+
const failed = outcomes.find((outcome) => outcome.status === "rejected");
|
|
9528
|
+
if (failed) throw failed.reason;
|
|
9553
9529
|
}
|
|
9554
|
-
|
|
9530
|
+
startOnLane(workflowId, task, hook) {
|
|
9531
|
+
if (this.ticked.has(workflowId)) return void 0;
|
|
9532
|
+
this.ticked.add(workflowId);
|
|
9533
|
+
return (hook ? this.hookSlots.run(() => this.enqueue(workflowId, task)) : this.enqueue(workflowId, task)).finally(() => this.ticked.delete(workflowId));
|
|
9534
|
+
}
|
|
9535
|
+
startOneEnableRetry() {
|
|
9555
9536
|
const now = this.now().getTime();
|
|
9556
|
-
const next = [...this.enableRetries].filter(([, retry]) => retry.at <= now).sort((a, b) => a[1].at - b[1].at).at(0);
|
|
9557
|
-
if (!next) return;
|
|
9537
|
+
const next = [...this.enableRetries].filter(([id, retry]) => retry.at <= now && !this.ticked.has(id)).sort((a, b) => a[1].at - b[1].at).at(0);
|
|
9538
|
+
if (!next) return void 0;
|
|
9558
9539
|
const [workflowId, retry] = next;
|
|
9559
9540
|
const binding = this.bindings.get(workflowId);
|
|
9560
9541
|
if (!binding) {
|
|
9561
9542
|
this.enableRetries.delete(workflowId);
|
|
9562
9543
|
return;
|
|
9563
9544
|
}
|
|
9545
|
+
return this.startOnLane(workflowId, () => this.retryEnable(workflowId, retry, binding, now), true);
|
|
9546
|
+
}
|
|
9547
|
+
async retryEnable(workflowId, retry, binding, now) {
|
|
9564
9548
|
try {
|
|
9565
9549
|
await this.enable(binding);
|
|
9566
9550
|
} catch (error) {
|
|
@@ -9575,7 +9559,7 @@ var TriggerSupervisor = class {
|
|
|
9575
9559
|
logger$2.error(`Enable retry for workflow ${workflowId} threw`, error);
|
|
9576
9560
|
}
|
|
9577
9561
|
}
|
|
9578
|
-
async
|
|
9562
|
+
async startDue(started) {
|
|
9579
9563
|
const store = await this.store();
|
|
9580
9564
|
if (!store) {
|
|
9581
9565
|
if (!this.warnedMissingJournal) {
|
|
@@ -9584,37 +9568,35 @@ var TriggerSupervisor = class {
|
|
|
9584
9568
|
}
|
|
9585
9569
|
return;
|
|
9586
9570
|
}
|
|
9587
|
-
const
|
|
9588
|
-
for (const row of
|
|
9571
|
+
const ordered = [...await store.listDueTriggerStates(this.now().toISOString())].sort((a, b) => Number(!this.isScheduleRow(a)) - Number(!this.isScheduleRow(b)));
|
|
9572
|
+
for (const row of ordered) {
|
|
9589
9573
|
this.throwIfStopped();
|
|
9590
9574
|
const binding = this.bindings.get(row.workflow_id);
|
|
9591
9575
|
if (!binding) {
|
|
9592
|
-
|
|
9576
|
+
started.push(this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false));
|
|
9593
9577
|
continue;
|
|
9594
9578
|
}
|
|
9595
|
-
|
|
9596
|
-
else await this.poll(store, row, binding);
|
|
9579
|
+
started.push(isSchedule(binding) ? this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.fireSchedule(store, row, binding)), false) : this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.poll(store, row, binding)), true));
|
|
9597
9580
|
}
|
|
9598
9581
|
this.throwIfStopped();
|
|
9599
|
-
await this.
|
|
9582
|
+
started.push(...await this.startRenewals(store));
|
|
9600
9583
|
this.throwIfStopped();
|
|
9601
|
-
|
|
9584
|
+
started.push(this.startOneEnableRetry());
|
|
9602
9585
|
}
|
|
9603
|
-
|
|
9604
|
-
|
|
9605
|
-
|
|
9606
|
-
|
|
9586
|
+
whileBound(binding, task) {
|
|
9587
|
+
return () => this.bindings.get(binding.workflowId) === binding ? task() : Promise.resolve();
|
|
9588
|
+
}
|
|
9589
|
+
isScheduleRow(row) {
|
|
9590
|
+
const binding = this.bindings.get(row.workflow_id);
|
|
9591
|
+
return binding !== void 0 && isSchedule(binding);
|
|
9592
|
+
}
|
|
9593
|
+
async startRenewals(store) {
|
|
9594
|
+
return (await store.listDueTriggerRenewals(this.now().toISOString())).map((row) => {
|
|
9607
9595
|
const binding = this.bindings.get(row.workflow_id);
|
|
9608
|
-
if (!binding)
|
|
9609
|
-
|
|
9610
|
-
|
|
9611
|
-
|
|
9612
|
-
if (isSchedule(binding)) {
|
|
9613
|
-
await store.setTriggerRenewAt(row.workflow_id, null);
|
|
9614
|
-
continue;
|
|
9615
|
-
}
|
|
9616
|
-
await this.renew(store, row, binding);
|
|
9617
|
-
}
|
|
9596
|
+
if (!binding) return this.startOnLane(row.workflow_id, () => store.setTriggerStatus(row.workflow_id, "DISABLED"), false);
|
|
9597
|
+
if (isSchedule(binding)) return this.startOnLane(row.workflow_id, () => store.setTriggerRenewAt(row.workflow_id, null), false);
|
|
9598
|
+
return this.startOnLane(row.workflow_id, this.whileBound(binding, () => this.renew(store, row, binding)), true);
|
|
9599
|
+
});
|
|
9618
9600
|
}
|
|
9619
9601
|
async renew(store, row, binding) {
|
|
9620
9602
|
const now = this.now();
|
|
@@ -9676,8 +9658,11 @@ var TriggerSupervisor = class {
|
|
|
9676
9658
|
throw error;
|
|
9677
9659
|
});
|
|
9678
9660
|
if (!Array.isArray(result.output)) throw new Error(`Trigger run returned ${typeof result.output}, expected an array`);
|
|
9679
|
-
await
|
|
9680
|
-
|
|
9661
|
+
await this.fireItems(store, binding, result.output, now, rewind, {
|
|
9662
|
+
workflowId: row.workflow_id,
|
|
9663
|
+
storeState: VESTIGIAL_STORE_STATE,
|
|
9664
|
+
nextPollAtIso: new Date(now.getTime() + row.interval_ms).toISOString()
|
|
9665
|
+
});
|
|
9681
9666
|
} catch (error) {
|
|
9682
9667
|
if (!refused) await rewind();
|
|
9683
9668
|
if (this.stopped) throw error;
|
|
@@ -9688,8 +9673,11 @@ var TriggerSupervisor = class {
|
|
|
9688
9673
|
logger$2.warn(`Poll failed for workflow ${row.workflow_id} (${failures}x): @error`, message);
|
|
9689
9674
|
}
|
|
9690
9675
|
}
|
|
9691
|
-
async fireItems(store, binding, items, now, rewind) {
|
|
9692
|
-
if (items.length === 0)
|
|
9676
|
+
async fireItems(store, binding, items, now, rewind, success) {
|
|
9677
|
+
if (items.length === 0) {
|
|
9678
|
+
if (success) await store.recordPollSuccess(success.workflowId, success.storeState, now.toISOString(), success.nextPollAtIso);
|
|
9679
|
+
return;
|
|
9680
|
+
}
|
|
9693
9681
|
const kind = pieceTriggerKind(binding.block);
|
|
9694
9682
|
const journaled = (await store.journalTriggerItems(items.map((item) => ({
|
|
9695
9683
|
dedupeKey: extractDedupeKey(item),
|
|
@@ -9698,7 +9686,7 @@ var TriggerSupervisor = class {
|
|
|
9698
9686
|
triggerKind: kind,
|
|
9699
9687
|
triggerPayload: item
|
|
9700
9688
|
}
|
|
9701
|
-
})), DEDUPE_TTL_MS, now.toISOString())).flatMap((runId, index) => runId ? [{
|
|
9689
|
+
})), DEDUPE_TTL_MS, now.toISOString(), success)).flatMap((runId, index) => runId ? [{
|
|
9702
9690
|
runId,
|
|
9703
9691
|
item: items[index]
|
|
9704
9692
|
}] : []);
|
|
@@ -9721,6 +9709,25 @@ var TriggerSupervisor = class {
|
|
|
9721
9709
|
}
|
|
9722
9710
|
}
|
|
9723
9711
|
};
|
|
9712
|
+
var Slots = class {
|
|
9713
|
+
limit;
|
|
9714
|
+
active = 0;
|
|
9715
|
+
waiting = [];
|
|
9716
|
+
constructor(limit) {
|
|
9717
|
+
this.limit = limit;
|
|
9718
|
+
}
|
|
9719
|
+
async run(task) {
|
|
9720
|
+
if (this.active >= Math.max(1, this.limit)) await new Promise((resolve) => this.waiting.push(resolve));
|
|
9721
|
+
else this.active++;
|
|
9722
|
+
try {
|
|
9723
|
+
return await task();
|
|
9724
|
+
} finally {
|
|
9725
|
+
const next = this.waiting.shift();
|
|
9726
|
+
if (next) next();
|
|
9727
|
+
else this.active--;
|
|
9728
|
+
}
|
|
9729
|
+
}
|
|
9730
|
+
};
|
|
9724
9731
|
//#endregion
|
|
9725
9732
|
//#region src/reactor/webhook.ts
|
|
9726
9733
|
const WEBHOOK = "Webhook";
|
|
@@ -9948,6 +9955,7 @@ function bindingTarget(binding) {
|
|
|
9948
9955
|
}
|
|
9949
9956
|
const RUNS_SCAN_LIMIT = 1e3;
|
|
9950
9957
|
const WEBHOOK_TEST_TIMEOUT_MS = 3e5;
|
|
9958
|
+
const PENDING_SWEEP_INTERVAL_MS = 6e4;
|
|
9951
9959
|
function draftBlocks(state) {
|
|
9952
9960
|
return [...state.trigger ? [{
|
|
9953
9961
|
id: state.trigger.id,
|
|
@@ -10118,6 +10126,10 @@ function configRecord(config) {
|
|
|
10118
10126
|
if (config && typeof config === "object" && !Array.isArray(config)) return config;
|
|
10119
10127
|
return {};
|
|
10120
10128
|
}
|
|
10129
|
+
const WORKFLOW_CACHE_MAX = 1e3;
|
|
10130
|
+
function globalRevision(document) {
|
|
10131
|
+
return document.header.revision?.global ?? 0;
|
|
10132
|
+
}
|
|
10121
10133
|
function runJournalName(stateName, documentName) {
|
|
10122
10134
|
return stateName?.trim() ? stateName : documentName ?? "";
|
|
10123
10135
|
}
|
|
@@ -10129,6 +10141,9 @@ var WorkflowRuntimeService = class {
|
|
|
10129
10141
|
executor;
|
|
10130
10142
|
pieceWorkers;
|
|
10131
10143
|
unobservePool;
|
|
10144
|
+
runDispatcher;
|
|
10145
|
+
unobserveDispatcher;
|
|
10146
|
+
eventLoop = new EventLoopMonitor();
|
|
10132
10147
|
storePromise;
|
|
10133
10148
|
storeError;
|
|
10134
10149
|
storeOpening = false;
|
|
@@ -10624,10 +10639,35 @@ var WorkflowRuntimeService = class {
|
|
|
10624
10639
|
await this.updateRegistration(workflowId, carried, onlyIfChanged);
|
|
10625
10640
|
return;
|
|
10626
10641
|
}
|
|
10642
|
+
const generation = this.workflowGeneration(workflowId);
|
|
10627
10643
|
const document = await this.host.reactorClient.get(workflowId);
|
|
10644
|
+
if (document.header.documentType === WORKFLOW_DOCUMENT_TYPE) this.cacheLoaded(workflowId, generation, document);
|
|
10628
10645
|
await this.updateRegistration(workflowId, document.state.global, onlyIfChanged);
|
|
10629
10646
|
}
|
|
10630
10647
|
seenOps = /* @__PURE__ */ new Set();
|
|
10648
|
+
workflowCache = /* @__PURE__ */ new Map();
|
|
10649
|
+
workflowGenerations = /* @__PURE__ */ new Map();
|
|
10650
|
+
workflowGeneration(workflowId) {
|
|
10651
|
+
return this.workflowGenerations.get(workflowId) ?? 0;
|
|
10652
|
+
}
|
|
10653
|
+
cacheLoaded(workflowId, generation, document) {
|
|
10654
|
+
if (this.workflowGeneration(workflowId) !== generation) return;
|
|
10655
|
+
this.cacheWorkflow(workflowId, {
|
|
10656
|
+
state: document.state.global,
|
|
10657
|
+
name: document.header.name,
|
|
10658
|
+
revision: globalRevision(document)
|
|
10659
|
+
});
|
|
10660
|
+
}
|
|
10661
|
+
cacheWorkflow(workflowId, entry) {
|
|
10662
|
+
const held = this.workflowCache.get(workflowId);
|
|
10663
|
+
if (held && held.revision > entry.revision) return;
|
|
10664
|
+
this.workflowCache.delete(workflowId);
|
|
10665
|
+
this.workflowCache.set(workflowId, entry);
|
|
10666
|
+
if (this.workflowCache.size > WORKFLOW_CACHE_MAX) {
|
|
10667
|
+
const oldest = this.workflowCache.keys().next().value;
|
|
10668
|
+
if (oldest !== void 0) this.workflowCache.delete(oldest);
|
|
10669
|
+
}
|
|
10670
|
+
}
|
|
10631
10671
|
seenOpsQueue = [];
|
|
10632
10672
|
alreadySeen(key) {
|
|
10633
10673
|
if (this.seenOps.has(key)) return true;
|
|
@@ -10663,12 +10703,22 @@ var WorkflowRuntimeService = class {
|
|
|
10663
10703
|
context
|
|
10664
10704
|
});
|
|
10665
10705
|
if (this.alreadySeen(opKey)) continue;
|
|
10706
|
+
if (context.documentType === WORKFLOW_DOCUMENT_TYPE) this.workflowGenerations.set(context.documentId, this.workflowGeneration(context.documentId) + 1);
|
|
10666
10707
|
if (context.scope === DOCUMENT_SCOPE) {
|
|
10708
|
+
this.workflowCache.delete(context.documentId);
|
|
10667
10709
|
await this.matchDocumentLifecycle(operation, context, hints);
|
|
10668
10710
|
await this.forgetDeletedWorkflow(operation, context);
|
|
10669
10711
|
continue;
|
|
10670
10712
|
}
|
|
10671
10713
|
if (context.documentType === "powerhouse/workflow") {
|
|
10714
|
+
const cached = this.workflowCache.get(context.documentId);
|
|
10715
|
+
const carried = cached && operation.error === void 0 ? parseWorkflowState(operation.resultingState) : void 0;
|
|
10716
|
+
if (cached && carried) this.cacheWorkflow(context.documentId, {
|
|
10717
|
+
state: carried,
|
|
10718
|
+
name: cached.name,
|
|
10719
|
+
revision: operation.index + 1
|
|
10720
|
+
});
|
|
10721
|
+
else if (cached) this.workflowCache.delete(context.documentId);
|
|
10672
10722
|
const rechecked = await this.recheckReactorAccess(context.documentId, operation);
|
|
10673
10723
|
await this.refreshRegistration(context.documentId, operation.resultingState, !rechecked);
|
|
10674
10724
|
}
|
|
@@ -10778,7 +10828,7 @@ var WorkflowRuntimeService = class {
|
|
|
10778
10828
|
this.logger.info(`${kind} fired workflow ${workflowId}: ${run.status}`);
|
|
10779
10829
|
}, (error) => {
|
|
10780
10830
|
if (enqueuedRunId && isShutdownRefusal(error)) {
|
|
10781
|
-
this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left to
|
|
10831
|
+
this.logger.warn(`${kind} fire of workflow ${workflowId} was refused by the shutdown; run ${enqueuedRunId} is left for the next owner to start`);
|
|
10782
10832
|
return;
|
|
10783
10833
|
}
|
|
10784
10834
|
this.logger.error(`${kind} run failed for workflow ${workflowId}: @error`, truncateForLog(error instanceof Error ? error.message : String(error)));
|
|
@@ -10840,6 +10890,7 @@ var WorkflowRuntimeService = class {
|
|
|
10840
10890
|
if (context.documentType !== WORKFLOW_DOCUMENT_TYPE) return;
|
|
10841
10891
|
const workflowId = stringField(inputRecord(operation.action.input), "documentId") ?? context.documentId;
|
|
10842
10892
|
this.disarmDeleted(workflowId);
|
|
10893
|
+
this.workflowCache.delete(workflowId);
|
|
10843
10894
|
try {
|
|
10844
10895
|
await (await this.store())?.deleteDedupe(workflowId);
|
|
10845
10896
|
} catch (error) {
|
|
@@ -10941,6 +10992,8 @@ var WorkflowRuntimeService = class {
|
|
|
10941
10992
|
webhookUrlFor: async (workflowId) => (await this.mintWebhookEndpoint(workflowId))?.url,
|
|
10942
10993
|
reactorAccess: (binding, requireReactor, runUser) => this.triggerReactorAccess(binding, requireReactor, runUser),
|
|
10943
10994
|
models: this.models,
|
|
10995
|
+
worker: this.workers().session("trigger"),
|
|
10996
|
+
hookConcurrency: this.workers().triggerSlots,
|
|
10944
10997
|
telemetry: this.telemetry,
|
|
10945
10998
|
cacheDir: bundleCacheDir(),
|
|
10946
10999
|
resolver: pieceResolver(),
|
|
@@ -10960,8 +11013,29 @@ var WorkflowRuntimeService = class {
|
|
|
10960
11013
|
}
|
|
10961
11014
|
startTriggerSupervisor() {
|
|
10962
11015
|
this.supervisor().start();
|
|
11016
|
+
this.admitAbandonedRuns();
|
|
11017
|
+
this.pendingSweepTimer ??= setInterval(() => {
|
|
11018
|
+
this.admitAbandonedRuns(PENDING_SWEEP_INTERVAL_MS);
|
|
11019
|
+
}, PENDING_SWEEP_INTERVAL_MS);
|
|
11020
|
+
this.pendingSweepTimer.unref();
|
|
11021
|
+
}
|
|
11022
|
+
pendingSweepTimer;
|
|
11023
|
+
admittingAbandoned = false;
|
|
11024
|
+
async admitAbandonedRuns(minAgeMs) {
|
|
11025
|
+
if (this.admittingAbandoned || this.closed) return;
|
|
11026
|
+
this.admittingAbandoned = true;
|
|
11027
|
+
try {
|
|
11028
|
+
const admitted = await (await this.store())?.admitAbandonedRuns(minAgeMs === void 0 ? {} : { enqueuedBefore: new Date(Date.now() - minAgeMs) });
|
|
11029
|
+
for (const run of admitted ?? []) this.startFiring(run.workflowId, run.triggerPayload, run.triggerKind, run.runId);
|
|
11030
|
+
} catch (error) {
|
|
11031
|
+
this.logger.error("Could not admit the runs a stopped reactor journaled: @error", error);
|
|
11032
|
+
} finally {
|
|
11033
|
+
this.admittingAbandoned = false;
|
|
11034
|
+
}
|
|
10963
11035
|
}
|
|
10964
11036
|
stopTriggerSupervisor() {
|
|
11037
|
+
clearInterval(this.pendingSweepTimer);
|
|
11038
|
+
this.pendingSweepTimer = void 0;
|
|
10965
11039
|
this.triggerSupervisor?.stop();
|
|
10966
11040
|
}
|
|
10967
11041
|
closed = false;
|
|
@@ -10971,6 +11045,10 @@ var WorkflowRuntimeService = class {
|
|
|
10971
11045
|
shutdown() {
|
|
10972
11046
|
this.closed = true;
|
|
10973
11047
|
this.runGate.close();
|
|
11048
|
+
this.runDispatcher?.close();
|
|
11049
|
+
this.eventLoop.stop();
|
|
11050
|
+
this.unobserveDispatcher?.();
|
|
11051
|
+
this.unobserveDispatcher = void 0;
|
|
10974
11052
|
clearInterval(this.retentionTimer);
|
|
10975
11053
|
for (const { timer } of this.resolutionRetries.values()) clearTimeout(timer);
|
|
10976
11054
|
this.resolutionRetries.clear();
|
|
@@ -10979,7 +11057,6 @@ var WorkflowRuntimeService = class {
|
|
|
10979
11057
|
this.pieceWorkers?.dispose();
|
|
10980
11058
|
this.unobservePool?.();
|
|
10981
11059
|
this.unobservePool = void 0;
|
|
10982
|
-
this.designWorker?.dispose();
|
|
10983
11060
|
this.designWorker = void 0;
|
|
10984
11061
|
}
|
|
10985
11062
|
async triggerStates(ctx) {
|
|
@@ -11266,10 +11343,7 @@ var WorkflowRuntimeService = class {
|
|
|
11266
11343
|
descriptors = /* @__PURE__ */ new Map();
|
|
11267
11344
|
designWorker;
|
|
11268
11345
|
createDesignWorker() {
|
|
11269
|
-
return
|
|
11270
|
-
models: this.models,
|
|
11271
|
-
telemetry: this.telemetry
|
|
11272
|
-
});
|
|
11346
|
+
return this.workers().session("design");
|
|
11273
11347
|
}
|
|
11274
11348
|
designEgress = configuredEgress() ?? DEFAULT_EGRESS_POLICY;
|
|
11275
11349
|
describing = /* @__PURE__ */ new Map();
|
|
@@ -12068,17 +12142,51 @@ var WorkflowRuntimeService = class {
|
|
|
12068
12142
|
}
|
|
12069
12143
|
workers() {
|
|
12070
12144
|
if (this.pieceWorkers) return this.pieceWorkers;
|
|
12145
|
+
const env = (name) => Number(process.env[name]) || void 0;
|
|
12146
|
+
const count = (name) => {
|
|
12147
|
+
const value = Number(process.env[name] || NaN);
|
|
12148
|
+
return value >= 0 ? value : void 0;
|
|
12149
|
+
};
|
|
12071
12150
|
const pool = new PieceWorkerPool({
|
|
12072
12151
|
models: this.models,
|
|
12073
|
-
size:
|
|
12074
|
-
maxQueueDepth:
|
|
12152
|
+
size: env("PH_WORKFLOWS_RUN_CONCURRENCY"),
|
|
12153
|
+
maxQueueDepth: env("PH_WORKFLOWS_RUN_QUEUE_DEPTH"),
|
|
12154
|
+
heapMb: env("PH_WORKFLOWS_WORKER_MEMORY_MB"),
|
|
12155
|
+
maxRequests: env("PH_WORKFLOWS_WORKER_MAX_REQUESTS"),
|
|
12156
|
+
minWarm: count("PH_WORKFLOWS_WORKER_MIN"),
|
|
12157
|
+
triggerSlots: env("PH_WORKFLOWS_TRIGGER_SLOTS"),
|
|
12158
|
+
reactorThreads: this.host.reactorThreads,
|
|
12075
12159
|
telemetry: this.telemetry
|
|
12076
12160
|
});
|
|
12161
|
+
if (this.closed) {
|
|
12162
|
+
pool.dispose();
|
|
12163
|
+
return this.pieceWorkers = pool;
|
|
12164
|
+
}
|
|
12077
12165
|
this.unobservePool = this.telemetry.observeWorkerPool(() => pool.stats());
|
|
12078
12166
|
return this.pieceWorkers = pool;
|
|
12079
12167
|
}
|
|
12168
|
+
dispatcher() {
|
|
12169
|
+
if (this.runDispatcher) return this.runDispatcher;
|
|
12170
|
+
const configured = Number(process.env.PH_WORKFLOWS_ADMIT_MAX_ELU);
|
|
12171
|
+
const dispatcher = new RunDispatcher({
|
|
12172
|
+
maxInFlight: Number(process.env.PH_WORKFLOWS_RUNS_IN_FLIGHT) || 8 * this.workers().size,
|
|
12173
|
+
maxElu: configured > 0 && configured <= 1 ? configured : DEFAULT_ADMIT_MAX_ELU,
|
|
12174
|
+
loop: this.eventLoop
|
|
12175
|
+
});
|
|
12176
|
+
this.runDispatcher = dispatcher;
|
|
12177
|
+
if (this.closed) {
|
|
12178
|
+
dispatcher.close();
|
|
12179
|
+
return dispatcher;
|
|
12180
|
+
}
|
|
12181
|
+
this.eventLoop.start();
|
|
12182
|
+
this.unobserveDispatcher = this.telemetry.observeDispatcher(() => ({
|
|
12183
|
+
...dispatcher.stats(),
|
|
12184
|
+
utilization: this.eventLoop.utilization()
|
|
12185
|
+
}));
|
|
12186
|
+
return dispatcher;
|
|
12187
|
+
}
|
|
12080
12188
|
async fire(workflowId, triggerPayload, triggerKind = "manual", resume, ctx, enqueuedRunId) {
|
|
12081
|
-
if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId);
|
|
12189
|
+
if (this.closed) return this.refuseClosed(await this.store(), workflowId, enqueuedRunId, triggerKind);
|
|
12082
12190
|
const started = performance.now();
|
|
12083
12191
|
const attributes = {
|
|
12084
12192
|
"workflow.id": workflowId,
|
|
@@ -12113,18 +12221,29 @@ var WorkflowRuntimeService = class {
|
|
|
12113
12221
|
let documentName;
|
|
12114
12222
|
let definition;
|
|
12115
12223
|
let runUser;
|
|
12224
|
+
let readGeneration;
|
|
12116
12225
|
try {
|
|
12117
|
-
if (triggerKind === "manual") await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12226
|
+
if (triggerKind === "manual" && !enqueuedRunId) await this.assertCanReadDocument(workflowId, ctx).catch((error) => {
|
|
12118
12227
|
run.refused = true;
|
|
12119
12228
|
throw error;
|
|
12120
12229
|
});
|
|
12121
|
-
const
|
|
12122
|
-
if (
|
|
12123
|
-
|
|
12124
|
-
|
|
12230
|
+
const cached = triggerKind === "manual" || resume ? void 0 : this.workflowCache.get(workflowId);
|
|
12231
|
+
if (cached) {
|
|
12232
|
+
state = cached.state;
|
|
12233
|
+
documentName = cached.name;
|
|
12234
|
+
readGeneration = this.workflowGeneration(workflowId);
|
|
12235
|
+
} else {
|
|
12236
|
+
const generation = this.workflowGeneration(workflowId);
|
|
12237
|
+
readGeneration = generation;
|
|
12238
|
+
const document = await this.telemetry.phase("load", {}, () => this.host.reactorClient.get(workflowId));
|
|
12239
|
+
if (document.header.documentType !== "powerhouse/workflow") {
|
|
12240
|
+
run.refused = true;
|
|
12241
|
+
throw new Error(`Document "${workflowId}" is not a powerhouse/workflow`);
|
|
12242
|
+
}
|
|
12243
|
+
state = document.state.global;
|
|
12244
|
+
documentName = document.header.name;
|
|
12245
|
+
this.cacheLoaded(workflowId, generation, document);
|
|
12125
12246
|
}
|
|
12126
|
-
state = document.state.global;
|
|
12127
|
-
documentName = document.header.name;
|
|
12128
12247
|
if (state.status !== "ENABLED") {
|
|
12129
12248
|
run.refused = true;
|
|
12130
12249
|
throw new Error(`Workflow is ${state.status}; only ENABLED workflows can fire`);
|
|
@@ -12139,6 +12258,19 @@ var WorkflowRuntimeService = class {
|
|
|
12139
12258
|
if (enqueuedRunId) await store?.failRun(enqueuedRunId, error instanceof Error ? error.message : String(error), errorNameOf(error));
|
|
12140
12259
|
throw error;
|
|
12141
12260
|
}
|
|
12261
|
+
let journaling;
|
|
12262
|
+
const journalOnQueue = () => {
|
|
12263
|
+
if (triggerKind !== "manual" || enqueuedRunId || resume || journaling) return;
|
|
12264
|
+
if (!store) return;
|
|
12265
|
+
journaling = store.enqueueRun({
|
|
12266
|
+
workflowId,
|
|
12267
|
+
triggerKind,
|
|
12268
|
+
triggerPayload
|
|
12269
|
+
}).then((runId) => {
|
|
12270
|
+
enqueuedRunId = runId;
|
|
12271
|
+
});
|
|
12272
|
+
journaling.catch(() => void 0);
|
|
12273
|
+
};
|
|
12142
12274
|
const connections = declaredConnectionIds(definition);
|
|
12143
12275
|
const executor = this.blockExecutor(store);
|
|
12144
12276
|
const runnable = runnableDefinition(state);
|
|
@@ -12159,30 +12291,72 @@ var WorkflowRuntimeService = class {
|
|
|
12159
12291
|
throw error;
|
|
12160
12292
|
}
|
|
12161
12293
|
if (parked) return skipped(parked, "parked");
|
|
12162
|
-
const admission = await this.runGate.admit(workflowId, policy);
|
|
12294
|
+
const admission = await this.runGate.admit(workflowId, policy, journalOnQueue);
|
|
12295
|
+
try {
|
|
12296
|
+
await journaling;
|
|
12297
|
+
} catch (error) {
|
|
12298
|
+
if (admission.admitted) admission.release();
|
|
12299
|
+
throw error;
|
|
12300
|
+
}
|
|
12163
12301
|
if (this.closed) {
|
|
12164
12302
|
if (admission.admitted) admission.release();
|
|
12165
|
-
return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12303
|
+
return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
|
|
12166
12304
|
}
|
|
12167
12305
|
if (!admission.admitted) {
|
|
12168
|
-
if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId);
|
|
12306
|
+
if (admission.refusal === "closed") return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
|
|
12169
12307
|
return skipped(admission.reason, admission.refusal);
|
|
12170
12308
|
}
|
|
12171
|
-
|
|
12172
|
-
let stale;
|
|
12309
|
+
const staleAfter = async () => {
|
|
12173
12310
|
try {
|
|
12174
|
-
|
|
12311
|
+
return await this.staleAfterWait(workflowId, triggerKind, runnable.version, readGeneration);
|
|
12175
12312
|
} catch (error) {
|
|
12176
|
-
|
|
12313
|
+
return `Skipped: the workflow could not be read again after this firing waited for its slot: ${error instanceof Error ? error.message : String(error)}`;
|
|
12177
12314
|
}
|
|
12315
|
+
};
|
|
12316
|
+
if (admission.waited) {
|
|
12317
|
+
const stale = await staleAfter();
|
|
12178
12318
|
if (stale) {
|
|
12179
12319
|
admission.release();
|
|
12180
12320
|
return skipped(stale, "stale");
|
|
12181
12321
|
}
|
|
12182
12322
|
}
|
|
12323
|
+
const expired = () => skipped(`Skipped: this firing waited past its runTimeoutSeconds (${policy.runTimeoutSeconds}s) for a concurrency slot and was cancelled without running`, "expired");
|
|
12183
12324
|
if (deadline !== void 0 && Date.now() >= deadline) {
|
|
12184
12325
|
admission.release();
|
|
12185
|
-
return
|
|
12326
|
+
return expired();
|
|
12327
|
+
}
|
|
12328
|
+
let dispatched;
|
|
12329
|
+
let dispatcherWaited = false;
|
|
12330
|
+
try {
|
|
12331
|
+
dispatched = await this.dispatcher().admit(workflowId, () => {
|
|
12332
|
+
dispatcherWaited = true;
|
|
12333
|
+
journalOnQueue();
|
|
12334
|
+
});
|
|
12335
|
+
} catch (error) {
|
|
12336
|
+
await journaling?.catch(() => void 0);
|
|
12337
|
+
admission.release();
|
|
12338
|
+
if (this.closed) return this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind);
|
|
12339
|
+
throw error;
|
|
12340
|
+
}
|
|
12341
|
+
try {
|
|
12342
|
+
await journaling;
|
|
12343
|
+
} catch (error) {
|
|
12344
|
+
dispatched();
|
|
12345
|
+
admission.release();
|
|
12346
|
+
throw error;
|
|
12347
|
+
}
|
|
12348
|
+
if (this.closed || deadline !== void 0 && Date.now() >= deadline) {
|
|
12349
|
+
dispatched();
|
|
12350
|
+
admission.release();
|
|
12351
|
+
return this.closed ? this.refuseClosed(store, workflowId, enqueuedRunId, triggerKind) : expired();
|
|
12352
|
+
}
|
|
12353
|
+
if (dispatcherWaited) {
|
|
12354
|
+
const stale = await staleAfter();
|
|
12355
|
+
if (stale) {
|
|
12356
|
+
dispatched();
|
|
12357
|
+
admission.release();
|
|
12358
|
+
return skipped(stale, "stale");
|
|
12359
|
+
}
|
|
12186
12360
|
}
|
|
12187
12361
|
let runId = enqueuedRunId ?? null;
|
|
12188
12362
|
let journalFailed = false;
|
|
@@ -12207,6 +12381,7 @@ var WorkflowRuntimeService = class {
|
|
|
12207
12381
|
}) ?? null;
|
|
12208
12382
|
});
|
|
12209
12383
|
if (runId) run.span.setAttribute("run.id", runId);
|
|
12384
|
+
if (this.closed) throw new WorkflowRuntimeClosedError(workflowId);
|
|
12210
12385
|
session = this.workers().session();
|
|
12211
12386
|
const { variables, secretValues } = await resolveVariables(runnable.variables, this.secretProvider());
|
|
12212
12387
|
const journal = store;
|
|
@@ -12269,6 +12444,7 @@ var WorkflowRuntimeService = class {
|
|
|
12269
12444
|
} finally {
|
|
12270
12445
|
session?.close();
|
|
12271
12446
|
admission.release();
|
|
12447
|
+
dispatched();
|
|
12272
12448
|
}
|
|
12273
12449
|
}
|
|
12274
12450
|
runGate = new WorkflowRunGate();
|
|
@@ -12282,7 +12458,8 @@ var WorkflowRuntimeService = class {
|
|
|
12282
12458
|
return `Skipped: this workflow is PARKED (${park.reason})`;
|
|
12283
12459
|
}
|
|
12284
12460
|
/** Why a firing that waited for its slot may no longer run, if it may not. */
|
|
12285
|
-
async staleAfterWait(workflowId, triggerKind, publishedVersion) {
|
|
12461
|
+
async staleAfterWait(workflowId, triggerKind, publishedVersion, readGeneration) {
|
|
12462
|
+
if (this.workflowGeneration(workflowId) === readGeneration) return this.parkedFiring(workflowId, triggerKind, publishedVersion);
|
|
12286
12463
|
const state = (await this.host.reactorClient.get(workflowId)).state.global;
|
|
12287
12464
|
if (state.status !== "ENABLED") return `Skipped: the workflow became ${state.status} while this firing waited for its slot`;
|
|
12288
12465
|
if (runnableDefinition(state).version !== publishedVersion) return "Skipped: the workflow was re-published while this firing waited for its slot";
|
|
@@ -12314,10 +12491,18 @@ var WorkflowRuntimeService = class {
|
|
|
12314
12491
|
* of bug this work package exists to stamp out. An already-enqueued row is
|
|
12315
12492
|
* closed out in place, so nothing is left PENDING for a sweep to find.
|
|
12316
12493
|
*/
|
|
12317
|
-
async refuseClosed(store, workflowId, enqueuedRunId) {
|
|
12494
|
+
async refuseClosed(store, workflowId, enqueuedRunId, triggerKind) {
|
|
12318
12495
|
const error = new WorkflowRuntimeClosedError(workflowId);
|
|
12319
12496
|
if (enqueuedRunId) {
|
|
12320
12497
|
const redelivered = this.heldReplies.get(enqueuedRunId)?.() ?? false;
|
|
12498
|
+
if (!redelivered && triggerKind !== "manual") {
|
|
12499
|
+
try {
|
|
12500
|
+
await store?.releaseAdmission(enqueuedRunId);
|
|
12501
|
+
} catch (failure) {
|
|
12502
|
+
this.logger.warn(`Could not give back the boot admission of run ${enqueuedRunId}: @error`, failure);
|
|
12503
|
+
}
|
|
12504
|
+
throw error;
|
|
12505
|
+
}
|
|
12321
12506
|
try {
|
|
12322
12507
|
if (redelivered) await store?.cancelRun(enqueuedRunId, WEBHOOK_REDELIVERED_REASON);
|
|
12323
12508
|
else await store?.failRun(enqueuedRunId, error.message, errorNameOf(error));
|