@powerhousedao/switchboard 6.2.3-dev.50 → 6.2.3-dev.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.tsbuild/test/boot-workflow-singleton-conflict.test.d.ts +2 -0
- package/.tsbuild/test/boot-workflow-singleton-conflict.test.d.ts.map +1 -0
- package/.tsbuild/test/boot-workflow-singleton-retry.test.d.ts +2 -0
- package/.tsbuild/test/boot-workflow-singleton-retry.test.d.ts.map +1 -0
- package/.tsbuild/tsconfig.tsbuildinfo +1 -1
- package/CHANGELOG.md +32 -0
- package/dist/index.mjs +1 -1
- package/dist/{server-Zz7SqTwO.mjs → server-J9fpd42v.mjs} +262 -48
- package/dist/server-J9fpd42v.mjs.map +1 -0
- package/dist/server.d.mts +3 -2
- package/dist/server.d.mts.map +1 -1
- package/dist/server.mjs +3 -3
- package/package.json +15 -15
- package/test/boot-workflow-singleton-conflict.test.ts +107 -0
- package/test/boot-workflow-singleton-retry.test.ts +133 -0
- package/test/pglite-dialect.test.ts +58 -0
- package/test/workflow/resolvers.test.ts +35 -0
- package/test/workflow-reactor-enforcement.test.ts +3 -0
- package/test/workflow-read-gate.test.ts +3 -0
- package/test/workflow-runtime.test.ts +502 -7
- package/dist/server-Zz7SqTwO.mjs.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,35 @@
|
|
|
1
|
+
## 6.2.3-dev.51 (2026-10-10)
|
|
2
|
+
|
|
3
|
+
### 🚀 Features
|
|
4
|
+
|
|
5
|
+
- **ph-cli:** keep native and WebAssembly dependencies external and list them as externalDependencies ([e3b616e67b](https://github.com/powerhouse-inc/powerhouse/commit/e3b616e67b))
|
|
6
|
+
- **workflow:** polish the block picker after a visual review ([67bf5a96f7](https://github.com/powerhouse-inc/powerhouse/commit/67bf5a96f7))
|
|
7
|
+
- **workflow:** group block picker search by piece and add source tabs ([723d7a0660](https://github.com/powerhouse-inc/powerhouse/commit/723d7a0660))
|
|
8
|
+
- **reactor-router:** the routing IReactorClient over IRoutableBackend ([772aefe6ec](https://github.com/powerhouse-inc/powerhouse/commit/772aefe6ec))
|
|
9
|
+
- **switchboard:** report the composed workflow runtime through the facts sink ([da552e5aee](https://github.com/powerhouse-inc/powerhouse/commit/da552e5aee))
|
|
10
|
+
- **switchboard:** stop the workflow runtime when the singleton is lost ([3936e68b76](https://github.com/powerhouse-inc/powerhouse/commit/3936e68b76))
|
|
11
|
+
- **reactor-workflow:** claim the workflow singleton before composing ([98a51ff9ab](https://github.com/powerhouse-inc/powerhouse/commit/98a51ff9ab))
|
|
12
|
+
|
|
13
|
+
### 🩹 Fixes
|
|
14
|
+
|
|
15
|
+
- **workflow:** address block picker review findings ([1c69224191](https://github.com/powerhouse-inc/powerhouse/commit/1c69224191))
|
|
16
|
+
- **switchboard,connect:** declare the reactor store's storage facts ([28d9d364a9](https://github.com/powerhouse-inc/powerhouse/commit/28d9d364a9))
|
|
17
|
+
- **switchboard:** take the trigger read model off the coordinator with its runtime ([363772ed4e](https://github.com/powerhouse-inc/powerhouse/commit/363772ed4e))
|
|
18
|
+
- **switchboard:** mount the workflow GraphQL API a late claim registers ([92e76a7207](https://github.com/powerhouse-inc/powerhouse/commit/92e76a7207))
|
|
19
|
+
- **reactor-workflow:** tell a self-fence from a takeover, and restart on the former ([d237e1ac51](https://github.com/powerhouse-inc/powerhouse/commit/d237e1ac51))
|
|
20
|
+
- **switchboard:** retry the workflow singleton after a refused boot ([62b41a8023](https://github.com/powerhouse-inc/powerhouse/commit/62b41a8023))
|
|
21
|
+
- **switchboard:** stop the runtime a failed compose had already built ([fd4012b3d7](https://github.com/powerhouse-inc/powerhouse/commit/fd4012b3d7))
|
|
22
|
+
- **reactor-workflow:** fence a singleton holder that cannot renew ([bc4b81edca](https://github.com/powerhouse-inc/powerhouse/commit/bc4b81edca))
|
|
23
|
+
- **reactor-workflow:** never take the singleton from a live holder of the same name ([9f6750789d](https://github.com/powerhouse-inc/powerhouse/commit/9f6750789d))
|
|
24
|
+
- **reactor-workflow:** renew the singleton lease from the claim ([7ef58a3585](https://github.com/powerhouse-inc/powerhouse/commit/7ef58a3585))
|
|
25
|
+
- **switchboard:** boot without workflows when another process holds the singleton ([33f8dd3c53](https://github.com/powerhouse-inc/powerhouse/commit/33f8dd3c53))
|
|
26
|
+
|
|
27
|
+
### ❤️ Thank You
|
|
28
|
+
|
|
29
|
+
- acaldas
|
|
30
|
+
- Benjamin Jordan
|
|
31
|
+
- Claude Opus 5.5
|
|
32
|
+
|
|
1
33
|
## 6.2.3-dev.50 (2026-10-09)
|
|
2
34
|
|
|
3
35
|
### 🚀 Features
|
package/dist/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="d30347ec-a382-56a4-b3ef-bdacc9d409fd")}catch(e){}}();
|
|
4
|
-
import { a as StoredDocumentsRefusedError, c as initProfilerFromEnv, i as startSwitchboard, l as parseForcePgVersion } from "./server-
|
|
4
|
+
import { a as StoredDocumentsRefusedError, c as initProfilerFromEnv, i as startSwitchboard, l as parseForcePgVersion } from "./server-J9fpd42v.mjs";
|
|
5
5
|
import { metrics } from "@opentelemetry/api";
|
|
6
6
|
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
7
7
|
import { ExpressInstrumentation } from "@opentelemetry/instrumentation-express";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
|
-
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="
|
|
2
|
+
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="e3d937b6-2957-5fe0-8fc9-bf4fd0c18c93")}catch(e){}}();
|
|
3
3
|
import { n as addDefaultReactorDrive, r as isPostgresUrl, t as addDefaultDrive } from "./utils-uaBlzyjS.mjs";
|
|
4
4
|
import { createRequire, register } from "node:module";
|
|
5
5
|
import { metrics, trace } from "@opentelemetry/api";
|
|
@@ -11,7 +11,7 @@ import { getConfig } from "@powerhousedao/config/node";
|
|
|
11
11
|
import { existsSync, promises, realpathSync } from "node:fs";
|
|
12
12
|
import path, { resolve } from "node:path";
|
|
13
13
|
import { createDurableNodeFs, preparePgliteDataDir, removeStalePgliteFiles, resolvePgliteFsync, syncTree } from "@powerhousedao/reactor-api/pglite-node";
|
|
14
|
-
import { ChannelScheme, DriveCollectionId, EventBus, HardenedPGliteDialect, REACTOR_SCHEMA, ReactorBuilder, ReactorClientBuilder, UnsupportedStoredProtocolError, createHybridProjectionCoordinatorFactory, instrumentPgPool, parseDriveUrl, supportsLiveReadModelRegistration } from "@powerhousedao/reactor";
|
|
14
|
+
import { ChannelScheme, DriveCollectionId, EventBus, HardenedPGliteDialect, IN_MEMORY_PGLITE_STORAGE_FACTS, PGLITE_PATH_STORAGE_FACTS, POSTGRES_STORAGE_FACTS, REACTOR_SCHEMA, ReactorBuilder, ReactorClientBuilder, UnsupportedStoredProtocolError, createHybridProjectionCoordinatorFactory, instrumentPgPool, parseDriveUrl, supportsLiveReadModelRegistration } from "@powerhousedao/reactor";
|
|
15
15
|
import { AuthorizationPolicy, BaseSubgraph, ForbiddenError, HttpPackageLoader, ImportPackageLoader, PGLITE_UTC_PARSERS, PackageManagementService, PackagesSubgraph, PrivacyPermissionAdapter, callerSubject, createCanonicalDocumentIdResolver, getUniqueDocumentModels, initializeAndStartAPI, resolveRenownConfig } from "@powerhousedao/reactor-api";
|
|
16
16
|
import { httpsHooksPath } from "@powerhousedao/reactor-api/https-hooks";
|
|
17
17
|
import { AttachmentAlreadyExists, AttachmentNotFound, AttachmentPending, AttachmentReferenceReadModel, AttachmentSchemaCompiler, HashMismatch, InvalidAttachmentMetadata, InvalidAttachmentRef, ReservationNotFound, SizeMismatch, UploadTooLarge, createRef, createRemoteAttachmentService, parseAttachmentDownloadTarget, parseRef, validateReserveMetadata } from "@powerhousedao/reactor-attachments";
|
|
@@ -1363,6 +1363,14 @@ function callbackUrlOf(http) {
|
|
|
1363
1363
|
}
|
|
1364
1364
|
//#endregion
|
|
1365
1365
|
//#region src/workflow/resolvers.ts
|
|
1366
|
+
const PIECE_SOURCES = [
|
|
1367
|
+
"local",
|
|
1368
|
+
"registry",
|
|
1369
|
+
"activepieces"
|
|
1370
|
+
];
|
|
1371
|
+
function isPieceSource(value) {
|
|
1372
|
+
return PIECE_SOURCES.includes(value);
|
|
1373
|
+
}
|
|
1366
1374
|
function blockRef(input) {
|
|
1367
1375
|
if (input.kind !== "action" && input.kind !== "trigger") throw new GraphQLError(`A block's kind is "action" or "trigger", got "${input.kind}"`);
|
|
1368
1376
|
return {
|
|
@@ -1516,7 +1524,17 @@ const getResolvers = (runtime, authorizationService, oauth, access = UNKNOWN_ACC
|
|
|
1516
1524
|
stepOutputTree: (_parent, args, ctx) => runtime.stepOutputTree(args.workflowId, args.stepId, ctx),
|
|
1517
1525
|
blockResolutions: (_parent, args, ctx) => runtime.blockResolutions(args.workflowId, ctx),
|
|
1518
1526
|
pieceDetail: (_parent, args) => runtime.pieceDetail(args.packageName, args.version ?? void 0),
|
|
1519
|
-
|
|
1527
|
+
searchPieces: (_parent, args) => {
|
|
1528
|
+
if (args.kind !== "action" && args.kind !== "trigger") throw new GraphQLError(`A search's kind is "action" or "trigger", got "${args.kind}"`);
|
|
1529
|
+
const unknown = (args.sources ?? []).filter((source) => !isPieceSource(source));
|
|
1530
|
+
if (unknown.length > 0) throw new GraphQLError(`A search's sources are ${PIECE_SOURCES.join(", ")}, got ${unknown.join(", ")}`);
|
|
1531
|
+
return runtime.searchPieces(args.query, {
|
|
1532
|
+
kind: args.kind,
|
|
1533
|
+
sources: (args.sources ?? []).filter(isPieceSource),
|
|
1534
|
+
categories: args.categories ?? void 0,
|
|
1535
|
+
limit: args.limit ?? void 0
|
|
1536
|
+
});
|
|
1537
|
+
},
|
|
1520
1538
|
connections: (_parent, _args, ctx) => runtime.connections(ctx),
|
|
1521
1539
|
webhookEndpoint: (_parent, args, ctx) => syncAware(() => runtime.webhookEndpoint(args.workflowId, ctx, { driveId: args.driveId ?? void 0 })),
|
|
1522
1540
|
secret: async (_parent, args) => {
|
|
@@ -1714,10 +1732,20 @@ const schema = gql`
|
|
|
1714
1732
|
version: String
|
|
1715
1733
|
): Unknown
|
|
1716
1734
|
"""
|
|
1717
|
-
|
|
1718
|
-
first use; poll while status is "indexing".
|
|
1719
|
-
"""
|
|
1720
|
-
|
|
1735
|
+
Piece and block search across the catalog, grouped by piece. The index
|
|
1736
|
+
builds lazily on first use; poll while status is "indexing".
|
|
1737
|
+
"""
|
|
1738
|
+
searchPieces(
|
|
1739
|
+
query: String!
|
|
1740
|
+
"action | trigger"
|
|
1741
|
+
kind: String!
|
|
1742
|
+
"local | registry | activepieces; all when omitted."
|
|
1743
|
+
sources: [String!]
|
|
1744
|
+
"Category ids, any of; all when omitted."
|
|
1745
|
+
categories: [String!]
|
|
1746
|
+
"Pieces returned."
|
|
1747
|
+
limit: Int
|
|
1748
|
+
): PieceSearchResult!
|
|
1721
1749
|
"""
|
|
1722
1750
|
Health of every registered piece trigger (poll schedule, errors).
|
|
1723
1751
|
"""
|
|
@@ -1834,10 +1862,28 @@ const schema = gql`
|
|
|
1834
1862
|
latestVersion: String
|
|
1835
1863
|
}
|
|
1836
1864
|
|
|
1837
|
-
type
|
|
1865
|
+
type PieceSearchMatch {
|
|
1866
|
+
pieceName: String!
|
|
1867
|
+
pieceVersion: String!
|
|
1868
|
+
displayName: String!
|
|
1869
|
+
description: String!
|
|
1870
|
+
logoUrl: String!
|
|
1871
|
+
categories: [String!]!
|
|
1872
|
+
"local | registry | activepieces"
|
|
1873
|
+
source: String!
|
|
1874
|
+
deprecated: Boolean
|
|
1875
|
+
"Why no block of the piece can run on this reactor."
|
|
1876
|
+
unsupported: String
|
|
1877
|
+
"Every query token matched the piece's own name."
|
|
1878
|
+
namedPiece: Boolean!
|
|
1879
|
+
"Matching blocks of the kind asked for, best first."
|
|
1880
|
+
blocks: [BlockSearchHit!]!
|
|
1881
|
+
}
|
|
1882
|
+
|
|
1883
|
+
type PieceSearchResult {
|
|
1838
1884
|
"ready | indexing | error"
|
|
1839
1885
|
status: String!
|
|
1840
|
-
|
|
1886
|
+
pieces: [PieceSearchMatch!]!
|
|
1841
1887
|
indexedPieces: Int!
|
|
1842
1888
|
error: String
|
|
1843
1889
|
}
|
|
@@ -2111,6 +2157,10 @@ const WORKFLOW_PACKAGE_NAME = "@powerhousedao/workflow";
|
|
|
2111
2157
|
const WORKFLOW_TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
|
|
2112
2158
|
/** The env var and OpenFeature flag key that turns workflows on. */
|
|
2113
2159
|
const PH_WORKFLOWS_ENABLED = "PH_WORKFLOWS_ENABLED";
|
|
2160
|
+
const SINGLETON_LOST = {
|
|
2161
|
+
status: "unavailable",
|
|
2162
|
+
reason: "workflow-singleton-lost"
|
|
2163
|
+
};
|
|
2114
2164
|
/** Precedence, unchanged from the reactor-api resolver this replaces: the
|
|
2115
2165
|
* host's option, then PH_WORKFLOWS_ENABLED, then the config file, then off. */
|
|
2116
2166
|
async function resolveWorkflowsEnabled({ featureFlags, override, configEnabled = false, env = process.env }) {
|
|
@@ -2127,6 +2177,88 @@ async function assertWorkflowPackageLoadable(load = () => import("@powerhousedao
|
|
|
2127
2177
|
throw new Error(`Workflows are enabled but ${WORKFLOW_PACKAGE_NAME} could not be loaded`, { cause: error });
|
|
2128
2178
|
}
|
|
2129
2179
|
}
|
|
2180
|
+
/**
|
|
2181
|
+
* Whether a composition failure was the workflow singleton refusing this
|
|
2182
|
+
* process, rather than something that should take the boot down.
|
|
2183
|
+
*
|
|
2184
|
+
* Matched by NAME: the engine loads lazily, and importing it here for the
|
|
2185
|
+
* constructor would defeat that. The fields the engine's error carries are
|
|
2186
|
+
* typed optional for the same reason.
|
|
2187
|
+
*/
|
|
2188
|
+
function isWorkflowSingletonConflict(error) {
|
|
2189
|
+
return error instanceof Error && error.name === "WorkflowSingletonConflictError";
|
|
2190
|
+
}
|
|
2191
|
+
/**
|
|
2192
|
+
* Retries the singleton claim for a host that booted without it, and hands
|
|
2193
|
+
* the runtime over once a claim succeeds.
|
|
2194
|
+
*
|
|
2195
|
+
* Without it a rolling deploy whose slots have different owner names ends
|
|
2196
|
+
* with nobody running workflows: the new pod is refused while the old one
|
|
2197
|
+
* still holds the lease, and nothing retries once the old pod releases it.
|
|
2198
|
+
* Only for a host that never composed; one that lost the lease after
|
|
2199
|
+
* composing stays without workflows until it restarts.
|
|
2200
|
+
*/
|
|
2201
|
+
function retryWorkflowSingleton(options) {
|
|
2202
|
+
const intervalMs = options.intervalMs ?? 2e4;
|
|
2203
|
+
let stopped = false;
|
|
2204
|
+
let timer;
|
|
2205
|
+
let attempt;
|
|
2206
|
+
const run = async () => {
|
|
2207
|
+
let workflows;
|
|
2208
|
+
try {
|
|
2209
|
+
workflows = await options.compose();
|
|
2210
|
+
} catch (error) {
|
|
2211
|
+
if (!isWorkflowSingletonConflict(error)) options.logger.error("Composing the workflow runtime after the singleton came free failed: @error", error);
|
|
2212
|
+
schedule();
|
|
2213
|
+
return;
|
|
2214
|
+
}
|
|
2215
|
+
if (stopped) {
|
|
2216
|
+
await workflows.stop();
|
|
2217
|
+
return;
|
|
2218
|
+
}
|
|
2219
|
+
options.logger.info("Claimed the workflow singleton after boot; starting the workflow runtime");
|
|
2220
|
+
await options.onComposed(workflows);
|
|
2221
|
+
};
|
|
2222
|
+
const schedule = () => {
|
|
2223
|
+
if (stopped) return;
|
|
2224
|
+
timer = setTimeout(() => {
|
|
2225
|
+
timer = void 0;
|
|
2226
|
+
attempt = run().finally(() => {
|
|
2227
|
+
attempt = void 0;
|
|
2228
|
+
});
|
|
2229
|
+
}, intervalMs);
|
|
2230
|
+
timer.unref();
|
|
2231
|
+
};
|
|
2232
|
+
schedule();
|
|
2233
|
+
return { async stop() {
|
|
2234
|
+
stopped = true;
|
|
2235
|
+
if (timer) clearTimeout(timer);
|
|
2236
|
+
timer = void 0;
|
|
2237
|
+
await attempt?.catch(() => void 0);
|
|
2238
|
+
} };
|
|
2239
|
+
}
|
|
2240
|
+
/**
|
|
2241
|
+
* What a host does once its composed runtime has shut down for losing the
|
|
2242
|
+
* workflow singleton.
|
|
2243
|
+
*
|
|
2244
|
+
* Taken over by another claim: that process runs workflows now, so this one
|
|
2245
|
+
* stays without them until it restarts. Unrenewable with no takeover seen (a
|
|
2246
|
+
* database blip): nothing else runs them, so the process goes down through
|
|
2247
|
+
* the fatal shutdown and its supervisor restarts it, which re-claims at boot.
|
|
2248
|
+
* A host with no fatal shutdown installed (embedded) stays without workflows
|
|
2249
|
+
* and says a restart is needed.
|
|
2250
|
+
*/
|
|
2251
|
+
function workflowSingletonLossHandler(logger, fatal = triggerFatalShutdown) {
|
|
2252
|
+
return (loss) => {
|
|
2253
|
+
if (loss.reason === "taken") {
|
|
2254
|
+
logger.error(`Another process ("${loss.heldBy ?? "unknown"}") took the workflow singleton. This Switchboard has stopped its workflow runtime and runs no workflows until it is restarted; everything else serves normally.`);
|
|
2255
|
+
return;
|
|
2256
|
+
}
|
|
2257
|
+
const error = /* @__PURE__ */ new Error(`The workflow singleton lease could not be renewed for ${Math.round(loss.silentMs)}ms; no other process was seen taking it over`);
|
|
2258
|
+
if (fatal("Workflow singleton lease unrenewable", error)) return;
|
|
2259
|
+
logger.error(`${error.message}. This Switchboard has stopped its workflow runtime and runs no workflows until it is restarted.`);
|
|
2260
|
+
};
|
|
2261
|
+
}
|
|
2130
2262
|
function modelManifestSource(builder) {
|
|
2131
2263
|
return {
|
|
2132
2264
|
modelManifest: () => builder.getResolvedModelManifest() ?? [],
|
|
@@ -2235,7 +2367,7 @@ function attachmentReadPolicy(access, retryMs = REFERENCE_INDEX_RETRY_MS) {
|
|
|
2235
2367
|
return false;
|
|
2236
2368
|
};
|
|
2237
2369
|
}
|
|
2238
|
-
async function registerWorkflowTriggersReadModel(engine, runtime, clientModule) {
|
|
2370
|
+
async function registerWorkflowTriggersReadModel(engine, runtime, clientModule, registration) {
|
|
2239
2371
|
const reactorModule = clientModule?.reactorModule;
|
|
2240
2372
|
if (!reactorModule) return {
|
|
2241
2373
|
status: "unavailable",
|
|
@@ -2248,7 +2380,9 @@ async function registerWorkflowTriggersReadModel(engine, runtime, clientModule)
|
|
|
2248
2380
|
};
|
|
2249
2381
|
const readModel = new engine.WorkflowTriggersReadModel(reactorModule.database.withSchema(REACTOR_SCHEMA), reactorModule.operationIndex, reactorModule.writeCache, reactorModule.processorManagerConsistencyTracker, runtime);
|
|
2250
2382
|
await readModel.init();
|
|
2383
|
+
if (registration.tornDown) return SINGLETON_LOST;
|
|
2251
2384
|
coordinator.addReadModel(readModel, engine.WORKFLOW_TRIGGERS_READ_MODEL_STAGE);
|
|
2385
|
+
registration.detach = () => coordinator.removeReadModel(readModel);
|
|
2252
2386
|
return { status: "available" };
|
|
2253
2387
|
}
|
|
2254
2388
|
function bindPackagePieces(registry, source) {
|
|
@@ -2266,8 +2400,37 @@ async function composeWorkflowRuntime(deps) {
|
|
|
2266
2400
|
} catch (error) {
|
|
2267
2401
|
throw new Error("Workflows are enabled but @powerhousedao/reactor-workflow could not be loaded", { cause: error });
|
|
2268
2402
|
}
|
|
2403
|
+
const loss = { lost: false };
|
|
2404
|
+
const lease = deps.singletonLease === false ? void 0 : await engine.acquireWorkflowSingletonLease({
|
|
2405
|
+
relationalDb: deps.relationalDb,
|
|
2406
|
+
logger: deps.logger,
|
|
2407
|
+
...deps.storageId ? { storageId: deps.storageId } : {},
|
|
2408
|
+
selfFence: deps.exclusiveJournal !== true,
|
|
2409
|
+
onLost: (lost) => {
|
|
2410
|
+
loss.lost = true;
|
|
2411
|
+
if (lost.reason === "taken") loss.heldBy = lost.heldBy;
|
|
2412
|
+
loss.tearDown?.();
|
|
2413
|
+
if (loss.composed) deps.onSingletonLost?.(lost);
|
|
2414
|
+
}
|
|
2415
|
+
});
|
|
2416
|
+
try {
|
|
2417
|
+
const composed = await composeClaimed(engine, deps, lease, loss);
|
|
2418
|
+
if (loss.lost) throw singletonLostError(engine, loss, lease);
|
|
2419
|
+
loss.composed = true;
|
|
2420
|
+
return composed;
|
|
2421
|
+
} catch (error) {
|
|
2422
|
+
loss.tearDown?.();
|
|
2423
|
+
await lease?.release();
|
|
2424
|
+
throw error;
|
|
2425
|
+
}
|
|
2426
|
+
}
|
|
2427
|
+
function singletonLostError(engine, loss, lease) {
|
|
2428
|
+
return new engine.WorkflowSingletonConflictError(loss.heldBy ?? "unknown", "unknown", lease?.owner ?? "unknown");
|
|
2429
|
+
}
|
|
2430
|
+
async function composeClaimed(engine, deps, lease, loss) {
|
|
2269
2431
|
engine.setPieceRegistryUrl(deps.pieceRegistryUrl);
|
|
2270
2432
|
if (deps.pieces) bindPackagePieces(engine.packagePieces, deps.pieces);
|
|
2433
|
+
if (loss.lost) throw singletonLostError(engine, loss, lease);
|
|
2271
2434
|
const access = reactorAccessOf(deps.clientModule);
|
|
2272
2435
|
const models = deps.models;
|
|
2273
2436
|
const runtime = engine.createWorkflowRuntime({
|
|
@@ -2290,24 +2453,40 @@ async function composeWorkflowRuntime(deps) {
|
|
|
2290
2453
|
logger: deps.logger,
|
|
2291
2454
|
...deps.telemetry ? { telemetry: deps.telemetry } : {}
|
|
2292
2455
|
});
|
|
2293
|
-
const
|
|
2456
|
+
const routes = {};
|
|
2457
|
+
const registration = { tornDown: false };
|
|
2458
|
+
const tearDown = () => {
|
|
2459
|
+
if (registration.tornDown) return;
|
|
2460
|
+
registration.tornDown = true;
|
|
2461
|
+
routes.oauthCallback?.dispose();
|
|
2462
|
+
runtime.shutdown();
|
|
2463
|
+
registration.detach?.();
|
|
2464
|
+
};
|
|
2465
|
+
loss.tearDown = tearDown;
|
|
2466
|
+
const triggers = await registerWorkflowTriggersReadModel(engine, runtime, deps.clientModule, registration);
|
|
2467
|
+
if (registration.tornDown) throw singletonLostError(engine, loss, lease);
|
|
2294
2468
|
if (triggers.status === "available") deps.logger.info(`Workflow trigger read model registered (${engine.WORKFLOW_TRIGGERS_READ_MODEL}, ${engine.WORKFLOW_TRIGGERS_READ_MODEL_STAGE})`);
|
|
2295
2469
|
else deps.logger.error("Workflow document triggers are NOT armed (@reason): this reactor's read-model coordinator takes no live registration, so no document operation reaches the runtime. Webhook and schedule triggers are unaffected.", triggers.reason);
|
|
2296
2470
|
let stopped = false;
|
|
2297
|
-
|
|
2471
|
+
if (deps.http) routes.oauthCallback = registerOAuthCallback(deps.http, runtime);
|
|
2298
2472
|
return {
|
|
2299
2473
|
subgraph: createWorkflowRuntimeSubgraph(runtime, deps.http ? { callbackUrl: callbackUrlOf(deps.http) } : void 0, access),
|
|
2300
|
-
triggers
|
|
2474
|
+
get triggers() {
|
|
2475
|
+
return loss.lost ? SINGLETON_LOST : triggers;
|
|
2476
|
+
},
|
|
2477
|
+
...lease ? { singletonOwner: lease.owner } : {},
|
|
2301
2478
|
async start() {
|
|
2479
|
+
if (registration.tornDown) return;
|
|
2302
2480
|
await runtime.registerWebhookEndpoint();
|
|
2481
|
+
if (loss.lost || stopped) return;
|
|
2303
2482
|
runtime.startTriggerSupervisor();
|
|
2483
|
+
runtime.warmPieceSearch();
|
|
2304
2484
|
},
|
|
2305
|
-
stop() {
|
|
2306
|
-
if (stopped) return
|
|
2485
|
+
async stop() {
|
|
2486
|
+
if (stopped) return;
|
|
2307
2487
|
stopped = true;
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
return Promise.resolve();
|
|
2488
|
+
tearDown();
|
|
2489
|
+
await lease?.release();
|
|
2311
2490
|
}
|
|
2312
2491
|
};
|
|
2313
2492
|
}
|
|
@@ -2785,7 +2964,8 @@ async function createReactorKysely(opts) {
|
|
|
2785
2964
|
logger.info(`Using PostgreSQL for reactor storage (host pool max ${poolSize})`);
|
|
2786
2965
|
return {
|
|
2787
2966
|
kysely: new Kysely({ dialect: new PostgresDialect({ pool }) }),
|
|
2788
|
-
poolInstrumentation
|
|
2967
|
+
poolInstrumentation,
|
|
2968
|
+
storageFacts: POSTGRES_STORAGE_FACTS
|
|
2789
2969
|
};
|
|
2790
2970
|
}
|
|
2791
2971
|
if (!reactorPgliteDir || reactorPgliteMajor === null) throw new Error("Reactor PGLite directory not resolved");
|
|
@@ -2794,7 +2974,8 @@ async function createReactorKysely(opts) {
|
|
|
2794
2974
|
logger.info(inMemory ? `Using in-memory PGlite (PG${reactorPgliteMajor}) for reactor storage [PH_PGLITE_IN_MEMORY=1]` : `Using PGlite (PG${reactorPgliteMajor}) for reactor storage at ${reactorPgliteDir}`);
|
|
2795
2975
|
return {
|
|
2796
2976
|
kysely: new Kysely({ dialect: new ClosablePGliteDialect(pglite, reactorPgliteDialectOptions(logger)) }),
|
|
2797
|
-
poolInstrumentation: void 0
|
|
2977
|
+
poolInstrumentation: void 0,
|
|
2978
|
+
storageFacts: inMemory ? IN_MEMORY_PGLITE_STORAGE_FACTS : PGLITE_PATH_STORAGE_FACTS
|
|
2798
2979
|
};
|
|
2799
2980
|
}
|
|
2800
2981
|
/** Derive the remote attachment service config for switchboard's own `/attachments/*` API. */
|
|
@@ -2906,7 +3087,7 @@ async function initServer(serverPort, options, renown, renownConfig) {
|
|
|
2906
3087
|
attachmentReferenceProjection
|
|
2907
3088
|
};
|
|
2908
3089
|
}
|
|
2909
|
-
const { kysely: baseKysely, poolInstrumentation } = await createReactorKysely({
|
|
3090
|
+
const { kysely: baseKysely, poolInstrumentation, storageFacts } = await createReactorKysely({
|
|
2910
3091
|
reactorDbUrl,
|
|
2911
3092
|
reactorPgliteDir,
|
|
2912
3093
|
reactorPgliteMajor,
|
|
@@ -2919,7 +3100,7 @@ async function initServer(serverPort, options, renown, renownConfig) {
|
|
|
2919
3100
|
if (hasSkipThreshold) logger.info(`Reactor maxSkipThreshold set to ${maxSkipThreshold}`);
|
|
2920
3101
|
const { flags: reactorFeatureFlags, enabled: enabledFeatureFlags } = resolveReactorFeatureFlags(process.env);
|
|
2921
3102
|
if (enabledFeatureFlags.length > 0) logger.info(`Reactor feature flags enabled: ${enabledFeatureFlags.join(", ")}`);
|
|
2922
|
-
const reactorBuilder = new ReactorBuilder().withEventBus(new EventBus()).withKysely(baseKysely).withFeatures({ legacyProcessorIds: process.env.REACTOR_LEGACY_PROCESSOR_IDS !== "false" }).withUnsupportedStoredDocuments(unsupportedStoredDocuments);
|
|
3103
|
+
const reactorBuilder = new ReactorBuilder().withEventBus(new EventBus()).withKysely(baseKysely).withStorageFacts(storageFacts).withFeatures({ legacyProcessorIds: process.env.REACTOR_LEGACY_PROCESSOR_IDS !== "false" }).withUnsupportedStoredDocuments(unsupportedStoredDocuments);
|
|
2923
3104
|
if (poolInstrumentation) reactorBuilder.withInstrumentedPool(poolInstrumentation);
|
|
2924
3105
|
const clientBuilder = new ReactorClientBuilder().withReactorBuilder(reactorBuilder);
|
|
2925
3106
|
const vetraDocumentModels = dev ? Object.values(await import("@powerhousedao/vetra/document-models")).filter((m) => typeof m === "object" && m !== null && "documentModel" in m && "reducer" in m) : [];
|
|
@@ -3102,27 +3283,31 @@ async function initServer(serverPort, options, renown, renownConfig) {
|
|
|
3102
3283
|
const { client, graphqlManager, documentModelRegistry } = api;
|
|
3103
3284
|
const lateSubgraphs = [];
|
|
3104
3285
|
let workflows;
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3286
|
+
let workflowsRetry;
|
|
3287
|
+
const composeWorkflows = () => composeWorkflowRuntime({
|
|
3288
|
+
reactorClient: client,
|
|
3289
|
+
clientModule: options.reactor ?? ownedReactorModule,
|
|
3290
|
+
relationalDb: api.relationalDb,
|
|
3291
|
+
secretsKeyFile: readModelPgliteDir === null ? false : void 0,
|
|
3292
|
+
storageId: readModelPgliteDir === null ? readModelPath : path$1.resolve(readModelPath),
|
|
3293
|
+
attachments: createAttachmentClient(api.attachments.service),
|
|
3294
|
+
attachmentAccess: api.attachmentAccess,
|
|
3295
|
+
webhooks: api.httpRoutes.scopeFor(WORKFLOW_PACKAGE_NAME).webhooks,
|
|
3296
|
+
http: api.httpRoutes.scopeFor(WORKFLOW_PACKAGE_NAME),
|
|
3297
|
+
authorizationService: api.authorizationService,
|
|
3298
|
+
pieces: api.packageManager,
|
|
3299
|
+
pieceRegistryUrl: registryUrl,
|
|
3300
|
+
models: workerModels,
|
|
3301
|
+
logger: logger.child(["workflow-runtime"]),
|
|
3302
|
+
exclusiveJournal: readModelPgliteDir !== null,
|
|
3303
|
+
onSingletonLost: workflowSingletonLossHandler(logger),
|
|
3304
|
+
telemetry: {
|
|
3305
|
+
tracer: trace.getTracer(WORKFLOW_TELEMETRY_SCOPE),
|
|
3306
|
+
meter: metrics.getMeter(WORKFLOW_TELEMETRY_SCOPE)
|
|
3307
|
+
}
|
|
3308
|
+
});
|
|
3309
|
+
const registerWorkflowSubgraph = (composed) => {
|
|
3310
|
+
const WorkflowRuntimeSubgraph = composed.subgraph;
|
|
3126
3311
|
const workflowSubgraph = new WorkflowRuntimeSubgraph({
|
|
3127
3312
|
reactorClient: client,
|
|
3128
3313
|
http: graphqlManager.scopeForPackage(WORKFLOW_PACKAGE_NAME),
|
|
@@ -3134,11 +3319,36 @@ async function initServer(serverPort, options, renown, renownConfig) {
|
|
|
3134
3319
|
path: graphqlManager.getBasePath(),
|
|
3135
3320
|
attachments: api.attachmentClientProvider
|
|
3136
3321
|
});
|
|
3137
|
-
|
|
3322
|
+
return graphqlManager.registerSubgraphInstance(workflowSubgraph, "graphql", false).catch((error) => {
|
|
3138
3323
|
logger.error("Failed to register workflow-runtime subgraph: @error", error);
|
|
3139
|
-
})
|
|
3324
|
+
});
|
|
3325
|
+
};
|
|
3326
|
+
if (workflowsEnabled) try {
|
|
3327
|
+
workflows = await composeWorkflows();
|
|
3328
|
+
} catch (error) {
|
|
3329
|
+
if (!isWorkflowSingletonConflict(error)) throw error;
|
|
3330
|
+
logger.warn(`Workflows are enabled but another live process ("${error.owner ?? "unknown"}") holds the workflow singleton until ${error.expiresAt ?? "unknown"}. This Switchboard has booted WITHOUT the workflow runtime: no trigger of any kind fires here, and the workflow GraphQL face is absent. Everything else serves normally. Stop the other owner, or set PH_WORKFLOWS_SINGLETON_OWNER to the same stable name on the slot that owns workflows. This Switchboard retries the claim and starts workflows once the lease comes free.`);
|
|
3331
|
+
workflowsRetry = retryWorkflowSingleton({
|
|
3332
|
+
compose: composeWorkflows,
|
|
3333
|
+
onComposed: async (composed) => {
|
|
3334
|
+
workflows = composed;
|
|
3335
|
+
await registerWorkflowSubgraph(composed);
|
|
3336
|
+
try {
|
|
3337
|
+
await graphqlManager.updateRouter();
|
|
3338
|
+
} catch (error) {
|
|
3339
|
+
logger.error("Mounting the late workflow-runtime subgraph failed: @error", error);
|
|
3340
|
+
}
|
|
3341
|
+
await composed.start();
|
|
3342
|
+
logger.info("Workflow runtime started");
|
|
3343
|
+
},
|
|
3344
|
+
logger
|
|
3345
|
+
});
|
|
3346
|
+
}
|
|
3347
|
+
if (workflows) {
|
|
3348
|
+
lateSubgraphs.push(registerWorkflowSubgraph(workflows));
|
|
3140
3349
|
await workflows.start();
|
|
3141
3350
|
logger.info("Workflow runtime started");
|
|
3351
|
+
api.inspection?.facts.setWorkflows(true);
|
|
3142
3352
|
}
|
|
3143
3353
|
let privacy;
|
|
3144
3354
|
if (privacyConfig.enabled) {
|
|
@@ -3155,6 +3365,7 @@ async function initServer(serverPort, options, renown, renownConfig) {
|
|
|
3155
3365
|
logger: logger.child(["privacy"])
|
|
3156
3366
|
});
|
|
3157
3367
|
} catch (error) {
|
|
3368
|
+
await workflowsRetry?.stop();
|
|
3158
3369
|
await workflows?.stop();
|
|
3159
3370
|
await abortBoot(api);
|
|
3160
3371
|
throw error;
|
|
@@ -3162,6 +3373,7 @@ async function initServer(serverPort, options, renown, renownConfig) {
|
|
|
3162
3373
|
}
|
|
3163
3374
|
const shutdown = async () => {
|
|
3164
3375
|
await privacy?.stop();
|
|
3376
|
+
await workflowsRetry?.stop();
|
|
3165
3377
|
await workflows?.stop();
|
|
3166
3378
|
await api.dispose();
|
|
3167
3379
|
};
|
|
@@ -3255,7 +3467,9 @@ async function initServer(serverPort, options, renown, renownConfig) {
|
|
|
3255
3467
|
reactor: client,
|
|
3256
3468
|
attachmentService,
|
|
3257
3469
|
attachmentReferenceProjection: api.attachmentReferenceProjection,
|
|
3258
|
-
workflowTriggers
|
|
3470
|
+
get workflowTriggers() {
|
|
3471
|
+
return workflows?.triggers;
|
|
3472
|
+
},
|
|
3259
3473
|
workflowsEnabled,
|
|
3260
3474
|
modelManifest: () => workerModels?.modelManifest() ?? [],
|
|
3261
3475
|
privacy: privacy ? { erasure: privacy.erasure } : void 0,
|
|
@@ -3331,5 +3545,5 @@ if (import.meta.main) await startSwitchboard({ fatalErrorShutdown: true });
|
|
|
3331
3545
|
//#endregion
|
|
3332
3546
|
export { StoredDocumentsRefusedError as a, initProfilerFromEnv as c, startSwitchboard as i, parseForcePgVersion as l, deriveAttachmentServiceConfig as n, resolveUnsupportedStoredDocuments as o, isPortAvailable as r, applySwitchboardReactorDefaults as s, createReactorKysely as t };
|
|
3333
3547
|
|
|
3334
|
-
//# sourceMappingURL=server-
|
|
3335
|
-
//# debugId=
|
|
3548
|
+
//# sourceMappingURL=server-J9fpd42v.mjs.map
|
|
3549
|
+
//# debugId=e3d937b6-2957-5fe0-8fc9-bf4fd0c18c93
|