@powerhousedao/reactor 6.2.2-dev.87 → 6.2.2-dev.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -332,12 +332,10 @@ function runProjectionWorker(parentPort, overrides = {}) {
332
332
  case "init":
333
333
  handleInit(msg).catch((err) => {
334
334
  post({
335
- type: "log",
335
+ type: "init-failed",
336
+ correlationId: msg.correlationId,
336
337
  shardId,
337
- level: "error",
338
- message: "projection worker init failed",
339
- args: [errorToInfo(err)],
340
- timestamp: Date.now()
338
+ error: errorToInfo(err)
341
339
  });
342
340
  });
343
341
  break;
@@ -1 +1 @@
1
- {"version":3,"file":"projection-entry.js","names":[],"sources":["../src/projection/projection-worker/build-projection-stack.ts","../src/projection/projection-worker/run-projection-worker.ts","../src/projection/projection-worker/projection-entry.ts"],"sourcesContent":["/**\n * In-worker projection stack builder.\n *\n * Mirrors {@link buildWorkerExecutor} but for the projection-side\n * read models. The worker owns one full copy of the storage stack — read\n * cache, operation index, document-meta cache — bound to its own\n * pg.Pool/Kysely, plus an in-process `ReadModelCoordinator` that\n * subscribes to a local `EventBus`.\n *\n * The host relays JOB_WRITE_READY into the local bus by calling\n * `relayWriteReady`, and forwards the local bus's JOB_READ_READY plus\n * READMODEL_* events back to the host bus via the IPC bridge.\n */\n\nimport type { DocumentModelModule } from \"@powerhousedao/shared/document-model\";\nimport type { ILogger } from \"document-model\";\nimport type { Kysely } from \"kysely\";\nimport { CollectionMembershipCache } from \"../../cache/collection-membership-cache.js\";\nimport { DocumentMetaCache } from \"../../cache/document-meta-cache.js\";\nimport { KyselyOperationIndex } from \"../../cache/kysely-operation-index.js\";\nimport { KyselyWriteCache } from \"../../cache/kysely-write-cache.js\";\nimport type { WriteCacheConfig } from \"../../cache/write-cache-types.js\";\nimport type { Database } from \"../../core/types.js\";\nimport { EventBus } from \"../../events/event-bus.js\";\nimport {\n ReactorEventTypes,\n type JobReadReadyEvent,\n type JobWriteReadyEvent,\n type ReadModelBatchCompletedEvent,\n type ReadModelIndexedEvent,\n type Unsubscribe,\n} from \"../../events/types.js\";\nimport {\n defaultLoadFactory,\n type BuildWorkerExecutorOptions,\n} from \"../../executor/worker/build-worker-executor.js\";\nimport type {\n FactorySpec,\n ModelManifestEntry,\n} from \"../../executor/worker/protocol.js\";\nimport { ReadModelCoordinator } from \"../../read-models/coordinator.js\";\nimport { KyselyDocumentView } from \"../../read-models/document-view.js\";\nimport type { IReadModel } from \"../../read-models/interfaces.js\";\nimport { DocumentModelRegistry } from \"../../registry/implementation.js\";\nimport { ConsistencyTracker } from \"../../shared/consistency-tracker.js\";\nimport {\n KyselyDocumentIndexer,\n type IndexerDatabase,\n} from \"../../storage/kysely/document-indexer.js\";\nimport { KyselyKeyframeStore } from \"../../storage/kysely/keyframe-store.js\";\nimport { KyselyOperationStore } from \"../../storage/kysely/store.js\";\nimport type { Database as StorageDatabase } from \"../../storage/kysely/types.js\";\nimport { REACTOR_SCHEMA } from \"../../storage/migrations/migrator.js\";\nimport type {\n BuiltInReadModelKind,\n ProjectionInitMessage,\n} from \"../protocol.js\";\n\nexport type ProjectionStackEvents = {\n onReadReady: (event: JobReadReadyEvent) => void;\n onReadModelIndexed: (event: ReadModelIndexedEvent) => void;\n onBatchCompleted: (event: ReadModelBatchCompletedEvent) => void;\n};\n\nexport type ProjectionStack = {\n registry: DocumentModelRegistry;\n coordinator: ReadModelCoordinator;\n eventBus: EventBus;\n relayWriteReady(event: JobWriteReadyEvent): Promise<void>;\n getChainDepth(): number;\n drain(): Promise<void>;\n shutdown(): Promise<void>;\n};\n\nexport type BuildProjectionStackOptions = {\n init: ProjectionInitMessage;\n database: Kysely<Database>;\n logger: ILogger;\n events: ProjectionStackEvents;\n loadFactory?: BuildWorkerExecutorOptions[\"loadFactory\"];\n driveContainerTypes?: ReadonlySet<string>;\n};\n\nasync function loadModelManifest(\n entries: ModelManifestEntry[],\n loadFactory: (spec: FactorySpec) => Promise<unknown>,\n registry: DocumentModelRegistry,\n logger: ILogger,\n): Promise<void> {\n for (const entry of entries) {\n let module: DocumentModelModule;\n try {\n module = (await loadFactory(entry.spec)) as DocumentModelModule;\n } catch (error) {\n logger.error(\n \"projection worker failed to load document model: @entry @error\",\n entry,\n error,\n );\n throw error;\n }\n const [result] = registry.registerModules(module);\n if (result.status === \"error\") {\n logger.error(\n \"projection worker failed to register document model: @entry @error\",\n entry,\n result.error,\n );\n throw result.error;\n }\n }\n}\n\nfunction instantiateReadModel(\n kind: BuiltInReadModelKind,\n database: Kysely<Database>,\n operationStore: KyselyOperationStore,\n operationIndex: KyselyOperationIndex,\n writeCache: KyselyWriteCache,\n): IReadModel {\n switch (kind) {\n case \"document-view\": {\n return new KyselyDocumentView(\n // @ts-expect-error - Database superset\n database,\n operationStore,\n operationIndex,\n writeCache,\n new ConsistencyTracker(),\n // The init payload carries no feature flags, so this view keeps hiding a\n // deleted document. Read-side only, so it cannot diverge state.\n false,\n );\n }\n case \"document-indexer\": {\n return new KyselyDocumentIndexer(\n database as unknown as Kysely<IndexerDatabase>,\n operationIndex,\n writeCache,\n new ConsistencyTracker(),\n );\n }\n default: {\n const exhaustive: never = kind;\n throw new Error(\n `unknown built-in read model kind: ${String(exhaustive)}`,\n );\n }\n }\n}\n\nasync function initReadModels(\n models: IReadModel[],\n logger: ILogger,\n): Promise<void> {\n for (const model of models) {\n const maybeInit = model as IReadModel & { init?: () => Promise<void> };\n if (typeof maybeInit.init !== \"function\") {\n continue;\n }\n try {\n await maybeInit.init();\n } catch (error) {\n logger.error(\n \"projection worker read model init failed: @name @error\",\n model.name,\n error,\n );\n throw error;\n }\n }\n}\n\nexport async function buildProjectionStack(\n options: BuildProjectionStackOptions,\n): Promise<ProjectionStack> {\n const { init, database: baseDatabase, logger, events } = options;\n const loadFactory = options.loadFactory ?? defaultLoadFactory;\n\n const registry = new DocumentModelRegistry();\n await loadModelManifest(init.models, loadFactory, registry, logger);\n\n const database = baseDatabase.withSchema(REACTOR_SCHEMA);\n const operationStore = new KyselyOperationStore(\n database as unknown as Kysely<StorageDatabase>,\n );\n const keyframeStore = new KyselyKeyframeStore(\n database as unknown as Kysely<StorageDatabase>,\n );\n\n const cacheConfig: WriteCacheConfig = {\n maxDocuments: 100,\n ringBufferSize: 10,\n keyframeInterval: 10,\n };\n const writeCache = new KyselyWriteCache(\n keyframeStore,\n operationStore,\n registry,\n cacheConfig,\n );\n await writeCache.startup();\n\n const operationIndex = new KyselyOperationIndex(\n database as unknown as Kysely<StorageDatabase>,\n );\n\n const documentMetaCache = new DocumentMetaCache(operationStore, {\n maxDocuments: 1000,\n });\n await documentMetaCache.startup();\n\n const collectionMembershipCache = new CollectionMembershipCache(\n operationIndex,\n );\n void collectionMembershipCache;\n void documentMetaCache;\n\n const preReady = init.preReadyKinds.map((kind) =>\n instantiateReadModel(\n kind,\n database,\n operationStore,\n operationIndex,\n writeCache,\n ),\n );\n const postReady = init.postReadyKinds.map((kind) =>\n instantiateReadModel(\n kind,\n database,\n operationStore,\n operationIndex,\n writeCache,\n ),\n );\n\n await initReadModels([...preReady, ...postReady], logger);\n\n const eventBus = new EventBus();\n const subscriptions: Unsubscribe[] = [];\n\n subscriptions.push(\n eventBus.subscribe(\n ReactorEventTypes.JOB_READ_READY,\n (_t: number, event: JobReadReadyEvent) => {\n events.onReadReady(event);\n },\n ),\n );\n subscriptions.push(\n eventBus.subscribe(\n ReactorEventTypes.READMODEL_INDEXED,\n (_t: number, event: ReadModelIndexedEvent) => {\n events.onReadModelIndexed(event);\n },\n ),\n );\n subscriptions.push(\n eventBus.subscribe(\n ReactorEventTypes.READMODEL_BATCH_COMPLETED,\n (_t: number, event: ReadModelBatchCompletedEvent) => {\n events.onBatchCompleted(event);\n },\n ),\n );\n\n const coordinator = new ReadModelCoordinator(eventBus, preReady, postReady);\n coordinator.start();\n\n return {\n registry,\n coordinator,\n eventBus,\n async relayWriteReady(event: JobWriteReadyEvent): Promise<void> {\n await eventBus.emit(ReactorEventTypes.JOB_WRITE_READY, event);\n },\n getChainDepth(): number {\n return coordinator.getChainDepth();\n },\n async drain(): Promise<void> {\n await coordinator.drain();\n },\n shutdown(): Promise<void> {\n coordinator.stop();\n for (const unsub of subscriptions) {\n unsub();\n }\n return Promise.resolve();\n },\n };\n}\n","import { ConsoleLogger } from \"document-model\";\nimport type { Kysely } from \"kysely\";\nimport type { MessagePort } from \"node:worker_threads\";\nimport type { Database } from \"../../core/types.js\";\nimport { createForwardingLogger } from \"../../executor/worker/forwarding-logger.js\";\nimport type { DbConfig } from \"../../executor/worker/protocol.js\";\nimport { errorToInfo } from \"../../executor/worker/sanitize.js\";\nimport type {\n JobReadReadyEvent,\n JobWriteReadyEvent,\n ReadModelBatchCompletedEvent,\n ReadModelIndexedEvent,\n} from \"../../events/types.js\";\nimport {\n instrumentPgPool,\n type PoolInstrumentation,\n} from \"../../storage/pool-instrumentation.js\";\nimport type {\n ProjectionInitMessage,\n ProjectionParentMessage,\n ProjectionWorkerMessage,\n} from \"../protocol.js\";\nimport {\n buildProjectionStack,\n type ProjectionStack,\n} from \"./build-projection-stack.js\";\n\nconst POOL_SAMPLE_INTERVAL_MS = 1_000;\n\n/**\n * Closeable handle around the worker's Postgres pool. Decoupled from the\n * default factory so tests can swap in PGlite via `RunProjectionWorkerOverrides`.\n *\n * When the worker owns a real pg.Pool, the handle exposes a\n * {@link PoolInstrumentation} so the run loop can forward acquire-wait\n * samples to the host. Tests that swap in PGlite leave this undefined.\n */\nexport type ProjectionWorkerDatabaseHandle = {\n kysely: Kysely<Database>;\n poolInstrumentation?: PoolInstrumentation;\n shutdown(): Promise<void>;\n};\n\nexport type RunProjectionWorkerOverrides = {\n createDatabase?: (\n config: DbConfig,\n shardId: string,\n ) => Promise<ProjectionWorkerDatabaseHandle>;\n loadFactory?: Parameters<typeof buildProjectionStack>[0][\"loadFactory\"];\n beforeBuildStack?: (db: Kysely<Database>) => Promise<void>;\n};\n\nasync function defaultCreateDatabase(\n config: DbConfig,\n shardId: string,\n): Promise<ProjectionWorkerDatabaseHandle> {\n const { Kysely, PostgresDialect } = await import(\"kysely\");\n const pgModule = await import(\"pg\");\n const Pool = pgModule.default.Pool;\n const pool = new Pool({\n host: config.host,\n port: config.port,\n database: config.database,\n user: config.user,\n password: config.password,\n ssl: config.ssl ? { rejectUnauthorized: false } : undefined,\n application_name: config.applicationName ?? shardId,\n max: config.poolSize,\n connectionTimeoutMillis: config.connectionTimeoutMillis,\n idleTimeoutMillis: config.idleTimeoutMillis,\n });\n const poolInstrumentation = instrumentPgPool(pool, shardId);\n const kysely = new Kysely<Database>({\n dialect: new PostgresDialect({ pool }),\n });\n return {\n kysely,\n poolInstrumentation,\n async shutdown(): Promise<void> {\n try {\n await kysely.destroy();\n } catch {\n // best-effort\n }\n try {\n await pool.end();\n } catch {\n // best-effort\n }\n },\n };\n}\n\n/**\n * Drives the projection worker's message loop. Owns lifecycle of the\n * database handle and the projection stack. The default factory builds a\n * real Postgres pool; tests inject overrides for an in-process PGlite path.\n */\nexport function runProjectionWorker(\n parentPort: MessagePort,\n overrides: RunProjectionWorkerOverrides = {},\n): void {\n let shardId = \"\";\n let initCompleted = false;\n let stack: ProjectionStack | null = null;\n let database: ProjectionWorkerDatabaseHandle | null = null;\n let depthTimer: NodeJS.Timeout | null = null;\n let lastReportedDepth = -1;\n let poolSampleTimer: NodeJS.Timeout | null = null;\n let pendingPoolSamples: number[] = [];\n let detachPoolListener: (() => void) | null = null;\n\n function post(msg: ProjectionWorkerMessage): void {\n parentPort.postMessage(msg);\n }\n\n function startPoolReporter(instrumentation: PoolInstrumentation): void {\n detachPoolListener = instrumentation.onAcquire((durationMs) => {\n pendingPoolSamples.push(durationMs);\n });\n poolSampleTimer = setInterval(() => {\n if (pendingPoolSamples.length === 0) {\n return;\n }\n const durations = pendingPoolSamples;\n pendingPoolSamples = [];\n const stats = instrumentation.getStats();\n post({\n type: \"pool-acquire-samples\",\n shardId,\n poolName: instrumentation.name,\n timestamp: Date.now(),\n durations,\n size: stats.size,\n idle: stats.idle,\n waiting: stats.waiting,\n });\n }, POOL_SAMPLE_INTERVAL_MS);\n poolSampleTimer.unref();\n }\n\n function stopPoolReporter(): void {\n if (detachPoolListener) {\n detachPoolListener();\n detachPoolListener = null;\n }\n if (poolSampleTimer) {\n clearInterval(poolSampleTimer);\n poolSampleTimer = null;\n }\n pendingPoolSamples = [];\n }\n\n const logger = createForwardingLogger((msg) => post({ ...msg, shardId }));\n\n process.on(\"uncaughtException\", (err: unknown) => {\n try {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker uncaughtException\",\n args: [errorToInfo(err)],\n timestamp: Date.now(),\n });\n } catch {\n // nothing left to do\n }\n throw err;\n });\n\n process.on(\"unhandledRejection\", (reason: unknown) => {\n try {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker unhandledRejection\",\n args: [errorToInfo(reason)],\n timestamp: Date.now(),\n });\n } catch {\n // nothing left to do\n }\n });\n\n function startDepthReporter(intervalMs: number): void {\n if (intervalMs <= 0) {\n return;\n }\n depthTimer = setInterval(() => {\n if (!stack) {\n return;\n }\n const depth = stack.getChainDepth();\n if (depth === lastReportedDepth) {\n return;\n }\n lastReportedDepth = depth;\n post({\n type: \"chain-depth\",\n shardId,\n depth,\n timestamp: Date.now(),\n });\n }, intervalMs);\n depthTimer.unref();\n }\n\n function stopDepthReporter(): void {\n if (depthTimer) {\n clearInterval(depthTimer);\n depthTimer = null;\n }\n }\n\n async function handleInit(msg: ProjectionInitMessage): Promise<void> {\n shardId = msg.shardId;\n const createDb = overrides.createDatabase ?? defaultCreateDatabase;\n database = await createDb(msg.db, msg.shardId);\n if (overrides.beforeBuildStack) {\n await overrides.beforeBuildStack(database.kysely);\n }\n stack = await buildProjectionStack({\n init: msg,\n database: database.kysely,\n logger: new ConsoleLogger([`projection-shard:${msg.shardId}`]),\n loadFactory: overrides.loadFactory,\n events: {\n onReadReady: (event: JobReadReadyEvent) => {\n post({\n type: \"read-ready\",\n shardId,\n jobId: event.jobId,\n operations: event.operations,\n });\n },\n onReadModelIndexed: (event: ReadModelIndexedEvent) => {\n post({\n type: \"readmodel-indexed\",\n shardId,\n jobId: event.jobId,\n readModelName: event.readModelName,\n stage: event.stage,\n durationMs: event.durationMs,\n operationCount: event.operationCount,\n success: event.success,\n });\n },\n onBatchCompleted: (event: ReadModelBatchCompletedEvent) => {\n post({\n type: \"readmodel-batch-completed\",\n shardId,\n jobId: event.jobId,\n batchSize: event.batchSize,\n chainWaitDurationMs: event.chainWaitDurationMs,\n preReadyDurationMs: event.preReadyDurationMs,\n emitDurationMs: event.emitDurationMs,\n postReadyDurationMs: event.postReadyDurationMs,\n });\n },\n },\n });\n initCompleted = true;\n startDepthReporter(msg.chainDepthReportIntervalMs);\n if (database.poolInstrumentation) {\n startPoolReporter(database.poolInstrumentation);\n }\n logger.info(\"projection worker initialized: @shardId\", msg.shardId);\n post({ type: \"ready\", correlationId: msg.correlationId, shardId });\n }\n\n async function handleWriteReady(\n msg: Extract<ProjectionParentMessage, { type: \"write-ready\" }>,\n ): Promise<void> {\n if (!stack) {\n logger.warn(\n \"write-ready received before init on shard @shardId\",\n shardId,\n );\n return;\n }\n const event: JobWriteReadyEvent = {\n jobId: msg.jobId,\n operations: msg.operations,\n jobMeta: msg.jobMeta,\n collectionMemberships: msg.collectionMemberships,\n };\n await stack.relayWriteReady(event);\n }\n\n async function handleDrain(correlationId: string): Promise<void> {\n if (stack) {\n await stack.drain();\n }\n post({ type: \"drained\", correlationId, shardId });\n }\n\n async function shutdownStack(): Promise<void> {\n stopDepthReporter();\n stopPoolReporter();\n if (stack) {\n try {\n await stack.drain();\n } catch (error) {\n logger.warn(\n \"projection worker drain failed during shutdown: @error\",\n error,\n );\n }\n try {\n await stack.shutdown();\n } catch (error) {\n logger.warn(\"projection worker stack shutdown failed: @error\", error);\n }\n stack = null;\n }\n if (database) {\n await database.shutdown();\n database = null;\n }\n }\n\n function handleParentMessage(msg: ProjectionParentMessage): void {\n switch (msg.type) {\n case \"init\": {\n handleInit(msg).catch((err: unknown) => {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker init failed\",\n args: [errorToInfo(err)],\n timestamp: Date.now(),\n });\n });\n break;\n }\n case \"write-ready\": {\n if (!initCompleted) {\n logger.warn(\n \"write-ready received before init on shard @shardId\",\n shardId,\n );\n break;\n }\n handleWriteReady(msg).catch((err: unknown) => {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker write-ready failed\",\n args: [errorToInfo(err)],\n timestamp: Date.now(),\n });\n });\n break;\n }\n case \"drain\": {\n handleDrain(msg.correlationId).catch((err: unknown) => {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker drain failed\",\n args: [errorToInfo(err)],\n timestamp: Date.now(),\n });\n });\n break;\n }\n case \"shutdown\": {\n logger.info(\"projection worker shutting down: @shardId\", shardId);\n void shutdownStack().finally(() => {\n post({\n type: \"log\",\n shardId,\n level: \"info\",\n message: \"projection worker shutdown\",\n args: [],\n timestamp: Date.now(),\n });\n process.exit(0);\n });\n break;\n }\n default: {\n const exhaustive: never = msg;\n void exhaustive;\n break;\n }\n }\n }\n\n parentPort.on(\"message\", handleParentMessage);\n\n const harness = {\n handleParentMessage,\n get initCompleted(): boolean {\n return initCompleted;\n },\n get shardId(): string {\n return shardId;\n },\n };\n (\n parentPort as unknown as { __reactorProjectionWorkerHarness?: unknown }\n ).__reactorProjectionWorkerHarness = harness;\n}\n","import { isMainThread, parentPort } from \"node:worker_threads\";\nimport { runProjectionWorker } from \"./run-projection-worker.js\";\n\nif (isMainThread || parentPort === null) {\n throw new Error(\"projection-worker entry.ts must be run as a worker thread\");\n}\n\nrunProjectionWorker(parentPort);\n"],"mappings":";;;;;;;;AAmFA,eAAe,kBACb,SACA,aACA,UACA,QACe;AACf,MAAK,MAAM,SAAS,SAAS;EAC3B,IAAI;AACJ,MAAI;AACF,YAAU,MAAM,YAAY,MAAM,KAAK;WAChC,OAAO;AACd,UAAO,MACL,kEACA,OACA,MACD;AACD,SAAM;;EAER,MAAM,CAAC,UAAU,SAAS,gBAAgB,OAAO;AACjD,MAAI,OAAO,WAAW,SAAS;AAC7B,UAAO,MACL,sEACA,OACA,OAAO,MACR;AACD,SAAM,OAAO;;;;AAKnB,SAAS,qBACP,MACA,UACA,gBACA,gBACA,YACY;AACZ,SAAQ,MAAR;EACE,KAAK,gBACH,QAAO,IAAI,mBAET,UACA,gBACA,gBACA,YACA,IAAI,oBAAoB,EAGxB,MACD;EAEH,KAAK,mBACH,QAAO,IAAI,sBACT,UACA,gBACA,YACA,IAAI,oBAAoB,CACzB;EAEH,SAAS;GACP,MAAM,aAAoB;AAC1B,SAAM,IAAI,MACR,qCAAqC,OAAO,WAAW,GACxD;;;;AAKP,eAAe,eACb,QACA,QACe;AACf,MAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,YAAY;AAClB,MAAI,OAAO,UAAU,SAAS,WAC5B;AAEF,MAAI;AACF,SAAM,UAAU,MAAM;WACf,OAAO;AACd,UAAO,MACL,0DACA,MAAM,MACN,MACD;AACD,SAAM;;;;AAKZ,eAAsB,qBACpB,SAC0B;CAC1B,MAAM,EAAE,MAAM,UAAU,cAAc,QAAQ,WAAW;CACzD,MAAM,cAAc,QAAQ,eAAe;CAE3C,MAAM,WAAW,IAAI,uBAAuB;AAC5C,OAAM,kBAAkB,KAAK,QAAQ,aAAa,UAAU,OAAO;CAEnE,MAAM,WAAW,aAAa,WAAW,eAAe;CACxD,MAAM,iBAAiB,IAAI,qBACzB,SACD;CAUD,MAAM,aAAa,IAAI,iBATD,IAAI,oBACxB,SACD,EASC,gBACA,UARoC;EACpC,cAAc;EACd,gBAAgB;EAChB,kBAAkB;EACnB,CAMA;AACD,OAAM,WAAW,SAAS;CAE1B,MAAM,iBAAiB,IAAI,qBACzB,SACD;AAKD,OAH0B,IAAI,kBAAkB,gBAAgB,EAC9D,cAAc,KACf,CAAC,CACsB,SAAS;AAEC,KAAI,0BACpC,eACD;CAID,MAAM,WAAW,KAAK,cAAc,KAAK,SACvC,qBACE,MACA,UACA,gBACA,gBACA,WACD,CACF;CACD,MAAM,YAAY,KAAK,eAAe,KAAK,SACzC,qBACE,MACA,UACA,gBACA,gBACA,WACD,CACF;AAED,OAAM,eAAe,CAAC,GAAG,UAAU,GAAG,UAAU,EAAE,OAAO;CAEzD,MAAM,WAAW,IAAI,UAAU;CAC/B,MAAM,gBAA+B,EAAE;AAEvC,eAAc,KACZ,SAAS,UACP,kBAAkB,iBACjB,IAAY,UAA6B;AACxC,SAAO,YAAY,MAAM;GAE5B,CACF;AACD,eAAc,KACZ,SAAS,UACP,kBAAkB,oBACjB,IAAY,UAAiC;AAC5C,SAAO,mBAAmB,MAAM;GAEnC,CACF;AACD,eAAc,KACZ,SAAS,UACP,kBAAkB,4BACjB,IAAY,UAAwC;AACnD,SAAO,iBAAiB,MAAM;GAEjC,CACF;CAED,MAAM,cAAc,IAAI,qBAAqB,UAAU,UAAU,UAAU;AAC3E,aAAY,OAAO;AAEnB,QAAO;EACL;EACA;EACA;EACA,MAAM,gBAAgB,OAA0C;AAC9D,SAAM,SAAS,KAAK,kBAAkB,iBAAiB,MAAM;;EAE/D,gBAAwB;AACtB,UAAO,YAAY,eAAe;;EAEpC,MAAM,QAAuB;AAC3B,SAAM,YAAY,OAAO;;EAE3B,WAA0B;AACxB,eAAY,MAAM;AAClB,QAAK,MAAM,SAAS,cAClB,QAAO;AAET,UAAO,QAAQ,SAAS;;EAE3B;;;;ACvQH,MAAM,0BAA0B;AAyBhC,eAAe,sBACb,QACA,SACyC;CACzC,MAAM,EAAE,QAAQ,oBAAoB,MAAM,OAAO;CAEjD,MAAM,QADW,MAAM,OAAO,OACR,QAAQ;CAC9B,MAAM,OAAO,IAAI,KAAK;EACpB,MAAM,OAAO;EACb,MAAM,OAAO;EACb,UAAU,OAAO;EACjB,MAAM,OAAO;EACb,UAAU,OAAO;EACjB,KAAK,OAAO,MAAM,EAAE,oBAAoB,OAAO,GAAG,KAAA;EAClD,kBAAkB,OAAO,mBAAmB;EAC5C,KAAK,OAAO;EACZ,yBAAyB,OAAO;EAChC,mBAAmB,OAAO;EAC3B,CAAC;CACF,MAAM,sBAAsB,iBAAiB,MAAM,QAAQ;CAC3D,MAAM,SAAS,IAAI,OAAiB,EAClC,SAAS,IAAI,gBAAgB,EAAE,MAAM,CAAC,EACvC,CAAC;AACF,QAAO;EACL;EACA;EACA,MAAM,WAA0B;AAC9B,OAAI;AACF,UAAM,OAAO,SAAS;WAChB;AAGR,OAAI;AACF,UAAM,KAAK,KAAK;WACV;;EAIX;;;;;;;AAQH,SAAgB,oBACd,YACA,YAA0C,EAAE,EACtC;CACN,IAAI,UAAU;CACd,IAAI,gBAAgB;CACpB,IAAI,QAAgC;CACpC,IAAI,WAAkD;CACtD,IAAI,aAAoC;CACxC,IAAI,oBAAoB;CACxB,IAAI,kBAAyC;CAC7C,IAAI,qBAA+B,EAAE;CACrC,IAAI,qBAA0C;CAE9C,SAAS,KAAK,KAAoC;AAChD,aAAW,YAAY,IAAI;;CAG7B,SAAS,kBAAkB,iBAA4C;AACrE,uBAAqB,gBAAgB,WAAW,eAAe;AAC7D,sBAAmB,KAAK,WAAW;IACnC;AACF,oBAAkB,kBAAkB;AAClC,OAAI,mBAAmB,WAAW,EAChC;GAEF,MAAM,YAAY;AAClB,wBAAqB,EAAE;GACvB,MAAM,QAAQ,gBAAgB,UAAU;AACxC,QAAK;IACH,MAAM;IACN;IACA,UAAU,gBAAgB;IAC1B,WAAW,KAAK,KAAK;IACrB;IACA,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,SAAS,MAAM;IAChB,CAAC;KACD,wBAAwB;AAC3B,kBAAgB,OAAO;;CAGzB,SAAS,mBAAyB;AAChC,MAAI,oBAAoB;AACtB,uBAAoB;AACpB,wBAAqB;;AAEvB,MAAI,iBAAiB;AACnB,iBAAc,gBAAgB;AAC9B,qBAAkB;;AAEpB,uBAAqB,EAAE;;CAGzB,MAAM,SAAS,wBAAwB,QAAQ,KAAK;EAAE,GAAG;EAAK;EAAS,CAAC,CAAC;AAEzE,SAAQ,GAAG,sBAAsB,QAAiB;AAChD,MAAI;AACF,QAAK;IACH,MAAM;IACN;IACA,OAAO;IACP,SAAS;IACT,MAAM,CAAC,YAAY,IAAI,CAAC;IACxB,WAAW,KAAK,KAAK;IACtB,CAAC;UACI;AAGR,QAAM;GACN;AAEF,SAAQ,GAAG,uBAAuB,WAAoB;AACpD,MAAI;AACF,QAAK;IACH,MAAM;IACN;IACA,OAAO;IACP,SAAS;IACT,MAAM,CAAC,YAAY,OAAO,CAAC;IAC3B,WAAW,KAAK,KAAK;IACtB,CAAC;UACI;GAGR;CAEF,SAAS,mBAAmB,YAA0B;AACpD,MAAI,cAAc,EAChB;AAEF,eAAa,kBAAkB;AAC7B,OAAI,CAAC,MACH;GAEF,MAAM,QAAQ,MAAM,eAAe;AACnC,OAAI,UAAU,kBACZ;AAEF,uBAAoB;AACpB,QAAK;IACH,MAAM;IACN;IACA;IACA,WAAW,KAAK,KAAK;IACtB,CAAC;KACD,WAAW;AACd,aAAW,OAAO;;CAGpB,SAAS,oBAA0B;AACjC,MAAI,YAAY;AACd,iBAAc,WAAW;AACzB,gBAAa;;;CAIjB,eAAe,WAAW,KAA2C;AACnE,YAAU,IAAI;AAEd,aAAW,OADM,UAAU,kBAAkB,uBACnB,IAAI,IAAI,IAAI,QAAQ;AAC9C,MAAI,UAAU,iBACZ,OAAM,UAAU,iBAAiB,SAAS,OAAO;AAEnD,UAAQ,MAAM,qBAAqB;GACjC,MAAM;GACN,UAAU,SAAS;GACnB,QAAQ,IAAI,cAAc,CAAC,oBAAoB,IAAI,UAAU,CAAC;GAC9D,aAAa,UAAU;GACvB,QAAQ;IACN,cAAc,UAA6B;AACzC,UAAK;MACH,MAAM;MACN;MACA,OAAO,MAAM;MACb,YAAY,MAAM;MACnB,CAAC;;IAEJ,qBAAqB,UAAiC;AACpD,UAAK;MACH,MAAM;MACN;MACA,OAAO,MAAM;MACb,eAAe,MAAM;MACrB,OAAO,MAAM;MACb,YAAY,MAAM;MAClB,gBAAgB,MAAM;MACtB,SAAS,MAAM;MAChB,CAAC;;IAEJ,mBAAmB,UAAwC;AACzD,UAAK;MACH,MAAM;MACN;MACA,OAAO,MAAM;MACb,WAAW,MAAM;MACjB,qBAAqB,MAAM;MAC3B,oBAAoB,MAAM;MAC1B,gBAAgB,MAAM;MACtB,qBAAqB,MAAM;MAC5B,CAAC;;IAEL;GACF,CAAC;AACF,kBAAgB;AAChB,qBAAmB,IAAI,2BAA2B;AAClD,MAAI,SAAS,oBACX,mBAAkB,SAAS,oBAAoB;AAEjD,SAAO,KAAK,2CAA2C,IAAI,QAAQ;AACnE,OAAK;GAAE,MAAM;GAAS,eAAe,IAAI;GAAe;GAAS,CAAC;;CAGpE,eAAe,iBACb,KACe;AACf,MAAI,CAAC,OAAO;AACV,UAAO,KACL,sDACA,QACD;AACD;;EAEF,MAAM,QAA4B;GAChC,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,SAAS,IAAI;GACb,uBAAuB,IAAI;GAC5B;AACD,QAAM,MAAM,gBAAgB,MAAM;;CAGpC,eAAe,YAAY,eAAsC;AAC/D,MAAI,MACF,OAAM,MAAM,OAAO;AAErB,OAAK;GAAE,MAAM;GAAW;GAAe;GAAS,CAAC;;CAGnD,eAAe,gBAA+B;AAC5C,qBAAmB;AACnB,oBAAkB;AAClB,MAAI,OAAO;AACT,OAAI;AACF,UAAM,MAAM,OAAO;YACZ,OAAO;AACd,WAAO,KACL,0DACA,MACD;;AAEH,OAAI;AACF,UAAM,MAAM,UAAU;YACf,OAAO;AACd,WAAO,KAAK,mDAAmD,MAAM;;AAEvE,WAAQ;;AAEV,MAAI,UAAU;AACZ,SAAM,SAAS,UAAU;AACzB,cAAW;;;CAIf,SAAS,oBAAoB,KAAoC;AAC/D,UAAQ,IAAI,MAAZ;GACE,KAAK;AACH,eAAW,IAAI,CAAC,OAAO,QAAiB;AACtC,UAAK;MACH,MAAM;MACN;MACA,OAAO;MACP,SAAS;MACT,MAAM,CAAC,YAAY,IAAI,CAAC;MACxB,WAAW,KAAK,KAAK;MACtB,CAAC;MACF;AACF;GAEF,KAAK;AACH,QAAI,CAAC,eAAe;AAClB,YAAO,KACL,sDACA,QACD;AACD;;AAEF,qBAAiB,IAAI,CAAC,OAAO,QAAiB;AAC5C,UAAK;MACH,MAAM;MACN;MACA,OAAO;MACP,SAAS;MACT,MAAM,CAAC,YAAY,IAAI,CAAC;MACxB,WAAW,KAAK,KAAK;MACtB,CAAC;MACF;AACF;GAEF,KAAK;AACH,gBAAY,IAAI,cAAc,CAAC,OAAO,QAAiB;AACrD,UAAK;MACH,MAAM;MACN;MACA,OAAO;MACP,SAAS;MACT,MAAM,CAAC,YAAY,IAAI,CAAC;MACxB,WAAW,KAAK,KAAK;MACtB,CAAC;MACF;AACF;GAEF,KAAK;AACH,WAAO,KAAK,6CAA6C,QAAQ;AAC5D,mBAAe,CAAC,cAAc;AACjC,UAAK;MACH,MAAM;MACN;MACA,OAAO;MACP,SAAS;MACT,MAAM,EAAE;MACR,WAAW,KAAK,KAAK;MACtB,CAAC;AACF,aAAQ,KAAK,EAAE;MACf;AACF;GAEF,QAGE;;;AAKN,YAAW,GAAG,WAAW,oBAAoB;AAY3C,YACA,mCAXc;EACd;EACA,IAAI,gBAAyB;AAC3B,UAAO;;EAET,IAAI,UAAkB;AACpB,UAAO;;EAEV;;;;ACjZH,IAAI,gBAAgB,eAAe,KACjC,OAAM,IAAI,MAAM,4DAA4D;AAG9E,oBAAoB,WAAW"}
1
+ {"version":3,"file":"projection-entry.js","names":[],"sources":["../src/projection/projection-worker/build-projection-stack.ts","../src/projection/projection-worker/run-projection-worker.ts","../src/projection/projection-worker/projection-entry.ts"],"sourcesContent":["/**\n * In-worker projection stack builder.\n *\n * Mirrors {@link buildWorkerExecutor} but for the projection-side\n * read models. The worker owns one full copy of the storage stack — read\n * cache, operation index, document-meta cache — bound to its own\n * pg.Pool/Kysely, plus an in-process `ReadModelCoordinator` that\n * subscribes to a local `EventBus`.\n *\n * The host relays JOB_WRITE_READY into the local bus by calling\n * `relayWriteReady`, and forwards the local bus's JOB_READ_READY plus\n * READMODEL_* events back to the host bus via the IPC bridge.\n */\n\nimport type { DocumentModelModule } from \"@powerhousedao/shared/document-model\";\nimport type { ILogger } from \"document-model\";\nimport type { Kysely } from \"kysely\";\nimport { CollectionMembershipCache } from \"../../cache/collection-membership-cache.js\";\nimport { DocumentMetaCache } from \"../../cache/document-meta-cache.js\";\nimport { KyselyOperationIndex } from \"../../cache/kysely-operation-index.js\";\nimport { KyselyWriteCache } from \"../../cache/kysely-write-cache.js\";\nimport type { WriteCacheConfig } from \"../../cache/write-cache-types.js\";\nimport type { Database } from \"../../core/types.js\";\nimport { EventBus } from \"../../events/event-bus.js\";\nimport {\n ReactorEventTypes,\n type JobReadReadyEvent,\n type JobWriteReadyEvent,\n type ReadModelBatchCompletedEvent,\n type ReadModelIndexedEvent,\n type Unsubscribe,\n} from \"../../events/types.js\";\nimport {\n defaultLoadFactory,\n type BuildWorkerExecutorOptions,\n} from \"../../executor/worker/build-worker-executor.js\";\nimport type {\n FactorySpec,\n ModelManifestEntry,\n} from \"../../executor/worker/protocol.js\";\nimport { ReadModelCoordinator } from \"../../read-models/coordinator.js\";\nimport { KyselyDocumentView } from \"../../read-models/document-view.js\";\nimport type { IReadModel } from \"../../read-models/interfaces.js\";\nimport { DocumentModelRegistry } from \"../../registry/implementation.js\";\nimport { ConsistencyTracker } from \"../../shared/consistency-tracker.js\";\nimport {\n KyselyDocumentIndexer,\n type IndexerDatabase,\n} from \"../../storage/kysely/document-indexer.js\";\nimport { KyselyKeyframeStore } from \"../../storage/kysely/keyframe-store.js\";\nimport { KyselyOperationStore } from \"../../storage/kysely/store.js\";\nimport type { Database as StorageDatabase } from \"../../storage/kysely/types.js\";\nimport { REACTOR_SCHEMA } from \"../../storage/migrations/migrator.js\";\nimport type {\n BuiltInReadModelKind,\n ProjectionInitMessage,\n} from \"../protocol.js\";\n\nexport type ProjectionStackEvents = {\n onReadReady: (event: JobReadReadyEvent) => void;\n onReadModelIndexed: (event: ReadModelIndexedEvent) => void;\n onBatchCompleted: (event: ReadModelBatchCompletedEvent) => void;\n};\n\nexport type ProjectionStack = {\n registry: DocumentModelRegistry;\n coordinator: ReadModelCoordinator;\n eventBus: EventBus;\n relayWriteReady(event: JobWriteReadyEvent): Promise<void>;\n getChainDepth(): number;\n drain(): Promise<void>;\n shutdown(): Promise<void>;\n};\n\nexport type BuildProjectionStackOptions = {\n init: ProjectionInitMessage;\n database: Kysely<Database>;\n logger: ILogger;\n events: ProjectionStackEvents;\n loadFactory?: BuildWorkerExecutorOptions[\"loadFactory\"];\n driveContainerTypes?: ReadonlySet<string>;\n};\n\nasync function loadModelManifest(\n entries: ModelManifestEntry[],\n loadFactory: (spec: FactorySpec) => Promise<unknown>,\n registry: DocumentModelRegistry,\n logger: ILogger,\n): Promise<void> {\n for (const entry of entries) {\n let module: DocumentModelModule;\n try {\n module = (await loadFactory(entry.spec)) as DocumentModelModule;\n } catch (error) {\n logger.error(\n \"projection worker failed to load document model: @entry @error\",\n entry,\n error,\n );\n throw error;\n }\n const [result] = registry.registerModules(module);\n if (result.status === \"error\") {\n logger.error(\n \"projection worker failed to register document model: @entry @error\",\n entry,\n result.error,\n );\n throw result.error;\n }\n }\n}\n\nfunction instantiateReadModel(\n kind: BuiltInReadModelKind,\n database: Kysely<Database>,\n operationStore: KyselyOperationStore,\n operationIndex: KyselyOperationIndex,\n writeCache: KyselyWriteCache,\n): IReadModel {\n switch (kind) {\n case \"document-view\": {\n return new KyselyDocumentView(\n // @ts-expect-error - Database superset\n database,\n operationStore,\n operationIndex,\n writeCache,\n new ConsistencyTracker(),\n // The init payload carries no feature flags, so this view keeps hiding a\n // deleted document. Read-side only, so it cannot diverge state.\n false,\n );\n }\n case \"document-indexer\": {\n return new KyselyDocumentIndexer(\n database as unknown as Kysely<IndexerDatabase>,\n operationIndex,\n writeCache,\n new ConsistencyTracker(),\n );\n }\n default: {\n const exhaustive: never = kind;\n throw new Error(\n `unknown built-in read model kind: ${String(exhaustive)}`,\n );\n }\n }\n}\n\nasync function initReadModels(\n models: IReadModel[],\n logger: ILogger,\n): Promise<void> {\n for (const model of models) {\n const maybeInit = model as IReadModel & { init?: () => Promise<void> };\n if (typeof maybeInit.init !== \"function\") {\n continue;\n }\n try {\n await maybeInit.init();\n } catch (error) {\n logger.error(\n \"projection worker read model init failed: @name @error\",\n model.name,\n error,\n );\n throw error;\n }\n }\n}\n\nexport async function buildProjectionStack(\n options: BuildProjectionStackOptions,\n): Promise<ProjectionStack> {\n const { init, database: baseDatabase, logger, events } = options;\n const loadFactory = options.loadFactory ?? defaultLoadFactory;\n\n const registry = new DocumentModelRegistry();\n await loadModelManifest(init.models, loadFactory, registry, logger);\n\n const database = baseDatabase.withSchema(REACTOR_SCHEMA);\n const operationStore = new KyselyOperationStore(\n database as unknown as Kysely<StorageDatabase>,\n );\n const keyframeStore = new KyselyKeyframeStore(\n database as unknown as Kysely<StorageDatabase>,\n );\n\n const cacheConfig: WriteCacheConfig = {\n maxDocuments: 100,\n ringBufferSize: 10,\n keyframeInterval: 10,\n };\n const writeCache = new KyselyWriteCache(\n keyframeStore,\n operationStore,\n registry,\n cacheConfig,\n );\n await writeCache.startup();\n\n const operationIndex = new KyselyOperationIndex(\n database as unknown as Kysely<StorageDatabase>,\n );\n\n const documentMetaCache = new DocumentMetaCache(operationStore, {\n maxDocuments: 1000,\n });\n await documentMetaCache.startup();\n\n const collectionMembershipCache = new CollectionMembershipCache(\n operationIndex,\n );\n void collectionMembershipCache;\n void documentMetaCache;\n\n const preReady = init.preReadyKinds.map((kind) =>\n instantiateReadModel(\n kind,\n database,\n operationStore,\n operationIndex,\n writeCache,\n ),\n );\n const postReady = init.postReadyKinds.map((kind) =>\n instantiateReadModel(\n kind,\n database,\n operationStore,\n operationIndex,\n writeCache,\n ),\n );\n\n await initReadModels([...preReady, ...postReady], logger);\n\n const eventBus = new EventBus();\n const subscriptions: Unsubscribe[] = [];\n\n subscriptions.push(\n eventBus.subscribe(\n ReactorEventTypes.JOB_READ_READY,\n (_t: number, event: JobReadReadyEvent) => {\n events.onReadReady(event);\n },\n ),\n );\n subscriptions.push(\n eventBus.subscribe(\n ReactorEventTypes.READMODEL_INDEXED,\n (_t: number, event: ReadModelIndexedEvent) => {\n events.onReadModelIndexed(event);\n },\n ),\n );\n subscriptions.push(\n eventBus.subscribe(\n ReactorEventTypes.READMODEL_BATCH_COMPLETED,\n (_t: number, event: ReadModelBatchCompletedEvent) => {\n events.onBatchCompleted(event);\n },\n ),\n );\n\n const coordinator = new ReadModelCoordinator(eventBus, preReady, postReady);\n coordinator.start();\n\n return {\n registry,\n coordinator,\n eventBus,\n async relayWriteReady(event: JobWriteReadyEvent): Promise<void> {\n await eventBus.emit(ReactorEventTypes.JOB_WRITE_READY, event);\n },\n getChainDepth(): number {\n return coordinator.getChainDepth();\n },\n async drain(): Promise<void> {\n await coordinator.drain();\n },\n shutdown(): Promise<void> {\n coordinator.stop();\n for (const unsub of subscriptions) {\n unsub();\n }\n return Promise.resolve();\n },\n };\n}\n","import { ConsoleLogger } from \"document-model\";\nimport type { Kysely } from \"kysely\";\nimport type { MessagePort } from \"node:worker_threads\";\nimport type { Database } from \"../../core/types.js\";\nimport { createForwardingLogger } from \"../../executor/worker/forwarding-logger.js\";\nimport type { DbConfig } from \"../../executor/worker/protocol.js\";\nimport { errorToInfo } from \"../../executor/worker/sanitize.js\";\nimport type {\n JobReadReadyEvent,\n JobWriteReadyEvent,\n ReadModelBatchCompletedEvent,\n ReadModelIndexedEvent,\n} from \"../../events/types.js\";\nimport {\n instrumentPgPool,\n type PoolInstrumentation,\n} from \"../../storage/pool-instrumentation.js\";\nimport type {\n ProjectionInitMessage,\n ProjectionParentMessage,\n ProjectionWorkerMessage,\n} from \"../protocol.js\";\nimport {\n buildProjectionStack,\n type ProjectionStack,\n} from \"./build-projection-stack.js\";\n\nconst POOL_SAMPLE_INTERVAL_MS = 1_000;\n\n/**\n * Closeable handle around the worker's Postgres pool. Decoupled from the\n * default factory so tests can swap in PGlite via `RunProjectionWorkerOverrides`.\n *\n * When the worker owns a real pg.Pool, the handle exposes a\n * {@link PoolInstrumentation} so the run loop can forward acquire-wait\n * samples to the host. Tests that swap in PGlite leave this undefined.\n */\nexport type ProjectionWorkerDatabaseHandle = {\n kysely: Kysely<Database>;\n poolInstrumentation?: PoolInstrumentation;\n shutdown(): Promise<void>;\n};\n\nexport type RunProjectionWorkerOverrides = {\n createDatabase?: (\n config: DbConfig,\n shardId: string,\n ) => Promise<ProjectionWorkerDatabaseHandle>;\n loadFactory?: Parameters<typeof buildProjectionStack>[0][\"loadFactory\"];\n beforeBuildStack?: (db: Kysely<Database>) => Promise<void>;\n};\n\nasync function defaultCreateDatabase(\n config: DbConfig,\n shardId: string,\n): Promise<ProjectionWorkerDatabaseHandle> {\n const { Kysely, PostgresDialect } = await import(\"kysely\");\n const pgModule = await import(\"pg\");\n const Pool = pgModule.default.Pool;\n const pool = new Pool({\n host: config.host,\n port: config.port,\n database: config.database,\n user: config.user,\n password: config.password,\n ssl: config.ssl ? { rejectUnauthorized: false } : undefined,\n application_name: config.applicationName ?? shardId,\n max: config.poolSize,\n connectionTimeoutMillis: config.connectionTimeoutMillis,\n idleTimeoutMillis: config.idleTimeoutMillis,\n });\n const poolInstrumentation = instrumentPgPool(pool, shardId);\n const kysely = new Kysely<Database>({\n dialect: new PostgresDialect({ pool }),\n });\n return {\n kysely,\n poolInstrumentation,\n async shutdown(): Promise<void> {\n try {\n await kysely.destroy();\n } catch {\n // best-effort\n }\n try {\n await pool.end();\n } catch {\n // best-effort\n }\n },\n };\n}\n\n/**\n * Drives the projection worker's message loop. Owns lifecycle of the\n * database handle and the projection stack. The default factory builds a\n * real Postgres pool; tests inject overrides for an in-process PGlite path.\n */\nexport function runProjectionWorker(\n parentPort: MessagePort,\n overrides: RunProjectionWorkerOverrides = {},\n): void {\n let shardId = \"\";\n let initCompleted = false;\n let stack: ProjectionStack | null = null;\n let database: ProjectionWorkerDatabaseHandle | null = null;\n let depthTimer: NodeJS.Timeout | null = null;\n let lastReportedDepth = -1;\n let poolSampleTimer: NodeJS.Timeout | null = null;\n let pendingPoolSamples: number[] = [];\n let detachPoolListener: (() => void) | null = null;\n\n function post(msg: ProjectionWorkerMessage): void {\n parentPort.postMessage(msg);\n }\n\n function startPoolReporter(instrumentation: PoolInstrumentation): void {\n detachPoolListener = instrumentation.onAcquire((durationMs) => {\n pendingPoolSamples.push(durationMs);\n });\n poolSampleTimer = setInterval(() => {\n if (pendingPoolSamples.length === 0) {\n return;\n }\n const durations = pendingPoolSamples;\n pendingPoolSamples = [];\n const stats = instrumentation.getStats();\n post({\n type: \"pool-acquire-samples\",\n shardId,\n poolName: instrumentation.name,\n timestamp: Date.now(),\n durations,\n size: stats.size,\n idle: stats.idle,\n waiting: stats.waiting,\n });\n }, POOL_SAMPLE_INTERVAL_MS);\n poolSampleTimer.unref();\n }\n\n function stopPoolReporter(): void {\n if (detachPoolListener) {\n detachPoolListener();\n detachPoolListener = null;\n }\n if (poolSampleTimer) {\n clearInterval(poolSampleTimer);\n poolSampleTimer = null;\n }\n pendingPoolSamples = [];\n }\n\n const logger = createForwardingLogger((msg) => post({ ...msg, shardId }));\n\n process.on(\"uncaughtException\", (err: unknown) => {\n try {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker uncaughtException\",\n args: [errorToInfo(err)],\n timestamp: Date.now(),\n });\n } catch {\n // nothing left to do\n }\n throw err;\n });\n\n process.on(\"unhandledRejection\", (reason: unknown) => {\n try {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker unhandledRejection\",\n args: [errorToInfo(reason)],\n timestamp: Date.now(),\n });\n } catch {\n // nothing left to do\n }\n });\n\n function startDepthReporter(intervalMs: number): void {\n if (intervalMs <= 0) {\n return;\n }\n depthTimer = setInterval(() => {\n if (!stack) {\n return;\n }\n const depth = stack.getChainDepth();\n if (depth === lastReportedDepth) {\n return;\n }\n lastReportedDepth = depth;\n post({\n type: \"chain-depth\",\n shardId,\n depth,\n timestamp: Date.now(),\n });\n }, intervalMs);\n depthTimer.unref();\n }\n\n function stopDepthReporter(): void {\n if (depthTimer) {\n clearInterval(depthTimer);\n depthTimer = null;\n }\n }\n\n async function handleInit(msg: ProjectionInitMessage): Promise<void> {\n shardId = msg.shardId;\n const createDb = overrides.createDatabase ?? defaultCreateDatabase;\n database = await createDb(msg.db, msg.shardId);\n if (overrides.beforeBuildStack) {\n await overrides.beforeBuildStack(database.kysely);\n }\n stack = await buildProjectionStack({\n init: msg,\n database: database.kysely,\n logger: new ConsoleLogger([`projection-shard:${msg.shardId}`]),\n loadFactory: overrides.loadFactory,\n events: {\n onReadReady: (event: JobReadReadyEvent) => {\n post({\n type: \"read-ready\",\n shardId,\n jobId: event.jobId,\n operations: event.operations,\n });\n },\n onReadModelIndexed: (event: ReadModelIndexedEvent) => {\n post({\n type: \"readmodel-indexed\",\n shardId,\n jobId: event.jobId,\n readModelName: event.readModelName,\n stage: event.stage,\n durationMs: event.durationMs,\n operationCount: event.operationCount,\n success: event.success,\n });\n },\n onBatchCompleted: (event: ReadModelBatchCompletedEvent) => {\n post({\n type: \"readmodel-batch-completed\",\n shardId,\n jobId: event.jobId,\n batchSize: event.batchSize,\n chainWaitDurationMs: event.chainWaitDurationMs,\n preReadyDurationMs: event.preReadyDurationMs,\n emitDurationMs: event.emitDurationMs,\n postReadyDurationMs: event.postReadyDurationMs,\n });\n },\n },\n });\n initCompleted = true;\n startDepthReporter(msg.chainDepthReportIntervalMs);\n if (database.poolInstrumentation) {\n startPoolReporter(database.poolInstrumentation);\n }\n logger.info(\"projection worker initialized: @shardId\", msg.shardId);\n post({ type: \"ready\", correlationId: msg.correlationId, shardId });\n }\n\n async function handleWriteReady(\n msg: Extract<ProjectionParentMessage, { type: \"write-ready\" }>,\n ): Promise<void> {\n if (!stack) {\n logger.warn(\n \"write-ready received before init on shard @shardId\",\n shardId,\n );\n return;\n }\n const event: JobWriteReadyEvent = {\n jobId: msg.jobId,\n operations: msg.operations,\n jobMeta: msg.jobMeta,\n collectionMemberships: msg.collectionMemberships,\n };\n await stack.relayWriteReady(event);\n }\n\n async function handleDrain(correlationId: string): Promise<void> {\n if (stack) {\n await stack.drain();\n }\n post({ type: \"drained\", correlationId, shardId });\n }\n\n async function shutdownStack(): Promise<void> {\n stopDepthReporter();\n stopPoolReporter();\n if (stack) {\n try {\n await stack.drain();\n } catch (error) {\n logger.warn(\n \"projection worker drain failed during shutdown: @error\",\n error,\n );\n }\n try {\n await stack.shutdown();\n } catch (error) {\n logger.warn(\"projection worker stack shutdown failed: @error\", error);\n }\n stack = null;\n }\n if (database) {\n await database.shutdown();\n database = null;\n }\n }\n\n function handleParentMessage(msg: ProjectionParentMessage): void {\n switch (msg.type) {\n case \"init\": {\n // Terminal: the worker has no stack, so it can project nothing. The\n // parent rejects startup on this message and terminates the thread;\n // exiting here would race the message and lose the cause.\n handleInit(msg).catch((err: unknown) => {\n post({\n type: \"init-failed\",\n correlationId: msg.correlationId,\n shardId,\n error: errorToInfo(err),\n });\n });\n break;\n }\n case \"write-ready\": {\n if (!initCompleted) {\n logger.warn(\n \"write-ready received before init on shard @shardId\",\n shardId,\n );\n break;\n }\n handleWriteReady(msg).catch((err: unknown) => {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker write-ready failed\",\n args: [errorToInfo(err)],\n timestamp: Date.now(),\n });\n });\n break;\n }\n case \"drain\": {\n handleDrain(msg.correlationId).catch((err: unknown) => {\n post({\n type: \"log\",\n shardId,\n level: \"error\",\n message: \"projection worker drain failed\",\n args: [errorToInfo(err)],\n timestamp: Date.now(),\n });\n });\n break;\n }\n case \"shutdown\": {\n logger.info(\"projection worker shutting down: @shardId\", shardId);\n void shutdownStack().finally(() => {\n post({\n type: \"log\",\n shardId,\n level: \"info\",\n message: \"projection worker shutdown\",\n args: [],\n timestamp: Date.now(),\n });\n process.exit(0);\n });\n break;\n }\n default: {\n const exhaustive: never = msg;\n void exhaustive;\n break;\n }\n }\n }\n\n parentPort.on(\"message\", handleParentMessage);\n\n const harness = {\n handleParentMessage,\n get initCompleted(): boolean {\n return initCompleted;\n },\n get shardId(): string {\n return shardId;\n },\n };\n (\n parentPort as unknown as { __reactorProjectionWorkerHarness?: unknown }\n ).__reactorProjectionWorkerHarness = harness;\n}\n","import { isMainThread, parentPort } from \"node:worker_threads\";\nimport { runProjectionWorker } from \"./run-projection-worker.js\";\n\nif (isMainThread || parentPort === null) {\n throw new Error(\"projection-worker entry.ts must be run as a worker thread\");\n}\n\nrunProjectionWorker(parentPort);\n"],"mappings":";;;;;;;;AAmFA,eAAe,kBACb,SACA,aACA,UACA,QACe;AACf,MAAK,MAAM,SAAS,SAAS;EAC3B,IAAI;AACJ,MAAI;AACF,YAAU,MAAM,YAAY,MAAM,KAAK;WAChC,OAAO;AACd,UAAO,MACL,kEACA,OACA,MACD;AACD,SAAM;;EAER,MAAM,CAAC,UAAU,SAAS,gBAAgB,OAAO;AACjD,MAAI,OAAO,WAAW,SAAS;AAC7B,UAAO,MACL,sEACA,OACA,OAAO,MACR;AACD,SAAM,OAAO;;;;AAKnB,SAAS,qBACP,MACA,UACA,gBACA,gBACA,YACY;AACZ,SAAQ,MAAR;EACE,KAAK,gBACH,QAAO,IAAI,mBAET,UACA,gBACA,gBACA,YACA,IAAI,oBAAoB,EAGxB,MACD;EAEH,KAAK,mBACH,QAAO,IAAI,sBACT,UACA,gBACA,YACA,IAAI,oBAAoB,CACzB;EAEH,SAAS;GACP,MAAM,aAAoB;AAC1B,SAAM,IAAI,MACR,qCAAqC,OAAO,WAAW,GACxD;;;;AAKP,eAAe,eACb,QACA,QACe;AACf,MAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,YAAY;AAClB,MAAI,OAAO,UAAU,SAAS,WAC5B;AAEF,MAAI;AACF,SAAM,UAAU,MAAM;WACf,OAAO;AACd,UAAO,MACL,0DACA,MAAM,MACN,MACD;AACD,SAAM;;;;AAKZ,eAAsB,qBACpB,SAC0B;CAC1B,MAAM,EAAE,MAAM,UAAU,cAAc,QAAQ,WAAW;CACzD,MAAM,cAAc,QAAQ,eAAe;CAE3C,MAAM,WAAW,IAAI,uBAAuB;AAC5C,OAAM,kBAAkB,KAAK,QAAQ,aAAa,UAAU,OAAO;CAEnE,MAAM,WAAW,aAAa,WAAW,eAAe;CACxD,MAAM,iBAAiB,IAAI,qBACzB,SACD;CAUD,MAAM,aAAa,IAAI,iBATD,IAAI,oBACxB,SACD,EASC,gBACA,UARoC;EACpC,cAAc;EACd,gBAAgB;EAChB,kBAAkB;EACnB,CAMA;AACD,OAAM,WAAW,SAAS;CAE1B,MAAM,iBAAiB,IAAI,qBACzB,SACD;AAKD,OAH0B,IAAI,kBAAkB,gBAAgB,EAC9D,cAAc,KACf,CAAC,CACsB,SAAS;AAEC,KAAI,0BACpC,eACD;CAID,MAAM,WAAW,KAAK,cAAc,KAAK,SACvC,qBACE,MACA,UACA,gBACA,gBACA,WACD,CACF;CACD,MAAM,YAAY,KAAK,eAAe,KAAK,SACzC,qBACE,MACA,UACA,gBACA,gBACA,WACD,CACF;AAED,OAAM,eAAe,CAAC,GAAG,UAAU,GAAG,UAAU,EAAE,OAAO;CAEzD,MAAM,WAAW,IAAI,UAAU;CAC/B,MAAM,gBAA+B,EAAE;AAEvC,eAAc,KACZ,SAAS,UACP,kBAAkB,iBACjB,IAAY,UAA6B;AACxC,SAAO,YAAY,MAAM;GAE5B,CACF;AACD,eAAc,KACZ,SAAS,UACP,kBAAkB,oBACjB,IAAY,UAAiC;AAC5C,SAAO,mBAAmB,MAAM;GAEnC,CACF;AACD,eAAc,KACZ,SAAS,UACP,kBAAkB,4BACjB,IAAY,UAAwC;AACnD,SAAO,iBAAiB,MAAM;GAEjC,CACF;CAED,MAAM,cAAc,IAAI,qBAAqB,UAAU,UAAU,UAAU;AAC3E,aAAY,OAAO;AAEnB,QAAO;EACL;EACA;EACA;EACA,MAAM,gBAAgB,OAA0C;AAC9D,SAAM,SAAS,KAAK,kBAAkB,iBAAiB,MAAM;;EAE/D,gBAAwB;AACtB,UAAO,YAAY,eAAe;;EAEpC,MAAM,QAAuB;AAC3B,SAAM,YAAY,OAAO;;EAE3B,WAA0B;AACxB,eAAY,MAAM;AAClB,QAAK,MAAM,SAAS,cAClB,QAAO;AAET,UAAO,QAAQ,SAAS;;EAE3B;;;;ACvQH,MAAM,0BAA0B;AAyBhC,eAAe,sBACb,QACA,SACyC;CACzC,MAAM,EAAE,QAAQ,oBAAoB,MAAM,OAAO;CAEjD,MAAM,QADW,MAAM,OAAO,OACR,QAAQ;CAC9B,MAAM,OAAO,IAAI,KAAK;EACpB,MAAM,OAAO;EACb,MAAM,OAAO;EACb,UAAU,OAAO;EACjB,MAAM,OAAO;EACb,UAAU,OAAO;EACjB,KAAK,OAAO,MAAM,EAAE,oBAAoB,OAAO,GAAG,KAAA;EAClD,kBAAkB,OAAO,mBAAmB;EAC5C,KAAK,OAAO;EACZ,yBAAyB,OAAO;EAChC,mBAAmB,OAAO;EAC3B,CAAC;CACF,MAAM,sBAAsB,iBAAiB,MAAM,QAAQ;CAC3D,MAAM,SAAS,IAAI,OAAiB,EAClC,SAAS,IAAI,gBAAgB,EAAE,MAAM,CAAC,EACvC,CAAC;AACF,QAAO;EACL;EACA;EACA,MAAM,WAA0B;AAC9B,OAAI;AACF,UAAM,OAAO,SAAS;WAChB;AAGR,OAAI;AACF,UAAM,KAAK,KAAK;WACV;;EAIX;;;;;;;AAQH,SAAgB,oBACd,YACA,YAA0C,EAAE,EACtC;CACN,IAAI,UAAU;CACd,IAAI,gBAAgB;CACpB,IAAI,QAAgC;CACpC,IAAI,WAAkD;CACtD,IAAI,aAAoC;CACxC,IAAI,oBAAoB;CACxB,IAAI,kBAAyC;CAC7C,IAAI,qBAA+B,EAAE;CACrC,IAAI,qBAA0C;CAE9C,SAAS,KAAK,KAAoC;AAChD,aAAW,YAAY,IAAI;;CAG7B,SAAS,kBAAkB,iBAA4C;AACrE,uBAAqB,gBAAgB,WAAW,eAAe;AAC7D,sBAAmB,KAAK,WAAW;IACnC;AACF,oBAAkB,kBAAkB;AAClC,OAAI,mBAAmB,WAAW,EAChC;GAEF,MAAM,YAAY;AAClB,wBAAqB,EAAE;GACvB,MAAM,QAAQ,gBAAgB,UAAU;AACxC,QAAK;IACH,MAAM;IACN;IACA,UAAU,gBAAgB;IAC1B,WAAW,KAAK,KAAK;IACrB;IACA,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,SAAS,MAAM;IAChB,CAAC;KACD,wBAAwB;AAC3B,kBAAgB,OAAO;;CAGzB,SAAS,mBAAyB;AAChC,MAAI,oBAAoB;AACtB,uBAAoB;AACpB,wBAAqB;;AAEvB,MAAI,iBAAiB;AACnB,iBAAc,gBAAgB;AAC9B,qBAAkB;;AAEpB,uBAAqB,EAAE;;CAGzB,MAAM,SAAS,wBAAwB,QAAQ,KAAK;EAAE,GAAG;EAAK;EAAS,CAAC,CAAC;AAEzE,SAAQ,GAAG,sBAAsB,QAAiB;AAChD,MAAI;AACF,QAAK;IACH,MAAM;IACN;IACA,OAAO;IACP,SAAS;IACT,MAAM,CAAC,YAAY,IAAI,CAAC;IACxB,WAAW,KAAK,KAAK;IACtB,CAAC;UACI;AAGR,QAAM;GACN;AAEF,SAAQ,GAAG,uBAAuB,WAAoB;AACpD,MAAI;AACF,QAAK;IACH,MAAM;IACN;IACA,OAAO;IACP,SAAS;IACT,MAAM,CAAC,YAAY,OAAO,CAAC;IAC3B,WAAW,KAAK,KAAK;IACtB,CAAC;UACI;GAGR;CAEF,SAAS,mBAAmB,YAA0B;AACpD,MAAI,cAAc,EAChB;AAEF,eAAa,kBAAkB;AAC7B,OAAI,CAAC,MACH;GAEF,MAAM,QAAQ,MAAM,eAAe;AACnC,OAAI,UAAU,kBACZ;AAEF,uBAAoB;AACpB,QAAK;IACH,MAAM;IACN;IACA;IACA,WAAW,KAAK,KAAK;IACtB,CAAC;KACD,WAAW;AACd,aAAW,OAAO;;CAGpB,SAAS,oBAA0B;AACjC,MAAI,YAAY;AACd,iBAAc,WAAW;AACzB,gBAAa;;;CAIjB,eAAe,WAAW,KAA2C;AACnE,YAAU,IAAI;AAEd,aAAW,OADM,UAAU,kBAAkB,uBACnB,IAAI,IAAI,IAAI,QAAQ;AAC9C,MAAI,UAAU,iBACZ,OAAM,UAAU,iBAAiB,SAAS,OAAO;AAEnD,UAAQ,MAAM,qBAAqB;GACjC,MAAM;GACN,UAAU,SAAS;GACnB,QAAQ,IAAI,cAAc,CAAC,oBAAoB,IAAI,UAAU,CAAC;GAC9D,aAAa,UAAU;GACvB,QAAQ;IACN,cAAc,UAA6B;AACzC,UAAK;MACH,MAAM;MACN;MACA,OAAO,MAAM;MACb,YAAY,MAAM;MACnB,CAAC;;IAEJ,qBAAqB,UAAiC;AACpD,UAAK;MACH,MAAM;MACN;MACA,OAAO,MAAM;MACb,eAAe,MAAM;MACrB,OAAO,MAAM;MACb,YAAY,MAAM;MAClB,gBAAgB,MAAM;MACtB,SAAS,MAAM;MAChB,CAAC;;IAEJ,mBAAmB,UAAwC;AACzD,UAAK;MACH,MAAM;MACN;MACA,OAAO,MAAM;MACb,WAAW,MAAM;MACjB,qBAAqB,MAAM;MAC3B,oBAAoB,MAAM;MAC1B,gBAAgB,MAAM;MACtB,qBAAqB,MAAM;MAC5B,CAAC;;IAEL;GACF,CAAC;AACF,kBAAgB;AAChB,qBAAmB,IAAI,2BAA2B;AAClD,MAAI,SAAS,oBACX,mBAAkB,SAAS,oBAAoB;AAEjD,SAAO,KAAK,2CAA2C,IAAI,QAAQ;AACnE,OAAK;GAAE,MAAM;GAAS,eAAe,IAAI;GAAe;GAAS,CAAC;;CAGpE,eAAe,iBACb,KACe;AACf,MAAI,CAAC,OAAO;AACV,UAAO,KACL,sDACA,QACD;AACD;;EAEF,MAAM,QAA4B;GAChC,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,SAAS,IAAI;GACb,uBAAuB,IAAI;GAC5B;AACD,QAAM,MAAM,gBAAgB,MAAM;;CAGpC,eAAe,YAAY,eAAsC;AAC/D,MAAI,MACF,OAAM,MAAM,OAAO;AAErB,OAAK;GAAE,MAAM;GAAW;GAAe;GAAS,CAAC;;CAGnD,eAAe,gBAA+B;AAC5C,qBAAmB;AACnB,oBAAkB;AAClB,MAAI,OAAO;AACT,OAAI;AACF,UAAM,MAAM,OAAO;YACZ,OAAO;AACd,WAAO,KACL,0DACA,MACD;;AAEH,OAAI;AACF,UAAM,MAAM,UAAU;YACf,OAAO;AACd,WAAO,KAAK,mDAAmD,MAAM;;AAEvE,WAAQ;;AAEV,MAAI,UAAU;AACZ,SAAM,SAAS,UAAU;AACzB,cAAW;;;CAIf,SAAS,oBAAoB,KAAoC;AAC/D,UAAQ,IAAI,MAAZ;GACE,KAAK;AAIH,eAAW,IAAI,CAAC,OAAO,QAAiB;AACtC,UAAK;MACH,MAAM;MACN,eAAe,IAAI;MACnB;MACA,OAAO,YAAY,IAAI;MACxB,CAAC;MACF;AACF;GAEF,KAAK;AACH,QAAI,CAAC,eAAe;AAClB,YAAO,KACL,sDACA,QACD;AACD;;AAEF,qBAAiB,IAAI,CAAC,OAAO,QAAiB;AAC5C,UAAK;MACH,MAAM;MACN;MACA,OAAO;MACP,SAAS;MACT,MAAM,CAAC,YAAY,IAAI,CAAC;MACxB,WAAW,KAAK,KAAK;MACtB,CAAC;MACF;AACF;GAEF,KAAK;AACH,gBAAY,IAAI,cAAc,CAAC,OAAO,QAAiB;AACrD,UAAK;MACH,MAAM;MACN;MACA,OAAO;MACP,SAAS;MACT,MAAM,CAAC,YAAY,IAAI,CAAC;MACxB,WAAW,KAAK,KAAK;MACtB,CAAC;MACF;AACF;GAEF,KAAK;AACH,WAAO,KAAK,6CAA6C,QAAQ;AAC5D,mBAAe,CAAC,cAAc;AACjC,UAAK;MACH,MAAM;MACN;MACA,OAAO;MACP,SAAS;MACT,MAAM,EAAE;MACR,WAAW,KAAK,KAAK;MACtB,CAAC;AACF,aAAQ,KAAK,EAAE;MACf;AACF;GAEF,QAGE;;;AAKN,YAAW,GAAG,WAAW,oBAAoB;AAY3C,YACA,mCAXc;EACd;EACA,IAAI,gBAAyB;AAC3B,UAAO;;EAET,IAAI,UAAkB;AACpB,UAAO;;EAEV;;;;AClZH,IAAI,gBAAgB,eAAe,KACjC,OAAM,IAAI,MAAM,4DAA4D;AAG9E,oBAAoB,WAAW"}
@@ -1,4 +1,5 @@
1
1
  import { n as ReactorEventTypes } from "./types-DMKLa0Ok.js";
2
+ import { t as fromErrorInfo } from "./error-info-Cpu4OY3o.js";
2
3
  import { childLogger } from "document-model";
3
4
  import { randomUUID } from "node:crypto";
4
5
  //#region src/projection/projection-shard-manager.ts
@@ -100,6 +101,7 @@ var ProjectionShardManager = class {
100
101
  transport.on("exit", state.onExit);
101
102
  this.shards.push(state);
102
103
  const correlationId = randomUUID();
104
+ state.initCorrelationId = correlationId;
103
105
  const initPromise = new Promise((resolve, reject) => {
104
106
  const timer = setTimeout(() => {
105
107
  this.initPromises.delete(correlationId);
@@ -144,11 +146,13 @@ var ProjectionShardManager = class {
144
146
  }
145
147
  this.isRunning = false;
146
148
  }
149
+ /** Waits on ready shards only; `handleTransportExit` releases one that dies. */
147
150
  async drain() {
148
- if (this.shards.length === 0) return;
151
+ const readyShards = this.shards.filter((s) => s.ready);
152
+ if (readyShards.length === 0) return;
149
153
  const drainTimeoutMs = this.config.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
150
154
  const correlationId = randomUUID();
151
- const remaining = new Set(this.shards.map((s) => s.shardId));
155
+ const remaining = new Set(readyShards.map((s) => s.shardId));
152
156
  const promise = new Promise((resolve, reject) => {
153
157
  const timer = setTimeout(() => {
154
158
  this.pendingDrains.delete(correlationId);
@@ -161,12 +165,16 @@ var ProjectionShardManager = class {
161
165
  timer
162
166
  });
163
167
  });
164
- for (const shard of this.shards) shard.transport.postMessage({
168
+ for (const shard of readyShards) shard.transport.postMessage({
165
169
  type: "drain",
166
170
  correlationId
167
171
  });
168
172
  await promise;
169
173
  }
174
+ /** Emits JOB_READ_READY on the host bus; an `onReadReady` hook awaits this. */
175
+ emitReadReady(event) {
176
+ return this.hostBus.emit(ReactorEventTypes.JOB_READ_READY, event);
177
+ }
170
178
  getChainDepth() {
171
179
  let total = 0;
172
180
  for (const shard of this.shards) total += shard.lastDepth;
@@ -182,6 +190,7 @@ var ProjectionShardManager = class {
182
190
  async shutdown() {
183
191
  this.isShuttingDown = true;
184
192
  this.stop();
193
+ for (const shard of this.shards) this.failPendingInit(shard, /* @__PURE__ */ new Error(`projection shard ${shard.shardId} was shut down before becoming ready`));
185
194
  const graceMs = this.config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;
186
195
  const correlationId = randomUUID();
187
196
  for (const shard of this.shards) try {
@@ -263,6 +272,10 @@ var ProjectionShardManager = class {
263
272
  case "ready":
264
273
  this.handleReady(shard, msg.correlationId);
265
274
  return;
275
+ case "init-failed":
276
+ this.logger.error("projection shard @shardId failed to initialize: @error", shard.shardId, msg.error.message);
277
+ this.failPendingInit(shard, fromErrorInfo(msg.error));
278
+ return;
266
279
  case "read-ready":
267
280
  this.relayReadReady({
268
281
  jobId: msg.jobId,
@@ -334,8 +347,26 @@ var ProjectionShardManager = class {
334
347
  });
335
348
  shard.poolInstrumentation.pushSamples(msg.durations);
336
349
  }
350
+ /**
351
+ * Settles a shard's unresolved `init` with `error`, clearing its timer.
352
+ * Returns false when the shard has no init outstanding, which is every
353
+ * failure after startup. `startup()` awaits all init promises together, so
354
+ * the rejection is always attached.
355
+ */
356
+ failPendingInit(shard, error) {
357
+ const correlationId = shard.initCorrelationId;
358
+ if (correlationId === void 0) return false;
359
+ shard.initCorrelationId = void 0;
360
+ const pending = this.initPromises.get(correlationId);
361
+ if (!pending) return false;
362
+ this.initPromises.delete(correlationId);
363
+ clearTimeout(pending.timer);
364
+ pending.reject(error);
365
+ return true;
366
+ }
337
367
  handleReady(shard, correlationId) {
338
368
  shard.ready = true;
369
+ shard.initCorrelationId = void 0;
339
370
  const pending = this.initPromises.get(correlationId);
340
371
  if (!pending) return;
341
372
  this.initPromises.delete(correlationId);
@@ -345,12 +376,15 @@ var ProjectionShardManager = class {
345
376
  handleDrained(msg) {
346
377
  const pending = this.pendingDrains.get(msg.correlationId);
347
378
  if (!pending) return;
348
- pending.remaining.delete(msg.shardId);
349
- if (pending.remaining.size === 0) {
350
- this.pendingDrains.delete(msg.correlationId);
351
- clearTimeout(pending.timer);
352
- pending.resolve();
353
- }
379
+ this.releaseDrain(msg.correlationId, pending, msg.shardId);
380
+ }
381
+ /** Removes `shardId` from a pending drain and settles it once no shard remains. */
382
+ releaseDrain(correlationId, pending, shardId) {
383
+ pending.remaining.delete(shardId);
384
+ if (pending.remaining.size > 0) return;
385
+ this.pendingDrains.delete(correlationId);
386
+ clearTimeout(pending.timer);
387
+ pending.resolve();
354
388
  }
355
389
  handleLog(shard, msg) {
356
390
  switch (msg.level) {
@@ -371,6 +405,7 @@ var ProjectionShardManager = class {
371
405
  }
372
406
  handleTransportError(shard, err) {
373
407
  this.logger.error("projection shard transport error @shardId: @error", shard.shardId, err);
408
+ if (this.failPendingInit(shard, err)) return;
374
409
  if (!this.isShuttingDown) this.config.onShardFatal?.(shard.shardId, err);
375
410
  }
376
411
  handleTransportExit(shard, code) {
@@ -378,12 +413,19 @@ var ProjectionShardManager = class {
378
413
  shard.ready = false;
379
414
  const abandoned = [...shard.pendingCoordinates.keys()];
380
415
  shard.pendingCoordinates.clear();
416
+ for (const [correlationId, pending] of this.pendingDrains) this.releaseDrain(correlationId, pending, shard.shardId);
417
+ if (this.failPendingInit(shard, /* @__PURE__ */ new Error(`projection shard ${shard.shardId} exited with code ${code} before becoming ready`))) return;
381
418
  if (!wasReady || this.isShuttingDown) return;
382
419
  this.logger.error("projection shard exited unexpectedly @shardId code=@code, abandoning @count in-flight job(s): @jobIds", shard.shardId, code, abandoned.length, abandoned.join(", "));
383
420
  this.config.onShardFatal?.(shard.shardId, /* @__PURE__ */ new Error(`projection shard ${shard.shardId} exited with code ${code}; ${abandoned.length} in-flight job(s) will never be projected` + (abandoned.length > 0 ? `: ${abandoned.join(", ")}` : "")));
384
421
  }
385
422
  relayReadReady(event) {
386
- this.hostBus.emit(ReactorEventTypes.JOB_READ_READY, event).catch((err) => this.logger.error("host JOB_READ_READY emit failed for job @jobId: @error", event.jobId, err));
423
+ const hook = this.config.onReadReady;
424
+ if (hook) {
425
+ hook(event);
426
+ return;
427
+ }
428
+ this.emitReadReady(event).catch((err) => this.logger.error("host JOB_READ_READY emit failed for job @jobId: @error", event.jobId, err));
387
429
  }
388
430
  relayReadModelIndexed(event) {
389
431
  this.hostBus.emit(ReactorEventTypes.READMODEL_INDEXED, event).catch((err) => this.logger.error("host READMODEL_INDEXED emit failed for job @jobId: @error", event.jobId, err));
@@ -395,4 +437,4 @@ var ProjectionShardManager = class {
395
437
  //#endregion
396
438
  export { ProjectionShardManager };
397
439
 
398
- //# sourceMappingURL=projection-shard-manager-D6KYcEZi.js.map
440
+ //# sourceMappingURL=projection-shard-manager-CPQc5XHL.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"projection-shard-manager-CPQc5XHL.js","names":[],"sources":["../src/projection/projection-shard-manager.ts"],"sourcesContent":["import type { OperationWithContext } from \"@powerhousedao/shared/document-model\";\nimport { childLogger, type ILogger } from \"document-model\";\nimport { randomUUID } from \"node:crypto\";\nimport { fromErrorInfo } from \"../executor/worker/error-info.js\";\nimport type { IEventBus } from \"../events/interfaces.js\";\nimport {\n ReactorEventTypes,\n type JobReadReadyEvent,\n type JobWriteReadyEvent,\n type ReadModelBatchCompletedEvent,\n type ReadModelIndexedEvent,\n type Unsubscribe,\n} from \"../events/types.js\";\nimport type {\n IReadModel,\n IReadModelCoordinator,\n} from \"../read-models/interfaces.js\";\nimport type { IConsistencyTracker } from \"../shared/consistency-tracker.js\";\nimport type { ConsistencyCoordinate } from \"../shared/types.js\";\nimport type { ForwardingPoolInstrumentation } from \"../storage/pool-instrumentation.js\";\nimport type {\n BuiltInReadModelKind,\n ChainDepthReport,\n DbConfig,\n ModelManifestEntry,\n ProjectionDrainedMessage,\n ProjectionInitMessage,\n ProjectionParentMessage,\n ProjectionPoolAcquireSamplesMessage,\n ProjectionReadModelIndexedMessage,\n ProjectionWorkerMessage,\n} from \"./protocol.js\";\nimport type { IProjectionTransport } from \"./transport.js\";\n\nconst DEFAULT_INIT_TIMEOUT_MS = 30_000;\nconst DEFAULT_SHUTDOWN_GRACE_MS = 5_000;\nconst DEFAULT_DRAIN_TIMEOUT_MS = 30_000;\nconst DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS = 250;\n\nconst FNV_OFFSET_BASIS = 0x811c9dc5;\nconst FNV_PRIME = 0x01000193;\n\nfunction bucketFor(documentId: string, numWorkers: number): number {\n if (numWorkers < 1) {\n throw new Error(`bucketFor: numWorkers must be >= 1 (got ${numWorkers})`);\n }\n let hash = FNV_OFFSET_BASIS;\n for (let i = 0; i < documentId.length; i++) {\n hash ^= documentId.charCodeAt(i);\n hash = Math.imul(hash, FNV_PRIME);\n }\n return (hash >>> 0) % numWorkers;\n}\n\n/**\n * Maps operations to the coordinates a consistency tracker is keyed by.\n * Same shape `BaseReadModel.updateConsistencyTracker` produces in-process.\n */\nfunction toConsistencyCoordinates(\n operations: OperationWithContext[],\n): ConsistencyCoordinate[] {\n const coordinates: ConsistencyCoordinate[] = [];\n for (let i = 0; i < operations.length; i++) {\n const item = operations[i]!;\n coordinates.push({\n documentId: item.context.documentId,\n scope: item.context.scope,\n branch: item.context.branch,\n operationIndex: item.operation.index,\n });\n }\n return coordinates;\n}\n\n/**\n * Factory that builds one projection-worker transport. Mirrors\n * `WorkerFactory` from the executor pool: lets tests inject fake\n * transports without spawning real worker threads.\n */\nexport type ProjectionWorkerFactory = (\n shardIndex: number,\n shardId: string,\n) => IProjectionTransport;\n\nexport type ProjectionShardHooks = {\n /** Takes over the host-bus JOB_READ_READY emit; call `emitReadReady` to emit. */\n onReadReady?: (event: JobReadReadyEvent) => void;\n /**\n * Fired when a shard errors, when it exits after having been ready, and\n * for every JOB_WRITE_READY dropped because its shard is not ready.\n *\n * There is no respawn path: once a shard stops being ready it never\n * projects again, so buffering the dropped work would only grow without\n * bound. A host that cares wires this to its shutdown path, so the process\n * restarts and each read model catches up from `ViewState.lastOrdinal` in\n * `BaseReadModel.init`. May fire repeatedly — handlers must be idempotent.\n */\n onShardFatal?: (shardId: string, reason: Error) => void;\n};\n\nexport type ProjectionShardManagerConfig = ProjectionShardHooks & {\n shardCount: number;\n db: DbConfig;\n models: ModelManifestEntry[];\n preReadyKinds: BuiltInReadModelKind[];\n postReadyKinds: BuiltInReadModelKind[];\n factory: ProjectionWorkerFactory;\n logger: ILogger;\n hostBus: IEventBus;\n initTimeoutMs?: number;\n shutdownGraceMs?: number;\n drainTimeoutMs?: number;\n chainDepthReportIntervalMs?: number;\n /**\n * Host-side forwarding instrumentations indexed by shard index. The\n * manager routes each shard's `pool-acquire-samples` message to the\n * matching forwarder so the host's OpenTelemetry instrumentation records\n * acquire-wait latencies as if each shard's pg.Pool were local.\n */\n poolInstrumentations?: ForwardingPoolInstrumentation[];\n /**\n * The host's consistency trackers for the built-in read models the shards\n * run, keyed by kind. The keys double as read-model names (see\n * `DOCUMENT_VIEW_READ_MODEL` / `DOCUMENT_INDEXER_READ_MODEL` in\n * `read-models/names.js`), which is how a relayed `readmodel-indexed`\n * message is matched to a tracker.\n *\n * The host's copies of these read models are never fed an operation under\n * sharding, so without this every read carrying a consistency token waits\n * on a tracker that can never advance — `ConsistencyTracker.waitFor` arms\n * no timer when `timeoutMs` is undefined, which is what every\n * `document-view` call site passes. The worker committed those rows to the\n * same tables the host reads from, so advancing here is exact, not a fudge.\n */\n consistencyTrackers?: Partial<\n Record<BuiltInReadModelKind, IConsistencyTracker>\n >;\n};\n\ntype ShardState = {\n shardIndex: number;\n shardId: string;\n transport: IProjectionTransport;\n ready: boolean;\n lastDepth: number;\n lastDepthAt: number;\n poolInstrumentation?: ForwardingPoolInstrumentation;\n /**\n * Consistency coordinates of the jobs this shard is still projecting,\n * keyed by jobId. Relayed `readmodel-indexed` messages carry only a count,\n * so the host keeps the coordinates here to advance its trackers from\n * them; the operations themselves are not retained. Empty unless\n * `consistencyTrackers` is configured; entries live from dispatch to\n * `readmodel-batch-completed`, and are dropped when the shard exits.\n */\n pendingCoordinates: Map<string, ConsistencyCoordinate[]>;\n /**\n * Correlation id of this shard's `init` while it is unsettled; cleared once\n * the shard reports ready, fails, or dies. Lets a transport error or a\n * premature exit settle the shard's own pending init instead of leaving\n * `startup()` to time out on a worker that will never answer.\n */\n initCorrelationId?: string;\n onMessage: (msg: ProjectionWorkerMessage) => void;\n onError: (err: Error) => void;\n onExit: (code: number) => void;\n};\n\ntype PendingDrain = {\n resolve: () => void;\n reject: (err: Error) => void;\n remaining: Set<string>;\n timer: NodeJS.Timeout;\n};\n\n/**\n * Host-side coordinator for N sharded projection workers.\n *\n * Implements {@link IReadModelCoordinator} so it slots into the same\n * `readModelCoordinator` field on the reactor module as the in-process\n * {@link ReadModelCoordinator}. The host subscribes to JOB_WRITE_READY\n * exactly once; events are routed to a single shard by\n * `bucketFor(documentId, shardCount)`. Each worker maintains the\n * per-queueKey serial chain locally and forwards JOB_READ_READY and\n * READMODEL_* events back to the host for the rest of the reactor (sync\n * manager, awaiters, observers) to consume on the host bus.\n *\n * @see Sharded projection workers sub-feature brief\n * (Powerhouse board wiki id: eb26f01f-8f68-4918-a6f6-ac7a4679b533)\n */\nexport class ProjectionShardManager implements IReadModelCoordinator {\n readonly readModels: IReadModel[] = [];\n\n private readonly config: ProjectionShardManagerConfig;\n private readonly logger: ILogger;\n private readonly hostBus: IEventBus;\n private readonly shards: ShardState[] = [];\n private readonly initPromises = new Map<\n string,\n { resolve: () => void; reject: (err: Error) => void; timer: NodeJS.Timeout }\n >();\n private readonly pendingDrains = new Map<string, PendingDrain>();\n private readonly trackersByReadModelName = new Map<\n string,\n IConsistencyTracker\n >();\n private hostSubscription?: Unsubscribe;\n private isRunning = false;\n private started = false;\n private isShuttingDown = false;\n\n constructor(config: ProjectionShardManagerConfig) {\n if (config.shardCount < 1) {\n throw new Error(\n `ProjectionShardManager: shardCount must be >= 1 (got ${config.shardCount})`,\n );\n }\n this.config = config;\n this.logger = childLogger([\"reactor\", \"projection-shard-manager\"]);\n this.hostBus = config.hostBus;\n const trackers = config.consistencyTrackers ?? {};\n for (const kind of Object.keys(trackers) as BuiltInReadModelKind[]) {\n const tracker = trackers[kind];\n if (tracker) {\n this.trackersByReadModelName.set(kind, tracker);\n }\n }\n }\n\n async startup(): Promise<void> {\n if (this.started) {\n return;\n }\n this.started = true;\n const initTimeoutMs = this.config.initTimeoutMs ?? DEFAULT_INIT_TIMEOUT_MS;\n const reportIntervalMs =\n this.config.chainDepthReportIntervalMs ??\n DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS;\n\n const initPromises: Promise<void>[] = [];\n for (let i = 0; i < this.config.shardCount; i++) {\n const shardId = `projection-shard-${i}`;\n const transport = this.config.factory(i, shardId);\n const state: ShardState = {\n shardIndex: i,\n shardId,\n transport,\n ready: false,\n lastDepth: 0,\n lastDepthAt: 0,\n poolInstrumentation: this.config.poolInstrumentations?.[i],\n pendingCoordinates: new Map(),\n onMessage: (msg) => this.handleWorkerMessage(state, msg),\n onError: (err) => this.handleTransportError(state, err),\n onExit: (code) => this.handleTransportExit(state, code),\n };\n transport.on(\"message\", state.onMessage);\n transport.on(\"error\", state.onError);\n transport.on(\"exit\", state.onExit);\n this.shards.push(state);\n\n const correlationId = randomUUID();\n state.initCorrelationId = correlationId;\n const initPromise = new Promise<void>((resolve, reject) => {\n const timer = setTimeout(() => {\n this.initPromises.delete(correlationId);\n reject(\n new Error(\n `projection shard ${shardId} did not become ready within ${initTimeoutMs}ms`,\n ),\n );\n }, initTimeoutMs);\n this.initPromises.set(correlationId, { resolve, reject, timer });\n });\n\n const init: ProjectionInitMessage = {\n type: \"init\",\n correlationId,\n shardId,\n shardIndex: i,\n shardCount: this.config.shardCount,\n db: this.config.db,\n models: this.config.models,\n preReadyKinds: this.config.preReadyKinds,\n postReadyKinds: this.config.postReadyKinds,\n chainDepthReportIntervalMs: reportIntervalMs,\n };\n transport.postMessage(init);\n initPromises.push(initPromise);\n }\n\n await Promise.all(initPromises);\n this.logger.info(\n \"projection shard manager ready: @count shards\",\n this.shards.length,\n );\n }\n\n start(): void {\n if (this.isRunning) {\n return;\n }\n this.hostSubscription = this.hostBus.subscribe(\n ReactorEventTypes.JOB_WRITE_READY,\n (_t: number, event: JobWriteReadyEvent) => {\n this.routeWriteReady(event);\n },\n );\n this.isRunning = true;\n }\n\n stop(): void {\n if (!this.isRunning) {\n return;\n }\n if (this.hostSubscription) {\n this.hostSubscription();\n this.hostSubscription = undefined;\n }\n this.isRunning = false;\n }\n\n /** Waits on ready shards only; `handleTransportExit` releases one that dies. */\n async drain(): Promise<void> {\n const readyShards = this.shards.filter((s) => s.ready);\n if (readyShards.length === 0) {\n return;\n }\n const drainTimeoutMs =\n this.config.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;\n const correlationId = randomUUID();\n const remaining = new Set(readyShards.map((s) => s.shardId));\n const promise = new Promise<void>((resolve, reject) => {\n const timer = setTimeout(() => {\n this.pendingDrains.delete(correlationId);\n reject(\n new Error(\n `projection shards did not drain within ${drainTimeoutMs}ms (remaining: ${[\n ...remaining,\n ].join(\", \")})`,\n ),\n );\n }, drainTimeoutMs);\n this.pendingDrains.set(correlationId, {\n resolve,\n reject,\n remaining,\n timer,\n });\n });\n for (const shard of readyShards) {\n shard.transport.postMessage({ type: \"drain\", correlationId });\n }\n await promise;\n }\n\n /** Emits JOB_READ_READY on the host bus; an `onReadReady` hook awaits this. */\n emitReadReady(event: JobReadReadyEvent): Promise<void> {\n return this.hostBus.emit(ReactorEventTypes.JOB_READ_READY, event);\n }\n\n getChainDepth(): number {\n let total = 0;\n for (const shard of this.shards) {\n total += shard.lastDepth;\n }\n return total;\n }\n\n getShardDepths(): ChainDepthReport[] {\n return this.shards.map((shard) => ({\n shardId: shard.shardId,\n depth: shard.lastDepth,\n timestamp: shard.lastDepthAt,\n }));\n }\n\n async shutdown(): Promise<void> {\n // Workers exit as a consequence of this call, so their `exit` events are\n // expected from here on and must not be reported as fatal.\n this.isShuttingDown = true;\n this.stop();\n // An init timer left armed by a shard that never reported keeps the\n // process alive for the whole init timeout after shutdown returns.\n for (const shard of this.shards) {\n this.failPendingInit(\n shard,\n new Error(\n `projection shard ${shard.shardId} was shut down before becoming ready`,\n ),\n );\n }\n const graceMs = this.config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;\n const correlationId = randomUUID();\n for (const shard of this.shards) {\n try {\n const msg: ProjectionParentMessage = {\n type: \"shutdown\",\n correlationId,\n graceMs,\n };\n shard.transport.postMessage(msg);\n } catch (err) {\n this.logger.warn(\n \"projection shard postMessage(shutdown) failed for @shardId: @error\",\n shard.shardId,\n err,\n );\n }\n }\n const terminationDeadline = Date.now() + graceMs;\n while (\n this.shards.some((s) => s.ready) &&\n Date.now() < terminationDeadline\n ) {\n await new Promise<void>((resolve) => setTimeout(resolve, 25));\n }\n for (const shard of this.shards) {\n try {\n await shard.transport.terminate();\n } catch (err) {\n this.logger.warn(\n \"projection shard terminate failed for @shardId: @error\",\n shard.shardId,\n err,\n );\n }\n shard.transport.off(\"message\", shard.onMessage);\n shard.transport.off(\"error\", shard.onError);\n shard.transport.off(\"exit\", shard.onExit);\n }\n this.shards.length = 0;\n }\n\n private routeWriteReady(event: JobWriteReadyEvent): void {\n if (event.operations.length === 0) {\n // No shard can be selected (there is no documentId) and no read model\n // has work to do, but JOB_READ_READY is the job's terminal signal:\n // InMemoryJobTracker and every JobAwaiter hang without it. Mirrors\n // ReadModelCoordinator.emitEmptyReadReady.\n this.relayReadReady({ jobId: event.jobId, operations: [] });\n this.relayBatchCompleted({\n jobId: event.jobId,\n batchSize: 0,\n chainWaitDurationMs: 0,\n preReadyDurationMs: 0,\n emitDurationMs: 0,\n postReadyDurationMs: 0,\n });\n return;\n }\n const documentId = event.operations[0]!.context.documentId;\n const index = bucketFor(documentId, this.shards.length);\n const shard = this.shards[index]!;\n if (!shard.ready) {\n this.dropWriteReady(shard, event, documentId);\n return;\n }\n if (this.trackersByReadModelName.size > 0) {\n shard.pendingCoordinates.set(\n event.jobId,\n toConsistencyCoordinates(event.operations),\n );\n }\n shard.transport.postMessage({\n type: \"write-ready\",\n jobId: event.jobId,\n operations: event.operations,\n jobMeta: event.jobMeta,\n collectionMemberships: event.collectionMemberships,\n });\n }\n\n /**\n * A shard stops being ready only when it dies, and nothing respawns it, so\n * this job's projection is genuinely lost. Buffering would grow without\n * bound behind a shard that never comes back, so the batch is dropped —\n * loudly, naming the job and document, and through `onShardFatal` so a host\n * can restart rather than serve stale read models.\n *\n * JOB_FAILED is deliberately not emitted: every other emitter uses it for a\n * job whose operations were *not* written (see\n * `executor/job-result-handler.ts`), and these were written and are\n * durable. Marking the job FAILED would invite the caller to re-submit a\n * write that already landed.\n */\n private dropWriteReady(\n shard: ShardState,\n event: JobWriteReadyEvent,\n documentId: string,\n ): void {\n const reason = new Error(\n `projection shard ${shard.shardId} is not ready: JOB_WRITE_READY for job ${event.jobId} ` +\n `(document ${documentId}, ${event.operations.length} operation(s)) was dropped and will ` +\n `never be projected; the operations are written and durable, but this shard's read models ` +\n `are now behind`,\n );\n this.logger.error(\n \"dropping JOB_WRITE_READY for job @jobId on shard @shardId: @error\",\n event.jobId,\n shard.shardId,\n reason,\n );\n this.config.onShardFatal?.(shard.shardId, reason);\n }\n\n private handleWorkerMessage(\n shard: ShardState,\n msg: ProjectionWorkerMessage,\n ): void {\n switch (msg.type) {\n case \"ready\":\n this.handleReady(shard, msg.correlationId);\n return;\n case \"init-failed\":\n this.logger.error(\n \"projection shard @shardId failed to initialize: @error\",\n shard.shardId,\n msg.error.message,\n );\n this.failPendingInit(shard, fromErrorInfo(msg.error));\n return;\n case \"read-ready\":\n this.relayReadReady({\n jobId: msg.jobId,\n operations: msg.operations,\n });\n return;\n case \"readmodel-indexed\":\n this.advanceConsistencyTrackers(shard, msg);\n this.relayReadModelIndexed({\n jobId: msg.jobId,\n readModelName: msg.readModelName,\n stage: msg.stage,\n durationMs: msg.durationMs,\n operationCount: msg.operationCount,\n success: msg.success,\n });\n return;\n case \"readmodel-batch-completed\":\n shard.pendingCoordinates.delete(msg.jobId);\n this.relayBatchCompleted({\n jobId: msg.jobId,\n batchSize: msg.batchSize,\n chainWaitDurationMs: msg.chainWaitDurationMs,\n preReadyDurationMs: msg.preReadyDurationMs,\n emitDurationMs: msg.emitDurationMs,\n postReadyDurationMs: msg.postReadyDurationMs,\n });\n return;\n case \"chain-depth\":\n shard.lastDepth = msg.depth;\n shard.lastDepthAt = msg.timestamp;\n return;\n case \"pool-acquire-samples\":\n this.handlePoolAcquireSamples(shard, msg);\n return;\n case \"drained\":\n this.handleDrained(msg);\n return;\n case \"log\":\n this.handleLog(shard, msg);\n return;\n default: {\n const exhaustive: never = msg;\n void exhaustive;\n return;\n }\n }\n }\n\n /**\n * Advances the host's tracker for the read model the shard just indexed.\n *\n * The shard writes to the same tables the host reads, so once it reports a\n * successful index the host's read path really is consistent to those\n * coordinates. Gated on `success` for parity with\n * `BaseReadModel.indexOperations`, which updates its tracker only after\n * `commitOperations` returns. The worker posts this before its\n * JOB_READ_READY, matching the in-process ordering.\n */\n private advanceConsistencyTrackers(\n shard: ShardState,\n msg: ProjectionReadModelIndexedMessage,\n ): void {\n if (!msg.success) {\n return;\n }\n const tracker = this.trackersByReadModelName.get(msg.readModelName);\n if (!tracker) {\n return;\n }\n const coordinates = shard.pendingCoordinates.get(msg.jobId);\n if (!coordinates || coordinates.length === 0) {\n return;\n }\n tracker.update(coordinates);\n }\n\n private handlePoolAcquireSamples(\n shard: ShardState,\n msg: ProjectionPoolAcquireSamplesMessage,\n ): void {\n if (!shard.poolInstrumentation) {\n return;\n }\n shard.poolInstrumentation.updateStats({\n size: msg.size,\n idle: msg.idle,\n waiting: msg.waiting,\n });\n shard.poolInstrumentation.pushSamples(msg.durations);\n }\n\n /**\n * Settles a shard's unresolved `init` with `error`, clearing its timer.\n * Returns false when the shard has no init outstanding, which is every\n * failure after startup. `startup()` awaits all init promises together, so\n * the rejection is always attached.\n */\n private failPendingInit(shard: ShardState, error: Error): boolean {\n const correlationId = shard.initCorrelationId;\n if (correlationId === undefined) {\n return false;\n }\n shard.initCorrelationId = undefined;\n const pending = this.initPromises.get(correlationId);\n if (!pending) {\n return false;\n }\n this.initPromises.delete(correlationId);\n clearTimeout(pending.timer);\n pending.reject(error);\n return true;\n }\n\n private handleReady(shard: ShardState, correlationId: string): void {\n shard.ready = true;\n shard.initCorrelationId = undefined;\n const pending = this.initPromises.get(correlationId);\n if (!pending) {\n return;\n }\n this.initPromises.delete(correlationId);\n clearTimeout(pending.timer);\n pending.resolve();\n }\n\n private handleDrained(msg: ProjectionDrainedMessage): void {\n const pending = this.pendingDrains.get(msg.correlationId);\n if (!pending) {\n return;\n }\n this.releaseDrain(msg.correlationId, pending, msg.shardId);\n }\n\n /** Removes `shardId` from a pending drain and settles it once no shard remains. */\n private releaseDrain(\n correlationId: string,\n pending: PendingDrain,\n shardId: string,\n ): void {\n pending.remaining.delete(shardId);\n if (pending.remaining.size > 0) {\n return;\n }\n this.pendingDrains.delete(correlationId);\n clearTimeout(pending.timer);\n pending.resolve();\n }\n\n private handleLog(\n shard: ShardState,\n msg: Extract<ProjectionWorkerMessage, { type: \"log\" }>,\n ): void {\n switch (msg.level) {\n case \"debug\":\n this.logger.debug(msg.message, ...msg.args);\n return;\n case \"info\":\n this.logger.info(msg.message, ...msg.args);\n return;\n case \"warn\":\n this.logger.warn(msg.message, ...msg.args);\n return;\n case \"error\":\n this.logger.error(msg.message, ...msg.args);\n return;\n default: {\n const exhaustive: never = msg.level;\n void exhaustive;\n }\n }\n void shard;\n }\n\n private handleTransportError(shard: ShardState, err: Error): void {\n this.logger.error(\n \"projection shard transport error @shardId: @error\",\n shard.shardId,\n err,\n );\n // A shard that dies during startup is reported through `startup()`, not\n // through the fatal hook: the host is still building and its shutdown\n // path (which the hook usually triggers) does not exist yet.\n if (this.failPendingInit(shard, err)) {\n return;\n }\n if (!this.isShuttingDown) {\n this.config.onShardFatal?.(shard.shardId, err);\n }\n }\n\n private handleTransportExit(shard: ShardState, code: number): void {\n const wasReady = shard.ready;\n shard.ready = false;\n // Nothing respawns the shard, so every job it had in flight is lost with\n // it. Release them rather than pin their operations for the process's\n // remaining life.\n const abandoned = [...shard.pendingCoordinates.keys()];\n shard.pendingCoordinates.clear();\n for (const [correlationId, pending] of this.pendingDrains) {\n this.releaseDrain(correlationId, pending, shard.shardId);\n }\n if (\n this.failPendingInit(\n shard,\n new Error(\n `projection shard ${shard.shardId} exited with code ${code} before becoming ready`,\n ),\n )\n ) {\n return;\n }\n if (!wasReady || this.isShuttingDown) {\n return;\n }\n this.logger.error(\n \"projection shard exited unexpectedly @shardId code=@code, abandoning @count in-flight job(s): @jobIds\",\n shard.shardId,\n code,\n abandoned.length,\n abandoned.join(\", \"),\n );\n this.config.onShardFatal?.(\n shard.shardId,\n new Error(\n `projection shard ${shard.shardId} exited with code ${code}; ` +\n `${abandoned.length} in-flight job(s) will never be projected` +\n (abandoned.length > 0 ? `: ${abandoned.join(\", \")}` : \"\"),\n ),\n );\n }\n\n private relayReadReady(event: JobReadReadyEvent): void {\n const hook = this.config.onReadReady;\n if (hook) {\n hook(event);\n return;\n }\n void this.emitReadReady(event).catch((err: unknown) =>\n this.logger.error(\n \"host JOB_READ_READY emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n\n private relayReadModelIndexed(event: ReadModelIndexedEvent): void {\n void this.hostBus\n .emit(ReactorEventTypes.READMODEL_INDEXED, event)\n .catch((err: unknown) =>\n this.logger.error(\n \"host READMODEL_INDEXED emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n\n private relayBatchCompleted(event: ReadModelBatchCompletedEvent): void {\n void this.hostBus\n .emit(ReactorEventTypes.READMODEL_BATCH_COMPLETED, event)\n .catch((err: unknown) =>\n this.logger.error(\n \"host READMODEL_BATCH_COMPLETED emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n}\n"],"mappings":";;;;;AAkCA,MAAM,0BAA0B;AAChC,MAAM,4BAA4B;AAClC,MAAM,2BAA2B;AACjC,MAAM,yCAAyC;AAE/C,MAAM,mBAAmB;AACzB,MAAM,YAAY;AAElB,SAAS,UAAU,YAAoB,YAA4B;AACjE,KAAI,aAAa,EACf,OAAM,IAAI,MAAM,2CAA2C,WAAW,GAAG;CAE3E,IAAI,OAAO;AACX,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,UAAQ,WAAW,WAAW,EAAE;AAChC,SAAO,KAAK,KAAK,MAAM,UAAU;;AAEnC,SAAQ,SAAS,KAAK;;;;;;AAOxB,SAAS,yBACP,YACyB;CACzB,MAAM,cAAuC,EAAE;AAC/C,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;EAC1C,MAAM,OAAO,WAAW;AACxB,cAAY,KAAK;GACf,YAAY,KAAK,QAAQ;GACzB,OAAO,KAAK,QAAQ;GACpB,QAAQ,KAAK,QAAQ;GACrB,gBAAgB,KAAK,UAAU;GAChC,CAAC;;AAEJ,QAAO;;;;;;;;;;;;;;;;;AAuHT,IAAa,yBAAb,MAAqE;CACnE,aAAoC,EAAE;CAEtC;CACA;CACA;CACA,SAAwC,EAAE;CAC1C,+BAAgC,IAAI,KAGjC;CACH,gCAAiC,IAAI,KAA2B;CAChE,0CAA2C,IAAI,KAG5C;CACH;CACA,YAAoB;CACpB,UAAkB;CAClB,iBAAyB;CAEzB,YAAY,QAAsC;AAChD,MAAI,OAAO,aAAa,EACtB,OAAM,IAAI,MACR,wDAAwD,OAAO,WAAW,GAC3E;AAEH,OAAK,SAAS;AACd,OAAK,SAAS,YAAY,CAAC,WAAW,2BAA2B,CAAC;AAClE,OAAK,UAAU,OAAO;EACtB,MAAM,WAAW,OAAO,uBAAuB,EAAE;AACjD,OAAK,MAAM,QAAQ,OAAO,KAAK,SAAS,EAA4B;GAClE,MAAM,UAAU,SAAS;AACzB,OAAI,QACF,MAAK,wBAAwB,IAAI,MAAM,QAAQ;;;CAKrD,MAAM,UAAyB;AAC7B,MAAI,KAAK,QACP;AAEF,OAAK,UAAU;EACf,MAAM,gBAAgB,KAAK,OAAO,iBAAiB;EACnD,MAAM,mBACJ,KAAK,OAAO,8BACZ;EAEF,MAAM,eAAgC,EAAE;AACxC,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,YAAY,KAAK;GAC/C,MAAM,UAAU,oBAAoB;GACpC,MAAM,YAAY,KAAK,OAAO,QAAQ,GAAG,QAAQ;GACjD,MAAM,QAAoB;IACxB,YAAY;IACZ;IACA;IACA,OAAO;IACP,WAAW;IACX,aAAa;IACb,qBAAqB,KAAK,OAAO,uBAAuB;IACxD,oCAAoB,IAAI,KAAK;IAC7B,YAAY,QAAQ,KAAK,oBAAoB,OAAO,IAAI;IACxD,UAAU,QAAQ,KAAK,qBAAqB,OAAO,IAAI;IACvD,SAAS,SAAS,KAAK,oBAAoB,OAAO,KAAK;IACxD;AACD,aAAU,GAAG,WAAW,MAAM,UAAU;AACxC,aAAU,GAAG,SAAS,MAAM,QAAQ;AACpC,aAAU,GAAG,QAAQ,MAAM,OAAO;AAClC,QAAK,OAAO,KAAK,MAAM;GAEvB,MAAM,gBAAgB,YAAY;AAClC,SAAM,oBAAoB;GAC1B,MAAM,cAAc,IAAI,SAAe,SAAS,WAAW;IACzD,MAAM,QAAQ,iBAAiB;AAC7B,UAAK,aAAa,OAAO,cAAc;AACvC,4BACE,IAAI,MACF,oBAAoB,QAAQ,+BAA+B,cAAc,IAC1E,CACF;OACA,cAAc;AACjB,SAAK,aAAa,IAAI,eAAe;KAAE;KAAS;KAAQ;KAAO,CAAC;KAChE;GAEF,MAAM,OAA8B;IAClC,MAAM;IACN;IACA;IACA,YAAY;IACZ,YAAY,KAAK,OAAO;IACxB,IAAI,KAAK,OAAO;IAChB,QAAQ,KAAK,OAAO;IACpB,eAAe,KAAK,OAAO;IAC3B,gBAAgB,KAAK,OAAO;IAC5B,4BAA4B;IAC7B;AACD,aAAU,YAAY,KAAK;AAC3B,gBAAa,KAAK,YAAY;;AAGhC,QAAM,QAAQ,IAAI,aAAa;AAC/B,OAAK,OAAO,KACV,iDACA,KAAK,OAAO,OACb;;CAGH,QAAc;AACZ,MAAI,KAAK,UACP;AAEF,OAAK,mBAAmB,KAAK,QAAQ,UACnC,kBAAkB,kBACjB,IAAY,UAA8B;AACzC,QAAK,gBAAgB,MAAM;IAE9B;AACD,OAAK,YAAY;;CAGnB,OAAa;AACX,MAAI,CAAC,KAAK,UACR;AAEF,MAAI,KAAK,kBAAkB;AACzB,QAAK,kBAAkB;AACvB,QAAK,mBAAmB,KAAA;;AAE1B,OAAK,YAAY;;;CAInB,MAAM,QAAuB;EAC3B,MAAM,cAAc,KAAK,OAAO,QAAQ,MAAM,EAAE,MAAM;AACtD,MAAI,YAAY,WAAW,EACzB;EAEF,MAAM,iBACJ,KAAK,OAAO,kBAAkB;EAChC,MAAM,gBAAgB,YAAY;EAClC,MAAM,YAAY,IAAI,IAAI,YAAY,KAAK,MAAM,EAAE,QAAQ,CAAC;EAC5D,MAAM,UAAU,IAAI,SAAe,SAAS,WAAW;GACrD,MAAM,QAAQ,iBAAiB;AAC7B,SAAK,cAAc,OAAO,cAAc;AACxC,2BACE,IAAI,MACF,0CAA0C,eAAe,iBAAiB,CACxE,GAAG,UACJ,CAAC,KAAK,KAAK,CAAC,GACd,CACF;MACA,eAAe;AAClB,QAAK,cAAc,IAAI,eAAe;IACpC;IACA;IACA;IACA;IACD,CAAC;IACF;AACF,OAAK,MAAM,SAAS,YAClB,OAAM,UAAU,YAAY;GAAE,MAAM;GAAS;GAAe,CAAC;AAE/D,QAAM;;;CAIR,cAAc,OAAyC;AACrD,SAAO,KAAK,QAAQ,KAAK,kBAAkB,gBAAgB,MAAM;;CAGnE,gBAAwB;EACtB,IAAI,QAAQ;AACZ,OAAK,MAAM,SAAS,KAAK,OACvB,UAAS,MAAM;AAEjB,SAAO;;CAGT,iBAAqC;AACnC,SAAO,KAAK,OAAO,KAAK,WAAW;GACjC,SAAS,MAAM;GACf,OAAO,MAAM;GACb,WAAW,MAAM;GAClB,EAAE;;CAGL,MAAM,WAA0B;AAG9B,OAAK,iBAAiB;AACtB,OAAK,MAAM;AAGX,OAAK,MAAM,SAAS,KAAK,OACvB,MAAK,gBACH,uBACA,IAAI,MACF,oBAAoB,MAAM,QAAQ,sCACnC,CACF;EAEH,MAAM,UAAU,KAAK,OAAO,mBAAmB;EAC/C,MAAM,gBAAgB,YAAY;AAClC,OAAK,MAAM,SAAS,KAAK,OACvB,KAAI;GACF,MAAM,MAA+B;IACnC,MAAM;IACN;IACA;IACD;AACD,SAAM,UAAU,YAAY,IAAI;WACzB,KAAK;AACZ,QAAK,OAAO,KACV,sEACA,MAAM,SACN,IACD;;EAGL,MAAM,sBAAsB,KAAK,KAAK,GAAG;AACzC,SACE,KAAK,OAAO,MAAM,MAAM,EAAE,MAAM,IAChC,KAAK,KAAK,GAAG,oBAEb,OAAM,IAAI,SAAe,YAAY,WAAW,SAAS,GAAG,CAAC;AAE/D,OAAK,MAAM,SAAS,KAAK,QAAQ;AAC/B,OAAI;AACF,UAAM,MAAM,UAAU,WAAW;YAC1B,KAAK;AACZ,SAAK,OAAO,KACV,0DACA,MAAM,SACN,IACD;;AAEH,SAAM,UAAU,IAAI,WAAW,MAAM,UAAU;AAC/C,SAAM,UAAU,IAAI,SAAS,MAAM,QAAQ;AAC3C,SAAM,UAAU,IAAI,QAAQ,MAAM,OAAO;;AAE3C,OAAK,OAAO,SAAS;;CAGvB,gBAAwB,OAAiC;AACvD,MAAI,MAAM,WAAW,WAAW,GAAG;AAKjC,QAAK,eAAe;IAAE,OAAO,MAAM;IAAO,YAAY,EAAE;IAAE,CAAC;AAC3D,QAAK,oBAAoB;IACvB,OAAO,MAAM;IACb,WAAW;IACX,qBAAqB;IACrB,oBAAoB;IACpB,gBAAgB;IAChB,qBAAqB;IACtB,CAAC;AACF;;EAEF,MAAM,aAAa,MAAM,WAAW,GAAI,QAAQ;EAChD,MAAM,QAAQ,UAAU,YAAY,KAAK,OAAO,OAAO;EACvD,MAAM,QAAQ,KAAK,OAAO;AAC1B,MAAI,CAAC,MAAM,OAAO;AAChB,QAAK,eAAe,OAAO,OAAO,WAAW;AAC7C;;AAEF,MAAI,KAAK,wBAAwB,OAAO,EACtC,OAAM,mBAAmB,IACvB,MAAM,OACN,yBAAyB,MAAM,WAAW,CAC3C;AAEH,QAAM,UAAU,YAAY;GAC1B,MAAM;GACN,OAAO,MAAM;GACb,YAAY,MAAM;GAClB,SAAS,MAAM;GACf,uBAAuB,MAAM;GAC9B,CAAC;;;;;;;;;;;;;;;CAgBJ,eACE,OACA,OACA,YACM;EACN,MAAM,yBAAS,IAAI,MACjB,oBAAoB,MAAM,QAAQ,yCAAyC,MAAM,MAAM,aACxE,WAAW,IAAI,MAAM,WAAW,OAAO,6IAGvD;AACD,OAAK,OAAO,MACV,qEACA,MAAM,OACN,MAAM,SACN,OACD;AACD,OAAK,OAAO,eAAe,MAAM,SAAS,OAAO;;CAGnD,oBACE,OACA,KACM;AACN,UAAQ,IAAI,MAAZ;GACE,KAAK;AACH,SAAK,YAAY,OAAO,IAAI,cAAc;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,MACV,0DACA,MAAM,SACN,IAAI,MAAM,QACX;AACD,SAAK,gBAAgB,OAAO,cAAc,IAAI,MAAM,CAAC;AACrD;GACF,KAAK;AACH,SAAK,eAAe;KAClB,OAAO,IAAI;KACX,YAAY,IAAI;KACjB,CAAC;AACF;GACF,KAAK;AACH,SAAK,2BAA2B,OAAO,IAAI;AAC3C,SAAK,sBAAsB;KACzB,OAAO,IAAI;KACX,eAAe,IAAI;KACnB,OAAO,IAAI;KACX,YAAY,IAAI;KAChB,gBAAgB,IAAI;KACpB,SAAS,IAAI;KACd,CAAC;AACF;GACF,KAAK;AACH,UAAM,mBAAmB,OAAO,IAAI,MAAM;AAC1C,SAAK,oBAAoB;KACvB,OAAO,IAAI;KACX,WAAW,IAAI;KACf,qBAAqB,IAAI;KACzB,oBAAoB,IAAI;KACxB,gBAAgB,IAAI;KACpB,qBAAqB,IAAI;KAC1B,CAAC;AACF;GACF,KAAK;AACH,UAAM,YAAY,IAAI;AACtB,UAAM,cAAc,IAAI;AACxB;GACF,KAAK;AACH,SAAK,yBAAyB,OAAO,IAAI;AACzC;GACF,KAAK;AACH,SAAK,cAAc,IAAI;AACvB;GACF,KAAK;AACH,SAAK,UAAU,OAAO,IAAI;AAC1B;GACF,QAGE;;;;;;;;;;;;;CAeN,2BACE,OACA,KACM;AACN,MAAI,CAAC,IAAI,QACP;EAEF,MAAM,UAAU,KAAK,wBAAwB,IAAI,IAAI,cAAc;AACnE,MAAI,CAAC,QACH;EAEF,MAAM,cAAc,MAAM,mBAAmB,IAAI,IAAI,MAAM;AAC3D,MAAI,CAAC,eAAe,YAAY,WAAW,EACzC;AAEF,UAAQ,OAAO,YAAY;;CAG7B,yBACE,OACA,KACM;AACN,MAAI,CAAC,MAAM,oBACT;AAEF,QAAM,oBAAoB,YAAY;GACpC,MAAM,IAAI;GACV,MAAM,IAAI;GACV,SAAS,IAAI;GACd,CAAC;AACF,QAAM,oBAAoB,YAAY,IAAI,UAAU;;;;;;;;CAStD,gBAAwB,OAAmB,OAAuB;EAChE,MAAM,gBAAgB,MAAM;AAC5B,MAAI,kBAAkB,KAAA,EACpB,QAAO;AAET,QAAM,oBAAoB,KAAA;EAC1B,MAAM,UAAU,KAAK,aAAa,IAAI,cAAc;AACpD,MAAI,CAAC,QACH,QAAO;AAET,OAAK,aAAa,OAAO,cAAc;AACvC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,OAAO,MAAM;AACrB,SAAO;;CAGT,YAAoB,OAAmB,eAA6B;AAClE,QAAM,QAAQ;AACd,QAAM,oBAAoB,KAAA;EAC1B,MAAM,UAAU,KAAK,aAAa,IAAI,cAAc;AACpD,MAAI,CAAC,QACH;AAEF,OAAK,aAAa,OAAO,cAAc;AACvC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,SAAS;;CAGnB,cAAsB,KAAqC;EACzD,MAAM,UAAU,KAAK,cAAc,IAAI,IAAI,cAAc;AACzD,MAAI,CAAC,QACH;AAEF,OAAK,aAAa,IAAI,eAAe,SAAS,IAAI,QAAQ;;;CAI5D,aACE,eACA,SACA,SACM;AACN,UAAQ,UAAU,OAAO,QAAQ;AACjC,MAAI,QAAQ,UAAU,OAAO,EAC3B;AAEF,OAAK,cAAc,OAAO,cAAc;AACxC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,SAAS;;CAGnB,UACE,OACA,KACM;AACN,UAAQ,IAAI,OAAZ;GACE,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,QAC4B,KAAI;;;CAOpC,qBAA6B,OAAmB,KAAkB;AAChE,OAAK,OAAO,MACV,qDACA,MAAM,SACN,IACD;AAID,MAAI,KAAK,gBAAgB,OAAO,IAAI,CAClC;AAEF,MAAI,CAAC,KAAK,eACR,MAAK,OAAO,eAAe,MAAM,SAAS,IAAI;;CAIlD,oBAA4B,OAAmB,MAAoB;EACjE,MAAM,WAAW,MAAM;AACvB,QAAM,QAAQ;EAId,MAAM,YAAY,CAAC,GAAG,MAAM,mBAAmB,MAAM,CAAC;AACtD,QAAM,mBAAmB,OAAO;AAChC,OAAK,MAAM,CAAC,eAAe,YAAY,KAAK,cAC1C,MAAK,aAAa,eAAe,SAAS,MAAM,QAAQ;AAE1D,MACE,KAAK,gBACH,uBACA,IAAI,MACF,oBAAoB,MAAM,QAAQ,oBAAoB,KAAK,wBAC5D,CACF,CAED;AAEF,MAAI,CAAC,YAAY,KAAK,eACpB;AAEF,OAAK,OAAO,MACV,yGACA,MAAM,SACN,MACA,UAAU,QACV,UAAU,KAAK,KAAK,CACrB;AACD,OAAK,OAAO,eACV,MAAM,yBACN,IAAI,MACF,oBAAoB,MAAM,QAAQ,oBAAoB,KAAK,IACtD,UAAU,OAAO,8CACnB,UAAU,SAAS,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,IACzD,CACF;;CAGH,eAAuB,OAAgC;EACrD,MAAM,OAAO,KAAK,OAAO;AACzB,MAAI,MAAM;AACR,QAAK,MAAM;AACX;;AAEG,OAAK,cAAc,MAAM,CAAC,OAAO,QACpC,KAAK,OAAO,MACV,0DACA,MAAM,OACN,IACD,CACF;;CAGH,sBAA8B,OAAoC;AAC3D,OAAK,QACP,KAAK,kBAAkB,mBAAmB,MAAM,CAChD,OAAO,QACN,KAAK,OAAO,MACV,6DACA,MAAM,OACN,IACD,CACF;;CAGL,oBAA4B,OAA2C;AAChE,OAAK,QACP,KAAK,kBAAkB,2BAA2B,MAAM,CACxD,OAAO,QACN,KAAK,OAAO,MACV,qEACA,MAAM,OACN,IACD,CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@powerhousedao/reactor",
3
- "version": "6.2.2-dev.87",
3
+ "version": "6.2.2-dev.88",
4
4
  "description": "Core Powerhouse reactor — stores documents, resolves conflicts, and reprocesses operation histories. Supports local, cloud, and decentralized storage backends.",
5
5
  "repository": {
6
6
  "url": "https://github.com/powerhouse-inc/powerhouse",
@@ -42,11 +42,11 @@
42
42
  "@electric-sql/pglite": "0.3.15",
43
43
  "pg": "8.18.0",
44
44
  "@sindresorhus/fnv1a": "3.1.0",
45
- "@powerhousedao/shared": "6.2.2-dev.87",
46
- "document-model": "6.2.2-dev.87"
45
+ "@powerhousedao/shared": "6.2.2-dev.88",
46
+ "document-model": "6.2.2-dev.88"
47
47
  },
48
48
  "devDependencies": {
49
- "@powerhousedao/pglite-fs": "6.2.2-dev.87",
49
+ "@powerhousedao/pglite-fs": "6.2.2-dev.88",
50
50
  "@types/pg": "8.16.0",
51
51
  "@vitest/coverage-v8": "4.1.1",
52
52
  "prettier": "3.8.1",
@@ -1 +0,0 @@
1
- {"version":3,"file":"projection-shard-manager-D6KYcEZi.js","names":[],"sources":["../src/projection/projection-shard-manager.ts"],"sourcesContent":["import type { OperationWithContext } from \"@powerhousedao/shared/document-model\";\nimport { childLogger, type ILogger } from \"document-model\";\nimport { randomUUID } from \"node:crypto\";\nimport type { IEventBus } from \"../events/interfaces.js\";\nimport {\n ReactorEventTypes,\n type JobReadReadyEvent,\n type JobWriteReadyEvent,\n type ReadModelBatchCompletedEvent,\n type ReadModelIndexedEvent,\n type Unsubscribe,\n} from \"../events/types.js\";\nimport type {\n IReadModel,\n IReadModelCoordinator,\n} from \"../read-models/interfaces.js\";\nimport type { IConsistencyTracker } from \"../shared/consistency-tracker.js\";\nimport type { ConsistencyCoordinate } from \"../shared/types.js\";\nimport type { ForwardingPoolInstrumentation } from \"../storage/pool-instrumentation.js\";\nimport type {\n BuiltInReadModelKind,\n ChainDepthReport,\n DbConfig,\n ModelManifestEntry,\n ProjectionDrainedMessage,\n ProjectionInitMessage,\n ProjectionParentMessage,\n ProjectionPoolAcquireSamplesMessage,\n ProjectionReadModelIndexedMessage,\n ProjectionWorkerMessage,\n} from \"./protocol.js\";\nimport type { IProjectionTransport } from \"./transport.js\";\n\nconst DEFAULT_INIT_TIMEOUT_MS = 30_000;\nconst DEFAULT_SHUTDOWN_GRACE_MS = 5_000;\nconst DEFAULT_DRAIN_TIMEOUT_MS = 30_000;\nconst DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS = 250;\n\nconst FNV_OFFSET_BASIS = 0x811c9dc5;\nconst FNV_PRIME = 0x01000193;\n\nfunction bucketFor(documentId: string, numWorkers: number): number {\n if (numWorkers < 1) {\n throw new Error(`bucketFor: numWorkers must be >= 1 (got ${numWorkers})`);\n }\n let hash = FNV_OFFSET_BASIS;\n for (let i = 0; i < documentId.length; i++) {\n hash ^= documentId.charCodeAt(i);\n hash = Math.imul(hash, FNV_PRIME);\n }\n return (hash >>> 0) % numWorkers;\n}\n\n/**\n * Maps operations to the coordinates a consistency tracker is keyed by.\n * Same shape `BaseReadModel.updateConsistencyTracker` produces in-process.\n */\nfunction toConsistencyCoordinates(\n operations: OperationWithContext[],\n): ConsistencyCoordinate[] {\n const coordinates: ConsistencyCoordinate[] = [];\n for (let i = 0; i < operations.length; i++) {\n const item = operations[i]!;\n coordinates.push({\n documentId: item.context.documentId,\n scope: item.context.scope,\n branch: item.context.branch,\n operationIndex: item.operation.index,\n });\n }\n return coordinates;\n}\n\n/**\n * Factory that builds one projection-worker transport. Mirrors\n * `WorkerFactory` from the executor pool: lets tests inject fake\n * transports without spawning real worker threads.\n */\nexport type ProjectionWorkerFactory = (\n shardIndex: number,\n shardId: string,\n) => IProjectionTransport;\n\nexport type ProjectionShardManagerConfig = {\n shardCount: number;\n db: DbConfig;\n models: ModelManifestEntry[];\n preReadyKinds: BuiltInReadModelKind[];\n postReadyKinds: BuiltInReadModelKind[];\n factory: ProjectionWorkerFactory;\n logger: ILogger;\n hostBus: IEventBus;\n initTimeoutMs?: number;\n shutdownGraceMs?: number;\n drainTimeoutMs?: number;\n chainDepthReportIntervalMs?: number;\n /**\n * Host-side forwarding instrumentations indexed by shard index. The\n * manager routes each shard's `pool-acquire-samples` message to the\n * matching forwarder so the host's OpenTelemetry instrumentation records\n * acquire-wait latencies as if each shard's pg.Pool were local.\n */\n poolInstrumentations?: ForwardingPoolInstrumentation[];\n /**\n * The host's consistency trackers for the built-in read models the shards\n * run, keyed by kind. The keys double as read-model names (see\n * `DOCUMENT_VIEW_READ_MODEL` / `DOCUMENT_INDEXER_READ_MODEL` in\n * `read-models/names.js`), which is how a relayed `readmodel-indexed`\n * message is matched to a tracker.\n *\n * The host's copies of these read models are never fed an operation under\n * sharding, so without this every read carrying a consistency token waits\n * on a tracker that can never advance — `ConsistencyTracker.waitFor` arms\n * no timer when `timeoutMs` is undefined, which is what every\n * `document-view` call site passes. The worker committed those rows to the\n * same tables the host reads from, so advancing here is exact, not a fudge.\n */\n consistencyTrackers?: Partial<\n Record<BuiltInReadModelKind, IConsistencyTracker>\n >;\n /**\n * Fired when a shard errors, when it exits after having been ready, and\n * for every JOB_WRITE_READY dropped because its shard is not ready.\n *\n * There is no respawn path: once a shard stops being ready it never\n * projects again, so buffering the dropped work would only grow without\n * bound. A host that cares wires this to its shutdown path, so the process\n * restarts and each read model catches up from `ViewState.lastOrdinal` in\n * `BaseReadModel.init`. May fire repeatedly — handlers must be idempotent.\n */\n onShardFatal?: (shardId: string, reason: Error) => void;\n};\n\ntype ShardState = {\n shardIndex: number;\n shardId: string;\n transport: IProjectionTransport;\n ready: boolean;\n lastDepth: number;\n lastDepthAt: number;\n poolInstrumentation?: ForwardingPoolInstrumentation;\n /**\n * Consistency coordinates of the jobs this shard is still projecting,\n * keyed by jobId. Relayed `readmodel-indexed` messages carry only a count,\n * so the host keeps the coordinates here to advance its trackers from\n * them; the operations themselves are not retained. Empty unless\n * `consistencyTrackers` is configured; entries live from dispatch to\n * `readmodel-batch-completed`, and are dropped when the shard exits.\n */\n pendingCoordinates: Map<string, ConsistencyCoordinate[]>;\n onMessage: (msg: ProjectionWorkerMessage) => void;\n onError: (err: Error) => void;\n onExit: (code: number) => void;\n};\n\ntype PendingDrain = {\n resolve: () => void;\n reject: (err: Error) => void;\n remaining: Set<string>;\n timer: NodeJS.Timeout;\n};\n\n/**\n * Host-side coordinator for N sharded projection workers.\n *\n * Implements {@link IReadModelCoordinator} so it slots into the same\n * `readModelCoordinator` field on the reactor module as the in-process\n * {@link ReadModelCoordinator}. The host subscribes to JOB_WRITE_READY\n * exactly once; events are routed to a single shard by\n * `bucketFor(documentId, shardCount)`. Each worker maintains the\n * per-queueKey serial chain locally and forwards JOB_READ_READY and\n * READMODEL_* events back to the host for the rest of the reactor (sync\n * manager, awaiters, observers) to consume on the host bus.\n *\n * @see Sharded projection workers sub-feature brief\n * (Powerhouse board wiki id: eb26f01f-8f68-4918-a6f6-ac7a4679b533)\n */\nexport class ProjectionShardManager implements IReadModelCoordinator {\n readonly readModels: IReadModel[] = [];\n\n private readonly config: ProjectionShardManagerConfig;\n private readonly logger: ILogger;\n private readonly hostBus: IEventBus;\n private readonly shards: ShardState[] = [];\n private readonly initPromises = new Map<\n string,\n { resolve: () => void; reject: (err: Error) => void; timer: NodeJS.Timeout }\n >();\n private readonly pendingDrains = new Map<string, PendingDrain>();\n private readonly trackersByReadModelName = new Map<\n string,\n IConsistencyTracker\n >();\n private hostSubscription?: Unsubscribe;\n private isRunning = false;\n private started = false;\n private isShuttingDown = false;\n\n constructor(config: ProjectionShardManagerConfig) {\n if (config.shardCount < 1) {\n throw new Error(\n `ProjectionShardManager: shardCount must be >= 1 (got ${config.shardCount})`,\n );\n }\n this.config = config;\n this.logger = childLogger([\"reactor\", \"projection-shard-manager\"]);\n this.hostBus = config.hostBus;\n const trackers = config.consistencyTrackers ?? {};\n for (const kind of Object.keys(trackers) as BuiltInReadModelKind[]) {\n const tracker = trackers[kind];\n if (tracker) {\n this.trackersByReadModelName.set(kind, tracker);\n }\n }\n }\n\n async startup(): Promise<void> {\n if (this.started) {\n return;\n }\n this.started = true;\n const initTimeoutMs = this.config.initTimeoutMs ?? DEFAULT_INIT_TIMEOUT_MS;\n const reportIntervalMs =\n this.config.chainDepthReportIntervalMs ??\n DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS;\n\n const initPromises: Promise<void>[] = [];\n for (let i = 0; i < this.config.shardCount; i++) {\n const shardId = `projection-shard-${i}`;\n const transport = this.config.factory(i, shardId);\n const state: ShardState = {\n shardIndex: i,\n shardId,\n transport,\n ready: false,\n lastDepth: 0,\n lastDepthAt: 0,\n poolInstrumentation: this.config.poolInstrumentations?.[i],\n pendingCoordinates: new Map(),\n onMessage: (msg) => this.handleWorkerMessage(state, msg),\n onError: (err) => this.handleTransportError(state, err),\n onExit: (code) => this.handleTransportExit(state, code),\n };\n transport.on(\"message\", state.onMessage);\n transport.on(\"error\", state.onError);\n transport.on(\"exit\", state.onExit);\n this.shards.push(state);\n\n const correlationId = randomUUID();\n const initPromise = new Promise<void>((resolve, reject) => {\n const timer = setTimeout(() => {\n this.initPromises.delete(correlationId);\n reject(\n new Error(\n `projection shard ${shardId} did not become ready within ${initTimeoutMs}ms`,\n ),\n );\n }, initTimeoutMs);\n this.initPromises.set(correlationId, { resolve, reject, timer });\n });\n\n const init: ProjectionInitMessage = {\n type: \"init\",\n correlationId,\n shardId,\n shardIndex: i,\n shardCount: this.config.shardCount,\n db: this.config.db,\n models: this.config.models,\n preReadyKinds: this.config.preReadyKinds,\n postReadyKinds: this.config.postReadyKinds,\n chainDepthReportIntervalMs: reportIntervalMs,\n };\n transport.postMessage(init);\n initPromises.push(initPromise);\n }\n\n await Promise.all(initPromises);\n this.logger.info(\n \"projection shard manager ready: @count shards\",\n this.shards.length,\n );\n }\n\n start(): void {\n if (this.isRunning) {\n return;\n }\n this.hostSubscription = this.hostBus.subscribe(\n ReactorEventTypes.JOB_WRITE_READY,\n (_t: number, event: JobWriteReadyEvent) => {\n this.routeWriteReady(event);\n },\n );\n this.isRunning = true;\n }\n\n stop(): void {\n if (!this.isRunning) {\n return;\n }\n if (this.hostSubscription) {\n this.hostSubscription();\n this.hostSubscription = undefined;\n }\n this.isRunning = false;\n }\n\n async drain(): Promise<void> {\n if (this.shards.length === 0) {\n return;\n }\n const drainTimeoutMs =\n this.config.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;\n const correlationId = randomUUID();\n const remaining = new Set(this.shards.map((s) => s.shardId));\n const promise = new Promise<void>((resolve, reject) => {\n const timer = setTimeout(() => {\n this.pendingDrains.delete(correlationId);\n reject(\n new Error(\n `projection shards did not drain within ${drainTimeoutMs}ms (remaining: ${[\n ...remaining,\n ].join(\", \")})`,\n ),\n );\n }, drainTimeoutMs);\n this.pendingDrains.set(correlationId, {\n resolve,\n reject,\n remaining,\n timer,\n });\n });\n for (const shard of this.shards) {\n shard.transport.postMessage({ type: \"drain\", correlationId });\n }\n await promise;\n }\n\n getChainDepth(): number {\n let total = 0;\n for (const shard of this.shards) {\n total += shard.lastDepth;\n }\n return total;\n }\n\n getShardDepths(): ChainDepthReport[] {\n return this.shards.map((shard) => ({\n shardId: shard.shardId,\n depth: shard.lastDepth,\n timestamp: shard.lastDepthAt,\n }));\n }\n\n async shutdown(): Promise<void> {\n // Workers exit as a consequence of this call, so their `exit` events are\n // expected from here on and must not be reported as fatal.\n this.isShuttingDown = true;\n this.stop();\n const graceMs = this.config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;\n const correlationId = randomUUID();\n for (const shard of this.shards) {\n try {\n const msg: ProjectionParentMessage = {\n type: \"shutdown\",\n correlationId,\n graceMs,\n };\n shard.transport.postMessage(msg);\n } catch (err) {\n this.logger.warn(\n \"projection shard postMessage(shutdown) failed for @shardId: @error\",\n shard.shardId,\n err,\n );\n }\n }\n const terminationDeadline = Date.now() + graceMs;\n while (\n this.shards.some((s) => s.ready) &&\n Date.now() < terminationDeadline\n ) {\n await new Promise<void>((resolve) => setTimeout(resolve, 25));\n }\n for (const shard of this.shards) {\n try {\n await shard.transport.terminate();\n } catch (err) {\n this.logger.warn(\n \"projection shard terminate failed for @shardId: @error\",\n shard.shardId,\n err,\n );\n }\n shard.transport.off(\"message\", shard.onMessage);\n shard.transport.off(\"error\", shard.onError);\n shard.transport.off(\"exit\", shard.onExit);\n }\n this.shards.length = 0;\n }\n\n private routeWriteReady(event: JobWriteReadyEvent): void {\n if (event.operations.length === 0) {\n // No shard can be selected (there is no documentId) and no read model\n // has work to do, but JOB_READ_READY is the job's terminal signal:\n // InMemoryJobTracker and every JobAwaiter hang without it. Mirrors\n // ReadModelCoordinator.emitEmptyReadReady.\n this.relayReadReady({ jobId: event.jobId, operations: [] });\n this.relayBatchCompleted({\n jobId: event.jobId,\n batchSize: 0,\n chainWaitDurationMs: 0,\n preReadyDurationMs: 0,\n emitDurationMs: 0,\n postReadyDurationMs: 0,\n });\n return;\n }\n const documentId = event.operations[0]!.context.documentId;\n const index = bucketFor(documentId, this.shards.length);\n const shard = this.shards[index]!;\n if (!shard.ready) {\n this.dropWriteReady(shard, event, documentId);\n return;\n }\n if (this.trackersByReadModelName.size > 0) {\n shard.pendingCoordinates.set(\n event.jobId,\n toConsistencyCoordinates(event.operations),\n );\n }\n shard.transport.postMessage({\n type: \"write-ready\",\n jobId: event.jobId,\n operations: event.operations,\n jobMeta: event.jobMeta,\n collectionMemberships: event.collectionMemberships,\n });\n }\n\n /**\n * A shard stops being ready only when it dies, and nothing respawns it, so\n * this job's projection is genuinely lost. Buffering would grow without\n * bound behind a shard that never comes back, so the batch is dropped —\n * loudly, naming the job and document, and through `onShardFatal` so a host\n * can restart rather than serve stale read models.\n *\n * JOB_FAILED is deliberately not emitted: every other emitter uses it for a\n * job whose operations were *not* written (see\n * `executor/job-result-handler.ts`), and these were written and are\n * durable. Marking the job FAILED would invite the caller to re-submit a\n * write that already landed.\n */\n private dropWriteReady(\n shard: ShardState,\n event: JobWriteReadyEvent,\n documentId: string,\n ): void {\n const reason = new Error(\n `projection shard ${shard.shardId} is not ready: JOB_WRITE_READY for job ${event.jobId} ` +\n `(document ${documentId}, ${event.operations.length} operation(s)) was dropped and will ` +\n `never be projected; the operations are written and durable, but this shard's read models ` +\n `are now behind`,\n );\n this.logger.error(\n \"dropping JOB_WRITE_READY for job @jobId on shard @shardId: @error\",\n event.jobId,\n shard.shardId,\n reason,\n );\n this.config.onShardFatal?.(shard.shardId, reason);\n }\n\n private handleWorkerMessage(\n shard: ShardState,\n msg: ProjectionWorkerMessage,\n ): void {\n switch (msg.type) {\n case \"ready\":\n this.handleReady(shard, msg.correlationId);\n return;\n case \"read-ready\":\n this.relayReadReady({\n jobId: msg.jobId,\n operations: msg.operations,\n });\n return;\n case \"readmodel-indexed\":\n this.advanceConsistencyTrackers(shard, msg);\n this.relayReadModelIndexed({\n jobId: msg.jobId,\n readModelName: msg.readModelName,\n stage: msg.stage,\n durationMs: msg.durationMs,\n operationCount: msg.operationCount,\n success: msg.success,\n });\n return;\n case \"readmodel-batch-completed\":\n shard.pendingCoordinates.delete(msg.jobId);\n this.relayBatchCompleted({\n jobId: msg.jobId,\n batchSize: msg.batchSize,\n chainWaitDurationMs: msg.chainWaitDurationMs,\n preReadyDurationMs: msg.preReadyDurationMs,\n emitDurationMs: msg.emitDurationMs,\n postReadyDurationMs: msg.postReadyDurationMs,\n });\n return;\n case \"chain-depth\":\n shard.lastDepth = msg.depth;\n shard.lastDepthAt = msg.timestamp;\n return;\n case \"pool-acquire-samples\":\n this.handlePoolAcquireSamples(shard, msg);\n return;\n case \"drained\":\n this.handleDrained(msg);\n return;\n case \"log\":\n this.handleLog(shard, msg);\n return;\n default: {\n const exhaustive: never = msg;\n void exhaustive;\n return;\n }\n }\n }\n\n /**\n * Advances the host's tracker for the read model the shard just indexed.\n *\n * The shard writes to the same tables the host reads, so once it reports a\n * successful index the host's read path really is consistent to those\n * coordinates. Gated on `success` for parity with\n * `BaseReadModel.indexOperations`, which updates its tracker only after\n * `commitOperations` returns. The worker posts this before its\n * JOB_READ_READY, matching the in-process ordering.\n */\n private advanceConsistencyTrackers(\n shard: ShardState,\n msg: ProjectionReadModelIndexedMessage,\n ): void {\n if (!msg.success) {\n return;\n }\n const tracker = this.trackersByReadModelName.get(msg.readModelName);\n if (!tracker) {\n return;\n }\n const coordinates = shard.pendingCoordinates.get(msg.jobId);\n if (!coordinates || coordinates.length === 0) {\n return;\n }\n tracker.update(coordinates);\n }\n\n private handlePoolAcquireSamples(\n shard: ShardState,\n msg: ProjectionPoolAcquireSamplesMessage,\n ): void {\n if (!shard.poolInstrumentation) {\n return;\n }\n shard.poolInstrumentation.updateStats({\n size: msg.size,\n idle: msg.idle,\n waiting: msg.waiting,\n });\n shard.poolInstrumentation.pushSamples(msg.durations);\n }\n\n private handleReady(shard: ShardState, correlationId: string): void {\n shard.ready = true;\n const pending = this.initPromises.get(correlationId);\n if (!pending) {\n return;\n }\n this.initPromises.delete(correlationId);\n clearTimeout(pending.timer);\n pending.resolve();\n }\n\n private handleDrained(msg: ProjectionDrainedMessage): void {\n const pending = this.pendingDrains.get(msg.correlationId);\n if (!pending) {\n return;\n }\n pending.remaining.delete(msg.shardId);\n if (pending.remaining.size === 0) {\n this.pendingDrains.delete(msg.correlationId);\n clearTimeout(pending.timer);\n pending.resolve();\n }\n }\n\n private handleLog(\n shard: ShardState,\n msg: Extract<ProjectionWorkerMessage, { type: \"log\" }>,\n ): void {\n switch (msg.level) {\n case \"debug\":\n this.logger.debug(msg.message, ...msg.args);\n return;\n case \"info\":\n this.logger.info(msg.message, ...msg.args);\n return;\n case \"warn\":\n this.logger.warn(msg.message, ...msg.args);\n return;\n case \"error\":\n this.logger.error(msg.message, ...msg.args);\n return;\n default: {\n const exhaustive: never = msg.level;\n void exhaustive;\n }\n }\n void shard;\n }\n\n private handleTransportError(shard: ShardState, err: Error): void {\n this.logger.error(\n \"projection shard transport error @shardId: @error\",\n shard.shardId,\n err,\n );\n if (!this.isShuttingDown) {\n this.config.onShardFatal?.(shard.shardId, err);\n }\n }\n\n private handleTransportExit(shard: ShardState, code: number): void {\n const wasReady = shard.ready;\n shard.ready = false;\n // Nothing respawns the shard, so every job it had in flight is lost with\n // it. Release them rather than pin their operations for the process's\n // remaining life.\n const abandoned = [...shard.pendingCoordinates.keys()];\n shard.pendingCoordinates.clear();\n if (!wasReady || this.isShuttingDown) {\n return;\n }\n this.logger.error(\n \"projection shard exited unexpectedly @shardId code=@code, abandoning @count in-flight job(s): @jobIds\",\n shard.shardId,\n code,\n abandoned.length,\n abandoned.join(\", \"),\n );\n this.config.onShardFatal?.(\n shard.shardId,\n new Error(\n `projection shard ${shard.shardId} exited with code ${code}; ` +\n `${abandoned.length} in-flight job(s) will never be projected` +\n (abandoned.length > 0 ? `: ${abandoned.join(\", \")}` : \"\"),\n ),\n );\n }\n\n private relayReadReady(event: JobReadReadyEvent): void {\n void this.hostBus\n .emit(ReactorEventTypes.JOB_READ_READY, event)\n .catch((err: unknown) =>\n this.logger.error(\n \"host JOB_READ_READY emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n\n private relayReadModelIndexed(event: ReadModelIndexedEvent): void {\n void this.hostBus\n .emit(ReactorEventTypes.READMODEL_INDEXED, event)\n .catch((err: unknown) =>\n this.logger.error(\n \"host READMODEL_INDEXED emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n\n private relayBatchCompleted(event: ReadModelBatchCompletedEvent): void {\n void this.hostBus\n .emit(ReactorEventTypes.READMODEL_BATCH_COMPLETED, event)\n .catch((err: unknown) =>\n this.logger.error(\n \"host READMODEL_BATCH_COMPLETED emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n}\n"],"mappings":";;;;AAiCA,MAAM,0BAA0B;AAChC,MAAM,4BAA4B;AAClC,MAAM,2BAA2B;AACjC,MAAM,yCAAyC;AAE/C,MAAM,mBAAmB;AACzB,MAAM,YAAY;AAElB,SAAS,UAAU,YAAoB,YAA4B;AACjE,KAAI,aAAa,EACf,OAAM,IAAI,MAAM,2CAA2C,WAAW,GAAG;CAE3E,IAAI,OAAO;AACX,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,UAAQ,WAAW,WAAW,EAAE;AAChC,SAAO,KAAK,KAAK,MAAM,UAAU;;AAEnC,SAAQ,SAAS,KAAK;;;;;;AAOxB,SAAS,yBACP,YACyB;CACzB,MAAM,cAAuC,EAAE;AAC/C,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;EAC1C,MAAM,OAAO,WAAW;AACxB,cAAY,KAAK;GACf,YAAY,KAAK,QAAQ;GACzB,OAAO,KAAK,QAAQ;GACpB,QAAQ,KAAK,QAAQ;GACrB,gBAAgB,KAAK,UAAU;GAChC,CAAC;;AAEJ,QAAO;;;;;;;;;;;;;;;;;AA2GT,IAAa,yBAAb,MAAqE;CACnE,aAAoC,EAAE;CAEtC;CACA;CACA;CACA,SAAwC,EAAE;CAC1C,+BAAgC,IAAI,KAGjC;CACH,gCAAiC,IAAI,KAA2B;CAChE,0CAA2C,IAAI,KAG5C;CACH;CACA,YAAoB;CACpB,UAAkB;CAClB,iBAAyB;CAEzB,YAAY,QAAsC;AAChD,MAAI,OAAO,aAAa,EACtB,OAAM,IAAI,MACR,wDAAwD,OAAO,WAAW,GAC3E;AAEH,OAAK,SAAS;AACd,OAAK,SAAS,YAAY,CAAC,WAAW,2BAA2B,CAAC;AAClE,OAAK,UAAU,OAAO;EACtB,MAAM,WAAW,OAAO,uBAAuB,EAAE;AACjD,OAAK,MAAM,QAAQ,OAAO,KAAK,SAAS,EAA4B;GAClE,MAAM,UAAU,SAAS;AACzB,OAAI,QACF,MAAK,wBAAwB,IAAI,MAAM,QAAQ;;;CAKrD,MAAM,UAAyB;AAC7B,MAAI,KAAK,QACP;AAEF,OAAK,UAAU;EACf,MAAM,gBAAgB,KAAK,OAAO,iBAAiB;EACnD,MAAM,mBACJ,KAAK,OAAO,8BACZ;EAEF,MAAM,eAAgC,EAAE;AACxC,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,YAAY,KAAK;GAC/C,MAAM,UAAU,oBAAoB;GACpC,MAAM,YAAY,KAAK,OAAO,QAAQ,GAAG,QAAQ;GACjD,MAAM,QAAoB;IACxB,YAAY;IACZ;IACA;IACA,OAAO;IACP,WAAW;IACX,aAAa;IACb,qBAAqB,KAAK,OAAO,uBAAuB;IACxD,oCAAoB,IAAI,KAAK;IAC7B,YAAY,QAAQ,KAAK,oBAAoB,OAAO,IAAI;IACxD,UAAU,QAAQ,KAAK,qBAAqB,OAAO,IAAI;IACvD,SAAS,SAAS,KAAK,oBAAoB,OAAO,KAAK;IACxD;AACD,aAAU,GAAG,WAAW,MAAM,UAAU;AACxC,aAAU,GAAG,SAAS,MAAM,QAAQ;AACpC,aAAU,GAAG,QAAQ,MAAM,OAAO;AAClC,QAAK,OAAO,KAAK,MAAM;GAEvB,MAAM,gBAAgB,YAAY;GAClC,MAAM,cAAc,IAAI,SAAe,SAAS,WAAW;IACzD,MAAM,QAAQ,iBAAiB;AAC7B,UAAK,aAAa,OAAO,cAAc;AACvC,4BACE,IAAI,MACF,oBAAoB,QAAQ,+BAA+B,cAAc,IAC1E,CACF;OACA,cAAc;AACjB,SAAK,aAAa,IAAI,eAAe;KAAE;KAAS;KAAQ;KAAO,CAAC;KAChE;GAEF,MAAM,OAA8B;IAClC,MAAM;IACN;IACA;IACA,YAAY;IACZ,YAAY,KAAK,OAAO;IACxB,IAAI,KAAK,OAAO;IAChB,QAAQ,KAAK,OAAO;IACpB,eAAe,KAAK,OAAO;IAC3B,gBAAgB,KAAK,OAAO;IAC5B,4BAA4B;IAC7B;AACD,aAAU,YAAY,KAAK;AAC3B,gBAAa,KAAK,YAAY;;AAGhC,QAAM,QAAQ,IAAI,aAAa;AAC/B,OAAK,OAAO,KACV,iDACA,KAAK,OAAO,OACb;;CAGH,QAAc;AACZ,MAAI,KAAK,UACP;AAEF,OAAK,mBAAmB,KAAK,QAAQ,UACnC,kBAAkB,kBACjB,IAAY,UAA8B;AACzC,QAAK,gBAAgB,MAAM;IAE9B;AACD,OAAK,YAAY;;CAGnB,OAAa;AACX,MAAI,CAAC,KAAK,UACR;AAEF,MAAI,KAAK,kBAAkB;AACzB,QAAK,kBAAkB;AACvB,QAAK,mBAAmB,KAAA;;AAE1B,OAAK,YAAY;;CAGnB,MAAM,QAAuB;AAC3B,MAAI,KAAK,OAAO,WAAW,EACzB;EAEF,MAAM,iBACJ,KAAK,OAAO,kBAAkB;EAChC,MAAM,gBAAgB,YAAY;EAClC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,KAAK,MAAM,EAAE,QAAQ,CAAC;EAC5D,MAAM,UAAU,IAAI,SAAe,SAAS,WAAW;GACrD,MAAM,QAAQ,iBAAiB;AAC7B,SAAK,cAAc,OAAO,cAAc;AACxC,2BACE,IAAI,MACF,0CAA0C,eAAe,iBAAiB,CACxE,GAAG,UACJ,CAAC,KAAK,KAAK,CAAC,GACd,CACF;MACA,eAAe;AAClB,QAAK,cAAc,IAAI,eAAe;IACpC;IACA;IACA;IACA;IACD,CAAC;IACF;AACF,OAAK,MAAM,SAAS,KAAK,OACvB,OAAM,UAAU,YAAY;GAAE,MAAM;GAAS;GAAe,CAAC;AAE/D,QAAM;;CAGR,gBAAwB;EACtB,IAAI,QAAQ;AACZ,OAAK,MAAM,SAAS,KAAK,OACvB,UAAS,MAAM;AAEjB,SAAO;;CAGT,iBAAqC;AACnC,SAAO,KAAK,OAAO,KAAK,WAAW;GACjC,SAAS,MAAM;GACf,OAAO,MAAM;GACb,WAAW,MAAM;GAClB,EAAE;;CAGL,MAAM,WAA0B;AAG9B,OAAK,iBAAiB;AACtB,OAAK,MAAM;EACX,MAAM,UAAU,KAAK,OAAO,mBAAmB;EAC/C,MAAM,gBAAgB,YAAY;AAClC,OAAK,MAAM,SAAS,KAAK,OACvB,KAAI;GACF,MAAM,MAA+B;IACnC,MAAM;IACN;IACA;IACD;AACD,SAAM,UAAU,YAAY,IAAI;WACzB,KAAK;AACZ,QAAK,OAAO,KACV,sEACA,MAAM,SACN,IACD;;EAGL,MAAM,sBAAsB,KAAK,KAAK,GAAG;AACzC,SACE,KAAK,OAAO,MAAM,MAAM,EAAE,MAAM,IAChC,KAAK,KAAK,GAAG,oBAEb,OAAM,IAAI,SAAe,YAAY,WAAW,SAAS,GAAG,CAAC;AAE/D,OAAK,MAAM,SAAS,KAAK,QAAQ;AAC/B,OAAI;AACF,UAAM,MAAM,UAAU,WAAW;YAC1B,KAAK;AACZ,SAAK,OAAO,KACV,0DACA,MAAM,SACN,IACD;;AAEH,SAAM,UAAU,IAAI,WAAW,MAAM,UAAU;AAC/C,SAAM,UAAU,IAAI,SAAS,MAAM,QAAQ;AAC3C,SAAM,UAAU,IAAI,QAAQ,MAAM,OAAO;;AAE3C,OAAK,OAAO,SAAS;;CAGvB,gBAAwB,OAAiC;AACvD,MAAI,MAAM,WAAW,WAAW,GAAG;AAKjC,QAAK,eAAe;IAAE,OAAO,MAAM;IAAO,YAAY,EAAE;IAAE,CAAC;AAC3D,QAAK,oBAAoB;IACvB,OAAO,MAAM;IACb,WAAW;IACX,qBAAqB;IACrB,oBAAoB;IACpB,gBAAgB;IAChB,qBAAqB;IACtB,CAAC;AACF;;EAEF,MAAM,aAAa,MAAM,WAAW,GAAI,QAAQ;EAChD,MAAM,QAAQ,UAAU,YAAY,KAAK,OAAO,OAAO;EACvD,MAAM,QAAQ,KAAK,OAAO;AAC1B,MAAI,CAAC,MAAM,OAAO;AAChB,QAAK,eAAe,OAAO,OAAO,WAAW;AAC7C;;AAEF,MAAI,KAAK,wBAAwB,OAAO,EACtC,OAAM,mBAAmB,IACvB,MAAM,OACN,yBAAyB,MAAM,WAAW,CAC3C;AAEH,QAAM,UAAU,YAAY;GAC1B,MAAM;GACN,OAAO,MAAM;GACb,YAAY,MAAM;GAClB,SAAS,MAAM;GACf,uBAAuB,MAAM;GAC9B,CAAC;;;;;;;;;;;;;;;CAgBJ,eACE,OACA,OACA,YACM;EACN,MAAM,yBAAS,IAAI,MACjB,oBAAoB,MAAM,QAAQ,yCAAyC,MAAM,MAAM,aACxE,WAAW,IAAI,MAAM,WAAW,OAAO,6IAGvD;AACD,OAAK,OAAO,MACV,qEACA,MAAM,OACN,MAAM,SACN,OACD;AACD,OAAK,OAAO,eAAe,MAAM,SAAS,OAAO;;CAGnD,oBACE,OACA,KACM;AACN,UAAQ,IAAI,MAAZ;GACE,KAAK;AACH,SAAK,YAAY,OAAO,IAAI,cAAc;AAC1C;GACF,KAAK;AACH,SAAK,eAAe;KAClB,OAAO,IAAI;KACX,YAAY,IAAI;KACjB,CAAC;AACF;GACF,KAAK;AACH,SAAK,2BAA2B,OAAO,IAAI;AAC3C,SAAK,sBAAsB;KACzB,OAAO,IAAI;KACX,eAAe,IAAI;KACnB,OAAO,IAAI;KACX,YAAY,IAAI;KAChB,gBAAgB,IAAI;KACpB,SAAS,IAAI;KACd,CAAC;AACF;GACF,KAAK;AACH,UAAM,mBAAmB,OAAO,IAAI,MAAM;AAC1C,SAAK,oBAAoB;KACvB,OAAO,IAAI;KACX,WAAW,IAAI;KACf,qBAAqB,IAAI;KACzB,oBAAoB,IAAI;KACxB,gBAAgB,IAAI;KACpB,qBAAqB,IAAI;KAC1B,CAAC;AACF;GACF,KAAK;AACH,UAAM,YAAY,IAAI;AACtB,UAAM,cAAc,IAAI;AACxB;GACF,KAAK;AACH,SAAK,yBAAyB,OAAO,IAAI;AACzC;GACF,KAAK;AACH,SAAK,cAAc,IAAI;AACvB;GACF,KAAK;AACH,SAAK,UAAU,OAAO,IAAI;AAC1B;GACF,QAGE;;;;;;;;;;;;;CAeN,2BACE,OACA,KACM;AACN,MAAI,CAAC,IAAI,QACP;EAEF,MAAM,UAAU,KAAK,wBAAwB,IAAI,IAAI,cAAc;AACnE,MAAI,CAAC,QACH;EAEF,MAAM,cAAc,MAAM,mBAAmB,IAAI,IAAI,MAAM;AAC3D,MAAI,CAAC,eAAe,YAAY,WAAW,EACzC;AAEF,UAAQ,OAAO,YAAY;;CAG7B,yBACE,OACA,KACM;AACN,MAAI,CAAC,MAAM,oBACT;AAEF,QAAM,oBAAoB,YAAY;GACpC,MAAM,IAAI;GACV,MAAM,IAAI;GACV,SAAS,IAAI;GACd,CAAC;AACF,QAAM,oBAAoB,YAAY,IAAI,UAAU;;CAGtD,YAAoB,OAAmB,eAA6B;AAClE,QAAM,QAAQ;EACd,MAAM,UAAU,KAAK,aAAa,IAAI,cAAc;AACpD,MAAI,CAAC,QACH;AAEF,OAAK,aAAa,OAAO,cAAc;AACvC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,SAAS;;CAGnB,cAAsB,KAAqC;EACzD,MAAM,UAAU,KAAK,cAAc,IAAI,IAAI,cAAc;AACzD,MAAI,CAAC,QACH;AAEF,UAAQ,UAAU,OAAO,IAAI,QAAQ;AACrC,MAAI,QAAQ,UAAU,SAAS,GAAG;AAChC,QAAK,cAAc,OAAO,IAAI,cAAc;AAC5C,gBAAa,QAAQ,MAAM;AAC3B,WAAQ,SAAS;;;CAIrB,UACE,OACA,KACM;AACN,UAAQ,IAAI,OAAZ;GACE,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,QAC4B,KAAI;;;CAOpC,qBAA6B,OAAmB,KAAkB;AAChE,OAAK,OAAO,MACV,qDACA,MAAM,SACN,IACD;AACD,MAAI,CAAC,KAAK,eACR,MAAK,OAAO,eAAe,MAAM,SAAS,IAAI;;CAIlD,oBAA4B,OAAmB,MAAoB;EACjE,MAAM,WAAW,MAAM;AACvB,QAAM,QAAQ;EAId,MAAM,YAAY,CAAC,GAAG,MAAM,mBAAmB,MAAM,CAAC;AACtD,QAAM,mBAAmB,OAAO;AAChC,MAAI,CAAC,YAAY,KAAK,eACpB;AAEF,OAAK,OAAO,MACV,yGACA,MAAM,SACN,MACA,UAAU,QACV,UAAU,KAAK,KAAK,CACrB;AACD,OAAK,OAAO,eACV,MAAM,yBACN,IAAI,MACF,oBAAoB,MAAM,QAAQ,oBAAoB,KAAK,IACtD,UAAU,OAAO,8CACnB,UAAU,SAAS,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,IACzD,CACF;;CAGH,eAAuB,OAAgC;AAChD,OAAK,QACP,KAAK,kBAAkB,gBAAgB,MAAM,CAC7C,OAAO,QACN,KAAK,OAAO,MACV,0DACA,MAAM,OACN,IACD,CACF;;CAGL,sBAA8B,OAAoC;AAC3D,OAAK,QACP,KAAK,kBAAkB,mBAAmB,MAAM,CAChD,OAAO,QACN,KAAK,OAAO,MACV,6DACA,MAAM,OACN,IACD,CACF;;CAGL,oBAA4B,OAA2C;AAChE,OAAK,QACP,KAAK,kBAAkB,2BAA2B,MAAM,CACxD,OAAO,QACN,KAAK,OAAO,MACV,qEACA,MAAM,OACN,IACD,CACF"}