npm - @powerhousedao/reactor - Versions diffs - 6.2.3-dev.2 → 6.2.3-dev.21 - Mend

@powerhousedao/reactor 6.2.3-dev.2 → 6.2.3-dev.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (22) hide show

package/dist/projection-shard-manager-CPQc5XHL.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"projection-shard-manager-CPQc5XHL.js","names":[],"sources":["../src/projection/projection-shard-manager.ts"],"sourcesContent":["import type { OperationWithContext } from \"@powerhousedao/shared/document-model\";\nimport { childLogger, type ILogger } from \"document-model\";\nimport { randomUUID } from \"node:crypto\";\nimport { fromErrorInfo } from \"../executor/worker/error-info.js\";\nimport type { IEventBus } from \"../events/interfaces.js\";\nimport {\n ReactorEventTypes,\n type JobReadReadyEvent,\n type JobWriteReadyEvent,\n type ReadModelBatchCompletedEvent,\n type ReadModelIndexedEvent,\n type Unsubscribe,\n} from \"../events/types.js\";\nimport type {\n IReadModel,\n IReadModelCoordinator,\n} from \"../read-models/interfaces.js\";\nimport type { IConsistencyTracker } from \"../shared/consistency-tracker.js\";\nimport type { ConsistencyCoordinate } from \"../shared/types.js\";\nimport type { ForwardingPoolInstrumentation } from \"../storage/pool-instrumentation.js\";\nimport type {\n BuiltInReadModelKind,\n ChainDepthReport,\n DbConfig,\n ModelManifestEntry,\n ProjectionDrainedMessage,\n ProjectionInitMessage,\n ProjectionParentMessage,\n ProjectionPoolAcquireSamplesMessage,\n ProjectionReadModelIndexedMessage,\n ProjectionWorkerMessage,\n} from \"./protocol.js\";\nimport type { IProjectionTransport } from \"./transport.js\";\n\nconst DEFAULT_INIT_TIMEOUT_MS = 30_000;\nconst DEFAULT_SHUTDOWN_GRACE_MS = 5_000;\nconst DEFAULT_DRAIN_TIMEOUT_MS = 30_000;\nconst DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS = 250;\n\nconst FNV_OFFSET_BASIS = 0x811c9dc5;\nconst FNV_PRIME = 0x01000193;\n\nfunction bucketFor(documentId: string, numWorkers: number): number {\n if (numWorkers < 1) {\n throw new Error(`bucketFor: numWorkers must be >= 1 (got ${numWorkers})`);\n }\n let hash = FNV_OFFSET_BASIS;\n for (let i = 0; i < documentId.length; i++) {\n hash ^= documentId.charCodeAt(i);\n hash = Math.imul(hash, FNV_PRIME);\n }\n return (hash >>> 0) % numWorkers;\n}\n\n/**\n * Maps operations to the coordinates a consistency tracker is keyed by.\n * Same shape `BaseReadModel.updateConsistencyTracker` produces in-process.\n */\nfunction toConsistencyCoordinates(\n operations: OperationWithContext[],\n): ConsistencyCoordinate[] {\n const coordinates: ConsistencyCoordinate[] = [];\n for (let i = 0; i < operations.length; i++) {\n const item = operations[i]!;\n coordinates.push({\n documentId: item.context.documentId,\n scope: item.context.scope,\n branch: item.context.branch,\n operationIndex: item.operation.index,\n });\n }\n return coordinates;\n}\n\n/**\n * Factory that builds one projection-worker transport. Mirrors\n * `WorkerFactory` from the executor pool: lets tests inject fake\n * transports without spawning real worker threads.\n */\nexport type ProjectionWorkerFactory = (\n shardIndex: number,\n shardId: string,\n) => IProjectionTransport;\n\nexport type ProjectionShardHooks = {\n /** Takes over the host-bus JOB_READ_READY emit; call `emitReadReady` to emit. */\n onReadReady?: (event: JobReadReadyEvent) => void;\n /**\n * Fired when a shard errors, when it exits after having been ready, and\n * for every JOB_WRITE_READY dropped because its shard is not ready.\n *\n * There is no respawn path: once a shard stops being ready it never\n * projects again, so buffering the dropped work would only grow without\n * bound. A host that cares wires this to its shutdown path, so the process\n * restarts and each read model catches up from `ViewState.lastOrdinal` in\n * `BaseReadModel.init`. May fire repeatedly — handlers must be idempotent.\n */\n onShardFatal?: (shardId: string, reason: Error) => void;\n};\n\nexport type ProjectionShardManagerConfig = ProjectionShardHooks & {\n shardCount: number;\n db: DbConfig;\n models: ModelManifestEntry[];\n preReadyKinds: BuiltInReadModelKind[];\n postReadyKinds: BuiltInReadModelKind[];\n factory: ProjectionWorkerFactory;\n logger: ILogger;\n hostBus: IEventBus;\n initTimeoutMs?: number;\n shutdownGraceMs?: number;\n drainTimeoutMs?: number;\n chainDepthReportIntervalMs?: number;\n /**\n * Host-side forwarding instrumentations indexed by shard index. The\n * manager routes each shard's `pool-acquire-samples` message to the\n * matching forwarder so the host's OpenTelemetry instrumentation records\n * acquire-wait latencies as if each shard's pg.Pool were local.\n */\n poolInstrumentations?: ForwardingPoolInstrumentation[];\n /**\n * The host's consistency trackers for the built-in read models the shards\n * run, keyed by kind. The keys double as read-model names (see\n * `DOCUMENT_VIEW_READ_MODEL` / `DOCUMENT_INDEXER_READ_MODEL` in\n * `read-models/names.js`), which is how a relayed `readmodel-indexed`\n * message is matched to a tracker.\n *\n * The host's copies of these read models are never fed an operation under\n * sharding, so without this every read carrying a consistency token waits\n * on a tracker that can never advance — `ConsistencyTracker.waitFor` arms\n * no timer when `timeoutMs` is undefined, which is what every\n * `document-view` call site passes. The worker committed those rows to the\n * same tables the host reads from, so advancing here is exact, not a fudge.\n */\n consistencyTrackers?: Partial<\n Record<BuiltInReadModelKind, IConsistencyTracker>\n >;\n};\n\ntype ShardState = {\n shardIndex: number;\n shardId: string;\n transport: IProjectionTransport;\n ready: boolean;\n lastDepth: number;\n lastDepthAt: number;\n poolInstrumentation?: ForwardingPoolInstrumentation;\n /**\n * Consistency coordinates of the jobs this shard is still projecting,\n * keyed by jobId. Relayed `readmodel-indexed` messages carry only a count,\n * so the host keeps the coordinates here to advance its trackers from\n * them; the operations themselves are not retained. Empty unless\n * `consistencyTrackers` is configured; entries live from dispatch to\n * `readmodel-batch-completed`, and are dropped when the shard exits.\n */\n pendingCoordinates: Map<string, ConsistencyCoordinate[]>;\n /**\n * Correlation id of this shard's `init` while it is unsettled; cleared once\n * the shard reports ready, fails, or dies. Lets a transport error or a\n * premature exit settle the shard's own pending init instead of leaving\n * `startup()` to time out on a worker that will never answer.\n */\n initCorrelationId?: string;\n onMessage: (msg: ProjectionWorkerMessage) => void;\n onError: (err: Error) => void;\n onExit: (code: number) => void;\n};\n\ntype PendingDrain = {\n resolve: () => void;\n reject: (err: Error) => void;\n remaining: Set<string>;\n timer: NodeJS.Timeout;\n};\n\n/**\n * Host-side coordinator for N sharded projection workers.\n *\n * Implements {@link IReadModelCoordinator} so it slots into the same\n * `readModelCoordinator` field on the reactor module as the in-process\n * {@link ReadModelCoordinator}. The host subscribes to JOB_WRITE_READY\n * exactly once; events are routed to a single shard by\n * `bucketFor(documentId, shardCount)`. Each worker maintains the\n * per-queueKey serial chain locally and forwards JOB_READ_READY and\n * READMODEL_* events back to the host for the rest of the reactor (sync\n * manager, awaiters, observers) to consume on the host bus.\n *\n * @see Sharded projection workers sub-feature brief\n * (Powerhouse board wiki id: eb26f01f-8f68-4918-a6f6-ac7a4679b533)\n */\nexport class ProjectionShardManager implements IReadModelCoordinator {\n readonly readModels: IReadModel[] = [];\n\n private readonly config: ProjectionShardManagerConfig;\n private readonly logger: ILogger;\n private readonly hostBus: IEventBus;\n private readonly shards: ShardState[] = [];\n private readonly initPromises = new Map<\n string,\n { resolve: () => void; reject: (err: Error) => void; timer: NodeJS.Timeout }\n >();\n private readonly pendingDrains = new Map<string, PendingDrain>();\n private readonly trackersByReadModelName = new Map<\n string,\n IConsistencyTracker\n >();\n private hostSubscription?: Unsubscribe;\n private isRunning = false;\n private started = false;\n private isShuttingDown = false;\n\n constructor(config: ProjectionShardManagerConfig) {\n if (config.shardCount < 1) {\n throw new Error(\n `ProjectionShardManager: shardCount must be >= 1 (got ${config.shardCount})`,\n );\n }\n this.config = config;\n this.logger = childLogger([\"reactor\", \"projection-shard-manager\"]);\n this.hostBus = config.hostBus;\n const trackers = config.consistencyTrackers ?? {};\n for (const kind of Object.keys(trackers) as BuiltInReadModelKind[]) {\n const tracker = trackers[kind];\n if (tracker) {\n this.trackersByReadModelName.set(kind, tracker);\n }\n }\n }\n\n async startup(): Promise<void> {\n if (this.started) {\n return;\n }\n this.started = true;\n const initTimeoutMs = this.config.initTimeoutMs ?? DEFAULT_INIT_TIMEOUT_MS;\n const reportIntervalMs =\n this.config.chainDepthReportIntervalMs ??\n DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS;\n\n const initPromises: Promise<void>[] = [];\n for (let i = 0; i < this.config.shardCount; i++) {\n const shardId = `projection-shard-${i}`;\n const transport = this.config.factory(i, shardId);\n const state: ShardState = {\n shardIndex: i,\n shardId,\n transport,\n ready: false,\n lastDepth: 0,\n lastDepthAt: 0,\n poolInstrumentation: this.config.poolInstrumentations?.[i],\n pendingCoordinates: new Map(),\n onMessage: (msg) => this.handleWorkerMessage(state, msg),\n onError: (err) => this.handleTransportError(state, err),\n onExit: (code) => this.handleTransportExit(state, code),\n };\n transport.on(\"message\", state.onMessage);\n transport.on(\"error\", state.onError);\n transport.on(\"exit\", state.onExit);\n this.shards.push(state);\n\n const correlationId = randomUUID();\n state.initCorrelationId = correlationId;\n const initPromise = new Promise<void>((resolve, reject) => {\n const timer = setTimeout(() => {\n this.initPromises.delete(correlationId);\n reject(\n new Error(\n `projection shard ${shardId} did not become ready within ${initTimeoutMs}ms`,\n ),\n );\n }, initTimeoutMs);\n this.initPromises.set(correlationId, { resolve, reject, timer });\n });\n\n const init: ProjectionInitMessage = {\n type: \"init\",\n correlationId,\n shardId,\n shardIndex: i,\n shardCount: this.config.shardCount,\n db: this.config.db,\n models: this.config.models,\n preReadyKinds: this.config.preReadyKinds,\n postReadyKinds: this.config.postReadyKinds,\n chainDepthReportIntervalMs: reportIntervalMs,\n };\n transport.postMessage(init);\n initPromises.push(initPromise);\n }\n\n await Promise.all(initPromises);\n this.logger.info(\n \"projection shard manager ready: @count shards\",\n this.shards.length,\n );\n }\n\n start(): void {\n if (this.isRunning) {\n return;\n }\n this.hostSubscription = this.hostBus.subscribe(\n ReactorEventTypes.JOB_WRITE_READY,\n (_t: number, event: JobWriteReadyEvent) => {\n this.routeWriteReady(event);\n },\n );\n this.isRunning = true;\n }\n\n stop(): void {\n if (!this.isRunning) {\n return;\n }\n if (this.hostSubscription) {\n this.hostSubscription();\n this.hostSubscription = undefined;\n }\n this.isRunning = false;\n }\n\n /** Waits on ready shards only; `handleTransportExit` releases one that dies. */\n async drain(): Promise<void> {\n const readyShards = this.shards.filter((s) => s.ready);\n if (readyShards.length === 0) {\n return;\n }\n const drainTimeoutMs =\n this.config.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;\n const correlationId = randomUUID();\n const remaining = new Set(readyShards.map((s) => s.shardId));\n const promise = new Promise<void>((resolve, reject) => {\n const timer = setTimeout(() => {\n this.pendingDrains.delete(correlationId);\n reject(\n new Error(\n `projection shards did not drain within ${drainTimeoutMs}ms (remaining: ${[\n ...remaining,\n ].join(\", \")})`,\n ),\n );\n }, drainTimeoutMs);\n this.pendingDrains.set(correlationId, {\n resolve,\n reject,\n remaining,\n timer,\n });\n });\n for (const shard of readyShards) {\n shard.transport.postMessage({ type: \"drain\", correlationId });\n }\n await promise;\n }\n\n /** Emits JOB_READ_READY on the host bus; an `onReadReady` hook awaits this. */\n emitReadReady(event: JobReadReadyEvent): Promise<void> {\n return this.hostBus.emit(ReactorEventTypes.JOB_READ_READY, event);\n }\n\n getChainDepth(): number {\n let total = 0;\n for (const shard of this.shards) {\n total += shard.lastDepth;\n }\n return total;\n }\n\n getShardDepths(): ChainDepthReport[] {\n return this.shards.map((shard) => ({\n shardId: shard.shardId,\n depth: shard.lastDepth,\n timestamp: shard.lastDepthAt,\n }));\n }\n\n async shutdown(): Promise<void> {\n // Workers exit as a consequence of this call, so their `exit` events are\n // expected from here on and must not be reported as fatal.\n this.isShuttingDown = true;\n this.stop();\n // An init timer left armed by a shard that never reported keeps the\n // process alive for the whole init timeout after shutdown returns.\n for (const shard of this.shards) {\n this.failPendingInit(\n shard,\n new Error(\n `projection shard ${shard.shardId} was shut down before becoming ready`,\n ),\n );\n }\n const graceMs = this.config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;\n const correlationId = randomUUID();\n for (const shard of this.shards) {\n try {\n const msg: ProjectionParentMessage = {\n type: \"shutdown\",\n correlationId,\n graceMs,\n };\n shard.transport.postMessage(msg);\n } catch (err) {\n this.logger.warn(\n \"projection shard postMessage(shutdown) failed for @shardId: @error\",\n shard.shardId,\n err,\n );\n }\n }\n const terminationDeadline = Date.now() + graceMs;\n while (\n this.shards.some((s) => s.ready) &&\n Date.now() < terminationDeadline\n ) {\n await new Promise<void>((resolve) => setTimeout(resolve, 25));\n }\n for (const shard of this.shards) {\n try {\n await shard.transport.terminate();\n } catch (err) {\n this.logger.warn(\n \"projection shard terminate failed for @shardId: @error\",\n shard.shardId,\n err,\n );\n }\n shard.transport.off(\"message\", shard.onMessage);\n shard.transport.off(\"error\", shard.onError);\n shard.transport.off(\"exit\", shard.onExit);\n }\n this.shards.length = 0;\n }\n\n private routeWriteReady(event: JobWriteReadyEvent): void {\n if (event.operations.length === 0) {\n // No shard can be selected (there is no documentId) and no read model\n // has work to do, but JOB_READ_READY is the job's terminal signal:\n // InMemoryJobTracker and every JobAwaiter hang without it. Mirrors\n // ReadModelCoordinator.emitEmptyReadReady.\n this.relayReadReady({ jobId: event.jobId, operations: [] });\n this.relayBatchCompleted({\n jobId: event.jobId,\n batchSize: 0,\n chainWaitDurationMs: 0,\n preReadyDurationMs: 0,\n emitDurationMs: 0,\n postReadyDurationMs: 0,\n });\n return;\n }\n const documentId = event.operations[0]!.context.documentId;\n const index = bucketFor(documentId, this.shards.length);\n const shard = this.shards[index]!;\n if (!shard.ready) {\n this.dropWriteReady(shard, event, documentId);\n return;\n }\n if (this.trackersByReadModelName.size > 0) {\n shard.pendingCoordinates.set(\n event.jobId,\n toConsistencyCoordinates(event.operations),\n );\n }\n shard.transport.postMessage({\n type: \"write-ready\",\n jobId: event.jobId,\n operations: event.operations,\n jobMeta: event.jobMeta,\n collectionMemberships: event.collectionMemberships,\n });\n }\n\n /**\n * A shard stops being ready only when it dies, and nothing respawns it, so\n * this job's projection is genuinely lost. Buffering would grow without\n * bound behind a shard that never comes back, so the batch is dropped —\n * loudly, naming the job and document, and through `onShardFatal` so a host\n * can restart rather than serve stale read models.\n *\n * JOB_FAILED is deliberately not emitted: every other emitter uses it for a\n * job whose operations were *not* written (see\n * `executor/job-result-handler.ts`), and these were written and are\n * durable. Marking the job FAILED would invite the caller to re-submit a\n * write that already landed.\n */\n private dropWriteReady(\n shard: ShardState,\n event: JobWriteReadyEvent,\n documentId: string,\n ): void {\n const reason = new Error(\n `projection shard ${shard.shardId} is not ready: JOB_WRITE_READY for job ${event.jobId} ` +\n `(document ${documentId}, ${event.operations.length} operation(s)) was dropped and will ` +\n `never be projected; the operations are written and durable, but this shard's read models ` +\n `are now behind`,\n );\n this.logger.error(\n \"dropping JOB_WRITE_READY for job @jobId on shard @shardId: @error\",\n event.jobId,\n shard.shardId,\n reason,\n );\n this.config.onShardFatal?.(shard.shardId, reason);\n }\n\n private handleWorkerMessage(\n shard: ShardState,\n msg: ProjectionWorkerMessage,\n ): void {\n switch (msg.type) {\n case \"ready\":\n this.handleReady(shard, msg.correlationId);\n return;\n case \"init-failed\":\n this.logger.error(\n \"projection shard @shardId failed to initialize: @error\",\n shard.shardId,\n msg.error.message,\n );\n this.failPendingInit(shard, fromErrorInfo(msg.error));\n return;\n case \"read-ready\":\n this.relayReadReady({\n jobId: msg.jobId,\n operations: msg.operations,\n });\n return;\n case \"readmodel-indexed\":\n this.advanceConsistencyTrackers(shard, msg);\n this.relayReadModelIndexed({\n jobId: msg.jobId,\n readModelName: msg.readModelName,\n stage: msg.stage,\n durationMs: msg.durationMs,\n operationCount: msg.operationCount,\n success: msg.success,\n });\n return;\n case \"readmodel-batch-completed\":\n shard.pendingCoordinates.delete(msg.jobId);\n this.relayBatchCompleted({\n jobId: msg.jobId,\n batchSize: msg.batchSize,\n chainWaitDurationMs: msg.chainWaitDurationMs,\n preReadyDurationMs: msg.preReadyDurationMs,\n emitDurationMs: msg.emitDurationMs,\n postReadyDurationMs: msg.postReadyDurationMs,\n });\n return;\n case \"chain-depth\":\n shard.lastDepth = msg.depth;\n shard.lastDepthAt = msg.timestamp;\n return;\n case \"pool-acquire-samples\":\n this.handlePoolAcquireSamples(shard, msg);\n return;\n case \"drained\":\n this.handleDrained(msg);\n return;\n case \"log\":\n this.handleLog(shard, msg);\n return;\n default: {\n const exhaustive: never = msg;\n void exhaustive;\n return;\n }\n }\n }\n\n /**\n * Advances the host's tracker for the read model the shard just indexed.\n *\n * The shard writes to the same tables the host reads, so once it reports a\n * successful index the host's read path really is consistent to those\n * coordinates. Gated on `success` for parity with\n * `BaseReadModel.indexOperations`, which updates its tracker only after\n * `commitOperations` returns. The worker posts this before its\n * JOB_READ_READY, matching the in-process ordering.\n */\n private advanceConsistencyTrackers(\n shard: ShardState,\n msg: ProjectionReadModelIndexedMessage,\n ): void {\n if (!msg.success) {\n return;\n }\n const tracker = this.trackersByReadModelName.get(msg.readModelName);\n if (!tracker) {\n return;\n }\n const coordinates = shard.pendingCoordinates.get(msg.jobId);\n if (!coordinates || coordinates.length === 0) {\n return;\n }\n tracker.update(coordinates);\n }\n\n private handlePoolAcquireSamples(\n shard: ShardState,\n msg: ProjectionPoolAcquireSamplesMessage,\n ): void {\n if (!shard.poolInstrumentation) {\n return;\n }\n shard.poolInstrumentation.updateStats({\n size: msg.size,\n idle: msg.idle,\n waiting: msg.waiting,\n });\n shard.poolInstrumentation.pushSamples(msg.durations);\n }\n\n /**\n * Settles a shard's unresolved `init` with `error`, clearing its timer.\n * Returns false when the shard has no init outstanding, which is every\n * failure after startup. `startup()` awaits all init promises together, so\n * the rejection is always attached.\n */\n private failPendingInit(shard: ShardState, error: Error): boolean {\n const correlationId = shard.initCorrelationId;\n if (correlationId === undefined) {\n return false;\n }\n shard.initCorrelationId = undefined;\n const pending = this.initPromises.get(correlationId);\n if (!pending) {\n return false;\n }\n this.initPromises.delete(correlationId);\n clearTimeout(pending.timer);\n pending.reject(error);\n return true;\n }\n\n private handleReady(shard: ShardState, correlationId: string): void {\n shard.ready = true;\n shard.initCorrelationId = undefined;\n const pending = this.initPromises.get(correlationId);\n if (!pending) {\n return;\n }\n this.initPromises.delete(correlationId);\n clearTimeout(pending.timer);\n pending.resolve();\n }\n\n private handleDrained(msg: ProjectionDrainedMessage): void {\n const pending = this.pendingDrains.get(msg.correlationId);\n if (!pending) {\n return;\n }\n this.releaseDrain(msg.correlationId, pending, msg.shardId);\n }\n\n /** Removes `shardId` from a pending drain and settles it once no shard remains. */\n private releaseDrain(\n correlationId: string,\n pending: PendingDrain,\n shardId: string,\n ): void {\n pending.remaining.delete(shardId);\n if (pending.remaining.size > 0) {\n return;\n }\n this.pendingDrains.delete(correlationId);\n clearTimeout(pending.timer);\n pending.resolve();\n }\n\n private handleLog(\n shard: ShardState,\n msg: Extract<ProjectionWorkerMessage, { type: \"log\" }>,\n ): void {\n switch (msg.level) {\n case \"debug\":\n this.logger.debug(msg.message, ...msg.args);\n return;\n case \"info\":\n this.logger.info(msg.message, ...msg.args);\n return;\n case \"warn\":\n this.logger.warn(msg.message, ...msg.args);\n return;\n case \"error\":\n this.logger.error(msg.message, ...msg.args);\n return;\n default: {\n const exhaustive: never = msg.level;\n void exhaustive;\n }\n }\n void shard;\n }\n\n private handleTransportError(shard: ShardState, err: Error): void {\n this.logger.error(\n \"projection shard transport error @shardId: @error\",\n shard.shardId,\n err,\n );\n // A shard that dies during startup is reported through `startup()`, not\n // through the fatal hook: the host is still building and its shutdown\n // path (which the hook usually triggers) does not exist yet.\n if (this.failPendingInit(shard, err)) {\n return;\n }\n if (!this.isShuttingDown) {\n this.config.onShardFatal?.(shard.shardId, err);\n }\n }\n\n private handleTransportExit(shard: ShardState, code: number): void {\n const wasReady = shard.ready;\n shard.ready = false;\n // Nothing respawns the shard, so every job it had in flight is lost with\n // it. Release them rather than pin their operations for the process's\n // remaining life.\n const abandoned = [...shard.pendingCoordinates.keys()];\n shard.pendingCoordinates.clear();\n for (const [correlationId, pending] of this.pendingDrains) {\n this.releaseDrain(correlationId, pending, shard.shardId);\n }\n if (\n this.failPendingInit(\n shard,\n new Error(\n `projection shard ${shard.shardId} exited with code ${code} before becoming ready`,\n ),\n )\n ) {\n return;\n }\n if (!wasReady || this.isShuttingDown) {\n return;\n }\n this.logger.error(\n \"projection shard exited unexpectedly @shardId code=@code, abandoning @count in-flight job(s): @jobIds\",\n shard.shardId,\n code,\n abandoned.length,\n abandoned.join(\", \"),\n );\n this.config.onShardFatal?.(\n shard.shardId,\n new Error(\n `projection shard ${shard.shardId} exited with code ${code}; ` +\n `${abandoned.length} in-flight job(s) will never be projected` +\n (abandoned.length > 0 ? `: ${abandoned.join(\", \")}` : \"\"),\n ),\n );\n }\n\n private relayReadReady(event: JobReadReadyEvent): void {\n const hook = this.config.onReadReady;\n if (hook) {\n hook(event);\n return;\n }\n void this.emitReadReady(event).catch((err: unknown) =>\n this.logger.error(\n \"host JOB_READ_READY emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n\n private relayReadModelIndexed(event: ReadModelIndexedEvent): void {\n void this.hostBus\n .emit(ReactorEventTypes.READMODEL_INDEXED, event)\n .catch((err: unknown) =>\n this.logger.error(\n \"host READMODEL_INDEXED emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n\n private relayBatchCompleted(event: ReadModelBatchCompletedEvent): void {\n void this.hostBus\n .emit(ReactorEventTypes.READMODEL_BATCH_COMPLETED, event)\n .catch((err: unknown) =>\n this.logger.error(\n \"host READMODEL_BATCH_COMPLETED emit failed for job @jobId: @error\",\n event.jobId,\n err,\n ),\n );\n }\n}\n"],"mappings":";;;;;AAkCA,MAAM,0BAA0B;AAChC,MAAM,4BAA4B;AAClC,MAAM,2BAA2B;AACjC,MAAM,yCAAyC;AAE/C,MAAM,mBAAmB;AACzB,MAAM,YAAY;AAElB,SAAS,UAAU,YAAoB,YAA4B;AACjE,KAAI,aAAa,EACf,OAAM,IAAI,MAAM,2CAA2C,WAAW,GAAG;CAE3E,IAAI,OAAO;AACX,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,UAAQ,WAAW,WAAW,EAAE;AAChC,SAAO,KAAK,KAAK,MAAM,UAAU;;AAEnC,SAAQ,SAAS,KAAK;;;;;;AAOxB,SAAS,yBACP,YACyB;CACzB,MAAM,cAAuC,EAAE;AAC/C,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;EAC1C,MAAM,OAAO,WAAW;AACxB,cAAY,KAAK;GACf,YAAY,KAAK,QAAQ;GACzB,OAAO,KAAK,QAAQ;GACpB,QAAQ,KAAK,QAAQ;GACrB,gBAAgB,KAAK,UAAU;GAChC,CAAC;;AAEJ,QAAO;;;;;;;;;;;;;;;;;AAuHT,IAAa,yBAAb,MAAqE;CACnE,aAAoC,EAAE;CAEtC;CACA;CACA;CACA,SAAwC,EAAE;CAC1C,+BAAgC,IAAI,KAGjC;CACH,gCAAiC,IAAI,KAA2B;CAChE,0CAA2C,IAAI,KAG5C;CACH;CACA,YAAoB;CACpB,UAAkB;CAClB,iBAAyB;CAEzB,YAAY,QAAsC;AAChD,MAAI,OAAO,aAAa,EACtB,OAAM,IAAI,MACR,wDAAwD,OAAO,WAAW,GAC3E;AAEH,OAAK,SAAS;AACd,OAAK,SAAS,YAAY,CAAC,WAAW,2BAA2B,CAAC;AAClE,OAAK,UAAU,OAAO;EACtB,MAAM,WAAW,OAAO,uBAAuB,EAAE;AACjD,OAAK,MAAM,QAAQ,OAAO,KAAK,SAAS,EAA4B;GAClE,MAAM,UAAU,SAAS;AACzB,OAAI,QACF,MAAK,wBAAwB,IAAI,MAAM,QAAQ;;;CAKrD,MAAM,UAAyB;AAC7B,MAAI,KAAK,QACP;AAEF,OAAK,UAAU;EACf,MAAM,gBAAgB,KAAK,OAAO,iBAAiB;EACnD,MAAM,mBACJ,KAAK,OAAO,8BACZ;EAEF,MAAM,eAAgC,EAAE;AACxC,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,YAAY,KAAK;GAC/C,MAAM,UAAU,oBAAoB;GACpC,MAAM,YAAY,KAAK,OAAO,QAAQ,GAAG,QAAQ;GACjD,MAAM,QAAoB;IACxB,YAAY;IACZ;IACA;IACA,OAAO;IACP,WAAW;IACX,aAAa;IACb,qBAAqB,KAAK,OAAO,uBAAuB;IACxD,oCAAoB,IAAI,KAAK;IAC7B,YAAY,QAAQ,KAAK,oBAAoB,OAAO,IAAI;IACxD,UAAU,QAAQ,KAAK,qBAAqB,OAAO,IAAI;IACvD,SAAS,SAAS,KAAK,oBAAoB,OAAO,KAAK;IACxD;AACD,aAAU,GAAG,WAAW,MAAM,UAAU;AACxC,aAAU,GAAG,SAAS,MAAM,QAAQ;AACpC,aAAU,GAAG,QAAQ,MAAM,OAAO;AAClC,QAAK,OAAO,KAAK,MAAM;GAEvB,MAAM,gBAAgB,YAAY;AAClC,SAAM,oBAAoB;GAC1B,MAAM,cAAc,IAAI,SAAe,SAAS,WAAW;IACzD,MAAM,QAAQ,iBAAiB;AAC7B,UAAK,aAAa,OAAO,cAAc;AACvC,4BACE,IAAI,MACF,oBAAoB,QAAQ,+BAA+B,cAAc,IAC1E,CACF;OACA,cAAc;AACjB,SAAK,aAAa,IAAI,eAAe;KAAE;KAAS;KAAQ;KAAO,CAAC;KAChE;GAEF,MAAM,OAA8B;IAClC,MAAM;IACN;IACA;IACA,YAAY;IACZ,YAAY,KAAK,OAAO;IACxB,IAAI,KAAK,OAAO;IAChB,QAAQ,KAAK,OAAO;IACpB,eAAe,KAAK,OAAO;IAC3B,gBAAgB,KAAK,OAAO;IAC5B,4BAA4B;IAC7B;AACD,aAAU,YAAY,KAAK;AAC3B,gBAAa,KAAK,YAAY;;AAGhC,QAAM,QAAQ,IAAI,aAAa;AAC/B,OAAK,OAAO,KACV,iDACA,KAAK,OAAO,OACb;;CAGH,QAAc;AACZ,MAAI,KAAK,UACP;AAEF,OAAK,mBAAmB,KAAK,QAAQ,UACnC,kBAAkB,kBACjB,IAAY,UAA8B;AACzC,QAAK,gBAAgB,MAAM;IAE9B;AACD,OAAK,YAAY;;CAGnB,OAAa;AACX,MAAI,CAAC,KAAK,UACR;AAEF,MAAI,KAAK,kBAAkB;AACzB,QAAK,kBAAkB;AACvB,QAAK,mBAAmB,KAAA;;AAE1B,OAAK,YAAY;;;CAInB,MAAM,QAAuB;EAC3B,MAAM,cAAc,KAAK,OAAO,QAAQ,MAAM,EAAE,MAAM;AACtD,MAAI,YAAY,WAAW,EACzB;EAEF,MAAM,iBACJ,KAAK,OAAO,kBAAkB;EAChC,MAAM,gBAAgB,YAAY;EAClC,MAAM,YAAY,IAAI,IAAI,YAAY,KAAK,MAAM,EAAE,QAAQ,CAAC;EAC5D,MAAM,UAAU,IAAI,SAAe,SAAS,WAAW;GACrD,MAAM,QAAQ,iBAAiB;AAC7B,SAAK,cAAc,OAAO,cAAc;AACxC,2BACE,IAAI,MACF,0CAA0C,eAAe,iBAAiB,CACxE,GAAG,UACJ,CAAC,KAAK,KAAK,CAAC,GACd,CACF;MACA,eAAe;AAClB,QAAK,cAAc,IAAI,eAAe;IACpC;IACA;IACA;IACA;IACD,CAAC;IACF;AACF,OAAK,MAAM,SAAS,YAClB,OAAM,UAAU,YAAY;GAAE,MAAM;GAAS;GAAe,CAAC;AAE/D,QAAM;;;CAIR,cAAc,OAAyC;AACrD,SAAO,KAAK,QAAQ,KAAK,kBAAkB,gBAAgB,MAAM;;CAGnE,gBAAwB;EACtB,IAAI,QAAQ;AACZ,OAAK,MAAM,SAAS,KAAK,OACvB,UAAS,MAAM;AAEjB,SAAO;;CAGT,iBAAqC;AACnC,SAAO,KAAK,OAAO,KAAK,WAAW;GACjC,SAAS,MAAM;GACf,OAAO,MAAM;GACb,WAAW,MAAM;GAClB,EAAE;;CAGL,MAAM,WAA0B;AAG9B,OAAK,iBAAiB;AACtB,OAAK,MAAM;AAGX,OAAK,MAAM,SAAS,KAAK,OACvB,MAAK,gBACH,uBACA,IAAI,MACF,oBAAoB,MAAM,QAAQ,sCACnC,CACF;EAEH,MAAM,UAAU,KAAK,OAAO,mBAAmB;EAC/C,MAAM,gBAAgB,YAAY;AAClC,OAAK,MAAM,SAAS,KAAK,OACvB,KAAI;GACF,MAAM,MAA+B;IACnC,MAAM;IACN;IACA;IACD;AACD,SAAM,UAAU,YAAY,IAAI;WACzB,KAAK;AACZ,QAAK,OAAO,KACV,sEACA,MAAM,SACN,IACD;;EAGL,MAAM,sBAAsB,KAAK,KAAK,GAAG;AACzC,SACE,KAAK,OAAO,MAAM,MAAM,EAAE,MAAM,IAChC,KAAK,KAAK,GAAG,oBAEb,OAAM,IAAI,SAAe,YAAY,WAAW,SAAS,GAAG,CAAC;AAE/D,OAAK,MAAM,SAAS,KAAK,QAAQ;AAC/B,OAAI;AACF,UAAM,MAAM,UAAU,WAAW;YAC1B,KAAK;AACZ,SAAK,OAAO,KACV,0DACA,MAAM,SACN,IACD;;AAEH,SAAM,UAAU,IAAI,WAAW,MAAM,UAAU;AAC/C,SAAM,UAAU,IAAI,SAAS,MAAM,QAAQ;AAC3C,SAAM,UAAU,IAAI,QAAQ,MAAM,OAAO;;AAE3C,OAAK,OAAO,SAAS;;CAGvB,gBAAwB,OAAiC;AACvD,MAAI,MAAM,WAAW,WAAW,GAAG;AAKjC,QAAK,eAAe;IAAE,OAAO,MAAM;IAAO,YAAY,EAAE;IAAE,CAAC;AAC3D,QAAK,oBAAoB;IACvB,OAAO,MAAM;IACb,WAAW;IACX,qBAAqB;IACrB,oBAAoB;IACpB,gBAAgB;IAChB,qBAAqB;IACtB,CAAC;AACF;;EAEF,MAAM,aAAa,MAAM,WAAW,GAAI,QAAQ;EAChD,MAAM,QAAQ,UAAU,YAAY,KAAK,OAAO,OAAO;EACvD,MAAM,QAAQ,KAAK,OAAO;AAC1B,MAAI,CAAC,MAAM,OAAO;AAChB,QAAK,eAAe,OAAO,OAAO,WAAW;AAC7C;;AAEF,MAAI,KAAK,wBAAwB,OAAO,EACtC,OAAM,mBAAmB,IACvB,MAAM,OACN,yBAAyB,MAAM,WAAW,CAC3C;AAEH,QAAM,UAAU,YAAY;GAC1B,MAAM;GACN,OAAO,MAAM;GACb,YAAY,MAAM;GAClB,SAAS,MAAM;GACf,uBAAuB,MAAM;GAC9B,CAAC;;;;;;;;;;;;;;;CAgBJ,eACE,OACA,OACA,YACM;EACN,MAAM,yBAAS,IAAI,MACjB,oBAAoB,MAAM,QAAQ,yCAAyC,MAAM,MAAM,aACxE,WAAW,IAAI,MAAM,WAAW,OAAO,6IAGvD;AACD,OAAK,OAAO,MACV,qEACA,MAAM,OACN,MAAM,SACN,OACD;AACD,OAAK,OAAO,eAAe,MAAM,SAAS,OAAO;;CAGnD,oBACE,OACA,KACM;AACN,UAAQ,IAAI,MAAZ;GACE,KAAK;AACH,SAAK,YAAY,OAAO,IAAI,cAAc;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,MACV,0DACA,MAAM,SACN,IAAI,MAAM,QACX;AACD,SAAK,gBAAgB,OAAO,cAAc,IAAI,MAAM,CAAC;AACrD;GACF,KAAK;AACH,SAAK,eAAe;KAClB,OAAO,IAAI;KACX,YAAY,IAAI;KACjB,CAAC;AACF;GACF,KAAK;AACH,SAAK,2BAA2B,OAAO,IAAI;AAC3C,SAAK,sBAAsB;KACzB,OAAO,IAAI;KACX,eAAe,IAAI;KACnB,OAAO,IAAI;KACX,YAAY,IAAI;KAChB,gBAAgB,IAAI;KACpB,SAAS,IAAI;KACd,CAAC;AACF;GACF,KAAK;AACH,UAAM,mBAAmB,OAAO,IAAI,MAAM;AAC1C,SAAK,oBAAoB;KACvB,OAAO,IAAI;KACX,WAAW,IAAI;KACf,qBAAqB,IAAI;KACzB,oBAAoB,IAAI;KACxB,gBAAgB,IAAI;KACpB,qBAAqB,IAAI;KAC1B,CAAC;AACF;GACF,KAAK;AACH,UAAM,YAAY,IAAI;AACtB,UAAM,cAAc,IAAI;AACxB;GACF,KAAK;AACH,SAAK,yBAAyB,OAAO,IAAI;AACzC;GACF,KAAK;AACH,SAAK,cAAc,IAAI;AACvB;GACF,KAAK;AACH,SAAK,UAAU,OAAO,IAAI;AAC1B;GACF,QAGE;;;;;;;;;;;;;CAeN,2BACE,OACA,KACM;AACN,MAAI,CAAC,IAAI,QACP;EAEF,MAAM,UAAU,KAAK,wBAAwB,IAAI,IAAI,cAAc;AACnE,MAAI,CAAC,QACH;EAEF,MAAM,cAAc,MAAM,mBAAmB,IAAI,IAAI,MAAM;AAC3D,MAAI,CAAC,eAAe,YAAY,WAAW,EACzC;AAEF,UAAQ,OAAO,YAAY;;CAG7B,yBACE,OACA,KACM;AACN,MAAI,CAAC,MAAM,oBACT;AAEF,QAAM,oBAAoB,YAAY;GACpC,MAAM,IAAI;GACV,MAAM,IAAI;GACV,SAAS,IAAI;GACd,CAAC;AACF,QAAM,oBAAoB,YAAY,IAAI,UAAU;;;;;;;;CAStD,gBAAwB,OAAmB,OAAuB;EAChE,MAAM,gBAAgB,MAAM;AAC5B,MAAI,kBAAkB,KAAA,EACpB,QAAO;AAET,QAAM,oBAAoB,KAAA;EAC1B,MAAM,UAAU,KAAK,aAAa,IAAI,cAAc;AACpD,MAAI,CAAC,QACH,QAAO;AAET,OAAK,aAAa,OAAO,cAAc;AACvC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,OAAO,MAAM;AACrB,SAAO;;CAGT,YAAoB,OAAmB,eAA6B;AAClE,QAAM,QAAQ;AACd,QAAM,oBAAoB,KAAA;EAC1B,MAAM,UAAU,KAAK,aAAa,IAAI,cAAc;AACpD,MAAI,CAAC,QACH;AAEF,OAAK,aAAa,OAAO,cAAc;AACvC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,SAAS;;CAGnB,cAAsB,KAAqC;EACzD,MAAM,UAAU,KAAK,cAAc,IAAI,IAAI,cAAc;AACzD,MAAI,CAAC,QACH;AAEF,OAAK,aAAa,IAAI,eAAe,SAAS,IAAI,QAAQ;;;CAI5D,aACE,eACA,SACA,SACM;AACN,UAAQ,UAAU,OAAO,QAAQ;AACjC,MAAI,QAAQ,UAAU,OAAO,EAC3B;AAEF,OAAK,cAAc,OAAO,cAAc;AACxC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,SAAS;;CAGnB,UACE,OACA,KACM;AACN,UAAQ,IAAI,OAAZ;GACE,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,QAC4B,KAAI;;;CAOpC,qBAA6B,OAAmB,KAAkB;AAChE,OAAK,OAAO,MACV,qDACA,MAAM,SACN,IACD;AAID,MAAI,KAAK,gBAAgB,OAAO,IAAI,CAClC;AAEF,MAAI,CAAC,KAAK,eACR,MAAK,OAAO,eAAe,MAAM,SAAS,IAAI;;CAIlD,oBAA4B,OAAmB,MAAoB;EACjE,MAAM,WAAW,MAAM;AACvB,QAAM,QAAQ;EAId,MAAM,YAAY,CAAC,GAAG,MAAM,mBAAmB,MAAM,CAAC;AACtD,QAAM,mBAAmB,OAAO;AAChC,OAAK,MAAM,CAAC,eAAe,YAAY,KAAK,cAC1C,MAAK,aAAa,eAAe,SAAS,MAAM,QAAQ;AAE1D,MACE,KAAK,gBACH,uBACA,IAAI,MACF,oBAAoB,MAAM,QAAQ,oBAAoB,KAAK,wBAC5D,CACF,CAED;AAEF,MAAI,CAAC,YAAY,KAAK,eACpB;AAEF,OAAK,OAAO,MACV,yGACA,MAAM,SACN,MACA,UAAU,QACV,UAAU,KAAK,KAAK,CACrB;AACD,OAAK,OAAO,eACV,MAAM,yBACN,IAAI,MACF,oBAAoB,MAAM,QAAQ,oBAAoB,KAAK,IACtD,UAAU,OAAO,8CACnB,UAAU,SAAS,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,IACzD,CACF;;CAGH,eAAuB,OAAgC;EACrD,MAAM,OAAO,KAAK,OAAO;AACzB,MAAI,MAAM;AACR,QAAK,MAAM;AACX;;AAEG,OAAK,cAAc,MAAM,CAAC,OAAO,QACpC,KAAK,OAAO,MACV,0DACA,MAAM,OACN,IACD,CACF;;CAGH,sBAA8B,OAAoC;AAC3D,OAAK,QACP,KAAK,kBAAkB,mBAAmB,MAAM,CAChD,OAAO,QACN,KAAK,OAAO,MACV,6DACA,MAAM,OACN,IACD,CACF;;CAGL,oBAA4B,OAA2C;AAChE,OAAK,QACP,KAAK,kBAAkB,2BAA2B,MAAM,CACxD,OAAO,QACN,KAAK,OAAO,MACV,qEACA,MAAM,OACN,IACD,CACF"}