@powerhousedao/reactor 6.2.2-dev.82 → 6.2.2-dev.84

package/dist/{projection-shard-manager-BnQKFqQd.js → projection-shard-manager-D6KYcEZi.js}

@@ -18,6 +18,23 @@ function bucketFor(documentId, numWorkers) {
 	return (hash >>> 0) % numWorkers;
 }
 /**
+* Maps operations to the coordinates a consistency tracker is keyed by.
+* Same shape `BaseReadModel.updateConsistencyTracker` produces in-process.
+*/
+function toConsistencyCoordinates(operations) {
+	const coordinates = [];
+	for (let i = 0; i < operations.length; i++) {
+		const item = operations[i];
+		coordinates.push({
+			documentId: item.context.documentId,
+			scope: item.context.scope,
+			branch: item.context.branch,
+			operationIndex: item.operation.index
+		});
+	}
+	return coordinates;
+}
+/**
 * Host-side coordinator for N sharded projection workers.
 *
 * Implements {@link IReadModelCoordinator} so it slots into the same
@@ -40,14 +57,21 @@ var ProjectionShardManager = class {
 	shards = [];
 	initPromises = /* @__PURE__ */ new Map();
 	pendingDrains = /* @__PURE__ */ new Map();
+	trackersByReadModelName = /* @__PURE__ */ new Map();
 	hostSubscription;
 	isRunning = false;
 	started = false;
+	isShuttingDown = false;
 	constructor(config) {
 		if (config.shardCount < 1) throw new Error(`ProjectionShardManager: shardCount must be >= 1 (got ${config.shardCount})`);
 		this.config = config;
 		this.logger = childLogger(["reactor", "projection-shard-manager"]);
 		this.hostBus = config.hostBus;
+		const trackers = config.consistencyTrackers ?? {};
+		for (const kind of Object.keys(trackers)) {
+			const tracker = trackers[kind];
+			if (tracker) this.trackersByReadModelName.set(kind, tracker);
+		}
 	}
 	async startup() {
 		if (this.started) return;
@@ -66,6 +90,7 @@ var ProjectionShardManager = class {
 				lastDepth: 0,
 				lastDepthAt: 0,
 				poolInstrumentation: this.config.poolInstrumentations?.[i],
+				pendingCoordinates: /* @__PURE__ */ new Map(),
 				onMessage: (msg) => this.handleWorkerMessage(state, msg),
 				onError: (err) => this.handleTransportError(state, err),
 				onExit: (code) => this.handleTransportExit(state, code)
@@ -155,6 +180,7 @@ var ProjectionShardManager = class {
 		}));
 	}
 	async shutdown() {
+		this.isShuttingDown = true;
 		this.stop();
 		const graceMs = this.config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;
 		const correlationId = randomUUID();
@@ -183,14 +209,29 @@ var ProjectionShardManager = class {
 		this.shards.length = 0;
 	}
 	routeWriteReady(event) {
-		if (event.operations.length === 0) return;
+		if (event.operations.length === 0) {
+			this.relayReadReady({
+				jobId: event.jobId,
+				operations: []
+			});
+			this.relayBatchCompleted({
+				jobId: event.jobId,
+				batchSize: 0,
+				chainWaitDurationMs: 0,
+				preReadyDurationMs: 0,
+				emitDurationMs: 0,
+				postReadyDurationMs: 0
+			});
+			return;
+		}
 		const documentId = event.operations[0].context.documentId;
 		const index = bucketFor(documentId, this.shards.length);
 		const shard = this.shards[index];
 		if (!shard.ready) {
-			this.logger.warn("dropping JOB_WRITE_READY: shard @index not ready", index);
+			this.dropWriteReady(shard, event, documentId);
 			return;
 		}
+		if (this.trackersByReadModelName.size > 0) shard.pendingCoordinates.set(event.jobId, toConsistencyCoordinates(event.operations));
 		shard.transport.postMessage({
 			type: "write-ready",
 			jobId: event.jobId,
@@ -199,6 +240,24 @@ var ProjectionShardManager = class {
 			collectionMemberships: event.collectionMemberships
 		});
 	}
+	/**
+	* A shard stops being ready only when it dies, and nothing respawns it, so
+	* this job's projection is genuinely lost. Buffering would grow without
+	* bound behind a shard that never comes back, so the batch is dropped —
+	* loudly, naming the job and document, and through `onShardFatal` so a host
+	* can restart rather than serve stale read models.
+	*
+	* JOB_FAILED is deliberately not emitted: every other emitter uses it for a
+	* job whose operations were *not* written (see
+	* `executor/job-result-handler.ts`), and these were written and are
+	* durable. Marking the job FAILED would invite the caller to re-submit a
+	* write that already landed.
+	*/
+	dropWriteReady(shard, event, documentId) {
+		const reason = /* @__PURE__ */ new Error(`projection shard ${shard.shardId} is not ready: JOB_WRITE_READY for job ${event.jobId} (document ${documentId}, ${event.operations.length} operation(s)) was dropped and will never be projected; the operations are written and durable, but this shard's read models are now behind`);
+		this.logger.error("dropping JOB_WRITE_READY for job @jobId on shard @shardId: @error", event.jobId, shard.shardId, reason);
+		this.config.onShardFatal?.(shard.shardId, reason);
+	}
 	handleWorkerMessage(shard, msg) {
 		switch (msg.type) {
 			case "ready":
@@ -211,6 +270,7 @@ var ProjectionShardManager = class {
 				});
 				return;
 			case "readmodel-indexed":
+				this.advanceConsistencyTrackers(shard, msg);
 				this.relayReadModelIndexed({
 					jobId: msg.jobId,
 					readModelName: msg.readModelName,
@@ -221,6 +281,7 @@ var ProjectionShardManager = class {
 				});
 				return;
 			case "readmodel-batch-completed":
+				shard.pendingCoordinates.delete(msg.jobId);
 				this.relayBatchCompleted({
 					jobId: msg.jobId,
 					batchSize: msg.batchSize,
@@ -246,6 +307,24 @@ var ProjectionShardManager = class {
 			default: return;
 		}
 	}
+	/**
+	* Advances the host's tracker for the read model the shard just indexed.
+	*
+	* The shard writes to the same tables the host reads, so once it reports a
+	* successful index the host's read path really is consistent to those
+	* coordinates. Gated on `success` for parity with
+	* `BaseReadModel.indexOperations`, which updates its tracker only after
+	* `commitOperations` returns. The worker posts this before its
+	* JOB_READ_READY, matching the in-process ordering.
+	*/
+	advanceConsistencyTrackers(shard, msg) {
+		if (!msg.success) return;
+		const tracker = this.trackersByReadModelName.get(msg.readModelName);
+		if (!tracker) return;
+		const coordinates = shard.pendingCoordinates.get(msg.jobId);
+		if (!coordinates || coordinates.length === 0) return;
+		tracker.update(coordinates);
+	}
 	handlePoolAcquireSamples(shard, msg) {
 		if (!shard.poolInstrumentation) return;
 		shard.poolInstrumentation.updateStats({
@@ -292,10 +371,16 @@ var ProjectionShardManager = class {
 	}
 	handleTransportError(shard, err) {
 		this.logger.error("projection shard transport error @shardId: @error", shard.shardId, err);
+		if (!this.isShuttingDown) this.config.onShardFatal?.(shard.shardId, err);
 	}
 	handleTransportExit(shard, code) {
-		if (shard.ready) this.logger.warn("projection shard exited unexpectedly @shardId code=@code", shard.shardId, code);
+		const wasReady = shard.ready;
 		shard.ready = false;
+		const abandoned = [...shard.pendingCoordinates.keys()];
+		shard.pendingCoordinates.clear();
+		if (!wasReady || this.isShuttingDown) return;
+		this.logger.error("projection shard exited unexpectedly @shardId code=@code, abandoning @count in-flight job(s): @jobIds", shard.shardId, code, abandoned.length, abandoned.join(", "));
+		this.config.onShardFatal?.(shard.shardId, /* @__PURE__ */ new Error(`projection shard ${shard.shardId} exited with code ${code}; ${abandoned.length} in-flight job(s) will never be projected` + (abandoned.length > 0 ? `: ${abandoned.join(", ")}` : "")));
 	}
 	relayReadReady(event) {
 		this.hostBus.emit(ReactorEventTypes.JOB_READ_READY, event).catch((err) => this.logger.error("host JOB_READ_READY emit failed for job @jobId: @error", event.jobId, err));
@@ -310,4 +395,4 @@ var ProjectionShardManager = class {
 //#endregion
 export { ProjectionShardManager };
 
-//# sourceMappingURL=projection-shard-manager-BnQKFqQd.js.map
+//# sourceMappingURL=projection-shard-manager-D6KYcEZi.js.map

package/dist/projection-shard-manager-D6KYcEZi.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"projection-shard-manager-D6KYcEZi.js","names":[],"sources":["../src/projection/projection-shard-manager.ts"],"sourcesContent":["import type { OperationWithContext } from \"@powerhousedao/shared/document-model\";\nimport { childLogger, type ILogger } from \"document-model\";\nimport { randomUUID } from \"node:crypto\";\nimport type { IEventBus } from \"../events/interfaces.js\";\nimport {\n  ReactorEventTypes,\n  type JobReadReadyEvent,\n  type JobWriteReadyEvent,\n  type ReadModelBatchCompletedEvent,\n  type ReadModelIndexedEvent,\n  type Unsubscribe,\n} from \"../events/types.js\";\nimport type {\n  IReadModel,\n  IReadModelCoordinator,\n} from \"../read-models/interfaces.js\";\nimport type { IConsistencyTracker } from \"../shared/consistency-tracker.js\";\nimport type { ConsistencyCoordinate } from \"../shared/types.js\";\nimport type { ForwardingPoolInstrumentation } from \"../storage/pool-instrumentation.js\";\nimport type {\n  BuiltInReadModelKind,\n  ChainDepthReport,\n  DbConfig,\n  ModelManifestEntry,\n  ProjectionDrainedMessage,\n  ProjectionInitMessage,\n  ProjectionParentMessage,\n  ProjectionPoolAcquireSamplesMessage,\n  ProjectionReadModelIndexedMessage,\n  ProjectionWorkerMessage,\n} from \"./protocol.js\";\nimport type { IProjectionTransport } from \"./transport.js\";\n\nconst DEFAULT_INIT_TIMEOUT_MS = 30_000;\nconst DEFAULT_SHUTDOWN_GRACE_MS = 5_000;\nconst DEFAULT_DRAIN_TIMEOUT_MS = 30_000;\nconst DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS = 250;\n\nconst FNV_OFFSET_BASIS = 0x811c9dc5;\nconst FNV_PRIME = 0x01000193;\n\nfunction bucketFor(documentId: string, numWorkers: number): number {\n  if (numWorkers < 1) {\n    throw new Error(`bucketFor: numWorkers must be >= 1 (got ${numWorkers})`);\n  }\n  let hash = FNV_OFFSET_BASIS;\n  for (let i = 0; i < documentId.length; i++) {\n    hash ^= documentId.charCodeAt(i);\n    hash = Math.imul(hash, FNV_PRIME);\n  }\n  return (hash >>> 0) % numWorkers;\n}\n\n/**\n * Maps operations to the coordinates a consistency tracker is keyed by.\n * Same shape `BaseReadModel.updateConsistencyTracker` produces in-process.\n */\nfunction toConsistencyCoordinates(\n  operations: OperationWithContext[],\n): ConsistencyCoordinate[] {\n  const coordinates: ConsistencyCoordinate[] = [];\n  for (let i = 0; i < operations.length; i++) {\n    const item = operations[i]!;\n    coordinates.push({\n      documentId: item.context.documentId,\n      scope: item.context.scope,\n      branch: item.context.branch,\n      operationIndex: item.operation.index,\n    });\n  }\n  return coordinates;\n}\n\n/**\n * Factory that builds one projection-worker transport. Mirrors\n * `WorkerFactory` from the executor pool: lets tests inject fake\n * transports without spawning real worker threads.\n */\nexport type ProjectionWorkerFactory = (\n  shardIndex: number,\n  shardId: string,\n) => IProjectionTransport;\n\nexport type ProjectionShardManagerConfig = {\n  shardCount: number;\n  db: DbConfig;\n  models: ModelManifestEntry[];\n  preReadyKinds: BuiltInReadModelKind[];\n  postReadyKinds: BuiltInReadModelKind[];\n  factory: ProjectionWorkerFactory;\n  logger: ILogger;\n  hostBus: IEventBus;\n  initTimeoutMs?: number;\n  shutdownGraceMs?: number;\n  drainTimeoutMs?: number;\n  chainDepthReportIntervalMs?: number;\n  /**\n   * Host-side forwarding instrumentations indexed by shard index. The\n   * manager routes each shard's `pool-acquire-samples` message to the\n   * matching forwarder so the host's OpenTelemetry instrumentation records\n   * acquire-wait latencies as if each shard's pg.Pool were local.\n   */\n  poolInstrumentations?: ForwardingPoolInstrumentation[];\n  /**\n   * The host's consistency trackers for the built-in read models the shards\n   * run, keyed by kind. The keys double as read-model names (see\n   * `DOCUMENT_VIEW_READ_MODEL` / `DOCUMENT_INDEXER_READ_MODEL` in\n   * `read-models/names.js`), which is how a relayed `readmodel-indexed`\n   * message is matched to a tracker.\n   *\n   * The host's copies of these read models are never fed an operation under\n   * sharding, so without this every read carrying a consistency token waits\n   * on a tracker that can never advance — `ConsistencyTracker.waitFor` arms\n   * no timer when `timeoutMs` is undefined, which is what every\n   * `document-view` call site passes. The worker committed those rows to the\n   * same tables the host reads from, so advancing here is exact, not a fudge.\n   */\n  consistencyTrackers?: Partial<\n    Record<BuiltInReadModelKind, IConsistencyTracker>\n  >;\n  /**\n   * Fired when a shard errors, when it exits after having been ready, and\n   * for every JOB_WRITE_READY dropped because its shard is not ready.\n   *\n   * There is no respawn path: once a shard stops being ready it never\n   * projects again, so buffering the dropped work would only grow without\n   * bound. A host that cares wires this to its shutdown path, so the process\n   * restarts and each read model catches up from `ViewState.lastOrdinal` in\n   * `BaseReadModel.init`. May fire repeatedly — handlers must be idempotent.\n   */\n  onShardFatal?: (shardId: string, reason: Error) => void;\n};\n\ntype ShardState = {\n  shardIndex: number;\n  shardId: string;\n  transport: IProjectionTransport;\n  ready: boolean;\n  lastDepth: number;\n  lastDepthAt: number;\n  poolInstrumentation?: ForwardingPoolInstrumentation;\n  /**\n   * Consistency coordinates of the jobs this shard is still projecting,\n   * keyed by jobId. Relayed `readmodel-indexed` messages carry only a count,\n   * so the host keeps the coordinates here to advance its trackers from\n   * them; the operations themselves are not retained. Empty unless\n   * `consistencyTrackers` is configured; entries live from dispatch to\n   * `readmodel-batch-completed`, and are dropped when the shard exits.\n   */\n  pendingCoordinates: Map<string, ConsistencyCoordinate[]>;\n  onMessage: (msg: ProjectionWorkerMessage) => void;\n  onError: (err: Error) => void;\n  onExit: (code: number) => void;\n};\n\ntype PendingDrain = {\n  resolve: () => void;\n  reject: (err: Error) => void;\n  remaining: Set<string>;\n  timer: NodeJS.Timeout;\n};\n\n/**\n * Host-side coordinator for N sharded projection workers.\n *\n * Implements {@link IReadModelCoordinator} so it slots into the same\n * `readModelCoordinator` field on the reactor module as the in-process\n * {@link ReadModelCoordinator}. The host subscribes to JOB_WRITE_READY\n * exactly once; events are routed to a single shard by\n * `bucketFor(documentId, shardCount)`. Each worker maintains the\n * per-queueKey serial chain locally and forwards JOB_READ_READY and\n * READMODEL_* events back to the host for the rest of the reactor (sync\n * manager, awaiters, observers) to consume on the host bus.\n *\n * @see Sharded projection workers sub-feature brief\n *   (Powerhouse board wiki id: eb26f01f-8f68-4918-a6f6-ac7a4679b533)\n */\nexport class ProjectionShardManager implements IReadModelCoordinator {\n  readonly readModels: IReadModel[] = [];\n\n  private readonly config: ProjectionShardManagerConfig;\n  private readonly logger: ILogger;\n  private readonly hostBus: IEventBus;\n  private readonly shards: ShardState[] = [];\n  private readonly initPromises = new Map<\n    string,\n    { resolve: () => void; reject: (err: Error) => void; timer: NodeJS.Timeout }\n  >();\n  private readonly pendingDrains = new Map<string, PendingDrain>();\n  private readonly trackersByReadModelName = new Map<\n    string,\n    IConsistencyTracker\n  >();\n  private hostSubscription?: Unsubscribe;\n  private isRunning = false;\n  private started = false;\n  private isShuttingDown = false;\n\n  constructor(config: ProjectionShardManagerConfig) {\n    if (config.shardCount < 1) {\n      throw new Error(\n        `ProjectionShardManager: shardCount must be >= 1 (got ${config.shardCount})`,\n      );\n    }\n    this.config = config;\n    this.logger = childLogger([\"reactor\", \"projection-shard-manager\"]);\n    this.hostBus = config.hostBus;\n    const trackers = config.consistencyTrackers ?? {};\n    for (const kind of Object.keys(trackers) as BuiltInReadModelKind[]) {\n      const tracker = trackers[kind];\n      if (tracker) {\n        this.trackersByReadModelName.set(kind, tracker);\n      }\n    }\n  }\n\n  async startup(): Promise<void> {\n    if (this.started) {\n      return;\n    }\n    this.started = true;\n    const initTimeoutMs = this.config.initTimeoutMs ?? DEFAULT_INIT_TIMEOUT_MS;\n    const reportIntervalMs =\n      this.config.chainDepthReportIntervalMs ??\n      DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS;\n\n    const initPromises: Promise<void>[] = [];\n    for (let i = 0; i < this.config.shardCount; i++) {\n      const shardId = `projection-shard-${i}`;\n      const transport = this.config.factory(i, shardId);\n      const state: ShardState = {\n        shardIndex: i,\n        shardId,\n        transport,\n        ready: false,\n        lastDepth: 0,\n        lastDepthAt: 0,\n        poolInstrumentation: this.config.poolInstrumentations?.[i],\n        pendingCoordinates: new Map(),\n        onMessage: (msg) => this.handleWorkerMessage(state, msg),\n        onError: (err) => this.handleTransportError(state, err),\n        onExit: (code) => this.handleTransportExit(state, code),\n      };\n      transport.on(\"message\", state.onMessage);\n      transport.on(\"error\", state.onError);\n      transport.on(\"exit\", state.onExit);\n      this.shards.push(state);\n\n      const correlationId = randomUUID();\n      const initPromise = new Promise<void>((resolve, reject) => {\n        const timer = setTimeout(() => {\n          this.initPromises.delete(correlationId);\n          reject(\n            new Error(\n              `projection shard ${shardId} did not become ready within ${initTimeoutMs}ms`,\n            ),\n          );\n        }, initTimeoutMs);\n        this.initPromises.set(correlationId, { resolve, reject, timer });\n      });\n\n      const init: ProjectionInitMessage = {\n        type: \"init\",\n        correlationId,\n        shardId,\n        shardIndex: i,\n        shardCount: this.config.shardCount,\n        db: this.config.db,\n        models: this.config.models,\n        preReadyKinds: this.config.preReadyKinds,\n        postReadyKinds: this.config.postReadyKinds,\n        chainDepthReportIntervalMs: reportIntervalMs,\n      };\n      transport.postMessage(init);\n      initPromises.push(initPromise);\n    }\n\n    await Promise.all(initPromises);\n    this.logger.info(\n      \"projection shard manager ready: @count shards\",\n      this.shards.length,\n    );\n  }\n\n  start(): void {\n    if (this.isRunning) {\n      return;\n    }\n    this.hostSubscription = this.hostBus.subscribe(\n      ReactorEventTypes.JOB_WRITE_READY,\n      (_t: number, event: JobWriteReadyEvent) => {\n        this.routeWriteReady(event);\n      },\n    );\n    this.isRunning = true;\n  }\n\n  stop(): void {\n    if (!this.isRunning) {\n      return;\n    }\n    if (this.hostSubscription) {\n      this.hostSubscription();\n      this.hostSubscription = undefined;\n    }\n    this.isRunning = false;\n  }\n\n  async drain(): Promise<void> {\n    if (this.shards.length === 0) {\n      return;\n    }\n    const drainTimeoutMs =\n      this.config.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;\n    const correlationId = randomUUID();\n    const remaining = new Set(this.shards.map((s) => s.shardId));\n    const promise = new Promise<void>((resolve, reject) => {\n      const timer = setTimeout(() => {\n        this.pendingDrains.delete(correlationId);\n        reject(\n          new Error(\n            `projection shards did not drain within ${drainTimeoutMs}ms (remaining: ${[\n              ...remaining,\n            ].join(\", \")})`,\n          ),\n        );\n      }, drainTimeoutMs);\n      this.pendingDrains.set(correlationId, {\n        resolve,\n        reject,\n        remaining,\n        timer,\n      });\n    });\n    for (const shard of this.shards) {\n      shard.transport.postMessage({ type: \"drain\", correlationId });\n    }\n    await promise;\n  }\n\n  getChainDepth(): number {\n    let total = 0;\n    for (const shard of this.shards) {\n      total += shard.lastDepth;\n    }\n    return total;\n  }\n\n  getShardDepths(): ChainDepthReport[] {\n    return this.shards.map((shard) => ({\n      shardId: shard.shardId,\n      depth: shard.lastDepth,\n      timestamp: shard.lastDepthAt,\n    }));\n  }\n\n  async shutdown(): Promise<void> {\n    // Workers exit as a consequence of this call, so their `exit` events are\n    // expected from here on and must not be reported as fatal.\n    this.isShuttingDown = true;\n    this.stop();\n    const graceMs = this.config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;\n    const correlationId = randomUUID();\n    for (const shard of this.shards) {\n      try {\n        const msg: ProjectionParentMessage = {\n          type: \"shutdown\",\n          correlationId,\n          graceMs,\n        };\n        shard.transport.postMessage(msg);\n      } catch (err) {\n        this.logger.warn(\n          \"projection shard postMessage(shutdown) failed for @shardId: @error\",\n          shard.shardId,\n          err,\n        );\n      }\n    }\n    const terminationDeadline = Date.now() + graceMs;\n    while (\n      this.shards.some((s) => s.ready) &&\n      Date.now() < terminationDeadline\n    ) {\n      await new Promise<void>((resolve) => setTimeout(resolve, 25));\n    }\n    for (const shard of this.shards) {\n      try {\n        await shard.transport.terminate();\n      } catch (err) {\n        this.logger.warn(\n          \"projection shard terminate failed for @shardId: @error\",\n          shard.shardId,\n          err,\n        );\n      }\n      shard.transport.off(\"message\", shard.onMessage);\n      shard.transport.off(\"error\", shard.onError);\n      shard.transport.off(\"exit\", shard.onExit);\n    }\n    this.shards.length = 0;\n  }\n\n  private routeWriteReady(event: JobWriteReadyEvent): void {\n    if (event.operations.length === 0) {\n      // No shard can be selected (there is no documentId) and no read model\n      // has work to do, but JOB_READ_READY is the job's terminal signal:\n      // InMemoryJobTracker and every JobAwaiter hang without it. Mirrors\n      // ReadModelCoordinator.emitEmptyReadReady.\n      this.relayReadReady({ jobId: event.jobId, operations: [] });\n      this.relayBatchCompleted({\n        jobId: event.jobId,\n        batchSize: 0,\n        chainWaitDurationMs: 0,\n        preReadyDurationMs: 0,\n        emitDurationMs: 0,\n        postReadyDurationMs: 0,\n      });\n      return;\n    }\n    const documentId = event.operations[0]!.context.documentId;\n    const index = bucketFor(documentId, this.shards.length);\n    const shard = this.shards[index]!;\n    if (!shard.ready) {\n      this.dropWriteReady(shard, event, documentId);\n      return;\n    }\n    if (this.trackersByReadModelName.size > 0) {\n      shard.pendingCoordinates.set(\n        event.jobId,\n        toConsistencyCoordinates(event.operations),\n      );\n    }\n    shard.transport.postMessage({\n      type: \"write-ready\",\n      jobId: event.jobId,\n      operations: event.operations,\n      jobMeta: event.jobMeta,\n      collectionMemberships: event.collectionMemberships,\n    });\n  }\n\n  /**\n   * A shard stops being ready only when it dies, and nothing respawns it, so\n   * this job's projection is genuinely lost. Buffering would grow without\n   * bound behind a shard that never comes back, so the batch is dropped —\n   * loudly, naming the job and document, and through `onShardFatal` so a host\n   * can restart rather than serve stale read models.\n   *\n   * JOB_FAILED is deliberately not emitted: every other emitter uses it for a\n   * job whose operations were *not* written (see\n   * `executor/job-result-handler.ts`), and these were written and are\n   * durable. Marking the job FAILED would invite the caller to re-submit a\n   * write that already landed.\n   */\n  private dropWriteReady(\n    shard: ShardState,\n    event: JobWriteReadyEvent,\n    documentId: string,\n  ): void {\n    const reason = new Error(\n      `projection shard ${shard.shardId} is not ready: JOB_WRITE_READY for job ${event.jobId} ` +\n        `(document ${documentId}, ${event.operations.length} operation(s)) was dropped and will ` +\n        `never be projected; the operations are written and durable, but this shard's read models ` +\n        `are now behind`,\n    );\n    this.logger.error(\n      \"dropping JOB_WRITE_READY for job @jobId on shard @shardId: @error\",\n      event.jobId,\n      shard.shardId,\n      reason,\n    );\n    this.config.onShardFatal?.(shard.shardId, reason);\n  }\n\n  private handleWorkerMessage(\n    shard: ShardState,\n    msg: ProjectionWorkerMessage,\n  ): void {\n    switch (msg.type) {\n      case \"ready\":\n        this.handleReady(shard, msg.correlationId);\n        return;\n      case \"read-ready\":\n        this.relayReadReady({\n          jobId: msg.jobId,\n          operations: msg.operations,\n        });\n        return;\n      case \"readmodel-indexed\":\n        this.advanceConsistencyTrackers(shard, msg);\n        this.relayReadModelIndexed({\n          jobId: msg.jobId,\n          readModelName: msg.readModelName,\n          stage: msg.stage,\n          durationMs: msg.durationMs,\n          operationCount: msg.operationCount,\n          success: msg.success,\n        });\n        return;\n      case \"readmodel-batch-completed\":\n        shard.pendingCoordinates.delete(msg.jobId);\n        this.relayBatchCompleted({\n          jobId: msg.jobId,\n          batchSize: msg.batchSize,\n          chainWaitDurationMs: msg.chainWaitDurationMs,\n          preReadyDurationMs: msg.preReadyDurationMs,\n          emitDurationMs: msg.emitDurationMs,\n          postReadyDurationMs: msg.postReadyDurationMs,\n        });\n        return;\n      case \"chain-depth\":\n        shard.lastDepth = msg.depth;\n        shard.lastDepthAt = msg.timestamp;\n        return;\n      case \"pool-acquire-samples\":\n        this.handlePoolAcquireSamples(shard, msg);\n        return;\n      case \"drained\":\n        this.handleDrained(msg);\n        return;\n      case \"log\":\n        this.handleLog(shard, msg);\n        return;\n      default: {\n        const exhaustive: never = msg;\n        void exhaustive;\n        return;\n      }\n    }\n  }\n\n  /**\n   * Advances the host's tracker for the read model the shard just indexed.\n   *\n   * The shard writes to the same tables the host reads, so once it reports a\n   * successful index the host's read path really is consistent to those\n   * coordinates. Gated on `success` for parity with\n   * `BaseReadModel.indexOperations`, which updates its tracker only after\n   * `commitOperations` returns. The worker posts this before its\n   * JOB_READ_READY, matching the in-process ordering.\n   */\n  private advanceConsistencyTrackers(\n    shard: ShardState,\n    msg: ProjectionReadModelIndexedMessage,\n  ): void {\n    if (!msg.success) {\n      return;\n    }\n    const tracker = this.trackersByReadModelName.get(msg.readModelName);\n    if (!tracker) {\n      return;\n    }\n    const coordinates = shard.pendingCoordinates.get(msg.jobId);\n    if (!coordinates || coordinates.length === 0) {\n      return;\n    }\n    tracker.update(coordinates);\n  }\n\n  private handlePoolAcquireSamples(\n    shard: ShardState,\n    msg: ProjectionPoolAcquireSamplesMessage,\n  ): void {\n    if (!shard.poolInstrumentation) {\n      return;\n    }\n    shard.poolInstrumentation.updateStats({\n      size: msg.size,\n      idle: msg.idle,\n      waiting: msg.waiting,\n    });\n    shard.poolInstrumentation.pushSamples(msg.durations);\n  }\n\n  private handleReady(shard: ShardState, correlationId: string): void {\n    shard.ready = true;\n    const pending = this.initPromises.get(correlationId);\n    if (!pending) {\n      return;\n    }\n    this.initPromises.delete(correlationId);\n    clearTimeout(pending.timer);\n    pending.resolve();\n  }\n\n  private handleDrained(msg: ProjectionDrainedMessage): void {\n    const pending = this.pendingDrains.get(msg.correlationId);\n    if (!pending) {\n      return;\n    }\n    pending.remaining.delete(msg.shardId);\n    if (pending.remaining.size === 0) {\n      this.pendingDrains.delete(msg.correlationId);\n      clearTimeout(pending.timer);\n      pending.resolve();\n    }\n  }\n\n  private handleLog(\n    shard: ShardState,\n    msg: Extract<ProjectionWorkerMessage, { type: \"log\" }>,\n  ): void {\n    switch (msg.level) {\n      case \"debug\":\n        this.logger.debug(msg.message, ...msg.args);\n        return;\n      case \"info\":\n        this.logger.info(msg.message, ...msg.args);\n        return;\n      case \"warn\":\n        this.logger.warn(msg.message, ...msg.args);\n        return;\n      case \"error\":\n        this.logger.error(msg.message, ...msg.args);\n        return;\n      default: {\n        const exhaustive: never = msg.level;\n        void exhaustive;\n      }\n    }\n    void shard;\n  }\n\n  private handleTransportError(shard: ShardState, err: Error): void {\n    this.logger.error(\n      \"projection shard transport error @shardId: @error\",\n      shard.shardId,\n      err,\n    );\n    if (!this.isShuttingDown) {\n      this.config.onShardFatal?.(shard.shardId, err);\n    }\n  }\n\n  private handleTransportExit(shard: ShardState, code: number): void {\n    const wasReady = shard.ready;\n    shard.ready = false;\n    // Nothing respawns the shard, so every job it had in flight is lost with\n    // it. Release them rather than pin their operations for the process's\n    // remaining life.\n    const abandoned = [...shard.pendingCoordinates.keys()];\n    shard.pendingCoordinates.clear();\n    if (!wasReady || this.isShuttingDown) {\n      return;\n    }\n    this.logger.error(\n      \"projection shard exited unexpectedly @shardId code=@code, abandoning @count in-flight job(s): @jobIds\",\n      shard.shardId,\n      code,\n      abandoned.length,\n      abandoned.join(\", \"),\n    );\n    this.config.onShardFatal?.(\n      shard.shardId,\n      new Error(\n        `projection shard ${shard.shardId} exited with code ${code}; ` +\n          `${abandoned.length} in-flight job(s) will never be projected` +\n          (abandoned.length > 0 ? `: ${abandoned.join(\", \")}` : \"\"),\n      ),\n    );\n  }\n\n  private relayReadReady(event: JobReadReadyEvent): void {\n    void this.hostBus\n      .emit(ReactorEventTypes.JOB_READ_READY, event)\n      .catch((err: unknown) =>\n        this.logger.error(\n          \"host JOB_READ_READY emit failed for job @jobId: @error\",\n          event.jobId,\n          err,\n        ),\n      );\n  }\n\n  private relayReadModelIndexed(event: ReadModelIndexedEvent): void {\n    void this.hostBus\n      .emit(ReactorEventTypes.READMODEL_INDEXED, event)\n      .catch((err: unknown) =>\n        this.logger.error(\n          \"host READMODEL_INDEXED emit failed for job @jobId: @error\",\n          event.jobId,\n          err,\n        ),\n      );\n  }\n\n  private relayBatchCompleted(event: ReadModelBatchCompletedEvent): void {\n    void this.hostBus\n      .emit(ReactorEventTypes.READMODEL_BATCH_COMPLETED, event)\n      .catch((err: unknown) =>\n        this.logger.error(\n          \"host READMODEL_BATCH_COMPLETED emit failed for job @jobId: @error\",\n          event.jobId,\n          err,\n        ),\n      );\n  }\n}\n"],"mappings":";;;;AAiCA,MAAM,0BAA0B;AAChC,MAAM,4BAA4B;AAClC,MAAM,2BAA2B;AACjC,MAAM,yCAAyC;AAE/C,MAAM,mBAAmB;AACzB,MAAM,YAAY;AAElB,SAAS,UAAU,YAAoB,YAA4B;AACjE,KAAI,aAAa,EACf,OAAM,IAAI,MAAM,2CAA2C,WAAW,GAAG;CAE3E,IAAI,OAAO;AACX,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,UAAQ,WAAW,WAAW,EAAE;AAChC,SAAO,KAAK,KAAK,MAAM,UAAU;;AAEnC,SAAQ,SAAS,KAAK;;;;;;AAOxB,SAAS,yBACP,YACyB;CACzB,MAAM,cAAuC,EAAE;AAC/C,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;EAC1C,MAAM,OAAO,WAAW;AACxB,cAAY,KAAK;GACf,YAAY,KAAK,QAAQ;GACzB,OAAO,KAAK,QAAQ;GACpB,QAAQ,KAAK,QAAQ;GACrB,gBAAgB,KAAK,UAAU;GAChC,CAAC;;AAEJ,QAAO;;;;;;;;;;;;;;;;;AA2GT,IAAa,yBAAb,MAAqE;CACnE,aAAoC,EAAE;CAEtC;CACA;CACA;CACA,SAAwC,EAAE;CAC1C,+BAAgC,IAAI,KAGjC;CACH,gCAAiC,IAAI,KAA2B;CAChE,0CAA2C,IAAI,KAG5C;CACH;CACA,YAAoB;CACpB,UAAkB;CAClB,iBAAyB;CAEzB,YAAY,QAAsC;AAChD,MAAI,OAAO,aAAa,EACtB,OAAM,IAAI,MACR,wDAAwD,OAAO,WAAW,GAC3E;AAEH,OAAK,SAAS;AACd,OAAK,SAAS,YAAY,CAAC,WAAW,2BAA2B,CAAC;AAClE,OAAK,UAAU,OAAO;EACtB,MAAM,WAAW,OAAO,uBAAuB,EAAE;AACjD,OAAK,MAAM,QAAQ,OAAO,KAAK,SAAS,EAA4B;GAClE,MAAM,UAAU,SAAS;AACzB,OAAI,QACF,MAAK,wBAAwB,IAAI,MAAM,QAAQ;;;CAKrD,MAAM,UAAyB;AAC7B,MAAI,KAAK,QACP;AAEF,OAAK,UAAU;EACf,MAAM,gBAAgB,KAAK,OAAO,iBAAiB;EACnD,MAAM,mBACJ,KAAK,OAAO,8BACZ;EAEF,MAAM,eAAgC,EAAE;AACxC,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,YAAY,KAAK;GAC/C,MAAM,UAAU,oBAAoB;GACpC,MAAM,YAAY,KAAK,OAAO,QAAQ,GAAG,QAAQ;GACjD,MAAM,QAAoB;IACxB,YAAY;IACZ;IACA;IACA,OAAO;IACP,WAAW;IACX,aAAa;IACb,qBAAqB,KAAK,OAAO,uBAAuB;IACxD,oCAAoB,IAAI,KAAK;IAC7B,YAAY,QAAQ,KAAK,oBAAoB,OAAO,IAAI;IACxD,UAAU,QAAQ,KAAK,qBAAqB,OAAO,IAAI;IACvD,SAAS,SAAS,KAAK,oBAAoB,OAAO,KAAK;IACxD;AACD,aAAU,GAAG,WAAW,MAAM,UAAU;AACxC,aAAU,GAAG,SAAS,MAAM,QAAQ;AACpC,aAAU,GAAG,QAAQ,MAAM,OAAO;AAClC,QAAK,OAAO,KAAK,MAAM;GAEvB,MAAM,gBAAgB,YAAY;GAClC,MAAM,cAAc,IAAI,SAAe,SAAS,WAAW;IACzD,MAAM,QAAQ,iBAAiB;AAC7B,UAAK,aAAa,OAAO,cAAc;AACvC,4BACE,IAAI,MACF,oBAAoB,QAAQ,+BAA+B,cAAc,IAC1E,CACF;OACA,cAAc;AACjB,SAAK,aAAa,IAAI,eAAe;KAAE;KAAS;KAAQ;KAAO,CAAC;KAChE;GAEF,MAAM,OAA8B;IAClC,MAAM;IACN;IACA;IACA,YAAY;IACZ,YAAY,KAAK,OAAO;IACxB,IAAI,KAAK,OAAO;IAChB,QAAQ,KAAK,OAAO;IACpB,eAAe,KAAK,OAAO;IAC3B,gBAAgB,KAAK,OAAO;IAC5B,4BAA4B;IAC7B;AACD,aAAU,YAAY,KAAK;AAC3B,gBAAa,KAAK,YAAY;;AAGhC,QAAM,QAAQ,IAAI,aAAa;AAC/B,OAAK,OAAO,KACV,iDACA,KAAK,OAAO,OACb;;CAGH,QAAc;AACZ,MAAI,KAAK,UACP;AAEF,OAAK,mBAAmB,KAAK,QAAQ,UACnC,kBAAkB,kBACjB,IAAY,UAA8B;AACzC,QAAK,gBAAgB,MAAM;IAE9B;AACD,OAAK,YAAY;;CAGnB,OAAa;AACX,MAAI,CAAC,KAAK,UACR;AAEF,MAAI,KAAK,kBAAkB;AACzB,QAAK,kBAAkB;AACvB,QAAK,mBAAmB,KAAA;;AAE1B,OAAK,YAAY;;CAGnB,MAAM,QAAuB;AAC3B,MAAI,KAAK,OAAO,WAAW,EACzB;EAEF,MAAM,iBACJ,KAAK,OAAO,kBAAkB;EAChC,MAAM,gBAAgB,YAAY;EAClC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,KAAK,MAAM,EAAE,QAAQ,CAAC;EAC5D,MAAM,UAAU,IAAI,SAAe,SAAS,WAAW;GACrD,MAAM,QAAQ,iBAAiB;AAC7B,SAAK,cAAc,OAAO,cAAc;AACxC,2BACE,IAAI,MACF,0CAA0C,eAAe,iBAAiB,CACxE,GAAG,UACJ,CAAC,KAAK,KAAK,CAAC,GACd,CACF;MACA,eAAe;AAClB,QAAK,cAAc,IAAI,eAAe;IACpC;IACA;IACA;IACA;IACD,CAAC;IACF;AACF,OAAK,MAAM,SAAS,KAAK,OACvB,OAAM,UAAU,YAAY;GAAE,MAAM;GAAS;GAAe,CAAC;AAE/D,QAAM;;CAGR,gBAAwB;EACtB,IAAI,QAAQ;AACZ,OAAK,MAAM,SAAS,KAAK,OACvB,UAAS,MAAM;AAEjB,SAAO;;CAGT,iBAAqC;AACnC,SAAO,KAAK,OAAO,KAAK,WAAW;GACjC,SAAS,MAAM;GACf,OAAO,MAAM;GACb,WAAW,MAAM;GAClB,EAAE;;CAGL,MAAM,WAA0B;AAG9B,OAAK,iBAAiB;AACtB,OAAK,MAAM;EACX,MAAM,UAAU,KAAK,OAAO,mBAAmB;EAC/C,MAAM,gBAAgB,YAAY;AAClC,OAAK,MAAM,SAAS,KAAK,OACvB,KAAI;GACF,MAAM,MAA+B;IACnC,MAAM;IACN;IACA;IACD;AACD,SAAM,UAAU,YAAY,IAAI;WACzB,KAAK;AACZ,QAAK,OAAO,KACV,sEACA,MAAM,SACN,IACD;;EAGL,MAAM,sBAAsB,KAAK,KAAK,GAAG;AACzC,SACE,KAAK,OAAO,MAAM,MAAM,EAAE,MAAM,IAChC,KAAK,KAAK,GAAG,oBAEb,OAAM,IAAI,SAAe,YAAY,WAAW,SAAS,GAAG,CAAC;AAE/D,OAAK,MAAM,SAAS,KAAK,QAAQ;AAC/B,OAAI;AACF,UAAM,MAAM,UAAU,WAAW;YAC1B,KAAK;AACZ,SAAK,OAAO,KACV,0DACA,MAAM,SACN,IACD;;AAEH,SAAM,UAAU,IAAI,WAAW,MAAM,UAAU;AAC/C,SAAM,UAAU,IAAI,SAAS,MAAM,QAAQ;AAC3C,SAAM,UAAU,IAAI,QAAQ,MAAM,OAAO;;AAE3C,OAAK,OAAO,SAAS;;CAGvB,gBAAwB,OAAiC;AACvD,MAAI,MAAM,WAAW,WAAW,GAAG;AAKjC,QAAK,eAAe;IAAE,OAAO,MAAM;IAAO,YAAY,EAAE;IAAE,CAAC;AAC3D,QAAK,oBAAoB;IACvB,OAAO,MAAM;IACb,WAAW;IACX,qBAAqB;IACrB,oBAAoB;IACpB,gBAAgB;IAChB,qBAAqB;IACtB,CAAC;AACF;;EAEF,MAAM,aAAa,MAAM,WAAW,GAAI,QAAQ;EAChD,MAAM,QAAQ,UAAU,YAAY,KAAK,OAAO,OAAO;EACvD,MAAM,QAAQ,KAAK,OAAO;AAC1B,MAAI,CAAC,MAAM,OAAO;AAChB,QAAK,eAAe,OAAO,OAAO,WAAW;AAC7C;;AAEF,MAAI,KAAK,wBAAwB,OAAO,EACtC,OAAM,mBAAmB,IACvB,MAAM,OACN,yBAAyB,MAAM,WAAW,CAC3C;AAEH,QAAM,UAAU,YAAY;GAC1B,MAAM;GACN,OAAO,MAAM;GACb,YAAY,MAAM;GAClB,SAAS,MAAM;GACf,uBAAuB,MAAM;GAC9B,CAAC;;;;;;;;;;;;;;;CAgBJ,eACE,OACA,OACA,YACM;EACN,MAAM,yBAAS,IAAI,MACjB,oBAAoB,MAAM,QAAQ,yCAAyC,MAAM,MAAM,aACxE,WAAW,IAAI,MAAM,WAAW,OAAO,6IAGvD;AACD,OAAK,OAAO,MACV,qEACA,MAAM,OACN,MAAM,SACN,OACD;AACD,OAAK,OAAO,eAAe,MAAM,SAAS,OAAO;;CAGnD,oBACE,OACA,KACM;AACN,UAAQ,IAAI,MAAZ;GACE,KAAK;AACH,SAAK,YAAY,OAAO,IAAI,cAAc;AAC1C;GACF,KAAK;AACH,SAAK,eAAe;KAClB,OAAO,IAAI;KACX,YAAY,IAAI;KACjB,CAAC;AACF;GACF,KAAK;AACH,SAAK,2BAA2B,OAAO,IAAI;AAC3C,SAAK,sBAAsB;KACzB,OAAO,IAAI;KACX,eAAe,IAAI;KACnB,OAAO,IAAI;KACX,YAAY,IAAI;KAChB,gBAAgB,IAAI;KACpB,SAAS,IAAI;KACd,CAAC;AACF;GACF,KAAK;AACH,UAAM,mBAAmB,OAAO,IAAI,MAAM;AAC1C,SAAK,oBAAoB;KACvB,OAAO,IAAI;KACX,WAAW,IAAI;KACf,qBAAqB,IAAI;KACzB,oBAAoB,IAAI;KACxB,gBAAgB,IAAI;KACpB,qBAAqB,IAAI;KAC1B,CAAC;AACF;GACF,KAAK;AACH,UAAM,YAAY,IAAI;AACtB,UAAM,cAAc,IAAI;AACxB;GACF,KAAK;AACH,SAAK,yBAAyB,OAAO,IAAI;AACzC;GACF,KAAK;AACH,SAAK,cAAc,IAAI;AACvB;GACF,KAAK;AACH,SAAK,UAAU,OAAO,IAAI;AAC1B;GACF,QAGE;;;;;;;;;;;;;CAeN,2BACE,OACA,KACM;AACN,MAAI,CAAC,IAAI,QACP;EAEF,MAAM,UAAU,KAAK,wBAAwB,IAAI,IAAI,cAAc;AACnE,MAAI,CAAC,QACH;EAEF,MAAM,cAAc,MAAM,mBAAmB,IAAI,IAAI,MAAM;AAC3D,MAAI,CAAC,eAAe,YAAY,WAAW,EACzC;AAEF,UAAQ,OAAO,YAAY;;CAG7B,yBACE,OACA,KACM;AACN,MAAI,CAAC,MAAM,oBACT;AAEF,QAAM,oBAAoB,YAAY;GACpC,MAAM,IAAI;GACV,MAAM,IAAI;GACV,SAAS,IAAI;GACd,CAAC;AACF,QAAM,oBAAoB,YAAY,IAAI,UAAU;;CAGtD,YAAoB,OAAmB,eAA6B;AAClE,QAAM,QAAQ;EACd,MAAM,UAAU,KAAK,aAAa,IAAI,cAAc;AACpD,MAAI,CAAC,QACH;AAEF,OAAK,aAAa,OAAO,cAAc;AACvC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,SAAS;;CAGnB,cAAsB,KAAqC;EACzD,MAAM,UAAU,KAAK,cAAc,IAAI,IAAI,cAAc;AACzD,MAAI,CAAC,QACH;AAEF,UAAQ,UAAU,OAAO,IAAI,QAAQ;AACrC,MAAI,QAAQ,UAAU,SAAS,GAAG;AAChC,QAAK,cAAc,OAAO,IAAI,cAAc;AAC5C,gBAAa,QAAQ,MAAM;AAC3B,WAAQ,SAAS;;;CAIrB,UACE,OACA,KACM;AACN,UAAQ,IAAI,OAAZ;GACE,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,QAC4B,KAAI;;;CAOpC,qBAA6B,OAAmB,KAAkB;AAChE,OAAK,OAAO,MACV,qDACA,MAAM,SACN,IACD;AACD,MAAI,CAAC,KAAK,eACR,MAAK,OAAO,eAAe,MAAM,SAAS,IAAI;;CAIlD,oBAA4B,OAAmB,MAAoB;EACjE,MAAM,WAAW,MAAM;AACvB,QAAM,QAAQ;EAId,MAAM,YAAY,CAAC,GAAG,MAAM,mBAAmB,MAAM,CAAC;AACtD,QAAM,mBAAmB,OAAO;AAChC,MAAI,CAAC,YAAY,KAAK,eACpB;AAEF,OAAK,OAAO,MACV,yGACA,MAAM,SACN,MACA,UAAU,QACV,UAAU,KAAK,KAAK,CACrB;AACD,OAAK,OAAO,eACV,MAAM,yBACN,IAAI,MACF,oBAAoB,MAAM,QAAQ,oBAAoB,KAAK,IACtD,UAAU,OAAO,8CACnB,UAAU,SAAS,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,IACzD,CACF;;CAGH,eAAuB,OAAgC;AAChD,OAAK,QACP,KAAK,kBAAkB,gBAAgB,MAAM,CAC7C,OAAO,QACN,KAAK,OAAO,MACV,0DACA,MAAM,OACN,IACD,CACF;;CAGL,sBAA8B,OAAoC;AAC3D,OAAK,QACP,KAAK,kBAAkB,mBAAmB,MAAM,CAChD,OAAO,QACN,KAAK,OAAO,MACV,6DACA,MAAM,OACN,IACD,CACF;;CAGL,oBAA4B,OAA2C;AAChE,OAAK,QACP,KAAK,kBAAkB,2BAA2B,MAAM,CACxD,OAAO,QACN,KAAK,OAAO,MACV,qEACA,MAAM,OACN,IACD,CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powerhousedao/reactor",
-  "version": "6.2.2-dev.82",
+  "version": "6.2.2-dev.84",
   "description": "Core Powerhouse reactor — stores documents, resolves conflicts, and reprocesses operation histories. Supports local, cloud, and decentralized storage backends.",
   "repository": {
     "url": "https://github.com/powerhouse-inc/powerhouse",
@@ -42,11 +42,11 @@
     "@electric-sql/pglite": "0.3.15",
     "pg": "8.18.0",
     "@sindresorhus/fnv1a": "3.1.0",
-    "@powerhousedao/shared": "6.2.2-dev.82",
-    "document-model": "6.2.2-dev.82"
+    "@powerhousedao/shared": "6.2.2-dev.84",
+    "document-model": "6.2.2-dev.84"
   },
   "devDependencies": {
-    "@powerhousedao/pglite-fs": "6.2.2-dev.82",
+    "@powerhousedao/pglite-fs": "6.2.2-dev.84",
     "@types/pg": "8.16.0",
     "@vitest/coverage-v8": "4.1.1",
     "prettier": "3.8.1",

package/dist/projection-shard-manager-BnQKFqQd.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"projection-shard-manager-BnQKFqQd.js","names":[],"sources":["../src/projection/projection-shard-manager.ts"],"sourcesContent":["import { childLogger, type ILogger } from \"document-model\";\nimport { randomUUID } from \"node:crypto\";\nimport type { IEventBus } from \"../events/interfaces.js\";\nimport {\n  ReactorEventTypes,\n  type JobReadReadyEvent,\n  type JobWriteReadyEvent,\n  type ReadModelBatchCompletedEvent,\n  type ReadModelIndexedEvent,\n  type Unsubscribe,\n} from \"../events/types.js\";\nimport type {\n  IReadModel,\n  IReadModelCoordinator,\n} from \"../read-models/interfaces.js\";\nimport type { ForwardingPoolInstrumentation } from \"../storage/pool-instrumentation.js\";\nimport type {\n  BuiltInReadModelKind,\n  ChainDepthReport,\n  DbConfig,\n  ModelManifestEntry,\n  ProjectionDrainedMessage,\n  ProjectionInitMessage,\n  ProjectionParentMessage,\n  ProjectionPoolAcquireSamplesMessage,\n  ProjectionWorkerMessage,\n} from \"./protocol.js\";\nimport type { IProjectionTransport } from \"./transport.js\";\n\nconst DEFAULT_INIT_TIMEOUT_MS = 30_000;\nconst DEFAULT_SHUTDOWN_GRACE_MS = 5_000;\nconst DEFAULT_DRAIN_TIMEOUT_MS = 30_000;\nconst DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS = 250;\n\nconst FNV_OFFSET_BASIS = 0x811c9dc5;\nconst FNV_PRIME = 0x01000193;\n\nfunction bucketFor(documentId: string, numWorkers: number): number {\n  if (numWorkers < 1) {\n    throw new Error(`bucketFor: numWorkers must be >= 1 (got ${numWorkers})`);\n  }\n  let hash = FNV_OFFSET_BASIS;\n  for (let i = 0; i < documentId.length; i++) {\n    hash ^= documentId.charCodeAt(i);\n    hash = Math.imul(hash, FNV_PRIME);\n  }\n  return (hash >>> 0) % numWorkers;\n}\n\n/**\n * Factory that builds one projection-worker transport. Mirrors\n * `WorkerFactory` from the executor pool: lets tests inject fake\n * transports without spawning real worker threads.\n */\nexport type ProjectionWorkerFactory = (\n  shardIndex: number,\n  shardId: string,\n) => IProjectionTransport;\n\nexport type ProjectionShardManagerConfig = {\n  shardCount: number;\n  db: DbConfig;\n  models: ModelManifestEntry[];\n  preReadyKinds: BuiltInReadModelKind[];\n  postReadyKinds: BuiltInReadModelKind[];\n  factory: ProjectionWorkerFactory;\n  logger: ILogger;\n  hostBus: IEventBus;\n  initTimeoutMs?: number;\n  shutdownGraceMs?: number;\n  drainTimeoutMs?: number;\n  chainDepthReportIntervalMs?: number;\n  /**\n   * Host-side forwarding instrumentations indexed by shard index. The\n   * manager routes each shard's `pool-acquire-samples` message to the\n   * matching forwarder so the host's OpenTelemetry instrumentation records\n   * acquire-wait latencies as if each shard's pg.Pool were local.\n   */\n  poolInstrumentations?: ForwardingPoolInstrumentation[];\n};\n\ntype ShardState = {\n  shardIndex: number;\n  shardId: string;\n  transport: IProjectionTransport;\n  ready: boolean;\n  lastDepth: number;\n  lastDepthAt: number;\n  poolInstrumentation?: ForwardingPoolInstrumentation;\n  onMessage: (msg: ProjectionWorkerMessage) => void;\n  onError: (err: Error) => void;\n  onExit: (code: number) => void;\n};\n\ntype PendingDrain = {\n  resolve: () => void;\n  reject: (err: Error) => void;\n  remaining: Set<string>;\n  timer: NodeJS.Timeout;\n};\n\n/**\n * Host-side coordinator for N sharded projection workers.\n *\n * Implements {@link IReadModelCoordinator} so it slots into the same\n * `readModelCoordinator` field on the reactor module as the in-process\n * {@link ReadModelCoordinator}. The host subscribes to JOB_WRITE_READY\n * exactly once; events are routed to a single shard by\n * `bucketFor(documentId, shardCount)`. Each worker maintains the\n * per-queueKey serial chain locally and forwards JOB_READ_READY and\n * READMODEL_* events back to the host for the rest of the reactor (sync\n * manager, awaiters, observers) to consume on the host bus.\n *\n * @see Sharded projection workers sub-feature brief\n *   (Powerhouse board wiki id: eb26f01f-8f68-4918-a6f6-ac7a4679b533)\n */\nexport class ProjectionShardManager implements IReadModelCoordinator {\n  readonly readModels: IReadModel[] = [];\n\n  private readonly config: ProjectionShardManagerConfig;\n  private readonly logger: ILogger;\n  private readonly hostBus: IEventBus;\n  private readonly shards: ShardState[] = [];\n  private readonly initPromises = new Map<\n    string,\n    { resolve: () => void; reject: (err: Error) => void; timer: NodeJS.Timeout }\n  >();\n  private readonly pendingDrains = new Map<string, PendingDrain>();\n  private hostSubscription?: Unsubscribe;\n  private isRunning = false;\n  private started = false;\n\n  constructor(config: ProjectionShardManagerConfig) {\n    if (config.shardCount < 1) {\n      throw new Error(\n        `ProjectionShardManager: shardCount must be >= 1 (got ${config.shardCount})`,\n      );\n    }\n    this.config = config;\n    this.logger = childLogger([\"reactor\", \"projection-shard-manager\"]);\n    this.hostBus = config.hostBus;\n  }\n\n  async startup(): Promise<void> {\n    if (this.started) {\n      return;\n    }\n    this.started = true;\n    const initTimeoutMs = this.config.initTimeoutMs ?? DEFAULT_INIT_TIMEOUT_MS;\n    const reportIntervalMs =\n      this.config.chainDepthReportIntervalMs ??\n      DEFAULT_CHAIN_DEPTH_REPORT_INTERVAL_MS;\n\n    const initPromises: Promise<void>[] = [];\n    for (let i = 0; i < this.config.shardCount; i++) {\n      const shardId = `projection-shard-${i}`;\n      const transport = this.config.factory(i, shardId);\n      const state: ShardState = {\n        shardIndex: i,\n        shardId,\n        transport,\n        ready: false,\n        lastDepth: 0,\n        lastDepthAt: 0,\n        poolInstrumentation: this.config.poolInstrumentations?.[i],\n        onMessage: (msg) => this.handleWorkerMessage(state, msg),\n        onError: (err) => this.handleTransportError(state, err),\n        onExit: (code) => this.handleTransportExit(state, code),\n      };\n      transport.on(\"message\", state.onMessage);\n      transport.on(\"error\", state.onError);\n      transport.on(\"exit\", state.onExit);\n      this.shards.push(state);\n\n      const correlationId = randomUUID();\n      const initPromise = new Promise<void>((resolve, reject) => {\n        const timer = setTimeout(() => {\n          this.initPromises.delete(correlationId);\n          reject(\n            new Error(\n              `projection shard ${shardId} did not become ready within ${initTimeoutMs}ms`,\n            ),\n          );\n        }, initTimeoutMs);\n        this.initPromises.set(correlationId, { resolve, reject, timer });\n      });\n\n      const init: ProjectionInitMessage = {\n        type: \"init\",\n        correlationId,\n        shardId,\n        shardIndex: i,\n        shardCount: this.config.shardCount,\n        db: this.config.db,\n        models: this.config.models,\n        preReadyKinds: this.config.preReadyKinds,\n        postReadyKinds: this.config.postReadyKinds,\n        chainDepthReportIntervalMs: reportIntervalMs,\n      };\n      transport.postMessage(init);\n      initPromises.push(initPromise);\n    }\n\n    await Promise.all(initPromises);\n    this.logger.info(\n      \"projection shard manager ready: @count shards\",\n      this.shards.length,\n    );\n  }\n\n  start(): void {\n    if (this.isRunning) {\n      return;\n    }\n    this.hostSubscription = this.hostBus.subscribe(\n      ReactorEventTypes.JOB_WRITE_READY,\n      (_t: number, event: JobWriteReadyEvent) => {\n        this.routeWriteReady(event);\n      },\n    );\n    this.isRunning = true;\n  }\n\n  stop(): void {\n    if (!this.isRunning) {\n      return;\n    }\n    if (this.hostSubscription) {\n      this.hostSubscription();\n      this.hostSubscription = undefined;\n    }\n    this.isRunning = false;\n  }\n\n  async drain(): Promise<void> {\n    if (this.shards.length === 0) {\n      return;\n    }\n    const drainTimeoutMs =\n      this.config.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;\n    const correlationId = randomUUID();\n    const remaining = new Set(this.shards.map((s) => s.shardId));\n    const promise = new Promise<void>((resolve, reject) => {\n      const timer = setTimeout(() => {\n        this.pendingDrains.delete(correlationId);\n        reject(\n          new Error(\n            `projection shards did not drain within ${drainTimeoutMs}ms (remaining: ${[\n              ...remaining,\n            ].join(\", \")})`,\n          ),\n        );\n      }, drainTimeoutMs);\n      this.pendingDrains.set(correlationId, {\n        resolve,\n        reject,\n        remaining,\n        timer,\n      });\n    });\n    for (const shard of this.shards) {\n      shard.transport.postMessage({ type: \"drain\", correlationId });\n    }\n    await promise;\n  }\n\n  getChainDepth(): number {\n    let total = 0;\n    for (const shard of this.shards) {\n      total += shard.lastDepth;\n    }\n    return total;\n  }\n\n  getShardDepths(): ChainDepthReport[] {\n    return this.shards.map((shard) => ({\n      shardId: shard.shardId,\n      depth: shard.lastDepth,\n      timestamp: shard.lastDepthAt,\n    }));\n  }\n\n  async shutdown(): Promise<void> {\n    this.stop();\n    const graceMs = this.config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS;\n    const correlationId = randomUUID();\n    for (const shard of this.shards) {\n      try {\n        const msg: ProjectionParentMessage = {\n          type: \"shutdown\",\n          correlationId,\n          graceMs,\n        };\n        shard.transport.postMessage(msg);\n      } catch (err) {\n        this.logger.warn(\n          \"projection shard postMessage(shutdown) failed for @shardId: @error\",\n          shard.shardId,\n          err,\n        );\n      }\n    }\n    const terminationDeadline = Date.now() + graceMs;\n    while (\n      this.shards.some((s) => s.ready) &&\n      Date.now() < terminationDeadline\n    ) {\n      await new Promise<void>((resolve) => setTimeout(resolve, 25));\n    }\n    for (const shard of this.shards) {\n      try {\n        await shard.transport.terminate();\n      } catch (err) {\n        this.logger.warn(\n          \"projection shard terminate failed for @shardId: @error\",\n          shard.shardId,\n          err,\n        );\n      }\n      shard.transport.off(\"message\", shard.onMessage);\n      shard.transport.off(\"error\", shard.onError);\n      shard.transport.off(\"exit\", shard.onExit);\n    }\n    this.shards.length = 0;\n  }\n\n  private routeWriteReady(event: JobWriteReadyEvent): void {\n    if (event.operations.length === 0) {\n      return;\n    }\n    const documentId = event.operations[0]!.context.documentId;\n    const index = bucketFor(documentId, this.shards.length);\n    const shard = this.shards[index]!;\n    if (!shard.ready) {\n      this.logger.warn(\n        \"dropping JOB_WRITE_READY: shard @index not ready\",\n        index,\n      );\n      return;\n    }\n    shard.transport.postMessage({\n      type: \"write-ready\",\n      jobId: event.jobId,\n      operations: event.operations,\n      jobMeta: event.jobMeta,\n      collectionMemberships: event.collectionMemberships,\n    });\n  }\n\n  private handleWorkerMessage(\n    shard: ShardState,\n    msg: ProjectionWorkerMessage,\n  ): void {\n    switch (msg.type) {\n      case \"ready\":\n        this.handleReady(shard, msg.correlationId);\n        return;\n      case \"read-ready\":\n        this.relayReadReady({\n          jobId: msg.jobId,\n          operations: msg.operations,\n        });\n        return;\n      case \"readmodel-indexed\":\n        this.relayReadModelIndexed({\n          jobId: msg.jobId,\n          readModelName: msg.readModelName,\n          stage: msg.stage,\n          durationMs: msg.durationMs,\n          operationCount: msg.operationCount,\n          success: msg.success,\n        });\n        return;\n      case \"readmodel-batch-completed\":\n        this.relayBatchCompleted({\n          jobId: msg.jobId,\n          batchSize: msg.batchSize,\n          chainWaitDurationMs: msg.chainWaitDurationMs,\n          preReadyDurationMs: msg.preReadyDurationMs,\n          emitDurationMs: msg.emitDurationMs,\n          postReadyDurationMs: msg.postReadyDurationMs,\n        });\n        return;\n      case \"chain-depth\":\n        shard.lastDepth = msg.depth;\n        shard.lastDepthAt = msg.timestamp;\n        return;\n      case \"pool-acquire-samples\":\n        this.handlePoolAcquireSamples(shard, msg);\n        return;\n      case \"drained\":\n        this.handleDrained(msg);\n        return;\n      case \"log\":\n        this.handleLog(shard, msg);\n        return;\n      default: {\n        const exhaustive: never = msg;\n        void exhaustive;\n        return;\n      }\n    }\n  }\n\n  private handlePoolAcquireSamples(\n    shard: ShardState,\n    msg: ProjectionPoolAcquireSamplesMessage,\n  ): void {\n    if (!shard.poolInstrumentation) {\n      return;\n    }\n    shard.poolInstrumentation.updateStats({\n      size: msg.size,\n      idle: msg.idle,\n      waiting: msg.waiting,\n    });\n    shard.poolInstrumentation.pushSamples(msg.durations);\n  }\n\n  private handleReady(shard: ShardState, correlationId: string): void {\n    shard.ready = true;\n    const pending = this.initPromises.get(correlationId);\n    if (!pending) {\n      return;\n    }\n    this.initPromises.delete(correlationId);\n    clearTimeout(pending.timer);\n    pending.resolve();\n  }\n\n  private handleDrained(msg: ProjectionDrainedMessage): void {\n    const pending = this.pendingDrains.get(msg.correlationId);\n    if (!pending) {\n      return;\n    }\n    pending.remaining.delete(msg.shardId);\n    if (pending.remaining.size === 0) {\n      this.pendingDrains.delete(msg.correlationId);\n      clearTimeout(pending.timer);\n      pending.resolve();\n    }\n  }\n\n  private handleLog(\n    shard: ShardState,\n    msg: Extract<ProjectionWorkerMessage, { type: \"log\" }>,\n  ): void {\n    switch (msg.level) {\n      case \"debug\":\n        this.logger.debug(msg.message, ...msg.args);\n        return;\n      case \"info\":\n        this.logger.info(msg.message, ...msg.args);\n        return;\n      case \"warn\":\n        this.logger.warn(msg.message, ...msg.args);\n        return;\n      case \"error\":\n        this.logger.error(msg.message, ...msg.args);\n        return;\n      default: {\n        const exhaustive: never = msg.level;\n        void exhaustive;\n      }\n    }\n    void shard;\n  }\n\n  private handleTransportError(shard: ShardState, err: Error): void {\n    this.logger.error(\n      \"projection shard transport error @shardId: @error\",\n      shard.shardId,\n      err,\n    );\n  }\n\n  private handleTransportExit(shard: ShardState, code: number): void {\n    if (shard.ready) {\n      this.logger.warn(\n        \"projection shard exited unexpectedly @shardId code=@code\",\n        shard.shardId,\n        code,\n      );\n    }\n    shard.ready = false;\n  }\n\n  private relayReadReady(event: JobReadReadyEvent): void {\n    void this.hostBus\n      .emit(ReactorEventTypes.JOB_READ_READY, event)\n      .catch((err: unknown) =>\n        this.logger.error(\n          \"host JOB_READ_READY emit failed for job @jobId: @error\",\n          event.jobId,\n          err,\n        ),\n      );\n  }\n\n  private relayReadModelIndexed(event: ReadModelIndexedEvent): void {\n    void this.hostBus\n      .emit(ReactorEventTypes.READMODEL_INDEXED, event)\n      .catch((err: unknown) =>\n        this.logger.error(\n          \"host READMODEL_INDEXED emit failed for job @jobId: @error\",\n          event.jobId,\n          err,\n        ),\n      );\n  }\n\n  private relayBatchCompleted(event: ReadModelBatchCompletedEvent): void {\n    void this.hostBus\n      .emit(ReactorEventTypes.READMODEL_BATCH_COMPLETED, event)\n      .catch((err: unknown) =>\n        this.logger.error(\n          \"host READMODEL_BATCH_COMPLETED emit failed for job @jobId: @error\",\n          event.jobId,\n          err,\n        ),\n      );\n  }\n}\n"],"mappings":";;;;AA6BA,MAAM,0BAA0B;AAChC,MAAM,4BAA4B;AAClC,MAAM,2BAA2B;AACjC,MAAM,yCAAyC;AAE/C,MAAM,mBAAmB;AACzB,MAAM,YAAY;AAElB,SAAS,UAAU,YAAoB,YAA4B;AACjE,KAAI,aAAa,EACf,OAAM,IAAI,MAAM,2CAA2C,WAAW,GAAG;CAE3E,IAAI,OAAO;AACX,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,UAAQ,WAAW,WAAW,EAAE;AAChC,SAAO,KAAK,KAAK,MAAM,UAAU;;AAEnC,SAAQ,SAAS,KAAK;;;;;;;;;;;;;;;;;AAsExB,IAAa,yBAAb,MAAqE;CACnE,aAAoC,EAAE;CAEtC;CACA;CACA;CACA,SAAwC,EAAE;CAC1C,+BAAgC,IAAI,KAGjC;CACH,gCAAiC,IAAI,KAA2B;CAChE;CACA,YAAoB;CACpB,UAAkB;CAElB,YAAY,QAAsC;AAChD,MAAI,OAAO,aAAa,EACtB,OAAM,IAAI,MACR,wDAAwD,OAAO,WAAW,GAC3E;AAEH,OAAK,SAAS;AACd,OAAK,SAAS,YAAY,CAAC,WAAW,2BAA2B,CAAC;AAClE,OAAK,UAAU,OAAO;;CAGxB,MAAM,UAAyB;AAC7B,MAAI,KAAK,QACP;AAEF,OAAK,UAAU;EACf,MAAM,gBAAgB,KAAK,OAAO,iBAAiB;EACnD,MAAM,mBACJ,KAAK,OAAO,8BACZ;EAEF,MAAM,eAAgC,EAAE;AACxC,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,YAAY,KAAK;GAC/C,MAAM,UAAU,oBAAoB;GACpC,MAAM,YAAY,KAAK,OAAO,QAAQ,GAAG,QAAQ;GACjD,MAAM,QAAoB;IACxB,YAAY;IACZ;IACA;IACA,OAAO;IACP,WAAW;IACX,aAAa;IACb,qBAAqB,KAAK,OAAO,uBAAuB;IACxD,YAAY,QAAQ,KAAK,oBAAoB,OAAO,IAAI;IACxD,UAAU,QAAQ,KAAK,qBAAqB,OAAO,IAAI;IACvD,SAAS,SAAS,KAAK,oBAAoB,OAAO,KAAK;IACxD;AACD,aAAU,GAAG,WAAW,MAAM,UAAU;AACxC,aAAU,GAAG,SAAS,MAAM,QAAQ;AACpC,aAAU,GAAG,QAAQ,MAAM,OAAO;AAClC,QAAK,OAAO,KAAK,MAAM;GAEvB,MAAM,gBAAgB,YAAY;GAClC,MAAM,cAAc,IAAI,SAAe,SAAS,WAAW;IACzD,MAAM,QAAQ,iBAAiB;AAC7B,UAAK,aAAa,OAAO,cAAc;AACvC,4BACE,IAAI,MACF,oBAAoB,QAAQ,+BAA+B,cAAc,IAC1E,CACF;OACA,cAAc;AACjB,SAAK,aAAa,IAAI,eAAe;KAAE;KAAS;KAAQ;KAAO,CAAC;KAChE;GAEF,MAAM,OAA8B;IAClC,MAAM;IACN;IACA;IACA,YAAY;IACZ,YAAY,KAAK,OAAO;IACxB,IAAI,KAAK,OAAO;IAChB,QAAQ,KAAK,OAAO;IACpB,eAAe,KAAK,OAAO;IAC3B,gBAAgB,KAAK,OAAO;IAC5B,4BAA4B;IAC7B;AACD,aAAU,YAAY,KAAK;AAC3B,gBAAa,KAAK,YAAY;;AAGhC,QAAM,QAAQ,IAAI,aAAa;AAC/B,OAAK,OAAO,KACV,iDACA,KAAK,OAAO,OACb;;CAGH,QAAc;AACZ,MAAI,KAAK,UACP;AAEF,OAAK,mBAAmB,KAAK,QAAQ,UACnC,kBAAkB,kBACjB,IAAY,UAA8B;AACzC,QAAK,gBAAgB,MAAM;IAE9B;AACD,OAAK,YAAY;;CAGnB,OAAa;AACX,MAAI,CAAC,KAAK,UACR;AAEF,MAAI,KAAK,kBAAkB;AACzB,QAAK,kBAAkB;AACvB,QAAK,mBAAmB,KAAA;;AAE1B,OAAK,YAAY;;CAGnB,MAAM,QAAuB;AAC3B,MAAI,KAAK,OAAO,WAAW,EACzB;EAEF,MAAM,iBACJ,KAAK,OAAO,kBAAkB;EAChC,MAAM,gBAAgB,YAAY;EAClC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,KAAK,MAAM,EAAE,QAAQ,CAAC;EAC5D,MAAM,UAAU,IAAI,SAAe,SAAS,WAAW;GACrD,MAAM,QAAQ,iBAAiB;AAC7B,SAAK,cAAc,OAAO,cAAc;AACxC,2BACE,IAAI,MACF,0CAA0C,eAAe,iBAAiB,CACxE,GAAG,UACJ,CAAC,KAAK,KAAK,CAAC,GACd,CACF;MACA,eAAe;AAClB,QAAK,cAAc,IAAI,eAAe;IACpC;IACA;IACA;IACA;IACD,CAAC;IACF;AACF,OAAK,MAAM,SAAS,KAAK,OACvB,OAAM,UAAU,YAAY;GAAE,MAAM;GAAS;GAAe,CAAC;AAE/D,QAAM;;CAGR,gBAAwB;EACtB,IAAI,QAAQ;AACZ,OAAK,MAAM,SAAS,KAAK,OACvB,UAAS,MAAM;AAEjB,SAAO;;CAGT,iBAAqC;AACnC,SAAO,KAAK,OAAO,KAAK,WAAW;GACjC,SAAS,MAAM;GACf,OAAO,MAAM;GACb,WAAW,MAAM;GAClB,EAAE;;CAGL,MAAM,WAA0B;AAC9B,OAAK,MAAM;EACX,MAAM,UAAU,KAAK,OAAO,mBAAmB;EAC/C,MAAM,gBAAgB,YAAY;AAClC,OAAK,MAAM,SAAS,KAAK,OACvB,KAAI;GACF,MAAM,MAA+B;IACnC,MAAM;IACN;IACA;IACD;AACD,SAAM,UAAU,YAAY,IAAI;WACzB,KAAK;AACZ,QAAK,OAAO,KACV,sEACA,MAAM,SACN,IACD;;EAGL,MAAM,sBAAsB,KAAK,KAAK,GAAG;AACzC,SACE,KAAK,OAAO,MAAM,MAAM,EAAE,MAAM,IAChC,KAAK,KAAK,GAAG,oBAEb,OAAM,IAAI,SAAe,YAAY,WAAW,SAAS,GAAG,CAAC;AAE/D,OAAK,MAAM,SAAS,KAAK,QAAQ;AAC/B,OAAI;AACF,UAAM,MAAM,UAAU,WAAW;YAC1B,KAAK;AACZ,SAAK,OAAO,KACV,0DACA,MAAM,SACN,IACD;;AAEH,SAAM,UAAU,IAAI,WAAW,MAAM,UAAU;AAC/C,SAAM,UAAU,IAAI,SAAS,MAAM,QAAQ;AAC3C,SAAM,UAAU,IAAI,QAAQ,MAAM,OAAO;;AAE3C,OAAK,OAAO,SAAS;;CAGvB,gBAAwB,OAAiC;AACvD,MAAI,MAAM,WAAW,WAAW,EAC9B;EAEF,MAAM,aAAa,MAAM,WAAW,GAAI,QAAQ;EAChD,MAAM,QAAQ,UAAU,YAAY,KAAK,OAAO,OAAO;EACvD,MAAM,QAAQ,KAAK,OAAO;AAC1B,MAAI,CAAC,MAAM,OAAO;AAChB,QAAK,OAAO,KACV,oDACA,MACD;AACD;;AAEF,QAAM,UAAU,YAAY;GAC1B,MAAM;GACN,OAAO,MAAM;GACb,YAAY,MAAM;GAClB,SAAS,MAAM;GACf,uBAAuB,MAAM;GAC9B,CAAC;;CAGJ,oBACE,OACA,KACM;AACN,UAAQ,IAAI,MAAZ;GACE,KAAK;AACH,SAAK,YAAY,OAAO,IAAI,cAAc;AAC1C;GACF,KAAK;AACH,SAAK,eAAe;KAClB,OAAO,IAAI;KACX,YAAY,IAAI;KACjB,CAAC;AACF;GACF,KAAK;AACH,SAAK,sBAAsB;KACzB,OAAO,IAAI;KACX,eAAe,IAAI;KACnB,OAAO,IAAI;KACX,YAAY,IAAI;KAChB,gBAAgB,IAAI;KACpB,SAAS,IAAI;KACd,CAAC;AACF;GACF,KAAK;AACH,SAAK,oBAAoB;KACvB,OAAO,IAAI;KACX,WAAW,IAAI;KACf,qBAAqB,IAAI;KACzB,oBAAoB,IAAI;KACxB,gBAAgB,IAAI;KACpB,qBAAqB,IAAI;KAC1B,CAAC;AACF;GACF,KAAK;AACH,UAAM,YAAY,IAAI;AACtB,UAAM,cAAc,IAAI;AACxB;GACF,KAAK;AACH,SAAK,yBAAyB,OAAO,IAAI;AACzC;GACF,KAAK;AACH,SAAK,cAAc,IAAI;AACvB;GACF,KAAK;AACH,SAAK,UAAU,OAAO,IAAI;AAC1B;GACF,QAGE;;;CAKN,yBACE,OACA,KACM;AACN,MAAI,CAAC,MAAM,oBACT;AAEF,QAAM,oBAAoB,YAAY;GACpC,MAAM,IAAI;GACV,MAAM,IAAI;GACV,SAAS,IAAI;GACd,CAAC;AACF,QAAM,oBAAoB,YAAY,IAAI,UAAU;;CAGtD,YAAoB,OAAmB,eAA6B;AAClE,QAAM,QAAQ;EACd,MAAM,UAAU,KAAK,aAAa,IAAI,cAAc;AACpD,MAAI,CAAC,QACH;AAEF,OAAK,aAAa,OAAO,cAAc;AACvC,eAAa,QAAQ,MAAM;AAC3B,UAAQ,SAAS;;CAGnB,cAAsB,KAAqC;EACzD,MAAM,UAAU,KAAK,cAAc,IAAI,IAAI,cAAc;AACzD,MAAI,CAAC,QACH;AAEF,UAAQ,UAAU,OAAO,IAAI,QAAQ;AACrC,MAAI,QAAQ,UAAU,SAAS,GAAG;AAChC,QAAK,cAAc,OAAO,IAAI,cAAc;AAC5C,gBAAa,QAAQ,MAAM;AAC3B,WAAQ,SAAS;;;CAIrB,UACE,OACA,KACM;AACN,UAAQ,IAAI,OAAZ;GACE,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,KAAK,IAAI,SAAS,GAAG,IAAI,KAAK;AAC1C;GACF,KAAK;AACH,SAAK,OAAO,MAAM,IAAI,SAAS,GAAG,IAAI,KAAK;AAC3C;GACF,QAC4B,KAAI;;;CAOpC,qBAA6B,OAAmB,KAAkB;AAChE,OAAK,OAAO,MACV,qDACA,MAAM,SACN,IACD;;CAGH,oBAA4B,OAAmB,MAAoB;AACjE,MAAI,MAAM,MACR,MAAK,OAAO,KACV,4DACA,MAAM,SACN,KACD;AAEH,QAAM,QAAQ;;CAGhB,eAAuB,OAAgC;AAChD,OAAK,QACP,KAAK,kBAAkB,gBAAgB,MAAM,CAC7C,OAAO,QACN,KAAK,OAAO,MACV,0DACA,MAAM,OACN,IACD,CACF;;CAGL,sBAA8B,OAAoC;AAC3D,OAAK,QACP,KAAK,kBAAkB,mBAAmB,MAAM,CAChD,OAAO,QACN,KAAK,OAAO,MACV,6DACA,MAAM,OACN,IACD,CACF;;CAGL,oBAA4B,OAA2C;AAChE,OAAK,QACP,KAAK,kBAAkB,2BAA2B,MAAM,CACxD,OAAO,QACN,KAAK,OAAO,MACV,qEACA,MAAM,OACN,IACD,CACF"}