@powerhousedao/reactor 6.2.3-dev.21 → 6.2.3-dev.23

package/dist/index.js

@@ -1,7 +1,7 @@
 import { $ as parsePagingOptions, A as targetDocumentId, B as DuplicateOperationError, C as InvalidModuleError, D as createEmptyConsistencyToken, E as createConsistencyToken, F as buildDecisionModel, G as AuthorizationDeniedError, H as RevisionMismatchError, I as APPEND_CONDITION_FAILED_PREFIX, J as ExcessiveReshuffleError, K as DocumentDeletedError, L as AppendConditionFailedError, M as selectDecisionModel, N as documentDecisionModel, O as submittedActionIds, P as authDecisionModel, Q as matchesScope, R as DocumentAlreadyExistsError, S as DuplicateModuleError, T as GATED_DOCUMENT_ACTIONS, U as AuthEnforcementDisabledError, V as OptimisticLockError, W as AuthTimestampNotMonotonicError, X as RelationshipNotFoundError, Y as InvalidOperationTimestampError, Z as UpgradePreconditionFailedError, _ as DocumentMetaCache, a as createForwardingPoolInstrumentation, b as JobExecutorEventTypes, c as KyselyKeyframeStore, d as DriveCollectionId, et as throwIfAborted, f as KyselyExecutionScope, g as KyselyOperationIndex, h as KyselyWriteCache, i as runMigrations, j as decideAtHead, k as summarizeSubmittedActions, l as DocumentModelRegistry, m as EventBus, n as REACTOR_SCHEMA, o as instrumentPgPool, p as resolveFeatureFlags, q as DocumentNotFoundError, r as getMigrationStatus, s as KyselyOperationStore, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as SimpleJobExecutor, v as CollectionMembershipCache, w as ModuleNotFoundError, x as DuplicateManifestError, y as DEFAULT_DEFERRED_JOB_TTL_MS, z as DocumentExistence } from "./drive-container-types-CS5IxDiA.js";
 import { n as ReactorEventTypes, t as EventBusAggregateError } from "./types-DMKLa0Ok.js";
 import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-D3S6Eysd.js";
-import { a as KyselyDocumentView, c as ReadModelCoordinator, d as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, f as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as BaseReadModel, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as unchunkedReadModelIndexingConfig, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as DEFAULT_COMMIT_CHUNK_SIZE } from "./document-indexer-C1oY6CaR.js";
+import { a as KyselyDocumentView, c as ReadModelCoordinator, d as DEFAULT_READ_MODEL_YIELD_DEADLINE_MS, f as defaultReadModelIndexingConfig, i as DeletedDocumentRead, l as BaseReadModel, n as ConsistencyTracker, o as DOCUMENT_INDEXER_READ_MODEL, p as unchunkedReadModelIndexingConfig, r as makeConsistencyKey, s as DOCUMENT_VIEW_READ_MODEL, t as KyselyDocumentIndexer, u as DEFAULT_COMMIT_CHUNK_SIZE } from "./document-indexer-CQzTEE1o.js";
 import { n as errorToInfo, r as sanitizeArg, t as createForwardingLogger } from "./forwarding-logger-BBkMSxuJ.js";
 import { t as workerEntryPath } from "./worker-g03lNM42.js";
 import { DowngradeNotSupportedError, MAX_AUTH_GRANTS, UnsupportedDocumentModelVersionError, actions, actions as documentActions, createPresignedHeader, decide, deserializeSignature, garbageCollect, generateId, groupDocumentType, groupMembershipActionTypes, hashDocumentStateForScope, normalizeDocumentModelVersion, sortOperations, toTransportAction } from "@powerhousedao/shared/document-model";
@@ -2727,24 +2727,267 @@ function resolveProcessorSlots(records, legacy) {
 	});
 }
 //#endregion
+//#region src/processors/processor-queue.ts
+function isDelivery(task) {
+	return task?.kind === "live" || task?.kind === "advance";
+}
+function errorMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+function lowestOf(ops) {
+	let lowest = ops[0].context.ordinal;
+	for (const op of ops) lowest = Math.min(lowest, op.context.ordinal);
+	return lowest;
+}
+function highestOf(ops) {
+	let highest = 0;
+	for (const op of ops) highest = Math.max(highest, op.context.ordinal);
+	return highest;
+}
+var ProcessorQueue = class {
+	tasks = [];
+	running = false;
+	closed = false;
+	replaysAhead = 0;
+	unrouted = /* @__PURE__ */ new Set();
+	constructor(options) {
+		this.options = options;
+	}
+	get isClosed() {
+		return this.closed;
+	}
+	/** Resolves at once behind a replay, so a pass never waits out a backfill. */
+	live(ops) {
+		const delivered = this.push((done) => ({
+			kind: "live",
+			ops,
+			done
+		}));
+		return this.replaysAhead > 0 ? Promise.resolve() : delivered;
+	}
+	/** Raises the cursor past a batch with nothing for this processor. */
+	advance(through) {
+		const last = this.tasks.at(-1);
+		if (last?.kind === "advance") {
+			last.through = Math.max(last.through, through);
+			return Promise.resolve();
+		}
+		return this.push((done) => ({
+			kind: "advance",
+			through,
+			done
+		}));
+	}
+	/** Replays from the cursor as it stands when the task runs. */
+	backfill() {
+		return this.replay("backfill", () => this.runBackfill());
+	}
+	/** Clears an error and replays from the cursor. No-op when active. */
+	retry() {
+		return this.replay("retry", async () => {
+			const { cursor } = this.options;
+			if (cursor.status !== "errored") return;
+			cursor.status = "active";
+			cursor.lastError = void 0;
+			cursor.lastErrorTimestamp = void 0;
+			await this.persist();
+			await this.runBackfill();
+		});
+	}
+	/** Stops cursor writes now; queued tasks still run before the disconnect. */
+	close() {
+		if (this.closed) return Promise.resolve();
+		const disconnect = this.enqueue("disconnect", async () => {
+			try {
+				await this.options.processor.onDisconnect();
+			} catch (error) {
+				this.options.logger.error("Error disconnecting processor '@ProcessorId': @Error", this.options.processorId, error);
+			}
+		});
+		this.closed = true;
+		return disconnect;
+	}
+	replay(kind, run) {
+		if (this.closed) return Promise.resolve();
+		this.replaysAhead++;
+		return this.enqueue(kind, async () => {
+			try {
+				await run();
+			} finally {
+				this.replaysAhead--;
+			}
+		});
+	}
+	enqueue(kind, run) {
+		return this.push((done) => ({
+			kind,
+			run,
+			done
+		}));
+	}
+	push(task) {
+		if (this.closed) return Promise.resolve();
+		return new Promise((resolve) => {
+			this.tasks.push(task(resolve));
+			if (this.running) return;
+			this.running = true;
+			queueMicrotask(() => void this.drain());
+		});
+	}
+	async drain() {
+		while (this.tasks.length > 0) {
+			const task = this.tasks.shift();
+			const batch = [];
+			if (isDelivery(task)) {
+				batch.push(task);
+				let size = task.kind === "live" ? task.ops.length : 0;
+				let next = this.tasks[0];
+				while (isDelivery(next) && size < 500) {
+					if (next.kind === "live") size += next.ops.length;
+					batch.push(next);
+					this.tasks.shift();
+					next = this.tasks[0];
+				}
+			}
+			try {
+				await (isDelivery(task) ? this.deliverLive(batch) : task.run());
+			} catch (error) {
+				this.options.logger.error("Processor '@ProcessorId' task '@Kind' failed: @Error", this.options.processorId, task.kind, error);
+			} finally {
+				task.done();
+				for (const merged of batch.slice(1)) merged.done();
+			}
+		}
+		this.running = false;
+	}
+	async deliverLive(batch) {
+		const { cursor, floor } = this.options;
+		const ops = [];
+		let through = 0;
+		for (const task of batch) if (task.kind === "live") ops.push(...task.ops);
+		else through = Math.max(through, task.through);
+		const fresh = ops.filter((op) => {
+			const ordinal = op.context.ordinal;
+			if (ordinal <= floor) return false;
+			return !this.unrouted.delete(ordinal);
+		});
+		if (fresh.length === 0) {
+			await this.raiseCursor(through);
+			return;
+		}
+		if (cursor.status !== "active") {
+			await this.parkBelow(fresh);
+			return;
+		}
+		if (!await this.deliver(fresh)) {
+			await this.parkBelow(fresh);
+			return;
+		}
+		await this.raiseCursor(Math.max(highestOf(fresh), through));
+	}
+	async runBackfill() {
+		const { cursor, filter, floor } = this.options;
+		if (cursor.status !== "active") return;
+		let page;
+		try {
+			page = await this.options.readSince(cursor.lastOrdinal);
+		} catch (error) {
+			await this.fail(error, "reading backfill");
+			return;
+		}
+		while (page.results.length > 0) {
+			if (this.closed) return;
+			const matching = page.results.filter((op) => op.context.ordinal > floor && matchesFilter(op, filter));
+			if (matching.length > 0) {
+				if (!await this.deliver(matching)) {
+					await this.persist();
+					return;
+				}
+				this.dedupeQueued(matching);
+			}
+			await this.raiseCursor(highestOf(page.results));
+			if (!page.next) break;
+			try {
+				page = await page.next();
+			} catch (error) {
+				await this.fail(error, "reading backfill");
+				return;
+			}
+		}
+	}
+	dedupeQueued(delivered) {
+		const routed = this.options.routedThrough();
+		const queued = /* @__PURE__ */ new Set();
+		for (const op of delivered) {
+			const ordinal = op.context.ordinal;
+			if (ordinal > routed) this.unrouted.add(ordinal);
+			else queued.add(ordinal);
+		}
+		if (queued.size === 0) return;
+		for (const task of this.tasks) {
+			if (task.kind !== "live") continue;
+			task.ops = task.ops.filter((op) => !queued.has(op.context.ordinal));
+		}
+	}
+	async deliver(ops) {
+		try {
+			await this.options.processor.onOperations(ops);
+			return true;
+		} catch (error) {
+			this.markErrored(error);
+			this.options.logger.error("Processor '@ProcessorId' failed at ordinal @Ordinal: @Error", this.options.processorId, lowestOf(ops), error);
+			return false;
+		}
+	}
+	async fail(error, during) {
+		this.markErrored(error);
+		await this.persist();
+		this.options.logger.error("Processor '@ProcessorId' failed @During: @Error", this.options.processorId, during, error);
+	}
+	markErrored(error) {
+		const { cursor } = this.options;
+		cursor.status = "errored";
+		cursor.lastError = errorMessage(error);
+		cursor.lastErrorTimestamp = /* @__PURE__ */ new Date();
+	}
+	async raiseCursor(through) {
+		const { cursor } = this.options;
+		if (cursor.status !== "active") return;
+		if (through <= cursor.lastOrdinal) return;
+		cursor.lastOrdinal = through;
+		await this.persist();
+	}
+	async parkBelow(missed) {
+		const { cursor } = this.options;
+		cursor.lastOrdinal = Math.min(cursor.lastOrdinal, lowestOf(missed) - 1);
+		await this.persist();
+	}
+	async persist() {
+		if (this.closed) return;
+		const { cursor } = this.options;
+		try {
+			await this.options.persist({
+				lastOrdinal: cursor.lastOrdinal,
+				status: cursor.status,
+				lastError: cursor.lastError,
+				lastErrorTimestamp: cursor.lastErrorTimestamp
+			});
+		} catch (error) {
+			this.options.logger.error("Failed to persist cursor for '@ProcessorId': @Error", this.options.processorId, error);
+		}
+	}
+};
+//#endregion
 //#region src/processors/processor-manager.ts
-/**
-* Manages processor lifecycle based on operations.
-* Extends BaseReadModel to receive operations from ReadModelCoordinator.
-*
-* Responsibilities:
-* 1. Detect drive creation from CREATE_DOCUMENT operations
-* 2. Create processors for each drive using registered factories
-* 3. Route operations to matching processors based on filters
-* 4. Clean up processors when drives are deleted or factories are unregistered
-* 5. Track per-processor cursors for failure recovery and backfill
-*/
 var ProcessorManager = class extends BaseReadModel {
 	factoryRegistry = /* @__PURE__ */ new Map();
 	processorsByDrive = /* @__PURE__ */ new Map();
-	factoryToProcessors = /* @__PURE__ */ new Map();
+	pendingSlots = /* @__PURE__ */ new Set();
 	knownDrives = /* @__PURE__ */ new Map();
+	highestRoutedOrdinal = 0;
 	cursorCache = /* @__PURE__ */ new Map();
+	cursorWrites = /* @__PURE__ */ new Map();
+	draining = /* @__PURE__ */ new Map();
 	logger;
 	driveContainerTypes;
 	legacyProcessorIds;
@@ -2759,39 +3002,31 @@ var ProcessorManager = class extends BaseReadModel {
 		this.legacyProcessorIds = options.legacyProcessorIds ?? true;
 	}
 	async init() {
-		await super.init();
 		await this.loadAllCursors();
+		await super.init();
 		await this.discoverExistingDrives();
 	}
 	async commitOperations(items) {
-		await this.detectAndRegisterNewDrives(items);
-		await this.detectAndCleanupDeletedDrives(items);
-		await this.routeOperationsToProcessors(items);
+		const { runs, reserved } = this.detectNewDrives(items);
+		const disconnects = this.detectDeletedDrives(items);
+		const deliveries = this.enqueueRouted(items, reserved);
+		await Promise.all([
+			...runs.map((run) => run()),
+			...disconnects,
+			...deliveries
+		]);
 	}
 	async registerFactory(identifier, factory) {
-		if (this.factoryRegistry.has(identifier)) await this.unregisterFactory(identifier);
+		const removals = this.removeFactory(identifier);
 		this.factoryRegistry.set(identifier, factory);
-		this.factoryToProcessors.set(identifier, /* @__PURE__ */ new Map());
-		for (const [driveId, documentType] of this.knownDrives) {
-			const driveHeader = createMinimalDriveHeader(driveId, documentType);
-			await this.createProcessorsForDrive(driveId, identifier, factory, driveHeader);
-		}
+		const previous = this.draining.get(identifier);
+		const creationOrdinal = this.highWater() + 1;
+		const runs = [];
+		for (const [driveId, documentType] of this.knownDrives) runs.push(this.reserveSlot(identifier, factory, driveId, createMinimalDriveHeader(driveId, documentType), creationOrdinal, void 0, false, previous).run);
+		await Promise.all([...removals, ...runs.map((run) => run())]);
 	}
 	async unregisterFactory(identifier) {
-		const factoryProcessors = this.factoryToProcessors.get(identifier);
-		if (!factoryProcessors) return;
-		for (const [driveId, tracked] of factoryProcessors) {
-			for (const t of tracked) await this.safeDisconnect(t.record.processor);
-			const driveProcessors = this.processorsByDrive.get(driveId);
-			if (driveProcessors) {
-				const remaining = driveProcessors.filter((p) => !tracked.includes(p));
-				if (remaining.length > 0) this.processorsByDrive.set(driveId, remaining);
-				else this.processorsByDrive.delete(driveId);
-			}
-		}
-		await this.deleteProcessorCursors({ factoryId: identifier });
-		this.factoryToProcessors.delete(identifier);
-		this.factoryRegistry.delete(identifier);
+		await Promise.all(this.removeFactory(identifier));
 	}
 	get(processorId) {
 		for (const tracked of this.allTrackedProcessors()) if (tracked.processorId === processorId) return tracked;
@@ -2799,208 +3034,283 @@ var ProcessorManager = class extends BaseReadModel {
 	getAll() {
 		return Array.from(this.allTrackedProcessors());
 	}
-	*allTrackedProcessors() {
-		for (const tracked of this.processorsByDrive.values()) yield* tracked;
-	}
-	async detectAndRegisterNewDrives(operations) {
-		for (const op of operations) {
+	/** Synchronous: reserves a slot per registered factory for each new drive. */
+	detectNewDrives(items) {
+		const runs = [];
+		const reserved = /* @__PURE__ */ new Set();
+		for (const op of items) {
 			if (!this.isDriveCreation(op)) continue;
 			const driveId = op.context.documentId;
 			if (this.knownDrives.has(driveId)) continue;
 			this.knownDrives.set(driveId, op.context.documentType);
 			const driveHeader = extractDriveHeader(op);
 			if (!driveHeader) continue;
-			for (const [identifier, factory] of this.factoryRegistry) await this.createProcessorsForDrive(driveId, identifier, factory, driveHeader);
+			for (const [identifier, factory] of this.factoryRegistry) {
+				const { slot, run } = this.reserveSlot(identifier, factory, driveId, driveHeader, op.context.ordinal, items, true);
+				reserved.add(slot);
+				runs.push(run);
+			}
 		}
+		return {
+			runs,
+			reserved
+		};
 	}
-	isDriveCreation(op) {
-		return op.operation.action.type === "CREATE_DOCUMENT" && this.driveContainerTypes.has(op.context.documentType);
-	}
-	async detectAndCleanupDeletedDrives(operations) {
-		for (const op of operations) {
+	/** Synchronous: drops a deleted drive's processors from the tables. */
+	detectDeletedDrives(items) {
+		const pending = [];
+		for (const op of items) {
 			if (!isDriveDeletion(op)) continue;
 			const driveId = extractDeletedDocumentId(op);
 			if (!driveId || !this.knownDrives.has(driveId)) continue;
-			if (!this.isDeletedDocumentADrive(driveId)) continue;
-			await this.cleanupDriveProcessors(driveId);
 			this.knownDrives.delete(driveId);
+			for (const slot of this.pendingSlots) if (slot.driveId === driveId) this.pendingSlots.delete(slot);
+			for (const { queue } of this.processorsByDrive.get(driveId) ?? []) queue.close();
+			this.processorsByDrive.delete(driveId);
+			pending.push(...this.deleteCursors((row) => row.driveId === driveId));
+		}
+		return pending;
+	}
+	/** Synchronous: puts each processor's share of the batch on its queue. */
+	enqueueRouted(items, reserved) {
+		if (items.length === 0) return [];
+		let lowest = items[0].context.ordinal;
+		let highest = 0;
+		for (const item of items) {
+			lowest = Math.min(lowest, item.context.ordinal);
+			highest = Math.max(highest, item.context.ordinal);
+		}
+		this.highestRoutedOrdinal = Math.max(this.highestRoutedOrdinal, highest);
+		for (const slot of this.pendingSlots) {
+			if (reserved.has(slot)) continue;
+			slot.lowestRoutedOrdinal = Math.min(slot.lowestRoutedOrdinal ?? lowest, lowest);
+		}
+		const deliveries = [];
+		for (const { tracked, queue } of this.allBound()) {
+			const matching = items.filter((op) => matchesFilter(op, tracked.record.filter));
+			if (matching.length > 0) deliveries.push(queue.live(matching));
+			else queue.advance(highest);
+		}
+		return deliveries;
+	}
+	/** Synchronous: binds records, or discards them if the slot was cancelled. */
+	bind(slot, records, creationOrdinal, creationItems) {
+		const released = this.pendingSlots.delete(slot);
+		if (!records || records.length === 0) {
+			const settled = Promise.resolve();
+			return {
+				persisted: settled,
+				delivered: settled
+			};
+		}
+		if (!released) {
+			const discarded = records.map((record) => this.discard(slot, record.processor.onDisconnect.bind(record.processor)));
+			const settled = Promise.all(discarded).then(() => void 0);
+			return {
+				persisted: settled,
+				delivered: settled
+			};
 		}
+		const { factoryId, driveId, lowestRoutedOrdinal } = slot;
+		const ids = resolveProcessorSlots(records, this.legacyProcessorIds);
+		const bound = records.map((record, i) => this.track(slot, record, i, `${factoryId}:${driveId}:${ids[i]}`, creationOrdinal, lowestRoutedOrdinal));
+		const drive = this.processorsByDrive.get(driveId) ?? [];
+		this.processorsByDrive.set(driveId, [...drive, ...bound]);
+		const liveIds = new Set(bound.map((b) => b.tracked.processorId));
+		const writes = this.deleteCursors((row) => row.factoryId === factoryId && row.driveId === driveId && !liveIds.has(row.processorId));
+		for (const { tracked } of bound) writes.push(this.writeCursor(tracked, tracked));
+		const deliveries = [];
+		for (const { tracked, queue } of bound) {
+			const missed = tracked.lastOrdinal < slot.reservedAt || lowestRoutedOrdinal !== void 0;
+			if (tracked.status === "active" && missed) deliveries.push(queue.backfill());
+			const matching = creationItems?.filter((op) => matchesFilter(op, tracked.record.filter));
+			if (matching && matching.length > 0) deliveries.push(queue.live(matching));
+		}
+		return {
+			persisted: Promise.all(writes).then(() => void 0),
+			delivered: Promise.all(deliveries).then(() => void 0)
+		};
 	}
-	async discoverExistingDrives() {
-		const drives = await this.db.selectFrom("DocumentSnapshot").select(["documentId", "documentType"]).where("documentType", "in", [...this.driveContainerTypes]).where("isDeleted", "=", false).execute();
-		for (const drive of drives) this.knownDrives.set(drive.documentId, drive.documentType);
+	/** Synchronous: removes a factory's slots, processors and cursor rows. */
+	removeFactory(identifier) {
+		if (!this.factoryRegistry.delete(identifier)) return [];
+		const closing = [];
+		for (const slot of this.pendingSlots) {
+			if (slot.factoryId !== identifier) continue;
+			this.pendingSlots.delete(slot);
+			if (slot.settled) closing.push(slot.settled);
+		}
+		for (const [driveId, drive] of this.processorsByDrive) {
+			const remaining = [];
+			for (const b of drive) if (b.tracked.factoryId === identifier) closing.push(b.queue.close());
+			else remaining.push(b);
+			if (remaining.length > 0) this.processorsByDrive.set(driveId, remaining);
+			else this.processorsByDrive.delete(driveId);
+		}
+		if (closing.length > 0) {
+			const drained = Promise.all([this.draining.get(identifier), ...closing]).then(() => void 0);
+			this.draining.set(identifier, drained);
+			drained.then(() => {
+				if (this.draining.get(identifier) === drained) this.draining.delete(identifier);
+			});
+		}
+		return this.deleteCursors((row) => row.factoryId === identifier);
 	}
-	isDeletedDocumentADrive(documentId) {
-		return this.knownDrives.has(documentId);
+	reserveSlot(factoryId, factory, driveId, driveHeader, creationOrdinal, creationItems, awaitDelivery, previous) {
+		const slot = {
+			factoryId,
+			driveId,
+			reservedAt: this.highWater(),
+			lowestRoutedOrdinal: void 0,
+			settled: void 0
+		};
+		this.pendingSlots.add(slot);
+		const run = async () => {
+			const bound = (async () => {
+				await previous;
+				const records = await this.runFactory(slot, factory, driveHeader);
+				return this.bind(slot, records, creationOrdinal, creationItems);
+			})();
+			slot.settled = bound.then(({ persisted }) => persisted);
+			const { persisted, delivered } = await bound;
+			await (awaitDelivery ? Promise.all([persisted, delivered]) : persisted);
+		};
+		return {
+			slot,
+			run
+		};
 	}
-	async createProcessorsForDrive(driveId, identifier, factory, driveHeader) {
-		let records;
+	async runFactory(slot, factory, driveHeader) {
 		try {
-			records = await factory(driveHeader);
+			return await factory(driveHeader);
 		} catch (error) {
-			this.logger.error("Factory '@FactoryId' failed for drive '@DriveId': @Error", identifier, driveId, error);
+			this.logger.error("Factory '@FactoryId' failed for drive '@DriveId': @Error", slot.factoryId, slot.driveId, error);
 			return;
 		}
-		if (records.length === 0) return;
-		const trackedList = [];
-		const slots = resolveProcessorSlots(records, this.legacyProcessorIds);
-		for (let i = 0; i < records.length; i++) {
-			const record = records[i];
-			const processorId = `${identifier}:${driveId}:${slots[i]}`;
-			const cached = this.cursorCache.get(processorId);
-			let lastOrdinal;
-			let status;
-			let lastError;
-			let lastErrorTimestamp;
-			if (cached) {
-				lastOrdinal = cached.lastOrdinal;
-				status = cached.status;
-				lastError = cached.lastError ?? void 0;
-				lastErrorTimestamp = cached.lastErrorTimestamp ?? void 0;
-			} else {
-				lastOrdinal = (record.startFrom ?? "beginning") === "current" ? this.lastOrdinal : 0;
-				status = "active";
-				lastError = void 0;
-				lastErrorTimestamp = void 0;
-			}
-			const tracked = {
-				processorId,
-				factoryId: identifier,
-				driveId,
-				processorIndex: i,
-				record,
-				lastOrdinal,
-				status,
-				lastError,
-				lastErrorTimestamp,
-				retry: () => this.retryProcessor(tracked)
+	}
+	track(slot, record, processorIndex, processorId, creationOrdinal, lowestRoutedOrdinal) {
+		const cached = this.cursorCache.get(processorId);
+		let floor = 0;
+		let cursor;
+		if (cached) cursor = {
+			lastOrdinal: cached.lastOrdinal,
+			status: cached.status,
+			lastError: cached.lastError ?? void 0,
+			lastErrorTimestamp: cached.lastErrorTimestamp ?? void 0
+		};
+		else {
+			if (record.startFrom === "current") floor = creationOrdinal - 1;
+			cursor = {
+				lastOrdinal: floor,
+				status: "active",
+				lastError: void 0,
+				lastErrorTimestamp: void 0
 			};
-			trackedList.push(tracked);
-			await this.saveProcessorCursor(tracked);
-		}
-		const liveIds = new Set(trackedList.map((t) => t.processorId));
-		await this.db.deleteFrom("ProcessorCursor").where("factoryId", "=", identifier).where("driveId", "=", driveId).where("processorId", "not in", [...liveIds]).execute();
-		for (const [id, row] of this.cursorCache) if (row.factoryId === identifier && row.driveId === driveId && !liveIds.has(id)) this.cursorCache.delete(id);
-		const factoryProcessors = this.factoryToProcessors.get(identifier);
-		if (factoryProcessors) factoryProcessors.set(driveId, trackedList);
-		const existingDriveProcessors = this.processorsByDrive.get(driveId) ?? [];
-		this.processorsByDrive.set(driveId, [...existingDriveProcessors, ...trackedList]);
-		for (const tracked of trackedList) if (tracked.status === "active" && tracked.lastOrdinal < this.lastOrdinal) await this.backfillProcessor(tracked);
-	}
-	async backfillProcessor(tracked) {
-		let page = await this.operationIndex.getSinceOrdinal(tracked.lastOrdinal);
-		while (page.results.length > 0) {
-			const matching = page.results.filter((op) => matchesFilter(op, tracked.record.filter));
-			if (matching.length > 0) try {
-				await tracked.record.processor.onOperations(matching);
-			} catch (error) {
-				tracked.status = "errored";
-				tracked.lastError = error instanceof Error ? error.message : String(error);
-				tracked.lastErrorTimestamp = /* @__PURE__ */ new Date();
-				await this.safeSaveProcessorCursor(tracked);
-				this.logger.error("Processor '@ProcessorId' failed during backfill at ordinal @Ordinal: @Error", tracked.processorId, tracked.lastOrdinal, error);
-				return;
-			}
-			tracked.lastOrdinal = page.results[page.results.length - 1].context.ordinal;
-			await this.safeSaveProcessorCursor(tracked);
-			if (!page.next) break;
-			page = await page.next();
-		}
-	}
-	async retryProcessor(tracked) {
-		if (tracked.status !== "errored") return;
-		tracked.status = "active";
-		tracked.lastError = void 0;
-		tracked.lastErrorTimestamp = void 0;
-		await this.saveProcessorCursor(tracked);
-		await this.backfillProcessor(tracked);
-	}
-	async cleanupDriveProcessors(driveId) {
-		const processors = this.processorsByDrive.get(driveId);
-		if (!processors) return;
-		for (const tracked of processors) await this.safeDisconnect(tracked.record.processor);
-		this.processorsByDrive.delete(driveId);
-		for (const factoryProcessors of this.factoryToProcessors.values()) factoryProcessors.delete(driveId);
-		await this.deleteProcessorCursors({ driveId });
-	}
-	async safeDisconnect(processor) {
+		}
+		if (lowestRoutedOrdinal !== void 0) cursor.lastOrdinal = Math.max(floor, Math.min(cursor.lastOrdinal, lowestRoutedOrdinal - 1));
+		const tracked = {
+			processorId,
+			factoryId: slot.factoryId,
+			driveId: slot.driveId,
+			processorIndex,
+			record,
+			...cursor,
+			retry: () => queue.retry()
+		};
+		const queue = new ProcessorQueue({
+			processorId,
+			processor: record.processor,
+			filter: record.filter,
+			cursor: tracked,
+			floor,
+			readSince: (ordinal) => this.operationIndex.getSinceOrdinal(ordinal),
+			routedThrough: () => this.highWater(),
+			persist: (state) => this.writeCursor(tracked, state),
+			logger: this.logger
+		});
+		return {
+			tracked,
+			queue
+		};
+	}
+	async discard(slot, disconnect) {
 		try {
-			await processor.onDisconnect();
+			await disconnect();
 		} catch (error) {
-			this.logger.error("Error disconnecting processor: @Error", error);
+			this.logger.error("Error disconnecting discarded processor for '@FactoryId' on '@DriveId': @Error", slot.factoryId, slot.driveId, error);
 		}
 	}
-	async routeOperationsToProcessors(operations) {
-		let maxOrdinal = 0;
-		for (const op of operations) maxOrdinal = Math.max(maxOrdinal, op.context.ordinal);
-		const allTracked = Array.from(this.allTrackedProcessors());
-		await Promise.all(allTracked.map(async (tracked) => {
-			if (tracked.status !== "active") return;
-			const matching = operations.filter((op) => op.context.ordinal > tracked.lastOrdinal).filter((op) => matchesFilter(op, tracked.record.filter));
-			if (matching.length > 0) try {
-				await tracked.record.processor.onOperations(matching);
-			} catch (error) {
-				tracked.status = "errored";
-				tracked.lastError = error instanceof Error ? error.message : String(error);
-				tracked.lastErrorTimestamp = /* @__PURE__ */ new Date();
-				await this.safeSaveProcessorCursor(tracked);
-				this.logger.error("Processor '@ProcessorId' failed at ordinal @Ordinal: @Error", tracked.processorId, tracked.lastOrdinal, error);
-				return;
-			}
-			tracked.lastOrdinal = maxOrdinal;
-			await this.safeSaveProcessorCursor(tracked);
-		}));
+	highWater() {
+		return Math.max(this.lastOrdinal, this.highestRoutedOrdinal);
+	}
+	*allBound() {
+		for (const drive of this.processorsByDrive.values()) yield* drive;
+	}
+	*allTrackedProcessors() {
+		for (const { tracked } of this.allBound()) yield tracked;
+	}
+	isDriveCreation(op) {
+		return op.operation.action.type === "CREATE_DOCUMENT" && this.driveContainerTypes.has(op.context.documentType);
+	}
+	async discoverExistingDrives() {
+		const drives = await this.db.selectFrom("DocumentSnapshot").select(["documentId", "documentType"]).where("documentType", "in", [...this.driveContainerTypes]).where("isDeleted", "=", false).execute();
+		for (const drive of drives) this.knownDrives.set(drive.documentId, drive.documentType);
 	}
 	async loadAllCursors() {
 		const rows = await this.db.selectFrom("ProcessorCursor").selectAll().execute();
 		for (const row of rows) this.cursorCache.set(row.processorId, row);
 	}
-	async safeSaveProcessorCursor(tracked) {
-		try {
-			await this.saveProcessorCursor(tracked);
-		} catch (error) {
-			this.logger.error("Failed to persist cursor for '@ProcessorId': @Error", tracked.processorId, error);
-		}
+	lane(processorId, write) {
+		const next = (this.cursorWrites.get(processorId) ?? Promise.resolve()).then(write).then(() => void 0, (error) => {
+			this.logger.error("Failed to write cursor for '@ProcessorId': @Error", processorId, error);
+		});
+		this.cursorWrites.set(processorId, next);
+		next.then(() => {
+			if (this.cursorWrites.get(processorId) === next) this.cursorWrites.delete(processorId);
+		});
+		return next;
 	}
-	async saveProcessorCursor(tracked) {
-		await this.db.insertInto("ProcessorCursor").values({
+	writeCursor(tracked, state) {
+		const now = /* @__PURE__ */ new Date();
+		const row = {
 			processorId: tracked.processorId,
 			factoryId: tracked.factoryId,
 			driveId: tracked.driveId,
 			processorIndex: tracked.processorIndex,
-			lastOrdinal: tracked.lastOrdinal,
-			status: tracked.status,
-			lastError: tracked.lastError ?? null,
-			lastErrorTimestamp: tracked.lastErrorTimestamp ?? null,
-			updatedAt: /* @__PURE__ */ new Date()
+			lastOrdinal: state.lastOrdinal,
+			status: state.status,
+			lastError: state.lastError ?? null,
+			lastErrorTimestamp: state.lastErrorTimestamp ?? null,
+			createdAt: now,
+			updatedAt: now
+		};
+		this.cursorCache.set(row.processorId, row);
+		return this.lane(row.processorId, () => this.db.insertInto("ProcessorCursor").values({
+			processorId: row.processorId,
+			factoryId: row.factoryId,
+			driveId: row.driveId,
+			processorIndex: row.processorIndex,
+			lastOrdinal: row.lastOrdinal,
+			status: row.status,
+			lastError: row.lastError,
+			lastErrorTimestamp: row.lastErrorTimestamp,
+			updatedAt: now
 		}).onConflict((oc) => oc.column("processorId").doUpdateSet({
-			lastOrdinal: tracked.lastOrdinal,
-			status: tracked.status,
-			lastError: tracked.lastError ?? null,
-			lastErrorTimestamp: tracked.lastErrorTimestamp ?? null,
-			updatedAt: /* @__PURE__ */ new Date()
-		})).execute();
-		this.cursorCache.set(tracked.processorId, {
-			processorId: tracked.processorId,
-			factoryId: tracked.factoryId,
-			driveId: tracked.driveId,
-			processorIndex: tracked.processorIndex,
-			lastOrdinal: tracked.lastOrdinal,
-			status: tracked.status,
-			lastError: tracked.lastError ?? null,
-			lastErrorTimestamp: tracked.lastErrorTimestamp ?? null,
-			createdAt: /* @__PURE__ */ new Date(),
-			updatedAt: /* @__PURE__ */ new Date()
-		});
-	}
-	async deleteProcessorCursors(filter) {
-		if ("factoryId" in filter) {
-			await this.db.deleteFrom("ProcessorCursor").where("factoryId", "=", filter.factoryId).execute();
-			for (const [id, row] of this.cursorCache) if (row.factoryId === filter.factoryId) this.cursorCache.delete(id);
-		} else {
-			await this.db.deleteFrom("ProcessorCursor").where("driveId", "=", filter.driveId).execute();
-			for (const [id, row] of this.cursorCache) if (row.driveId === filter.driveId) this.cursorCache.delete(id);
-		}
+			lastOrdinal: row.lastOrdinal,
+			status: row.status,
+			lastError: row.lastError,
+			lastErrorTimestamp: row.lastErrorTimestamp,
+			updatedAt: now
+		})).execute());
+	}
+	deleteCursors(matches) {
+		const deletes = [];
+		for (const [processorId, row] of this.cursorCache) {
+			if (!matches(row)) continue;
+			this.cursorCache.delete(processorId);
+			deletes.push(this.lane(processorId, () => this.db.deleteFrom("ProcessorCursor").where("processorId", "=", processorId).execute()));
+		}
+		return deletes;
 	}
 };
 //#endregion
@@ -8506,7 +8816,7 @@ var ReactorBuilder = class {
 			}
 			this.logger.info("Shutdown complete");
 			nodeProcess.exit = realExit;
-			realExit(pendingExitCode ?? 0);
+			realExit(pendingExitCode || Number(nodeProcess.exitCode ?? 0));
 		};
 		nodeProcess.prependListener("SIGINT", () => void handler("SIGINT"));
 		nodeProcess.prependListener("SIGTERM", () => void handler("SIGTERM"));