@powerhousedao/reactor 6.0.0-dev.170 → 6.0.0-dev.172

package/dist/index.js

@@ -570,10 +570,8 @@ var ReactorClient = class {
 				limit: paging.limit
 			};
 			const scopeResult = (await this.reactor.getOperations(documentId, scopeView, filter, scopePaging, void 0, signal))[scopeName];
-			if (scopeResult) {
-				allOperations.push(...scopeResult.results);
-				if (scopeResult.nextCursor) activeCursors[scopeName] = scopeResult.nextCursor;
-			}
+			allOperations.push(...scopeResult.results);
+			if (scopeResult.nextCursor) activeCursors[scopeName] = scopeResult.nextCursor;
 		}
 		allOperations.sort((a, b) => a.index - b.index);
 		return {
@@ -2724,6 +2722,17 @@ var SimpleJobExecutorManager = class {
 	}
 };
 //#endregion
+//#region src/shared/utils.ts
+function matchesScope(view = {}, scope) {
+	if (view.scopes) return view.scopes.includes(scope);
+	return true;
+}
+function yieldToMain() {
+	const s = globalThis.scheduler;
+	if (s?.yield) return s.yield();
+	return new Promise((resolve) => setTimeout(resolve, 0));
+}
+//#endregion
 //#region src/utils/reshuffle.ts
 const STRICT_ORDER_ACTION_TYPES = new Set([
 	"CREATE_DOCUMENT",
@@ -3143,7 +3152,8 @@ var SimpleJobExecutor = class {
 			maxConcurrency: config.maxConcurrency ?? 1,
 			jobTimeoutMs: config.jobTimeoutMs ?? 3e4,
 			retryBaseDelayMs: config.retryBaseDelayMs ?? 100,
-			retryMaxDelayMs: config.retryMaxDelayMs ?? 5e3
+			retryMaxDelayMs: config.retryMaxDelayMs ?? 5e3,
+			yieldDeadlineMs: config.yieldDeadlineMs ?? 50
 		};
 		this.signatureVerifierModule = new SignatureVerifier(signatureVerifier);
 		this.documentActionHandler = new DocumentActionHandler(registry, logger);
@@ -3247,6 +3257,7 @@ var SimpleJobExecutor = class {
 			operationsWithContext,
 			error: /* @__PURE__ */ new Error(`Invalid timestamp "${action.timestampUtcMs}" on action ${action.type} (id: ${action.id})`)
 		};
+		let lastYield = performance.now();
 		for (let actionIndex = 0; actionIndex < actions.length; actionIndex++) {
 			const action = actions[actionIndex];
 			const skip = skipValues?.[actionIndex] ?? 0;
@@ -3259,6 +3270,16 @@ var SimpleJobExecutor = class {
 				operationsWithContext,
 				error: error.error
 			};
+			if (performance.now() - lastYield > this.config.yieldDeadlineMs) {
+				await yieldToMain();
+				lastYield = performance.now();
+				if (signal?.aborted) return {
+					success: false,
+					generatedOperations,
+					operationsWithContext,
+					error: /* @__PURE__ */ new Error("Aborted")
+				};
+			}
 		}
 		return {
 			success: true,
@@ -6537,6 +6558,113 @@ function mergeCollectionMemberships(events) {
 	}
 	return mergedMemberships;
 }
+/**
+* Chunks sync operations into batches that respect dependency-connected
+* components. SyncOps linked by jobDependencies are kept in the same chunk.
+* If a connected component exceeds maxSize, it is split by topological order.
+*/
+function chunkSyncOperations(items, maxSize) {
+	if (items.length === 0) return [];
+	if (items.length <= maxSize) return [items];
+	const parent = items.map((_, i) => i);
+	const rank = new Array(items.length).fill(0);
+	function find(x) {
+		while (parent[x] !== x) {
+			parent[x] = parent[parent[x]];
+			x = parent[x];
+		}
+		return x;
+	}
+	function union(a, b) {
+		const ra = find(a);
+		const rb = find(b);
+		if (ra === rb) return;
+		if (rank[ra] < rank[rb]) parent[ra] = rb;
+		else if (rank[ra] > rank[rb]) parent[rb] = ra;
+		else {
+			parent[rb] = ra;
+			rank[ra]++;
+		}
+	}
+	const jobIdToIndex = /* @__PURE__ */ new Map();
+	for (let i = 0; i < items.length; i++) jobIdToIndex.set(items[i].syncOp.jobId, i);
+	for (let i = 0; i < items.length; i++) for (const dep of items[i].syncOp.jobDependencies) {
+		const depIdx = jobIdToIndex.get(dep);
+		if (depIdx !== void 0) union(i, depIdx);
+	}
+	const componentMap = /* @__PURE__ */ new Map();
+	for (let i = 0; i < items.length; i++) {
+		const root = find(i);
+		let component = componentMap.get(root);
+		if (!component) {
+			component = [];
+			componentMap.set(root, component);
+		}
+		component.push(items[i]);
+	}
+	const components = [...componentMap.values()];
+	const chunks = [];
+	let currentChunk = [];
+	for (const component of components) {
+		if (component.length > maxSize) {
+			if (currentChunk.length > 0) {
+				chunks.push(currentChunk);
+				currentChunk = [];
+			}
+			for (const subChunk of splitComponent(component, maxSize)) chunks.push(subChunk);
+			continue;
+		}
+		if (currentChunk.length + component.length > maxSize) {
+			if (currentChunk.length > 0) chunks.push(currentChunk);
+			currentChunk = [...component];
+		} else currentChunk.push(...component);
+	}
+	if (currentChunk.length > 0) chunks.push(currentChunk);
+	return chunks;
+}
+/**
+* Splits an oversized connected component into chunks by topological order.
+* Cross-chunk dependency references are handled by the caller's dep filter.
+*/
+function splitComponent(items, maxSize) {
+	const jobIdToItem = /* @__PURE__ */ new Map();
+	const jobIds = /* @__PURE__ */ new Set();
+	for (const item of items) {
+		jobIdToItem.set(item.syncOp.jobId, item);
+		jobIds.add(item.syncOp.jobId);
+	}
+	const inDegree = /* @__PURE__ */ new Map();
+	const adjacency = /* @__PURE__ */ new Map();
+	for (const item of items) {
+		const key = item.syncOp.jobId;
+		if (!inDegree.has(key)) inDegree.set(key, 0);
+		if (!adjacency.has(key)) adjacency.set(key, []);
+		for (const dep of item.syncOp.jobDependencies) if (jobIds.has(dep)) {
+			inDegree.set(key, (inDegree.get(key) ?? 0) + 1);
+			if (!adjacency.has(dep)) adjacency.set(dep, []);
+			adjacency.get(dep).push(key);
+		}
+	}
+	const queue = [];
+	for (const [key, degree] of inDegree) if (degree === 0) queue.push(key);
+	const sorted = [];
+	while (queue.length > 0) {
+		const key = queue.shift();
+		sorted.push(jobIdToItem.get(key));
+		for (const neighbor of adjacency.get(key) ?? []) {
+			const newDegree = (inDegree.get(neighbor) ?? 1) - 1;
+			inDegree.set(neighbor, newDegree);
+			if (newDegree === 0) queue.push(neighbor);
+		}
+	}
+	if (sorted.length < items.length) {
+		const sortedIds = new Set(sorted.map((item) => item.syncOp.jobId));
+		for (const item of items) if (!sortedIds.has(item.syncOp.jobId)) sorted.push(item);
+	}
+	const chunks = [];
+	for (let i = 0; i < sorted.length; i += maxSize) chunks.push(sorted.slice(i, i + maxSize));
+	return chunks;
+}
 //#endregion
 //#region src/sync/channels/interval-poll-timer.ts
 const DEFAULT_CONFIG = {
@@ -6788,6 +6916,7 @@ var GqlRequestChannel = class {
 	pushBlocked = false;
 	isPushing = false;
 	pendingDrain = false;
+	receivingPages = false;
 	connectionState = "connecting";
 	connectionStateCallbacks = /* @__PURE__ */ new Set();
 	constructor(logger, channelId, remoteName, cursorStorage, config, operationIndex, pollTimer) {
@@ -6822,6 +6951,10 @@ var GqlRequestChannel = class {
 				return;
 			}
 			if (this.pushBlocked) return;
+			if (this.receivingPages) {
+				this.pendingDrain = true;
+				return;
+			}
 			this.attemptPush(syncOps);
 		});
 		this.outbox.onRemoved((syncOps) => {
@@ -6875,7 +7008,8 @@ var GqlRequestChannel = class {
 			lastSuccessUtcMs: this.lastSuccessUtcMs ?? 0,
 			lastFailureUtcMs: this.lastFailureUtcMs ?? 0,
 			pushBlocked: this.pushBlocked,
-			pushFailureCount: this.pushFailureCount
+			pushFailureCount: this.pushFailureCount,
+			receivingPages: this.receivingPages
 		};
 	}
 	onConnectionStateChange(callback) {
@@ -6923,7 +7057,7 @@ var GqlRequestChannel = class {
 			if (!this.handlePollError(error)) throw error;
 			return;
 		}
-		const { envelopes, ackOrdinal, deadLetters } = response;
+		const { envelopes, ackOrdinal, deadLetters, hasMore } = response;
 		if (ackOrdinal > 0) trimMailboxFromAckOrdinal(this.outbox, ackOrdinal);
 		const sortedEnvelopes = sortEnvelopesByFirstOperationTimestamp(envelopes);
 		const allSyncOps = [];
@@ -6935,6 +7069,11 @@ var GqlRequestChannel = class {
 		const consolidated = allSyncOps.length > 1 ? consolidateSyncOperations(allSyncOps) : allSyncOps;
 		if (consolidated.length > 0) this.inbox.add(...consolidated);
 		if (deadLetters.length > 0) this.handleRemoteDeadLetters(deadLetters);
+		if (hasMore) this.receivingPages = true;
+		else if (this.receivingPages) {
+			this.receivingPages = false;
+			this.drainOutbox();
+		}
 		this.lastSuccessUtcMs = Date.now();
 		this.failureCount = 0;
 		this.transitionConnectionState("connected");
@@ -7084,6 +7223,7 @@ var GqlRequestChannel = class {
             scopes
             operationCount
           }
+          hasMore
         }
       }
     `;
@@ -7096,7 +7236,8 @@ var GqlRequestChannel = class {
 		return {
 			envelopes: response.pollSyncEnvelopes.envelopes,
 			ackOrdinal: response.pollSyncEnvelopes.ackOrdinal,
-			deadLetters: response.pollSyncEnvelopes.deadLetters ?? []
+			deadLetters: response.pollSyncEnvelopes.deadLetters ?? [],
+			hasMore: response.pollSyncEnvelopes.hasMore
 		};
 	}
 	/**
@@ -7452,7 +7593,8 @@ var GqlResponseChannel = class {
 			lastSuccessUtcMs: 0,
 			lastFailureUtcMs: 0,
 			pushBlocked: false,
-			pushFailureCount: 0
+			pushFailureCount: 0,
+			receivingPages: false
 		};
 	}
 	onConnectionStateChange(callback) {
@@ -8033,6 +8175,15 @@ function deriveStatus(inboxCount, outboxCount, errorCount) {
 }
 //#endregion
 //#region src/sync/sync-manager.ts
+var OutboxMode = /* @__PURE__ */ function(OutboxMode) {
+	OutboxMode["Backfill"] = "backfill";
+	OutboxMode["BatchTriggered"] = "batch-triggered";
+	return OutboxMode;
+}(OutboxMode || {});
+const defaultSyncManagerConfig = {
+	maxDeadLettersPerRemote: 100,
+	maxInboxBatchSize: 32
+};
 var SyncManager = class {
 	logger;
 	remoteStorage;
@@ -8051,10 +8202,11 @@ var SyncManager = class {
 	failedEventUnsubscribe;
 	batchAggregator;
 	syncStatusTracker;
-	maxDeadLettersPerRemote;
+	config;
 	connectionStateUnsubscribes = /* @__PURE__ */ new Map();
 	quarantinedDocumentIds = /* @__PURE__ */ new Set();
-	constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, maxDeadLettersPerRemote = 100) {
+	backfillAbortControllers = /* @__PURE__ */ new Map();
+	constructor(logger, remoteStorage, cursorStorage, deadLetterStorage, channelFactory, operationIndex, reactor, eventBus, config = {}) {
 		this.logger = logger;
 		this.remoteStorage = remoteStorage;
 		this.cursorStorage = cursorStorage;
@@ -8063,7 +8215,10 @@ var SyncManager = class {
 		this.operationIndex = operationIndex;
 		this.reactor = reactor;
 		this.eventBus = eventBus;
-		this.maxDeadLettersPerRemote = maxDeadLettersPerRemote;
+		this.config = {
+			...defaultSyncManagerConfig,
+			...config
+		};
 		this.remotes = /* @__PURE__ */ new Map();
 		this.awaiter = new JobAwaiter(eventBus, (jobId, signal) => reactor.getJobStatus(jobId, signal));
 		this.syncAwaiter = new SyncAwaiter(eventBus);
@@ -8101,7 +8256,16 @@ var SyncManager = class {
 				continue;
 			}
 			const outboxAckOrdinal = remote.channel.outbox.ackOrdinal;
-			if (outboxAckOrdinal > 0) await this.updateOutbox(remote, outboxAckOrdinal);
+			if (outboxAckOrdinal > 0) {
+				const backfillController = new AbortController();
+				this.backfillAbortControllers.set(record.name, backfillController);
+				this.updateOutbox(remote, outboxAckOrdinal, OutboxMode.Backfill, backfillController.signal).catch((error) => {
+					if (backfillController.signal.aborted) return;
+					this.logger.error("Backfill failed for remote @RemoteName: @Error", remote.name, error instanceof Error ? error : new Error(String(error)));
+				}).finally(() => {
+					this.backfillAbortControllers.delete(record.name);
+				});
+			}
 		}
 		this.eventUnsubscribe = this.eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, async (_type, event) => this.batchAggregator.enqueueWriteReady(event));
 		this.failedEventUnsubscribe = this.eventBus.subscribe(ReactorEventTypes.JOB_FAILED, async (_type, event) => this.batchAggregator.handleJobFailed(event));
@@ -8109,6 +8273,8 @@ var SyncManager = class {
 	shutdown() {
 		this.isShutdown = true;
 		this.abortController.abort();
+		for (const controller of this.backfillAbortControllers.values()) controller.abort();
+		this.backfillAbortControllers.clear();
 		this.batchAggregator.clear();
 		if (this.eventUnsubscribe) {
 			this.eventUnsubscribe();
@@ -8181,12 +8347,24 @@ var SyncManager = class {
 			await this.remoteStorage.remove(name);
 			throw error;
 		}
-		await this.updateOutbox(remote, 0);
+		const backfillController = new AbortController();
+		this.backfillAbortControllers.set(name, backfillController);
+		this.updateOutbox(remote, 0, OutboxMode.Backfill, backfillController.signal).catch((error) => {
+			if (backfillController.signal.aborted) return;
+			this.logger.error("Backfill failed for remote @RemoteName: @Error", remote.name, error instanceof Error ? error : new Error(String(error)));
+		}).finally(() => {
+			this.backfillAbortControllers.delete(name);
+		});
 		return remote;
 	}
 	async remove(name) {
 		const remote = this.remotes.get(name);
 		if (!remote) throw new Error(`Remote with name '${name}' does not exist`);
+		const backfillController = this.backfillAbortControllers.get(name);
+		if (backfillController) {
+			backfillController.abort();
+			this.backfillAbortControllers.delete(name);
+		}
 		await remote.channel.shutdown();
 		await this.remoteStorage.remove(name);
 		await this.cursorStorage.remove(name);
@@ -8251,8 +8429,8 @@ var SyncManager = class {
 				}).catch(() => {});
 			}
 			const items = remote.channel.deadLetter.items;
-			if (items.length > this.maxDeadLettersPerRemote) {
-				const excessCount = items.length - this.maxDeadLettersPerRemote;
+			if (items.length > this.config.maxDeadLettersPerRemote) {
+				const excessCount = items.length - this.config.maxDeadLettersPerRemote;
 				const toEvict = items.slice(0, excessCount);
 				remote.channel.deadLetter.remove(...toEvict);
 			}
@@ -8263,7 +8441,7 @@ var SyncManager = class {
 		try {
 			records = (await this.deadLetterStorage.list(remote.name, {
 				cursor: "0",
-				limit: this.maxDeadLettersPerRemote
+				limit: this.config.maxDeadLettersPerRemote
 			})).results;
 		} catch (error) {
 			this.logger.error("Failed to load dead letters for remote (@name, @error)", remote.name, error instanceof Error ? error.message : String(error));
@@ -8292,7 +8470,7 @@ var SyncManager = class {
 			for (const remote of remotes) if (!affectedRemotes.includes(remote)) affectedRemotes.push(remote);
 		}
 		for (const remote of affectedRemotes) trimMailboxFromBatch(remote.channel.inbox, batch);
-		for (const remote of affectedRemotes) await this.updateOutbox(remote, remote.channel.outbox.latestOrdinal);
+		for (const remote of affectedRemotes) await this.updateOutbox(remote, remote.channel.outbox.latestOrdinal, OutboxMode.BatchTriggered);
 	}
 	handleInboxAdded(remote, syncOps) {
 		if (this.isShutdown) return;
@@ -8303,10 +8481,19 @@ var SyncManager = class {
 		for (const syncOp of eligible) if (syncOp.jobId) keyed.push(syncOp);
 		else nonKeyed.push(syncOp);
 		for (const syncOp of nonKeyed) this.applyInboxJob(remote, syncOp);
-		if (keyed.length > 0) this.applyInboxBatch(keyed.map((syncOp) => ({
-			remote,
-			syncOp
-		})));
+		if (keyed.length > 0) {
+			const chunks = chunkSyncOperations(keyed.map((syncOp) => ({
+				remote,
+				syncOp
+			})), this.config.maxInboxBatchSize);
+			this.processInboxChunks(chunks);
+		}
+	}
+	async processInboxChunks(chunks) {
+		for (const chunk of chunks) {
+			if (this.isShutdown) return;
+			await this.applyInboxBatch(chunk);
+		}
 	}
 	async applyInboxJob(remote, syncOp) {
 		const operations = syncOp.operations.map((op) => op.operation);
@@ -8348,13 +8535,14 @@ var SyncManager = class {
 	}
 	async applyInboxBatch(items) {
 		const sourceRemote = items[0].remote.name;
+		const chunkKeys = new Set(items.map(({ syncOp }) => syncOp.jobId));
 		const request = { jobs: items.map(({ syncOp }) => ({
 			key: syncOp.jobId,
 			documentId: syncOp.documentId,
 			scope: syncOp.scopes[0],
 			branch: syncOp.branch,
 			operations: syncOp.operations.map((op) => op.operation),
-			dependsOn: syncOp.jobDependencies.filter(Boolean)
+			dependsOn: syncOp.jobDependencies.filter((dep) => dep && chunkKeys.has(dep))
 		})) };
 		let result;
 		try {
@@ -8401,13 +8589,15 @@ var SyncManager = class {
 			remote.channel.inbox.remove(syncOp);
 		}
 	}
-	async updateOutbox(remote, ackOrdinal) {
+	async updateOutbox(remote, ackOrdinal, mode = OutboxMode.Backfill, signal) {
+		const composedSignal = signal ? AbortSignal.any([signal, this.abortController.signal]) : this.abortController.signal;
 		let maxOrdinal = ackOrdinal;
 		const lastJobByDoc = /* @__PURE__ */ new Map();
 		const sinceTimestamp = remote.options.sinceTimestampUtcMs;
-		let page = await this.operationIndex.find(remote.collectionId, ackOrdinal, { excludeSourceRemote: remote.name }, void 0, this.abortController.signal);
+		let page = await this.operationIndex.find(remote.collectionId, ackOrdinal, { excludeSourceRemote: remote.name }, void 0, composedSignal);
 		let hasMore;
 		do {
+			if (composedSignal.aborted) return;
 			for (const entry of page.results) maxOrdinal = Math.max(maxOrdinal, entry.ordinal ?? 0);
 			let operations = page.results.map((entry) => toOperationWithContext(entry));
 			if (sinceTimestamp && sinceTimestamp !== "0") operations = operations.filter((op) => op.operation.timestampUtcMs >= sinceTimestamp);
@@ -8421,12 +8611,17 @@ var SyncManager = class {
 				});
 				const batches = batchOperationsByDocument(operations);
 				const syncOps = [];
+				let prevChainJobId;
 				for (const batch of batches) {
 					const jobId = crypto.randomUUID();
 					const prevJobId = lastJobByDoc.get(batch.documentId);
-					const syncOp = new SyncOperation(crypto.randomUUID(), jobId, prevJobId ? [prevJobId] : [], remote.name, batch.documentId, [batch.scope], batch.branch, batch.operations);
+					const deps = [];
+					if (prevJobId) deps.push(prevJobId);
+					if (mode === OutboxMode.BatchTriggered && prevChainJobId && prevChainJobId !== prevJobId) deps.push(prevChainJobId);
+					const syncOp = new SyncOperation(crypto.randomUUID(), jobId, deps, remote.name, batch.documentId, [batch.scope], batch.branch, batch.operations);
 					syncOps.push(syncOp);
 					lastJobByDoc.set(batch.documentId, jobId);
+					if (mode === OutboxMode.BatchTriggered) prevChainJobId = jobId;
 				}
 				remote.channel.outbox.add(...syncOps);
 			}
@@ -8443,7 +8638,7 @@ var SyncBuilder = class {
 	remoteStorage;
 	cursorStorage;
 	deadLetterStorage;
-	maxDeadLettersPerRemote = 100;
+	config = {};
 	withChannelFactory(factory) {
 		this.channelFactory = factory;
 		return this;
@@ -8461,7 +8656,11 @@ var SyncBuilder = class {
 		return this;
 	}
 	withMaxDeadLettersPerRemote(limit) {
-		this.maxDeadLettersPerRemote = limit;
+		this.config.maxDeadLettersPerRemote = limit;
+		return this;
+	}
+	withMaxInboxBatchSize(limit) {
+		this.config.maxInboxBatchSize = limit;
 		return this;
 	}
 	build(reactor, logger, operationIndex, eventBus, db) {
@@ -8472,7 +8671,7 @@ var SyncBuilder = class {
 		const remoteStorage = this.remoteStorage ?? new KyselySyncRemoteStorage(db);
 		const cursorStorage = this.cursorStorage ?? new KyselySyncCursorStorage(db);
 		const deadLetterStorage = this.deadLetterStorage ?? new KyselySyncDeadLetterStorage(db);
-		const syncManager = new SyncManager(logger, remoteStorage, cursorStorage, deadLetterStorage, this.channelFactory, operationIndex, reactor, eventBus, this.maxDeadLettersPerRemote);
+		const syncManager = new SyncManager(logger, remoteStorage, cursorStorage, deadLetterStorage, this.channelFactory, operationIndex, reactor, eventBus, this.config);
 		return {
 			remoteStorage,
 			cursorStorage,
@@ -8514,12 +8713,6 @@ function createMutableShutdownStatus(initialState = false) {
 	];
 }
 //#endregion
-//#region src/shared/utils.ts
-function matchesScope(view = {}, scope) {
-	if (view.scopes) return view.scopes.includes(scope);
-	return true;
-}
-//#endregion
 //#region src/core/types.ts
 var AbortError = class extends Error {
 	constructor(message) {