@powerhousedao/reactor 6.2.2-dev.4 → 6.2.2-dev.40

package/dist/index.js

@@ -1,10 +1,10 @@
-import { A as parsePagingOptions, C as DuplicateManifestError, D as DocumentDeletedError, E as ModuleNotFoundError, O as DocumentNotFoundError, S as CollectionMembershipCache, T as InvalidModuleError, _ as KyselyWriteCache, a as createForwardingPoolInstrumentation, b as createConsistencyToken, c as DuplicateOperationError, d as KyselyKeyframeStore, f as DocumentModelRegistry, g as EventBus, h as KyselyExecutionScope, i as runMigrations, j as throwIfAborted, k as matchesScope, l as OptimisticLockError, m as DriveCollectionId, n as REACTOR_SCHEMA, o as instrumentPgPool, p as SimpleJobExecutor, r as getMigrationStatus, s as KyselyOperationStore, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as RevisionMismatchError, v as KyselyOperationIndex, w as DuplicateModuleError, x as createEmptyConsistencyToken, y as DocumentMetaCache } from "./drive-container-types-DpJp2AmE.js";
+import { A as OptimisticLockError, B as ExcessiveReshuffleError, C as DocumentMetaCache, D as APPEND_CONDITION_FAILED_PREFIX, E as CollectionMembershipCache, F as ModuleNotFoundError, H as matchesScope, I as AuthTimestampNotMonotonicError, L as AuthorizationDeniedError, M as DuplicateManifestError, N as DuplicateModuleError, O as AppendConditionFailedError, P as InvalidModuleError, R as DocumentDeletedError, S as KyselyOperationIndex, T as createEmptyConsistencyToken, U as parsePagingOptions, V as InvalidOperationTimestampError, W as throwIfAborted, _ as KyselyExecutionScope, a as createForwardingPoolInstrumentation, b as EventBus, c as KyselyKeyframeStore, d as DriveCollectionId, f as decideAtHead, g as authDecisionModel, h as buildDecisionModel, i as runMigrations, j as RevisionMismatchError, k as DuplicateOperationError, l as DocumentModelRegistry, m as documentDecisionModel, n as REACTOR_SCHEMA, o as instrumentPgPool, p as selectDecisionModel, r as getMigrationStatus, s as KyselyOperationStore, t as DEFAULT_DRIVE_CONTAINER_TYPES, u as SimpleJobExecutor, v as FLAG_PREREQUISITES, w as createConsistencyToken, x as KyselyWriteCache, y as validateFeatureFlags, z as DocumentNotFoundError } from "./drive-container-types-BJCKXJwH.js";
 import { n as ReactorEventTypes, t as EventBusAggregateError } from "./types-DMKLa0Ok.js";
 import { i as WorkerInitFailedError, r as WorkerExitedError, t as WorkerAbortTimeoutError } from "./errors-D3S6Eysd.js";
-import { a as ReadModelCoordinator, i as KyselyDocumentView, n as ConsistencyTracker, o as BaseReadModel, r as makeConsistencyKey, t as KyselyDocumentIndexer } from "./document-indexer-FGJmRAdX.js";
+import { a as ReadModelCoordinator, i as KyselyDocumentView, n as ConsistencyTracker, o as BaseReadModel, r as makeConsistencyKey, t as KyselyDocumentIndexer } from "./document-indexer-C0GB0b8Q.js";
 import { n as errorToInfo, r as sanitizeArg, t as createForwardingLogger } from "./forwarding-logger-BBkMSxuJ.js";
-import { t as workerEntryPath } from "./worker-DBJOv8Gp.js";
-import { actions, actions as documentActions, createPresignedHeader, generateId, hashDocumentStateForScope, replayDocument } from "@powerhousedao/shared/document-model";
+import { t as workerEntryPath } from "./worker-jEJW6_j7.js";
+import { actions, actions as documentActions, createPresignedHeader, decide, garbageCollect, generateId, hashDocumentStateForScope, replayDocument, sortOperations } from "@powerhousedao/shared/document-model";
 import { addFile, addFolder, copyNode, deleteNode, driveCreateDocument, generateNodesCopy, getDescendants, handleTargetNameCollisions, isFileNode, isFolderNode, moveNode, updateNode } from "@powerhousedao/shared/document-drive";
 import { v4 } from "uuid";
 import { ConsoleLogger, childLogger } from "document-model";
@@ -190,10 +190,12 @@ function topologicalSort(jobs) {
 */
 function toErrorInfo$1(error) {
 	if (error instanceof Error) return {
+		name: error.name,
 		message: error.message,
 		stack: error.stack || (/* @__PURE__ */ new Error()).stack || ""
 	};
 	return {
+		name: "Error",
 		message: error,
 		stack: (/* @__PURE__ */ new Error()).stack || ""
 	};
@@ -680,6 +682,45 @@ let DocumentChangeType = /* @__PURE__ */ function(DocumentChangeType) {
 	return DocumentChangeType;
 }({});
 //#endregion
+//#region src/client/util.ts
+const ALWAYS_READABLE_SCOPES = new Set(["auth", "document"]);
+function authSubjectFromSigner(signer) {
+	return {
+		address: signer.user?.address,
+		key: signer.app?.key
+	};
+}
+function canReadScope(auth, subject, scope) {
+	return ALWAYS_READABLE_SCOPES.has(scope) || decide(auth, subject, {
+		verb: "read",
+		scope
+	}) === "allow";
+}
+function withAuthScope(view) {
+	if (view?.scopes && view.scopes.length > 0) return {
+		...view,
+		scopes: [...new Set([...view.scopes, "auth"])]
+	};
+	return view;
+}
+function filterReadableScopes(document, subject) {
+	const state = document.state;
+	if (!state) return document;
+	const auth = document.state.auth;
+	const readable = (scope) => canReadScope(auth, subject, scope);
+	return {
+		...document,
+		state: keepReadableScopes(state, readable),
+		initialState: keepReadableScopes(document.initialState, readable)
+	};
+}
+function keepReadableScopes(scopes, readable) {
+	if (!scopes) return scopes;
+	const kept = {};
+	for (const scope of Object.keys(scopes)) if (readable(scope)) kept[scope] = scopes[scope];
+	return kept;
+}
+//#endregion
 //#region src/client/reactor-client.ts
 /**
 * ReactorClient implementation that wraps lower-level APIs to provide
@@ -711,6 +752,9 @@ var ReactorClient = class {
 		this.drives = new DriveClient(this, logger, reactor, signer);
 		this.logger.verbose("ReactorClient initialized");
 	}
+	readSubject(subject) {
+		return subject ?? authSubjectFromSigner(this.signer);
+	}
 	/**
 	* Retrieves a list of document model modules.
 	*/
@@ -734,15 +778,15 @@ var ReactorClient = class {
 	*/
 	async get(identifier, view, signal) {
 		this.logger.verbose("get(@identifier, @view)", identifier, view);
-		return await this.reactor.getByIdOrSlug(identifier, view, void 0, signal);
+		return filterReadableScopes(await this.reactor.getByIdOrSlug(identifier, withAuthScope(view), void 0, signal), this.readSubject(view?.subject));
 	}
 	/**
 	* Resolves an identifier (id or slug) to the canonical document id, using the
 	* same lookup as the data path. Resolves against the "main" branch. Throws if
 	* the identifier cannot be resolved or is ambiguous.
 	*/
-	async resolveIdOrSlug(identifier, signal) {
-		return this.documentView.resolveIdOrSlug(identifier, void 0, void 0, signal);
+	async resolveIdOrSlug(identifier, view, signal) {
+		return this.documentView.resolveIdOrSlug(identifier, view, void 0, signal);
 	}
 	/**
 	* Retrieves operations for a document
@@ -750,8 +794,15 @@ var ReactorClient = class {
 	async getOperations(documentIdentifier, view, filter, paging, signal) {
 		this.logger.verbose("getOperations(@documentIdentifier, @view, @filter, @paging)", documentIdentifier, view, filter, paging);
 		const documentId = await this.documentView.resolveIdOrSlug(documentIdentifier, view, void 0, signal);
-		if (paging?.cursor && isCompositeCursor(paging.cursor)) return this.getOperationsWithCompositeCursor(documentId, view, filter, paging, signal);
+		const authDoc = await this.reactor.getByIdOrSlug(documentId, {
+			scopes: ["auth"],
+			branch: view?.branch
+		}, void 0, signal);
+		const subject = this.readSubject(view?.subject);
+		const canRead = (scope) => canReadScope(authDoc?.state.auth, subject, scope);
+		if (paging?.cursor && isCompositeCursor(paging.cursor)) return this.getOperationsWithCompositeCursor(documentId, view, filter, paging, signal, canRead);
 		const operationsByScope = await this.reactor.getOperations(documentId, view, filter, paging, void 0, signal);
+		for (const scope of Object.keys(operationsByScope)) if (!canRead(scope)) delete operationsByScope[scope];
 		const scopeEntries = Object.entries(operationsByScope);
 		const effectivePaging = paging || {
 			cursor: "0",
@@ -779,11 +830,12 @@ var ReactorClient = class {
 			nextCursor: Object.keys(activeCursors).length > 0 ? encodeCompositeCursor(activeCursors) : void 0
 		};
 	}
-	async getOperationsWithCompositeCursor(documentId, view, filter, paging, signal) {
+	async getOperationsWithCompositeCursor(documentId, view, filter, paging, signal, canRead) {
 		const scopeCursors = decodeCompositeCursor(paging.cursor);
 		const allOperations = [];
 		const activeCursors = {};
 		for (const [scopeName, cursor] of Object.entries(scopeCursors)) {
+			if (!canRead(scopeName)) continue;
 			const scopeView = {
 				...view,
 				scopes: [scopeName]
@@ -817,7 +869,7 @@ var ReactorClient = class {
 				limit: 0
 			}
 		};
-		return this.reactor.find({ ids: targetIds }, view, paging, void 0, signal);
+		return this.find({ ids: targetIds }, view, paging, signal);
 	}
 	/**
 	* Retrieves incoming relationships of a given type to a target document.
@@ -833,14 +885,19 @@ var ReactorClient = class {
 				limit: 0
 			}
 		};
-		return this.reactor.find({ ids: sourceIds }, view, paging, void 0, signal);
+		return this.find({ ids: sourceIds }, view, paging, signal);
 	}
 	/**
 	* Filters documents by criteria and returns a list of them
 	*/
 	async find(search, view, paging, signal) {
 		this.logger.verbose("find(@search, @view, @paging)", search, view, paging);
-		return this.reactor.find(search, view, paging, void 0, signal);
+		const results = await this.reactor.find(search, withAuthScope(view), paging, void 0, signal);
+		const readSubject = this.readSubject(view?.subject);
+		return {
+			...results,
+			results: results.results.map((doc) => filterReadableScopes(doc, readSubject))
+		};
 	}
 	/**
 	* Creates a document and waits for completion
@@ -1078,13 +1135,15 @@ var ReactorClient = class {
 	*/
 	subscribe(search, callback, view) {
 		this.logger.verbose("subscribe(@search, @view)", search, view);
+		const subject = this.readSubject(view?.subject);
+		const readable = (document) => filterReadableScopes(document, subject);
 		const unsubscribeCreated = this.subscriptionManager.onDocumentCreated((result) => {
 			(async () => {
 				try {
-					const documents = await Promise.all(result.results.map((id) => this.reactor.get(id, view, void 0, void 0)));
+					const documents = await Promise.all(result.results.map((id) => this.reactor.get(id, withAuthScope(view), void 0, void 0)));
 					callback({
 						type: DocumentChangeType.Created,
-						documents
+						documents: documents.map(readable)
 					});
 				} catch {}
 			})();
@@ -1099,7 +1158,7 @@ var ReactorClient = class {
 		const unsubscribeUpdated = this.subscriptionManager.onDocumentStateUpdated((result) => {
 			callback({
 				type: DocumentChangeType.Updated,
-				documents: result.results
+				documents: result.results.map(readable)
 			});
 		}, search, view);
 		const unsubscribeRelationship = this.subscriptionManager.onRelationshipChanged((parentId, childId, changeType) => {
@@ -1144,17 +1203,33 @@ let JobQueueState = /* @__PURE__ */ function(JobQueueState) {
 	return JobQueueState;
 }({});
 /**
+* How a retry is accounted against the job's retry limit.
+* - `CountAgainstLimit` (default): a fault; the job eventually exhausts its
+*   retries and fails terminally.
+* - `ExemptFromLimit`: not a fault, so the attempt is not charged to the job.
+*   Used for concurrency conflicts, where the retry does new work.
+*/
+let RetryAccounting = /* @__PURE__ */ function(RetryAccounting) {
+	RetryAccounting["CountAgainstLimit"] = "count-against-limit";
+	RetryAccounting["ExemptFromLimit"] = "exempt-from-limit";
+	return RetryAccounting;
+}({});
+/**
 * Event types for the queue system
 */
 const QueueEventTypes = { JOB_AVAILABLE: 1e4 };
 //#endregion
 //#region src/executor/job-result-handler.ts
+/** Conflict retries a job may take without charging its retry limit. */
+const MAX_EXEMPT_CONFLICT_RETRIES = 20;
 function toErrorInfo(error) {
 	if (error instanceof Error) return {
+		name: error.name,
 		message: error.message,
 		stack: error.stack || (/* @__PURE__ */ new Error()).stack || ""
 	};
 	return {
+		name: "Error",
 		message: error,
 		stack: (/* @__PURE__ */ new Error()).stack || ""
 	};
@@ -1187,12 +1262,19 @@ var JobResultHandler = class {
 				} catch {}
 			}
 		}
+		if (result.error && AppendConditionFailedError.isError(result.error) && this.countConflicts(handle.job) < MAX_EXEMPT_CONFLICT_RETRIES) {
+			const errorInfo = toErrorInfo(result.error);
+			try {
+				await this.queue.retryJob(handle.job.id, errorInfo, RetryAccounting.ExemptFromLimit);
+				return;
+			} catch {}
+		}
 		if (result.error && DocumentNotFoundError.isError(result.error)) {
 			handle.defer();
 			callbacks.deferJob(handle.job.documentId, handle.job);
 			return;
 		}
-		if (result.error && DocumentDeletedError.isError(result.error)) {
+		if (result.error && (DocumentDeletedError.isError(result.error) || AuthorizationDeniedError.isError(result.error) || AuthTimestampNotMonotonicError.isError(result.error) || InvalidOperationTimestampError.isError(result.error) || ExcessiveReshuffleError.isError(result.error))) {
 			const errorInfo = toErrorInfo(result.error);
 			this.jobTracker.markFailed(handle.job.id, errorInfo, handle.job);
 			this.eventBus.emit(ReactorEventTypes.JOB_FAILED, {
@@ -1230,6 +1312,12 @@ var JobResultHandler = class {
 			handle.fail(fullErrorInfo);
 		}
 	}
+	/** How many times this job has already lost an append-condition race. */
+	countConflicts(job) {
+		let conflicts = 0;
+		for (const error of job.errorHistory) if (AppendConditionFailedError.isFailureMessage(error.message)) conflicts++;
+		return conflicts;
+	}
 	hasCreateDocumentAction(job) {
 		for (const action of job.actions) if (action.type === "CREATE_DOCUMENT") return true;
 		for (const operation of job.operations) if (operation.action.type === "CREATE_DOCUMENT") return true;
@@ -1245,6 +1333,7 @@ var JobResultHandler = class {
 			stackLines.push(`[Attempt ${index + 1}] Stack trace:\n${error.stack}`);
 		});
 		return {
+			name: currentError.name,
 			message: messageLines.join("\n"),
 			stack: stackLines.join("\n\n")
 		};
@@ -1850,6 +1939,7 @@ var InMemoryJobTracker = class {
 	}
 	handleJobFailed(event) {
 		this.markFailed(event.jobId, {
+			name: event.error.name,
 			message: event.error.message,
 			stack: event.error.stack || ""
 		}, event.job);
@@ -2311,10 +2401,12 @@ var InMemoryQueue = class {
 	}
 	toErrorInfo(error) {
 		if (error instanceof Error) return {
+			name: error.name,
 			message: error.message,
 			stack: error.stack || (/* @__PURE__ */ new Error()).stack || ""
 		};
 		return {
+			name: "Error",
 			message: error,
 			stack: (/* @__PURE__ */ new Error()).stack || ""
 		};
@@ -2402,6 +2494,7 @@ var InMemoryQueue = class {
 			await this.resolver.ensureModelLoaded(documentType);
 		} catch {
 			await this.failJob(job.id, {
+				name: "Error",
 				message: `Failed to load document model for type: ${documentType}`,
 				stack: (/* @__PURE__ */ new Error()).stack || ""
 			});
@@ -2587,7 +2680,7 @@ var InMemoryQueue = class {
 		if (documentId) this.markJobComplete(jobId, documentId);
 		this.jobIndex.delete(jobId);
 	}
-	async retryJob(jobId, error) {
+	async retryJob(jobId, error, accounting = RetryAccounting.CountAgainstLimit) {
 		const job = this.jobIndex.get(jobId);
 		if (!job) return;
 		job.lastError = error;
@@ -2596,9 +2689,10 @@ var InMemoryQueue = class {
 		this.jobIndex.delete(jobId);
 		this.jobIdToQueueKey.delete(jobId);
 		if (error) job.errorHistory.push(error);
+		const retryCount = job.retryCount || 0;
 		const updatedJob = {
 			...job,
-			retryCount: (job.retryCount || 0) + 1,
+			retryCount: accounting === RetryAccounting.CountAgainstLimit ? retryCount + 1 : retryCount,
 			lastError: error
 		};
 		await this.enqueue(updatedJob);
@@ -3437,11 +3531,18 @@ var PollingChannelError = class extends Error {
 var ChannelError = class extends Error {
 	source;
 	error;
-	constructor(source, error) {
+	/**
+	* The classification when something other than the error carries it. Absent
+	* means derive it from `error.name`; a dead letter mirrored from a peer sets it,
+	* because only the message crosses the wire.
+	*/
+	errorType;
+	constructor(source, error, errorType) {
 		super(`ChannelError[${source}]: ${error.message}`);
 		this.name = "ChannelError";
 		this.source = source;
 		this.error = error;
+		this.errorType = errorType;
 	}
 };
 //#endregion
@@ -3646,6 +3747,7 @@ function toOperationWithContext(entry) {
 			skip: entry.skip,
 			hash: entry.hash,
 			timestampUtcMs: entry.timestampUtcMs,
+			deniedReason: entry.deniedReason,
 			action: entry.action
 		},
 		context: {
@@ -3811,6 +3913,33 @@ function splitComponent(items, maxSize) {
 	for (let i = 0; i < sorted.length; i += maxSize) chunks.push(sorted.slice(i, i + maxSize));
 	return chunks;
 }
+/**
+* Classifies a failure by error name rather than `instanceof`, because a failure
+* that crossed the pooled-worker boundary arrives as plain data.
+*/
+function classifyJobFailure(errorName) {
+	switch (errorName) {
+		case "AuthTimestampNotMonotonicError": return "AUTH_TIMESTAMP_NOT_MONOTONIC";
+		case "InvalidOperationTimestampError": return "INVALID_TIMESTAMP";
+		case "ExcessiveReshuffleError": return "EXCESSIVE_SHUFFLE";
+		case "InvalidSignatureError": return "SIGNATURE_INVALID";
+		case "HashMismatchError": return "HASH_MISMATCH";
+		default: return "UNCLASSIFIED";
+	}
+}
+/** The explicit type when something else carried it, else derived by name. */
+function syncOperationErrorType(error) {
+	return error?.errorType ?? classifyJobFailure(error?.error?.name ?? "Error");
+}
+/** Dead-letter types that must not stop the document syncing. */
+const NON_QUARANTINING_ERROR_TYPES = new Set(["AUTH_TIMESTAMP_NOT_MONOTONIC"]);
+/**
+* A held auth operation must not quarantine: reconciling the two policies needs
+* the traffic a quarantine would stop.
+*/
+function quarantinesDocument(errorType) {
+	return !NON_QUARANTINING_ERROR_TYPES.has(errorType);
+}
 //#endregion
 //#region src/sync/channels/interval-poll-timer.ts
 const DEFAULT_CONFIG = {
@@ -3973,6 +4102,7 @@ function serializeEnvelope(envelope) {
 				hash: opWithContext.operation.hash,
 				skip: opWithContext.operation.skip,
 				error: opWithContext.operation.error,
+				...opWithContext.operation.deniedReason !== void 0 ? { deniedReason: opWithContext.operation.deniedReason } : {},
 				id: opWithContext.operation.id,
 				action: serializeAction(opWithContext.operation.action)
 			},
@@ -4000,21 +4130,33 @@ function deserializeSignature(sig) {
 	return sig.split(", ");
 }
 /**
-* Deserializes signatures in an operation's signer context from strings back to tuples.
-*
-* When operations are transported via GraphQL, signatures are serialized as comma-separated
-* strings. This function restores them to the Signature tuple format required for verification.
+* `isDenied` tests strictly against undefined, so a null left in place would mark
+* every synced operation denied. The key is removed rather than set to undefined,
+* so an operation that arrived without it stays identical to the one sent.
 */
-function deserializeOperationSignatures(opWithContext) {
-	const signer = opWithContext.operation.action.context?.signer;
-	if (!signer?.signatures || signer.signatures.length === 0) return opWithContext;
+function normalizeAbsentFields(operation) {
+	const wire = operation;
+	if (wire.error !== null && wire.deniedReason !== null) return operation;
+	const normalized = { ...operation };
+	if (wire.error === null) delete normalized.error;
+	if (wire.deniedReason === null) delete normalized.deniedReason;
+	return normalized;
+}
+/** Restores signature tuples and null-valued optional fields to undefined. */
+function deserializeOperation(opWithContext) {
+	const operation = normalizeAbsentFields(opWithContext.operation);
+	const signer = operation.action.context?.signer;
+	if (!signer?.signatures || signer.signatures.length === 0) return {
+		...opWithContext,
+		operation
+	};
 	const deserializedSignatures = signer.signatures.map(deserializeSignature);
 	const deserializedOperation = {
-		...opWithContext.operation,
+		...operation,
 		action: {
-			...opWithContext.operation.action,
+			...operation.action,
 			context: {
-				...opWithContext.operation.action.context,
+				...operation.action.context,
 				signer: {
 					...signer,
 					signatures: deserializedSignatures
@@ -4041,7 +4183,7 @@ function deserializeOperationSignatures(opWithContext) {
 */
 function envelopesToSyncOperations(envelope, remoteName) {
 	if (!envelope.operations || envelope.operations.length === 0) return [];
-	return batchOperationsByDocument(envelope.operations.map(deserializeOperationSignatures)).map((batch) => {
+	return batchOperationsByDocument(envelope.operations.map(deserializeOperation)).map((batch) => {
 		return new SyncOperation(`syncop-${envelope.channelMeta.id}-${Date.now()}-${syncOpCounter++}`, envelope.key ?? "", (envelope.dependsOn ?? []).filter(Boolean), remoteName, batch.documentId, [batch.scope], batch.branch, batch.operations);
 	});
 }
@@ -4053,6 +4195,12 @@ const getLatestAppliedOrdinal = (syncOps) => {
 //#endregion
 //#region src/sync/channels/gql-req-channel.ts
 /**
+* Fields the auth projection added to the sync schema. A remote that predates
+* them rejects the whole query for naming one, so they are selected only while
+* the remote is known to serve them.
+*/
+const DECISION_FIELDS = ["deniedReason", "errorType"];
+/**
 * GraphQL-based synchronization channel for network communication between reactors.
 */
 var GqlRequestChannel = class {
@@ -4079,6 +4227,8 @@ var GqlRequestChannel = class {
 	isPushing = false;
 	pendingDrain = false;
 	receivingPages = false;
+	/** Cleared for good the first time the remote rejects {@link DECISION_FIELDS}. */
+	peerServesDecisionFields = true;
 	isRecovering = false;
 	connectionState = "connecting";
 	/** Latest unrecoverable error was an auth rejection; cleared on connect. */
@@ -4257,7 +4407,7 @@ var GqlRequestChannel = class {
 		const syncOps = [];
 		for (const dl of deadLetters) {
 			const syncOp = new SyncOperation(crypto.randomUUID(), dl.jobId, [], this.remoteName, dl.documentId, dl.scopes, dl.branch, []);
-			syncOp.failed(new ChannelError(ChannelErrorSource.Outbox, new Error(dl.error)));
+			syncOp.failed(new ChannelError(ChannelErrorSource.Outbox, new Error(dl.error), dl.errorType ?? void 0));
 			syncOps.push(syncOp);
 		}
 		this.deadLetter.add(...syncOps);
@@ -4349,7 +4499,44 @@ var GqlRequestChannel = class {
 	* Queries the remote GraphQL endpoint for sync envelopes.
 	*/
 	async pollSyncEnvelopes(ackOrdinal, latestOrdinal) {
-		const query = `
+		const variables = {
+			channelId: this.channelId,
+			outboxAck: ackOrdinal,
+			outboxLatest: latestOrdinal
+		};
+		let response;
+		try {
+			response = await this.executeGraphQL(this.pollQuery(this.peerServesDecisionFields), variables);
+		} catch (error) {
+			if (!this.rejectsDecisionFields(error)) throw error;
+			this.logger.warn("Remote @channelId does not serve deniedReason/errorType; polling without them. The remote is on an older schema, so it has neither to report.", this.channelId);
+			this.peerServesDecisionFields = false;
+			response = await this.executeGraphQL(this.pollQuery(false), variables);
+		}
+		return {
+			envelopes: response.pollSyncEnvelopes.envelopes,
+			ackOrdinal: response.pollSyncEnvelopes.ackOrdinal,
+			deadLetters: response.pollSyncEnvelopes.deadLetters ?? [],
+			hasMore: response.pollSyncEnvelopes.hasMore
+		};
+	}
+	/**
+	* True when the remote rejected the query for naming a field it does not
+	* have. Selecting an unknown field fails validation for the whole query, so
+	* an unhandled one takes the channel's polling down until the process
+	* restarts rather than degrading.
+	*/
+	rejectsDecisionFields(error) {
+		if (!this.peerServesDecisionFields) return false;
+		if (!(error instanceof GraphQLRequestError) || error.category !== "graphql") return false;
+		return DECISION_FIELDS.some((field) => error.message.includes(field));
+	}
+	/**
+	* The poll query. `withDecisionFields` selects the two fields added with the
+	* auth projection; a remote on the previous schema is polled without them.
+	*/
+	pollQuery(withDecisionFields) {
+		return `
       query PollSyncEnvelopes($channelId: String!, $outboxAck: Int!, $outboxLatest: Int!) {
         pollSyncEnvelopes(channelId: $channelId, outboxAck: $outboxAck, outboxLatest: $outboxLatest) {
           envelopes {
@@ -4364,6 +4551,7 @@ var GqlRequestChannel = class {
                 hash
                 skip
                 error
+                ${withDecisionFields ? "deniedReason" : ""}
                 id
                 action {
                   id
@@ -4407,6 +4595,7 @@ var GqlRequestChannel = class {
           deadLetters {
             documentId
             error
+            ${withDecisionFields ? "errorType" : ""}
             jobId
             branch
             scopes
@@ -4416,18 +4605,6 @@ var GqlRequestChannel = class {
         }
       }
     `;
-		const variables = {
-			channelId: this.channelId,
-			outboxAck: ackOrdinal,
-			outboxLatest: latestOrdinal
-		};
-		const response = await this.executeGraphQL(query, variables);
-		return {
-			envelopes: response.pollSyncEnvelopes.envelopes,
-			ackOrdinal: response.pollSyncEnvelopes.ackOrdinal,
-			deadLetters: response.pollSyncEnvelopes.deadLetters ?? [],
-			hasMore: response.pollSyncEnvelopes.hasMore
-		};
 	}
 	/**
 	* Registers or updates this channel on the remote server via GraphQL mutation.
@@ -4909,7 +5086,8 @@ function rowToDeadLetterRecord(row) {
 		branch: row.branch,
 		operations: row.operations,
 		errorSource: row.error_source,
-		errorMessage: row.error_message
+		errorMessage: row.error_message,
+		errorType: row.error_type ?? "UNCLASSIFIED"
 	};
 }
 function deadLetterRecordToRow(record) {
@@ -4923,7 +5101,8 @@ function deadLetterRecordToRow(record) {
 		branch: record.branch,
 		operations: JSON.stringify(record.operations),
 		error_source: record.errorSource,
-		error_message: record.errorMessage
+		error_message: record.errorMessage,
+		error_type: record.errorType
 	};
 }
 /**
@@ -4980,9 +5159,11 @@ var KyselySyncDeadLetterStorage = class {
 	}
 	async listQuarantinedDocumentIds(signal) {
 		if (signal?.aborted) throw new Error("Operation aborted");
-		const rows = await this.db.selectFrom("sync_dead_letters").select("document_id").distinct().execute();
+		const rows = await this.db.selectFrom("sync_dead_letters").select(["document_id", "error_type"]).distinct().execute();
 		if (signal?.aborted) throw new Error("Operation aborted");
-		return rows.map((row) => row.document_id);
+		const quarantined = /* @__PURE__ */ new Set();
+		for (const row of rows) if (quarantinesDocument(row.error_type ?? "UNCLASSIFIED")) quarantined.add(row.document_id);
+		return [...quarantined];
 	}
 };
 //#endregion
@@ -5655,7 +5836,8 @@ var SyncManager = class {
 		remote.channel.deadLetter.onAdded((syncOps) => {
 			for (const syncOp of syncOps) {
 				this.logger.error("Dead letter (@remote, @documentId, @jobId, @error, @dependencies)", remote.meta.name, syncOp.documentId, syncOp.jobId, syncOp.error?.message ?? "unknown", syncOp.jobDependencies);
-				this.quarantinedDocumentIds.add(syncOp.documentId);
+				const errorType = syncOperationErrorType(syncOp.error);
+				if (quarantinesDocument(errorType)) this.quarantinedDocumentIds.add(syncOp.documentId);
 				const record = {
 					id: syncOp.id,
 					jobId: syncOp.jobId,
@@ -5666,7 +5848,8 @@ var SyncManager = class {
 					branch: syncOp.branch,
 					operations: syncOp.operations,
 					errorSource: syncOp.error?.source ?? ChannelErrorSource.None,
-					errorMessage: syncOp.error?.error.message ?? "unknown"
+					errorMessage: syncOp.error?.error.message ?? "unknown",
+					errorType
 				};
 				this.deadLetterStorage.add(record).catch((err) => {
 					this.logger.error("Failed to persist dead letter (@id, @error)", record.id, err instanceof Error ? err.message : String(err));
@@ -5676,7 +5859,8 @@ var SyncManager = class {
 					jobId: record.jobId,
 					remoteName: record.remoteName,
 					documentId: record.documentId,
-					errorSource: record.errorSource
+					errorSource: record.errorSource,
+					errorType: record.errorType
 				}).catch(() => {});
 			}
 			const items = remote.channel.deadLetter.items;
@@ -5703,7 +5887,7 @@ var SyncManager = class {
 		const syncOps = [];
 		for (const record of records) {
 			const syncOp = new SyncOperation(record.id, record.jobId, record.jobDependencies, record.remoteName, record.documentId, record.scopes, record.branch, record.operations);
-			syncOp.failed(new ChannelError(record.errorSource, new Error(record.errorMessage)));
+			syncOp.failed(new ChannelError(record.errorSource, new Error(record.errorMessage), record.errorType));
 			syncOps.push(syncOp);
 		}
 		remote.channel.deadLetter.add(...syncOps);
@@ -5784,8 +5968,7 @@ var SyncManager = class {
 		if (completedJobInfo.status === JobStatus.FAILED) {
 			const errorMessage = completedJobInfo.error?.message || "Unknown error";
 			this.logger.error("Failed to apply operations from inbox (@remote, @documentId, @jobId, @error)", remote.meta.name, syncOp.documentId, completedJobInfo.id, errorMessage);
-			const error = new ChannelError(ChannelErrorSource.Inbox, /* @__PURE__ */ new Error(`Failed to apply operations: ${errorMessage}`));
-			syncOp.failed(error);
+			syncOp.failed(this.inboxFailure(completedJobInfo.error));
 			remote.channel.deadLetter.add(syncOp);
 		} else syncOp.executed();
 		remote.channel.inbox.remove(syncOp);
@@ -5862,14 +6045,26 @@ var SyncManager = class {
 			}
 			if (this.isShutdown) return;
 			if (completedJobInfo.status === JobStatus.FAILED) {
-				const errorMessage = completedJobInfo.error?.message || "Unknown error";
-				const channelError = new ChannelError(ChannelErrorSource.Inbox, /* @__PURE__ */ new Error(`Failed to apply operations: ${errorMessage}`));
-				syncOp.failed(channelError);
+				syncOp.failed(this.inboxFailure(completedJobInfo.error));
 				remote.channel.deadLetter.add(syncOp);
 			} else syncOp.executed();
 			remote.channel.inbox.remove(syncOp);
 		}
 	}
+	/**
+	* The dead letter for a load job the executor failed.
+	*
+	* The classification is passed explicitly because it cannot be recovered
+	* downstream: the wrapper carries the failure's message, not the failure, so
+	* deriving it from the wrapper's own name would classify every one of these
+	* as unclassified and quarantine the document. A held auth operation must
+	* keep syncing, because reconciling the two policies needs the traffic a
+	* quarantine would stop.
+	*/
+	inboxFailure(error) {
+		const message = error?.message || "Unknown error";
+		return new ChannelError(ChannelErrorSource.Inbox, /* @__PURE__ */ new Error(`Failed to apply operations: ${message}`), classifyJobFailure(error?.name ?? "Error"));
+	}
 	async updateOutbox(remote, ackOrdinal, mode = OutboxMode.Backfill, signal) {
 		const composedSignal = signal ? AbortSignal.any([signal, this.abortController.signal]) : this.abortController.signal;
 		let maxOrdinal = ackOrdinal;
@@ -6769,6 +6964,8 @@ var ReactorBuilder = class {
 	}
 	async buildModule() {
 		if (!this.logger) this.logger = new ConsoleLogger(["reactor"]);
+		validateFeatureFlags(this.executorConfig.featureFlags ?? {}, FLAG_PREREQUISITES);
+		if (this.projectionShardConfig !== void 0 && this.readModelFactories.length > 0) throw new Error("withProjectionShards does not support read models registered through withReadModelFactory; projection workers cannot receive host-only factory dependencies");
 		const resolvedSources = await resolveModelSources(this.documentModelSources);
 		if (this.workerPool) {
 			if (resolvedSources.manifest.length === 0) throw new Error("withWorkerPool requires at least one worker-importable document-model source ({ filePath } or { packageName }).");
@@ -6826,7 +7023,7 @@ var ReactorBuilder = class {
 		await executorManager.start(executorStartCount);
 		const readModelInstances = Array.from(new Set([...this.readModels]));
 		const documentViewConsistencyTracker = new ConsistencyTracker();
-		const documentView = new KyselyDocumentView(database, operationStore, operationIndex, writeCache, documentViewConsistencyTracker);
+		const documentView = new KyselyDocumentView(database, operationStore, operationIndex, writeCache, documentViewConsistencyTracker, this.executorConfig.featureFlags?.documentDecisions ?? false);
 		try {
 			await documentView.init();
 		} catch (error) {
@@ -6852,6 +7049,7 @@ var ReactorBuilder = class {
 		}
 		for (const factory of this.readModelFactories) {
 			const readModel = await factory({
+				documentModelRegistry,
 				operationIndex,
 				writeCache,
 				processorManagerConsistencyTracker
@@ -6962,9 +7160,9 @@ var ReactorBuilder = class {
 	*/
 	async createDefaultWorkerFactory(numWorkers, db, signatureVerifier) {
 		const [{ WorkerHandle }, { createThreadTransport }, { workerEntryPath }] = await Promise.all([
-			import("./worker-handle-B1w03nRA.js"),
+			import("./worker-handle-CrERzl8s.js"),
 			import("./transport-ByGviWdZ.js"),
-			import("./worker-DBJOv8Gp.js").then((n) => n.n)
+			import("./worker-jEJW6_j7.js").then((n) => n.n)
 		]);
 		const poolConfig = {
 			enabled: true,
@@ -6985,7 +7183,8 @@ var ReactorBuilder = class {
 					poolConfig,
 					db,
 					signatureVerifier,
-					models
+					models,
+					executorConfig: this.executorConfig
 				},
 				logger,
 				poolInstrumentation
@@ -7225,6 +7424,43 @@ function driveIdFromUrl(url) {
 	return url.split("/").pop() ?? "";
 }
 //#endregion
+//#region src/decision/stream-order.ts
+/**
+* The first pair of effective operations whose stored order contradicts their
+* timestamps, or undefined when the stream is in position order.
+*
+* Such a stream cannot be walked, and the auth stream is never reshuffled once
+* the monotonic rule is on, so run this before enabling enforcement on a fleet.
+*
+* `requireStrict` additionally rejects a tie, which is what the auth stream's
+* monotonic rule requires and what the walk alone does not care about.
+*/
+function firstOutOfOrderPair(operations, options) {
+	const requireStrict = options?.requireStrict ?? false;
+	const effective = garbageCollect(sortOperations([...operations]));
+	for (let i = 1; i < effective.length; i++) {
+		const previous = effective[i - 1];
+		const current = effective[i];
+		const previousAt = Date.parse(previous.timestampUtcMs);
+		const currentAt = Date.parse(current.timestampUtcMs);
+		if (currentAt < previousAt) return {
+			previous,
+			current,
+			kind: "descending"
+		};
+		if (requireStrict && currentAt === previousAt) return {
+			previous,
+			current,
+			kind: "tied"
+		};
+	}
+}
+//#endregion
+//#region src/read-models/interfaces.ts
+function supportsLiveReadModelRegistration(coordinator) {
+	return "addReadModel" in coordinator && typeof coordinator.addReadModel === "function";
+}
+//#endregion
 //#region src/admin/passthrough-keyframe-store.ts
 const passthroughKeyframeStore = {
 	putKeyframe: () => Promise.resolve(),
@@ -7250,6 +7486,7 @@ var DocumentIntegrityService = class {
 	async validateDocument(documentId, branch = "main", signal) {
 		const keyframeIssues = [];
 		const snapshotIssues = [];
+		const streamOrderIssues = await this.findStreamOrderIssues(documentId, branch, signal);
 		const replayCache = new KyselyWriteCache(passthroughKeyframeStore, this.operationStore, this.documentModelRegistry, {
 			maxDocuments: 1,
 			ringBufferSize: 1,
@@ -7276,9 +7513,10 @@ var DocumentIntegrityService = class {
 		} catch {
 			return {
 				documentId,
-				isConsistent: keyframeIssues.length === 0,
+				isConsistent: keyframeIssues.length === 0 && streamOrderIssues.length === 0,
 				keyframeIssues,
-				snapshotIssues
+				snapshotIssues,
+				streamOrderIssues
 			};
 		}
 		const revisions = await this.operationStore.getRevisions(documentId, branch, signal);
@@ -7304,9 +7542,10 @@ var DocumentIntegrityService = class {
 		}
 		return {
 			documentId,
-			isConsistent: keyframeIssues.length === 0 && snapshotIssues.length === 0,
+			isConsistent: keyframeIssues.length === 0 && snapshotIssues.length === 0 && streamOrderIssues.length === 0,
 			keyframeIssues,
-			snapshotIssues
+			snapshotIssues,
+			streamOrderIssues
 		};
 	}
 	async rebuildKeyframes(documentId, branch = "main", signal) {
@@ -7328,12 +7567,26 @@ var DocumentIntegrityService = class {
 			scopesInvalidated: scopes.length
 		};
 	}
+	async findStreamOrderIssues(documentId, branch, signal) {
+		const scopes = await this.discoverScopes(documentId, branch, signal);
+		const issues = [];
+		for (const scope of scopes) {
+			throwIfAborted(signal);
+			const pair = firstOutOfOrderPair((await this.operationStore.getSince(documentId, scope, branch, -1, void 0, void 0, signal)).results, { requireStrict: scope === "auth" });
+			if (pair !== void 0) issues.push({
+				scope,
+				branch,
+				...pair
+			});
+		}
+		return issues;
+	}
 	async discoverScopes(documentId, branch, signal) {
 		const revisions = await this.operationStore.getRevisions(documentId, branch, signal);
 		return Object.keys(revisions.revision);
 	}
 };
 //#endregion
-export { BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentChangeType, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RevisionMismatchError, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncStatus, SyncStatusTracker, addRelationshipAction, batchOperationsByDocument, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createMutableShutdownStatus, createRelationalDb, deleteDocumentAction, documentActions, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, getMigrationStatus, instrumentPgPool, isDriveAuthError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, removeRelationshipAction, runMigrations, sanitizeArg, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
+export { APPEND_CONDITION_FAILED_PREFIX, AppendConditionFailedError, BaseReadModel, ChannelError, ChannelErrorSource, ChannelScheme, ConsistencyTracker, DEFAULT_DRIVE_CONTAINER_TYPES, DRIVE_AUTH_ERROR_MESSAGES, DefaultSubscriptionErrorHandler, DocumentChangeType, DocumentIntegrityService, DocumentModelRegistry, DocumentModelResolver, DriveClient, DriveCollectionId, DuplicateManifestError, DuplicateModuleError, DuplicateOperationError, EventBus, EventBusAggregateError, GqlRequestChannel, GqlRequestChannelFactory, GqlResponseChannel, GqlResponseChannelFactory, SimpleJobExecutor as InMemoryJobExecutor, SimpleJobExecutor, InMemoryJobTracker, InMemoryQueue, IntervalPollTimer, InvalidModuleError, JobAwaiter, JobExecutorEventTypes, JobStatus, KyselyDocumentIndexer, KyselyDocumentView, KyselyKeyframeStore, KyselyOperationStore, KyselySyncCursorStorage, KyselySyncRemoteStorage, KyselyWriteCache, Mailbox, ModuleNotFoundError, NullDocumentModelResolver, OptimisticLockError, PollBehavior, PollingChannelError, ProcessorManager, PropagationMode, QueueEventTypes, REACTOR_SCHEMA, Reactor, ReactorBuilder, ReactorClient, ReactorClientBuilder, ReactorEventTypes, ReactorSubscriptionManager, ReadModelCoordinator, RelationalDbProcessor, RelationshipChangeType, RetryAccounting, RevisionMismatchError, SimpleJobExecutorManager, SyncBuilder, SyncEventTypes, SyncOperation, SyncOperationAggregateError, SyncOperationStatus, SyncStatus, SyncStatusTracker, addRelationshipAction, authDecisionModel, batchOperationsByDocument, buildDecisionModel, classifyJobFailure, consolidateSyncOperations, createDocumentAction, createForwardingLogger, createMutableShutdownStatus, createRelationalDb, decideAtHead, deleteDocumentAction, documentActions, documentDecisionModel, driveIdFromUrl, envelopesToSyncOperations, errorToInfo, firstOutOfOrderPair, getMigrationStatus, instrumentPgPool, isDriveAuthError, makeConsistencyKey, parseDriveUrl, parsePagingOptions, quarantinesDocument, removeRelationshipAction, runMigrations, sanitizeArg, selectDecisionModel, supportsLiveReadModelRegistration, syncOperationErrorType, trimMailboxFromAckOrdinal, updateRelationshipAction, upgradeDocumentAction, workerEntryPath };
 
 //# sourceMappingURL=index.js.map