@ddd-ts/event-sourcing-firestore 0.0.37 → 0.0.39

package/dist/projection/firestore.projector.mjs

@@ -0,0 +1,533 @@
+import { EventCoordinator } from "./event-coordinator.mjs";
+import { Cursor, EventId, Lock, ProjectedStreamReader } from "@ddd-ts/core";
+import { DefaultConverter } from "@ddd-ts/store-firestore";
+import { FieldValue, Timestamp } from "firebase-admin/firestore";
+import { Mapping, MicrosecondTimestamp, Optional, Shape } from "@ddd-ts/shape";
+
+//#region src/projection/firestore.projector.ts
+const Status = {
+	SUCCESS: "OK",
+	FAILURE: "FAILURE",
+	DEFERRED: "DEFERRED"
+};
+const TaskState = {
+	ENQUEUED: "ENQUEUED",
+	PROCESSED: "PROCESSED",
+	MISSING: "MISSING"
+};
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const RETENTION = MicrosecondTimestamp.MONTH;
+var FirestoreProjector = class {
+	_unclaim = true;
+	constructor(projection, reader, queue, config = {
+		retry: {
+			attempts: 10,
+			minDelay: 10,
+			maxDelay: 200,
+			backoff: 1.5
+		},
+		enqueue: { batchSize: 100 },
+		onProcessError: (error) => {
+			console.error("Error processing event:", error);
+		},
+		onEnqueueError: (error) => {
+			console.error("Error enqueuing tasks:", error);
+		}
+	}) {
+		this.projection = projection;
+		this.reader = reader;
+		this.queue = queue;
+		this.config = config;
+	}
+	async *breathe() {
+		const { attempts, minDelay, maxDelay, backoff } = this.config.retry;
+		for (let i = 0; i < attempts; i++) {
+			const reset = () => {
+				i--;
+			};
+			yield [i, reset];
+			const margin = maxDelay - minDelay;
+			const jitter = Math.random() * margin;
+			await wait((backoff * i + 1) * minDelay + jitter);
+		}
+	}
+	eventCoordinators = /* @__PURE__ */ new Map();
+	getEventCoordinator(checkpointId) {
+		const key = checkpointId.serialize();
+		let coordinator = this.eventCoordinators.get(key);
+		if (!coordinator) {
+			coordinator = new EventCoordinator();
+			coordinator.onEmpty(() => this.eventCoordinators.delete(key));
+			this.eventCoordinators.set(key, coordinator);
+		}
+		return coordinator;
+	}
+	async handle(savedChange) {
+		const checkpointId = this.projection.getCheckpointId(savedChange);
+		const eventCoordinator = this.getEventCoordinator(checkpointId);
+		eventCoordinator.addEvent(savedChange);
+		await eventCoordinator.waitCurrentEvent();
+		if (!eventCoordinator.canProceed(savedChange)) {
+			eventCoordinator.cleanEvent(savedChange);
+			return;
+		}
+		const disposeEventCoordinator = eventCoordinator.start(savedChange);
+		const target = await this.getCursor(savedChange);
+		if (!target) {
+			disposeEventCoordinator();
+			throw new Error(`Cursor not found for event ${savedChange.id.serialize()}`);
+		}
+		const errors = [];
+		for await (const [attempt, reset] of this.breathe()) {
+			const source = this.projection.getSource(savedChange);
+			const [status, message] = await this.attempt(source, checkpointId, target);
+			if (status === Status.DEFERRED) {
+				reset();
+				continue;
+			}
+			if (status === Status.SUCCESS) {
+				await this.queue.cleanup(checkpointId);
+				disposeEventCoordinator();
+				return;
+			}
+			errors.push(message);
+		}
+		disposeEventCoordinator();
+		throw new Error(`Failed to handle event ${savedChange.id.serialize()}: ${errors.join(", ")}`);
+	}
+	async getCursor(savedChange) {
+		return await this.reader.getCursor(savedChange);
+	}
+	async attempt(source, checkpointId, target) {
+		const headCursor = await this.getQueueHead(checkpointId);
+		const isTargetAfterHead = !headCursor || target.isAfter(headCursor);
+		if (isTargetAfterHead) {
+			const [status, message] = await this.enqueue(source, headCursor, checkpointId, target);
+			if (status === Status.DEFERRED) return [Status.DEFERRED, message];
+		}
+		if (!isTargetAfterHead) {
+			const processed = await this.checkIsProcessed(checkpointId, target);
+			if (processed === TaskState.PROCESSED) return [Status.SUCCESS, "Target event already processed"];
+			if (processed === TaskState.MISSING) {
+				const [status, message] = await this.enqueueOne(checkpointId, target);
+				if (status === Status.DEFERRED) return [Status.FAILURE, message];
+			}
+		}
+		const unprocessed = await this.getUnprocessed(checkpointId);
+		if (!unprocessed.length) return [Status.FAILURE, "No unprocessed tasks found"];
+		const batch = Task.batch(unprocessed);
+		if (!batch.length) return [Status.DEFERRED, "No tasks available to claim, deferring"];
+		const claimer = ClaimerId.generate();
+		const [status, message] = await this.claimTasks(checkpointId, claimer, batch);
+		if (status === Status.FAILURE) return [Status.DEFERRED, message];
+		return await this.processEvents(checkpointId, claimer, target.eventId);
+	}
+	async getQueueHead(checkpointId) {
+		return await this.queue.head(checkpointId);
+	}
+	async readSourceStream(source, head, checkpointId, target) {
+		const shard = checkpointId.shard();
+		const headCursor = head;
+		const limit = this.config.enqueue.batchSize;
+		return this.reader.slice(source, shard, headCursor, target, limit);
+	}
+	async enqueue(source, head, checkpointId, target) {
+		const tasks = (await this.readSourceStream(source, head, checkpointId, target)).map((e) => {
+			const settings = this.projection.getTaskSettings(e);
+			return Task.new(e, settings);
+		});
+		return await this.queue.enqueue(checkpointId, tasks);
+	}
+	async enqueueOne(checkpointId, target) {
+		const event = await this.reader.get(target);
+		if (!event) throw new Error(`Event not found for cursor ${target.ref}`);
+		const settings = this.projection.getTaskSettings(event);
+		const task = Task.new(event, settings);
+		return await this.queue.enqueue(checkpointId, [task]);
+	}
+	async checkIsProcessed(checkpointId, cursor) {
+		return await this.queue.isProcessed(checkpointId, cursor);
+	}
+	async getUnprocessed(checkpointId) {
+		return await this.queue.unprocessed(checkpointId);
+	}
+	async claimTasks(checkpointId, claimer, batch) {
+		try {
+			await this.queue.claim(checkpointId, claimer, batch);
+			return [Status.SUCCESS, "Tasks claimed successfully"];
+		} catch (e) {
+			return [Status.FAILURE, e];
+		}
+	}
+	async processEvents(checkpointId, claimer, targetEventId) {
+		const tasks = await this.queue.claimed(checkpointId, claimer);
+		const filtered = (await Promise.all(tasks.map((t) => this.reader.get(t.cursor)))).filter((t) => !!t);
+		if (!filtered.length) return [Status.DEFERRED, "No events to process in claimed tasks"];
+		const context = {
+			onProcessed: this.queue.processed.bind(this.queue, claimer),
+			checkpointId,
+			assertBeforeInsert: this.assertBeforeInsert.bind(this, checkpointId, claimer, filtered)
+		};
+		const hasTarget = tasks.some((t) => t.id.equals(targetEventId));
+		try {
+			if ((await this.projection.process(filtered, context)).some((id) => id?.equals(targetEventId))) return [Status.SUCCESS, "Target event processed successfully"];
+			return [Status.DEFERRED, "Target event not processed yet"];
+		} catch (e) {
+			this.config.onProcessError(e);
+			if (this._unclaim) await this.queue.unclaim(checkpointId, tasks);
+			if (!hasTarget) return [Status.DEFERRED, "Target event not in claimed batch, deferring"];
+			return [Status.FAILURE, e];
+		}
+	}
+	async assertBeforeInsert(checkpointId, claimer, events) {
+		const claimedTasks = await this.queue.claimed(checkpointId, claimer);
+		const claimedTasksMap = new Map(claimedTasks.map((t) => [t.id.serialize(), t]));
+		for (const event of events) {
+			const task = claimedTasksMap.get(event.id.serialize());
+			if (!task) throw new Error(`Task not found for event ${event.id.serialize()} in claimer ${claimer.serialize()}`);
+			if (task.claimIds?.[0] !== claimer.serialize()) throw new Error(`Task ${task.id.serialize()} claimer mismatch: expected ${claimer.serialize()}, found ${task.claimIds?.[0]}`);
+		}
+	}
+};
+var AlreadyEnqueuedError = class extends Error {
+	constructor() {
+		super("Tasks already enqueued");
+		this.name = "AlreadyEnqueuedError";
+	}
+};
+var FirestoreQueueStore = class {
+	converter = new DefaultConverter();
+	collection;
+	constructor(db) {
+		this.db = db;
+		this.collection = db.collection("checkpoints");
+	}
+	timestampToMicroseconds(timestamp) {
+		return new MicrosecondTimestamp(BigInt(timestamp.seconds) * BigInt(1e6) + BigInt(timestamp.nanoseconds) / BigInt(1e3));
+	}
+	microsecondsToTimestamp(microseconds) {
+		const seconds = BigInt(microseconds.micros) / 1000000n;
+		const nanoseconds = BigInt(microseconds.micros) % 1000000n * 1000n;
+		return new Timestamp(Number(seconds), Number(nanoseconds));
+	}
+	async enqueue(checkpointId, tasks) {
+		const batch = this.collection.firestore.batch();
+		for (const task of tasks) {
+			const ref = this.queued(checkpointId, task.id);
+			batch.create(ref, {
+				ref: this.db.doc(task.cursor.ref),
+				...this.converter.toFirestore(task.serialize())
+			});
+		}
+		try {
+			await batch.commit();
+			return [Status.SUCCESS, "Tasks enqueued successfully"];
+		} catch (err) {
+			if (err.code === 6) return [Status.DEFERRED, new AlreadyEnqueuedError()];
+			return [Status.DEFERRED, err];
+		}
+	}
+	async claim(checkpointId, claimer, tasks) {
+		const batch = this.collection.firestore.batch();
+		for (const task of tasks) {
+			if (task.claimIds.length > 0) throw new Error(`Task ${task.id.serialize()} is already claimed by ${task.claimIds.join(", ")}`);
+			const ref = this.queued(checkpointId, task.id);
+			batch.update(ref, {
+				claimer: claimer.serialize(),
+				claimedAt: FieldValue.serverTimestamp(),
+				[`claimsMetadata.${claimer.serialize()}`]: { claimedAt: FieldValue.serverTimestamp() },
+				claimIds: FieldValue.arrayUnion(claimer.serialize()),
+				attempts: FieldValue.increment(1),
+				remaining: FieldValue.increment(-1)
+			}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
+		}
+		await batch.commit();
+	}
+	async head(checkpointId) {
+		const headDoc = (await this.queue(checkpointId).orderBy("occurredAt", "desc").orderBy("revision", "desc").limit(1).get()).docs[0];
+		if (!headDoc) return;
+		const headData = this.converter.fromFirestoreSnapshot(headDoc);
+		if (!headData) return;
+		return headDoc ? Cursor.deserialize({
+			ref: headData.ref,
+			occurredAt: headData.occurredAt,
+			revision: headData.revision,
+			eventId: headData.id
+		}) : void 0;
+	}
+	async unprocessed(checkpointId) {
+		const tasks = (await this.queue(checkpointId).where("processed", "==", false).where("remaining", ">", 0).orderBy("occurredAt", "asc").orderBy("revision", "asc").limit(100).get()).docs.map((doc) => {
+			const data = this.converter.fromFirestoreSnapshot(doc);
+			const timestamp = doc.updateTime ? this.timestampToMicroseconds(doc.updateTime) : void 0;
+			return Task.deserializeWithLastUpdateTime(data, timestamp);
+		});
+		const expiredTasks = [];
+		for (const task of tasks) {
+			const originalClaimIds = task.claimIds;
+			task.checkTimeout();
+			if (originalClaimIds.length > task.claimIds.length) expiredTasks.push(task);
+		}
+		if (expiredTasks.length > 0) {
+			const batch = this.collection.firestore.batch();
+			for (const task of expiredTasks) {
+				const ref = this.queued(checkpointId, task.id);
+				batch.update(ref, {
+					claimer: FieldValue.delete(),
+					claimedAt: FieldValue.delete(),
+					claimIds: task.claimIds
+				}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
+			}
+			await batch.commit();
+		}
+		return tasks;
+	}
+	async claimed(checkpointId, claimer) {
+		return (await this.queue(checkpointId).where("claimIds", "array-contains", claimer.serialize()).orderBy("occurredAt", "asc").orderBy("revision", "asc").get()).docs.map((doc) => {
+			const data = this.converter.fromFirestoreSnapshot(doc);
+			const timestamp = doc.updateTime ? this.timestampToMicroseconds(doc.updateTime) : void 0;
+			return Task.deserializeWithLastUpdateTime(data, timestamp);
+		}).filter((task) => task.claimIds[0] === claimer.serialize());
+	}
+	async unclaim(checkpointId, tasks) {
+		const batch = this.collection.firestore.batch();
+		for (const task of tasks) {
+			const ref = this.queued(checkpointId, task.id);
+			batch.update(ref, {
+				claimer: FieldValue.delete(),
+				claimedAt: FieldValue.delete(),
+				claimIds: FieldValue.arrayRemove(task.currentClaimId)
+			}, { lastUpdateTime: this.microsecondsToTimestamp(task.lastUpdateTime) });
+		}
+		await batch.commit();
+	}
+	/**
+	* If the task exists, then looks for a processed flag.
+	* If not found, check if the cursor is older than the retention time for processed event.
+	* If so, consider it processed.
+	* Otherwise, consider it missing.
+	*/
+	async isProcessed(checkpointId, cursor) {
+		const doc = await this.queued(checkpointId, cursor.eventId).get();
+		if (doc.exists) {
+			const data = doc.data();
+			if (!data) throw new Error("No data in queued document");
+			if (data.processed === true) return TaskState.PROCESSED;
+			return TaskState.ENQUEUED;
+		}
+		const lastRetention = MicrosecondTimestamp.now().sub(RETENTION);
+		if (cursor.isOlderThan(lastRetention)) return TaskState.PROCESSED;
+		return TaskState.MISSING;
+	}
+	checkpoint(id) {
+		return this.collection.doc(id.name).collection("shards");
+	}
+	queue(id) {
+		return this.checkpoint(id).doc(id.shard()).collection("queue");
+	}
+	queued(id, eventId) {
+		return this.queue(id).doc(eventId.serialize());
+	}
+	async processed(claimerId, id, eventIds, context = {}) {
+		const { transaction: trx, batchWriter } = context;
+		if (trx) {
+			for (const eventId of eventIds) {
+				const ref = this.queued(id, eventId);
+				trx.transaction.update(ref, {
+					processed: true,
+					[`claimsMetadata.${claimerId.serialize()}.processedAt`]: FieldValue.serverTimestamp()
+				});
+			}
+			return;
+		}
+		await Promise.all(eventIds.map((eventId) => this.queued(id, eventId).update({
+			processed: true,
+			[`claimsMetadata.${claimerId.serialize()}.processedAt`]: FieldValue.serverTimestamp()
+		})));
+	}
+	async getTailCursor(id) {
+		const tailDoc = (await this.queue(id).where("remaining", ">", 0).orderBy("occurredAt", "asc").orderBy("revision", "asc").limit(1).get()).docs[0];
+		if (!tailDoc) return;
+		const tailData = this.converter.fromFirestoreSnapshot(tailDoc);
+		if (!tailData) return;
+		return tailDoc ? Cursor.deserialize({
+			ref: tailData.ref,
+			occurredAt: tailData.occurredAt,
+			revision: tailData.revision,
+			eventId: tailData.id
+		}) : void 0;
+	}
+	async cleanup(id) {
+		const aMonthAgo = MicrosecondTimestamp.now().sub(MicrosecondTimestamp.WEEK.mult(4));
+		const query = this.queue(id).where("remaining", ">", 0).where("occurredAt", "<", aMonthAgo.serialize()).orderBy("occurredAt", "asc").orderBy("revision", "asc");
+		const TRAIL = 1;
+		const snapshot = await query.get();
+		if (snapshot.size < TRAIL) return;
+		const stopper = snapshot.docs.findIndex((doc) => !doc.data().processed);
+		const cleanable = snapshot.docs.slice(0, stopper);
+		const cleaning = cleanable.slice(0, cleanable.length - TRAIL);
+		if (cleaning.length === 0) return;
+		const batch = this.collection.firestore.batch();
+		for (const queued of cleaning) batch.delete(queued.ref);
+		await batch.commit();
+	}
+	async flush(id) {
+		const stream = this.queue(id).stream();
+		const writer = this.collection.firestore.bulkWriter();
+		for await (const queued of stream) writer.delete(queued.ref);
+		await writer.close();
+	}
+	/**
+	* This method adds a fake processed event to the queue.
+	* It is useful for initializing the tail cursor of a new projection, at the
+	* same time as the projection's initial state is created, reset, or updated.
+	* By default, it will use the current time as the occurredAt timestamp.
+	* You can override this by providing a specific timestamp.
+	*
+	* This ensures that the projection can start processing new events from the
+	* correct point in time, avoiding reprocessing of old events.
+	*/
+	async seed(checkpointId) {
+		const cursor = new Cursor({
+			ref: "seed",
+			occurredAt: MicrosecondTimestamp.now(),
+			revision: 0,
+			eventId: EventId.generate()
+		});
+		const task = new Task({
+			id: EventId.generate(),
+			ref: "Seed",
+			occurredAt: cursor.occurredAt,
+			revision: cursor.revision,
+			attempts: 0,
+			processed: true,
+			claimer: void 0,
+			claimedAt: void 0,
+			claimsMetadata: {},
+			claimIds: [],
+			lock: new Lock({}),
+			remaining: 1,
+			claimTimeout: 0,
+			skipAfter: 0,
+			isolateAfter: 0,
+			lastUpdateTime: void 0
+		});
+		try {
+			const serialized = task.serialize();
+			const converted = this.converter.toFirestore(serialized);
+			await this.queued(checkpointId, task.cursor.eventId).create(converted);
+		} catch (e) {
+			if (!(e instanceof AlreadyEnqueuedError)) throw e;
+		}
+	}
+};
+var ClaimerId = class extends EventId {};
+var Task = class Task extends Shape({
+	id: EventId,
+	ref: String,
+	occurredAt: MicrosecondTimestamp,
+	revision: Number,
+	attempts: Number,
+	processed: Boolean,
+	claimer: Optional(String),
+	claimedAt: Optional(MicrosecondTimestamp),
+	claimsMetadata: Mapping([{
+		claimedAt: MicrosecondTimestamp,
+		processedAt: Optional(MicrosecondTimestamp)
+	}]),
+	claimIds: [String],
+	lock: Lock,
+	skipAfter: Number,
+	remaining: Number,
+	isolateAfter: Number,
+	claimTimeout: Number,
+	lastUpdateTime: Optional(MicrosecondTimestamp)
+}) {
+	get cursor() {
+		return new Cursor({
+			ref: this.ref,
+			occurredAt: this.occurredAt,
+			revision: this.revision,
+			eventId: this.id
+		});
+	}
+	static new(fact, config) {
+		return new Task({
+			id: fact.id,
+			attempts: 0,
+			claimer: void 0,
+			processed: false,
+			claimedAt: void 0,
+			claimsMetadata: {},
+			claimIds: [],
+			lock: config.lock,
+			claimTimeout: config.claimTimeout,
+			skipAfter: config.skipAfter,
+			isolateAfter: config.isolateAfter,
+			remaining: config.skipAfter,
+			ref: fact.ref,
+			revision: fact.revision,
+			occurredAt: fact.occurredAt,
+			lastUpdateTime: void 0
+		});
+	}
+	get currentClaimId() {
+		return this.claimIds.at(-1);
+	}
+	get isProcessing() {
+		return this.currentClaimId !== void 0;
+	}
+	get isProcessed() {
+		return !!this.processed;
+	}
+	get shouldSkip() {
+		return this.attempts > this.skipAfter;
+	}
+	get shouldIsolate() {
+		return this.attempts > this.isolateAfter;
+	}
+	checkTimeout() {
+		const claimer = this.currentClaimId;
+		if (!claimer) return;
+		const claimInfo = this.claimsMetadata[claimer];
+		if (!claimInfo || !claimInfo.claimedAt) return;
+		if (MicrosecondTimestamp.now().micros - claimInfo.claimedAt.micros > BigInt(this.claimTimeout) * 1000n) this.claimIds = this.claimIds.filter((id) => id !== claimer);
+	}
+	static deserializeWithLastUpdateTime(data, timestamp) {
+		return Task.deserialize({
+			...data,
+			lastUpdateTime: timestamp
+		});
+	}
+	static batch(tasks) {
+		const locks = [];
+		const batchLocks = [];
+		const batch = [];
+		for (const task of tasks) {
+			if (task.shouldSkip) continue;
+			if (task.shouldIsolate) {
+				if (batch.length > 0) return batch;
+				batch.push(task);
+				return batch;
+			}
+			if (locks.some((l) => l.restrains(task.lock))) {
+				locks.push(task.lock);
+				continue;
+			}
+			if (batchLocks.some((l) => l.restrains(task.lock, false))) {
+				locks.push(task.lock);
+				continue;
+			}
+			if (task.isProcessed) continue;
+			if (task.isProcessing) {
+				locks.push(task.lock);
+				continue;
+			}
+			batch.push(task);
+			batchLocks.push(task.lock);
+		}
+		return batch;
+	}
+};
+
+//#endregion
+export { AlreadyEnqueuedError, ClaimerId, FirestoreProjector, FirestoreQueueStore, Task };