@gscdump/engine 0.40.2 → 1.0.0

package/dist/adapters/r2-manifest.mjs

@@ -1,364 +0,0 @@
-import { inferSearchType } from "../_chunks/layout.mjs";
-import { engineErrorToException, engineErrors } from "../errors.mjs";
-import { matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter } from "../_chunks/manifest-store-utils.mjs";
-import { err, ok, unwrapResult } from "gscdump/result";
-const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
-const CAS_BACKOFF_BASE_MS = 5;
-const CAS_BACKOFF_CAP_MS = 250;
-const SHARD_IO_CONCURRENCY = 8;
-const R2_DELETE_BATCH_SIZE = 1e3;
-async function casBackoff(attempt) {
-	const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
-	await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
-}
-async function mapWithConcurrency(items, concurrency, fn) {
-	if (items.length === 0) return [];
-	const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency)));
-	const results = Array.from({ length: items.length }, () => void 0);
-	let nextIndex = 0;
-	async function worker() {
-		while (true) {
-			const index = nextIndex++;
-			if (index >= items.length) return;
-			results[index] = await fn(items[index], index);
-		}
-	}
-	await Promise.all(Array.from({ length: workerCount }, worker));
-	return results;
-}
-function defaultSnapshotId() {
-	return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
-}
-function shardPrefix(userId, siteId, table) {
-	return `u_${userId}/manifest/${siteId}/${table}/`;
-}
-function headKey(userId, siteId, table) {
-	return `${shardPrefix(userId, siteId, table)}HEAD`;
-}
-function snapshotKey(userId, siteId, table, snapshotId) {
-	return `${shardPrefix(userId, siteId, table)}v${snapshotId}.json`;
-}
-function emptySnapshot() {
-	return {
-		version: 1,
-		entries: [],
-		watermarks: [],
-		syncStates: []
-	};
-}
-function shardScopesFromEntries(entries) {
-	const out = /* @__PURE__ */ new Set();
-	for (const e of entries) {
-		if (e.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId; cross-site entries are unshardable");
-		out.add(`${e.siteId}\0${e.table}`);
-	}
-	return out;
-}
-function createR2ManifestStore(opts) {
-	const { bucket, userId } = opts;
-	const newSnapshotId = opts.newSnapshotId ?? defaultSnapshotId;
-	const now = opts.now ?? (() => Date.now());
-	const maxRetries = opts.maxRetries ?? 16;
-	const onEvent = opts.onEvent;
-	async function readShard(siteId, table) {
-		const head = await bucket.get(headKey(userId, siteId, table));
-		if (!head) return {
-			snapshot: emptySnapshot(),
-			headEtag: void 0
-		};
-		const snapshotId = (await head.text()).trim();
-		if (!snapshotId) return {
-			snapshot: emptySnapshot(),
-			headEtag: head.etag
-		};
-		const snap = await bucket.get(snapshotKey(userId, siteId, table, snapshotId));
-		if (!snap) return {
-			snapshot: emptySnapshot(),
-			headEtag: head.etag
-		};
-		const parsed = JSON.parse(await snap.text());
-		if (parsed.version !== 1) throw new Error(`unsupported manifest snapshot version: ${parsed.version}`);
-		return {
-			snapshot: parsed,
-			headEtag: head.etag
-		};
-	}
-	async function writeShardResult(siteId, table, snapshot, headEtag, attempt) {
-		const id = newSnapshotId();
-		const snapKey = snapshotKey(userId, siteId, table, id);
-		await bucket.put(snapKey, JSON.stringify(snapshot));
-		const conditional = headEtag ? { onlyIf: { etagMatches: headEtag } } : { onlyIf: { etagDoesNotMatch: "*" } };
-		return await bucket.put(headKey(userId, siteId, table), id, conditional) !== null ? ok(void 0) : err(engineErrors.manifestCasRoundLost(siteId, table, attempt));
-	}
-	async function mutateShardResult(siteId, table, mutate) {
-		let attempt = 0;
-		while (attempt < maxRetries) {
-			onEvent?.({
-				kind: "cas-attempt",
-				siteId,
-				table,
-				attempt
-			});
-			const { snapshot, headEtag } = await readShard(siteId, table);
-			await mutate(snapshot);
-			const round = await writeShardResult(siteId, table, snapshot, headEtag, attempt);
-			if (round.ok) {
-				onEvent?.({
-					kind: "cas-committed",
-					siteId,
-					table,
-					attempts: attempt + 1
-				});
-				return round;
-			}
-			onEvent?.({
-				kind: "cas-rejected",
-				siteId,
-				table,
-				attempt
-			});
-			attempt++;
-			if (attempt < maxRetries) await casBackoff(attempt);
-		}
-		return err(engineErrors.manifestCasExhausted(siteId, table, maxRetries));
-	}
-	async function mutateShard(siteId, table, mutate) {
-		return unwrapResult(await mutateShardResult(siteId, table, mutate), engineErrorToException);
-	}
-	async function listShards(siteId) {
-		const shards = [];
-		let cursor;
-		const prefix = siteId === void 0 ? `u_${userId}/manifest/` : `u_${userId}/manifest/${siteId}/`;
-		do {
-			const res = await bucket.list({
-				prefix,
-				cursor,
-				limit: 1e3
-			});
-			for (const obj of res.objects) {
-				const m = SHARD_RE.exec(obj.key);
-				if (m?.groups) shards.push({
-					siteId: m.groups.siteId,
-					table: m.groups.table
-				});
-			}
-			cursor = res.truncated ? res.cursor : void 0;
-		} while (cursor);
-		return shards;
-	}
-	async function shardsForFilter(filter) {
-		if (filter.siteId !== void 0 && filter.table !== void 0) return [{
-			siteId: filter.siteId,
-			table: filter.table
-		}];
-		return (await listShards(filter.siteId)).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
-	}
-	function assertScopedUser(got, op) {
-		if (got !== userId) throw new Error(`${op}: R2 manifest store is scoped to userId=${userId}, got ${got}`);
-	}
-	async function readEntriesAcrossShards(filter, includeRetired) {
-		assertScopedUser(filter.userId, includeRetired ? "listAll" : "listLive");
-		return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-			const { snapshot } = await readShard(siteId, table);
-			const entries = [];
-			for (const entry of snapshot.entries) {
-				if (!includeRetired && entry.retiredAt !== void 0) continue;
-				if (matchesManifestEntryFilter(entry, filter, { ignoreUserId: true })) entries.push(entry);
-			}
-			return entries;
-		})).flat();
-	}
-	function groupBySiteTable(entries) {
-		const out = /* @__PURE__ */ new Map();
-		for (const e of entries) {
-			const key = `${e.siteId}\0${e.table}`;
-			if (!out.has(key)) out.set(key, []);
-			out.get(key).push(e);
-		}
-		return out;
-	}
-	async function registerVersionsImpl(newEntries, superseding) {
-		if (newEntries.length === 0 && (!superseding || superseding.length === 0)) return;
-		const supersededAt = newEntries[0]?.createdAt ?? now();
-		const byShard = /* @__PURE__ */ new Map();
-		function bucket(entry, kind) {
-			assertScopedUser(entry.userId, "registerVersions");
-			if (entry.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId");
-			const key = `${entry.siteId}\0${entry.table}`;
-			let bag = byShard.get(key);
-			if (!bag) {
-				bag = {
-					newEntries: [],
-					superseding: []
-				};
-				byShard.set(key, bag);
-			}
-			if (kind === "new") bag.newEntries.push(entry);
-			else bag.superseding.push(entry);
-		}
-		for (const e of newEntries) bucket(e, "new");
-		if (superseding) for (const e of superseding) bucket(e, "super");
-		await mapWithConcurrency([...byShard], SHARD_IO_CONCURRENCY, async ([shardKey, { newEntries: news, superseding: supers }]) => {
-			const [siteId, table] = shardKey.split("\0");
-			await mutateShard(siteId, table, (snap) => {
-				const byObjectKey = new Map(snap.entries.map((e) => [e.objectKey, e]));
-				for (const s of supers) {
-					const existing = byObjectKey.get(s.objectKey);
-					if (existing && existing.retiredAt === void 0) byObjectKey.set(s.objectKey, {
-						...existing,
-						retiredAt: supersededAt
-					});
-				}
-				for (const n of news) byObjectKey.set(n.objectKey, n);
-				snap.entries = Array.from(byObjectKey.values());
-			});
-		});
-	}
-	return {
-		async listLive(filter) {
-			return readEntriesAcrossShards(filter, false);
-		},
-		async listAll(filter) {
-			return readEntriesAcrossShards(filter, true);
-		},
-		async registerVersion(entry, superseding) {
-			return registerVersionsImpl([entry], superseding);
-		},
-		async registerVersions(entries, superseding) {
-			shardScopesFromEntries(entries);
-			return registerVersionsImpl(entries, superseding);
-		},
-		async listRetired(olderThan) {
-			return (await mapWithConcurrency(await listShards(), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const retired = [];
-				for (const e of snapshot.entries) if (e.retiredAt !== void 0 && e.retiredAt <= olderThan) retired.push(e);
-				return retired;
-			})).flat();
-		},
-		async delete(toDelete) {
-			await mapWithConcurrency([...groupBySiteTable(toDelete)], SHARD_IO_CONCURRENCY, async ([shardKey, entries]) => {
-				const [siteId, table] = shardKey.split("\0");
-				await mutateShard(siteId, table, (snap) => {
-					const drop = new Set(entries.map((e) => e.objectKey));
-					snap.entries = snap.entries.filter((e) => !drop.has(e.objectKey));
-				});
-			});
-		},
-		async getWatermarks(filter) {
-			assertScopedUser(filter.userId, "getWatermarks");
-			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const watermarks = [];
-				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter, { ignoreUserId: true })) watermarks.push(w);
-				return watermarks;
-			})).flat();
-		},
-		async bumpWatermark(scope, date, at) {
-			assertScopedUser(scope.userId, "bumpWatermark");
-			if (scope.siteId === void 0) throw new Error("R2 manifest store requires watermarks to carry siteId");
-			const ts = at ?? now();
-			const scopeSearchType = inferSearchType(scope);
-			await mutateShard(scope.siteId, scope.table, (snap) => {
-				const idx = snap.watermarks.findIndex((w) => w.userId === userId && w.siteId === scope.siteId && w.table === scope.table && inferSearchType(w) === scopeSearchType);
-				if (idx === -1) {
-					snap.watermarks.push({
-						userId,
-						siteId: scope.siteId,
-						table: scope.table,
-						...scope.searchType !== void 0 ? { searchType: scope.searchType } : {},
-						newestDateSynced: date,
-						oldestDateSynced: date,
-						lastSyncAt: ts
-					});
-					return;
-				}
-				const w = snap.watermarks[idx];
-				const newest = date > w.newestDateSynced ? date : w.newestDateSynced;
-				const oldest = date < w.oldestDateSynced ? date : w.oldestDateSynced;
-				const lastSyncAt = ts > w.lastSyncAt ? ts : w.lastSyncAt;
-				snap.watermarks[idx] = {
-					...w,
-					newestDateSynced: newest,
-					oldestDateSynced: oldest,
-					lastSyncAt
-				};
-			});
-		},
-		async getSyncStates(filter) {
-			assertScopedUser(filter.userId, "getSyncStates");
-			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const states = [];
-				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter, { ignoreUserId: true })) states.push(s);
-				return states;
-			})).flat();
-		},
-		async setSyncState(scope, state, detail) {
-			assertScopedUser(scope.userId, "setSyncState");
-			if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");
-			const at = detail?.at ?? now();
-			const scopeSearchType = inferSearchType(scope);
-			await mutateShard(scope.siteId, scope.table, (snap) => {
-				const idx = snap.syncStates.findIndex((s) => s.userId === userId && s.siteId === scope.siteId && s.table === scope.table && s.date === scope.date && inferSearchType(s) === scopeSearchType);
-				if (idx === -1) {
-					snap.syncStates.push({
-						userId,
-						siteId: scope.siteId,
-						table: scope.table,
-						date: scope.date,
-						state,
-						updatedAt: at,
-						attempts: 1,
-						error: detail?.error,
-						...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
-					});
-					return;
-				}
-				const prev = snap.syncStates[idx];
-				const attempts = state === "inflight" && prev.state !== "inflight" ? prev.attempts + 1 : prev.attempts;
-				const error = state === "done" ? void 0 : detail?.error ?? prev.error;
-				snap.syncStates[idx] = {
-					...prev,
-					state,
-					updatedAt: at,
-					attempts,
-					error
-				};
-			});
-		},
-		async withLock(_scope, fn) {
-			return fn();
-		},
-		async purgeTenant(filter) {
-			if (filter.userId !== userId) throw new Error(`purgeTenant: store is scoped to userId=${userId}, got ${filter.userId}`);
-			const purged = await mapWithConcurrency(await shardsForFilter({ siteId: filter.siteId }), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const prefix = shardPrefix(userId, siteId, table);
-				const keys = [];
-				let cursor;
-				do {
-					const res = await bucket.list({
-						prefix,
-						cursor,
-						limit: 1e3
-					});
-					for (const obj of res.objects) keys.push(obj.key);
-					cursor = res.truncated ? res.cursor : void 0;
-				} while (cursor);
-				for (let offset = 0; offset < keys.length; offset += R2_DELETE_BATCH_SIZE) await bucket.delete(keys.slice(offset, offset + R2_DELETE_BATCH_SIZE));
-				return {
-					entriesRemoved: snapshot.entries.length,
-					watermarksRemoved: snapshot.watermarks.length,
-					syncStatesRemoved: snapshot.syncStates.length
-				};
-			});
-			return {
-				entriesRemoved: purged.reduce((sum, result) => sum + result.entriesRemoved, 0),
-				watermarksRemoved: purged.reduce((sum, result) => sum + result.watermarksRemoved, 0),
-				syncStatesRemoved: purged.reduce((sum, result) => sum + result.syncStatesRemoved, 0)
-			};
-		}
-	};
-}
-export { createR2ManifestStore };

package/dist/compaction-public.d.mts

@@ -1,15 +0,0 @@
-import { dedupeOverlappingTiers, splitOverlappingTiers } from "./_chunks/storage.mjs";
-/**
- * Host-policy predicate: true once a table's live raw-daily count crosses the
- * engine's daily→weekly compaction gate. Wraps the internal threshold so hosts
- * decide "is compaction due?" without importing the constant or the counter.
- *
- * Pure — pass the entries the host already fetched (typically via its own cached
- * manifest store, so the hot-path check stays on the host's cache rather than
- * forcing an uncached read through the engine).
- */
-declare function isRawDailyCompactionDue(entries: ReadonlyArray<{
-  tier?: string | null;
-  partition: string;
-}>): boolean;
-export { dedupeOverlappingTiers, isRawDailyCompactionDue, splitOverlappingTiers };

package/dist/compaction-public.mjs

@@ -1,5 +0,0 @@
-import { countRawDailies, dedupeOverlappingTiers, splitOverlappingTiers } from "./_chunks/compaction.mjs";
-function isRawDailyCompactionDue(entries) {
-	return countRawDailies(entries) > 7;
-}
-export { dedupeOverlappingTiers, isRawDailyCompactionDue, splitOverlappingTiers };

package/dist/snapshot.d.mts

@@ -1,2 +0,0 @@
-import { SnapshotIndex } from "./_chunks/snapshot.mjs";
-export { SnapshotIndex };

package/dist/snapshot.mjs

@@ -1 +0,0 @@
-export {};