@gscdump/engine 0.4.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Harlan Wilton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,53 @@
+## @gscdump/engine
+
+Append-only Parquet/DuckDB storage engine for the gscdump pipeline. Owns the storage runtime, planner, schema, and adapters that were previously bundled into `gscdump`.
+
+Edge consumers stay on [`gscdump`](../gscdump). Anything that needs to read/write Parquet, run the DuckDB executor, or attach a snapshot lives here.
+
+## Install
+
+```bash
+npm install @gscdump/engine
+```
+
+Optional peers (install only what your runtime needs):
+
+- `@duckdb/duckdb-wasm` — DuckDB-WASM executor (browser + node-blocking shim).
+- `hyparquet`, `hyparquet-writer` — pure-JS Parquet codec for environments without a native build.
+
+## Subpath exports
+
+| Subpath | Purpose |
+|---|---|
+| `@gscdump/engine` | Barrel: `createStorageEngine`, codec/executor factories, all storage contracts. |
+| `@gscdump/engine/contracts` | `StorageEngine`, `Row`, `TableName`, `WriteCtx`, `ManifestEntry`, ... |
+| `@gscdump/engine/schema` | `SCHEMAS`, `allTables`, `inferTable`, column metadata. |
+| `@gscdump/engine/planner` | `resolveToSQL`, `enumeratePartitions`, partition planning. |
+| `@gscdump/engine/snapshot` | `SnapshotIndex` contract for hot/cold snapshot files. |
+| `@gscdump/engine/ingest` | GSC row → storage row helpers (`createRowAccumulator`, `transformGscRow`). |
+| `@gscdump/engine/sql` | SQL literal binding helpers (`bindLiterals`, `formatLiteral`). |
+| `@gscdump/engine/node` | Node-only DuckDB handle. |
+| `@gscdump/engine/filesystem` | Node-only `DataSource` + `ManifestStore` adapters. |
+| `@gscdump/engine/http` | Read-only HTTP `DataSource` (signed URLs, Range requests). |
+| `@gscdump/engine/hyparquet` | Pure-JS `ParquetCodec`. |
+| `@gscdump/engine/r2` | Cloudflare R2 `DataSource` (structurally typed against `R2Bucket`). |
+
+## Stability
+
+| Surface | Stability |
+|---|---|
+| `createStorageEngine` and storage contracts (`StorageEngine`, `Row`, `WriteCtx`, ...) | Public |
+| Adapters (`/node`, `/filesystem`, `/http`, `/hyparquet`, `/r2`) | Public |
+| Planner (`resolveToSQL`, `enumeratePartitions`) | Public |
+| Schema (`SCHEMAS`, `allTables`, ...) | Public |
+| Internals reached through `@gscdump/engine/<file>` paths not listed above | Private — may break between minors |
+
+## Related
+
+- [`gscdump`](../gscdump) — REST client + query builder (edge-safe peer dep).
+- [`@gscdump/analysis`](../analysis) — analyzers; consumes `StorageEngine` via `createEngine` factories.
+- [`@gscdump/cli`](../cli) — CLI wrapping engine + analysis.
+
+## License
+
+[MIT](../../LICENSE)

package/dist/adapters/duckdb-node.d.mts

@@ -0,0 +1,19 @@
+import { Row } from "gscdump/contracts";
+interface DuckDBHandle {
+  query: (sql: string, params?: unknown[]) => Promise<Row[]>;
+  registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
+  copyFileToBuffer: (name: string) => Promise<Uint8Array>;
+  dropFiles: (names: string[]) => Promise<void>;
+  /**
+   * Returns a unique path suitable for `COPY TO '…'` + `copyFileToBuffer`.
+   * In Node this is an absolute path under `os.tmpdir()` so DuckDB doesn't
+   * litter the CWD; in browsers/Workers it's a plain virtual-FS name.
+   */
+  makeTempPath: (ext: string) => string;
+}
+interface NodeDuckDBOptions {
+  verbose?: boolean;
+}
+declare function createNodeDuckDBHandle(opts?: NodeDuckDBOptions): DuckDBHandle;
+declare function resetNodeDuckDB(): void;
+export { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB };

package/dist/adapters/duckdb-node.mjs

@@ -0,0 +1,78 @@
+import { createRequire } from "node:module";
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
+function arrowToRows(result) {
+	const r = result;
+	const arr = Array.isArray(r) ? r : typeof r?.toArray === "function" ? r.toArray() : [];
+	if (!arr || arr.length === 0) return [];
+	if (typeof arr[0]?.toJSON === "function") return arr.map((r) => r.toJSON());
+	return arr;
+}
+const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
+let singleton = null;
+function bundles() {
+	return {
+		mvp: {
+			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm"),
+			mainWorker: null
+		},
+		eh: {
+			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-eh.wasm"),
+			mainWorker: null
+		}
+	};
+}
+async function initialize(opts) {
+	const logger = opts.verbose ? new ConsoleLogger() : new VoidLogger();
+	const db = await createDuckDB(bundles(), logger, NODE_RUNTIME);
+	await db.instantiate();
+	return {
+		db,
+		conn: db.connect()
+	};
+}
+function createNodeDuckDBHandle(opts = {}) {
+	if (!singleton) singleton = initialize(opts);
+	return {
+		async query(sql, params) {
+			const { conn } = await singleton;
+			if (!params || params.length === 0) return arrowToRows(conn.query(sql));
+			const stmt = conn.prepare(sql);
+			try {
+				return arrowToRows(stmt.query(...params));
+			} finally {
+				stmt.close();
+			}
+		},
+		async registerFileBuffer(name, bytes) {
+			const { db } = await singleton;
+			db.registerFileBuffer(name, bytes);
+		},
+		async copyFileToBuffer(name) {
+			const { db } = await singleton;
+			return db.copyFileToBuffer(name);
+		},
+		async dropFiles(names) {
+			const { db } = await singleton;
+			for (const name of names) {
+				try {
+					db.dropFile(name);
+				} catch {}
+				try {
+					unlinkSync(name);
+				} catch {}
+			}
+		},
+		makeTempPath(ext) {
+			return join(tmpdir(), `gscdump-${Math.random().toString(36).slice(2, 10)}.${ext}`);
+		}
+	};
+}
+function resetNodeDuckDB() {
+	singleton = null;
+}
+export { createNodeDuckDBHandle, resetNodeDuckDB };

package/dist/adapters/filesystem.d.mts

@@ -0,0 +1,206 @@
+import { TableName } from "gscdump/contracts";
+import { SearchType } from "gscdump/query";
+/**
+ * Compaction tier of a manifest entry. Determines which compactor stage may
+ * pick it up as input:
+ * - `raw`: per-day file produced by `writeDay`. Eligible for raw→d7 merge at 7d.
+ * - `d7`: weekly compaction output. Eligible for d7→d30 merge at 30d.
+ * - `d30`: monthly compaction output (matches the legacy `monthly/` partition
+ *   shape — pre-tier entries are read as `d30`). Eligible for d30→d90 at 90d.
+ * - `d90`: quarterly cold-tier output. Terminal; never recompacted.
+ *
+ * Without an explicit tier, entries written before this field landed default
+ * to `raw` for `daily/` partitions and `d30` for `monthly/` partitions, so
+ * the tiered compactor picks the right inputs without a backfill rewrite.
+ */
+type CompactionTier = 'raw' | 'd7' | 'd30' | 'd90';
+interface ManifestEntry {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+  partition: string;
+  objectKey: string;
+  rowCount: number;
+  bytes: number;
+  createdAt: number;
+  retiredAt?: number;
+  /** Table schema version at write time. Omitted on pre-#27 entries — treat as 1. */
+  schemaVersion?: number;
+  /**
+   * Compaction tier. Omitted on entries written before tiered compaction —
+   * treat as `raw` for `daily/` partitions and `d30` for `monthly/` partitions
+   * (see {@link inferLegacyTier}).
+   */
+  tier?: CompactionTier;
+  /**
+   * GSC search-type this entry covers (web | discover | news | googleNews |
+   * image | video). Omitted on entries written before per-type partitioning
+   * landed — treat as `web` (see {@link inferSearchType}). Compaction merges
+   * only entries with the same searchType.
+   */
+  searchType?: SearchType;
+}
+interface ListLiveFilter {
+  userId: string;
+  siteId?: string;
+  table?: TableName;
+  partitions?: string[];
+  /**
+   * Narrow to a single compaction tier. Tier-aware compaction stages set this
+   * so the store doesn't have to return (and the caller doesn't have to scan)
+   * the entire manifest just to compact the raw cohort. Legacy entries without
+   * an explicit `tier` field match on {@link inferLegacyTier}.
+   */
+  tier?: CompactionTier;
+}
+interface DataSource {
+  read: (key: string, range?: {
+    offset: number;
+    length: number;
+  }, signal?: AbortSignal) => Promise<Uint8Array>;
+  write: (key: string, bytes: Uint8Array) => Promise<void>;
+  delete: (keys: string[]) => Promise<void>;
+  /**
+   * One-shot listing under a prefix. Implementations may cap the number of
+   * returned keys (typically 10k) — callers iterating full tenant space
+   * should prefer `streamList` when available or narrow the prefix.
+   */
+  list: (prefix: string) => Promise<string[]>;
+  /**
+   * Per-key URI probe. Returns a URI string DuckDB's `httpfs` (or an
+   * equivalent engine that fetches its own I/O) can read directly, or
+   * `undefined` if the key isn't URI-resolvable on this backend and the
+   * caller must fall back to `read(key)` for the bytes.
+   *
+   * Contracts:
+   * - When defined, the returned URI MUST yield byte-identical content to
+   *   `read(key)`. Callers rely on this for correctness.
+   * - Backends with a native URI for every key (filesystem: absolute path,
+   *   R2 via `httpfs`: signed URL) may always return a string.
+   * - Backends without a native URI shape (in-memory) omit the method or
+   *   return `undefined` per call.
+   * - Mixed-per-query is allowed: some keys in one query may return a URI,
+   *   others may not; the executor branches per key.
+   */
+  uri?: (key: string) => string | undefined;
+  /**
+   * Optional — probe the byte size of a key without reading it. Used by
+   * the engine to fill in `WriteResult.bytes` when a codec reports 0 or
+   * unknown but the file is non-trivial.
+   */
+  head?: (key: string) => Promise<{
+    bytes: number;
+  } | undefined>;
+  /**
+   * Optional streaming variant of `list`. Implementations that page
+   * backing-store results (R2, S3) should implement this and yield keys
+   * lazily. `list` may return up to an adapter-defined cap (typically
+   * 10k keys); callers iterating full tenant space must prefer
+   * `streamList` when available, or chunk by narrower prefixes.
+   */
+  streamList?: (prefix: string) => AsyncIterable<string>;
+}
+interface WatermarkScope {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+}
+interface Watermark extends WatermarkScope {
+  newestDateSynced: string;
+  oldestDateSynced: string;
+  lastSyncAt: number;
+}
+interface WatermarkFilter {
+  userId: string;
+  siteId?: string;
+  table?: TableName;
+}
+type SyncStateKind = 'pending' | 'inflight' | 'done' | 'failed';
+interface SyncStateScope {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+  date: string;
+  /**
+   * GSC search-type this sync state covers. Omitted = `web` (the legacy
+   * default; matches pre-#5 sync states stored before per-type sync landed).
+   * Lookups must compare via {@link inferSearchType} so a missing field
+   * matches an explicit `'web'` and vice versa.
+   */
+  searchType?: SearchType;
+}
+interface SyncState extends SyncStateScope {
+  state: SyncStateKind;
+  updatedAt: number;
+  attempts: number;
+  error?: string;
+}
+interface SyncStateFilter {
+  userId: string;
+  siteId?: string;
+  table?: TableName;
+  state?: SyncStateKind;
+  searchType?: SearchType;
+}
+interface SyncStateDetail {
+  at?: number;
+  error?: string;
+}
+interface LockScope {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+  partition: string;
+}
+interface PurgeFilter {
+  userId: string;
+  siteId?: string;
+}
+interface ManifestPurgeResult {
+  entriesRemoved: number;
+  watermarksRemoved: number;
+  syncStatesRemoved: number;
+}
+interface ManifestStore {
+  listLive: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
+  listAll: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
+  registerVersion: (entry: ManifestEntry, superseding?: ManifestEntry[]) => Promise<void>;
+  registerVersions: (entries: ManifestEntry[], superseding?: ManifestEntry[]) => Promise<void>;
+  listRetired: (olderThan: number) => Promise<ManifestEntry[]>;
+  delete: (entries: ManifestEntry[]) => Promise<void>;
+  getWatermarks: (filter: WatermarkFilter) => Promise<Watermark[]>;
+  bumpWatermark: (scope: WatermarkScope, date: string, at?: number) => Promise<void>;
+  getSyncStates: (filter: SyncStateFilter) => Promise<SyncState[]>;
+  setSyncState: (scope: SyncStateScope, state: SyncStateKind, detail?: SyncStateDetail) => Promise<void>;
+  /**
+   * Serialize concurrent writers against the same scope. Held across the
+   * write+register window so GC (orphan sweep) won't delete bytes that are
+   * midway between `dataSource.write` and `manifestStore.registerVersion`.
+   * Scope = tenant × table × partition.
+   */
+  withLock: <T>(scope: LockScope, fn: () => Promise<T>) => Promise<T>;
+  /**
+   * GDPR-grade tenant purge. Removes every manifest entry, watermark, and
+   * sync-state record matching the filter. Does NOT touch the underlying
+   * data-source bytes; callers (typically {@link StorageEngine.purgeTenant})
+   * must sweep the tenant prefix separately before invoking this so that
+   * mid-flight failures can't leave orphan parquet with no manifest record.
+   *
+   * On stores with CAS-backed sharding (R2 manifest) this may issue one
+   * mutation per shard. On read-only stores (HTTP) this throws.
+   */
+  purgeTenant: (filter: PurgeFilter) => Promise<ManifestPurgeResult>;
+}
+interface FilesystemDataSourceOptions {
+  rootDir: string;
+}
+declare function createFilesystemDataSource(opts: FilesystemDataSourceOptions): DataSource;
+interface FilesystemManifestStoreOptions {
+  path: string;
+}
+declare function createFilesystemManifestStore(opts: FilesystemManifestStoreOptions): ManifestStore;
+declare function filesystemStats(rootDir: string): Promise<{
+  files: number;
+  bytes: number;
+}>;
+export { FilesystemDataSourceOptions, FilesystemManifestStoreOptions, createFilesystemDataSource, createFilesystemManifestStore, filesystemStats };

package/dist/adapters/filesystem.mjs

@@ -0,0 +1,320 @@
+import { Buffer } from "node:buffer";
+import { randomBytes } from "node:crypto";
+import { mkdir, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { lock } from "proper-lockfile";
+function inferSearchType(entry) {
+	return entry.searchType ?? "web";
+}
+function inferLegacyTier(entry) {
+	if (entry.tier !== void 0) return entry.tier;
+	if (entry.partition.startsWith("daily/")) return "raw";
+	if (entry.partition.startsWith("monthly/")) return "d30";
+}
+function createFilesystemDataSource(opts) {
+	const root = resolve(opts.rootDir);
+	function pathFor(key) {
+		const resolved = resolve(root, key);
+		if (!resolved.startsWith(`${root}/`) && resolved !== root) throw new Error(`path escapes root: ${key}`);
+		return resolved;
+	}
+	return {
+		async read(key, range, signal) {
+			const bytes = await readFile(pathFor(key), { signal });
+			if (!range) return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+			const sliced = bytes.subarray(range.offset, range.offset + range.length);
+			return new Uint8Array(sliced.buffer, sliced.byteOffset, sliced.byteLength);
+		},
+		async write(key, bytes) {
+			const path = pathFor(key);
+			await mkdir(dirname(path), { recursive: true });
+			await writeFile(path, Buffer.from(bytes));
+		},
+		async delete(keys) {
+			await Promise.all(keys.map(async (k) => {
+				await rm(pathFor(k), { force: true });
+			}));
+		},
+		async list(prefix) {
+			const full = resolve(root, prefix);
+			const out = [];
+			await walk(full, out);
+			return out.map((p) => p.slice(root.length + 1));
+		},
+		async *streamList(prefix) {
+			const full = resolve(root, prefix);
+			for await (const p of walkStream(full)) yield p.slice(root.length + 1);
+		},
+		async head(key) {
+			return stat(pathFor(key)).then((s) => ({ bytes: s.size }), (err) => {
+				if (err.code === "ENOENT") return void 0;
+				throw err;
+			});
+		},
+		uri(key) {
+			return pathFor(key);
+		}
+	};
+}
+async function* walkStream(dir) {
+	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
+		if (err.code === "ENOENT") return [];
+		throw err;
+	});
+	for (const entry of entries) {
+		const p = join(dir, String(entry.name));
+		if (entry.isDirectory()) yield* walkStream(p);
+		else yield p;
+	}
+}
+async function walk(dir, out) {
+	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
+		if (err.code === "ENOENT") return [];
+		throw err;
+	});
+	for (const entry of entries) {
+		const p = join(dir, String(entry.name));
+		if (entry.isDirectory()) await walk(p, out);
+		else out.push(p);
+	}
+}
+function watermarkKey(w) {
+	return `${w.userId}|${w.siteId ?? ""}|${w.table}`;
+}
+function matchesWatermarkFilter(w, filter) {
+	if (w.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && w.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && w.table !== filter.table) return false;
+	return true;
+}
+function syncStateKey(s) {
+	return `${s.userId}|${s.siteId ?? ""}|${s.table}|${s.date}|${inferSearchType(s)}`;
+}
+function matchesSyncStateFilter(s, filter) {
+	if (s.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && s.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && s.table !== filter.table) return false;
+	if (filter.state !== void 0 && s.state !== filter.state) return false;
+	if (filter.searchType !== void 0 && inferSearchType(s) !== filter.searchType) return false;
+	return true;
+}
+function mergeSyncState(existing, scope, state, detail) {
+	const at = detail?.at ?? Date.now();
+	const attemptsBump = state === "inflight" ? 1 : 0;
+	if (!existing) return {
+		userId: scope.userId,
+		siteId: scope.siteId,
+		table: scope.table,
+		date: scope.date,
+		state,
+		updatedAt: at,
+		attempts: attemptsBump,
+		error: detail?.error,
+		...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
+	};
+	return {
+		...existing,
+		state,
+		updatedAt: at,
+		attempts: existing.attempts + attemptsBump,
+		error: state === "done" ? void 0 : detail?.error ?? existing.error
+	};
+}
+function matchesFilter(entry, filter) {
+	if (entry.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && entry.table !== filter.table) return false;
+	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
+	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
+	return true;
+}
+function lockFileFor(locksDir, scope) {
+	return join(locksDir, `${`${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`.replace(/[^\w.-]/g, "_")}.lock`);
+}
+function createFilesystemManifestStore(opts) {
+	const manifestPath = resolve(opts.path);
+	const locksDir = join(dirname(manifestPath), "locks");
+	async function load() {
+		const content = await readFile(manifestPath, "utf8").catch((err) => {
+			if (err.code === "ENOENT") return null;
+			throw err;
+		});
+		if (content === null) return {
+			version: 1,
+			entries: []
+		};
+		const parsed = JSON.parse(content);
+		if (parsed.version !== 1) throw new Error(`unsupported manifest version ${parsed.version}`);
+		return parsed;
+	}
+	async function save(data) {
+		await mkdir(dirname(manifestPath), { recursive: true });
+		const tmp = `${manifestPath}.${randomBytes(6).toString("hex")}.tmp`;
+		await writeFile(tmp, JSON.stringify(data), "utf8");
+		await rename(tmp, manifestPath).catch(async (err) => {
+			await unlink(tmp).catch(() => {});
+			throw err;
+		});
+	}
+	const queue = [];
+	let running = false;
+	function enqueue(fn) {
+		return new Promise((resolvePromise, rejectPromise) => {
+			queue.push(async () => {
+				await fn().then(resolvePromise, rejectPromise);
+			});
+			drain();
+		});
+	}
+	async function drain() {
+		if (running) return;
+		running = true;
+		while (queue.length > 0) await queue.shift()().catch(() => {});
+		running = false;
+	}
+	function entryKey(e) {
+		return e.objectKey;
+	}
+	async function registerVersionsImpl(newEntries, superseding) {
+		const data = await load();
+		const supersededAt = newEntries[0]?.createdAt ?? Date.now();
+		const byKey = new Map(data.entries.map((e) => [entryKey(e), e]));
+		if (superseding) for (const s of superseding) {
+			const existing = byKey.get(entryKey(s));
+			if (existing && existing.retiredAt === void 0) byKey.set(entryKey(s), {
+				...existing,
+				retiredAt: supersededAt
+			});
+		}
+		for (const e of newEntries) byKey.set(entryKey(e), e);
+		data.entries = Array.from(byKey.values());
+		await save(data);
+	}
+	return {
+		async listLive(filter) {
+			return (await load()).entries.filter((e) => e.retiredAt === void 0 && matchesFilter(e, filter));
+		},
+		async listAll(filter) {
+			return (await load()).entries.filter((e) => matchesFilter(e, filter));
+		},
+		async registerVersion(entry, superseding) {
+			return enqueue(() => registerVersionsImpl([entry], superseding));
+		},
+		async registerVersions(entries, superseding) {
+			return enqueue(() => registerVersionsImpl(entries, superseding));
+		},
+		async listRetired(olderThan) {
+			return (await load()).entries.filter((e) => e.retiredAt !== void 0 && e.retiredAt <= olderThan);
+		},
+		async delete(toDelete) {
+			return enqueue(async () => {
+				const data = await load();
+				const toDeleteKeys = new Set(toDelete.map(entryKey));
+				data.entries = data.entries.filter((e) => !toDeleteKeys.has(entryKey(e)));
+				await save(data);
+			});
+		},
+		async getWatermarks(filter) {
+			return ((await load()).watermarks ?? []).filter((w) => matchesWatermarkFilter(w, filter));
+		},
+		async getSyncStates(filter) {
+			return ((await load()).syncStates ?? []).filter((s) => matchesSyncStateFilter(s, filter));
+		},
+		async setSyncState(scope, state, detail) {
+			return enqueue(async () => {
+				const data = await load();
+				const key = syncStateKey(scope);
+				const byKey = new Map((data.syncStates ?? []).map((s) => [syncStateKey(s), s]));
+				byKey.set(key, mergeSyncState(byKey.get(key), scope, state, detail));
+				data.syncStates = Array.from(byKey.values());
+				await save(data);
+			});
+		},
+		async withLock(scope, fn) {
+			await mkdir(locksDir, { recursive: true });
+			const path = lockFileFor(locksDir, scope);
+			await writeFile(path, "", { flag: "a" });
+			const release = await lock(path, {
+				realpath: false,
+				stale: 3e4,
+				retries: {
+					retries: 20,
+					minTimeout: 50,
+					maxTimeout: 500,
+					factor: 1.5
+				}
+			});
+			return await fn().finally(() => release().catch(() => {}));
+		},
+		async purgeTenant(filter) {
+			return enqueue(async () => {
+				const data = await load();
+				const matches = (r) => r.userId === filter.userId && (filter.siteId === void 0 || r.siteId === filter.siteId);
+				const before = {
+					entries: data.entries.length,
+					watermarks: (data.watermarks ?? []).length,
+					syncStates: (data.syncStates ?? []).length
+				};
+				data.entries = data.entries.filter((e) => !matches(e));
+				data.watermarks = (data.watermarks ?? []).filter((w) => !matches(w));
+				data.syncStates = (data.syncStates ?? []).filter((s) => !matches(s));
+				await save(data);
+				return {
+					entriesRemoved: before.entries - data.entries.length,
+					watermarksRemoved: before.watermarks - data.watermarks.length,
+					syncStatesRemoved: before.syncStates - data.syncStates.length
+				};
+			});
+		},
+		async bumpWatermark(scope, date, at) {
+			return enqueue(async () => {
+				const data = await load();
+				const key = watermarkKey(scope);
+				const byKey = new Map((data.watermarks ?? []).map((w) => [watermarkKey(w), w]));
+				const existing = byKey.get(key);
+				const nowMs = at ?? Date.now();
+				const next = existing ? {
+					...existing,
+					newestDateSynced: date > existing.newestDateSynced ? date : existing.newestDateSynced,
+					oldestDateSynced: date < existing.oldestDateSynced ? date : existing.oldestDateSynced,
+					lastSyncAt: nowMs > existing.lastSyncAt ? nowMs : existing.lastSyncAt
+				} : {
+					userId: scope.userId,
+					siteId: scope.siteId,
+					table: scope.table,
+					newestDateSynced: date,
+					oldestDateSynced: date,
+					lastSyncAt: nowMs
+				};
+				byKey.set(key, next);
+				data.watermarks = Array.from(byKey.values());
+				await save(data);
+			});
+		}
+	};
+}
+async function filesystemStats(rootDir) {
+	const keys = [];
+	await walkForStats(resolve(rootDir), keys);
+	let bytes = 0;
+	for (const k of keys) {
+		const s = await stat(k);
+		bytes += s.size;
+	}
+	return {
+		files: keys.length,
+		bytes
+	};
+}
+async function walkForStats(dir, out) {
+	const entries = await readdir(dir, { withFileTypes: true }).catch((err) => {
+		if (err.code === "ENOENT") return [];
+		throw err;
+	});
+	for (const entry of entries) {
+		const p = join(dir, String(entry.name));
+		if (entry.isDirectory()) await walkForStats(p, out);
+		else out.push(p);
+	}
+}
+export { createFilesystemDataSource, createFilesystemManifestStore, filesystemStats };