@gscdump/engine 0.33.0 → 0.33.3

package/dist/_chunks/engine.mjs

@@ -304,6 +304,12 @@ function txLeaf(leaf, columns) {
 	if (!column || !columns.has(column)) return null;
 	return { [column]: { $eq: leaf.expression } };
 }
+function combineFilters(parts, groupType) {
+	const first = parts[0];
+	if (!first) return null;
+	if (parts.length === 1) return first;
+	return groupType === "or" ? { $or: [...parts] } : { $and: [...parts] };
+}
 function txExact(node, columns) {
 	const groupType = node._groupType ?? "and";
 	const leafParts = [];
@@ -314,7 +320,7 @@ function txExact(node, columns) {
 	}
 	if (groupType === "or") {
 		if (node._nestedGroups?.length || leafParts.length === 0) return null;
-		return leafParts.length === 1 ? leafParts[0] : { $or: leafParts };
+		return combineFilters(leafParts, "or");
 	}
 	const parts = leafParts;
 	for (const group of node._nestedGroups ?? []) {
@@ -323,7 +329,7 @@ function txExact(node, columns) {
 		parts.push(t);
 	}
 	if (parts.length === 0) return null;
-	return parts.length === 1 ? parts[0] : { $and: parts };
+	return combineFilters(parts, "and");
 }
 function extractParquetPushdown(state, table) {
 	const filter = state?.filter;
@@ -341,7 +347,7 @@ function extractParquetPushdown(state, table) {
 		if (t) parts.push(t);
 	}
 	if (parts.length === 0) return void 0;
-	return parts.length === 1 ? parts[0] : { $and: parts };
+	return combineFilters(parts, "and") ?? void 0;
 }
 const URL_PURGE_TABLES = ["pages", "page_queries"];
 const MAX_DAY_BYTES = 100 * 1024 * 1024;

package/dist/_chunks/entities.mjs

@@ -737,6 +737,7 @@ function createSitemapStore(opts) {
 				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
 				if (!m) continue;
 				const date = m[1];
+				if (!date) continue;
 				if (from && date < from) continue;
 				if (to && date > to) continue;
 				const bytes = await readOptional(ds, key);
@@ -763,9 +764,11 @@ function createSitemapStore(opts) {
 			for (const key of deltaKeys) {
 				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
 				if (!m) continue;
-				const list = deltasByFeed.get(m[2]) ?? [];
+				const feedpathHash = m[2];
+				if (!feedpathHash) continue;
+				const list = deltasByFeed.get(feedpathHash) ?? [];
 				list.push(key);
-				deltasByFeed.set(m[2], list);
+				deltasByFeed.set(feedpathHash, list);
 			}
 			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
 				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);

package/dist/_chunks/manifest-store-utils.mjs

@@ -0,0 +1,56 @@
+import { inferLegacyTier, inferSearchType } from "./layout.mjs";
+function manifestEntryKey(entry) {
+	return entry.objectKey;
+}
+function watermarkKey(watermark) {
+	return `${watermark.userId}|${watermark.siteId ?? ""}|${watermark.table}`;
+}
+function syncStateKey(state) {
+	return `${state.userId}|${state.siteId ?? ""}|${state.table}|${state.date}|${inferSearchType(state)}`;
+}
+function matchesManifestEntryFilter(entry, filter, options = {}) {
+	if (!options.ignoreUserId && entry.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && entry.table !== filter.table) return false;
+	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
+	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
+	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
+	return true;
+}
+function matchesWatermarkFilter(watermark, filter, options = {}) {
+	if (!options.ignoreUserId && watermark.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && watermark.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && watermark.table !== filter.table) return false;
+	return true;
+}
+function matchesSyncStateFilter(state, filter, options = {}) {
+	if (!options.ignoreUserId && state.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && state.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && state.table !== filter.table) return false;
+	if (filter.state !== void 0 && state.state !== filter.state) return false;
+	if (filter.searchType !== void 0 && inferSearchType(state) !== filter.searchType) return false;
+	return true;
+}
+function mergeSyncState(existing, scope, state, detail) {
+	const at = detail?.at ?? Date.now();
+	const attemptsBump = state === "inflight" ? 1 : 0;
+	if (!existing) return {
+		userId: scope.userId,
+		siteId: scope.siteId,
+		table: scope.table,
+		date: scope.date,
+		state,
+		updatedAt: at,
+		attempts: attemptsBump,
+		error: detail?.error,
+		...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
+	};
+	return {
+		...existing,
+		state,
+		updatedAt: at,
+		attempts: existing.attempts + attemptsBump,
+		error: state === "done" ? void 0 : detail?.error ?? existing.error
+	};
+}
+export { manifestEntryKey, matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter, mergeSyncState, syncStateKey, watermarkKey };

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
+import { manifestEntryKey, matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter, mergeSyncState, syncStateKey, watermarkKey } from "../_chunks/manifest-store-utils.mjs";
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";
@@ -29,13 +29,13 @@ function createFilesystemDataSource(opts) {
 			}));
 		},
 		async list(prefix) {
-			const full = resolve(root, prefix);
+			const full = pathFor(prefix);
 			const out = [];
 			await walk(full, out);
 			return out.map((p) => p.slice(root.length + 1));
 		},
 		async *streamList(prefix) {
-			const full = resolve(root, prefix);
+			const full = pathFor(prefix);
 			for await (const p of walkStream(full)) yield p.slice(root.length + 1);
 		},
 		async head(key) {
@@ -71,57 +71,6 @@ async function walk(dir, out) {
 		else out.push(p);
 	}
 }
-function watermarkKey(w) {
-	return `${w.userId}|${w.siteId ?? ""}|${w.table}`;
-}
-function matchesWatermarkFilter(w, filter) {
-	if (w.userId !== filter.userId) return false;
-	if (filter.siteId !== void 0 && w.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && w.table !== filter.table) return false;
-	return true;
-}
-function syncStateKey(s) {
-	return `${s.userId}|${s.siteId ?? ""}|${s.table}|${s.date}|${inferSearchType(s)}`;
-}
-function matchesSyncStateFilter(s, filter) {
-	if (s.userId !== filter.userId) return false;
-	if (filter.siteId !== void 0 && s.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && s.table !== filter.table) return false;
-	if (filter.state !== void 0 && s.state !== filter.state) return false;
-	if (filter.searchType !== void 0 && inferSearchType(s) !== filter.searchType) return false;
-	return true;
-}
-function mergeSyncState(existing, scope, state, detail) {
-	const at = detail?.at ?? Date.now();
-	const attemptsBump = state === "inflight" ? 1 : 0;
-	if (!existing) return {
-		userId: scope.userId,
-		siteId: scope.siteId,
-		table: scope.table,
-		date: scope.date,
-		state,
-		updatedAt: at,
-		attempts: attemptsBump,
-		error: detail?.error,
-		...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
-	};
-	return {
-		...existing,
-		state,
-		updatedAt: at,
-		attempts: existing.attempts + attemptsBump,
-		error: state === "done" ? void 0 : detail?.error ?? existing.error
-	};
-}
-function matchesFilter(entry, filter) {
-	if (entry.userId !== filter.userId) return false;
-	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && entry.table !== filter.table) return false;
-	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
-	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
-	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
-	return true;
-}
 function lockFileFor(locksDir, scope) {
 	return join(locksDir, `${`${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`.replace(/[^\w.-]/g, "_")}.lock`);
 }
@@ -166,30 +115,27 @@ function createFilesystemManifestStore(opts) {
 		while (queue.length > 0) await queue.shift()().catch(() => {});
 		running = false;
 	}
-	function entryKey(e) {
-		return e.objectKey;
-	}
 	async function registerVersionsImpl(newEntries, superseding) {
 		const data = await load();
 		const supersededAt = newEntries[0]?.createdAt ?? Date.now();
-		const byKey = new Map(data.entries.map((e) => [entryKey(e), e]));
+		const byKey = new Map(data.entries.map((e) => [manifestEntryKey(e), e]));
 		if (superseding) for (const s of superseding) {
-			const existing = byKey.get(entryKey(s));
-			if (existing && existing.retiredAt === void 0) byKey.set(entryKey(s), {
+			const existing = byKey.get(manifestEntryKey(s));
+			if (existing && existing.retiredAt === void 0) byKey.set(manifestEntryKey(s), {
 				...existing,
 				retiredAt: supersededAt
 			});
 		}
-		for (const e of newEntries) byKey.set(entryKey(e), e);
+		for (const e of newEntries) byKey.set(manifestEntryKey(e), e);
 		data.entries = Array.from(byKey.values());
 		await save(data);
 	}
 	return {
 		async listLive(filter) {
-			return (await load()).entries.filter((e) => e.retiredAt === void 0 && matchesFilter(e, filter));
+			return (await load()).entries.filter((e) => e.retiredAt === void 0 && matchesManifestEntryFilter(e, filter));
 		},
 		async listAll(filter) {
-			return (await load()).entries.filter((e) => matchesFilter(e, filter));
+			return (await load()).entries.filter((e) => matchesManifestEntryFilter(e, filter));
 		},
 		async registerVersion(entry, superseding) {
 			return enqueue(() => registerVersionsImpl([entry], superseding));
@@ -203,8 +149,8 @@ function createFilesystemManifestStore(opts) {
 		async delete(toDelete) {
 			return enqueue(async () => {
 				const data = await load();
-				const toDeleteKeys = new Set(toDelete.map(entryKey));
-				data.entries = data.entries.filter((e) => !toDeleteKeys.has(entryKey(e)));
+				const toDeleteKeys = new Set(toDelete.map(manifestEntryKey));
+				data.entries = data.entries.filter((e) => !toDeleteKeys.has(manifestEntryKey(e)));
 				await save(data);
 			});
 		},

package/dist/adapters/hyparquet.mjs

@@ -124,8 +124,11 @@ function naturalKeyFor(table, row) {
 }
 function encodeOrderedRows(rows, columns, rowGroupSize) {
 	const schema = buildWriteSchema(columns);
-	const isDate = columns.map((col) => col.type === "DATE");
-	const types = columns.map((col) => basicTypeFor(col.type));
+	const codecs = columns.map((col) => ({
+		name: col.name,
+		isDate: col.type === "DATE",
+		type: basicTypeFor(col.type)
+	}));
 	const columnSpecs = columns.map((col) => ({
 		name: col.name,
 		nullable: col.nullable,
@@ -134,10 +137,7 @@ function encodeOrderedRows(rows, columns, rowGroupSize) {
 	function* coercedRows() {
 		for (const r of rows) {
 			const out = {};
-			for (let c = 0; c < columns.length; c++) {
-				const name = columns[c].name;
-				out[name] = isDate[c] ? toEpochDays(r[name]) : coerceValue(r[name], types[c]);
-			}
+			for (const codec of codecs) out[codec.name] = codec.isDate ? toEpochDays(r[codec.name]) : coerceValue(r[codec.name], codec.type);
 			yield out;
 		}
 	}

package/dist/adapters/r2-manifest.mjs

@@ -1,5 +1,6 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
+import { inferSearchType } from "../_chunks/layout.mjs";
 import { engineErrorToException, engineErrors } from "../errors.mjs";
+import { matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter } from "../_chunks/manifest-store-utils.mjs";
 import { err, ok, unwrapResult } from "gscdump/result";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 const CAS_BACKOFF_BASE_MS = 5;
@@ -52,26 +53,6 @@ function shardScopesFromEntries(entries) {
 	}
 	return out;
 }
-function matchesEntryFilter(entry, filter) {
-	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && entry.table !== filter.table) return false;
-	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
-	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
-	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
-	return true;
-}
-function matchesWatermarkFilter(w, filter) {
-	if (filter.siteId !== void 0 && w.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && w.table !== filter.table) return false;
-	return true;
-}
-function matchesSyncStateFilter(s, filter) {
-	if (filter.siteId !== void 0 && s.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && s.table !== filter.table) return false;
-	if (filter.state !== void 0 && s.state !== filter.state) return false;
-	if (filter.searchType !== void 0 && inferSearchType(s) !== filter.searchType) return false;
-	return true;
-}
 function createR2ManifestStore(opts) {
 	const { bucket, userId } = opts;
 	const newSnapshotId = opts.newSnapshotId ?? defaultSnapshotId;
@@ -170,13 +151,17 @@ function createR2ManifestStore(opts) {
 		}];
 		return (await listShards()).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
 	}
+	function assertScopedUser(got, op) {
+		if (got !== userId) throw new Error(`${op}: R2 manifest store is scoped to userId=${userId}, got ${got}`);
+	}
 	async function readEntriesAcrossShards(filter, includeRetired) {
+		assertScopedUser(filter.userId, includeRetired ? "listAll" : "listLive");
 		return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 			const { snapshot } = await readShard(siteId, table);
 			const entries = [];
 			for (const entry of snapshot.entries) {
 				if (!includeRetired && entry.retiredAt !== void 0) continue;
-				if (matchesEntryFilter(entry, filter)) entries.push(entry);
+				if (matchesManifestEntryFilter(entry, filter, { ignoreUserId: true })) entries.push(entry);
 			}
 			return entries;
 		})).flat();
@@ -195,6 +180,7 @@ function createR2ManifestStore(opts) {
 		const supersededAt = newEntries[0]?.createdAt ?? now();
 		const byShard = /* @__PURE__ */ new Map();
 		function bucket(entry, kind) {
+			assertScopedUser(entry.userId, "registerVersions");
 			if (entry.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId");
 			const key = `${entry.siteId}\0${entry.table}`;
 			let bag = byShard.get(key);
@@ -258,14 +244,16 @@ function createR2ManifestStore(opts) {
 			});
 		},
 		async getWatermarks(filter) {
+			assertScopedUser(filter.userId, "getWatermarks");
 			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
 				const watermarks = [];
-				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter)) watermarks.push(w);
+				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter, { ignoreUserId: true })) watermarks.push(w);
 				return watermarks;
 			})).flat();
 		},
 		async bumpWatermark(scope, date, at) {
+			assertScopedUser(scope.userId, "bumpWatermark");
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires watermarks to carry siteId");
 			const ts = at ?? now();
 			await mutateShard(scope.siteId, scope.table, (snap) => {
@@ -294,14 +282,16 @@ function createR2ManifestStore(opts) {
 			});
 		},
 		async getSyncStates(filter) {
+			assertScopedUser(filter.userId, "getSyncStates");
 			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
 				const states = [];
-				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter)) states.push(s);
+				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter, { ignoreUserId: true })) states.push(s);
 				return states;
 			})).flat();
 		},
 		async setSyncState(scope, state, detail) {
+			assertScopedUser(scope.userId, "setSyncState");
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");
 			const at = detail?.at ?? now();
 			const scopeSearchType = inferSearchType(scope);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.33.0",
+  "version": "0.33.3",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.33.0",
-    "gscdump": "0.33.0"
+    "@gscdump/contracts": "0.33.3",
+    "gscdump": "0.33.3"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",