@gscdump/engine 0.32.12 → 0.33.3

package/dist/_chunks/engine.mjs

@@ -6,6 +6,20 @@ import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mj
 import { sqlEscape } from "../sql-bind.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
+const BUFFER_READ_BATCH_SIZE = 8;
+async function registerBufferedFiles(db, files, dataSource, registered, signal) {
+	for (let i = 0; i < files.length; i += BUFFER_READ_BATCH_SIZE) {
+		signal?.throwIfAborted();
+		const batch = files.slice(i, i + BUFFER_READ_BATCH_SIZE);
+		const buffers = await Promise.all(batch.map((file) => dataSource.read(file.key, void 0, signal)));
+		for (let j = 0; j < batch.length; j++) {
+			signal?.throwIfAborted();
+			const file = batch[j];
+			await db.registerFileBuffer(file.name, buffers[j]);
+			registered.push(file.name);
+		}
+	}
+}
 async function encodeBytes(db, table, rows) {
 	const inName = db.makeTempPath("json");
 	const outName = db.makeTempPath("parquet");
@@ -72,17 +86,19 @@ function createDuckDBCodec(factory) {
 					await db.dropFiles([outName]);
 				}
 			}
-			const inputs = await Promise.all(inputKeys.map((k) => dataSource.read(k)));
 			const inNames = [];
 			const outName = db.makeTempPath("parquet");
 			const registered = [];
-			for (let i = 0; i < inputs.length; i++) {
+			const bufferedInputs = inputKeys.map((key) => {
 				const name = db.makeTempPath("parquet");
-				await db.registerFileBuffer(name, inputs[i]);
 				inNames.push(name);
-				registered.push(name);
-			}
+				return {
+					key,
+					name
+				};
+			});
 			try {
+				await registerBufferedFiles(db, bufferedInputs, dataSource, registered);
 				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
 				await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
 				registered.push(outName);
@@ -127,23 +143,26 @@ function createDuckDBExecutor(factory) {
 		const registered = [];
 		const totalFiles = Object.values(fileKeys).reduce((n, keys) => n + keys.length, 0);
 		const endRegister = profiler?.start("files.register", { files: totalFiles });
-		await Promise.all(Object.entries(fileKeys).map(async ([name, keys]) => {
-			const uris = keys.map((key) => dataSource.uri?.(key));
-			const buffers = await Promise.all(keys.map((key, i) => uris[i] !== void 0 ? Promise.resolve(void 0) : dataSource.read(key, void 0, signal)));
-			const resolved = [];
-			for (let i = 0; i < keys.length; i++) {
-				const uri = uris[i];
-				if (uri !== void 0) resolved.push(uri);
-				else {
-					await db.registerFileBuffer(keys[i], buffers[i]);
-					registered.push(keys[i]);
-					resolved.push(keys[i]);
-				}
-			}
-			placeholders[name] = resolved;
-		}));
-		endRegister?.({ buffered: registered.length });
 		try {
+			await Promise.all(Object.entries(fileKeys).map(async ([name, keys]) => {
+				const resolved = [];
+				const buffered = [];
+				for (let i = 0; i < keys.length; i++) {
+					const key = keys[i];
+					const uri = dataSource.uri?.(key);
+					if (uri !== void 0) resolved.push(uri);
+					else {
+						buffered.push({
+							key,
+							name: key
+						});
+						resolved.push(key);
+					}
+				}
+				await registerBufferedFiles(db, buffered, dataSource, registered, signal);
+				placeholders[name] = resolved;
+			}));
+			endRegister?.({ buffered: registered.length });
 			signal?.throwIfAborted();
 			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);
 			const endQuery = profiler?.start("query.run");
@@ -274,13 +293,23 @@ async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
 	}
 	return { deleted: retired.length + sweptOrphans + hourlyDeleted };
 }
-const PUSHABLE_COLUMN = { query: "query" };
+const PUSHABLE_COLUMN = {
+	country: "country",
+	query: "query",
+	searchAppearance: "searchAppearance"
+};
 function txLeaf(leaf, columns) {
 	if (leaf.operator !== "equals") return null;
 	const column = PUSHABLE_COLUMN[leaf.dimension];
 	if (!column || !columns.has(column)) return null;
 	return { [column]: { $eq: leaf.expression } };
 }
+function combineFilters(parts, groupType) {
+	const first = parts[0];
+	if (!first) return null;
+	if (parts.length === 1) return first;
+	return groupType === "or" ? { $or: [...parts] } : { $and: [...parts] };
+}
 function txExact(node, columns) {
 	const groupType = node._groupType ?? "and";
 	const leafParts = [];
@@ -291,7 +320,7 @@ function txExact(node, columns) {
 	}
 	if (groupType === "or") {
 		if (node._nestedGroups?.length || leafParts.length === 0) return null;
-		return leafParts.length === 1 ? leafParts[0] : { $or: leafParts };
+		return combineFilters(leafParts, "or");
 	}
 	const parts = leafParts;
 	for (const group of node._nestedGroups ?? []) {
@@ -300,7 +329,7 @@ function txExact(node, columns) {
 		parts.push(t);
 	}
 	if (parts.length === 0) return null;
-	return parts.length === 1 ? parts[0] : { $and: parts };
+	return combineFilters(parts, "and");
 }
 function extractParquetPushdown(state, table) {
 	const filter = state?.filter;
@@ -318,7 +347,7 @@ function extractParquetPushdown(state, table) {
 		if (t) parts.push(t);
 	}
 	if (parts.length === 0) return void 0;
-	return parts.length === 1 ? parts[0] : { $and: parts };
+	return combineFilters(parts, "and") ?? void 0;
 }
 const URL_PURGE_TABLES = ["pages", "page_queries"];
 const MAX_DAY_BYTES = 100 * 1024 * 1024;

package/dist/_chunks/entities.mjs

@@ -737,6 +737,7 @@ function createSitemapStore(opts) {
 				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
 				if (!m) continue;
 				const date = m[1];
+				if (!date) continue;
 				if (from && date < from) continue;
 				if (to && date > to) continue;
 				const bytes = await readOptional(ds, key);
@@ -763,9 +764,11 @@ function createSitemapStore(opts) {
 			for (const key of deltaKeys) {
 				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
 				if (!m) continue;
-				const list = deltasByFeed.get(m[2]) ?? [];
+				const feedpathHash = m[2];
+				if (!feedpathHash) continue;
+				const list = deltasByFeed.get(feedpathHash) ?? [];
 				list.push(key);
-				deltasByFeed.set(m[2], list);
+				deltasByFeed.set(feedpathHash, list);
 			}
 			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
 				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);

package/dist/_chunks/manifest-store-utils.mjs

@@ -0,0 +1,56 @@
+import { inferLegacyTier, inferSearchType } from "./layout.mjs";
+function manifestEntryKey(entry) {
+	return entry.objectKey;
+}
+function watermarkKey(watermark) {
+	return `${watermark.userId}|${watermark.siteId ?? ""}|${watermark.table}`;
+}
+function syncStateKey(state) {
+	return `${state.userId}|${state.siteId ?? ""}|${state.table}|${state.date}|${inferSearchType(state)}`;
+}
+function matchesManifestEntryFilter(entry, filter, options = {}) {
+	if (!options.ignoreUserId && entry.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && entry.table !== filter.table) return false;
+	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
+	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
+	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
+	return true;
+}
+function matchesWatermarkFilter(watermark, filter, options = {}) {
+	if (!options.ignoreUserId && watermark.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && watermark.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && watermark.table !== filter.table) return false;
+	return true;
+}
+function matchesSyncStateFilter(state, filter, options = {}) {
+	if (!options.ignoreUserId && state.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && state.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && state.table !== filter.table) return false;
+	if (filter.state !== void 0 && state.state !== filter.state) return false;
+	if (filter.searchType !== void 0 && inferSearchType(state) !== filter.searchType) return false;
+	return true;
+}
+function mergeSyncState(existing, scope, state, detail) {
+	const at = detail?.at ?? Date.now();
+	const attemptsBump = state === "inflight" ? 1 : 0;
+	if (!existing) return {
+		userId: scope.userId,
+		siteId: scope.siteId,
+		table: scope.table,
+		date: scope.date,
+		state,
+		updatedAt: at,
+		attempts: attemptsBump,
+		error: detail?.error,
+		...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
+	};
+	return {
+		...existing,
+		state,
+		updatedAt: at,
+		attempts: existing.attempts + attemptsBump,
+		error: state === "done" ? void 0 : detail?.error ?? existing.error
+	};
+}
+export { manifestEntryKey, matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter, mergeSyncState, syncStateKey, watermarkKey };

package/dist/_chunks/pg-adapter.d.mts

@@ -15,11 +15,25 @@ declare const pgResolverAdapter: ResolverAdapter<PgTableKey>;
  */
 interface ResolverAdapterOptions {
   /**
-   * Opt-in canonical-primary correctness: fold NULL/'' `query_canonical` back
-   * to the raw `query` so canonical is a total GROUP BY / join key. Default
-   * false preserves the legacy raw-column behaviour. See ADR-0018.
+   * Deprecated compatibility flag. Fact tables no longer carry
+   * `query_canonical`; canonical reads derive from `query_dim`, or from a
+   * canonical rollup relation when `queryCanonicalSource: 'column'` is set.
    */
   canonicalFallback?: boolean;
+  /**
+   * `queryDim` reads canonical from a joined query dimension. `column` is only
+   * for derived canonical rollup relations whose primary relation already
+   * carries a null-free `query_canonical` output column.
+   */
+  queryCanonicalSource?: 'queryDim' | 'column';
+}
+interface R2SqlResolverAdapterOptions extends ResolverAdapterOptions {
+  /**
+   * R2 SQL string partition equality can undercount on identity partitions;
+   * string-encoded catalogs use CONCAT(col, '') in partition predicates. Int
+   * catalogs do not need the workaround and keep bare equality for pruning.
+   */
+  partitionKeyEncoding?: 'string' | 'int';
 }
 declare function createParquetResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
 /**
@@ -33,4 +47,16 @@ declare function createParquetResolverAdapter(options?: ResolverAdapterOptions):
  * `${namespace}.pages`) before sending to R2 SQL.
  */
 declare function createIcebergResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
-export { PgTableKey, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter };
+/**
+ * R2 SQL adapter for the Iceberg fact tables.
+ *
+ * It shares the multi-tenant Iceberg schema with `createIcebergResolverAdapter`
+ * but models R2 SQL's narrower execution surface: no window-total plans and no
+ * comparison joins. Int-partition catalogs are the default and emit bare
+ * equality predicates for pruning. Legacy string-partition catalogs must pass
+ * `partitionKeyEncoding: 'string'` to emit `CONCAT(partition_col, '') = ?`,
+ * working around R2 SQL's partition-string equality undercount while preserving
+ * bound params.
+ */
+declare function createR2SqlResolverAdapter(options?: R2SqlResolverAdapterOptions): ResolverAdapter<PgTableKey>;
+export { PgTableKey, R2SqlResolverAdapterOptions, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter };