@gscdump/engine 1.4.0 → 1.4.2

package/dist/{_chunks/schema.mjs → drizzle-schema.mjs}

@@ -1,4 +1,8 @@
-import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
+import { date } from "drizzle-orm/pg-core/columns/date";
+import { doublePrecision } from "drizzle-orm/pg-core/columns/double-precision";
+import { integer } from "drizzle-orm/pg-core/columns/integer";
+import { varchar } from "drizzle-orm/pg-core/columns/varchar";
+import { pgTable } from "drizzle-orm/pg-core/table";
 function metricCols() {
 	return {
 		clicks: integer("clicks").notNull(),
@@ -180,80 +184,4 @@ const TABLE_METADATA = {
 		version: 2
 	}
 };
-function pgSqlTypeToColumnType(sqlType) {
-	const t = sqlType.toLowerCase();
-	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
-	if (t === "date" || t.startsWith("timestamp")) return "DATE";
-	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
-	if (t === "bigint" || t === "int8") return "BIGINT";
-	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
-	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
-}
-function tableSchemaFrom(tableName) {
-	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
-		name: col.name,
-		type: pgSqlTypeToColumnType(col.getSQLType()),
-		nullable: !col.notNull
-	}));
-	const meta = TABLE_METADATA[tableName];
-	return {
-		name: tableName,
-		columns,
-		sortKey: meta.sortKey,
-		version: meta.version
-	};
-}
-const METRIC_TABLES = [
-	"pages",
-	"queries",
-	"countries",
-	"page_queries",
-	"dates",
-	"search_appearance",
-	"search_appearance_pages",
-	"search_appearance_queries",
-	"search_appearance_page_queries",
-	"hourly_pages"
-];
-const SCHEMAS = Object.fromEntries(METRIC_TABLES.map((t) => [t, tableSchemaFrom(t)]));
-function currentSchemaVersion(table) {
-	return SCHEMAS[table].version;
-}
-function dateColumnsFor(table) {
-	return SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
-}
-function allTables() {
-	return METRIC_TABLES;
-}
-function inferTable(dimensions) {
-	const dims = new Set(dimensions);
-	const hasPage = dims.has("page");
-	const hasQuery = dims.has("query");
-	if (dims.has("searchAppearance")) return hasPage && hasQuery ? "search_appearance_page_queries" : hasPage ? "search_appearance_pages" : hasQuery ? "search_appearance_queries" : "search_appearance";
-	if (hasPage && hasQuery) return "page_queries";
-	if (hasQuery) return "queries";
-	if (hasPage) return "pages";
-	if (dims.has("country")) return "countries";
-	if (dims.has("device")) return "dates";
-	return "dates";
-}
-function naturalKeyColumns(table) {
-	return TABLE_METADATA[table].sortKey;
-}
-function dedupeByNaturalKey(table, rows) {
-	const key = TABLE_METADATA[table].sortKey;
-	if (key.length === 0) return rows.slice();
-	const seen = /* @__PURE__ */ new Map();
-	for (const row of rows) {
-		const r = row;
-		const k = key.map((col) => `${r[col] ?? ""}`).join("\0");
-		seen.set(k, row);
-	}
-	return [...seen.values()];
-}
-function dimensionToColumn(dim, _table) {
-	if (dim === "page") return "url";
-	if (dim === "queryCanonical") return "query_canonical";
-	return dim;
-}
-export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+export { TABLE_METADATA, countries, dates, drizzleSchema, hourly_pages, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/{_chunks/snapshot.d.mts → duckdb.d.mts}

@@ -24,17 +24,4 @@ interface DuckDBReadOptions {
 }
 declare function createDuckDBCodec(factory: DuckDBFactory, options?: DuckDBReadOptions): ParquetCodec;
 declare function createDuckDBExecutor(factory: DuckDBFactory, options?: DuckDBReadOptions): QueryExecutor;
-/**
- * Describes a hot/cold snapshot set. Produced by the snapshot builder,
- * consumed by `attachSnapshotIndex`. Filenames are derived from `cold`
- * via `cold-${yearMonth}.duckdb`; hot is always `hot.duckdb` when
- * `hot: true`.
- */
-interface SnapshotIndex {
-  version: 1;
-  builtAt: string;
-  cold: string[];
-  hot: boolean;
-  hotDays: number;
-}
-export { DuckDBFactory, DuckDBHandle, SnapshotIndex, createDuckDBCodec, createDuckDBExecutor };
+export { DuckDBFactory, DuckDBHandle, DuckDBReadOptions, createDuckDBCodec, createDuckDBExecutor };

package/dist/duckdb.mjs

@@ -0,0 +1,207 @@
+import { coerceRows } from "./coerce.mjs";
+import { TABLE_METADATA } from "./drizzle-schema.mjs";
+import { SCHEMAS, dateColumnsFor } from "./schema.mjs";
+import { dateReplaceClause as dateReplaceClause$1 } from "./sql-fragments.mjs";
+import { substituteNamedFiles } from "./parquet-plan.mjs";
+import { sqlEscape } from "./sql-bind.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+const DEFAULT_BUFFER_READ_CONCURRENCY = 16;
+const MAX_BUFFER_READ_CONCURRENCY = 64;
+function bufferReadConcurrency(value, fileCount) {
+	return Math.max(1, Math.min(fileCount || 1, typeof value === "number" && Number.isFinite(value) ? Math.floor(value) : DEFAULT_BUFFER_READ_CONCURRENCY, MAX_BUFFER_READ_CONCURRENCY));
+}
+async function registerBufferedFiles(db, files, dataSource, registered, signal, maxConcurrency) {
+	const batchSize = bufferReadConcurrency(maxConcurrency, files.length);
+	for (let i = 0; i < files.length; i += batchSize) {
+		signal?.throwIfAborted();
+		const batch = files.slice(i, i + batchSize);
+		const buffers = await Promise.all(batch.map((file) => dataSource.read(file.key, void 0, signal)));
+		for (let j = 0; j < batch.length; j++) {
+			signal?.throwIfAborted();
+			const file = batch[j];
+			await db.registerFileBuffer(file.name, buffers[j]);
+			registered.push(file.name);
+		}
+	}
+}
+async function encodeBytes(db, table, rows) {
+	const inName = db.makeTempPath("json");
+	const outName = db.makeTempPath("parquet");
+	const jsonBytes = encodeJsonBigintSafe(coerceRows(rows));
+	const registered = [];
+	await db.registerFileBuffer(inName, jsonBytes);
+	registered.push(inName);
+	try {
+		const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`;
+		await db.query(sql);
+		registered.push(outName);
+		return await db.copyFileToBuffer(outName);
+	} finally {
+		await db.dropFiles(registered);
+	}
+}
+async function decodeBytes(db, bytes, table) {
+	const name = db.makeTempPath("parquet");
+	await db.registerFileBuffer(name, bytes);
+	try {
+		return await db.query(`SELECT * ${dateReplaceClause(table)} FROM read_parquet('${sqlEscape(name)}')`);
+	} finally {
+		await db.dropFiles([name]);
+	}
+}
+function createDuckDBCodec(factory, options = {}) {
+	return {
+		async writeRows(ctx, rows, key, dataSource) {
+			const bytes = await encodeBytes(await factory.getDuckDB(), ctx.table, rows);
+			await dataSource.write(key, bytes);
+			return {
+				bytes: bytes.byteLength,
+				rowCount: rows.length
+			};
+		},
+		async readRows(ctx, key, dataSource) {
+			return decodeBytes(await factory.getDuckDB(), await dataSource.read(key), ctx.table);
+		},
+		async compactRows(ctx, inputKeys, outputKey, dataSource) {
+			const db = await factory.getDuckDB();
+			if (inputKeys.length === 0) {
+				const bytes = await encodeBytes(db, ctx.table, []);
+				await dataSource.write(outputKey, bytes);
+				return {
+					bytes: bytes.byteLength,
+					rowCount: 0
+				};
+			}
+			const inputUris = inputKeys.map((k) => dataSource.uri?.(k));
+			if (inputUris.every((u) => u !== void 0)) {
+				const outName = db.makeTempPath("parquet");
+				const fileList = inputUris.map((u) => `'${sqlEscape(u)}'`).join(", ");
+				try {
+					await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
+					const bytes = await db.copyFileToBuffer(outName);
+					const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
+					const rowCount = Number(countRows[0]?.n ?? 0);
+					await dataSource.write(outputKey, bytes);
+					return {
+						bytes: bytes.byteLength,
+						rowCount
+					};
+				} finally {
+					await db.dropFiles([outName]);
+				}
+			}
+			const inNames = [];
+			const outName = db.makeTempPath("parquet");
+			const registered = [];
+			const bufferedInputs = inputKeys.map((key) => {
+				const name = db.makeTempPath("parquet");
+				inNames.push(name);
+				return {
+					key,
+					name
+				};
+			});
+			try {
+				await registerBufferedFiles(db, bufferedInputs, dataSource, registered, void 0, options.bufferReadConcurrency);
+				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
+				await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
+				registered.push(outName);
+				const bytes = await db.copyFileToBuffer(outName);
+				const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
+				const rowCount = Number(countRows[0]?.n ?? 0);
+				await dataSource.write(outputKey, bytes);
+				return {
+					bytes: bytes.byteLength,
+					rowCount
+				};
+			} finally {
+				await db.dropFiles(registered);
+			}
+		}
+	};
+}
+const quoteCol = (c) => `"${c.replace(/"/g, "\"\"")}"`;
+function dedupedMergeSql(table, fileListSql) {
+	const base = `SELECT * FROM read_parquet([${fileListSql}], union_by_name = true)`;
+	const sortKey = SCHEMAS[table].sortKey;
+	const clusterKey = TABLE_METADATA[table].clusterKey;
+	const dedup = sortKey.length === 0 ? base : `${base} QUALIFY row_number() OVER (PARTITION BY ${sortKey.map(quoteCol).join(", ")}) = 1`;
+	if (clusterKey.length === 0) return dedup;
+	return `${dedup} ORDER BY ${clusterKey.map(quoteCol).join(", ")}`;
+}
+function rewriteEmptyFileSets(sql, placeholders, defaultTable, placeholderTables) {
+	let out = sql;
+	for (const [name, keys] of Object.entries(placeholders)) {
+		if (keys.length > 0) continue;
+		const emptyFallback = `(SELECT * FROM ${emptyTableSchema(placeholderTables?.[name] ?? defaultTable)} WHERE FALSE)`;
+		const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${name}\\}\\}\\s*(?:,\\s*union_by_name\\s*=\\s*true\\s*)?\\)`, "g");
+		out = out.replace(pattern, emptyFallback);
+	}
+	return out;
+}
+function createDuckDBExecutor(factory, options = {}) {
+	return { async execute({ sql, params, fileKeys, placeholderTables, dataSource, table, signal, profiler }) {
+		signal?.throwIfAborted();
+		const db = await factory.getDuckDB();
+		const placeholders = {};
+		const registered = [];
+		const totalFiles = Object.values(fileKeys).reduce((n, keys) => n + keys.length, 0);
+		const endRegister = profiler?.start("files.register", { files: totalFiles });
+		try {
+			const bufferedByName = /* @__PURE__ */ new Map();
+			for (const [name, keys] of Object.entries(fileKeys)) {
+				const resolved = [];
+				for (let i = 0; i < keys.length; i++) {
+					const key = keys[i];
+					const uri = dataSource.uri?.(key);
+					if (uri !== void 0) resolved.push(uri);
+					else {
+						if (!bufferedByName.has(key)) bufferedByName.set(key, {
+							key,
+							name: key
+						});
+						resolved.push(key);
+					}
+				}
+				placeholders[name] = resolved;
+			}
+			await registerBufferedFiles(db, [...bufferedByName.values()], dataSource, registered, signal, options.bufferReadConcurrency);
+			endRegister?.({ buffered: registered.length });
+			signal?.throwIfAborted();
+			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);
+			const endQuery = profiler?.start("query.run");
+			const rows = await db.query(finalSql, params);
+			endQuery?.({ rows: rows.length });
+			return {
+				rows,
+				sql: finalSql
+			};
+		} finally {
+			if (registered.length > 0) await db.dropFiles(registered);
+		}
+	} };
+}
+function emptyTableSchema(table) {
+	return `(FROM (VALUES ${placeholderValues(table)}) t(${columnList(table)}))`;
+}
+function dateReplaceClause(table) {
+	if (!table) return "";
+	return dateReplaceClause$1(dateColumnsFor(table), "string");
+}
+function columnList(table) {
+	return SCHEMAS[table].columns.map((c) => c.name).join(", ");
+}
+function placeholderValues(table) {
+	return `(${SCHEMAS[table].columns.map((c) => defaultForType(c.type)).join(", ")})`;
+}
+function defaultForType(t) {
+	if (t === "VARCHAR") return "''";
+	if (t === "DATE") return "DATE '1970-01-01'";
+	if (t === "INTEGER" || t === "BIGINT") return "0";
+	if (t === "DOUBLE") return "CAST(0 AS DOUBLE)";
+	return "NULL";
+}
+function columnsJson(table) {
+	return `{${SCHEMAS[table].columns.map((c) => `'${c.name}': '${c.type}'`).join(", ")}}`;
+}
+export { createDuckDBCodec, createDuckDBExecutor };

package/dist/engine.d.mts

@@ -0,0 +1,4 @@
+import { EngineOptions, StorageEngine } from "./storage.mjs";
+declare const MAX_DAY_BYTES: number;
+declare function createStorageEngine(opts: EngineOptions): StorageEngine;
+export { MAX_DAY_BYTES, createStorageEngine };