@gscdump/engine 0.4.0 → 0.6.2

package/dist/rollups.d.mts

@@ -1,51 +1,6 @@
-import { ColumnDef, Row, TableName, TenantCtx } from "gscdump/contracts";
-interface DataSource {
-  read: (key: string, range?: {
-    offset: number;
-    length: number;
-  }, signal?: AbortSignal) => Promise<Uint8Array>;
-  write: (key: string, bytes: Uint8Array) => Promise<void>;
-  delete: (keys: string[]) => Promise<void>;
-  /**
-   * One-shot listing under a prefix. Implementations may cap the number of
-   * returned keys (typically 10k) — callers iterating full tenant space
-   * should prefer `streamList` when available or narrow the prefix.
-   */
-  list: (prefix: string) => Promise<string[]>;
-  /**
-   * Per-key URI probe. Returns a URI string DuckDB's `httpfs` (or an
-   * equivalent engine that fetches its own I/O) can read directly, or
-   * `undefined` if the key isn't URI-resolvable on this backend and the
-   * caller must fall back to `read(key)` for the bytes.
-   *
-   * Contracts:
-   * - When defined, the returned URI MUST yield byte-identical content to
-   *   `read(key)`. Callers rely on this for correctness.
-   * - Backends with a native URI for every key (filesystem: absolute path,
-   *   R2 via `httpfs`: signed URL) may always return a string.
-   * - Backends without a native URI shape (in-memory) omit the method or
-   *   return `undefined` per call.
-   * - Mixed-per-query is allowed: some keys in one query may return a URI,
-   *   others may not; the executor branches per key.
-   */
-  uri?: (key: string) => string | undefined;
-  /**
-   * Optional — probe the byte size of a key without reading it. Used by
-   * the engine to fill in `WriteResult.bytes` when a codec reports 0 or
-   * unknown but the file is non-trivial.
-   */
-  head?: (key: string) => Promise<{
-    bytes: number;
-  } | undefined>;
-  /**
-   * Optional streaming variant of `list`. Implementations that page
-   * backing-store results (R2, S3) should implement this and yield keys
-   * lazily. `list` may return up to an adapter-defined cap (typically
-   * 10k keys); callers iterating full tenant space must prefer
-   * `streamList` when available, or chunk by narrower prefixes.
-   */
-  streamList?: (prefix: string) => AsyncIterable<string>;
-}
+import { N as TableName$1, a as DataSource, w as Row$1 } from "./_chunks/storage.mjs";
+import { t as ColumnDef } from "./_chunks/schema.mjs";
+import { TenantCtx } from "gscdump/contracts";
 interface RollupCtx extends TenantCtx {
   /** When the rollup was built. Stamped into payload + filename. */
   builtAt: number;
@@ -58,14 +13,14 @@ interface RollupEngine {
   runSQL: (opts: {
     ctx: TenantCtx;
     fileSets: Record<string, {
-      table: TableName;
+      table: TableName$1;
       partitions?: string[];
     }>;
-    table?: TableName;
+    table?: TableName$1;
     sql: string;
     params?: unknown[];
   }) => Promise<{
-    rows: Row[];
+    rows: Row$1[];
   }>;
 }
 /**

package/dist/rollups.mjs

@@ -1,213 +1,6 @@
+import { createIndexingMetadataStore } from "./entities.mjs";
+import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { MS_PER_DAY } from "gscdump";
-import { parquetWriteBuffer } from "hyparquet-writer";
-import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
-function metricCols() {
-	return {
-		clicks: integer("clicks").notNull(),
-		impressions: integer("impressions").notNull(),
-		sum_position: doublePrecision("sum_position").notNull()
-	};
-}
-const dateCol = () => date("date").notNull();
-const drizzleSchema = {
-	pages: pgTable("pages", {
-		url: varchar("url").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	keywords: pgTable("keywords", {
-		query: varchar("query").notNull(),
-		query_canonical: varchar("query_canonical"),
-		date: dateCol(),
-		...metricCols()
-	}),
-	countries: pgTable("countries", {
-		country: varchar("country").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	devices: pgTable("devices", {
-		device: varchar("device").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	page_keywords: pgTable("page_keywords", {
-		url: varchar("url").notNull(),
-		query: varchar("query").notNull(),
-		query_canonical: varchar("query_canonical"),
-		date: dateCol(),
-		...metricCols()
-	}),
-	search_appearance: pgTable("search_appearance", {
-		searchAppearance: varchar("searchAppearance").notNull(),
-		date: dateCol(),
-		...metricCols()
-	})
-};
-const TABLE_METADATA = {
-	pages: {
-		sortKey: ["date", "url"],
-		version: 1
-	},
-	keywords: {
-		sortKey: ["date", "query"],
-		version: 2
-	},
-	countries: {
-		sortKey: ["date", "country"],
-		version: 1
-	},
-	devices: {
-		sortKey: ["date", "device"],
-		version: 1
-	},
-	page_keywords: {
-		sortKey: [
-			"date",
-			"url",
-			"query"
-		],
-		version: 2
-	},
-	search_appearance: {
-		sortKey: ["date", "searchAppearance"],
-		version: 1
-	}
-};
-function pgSqlTypeToColumnType(sqlType) {
-	const t = sqlType.toLowerCase();
-	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
-	if (t === "date" || t.startsWith("timestamp")) return "DATE";
-	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
-	if (t === "bigint" || t === "int8") return "BIGINT";
-	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
-	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
-}
-function tableSchemaFrom(tableName) {
-	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
-		name: col.name,
-		type: pgSqlTypeToColumnType(col.getSQLType()),
-		nullable: !col.notNull
-	}));
-	const meta = TABLE_METADATA[tableName];
-	return {
-		name: tableName,
-		columns,
-		sortKey: meta.sortKey,
-		version: meta.version
-	};
-}
-Object.fromEntries([
-	"pages",
-	"keywords",
-	"countries",
-	"devices",
-	"page_keywords",
-	"search_appearance"
-].map((t) => [t, tableSchemaFrom(t)]));
-const ROW_GROUP_SIZE = 25e3;
-function basicTypeFor(colType) {
-	if (colType === "VARCHAR" || colType === "DATE") return "STRING";
-	if (colType === "BIGINT") return "INT64";
-	if (colType === "INTEGER") return "INT32";
-	if (colType === "DOUBLE") return "DOUBLE";
-	throw new Error(`unsupported column type for parquet encoding: ${colType}`);
-}
-function coerceValue(value, type) {
-	if (value === null || value === void 0) return null;
-	if (type === "STRING") return typeof value === "string" ? value : String(value);
-	if (type === "INT32") {
-		const n = typeof value === "number" ? value : Number(value);
-		if (!Number.isFinite(n)) throw new Error(`non-finite number for INT32: ${String(value)}`);
-		return Math.trunc(n);
-	}
-	if (type === "INT64") {
-		if (typeof value === "bigint") return value;
-		const n = typeof value === "number" ? value : Number(value);
-		if (!Number.isFinite(n)) throw new Error(`non-finite number for INT64: ${String(value)}`);
-		return BigInt(Math.trunc(n));
-	}
-	if (type === "DOUBLE") {
-		const n = typeof value === "number" ? value : Number(value);
-		if (!Number.isFinite(n)) throw new Error(`non-finite number for DOUBLE: ${String(value)}`);
-		return n;
-	}
-	return value;
-}
-function compareValues(a, b) {
-	if (a === b) return 0;
-	if (a === null || a === void 0) return -1;
-	if (b === null || b === void 0) return 1;
-	if (typeof a === "number" && typeof b === "number") return a - b;
-	return String(a) < String(b) ? -1 : 1;
-}
-function encodeRowsToParquetFlex(rows, opts) {
-	const { columns, sortKey = [], rowGroupSize = ROW_GROUP_SIZE } = opts;
-	const sorted = sortKey.length === 0 || rows.length <= 1 ? rows : [...rows].sort((a, b) => {
-		for (const col of sortKey) {
-			const cmp = compareValues(a[col], b[col]);
-			if (cmp !== 0) return cmp;
-		}
-		return 0;
-	});
-	const buffer = parquetWriteBuffer({
-		columnData: columns.map((col) => {
-			const type = basicTypeFor(col.type);
-			const data = sorted.map((r) => coerceValue(r[col.name], type));
-			return {
-				name: col.name,
-				data,
-				type,
-				nullable: col.nullable,
-				columnIndex: true
-			};
-		}),
-		rowGroupSize
-	});
-	return new Uint8Array(buffer);
-}
-function hashUrl(url) {
-	let hi = 2166136261;
-	let lo = 3421674724;
-	for (let i = 0; i < url.length; i++) {
-		const c = url.charCodeAt(i);
-		lo ^= c;
-		const loMul = Math.imul(lo, 435) >>> 0;
-		const carry = Math.floor(lo * 435 / 4294967296);
-		const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
-		lo = loMul;
-		hi = hiMul;
-	}
-	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
-}
-function indexingMetadataIndexKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/indexing/index.json` : `u_${ctx.userId}/entities/indexing/index.json`;
-}
-function createIndexingMetadataStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	async function readIndex(key) {
-		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => ({
-			version: 1,
-			records: {}
-		}));
-	}
-	return {
-		async writeBatch(ctx, records) {
-			if (records.length === 0) return;
-			const key = indexingMetadataIndexKey(ctx);
-			const index = await readIndex(key);
-			for (const r of records) index.records[hash(r.url)] = r;
-			await ds.write(key, new TextEncoder().encode(JSON.stringify(index)));
-		},
-		async loadIndex(ctx) {
-			return readIndex(indexingMetadataIndexKey(ctx));
-		},
-		async getLatest(ctx, url) {
-			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
-		}
-	};
-}
 function rollupPrefix(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
 }