@gscdump/engine 0.4.0

package/dist/schema.mjs

@@ -0,0 +1,139 @@
+import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
+function metricCols() {
+	return {
+		clicks: integer("clicks").notNull(),
+		impressions: integer("impressions").notNull(),
+		sum_position: doublePrecision("sum_position").notNull()
+	};
+}
+const dateCol = () => date("date").notNull();
+const pages = pgTable("pages", {
+	url: varchar("url").notNull(),
+	date: dateCol(),
+	...metricCols()
+});
+const keywords = pgTable("keywords", {
+	query: varchar("query").notNull(),
+	query_canonical: varchar("query_canonical"),
+	date: dateCol(),
+	...metricCols()
+});
+const countries = pgTable("countries", {
+	country: varchar("country").notNull(),
+	date: dateCol(),
+	...metricCols()
+});
+const devices = pgTable("devices", {
+	device: varchar("device").notNull(),
+	date: dateCol(),
+	...metricCols()
+});
+const page_keywords = pgTable("page_keywords", {
+	url: varchar("url").notNull(),
+	query: varchar("query").notNull(),
+	query_canonical: varchar("query_canonical"),
+	date: dateCol(),
+	...metricCols()
+});
+const search_appearance = pgTable("search_appearance", {
+	searchAppearance: varchar("searchAppearance").notNull(),
+	date: dateCol(),
+	...metricCols()
+});
+const drizzleSchema = {
+	pages,
+	keywords,
+	countries,
+	devices,
+	page_keywords,
+	search_appearance
+};
+const TABLE_METADATA = {
+	pages: {
+		sortKey: ["date", "url"],
+		version: 1
+	},
+	keywords: {
+		sortKey: ["date", "query"],
+		version: 2
+	},
+	countries: {
+		sortKey: ["date", "country"],
+		version: 1
+	},
+	devices: {
+		sortKey: ["date", "device"],
+		version: 1
+	},
+	page_keywords: {
+		sortKey: [
+			"date",
+			"url",
+			"query"
+		],
+		version: 2
+	},
+	search_appearance: {
+		sortKey: ["date", "searchAppearance"],
+		version: 1
+	}
+};
+function pgSqlTypeToColumnType(sqlType) {
+	const t = sqlType.toLowerCase();
+	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
+	if (t === "date" || t.startsWith("timestamp")) return "DATE";
+	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
+	if (t === "bigint" || t === "int8") return "BIGINT";
+	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
+	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
+}
+function tableSchemaFrom(tableName) {
+	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
+		name: col.name,
+		type: pgSqlTypeToColumnType(col.getSQLType()),
+		nullable: !col.notNull
+	}));
+	const meta = TABLE_METADATA[tableName];
+	return {
+		name: tableName,
+		columns,
+		sortKey: meta.sortKey,
+		version: meta.version
+	};
+}
+const METRIC_TABLES = [
+	"pages",
+	"keywords",
+	"countries",
+	"devices",
+	"page_keywords",
+	"search_appearance"
+];
+const SCHEMAS = Object.fromEntries(METRIC_TABLES.map((t) => [t, tableSchemaFrom(t)]));
+function currentSchemaVersion(table) {
+	return SCHEMAS[table].version;
+}
+function schemaFor(table) {
+	return SCHEMAS[table];
+}
+function allTables() {
+	return METRIC_TABLES;
+}
+function inferTable(dimensions) {
+	const dims = new Set(dimensions);
+	const hasPage = dims.has("page");
+	const hasQuery = dims.has("query");
+	if (hasPage && hasQuery) return "page_keywords";
+	if (hasQuery) return "keywords";
+	if (hasPage) return "pages";
+	if (dims.has("country")) return "countries";
+	if (dims.has("device")) return "devices";
+	if (dims.has("searchAppearance")) return "search_appearance";
+	return "keywords";
+}
+function dimensionToColumn(dim, _table) {
+	if (dim === "page") return "url";
+	if (dim === "queryCanonical") return "query_canonical";
+	return dim;
+}
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, devices, dimensionToColumn, drizzleSchema, inferTable, keywords, page_keywords, pages, schemaFor, search_appearance };

package/dist/scope.d.mts

@@ -0,0 +1,38 @@
+import { SQL } from "drizzle-orm";
+/**
+ * Structural subset of `ResolvedWindow` from `@gscdump/analysis/period`.
+ * Inlined here so engine doesn't take a downstream dependency on analysis;
+ * any object with `start`/`end` strings (and optional `days`) satisfies it.
+ */
+interface ResolvedWindow {
+  start: string;
+  end: string;
+  days?: number;
+}
+interface ScopedRunnerOptions {
+  siteId?: string;
+  window?: ResolvedWindow;
+  /** Inclusive lower bound for `date`. Ignored if `window` is supplied. */
+  startDate?: string;
+  /** Inclusive upper bound for `date`. Ignored if `window` is supplied. */
+  endDate?: string;
+}
+interface TableScope {
+  wherePredicates: SQL[];
+  window?: ResolvedWindow;
+  siteId?: string;
+}
+declare function buildTableScope(table: Record<string, any>, opts: ScopedRunnerOptions): TableScope;
+declare function mergeScope(scope: TableScope, ...extra: SQL[]): SQL | undefined;
+/**
+ * Bind `buildTableScope` + `mergeScope` to a specific drizzle schema. Engine
+ * adapters (`engine-sqlite`, `engine-wasm`) call this once at module load and
+ * re-export the returned `scopeFor` / `mergeScope` so consumers get a typed
+ * `keyof Schema` table parameter without each adapter re-implementing the
+ * pass-through wrapper.
+ */
+declare function createScopedHelpers<S extends Record<string, Record<string, any>>>(schema: S): {
+  scopeFor: (table: keyof S, opts: ScopedRunnerOptions) => TableScope;
+  mergeScope: typeof mergeScope;
+};
+export { ResolvedWindow, ScopedRunnerOptions, TableScope, buildTableScope, createScopedHelpers, mergeScope };

package/dist/scope.mjs

@@ -0,0 +1,28 @@
+import { and, eq, gte, lte } from "drizzle-orm";
+function buildTableScope(table, opts) {
+	const predicates = [];
+	if (opts.siteId && "site_id" in table) predicates.push(eq(table.site_id, opts.siteId));
+	if ("date" in table) {
+		const start = opts.window?.start ?? opts.startDate;
+		const end = opts.window?.end ?? opts.endDate;
+		if (start) predicates.push(gte(table.date, start));
+		if (end) predicates.push(lte(table.date, end));
+	}
+	return {
+		wherePredicates: predicates,
+		window: opts.window,
+		siteId: opts.siteId
+	};
+}
+function mergeScope(scope, ...extra) {
+	const all = [...scope.wherePredicates, ...extra].filter(Boolean);
+	if (all.length === 0) return void 0;
+	return and(...all);
+}
+function createScopedHelpers(schema) {
+	return {
+		scopeFor: (table, opts) => buildTableScope(schema[table], opts),
+		mergeScope
+	};
+}
+export { buildTableScope, createScopedHelpers, mergeScope };

package/dist/snapshot.d.mts

@@ -0,0 +1,14 @@
+/**
+ * Describes a hot/cold snapshot set. Produced by the snapshot builder,
+ * consumed by `attachSnapshotIndex`. Filenames are derived from `cold`
+ * via `cold-${yearMonth}.duckdb`; hot is always `hot.duckdb` when
+ * `hot: true`.
+ */
+interface SnapshotIndex {
+  version: 1;
+  builtAt: string;
+  cold: string[];
+  hot: boolean;
+  hotDays: number;
+}
+export { SnapshotIndex };

package/dist/snapshot.mjs

@@ -0,0 +1 @@
+export {};

package/dist/sql-bind.d.mts

@@ -0,0 +1,19 @@
+/** Escape single quotes for inlining inside a SQL string literal (SQL-standard `''` escaping). */
+declare function sqlEscape(s: string): string;
+declare function formatLiteral(value: unknown): string;
+/**
+ * Replace `?` and `$N` placeholders with inline SQL literals. Single-quoted
+ * string regions and SQL comments (`-- line`, `/* block *\/`) are left
+ * untouched — a `?` or `$1` inside `'foo?bar'` or a comment is not a
+ * placeholder. SQL-standard `''` escape handling; no `\`-escape or
+ * dialect-specific identifier quoting.
+ *
+ * `?` placeholders bind sequentially against `params`. `$N` (Postgres-style)
+ * binds explicitly to `params[N-1]`. The two styles must not be mixed in the
+ * same query.
+ *
+ * Throws when placeholder count and params length disagree, or when a `$N`
+ * index is out of range.
+ */
+declare function bindLiterals(sql: string, params: readonly unknown[]): string;
+export { bindLiterals, formatLiteral, sqlEscape };

package/dist/sql-bind.mjs

@@ -0,0 +1,92 @@
+function containsDisallowedControlChars(value) {
+	for (let i = 0; i < value.length; i++) {
+		const code = value.charCodeAt(i);
+		if (code >= 0 && code <= 8 || code === 11 || code === 12 || code >= 14 && code <= 31) return true;
+	}
+	return false;
+}
+function sqlEscape(s) {
+	return s.replace(/'/g, "''");
+}
+function formatLiteral(value) {
+	if (value == null) return "NULL";
+	if (typeof value === "number") {
+		if (!Number.isFinite(value)) throw new Error(`cannot inline non-finite number: ${value}`);
+		return String(value);
+	}
+	if (typeof value === "boolean") return value ? "TRUE" : "FALSE";
+	if (typeof value === "bigint") return value.toString();
+	if (value instanceof Date) return `'${value.toISOString()}'`;
+	if (typeof value === "string") {
+		if (containsDisallowedControlChars(value)) throw new Error("string literal contains disallowed control characters");
+		return `'${value.replace(/'/g, "''")}'`;
+	}
+	throw new Error(`cannot inline value of type ${typeof value}`);
+}
+function bindLiterals(sql, params) {
+	if (params.length === 0) return sql;
+	let out = "";
+	let i = 0;
+	let qmarkIdx = 0;
+	const usedDollar = /* @__PURE__ */ new Set();
+	let inString = false;
+	while (i < sql.length) {
+		const c = sql[i];
+		if (inString) {
+			out += c;
+			if (c === "'") {
+				if (sql[i + 1] === "'") {
+					out += "'";
+					i += 2;
+					continue;
+				}
+				inString = false;
+			}
+			i++;
+			continue;
+		}
+		if (c === "-" && sql[i + 1] === "-") {
+			const nl = sql.indexOf("\n", i + 2);
+			const end = nl === -1 ? sql.length : nl;
+			out += sql.slice(i, end);
+			i = end;
+			continue;
+		}
+		if (c === "/" && sql[i + 1] === "*") {
+			const close = sql.indexOf("*/", i + 2);
+			const end = close === -1 ? sql.length : close + 2;
+			out += sql.slice(i, end);
+			i = end;
+			continue;
+		}
+		if (c === "'") {
+			inString = true;
+			out += c;
+			i++;
+			continue;
+		}
+		if (c === "?") {
+			if (qmarkIdx >= params.length) throw new Error(`bindLiterals: more '?' placeholders than params (have ${params.length})`);
+			out += formatLiteral(params[qmarkIdx++]);
+			i++;
+			continue;
+		}
+		if (c === "$" && sql[i + 1] && sql[i + 1] >= "0" && sql[i + 1] <= "9") {
+			let j = i + 1;
+			while (j < sql.length && sql[j] >= "0" && sql[j] <= "9") j++;
+			const n = Number(sql.slice(i + 1, j));
+			if (n < 1 || n > params.length) throw new Error(`bindLiterals: $${n} out of range (have ${params.length} params)`);
+			usedDollar.add(n - 1);
+			out += formatLiteral(params[n - 1]);
+			i = j;
+			continue;
+		}
+		out += c;
+		i++;
+	}
+	if (qmarkIdx > 0 && usedDollar.size > 0) throw new Error("bindLiterals: cannot mix '?' and '$N' placeholders in the same query");
+	const used = qmarkIdx > 0 ? qmarkIdx : usedDollar.size;
+	if (used !== params.length) throw new Error(`bindLiterals: ${params.length - used} params unused`);
+	return out;
+}
+export { bindLiterals, formatLiteral, sqlEscape };

package/dist/sql-fragments.d.mts

@@ -0,0 +1,21 @@
+import { Metric } from "gscdump/query";
+/**
+ * Standard SQL `LIKE` escape. Backslash is the explicit ESCAPE char so
+ * literal `%`, `_`, and `\` survive a `contains` predicate unchanged.
+ */
+declare function escapeLike(value: string): string;
+/**
+ * Per-metric raw SQL aggregate expression.
+ *
+ * - `clicks` / `impressions` cast to DOUBLE so DuckDB SUM doesn't return
+ *   BIGINT (which loses through JSON / Workers RPC).
+ * - `position` reverses the ingestion offset (`sum_position = position - 1`
+ *   summed; divide by impressions and add 1 back).
+ */
+declare const METRIC_EXPR: Record<Metric, string>;
+/**
+ * Top-level page predicate: matches paths with at most one `/`. Parameterized
+ * on the resolved column expression so drizzle can pass a column ref.
+ */
+declare function topLevelPagePredicateSql(pathExpr: string): string;
+export { METRIC_EXPR, escapeLike, topLevelPagePredicateSql };

package/dist/sql-fragments.mjs

@@ -0,0 +1,13 @@
+function escapeLike(value) {
+	return value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
+}
+const METRIC_EXPR = {
+	clicks: "CAST(SUM(clicks) AS DOUBLE)",
+	impressions: "CAST(SUM(impressions) AS DOUBLE)",
+	ctr: "CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0)",
+	position: "SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1"
+};
+function topLevelPagePredicateSql(pathExpr) {
+	return `LENGTH(${pathExpr}) - LENGTH(REPLACE(${pathExpr}, '/', '')) <= 1`;
+}
+export { METRIC_EXPR, escapeLike, topLevelPagePredicateSql };

package/package.json

@@ -0,0 +1,168 @@
+{
+  "name": "@gscdump/engine",
+  "type": "module",
+  "version": "0.4.0",
+  "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
+  "author": {
+    "name": "Harlan Wilton",
+    "email": "harlan@harlanzw.com",
+    "url": "https://harlanzw.com/"
+  },
+  "license": "MIT",
+  "funding": "https://github.com/sponsors/harlan-zw",
+  "homepage": "https://github.com/harlan-zw/gscdump/tree/main/packages/engine#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/harlan-zw/gscdump.git",
+    "directory": "packages/engine"
+  },
+  "bugs": {
+    "url": "https://github.com/harlan-zw/gscdump/issues"
+  },
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    },
+    "./contracts": {
+      "types": "./dist/contracts.d.mts",
+      "import": "./dist/contracts.mjs"
+    },
+    "./snapshot": {
+      "types": "./dist/snapshot.d.mts",
+      "import": "./dist/snapshot.mjs"
+    },
+    "./planner": {
+      "types": "./dist/planner.d.mts",
+      "import": "./dist/planner.mjs"
+    },
+    "./schema": {
+      "types": "./dist/schema.d.mts",
+      "import": "./dist/schema.mjs"
+    },
+    "./ingest": {
+      "types": "./dist/ingest.d.mts",
+      "import": "./dist/ingest.mjs"
+    },
+    "./sql": {
+      "types": "./dist/sql-bind.d.mts",
+      "import": "./dist/sql-bind.mjs"
+    },
+    "./sql-fragments": {
+      "types": "./dist/sql-fragments.d.mts",
+      "import": "./dist/sql-fragments.mjs"
+    },
+    "./rollups": {
+      "types": "./dist/rollups.d.mts",
+      "import": "./dist/rollups.mjs"
+    },
+    "./entities": {
+      "types": "./dist/entities.d.mts",
+      "import": "./dist/entities.mjs"
+    },
+    "./node": {
+      "types": "./dist/adapters/duckdb-node.d.mts",
+      "import": "./dist/adapters/duckdb-node.mjs"
+    },
+    "./node-harness": {
+      "types": "./dist/adapters/node-harness.d.mts",
+      "import": "./dist/adapters/node-harness.mjs"
+    },
+    "./filesystem": {
+      "types": "./dist/adapters/filesystem.d.mts",
+      "import": "./dist/adapters/filesystem.mjs"
+    },
+    "./http": {
+      "types": "./dist/adapters/http.d.mts",
+      "import": "./dist/adapters/http.mjs"
+    },
+    "./hyparquet": {
+      "types": "./dist/adapters/hyparquet.d.mts",
+      "import": "./dist/adapters/hyparquet.mjs"
+    },
+    "./r2": {
+      "types": "./dist/adapters/r2.d.mts",
+      "import": "./dist/adapters/r2.mjs"
+    },
+    "./r2-manifest": {
+      "types": "./dist/adapters/r2-manifest.d.mts",
+      "import": "./dist/adapters/r2-manifest.mjs"
+    },
+    "./resolver": {
+      "types": "./dist/resolver/index.d.mts",
+      "import": "./dist/resolver/index.mjs"
+    },
+    "./scope": {
+      "types": "./dist/scope.d.mts",
+      "import": "./dist/scope.mjs"
+    },
+    "./arrow": {
+      "types": "./dist/arrow-utils.d.mts",
+      "import": "./dist/arrow-utils.mjs"
+    },
+    "./inspection-sqlite-node": {
+      "types": "./dist/adapters/inspection-sqlite-node.d.mts",
+      "import": "./dist/adapters/inspection-sqlite-node.mjs"
+    },
+    "./inspection-sqlite-browser": {
+      "types": "./dist/adapters/inspection-sqlite-browser.d.mts",
+      "import": "./dist/adapters/inspection-sqlite-browser.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "peerDependencies": {
+    "@duckdb/duckdb-wasm": "^1.32.0",
+    "better-sqlite3": "^12.9.0",
+    "hyparquet": "^1.25.1",
+    "hyparquet-writer": "^0.13.0",
+    "wa-sqlite": "^1.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@duckdb/duckdb-wasm": {
+      "optional": true
+    },
+    "better-sqlite3": {
+      "optional": true
+    },
+    "hyparquet": {
+      "optional": true
+    },
+    "hyparquet-writer": {
+      "optional": true
+    },
+    "wa-sqlite": {
+      "optional": true
+    }
+  },
+  "dependencies": {
+    "drizzle-orm": "^0.45.2",
+    "proper-lockfile": "^4.1.2",
+    "gscdump": "0.4.0"
+  },
+  "devDependencies": {
+    "@duckdb/duckdb-wasm": "^1.32.0",
+    "@types/proper-lockfile": "^4.1.4",
+    "aws4fetch": "^1.0.20",
+    "better-sqlite3": "^12.9.0",
+    "hyparquet": "^1.25.1",
+    "hyparquet-writer": "^0.13.0",
+    "tsx": "^4.19.2",
+    "vitest": "^4.1.5"
+  },
+  "scripts": {
+    "dev": "obuild --stub",
+    "build": "obuild",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest",
+    "r2-harness": "tsx scripts/r2-contention-harness.ts",
+    "backfill-audit": "tsx scripts/backfill-audit.ts"
+  }
+}