@gscdump/engine 0.6.1 → 0.6.3

package/dist/_chunks/compiler.mjs

@@ -0,0 +1,288 @@
+import { i as dimensionToColumn, r as currentSchemaVersion } from "./schema.mjs";
+import { a as mondayOfWeek, c as quarterOfMonth, d as weekPartition, i as inferSearchType, l as quarterPartition, n as dayPartition, o as monthPartition, s as objectKey } from "./storage.mjs";
+import { METRIC_EXPR, escapeLike, topLevelPagePredicateSql } from "../sql-fragments.mjs";
+import { buildLogicalPlan } from "gscdump/query/plan";
+import { MS_PER_DAY } from "gscdump";
+const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
+const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
+const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;
+const DEFAULT_THRESHOLDS = {
+	raw: 7,
+	d7: 30,
+	d30: 90
+};
+const PENDING_WINDOW_DAYS = 4;
+const STAGES = [
+	{
+		inputTier: "raw",
+		outputTier: "d7",
+		cutoffDays: DEFAULT_THRESHOLDS.raw,
+		bucketKey: (e) => {
+			const m = e.partition.match(DAILY_PARTITION_RE);
+			if (!m) return void 0;
+			return mondayOfWeek(m[1]);
+		},
+		bucketLatestMs: (monday) => Date.parse(`${monday}T00:00:00Z`) + 6 * MS_PER_DAY,
+		outputPartition: weekPartition
+	},
+	{
+		inputTier: "d7",
+		outputTier: "d30",
+		cutoffDays: DEFAULT_THRESHOLDS.d7,
+		bucketKey: (e) => {
+			const m = e.partition.match(WEEKLY_PARTITION_RE);
+			if (!m) return void 0;
+			return m[1].slice(0, 7);
+		},
+		bucketLatestMs: monthEndMs,
+		outputPartition: monthPartition
+	},
+	{
+		inputTier: "d30",
+		outputTier: "d90",
+		cutoffDays: DEFAULT_THRESHOLDS.d30,
+		bucketKey: (e) => {
+			const m = e.partition.match(MONTHLY_PARTITION_RE);
+			if (!m) return void 0;
+			return quarterOfMonth(m[1]);
+		},
+		bucketLatestMs: quarterEndMs,
+		outputPartition: quarterPartition
+	}
+];
+async function compactTieredImpl(deps, ctx, now, overrides = {}) {
+	const thresholds = {
+		...DEFAULT_THRESHOLDS,
+		...overrides
+	};
+	const stagesWithThresholds = STAGES.map((s) => ({
+		...s,
+		cutoffDays: s.outputTier === "d7" ? thresholds.raw : s.outputTier === "d30" ? thresholds.d7 : thresholds.d30
+	}));
+	for (const stage of stagesWithThresholds) await runStage(deps, ctx, stage, now);
+}
+async function runStage(deps, ctx, stage, now) {
+	const cutoff = now - Math.max(stage.cutoffDays, PENDING_WINDOW_DAYS) * MS_PER_DAY;
+	const candidates = await deps.manifestStore.listLive({
+		userId: ctx.userId,
+		siteId: ctx.siteId,
+		table: ctx.table,
+		tier: stage.inputTier
+	});
+	const buckets = /* @__PURE__ */ new Map();
+	for (const entry of candidates) {
+		const key = stage.bucketKey(entry);
+		if (!key) continue;
+		if (stage.bucketLatestMs(key) >= cutoff) continue;
+		const compositeKey = `${inferSearchType(entry)}\0${key}`;
+		if (!buckets.has(compositeKey)) buckets.set(compositeKey, []);
+		buckets.get(compositeKey).push(entry);
+	}
+	for (const [compositeKey, entries] of buckets) {
+		const [searchType, bucket] = compositeKey.split("\0");
+		const targetPartition = stage.outputPartition(bucket);
+		if (entries.length === 1 && entries[0].partition === targetPartition) continue;
+		await deps.manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition: targetPartition
+		}, async () => {
+			const key = objectKey(ctx, ctx.table, targetPartition, now, searchType);
+			const { bytes, rowCount } = await deps.codec.compactRows({ table: ctx.table }, entries.map((e) => e.objectKey), key, deps.dataSource);
+			const newEntry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition: targetPartition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: stage.outputTier,
+				...searchType !== "web" ? { searchType } : {}
+			};
+			await deps.manifestStore.registerVersion(newEntry, entries);
+		});
+	}
+}
+function enumeratePartitions(startDate, endDate) {
+	const out = [];
+	const [sy, sm, sd] = startDate.split("-").map(Number);
+	const [ey, em, ed] = endDate.split("-").map(Number);
+	const start = Date.UTC(sy, sm - 1, sd);
+	const end = Date.UTC(ey, em - 1, ed);
+	if (end < start) return out;
+	const seenWeeks = /* @__PURE__ */ new Set();
+	const seenMonths = /* @__PURE__ */ new Set();
+	const seenQuarters = /* @__PURE__ */ new Set();
+	for (let t = start; t <= end; t += 864e5) {
+		const d = new Date(t);
+		const y = d.getUTCFullYear();
+		const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+		const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
+		const isoMonth = `${y}-${m}`;
+		out.push(dayPartition(isoDay));
+		const monday = mondayOfWeek(isoDay);
+		if (!seenWeeks.has(monday)) {
+			seenWeeks.add(monday);
+			out.push(weekPartition(monday));
+		}
+		if (!seenMonths.has(isoMonth)) {
+			seenMonths.add(isoMonth);
+			out.push(monthPartition(isoMonth));
+		}
+		const quarter = quarterOfMonth(isoMonth);
+		if (!seenQuarters.has(quarter)) {
+			seenQuarters.add(quarter);
+			out.push(quarterPartition(quarter));
+		}
+	}
+	return out;
+}
+function monthEndMs(month) {
+	const [y, m] = month.split("-").map(Number);
+	return Date.UTC(y, m, 0, 23, 59, 59, 999);
+}
+function quarterEndMs(quarter) {
+	const [yStr, qStr] = quarter.split("-Q");
+	const y = Number(yStr);
+	const q = Number(qStr);
+	return Date.UTC(y, q * 3, 0, 23, 59, 59, 999);
+}
+const FILES_PLACEHOLDER = "{{FILES}}";
+function buildDimensionWhere(filters, table) {
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const column = dimensionToColumn(filter.dimension, table);
+		switch (filter.operator) {
+			case "equals":
+				clauses.push(`${column} = ?`);
+				params.push(filter.expression);
+				break;
+			case "notEquals":
+				clauses.push(`${column} != ?`);
+				params.push(filter.expression);
+				break;
+			case "contains":
+				clauses.push(`${column} LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "notContains":
+				clauses.push(`${column} NOT LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "includingRegex":
+				clauses.push(`regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+			case "excludingRegex":
+				clauses.push(`NOT regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.join(" AND "),
+		params
+	};
+}
+function buildTopLevelWhere(plan, table) {
+	if (!plan.specialFilters.topLevel) return "";
+	return topLevelPagePredicateSql(dimensionToColumn("page", table));
+}
+function buildHaving(filters) {
+	if (filters.length === 0) return {
+		clause: "",
+		params: []
+	};
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const expr = METRIC_EXPR[filter.metric];
+		switch (filter.operator) {
+			case "metricGte":
+				clauses.push(`${expr} >= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricGt":
+				clauses.push(`${expr} > ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLte":
+				clauses.push(`${expr} <= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLt":
+				clauses.push(`${expr} < ?`);
+				params.push(filter.expression);
+				break;
+			case "metricBetween":
+				clauses.push(`${expr} >= ? AND ${expr} <= ?`);
+				params.push(filter.expression, filter.expression2 ?? filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.length > 0 ? `HAVING ${clauses.join(" AND ")}` : "",
+		params
+	};
+}
+function compileLogicalQueryPlan(plan, table = plan.dataset) {
+	const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
+	const metricSelects = plan.metrics.map((metric) => `${METRIC_EXPR[metric]} AS ${metric}`);
+	const dimSelects = plan.groupByDimensions.map((dimension) => {
+		const column = dimensionToColumn(dimension, table);
+		return column !== dimension ? `${column} AS ${dimension}` : dimension;
+	});
+	const whereClauses = ["date >= ?", "date <= ?"];
+	const whereParams = [plan.dateRange.startDate, plan.dateRange.endDate];
+	const dimWhere = buildDimensionWhere(plan.dimensionFilters, table);
+	if (dimWhere.clause) {
+		whereClauses.push(dimWhere.clause);
+		whereParams.push(...dimWhere.params);
+	}
+	const topLevelClause = buildTopLevelWhere(plan, table);
+	if (topLevelClause) whereClauses.push(topLevelClause);
+	const having = buildHaving(plan.metricFilters);
+	const groupByCols = [...plan.groupByDimensions.map((dimension) => dimensionToColumn(dimension, table)), ...plan.hasDate ? ["date"] : []];
+	const groupBy = groupByCols.length > 0 ? `GROUP BY ${groupByCols.join(", ")}` : "";
+	const orderBy = plan.orderBy ? `ORDER BY ${plan.orderBy.column} ${plan.orderBy.dir.toUpperCase()}` : "ORDER BY clicks DESC";
+	const limit = `LIMIT ${plan.rowLimit ?? 1e3}`;
+	const offset = plan.startRow ? `OFFSET ${plan.startRow}` : "";
+	return {
+		sql: [
+			`SELECT ${[
+				...dimSelects,
+				...plan.hasDate ? ["date"] : [],
+				...metricSelects
+			].join(", ")}`,
+			`FROM read_parquet(${FILES_PLACEHOLDER}, union_by_name = true)`,
+			`WHERE ${whereClauses.join(" AND ")}`,
+			groupBy,
+			having.clause,
+			orderBy,
+			limit,
+			offset
+		].filter(Boolean).join(" ").replace(/\s+/g, " ").trim(),
+		params: [...whereParams, ...having.params],
+		partitions,
+		table,
+		filesPlaceholder: FILES_PLACEHOLDER
+	};
+}
+function resolveToSQL(state, table) {
+	const plan = buildLogicalPlan(state, { regex: true });
+	return compileLogicalQueryPlan(plan, table ?? plan.dataset);
+}
+function fileList(keys) {
+	return keys.length === 0 ? "[]" : `[${keys.map((key) => `'${key.replace(/'/g, "''")}'`).join(", ")}]`;
+}
+function substituteNamedFiles(sql, sets) {
+	let out = sql;
+	for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
+	return out;
+}
+export { compactTieredImpl as a, substituteNamedFiles as i, compileLogicalQueryPlan as n, enumeratePartitions as o, resolveToSQL as r, FILES_PLACEHOLDER as t };

package/dist/_chunks/duckdb.d.mts

@@ -0,0 +1,26 @@
+import { N as TableName, S as QueryExecutor, h as ParquetCodec, w as Row } from "./storage.mjs";
+interface DuckDBHandle {
+  query: (sql: string, params?: unknown[]) => Promise<Row[]>;
+  registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
+  copyFileToBuffer: (name: string) => Promise<Uint8Array>;
+  dropFiles: (names: string[]) => Promise<void>;
+  /**
+   * Returns a unique path suitable for `COPY TO '…'` + `copyFileToBuffer`.
+   * In Node this is an absolute path under `os.tmpdir()` so DuckDB doesn't
+   * litter the CWD; in browsers/Workers it's a plain virtual-FS name.
+   */
+  makeTempPath: (ext: string) => string;
+}
+interface DuckDBFactory {
+  getDuckDB: () => Promise<DuckDBHandle>;
+}
+declare function createDuckDBCodec(factory: DuckDBFactory): ParquetCodec;
+declare function createDuckDBExecutor(factory: DuckDBFactory): QueryExecutor;
+/**
+ * Canonical "empty-file" SELECT clause for a table. Codecs that need to
+ * emit a schema-correct empty Parquet can wrap this in:
+ *   `COPY (SELECT * FROM <clause> WHERE FALSE) TO '<key>' (FORMAT PARQUET)`
+ * to satisfy the ParquetCodec empty-rows invariant.
+ */
+declare function canonicalEmptyParquetSchema(table: TableName): string;
+export { createDuckDBExecutor as a, createDuckDBCodec as i, DuckDBHandle as n, canonicalEmptyParquetSchema as r, DuckDBFactory as t };