@gscdump/engine 1.3.2 → 1.4.1

package/dist/parquet-plan.mjs

@@ -0,0 +1,139 @@
+import { dimensionToColumn } from "./schema.mjs";
+import { enumeratePartitions } from "./compaction.mjs";
+import { METRIC_EXPR, escapeLike, topLevelPagePredicateSql } from "./sql-fragments.mjs";
+import { buildLogicalPlan } from "gscdump/query/plan";
+const FILES_PLACEHOLDER = "{{FILES}}";
+function buildDimensionWhere(filters, table) {
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const column = dimensionToColumn(filter.dimension, table);
+		switch (filter.operator) {
+			case "equals":
+				clauses.push(`${column} = ?`);
+				params.push(filter.expression);
+				break;
+			case "notEquals":
+				clauses.push(`${column} != ?`);
+				params.push(filter.expression);
+				break;
+			case "contains":
+				clauses.push(`${column} LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "notContains":
+				clauses.push(`${column} NOT LIKE ? ESCAPE '\\'`);
+				params.push(`%${escapeLike(filter.expression)}%`);
+				break;
+			case "includingRegex":
+				clauses.push(`regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+			case "excludingRegex":
+				clauses.push(`NOT regexp_matches(${column}, ?)`);
+				params.push(filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.join(" AND "),
+		params
+	};
+}
+function buildTopLevelWhere(plan, table) {
+	if (!plan.specialFilters.topLevel) return "";
+	return topLevelPagePredicateSql(dimensionToColumn("page", table));
+}
+function buildHaving(filters) {
+	if (filters.length === 0) return {
+		clause: "",
+		params: []
+	};
+	const clauses = [];
+	const params = [];
+	for (const filter of filters) {
+		const expr = METRIC_EXPR[filter.metric];
+		switch (filter.operator) {
+			case "metricGte":
+				clauses.push(`${expr} >= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricGt":
+				clauses.push(`${expr} > ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLte":
+				clauses.push(`${expr} <= ?`);
+				params.push(filter.expression);
+				break;
+			case "metricLt":
+				clauses.push(`${expr} < ?`);
+				params.push(filter.expression);
+				break;
+			case "metricBetween":
+				clauses.push(`${expr} >= ? AND ${expr} <= ?`);
+				params.push(filter.expression, filter.expression2 ?? filter.expression);
+				break;
+		}
+	}
+	return {
+		clause: clauses.length > 0 ? `HAVING ${clauses.join(" AND ")}` : "",
+		params
+	};
+}
+function compileLogicalQueryPlan(plan, table = plan.dataset) {
+	const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
+	const metricSelects = plan.metrics.map((metric) => `${METRIC_EXPR[metric]} AS ${metric}`);
+	const dimSelects = plan.groupByDimensions.map((dimension) => {
+		const column = dimensionToColumn(dimension, table);
+		return column !== dimension ? `${column} AS ${dimension}` : dimension;
+	});
+	const whereClauses = ["date >= ?", "date <= ?"];
+	const whereParams = [plan.dateRange.startDate, plan.dateRange.endDate];
+	const dimWhere = buildDimensionWhere(plan.dimensionFilters, table);
+	if (dimWhere.clause) {
+		whereClauses.push(dimWhere.clause);
+		whereParams.push(...dimWhere.params);
+	}
+	const topLevelClause = buildTopLevelWhere(plan, table);
+	if (topLevelClause) whereClauses.push(topLevelClause);
+	const having = buildHaving(plan.metricFilters);
+	const groupByCols = [...plan.groupByDimensions.map((dimension) => dimensionToColumn(dimension, table)), ...plan.hasDate ? ["date"] : []];
+	const groupBy = groupByCols.length > 0 ? `GROUP BY ${groupByCols.join(", ")}` : "";
+	const orderBy = plan.orderBy ? `ORDER BY ${plan.orderBy.column} ${plan.orderBy.dir.toUpperCase()}` : "ORDER BY clicks DESC";
+	const limit = `LIMIT ${plan.rowLimit ?? 1e3}`;
+	const offset = plan.startRow ? `OFFSET ${plan.startRow}` : "";
+	return {
+		sql: [
+			`SELECT ${[
+				...dimSelects,
+				...plan.hasDate ? ["date"] : [],
+				...metricSelects
+			].join(", ")}`,
+			`FROM read_parquet(${FILES_PLACEHOLDER}, union_by_name = true)`,
+			`WHERE ${whereClauses.join(" AND ")}`,
+			groupBy,
+			having.clause,
+			orderBy,
+			limit,
+			offset
+		].filter(Boolean).join(" ").replace(/\s+/g, " ").trim(),
+		params: [...whereParams, ...having.params],
+		partitions,
+		table,
+		filesPlaceholder: FILES_PLACEHOLDER
+	};
+}
+function resolveParquetSQL(state, table) {
+	const plan = buildLogicalPlan(state, { regex: true });
+	return compileLogicalQueryPlan(plan, table ?? plan.dataset);
+}
+function fileList(keys) {
+	return keys.length === 0 ? "[]" : `[${keys.map((key) => `'${key.replace(/'/g, "''")}'`).join(", ")}]`;
+}
+function substituteNamedFiles(sql, sets) {
+	let out = sql;
+	for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
+	return out;
+}
+export { FILES_PLACEHOLDER, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles };

package/dist/parquet-pushdown.mjs

@@ -0,0 +1,58 @@
+import { SCHEMAS } from "./schema.mjs";
+const PUSHABLE_COLUMN = {
+	country: "country",
+	query: "query",
+	searchAppearance: "searchAppearance"
+};
+function txLeaf(leaf, columns) {
+	if (leaf.operator !== "equals") return null;
+	const column = PUSHABLE_COLUMN[leaf.dimension];
+	if (!column || !columns.has(column)) return null;
+	return { [column]: { $eq: leaf.expression } };
+}
+function combineFilters(parts, groupType) {
+	const first = parts[0];
+	if (!first) return null;
+	if (parts.length === 1) return first;
+	return groupType === "or" ? { $or: [...parts] } : { $and: [...parts] };
+}
+function txExact(node, columns) {
+	const groupType = node._groupType ?? "and";
+	const leafParts = [];
+	for (const leaf of node._filters) {
+		const t = txLeaf(leaf, columns);
+		if (!t) return null;
+		leafParts.push(t);
+	}
+	if (groupType === "or") {
+		if (node._nestedGroups?.length || leafParts.length === 0) return null;
+		return combineFilters(leafParts, "or");
+	}
+	const parts = leafParts;
+	for (const group of node._nestedGroups ?? []) {
+		const t = txExact(group, columns);
+		if (!t) return null;
+		parts.push(t);
+	}
+	if (parts.length === 0) return null;
+	return combineFilters(parts, "and");
+}
+function extractParquetPushdown(state, table) {
+	const filter = state?.filter;
+	const schema = SCHEMAS[table];
+	if (!filter || !schema) return void 0;
+	const columns = new Set(schema.columns.map((c) => c.name));
+	if ((filter._groupType ?? "and") === "or") return txExact(filter, columns) ?? void 0;
+	const parts = [];
+	for (const leaf of filter._filters) {
+		const t = txLeaf(leaf, columns);
+		if (t) parts.push(t);
+	}
+	for (const group of filter._nestedGroups ?? []) {
+		const t = txExact(group, columns);
+		if (t) parts.push(t);
+	}
+	if (parts.length === 0) return void 0;
+	return combineFilters(parts, "and") ?? void 0;
+}
+export { extractParquetPushdown };

package/dist/period/index.d.mts

@@ -1,2 +1,53 @@
-import { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow } from "../_chunks/index.mjs";
+import { AnalysisParams } from "../analysis-types.mjs";
+type WindowPreset = 'last-7d' | 'last-28d' | 'last-30d' | 'last-90d' | 'last-180d' | 'last-365d' | 'mtd' | 'ytd' | 'custom';
+type ComparisonMode = 'none' | 'prev-period' | 'yoy';
+interface ResolveWindowOptions {
+  preset: WindowPreset;
+  comparison?: ComparisonMode;
+  anchor?: string;
+  start?: string;
+  end?: string;
+}
+interface ResolvedWindow {
+  start: string;
+  end: string;
+  days: number;
+  comparison?: {
+    start: string;
+    end: string;
+  };
+}
+interface AnalysisPeriod {
+  startDate: string;
+  endDate: string;
+}
+interface ComparisonPeriod {
+  current: AnalysisPeriod;
+  previous: AnalysisPeriod;
+}
+declare function defaultEndDate(): string;
+declare function periodOf(params: AnalysisParams): AnalysisPeriod;
+declare function comparisonOf(params: AnalysisParams): ComparisonPeriod;
+declare function resolveWindow(opts: ResolveWindowOptions): ResolvedWindow;
+interface PadTimeseriesOptions<T> {
+  /** ISO date (YYYY-MM-DD), inclusive lower bound. */
+  startDate: string;
+  /** ISO date (YYYY-MM-DD), inclusive upper bound. */
+  endDate: string;
+  /**
+   * Row to insert for missing dates. Defaults to `{ clicks: 0, impressions: 0, ctr: 0, position: 0 }`.
+   * The `date` field is set automatically.
+   */
+  fill?: Omit<T, 'date'>;
+  /** Row-field that carries the ISO date. Defaults to `date`. */
+  dateKey?: string;
+}
+type DateRowShape = Record<string, unknown> & {
+  date?: unknown;
+};
+/**
+ * Pad rows so every calendar day in `[startDate, endDate]` appears at least
+ * once. Existing dates keep all their rows (grouped timeseries safe).
+ */
+declare function padTimeseries<T extends DateRowShape = DateRowShape>(rows: readonly T[], options: PadTimeseriesOptions<T>): T[];
 export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };

package/dist/planner.d.mts

@@ -1,15 +1,3 @@
-import { TableName, enumeratePartitions } from "./_chunks/storage.mjs";
-import { LogicalQueryPlan } from "gscdump/query/plan";
-import { BuilderState } from "gscdump/query";
-interface ResolvedQuery {
-  sql: string;
-  params: unknown[];
-  partitions: string[];
-  table: TableName;
-  filesPlaceholder: string;
-}
-declare const FILES_PLACEHOLDER = "{{FILES}}";
-declare function compileLogicalQueryPlan(plan: LogicalQueryPlan, table?: TableName): ResolvedQuery;
-declare function resolveParquetSQL(state: BuilderState, table?: TableName): ResolvedQuery;
-declare function substituteNamedFiles(sql: string, sets: Record<string, string[]>): string;
+import { enumeratePartitions } from "./compaction.mjs";
+import { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles } from "./parquet-plan.mjs";
 export { FILES_PLACEHOLDER, type ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };

package/dist/planner.mjs

@@ -1,2 +1,3 @@
-import { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
+import { enumeratePartitions } from "./compaction.mjs";
+import { FILES_PLACEHOLDER, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles } from "./parquet-plan.mjs";
 export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };

package/dist/profile.d.mts

@@ -0,0 +1,21 @@
+import { QueryProfiler, QuerySpan } from "./storage.mjs";
+/**
+ * Build a {@link QueryProfiler} that records each closed span to `sink`.
+ *
+ * `start(name, meta)` stamps the open time and returns an `end` thunk; calling
+ * `end(extra)` records `{ name, ms, meta }` with `extra` merged over the
+ * open-time `meta` (so completion-only facts — row counts, buffered-file
+ * counts — land on the same span). `now` is injectable for deterministic
+ * tests; it defaults to `Date.now`.
+ */
+declare function createQueryProfiler(sink: (span: QuerySpan) => void, now?: () => number): QueryProfiler;
+/**
+ * A profiler that accumulates closed spans into an array — for tests, the CLI
+ * `query` command, or any ad-hoc "where did the time go" probe. The returned
+ * `spans` array is filled as spans close, in completion order.
+ */
+declare function collectSpans(now?: () => number): {
+  profiler: QueryProfiler;
+  spans: QuerySpan[];
+};
+export { collectSpans, createQueryProfiler };

package/dist/profile.mjs

@@ -0,0 +1,27 @@
+function createQueryProfiler(sink, now = () => Date.now()) {
+	return { start(name, meta) {
+		const t0 = now();
+		return (extra) => {
+			const merged = meta || extra ? {
+				...meta,
+				...extra
+			} : void 0;
+			sink(merged ? {
+				name,
+				ms: now() - t0,
+				meta: merged
+			} : {
+				name,
+				ms: now() - t0
+			});
+		};
+	} };
+}
+function collectSpans(now) {
+	const spans = [];
+	return {
+		profiler: createQueryProfiler((s) => spans.push(s), now),
+		spans
+	};
+}
+export { collectSpans, createQueryProfiler };

package/dist/query-dim.d.mts

@@ -0,0 +1,54 @@
+import { DataSource } from "./storage.mjs";
+import { queryDimMetaKey, queryDimParquetKey } from "./entity-keys.mjs";
+import { TenantCtx } from "@gscdump/contracts";
+interface QueryDimRecord {
+  query: string;
+  /** Lexical canonical, never empty: NULL/'' folds to the raw query. */
+  query_canonical: string;
+  /** Packed search-intent code (see `@gscdump/analysis` `encodeIntent`). */
+  intent_code: number;
+  normalizer_version: number;
+  intent_version: number;
+}
+/** JSON sidecar: versions + freshness, readable without decoding the parquet. */
+interface QueryDimMeta {
+  version: 1;
+  builtAt: number;
+  rowCount: number;
+  normalizerVersion: number;
+  intentVersion: number;
+}
+/**
+ * Injected derivation. `engine` never imports `@gscdump/analysis`; the host
+ * passes `normalizeQuery` / `classifyIntentCode` (e.g. `encodeIntent ∘
+ * classifyQueryIntent`) plus their version constants.
+ */
+interface QueryDimDeps {
+  normalizeQuery: (query: string) => string;
+  normalizerVersion: number;
+  /** Returns the packed intent code for a raw query. */
+  classifyIntentCode: (query: string) => number;
+  intentVersion: number;
+}
+/**
+ * Pure: distinct raw queries → dimension records. De-dupes, drops empties, and
+ * folds an empty/whitespace canonical back to the raw query so the dimension
+ * is total for read-time joins.
+ */
+declare function buildQueryDimRecords(queries: Iterable<string>, deps: QueryDimDeps): QueryDimRecord[];
+interface QueryDimStore {
+  parquetKey: (ctx: TenantCtx) => string;
+  /** Write the parquet + JSON sidecar. Last-write-wins; no history. */
+  write: (ctx: TenantCtx, records: readonly QueryDimRecord[], builtAt: number) => Promise<{
+    parquetKey: string;
+    rowCount: number;
+  }>;
+  /** Read the sidecar (versions + freshness), or null on first build. */
+  loadMeta: (ctx: TenantCtx) => Promise<QueryDimMeta | null>;
+  /** Decode the dimension rows (test/inspection; reads JOIN the parquet by key). */
+  loadRecords: (ctx: TenantCtx) => Promise<QueryDimRecord[]>;
+}
+declare function createQueryDimStore({ dataSource }: {
+  dataSource: DataSource;
+}): QueryDimStore;
+export { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore };

package/dist/query-dim.mjs

@@ -0,0 +1,91 @@
+import { queryDimMetaKey, queryDimParquetKey } from "./entity-keys.mjs";
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
+import { readOptional } from "./adapters/read-optional.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+const QUERY_DIM_COLUMNS = [
+	{
+		name: "query",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "query_canonical",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "intent_code",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "normalizer_version",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "intent_version",
+		type: "INTEGER",
+		nullable: false
+	}
+];
+function buildQueryDimRecords(queries, deps) {
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	for (const raw of queries) {
+		const query = String(raw);
+		if (query.trim() === "" || seen.has(query)) continue;
+		seen.add(query);
+		const canonical = deps.normalizeQuery(query);
+		out.push({
+			query,
+			query_canonical: canonical === "" ? query : canonical,
+			intent_code: deps.classifyIntentCode(query),
+			normalizer_version: deps.normalizerVersion,
+			intent_version: deps.intentVersion
+		});
+	}
+	return out;
+}
+function createQueryDimStore({ dataSource }) {
+	return {
+		parquetKey: queryDimParquetKey,
+		async write(ctx, records, builtAt) {
+			const parquetKey = queryDimParquetKey(ctx);
+			const bytes = encodeRowsToParquetFlex(records, {
+				columns: QUERY_DIM_COLUMNS,
+				sortKey: ["query"]
+			});
+			await dataSource.write(parquetKey, bytes);
+			const meta = {
+				version: 1,
+				builtAt,
+				rowCount: records.length,
+				normalizerVersion: records[0]?.normalizer_version ?? 0,
+				intentVersion: records[0]?.intent_version ?? 0
+			};
+			await dataSource.write(queryDimMetaKey(ctx), encodeJsonBigintSafe(meta));
+			return {
+				parquetKey,
+				rowCount: records.length
+			};
+		},
+		async loadMeta(ctx) {
+			const bytes = await readOptional(dataSource, queryDimMetaKey(ctx));
+			if (!bytes) return null;
+			return JSON.parse(new TextDecoder().decode(bytes));
+		},
+		async loadRecords(ctx) {
+			const bytes = await readOptional(dataSource, queryDimParquetKey(ctx));
+			if (!bytes) return [];
+			return (await decodeParquetToRows(bytes)).map((r) => ({
+				query: String(r.query),
+				query_canonical: String(r.query_canonical),
+				intent_code: Number(r.intent_code),
+				normalizer_version: Number(r.normalizer_version),
+				intent_version: Number(r.intent_version)
+			}));
+		}
+	};
+}
+export { buildQueryDimRecords, createQueryDimStore };

package/dist/report/define.d.mts

@@ -0,0 +1,8 @@
+import { DefineReportOptions, DefinedReport, ReportParams } from "./types.mjs";
+/**
+ * Mirror of `defineAnalyzer`. Pure factory: validates required fields,
+ * fills default `argsSpec`. No runtime behaviour — `runReport` consumes
+ * the returned object.
+ */
+declare function defineReport<P extends ReportParams = ReportParams>(opts: DefineReportOptions<P>): DefinedReport<P>;
+export { defineReport };

package/dist/report/define.mjs

@@ -0,0 +1,15 @@
+function defineReport(opts) {
+	if (!opts.id) throw new Error("defineReport: id is required");
+	if (!opts.plan) throw new Error(`defineReport(${opts.id}): plan is required`);
+	if (!opts.reduce) throw new Error(`defineReport(${opts.id}): reduce is required`);
+	return {
+		id: opts.id,
+		description: opts.description,
+		defaultPeriod: opts.defaultPeriod,
+		defaultComparison: opts.defaultComparison,
+		argsSpec: opts.argsSpec ?? {},
+		plan: opts.plan,
+		reduce: opts.reduce
+	};
+}
+export { defineReport };

package/dist/report/hash.d.mts

@@ -0,0 +1,13 @@
+import { ResolvedWindow } from "../period/index.mjs";
+import { ReportParams } from "./types.mjs";
+interface InputHashSeeds {
+  id: string;
+  site: string;
+  window: ResolvedWindow;
+  params: ReportParams;
+  registryVersion: string;
+}
+/** Stable JSON: sorts object keys at every level. Arrays preserve order. */
+declare function canonicalize(value: unknown): unknown;
+declare function computeInputHash(seeds: InputHashSeeds): Promise<string>;
+export { InputHashSeeds, canonicalize, computeInputHash };

package/dist/report/hash.mjs

@@ -0,0 +1,29 @@
+function canonicalize(value) {
+	if (value == null || typeof value !== "object") return value;
+	if (Array.isArray(value)) return value.map(canonicalize);
+	const out = {};
+	for (const k of Object.keys(value).sort()) out[k] = canonicalize(value[k]);
+	return out;
+}
+async function computeInputHash(seeds) {
+	const payload = JSON.stringify(canonicalize({
+		id: seeds.id,
+		site: seeds.site,
+		window: {
+			start: seeds.window.start,
+			end: seeds.window.end,
+			comparison: seeds.window.comparison ?? null
+		},
+		params: seeds.params,
+		registryVersion: seeds.registryVersion
+	}));
+	const bytes = new TextEncoder().encode(payload);
+	return bufferToHex(await globalThis.crypto.subtle.digest("SHA-256", bytes));
+}
+function bufferToHex(buffer) {
+	const bytes = new Uint8Array(buffer);
+	let out = "";
+	for (let i = 0; i < bytes.length; i++) out += bytes[i].toString(16).padStart(2, "0");
+	return out;
+}
+export { canonicalize, computeInputHash };