@gscdump/engine 0.6.1 → 0.6.3

package/dist/adapters/hyparquet.mjs

@@ -1,110 +1,6 @@
+import { s as TABLE_METADATA, t as SCHEMAS } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "hyparquet";
 import { parquetWriteBuffer } from "hyparquet-writer";
-import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
-function metricCols() {
-	return {
-		clicks: integer("clicks").notNull(),
-		impressions: integer("impressions").notNull(),
-		sum_position: doublePrecision("sum_position").notNull()
-	};
-}
-const dateCol = () => date("date").notNull();
-const drizzleSchema = {
-	pages: pgTable("pages", {
-		url: varchar("url").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	keywords: pgTable("keywords", {
-		query: varchar("query").notNull(),
-		query_canonical: varchar("query_canonical"),
-		date: dateCol(),
-		...metricCols()
-	}),
-	countries: pgTable("countries", {
-		country: varchar("country").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	devices: pgTable("devices", {
-		device: varchar("device").notNull(),
-		date: dateCol(),
-		...metricCols()
-	}),
-	page_keywords: pgTable("page_keywords", {
-		url: varchar("url").notNull(),
-		query: varchar("query").notNull(),
-		query_canonical: varchar("query_canonical"),
-		date: dateCol(),
-		...metricCols()
-	}),
-	search_appearance: pgTable("search_appearance", {
-		searchAppearance: varchar("searchAppearance").notNull(),
-		date: dateCol(),
-		...metricCols()
-	})
-};
-const TABLE_METADATA = {
-	pages: {
-		sortKey: ["date", "url"],
-		version: 1
-	},
-	keywords: {
-		sortKey: ["date", "query"],
-		version: 2
-	},
-	countries: {
-		sortKey: ["date", "country"],
-		version: 1
-	},
-	devices: {
-		sortKey: ["date", "device"],
-		version: 1
-	},
-	page_keywords: {
-		sortKey: [
-			"date",
-			"url",
-			"query"
-		],
-		version: 2
-	},
-	search_appearance: {
-		sortKey: ["date", "searchAppearance"],
-		version: 1
-	}
-};
-function pgSqlTypeToColumnType(sqlType) {
-	const t = sqlType.toLowerCase();
-	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
-	if (t === "date" || t.startsWith("timestamp")) return "DATE";
-	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
-	if (t === "bigint" || t === "int8") return "BIGINT";
-	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
-	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
-}
-function tableSchemaFrom(tableName) {
-	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
-		name: col.name,
-		type: pgSqlTypeToColumnType(col.getSQLType()),
-		nullable: !col.notNull
-	}));
-	const meta = TABLE_METADATA[tableName];
-	return {
-		name: tableName,
-		columns,
-		sortKey: meta.sortKey,
-		version: meta.version
-	};
-}
-const SCHEMAS = Object.fromEntries([
-	"pages",
-	"keywords",
-	"countries",
-	"devices",
-	"page_keywords",
-	"search_appearance"
-].map((t) => [t, tableSchemaFrom(t)]));
 const ROW_GROUP_SIZE = 25e3;
 function basicTypeFor(colType) {
 	if (colType === "VARCHAR" || colType === "DATE") return "STRING";

package/dist/adapters/inspection-sqlite-browser.d.mts

@@ -1,9 +1,3 @@
-interface InspectionSqlDriver {
-  exec: (sql: string) => void | Promise<void>;
-  run: (sql: string, params: unknown[]) => void | Promise<void>;
-  all: (sql: string, params: unknown[]) => unknown[] | Promise<unknown[]>;
-  serialize: () => Uint8Array | Promise<Uint8Array>;
-  close: () => void | Promise<void>;
-}
+import { InspectionSqlDriver } from "../entities.mjs";
 declare function createWaSqliteDriver(bytes: Uint8Array | undefined): Promise<InspectionSqlDriver>;
 export { createWaSqliteDriver };

package/dist/adapters/inspection-sqlite-node.d.mts

@@ -1,9 +1,3 @@
-interface InspectionSqlDriver {
-  exec: (sql: string) => void | Promise<void>;
-  run: (sql: string, params: unknown[]) => void | Promise<void>;
-  all: (sql: string, params: unknown[]) => unknown[] | Promise<unknown[]>;
-  serialize: () => Uint8Array | Promise<Uint8Array>;
-  close: () => void | Promise<void>;
-}
+import { InspectionSqlDriver } from "../entities.mjs";
 declare function createBetterSqliteDriver(bytes: Uint8Array | undefined): InspectionSqlDriver;
 export { createBetterSqliteDriver };

package/dist/adapters/inspection-sqlite-node.mjs

@@ -1,7 +1,7 @@
 import { createRequire } from "node:module";
-import { Buffer } from "node:buffer";
 import process from "node:process";
 import { fileURLToPath } from "node:url";
+import { Buffer } from "node:buffer";
 const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
 function loadBetterSqlite() {
 	const mod = require_("better-sqlite3");

package/dist/adapters/node-harness.d.mts

@@ -1,307 +1,4 @@
-import { Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx } from "gscdump/contracts";
-import { BuilderState, SearchType } from "gscdump/query";
-/**
- * Per-tier age threshold in days. Default ladder collapses on these gates:
- * - raw → d7 once a daily file is older than `raw` days (default 7).
- * - d7 → d30 once the entire weekly bucket sits behind `d7` days (default 30).
- * - d30 → d90 once the entire monthly bucket sits behind `d30` days (default 90).
- */
-interface CompactionThresholds {
-  raw?: number;
-  d7?: number;
-  d30?: number;
-}
-type ComparisonFilter = 'new' | 'lost' | 'improving' | 'declining';
-interface WriteCtx extends TenantCtx {
-  table: TableName;
-  date?: string;
-  now?: () => number;
-  /**
-   * GSC search-type partition this write belongs to. Defaults to `'web'`.
-   * Non-web values (`discover`, `news`, `googleNews`, `image`, `video`)
-   * cause the writer to insert the type into the object key path so files
-   * for different search types coexist without colliding.
-   */
-  searchType?: SearchType;
-}
-interface QueryCtx extends TenantCtx {
-  table?: TableName;
-  signal?: AbortSignal;
-}
-interface GcCtx {
-  now?: () => number;
-  userId?: string;
-  siteId?: string;
-}
-/**
- * Compaction tier of a manifest entry. Determines which compactor stage may
- * pick it up as input:
- * - `raw`: per-day file produced by `writeDay`. Eligible for raw→d7 merge at 7d.
- * - `d7`: weekly compaction output. Eligible for d7→d30 merge at 30d.
- * - `d30`: monthly compaction output (matches the legacy `monthly/` partition
- *   shape — pre-tier entries are read as `d30`). Eligible for d30→d90 at 90d.
- * - `d90`: quarterly cold-tier output. Terminal; never recompacted.
- *
- * Without an explicit tier, entries written before this field landed default
- * to `raw` for `daily/` partitions and `d30` for `monthly/` partitions, so
- * the tiered compactor picks the right inputs without a backfill rewrite.
- */
-type CompactionTier = 'raw' | 'd7' | 'd30' | 'd90';
-interface ManifestEntry {
-  userId: string;
-  siteId?: string;
-  table: TableName;
-  partition: string;
-  objectKey: string;
-  rowCount: number;
-  bytes: number;
-  createdAt: number;
-  retiredAt?: number;
-  /** Table schema version at write time. Omitted on pre-#27 entries — treat as 1. */
-  schemaVersion?: number;
-  /**
-   * Compaction tier. Omitted on entries written before tiered compaction —
-   * treat as `raw` for `daily/` partitions and `d30` for `monthly/` partitions
-   * (see {@link inferLegacyTier}).
-   */
-  tier?: CompactionTier;
-  /**
-   * GSC search-type this entry covers (web | discover | news | googleNews |
-   * image | video). Omitted on entries written before per-type partitioning
-   * landed — treat as `web` (see {@link inferSearchType}). Compaction merges
-   * only entries with the same searchType.
-   */
-  searchType?: SearchType;
-}
-interface ListLiveFilter {
-  userId: string;
-  siteId?: string;
-  table?: TableName;
-  partitions?: string[];
-  /**
-   * Narrow to a single compaction tier. Tier-aware compaction stages set this
-   * so the store doesn't have to return (and the caller doesn't have to scan)
-   * the entire manifest just to compact the raw cohort. Legacy entries without
-   * an explicit `tier` field match on {@link inferLegacyTier}.
-   */
-  tier?: CompactionTier;
-}
-interface DataSource {
-  read: (key: string, range?: {
-    offset: number;
-    length: number;
-  }, signal?: AbortSignal) => Promise<Uint8Array>;
-  write: (key: string, bytes: Uint8Array) => Promise<void>;
-  delete: (keys: string[]) => Promise<void>;
-  /**
-   * One-shot listing under a prefix. Implementations may cap the number of
-   * returned keys (typically 10k) — callers iterating full tenant space
-   * should prefer `streamList` when available or narrow the prefix.
-   */
-  list: (prefix: string) => Promise<string[]>;
-  /**
-   * Per-key URI probe. Returns a URI string DuckDB's `httpfs` (or an
-   * equivalent engine that fetches its own I/O) can read directly, or
-   * `undefined` if the key isn't URI-resolvable on this backend and the
-   * caller must fall back to `read(key)` for the bytes.
-   *
-   * Contracts:
-   * - When defined, the returned URI MUST yield byte-identical content to
-   *   `read(key)`. Callers rely on this for correctness.
-   * - Backends with a native URI for every key (filesystem: absolute path,
-   *   R2 via `httpfs`: signed URL) may always return a string.
-   * - Backends without a native URI shape (in-memory) omit the method or
-   *   return `undefined` per call.
-   * - Mixed-per-query is allowed: some keys in one query may return a URI,
-   *   others may not; the executor branches per key.
-   */
-  uri?: (key: string) => string | undefined;
-  /**
-   * Optional — probe the byte size of a key without reading it. Used by
-   * the engine to fill in `WriteResult.bytes` when a codec reports 0 or
-   * unknown but the file is non-trivial.
-   */
-  head?: (key: string) => Promise<{
-    bytes: number;
-  } | undefined>;
-  /**
-   * Optional streaming variant of `list`. Implementations that page
-   * backing-store results (R2, S3) should implement this and yield keys
-   * lazily. `list` may return up to an adapter-defined cap (typically
-   * 10k keys); callers iterating full tenant space must prefer
-   * `streamList` when available, or chunk by narrower prefixes.
-   */
-  streamList?: (prefix: string) => AsyncIterable<string>;
-}
-interface WatermarkScope {
-  userId: string;
-  siteId?: string;
-  table: TableName;
-}
-interface Watermark extends WatermarkScope {
-  newestDateSynced: string;
-  oldestDateSynced: string;
-  lastSyncAt: number;
-}
-interface WatermarkFilter {
-  userId: string;
-  siteId?: string;
-  table?: TableName;
-}
-type SyncStateKind = 'pending' | 'inflight' | 'done' | 'failed';
-interface SyncStateScope {
-  userId: string;
-  siteId?: string;
-  table: TableName;
-  date: string;
-  /**
-   * GSC search-type this sync state covers. Omitted = `web` (the legacy
-   * default; matches pre-#5 sync states stored before per-type sync landed).
-   * Lookups must compare via {@link inferSearchType} so a missing field
-   * matches an explicit `'web'` and vice versa.
-   */
-  searchType?: SearchType;
-}
-interface SyncState extends SyncStateScope {
-  state: SyncStateKind;
-  updatedAt: number;
-  attempts: number;
-  error?: string;
-}
-interface SyncStateFilter {
-  userId: string;
-  siteId?: string;
-  table?: TableName;
-  state?: SyncStateKind;
-  searchType?: SearchType;
-}
-interface SyncStateDetail {
-  at?: number;
-  error?: string;
-}
-interface PurgeResult {
-  userId: string;
-  siteId?: string;
-  prefix: string;
-  objectsDeleted: number;
-  entriesRemoved: number;
-  watermarksRemoved: number;
-  syncStatesRemoved: number;
-  at: number;
-}
-interface PurgeUrlsResult {
-  userId: string;
-  siteId?: string;
-  urlsRequested: number;
-  entriesRewritten: number;
-  rowsRemoved: number;
-  bytesAfter: number;
-  at: number;
-}
-interface QueryResult {
-  rows: Row[];
-  sql: string;
-  objectKeys: string[];
-}
-interface ComparisonResult {
-  rows: Row[];
-  totalCount: number;
-  totals: Record<string, unknown>;
-}
-interface ExtraResult {
-  key: string;
-  rows: Row[];
-}
-interface FileSetRef {
-  table: TableName;
-  partitions?: string[];
-}
-interface RunSQLOptions {
-  ctx: TenantCtx;
-  /**
-   * Named partition references. Each name becomes a `{{NAME}}` placeholder
-   * substituted into the SQL with the matching list of object keys. The
-   * canonical name is `FILES`; analyzers also use `FILES_PREV` for a prior
-   * window. Providing zero fileSets runs the SQL against no files.
-   */
-  fileSets: Record<string, FileSetRef>;
-  /** Schema-bearing table; defaults to the first fileSet's table. */
-  table?: TableName;
-  sql: string;
-  params?: unknown[];
-  signal?: AbortSignal;
-}
-interface StorageEngine {
-  writeDay: (ctx: WriteCtx, rows: Row[]) => Promise<void>;
-  query: (ctx: QueryCtx, state: BuilderState) => Promise<QueryResult>;
-  /**
-   * Two-window comparison query (resolver-compiled). Joins a `current` and
-   * `previous` window CTE on dimensions, applies an optional row filter
-   * (`new`/`lost`/`improving`/`declining`), and returns the merged rows plus
-   * total count and unfiltered totals.
-   *
-   * Tenant scoping comes from `ctx.userId`/`ctx.siteId` (manifest lookup) —
-   * the SQL itself is single-tenant against the parquet adapter, which has
-   * `includeSiteId: false`.
-   *
-   * Throws if `current` and `previous` resolve to different tables.
-   */
-  queryComparison: (ctx: QueryCtx, current: BuilderState, previous: BuilderState, filter?: ComparisonFilter) => Promise<ComparisonResult>;
-  /**
-   * Canonical-variant enrichment queries. Returns one result per extra
-   * surface; today only `queryCanonical` triggers an extra. Empty array
-   * when the state has no extras-eligible dimensions.
-   */
-  queryExtras: (ctx: QueryCtx, state: BuilderState) => Promise<ExtraResult[]>;
-  /**
-   * Run arbitrary SQL resolved against named partition sets. Composes
-   * manifest lookup + object reads + placeholder substitution + execution
-   * so callers don't need to reach into `ManifestStore`/`DataSource`
-   * directly.
-   */
-  runSQL: (opts: RunSQLOptions) => Promise<QueryResult>;
-  compactTiered: (ctx: WriteCtx, thresholds?: CompactionThresholds) => Promise<void>;
-  gcOrphans: (ctx: GcCtx, graceMs: number) => Promise<{
-    deleted: number;
-  }>;
-  /**
-   * GDPR-grade tenant purge. Deletes every object under the tenant prefix
-   * (parquet, rollups, entity stores), then removes manifest/watermark/
-   * sync-state records via {@link ManifestStore.purgeTenant}.
-   *
-   * Order matters: bytes are deleted before manifest entries, so a
-   * crash mid-purge leaves orphan manifest records (detectable via the
-   * normal orphan sweep) rather than orphan bytes with no record.
-   *
-   * Returns counters suitable for an audit log. Caller is responsible
-   * for persisting the audit entry.
-   */
-  purgeTenant: (ctx: TenantCtx) => Promise<PurgeResult>;
-  /**
-   * GDPR URL-matcher purge. Deletes rows whose `url` column matches one of
-   * `urls` across every live parquet entry for the tenant in tables that
-   * carry a `url` column (`pages`, `page_keywords`). Tables without a `url`
-   * column (`keywords`, `countries`, `devices`, `search_appearance`) are
-   * untouched — they never store per-URL data.
-   *
-   * For each affected entry the engine reads the file, filters the matching
-   * rows out, writes a replacement parquet at a new object key, and registers
-   * the new entry as a supersede of the old. Entries with no matches are
-   * left untouched. Entries with all rows matching are replaced by a
-   * schema-bearing empty-rows file.
-   *
-   * Narrower counterpart to {@link purgeTenant}: use this for a per-URL
-   * takedown request; use `purgeTenant` for full-account deletion.
-   */
-  purgeUrls: (ctx: TenantCtx, urls: readonly string[]) => Promise<PurgeUrlsResult>;
-  listLive: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
-  listAll: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
-  getWatermarks: (filter: WatermarkFilter) => Promise<Watermark[]>;
-  getSyncStates: (filter: SyncStateFilter) => Promise<SyncState[]>;
-  setSyncState: (scope: SyncStateScope, state: SyncStateKind, detail?: SyncStateDetail) => Promise<void>;
-  /** Read the raw bytes of a single object. Rarely needed outside the `dump` CLI. */
-  readObject: (key: string) => Promise<Uint8Array>;
-}
+import { D as StorageEngine, N as TableName, a as DataSource, w as Row } from "../_chunks/storage.mjs";
 interface NodeHarnessOptions {
   dataDir: string;
   /** Tenant user id. Defaults to `'local'` for single-user CLI installs. */
@@ -322,10 +19,10 @@ interface NodeHarness {
   runRawSql: (opts: {
     sql: string;
     siteUrl: string;
-    table: TableName$1;
+    table: TableName;
     params?: unknown[];
   }) => Promise<{
-    rows: Row$1[];
+    rows: Row[];
     sql: string;
     keys: string[];
   }>;