@gscdump/engine 3.5.0 → 3.6.1

package/dist/storage.d.mts

@@ -1,9 +1,9 @@
 import { CompactionThresholds } from "./compaction.mjs";
-import { ParquetQueryFilter } from "./node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/index.mjs";
-import "./node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
+import { ParquetQueryFilter } from "./node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/index.mjs";
+import "./node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/node.mjs";
 import { Grain, Grain as Grain$1, Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
 import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
-interface WriteCtx extends TenantCtx {
+export interface WriteCtx extends TenantCtx {
   table: TableName;
   date?: string;
   now?: () => number;
@@ -26,7 +26,7 @@ interface WriteCtx extends TenantCtx {
  * wall-clock cost and optional dimensional meta (file counts, row counts).
  * Emitted by an injected {@link QueryProfiler}; see `./profile.ts`.
  */
-interface QuerySpan {
+export interface QuerySpan {
   readonly name: string;
   readonly ms: number;
   readonly meta?: Readonly<Record<string, string | number | boolean>>;
@@ -37,10 +37,10 @@ interface QuerySpan {
  * Absent by default — every emit site optional-chains it, so an unprofiled
  * query pays nothing. Build one with `createQueryProfiler` / `collectSpans`.
  */
-interface QueryProfiler {
+export interface QueryProfiler {
   readonly start: (name: string, meta?: Record<string, string | number | boolean>) => (extra?: Record<string, string | number | boolean>) => void;
 }
-interface QueryCtx extends TenantCtx {
+export interface QueryCtx extends TenantCtx {
   table?: TableName;
   signal?: AbortSignal;
   /** Optional read-path profiler; forwarded into `runSQL` and the executor. */
@@ -60,7 +60,7 @@ interface QueryCtx extends TenantCtx {
    */
   grain?: Grain;
 }
-interface GcCtx {
+export interface GcCtx {
   now?: () => number;
   userId?: string;
   siteId?: string;
@@ -84,8 +84,8 @@ interface GcCtx {
  * to `raw` for `daily/` partitions and `d30` for `monthly/` partitions, so
  * the tiered compactor picks the right inputs without a backfill rewrite.
  */
-type CompactionTier = 'raw' | 'd7' | 'd30' | 'd90';
-interface ManifestEntry {
+export type CompactionTier = 'raw' | 'd7' | 'd30' | 'd90';
+export interface ManifestEntry {
   userId: string;
   siteId?: string;
   table: TableName;
@@ -111,7 +111,7 @@ interface ManifestEntry {
    */
   searchType?: SearchType;
 }
-interface ListLiveFilter {
+export interface ListLiveFilter {
   userId: string;
   siteId?: string;
   table?: TableName;
@@ -136,7 +136,7 @@ interface ListLiveFilter {
    */
   searchType?: SearchType;
 }
-interface DataSource {
+export interface DataSource {
   read: (key: string, range?: {
     offset: number;
     length: number;
@@ -183,7 +183,7 @@ interface DataSource {
    */
   streamList?: (prefix: string) => AsyncIterable<string>;
 }
-interface WatermarkScope {
+export interface WatermarkScope {
   userId: string;
   siteId?: string;
   table: TableName;
@@ -193,19 +193,19 @@ interface WatermarkScope {
    */
   searchType?: SearchType;
 }
-interface Watermark extends WatermarkScope {
+export interface Watermark extends WatermarkScope {
   newestDateSynced: string;
   oldestDateSynced: string;
   lastSyncAt: number;
 }
-interface WatermarkFilter {
+export interface WatermarkFilter {
   userId: string;
   siteId?: string;
   table?: TableName;
   searchType?: SearchType;
 }
-type SyncStateKind = 'pending' | 'inflight' | 'done' | 'failed';
-interface SyncStateScope {
+export type SyncStateKind = 'pending' | 'inflight' | 'done' | 'failed';
+export interface SyncStateScope {
   userId: string;
   siteId?: string;
   table: TableName;
@@ -218,39 +218,39 @@ interface SyncStateScope {
    */
   searchType?: SearchType;
 }
-interface SyncState extends SyncStateScope {
+export interface SyncState extends SyncStateScope {
   state: SyncStateKind;
   updatedAt: number;
   attempts: number;
   error?: string;
 }
-interface SyncStateFilter {
+export interface SyncStateFilter {
   userId: string;
   siteId?: string;
   table?: TableName;
   state?: SyncStateKind;
   searchType?: SearchType;
 }
-interface SyncStateDetail {
+export interface SyncStateDetail {
   at?: number;
   error?: string;
 }
-interface LockScope {
+export interface LockScope {
   userId: string;
   siteId?: string;
   table: TableName;
   partition: string;
 }
-interface PurgeFilter {
+export interface PurgeFilter {
   userId: string;
   siteId?: string;
 }
-interface ManifestPurgeResult {
+export interface ManifestPurgeResult {
   entriesRemoved: number;
   watermarksRemoved: number;
   syncStatesRemoved: number;
 }
-interface PurgeResult {
+export interface PurgeResult {
   userId: string;
   siteId?: string;
   prefix: string;
@@ -260,7 +260,7 @@ interface PurgeResult {
   syncStatesRemoved: number;
   at: number;
 }
-interface PurgeUrlsResult {
+export interface PurgeUrlsResult {
   userId: string;
   siteId?: string;
   urlsRequested: number;
@@ -269,7 +269,7 @@ interface PurgeUrlsResult {
   bytesAfter: number;
   at: number;
 }
-interface ManifestStore {
+export interface ManifestStore {
   listLive: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
   listAll: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
   registerVersion: (entry: ManifestEntry, superseding?: ManifestEntry[]) => Promise<void>;
@@ -300,11 +300,11 @@ interface ManifestStore {
    */
   purgeTenant: (filter: PurgeFilter) => Promise<ManifestPurgeResult>;
 }
-interface WriteResult {
+export interface WriteResult {
   bytes: number;
   rowCount: number;
 }
-interface CodecCtx {
+export interface CodecCtx {
   table: TableName;
 }
 /**
@@ -326,17 +326,17 @@ interface CodecCtx {
  * - `WriteResult.rowCount` MUST equal `rows.length` (or, for
  *   `compactRows`, the sum of input row counts).
  */
-interface ParquetCodec {
+export interface ParquetCodec {
   writeRows: (ctx: CodecCtx, rows: Row[], key: string, dataSource: DataSource) => Promise<WriteResult>;
   readRows: (ctx: CodecCtx, key: string, dataSource: DataSource) => Promise<Row[]>;
   compactRows: (ctx: CodecCtx, inputKeys: string[], outputKey: string, dataSource: DataSource) => Promise<WriteResult>;
 }
-interface QueryResult {
+export interface QueryResult {
   rows: Row[];
   sql: string;
   objectKeys: string[];
 }
-interface QueryExecuteOptions {
+export interface QueryExecuteOptions {
   sql: string;
   params: unknown[];
   /**
@@ -382,7 +382,7 @@ interface QueryExecuteOptions {
    */
   profiler?: QueryProfiler;
 }
-interface QueryExecuteResult {
+export interface QueryExecuteResult {
   rows: Row[];
   /** The final SQL actually run (after placeholder substitution). */
   sql: string;
@@ -402,10 +402,10 @@ interface QueryExecuteResult {
     resetRecommended?: boolean;
   };
 }
-interface QueryExecutor {
+export interface QueryExecutor {
   execute: (opts: QueryExecuteOptions) => Promise<QueryExecuteResult>;
 }
-interface FileSetRef {
+export interface FileSetRef {
   table: TableName;
   partitions?: string[];
   /**
@@ -418,7 +418,7 @@ interface FileSetRef {
    */
   keys?: string[];
 }
-interface RunSQLOptions {
+export interface RunSQLOptions {
   ctx: TenantCtx;
   /**
    * Named partition references. Each name becomes a `{{NAME}}` placeholder
@@ -453,7 +453,7 @@ interface RunSQLOptions {
    */
   profiler?: QueryProfiler;
 }
-interface StorageEngine {
+export interface StorageEngine {
   writeDay: (ctx: WriteCtx, rows: Row[]) => Promise<void>;
   /**
    * Read-merge-write a single-day hourly partition. Idempotent on
@@ -537,11 +537,11 @@ interface StorageEngine {
   /** Read the raw bytes of a single object. Rarely needed outside the `dump` CLI. */
   readObject: (key: string) => Promise<Uint8Array>;
 }
-interface EngineOptions {
+export interface EngineOptions {
   dataSource: DataSource;
   manifestStore: ManifestStore;
   codec: ParquetCodec;
   executor: QueryExecutor;
   now?: () => number;
 }
-export { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult };
+export type { Grain$1 as Grain, Row$1 as Row, SearchType$1 as SearchType, TableName$1 as TableName, TenantCtx$1 as TenantCtx };

package/dist/sync-config.d.mts

@@ -2,35 +2,34 @@ import { SearchType } from "./storage.mjs";
 import { EngineError } from "./errors.mjs";
 import { Result } from "gscdump/result";
 import { TableName } from "@gscdump/contracts";
-type SyncTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates'>;
-declare const TABLES_BY_SEARCH_TYPE: Record<SearchType, readonly SyncTableName[]>;
-declare function parseEnabledSearchTypes(raw: string | null | undefined): SearchType[];
+export type SyncTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates'>;
+export declare const TABLES_BY_SEARCH_TYPE: Record<SearchType, readonly SyncTableName[]>;
+export declare function parseEnabledSearchTypes(raw: string | null | undefined): SearchType[];
 /**
  * Errors-as-values core for {@link validateEnabledSearchTypes}: returns a typed
  * `invalid-search-types` `EngineError` instead of throwing, so a host saving the
  * persisted config can map a bad value to a 4xx rather than a 500.
  */
-declare function validateEnabledSearchTypesResult(value: unknown): Result<SearchType[], EngineError>;
-declare function validateEnabledSearchTypes(value: unknown): SearchType[];
-declare const TABLE_TIERS: {
+export declare function validateEnabledSearchTypesResult(value: unknown): Result<SearchType[], EngineError>;
+export declare function validateEnabledSearchTypes(value: unknown): SearchType[];
+export declare const TABLE_TIERS: {
   readonly pages: "critical";
   readonly queries: "critical";
   readonly countries: "standard";
   readonly dates: "standard";
   readonly page_queries: "extended";
 };
-type TieredTableName = keyof typeof TABLE_TIERS;
-type TableTier = 'critical' | 'standard' | 'extended';
-type DateWeight = 'fresh' | 'recent' | 'historical';
-declare function getTableTier(table: string): TableTier;
-declare function getTablesForTier(tier: TableTier): TieredTableName[];
-declare function getDateWeight(date: string, now?: Date): DateWeight;
-declare const TIER_PRIORITY: Record<TableTier, number>;
-declare const WEIGHT_PRIORITY: Record<DateWeight, number>;
-declare const MAX_GSC_PAGES_R2 = 40;
-declare const ROW_LIMIT_R2 = 25000;
-declare const MIN_SYNC_IMPRESSIONS = 1;
-declare const MIN_COUNTRY_IMPRESSIONS = 10;
-declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
-declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { DateWeight, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, TIER_PRIORITY, TableTier, TieredTableName, WEIGHT_PRIORITY, getDateWeight, getTableTier, getTablesForTier, parseEnabledSearchTypes, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export type TieredTableName = keyof typeof TABLE_TIERS;
+export type TableTier = 'critical' | 'standard' | 'extended';
+export type DateWeight = 'fresh' | 'recent' | 'historical';
+export declare function getTableTier(table: string): TableTier;
+export declare function getTablesForTier(tier: TableTier): TieredTableName[];
+export declare function getDateWeight(date: string, now?: Date): DateWeight;
+export declare const TIER_PRIORITY: Record<TableTier, number>;
+export declare const WEIGHT_PRIORITY: Record<DateWeight, number>;
+export declare const MAX_GSC_PAGES_R2 = 40;
+export declare const ROW_LIMIT_R2 = 25000;
+export declare const MIN_SYNC_IMPRESSIONS = 1;
+export declare const MIN_COUNTRY_IMPRESSIONS = 10;
+export declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
+export declare const MAX_TRACKED_URLS_PER_SITE = 200000;

package/dist/vendor/hysnappy-purejs.d.mts

@@ -23,7 +23,6 @@
  * decompressor immediately — no WASM compilation, so it is safe to call at
  * module top level inside `workerd`.
  */
-declare function snappyUncompressor(): (input: Uint8Array, outputLength: number) => Uint8Array;
+export declare function snappyUncompressor(): (input: Uint8Array, outputLength: number) => Uint8Array;
 /** Pure-JS stand-in for `hysnappy`'s `snappyUncompress(input, outputLength)`. */
-declare function snappyUncompress(input: Uint8Array, outputLength: number): Uint8Array;
-export { snappyUncompress, snappyUncompressor };
+export declare function snappyUncompress(input: Uint8Array, outputLength: number): Uint8Array;

package/dist/vendor/hysnappy-purejs.mjs

@@ -1,4 +1,4 @@
-import { snappyUncompress as snappyUncompress$1 } from "../node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/snappy.mjs";
+import { snappyUncompress as snappyUncompress$1 } from "../node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/snappy.mjs";
 function decode(input, outputLength) {
 	const output = new Uint8Array(outputLength);
 	snappyUncompress$1(input, output);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "3.5.0",
+  "version": "3.6.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,31 +181,33 @@
     "node": ">=22"
   },
   "peerDependencies": {
-    "@duckdb/duckdb-wasm": "^1.33.1-dev57.0"
+    "@duckdb/node-api": "^1.5.5-r.4"
   },
   "peerDependenciesMeta": {
-    "@duckdb/duckdb-wasm": {
+    "@duckdb/node-api": {
       "optional": true
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^3.5.0",
-    "@gscdump/lakehouse": "^3.5.0",
+    "@gscdump/contracts": "^3.6.1",
+    "@gscdump/lakehouse": "^3.6.1",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^3.5.0",
+    "gscdump": "^3.6.1",
+    "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",
-    "@types/node": "^26.4.0",
+    "@duckdb/node-api": "1.5.5-r.4",
+    "@types/node": "^26.5.1",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
-    "hyparquet": "^1.29.2",
+    "hyparquet": "^1.30.0",
     "hyparquet-writer": "^0.16.9",
-    "icebird": "^0.8.27",
+    "icebird": "^0.8.28",
     "tsx": "^4.23.13",
     "unstorage": "^1.17.5",
-    "vitest": "^4.1.11"
+    "vitest": "^5.0.0"
   },
   "scripts": {
     "dev": "obuild --stub",

package/dist/node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/sql/icebergDataSource.d.mts

@@ -1,2 +0,0 @@
-import "../../src/types.mjs";
-import "../../../../../squirreling@0.16.3/node_modules/squirreling/src/index.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/sql/icebergQuery.d.mts

@@ -1,2 +0,0 @@
-import "../../src/types.mjs";
-import "../../../../../squirreling@0.16.3/node_modules/squirreling/src/index.mjs";