@gscdump/engine 0.37.6 → 0.38.0

package/dist/_chunks/contracts.d.mts

@@ -1 +1 @@
-export { };
+import "./storage.mjs";

package/dist/_chunks/index.d.mts

@@ -3,6 +3,7 @@ import { EngineError } from "./errors.mjs";
 import { AnalysisParams, AnalysisResult } from "./analysis-types.mjs";
 import { ResolverAdapter } from "./types.mjs";
 import { AnalysisQuerySource, AnalysisSourceKind, AnalyzerRegistry, FileSet, QueryRow, SourceCapabilities } from "./registry.mjs";
+import "./contracts.mjs";
 import { PlannerCapabilities } from "gscdump/query/plan";
 import { BuilderState } from "gscdump/query";
 interface AttachedTableRunner {

package/dist/_chunks/libs/fzstd.mjs

@@ -1,4 +1,8 @@
-var ab = ArrayBuffer, u8 = Uint8Array, u16 = Uint16Array, i16 = Int16Array, i32 = Int32Array;
+var ab = ArrayBuffer;
+var u8 = Uint8Array;
+var u16 = Uint16Array;
+var i16 = Int16Array;
+var i32 = Int32Array;
 var slc = function(v, s, e) {
 	if (u8.prototype.slice) return u8.prototype.slice.call(v, s, e);
 	if (s == null || s < 0) s = 0;

package/dist/_chunks/libs/hyparquet-writer.d.mts

@@ -0,0 +1,23 @@
+interface Writer$1 {
+  buffer: ArrayBuffer;
+  view: DataView;
+  offset: number; // total bytes written
+  ensure(size: number): void;
+  flush?(): void | Promise<void>;
+  finish(): void | Promise<void>;
+  getBuffer(): ArrayBuffer;
+  getBytes(): Uint8Array;
+  appendUint8(value: number): void;
+  appendUint32(value: number): void;
+  appendInt32(value: number): void;
+  appendInt64(value: bigint): void;
+  appendFloat32(value: number): void;
+  appendFloat64(value: number): void;
+  appendBuffer(buffer: ArrayBuffer): void;
+  appendBytes(value: Uint8Array): void;
+  appendVarInt(value: number): void;
+  appendVarBigInt(value: bigint): void;
+  appendZigZag(value: number | bigint): void;
+}
+type Writer = Writer$1;
+export { Writer };

package/dist/_chunks/libs/icebird.d.mts

@@ -1,5 +1,5 @@
 import { AsyncBuffer } from "./hyparquet.mjs";
-import { Writer } from "hyparquet-writer/src/types.js";
+import { Writer } from "./hyparquet-writer.mjs";
 interface WriterOptions {
   /**
    * If `'*'`, the writer must create the object only if it does not already
@@ -24,6 +24,8 @@ interface RestCatalogContext {
   defaults: Record<string, string>;
   overrides: Record<string, string>;
   requestInit?: RequestInit;
+  /** Optional per-request auth hook (e.g. SigV4 signing). Called after merging requestInit. */
+  signRequest?: (url: string, init?: RequestInit) => Promise<RequestInit>;
 }
 interface FileCatalog {
   type: 'file';
@@ -136,7 +138,8 @@ interface Snapshot {
   manifests?: Manifest[];
   summary: {
     // spec: "value of these fields should be of string type"
-    operation: string; // 'spark.app.id'?: string
+    operation: string;
+    // 'spark.app.id'?: string
     'added-data-files'?: string;
     'added-records'?: string;
     'deleted-data-files'?: string;
@@ -185,9 +188,6 @@ interface EncryptionKey {
   'key-id': string;
   'key-metadata': string;
 }
-/**
- * Iceberg REST `TableRequirement`s recognized by `checkRequirements`.
- */
 interface MetadataLog {
   'timestamp-ms': number;
   'metadata-file': string;
@@ -206,7 +206,8 @@ interface Manifest {
   added_rows_count: bigint;
   existing_rows_count: bigint;
   deleted_rows_count: bigint;
-  partitions?: FieldSummary[]; // key_metadata?: unknown
+  partitions?: FieldSummary[];
+  // key_metadata?: unknown
   first_row_id?: bigint | number;
 }
 interface ManifestEntry {
@@ -235,7 +236,8 @@ interface DataFile {
   null_value_counts?: Record<number, bigint>;
   nan_value_counts?: Record<number, bigint>;
   lower_bounds?: Record<number, unknown>;
-  upper_bounds?: Record<number, unknown>; // key_metadata?: string
+  upper_bounds?: Record<number, unknown>;
+  // key_metadata?: string
   split_offsets?: bigint[];
   equality_ids?: number[];
   sort_order_id?: number;
@@ -244,44 +246,6 @@ interface DataFile {
   content_offset?: bigint;
   content_size_in_bytes?: bigint;
 }
-/**
- * Returns manifest entries for a snapshot. Defaults to the current snapshot;
- * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
- * snapshot log.
- *
- * @import {Resolver, TableMetadata, Manifest, ManifestEntry} from '../src/types.js'
- * @typedef {{ url: string, entries: ManifestEntry[] }[]} ManifestList
- * @param {object} options
- * @param {TableMetadata} options.metadata
- * @param {Resolver} [options.resolver]
- * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
- * @returns {Promise<ManifestList>}
- */
-declare function icebergManifests({
-  metadata,
-  resolver,
-  snapshotId,
-  partitionFilter
-}: {
-  metadata: TableMetadata;
-  resolver?: Resolver | undefined;
-  snapshotId?: number | bigint | undefined;
-  partitionFilter?: ManifestPartitionFilter | undefined;
-}): Promise<ManifestList>;
-/**
- * Predicate applied to each manifest's manifest-list `partitions` field-summary
- * array before entry fetch. Return `false` to skip the manifest's entry fetch.
- */
-type ManifestPartitionFilter = (partitions: Manifest['partitions'], partitionSpecId: number, manifest: Manifest) => boolean;
-/**
- * Returns manifest entries for a snapshot. Defaults to the current snapshot;
- * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
- * snapshot log.
- */
-type ManifestList = {
-  url: string;
-  entries: ManifestEntry[];
-}[];
 /**
  * Create a new table. REST: delegates to the catalog's create endpoint.
  * File: writes the initial `v1.metadata.json` and `version-hint.text` under
@@ -300,28 +264,17 @@ type ManifestList = {
  * @param {boolean} [options.stageCreate] - REST catalog only.
  * @returns {Promise<TableMetadata>}
  */
-declare function icebergCreateTable({
-  catalog,
-  namespace,
-  table,
-  tableUrl,
-  schema,
-  partitionSpec,
-  sortOrder,
-  properties,
-  formatVersion,
-  stageCreate
-}: {
+declare function icebergCreateTable({ catalog, namespace, table, tableUrl, schema, partitionSpec, sortOrder, properties, formatVersion, stageCreate }: {
   catalog: Catalog;
-  namespace?: string | string[] | undefined;
-  table?: string | undefined;
-  tableUrl?: string | undefined;
-  schema?: Schema | undefined;
-  partitionSpec?: PartitionSpec | undefined;
-  sortOrder?: SortOrder | undefined;
-  properties?: Record<string, string> | undefined;
-  formatVersion?: 2 | 3 | undefined;
-  stageCreate?: boolean | undefined;
+  namespace?: string | string[];
+  table?: string;
+  tableUrl?: string;
+  schema?: Schema;
+  partitionSpec?: PartitionSpec;
+  sortOrder?: SortOrder;
+  properties?: Record<string, string>;
+  formatVersion?: 2 | 3;
+  stageCreate?: boolean;
 }): Promise<TableMetadata>;
 /**
  * Load a single table. Returns the inline TableMetadata, the metadata
@@ -336,11 +289,36 @@ declare function icebergCreateTable({
  * @param {string} options.table
  * @returns {Promise<LoadTableResponse>}
  */
-declare function restCatalogLoadTable(ctx: RestCatalogContext, {
-  namespace,
-  table
-}: {
+declare function restCatalogLoadTable(ctx: RestCatalogContext, { namespace, table }: {
   namespace: string | string[];
   table: string;
 }): Promise<LoadTableResponse>;
+type ManifestList = {
+  url: string;
+  entries: ManifestEntry[];
+}[];
+/**
+ * Predicate applied to each manifest's manifest-list `partitions` field-summary
+ * array before entry fetch. Return `false` to skip the manifest's entry fetch.
+ */
+type ManifestPartitionFilter = (partitions: Manifest['partitions'], partitionSpecId: number, manifest: Manifest) => boolean;
+/**
+ * Returns manifest entries for a snapshot. Defaults to the current snapshot;
+ * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
+ * snapshot log.
+ *
+ * @import {Resolver, TableMetadata, Manifest, ManifestEntry} from '../src/types.js'
+ * @typedef {{ url: string, entries: ManifestEntry[] }[]} ManifestList
+ * @param {object} options
+ * @param {TableMetadata} options.metadata
+ * @param {Resolver} [options.resolver]
+ * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
+ * @returns {Promise<ManifestList>}
+ */
+declare function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }: {
+  metadata: TableMetadata;
+  resolver?: Resolver;
+  snapshotId?: number | bigint;
+  partitionFilter?: ManifestPartitionFilter;
+}): Promise<ManifestList>;
 export { icebergCreateTable, icebergManifests, restCatalogLoadTable };

package/dist/_chunks/libs/icebird.mjs

@@ -103,7 +103,23 @@ function readType(reader, type) {
 			for (let i = 0; i < count; i++) arr.push(readType(reader, type.items));
 		}
 		return arr;
-	} else if (typeof type === "object" && type.logicalType) if (type.logicalType === "date" && type.type === "int") {
+	} else if (typeof type === "object" && type.type === "map") {
+		const map = {};
+		while (true) {
+			let count = readZigZag(reader);
+			if (count === 0) break;
+			if (count < 0) {
+				count = -count;
+				readZigZag(reader);
+			}
+			for (let i = 0; i < count; i++) {
+				const key = readType(reader, "string");
+				map[key] = readType(reader, type.values);
+			}
+		}
+		return map;
+	} else if (typeof type === "object" && type.type === "enum") return type.symbols[readZigZag(reader)];
+	else if (typeof type === "object" && "logicalType" in type && type.logicalType) if (type.logicalType === "date" && type.type === "int") {
 		const value = readZigZag(reader);
 		return /* @__PURE__ */ new Date(value * 864e5);
 	} else if (type.logicalType === "time-millis" && type.type === "int") return readZigZag(reader);
@@ -152,7 +168,8 @@ function readType(reader, type) {
 		const text = new TextDecoder().decode(bytes);
 		reader.offset += length;
 		return text;
-	} else throw new Error(`unsupported type: ${type}`);
+	} else if (typeof type === "object" && typeof type.type === "string") return readType(reader, type.type);
+	else throw new Error(`unsupported type: ${JSON.stringify(type)}`);
 }
 function readFixed(reader, size) {
 	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, size);
@@ -422,7 +439,8 @@ function encodeNamespace(namespace) {
 async function restFetch(ctx, path, init) {
 	const prefixSegment = ctx.prefix ? `${ctx.prefix.replace(/^\/|\/$/g, "")}/` : "";
 	const fullUrl = `${ctx.url}/v1/${prefixSegment}${path}`;
-	const merged = mergeRequestInit(ctx.requestInit, init);
+	let merged = mergeRequestInit(ctx.requestInit, init);
+	if (ctx.signRequest) merged = await ctx.signRequest(fullUrl, merged);
 	const res = await fetch(fullUrl, merged);
 	if (!res.ok) await throwRestError(res);
 	return res;
@@ -868,4 +886,5 @@ new TextEncoder();
 	}
 	return () => new Promise((resolve) => setTimeout(resolve, 0));
 })();
+new TextEncoder();
 export { icebergCreateTable, icebergManifests, restCatalogLoadTable };

package/dist/contracts.d.mts

@@ -1,2 +1,3 @@
 import { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, ListLiveFilter, LockScope, ManifestEntry, ManifestStore, ParquetCodec, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult } from "./_chunks/storage.mjs";
+import "./_chunks/contracts.mjs";
 export type { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, ListLiveFilter, LockScope, ManifestEntry, ManifestStore, ParquetCodec, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult };

package/dist/entities.d.mts

@@ -50,9 +50,7 @@ interface QueryDimStore {
   /** Decode the dimension rows (test/inspection; reads JOIN the parquet by key). */
   loadRecords: (ctx: TenantCtx) => Promise<QueryDimRecord[]>;
 }
-declare function createQueryDimStore({
-  dataSource
-}: {
+declare function createQueryDimStore({ dataSource }: {
   dataSource: DataSource;
 }): QueryDimStore;
 /**

package/dist/ingest.mjs

@@ -219,7 +219,8 @@ function assembleDatesRow(date, totalsRow, deviceRows, queryGrainedImpressions)
 		sum_position_tablet: 0
 	};
 	for (const dr of deviceRows) {
-		const suffix = DEVICE_SUFFIX[String(dr.keys?.[1] ?? dr.keys?.[0] ?? "").toUpperCase()];
+		const deviceKey = String(dr.keys?.[1] ?? dr.keys?.[0] ?? "").toUpperCase();
+		const suffix = DEVICE_SUFFIX[deviceKey];
 		if (!suffix) continue;
 		const dImpr = dr.impressions || 0;
 		row[`clicks_${suffix}`] = dr.clicks || 0;

package/dist/rollups.d.mts

@@ -1,6 +1,7 @@
 import { DataSource, FileSetRef, Row as Row$1, TableName as TableName$1 } from "./_chunks/storage.mjs";
 import { ColumnDef as ColumnDef$1 } from "./_chunks/schema.mjs";
 import { EngineError } from "./_chunks/errors.mjs";
+import "./_chunks/contracts.mjs";
 import { SearchType } from "gscdump/query";
 import { TenantCtx } from "@gscdump/contracts";
 interface RollupCtx extends TenantCtx {
@@ -287,11 +288,13 @@ declare function runWindowed(opts: {
    * windowed `FILES`). Use to JOIN a non-windowed sidecar (e.g. the query
    * dimension parquet via `{ QUERY_DIM: { keys: [...] } }`) inside `sqlFor`.
    */
-  extraFileSets?: Record<string, FileSetRef>; /** Page each window's output by a total-order key. See `runPagedQuery`. */
+  extraFileSets?: Record<string, FileSetRef>;
+  /** Page each window's output by a total-order key. See `runPagedQuery`. */
   paginate?: {
     orderBy: string;
     pageRows: number;
-  }; /** Cap each window's day span (output-cardinality bound). See `planRollupWindows`. */
+  };
+  /** Cap each window's day span (output-cardinality bound). See `planRollupWindows`. */
   maxWindowDays?: number;
 }): Promise<Row$1[]>;
 /**
@@ -379,10 +382,14 @@ declare function rebuildCanonicalDailyResumable(opts: {
   dataSource: DataSource;
   searchType?: SearchType;
   builtAt: number;
-  windowOffset: number; /** Resume the `windowOffset` window at this shard offset (0 = window start). */
-  pageOffset?: number; /** Output rows per page (default `ROLLUP_PAGE_ROWS_DAILY`). Injectable for tests. */
-  pageRows?: number; /** Cap each input window's day span (default `DAILY_MAX_WINDOW_DAYS`). */
-  maxWindowDays?: number; /** Split each date window by raw-query hash before grouping (1 = no sharding). */
+  windowOffset: number;
+  /** Resume the `windowOffset` window at this shard offset (0 = window start). */
+  pageOffset?: number;
+  /** Output rows per page (default `ROLLUP_PAGE_ROWS_DAILY`). Injectable for tests. */
+  pageRows?: number;
+  /** Cap each input window's day span (default `DAILY_MAX_WINDOW_DAYS`). */
+  maxWindowDays?: number;
+  /** Split each date window by raw-query hash before grouping (1 = no sharding). */
   shardCount?: number;
   deadlineMs: number;
 }): Promise<{

package/dist/sink-node.mjs

@@ -38,7 +38,7 @@ function createIcebergOverwriteWriter(opts) {
 		const siteId = slice.ctx.siteId;
 		if (!siteId) throw new Error("overwriteSlice: slice.ctx.siteId is required for the Iceberg partition key");
 		const table = assertIcebergTable(slice.table);
-		const res = await backend({
+		const job = {
 			op: "overwrite",
 			catalogUri: catalog.catalogUri,
 			namespace: catalog.namespace,
@@ -50,7 +50,8 @@ function createIcebergOverwriteWriter(opts) {
 			searchType: slice.searchType,
 			date: slice.date,
 			rows
-		});
+		};
+		const res = await backend(job);
 		if (res.error) throw new Error(`overwriteSlice: PyIceberg backend failed: ${res.error}`);
 		return { rowCount: res.rowCount ?? rows.length };
 	}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.37.6",
+  "version": "0.38.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,21 +181,21 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.37.6",
-    "@gscdump/lakehouse": "0.37.6",
-    "gscdump": "0.37.6"
+    "@gscdump/lakehouse": "0.38.0",
+    "gscdump": "0.38.0",
+    "@gscdump/contracts": "0.38.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
-    "@types/node": "^26.1.0",
+    "@types/node": "^26.1.1",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.26.2",
     "hyparquet-writer": "^0.16.1",
-    "icebird": "^0.8.13",
-    "tsx": "^4.23.0",
+    "icebird": "^0.8.15",
+    "tsx": "^4.23.1",
     "unstorage": "^1.17.5",
-    "vitest": "^4.1.9"
+    "vitest": "^4.1.10"
   },
   "scripts": {
     "dev": "obuild --stub",