@gscdump/engine 0.40.1 → 1.0.0

package/README.md

@@ -27,7 +27,6 @@ Optional peers (install only what your runtime needs):
 | `@gscdump/engine/contracts` | `StorageEngine`, `Row`, `TableName`, `WriteCtx`, `ManifestEntry`, ... |
 | `@gscdump/engine/schema` | `SCHEMAS`, `allTables`, `inferTable`, column metadata. |
 | `@gscdump/engine/planner` | `resolveToSQL`, `enumeratePartitions`, partition planning. |
-| `@gscdump/engine/snapshot` | `SnapshotIndex` contract for hot/cold snapshot files. |
 | `@gscdump/engine/ingest` | GSC row → storage row helpers (`createRowAccumulator`, `transformGscRow`). |
 | `@gscdump/engine/sql` | SQL literal binding helpers (`bindLiterals`, `formatLiteral`). |
 | `@gscdump/engine/sql-fragments` | Reusable SQL fragments shared across analyzers. |
@@ -40,8 +39,7 @@ Optional peers (install only what your runtime needs):
 | `@gscdump/engine/filesystem` | Node-only `DataSource` + `ManifestStore` adapters. |
 | `@gscdump/engine/hyparquet` | Pure-JS `ParquetCodec`. |
 | `@gscdump/engine/r2` | Cloudflare R2 `DataSource` (structurally typed against `R2Bucket`). |
-| `@gscdump/engine/r2-manifest` | R2-backed `ManifestStore` for hosted deployments. |
-| `@gscdump/engine/iceberg` | Edge-safe GSC Iceberg schema/catalog/append sink. |
+| `@gscdump/engine/iceberg` | Edge-safe GSC dataset schemas, catalog wrappers, and append sink. Generic Iceberg APIs live in `@gscdump/lakehouse`. |
 | `@gscdump/engine/sink-node` | Node-only Iceberg overwrite/delete recovery writer. |
 | `@gscdump/engine/source` | Query-source contracts and factories. |
 
@@ -58,9 +56,11 @@ Optional peers (install only what your runtime needs):
 ## Related
 
 - [`gscdump`](../gscdump) — REST client + query builder (edge-safe peer dep).
-- [`@gscdump/analysis`](../analysis) — analyzers; consumes `StorageEngine` via `createEngine` factories.
+- [`@gscdump/analysis`](../analysis) — analyzers; consumes storage through `AnalysisQuerySource` adapters.
 - [`@gscdump/engine-duckdb-wasm`](../engine-duckdb-wasm) — DuckDB-WASM browser adapter.
 - [`@gscdump/engine-sqlite`](../engine-sqlite) — SQLite / D1 adapter.
+- [`@gscdump/lakehouse`](../lakehouse) — dataset-agnostic Iceberg catalog and
+  dataset registry APIs.
 - [`@gscdump/cli`](../cli) — CLI wrapping engine + analysis.
 
 ## License

package/dist/_chunks/duckdb.d.mts

@@ -1,4 +1,4 @@
-import { ParquetCodec, QueryExecutor, Row, TableName } from "./storage.mjs";
+import { ParquetCodec, QueryExecutor, Row } from "./storage.mjs";
 interface DuckDBHandle {
   query: (sql: string, params?: unknown[]) => Promise<Row[]>;
   registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
@@ -24,11 +24,4 @@ interface DuckDBReadOptions {
 }
 declare function createDuckDBCodec(factory: DuckDBFactory, options?: DuckDBReadOptions): ParquetCodec;
 declare function createDuckDBExecutor(factory: DuckDBFactory, options?: DuckDBReadOptions): QueryExecutor;
-/**
- * Canonical "empty-file" SELECT clause for a table. Codecs that need to
- * emit a schema-correct empty Parquet can wrap this in:
- *   `COPY (SELECT * FROM <clause> WHERE FALSE) TO '<key>' (FORMAT PARQUET, COMPRESSION ZSTD)`
- * to satisfy the ParquetCodec empty-rows invariant.
- */
-declare function canonicalEmptyParquetSchema(table: TableName): string;
-export { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor };
+export { DuckDBFactory, DuckDBHandle, createDuckDBCodec, createDuckDBExecutor };

package/dist/_chunks/engine.mjs

@@ -1,9 +1,8 @@
-import { coerceRows } from "./coerce.mjs";
 import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
+import { coerceRows } from "./coerce.mjs";
 import { SCHEMAS, TABLE_METADATA, currentSchemaVersion, dateColumnsFor, dedupeByNaturalKey } from "./schema.mjs";
-import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from "./compaction.mjs";
+import { compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, splitOverlappingTiers, substituteNamedFiles } from "./parquet-plan.mjs";
 import { dateReplaceClause as dateReplaceClause$1 } from "../sql-fragments.mjs";
-import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
 import { buildLogicalPlan } from "gscdump/query/plan";
@@ -187,9 +186,6 @@ function createDuckDBExecutor(factory, options = {}) {
 function emptyTableSchema(table) {
 	return `(FROM (VALUES ${placeholderValues(table)}) t(${columnList(table)}))`;
 }
-function canonicalEmptyParquetSchema(table) {
-	return emptyTableSchema(table);
-}
 function dateReplaceClause(table) {
 	if (!table) return "";
 	return dateReplaceClause$1(dateColumnsFor(table), "string");
@@ -732,4 +728,4 @@ function createStorageEngine(opts) {
 		readObject: (key) => dataSource.read(key)
 	};
 }
-export { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine };
+export { MAX_DAY_BYTES, createDuckDBCodec, createDuckDBExecutor, createStorageEngine };

package/dist/_chunks/entities.mjs

@@ -1052,4 +1052,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/_chunks/index.d.mts

@@ -1,8 +1,8 @@
 import { Row, SearchType as SearchType$1, StorageEngine, TenantCtx } from "./storage.mjs";
-import { EngineError } from "./errors.mjs";
 import { AnalysisParams, AnalysisResult } from "./analysis-types.mjs";
 import { ResolverAdapter } from "./types.mjs";
-import { AnalysisQuerySource, AnalysisSourceKind, AnalyzerRegistry, FileSet, QueryRow, SourceCapabilities } from "./registry.mjs";
+import { AnalysisQuerySource, AnalysisSourceKind, AnalyzerRegistry, QueryRow, SourceCapabilities } from "./registry.mjs";
+import { EngineError } from "./errors.mjs";
 import "./contracts.mjs";
 import { PlannerCapabilities } from "gscdump/query/plan";
 import { BuilderState } from "gscdump/query";
@@ -44,11 +44,6 @@ declare class AttachedTableMissingError extends Error {
   readonly engineError: EngineError;
   constructor(missing: readonly string[]);
 }
-/**
- * Swap `read_parquet({{KEY}}, union_by_name = true)` for `<schema>.<table>`.
- * Tolerates whitespace variation. Preserves the rest of the SQL verbatim.
- */
-declare function rewriteForTableSource(sql: string, schema: string, fileSets: Record<string, FileSet>): string;
 declare function createAttachedTableSource(runner: AttachedTableRunner, options: AttachedTableSourceOptions): AnalysisQuerySource;
 interface CreateSqlQuerySourceOptions<TKey extends string> {
   /** Debug-only identifier surfaced on the source for error messages. */
@@ -106,4 +101,4 @@ declare function runAnalyzerWithEngine(deps: {
   engine: StorageEngine;
 }, ctx: TenantCtx, params: AnalysisParams, registry: AnalyzerRegistry): Promise<AnalysisResult>;
 declare function queryRows<TRow = QueryRow>(source: AnalysisQuerySource, state: BuilderState): Promise<TRow[]>;
-export { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine };
+export { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };

package/dist/_chunks/index2.d.mts

@@ -26,7 +26,6 @@ interface ComparisonPeriod {
   previous: AnalysisPeriod;
 }
 declare function defaultEndDate(): string;
-declare function defaultStartDate(): string;
 declare function periodOf(params: AnalysisParams): AnalysisPeriod;
 declare function comparisonOf(params: AnalysisParams): ComparisonPeriod;
 declare function resolveWindow(opts: ResolveWindowOptions): ResolvedWindow;
@@ -51,4 +50,4 @@ type DateRowShape = Record<string, unknown> & {
  * once. Existing dates keep all their rows (grouped timeseries safe).
  */
 declare function padTimeseries<T extends DateRowShape = DateRowShape>(rows: readonly T[], options: PadTimeseriesOptions<T>): T[];
-export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow };
+export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };

package/dist/_chunks/libs/hyparquet-compressors.mjs

@@ -2793,4 +2793,4 @@ new Uint8Array([
 	5,
 	5
 ]);
-export { gunzip, snappyUncompress, snappyUncompressor };
+export { snappyUncompress, snappyUncompressor };

package/dist/_chunks/libs/hyparquet.mjs

@@ -913,68 +913,6 @@ function equals(a, b, strict = true) {
 	for (const k of aKeys) if (!equals(a[k], b[k], strict)) return false;
 	return true;
 }
-async function byteLengthFromUrlUsingGet(url, requestInit = {}, fetchFn = globalThis.fetch) {
-	const controller = new AbortController();
-	const headers = new Headers(requestInit.headers);
-	headers.set("Range", "bytes=0-0");
-	const res = await fetchFn(url, {
-		...requestInit,
-		headers,
-		signal: controller.signal
-	});
-	if (!res.ok) throw new Error(`fetch with range failed ${res.status}`);
-	if (res.status === 206) {
-		const contentRange = res.headers.get("Content-Range");
-		if (!contentRange) throw new Error("missing content-range header");
-		const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
-		if (!match) throw new Error(`invalid content-range header: ${contentRange}`);
-		return parseInt(match[1]);
-	}
-	if (res.status === 200) {
-		const contentLength = res.headers.get("Content-Length");
-		controller.abort();
-		if (contentLength) return parseInt(contentLength);
-	}
-	throw new Error("server does not support range requests and missing content-length");
-}
-async function byteLengthFromUrl(url, requestInit, customFetch) {
-	const fetch = customFetch ?? globalThis.fetch;
-	const res = await fetch(url, {
-		...requestInit,
-		method: "HEAD"
-	});
-	if (res.status === 403) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
-	if (!res.ok) throw new Error(`fetch head failed ${res.status}`);
-	const length = res.headers.get("Content-Length");
-	if (!length) return byteLengthFromUrlUsingGet(url, requestInit, fetch);
-	return parseInt(length);
-}
-async function asyncBufferFromUrl({ url, byteLength, requestInit, fetch: customFetch }) {
-	if (!url) throw new Error("missing url");
-	const fetch = customFetch ?? globalThis.fetch;
-	byteLength ??= await byteLengthFromUrl(url, requestInit, fetch);
-	let buffer = void 0;
-	const init = requestInit || {};
-	return {
-		byteLength,
-		async slice(start, end) {
-			if (buffer) return buffer.then((buffer) => buffer.slice(start, end));
-			const headers = new Headers(init.headers);
-			const endStr = end === void 0 ? "" : end - 1;
-			headers.set("Range", `bytes=${start}-${endStr}`);
-			const res = await fetch(url, {
-				...init,
-				headers
-			});
-			if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`);
-			if (res.status === 200) {
-				buffer = res.arrayBuffer();
-				return buffer.then((buffer) => buffer.slice(start, end));
-			} else if (res.status === 206) return res.arrayBuffer();
-			else throw new Error(`fetch received unexpected status code ${res.status}`);
-		}
-	};
-}
 function flatten(chunks) {
 	if (!chunks) return [];
 	if (chunks.length === 1) return chunks[0];
@@ -2655,4 +2593,4 @@ function toJson(obj) {
 	return obj;
 }
 new TextDecoder();
-export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes, asyncBufferFromUrl, getMaxDefinitionLevel, getSchemaPath, hashParquetValue, isListLike, isMapLike, parquetReadObjects, parseDecimal, snappyUncompress, toJson };
+export { BoundaryOrders, CompressionCodecs, ConvertedTypes, EdgeInterpolationAlgorithms, Encodings, FieldRepetitionTypes, PageTypes, ParquetTypes, getMaxDefinitionLevel, getSchemaPath, hashParquetValue, isListLike, isMapLike, parquetReadObjects, snappyUncompress, toJson };

package/dist/_chunks/libs/icebird.d.mts

@@ -42,11 +42,6 @@ interface FileCatalog {
   conditionalCommits?: boolean;
 }
 type Catalog = RestCatalogContext | FileCatalog;
-interface LoadTableResponse {
-  metadataLocation?: string;
-  metadata: TableMetadata;
-  config: Record<string, string>;
-}
 interface TableMetadata {
   'format-version': number;
   'table-uuid': string;
@@ -210,42 +205,12 @@ interface Manifest {
   // key_metadata?: unknown
   first_row_id?: bigint | number;
 }
-interface ManifestEntry {
-  status: 0 | 1 | 2; // 0=existing, 1=added, 2=deleted
-  snapshot_id?: bigint;
-  sequence_number?: bigint;
-  file_sequence_number?: bigint;
-  partition_spec_id?: number;
-  data_file: DataFile;
-}
 interface FieldSummary {
   contains_null: boolean;
   contains_nan?: boolean | null;
   lower_bound?: Uint8Array | null;
   upper_bound?: Uint8Array | null;
 }
-interface DataFile {
-  content: 0 | 1 | 2; // 0=data, 1=position_delete, 2=equality_delete
-  file_path: string;
-  file_format: 'avro' | 'orc' | 'parquet' | 'puffin';
-  partition: Record<string, unknown>; // keyed by partition-field name in Avro
-  record_count: bigint;
-  file_size_in_bytes: bigint;
-  column_sizes?: Record<number, bigint>;
-  value_counts?: Record<number, bigint>;
-  null_value_counts?: Record<number, bigint>;
-  nan_value_counts?: Record<number, bigint>;
-  lower_bounds?: Record<number, unknown>;
-  upper_bounds?: Record<number, unknown>;
-  // key_metadata?: string
-  split_offsets?: bigint[];
-  equality_ids?: number[];
-  sort_order_id?: number;
-  first_row_id?: bigint | number;
-  referenced_data_file?: string;
-  content_offset?: bigint;
-  content_size_in_bytes?: bigint;
-}
 /**
  * Create a new table. REST: delegates to the catalog's create endpoint.
  * File: writes the initial `v1.metadata.json` and `version-hint.text` under
@@ -276,49 +241,4 @@ declare function icebergCreateTable({ catalog, namespace, table, tableUrl, schem
   formatVersion?: 2 | 3;
   stageCreate?: boolean;
 }): Promise<TableMetadata>;
-/**
- * Load a single table. Returns the inline TableMetadata, the metadata
- * file location, and any per-table config the server returned.
- *
- * The returned `metadata` and `metadata.location` can be passed directly
- * into `icebergRead({ tableUrl: metadata.location, metadata })`.
- *
- * @param {RestCatalogContext} ctx
- * @param {object} options
- * @param {string | string[]} options.namespace
- * @param {string} options.table
- * @returns {Promise<LoadTableResponse>}
- */
-declare function restCatalogLoadTable(ctx: RestCatalogContext, { namespace, table }: {
-  namespace: string | string[];
-  table: string;
-}): Promise<LoadTableResponse>;
-type ManifestList = {
-  url: string;
-  entries: ManifestEntry[];
-}[];
-/**
- * Predicate applied to each manifest's manifest-list `partitions` field-summary
- * array before entry fetch. Return `false` to skip the manifest's entry fetch.
- */
-type ManifestPartitionFilter = (partitions: Manifest['partitions'], partitionSpecId: number, manifest: Manifest) => boolean;
-/**
- * Returns manifest entries for a snapshot. Defaults to the current snapshot;
- * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
- * snapshot log.
- *
- * @import {Resolver, TableMetadata, Manifest, ManifestEntry} from '../src/types.js'
- * @typedef {{ url: string, entries: ManifestEntry[] }[]} ManifestList
- * @param {object} options
- * @param {TableMetadata} options.metadata
- * @param {Resolver} [options.resolver]
- * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
- * @returns {Promise<ManifestList>}
- */
-declare function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }: {
-  metadata: TableMetadata;
-  resolver?: Resolver;
-  snapshotId?: number | bigint;
-  partitionFilter?: ManifestPartitionFilter;
-}): Promise<ManifestList>;
-export { icebergCreateTable, icebergManifests, restCatalogLoadTable };
+export { icebergCreateTable };

package/dist/_chunks/libs/icebird.mjs

@@ -1,189 +1,6 @@
-import { asyncBufferFromUrl, parseDecimal } from "./hyparquet.mjs";
-import { ByteWriter } from "./hyparquet-writer.mjs";
-import { gunzip } from "./hyparquet-compressors.mjs";
-function readZigZag(reader) {
-	let result = 0;
-	let shift = 0;
-	while (true) {
-		const byte = reader.view.getUint8(reader.offset++);
-		result |= (byte & 127) << shift;
-		if (!(byte & 128)) return result >>> 1 ^ -(result & 1);
-		shift += 7;
-	}
-}
-function readZigZagBigInt(reader) {
-	let result = 0n;
-	let shift = 0n;
-	while (true) {
-		const byte = reader.view.getUint8(reader.offset++);
-		result |= BigInt(byte & 127) << shift;
-		if (!(byte & 128)) return result >> 1n ^ -(result & 1n);
-		shift += 7n;
-	}
-}
-function readAvroString(reader) {
-	const length = readZigZag(reader);
-	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
-	reader.offset += length;
-	return new TextDecoder().decode(bytes);
-}
-function avroMetadata(reader) {
-	if (reader.view.getUint32(reader.offset) !== 1331849729) throw new Error("avro invalid magic bytes");
-	reader.offset += 4;
-	const metadata = {};
-	let mapCount = readZigZag(reader);
-	while (mapCount !== 0) {
-		if (mapCount < 0) {
-			mapCount = -mapCount;
-			readZigZag(reader);
-		}
-		for (let i = 0; i < mapCount; i++) {
-			const key = readAvroString(reader);
-			metadata[key] = readAvroString(reader);
-		}
-		mapCount = readZigZag(reader);
-	}
-	metadata["avro.schema"] = JSON.parse(metadata["avro.schema"]);
-	if (metadata["schema"]) metadata["schema"] = JSON.parse(metadata["schema"]);
-	if (metadata["iceberg.schema"]) metadata["iceberg.schema"] = JSON.parse(metadata["iceberg.schema"]);
-	const syncMarker = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
-	reader.offset += 16;
-	return {
-		metadata,
-		syncMarker
-	};
-}
-function avroRead({ reader, metadata, syncMarker }) {
-	const blocks = [];
-	while (reader.offset < reader.view.byteLength) {
-		let recordCount = readZigZag(reader);
-		if (recordCount === 0) break;
-		if (recordCount < 0) recordCount = -recordCount;
-		const blockSize = readZigZag(reader);
-		let data = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, blockSize);
-		reader.offset += blockSize;
-		const blockSync = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
-		reader.offset += 16;
-		for (let i = 0; i < 16; i++) if (blockSync[i] !== syncMarker[i]) throw new Error("sync marker does not match");
-		const codec = metadata["avro.codec"];
-		if (codec === "deflate") data = gunzip(data);
-		else if (codec !== "null") throw new Error(`unsupported codec: ${codec}`);
-		const { fields } = metadata["avro.schema"];
-		const dataReader = {
-			view: new DataView(data.buffer, data.byteOffset, data.byteLength),
-			offset: 0
-		};
-		for (let i = 0; i < recordCount; i++) {
-			const obj = {};
-			for (const field of fields) {
-				const value = readType(dataReader, field.type);
-				obj[field.name] = value;
-			}
-			blocks.push(obj);
-		}
-	}
-	return blocks;
-}
-function readType(reader, type) {
-	if (type === "null") return;
-	else if (Array.isArray(type)) return readType(reader, type[readZigZag(reader)]);
-	else if (typeof type === "object" && type.type === "record") {
-		const obj = {};
-		for (const subField of type.fields) obj[subField.name] = readType(reader, subField.type);
-		return obj;
-	} else if (typeof type === "object" && type.type === "array") {
-		const arr = [];
-		while (true) {
-			let count = readZigZag(reader);
-			if (count === 0) break;
-			if (count < 0) {
-				count = -count;
-				readZigZag(reader);
-			}
-			for (let i = 0; i < count; i++) arr.push(readType(reader, type.items));
-		}
-		return arr;
-	} else if (typeof type === "object" && type.type === "map") {
-		const map = {};
-		while (true) {
-			let count = readZigZag(reader);
-			if (count === 0) break;
-			if (count < 0) {
-				count = -count;
-				readZigZag(reader);
-			}
-			for (let i = 0; i < count; i++) {
-				const key = readType(reader, "string");
-				map[key] = readType(reader, type.values);
-			}
-		}
-		return map;
-	} else if (typeof type === "object" && type.type === "enum") return type.symbols[readZigZag(reader)];
-	else if (typeof type === "object" && "logicalType" in type && type.logicalType) if (type.logicalType === "date" && type.type === "int") {
-		const value = readZigZag(reader);
-		return /* @__PURE__ */ new Date(value * 864e5);
-	} else if (type.logicalType === "time-millis" && type.type === "int") return readZigZag(reader);
-	else if (type.logicalType === "time-micros" && type.type === "long") return readZigZagBigInt(reader);
-	else if (type.logicalType === "timestamp-millis" && type.type === "long") {
-		const value = readZigZagBigInt(reader);
-		return new Date(Number(value));
-	} else if (type.logicalType === "timestamp-micros" && type.type === "long") {
-		const value = readZigZagBigInt(reader);
-		return new Date(Number(value / 1000n));
-	} else if (type.logicalType === "timestamp-nanos" && type.type === "long") {
-		const value = readZigZagBigInt(reader);
-		return new Date(Number(value / 1000000n));
-	} else if (type.logicalType === "decimal" && "precision" in type) {
-		const bytes = type.type === "fixed" ? readFixed(reader, type.size) : readType(reader, type.type);
-		const factor = 10 ** -(type.scale || 0);
-		return parseDecimal(bytes) * factor;
-	} else if (type.logicalType === "uuid" && type.type === "fixed" && type.size === 16) return bytesToUuid(readFixed(reader, 16));
-	else {
-		console.warn(`unknown logical type: ${type.logicalType}`);
-		return type.type === "fixed" ? readFixed(reader, type.size) : readType(reader, type.type);
-	}
-	else if (typeof type === "object" && type.type === "fixed") return readFixed(reader, type.size);
-	else if (type === "boolean") {
-		const value = reader.view.getUint8(reader.offset) === 1;
-		reader.offset++;
-		return value;
-	} else if (type === "int") return readZigZag(reader);
-	else if (type === "long") return readZigZagBigInt(reader);
-	else if (type === "float") {
-		const value = reader.view.getFloat32(reader.offset, true);
-		reader.offset += 4;
-		return value;
-	} else if (type === "double") {
-		const value = reader.view.getFloat64(reader.offset, true);
-		reader.offset += 8;
-		return value;
-	} else if (type === "bytes") {
-		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
-		reader.offset += length;
-		return bytes;
-	} else if (type === "string") {
-		const length = readZigZag(reader);
-		const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
-		const text = new TextDecoder().decode(bytes);
-		reader.offset += length;
-		return text;
-	} else if (typeof type === "object" && typeof type.type === "string") return readType(reader, type.type);
-	else throw new Error(`unsupported type: ${JSON.stringify(type)}`);
-}
-function readFixed(reader, size) {
-	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, size);
-	reader.offset += size;
-	return bytes;
-}
-function bytesToUuid(bytes) {
-	let hex = "";
-	for (let i = 0; i < 16; i++) {
-		hex += bytes[i].toString(16).padStart(2, "0");
-		if (i === 3 || i === 5 || i === 7 || i === 9) hex += "-";
-	}
-	return hex;
-}
+import "./hyparquet.mjs";
+import "./hyparquet-writer.mjs";
+import "./hyparquet-compressors.mjs";
 function uuid4() {
 	if (globalThis.crypto?.randomUUID) return globalThis.crypto.randomUUID();
 	return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => {
@@ -191,72 +8,6 @@ function uuid4() {
 		return (c === "x" ? r : r & 3 | 8).toString(16);
 	});
 }
-function translateS3Url(url) {
-	if (url.startsWith("s3a://") || url.startsWith("s3://")) {
-		const rest = url.slice(url.indexOf("://") + 3);
-		const slashIndex = rest.indexOf("/");
-		if (slashIndex === -1) throw new Error("Invalid S3 URL, missing \"/\" after bucket");
-		return `https://${rest.slice(0, slashIndex)}.s3.amazonaws.com${rest.slice(slashIndex)}`;
-	}
-	return url;
-}
-function urlResolver({ requestInit } = {}) {
-	return {
-		reader(url, byteLength) {
-			return asyncBufferFromUrl({
-				url: translateS3Url(url),
-				byteLength,
-				requestInit
-			});
-		},
-		writer(url, options) {
-			const w = new ByteWriter();
-			w.finish = async function() {
-				const target = translateS3Url(url);
-				const body = w.getBytes().slice();
-				const headers = {};
-				if (requestInit?.headers) new Headers(requestInit.headers).forEach((v, k) => {
-					headers[k] = v;
-				});
-				if (options?.ifNoneMatch) headers["If-None-Match"] = options.ifNoneMatch;
-				const res = await fetch(target, {
-					...requestInit,
-					method: "PUT",
-					headers,
-					body
-				});
-				if (!res.ok) {
-					const err = /* @__PURE__ */ new Error(`PUT ${url}: ${res.status} ${res.statusText}`);
-					err.status = res.status;
-					throw err;
-				}
-			};
-			return w;
-		},
-		async deleter(url) {
-			const res = await fetch(translateS3Url(url), {
-				...requestInit,
-				method: "DELETE"
-			});
-			if (!res.ok && res.status !== 404) throw new Error(`DELETE ${url}: ${res.status} ${res.statusText}`);
-		}
-	};
-}
-async function fetchAvroRecords(url, resolver, byteLength) {
-	const lengthHint = byteLength !== void 0 && Number.isFinite(byteLength) ? byteLength : void 0;
-	const ab = await resolver.reader(url, lengthHint);
-	const buffer = await ab.slice(0, ab.byteLength);
-	const reader = {
-		view: new DataView(buffer),
-		offset: 0
-	};
-	const { metadata, syncMarker } = await avroMetadata(reader);
-	return await avroRead({
-		reader,
-		metadata,
-		syncMarker
-	});
-}
 const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
 function parseIcebergJson(text) {
 	let i = 0;
@@ -403,14 +154,6 @@ function stringifyIcebergJson(value, indent = 2) {
 	}
 	return serialize(value, 0);
 }
-async function restCatalogLoadTable(ctx, { namespace, table }) {
-	const body = parseIcebergJson(await (await restFetch(ctx, `namespaces/${encodeNamespace(namespace)}/tables/${encodeURIComponent(table)}`)).text());
-	return {
-		metadataLocation: body["metadata-location"],
-		metadata: body.metadata,
-		config: body.config ?? {}
-	};
-}
 async function restCatalogCreateTable(ctx, { namespace, table, schema, location, partitionSpec, writeOrder, stageCreate, properties }) {
 	const ns = encodeNamespace(namespace);
 	const body = {
@@ -765,75 +508,6 @@ function maxPartitionFieldId(partitionFields = []) {
 	for (const pf of partitionFields) if (max < pf["field-id"]) max = pf["field-id"];
 	return max;
 }
-async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }) {
-	resolver ??= urlResolver();
-	const rawTarget = snapshotId ?? metadata["current-snapshot-id"];
-	if (rawTarget == null || rawTarget < 0) throw new Error("No current snapshot id found in table metadata");
-	const targetId = BigInt(rawTarget);
-	const snapshot = metadata.snapshots?.find((s) => BigInt(s["snapshot-id"]) === targetId);
-	if (!snapshot) throw new Error(`Snapshot ${rawTarget} not found in metadata`);
-	let manifests = [];
-	if (snapshot["manifest-list"]) {
-		const manifestListUrl = snapshot["manifest-list"];
-		manifests = await fetchAvroRecords(manifestListUrl, resolver);
-	} else if (snapshot.manifests) manifests = snapshot.manifests;
-	else throw new Error("No manifest information found in snapshot");
-	if (partitionFilter) manifests = manifests.filter((manifest) => {
-		let keep = true;
-		try {
-			keep = partitionFilter(manifest.partitions, manifest.partition_spec_id ?? 0, manifest) !== false;
-		} catch {
-			keep = true;
-		}
-		return keep;
-	});
-	return await fetchManifests(manifests, resolver);
-}
-const MANIFEST_FETCH_CONCURRENCY = 8;
-async function fetchOneManifest(manifest, resolver) {
-	const url = manifest.manifest_path;
-	const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
-	for (const entry of entries) {
-		entry.partition_spec_id = manifest.partition_spec_id ?? 0;
-		if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number ?? 0n;
-		if (entry.status === 1) {
-			if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number;
-			if (entry.file_sequence_number === void 0) entry.file_sequence_number = manifest.sequence_number;
-		} else if (entry.sequence_number === void 0 || entry.file_sequence_number === void 0) throw new Error("iceberg manifest entry missing sequence number");
-	}
-	assignFirstRowIds(manifest, entries);
-	return {
-		url,
-		entries
-	};
-}
-async function fetchManifests(manifests, resolver) {
-	const results = new Array(manifests.length);
-	let next = 0;
-	async function worker() {
-		while (next < manifests.length) {
-			const i = next++;
-			results[i] = await fetchOneManifest(manifests[i], resolver);
-		}
-	}
-	const poolSize = Math.min(MANIFEST_FETCH_CONCURRENCY, manifests.length);
-	const workers = [];
-	for (let w = 0; w < poolSize; w++) workers.push(worker());
-	await Promise.all(workers);
-	return results;
-}
-function assignFirstRowIds(manifest, entries) {
-	if (manifest.content !== 0 || manifest.first_row_id == null) return;
-	let nextFirstRowId = BigInt(manifest.first_row_id);
-	for (const entry of entries) {
-		const dataFile = entry.data_file;
-		if (dataFile.content !== 0) continue;
-		if (dataFile.first_row_id == null) {
-			dataFile.first_row_id = nextFirstRowId;
-			nextFirstRowId += BigInt(dataFile.record_count);
-		}
-	}
-}
 Object.freeze({
 	maxAttempts: 50,
 	initialMs: 50,
@@ -887,4 +561,4 @@ new TextEncoder();
 	return () => new Promise((resolve) => setTimeout(resolve, 0));
 })();
 new TextEncoder();
-export { icebergCreateTable, icebergManifests, restCatalogLoadTable };
+export { icebergCreateTable };