@gscdump/engine 0.4.0

package/dist/adapters/http.d.mts

@@ -0,0 +1,227 @@
+import { TableName } from "gscdump/contracts";
+import { SearchType } from "gscdump/query";
+/**
+ * Compaction tier of a manifest entry. Determines which compactor stage may
+ * pick it up as input:
+ * - `raw`: per-day file produced by `writeDay`. Eligible for raw→d7 merge at 7d.
+ * - `d7`: weekly compaction output. Eligible for d7→d30 merge at 30d.
+ * - `d30`: monthly compaction output (matches the legacy `monthly/` partition
+ *   shape — pre-tier entries are read as `d30`). Eligible for d30→d90 at 90d.
+ * - `d90`: quarterly cold-tier output. Terminal; never recompacted.
+ *
+ * Without an explicit tier, entries written before this field landed default
+ * to `raw` for `daily/` partitions and `d30` for `monthly/` partitions, so
+ * the tiered compactor picks the right inputs without a backfill rewrite.
+ */
+type CompactionTier = 'raw' | 'd7' | 'd30' | 'd90';
+interface ManifestEntry {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+  partition: string;
+  objectKey: string;
+  rowCount: number;
+  bytes: number;
+  createdAt: number;
+  retiredAt?: number;
+  /** Table schema version at write time. Omitted on pre-#27 entries — treat as 1. */
+  schemaVersion?: number;
+  /**
+   * Compaction tier. Omitted on entries written before tiered compaction —
+   * treat as `raw` for `daily/` partitions and `d30` for `monthly/` partitions
+   * (see {@link inferLegacyTier}).
+   */
+  tier?: CompactionTier;
+  /**
+   * GSC search-type this entry covers (web | discover | news | googleNews |
+   * image | video). Omitted on entries written before per-type partitioning
+   * landed — treat as `web` (see {@link inferSearchType}). Compaction merges
+   * only entries with the same searchType.
+   */
+  searchType?: SearchType;
+}
+interface ListLiveFilter {
+  userId: string;
+  siteId?: string;
+  table?: TableName;
+  partitions?: string[];
+  /**
+   * Narrow to a single compaction tier. Tier-aware compaction stages set this
+   * so the store doesn't have to return (and the caller doesn't have to scan)
+   * the entire manifest just to compact the raw cohort. Legacy entries without
+   * an explicit `tier` field match on {@link inferLegacyTier}.
+   */
+  tier?: CompactionTier;
+}
+interface DataSource {
+  read: (key: string, range?: {
+    offset: number;
+    length: number;
+  }, signal?: AbortSignal) => Promise<Uint8Array>;
+  write: (key: string, bytes: Uint8Array) => Promise<void>;
+  delete: (keys: string[]) => Promise<void>;
+  /**
+   * One-shot listing under a prefix. Implementations may cap the number of
+   * returned keys (typically 10k) — callers iterating full tenant space
+   * should prefer `streamList` when available or narrow the prefix.
+   */
+  list: (prefix: string) => Promise<string[]>;
+  /**
+   * Per-key URI probe. Returns a URI string DuckDB's `httpfs` (or an
+   * equivalent engine that fetches its own I/O) can read directly, or
+   * `undefined` if the key isn't URI-resolvable on this backend and the
+   * caller must fall back to `read(key)` for the bytes.
+   *
+   * Contracts:
+   * - When defined, the returned URI MUST yield byte-identical content to
+   *   `read(key)`. Callers rely on this for correctness.
+   * - Backends with a native URI for every key (filesystem: absolute path,
+   *   R2 via `httpfs`: signed URL) may always return a string.
+   * - Backends without a native URI shape (in-memory) omit the method or
+   *   return `undefined` per call.
+   * - Mixed-per-query is allowed: some keys in one query may return a URI,
+   *   others may not; the executor branches per key.
+   */
+  uri?: (key: string) => string | undefined;
+  /**
+   * Optional — probe the byte size of a key without reading it. Used by
+   * the engine to fill in `WriteResult.bytes` when a codec reports 0 or
+   * unknown but the file is non-trivial.
+   */
+  head?: (key: string) => Promise<{
+    bytes: number;
+  } | undefined>;
+  /**
+   * Optional streaming variant of `list`. Implementations that page
+   * backing-store results (R2, S3) should implement this and yield keys
+   * lazily. `list` may return up to an adapter-defined cap (typically
+   * 10k keys); callers iterating full tenant space must prefer
+   * `streamList` when available, or chunk by narrower prefixes.
+   */
+  streamList?: (prefix: string) => AsyncIterable<string>;
+}
+interface WatermarkScope {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+}
+interface Watermark extends WatermarkScope {
+  newestDateSynced: string;
+  oldestDateSynced: string;
+  lastSyncAt: number;
+}
+interface WatermarkFilter {
+  userId: string;
+  siteId?: string;
+  table?: TableName;
+}
+type SyncStateKind = 'pending' | 'inflight' | 'done' | 'failed';
+interface SyncStateScope {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+  date: string;
+  /**
+   * GSC search-type this sync state covers. Omitted = `web` (the legacy
+   * default; matches pre-#5 sync states stored before per-type sync landed).
+   * Lookups must compare via {@link inferSearchType} so a missing field
+   * matches an explicit `'web'` and vice versa.
+   */
+  searchType?: SearchType;
+}
+interface SyncState extends SyncStateScope {
+  state: SyncStateKind;
+  updatedAt: number;
+  attempts: number;
+  error?: string;
+}
+interface SyncStateFilter {
+  userId: string;
+  siteId?: string;
+  table?: TableName;
+  state?: SyncStateKind;
+  searchType?: SearchType;
+}
+interface SyncStateDetail {
+  at?: number;
+  error?: string;
+}
+interface LockScope {
+  userId: string;
+  siteId?: string;
+  table: TableName;
+  partition: string;
+}
+interface PurgeFilter {
+  userId: string;
+  siteId?: string;
+}
+interface ManifestPurgeResult {
+  entriesRemoved: number;
+  watermarksRemoved: number;
+  syncStatesRemoved: number;
+}
+interface ManifestStore {
+  listLive: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
+  listAll: (filter: ListLiveFilter) => Promise<ManifestEntry[]>;
+  registerVersion: (entry: ManifestEntry, superseding?: ManifestEntry[]) => Promise<void>;
+  registerVersions: (entries: ManifestEntry[], superseding?: ManifestEntry[]) => Promise<void>;
+  listRetired: (olderThan: number) => Promise<ManifestEntry[]>;
+  delete: (entries: ManifestEntry[]) => Promise<void>;
+  getWatermarks: (filter: WatermarkFilter) => Promise<Watermark[]>;
+  bumpWatermark: (scope: WatermarkScope, date: string, at?: number) => Promise<void>;
+  getSyncStates: (filter: SyncStateFilter) => Promise<SyncState[]>;
+  setSyncState: (scope: SyncStateScope, state: SyncStateKind, detail?: SyncStateDetail) => Promise<void>;
+  /**
+   * Serialize concurrent writers against the same scope. Held across the
+   * write+register window so GC (orphan sweep) won't delete bytes that are
+   * midway between `dataSource.write` and `manifestStore.registerVersion`.
+   * Scope = tenant × table × partition.
+   */
+  withLock: <T>(scope: LockScope, fn: () => Promise<T>) => Promise<T>;
+  /**
+   * GDPR-grade tenant purge. Removes every manifest entry, watermark, and
+   * sync-state record matching the filter. Does NOT touch the underlying
+   * data-source bytes; callers (typically {@link StorageEngine.purgeTenant})
+   * must sweep the tenant prefix separately before invoking this so that
+   * mid-flight failures can't leave orphan parquet with no manifest record.
+   *
+   * On stores with CAS-backed sharding (R2 manifest) this may issue one
+   * mutation per shard. On read-only stores (HTTP) this throws.
+   */
+  purgeTenant: (filter: PurgeFilter) => Promise<ManifestPurgeResult>;
+}
+interface HttpDataSourceOptions {
+  /**
+   * Base URL to prefix each object key with. MUST NOT have a trailing slash.
+   * E.g. `https://pub-abcdef.r2.dev/gscdump-data`.
+   */
+  baseUrl: string;
+  /**
+   * Optional transformer that produces the final URL for a key. Use this when
+   * keys need signing (pre-signed URLs, per-request tokens, etc.). If omitted,
+   * the default is `${baseUrl}/${encodeKey(key)}` where forward slashes in the
+   * key are preserved.
+   */
+  signUrl?: (key: string) => string;
+  /**
+   * Whether `uri(key)` should return the HTTPS URL so DuckDB's httpfs can
+   * fetch directly. Default true (browser, Workers, anywhere httpfs is
+   * loaded). Set to false for environments where httpfs isn't available —
+   * the executor will fall back to `read(key)` and buffer the bytes itself.
+   */
+  useDuckDBHttpfs?: boolean;
+}
+declare function createHttpDataSource(opts: HttpDataSourceOptions): DataSource;
+interface HttpManifestStoreOptions {
+  /**
+   * URL of a JSON manifest snapshot. The response MUST be:
+   *   { version: 1, entries: ManifestEntry[], watermarks?: Watermark[] }
+   * (Matches the on-disk layout produced by the filesystem adapter.)
+   */
+  manifestUrl: string;
+  /** Override fetch for tests / custom origins. */
+  fetchImpl?: typeof fetch;
+}
+declare function createHttpManifestStore(opts: HttpManifestStoreOptions): ManifestStore;
+export { HttpDataSourceOptions, HttpManifestStoreOptions, createHttpDataSource, createHttpManifestStore };

package/dist/adapters/http.mjs

@@ -0,0 +1,119 @@
+function inferLegacyTier(entry) {
+	if (entry.tier !== void 0) return entry.tier;
+	if (entry.partition.startsWith("daily/")) return "raw";
+	if (entry.partition.startsWith("monthly/")) return "d30";
+}
+function readOnly(name) {
+	throw new Error(`http adapter is read-only: ${name} is not supported`);
+}
+function encodeKey(key) {
+	return key.split("/").map(encodeURIComponent).join("/");
+}
+const TRAILING_SLASH = /\/$/;
+function createHttpDataSource(opts) {
+	const base = opts.baseUrl.replace(TRAILING_SLASH, "");
+	const sign = opts.signUrl ?? ((key) => `${base}/${encodeKey(key)}`);
+	const useHttpfs = opts.useDuckDBHttpfs ?? true;
+	async function readBytes(key, range, signal) {
+		const url = sign(key);
+		const headers = {};
+		if (range) headers.Range = `bytes=${range.offset}-${range.offset + range.length - 1}`;
+		const res = await fetch(url, {
+			headers,
+			signal
+		});
+		if (!res.ok) throw new Error(`http read failed ${res.status} ${res.statusText} for ${url}`);
+		return new Uint8Array(await res.arrayBuffer());
+	}
+	return {
+		read: readBytes,
+		async write() {
+			readOnly("write");
+		},
+		async delete() {
+			readOnly("delete");
+		},
+		async list() {
+			readOnly("list");
+		},
+		async head(key) {
+			const res = await fetch(sign(key), { method: "HEAD" });
+			if (!res.ok) return void 0;
+			const len = res.headers.get("content-length");
+			return len == null ? void 0 : { bytes: Number(len) };
+		},
+		uri(key) {
+			return useHttpfs ? sign(key) : void 0;
+		}
+	};
+}
+function matchesFilter(entry, filter) {
+	if (entry.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && entry.table !== filter.table) return false;
+	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
+	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
+	return true;
+}
+function matchesWatermark(w, filter) {
+	if (w.userId !== filter.userId) return false;
+	if (filter.siteId !== void 0 && w.siteId !== filter.siteId) return false;
+	if (filter.table !== void 0 && w.table !== filter.table) return false;
+	return true;
+}
+function createHttpManifestStore(opts) {
+	const fetchImpl = opts.fetchImpl ?? fetch;
+	let cache = null;
+	async function load() {
+		if (!cache) cache = (async () => {
+			const res = await fetchImpl(opts.manifestUrl);
+			if (!res.ok) throw new Error(`manifest fetch failed ${res.status} ${res.statusText} for ${opts.manifestUrl}`);
+			const parsed = await res.json();
+			if (parsed.version !== 1) throw new Error(`unsupported manifest version ${parsed.version}`);
+			return parsed;
+		})();
+		return cache;
+	}
+	return {
+		async listLive(filter) {
+			const { entries } = await load();
+			return entries.filter((e) => e.retiredAt === void 0 && matchesFilter(e, filter));
+		},
+		async listAll(filter) {
+			const { entries } = await load();
+			return entries.filter((e) => matchesFilter(e, filter));
+		},
+		async getWatermarks(filter) {
+			const { watermarks = [] } = await load();
+			return watermarks.filter((w) => matchesWatermark(w, filter));
+		},
+		async getSyncStates(_filter) {
+			return [];
+		},
+		async listRetired() {
+			return [];
+		},
+		async registerVersion() {
+			readOnly("registerVersion");
+		},
+		async registerVersions() {
+			readOnly("registerVersions");
+		},
+		async delete() {
+			readOnly("delete");
+		},
+		async bumpWatermark() {
+			readOnly("bumpWatermark");
+		},
+		async setSyncState() {
+			readOnly("setSyncState");
+		},
+		async withLock(_, fn) {
+			return fn();
+		},
+		async purgeTenant() {
+			readOnly("purgeTenant");
+		}
+	};
+}
+export { createHttpDataSource, createHttpManifestStore };

package/dist/adapters/hyparquet.d.mts

@@ -0,0 +1,107 @@
+import { ColumnDef, Row, Row as Row$1, TableName, TableName as TableName$1 } from "gscdump/contracts";
+interface DataSource {
+  read: (key: string, range?: {
+    offset: number;
+    length: number;
+  }, signal?: AbortSignal) => Promise<Uint8Array>;
+  write: (key: string, bytes: Uint8Array) => Promise<void>;
+  delete: (keys: string[]) => Promise<void>;
+  /**
+   * One-shot listing under a prefix. Implementations may cap the number of
+   * returned keys (typically 10k) — callers iterating full tenant space
+   * should prefer `streamList` when available or narrow the prefix.
+   */
+  list: (prefix: string) => Promise<string[]>;
+  /**
+   * Per-key URI probe. Returns a URI string DuckDB's `httpfs` (or an
+   * equivalent engine that fetches its own I/O) can read directly, or
+   * `undefined` if the key isn't URI-resolvable on this backend and the
+   * caller must fall back to `read(key)` for the bytes.
+   *
+   * Contracts:
+   * - When defined, the returned URI MUST yield byte-identical content to
+   *   `read(key)`. Callers rely on this for correctness.
+   * - Backends with a native URI for every key (filesystem: absolute path,
+   *   R2 via `httpfs`: signed URL) may always return a string.
+   * - Backends without a native URI shape (in-memory) omit the method or
+   *   return `undefined` per call.
+   * - Mixed-per-query is allowed: some keys in one query may return a URI,
+   *   others may not; the executor branches per key.
+   */
+  uri?: (key: string) => string | undefined;
+  /**
+   * Optional — probe the byte size of a key without reading it. Used by
+   * the engine to fill in `WriteResult.bytes` when a codec reports 0 or
+   * unknown but the file is non-trivial.
+   */
+  head?: (key: string) => Promise<{
+    bytes: number;
+  } | undefined>;
+  /**
+   * Optional streaming variant of `list`. Implementations that page
+   * backing-store results (R2, S3) should implement this and yield keys
+   * lazily. `list` may return up to an adapter-defined cap (typically
+   * 10k keys); callers iterating full tenant space must prefer
+   * `streamList` when available, or chunk by narrower prefixes.
+   */
+  streamList?: (prefix: string) => AsyncIterable<string>;
+}
+interface WriteResult {
+  bytes: number;
+  rowCount: number;
+}
+interface CodecCtx {
+  table: TableName;
+}
+/**
+ * Key-oriented codec. Each method owns its I/O through `dataSource`:
+ * - Node / browser codecs read/write bytes via `dataSource.read` / `.write`.
+ * - Workers codecs let DuckDB's httpfs read/write remote URIs directly (via
+ *   `dataSource.uri`) and never materialise bytes in JS.
+ *
+ * The engine never touches bytes; it just hands rows + keys to the codec.
+ *
+ * Invariants every implementation MUST uphold:
+ * - `writeRows` with an empty `rows` array MUST still write a file
+ *   carrying the canonical column set for `ctx.table` — a schema-correct
+ *   empty file. No placeholder-column shortcuts; readers depend on the
+ *   schema being present for `union_by_name` merges.
+ * - `WriteResult.bytes` MUST be the real byte size written to the
+ *   data source (not 0, not an estimate) so the engine can enforce the
+ *   payload ceiling without a second `head` round-trip.
+ * - `WriteResult.rowCount` MUST equal `rows.length` (or, for
+ *   `compactRows`, the sum of input row counts).
+ */
+interface ParquetCodec {
+  writeRows: (ctx: CodecCtx, rows: Row[], key: string, dataSource: DataSource) => Promise<WriteResult>;
+  readRows: (ctx: CodecCtx, key: string, dataSource: DataSource) => Promise<Row[]>;
+  compactRows: (ctx: CodecCtx, inputKeys: string[], outputKey: string, dataSource: DataSource) => Promise<WriteResult>;
+}
+declare function encodeRowsToParquet(table: TableName$1, rows: readonly Row$1[]): Uint8Array;
+interface EncodeFlexOptions {
+  /** Columns defining the output schema + order. */
+  columns: readonly ColumnDef[];
+  /** Sort key columns (subset of `columns` by name). Empty = preserve input order. */
+  sortKey?: readonly string[];
+  /** Row-group size; smaller groups = more prunable DuckDB stats. Default 25000. */
+  rowGroupSize?: number;
+}
+/**
+ * Schema-free encoder for rollups + auxiliary tables whose column set isn't
+ * in `SCHEMAS`. Caller supplies column definitions; types must be one of
+ * `VARCHAR | DATE | BIGINT | INTEGER | DOUBLE` — same physical mappings as
+ * the canonical encoder so DuckDB `read_parquet(union_by_name = true)`
+ * merges cleanly with fact-table reads.
+ */
+declare function encodeRowsToParquetFlex(rows: readonly Row$1[], opts: EncodeFlexOptions): Uint8Array;
+declare function decodeParquetToRows(bytes: Uint8Array): Promise<Row$1[]>;
+interface HyparquetCodecOptions {
+  /**
+   * Override `readRows`. Useful when reads should be delegated to a faster
+   * engine (e.g. DuckDB-WASM via httpfs) while writes + compaction stay on
+   * hyparquet to avoid WASM linear-memory growth. Defaults to hyparquet.
+   */
+  readRows?: (ctx: CodecCtx, key: string, dataSource: DataSource) => Promise<Row$1[]>;
+}
+declare function createHyparquetCodec(options?: HyparquetCodecOptions): ParquetCodec;
+export { EncodeFlexOptions, HyparquetCodecOptions, createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };

package/dist/adapters/hyparquet.mjs

@@ -0,0 +1,250 @@
+import { parquetReadObjects } from "hyparquet";
+import { parquetWriteBuffer } from "hyparquet-writer";
+import { date, doublePrecision, getTableConfig, integer, pgTable, varchar } from "drizzle-orm/pg-core";
+function metricCols() {
+	return {
+		clicks: integer("clicks").notNull(),
+		impressions: integer("impressions").notNull(),
+		sum_position: doublePrecision("sum_position").notNull()
+	};
+}
+const dateCol = () => date("date").notNull();
+const drizzleSchema = {
+	pages: pgTable("pages", {
+		url: varchar("url").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	keywords: pgTable("keywords", {
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	countries: pgTable("countries", {
+		country: varchar("country").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	devices: pgTable("devices", {
+		device: varchar("device").notNull(),
+		date: dateCol(),
+		...metricCols()
+	}),
+	page_keywords: pgTable("page_keywords", {
+		url: varchar("url").notNull(),
+		query: varchar("query").notNull(),
+		query_canonical: varchar("query_canonical"),
+		date: dateCol(),
+		...metricCols()
+	}),
+	search_appearance: pgTable("search_appearance", {
+		searchAppearance: varchar("searchAppearance").notNull(),
+		date: dateCol(),
+		...metricCols()
+	})
+};
+const TABLE_METADATA = {
+	pages: {
+		sortKey: ["date", "url"],
+		version: 1
+	},
+	keywords: {
+		sortKey: ["date", "query"],
+		version: 2
+	},
+	countries: {
+		sortKey: ["date", "country"],
+		version: 1
+	},
+	devices: {
+		sortKey: ["date", "device"],
+		version: 1
+	},
+	page_keywords: {
+		sortKey: [
+			"date",
+			"url",
+			"query"
+		],
+		version: 2
+	},
+	search_appearance: {
+		sortKey: ["date", "searchAppearance"],
+		version: 1
+	}
+};
+function pgSqlTypeToColumnType(sqlType) {
+	const t = sqlType.toLowerCase();
+	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
+	if (t === "date" || t.startsWith("timestamp")) return "DATE";
+	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
+	if (t === "bigint" || t === "int8") return "BIGINT";
+	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
+	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
+}
+function tableSchemaFrom(tableName) {
+	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
+		name: col.name,
+		type: pgSqlTypeToColumnType(col.getSQLType()),
+		nullable: !col.notNull
+	}));
+	const meta = TABLE_METADATA[tableName];
+	return {
+		name: tableName,
+		columns,
+		sortKey: meta.sortKey,
+		version: meta.version
+	};
+}
+const SCHEMAS = Object.fromEntries([
+	"pages",
+	"keywords",
+	"countries",
+	"devices",
+	"page_keywords",
+	"search_appearance"
+].map((t) => [t, tableSchemaFrom(t)]));
+const ROW_GROUP_SIZE = 25e3;
+function basicTypeFor(colType) {
+	if (colType === "VARCHAR" || colType === "DATE") return "STRING";
+	if (colType === "BIGINT") return "INT64";
+	if (colType === "INTEGER") return "INT32";
+	if (colType === "DOUBLE") return "DOUBLE";
+	throw new Error(`unsupported column type for parquet encoding: ${colType}`);
+}
+function coerceValue(value, type) {
+	if (value === null || value === void 0) return null;
+	if (type === "STRING") return typeof value === "string" ? value : String(value);
+	if (type === "INT32") {
+		const n = typeof value === "number" ? value : Number(value);
+		if (!Number.isFinite(n)) throw new Error(`non-finite number for INT32: ${String(value)}`);
+		return Math.trunc(n);
+	}
+	if (type === "INT64") {
+		if (typeof value === "bigint") return value;
+		const n = typeof value === "number" ? value : Number(value);
+		if (!Number.isFinite(n)) throw new Error(`non-finite number for INT64: ${String(value)}`);
+		return BigInt(Math.trunc(n));
+	}
+	if (type === "DOUBLE") {
+		const n = typeof value === "number" ? value : Number(value);
+		if (!Number.isFinite(n)) throw new Error(`non-finite number for DOUBLE: ${String(value)}`);
+		return n;
+	}
+	return value;
+}
+function compareValues(a, b) {
+	if (a === b) return 0;
+	if (a === null || a === void 0) return -1;
+	if (b === null || b === void 0) return 1;
+	if (typeof a === "number" && typeof b === "number") return a - b;
+	return String(a) < String(b) ? -1 : 1;
+}
+function sortRowsBySortKey(table, rows) {
+	const sortKey = TABLE_METADATA[table].sortKey;
+	if (sortKey.length === 0 || rows.length <= 1) return rows;
+	const copy = rows.slice();
+	copy.sort((a, b) => {
+		for (const col of sortKey) {
+			const cmp = compareValues(a[col], b[col]);
+			if (cmp !== 0) return cmp;
+		}
+		return 0;
+	});
+	return copy;
+}
+function encodeRowsToParquet(table, rows) {
+	const schema = SCHEMAS[table];
+	const sorted = sortRowsBySortKey(table, rows);
+	const buffer = parquetWriteBuffer({
+		columnData: schema.columns.map((col) => {
+			const type = basicTypeFor(col.type);
+			const data = sorted.map((r) => coerceValue(r[col.name], type));
+			return {
+				name: col.name,
+				data,
+				type,
+				nullable: col.nullable,
+				columnIndex: true
+			};
+		}),
+		rowGroupSize: ROW_GROUP_SIZE
+	});
+	return new Uint8Array(buffer);
+}
+function encodeRowsToParquetFlex(rows, opts) {
+	const { columns, sortKey = [], rowGroupSize = ROW_GROUP_SIZE } = opts;
+	const sorted = sortKey.length === 0 || rows.length <= 1 ? rows : [...rows].sort((a, b) => {
+		for (const col of sortKey) {
+			const cmp = compareValues(a[col], b[col]);
+			if (cmp !== 0) return cmp;
+		}
+		return 0;
+	});
+	const buffer = parquetWriteBuffer({
+		columnData: columns.map((col) => {
+			const type = basicTypeFor(col.type);
+			const data = sorted.map((r) => coerceValue(r[col.name], type));
+			return {
+				name: col.name,
+				data,
+				type,
+				nullable: col.nullable,
+				columnIndex: true
+			};
+		}),
+		rowGroupSize
+	});
+	return new Uint8Array(buffer);
+}
+function asyncBufferFromBytes(bytes) {
+	const ab = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+	return {
+		byteLength: ab.byteLength,
+		slice(start, end) {
+			return ab.slice(start, end);
+		}
+	};
+}
+async function decodeParquetToRows(bytes) {
+	if (bytes.byteLength === 0) return [];
+	return await parquetReadObjects({ file: asyncBufferFromBytes(bytes) });
+}
+function createHyparquetCodec(options = {}) {
+	return {
+		async writeRows(ctx, rows, key, dataSource) {
+			const bytes = encodeRowsToParquet(ctx.table, rows);
+			await dataSource.write(key, bytes);
+			return {
+				bytes: bytes.byteLength,
+				rowCount: rows.length
+			};
+		},
+		readRows: options.readRows ?? (async (_ctx, key, dataSource) => {
+			return decodeParquetToRows(await dataSource.read(key));
+		}),
+		async compactRows(ctx, inputKeys, outputKey, dataSource) {
+			if (inputKeys.length === 0) {
+				const bytes = encodeRowsToParquet(ctx.table, []);
+				await dataSource.write(outputKey, bytes);
+				return {
+					bytes: bytes.byteLength,
+					rowCount: 0
+				};
+			}
+			const allRows = [];
+			for (const key of inputKeys) {
+				const rows = await decodeParquetToRows(await dataSource.read(key));
+				allRows.push(...rows);
+			}
+			const bytes = encodeRowsToParquet(ctx.table, allRows);
+			await dataSource.write(outputKey, bytes);
+			return {
+				bytes: bytes.byteLength,
+				rowCount: allRows.length
+			};
+		}
+	};
+}
+export { createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };