@gscdump/engine 5.3.0 → 5.6.0

package/dist/adapters/hyparquet.mjs

@@ -213,15 +213,24 @@ function asyncBufferFromBytes(bytes) {
 }
 async function decodeParquetToRows(bytes, opts = {}) {
 	const rows = [];
-	for await (const group of decodeParquetGroups(bytes, opts)) for (const row of group) rows.push(row);
+	for await (const group of decodeParquetGroups(bytes, opts)) for (let i = 0; i < group.length; i++) rows.push(group[i]);
 	return rows;
 }
 async function* decodeParquetGroups(bytes, opts = {}) {
 	if (bytes.byteLength === 0) return;
 	const file = asyncBufferFromBytes(bytes);
 	const metadata = await parquetMetadataAsync(file);
-	const present = new Set(parquetSchema(metadata).children.map((child) => child.element.name));
+	const tree = parquetSchema(metadata);
+	const present = /* @__PURE__ */ new Set();
+	const dateColumns = [];
+	let nested = false;
+	for (const child of tree.children) {
+		present.add(child.element.name);
+		if (child.children.length > 0) nested = true;
+		else if (isDateCapableElement(child.element)) dateColumns.push(child.element.name);
+	}
 	const columns = opts.columns?.filter((name) => present.has(name));
+	const normalizeColumns = nested ? null : dateColumns;
 	let groupStart = 0;
 	for (const group of metadata.row_groups) {
 		const groupEnd = groupStart + Number(group.num_rows);
@@ -243,15 +252,22 @@ async function* decodeParquetGroups(bytes, opts = {}) {
 				useBloomFilters: true,
 				usePageIndex: true
 			} : {}
-		}));
+		}), normalizeColumns);
 		groupStart = groupEnd;
 	}
 }
-function normalizeDecodedDates(rows) {
+function isDateCapableElement(element) {
+	if (element.type === "INT96") return true;
+	const converted = element.converted_type;
+	if (converted === "DATE" || converted === "TIMESTAMP_MILLIS" || converted === "TIMESTAMP_MICROS") return true;
+	return element.logical_type?.type === "TIMESTAMP";
+}
+function normalizeDecodedDates(rows, dateColumns) {
+	if (dateColumns !== null && dateColumns.length === 0) return rows;
 	const dates = /* @__PURE__ */ new Map();
 	for (const row of rows) {
 		const r = row;
-		for (const k in r) {
+		for (const k of dateColumns ?? Object.keys(r)) {
 			const v = r[k];
 			if (v instanceof Date) {
 				const day = Math.floor(v.getTime() / EPOCH_DAY_MS);

package/dist/iceberg/index.d.mts

@@ -1,5 +1,5 @@
-import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergTableName, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "./schema.mjs";
+import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergTableName, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "./schema.mjs";
 import { IcebergTableOpResult, ListIcebergDataFilesOptions, createIcebergTables, dropIcebergTables, icebergPartitionSpecFor, icebergSchemaFor, icebergSortOrderFor, listIcebergDataFiles } from "./catalog.mjs";
 import { IcebergAppendSinkOptions } from "../sink.mjs";
 import { IcebergAppendSink, createIcebergAppendSink } from "./append-sink.mjs";
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergTableName, type IcebergTableOpResult, type ListIcebergDataFilesOptions, SEARCH_TYPE_INT, assertIcebergTable, createIcebergAppendSink, createIcebergTables, dropIcebergTables, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isIcebergTable, listIcebergDataFiles };
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergTableName, type IcebergTableOpResult, type ListIcebergDataFilesOptions, SEARCH_TYPE_INT, assertIcebergTable, createIcebergAppendSink, createIcebergTables, dropIcebergTables, gscDataset, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isIcebergTable, listIcebergDataFiles };

package/dist/iceberg/index.mjs

@@ -1,4 +1,4 @@
-import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "./schema.mjs";
+import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "./schema.mjs";
 import { createIcebergTables, dropIcebergTables, icebergPartitionSpecFor, icebergSchemaFor, icebergSortOrderFor, listIcebergDataFiles } from "./catalog.mjs";
 import { createIcebergAppendSink } from "./append-sink.mjs";
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, createIcebergAppendSink, createIcebergTables, dropIcebergTables, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isIcebergTable, listIcebergDataFiles };
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, createIcebergAppendSink, createIcebergTables, dropIcebergTables, gscDataset, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isIcebergTable, listIcebergDataFiles };

package/dist/iceberg/schema.d.mts

@@ -1,4 +1,5 @@
 import { SearchType } from "../storage.mjs";
+import { IcebergDataset } from "@gscdump/lakehouse";
 import { IcebergColumn, IcebergPartitionField, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1 } from "@gscdump/lakehouse/schema";
 import { TableName } from "@gscdump/contracts";
 /** The 9 fact tables that exist as global Iceberg tables. */
@@ -38,6 +39,12 @@ export declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding)
 export declare const ICEBERG_FIELD_ID_BASE = 3;
 /** Shared partition spec — identical across every table. */
 export declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
+/**
+ * The `IcebergDataset` registry instance for one `gsc.*` table + encoding —
+ * the single source `./append-sink.ts`'s identity injection / dedupe key /
+ * cluster pre-sort and `./catalog.ts`'s create-table derivation read from.
+ */
+export declare function gscDataset(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergDataset;
 /**
  * Derive the full Iceberg table spec for a table + encoding. Thin wrapper
  * over {@link gscDataset}'s `tableSpec` (ADR-0021 R2-FIXES C5) — kept as a

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "5.3.0",
+  "version": "5.6.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^5.3.0",
-    "@gscdump/lakehouse": "^5.3.0",
+    "@gscdump/contracts": "^5.6.0",
+    "@gscdump/lakehouse": "^5.6.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^5.3.0",
+    "gscdump": "^5.6.0",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },