@gscdump/lakehouse 1.4.0 → 1.4.2

package/dist/schema.d.mts

@@ -1,2 +1,77 @@
-import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
-export { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding };
+/**
+ * Generic Iceberg schema/partition/identity type shapes — the dataset-agnostic
+ * half of `@gscdump/engine/iceberg/schema.ts` (ADR-0021, split per amendment 8).
+ *
+ * TYPES + PURE HELPERS ONLY. The GSC-specific frozen constants
+ * (`IcebergTableName`, `ICEBERG_TABLES`, `ICEBERG_SCHEMAS`, `SEARCH_TYPE_INT`,
+ * `ICEBERG_PARTITION_SPEC`) stay in the engine's own registry instance — they
+ * describe ONE dataset (`gsc.*`), not the mechanism.
+ */
+/** S3-compatible credentials for the Iceberg warehouse object store (R2 in prod). */
+interface IcebergS3Config {
+  /** S3 endpoint host (prod: the R2 S3 endpoint). */
+  endpoint: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  /** Defaults to `'auto'` (R2's region). */
+  region?: string;
+}
+/** Iceberg-native column type. */
+type IcebergColumnType = 'STRING' | 'INT' | 'LONG' | 'DOUBLE' | 'DATE' | 'BOOLEAN';
+/** icebird's lowercase Iceberg primitive types (subset we use). */
+type IcebergPrimitiveType = 'string' | 'int' | 'long' | 'double' | 'date' | 'boolean';
+declare const ICEBERG_TYPE_MAP: Record<IcebergColumnType, IcebergPrimitiveType>;
+interface IcebergColumn {
+  /** Column name as written into the Iceberg table (snake_case). */
+  name: string;
+  type: IcebergColumnType;
+  /** Iceberg field nullability. Partition identity columns are never null. */
+  required: boolean;
+  /**
+   * Stable Iceberg field id. Field ids — not names — are the schema-evolution
+   * identity in Iceberg; never reuse or renumber an id once a table is live.
+   */
+  fieldId: number;
+}
+/**
+ * Partition-key encoding for identity columns.
+ *
+ * - `'string'` (legacy): identity columns are STRING. Correct, but R2 SQL's
+ *   string min/max statistics are truncated in predicate pushdown, so a bare
+ *   `WHERE site_id='<uuid>'` UNDERCOUNTS.
+ * - `'int'`: identity columns are INT. Integer statistics are fixed-width and
+ *   never truncated, so `WHERE site_id=<n>` is both correct AND prunes.
+ *
+ * New catalogs are provisioned `'int'`; existing legacy catalogs stay `'string'`.
+ */
+type PartitionKeyEncoding = 'string' | 'int';
+/** Default for new Iceberg/R2 Data Catalog tables. */
+declare const DEFAULT_PARTITION_KEY_ENCODING: PartitionKeyEncoding;
+/** Iceberg partition transform applied to a source column. */
+type IcebergPartitionTransform = 'identity' | 'month';
+interface IcebergPartitionField {
+  /** Source column the transform reads. */
+  sourceColumn: string;
+  transform: IcebergPartitionTransform;
+  /** Partition field name as it appears in Iceberg metadata. */
+  name: string;
+}
+interface IcebergTableSpec {
+  namespace: string;
+  table: string;
+  columns: readonly IcebergColumn[];
+  /** Partition spec — identity columns must appear here. */
+  partitionSpec: readonly IcebergPartitionField[];
+  /**
+   * Natural-key columns: a row is uniquely identified by this tuple within
+   * its partition. Drives dedupe. Per amendment 3, the FULL dedupe key is
+   * identity columns + dims columns + `naturalKey` (not `naturalKey` alone —
+   * identity alone would collapse rows across sites/dims).
+   */
+  naturalKey: readonly string[];
+  /** Full dedupe identity: identity columns + dims columns + naturalKey. */
+  identityColumns: readonly string[];
+  /** Physical write-order columns (dimension-first) — undefined skips sort-order. */
+  clusterKey?: readonly string[];
+}
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding };

package/dist/schema.mjs

@@ -1,2 +1,10 @@
+const ICEBERG_TYPE_MAP = {
+	STRING: "string",
+	INT: "int",
+	LONG: "long",
+	DOUBLE: "double",
+	DATE: "date",
+	BOOLEAN: "boolean"
+};
 const DEFAULT_PARTITION_KEY_ENCODING = "int";
-export { DEFAULT_PARTITION_KEY_ENCODING };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP };

package/dist/unsafe-raw.d.mts

@@ -1,3 +1,7 @@
-import { icebergAppendRetrying } from "./_chunks/catalog.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/write/write.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/catalog/rest.mjs";
+import { icebergAppendRetrying } from "./catalog.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/s3.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/unsafe-raw.mjs

@@ -1,3 +1,7 @@
-import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
-import { icebergAppendRetrying } from "./_chunks/catalog.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/s3.mjs";
+import { icebergAppendRetrying } from "./catalog.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/vendor/hysnappy-purejs.mjs

@@ -0,0 +1,10 @@
+import { snappyUncompress } from "../_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/snappy.mjs";
+function decode(input, outputLength) {
+	const output = new Uint8Array(outputLength);
+	snappyUncompress(input, output);
+	return output;
+}
+function snappyUncompressor() {
+	return decode;
+}
+export { snappyUncompressor };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",
@@ -31,6 +31,16 @@
       "import": "./dist/bigint.mjs",
       "default": "./dist/bigint.mjs"
     },
+    "./dataset": {
+      "types": "./dist/dataset.d.mts",
+      "import": "./dist/dataset.mjs",
+      "default": "./dist/dataset.mjs"
+    },
+    "./date": {
+      "types": "./dist/date.d.mts",
+      "import": "./dist/date.mjs",
+      "default": "./dist/date.mjs"
+    },
     "./schema": {
       "types": "./dist/schema.d.mts",
       "import": "./dist/schema.mjs",