@gscdump/engine 0.35.1 → 0.35.3

package/dist/_chunks/resolver.mjs

@@ -282,7 +282,6 @@ function createSqlFragments(config) {
 			let col;
 			if (metric === "clicks") col = t.clicks;
 			else if (metric === "impressions") col = t.impressions;
-			else if (metric === "position") col = t.sum_position;
 			if (!col) continue;
 			const v = Number(f.expression);
 			switch (f.operator) {

package/dist/_chunks/schema2.mjs

@@ -1,5 +1,5 @@
-import { SCHEMAS } from "./schema.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1 } from "@gscdump/lakehouse";
+import { SCHEMAS, TABLE_METADATA } from "./schema.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, defineIcebergDataset } from "@gscdump/lakehouse";
 function mapColumnType(t) {
 	switch (t) {
 		case "VARCHAR": return "STRING";
@@ -72,27 +72,43 @@ const ICEBERG_PARTITION_SPEC = [
 		name: "date_month"
 	}
 ];
-function icebergTableSpec(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
-	const base = SCHEMAS[table];
-	const dataColumns = base.columns.map((col, i) => ({
+function dataColumnsFor(table) {
+	return SCHEMAS[table].columns.map((col) => ({
 		name: col.name,
 		type: mapColumnType(col.type),
-		required: !col.nullable,
-		fieldId: 3 + i
+		required: !col.nullable
 	}));
+}
+function searchTypeDim(encoding) {
+	return {
+		toPartitionValue: (v) => encoding === "int" ? SEARCH_TYPE_INT[v] : v,
+		boundEncoding: encoding === "int" ? "int32" : "string"
+	};
+}
+function gscDatasetDef(table, encoding) {
+	const meta = TABLE_METADATA[table];
 	return {
 		namespace: "gsc",
 		table,
-		columns: [...icebergPartitionColumns(encoding), ...dataColumns],
-		partitionSpec: ICEBERG_PARTITION_SPEC,
-		naturalKey: base.sortKey,
-		identityColumns: [
-			"site_id",
-			"search_type",
-			...base.sortKey
-		]
+		identity: {
+			kind: "site-int",
+			encoding
+		},
+		dims: { search_type: searchTypeDim(encoding) },
+		columns: dataColumnsFor(table),
+		partition: ICEBERG_PARTITION_SPEC,
+		naturalKey: meta.sortKey,
+		clusterKey: meta.clusterKey
 	};
 }
+const GSC_DATASETS_STRING = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, defineIcebergDataset(gscDatasetDef(t, "string"))]));
+const GSC_DATASETS_INT = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, defineIcebergDataset(gscDatasetDef(t, "int"))]));
+function gscDataset(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
+	return (encoding === "int" ? GSC_DATASETS_INT : GSC_DATASETS_STRING)[table];
+}
+function icebergTableSpec(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
+	return gscDataset(table, encoding).tableSpec;
+}
 const ICEBERG_SCHEMAS_STRING = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "string")]));
 const ICEBERG_SCHEMAS_INT = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "int")]));
 const ICEBERG_SCHEMAS = ICEBERG_SCHEMAS_INT;
@@ -107,4 +123,4 @@ function assertIcebergTable(table) {
 	if (!isIcebergTable(table)) throw new Error(`Unknown Iceberg table '${table}'. Expected one of: ${ICEBERG_TABLES.join(", ")}`);
 	return table;
 }
-export { DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };
+export { DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };

package/dist/_chunks/sink.d.mts

@@ -34,15 +34,18 @@ declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): reado
 /**
  * First field id used for per-table (non-partition) columns — immediately
  * after the two partition-identity columns (`site_id`=1, `search_type`=2).
+ * No longer read by `defineDataset` below (field ids are now assigned by
+ * `defineIcebergDataset` itself, identity-then-dims-then-columns) — kept as a
+ * documented, byte-identity-tested constant for existing external readers.
  */
 declare const ICEBERG_FIELD_ID_BASE = 3;
 /** Shared partition spec — identical across every table. */
 declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
 /**
- * Derive the full Iceberg table spec for a table from the engine `SCHEMAS`
- * (drizzle-derived column set) plus the shared partition-identity columns.
- * Field ids are assigned deterministically from `ICEBERG_FIELD_ID_BASE` in
- * column declaration order so the same schema always yields the same ids.
+ * Derive the full Iceberg table spec for a table + encoding. Thin wrapper
+ * over {@link gscDataset}'s `tableSpec` (ADR-0021 R2-FIXES C5) — kept as a
+ * named export so existing `@gscdump/engine/iceberg` consumers see no change
+ * to their import surface.
  *
  * CONTRACT NOTE: implementation agents must treat the RETURNED VALUE as the
  * source of truth — do not hand-list columns elsewhere.

package/dist/iceberg/index.mjs

@@ -1,6 +1,6 @@
 import { TABLE_METADATA } from "../_chunks/schema.mjs";
 import { engineErrors } from "../errors.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
 import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
 import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables as dropIcebergTables$1, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
@@ -126,7 +126,7 @@ function toIntPartitionSiteId(value) {
 }
 function dedupeByIdentity(table, records) {
 	if (records.length < 2) return records;
-	const key = ICEBERG_SCHEMAS[table].identityColumns;
+	const key = gscDataset(table, "int").tableSpec.identityColumns;
 	const seen = /* @__PURE__ */ new Map();
 	for (const rec of records) {
 		const k = key.map((col) => `${rec[col] ?? ""}`).join("\0");
@@ -135,7 +135,7 @@ function dedupeByIdentity(table, records) {
 	return seen.size === records.length ? records : [...seen.values()];
 }
 function sortByClusterKey(table, records) {
-	const cols = TABLE_METADATA[table].clusterKey;
+	const cols = gscDataset(table, "int").tableSpec.clusterKey ?? [];
 	if (cols.length === 0 || records.length < 2) return records;
 	return records.slice().sort((a, b) => {
 		for (const col of cols) {
@@ -154,7 +154,7 @@ function sortByClusterKey(table, records) {
 }
 function toRecords(slice, rows, encoding) {
 	const siteVal = encoding === "int" ? toIntPartitionSiteId(slice.ctx.siteId) : slice.ctx.siteId ?? "";
-	const searchVal = encoding === "int" ? SEARCH_TYPE_INT[slice.searchType] : slice.searchType;
+	const searchVal = gscDataset(slice.table, encoding).def.dims.search_type.toPartitionValue(slice.searchType);
 	return rows.map((row) => {
 		const out = {};
 		for (const k in row) out[k] = coerceJsonSafe(row[k]);

package/dist/ingest.mjs

@@ -26,7 +26,7 @@ function toPath(gscUrl) {
 	}
 }
 function toSumPosition(apiPosition, impressions) {
-	return ((apiPosition >= 1 ? apiPosition : 1) - 1) * Math.max(impressions, 1);
+	return ((apiPosition >= 1 ? apiPosition : 1) - 1) * impressions;
 }
 function transformGscRow(table, apiRow, options = {}) {
 	const keys = apiRow.keys;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.35.1",
+  "version": "0.35.3",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/lakehouse": "0.35.1",
-    "gscdump": "0.35.1",
-    "@gscdump/contracts": "0.35.1"
+    "@gscdump/contracts": "0.35.3",
+    "@gscdump/lakehouse": "0.35.3",
+    "gscdump": "0.35.3"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",