@gscdump/engine 7.6.0 → 7.7.0

package/dist/iceberg/catalog.d.mts

@@ -1,7 +1,7 @@
 import { EngineError } from "../errors.mjs";
 import { IcebergTableName, PartitionKeyEncoding } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/index.mjs";
 import { Result } from "gscdump/result";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, QueryProfiler } from "@gscdump/lakehouse";
 type IcebirdCreateTableOptions = Parameters<typeof icebergCreateTable>[0];

package/dist/iceberg/catalog.mjs

@@ -2,8 +2,8 @@ import { engineErrors } from "../errors.mjs";
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import "../schema.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_PARTITION_SPEC, ICEBERG_TABLES, icebergSchemasFor } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/index.mjs";
 import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
 import { icebergAppendRetrying } from "@gscdump/lakehouse/unsafe-raw";

package/dist/node_modules/.pnpm/{hyparquet-compressors@1.1.2 → hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d}/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -39,12 +39,15 @@ function gzipStart(input, i) {
 	for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
 	return i + (flag & 2);
 }
-function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
-	let out = output ?? /* @__PURE__ */ new Uint8Array(1024);
+function gunzip(input, output, inputIndex = 0, outputIndex = 0, limits) {
+	const maximum = limits?.maxOutputBytes;
+	if (limits && (limits.raw !== true || !Number.isSafeInteger(maximum) || maximum < 0 || output || inputIndex !== 0 || outputIndex !== 0)) throw new RangeError("Invalid bounded raw DEFLATE options");
+	let out = output ?? new Uint8Array(limits ? Math.min(1024, maximum) : 1024);
+	if (limits && !input.length) throw new Error("unexpected EOF");
 	if (!(input.length - inputIndex)) return out;
-	const payloadStart = gzipStart(input, inputIndex);
-	if (payloadStart === input.length - 8) return out;
-	if (payloadStart > input.length - 8) throw new Error("unexpected EOF");
+	const payloadStart = limits ? 0 : gzipStart(input, inputIndex);
+	if (!limits && payloadStart === input.length - 8) return out;
+	if (!limits && payloadStart > input.length - 8) throw new Error("unexpected EOF");
 	let pos = payloadStart * 8;
 	let final = 0;
 	let lengthBits = 0;
@@ -53,9 +56,10 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 	let distMap;
 	const totalBits = input.length * 8;
 	function ensureSize(length) {
+		if (limits && (!Number.isSafeInteger(length) || length > maximum)) throw new RangeError("DEFLATE output byte limit exceeded");
 		if (!output && length > out.length) {
 			const old = out;
-			out = new Uint8Array(Math.max(old.length * 2, length));
+			out = new Uint8Array(limits ? Math.min(maximum, Math.max(old.length * 2, length)) : Math.max(old.length * 2, length));
 			out.set(old);
 		}
 	}
@@ -67,6 +71,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			if (!type) {
 				const s = shft(pos) + 4;
 				const l = input[s - 4] | input[s - 3] << 8;
+				if (limits && (s > input.length || ((input[s - 2] | input[s - 1] << 8) ^ l) !== 65535)) throw new Error("invalid stored block length");
 				const t = s + l;
 				if (t > input.length) throw new Error("unexpected EOF");
 				ensureSize(outputIndex + l);
@@ -93,6 +98,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				const codeLengthMap = huffMap(codeLengthTree, codeLengthBits, 1);
 				for (let i = 0; i < tl;) {
 					const r = codeLengthMap[bits(input, pos, clbMask)];
+					if (limits && !r) throw new Error("invalid code length");
 					pos += r & 15;
 					const symbol = r >> 4;
 					if (symbol < 16) lengthDistanceTree[i++] = symbol;
@@ -100,6 +106,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 						let copy = 0;
 						let n = 0;
 						if (symbol === 16) {
+							if (limits && i === 0) throw new Error("invalid code length repeat");
 							n = 3 + bits(input, pos, 3);
 							pos += 2;
 							copy = lengthDistanceTree[i - 1];
@@ -110,6 +117,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 							n = 11 + bits(input, pos, 127);
 							pos += 7;
 						}
+						if (limits && i + n > tl) throw new Error("invalid code length repeat");
 						while (n--) lengthDistanceTree[i++] = copy;
 					}
 				}
@@ -122,7 +130,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			} else throw new Error("invalid block type");
 			if (pos > totalBits) throw new Error("unexpected EOF");
 		}
-		ensureSize(outputIndex + 131072);
+		ensureSize(limits ? Math.min(maximum, outputIndex + 131072) : outputIndex + 131072);
 		const lms = (1 << lengthBits) - 1;
 		const dms = (1 << distBits) - 1;
 		for (;;) {
@@ -131,8 +139,10 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			pos += code & 15;
 			if (pos > totalBits) throw new Error("unexpected EOF");
 			if (!code) throw new Error("invalid length/literal");
-			if (sym < 256) out[outputIndex++] = sym;
-			else if (sym === 256) {
+			if (sym < 256) {
+				if (limits) ensureSize(outputIndex + 1);
+				out[outputIndex++] = sym;
+			} else if (sym === 256) {
 				lengthMap = void 0;
 				break;
 			} else {
@@ -147,6 +157,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				const d = distMap[bits16(input, pos) & dms];
 				const dsym = d >> 4;
 				if (!d) throw new Error("invalid distance");
+				if (limits && dsym > 29) throw new Error("invalid distance symbol");
 				pos += d & 15;
 				let dt = fixedDistance[dsym];
 				if (dsym > 3) {
@@ -163,10 +174,12 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 		}
 		if (lengthMap) final = 1;
 	} while (!final);
-	if (outputIndex < out.length) {
+	if (limits && Math.ceil(pos / 8) !== input.length) throw new Error("trailing raw DEFLATE bytes");
+	if (!limits && outputIndex < out.length) {
 		const nextBlock = Math.ceil(pos / 8) + 8;
 		gunzip(input, out, nextBlock, outputIndex);
 	}
+	if (limits && outputIndex !== out.length) return out.slice(0, outputIndex);
 	if (!output) return out.subarray(0, outputIndex);
 	return out;
 }

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/avro/avro.metadata.mjs

@@ -1,2 +1,3 @@
+import "./avro.budget.mjs";
 Array.from({ length: 128 }, (_, i) => BigInt(i >>> 1 ^ -(i & 1)));
 export {};

package/dist/node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/avro/avro.read.mjs

@@ -0,0 +1,6 @@
+import "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
+import "../../../../../hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
+import "./avro.budget.mjs";
+import "./avro.metadata.mjs";
+new TextDecoder();
+export {};

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/fetch.mjs

@@ -1,6 +1,6 @@
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
 import "../../../../hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
-import "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import "./avro/avro.metadata.mjs";
 import "./avro/avro.read.mjs";
 import "./utils.mjs";

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/manifest.mjs

@@ -1,3 +1,4 @@
+import "./avro/avro.budget.mjs";
 import "./avro/avro.metadata.mjs";
 import "./avro/avro.read.mjs";
 import "./fetch.mjs";

package/dist/node_modules/.pnpm/{icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f → icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471}/node_modules/icebird/src/read.mjs

@@ -2,7 +2,7 @@ import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/schema.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/filter.mjs";
 import "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
-import "../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
+import "../../../../hyparquet-compressors@1.1.2_patch_hash_821c89dbb6181f0c34d9abebd844451458a4f16c04e7aa7a19abca0c09c2238d/node_modules/hyparquet-compressors/src/index.mjs";
 import "./utils.mjs";
 import "./delete.mjs";
 import "./fetch.mjs";

package/dist/node_modules/.pnpm/icebird@0.8.30_patch_hash_1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/geospatial.mjs

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "7.6.0",
+  "version": "7.7.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^7.6.0",
-    "@gscdump/lakehouse": "^7.6.0",
+    "@gscdump/contracts": "^7.7.0",
+    "@gscdump/lakehouse": "^7.7.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^7.6.0",
+    "gscdump": "^7.7.0",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },

package/dist/node_modules/.pnpm/icebird@0.8.30_patch_hash_a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/avro/avro.read.mjs

@@ -1,5 +0,0 @@
-import "../../../../../hyparquet@1.31.1/node_modules/hyparquet/src/convert.mjs";
-import "../../../../../hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/index.mjs";
-import "./avro.metadata.mjs";
-new TextDecoder();
-export {};