@gscdump/engine 1.4.0 → 1.4.2

package/dist/_chunks/libs/icebird.mjs

@@ -1,564 +0,0 @@
-import "./hyparquet.mjs";
-import "./hyparquet-writer.mjs";
-import "./hyparquet-compressors.mjs";
-function uuid4() {
-	if (globalThis.crypto?.randomUUID) return globalThis.crypto.randomUUID();
-	return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => {
-		const r = Math.random() * 16 | 0;
-		return (c === "x" ? r : r & 3 | 8).toString(16);
-	});
-}
-const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
-function parseIcebergJson(text) {
-	let i = 0;
-	function skipWs() {
-		while (i < text.length) {
-			const c = text.charCodeAt(i);
-			if (c !== 32 && c !== 9 && c !== 10 && c !== 13) break;
-			i++;
-		}
-	}
-	function parseString() {
-		if (text[i] !== "\"") throw new Error(`expected " at ${i}`);
-		i++;
-		let s = "";
-		while (i < text.length) {
-			const c = text[i++];
-			if (c === "\"") return s;
-			if (c !== "\\") {
-				s += c;
-				continue;
-			}
-			const e = text[i++];
-			if (e === "u") {
-				s += String.fromCharCode(parseInt(text.slice(i, i + 4), 16));
-				i += 4;
-			} else if (e === "n") s += "\n";
-			else if (e === "t") s += "	";
-			else if (e === "r") s += "\r";
-			else if (e === "b") s += "\b";
-			else if (e === "f") s += "\f";
-			else s += e;
-		}
-		throw new Error("unterminated string");
-	}
-	function parseNumber() {
-		const start = i;
-		if (text[i] === "-") i++;
-		while (text[i] >= "0" && text[i] <= "9") i++;
-		const intEnd = i;
-		let isFloat = false;
-		if (text[i] === ".") {
-			isFloat = true;
-			i++;
-			while (text[i] >= "0" && text[i] <= "9") i++;
-		}
-		if (text[i] === "e" || text[i] === "E") {
-			isFloat = true;
-			i++;
-			if (text[i] === "+" || text[i] === "-") i++;
-			while (text[i] >= "0" && text[i] <= "9") i++;
-		}
-		if (isFloat) return Number(text.slice(start, i));
-		const intStr = text.slice(start, intEnd);
-		if (intStr.length >= 16) {
-			const n = BigInt(intStr);
-			if (n > MAX_SAFE || n < -MAX_SAFE) return n;
-		}
-		return Number(intStr);
-	}
-	function parseLiteral(lit, val) {
-		if (text.slice(i, i + lit.length) !== lit) throw new Error(`bad literal at ${i}`);
-		i += lit.length;
-		return val;
-	}
-	function parseValue() {
-		skipWs();
-		const ch = text[i];
-		if (ch === "\"") return parseString();
-		if (ch === "{") return parseObject();
-		if (ch === "[") return parseArray();
-		if (ch === "t") return parseLiteral("true", true);
-		if (ch === "f") return parseLiteral("false", false);
-		if (ch === "n") return parseLiteral("null", null);
-		return parseNumber();
-	}
-	function parseObject() {
-		i++;
-		skipWs();
-		const obj = {};
-		if (text[i] === "}") {
-			i++;
-			return obj;
-		}
-		while (true) {
-			skipWs();
-			const key = parseString();
-			skipWs();
-			if (text[i] !== ":") throw new Error(`expected : at ${i}`);
-			i++;
-			obj[key] = parseValue();
-			skipWs();
-			if (text[i] === ",") {
-				i++;
-				continue;
-			}
-			if (text[i] === "}") {
-				i++;
-				return obj;
-			}
-			throw new Error(`expected , or } at ${i}`);
-		}
-	}
-	function parseArray() {
-		i++;
-		skipWs();
-		const arr = [];
-		if (text[i] === "]") {
-			i++;
-			return arr;
-		}
-		while (true) {
-			arr.push(parseValue());
-			skipWs();
-			if (text[i] === ",") {
-				i++;
-				continue;
-			}
-			if (text[i] === "]") {
-				i++;
-				return arr;
-			}
-			throw new Error(`expected , or ] at ${i}`);
-		}
-	}
-	const value = parseValue();
-	skipWs();
-	if (i !== text.length) throw new Error(`unexpected trailing input at ${i}`);
-	return value;
-}
-function stringifyIcebergJson(value, indent = 2) {
-	const pad = " ".repeat(indent);
-	function serialize(val, depth) {
-		if (typeof val === "bigint") return val.toString();
-		if (val === null || typeof val !== "object") return JSON.stringify(val);
-		const inner = pad.repeat(depth + 1);
-		const outer = pad.repeat(depth);
-		if (Array.isArray(val)) {
-			if (val.length === 0) return "[]";
-			return `[\n${val.map((v) => inner + serialize(v === void 0 ? null : v, depth + 1)).join(",\n")}\n${outer}]`;
-		}
-		const keys = Object.keys(val).filter((k) => val[k] !== void 0);
-		if (keys.length === 0) return "{}";
-		return `{\n${keys.map((k) => `${inner}${JSON.stringify(k)}: ${serialize(val[k], depth + 1)}`).join(",\n")}\n${outer}}`;
-	}
-	return serialize(value, 0);
-}
-async function restCatalogCreateTable(ctx, { namespace, table, schema, location, partitionSpec, writeOrder, stageCreate, properties }) {
-	const ns = encodeNamespace(namespace);
-	const body = {
-		name: table,
-		schema
-	};
-	if (location !== void 0) body.location = location;
-	if (partitionSpec !== void 0) body["partition-spec"] = partitionSpec;
-	if (writeOrder !== void 0) body["write-order"] = writeOrder;
-	if (stageCreate !== void 0) body["stage-create"] = stageCreate;
-	if (properties !== void 0) body.properties = properties;
-	const responseBody = parseIcebergJson(await (await restFetch(ctx, `namespaces/${ns}/tables`, {
-		method: "POST",
-		headers: { "content-type": "application/json" },
-		body: stringifyIcebergJson(body)
-	})).text());
-	return {
-		metadataLocation: responseBody["metadata-location"],
-		metadata: responseBody.metadata,
-		config: responseBody.config ?? {}
-	};
-}
-function encodeNamespace(namespace) {
-	return (Array.isArray(namespace) ? namespace : namespace.split(".")).map((p) => encodeURIComponent(p)).join("%1F");
-}
-async function restFetch(ctx, path, init) {
-	const prefixSegment = ctx.prefix ? `${ctx.prefix.replace(/^\/|\/$/g, "")}/` : "";
-	const fullUrl = `${ctx.url}/v1/${prefixSegment}${path}`;
-	let merged = mergeRequestInit(ctx.requestInit, init);
-	if (ctx.signRequest) merged = await ctx.signRequest(fullUrl, merged);
-	const res = await fetch(fullUrl, merged);
-	if (!res.ok) await throwRestError(res);
-	return res;
-}
-function mergeRequestInit(a, b) {
-	if (!a) return b;
-	if (!b) return a;
-	return {
-		...a,
-		...b,
-		headers: {
-			...headersToObject(a.headers),
-			...headersToObject(b.headers)
-		}
-	};
-}
-function headersToObject(h) {
-	if (!h) return {};
-	if (h instanceof Headers) {
-		const out = {};
-		h.forEach((v, k) => {
-			out[k] = v;
-		});
-		return out;
-	}
-	if (Array.isArray(h)) return Object.fromEntries(h);
-	return h;
-}
-async function throwRestError(res) {
-	let detail = "";
-	try {
-		const body = parseIcebergJson(await res.text());
-		if (body?.error) {
-			const { code, type, message } = body.error;
-			detail = `${code ?? res.status} ${type ?? ""}: ${message ?? ""}`.trim();
-		}
-	} catch {}
-	const err = new Error(detail || `${res.status} ${res.statusText}`);
-	err.status = res.status;
-	throw err;
-}
-function validateSchemaForVersion(schema, formatVersion) {
-	for (const field of schema.fields) validateFieldForVersion(field, formatVersion, field.name);
-}
-function maxFieldId(fields = []) {
-	let max = 0;
-	for (const field of fields) {
-		if (max < field.id) max = field.id;
-		const nested = maxNestedFieldId(field.type);
-		if (max < nested) max = nested;
-	}
-	return max;
-}
-function maxNestedFieldId(type) {
-	if (typeof type === "string") return 0;
-	if (type.type === "list") {
-		const elementId = type["element-id"] ?? 0;
-		return Math.max(elementId, maxNestedFieldId(type.element));
-	}
-	if (type.type === "map") {
-		const keyId = type["key-id"] ?? 0;
-		const valueId = type["value-id"] ?? 0;
-		return Math.max(keyId, valueId, maxNestedFieldId(type.key), maxNestedFieldId(type.value));
-	}
-	if (type.type === "struct") return maxFieldId(type.fields);
-	return 0;
-}
-const MAX_USER_FIELD_ID = 2147483447;
-function validateFieldForVersion(field, formatVersion, path) {
-	if (typeof field.id === "number" && field.id > MAX_USER_FIELD_ID) throw new Error(`field id ${field.id} is in the reserved range (> ${MAX_USER_FIELD_ID}) (field: ${path})`);
-	if (formatVersion < 3) {
-		checkTypeForV2(field.type, path);
-		if (field["initial-default"] !== void 0) throw new Error(`initial-default requires format-version 3 (field: ${path})`);
-		if (field["write-default"] !== void 0) throw new Error(`write-default requires format-version 3 (field: ${path})`);
-	} else checkV3Default(field, path);
-	checkNestedFieldsForVersion(field.type, formatVersion, path);
-}
-function checkTypeForV2(type, path) {
-	if (typeof type === "string") {
-		if (type === "unknown" || type === "variant" || type === "timestamp_ns" || type === "timestamptz_ns" || type === "geometry" || type.startsWith("geometry(") || type === "geography" || type.startsWith("geography(")) throw new Error(`type ${type} requires format-version 3 (field: ${path})`);
-		return;
-	}
-	if (type.type === "struct") for (const f of type.fields) checkTypeForV2(f.type, `${path}.${f.name}`);
-	else if (type.type === "list") checkTypeForV2(type.element, `${path}.element`);
-	else if (type.type === "map") {
-		checkTypeForV2(type.key, `${path}.key`);
-		checkTypeForV2(type.value, `${path}.value`);
-	}
-}
-function checkV3Default(field, path) {
-	const type = typeName(field.type);
-	if (!requiresNullDefault(type)) return;
-	for (const key of ["initial-default", "write-default"]) if (field[key] != null) throw new Error(`${key} for field ${path} of type ${type} must default to null`);
-}
-function checkNestedFieldsForVersion(type, formatVersion, path) {
-	if (typeof type === "string") return;
-	if (type.type === "struct") for (const f of type.fields) validateFieldForVersion(f, formatVersion, `${path}.${f.name}`);
-	else if (type.type === "list") {
-		checkReservedFieldId(type["element-id"], `${path}.element`);
-		checkNestedFieldsForVersion(type.element, formatVersion, `${path}.element`);
-	} else if (type.type === "map") {
-		checkReservedFieldId(type["key-id"], `${path}.key`);
-		checkReservedFieldId(type["value-id"], `${path}.value`);
-		checkNestedFieldsForVersion(type.key, formatVersion, `${path}.key`);
-		checkNestedFieldsForVersion(type.value, formatVersion, `${path}.value`);
-	}
-}
-function checkReservedFieldId(id, path) {
-	if (typeof id === "number" && id > MAX_USER_FIELD_ID) throw new Error(`field id ${id} is in the reserved range (> ${MAX_USER_FIELD_ID}) (field: ${path})`);
-}
-function typeName(type) {
-	return typeof type === "string" ? type : type.type;
-}
-function requiresNullDefault(type) {
-	return type === "unknown" || type === "variant" || type === "geometry" || type.startsWith("geometry(") || type === "geography" || type.startsWith("geography(");
-}
-function parseDecimalType(type) {
-	const m = /^decimal\((\d+),\s*(\d+)\)$/.exec(type);
-	if (!m) return void 0;
-	return {
-		precision: parseInt(m[1], 10),
-		scale: parseInt(m[2], 10)
-	};
-}
-function decimalRequiredBytes(precision) {
-	const limit = 10n ** BigInt(precision);
-	let n = 1;
-	let bound = 128n;
-	while (limit > bound) {
-		n++;
-		bound <<= 8n;
-	}
-	return n;
-}
-function parseTransform(transform) {
-	if (transform === "identity" || transform === "void" || transform === "year" || transform === "month" || transform === "day" || transform === "hour") return { kind: transform };
-	let m = /^bucket\[(\d+)\]$/.exec(transform);
-	if (m) {
-		const n = parseInt(m[1], 10);
-		if (n > 0) return {
-			kind: "bucket",
-			n
-		};
-	}
-	m = /^truncate\[(\d+)\]$/.exec(transform);
-	if (m) {
-		const w = parseInt(m[1], 10);
-		if (w > 0) return {
-			kind: "truncate",
-			w
-		};
-	}
-	throw new Error(`unsupported partition transform: ${transform}`);
-}
-function transformResultType(transform, sourceType) {
-	const parsed = parseTransform(transform);
-	validateTransformSource(parsed, sourceType);
-	switch (parsed.kind) {
-		case "identity":
-		case "truncate": return sourceType;
-		case "void": return "int";
-		case "year":
-		case "month":
-		case "day":
-		case "hour":
-		case "bucket": return "int";
-	}
-}
-function validateTransformSource(parsed, sourceType) {
-	const t = typeName(sourceType);
-	switch (parsed.kind) {
-		case "identity":
-			if (t === "variant" || t === "geometry" || t.startsWith("geometry(") || t === "geography" || t.startsWith("geography(")) throw new Error(`identity transform: unsupported source type ${t}`);
-			return;
-		case "void": return;
-		case "bucket":
-			if (t === "int" || t === "long" || t.startsWith("decimal(") || t === "date" || t === "time" || t === "timestamp" || t === "timestamptz" || t === "timestamp_ns" || t === "timestamptz_ns" || t === "string" || t === "uuid" || t.startsWith("fixed[") || t === "binary" || t === "fixed") return;
-			throw new Error(`bucket transform: unsupported source type ${t}`);
-		case "truncate":
-			if (t === "int" || t === "long" || t.startsWith("decimal(") || t === "string" || t === "binary") return;
-			throw new Error(`truncate transform: unsupported source type ${t}`);
-		case "year":
-		case "month":
-		case "day":
-			if (t === "date" || t === "timestamp" || t === "timestamptz" || t === "timestamp_ns" || t === "timestamptz_ns") return;
-			throw new Error(`${parsed.kind} transform: unsupported source type ${t}`);
-		case "hour":
-			if (t === "timestamp" || t === "timestamptz" || t === "timestamp_ns" || t === "timestamptz_ns") return;
-			throw new Error("hour transform: unsupported source type " + t);
-	}
-}
-function validatePartitionSpecForWrite(schema, partitionSpec, label = "partition spec") {
-	for (const pf of partitionSpec.fields) {
-		const sourceId = pf["source-id"];
-		if (sourceId === void 0) throw new Error(`${label}: partition field ${pf.name} is missing source-id`);
-		const sourceField = schema.fields.find((f) => f.id === sourceId);
-		if (!sourceField) throw new Error(`${label}: partition source field id ${sourceId} not found in schema`);
-		icebergTypeToAvro(transformResultType(pf.transform, sourceField.type), pf["field-id"]);
-	}
-}
-function icebergTypeToAvro(type, fieldId) {
-	const name = typeof type === "string" ? type : type.type;
-	const decimal = parseDecimalType(name);
-	if (decimal) return {
-		type: "fixed",
-		name: `r102_${fieldId}`,
-		size: decimalRequiredBytes(decimal.precision),
-		logicalType: "decimal",
-		precision: decimal.precision,
-		scale: decimal.scale
-	};
-	const fixed = /^fixed\[(\d+)\]$/.exec(name);
-	if (fixed) return {
-		type: "fixed",
-		name: `r102_${fieldId}`,
-		size: parseInt(fixed[1], 10)
-	};
-	switch (name) {
-		case "boolean": return "boolean";
-		case "int": return "int";
-		case "long": return "long";
-		case "float": return "float";
-		case "double": return "double";
-		case "string": return "string";
-		case "uuid": return {
-			type: "fixed",
-			name: `r102_${fieldId}`,
-			size: 16,
-			logicalType: "uuid"
-		};
-		case "binary": return "bytes";
-		case "date": return {
-			type: "int",
-			logicalType: "date"
-		};
-		case "time": return {
-			type: "long",
-			logicalType: "time-micros"
-		};
-		case "timestamp": return {
-			type: "long",
-			logicalType: "timestamp-micros",
-			"adjust-to-utc": false
-		};
-		case "timestamptz": return {
-			type: "long",
-			logicalType: "timestamp-micros",
-			"adjust-to-utc": true
-		};
-		case "timestamp_ns": return {
-			type: "long",
-			logicalType: "timestamp-nanos",
-			"adjust-to-utc": false
-		};
-		case "timestamptz_ns": return {
-			type: "long",
-			logicalType: "timestamp-nanos",
-			"adjust-to-utc": true
-		};
-		default: throw new Error(`unsupported partition source type: ${name}`);
-	}
-}
-async function icebergCreate({ tableUrl, resolver, schema, formatVersion, partitionSpec, sortOrder, properties, conditionalCommits }) {
-	if (!tableUrl) throw new Error("tableUrl is required");
-	if (formatVersion === void 0) {
-		const propVersion = properties?.["format-version"];
-		formatVersion = propVersion !== void 0 ? Number(propVersion) : 2;
-	}
-	if (formatVersion !== 2 && formatVersion !== 3) throw new Error(`unsupported format-version: ${formatVersion}`);
-	const metadataVersion = 1;
-	const metadataUrl = `${tableUrl}/metadata/v${metadataVersion}.metadata.json`;
-	const initialSchema = schema ?? {
-		type: "struct",
-		"schema-id": 0,
-		fields: []
-	};
-	validateSchemaForVersion(initialSchema, formatVersion);
-	const initialPartitionSpec = partitionSpec ?? {
-		"spec-id": 0,
-		fields: []
-	};
-	validatePartitionSpecForWrite(initialSchema, initialPartitionSpec);
-	const initialSortOrder = sortOrder ?? {
-		"order-id": 0,
-		fields: []
-	};
-	const metadata = {
-		"format-version": formatVersion,
-		"table-uuid": uuid4(),
-		location: tableUrl,
-		"last-sequence-number": 0,
-		"last-updated-ms": Date.now(),
-		"last-column-id": maxFieldId(initialSchema.fields),
-		"current-schema-id": initialSchema["schema-id"] ?? 0,
-		schemas: [initialSchema],
-		"default-spec-id": initialPartitionSpec["spec-id"],
-		"partition-specs": [initialPartitionSpec],
-		"last-partition-id": maxPartitionFieldId(initialPartitionSpec.fields),
-		"sort-orders": [initialSortOrder],
-		"default-sort-order-id": initialSortOrder["order-id"]
-	};
-	if (properties) metadata.properties = properties;
-	if (formatVersion >= 3) metadata["next-row-id"] = 0;
-	if (!resolver.writer) throw new Error("resolver.writer is required");
-	const metadataWriter = conditionalCommits ? resolver.writer(metadataUrl, { ifNoneMatch: "*" }) : resolver.writer(metadataUrl);
-	const metadataBytes = new TextEncoder().encode(stringifyIcebergJson(metadata));
-	metadataWriter.appendBytes(metadataBytes);
-	await metadataWriter.finish();
-	const versionHintUrl = `${tableUrl}/metadata/version-hint.text`;
-	try {
-		const versionHintWriter = resolver.writer(versionHintUrl);
-		const versionHintBytes = new TextEncoder().encode(String(metadataVersion));
-		versionHintWriter.appendBytes(versionHintBytes);
-		await versionHintWriter.finish();
-	} catch (err) {
-		if (!conditionalCommits) throw err;
-	}
-	return metadata;
-}
-function maxPartitionFieldId(partitionFields = []) {
-	let max = 0;
-	for (const pf of partitionFields) if (max < pf["field-id"]) max = pf["field-id"];
-	return max;
-}
-Object.freeze({
-	maxAttempts: 50,
-	initialMs: 50,
-	maxMs: 3e3,
-	factor: 2,
-	totalTimeoutMs: 1800 * 1e3
-});
-async function icebergCreateTable({ catalog, namespace, table, tableUrl, schema, partitionSpec, sortOrder, properties, formatVersion, stageCreate }) {
-	if (catalog.type === "rest") {
-		if (!namespace || !table) throw new Error("namespace and table are required for rest catalogs");
-		if (!schema) throw new Error("schema is required for rest catalogs");
-		const { metadata } = await restCatalogCreateTable(catalog, {
-			namespace,
-			table,
-			schema,
-			location: tableUrl,
-			partitionSpec,
-			writeOrder: sortOrder,
-			stageCreate,
-			properties
-		});
-		return metadata;
-	}
-	if (!tableUrl) throw new Error("tableUrl is required for file catalogs");
-	return await icebergCreate({
-		tableUrl,
-		resolver: catalog.resolver,
-		schema,
-		formatVersion,
-		partitionSpec,
-		sortOrder,
-		properties,
-		conditionalCommits: catalog.conditionalCommits
-	});
-}
-new TextEncoder();
-(() => {
-	if (typeof setImmediate === "function") return () => new Promise((resolve) => setImmediate(resolve));
-	if (typeof MessageChannel !== "undefined") {
-		const channel = new MessageChannel();
-		const queue = [];
-		channel.port1.onmessage = () => {
-			const resolve = queue.shift();
-			if (resolve) resolve();
-		};
-		return () => new Promise((resolve) => {
-			queue.push(resolve);
-			channel.port2.postMessage(0);
-		});
-	}
-	return () => new Promise((resolve) => setTimeout(resolve, 0));
-})();
-new TextEncoder();
-export { icebergCreateTable };