@gscdump/lakehouse 0.1.0 → 0.35.2

package/dist/THIRD-PARTY-LICENSES.md

@@ -5,37 +5,6 @@ Apache-2.0, BlueOak-1.0.0, MIT
 
 # Bundled Dependencies
 
-## @netlify/blobs
-
-License: MIT
-By: Netlify Inc.
-Repository: https://github.com/netlify/primitives
-
-> Copyright (c) 2022 Netlify <team@netlify.com>
-> 
-> MIT License
-> 
-> Permission is hereby granted, free of charge, to any person obtaining
-> a copy of this software and associated documentation files (the
-> "Software"), to deal in the Software without restriction, including
-> without limitation the rights to use, copy, modify, merge, publish,
-> distribute, sublicense, and/or sell copies of the Software, and to
-> permit persons to whom the Software is furnished to do so, subject to
-> the following conditions:
-> 
-> The above copyright notice and this permission notice shall be
-> included in all copies or substantial portions of the Software.
-> 
-> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-> LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-> OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-> WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
----------------------------------------
-
 ## chokidar
 
 License: MIT

package/dist/provisioning/index.d.mts

@@ -63,6 +63,45 @@ interface ProvisionedCatalogRef {
   bucket: string;
   namespace: string;
 }
+interface EnableCatalogMaintenanceOptions {
+  accountId: string;
+  /** R2-Data-Catalog-scoped token (credential-set / maintenance-configs). */
+  catalogToken: string;
+  bucket: string;
+  /** See {@link CloudflareProvisionCreds.compactionCredentialToken}. */
+  compactionCredentialToken?: string;
+  /** Injectable fetch — defaults to the global. */
+  fetch?: typeof fetch;
+  /** Injectable warn sink for best-effort maintenance failures. Defaults to `console.warn`. */
+  onWarn?: (message: string) => void;
+}
+/**
+ * Standalone credential-set + compaction/snapshot-expiry enable — the same two
+ * steps {@link selfProvisionCatalog} ends with, WITHOUT its bucket-create /
+ * CORS / catalog-enable side effects. For the "backfill maintenance onto an
+ * already-ready catalog" case (re-provision / admin-repair jobs), where
+ * re-running bucket-create or catalog-enable would be wasted CF calls, not
+ * just idempotent no-ops.
+ *
+ * R2-FIXES C3 note: this export postdates the version both nuxtseo's
+ * `provision-team-catalog.ts` and gscdump.com's `team-catalog-provisioner.ts`
+ * synced against (published 0.1.0 had no such entry point), so BOTH apps still
+ * hand-roll this exact sequence locally for their idempotent-backfill branch.
+ * Swap their local duplicates for this export on the next publish + bump.
+ */
+declare function enableCatalogMaintenance(opts: EnableCatalogMaintenanceOptions): Promise<void>;
+/**
+ * Generic `prefix + key` bucket-name normalization (R2-FIXES C3 TODO): lowercase,
+ * collapse non-`[a-z0-9-]` runs to a single hyphen, trim, clamp to the R2
+ * 63-char limit. Throws if the result isn't a valid bucket name.
+ *
+ * Neither app has switched over yet: nuxtseo's `teamCatalogBucketName` and
+ * gscdump.com's `teamCatalogBucket` predate this export AND encode slightly
+ * different schemes (gscdump.com additionally suffixes the partition-key
+ * encoding, e.g. `-int`) — see their own TODO(lakehouse@next) comments for the
+ * planned migration once a version carrying this export is published.
+ */
+declare function deriveBucketName(prefix: string, key: string): string;
 interface SelfProvisionOptions {
   creds: CloudflareProvisionCreds;
   bucket: string;
@@ -121,4 +160,4 @@ declare function adoptCatalog(remote: RemoteCatalogRef | null, opts?: AdoptCatal
  * pure validation/passthrough, no CF calls, no persistence.
  */
 declare function suppliedCatalog(ref: ProvisionedCatalogRef): ProvisionedCatalogRef;
-export { AdoptCatalogOptions, AdoptCatalogResult, type AllocateCatalogSiteIdOptions, type AllocateCatalogSiteIdResult, CloudflareProvisionCreds, ProvisionedCatalogRef, RemoteCatalogRef, SelfProvisionOptions, adoptCatalog, allocateCatalogSiteId, selfProvisionCatalog, suppliedCatalog };
+export { AdoptCatalogOptions, AdoptCatalogResult, type AllocateCatalogSiteIdOptions, type AllocateCatalogSiteIdResult, CloudflareProvisionCreds, EnableCatalogMaintenanceOptions, ProvisionedCatalogRef, RemoteCatalogRef, SelfProvisionOptions, adoptCatalog, allocateCatalogSiteId, deriveBucketName, enableCatalogMaintenance, selfProvisionCatalog, suppliedCatalog };

package/dist/provisioning/index.mjs

@@ -95,6 +95,18 @@ async function ensureCredential(fetchImpl, accountId, catalogToken, credentialTo
 		onWarn(`[lakehouse-provisioning] credential-set failed for ${bucket} — compaction stays inert (credential absent): ${err instanceof Error ? err.message : String(err)}`);
 	});
 }
+async function enableCatalogMaintenance(opts) {
+	const fetchImpl = opts.fetch ?? fetch;
+	const onWarn = opts.onWarn ?? ((m) => console.warn(m));
+	if (opts.compactionCredentialToken) await ensureCredential(fetchImpl, opts.accountId, opts.catalogToken, opts.compactionCredentialToken, opts.bucket, onWarn);
+	await enableMaintenance(fetchImpl, opts.accountId, opts.catalogToken, opts.bucket, onWarn);
+}
+const MAX_BUCKET_NAME = 63;
+function deriveBucketName(prefix, key) {
+	const bucket = `${prefix}${key}`.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/-+/g, "-").replace(/^-+|-+$/g, "").slice(0, MAX_BUCKET_NAME).replace(/-+$/g, "");
+	if (!/^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(bucket)) throw new Error(`Cannot derive a valid R2 bucket name from prefix "${prefix}" + key "${key}" (got "${bucket}")`);
+	return bucket;
+}
 async function selfProvisionCatalog(opts) {
 	const { creds, bucket, namespace, corsOrigins } = opts;
 	const fetchImpl = opts.fetch ?? fetch;
@@ -138,4 +150,4 @@ function suppliedCatalog(ref) {
 	if (!ref.catalogUri || !ref.warehouse || !ref.bucket) throw new Error("suppliedCatalog: catalogUri, warehouse and bucket are all required");
 	return ref;
 }
-export { adoptCatalog, allocateCatalogSiteId, selfProvisionCatalog, suppliedCatalog };
+export { adoptCatalog, allocateCatalogSiteId, deriveBucketName, enableCatalogMaintenance, selfProvisionCatalog, suppliedCatalog };

package/dist/unsafe-raw.d.mts

@@ -1,3 +1,93 @@
-import { icebergAppendRetrying, isCommitRateLimited } from "./_chunks/catalog.mjs";
+import { IcebergConnection, icebergAppendRetrying, isCommitRateLimited } from "./_chunks/catalog.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
-export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, isCommitRateLimited, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };
+/** One listed object under a data prefix. */
+interface SweepListedObject {
+  key: string;
+  uploaded: Date;
+}
+/** One page of a prefix listing. */
+interface SweepListPage {
+  objects: readonly SweepListedObject[];
+  truncated: boolean;
+  cursor?: string;
+}
+/**
+ * Minimal storage client `sweepUncommittedOrphans` needs: list-by-prefix +
+ * batch delete against the warehouse bucket. Deliberately NOT an S3 SDK
+ * client shape — this interface is structurally satisfied by Cloudflare's
+ * native `R2Bucket` Workers binding (`env.R2_ANALYTICS.list({ prefix })` /
+ * `env.R2_ANALYTICS.delete(keys)`), so a caller running inside a Worker can
+ * pass the binding straight through with no adapter. Any other
+ * S3-compatible client can be wrapped to this shape.
+ */
+interface SweepStorageClient {
+  list: (options: {
+    prefix: string;
+    cursor?: string;
+  }) => Promise<SweepListPage>;
+  delete: (keys: string[]) => Promise<void>;
+}
+interface SweepUncommittedOrphansOptions {
+  conn: IcebergConnection;
+  s3: SweepStorageClient;
+  /**
+   * Bare R2 bucket name backing `conn`'s warehouse. Every scanned table's
+   * `location` must resolve into this exact bucket; a table whose location
+   * points elsewhere is SKIPPED — never guessed at, never touched.
+   */
+  bucket: string;
+  /**
+   * Restrict the sweep to these tables; defaults to every table currently
+   * in `conn.namespace` (`conn` already carries the namespace scope, so
+   * there's no separate `namespace` field here).
+   */
+  tables?: readonly string[];
+  /** Only delete objects older than this many hours. Default 48. */
+  graceHours?: number;
+  /** Hard cap on deletions in one run. Default 500. */
+  maxDeletes?: number;
+  /** Compute the delete set but never call `s3.delete`. Default false. */
+  dryRun?: boolean;
+  /** Injectable clock (ms epoch). Defaults to `Date.now`. */
+  now?: () => number;
+}
+interface SweepUncommittedOrphansResult {
+  /** Tables whose retained snapshots were successfully walked. */
+  scannedTables: string[];
+  /** Tables skipped because their `location` didn't resolve into `bucket` — never scanned, never touched. */
+  skippedTables: string[];
+  /** Count of live (referenced-by-some-retained-snapshot) files found across `scannedTables`. */
+  liveFileCount: number;
+  /** Objects under a scanned table's data prefix with no snapshot reference anywhere, past `graceHours`. */
+  candidateCount: number;
+  /** Keys deleted (or, under `dryRun`, that WOULD be deleted). */
+  deleted: string[];
+  dryRun: boolean;
+  /** True when `candidateCount` exceeded `maxDeletes` — remaining orphans are left for the next run. */
+  cappedAtLimit: boolean;
+  /** Set when the sweep did nothing because the table scope listed zero tables (fail-closed — see class docs). */
+  reason?: 'zero-tables';
+}
+/**
+ * Sweep data files written under a table's `data/` prefix that were NEVER
+ * referenced by any snapshot the catalog still retains — a crashed/partial
+ * writer's leftovers. This is the one orphan class Cloudflare's own R2 Data
+ * Catalog snapshot-expiration GC does not cover.
+ *
+ * HARD SAFETY:
+ * - A listing failure (table load / manifest walk throwing) PROPAGATES.
+ *   It is never caught and treated as "no live files" — that would make
+ *   every real data file look orphaned. Callers must let this reject and
+ *   must not proceed to delete on catch.
+ * - Zero tables in scope is treated as "couldn't determine what's live",
+ *   not "nothing to protect" — the sweep no-ops with `reason: 'zero-tables'`
+ *   rather than listing/deleting anything.
+ * - Deletions are capped at `maxDeletes` per run (default 500).
+ * - `dryRun` computes the exact candidate set without ever calling `s3.delete`.
+ * - Only ever lists/deletes keys under a scanned table's own
+ *   `<table.location>/data/` prefix, and every candidate is re-checked to
+ *   actually start with that prefix before being considered for deletion —
+ *   never a bare bucket-root listing, never a key outside a data prefix.
+ */
+declare function sweepUncommittedOrphans(opts: SweepUncommittedOrphansOptions): Promise<SweepUncommittedOrphansResult>;
+export { type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, isCommitRateLimited, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver, sweepUncommittedOrphans };

package/dist/unsafe-raw.mjs

@@ -1,3 +1,112 @@
 import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
 import { icebergAppendRetrying, isCommitRateLimited } from "./_chunks/catalog.mjs";
-export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, isCommitRateLimited, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };
+const DEFAULT_GRACE_HOURS = 48;
+const DEFAULT_MAX_DELETES = 500;
+const HOUR_MS = 3600 * 1e3;
+function splitBucketKey(filePath) {
+	if (!filePath.startsWith("s3://")) return null;
+	const rest = filePath.slice(5);
+	const slash = rest.indexOf("/");
+	if (slash < 0) return null;
+	return {
+		bucket: rest.slice(0, slash),
+		key: rest.slice(slash + 1)
+	};
+}
+async function scanTable(conn, table, bucket) {
+	const { metadata } = await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table
+	});
+	const location = splitBucketKey(String(metadata.location ?? ""));
+	if (!location || location.bucket !== bucket) return {
+		dataPrefix: null,
+		liveKeys: /* @__PURE__ */ new Set()
+	};
+	const dataPrefix = `${location.key.replace(/\/+$/, "")}/data/`;
+	const liveKeys = /* @__PURE__ */ new Set();
+	const snapshots = metadata.snapshots ?? [];
+	for (const snapshot of snapshots) {
+		const manifests = await icebergManifests({
+			metadata,
+			resolver: conn.resolver,
+			snapshotId: snapshot["snapshot-id"]
+		});
+		for (const manifest of manifests) for (const entry of manifest.entries) {
+			if (entry.status === 2) continue;
+			const filePath = entry.data_file?.file_path;
+			if (!filePath) continue;
+			const parsed = splitBucketKey(filePath);
+			if (parsed && parsed.bucket === bucket) liveKeys.add(parsed.key);
+		}
+	}
+	return {
+		dataPrefix,
+		liveKeys
+	};
+}
+async function listAllUnderPrefix(s3, prefix) {
+	const out = [];
+	let cursor;
+	do {
+		const page = await s3.list({
+			prefix,
+			cursor
+		});
+		out.push(...page.objects);
+		cursor = page.truncated ? page.cursor : void 0;
+	} while (cursor);
+	return out;
+}
+async function sweepUncommittedOrphans(opts) {
+	const { conn, s3, bucket } = opts;
+	const graceHours = opts.graceHours ?? DEFAULT_GRACE_HOURS;
+	const maxDeletes = opts.maxDeletes ?? DEFAULT_MAX_DELETES;
+	const dryRun = opts.dryRun ?? false;
+	const graceCutoff = (opts.now ?? Date.now)() - graceHours * HOUR_MS;
+	const tables = opts.tables ?? (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name);
+	if (tables.length === 0) return {
+		scannedTables: [],
+		skippedTables: [],
+		liveFileCount: 0,
+		candidateCount: 0,
+		deleted: [],
+		dryRun,
+		cappedAtLimit: false,
+		reason: "zero-tables"
+	};
+	const scannedTables = [];
+	const skippedTables = [];
+	let liveFileCount = 0;
+	const candidates = [];
+	for (const table of tables) {
+		const { dataPrefix, liveKeys } = await scanTable(conn, table, bucket);
+		if (!dataPrefix) {
+			skippedTables.push(table);
+			continue;
+		}
+		scannedTables.push(table);
+		liveFileCount += liveKeys.size;
+		const listed = await listAllUnderPrefix(s3, dataPrefix);
+		for (const obj of listed) {
+			if (!obj.key.startsWith(dataPrefix)) continue;
+			if (liveKeys.has(obj.key)) continue;
+			if (obj.uploaded.getTime() > graceCutoff) continue;
+			candidates.push(obj);
+		}
+	}
+	const candidateCount = candidates.length;
+	const toDelete = candidates.slice(0, maxDeletes);
+	const cappedAtLimit = candidateCount > maxDeletes;
+	if (!dryRun && toDelete.length > 0) await s3.delete(toDelete.map((o) => o.key));
+	return {
+		scannedTables,
+		skippedTables,
+		liveFileCount,
+		candidateCount,
+		deleted: toDelete.map((o) => o.key),
+		dryRun,
+		cappedAtLimit
+	};
+}
+export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, isCommitRateLimited, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver, sweepUncommittedOrphans };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "0.1.0",
+  "version": "0.35.2",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",

package/dist/_chunks/libs/@netlify/blobs.d.mts

@@ -1,6 +0,0 @@
-declare global {
-  var netlifyBlobsContext: unknown;
-}
-/**
- * The context object that we expect in the environment.
- */