@gscdump/lakehouse 0.1.0

package/dist/_chunks/libs/icebird.d.mts

@@ -0,0 +1,549 @@
+import { AsyncBuffer } from "./hyparquet.mjs";
+import { Writer } from "hyparquet-writer/src/types.js";
+interface WriterOptions {
+  /**
+   * If `'*'`, the writer must create the object only if it does not already
+   * exist (S3 conditional write: `If-None-Match: *`). On collision the
+   * underlying `finish()` rejects with an error whose `status` is 412 (or 409
+   * on some providers). The HTTP/S3 `urlResolver` translates this to the
+   * `If-None-Match` header. Other resolvers may honor or ignore it.
+   */
+  ifNoneMatch?: '*';
+}
+interface Resolver {
+  reader: (path: string, byteLength?: number) => AsyncBuffer | Promise<AsyncBuffer>;
+  writer?: (path: string, options?: WriterOptions) => Writer;
+  deleter?: (path: string) => Promise<void>;
+}
+type Lister = (path: string) => Promise<string[]>;
+/** Catalogs */
+interface RestCatalogContext {
+  type: 'rest';
+  url: string;
+  prefix: string;
+  defaults: Record<string, string>;
+  overrides: Record<string, string>;
+  requestInit?: RequestInit;
+}
+interface FileCatalog {
+  type: 'file';
+  resolver: Resolver;
+  lister?: Lister;
+  /**
+   * Opt in to S3-safe metadata commits: every `vN.metadata.json` (the
+   * initial create and every subsequent commit) is written with
+   * `If-None-Match: *` and `version-hint.text` is best-effort. High-level
+   * write functions retry on 412/409 by reloading the latest metadata and
+   * re-staging. `icebergCreateTable` and `icebergTransaction` do not retry.
+   * Default false preserves backwards-compatible (overwrite) behavior.
+   */
+  conditionalCommits?: boolean;
+}
+type Catalog = RestCatalogContext | FileCatalog;
+interface TableIdentifier {
+  namespace: string[];
+  name: string;
+}
+interface LoadTableResponse {
+  metadataLocation?: string;
+  metadata: TableMetadata;
+  config: Record<string, string>;
+}
+interface TableMetadata {
+  'format-version': number;
+  'table-uuid': string;
+  location: string;
+  'last-sequence-number': number; // missing in V1, required in V2+
+  'last-updated-ms': number;
+  'last-column-id': number;
+  'current-schema-id': number; // optional in V1, required in V2+
+  schemas: Schema[]; // optional in V1, required in V2+
+  'default-spec-id': number; // optional in V1, required in V2+
+  'partition-specs': PartitionSpec[]; // optional in V1, required in V2+
+  'last-partition-id': number; // optional in V1, required in V2+
+  properties?: Record<string, string>;
+  'current-snapshot-id'?: number | bigint;
+  snapshots?: Snapshot[];
+  'snapshot-log'?: SnapshotLog[];
+  'metadata-log'?: MetadataLog[];
+  'sort-orders': SortOrder[]; // optional in V1, required in V2+
+  'default-sort-order-id': number; // optional in V1, required in V2+
+  refs?: Record<string, SnapshotRef>;
+  statistics?: TableStatistics[];
+  'partition-statistics'?: PartitionStatistics[];
+  'next-row-id'?: number | bigint; // required in V3
+  'encryption-keys'?: EncryptionKey[]; // V3
+}
+interface Schema {
+  type: 'struct';
+  'schema-id': number;
+  'identifier-field-ids'?: number[];
+  fields: Field[];
+}
+interface Field {
+  id: number;
+  name: string;
+  required: boolean;
+  type: IcebergType;
+  doc?: string;
+  'initial-default'?: any;
+  'write-default'?: any;
+}
+type IcebergType = 'unknown' | 'boolean' | 'int' | 'long' | 'float' | 'double' | 'date' | 'time' | 'timestamp' | 'timestamptz' | 'timestamp_ns' | 'timestamptz_ns' | 'string' | 'uuid' | `fixed[${number}]` | 'binary' | `decimal(${number},${number})` | `decimal(${number}, ${number})` | 'variant' | 'geometry' | `geometry(${string})` | 'geography' | `geography(${string})` | IcebergNestedType;
+type IcebergNestedType = Schema | {
+  type: 'list';
+  'element-id': number;
+  'element-required': boolean;
+  element: IcebergType;
+} | {
+  type: 'map';
+  'key-id': number;
+  key: IcebergType;
+  'value-id': number;
+  'value-required': boolean;
+  value: IcebergType;
+};
+interface PartitionSpec {
+  'spec-id': number;
+  fields: PartitionField[];
+}
+interface PartitionField {
+  'source-id'?: number;
+  'source-ids'?: number[];
+  'field-id': number;
+  name: string;
+  transform: PartitionTransform;
+}
+type PartitionTransform = 'identity' | `bucket[${number}]` | `truncate[${number}]` | 'year' | 'month' | 'day' | 'hour' | 'void' | string;
+interface PartitionStatistics {
+  'snapshot-id': bigint;
+  'statistics-path': string;
+  'file-size-in-bytes': bigint;
+}
+interface SortOrder {
+  'order-id': number;
+  'fields': SortField[];
+}
+interface SortField {
+  transform: string;
+  'source-id'?: number;
+  'source-ids'?: number[]; // V3
+  'direction': 'asc' | 'desc';
+  'null-order': 'nulls-first' | 'nulls-last';
+}
+interface Snapshot {
+  'snapshot-id': number | bigint;
+  'parent-snapshot-id'?: number | bigint;
+  'sequence-number': number;
+  'timestamp-ms': number;
+  'manifest-list': string;
+  manifests?: Manifest[];
+  summary: {
+    // spec: "value of these fields should be of string type"
+    operation: string; // 'spark.app.id'?: string
+    'added-data-files'?: string;
+    'added-records'?: string;
+    'deleted-data-files'?: string;
+    'deleted-records'?: string;
+    'removed-files-size'?: string;
+    'added-delete-files'?: string;
+    'removed-delete-files'?: string;
+    'added-position-deletes'?: string;
+    'removed-position-deletes'?: string;
+    'added-equality-deletes'?: string;
+    'removed-equality-deletes'?: string;
+    'added-dvs'?: string;
+    'removed-dvs'?: string;
+    'added-files-size'?: string;
+    'changed-partition-count'?: string;
+    'total-records'?: string;
+    'total-files-size'?: string;
+    'total-data-files'?: string;
+    'total-delete-files'?: string;
+    'total-position-deletes'?: string;
+    'total-equality-deletes'?: string;
+  };
+  'schema-id'?: number;
+  'first-row-id'?: number; // V3
+  'added-rows'?: number; // V3
+  'key-id'?: string; // V3
+}
+interface TableStatistics {
+  'snapshot-id': number | bigint;
+  'statistics-path': string;
+  'file-size-in-bytes': bigint;
+  'file-footer-size-in-bytes': bigint;
+}
+interface SnapshotLog {
+  'timestamp-ms': number;
+  'snapshot-id': number | bigint;
+}
+interface SnapshotRef {
+  'snapshot-id': number | bigint;
+  type: 'branch' | 'tag';
+  'min-snapshots-to-keep'?: number;
+  'max-snapshot-age-ms'?: number;
+  'max-ref-age-ms'?: number;
+}
+interface EncryptionKey {
+  'key-id': string;
+  'key-metadata': string;
+}
+/**
+ * Iceberg REST `TableRequirement`s recognized by `checkRequirements`.
+ */
+interface MetadataLog {
+  'timestamp-ms': number;
+  'metadata-file': string;
+}
+interface Manifest {
+  manifest_path: string;
+  manifest_length: bigint;
+  partition_spec_id: number;
+  content: 0 | 1; // 0=data, 1=deletes
+  sequence_number?: bigint;
+  min_sequence_number?: bigint;
+  added_snapshot_id: bigint;
+  added_files_count: number;
+  existing_files_count: number;
+  deleted_files_count: number;
+  added_rows_count: bigint;
+  existing_rows_count: bigint;
+  deleted_rows_count: bigint;
+  partitions?: FieldSummary[]; // key_metadata?: unknown
+  first_row_id?: bigint | number;
+}
+interface ManifestEntry {
+  status: 0 | 1 | 2; // 0=existing, 1=added, 2=deleted
+  snapshot_id?: bigint;
+  sequence_number?: bigint;
+  file_sequence_number?: bigint;
+  partition_spec_id?: number;
+  data_file: DataFile;
+}
+interface FieldSummary {
+  contains_null: boolean;
+  contains_nan?: boolean | null;
+  lower_bound?: Uint8Array | null;
+  upper_bound?: Uint8Array | null;
+}
+interface DataFile {
+  content: 0 | 1 | 2; // 0=data, 1=position_delete, 2=equality_delete
+  file_path: string;
+  file_format: 'avro' | 'orc' | 'parquet' | 'puffin';
+  partition: Record<string, unknown>; // keyed by partition-field name in Avro
+  record_count: bigint;
+  file_size_in_bytes: bigint;
+  column_sizes?: Record<number, bigint>;
+  value_counts?: Record<number, bigint>;
+  null_value_counts?: Record<number, bigint>;
+  nan_value_counts?: Record<number, bigint>;
+  lower_bounds?: Record<number, unknown>;
+  upper_bounds?: Record<number, unknown>; // key_metadata?: string
+  split_offsets?: bigint[];
+  equality_ids?: number[];
+  sort_order_id?: number;
+  first_row_id?: bigint | number;
+  referenced_data_file?: string;
+  content_offset?: bigint;
+  content_size_in_bytes?: bigint;
+}
+/**
+ * Build a SigV4-signing `Resolver` for private S3-compatible buckets (AWS S3,
+ * Cloudflare R2, MinIO, etc.). Signs every request with AWS Signature V4
+ * using static credentials. Works in browsers and Node via the Web Crypto API.
+ *
+ * Credentials are taken as inputs; profile/STS/IMDS chains are out of scope.
+ * For Iceberg REST catalogs that vend per-table credentials (R2, AWS Glue),
+ * pass the `s3.*` keys from `restCatalogLoadCredentials` or `LoadTable.config`.
+ *
+ * Accepts paths as `s3://bucket/key`, `s3a://bucket/key`, or full https URLs.
+ *
+ * @param {object} options
+ * @param {string} options.accessKeyId
+ * @param {string} options.secretAccessKey
+ * @param {string} [options.sessionToken] - For temporary credentials (STS/vended).
+ * @param {string} options.region - e.g. 'us-east-1', or 'auto' for R2.
+ * @param {string} [options.endpoint] - HTTPS endpoint root for non-AWS providers
+ *   (e.g. `https://<account>.r2.cloudflarestorage.com`). Defaults to AWS S3.
+ * @param {boolean} [options.pathStyle] - When true, URLs are
+ *   `<endpoint>/<bucket>/<key>`; when false (default), virtual-hosted-style
+ *   `<bucket>.<endpoint-host>/<key>`. R2 requires true.
+ * @returns {Resolver}
+ */
+declare function s3SignedResolver({
+  accessKeyId,
+  secretAccessKey,
+  sessionToken,
+  region,
+  endpoint,
+  pathStyle
+}: {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string | undefined;
+  region: string;
+  endpoint?: string | undefined;
+  pathStyle?: boolean | undefined;
+}): Resolver;
+/**
+ * Returns manifest entries for a snapshot. Defaults to the current snapshot;
+ * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
+ * snapshot log.
+ *
+ * @import {Resolver, TableMetadata, Manifest, ManifestEntry} from '../src/types.js'
+ * @typedef {{ url: string, entries: ManifestEntry[] }[]} ManifestList
+ * @param {object} options
+ * @param {TableMetadata} options.metadata
+ * @param {Resolver} [options.resolver]
+ * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
+ * @returns {Promise<ManifestList>}
+ */
+declare function icebergManifests({
+  metadata,
+  resolver,
+  snapshotId,
+  partitionFilter
+}: {
+  metadata: TableMetadata;
+  resolver?: Resolver | undefined;
+  snapshotId?: number | bigint | undefined;
+  partitionFilter?: ManifestPartitionFilter | undefined;
+}): Promise<ManifestList>;
+/**
+ * Predicate applied to each manifest's manifest-list `partitions` field-summary
+ * array before entry fetch. Return `false` to skip the manifest's entry fetch.
+ */
+type ManifestPartitionFilter = (partitions: Manifest['partitions'], partitionSpecId: number, manifest: Manifest) => boolean;
+/**
+ * Returns manifest entries for a snapshot. Defaults to the current snapshot;
+ * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
+ * snapshot log.
+ */
+type ManifestList = {
+  url: string;
+  entries: ManifestEntry[];
+}[];
+/**
+ * Append rows to a table in one call: load metadata, stage the parquet writes
+ * + manifest + new snapshot, commit through the catalog.
+ *
+ * @param {object} options
+ * @param {Catalog} options.catalog
+ * @param {string | string[]} [options.namespace] - REST catalog only.
+ * @param {string} [options.table] - REST catalog only.
+ * @param {string} [options.tableUrl] - File catalog only.
+ * @param {Resolver} [options.resolver]
+ * @param {Record<string, any>[]} options.records
+ * @param {number} [options.sortOrderId] - Sort order id to apply; defaults to the table default.
+ * @returns {Promise<TableMetadata>}
+ */
+declare function icebergAppend({
+  catalog,
+  namespace,
+  table,
+  tableUrl,
+  resolver,
+  records,
+  sortOrderId,
+  snapshotProperties
+}: {
+  catalog: Catalog;
+  namespace?: string | string[] | undefined;
+  table?: string | undefined;
+  tableUrl?: string | undefined;
+  resolver?: Resolver | undefined;
+  records: Record<string, any>[];
+  sortOrderId?: number | undefined;
+  snapshotProperties?: Record<string, string> | undefined;
+}): Promise<TableMetadata>;
+/**
+ * Create a new table. REST: delegates to the catalog's create endpoint.
+ * File: writes the initial `v1.metadata.json` and `version-hint.text` under
+ * `tableUrl` via `catalog.resolver`.
+ *
+ * @param {object} options
+ * @param {Catalog} options.catalog
+ * @param {string | string[]} [options.namespace] - REST catalog only.
+ * @param {string} [options.table] - REST catalog only.
+ * @param {string} [options.tableUrl] - File catalog only; also passed as `location` for REST.
+ * @param {Schema} [options.schema]
+ * @param {PartitionSpec} [options.partitionSpec]
+ * @param {SortOrder} [options.sortOrder]
+ * @param {Record<string, string>} [options.properties]
+ * @param {2 | 3} [options.formatVersion] - File catalog only.
+ * @param {boolean} [options.stageCreate] - REST catalog only.
+ * @returns {Promise<TableMetadata>}
+ */
+declare function icebergCreateTable({
+  catalog,
+  namespace,
+  table,
+  tableUrl,
+  schema,
+  partitionSpec,
+  sortOrder,
+  properties,
+  formatVersion,
+  stageCreate
+}: {
+  catalog: Catalog;
+  namespace?: string | string[] | undefined;
+  table?: string | undefined;
+  tableUrl?: string | undefined;
+  schema?: Schema | undefined;
+  partitionSpec?: PartitionSpec | undefined;
+  sortOrder?: SortOrder | undefined;
+  properties?: Record<string, string> | undefined;
+  formatVersion?: 2 | 3 | undefined;
+  stageCreate?: boolean | undefined;
+}): Promise<TableMetadata>;
+/**
+ * Drop a table. REST: delegates to the catalog's drop endpoint, forwarding
+ * `purgeRequested`. File: lists `metadata/` (and `data/` when `purgeRequested`)
+ * via `lister` and deletes each file via `catalog.resolver.deleter`. File
+ * purges do not recurse into partition subdirectories.
+ *
+ * @param {object} options
+ * @param {Catalog} options.catalog
+ * @param {string | string[]} [options.namespace] - REST catalog only.
+ * @param {string} [options.table] - REST catalog only.
+ * @param {string} [options.tableUrl] - File catalog only.
+ * @param {Lister} [options.lister] - File catalog only; required to enumerate files to delete.
+ * @param {boolean} [options.purgeRequested] - REST: forwarded to the server. File: also delete `data/`.
+ * @returns {Promise<void>}
+ */
+declare function icebergDropTable({
+  catalog,
+  namespace,
+  table,
+  tableUrl,
+  lister,
+  purgeRequested
+}: {
+  catalog: Catalog;
+  namespace?: string | string[] | undefined;
+  table?: string | undefined;
+  tableUrl?: string | undefined;
+  lister?: Lister | undefined;
+  purgeRequested?: boolean | undefined;
+}): Promise<void>;
+/**
+ * Iceberg REST Catalog client.
+ *
+ * Plain async functions over a stateless context object — no classes.
+ * The catalog client never imports from the read path; callers glue the
+ * two together by passing `metadata` and `metadata.location` from
+ * `restCatalogLoadTable` into `icebergRead`.
+ *
+ * @import {LoadTableResponse, PartitionSpec, RestCatalogContext, Schema, SortOrder, StorageCredential, TableIdentifier, TableMetadata, TableRequirement, TableUpdate} from '../../src/types.js'
+ */
+/**
+ * Connect to a REST catalog by fetching `/v1/config`.
+ * Returns a frozen context object that holds the prefix, defaults, overrides
+ * and the user-supplied requestInit (for auth) for use in subsequent calls.
+ *
+ * @param {object} options
+ * @param {string} options.url - catalog base URL, with or without trailing slash
+ * @param {string} [options.warehouse] - optional warehouse query param sent to /v1/config
+ * @param {RequestInit} [options.requestInit] - fetch options (e.g. Authorization header)
+ * @returns {Promise<RestCatalogContext>}
+ */
+declare function restCatalogConnect({
+  url,
+  warehouse,
+  requestInit
+}: {
+  url: string;
+  warehouse?: string | undefined;
+  requestInit?: RequestInit | undefined;
+}): Promise<RestCatalogContext>;
+/**
+ * List tables within a namespace.
+ * Follows pagination (`next-page-token`) until exhausted.
+ *
+ * @param {RestCatalogContext} ctx
+ * @param {object} options
+ * @param {string | string[]} options.namespace
+ * @returns {Promise<TableIdentifier[]>}
+ */
+declare function restCatalogListTables(ctx: RestCatalogContext, {
+  namespace
+}: {
+  namespace: string | string[];
+}): Promise<TableIdentifier[]>;
+/**
+ * Load a single table. Returns the inline TableMetadata, the metadata
+ * file location, and any per-table config the server returned.
+ *
+ * The returned `metadata` and `metadata.location` can be passed directly
+ * into `icebergRead({ tableUrl: metadata.location, metadata })`.
+ *
+ * @param {RestCatalogContext} ctx
+ * @param {object} options
+ * @param {string | string[]} options.namespace
+ * @param {string} options.table
+ * @returns {Promise<LoadTableResponse>}
+ */
+declare function restCatalogLoadTable(ctx: RestCatalogContext, {
+  namespace,
+  table
+}: {
+  namespace: string | string[];
+  table: string;
+}): Promise<LoadTableResponse>;
+/**
+ * Create a namespace. Returns the namespace as the server stored it (which may
+ * include defaulted properties).
+ *
+ * @param {RestCatalogContext} ctx
+ * @param {object} options
+ * @param {string | string[]} options.namespace
+ * @param {Record<string, string>} [options.properties]
+ * @returns {Promise<{namespace: string[], properties: Record<string, string>}>}
+ */
+declare function restCatalogCreateNamespace(ctx: RestCatalogContext, {
+  namespace,
+  properties
+}: {
+  namespace: string | string[];
+  properties?: Record<string, string> | undefined;
+}): Promise<{
+  namespace: string[];
+  properties: Record<string, string>;
+}>;
+/**
+ * Wrap a `Resolver` so reads of the same path share one HTTP fetch and
+ * subsequent range reads share one in-memory buffer. Writes and deletes
+ * through the same resolver invalidate the cache entry for their path on
+ * success, so a commit pipeline (`icebergAppend` → reload metadata) sees the
+ * new bytes without manual invalidation.
+ *
+ * - `reader(path)` is memoized by path. The returned buffer is passed through
+ *   `cachedAsyncBuffer` so range reads within a single file are also deduped.
+ *   `byteLength` is a fetch hint and not part of the cache key — the bytes
+ *   are the same either way.
+ * - `writer(path).finish()` invalidates the cache entry only when the
+ *   underlying finish resolves; a rejected finish (e.g. an `If-None-Match: *`
+ *   collision returning 412/409) leaves the cached bytes intact, since the
+ *   server-side object hasn't changed.
+ * - `deleter(path)` invalidates the cache entry on success.
+ * - `writer` and `deleter` are omitted when the base resolver omits them, so
+ *   wrapping a read-only resolver still yields a read-only resolver.
+ *
+ * Iceberg writes are almost entirely create-new-path (data parquets, manifest
+ * avros, snapshot avros, vN.metadata.json all live at fresh paths per
+ * commit). Only `version-hint.text` and equivalent catalog-pointer files are
+ * truly mutated in place, so path-level invalidation is sufficient to keep
+ * single-process write-then-read pipelines consistent.
+ *
+ * Cross-process freshness is out of scope: a reader in process B does not
+ * see writes committed by process A through a different cache. Use a
+ * short-lived resolver, an external coordination layer, or wrap with TTL /
+ * conditional-GET revalidation if you need that.
+ *
+ * @param {Resolver} base
+ * @returns {Resolver}
+ */
+declare function cachingResolver(base: Resolver): Resolver;
+export { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };