@gscdump/lakehouse 1.3.2 → 1.4.1

package/dist/_chunks/libs/icebird.d.mts

@@ -1,500 +0,0 @@
-import { AsyncBuffer } from "./hyparquet.mjs";
-import { Writer } from "./hyparquet-writer.mjs";
-interface WriterOptions {
-  /**
-   * If `'*'`, the writer must create the object only if it does not already
-   * exist (S3 conditional write: `If-None-Match: *`). On collision the
-   * underlying `finish()` rejects with an error whose `status` is 412 (or 409
-   * on some providers). The HTTP/S3 `urlResolver` translates this to the
-   * `If-None-Match` header. Other resolvers may honor or ignore it.
-   */
-  ifNoneMatch?: '*';
-}
-interface Resolver {
-  reader: (path: string, byteLength?: number) => AsyncBuffer | Promise<AsyncBuffer>;
-  writer?: (path: string, options?: WriterOptions) => Writer;
-  deleter?: (path: string) => Promise<void>;
-}
-type Lister = (path: string) => Promise<string[]>;
-/** Catalogs */
-interface RestCatalogContext {
-  type: 'rest';
-  url: string;
-  prefix: string;
-  defaults: Record<string, string>;
-  overrides: Record<string, string>;
-  requestInit?: RequestInit;
-  /** Optional per-request auth hook (e.g. SigV4 signing). Called after merging requestInit. */
-  signRequest?: (url: string, init?: RequestInit) => Promise<RequestInit>;
-}
-interface FileCatalog {
-  type: 'file';
-  resolver: Resolver;
-  lister?: Lister;
-  /**
-   * Opt in to S3-safe metadata commits: every `vN.metadata.json` (the
-   * initial create and every subsequent commit) is written with
-   * `If-None-Match: *` and `version-hint.text` is best-effort. High-level
-   * write functions retry on 412/409 by reloading the latest metadata and
-   * re-staging. `icebergCreateTable` and `icebergTransaction` do not retry.
-   * Default false preserves backwards-compatible (overwrite) behavior.
-   */
-  conditionalCommits?: boolean;
-}
-type Catalog = RestCatalogContext | FileCatalog;
-interface TableIdentifier {
-  namespace: string[];
-  name: string;
-}
-interface LoadTableResponse {
-  metadataLocation?: string;
-  metadata: TableMetadata;
-  config: Record<string, string>;
-}
-interface TableMetadata {
-  'format-version': number;
-  'table-uuid': string;
-  location: string;
-  'last-sequence-number': number; // missing in V1, required in V2+
-  'last-updated-ms': number;
-  'last-column-id': number;
-  'current-schema-id': number; // optional in V1, required in V2+
-  schemas: Schema[]; // optional in V1, required in V2+
-  'default-spec-id': number; // optional in V1, required in V2+
-  'partition-specs': PartitionSpec[]; // optional in V1, required in V2+
-  'last-partition-id': number; // optional in V1, required in V2+
-  properties?: Record<string, string>;
-  'current-snapshot-id'?: number | bigint;
-  snapshots?: Snapshot[];
-  'snapshot-log'?: SnapshotLog[];
-  'metadata-log'?: MetadataLog[];
-  'sort-orders': SortOrder[]; // optional in V1, required in V2+
-  'default-sort-order-id': number; // optional in V1, required in V2+
-  refs?: Record<string, SnapshotRef>;
-  statistics?: TableStatistics[];
-  'partition-statistics'?: PartitionStatistics[];
-  'next-row-id'?: number | bigint; // required in V3
-  'encryption-keys'?: EncryptionKey[]; // V3
-}
-interface Schema {
-  type: 'struct';
-  'schema-id': number;
-  'identifier-field-ids'?: number[];
-  fields: Field[];
-}
-interface Field {
-  id: number;
-  name: string;
-  required: boolean;
-  type: IcebergType;
-  doc?: string;
-  'initial-default'?: any;
-  'write-default'?: any;
-}
-type IcebergType = 'unknown' | 'boolean' | 'int' | 'long' | 'float' | 'double' | 'date' | 'time' | 'timestamp' | 'timestamptz' | 'timestamp_ns' | 'timestamptz_ns' | 'string' | 'uuid' | `fixed[${number}]` | 'binary' | `decimal(${number},${number})` | `decimal(${number}, ${number})` | 'variant' | 'geometry' | `geometry(${string})` | 'geography' | `geography(${string})` | IcebergNestedType;
-type IcebergNestedType = Schema | {
-  type: 'list';
-  'element-id': number;
-  'element-required': boolean;
-  element: IcebergType;
-} | {
-  type: 'map';
-  'key-id': number;
-  key: IcebergType;
-  'value-id': number;
-  'value-required': boolean;
-  value: IcebergType;
-};
-interface PartitionSpec {
-  'spec-id': number;
-  fields: PartitionField[];
-}
-interface PartitionField {
-  'source-id'?: number;
-  'source-ids'?: number[];
-  'field-id': number;
-  name: string;
-  transform: PartitionTransform;
-}
-type PartitionTransform = 'identity' | `bucket[${number}]` | `truncate[${number}]` | 'year' | 'month' | 'day' | 'hour' | 'void' | string;
-interface PartitionStatistics {
-  'snapshot-id': bigint;
-  'statistics-path': string;
-  'file-size-in-bytes': bigint;
-}
-interface SortOrder {
-  'order-id': number;
-  'fields': SortField[];
-}
-interface SortField {
-  transform: string;
-  'source-id'?: number;
-  'source-ids'?: number[]; // V3
-  'direction': 'asc' | 'desc';
-  'null-order': 'nulls-first' | 'nulls-last';
-}
-interface Snapshot {
-  'snapshot-id': number | bigint;
-  'parent-snapshot-id'?: number | bigint;
-  'sequence-number': number;
-  'timestamp-ms': number;
-  'manifest-list': string;
-  manifests?: Manifest[];
-  summary: {
-    // spec: "value of these fields should be of string type"
-    operation: string;
-    // 'spark.app.id'?: string
-    'added-data-files'?: string;
-    'added-records'?: string;
-    'deleted-data-files'?: string;
-    'deleted-records'?: string;
-    'removed-files-size'?: string;
-    'added-delete-files'?: string;
-    'removed-delete-files'?: string;
-    'added-position-deletes'?: string;
-    'removed-position-deletes'?: string;
-    'added-equality-deletes'?: string;
-    'removed-equality-deletes'?: string;
-    'added-dvs'?: string;
-    'removed-dvs'?: string;
-    'added-files-size'?: string;
-    'changed-partition-count'?: string;
-    'total-records'?: string;
-    'total-files-size'?: string;
-    'total-data-files'?: string;
-    'total-delete-files'?: string;
-    'total-position-deletes'?: string;
-    'total-equality-deletes'?: string;
-  };
-  'schema-id'?: number;
-  'first-row-id'?: number; // V3
-  'added-rows'?: number; // V3
-  'key-id'?: string; // V3
-}
-interface TableStatistics {
-  'snapshot-id': number | bigint;
-  'statistics-path': string;
-  'file-size-in-bytes': bigint;
-  'file-footer-size-in-bytes': bigint;
-}
-interface SnapshotLog {
-  'timestamp-ms': number;
-  'snapshot-id': number | bigint;
-}
-interface SnapshotRef {
-  'snapshot-id': number | bigint;
-  type: 'branch' | 'tag';
-  'min-snapshots-to-keep'?: number;
-  'max-snapshot-age-ms'?: number;
-  'max-ref-age-ms'?: number;
-}
-interface EncryptionKey {
-  'key-id': string;
-  'key-metadata': string;
-}
-interface MetadataLog {
-  'timestamp-ms': number;
-  'metadata-file': string;
-}
-interface Manifest {
-  manifest_path: string;
-  manifest_length: bigint;
-  partition_spec_id: number;
-  content: 0 | 1; // 0=data, 1=deletes
-  sequence_number?: bigint;
-  min_sequence_number?: bigint;
-  added_snapshot_id: bigint;
-  added_files_count: number;
-  existing_files_count: number;
-  deleted_files_count: number;
-  added_rows_count: bigint;
-  existing_rows_count: bigint;
-  deleted_rows_count: bigint;
-  partitions?: FieldSummary[];
-  // key_metadata?: unknown
-  first_row_id?: bigint | number;
-}
-interface ManifestEntry {
-  status: 0 | 1 | 2; // 0=existing, 1=added, 2=deleted
-  snapshot_id?: bigint;
-  sequence_number?: bigint;
-  file_sequence_number?: bigint;
-  partition_spec_id?: number;
-  data_file: DataFile;
-}
-interface FieldSummary {
-  contains_null: boolean;
-  contains_nan?: boolean | null;
-  lower_bound?: Uint8Array | null;
-  upper_bound?: Uint8Array | null;
-}
-interface DataFile {
-  content: 0 | 1 | 2; // 0=data, 1=position_delete, 2=equality_delete
-  file_path: string;
-  file_format: 'avro' | 'orc' | 'parquet' | 'puffin';
-  partition: Record<string, unknown>; // keyed by partition-field name in Avro
-  record_count: bigint;
-  file_size_in_bytes: bigint;
-  column_sizes?: Record<number, bigint>;
-  value_counts?: Record<number, bigint>;
-  null_value_counts?: Record<number, bigint>;
-  nan_value_counts?: Record<number, bigint>;
-  lower_bounds?: Record<number, unknown>;
-  upper_bounds?: Record<number, unknown>;
-  // key_metadata?: string
-  split_offsets?: bigint[];
-  equality_ids?: number[];
-  sort_order_id?: number;
-  first_row_id?: bigint | number;
-  referenced_data_file?: string;
-  content_offset?: bigint;
-  content_size_in_bytes?: bigint;
-}
-/**
- * Append rows to a table in one call: load metadata, stage the parquet writes
- * + manifest + new snapshot, commit through the catalog.
- *
- * @param {object} options
- * @param {Catalog} options.catalog
- * @param {string | string[]} [options.namespace] - REST catalog only.
- * @param {string} [options.table] - REST catalog only.
- * @param {string} [options.tableUrl] - File catalog only.
- * @param {Resolver} [options.resolver]
- * @param {Record<string, any>[]} options.records
- * @param {number} [options.sortOrderId] - Sort order id to apply; defaults to the table default.
- * @returns {Promise<TableMetadata>}
- */
-declare function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties }: {
-  catalog: Catalog;
-  namespace?: string | string[];
-  table?: string;
-  tableUrl?: string;
-  resolver?: Resolver;
-  records: Record<string, any>[];
-  sortOrderId?: number;
-  snapshotProperties?: Record<string, string>;
-}): Promise<TableMetadata>;
-/**
- * Create a new table. REST: delegates to the catalog's create endpoint.
- * File: writes the initial `v1.metadata.json` and `version-hint.text` under
- * `tableUrl` via `catalog.resolver`.
- *
- * @param {object} options
- * @param {Catalog} options.catalog
- * @param {string | string[]} [options.namespace] - REST catalog only.
- * @param {string} [options.table] - REST catalog only.
- * @param {string} [options.tableUrl] - File catalog only; also passed as `location` for REST.
- * @param {Schema} [options.schema]
- * @param {PartitionSpec} [options.partitionSpec]
- * @param {SortOrder} [options.sortOrder]
- * @param {Record<string, string>} [options.properties]
- * @param {2 | 3} [options.formatVersion] - File catalog only.
- * @param {boolean} [options.stageCreate] - REST catalog only.
- * @returns {Promise<TableMetadata>}
- */
-declare function icebergCreateTable({ catalog, namespace, table, tableUrl, schema, partitionSpec, sortOrder, properties, formatVersion, stageCreate }: {
-  catalog: Catalog;
-  namespace?: string | string[];
-  table?: string;
-  tableUrl?: string;
-  schema?: Schema;
-  partitionSpec?: PartitionSpec;
-  sortOrder?: SortOrder;
-  properties?: Record<string, string>;
-  formatVersion?: 2 | 3;
-  stageCreate?: boolean;
-}): Promise<TableMetadata>;
-/**
- * Drop a table. REST: delegates to the catalog's drop endpoint, forwarding
- * `purgeRequested`. File: lists `metadata/` (and `data/` when `purgeRequested`)
- * via `lister` and deletes each file via `catalog.resolver.deleter`. File
- * purges do not recurse into partition subdirectories.
- *
- * @param {object} options
- * @param {Catalog} options.catalog
- * @param {string | string[]} [options.namespace] - REST catalog only.
- * @param {string} [options.table] - REST catalog only.
- * @param {string} [options.tableUrl] - File catalog only.
- * @param {Lister} [options.lister] - File catalog only; required to enumerate files to delete.
- * @param {boolean} [options.purgeRequested] - REST: forwarded to the server. File: also delete `data/`.
- * @returns {Promise<void>}
- */
-declare function icebergDropTable({ catalog, namespace, table, tableUrl, lister, purgeRequested }: {
-  catalog: Catalog;
-  namespace?: string | string[];
-  table?: string;
-  tableUrl?: string;
-  lister?: Lister;
-  purgeRequested?: boolean;
-}): Promise<void>;
-/**
- * Iceberg REST Catalog client.
- *
- * Plain async functions over a stateless context object — no classes.
- * The catalog client never imports from the read path; callers glue the
- * two together by passing `metadata` and `metadata.location` from
- * `restCatalogLoadTable` into `icebergRead`.
- *
- * @import {LoadTableResponse, PartitionSpec, RestCatalogContext, Schema, SortOrder, StorageCredential, TableIdentifier, TableMetadata, TableRequirement, TableUpdate} from '../../src/types.js'
- */
-/**
- * Connect to a REST catalog by fetching `/v1/config`.
- * Returns a frozen context object that holds the prefix, defaults, overrides
- * and the user-supplied requestInit (for auth) for use in subsequent calls.
- *
- * @param {object} options
- * @param {string} options.url - catalog base URL, with or without trailing slash
- * @param {string} [options.warehouse] - optional warehouse query param sent to /v1/config
- * @param {RequestInit} [options.requestInit] - fetch options (e.g. Authorization header)
- * @param {(url: string, init?: RequestInit) => Promise<RequestInit>} [options.signRequest] - per-request auth hook
- * @returns {Promise<RestCatalogContext>}
- */
-declare function restCatalogConnect({ url, warehouse, requestInit, signRequest }: {
-  url: string;
-  warehouse?: string;
-  requestInit?: RequestInit;
-  signRequest?: (url: string, init?: RequestInit) => Promise<RequestInit>;
-}): Promise<RestCatalogContext>;
-/**
- * List tables within a namespace.
- * Follows pagination (`next-page-token`) until exhausted.
- *
- * @param {RestCatalogContext} ctx
- * @param {object} options
- * @param {string | string[]} options.namespace
- * @returns {Promise<TableIdentifier[]>}
- */
-declare function restCatalogListTables(ctx: RestCatalogContext, { namespace }: {
-  namespace: string | string[];
-}): Promise<TableIdentifier[]>;
-/**
- * Load a single table. Returns the inline TableMetadata, the metadata
- * file location, and any per-table config the server returned.
- *
- * The returned `metadata` and `metadata.location` can be passed directly
- * into `icebergRead({ tableUrl: metadata.location, metadata })`.
- *
- * @param {RestCatalogContext} ctx
- * @param {object} options
- * @param {string | string[]} options.namespace
- * @param {string} options.table
- * @returns {Promise<LoadTableResponse>}
- */
-declare function restCatalogLoadTable(ctx: RestCatalogContext, { namespace, table }: {
-  namespace: string | string[];
-  table: string;
-}): Promise<LoadTableResponse>;
-/**
- * Create a namespace. Returns the namespace as the server stored it (which may
- * include defaulted properties).
- *
- * @param {RestCatalogContext} ctx
- * @param {object} options
- * @param {string | string[]} options.namespace
- * @param {Record<string, string>} [options.properties]
- * @returns {Promise<{namespace: string[], properties: Record<string, string>}>}
- */
-declare function restCatalogCreateNamespace(ctx: RestCatalogContext, { namespace, properties }: {
-  namespace: string | string[];
-  properties?: Record<string, string>;
-}): Promise<{
-  namespace: string[];
-  properties: Record<string, string>;
-}>;
-/**
- * Wrap a `Resolver` so reads of the same path share one HTTP fetch and
- * subsequent range reads share one in-memory buffer. Writes and deletes
- * through the same resolver invalidate the cache entry for their path on
- * success, so a commit pipeline (`icebergAppend` → reload metadata) sees the
- * new bytes without manual invalidation.
- *
- * - `reader(path)` is memoized by path. The returned buffer is passed through
- *   `cachedAsyncBuffer` so range reads within a single file are also deduped.
- *   `byteLength` is a fetch hint and not part of the cache key — the bytes
- *   are the same either way.
- * - `writer(path).finish()` invalidates the cache entry only when the
- *   underlying finish resolves; a rejected finish (e.g. an `If-None-Match: *`
- *   collision returning 412/409) leaves the cached bytes intact, since the
- *   server-side object hasn't changed.
- * - `deleter(path)` invalidates the cache entry on success.
- * - `writer` and `deleter` are omitted when the base resolver omits them, so
- *   wrapping a read-only resolver still yields a read-only resolver.
- *
- * Iceberg writes are almost entirely create-new-path (data parquets, manifest
- * avros, snapshot avros, vN.metadata.json all live at fresh paths per
- * commit). Only `version-hint.text` and equivalent catalog-pointer files are
- * truly mutated in place, so path-level invalidation is sufficient to keep
- * single-process write-then-read pipelines consistent.
- *
- * Cross-process freshness is out of scope: a reader in process B does not
- * see writes committed by process A through a different cache. Use a
- * short-lived resolver, an external coordination layer, or wrap with TTL /
- * conditional-GET revalidation if you need that.
- *
- * @param {Resolver} base
- * @returns {Resolver}
- */
-declare function cachingResolver(base: Resolver): Resolver;
-/**
- * @import {Resolver} from '../src/types.js'
- */
-/**
- * Build a SigV4-signing `Resolver` for private S3-compatible buckets (AWS S3,
- * Cloudflare R2, MinIO, etc.). Signs every request with AWS Signature V4
- * using static credentials. Works in browsers and Node via the Web Crypto API.
- *
- * Credentials are taken as inputs; profile/STS/IMDS chains are out of scope.
- * For Iceberg REST catalogs that vend per-table credentials (R2, AWS Glue),
- * pass the `s3.*` keys from `restCatalogLoadCredentials` or `LoadTable.config`.
- *
- * Accepts paths as `s3://bucket/key`, `s3a://bucket/key`, or full https URLs.
- *
- * @param {object} options
- * @param {string} options.accessKeyId
- * @param {string} options.secretAccessKey
- * @param {string} [options.sessionToken] - For temporary credentials (STS/vended).
- * @param {string} options.region - e.g. 'us-east-1', or 'auto' for R2.
- * @param {string} [options.endpoint] - HTTPS endpoint root for non-AWS providers
- *   (e.g. `https://<account>.r2.cloudflarestorage.com`). Defaults to AWS S3.
- * @param {boolean} [options.pathStyle] - When true, URLs are
- *   `<endpoint>/<bucket>/<key>`; when false (default), virtual-hosted-style
- *   `<bucket>.<endpoint-host>/<key>`. R2 requires true.
- * @returns {Resolver}
- */
-declare function s3SignedResolver({ accessKeyId, secretAccessKey, sessionToken, region, endpoint, pathStyle }: {
-  accessKeyId: string;
-  secretAccessKey: string;
-  sessionToken?: string;
-  region: string;
-  endpoint?: string;
-  pathStyle?: boolean;
-}): Resolver;
-type ManifestList = {
-  url: string;
-  entries: ManifestEntry[];
-}[];
-/**
- * Predicate applied to each manifest's manifest-list `partitions` field-summary
- * array before entry fetch. Return `false` to skip the manifest's entry fetch.
- */
-type ManifestPartitionFilter = (partitions: Manifest['partitions'], partitionSpecId: number, manifest: Manifest) => boolean;
-/**
- * Returns manifest entries for a snapshot. Defaults to the current snapshot;
- * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
- * snapshot log.
- *
- * @import {Resolver, TableMetadata, Manifest, ManifestEntry} from '../src/types.js'
- * @typedef {{ url: string, entries: ManifestEntry[] }[]} ManifestList
- * @param {object} options
- * @param {TableMetadata} options.metadata
- * @param {Resolver} [options.resolver]
- * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
- * @returns {Promise<ManifestList>}
- */
-declare function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }: {
-  metadata: TableMetadata;
-  resolver?: Resolver;
-  snapshotId?: number | bigint;
-  partitionFilter?: ManifestPartitionFilter;
-}): Promise<ManifestList>;
-export { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };