@gscdump/engine 0.27.2 → 0.28.1

package/dist/_chunks/libs/icebird.d.mts

@@ -0,0 +1,441 @@
+import { AsyncBuffer } from "hyparquet";
+import { Writer } from "hyparquet-writer/src/types.js";
+interface WriterOptions {
+  /**
+   * If `'*'`, the writer must create the object only if it does not already
+   * exist (S3 conditional write: `If-None-Match: *`). On collision the
+   * underlying `finish()` rejects with an error whose `status` is 412 (or 409
+   * on some providers). The HTTP/S3 `urlResolver` translates this to the
+   * `If-None-Match` header. Other resolvers may honor or ignore it.
+   */
+  ifNoneMatch?: '*';
+}
+interface Resolver {
+  reader: (path: string, byteLength?: number) => AsyncBuffer | Promise<AsyncBuffer>;
+  writer?: (path: string, options?: WriterOptions) => Writer;
+  deleter?: (path: string) => Promise<void>;
+}
+type Lister = (path: string) => Promise<string[]>;
+/** Catalogs */
+interface RestCatalogContext {
+  type: 'rest';
+  url: string;
+  prefix: string;
+  defaults: Record<string, string>;
+  overrides: Record<string, string>;
+  requestInit?: RequestInit;
+}
+interface FileCatalog {
+  type: 'file';
+  resolver: Resolver;
+  lister?: Lister;
+  /**
+   * Opt in to S3-safe metadata commits: every `vN.metadata.json` (the
+   * initial create and every subsequent commit) is written with
+   * `If-None-Match: *` and `version-hint.text` is best-effort. High-level
+   * write functions retry on 412/409 by reloading the latest metadata and
+   * re-staging. `icebergCreateTable` and `icebergTransaction` do not retry.
+   * Default false preserves backwards-compatible (overwrite) behavior.
+   */
+  conditionalCommits?: boolean;
+}
+type Catalog = RestCatalogContext | FileCatalog;
+interface LoadTableResponse {
+  metadataLocation?: string;
+  metadata: TableMetadata;
+  config: Record<string, string>;
+}
+interface TableMetadata {
+  'format-version': number;
+  'table-uuid': string;
+  location: string;
+  'last-sequence-number': number; // missing in V1, required in V2+
+  'last-updated-ms': number;
+  'last-column-id': number;
+  'current-schema-id': number; // optional in V1, required in V2+
+  schemas: Schema[]; // optional in V1, required in V2+
+  'default-spec-id': number; // optional in V1, required in V2+
+  'partition-specs': PartitionSpec[]; // optional in V1, required in V2+
+  'last-partition-id': number; // optional in V1, required in V2+
+  properties?: Record<string, string>;
+  'current-snapshot-id'?: number | bigint;
+  snapshots?: Snapshot[];
+  'snapshot-log'?: SnapshotLog[];
+  'metadata-log'?: MetadataLog[];
+  'sort-orders': SortOrder[]; // optional in V1, required in V2+
+  'default-sort-order-id': number; // optional in V1, required in V2+
+  refs?: Record<string, SnapshotRef>;
+  statistics?: TableStatistics[];
+  'partition-statistics'?: PartitionStatistics[];
+  'next-row-id'?: number | bigint; // required in V3
+  'encryption-keys'?: EncryptionKey[]; // V3
+}
+interface Schema {
+  type: 'struct';
+  'schema-id': number;
+  'identifier-field-ids'?: number[];
+  fields: Field[];
+}
+interface Field {
+  id: number;
+  name: string;
+  required: boolean;
+  type: IcebergType;
+  doc?: string;
+  'initial-default'?: any;
+  'write-default'?: any;
+}
+type IcebergType = 'unknown' | 'boolean' | 'int' | 'long' | 'float' | 'double' | 'date' | 'time' | 'timestamp' | 'timestamptz' | 'timestamp_ns' | 'timestamptz_ns' | 'string' | 'uuid' | `fixed[${number}]` | 'binary' | `decimal(${number},${number})` | `decimal(${number}, ${number})` | 'variant' | 'geometry' | `geometry(${string})` | 'geography' | `geography(${string})` | IcebergNestedType;
+type IcebergNestedType = Schema | {
+  type: 'list';
+  'element-id': number;
+  'element-required': boolean;
+  element: IcebergType;
+} | {
+  type: 'map';
+  'key-id': number;
+  key: IcebergType;
+  'value-id': number;
+  'value-required': boolean;
+  value: IcebergType;
+};
+interface PartitionSpec {
+  'spec-id': number;
+  fields: PartitionField[];
+}
+interface PartitionField {
+  'source-id'?: number;
+  'source-ids'?: number[];
+  'field-id': number;
+  name: string;
+  transform: PartitionTransform;
+}
+type PartitionTransform = 'identity' | `bucket[${number}]` | `truncate[${number}]` | 'year' | 'month' | 'day' | 'hour' | 'void' | string;
+interface PartitionStatistics {
+  'snapshot-id': bigint;
+  'statistics-path': string;
+  'file-size-in-bytes': bigint;
+}
+interface SortOrder {
+  'order-id': number;
+  'fields': SortField[];
+}
+interface SortField {
+  transform: string;
+  'source-id'?: number;
+  'source-ids'?: number[]; // V3
+  'direction': 'asc' | 'desc';
+  'null-order': 'nulls-first' | 'nulls-last';
+}
+interface Snapshot {
+  'snapshot-id': number | bigint;
+  'parent-snapshot-id'?: number | bigint;
+  'sequence-number': number;
+  'timestamp-ms': number;
+  'manifest-list': string;
+  manifests?: Manifest$1[];
+  summary: {
+    // spec: "value of these fields should be of string type"
+    operation: string; // 'spark.app.id'?: string
+    'added-data-files'?: string;
+    'added-records'?: string;
+    'deleted-data-files'?: string;
+    'deleted-records'?: string;
+    'removed-files-size'?: string;
+    'added-delete-files'?: string;
+    'removed-delete-files'?: string;
+    'added-position-deletes'?: string;
+    'removed-position-deletes'?: string;
+    'added-equality-deletes'?: string;
+    'removed-equality-deletes'?: string;
+    'added-dvs'?: string;
+    'removed-dvs'?: string;
+    'added-files-size'?: string;
+    'changed-partition-count'?: string;
+    'total-records'?: string;
+    'total-files-size'?: string;
+    'total-data-files'?: string;
+    'total-delete-files'?: string;
+    'total-position-deletes'?: string;
+    'total-equality-deletes'?: string;
+  };
+  'schema-id'?: number;
+  'first-row-id'?: number; // V3
+  'added-rows'?: number; // V3
+  'key-id'?: string; // V3
+}
+interface TableStatistics {
+  'snapshot-id': number | bigint;
+  'statistics-path': string;
+  'file-size-in-bytes': bigint;
+  'file-footer-size-in-bytes': bigint;
+}
+interface SnapshotLog {
+  'timestamp-ms': number;
+  'snapshot-id': number | bigint;
+}
+interface SnapshotRef {
+  'snapshot-id': number | bigint;
+  type: 'branch' | 'tag';
+  'min-snapshots-to-keep'?: number;
+  'max-snapshot-age-ms'?: number;
+  'max-ref-age-ms'?: number;
+}
+interface EncryptionKey {
+  'key-id': string;
+  'key-metadata': string;
+}
+/**
+ * Iceberg REST `TableRequirement`s recognized by `checkRequirements`.
+ */
+interface MetadataLog {
+  'timestamp-ms': number;
+  'metadata-file': string;
+}
+interface Manifest$1 {
+  manifest_path: string;
+  manifest_length: bigint;
+  partition_spec_id: number;
+  content: 0 | 1; // 0=data, 1=deletes
+  sequence_number?: bigint;
+  min_sequence_number?: bigint;
+  added_snapshot_id: bigint;
+  added_files_count: number;
+  existing_files_count: number;
+  deleted_files_count: number;
+  added_rows_count: bigint;
+  existing_rows_count: bigint;
+  deleted_rows_count: bigint;
+  partitions?: FieldSummary[]; // key_metadata?: unknown
+  first_row_id?: bigint | number;
+}
+interface ManifestEntry {
+  status: 0 | 1 | 2; // 0=existing, 1=added, 2=deleted
+  snapshot_id?: bigint;
+  sequence_number?: bigint;
+  file_sequence_number?: bigint;
+  partition_spec_id?: number;
+  data_file: DataFile;
+}
+interface FieldSummary {
+  contains_null: boolean;
+  contains_nan?: boolean | null;
+  lower_bound?: Uint8Array | null;
+  upper_bound?: Uint8Array | null;
+}
+interface DataFile {
+  content: 0 | 1 | 2; // 0=data, 1=position_delete, 2=equality_delete
+  file_path: string;
+  file_format: 'avro' | 'orc' | 'parquet' | 'puffin';
+  partition: Record<string, unknown>; // keyed by partition-field name in Avro
+  record_count: bigint;
+  file_size_in_bytes: bigint;
+  column_sizes?: Record<number, bigint>;
+  value_counts?: Record<number, bigint>;
+  null_value_counts?: Record<number, bigint>;
+  nan_value_counts?: Record<number, bigint>;
+  lower_bounds?: Record<number, unknown>;
+  upper_bounds?: Record<number, unknown>; // key_metadata?: string
+  split_offsets?: bigint[];
+  equality_ids?: number[];
+  sort_order_id?: number;
+  first_row_id?: bigint | number;
+  referenced_data_file?: string;
+  content_offset?: bigint;
+  content_size_in_bytes?: bigint;
+}
+/**
+ * Returns manifest entries for a snapshot. Defaults to the current snapshot;
+ * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
+ * snapshot log.
+ *
+ * @import {Resolver, TableMetadata, Manifest, ManifestEntry} from '../src/types.js'
+ * @typedef {{ url: string, entries: ManifestEntry[] }[]} ManifestList
+ * @param {object} options
+ * @param {TableMetadata} options.metadata
+ * @param {Resolver} [options.resolver]
+ * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
+ * @returns {Promise<ManifestList>}
+ */
+declare function icebergManifests({
+  metadata,
+  resolver,
+  snapshotId,
+  partitionFilter
+}: {
+  metadata: TableMetadata;
+  resolver?: Resolver | undefined;
+  snapshotId?: number | bigint | undefined;
+  partitionFilter?: ManifestPartitionFilter | undefined;
+}): Promise<ManifestList>;
+/**
+ * Predicate applied to each manifest's manifest-list `partitions` field-summary
+ * array before entry fetch. Return `false` to skip the manifest's entry fetch.
+ */
+type ManifestPartitionFilter = (partitions: Manifest['partitions'], partitionSpecId: number, manifest: Manifest) => boolean;
+/**
+ * Returns manifest entries for a snapshot. Defaults to the current snapshot;
+ * pass `snapshotId` to time-travel to a prior snapshot in the metadata's
+ * snapshot log.
+ */
+type ManifestList = {
+  url: string;
+  entries: ManifestEntry[];
+}[];
+/**
+ * Append rows to a table in one call: load metadata, stage the parquet writes
+ * + manifest + new snapshot, commit through the catalog.
+ *
+ * @param {object} options
+ * @param {Catalog} options.catalog
+ * @param {string | string[]} [options.namespace] - REST catalog only.
+ * @param {string} [options.table] - REST catalog only.
+ * @param {string} [options.tableUrl] - File catalog only.
+ * @param {Resolver} [options.resolver]
+ * @param {Record<string, any>[]} options.records
+ * @param {number} [options.sortOrderId] - Sort order id to apply; defaults to the table default.
+ * @returns {Promise<TableMetadata>}
+ */
+declare function icebergAppend({
+  catalog,
+  namespace,
+  table,
+  tableUrl,
+  resolver,
+  records,
+  sortOrderId
+}: {
+  catalog: Catalog;
+  namespace?: string | string[] | undefined;
+  table?: string | undefined;
+  tableUrl?: string | undefined;
+  resolver?: Resolver | undefined;
+  records: Record<string, any>[];
+  sortOrderId?: number | undefined;
+}): Promise<TableMetadata>;
+/**
+ * Create a new table. REST: delegates to the catalog's create endpoint.
+ * File: writes the initial `v1.metadata.json` and `version-hint.text` under
+ * `tableUrl` via `catalog.resolver`.
+ *
+ * @param {object} options
+ * @param {Catalog} options.catalog
+ * @param {string | string[]} [options.namespace] - REST catalog only.
+ * @param {string} [options.table] - REST catalog only.
+ * @param {string} [options.tableUrl] - File catalog only; also passed as `location` for REST.
+ * @param {Schema} [options.schema]
+ * @param {PartitionSpec} [options.partitionSpec]
+ * @param {SortOrder} [options.sortOrder]
+ * @param {Record<string, string>} [options.properties]
+ * @param {2 | 3} [options.formatVersion] - File catalog only.
+ * @param {boolean} [options.stageCreate] - REST catalog only.
+ * @returns {Promise<TableMetadata>}
+ */
+declare function icebergCreateTable({
+  catalog,
+  namespace,
+  table,
+  tableUrl,
+  schema,
+  partitionSpec,
+  sortOrder,
+  properties,
+  formatVersion,
+  stageCreate
+}: {
+  catalog: Catalog;
+  namespace?: string | string[] | undefined;
+  table?: string | undefined;
+  tableUrl?: string | undefined;
+  schema?: Schema | undefined;
+  partitionSpec?: PartitionSpec | undefined;
+  sortOrder?: SortOrder | undefined;
+  properties?: Record<string, string> | undefined;
+  formatVersion?: 2 | 3 | undefined;
+  stageCreate?: boolean | undefined;
+}): Promise<TableMetadata>;
+/**
+ * Iceberg REST Catalog client.
+ *
+ * Plain async functions over a stateless context object — no classes.
+ * The catalog client never imports from the read path; callers glue the
+ * two together by passing `metadata` and `metadata.location` from
+ * `restCatalogLoadTable` into `icebergRead`.
+ *
+ * @import {LoadTableResponse, PartitionSpec, RestCatalogContext, Schema, SortOrder, StorageCredential, TableIdentifier, TableMetadata, TableRequirement, TableUpdate} from '../../src/types.js'
+ */
+/**
+ * Connect to a REST catalog by fetching `/v1/config`.
+ * Returns a frozen context object that holds the prefix, defaults, overrides
+ * and the user-supplied requestInit (for auth) for use in subsequent calls.
+ *
+ * @param {object} options
+ * @param {string} options.url - catalog base URL, with or without trailing slash
+ * @param {string} [options.warehouse] - optional warehouse query param sent to /v1/config
+ * @param {RequestInit} [options.requestInit] - fetch options (e.g. Authorization header)
+ * @returns {Promise<RestCatalogContext>}
+ */
+declare function restCatalogConnect({
+  url,
+  warehouse,
+  requestInit
+}: {
+  url: string;
+  warehouse?: string | undefined;
+  requestInit?: RequestInit | undefined;
+}): Promise<RestCatalogContext>;
+/**
+ * Load a single table. Returns the inline TableMetadata, the metadata
+ * file location, and any per-table config the server returned.
+ *
+ * The returned `metadata` and `metadata.location` can be passed directly
+ * into `icebergRead({ tableUrl: metadata.location, metadata })`.
+ *
+ * @param {RestCatalogContext} ctx
+ * @param {object} options
+ * @param {string | string[]} options.namespace
+ * @param {string} options.table
+ * @returns {Promise<LoadTableResponse>}
+ */
+declare function restCatalogLoadTable(ctx: RestCatalogContext, {
+  namespace,
+  table
+}: {
+  namespace: string | string[];
+  table: string;
+}): Promise<LoadTableResponse>;
+/**
+ * Wrap a `Resolver` so reads of the same path share one HTTP fetch and
+ * subsequent range reads share one in-memory buffer. Writes and deletes
+ * through the same resolver invalidate the cache entry for their path on
+ * success, so a commit pipeline (`icebergAppend` → reload metadata) sees the
+ * new bytes without manual invalidation.
+ *
+ * - `reader(path)` is memoized by path. The returned buffer is passed through
+ *   `cachedAsyncBuffer` so range reads within a single file are also deduped.
+ *   `byteLength` is a fetch hint and not part of the cache key — the bytes
+ *   are the same either way.
+ * - `writer(path).finish()` invalidates the cache entry only when the
+ *   underlying finish resolves; a rejected finish (e.g. an `If-None-Match: *`
+ *   collision returning 412/409) leaves the cached bytes intact, since the
+ *   server-side object hasn't changed.
+ * - `deleter(path)` invalidates the cache entry on success.
+ * - `writer` and `deleter` are omitted when the base resolver omits them, so
+ *   wrapping a read-only resolver still yields a read-only resolver.
+ *
+ * Iceberg writes are almost entirely create-new-path (data parquets, manifest
+ * avros, snapshot avros, vN.metadata.json all live at fresh paths per
+ * commit). Only `version-hint.text` and equivalent catalog-pointer files are
+ * truly mutated in place, so path-level invalidation is sufficient to keep
+ * single-process write-then-read pipelines consistent.
+ *
+ * Cross-process freshness is out of scope: a reader in process B does not
+ * see writes committed by process A through a different cache. Use a
+ * short-lived resolver, an external coordination layer, or wrap with TTL /
+ * conditional-GET revalidation if you need that.
+ *
+ * @param {Resolver} base
+ * @returns {Resolver}
+ */
+declare function cachingResolver(base: Resolver): Resolver;
+export { cachingResolver, icebergAppend, icebergCreateTable, icebergManifests, restCatalogConnect, restCatalogLoadTable };