@gscdump/engine 0.33.10 → 0.35.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/THIRD-PARTY-LICENSES.md +1 -394
- package/dist/_chunks/libs/hyparquet-writer.mjs +1 -112
- package/dist/_chunks/libs/hyparquet.mjs +1 -35
- package/dist/_chunks/libs/icebird.d.mts +1 -98
- package/dist/_chunks/libs/icebird.mjs +51 -2894
- package/dist/_chunks/resolver.mjs +2 -2
- package/dist/_chunks/schema2.mjs +44 -26
- package/dist/_chunks/sink.d.mts +62 -359
- package/dist/adapters/duckdb-node.mjs +3 -3
- package/dist/iceberg/index.mjs +35 -335
- package/dist/sink-node.d.mts +2 -24
- package/dist/sink-node.mjs +2 -85
- package/package.json +4 -3
- package/dist/_chunks/libs/chokidar.d.mts +0 -1
- package/dist/_chunks/libs/db0.d.mts +0 -1
- package/dist/_chunks/libs/denque.d.mts +0 -1
- package/dist/_chunks/libs/ioredis.d.mts +0 -1
- package/dist/_chunks/libs/lru-cache.d.mts +0 -1
- package/dist/_chunks/libs/unstorage.d.mts +0 -120
|
@@ -282,39 +282,6 @@ type ManifestList = {
|
|
|
282
282
|
url: string;
|
|
283
283
|
entries: ManifestEntry[];
|
|
284
284
|
}[];
|
|
285
|
-
/**
|
|
286
|
-
* Append rows to a table in one call: load metadata, stage the parquet writes
|
|
287
|
-
* + manifest + new snapshot, commit through the catalog.
|
|
288
|
-
*
|
|
289
|
-
* @param {object} options
|
|
290
|
-
* @param {Catalog} options.catalog
|
|
291
|
-
* @param {string | string[]} [options.namespace] - REST catalog only.
|
|
292
|
-
* @param {string} [options.table] - REST catalog only.
|
|
293
|
-
* @param {string} [options.tableUrl] - File catalog only.
|
|
294
|
-
* @param {Resolver} [options.resolver]
|
|
295
|
-
* @param {Record<string, any>[]} options.records
|
|
296
|
-
* @param {number} [options.sortOrderId] - Sort order id to apply; defaults to the table default.
|
|
297
|
-
* @returns {Promise<TableMetadata>}
|
|
298
|
-
*/
|
|
299
|
-
declare function icebergAppend({
|
|
300
|
-
catalog,
|
|
301
|
-
namespace,
|
|
302
|
-
table,
|
|
303
|
-
tableUrl,
|
|
304
|
-
resolver,
|
|
305
|
-
records,
|
|
306
|
-
sortOrderId,
|
|
307
|
-
snapshotProperties
|
|
308
|
-
}: {
|
|
309
|
-
catalog: Catalog;
|
|
310
|
-
namespace?: string | string[] | undefined;
|
|
311
|
-
table?: string | undefined;
|
|
312
|
-
tableUrl?: string | undefined;
|
|
313
|
-
resolver?: Resolver | undefined;
|
|
314
|
-
records: Record<string, any>[];
|
|
315
|
-
sortOrderId?: number | undefined;
|
|
316
|
-
snapshotProperties?: Record<string, string> | undefined;
|
|
317
|
-
}): Promise<TableMetadata>;
|
|
318
285
|
/**
|
|
319
286
|
* Create a new table. REST: delegates to the catalog's create endpoint.
|
|
320
287
|
* File: writes the initial `v1.metadata.json` and `version-hint.text` under
|
|
@@ -356,36 +323,6 @@ declare function icebergCreateTable({
|
|
|
356
323
|
formatVersion?: 2 | 3 | undefined;
|
|
357
324
|
stageCreate?: boolean | undefined;
|
|
358
325
|
}): Promise<TableMetadata>;
|
|
359
|
-
/**
|
|
360
|
-
* Iceberg REST Catalog client.
|
|
361
|
-
*
|
|
362
|
-
* Plain async functions over a stateless context object — no classes.
|
|
363
|
-
* The catalog client never imports from the read path; callers glue the
|
|
364
|
-
* two together by passing `metadata` and `metadata.location` from
|
|
365
|
-
* `restCatalogLoadTable` into `icebergRead`.
|
|
366
|
-
*
|
|
367
|
-
* @import {LoadTableResponse, PartitionSpec, RestCatalogContext, Schema, SortOrder, StorageCredential, TableIdentifier, TableMetadata, TableRequirement, TableUpdate} from '../../src/types.js'
|
|
368
|
-
*/
|
|
369
|
-
/**
|
|
370
|
-
* Connect to a REST catalog by fetching `/v1/config`.
|
|
371
|
-
* Returns a frozen context object that holds the prefix, defaults, overrides
|
|
372
|
-
* and the user-supplied requestInit (for auth) for use in subsequent calls.
|
|
373
|
-
*
|
|
374
|
-
* @param {object} options
|
|
375
|
-
* @param {string} options.url - catalog base URL, with or without trailing slash
|
|
376
|
-
* @param {string} [options.warehouse] - optional warehouse query param sent to /v1/config
|
|
377
|
-
* @param {RequestInit} [options.requestInit] - fetch options (e.g. Authorization header)
|
|
378
|
-
* @returns {Promise<RestCatalogContext>}
|
|
379
|
-
*/
|
|
380
|
-
declare function restCatalogConnect({
|
|
381
|
-
url,
|
|
382
|
-
warehouse,
|
|
383
|
-
requestInit
|
|
384
|
-
}: {
|
|
385
|
-
url: string;
|
|
386
|
-
warehouse?: string | undefined;
|
|
387
|
-
requestInit?: RequestInit | undefined;
|
|
388
|
-
}): Promise<RestCatalogContext>;
|
|
389
326
|
/**
|
|
390
327
|
* Load a single table. Returns the inline TableMetadata, the metadata
|
|
391
328
|
* file location, and any per-table config the server returned.
|
|
@@ -406,38 +343,4 @@ declare function restCatalogLoadTable(ctx: RestCatalogContext, {
|
|
|
406
343
|
namespace: string | string[];
|
|
407
344
|
table: string;
|
|
408
345
|
}): Promise<LoadTableResponse>;
|
|
409
|
-
|
|
410
|
-
* Wrap a `Resolver` so reads of the same path share one HTTP fetch and
|
|
411
|
-
* subsequent range reads share one in-memory buffer. Writes and deletes
|
|
412
|
-
* through the same resolver invalidate the cache entry for their path on
|
|
413
|
-
* success, so a commit pipeline (`icebergAppend` → reload metadata) sees the
|
|
414
|
-
* new bytes without manual invalidation.
|
|
415
|
-
*
|
|
416
|
-
* - `reader(path)` is memoized by path. The returned buffer is passed through
|
|
417
|
-
* `cachedAsyncBuffer` so range reads within a single file are also deduped.
|
|
418
|
-
* `byteLength` is a fetch hint and not part of the cache key — the bytes
|
|
419
|
-
* are the same either way.
|
|
420
|
-
* - `writer(path).finish()` invalidates the cache entry only when the
|
|
421
|
-
* underlying finish resolves; a rejected finish (e.g. an `If-None-Match: *`
|
|
422
|
-
* collision returning 412/409) leaves the cached bytes intact, since the
|
|
423
|
-
* server-side object hasn't changed.
|
|
424
|
-
* - `deleter(path)` invalidates the cache entry on success.
|
|
425
|
-
* - `writer` and `deleter` are omitted when the base resolver omits them, so
|
|
426
|
-
* wrapping a read-only resolver still yields a read-only resolver.
|
|
427
|
-
*
|
|
428
|
-
* Iceberg writes are almost entirely create-new-path (data parquets, manifest
|
|
429
|
-
* avros, snapshot avros, vN.metadata.json all live at fresh paths per
|
|
430
|
-
* commit). Only `version-hint.text` and equivalent catalog-pointer files are
|
|
431
|
-
* truly mutated in place, so path-level invalidation is sufficient to keep
|
|
432
|
-
* single-process write-then-read pipelines consistent.
|
|
433
|
-
*
|
|
434
|
-
* Cross-process freshness is out of scope: a reader in process B does not
|
|
435
|
-
* see writes committed by process A through a different cache. Use a
|
|
436
|
-
* short-lived resolver, an external coordination layer, or wrap with TTL /
|
|
437
|
-
* conditional-GET revalidation if you need that.
|
|
438
|
-
*
|
|
439
|
-
* @param {Resolver} base
|
|
440
|
-
* @returns {Resolver}
|
|
441
|
-
*/
|
|
442
|
-
declare function cachingResolver(base: Resolver): Resolver;
|
|
443
|
-
export { cachingResolver, icebergAppend, icebergCreateTable, icebergManifests, restCatalogConnect, restCatalogLoadTable };
|
|
346
|
+
export { icebergCreateTable, icebergManifests, restCatalogLoadTable };
|