@gscdump/lakehouse 7.8.0 → 7.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/fetch.d.mts +35 -1
- package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/index.d.mts +1 -1
- package/dist/catalog.d.mts +1 -1
- package/dist/icebird-resolver.d.mts +10 -0
- package/package.json +1 -1
|
@@ -1,2 +1,36 @@
|
|
|
1
1
|
import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
|
|
2
|
-
import "../src/types.mjs";
|
|
2
|
+
import { Resolver } from "../src/types.mjs";
|
|
3
|
+
/**
|
|
4
|
+
* Wrap a `Resolver` so reads of the same path share one HTTP fetch and
|
|
5
|
+
* subsequent range reads share one in-memory buffer. Writes and deletes
|
|
6
|
+
* through the same resolver invalidate the cache entry for their path on
|
|
7
|
+
* success, so a commit pipeline (`icebergAppend` → reload metadata) sees the
|
|
8
|
+
* new bytes without manual invalidation.
|
|
9
|
+
*
|
|
10
|
+
* - `reader(path)` is memoized by path. The returned buffer is passed through
|
|
11
|
+
* `cachedAsyncBuffer` so range reads within a single file are also deduped.
|
|
12
|
+
* `byteLength` is a fetch hint and not part of the cache key — the bytes
|
|
13
|
+
* are the same either way.
|
|
14
|
+
* - `writer(path).finish()` invalidates the cache entry only when the
|
|
15
|
+
* underlying finish resolves; a rejected finish (e.g. an `If-None-Match: *`
|
|
16
|
+
* collision returning 412/409) leaves the cached bytes intact, since the
|
|
17
|
+
* server-side object hasn't changed.
|
|
18
|
+
* - `deleter(path)` invalidates the cache entry on success.
|
|
19
|
+
* - `writer` and `deleter` are omitted when the base resolver omits them, so
|
|
20
|
+
* wrapping a read-only resolver still yields a read-only resolver.
|
|
21
|
+
*
|
|
22
|
+
* Iceberg writes are almost entirely create-new-path (data parquets, manifest
|
|
23
|
+
* avros, snapshot avros, vN.metadata.json all live at fresh paths per
|
|
24
|
+
* commit). Only `version-hint.text` and equivalent catalog-pointer files are
|
|
25
|
+
* truly mutated in place, so path-level invalidation is sufficient to keep
|
|
26
|
+
* single-process write-then-read pipelines consistent.
|
|
27
|
+
*
|
|
28
|
+
* Cross-process freshness is out of scope: a reader in process B does not
|
|
29
|
+
* see writes committed by process A through a different cache. Use a
|
|
30
|
+
* short-lived resolver, an external coordination layer, or wrap with TTL /
|
|
31
|
+
* conditional-GET revalidation if you need that.
|
|
32
|
+
*
|
|
33
|
+
* @param {Resolver} base
|
|
34
|
+
* @returns {Resolver}
|
|
35
|
+
*/
|
|
36
|
+
export declare function cachingResolver(base: Resolver): Resolver;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { icebergManifests } from "./manifest.mjs";
|
|
2
2
|
import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
|
|
3
|
+
import { cachingResolver } from "./fetch.mjs";
|
|
3
4
|
import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./catalog/rest.mjs";
|
|
4
5
|
import "./create.mjs";
|
|
5
6
|
import "./catalog/file.mjs";
|
|
6
|
-
import "./fetch.mjs";
|
|
7
7
|
import { s3SignedResolver } from "./s3.mjs";
|
|
8
8
|
import "../../../../squirreling@0.16.6/node_modules/squirreling/src/index.mjs";
|
|
9
9
|
import "./read.mjs";
|
package/dist/catalog.d.mts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { ManifestReadCache, ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
|
|
2
2
|
import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/write/write.mjs";
|
|
3
3
|
import { CatalogCache } from "./catalog-cache.mjs";
|
|
4
|
+
import { Resolver } from "./icebird-resolver.mjs";
|
|
4
5
|
import "./manifest-read-limits.mjs";
|
|
5
6
|
import { IcebergPrimitiveType, IcebergS3Config } from "./schema.mjs";
|
|
6
7
|
import { PartitionValueMatch } from "./partition-prune.mjs";
|
|
7
8
|
import { restCatalogConnect, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/catalog/rest.mjs";
|
|
8
|
-
import { Resolver } from "icebird/src/types.js";
|
|
9
9
|
/** A field in an icebird table `Schema`. */
|
|
10
10
|
export interface IcebergSchemaField {
|
|
11
11
|
id: number;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/fetch.mjs";
|
|
2
|
+
/**
|
|
3
|
+
* icebird's file `Resolver`.
|
|
4
|
+
*
|
|
5
|
+
* Derive it from `cachingResolver` and never import `icebird/src/types.js`.
|
|
6
|
+
* That specifier has no declaration under icebird's `types/` folder, so the
|
|
7
|
+
* declaration bundler keeps it as an external import. icebird is bundled and
|
|
8
|
+
* consumers do not install it, so the type would resolve to `any` for them.
|
|
9
|
+
*/
|
|
10
|
+
export type Resolver = ReturnType<typeof cachingResolver>;
|
package/package.json
CHANGED