@gscdump/lakehouse 4.2.1 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog-cache.d.mts +3 -1
- package/dist/catalog-cache.mjs +29 -4
- package/dist/catalog.d.mts +2 -0
- package/dist/catalog.mjs +38 -7
- package/package.json +1 -1
package/dist/catalog-cache.d.mts
CHANGED
|
@@ -13,12 +13,14 @@ export interface CatalogCache {
|
|
|
13
13
|
/** Optional warning sink for best-effort cache driver failures. */
|
|
14
14
|
onError?: (operation: 'get' | 'set' | 'remove', key: string, error: unknown) => void;
|
|
15
15
|
}
|
|
16
|
+
/** Outcome of one cache read, including misses that still permit a fresh load. */
|
|
17
|
+
export type CatalogCacheGetOutcome = 'hit' | 'miss' | 'expired' | 'invalid' | 'error';
|
|
16
18
|
/**
|
|
17
19
|
* Read a cached value. Returns `undefined` on a miss, an expired entry, a
|
|
18
20
|
* malformed box, or any driver error (the cache is best-effort: a read failure
|
|
19
21
|
* degrades to a fresh load, never to an error).
|
|
20
22
|
*/
|
|
21
|
-
export declare function cacheGet<T>(cache: CatalogCache, key: string, now: number): Promise<T | undefined>;
|
|
23
|
+
export declare function cacheGet<T>(cache: CatalogCache, key: string, now: number, onOutcome?: (outcome: CatalogCacheGetOutcome) => void): Promise<T | undefined>;
|
|
22
24
|
/**
|
|
23
25
|
* Write a cached value with an embedded expiry and a forwarded driver TTL.
|
|
24
26
|
*
|
package/dist/catalog-cache.mjs
CHANGED
|
@@ -6,12 +6,37 @@ function reportCatalogCacheError(cache, operation, key, error) {
|
|
|
6
6
|
console.warn(`[gscdump/lakehouse] cache error reporter failed during ${operation} for ${key}`, reportError);
|
|
7
7
|
}
|
|
8
8
|
}
|
|
9
|
-
async function cacheGet(cache, key, now) {
|
|
10
|
-
const
|
|
9
|
+
async function cacheGet(cache, key, now, onOutcome) {
|
|
10
|
+
const reportOutcome = (outcome) => {
|
|
11
|
+
try {
|
|
12
|
+
onOutcome?.(outcome);
|
|
13
|
+
} catch {}
|
|
14
|
+
};
|
|
15
|
+
const read = await cache.storage.getItem(key).then((boxed) => ({
|
|
16
|
+
_tag: "Ok",
|
|
17
|
+
boxed
|
|
18
|
+
}), (error) => {
|
|
11
19
|
reportCatalogCacheError(cache, "get", key, error);
|
|
12
|
-
return
|
|
20
|
+
return { _tag: "Error" };
|
|
13
21
|
});
|
|
14
|
-
if (
|
|
22
|
+
if (read._tag === "Error") {
|
|
23
|
+
reportOutcome("error");
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
const { boxed } = read;
|
|
27
|
+
if (!boxed) {
|
|
28
|
+
reportOutcome("miss");
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
if (typeof boxed.exp !== "number" || boxed.v === void 0) {
|
|
32
|
+
reportOutcome("invalid");
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
if (boxed.exp <= now) {
|
|
36
|
+
reportOutcome("expired");
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
reportOutcome("hit");
|
|
15
40
|
return boxed.v;
|
|
16
41
|
}
|
|
17
42
|
function cachePut(cache, key, value, ttlMs, now) {
|
package/dist/catalog.d.mts
CHANGED
|
@@ -93,6 +93,8 @@ export interface ConnectIcebergOptions {
|
|
|
93
93
|
cache?: CatalogCache;
|
|
94
94
|
/** Injectable clock for the cache TTL. Defaults to `Date.now`. */
|
|
95
95
|
clock?: () => number;
|
|
96
|
+
/** Optional connection-stage timings. Spans carry no catalog coordinates. */
|
|
97
|
+
profiler?: QueryProfiler;
|
|
96
98
|
}
|
|
97
99
|
/**
|
|
98
100
|
* Connect to the R2 Data Catalog: a REST catalog context + a signed S3
|
package/dist/catalog.mjs
CHANGED
|
@@ -35,6 +35,20 @@ function manifestReadCacheFor(conn, namespace, table, metadata) {
|
|
|
35
35
|
}
|
|
36
36
|
return cache;
|
|
37
37
|
}
|
|
38
|
+
function startConnectSpan(profiler, name) {
|
|
39
|
+
if (!profiler) return void 0;
|
|
40
|
+
try {
|
|
41
|
+
const end = profiler.start(name);
|
|
42
|
+
if (!end) return void 0;
|
|
43
|
+
return (meta) => {
|
|
44
|
+
try {
|
|
45
|
+
end(meta);
|
|
46
|
+
} catch {}
|
|
47
|
+
};
|
|
48
|
+
} catch {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
38
52
|
let icebirdWriteModule;
|
|
39
53
|
function useIcebirdWriteModule() {
|
|
40
54
|
icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs");
|
|
@@ -49,7 +63,12 @@ async function connectIcebergCatalog(config, opts = {}) {
|
|
|
49
63
|
const requestInit = { headers: { Authorization: `Bearer ${config.catalogToken}` } };
|
|
50
64
|
let catalog;
|
|
51
65
|
if (opts.cache) {
|
|
52
|
-
const
|
|
66
|
+
const endCache = startConnectSpan(opts.profiler, "catalog.config.cache");
|
|
67
|
+
const cacheRead = { outcome: "miss" };
|
|
68
|
+
const cached = await cacheGet(opts.cache, catalogConfigKey(config), now, (outcome) => {
|
|
69
|
+
cacheRead.outcome = outcome;
|
|
70
|
+
});
|
|
71
|
+
endCache?.({ outcome: cacheRead.outcome === "hit" && !cached ? "invalid" : cacheRead.outcome });
|
|
53
72
|
if (cached) catalog = Object.freeze({
|
|
54
73
|
type: "rest",
|
|
55
74
|
url: cached.url,
|
|
@@ -60,11 +79,18 @@ async function connectIcebergCatalog(config, opts = {}) {
|
|
|
60
79
|
});
|
|
61
80
|
}
|
|
62
81
|
if (!catalog) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
82
|
+
const endRest = startConnectSpan(opts.profiler, "catalog.config.rest");
|
|
83
|
+
let restOutcome = "error";
|
|
84
|
+
try {
|
|
85
|
+
catalog = await restCatalogConnect({
|
|
86
|
+
url: config.catalogUri,
|
|
87
|
+
warehouse: config.warehouse,
|
|
88
|
+
requestInit
|
|
89
|
+
});
|
|
90
|
+
restOutcome = "ok";
|
|
91
|
+
} finally {
|
|
92
|
+
endRest?.({ outcome: restOutcome });
|
|
93
|
+
}
|
|
68
94
|
if (opts.cache) {
|
|
69
95
|
const toCache = {
|
|
70
96
|
url: catalog.url,
|
|
@@ -72,7 +98,12 @@ async function connectIcebergCatalog(config, opts = {}) {
|
|
|
72
98
|
defaults: catalog.defaults,
|
|
73
99
|
overrides: catalog.overrides
|
|
74
100
|
};
|
|
75
|
-
|
|
101
|
+
const endWrite = startConnectSpan(opts.profiler, "catalog.config.write");
|
|
102
|
+
try {
|
|
103
|
+
await cachePut(opts.cache, catalogConfigKey(config), toCache, CATALOG_CONFIG_TTL_MS, now);
|
|
104
|
+
} finally {
|
|
105
|
+
endWrite?.({ deferred: opts.cache.defer != null });
|
|
106
|
+
}
|
|
76
107
|
}
|
|
77
108
|
}
|
|
78
109
|
const cacheScope = catalogCacheScope(config);
|
package/package.json
CHANGED