@gscdump/lakehouse 5.8.0 → 5.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.mjs +11 -1
- package/package.json +1 -1
package/dist/catalog.mjs
CHANGED
|
@@ -479,6 +479,7 @@ async function createIcebergAppendFileResolver(conn, table, dateColumn) {
|
|
|
479
479
|
entries: /* @__PURE__ */ new Map()
|
|
480
480
|
};
|
|
481
481
|
const immutableBytes = /* @__PURE__ */ new Map();
|
|
482
|
+
const retainedSizes = /* @__PURE__ */ new Map();
|
|
482
483
|
const maxRetainedBytes = 4194304;
|
|
483
484
|
const budgetFailure = { _tag: "ManifestBudgetExceeded" };
|
|
484
485
|
let retainedBytes = 0;
|
|
@@ -489,11 +490,20 @@ async function createIcebergAppendFileResolver(conn, table, dateColumn) {
|
|
|
489
490
|
let bytes = immutableBytes.get(url);
|
|
490
491
|
if (!bytes) {
|
|
491
492
|
bytes = Promise.resolve(conn.resolver.reader(url)).then(async (buffer) => {
|
|
492
|
-
if (!Number.isSafeInteger(buffer.byteLength) || buffer.byteLength < 0 ||
|
|
493
|
+
if (!Number.isSafeInteger(buffer.byteLength) || buffer.byteLength < 0 || buffer.byteLength > maxRetainedBytes) {
|
|
493
494
|
unavailable = true;
|
|
494
495
|
throw budgetFailure;
|
|
495
496
|
}
|
|
497
|
+
for (const [key, size] of retainedSizes) {
|
|
498
|
+
if (retainedBytes + buffer.byteLength <= maxRetainedBytes) break;
|
|
499
|
+
retainedSizes.delete(key);
|
|
500
|
+
immutableBytes.delete(key);
|
|
501
|
+
manifestCache.lists.delete(key);
|
|
502
|
+
manifestCache.entries.delete(key);
|
|
503
|
+
retainedBytes -= size;
|
|
504
|
+
}
|
|
496
505
|
retainedBytes += buffer.byteLength;
|
|
506
|
+
retainedSizes.set(url, buffer.byteLength);
|
|
497
507
|
const content = await buffer.slice(0, buffer.byteLength);
|
|
498
508
|
if (content.byteLength !== buffer.byteLength) {
|
|
499
509
|
unavailable = true;
|
package/package.json
CHANGED