@cloudbitmaps/core 0.8.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/azure/index.d.cts +1 -1
- package/dist/azure/index.d.ts +1 -1
- package/dist/cassandra/index.cjs.map +1 -1
- package/dist/cassandra/index.d.cts +1 -1
- package/dist/cassandra/index.d.ts +1 -1
- package/dist/cassandra/index.js +1 -1
- package/dist/{chunk-L54FL5DW.js → chunk-BD22WDAN.js} +34 -7
- package/dist/chunk-BD22WDAN.js.map +1 -0
- package/dist/{chunk-YEHMG6W2.js → chunk-OZ6LAFV4.js} +2 -2
- package/dist/chunk-OZ6LAFV4.js.map +1 -0
- package/dist/dynamodb/index.cjs +32 -5
- package/dist/dynamodb/index.cjs.map +1 -1
- package/dist/dynamodb/index.d.cts +1 -1
- package/dist/dynamodb/index.d.ts +1 -1
- package/dist/dynamodb/index.js +2 -2
- package/dist/gcs/index.d.cts +1 -1
- package/dist/gcs/index.d.ts +1 -1
- package/dist/index.cjs +417 -35
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +244 -18
- package/dist/index.d.ts +244 -18
- package/dist/index.js +379 -34
- package/dist/index.js.map +1 -1
- package/dist/mongodb/index.cjs.map +1 -1
- package/dist/mongodb/index.d.cts +1 -1
- package/dist/mongodb/index.d.ts +1 -1
- package/dist/mongodb/index.js +1 -1
- package/dist/mysql/index.cjs.map +1 -1
- package/dist/mysql/index.d.cts +1 -1
- package/dist/mysql/index.d.ts +1 -1
- package/dist/mysql/index.js +1 -1
- package/dist/{ports-5BQTGKze.d.cts → ports-CvcvJWf6.d.cts} +46 -7
- package/dist/{ports-5BQTGKze.d.ts → ports-CvcvJWf6.d.ts} +46 -7
- package/dist/postgres/index.cjs.map +1 -1
- package/dist/postgres/index.d.cts +1 -1
- package/dist/postgres/index.d.ts +1 -1
- package/dist/postgres/index.js +1 -1
- package/dist/redis/index.cjs.map +1 -1
- package/dist/redis/index.d.cts +1 -1
- package/dist/redis/index.d.ts +1 -1
- package/dist/redis/index.js +1 -1
- package/dist/s3/index.cjs +32 -5
- package/dist/s3/index.cjs.map +1 -1
- package/dist/s3/index.d.cts +1 -1
- package/dist/s3/index.d.ts +1 -1
- package/dist/s3/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-L54FL5DW.js.map +0 -1
- package/dist/chunk-YEHMG6W2.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -2153,6 +2153,7 @@ var CrbmColdChunkSource = class {
|
|
|
2153
2153
|
const record = await this.registry.get(ref);
|
|
2154
2154
|
if (record === null) return null;
|
|
2155
2155
|
if (record.status === "destroyed") return null;
|
|
2156
|
+
if (record.currentGen === null) return null;
|
|
2156
2157
|
return { generation: record.currentGen, wrappedDeks: record.wrappedDeks };
|
|
2157
2158
|
}
|
|
2158
2159
|
let maxGen = -1;
|
|
@@ -2276,6 +2277,11 @@ async function publishGeneration(registry, key, options = {}) {
|
|
|
2276
2277
|
throw new ValidationError(
|
|
2277
2278
|
`segment "${key.segment}" was destroyed (crypto-shredded) while generation ${key.generation} was being written \u2014 refusing to publish it; the written object is unreadable. Use a new segment.`
|
|
2278
2279
|
);
|
|
2280
|
+
} else if (record.currentGen === null) {
|
|
2281
|
+
await registry.compareAndSwap(key, record.token, {
|
|
2282
|
+
currentGen: key.generation,
|
|
2283
|
+
...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks }
|
|
2284
|
+
});
|
|
2279
2285
|
} else if (record.currentGen > key.generation) {
|
|
2280
2286
|
return false;
|
|
2281
2287
|
} else if (record.currentGen === key.generation) {
|
|
@@ -2290,7 +2296,7 @@ async function publishGeneration(registry, key, options = {}) {
|
|
|
2290
2296
|
}
|
|
2291
2297
|
}
|
|
2292
2298
|
const final = await registry.get(key);
|
|
2293
|
-
if (final !== null && final.currentGen >= key.generation)
|
|
2299
|
+
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation)
|
|
2294
2300
|
return final.currentGen === key.generation;
|
|
2295
2301
|
throw new WriteConflictError(
|
|
2296
2302
|
`publishGeneration: contention setting currentGen for ${key.segment}`
|
|
@@ -2359,6 +2365,11 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
2359
2365
|
`segment "${key.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
2360
2366
|
);
|
|
2361
2367
|
}
|
|
2368
|
+
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0 && options.keystore === void 0) {
|
|
2369
|
+
throw new KeyUnavailableError(
|
|
2370
|
+
`segment "${key.segment}" is encrypted but this bulk-load has no keystore \u2014 refusing to write a cleartext generation onto an encrypted segment. Pass the keystore holding its DEK.`
|
|
2371
|
+
);
|
|
2372
|
+
}
|
|
2362
2373
|
let crypto;
|
|
2363
2374
|
let newWrapped;
|
|
2364
2375
|
if (options.keystore !== void 0) {
|
|
@@ -2461,7 +2472,7 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2461
2472
|
survived: 0
|
|
2462
2473
|
};
|
|
2463
2474
|
}
|
|
2464
|
-
if (record === null) {
|
|
2475
|
+
if (record === null || record.currentGen === null) {
|
|
2465
2476
|
const pinned = await collectDirty(
|
|
2466
2477
|
deps.warm,
|
|
2467
2478
|
ref,
|
|
@@ -2480,7 +2491,15 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2480
2491
|
};
|
|
2481
2492
|
let aead;
|
|
2482
2493
|
let wrappedDeks;
|
|
2483
|
-
if (
|
|
2494
|
+
if (record?.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
2495
|
+
if (deps.keystore === void 0) {
|
|
2496
|
+
throw new KeyUnavailableError(
|
|
2497
|
+
`segment "${ref.segment}" is encrypted but compaction has no keystore`
|
|
2498
|
+
);
|
|
2499
|
+
}
|
|
2500
|
+
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
2501
|
+
wrappedDeks = record.wrappedDeks;
|
|
2502
|
+
} else if (deps.keystore !== void 0) {
|
|
2484
2503
|
const minted = await deps.keystore.createDek();
|
|
2485
2504
|
aead = minted.aead;
|
|
2486
2505
|
wrappedDeks = minted.wrapped;
|
|
@@ -2506,17 +2525,8 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2506
2525
|
if (wrote && tally !== void 0) {
|
|
2507
2526
|
await verifyGeneration(deps.cold, gen0, tally, cryptoAt(aead, 0));
|
|
2508
2527
|
}
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
currentGen: 0,
|
|
2512
|
-
dirtyChunkCount: 0,
|
|
2513
|
-
status: "active",
|
|
2514
|
-
wrappedDeks
|
|
2515
|
-
});
|
|
2516
|
-
} catch (err) {
|
|
2517
|
-
if (!isWriteConflictError(err)) throw err;
|
|
2518
|
-
}
|
|
2519
|
-
if (!wrote)
|
|
2528
|
+
const published = await publishGenZero(deps.registry, ref, record, wrappedDeks, wrote);
|
|
2529
|
+
if (!wrote || !published)
|
|
2520
2530
|
return {
|
|
2521
2531
|
...base,
|
|
2522
2532
|
compacted: false,
|
|
@@ -2680,6 +2690,43 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2680
2690
|
throw err;
|
|
2681
2691
|
}
|
|
2682
2692
|
}
|
|
2693
|
+
var PUBLISH_GEN0_ATTEMPTS = 5;
|
|
2694
|
+
async function publishGenZero(registry, ref, known, wrappedDeks, wrote) {
|
|
2695
|
+
if (!wrote && wrappedDeks !== void 0) return false;
|
|
2696
|
+
let current = known;
|
|
2697
|
+
for (let attempt = 0; attempt < PUBLISH_GEN0_ATTEMPTS; attempt += 1) {
|
|
2698
|
+
try {
|
|
2699
|
+
if (current === null) {
|
|
2700
|
+
await registry.create(ref, {
|
|
2701
|
+
currentGen: 0,
|
|
2702
|
+
dirtyChunkCount: 0,
|
|
2703
|
+
status: "active",
|
|
2704
|
+
wrappedDeks
|
|
2705
|
+
});
|
|
2706
|
+
} else if (current.status === "destroyed") {
|
|
2707
|
+
return false;
|
|
2708
|
+
} else if (current.currentGen === null) {
|
|
2709
|
+
await registry.compareAndSwap(ref, current.token, {
|
|
2710
|
+
currentGen: 0,
|
|
2711
|
+
dirtyChunkCount: 0,
|
|
2712
|
+
...wrappedDeks === void 0 ? {} : { wrappedDeks }
|
|
2713
|
+
});
|
|
2714
|
+
} else {
|
|
2715
|
+
return current.currentGen === 0 && sameWrappings(current.wrappedDeks, wrappedDeks);
|
|
2716
|
+
}
|
|
2717
|
+
return true;
|
|
2718
|
+
} catch (err) {
|
|
2719
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2720
|
+
current = await registry.get(ref);
|
|
2721
|
+
}
|
|
2722
|
+
}
|
|
2723
|
+
return false;
|
|
2724
|
+
}
|
|
2725
|
+
function sameWrappings(a, b) {
|
|
2726
|
+
if (a === void 0 || b === void 0) return a === b;
|
|
2727
|
+
if (a.length !== b.length) return false;
|
|
2728
|
+
return a.every((w, i) => w.keyId === b[i]?.keyId && w.wrapped === b[i]?.wrapped);
|
|
2729
|
+
}
|
|
2683
2730
|
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
2684
2731
|
const keep = Math.max(0, options.keep ?? 1);
|
|
2685
2732
|
const record = await deps.registry.get(ref);
|
|
@@ -2687,7 +2734,12 @@ async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
|
2687
2734
|
const current = record.currentGen;
|
|
2688
2735
|
const gens = [];
|
|
2689
2736
|
for await (const key of deps.cold.list(ref)) gens.push(key.generation);
|
|
2690
|
-
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : (
|
|
2737
|
+
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : current === null ? (
|
|
2738
|
+
// No Cold pointer yet, so "below current" selects nothing and there is nothing safe to infer: an
|
|
2739
|
+
// object here is either a bootstrap about to publish gen 0 or an orphan we cannot tell apart from it.
|
|
2740
|
+
// Deleting would race that publish into a dangling pointer. RECONCILE collects it on the next cycle.
|
|
2741
|
+
[]
|
|
2742
|
+
) : (
|
|
2691
2743
|
// Delete generations below current, except the newest `keep` of them (the grace window).
|
|
2692
2744
|
gens.filter((g) => g < current).sort((a, b) => b - a).slice(keep)
|
|
2693
2745
|
);
|
|
@@ -3033,9 +3085,21 @@ var MAX_GOVERNANCE_BYTES = 64 * 1024;
|
|
|
3033
3085
|
var MAX_WRAPPED_DEKS = 8;
|
|
3034
3086
|
var MAX_WRAPPED_DEK_BYTES = 4 * 1024;
|
|
3035
3087
|
function validateGeneration(gen) {
|
|
3088
|
+
if (gen === null) return;
|
|
3036
3089
|
if (!Number.isInteger(gen) || gen < 0) {
|
|
3037
|
-
throw new ValidationError(
|
|
3090
|
+
throw new ValidationError(
|
|
3091
|
+
`currentGen must be a non-negative integer or null (no Cold generation yet); got ${gen}`
|
|
3092
|
+
);
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
function validatePatchGeneration(patch) {
|
|
3096
|
+
if (!("currentGen" in patch)) return;
|
|
3097
|
+
if (patch.currentGen === void 0) {
|
|
3098
|
+
throw new ValidationError(
|
|
3099
|
+
`currentGen was present in the patch but undefined. Pass \`null\` to clear the pointer (the segment has no Cold generation), or omit the key to leave it unchanged \u2014 an undefined value is refused because it used to be a no-op and would now un-publish the segment's Cold data.`
|
|
3100
|
+
);
|
|
3038
3101
|
}
|
|
3102
|
+
validateGeneration(patch.currentGen);
|
|
3039
3103
|
}
|
|
3040
3104
|
function validateCount(count, field = "dirtyChunkCount") {
|
|
3041
3105
|
if (!Number.isInteger(count) || count < 0) {
|
|
@@ -3054,6 +3118,11 @@ function validateStatus(status) {
|
|
|
3054
3118
|
}
|
|
3055
3119
|
function validateGovernance(meta, field) {
|
|
3056
3120
|
if (meta === void 0) return;
|
|
3121
|
+
if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
|
|
3122
|
+
throw new ValidationError(
|
|
3123
|
+
`${field} must be a plain object (got ${meta === null ? "null" : typeof meta})`
|
|
3124
|
+
);
|
|
3125
|
+
}
|
|
3057
3126
|
let json;
|
|
3058
3127
|
try {
|
|
3059
3128
|
json = JSON.stringify(meta);
|
|
@@ -3094,7 +3163,7 @@ function validateNewRegistryRecord(rec) {
|
|
|
3094
3163
|
validateGovernance(rec.residency, "residency");
|
|
3095
3164
|
}
|
|
3096
3165
|
function validateRegistryPatch(patch) {
|
|
3097
|
-
|
|
3166
|
+
validatePatchGeneration(patch);
|
|
3098
3167
|
if (patch.dirtyChunkCount !== void 0) validateCount(patch.dirtyChunkCount);
|
|
3099
3168
|
if (patch.consecutiveFailures !== void 0)
|
|
3100
3169
|
validateCount(patch.consecutiveFailures, "consecutiveFailures");
|
|
@@ -3175,10 +3244,10 @@ function recordFromNew(ref, rec, now, token) {
|
|
|
3175
3244
|
};
|
|
3176
3245
|
}
|
|
3177
3246
|
function assertStoredRecordShape(r, ctx) {
|
|
3178
|
-
if (typeof r.segment !== "string" || r.namespace !== void 0 && typeof r.namespace !== "string" || typeof r.currentGen !== "number" || typeof r.dirtyChunkCount !== "number" || typeof r.status !== "string" || typeof r.createdAt !== "number" || typeof r.updatedAt !== "number") {
|
|
3247
|
+
if (typeof r.segment !== "string" || r.namespace !== void 0 && typeof r.namespace !== "string" || r.currentGen !== null && typeof r.currentGen !== "number" || typeof r.dirtyChunkCount !== "number" || typeof r.status !== "string" || typeof r.createdAt !== "number" || typeof r.updatedAt !== "number") {
|
|
3179
3248
|
throw new IntegrityError(`registry record is missing required fields: ${ctx}`);
|
|
3180
3249
|
}
|
|
3181
|
-
if (!Number.isInteger(r.currentGen) || r.currentGen < 0) {
|
|
3250
|
+
if (r.currentGen !== null && (!Number.isInteger(r.currentGen) || r.currentGen < 0)) {
|
|
3182
3251
|
throw new IntegrityError(`registry record has an invalid currentGen (${r.currentGen}): ${ctx}`);
|
|
3183
3252
|
}
|
|
3184
3253
|
if (!Number.isInteger(r.dirtyChunkCount) || r.dirtyChunkCount < 0) {
|
|
@@ -3203,12 +3272,22 @@ function assertStoredRecordShape(r, ctx) {
|
|
|
3203
3272
|
throw new IntegrityError(`registry record has an invalid keyId: ${ctx}`);
|
|
3204
3273
|
}
|
|
3205
3274
|
validateWrappedDeks(r.wrappedDeks, true);
|
|
3275
|
+
for (const field of ["retention", "residency"]) {
|
|
3276
|
+
const meta = r[field];
|
|
3277
|
+
if (meta !== void 0 && (meta === null || typeof meta !== "object" || Array.isArray(meta))) {
|
|
3278
|
+
throw new IntegrityError(`registry record has a non-object ${field}: ${ctx}`);
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3206
3281
|
}
|
|
3207
3282
|
function applyRegistryPatch(prev, patch, now, token) {
|
|
3208
3283
|
return {
|
|
3209
3284
|
namespace: prev.namespace,
|
|
3210
3285
|
segment: prev.segment,
|
|
3211
|
-
currentGen
|
|
3286
|
+
// `'currentGen' in patch`, NOT `patch.currentGen ?? prev.currentGen`. `null` is a legal, meaningful value
|
|
3287
|
+
// here (no Cold generation yet), and `??` treats it as absent — so the nullish form would silently ignore a
|
|
3288
|
+
// patch that clears the pointer and leave the old generation in place. The same reason `wrappedDeks` and
|
|
3289
|
+
// `keyId` below use presence: any field whose null is a *value* cannot be merged with `??`.
|
|
3290
|
+
currentGen: "currentGen" in patch ? patch.currentGen ?? null : prev.currentGen,
|
|
3212
3291
|
wrappedDeks: "wrappedDeks" in patch ? patch.wrappedDeks : prev.wrappedDeks,
|
|
3213
3292
|
keyId: "keyId" in patch ? patch.keyId : prev.keyId,
|
|
3214
3293
|
dirtyChunkCount: patch.dirtyChunkCount ?? prev.dirtyChunkCount,
|
|
@@ -4276,8 +4355,309 @@ async function eraseWarm(warm, ref, op = "destroySegment") {
|
|
|
4276
4355
|
return deleted;
|
|
4277
4356
|
}
|
|
4278
4357
|
|
|
4279
|
-
// src/core/
|
|
4358
|
+
// src/core/retention.ts
|
|
4359
|
+
var EXPIRES_AT = "expiresAt";
|
|
4360
|
+
var RETENTION_CAS_ATTEMPTS = 5;
|
|
4361
|
+
var MIN_EXPIRES_AT_MS = 1e12;
|
|
4362
|
+
function validateRetentionPolicy(policy) {
|
|
4363
|
+
const { expiresAt } = policy;
|
|
4364
|
+
if (typeof expiresAt !== "number" || !Number.isInteger(expiresAt)) {
|
|
4365
|
+
throw new ValidationError(
|
|
4366
|
+
`retention.expiresAt must be an integer epoch-ms; got ${String(expiresAt)}`
|
|
4367
|
+
);
|
|
4368
|
+
}
|
|
4369
|
+
if (expiresAt < MIN_EXPIRES_AT_MS) {
|
|
4370
|
+
throw new ValidationError(
|
|
4371
|
+
`retention.expiresAt (${expiresAt}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds, which would read as "already expired" and retire the segment on the next sweep. Use \`Date.now() + days * 86_400_000\`. To retire a segment now, call \`dropSegment\`.`
|
|
4372
|
+
);
|
|
4373
|
+
}
|
|
4374
|
+
}
|
|
4375
|
+
function readRetentionPolicy(meta) {
|
|
4376
|
+
if (meta === void 0) return null;
|
|
4377
|
+
if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "invalid";
|
|
4378
|
+
if (!(EXPIRES_AT in meta)) return null;
|
|
4379
|
+
const raw = meta[EXPIRES_AT];
|
|
4380
|
+
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < MIN_EXPIRES_AT_MS)
|
|
4381
|
+
return "invalid";
|
|
4382
|
+
return { expiresAt: raw };
|
|
4383
|
+
}
|
|
4384
|
+
async function getSegmentRetention(ref, deps) {
|
|
4385
|
+
const record = await deps.registry.get(ref);
|
|
4386
|
+
if (record === null || record.status === "destroyed") return null;
|
|
4387
|
+
return readRetentionPolicy(record.retention);
|
|
4388
|
+
}
|
|
4389
|
+
async function setSegmentRetention(ref, deps, policy) {
|
|
4390
|
+
validateRetentionPolicy(policy);
|
|
4391
|
+
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
4392
|
+
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
4393
|
+
const record = await deps.registry.get(ref);
|
|
4394
|
+
try {
|
|
4395
|
+
if (record === null) {
|
|
4396
|
+
await deps.registry.create(ref, {
|
|
4397
|
+
currentGen: null,
|
|
4398
|
+
retention: { [EXPIRES_AT]: policy.expiresAt }
|
|
4399
|
+
});
|
|
4400
|
+
return { ...base, expiresAt: policy.expiresAt, createdRow: true };
|
|
4401
|
+
}
|
|
4402
|
+
assertNotDestroyed(record, ref);
|
|
4403
|
+
await deps.registry.compareAndSwap(ref, record.token, {
|
|
4404
|
+
retention: { ...record.retention, [EXPIRES_AT]: policy.expiresAt }
|
|
4405
|
+
});
|
|
4406
|
+
return { ...base, expiresAt: policy.expiresAt, createdRow: false };
|
|
4407
|
+
} catch (err) {
|
|
4408
|
+
if (!isWriteConflictError(err)) throw err;
|
|
4409
|
+
}
|
|
4410
|
+
}
|
|
4411
|
+
throw new WriteConflictError(
|
|
4412
|
+
`setRetention: contention writing the retention policy for segment "${ref.segment}"`
|
|
4413
|
+
);
|
|
4414
|
+
}
|
|
4415
|
+
async function clearSegmentRetention(ref, deps) {
|
|
4416
|
+
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
4417
|
+
const record = await deps.registry.get(ref);
|
|
4418
|
+
if (record === null) return false;
|
|
4419
|
+
if (record.status === "destroyed") return false;
|
|
4420
|
+
if (record.retention === void 0 || !(EXPIRES_AT in record.retention)) return false;
|
|
4421
|
+
const rest = { ...record.retention };
|
|
4422
|
+
delete rest[EXPIRES_AT];
|
|
4423
|
+
try {
|
|
4424
|
+
await deps.registry.compareAndSwap(ref, record.token, {
|
|
4425
|
+
retention: Object.keys(rest).length === 0 ? void 0 : rest
|
|
4426
|
+
});
|
|
4427
|
+
return true;
|
|
4428
|
+
} catch (err) {
|
|
4429
|
+
if (!isWriteConflictError(err)) throw err;
|
|
4430
|
+
}
|
|
4431
|
+
}
|
|
4432
|
+
throw new WriteConflictError(
|
|
4433
|
+
`clearRetention: contention clearing the retention policy for segment "${ref.segment}"`
|
|
4434
|
+
);
|
|
4435
|
+
}
|
|
4436
|
+
function assertNotDestroyed(record, ref) {
|
|
4437
|
+
if (record.status === "destroyed") {
|
|
4438
|
+
throw new ValidationError(
|
|
4439
|
+
`segment "${ref.segment}" is destroyed (crypto-shredded) \u2014 refusing to set a retention policy on a tombstone; there is nothing left to retire. Use a new segment.`
|
|
4440
|
+
);
|
|
4441
|
+
}
|
|
4442
|
+
}
|
|
4443
|
+
|
|
4444
|
+
// src/core/registry-scan.ts
|
|
4280
4445
|
var DEFAULT_MAX_SCAN_SEGMENTS = 25e4;
|
|
4446
|
+
function validateMaxScanSegments(value, op) {
|
|
4447
|
+
if (!Number.isFinite(value) || value < 1) {
|
|
4448
|
+
throw new ValidationError(`${op}: maxScanSegments must be a finite number >= 1; got ${value}`);
|
|
4449
|
+
}
|
|
4450
|
+
}
|
|
4451
|
+
async function drainRegistry(registry, options) {
|
|
4452
|
+
const { maxScanSegments, op } = options;
|
|
4453
|
+
validateMaxScanSegments(maxScanSegments, op);
|
|
4454
|
+
const rows = [];
|
|
4455
|
+
for await (const rec of registry.list(options.namespace)) {
|
|
4456
|
+
if (rows.length >= maxScanSegments) {
|
|
4457
|
+
throw new BudgetExceededError(
|
|
4458
|
+
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Narrow it with \`namespace\`, or raise \`maxScanSegments\` if the fleet really is that large and the memory is available (a record is a few hundred bytes resident).`
|
|
4459
|
+
);
|
|
4460
|
+
}
|
|
4461
|
+
rows.push(rec);
|
|
4462
|
+
}
|
|
4463
|
+
return rows;
|
|
4464
|
+
}
|
|
4465
|
+
|
|
4466
|
+
// src/core/retention-sweep.ts
|
|
4467
|
+
var DEFAULT_RETIRE_LIMIT = 100;
|
|
4468
|
+
var DEFAULT_TOMBSTONE_GRACE_MS = 864e5;
|
|
4469
|
+
async function retireExpired(deps, options) {
|
|
4470
|
+
const now = options.now;
|
|
4471
|
+
if (!Number.isFinite(now)) {
|
|
4472
|
+
throw new ValidationError(
|
|
4473
|
+
`retireExpired: \`now\` must be a finite epoch-ms; got ${String(now)}`
|
|
4474
|
+
);
|
|
4475
|
+
}
|
|
4476
|
+
if (now < MIN_EXPIRES_AT_MS) {
|
|
4477
|
+
throw new ValidationError(
|
|
4478
|
+
`retireExpired: \`now\` (${now}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds. A sweep with a wrong-units clock treats every policy as expired.`
|
|
4479
|
+
);
|
|
4480
|
+
}
|
|
4481
|
+
const limit = options.limit ?? DEFAULT_RETIRE_LIMIT;
|
|
4482
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
4483
|
+
throw new ValidationError(`retireExpired: \`limit\` must be a positive integer; got ${limit}`);
|
|
4484
|
+
}
|
|
4485
|
+
const purgeTombstones = options.purgeTombstones ?? true;
|
|
4486
|
+
const grace = options.tombstoneGraceMs ?? DEFAULT_TOMBSTONE_GRACE_MS;
|
|
4487
|
+
if (!Number.isFinite(grace) || grace < 0) {
|
|
4488
|
+
throw new ValidationError(
|
|
4489
|
+
`retireExpired: \`tombstoneGraceMs\` must be a non-negative finite number; got ${String(grace)}`
|
|
4490
|
+
);
|
|
4491
|
+
}
|
|
4492
|
+
const maxScanSegments = options.maxScanSegments ?? DEFAULT_MAX_SCAN_SEGMENTS;
|
|
4493
|
+
if (!Number.isFinite(maxScanSegments) || maxScanSegments < 1) {
|
|
4494
|
+
throw new ValidationError(
|
|
4495
|
+
`retireExpired: \`maxScanSegments\` must be a finite number >= 1; got ${String(maxScanSegments)}`
|
|
4496
|
+
);
|
|
4497
|
+
}
|
|
4498
|
+
const dryRun = options.dryRun === true;
|
|
4499
|
+
const rows = await drainRegistry(deps.registry, {
|
|
4500
|
+
namespace: options.namespace,
|
|
4501
|
+
maxScanSegments,
|
|
4502
|
+
op: "retireExpired"
|
|
4503
|
+
});
|
|
4504
|
+
const entries = [];
|
|
4505
|
+
let eligible = 0;
|
|
4506
|
+
let retired = 0;
|
|
4507
|
+
let wouldRetire = 0;
|
|
4508
|
+
let tombstonesPurged = 0;
|
|
4509
|
+
let limited = false;
|
|
4510
|
+
let attempted = 0;
|
|
4511
|
+
for (const rec of rows) {
|
|
4512
|
+
const ref = { namespace: rec.namespace, segment: rec.segment };
|
|
4513
|
+
const base = { segment: rec.segment, namespace: rec.namespace };
|
|
4514
|
+
const policy = readRetentionPolicy(rec.retention);
|
|
4515
|
+
if (rec.status === "destroyed") {
|
|
4516
|
+
if (!purgeTombstones) continue;
|
|
4517
|
+
const retiredAt = retirementStamp(rec.retention);
|
|
4518
|
+
if (retiredAt === null) continue;
|
|
4519
|
+
if (now - retiredAt < grace) continue;
|
|
4520
|
+
if (attempted >= limit) {
|
|
4521
|
+
limited = true;
|
|
4522
|
+
break;
|
|
4523
|
+
}
|
|
4524
|
+
attempted += 1;
|
|
4525
|
+
try {
|
|
4526
|
+
if (!await isFullyReclaimed(deps, ref)) {
|
|
4527
|
+
if (!dryRun) await gcOrphanGenerations(ref, deps).catch(() => void 0);
|
|
4528
|
+
if (!await isFullyReclaimed(deps, ref)) {
|
|
4529
|
+
entries.push({ ...base, action: "skipped", reason: "tombstone-not-empty" });
|
|
4530
|
+
continue;
|
|
4531
|
+
}
|
|
4532
|
+
}
|
|
4533
|
+
if (!dryRun) await deps.registry.delete(ref);
|
|
4534
|
+
if (!dryRun) tombstonesPurged += 1;
|
|
4535
|
+
entries.push({
|
|
4536
|
+
...base,
|
|
4537
|
+
action: dryRun ? "would-purge-tombstone" : "purged-tombstone"
|
|
4538
|
+
});
|
|
4539
|
+
} catch (err) {
|
|
4540
|
+
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4541
|
+
}
|
|
4542
|
+
continue;
|
|
4543
|
+
}
|
|
4544
|
+
if (policy === null) continue;
|
|
4545
|
+
if (policy === "invalid") {
|
|
4546
|
+
entries.push({ ...base, action: "skipped", reason: "invalid-policy" });
|
|
4547
|
+
continue;
|
|
4548
|
+
}
|
|
4549
|
+
if (policy.expiresAt > now) continue;
|
|
4550
|
+
eligible += 1;
|
|
4551
|
+
if (attempted >= limit) {
|
|
4552
|
+
limited = true;
|
|
4553
|
+
break;
|
|
4554
|
+
}
|
|
4555
|
+
try {
|
|
4556
|
+
const live = await deps.registry.get(ref);
|
|
4557
|
+
const livePolicy = live === null ? null : readRetentionPolicy(live.retention);
|
|
4558
|
+
if (live === null || live.status === "destroyed" || livePolicy === null || livePolicy === "invalid" || livePolicy.expiresAt > now) {
|
|
4559
|
+
entries.push({ ...base, action: "skipped", reason: "policy-changed" });
|
|
4560
|
+
continue;
|
|
4561
|
+
}
|
|
4562
|
+
attempted += 1;
|
|
4563
|
+
const result = await dropSegment(ref, deps, {
|
|
4564
|
+
confirmSegment: rec.segment,
|
|
4565
|
+
dryRun,
|
|
4566
|
+
audit: options.audit
|
|
4567
|
+
});
|
|
4568
|
+
if (dryRun) {
|
|
4569
|
+
wouldRetire += 1;
|
|
4570
|
+
entries.push({ ...base, action: "would-retire", expiresAt: livePolicy.expiresAt, result });
|
|
4571
|
+
continue;
|
|
4572
|
+
}
|
|
4573
|
+
if (!result.dropped) {
|
|
4574
|
+
entries.push({
|
|
4575
|
+
...base,
|
|
4576
|
+
action: "skipped",
|
|
4577
|
+
reason: `failed: drop reported ${result.reason}`
|
|
4578
|
+
});
|
|
4579
|
+
continue;
|
|
4580
|
+
}
|
|
4581
|
+
retired += 1;
|
|
4582
|
+
entries.push({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
|
|
4583
|
+
if (result.warmRowsDeleted === 0 && result.generationsDeleted.length === 0) {
|
|
4584
|
+
await deps.registry.delete(ref).catch(() => void 0);
|
|
4585
|
+
continue;
|
|
4586
|
+
}
|
|
4587
|
+
await stampRetirement(deps.registry, ref, now).catch(() => void 0);
|
|
4588
|
+
} catch (err) {
|
|
4589
|
+
const after = await deps.registry.get(ref).catch(() => null);
|
|
4590
|
+
if (after?.status === "destroyed") {
|
|
4591
|
+
await stampRetirement(deps.registry, ref, now).catch(() => void 0);
|
|
4592
|
+
retired += 1;
|
|
4593
|
+
entries.push({
|
|
4594
|
+
...base,
|
|
4595
|
+
action: "retired",
|
|
4596
|
+
expiresAt: policy.expiresAt,
|
|
4597
|
+
result: {
|
|
4598
|
+
...base,
|
|
4599
|
+
dropped: true,
|
|
4600
|
+
warmRowsDeleted: 0,
|
|
4601
|
+
generationsDeleted: [],
|
|
4602
|
+
generationsRemaining: [],
|
|
4603
|
+
cryptoShredded: false,
|
|
4604
|
+
reason: void 0
|
|
4605
|
+
},
|
|
4606
|
+
fault: failureReason(err)
|
|
4607
|
+
});
|
|
4608
|
+
continue;
|
|
4609
|
+
}
|
|
4610
|
+
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4611
|
+
}
|
|
4612
|
+
}
|
|
4613
|
+
return {
|
|
4614
|
+
scanned: rows.length,
|
|
4615
|
+
eligible,
|
|
4616
|
+
retired,
|
|
4617
|
+
wouldRetire,
|
|
4618
|
+
tombstonesPurged,
|
|
4619
|
+
limited,
|
|
4620
|
+
dryRun,
|
|
4621
|
+
entries
|
|
4622
|
+
};
|
|
4623
|
+
}
|
|
4624
|
+
var RETIRED_AT = "retiredBySweepAt";
|
|
4625
|
+
function retirementStamp(meta) {
|
|
4626
|
+
if (meta === void 0 || meta === null || typeof meta !== "object" || Array.isArray(meta)) {
|
|
4627
|
+
return null;
|
|
4628
|
+
}
|
|
4629
|
+
const raw = meta[RETIRED_AT];
|
|
4630
|
+
return typeof raw === "number" && Number.isInteger(raw) && raw >= MIN_EXPIRES_AT_MS ? raw : null;
|
|
4631
|
+
}
|
|
4632
|
+
async function stampRetirement(registry, ref, now) {
|
|
4633
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4634
|
+
const rec = await registry.get(ref);
|
|
4635
|
+
if (rec === null || rec.status !== "destroyed") return;
|
|
4636
|
+
try {
|
|
4637
|
+
await registry.compareAndSwap(ref, rec.token, {
|
|
4638
|
+
retention: { ...rec.retention, [RETIRED_AT]: now }
|
|
4639
|
+
});
|
|
4640
|
+
return;
|
|
4641
|
+
} catch (err) {
|
|
4642
|
+
if (!isWriteConflictError(err)) throw err;
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
}
|
|
4646
|
+
async function isFullyReclaimed(deps, ref) {
|
|
4647
|
+
for await (const key of deps.cold.list(ref)) {
|
|
4648
|
+
return false;
|
|
4649
|
+
}
|
|
4650
|
+
for await (const row of deps.warm.listChunks(ref)) {
|
|
4651
|
+
return false;
|
|
4652
|
+
}
|
|
4653
|
+
return true;
|
|
4654
|
+
}
|
|
4655
|
+
function failureReason(err) {
|
|
4656
|
+
if (isWriteConflictError(err)) return "failed: contended";
|
|
4657
|
+
return `failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
4658
|
+
}
|
|
4659
|
+
|
|
4660
|
+
// src/core/consistency.ts
|
|
4281
4661
|
var DEFAULT_CHECK_CONCURRENCY = 8;
|
|
4282
4662
|
async function listGenerations2(cold, ref) {
|
|
4283
4663
|
const present = /* @__PURE__ */ new Set();
|
|
@@ -4290,26 +4670,18 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
4290
4670
|
throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
4291
4671
|
}
|
|
4292
4672
|
const maxScanSegments = options.maxScanSegments ?? DEFAULT_MAX_SCAN_SEGMENTS;
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
}
|
|
4298
|
-
const recs = [];
|
|
4299
|
-
for await (const rec of deps.registry.list(options.namespace)) {
|
|
4300
|
-
recs.push(rec);
|
|
4301
|
-
if (recs.length > maxScanSegments) {
|
|
4302
|
-
throw new BudgetExceededError(
|
|
4303
|
-
`checkConsistency would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Narrow it with \`namespace\`, or raise \`maxScanSegments\` if the fleet really is that large and the memory is available.`
|
|
4304
|
-
);
|
|
4305
|
-
}
|
|
4306
|
-
}
|
|
4673
|
+
const recs = await drainRegistry(deps.registry, {
|
|
4674
|
+
namespace: options.namespace,
|
|
4675
|
+
maxScanSegments,
|
|
4676
|
+
op: "checkConsistency"
|
|
4677
|
+
});
|
|
4307
4678
|
const results = await mapWithConcurrency(recs, concurrency, async (rec) => {
|
|
4308
4679
|
if (rec.status === "destroyed") return { kind: "ok" };
|
|
4309
4680
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
4310
4681
|
try {
|
|
4311
4682
|
const live = await deps.registry.get(ref);
|
|
4312
4683
|
if (!live || live.status === "destroyed") return { kind: "ok" };
|
|
4684
|
+
if (live.currentGen === null) return { kind: "ok" };
|
|
4313
4685
|
const present = await listGenerations2(deps.cold, ref);
|
|
4314
4686
|
if (present.has(live.currentGen)) return { kind: "ok" };
|
|
4315
4687
|
return {
|
|
@@ -4541,7 +4913,9 @@ exports.DEFAULT_MAX_SCAN_SEGMENTS = DEFAULT_MAX_SCAN_SEGMENTS;
|
|
|
4541
4913
|
exports.DEFAULT_MAX_WARM_SCAN_BYTES = DEFAULT_MAX_WARM_SCAN_BYTES;
|
|
4542
4914
|
exports.DEFAULT_OCC_BACKOFF = DEFAULT_OCC_BACKOFF;
|
|
4543
4915
|
exports.DEFAULT_PRICING = DEFAULT_PRICING;
|
|
4916
|
+
exports.DEFAULT_RETIRE_LIMIT = DEFAULT_RETIRE_LIMIT;
|
|
4544
4917
|
exports.DEFAULT_RETRY_POLICY = DEFAULT_RETRY_POLICY;
|
|
4918
|
+
exports.DEFAULT_TOMBSTONE_GRACE_MS = DEFAULT_TOMBSTONE_GRACE_MS;
|
|
4545
4919
|
exports.DEFAULT_WRITE_CONCURRENCY = DEFAULT_WRITE_CONCURRENCY;
|
|
4546
4920
|
exports.InProcessKeystore = InProcessKeystore;
|
|
4547
4921
|
exports.IntegrityError = IntegrityError;
|
|
@@ -4549,6 +4923,7 @@ exports.KeyUnavailableError = KeyUnavailableError;
|
|
|
4549
4923
|
exports.LocalFsColdDriver = LocalFsColdDriver;
|
|
4550
4924
|
exports.LocalFsRegistryDriver = LocalFsRegistryDriver;
|
|
4551
4925
|
exports.LocalFsWarmDriver = LocalFsWarmDriver;
|
|
4926
|
+
exports.MIN_EXPIRES_AT_MS = MIN_EXPIRES_AT_MS;
|
|
4552
4927
|
exports.MemoryColdChunkSource = MemoryColdChunkSource;
|
|
4553
4928
|
exports.MemoryColdDriver = MemoryColdDriver;
|
|
4554
4929
|
exports.MemoryRegistryDriver = MemoryRegistryDriver;
|
|
@@ -4573,14 +4948,17 @@ exports.aadFor = aadFor;
|
|
|
4573
4948
|
exports.bulkLoadCrbmGeneration = bulkLoadCrbmGeneration;
|
|
4574
4949
|
exports.checkBudget = checkBudget;
|
|
4575
4950
|
exports.chunkRefKey = chunkRefKey;
|
|
4951
|
+
exports.clearSegmentRetention = clearSegmentRetention;
|
|
4576
4952
|
exports.collectWithinBudget = collectWithinBudget;
|
|
4577
4953
|
exports.compactSegment = compactSegment;
|
|
4578
4954
|
exports.destroySegment = destroySegment;
|
|
4955
|
+
exports.drainRegistry = drainRegistry;
|
|
4579
4956
|
exports.dropSegment = dropSegment;
|
|
4580
4957
|
exports.eraseNamespace = eraseNamespace;
|
|
4581
4958
|
exports.estimateCost = estimateCost;
|
|
4582
4959
|
exports.findCompactable = findCompactable;
|
|
4583
4960
|
exports.gcOrphanGenerations = gcOrphanGenerations;
|
|
4961
|
+
exports.getSegmentRetention = getSegmentRetention;
|
|
4584
4962
|
exports.groundedReport = groundedReport;
|
|
4585
4963
|
exports.isCloudRoaringError = isCloudRoaringError;
|
|
4586
4964
|
exports.isIntegrityError = isIntegrityError;
|
|
@@ -4592,15 +4970,19 @@ exports.isWriteConflictError = isWriteConflictError;
|
|
|
4592
4970
|
exports.joinId = joinId;
|
|
4593
4971
|
exports.mapWithConcurrency = mapWithConcurrency;
|
|
4594
4972
|
exports.publishGeneration = publishGeneration;
|
|
4973
|
+
exports.readRetentionPolicy = readRetentionPolicy;
|
|
4595
4974
|
exports.resolveBudget = resolveBudget;
|
|
4596
4975
|
exports.resolvePerOpBudget = resolvePerOpBudget;
|
|
4976
|
+
exports.retireExpired = retireExpired;
|
|
4597
4977
|
exports.runCompactionCycle = runCompactionCycle;
|
|
4598
4978
|
exports.runConsistencyCheck = runConsistencyCheck;
|
|
4599
4979
|
exports.runExport = runExport;
|
|
4600
4980
|
exports.safeMetrics = safeMetrics;
|
|
4601
4981
|
exports.segmentKey = segmentKey;
|
|
4982
|
+
exports.setSegmentRetention = setSegmentRetention;
|
|
4602
4983
|
exports.splitId = splitId;
|
|
4603
4984
|
exports.validateCompactionOptions = validateCompactionOptions;
|
|
4985
|
+
exports.validateMaxScanSegments = validateMaxScanSegments;
|
|
4604
4986
|
exports.validateSegmentRef = validateSegmentRef;
|
|
4605
4987
|
exports.withRetry = withRetry;
|
|
4606
4988
|
exports.writeCrbmGeneration = writeCrbmGeneration;
|