@cloudbitmaps/core 0.8.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/azure/index.d.cts +1 -1
- package/dist/azure/index.d.ts +1 -1
- package/dist/cassandra/index.cjs.map +1 -1
- package/dist/cassandra/index.d.cts +1 -1
- package/dist/cassandra/index.d.ts +1 -1
- package/dist/cassandra/index.js +1 -1
- package/dist/{chunk-L54FL5DW.js → chunk-BD22WDAN.js} +34 -7
- package/dist/chunk-BD22WDAN.js.map +1 -0
- package/dist/{chunk-YEHMG6W2.js → chunk-OZ6LAFV4.js} +2 -2
- package/dist/chunk-OZ6LAFV4.js.map +1 -0
- package/dist/dynamodb/index.cjs +32 -5
- package/dist/dynamodb/index.cjs.map +1 -1
- package/dist/dynamodb/index.d.cts +1 -1
- package/dist/dynamodb/index.d.ts +1 -1
- package/dist/dynamodb/index.js +2 -2
- package/dist/gcs/index.d.cts +1 -1
- package/dist/gcs/index.d.ts +1 -1
- package/dist/index.cjs +417 -35
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +244 -18
- package/dist/index.d.ts +244 -18
- package/dist/index.js +379 -34
- package/dist/index.js.map +1 -1
- package/dist/mongodb/index.cjs.map +1 -1
- package/dist/mongodb/index.d.cts +1 -1
- package/dist/mongodb/index.d.ts +1 -1
- package/dist/mongodb/index.js +1 -1
- package/dist/mysql/index.cjs.map +1 -1
- package/dist/mysql/index.d.cts +1 -1
- package/dist/mysql/index.d.ts +1 -1
- package/dist/mysql/index.js +1 -1
- package/dist/{ports-5BQTGKze.d.cts → ports-CvcvJWf6.d.cts} +46 -7
- package/dist/{ports-5BQTGKze.d.ts → ports-CvcvJWf6.d.ts} +46 -7
- package/dist/postgres/index.cjs.map +1 -1
- package/dist/postgres/index.d.cts +1 -1
- package/dist/postgres/index.d.ts +1 -1
- package/dist/postgres/index.js +1 -1
- package/dist/redis/index.cjs.map +1 -1
- package/dist/redis/index.d.cts +1 -1
- package/dist/redis/index.d.ts +1 -1
- package/dist/redis/index.js +1 -1
- package/dist/s3/index.cjs +32 -5
- package/dist/s3/index.cjs.map +1 -1
- package/dist/s3/index.d.cts +1 -1
- package/dist/s3/index.d.ts +1 -1
- package/dist/s3/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-L54FL5DW.js.map +0 -1
- package/dist/chunk-YEHMG6W2.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { mapWithConcurrency } from './chunk-SNJVZ227.js';
|
|
2
2
|
export { mapWithConcurrency } from './chunk-SNJVZ227.js';
|
|
3
|
-
import { validateNewRegistryRecord, recordFromNew, validateRegistryPatch, applyRegistryPatch, registryCounterOf, parseRegistryEnvelope, serializeRegistryEnvelope } from './chunk-
|
|
4
|
-
import { NO_ROW } from './chunk-
|
|
5
|
-
export { NO_ROW } from './chunk-
|
|
3
|
+
import { validateNewRegistryRecord, recordFromNew, validateRegistryPatch, applyRegistryPatch, registryCounterOf, parseRegistryEnvelope, serializeRegistryEnvelope } from './chunk-BD22WDAN.js';
|
|
4
|
+
import { NO_ROW } from './chunk-OZ6LAFV4.js';
|
|
5
|
+
export { NO_ROW } from './chunk-OZ6LAFV4.js';
|
|
6
6
|
import { ValidationError, BudgetExceededError, isTransientError, isWriteConflictError, WriteConflictError, IntegrityError, UnsupportedError, CapabilityError, KeyUnavailableError, validateChunkRef, validateSegmentRef, isNotFoundError, NotFoundError, TransientError, namespacePart } from './chunk-NUIDEEFZ.js';
|
|
7
7
|
export { BudgetExceededError, CapabilityError, CloudRoaringError, IntegrityError, KeyUnavailableError, NotFoundError, TimeoutError, TransientError, UnsupportedError, ValidationError, WriteConflictError, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransientError, isValidationError, isWriteConflictError, validateSegmentRef } from './chunk-NUIDEEFZ.js';
|
|
8
8
|
import { createHash, randomUUID, randomBytes, createCipheriv, createDecipheriv } from 'crypto';
|
|
@@ -2043,6 +2043,7 @@ var CrbmColdChunkSource = class {
|
|
|
2043
2043
|
const record = await this.registry.get(ref);
|
|
2044
2044
|
if (record === null) return null;
|
|
2045
2045
|
if (record.status === "destroyed") return null;
|
|
2046
|
+
if (record.currentGen === null) return null;
|
|
2046
2047
|
return { generation: record.currentGen, wrappedDeks: record.wrappedDeks };
|
|
2047
2048
|
}
|
|
2048
2049
|
let maxGen = -1;
|
|
@@ -2166,6 +2167,11 @@ async function publishGeneration(registry, key, options = {}) {
|
|
|
2166
2167
|
throw new ValidationError(
|
|
2167
2168
|
`segment "${key.segment}" was destroyed (crypto-shredded) while generation ${key.generation} was being written \u2014 refusing to publish it; the written object is unreadable. Use a new segment.`
|
|
2168
2169
|
);
|
|
2170
|
+
} else if (record.currentGen === null) {
|
|
2171
|
+
await registry.compareAndSwap(key, record.token, {
|
|
2172
|
+
currentGen: key.generation,
|
|
2173
|
+
...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks }
|
|
2174
|
+
});
|
|
2169
2175
|
} else if (record.currentGen > key.generation) {
|
|
2170
2176
|
return false;
|
|
2171
2177
|
} else if (record.currentGen === key.generation) {
|
|
@@ -2180,7 +2186,7 @@ async function publishGeneration(registry, key, options = {}) {
|
|
|
2180
2186
|
}
|
|
2181
2187
|
}
|
|
2182
2188
|
const final = await registry.get(key);
|
|
2183
|
-
if (final !== null && final.currentGen >= key.generation)
|
|
2189
|
+
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation)
|
|
2184
2190
|
return final.currentGen === key.generation;
|
|
2185
2191
|
throw new WriteConflictError(
|
|
2186
2192
|
`publishGeneration: contention setting currentGen for ${key.segment}`
|
|
@@ -2249,6 +2255,11 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
2249
2255
|
`segment "${key.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
2250
2256
|
);
|
|
2251
2257
|
}
|
|
2258
|
+
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0 && options.keystore === void 0) {
|
|
2259
|
+
throw new KeyUnavailableError(
|
|
2260
|
+
`segment "${key.segment}" is encrypted but this bulk-load has no keystore \u2014 refusing to write a cleartext generation onto an encrypted segment. Pass the keystore holding its DEK.`
|
|
2261
|
+
);
|
|
2262
|
+
}
|
|
2252
2263
|
let crypto;
|
|
2253
2264
|
let newWrapped;
|
|
2254
2265
|
if (options.keystore !== void 0) {
|
|
@@ -2351,7 +2362,7 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2351
2362
|
survived: 0
|
|
2352
2363
|
};
|
|
2353
2364
|
}
|
|
2354
|
-
if (record === null) {
|
|
2365
|
+
if (record === null || record.currentGen === null) {
|
|
2355
2366
|
const pinned = await collectDirty(
|
|
2356
2367
|
deps.warm,
|
|
2357
2368
|
ref,
|
|
@@ -2370,7 +2381,15 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2370
2381
|
};
|
|
2371
2382
|
let aead;
|
|
2372
2383
|
let wrappedDeks;
|
|
2373
|
-
if (
|
|
2384
|
+
if (record?.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
2385
|
+
if (deps.keystore === void 0) {
|
|
2386
|
+
throw new KeyUnavailableError(
|
|
2387
|
+
`segment "${ref.segment}" is encrypted but compaction has no keystore`
|
|
2388
|
+
);
|
|
2389
|
+
}
|
|
2390
|
+
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
2391
|
+
wrappedDeks = record.wrappedDeks;
|
|
2392
|
+
} else if (deps.keystore !== void 0) {
|
|
2374
2393
|
const minted = await deps.keystore.createDek();
|
|
2375
2394
|
aead = minted.aead;
|
|
2376
2395
|
wrappedDeks = minted.wrapped;
|
|
@@ -2396,17 +2415,8 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2396
2415
|
if (wrote && tally !== void 0) {
|
|
2397
2416
|
await verifyGeneration(deps.cold, gen0, tally, cryptoAt(aead, 0));
|
|
2398
2417
|
}
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
currentGen: 0,
|
|
2402
|
-
dirtyChunkCount: 0,
|
|
2403
|
-
status: "active",
|
|
2404
|
-
wrappedDeks
|
|
2405
|
-
});
|
|
2406
|
-
} catch (err) {
|
|
2407
|
-
if (!isWriteConflictError(err)) throw err;
|
|
2408
|
-
}
|
|
2409
|
-
if (!wrote)
|
|
2418
|
+
const published = await publishGenZero(deps.registry, ref, record, wrappedDeks, wrote);
|
|
2419
|
+
if (!wrote || !published)
|
|
2410
2420
|
return {
|
|
2411
2421
|
...base,
|
|
2412
2422
|
compacted: false,
|
|
@@ -2570,6 +2580,43 @@ async function compactSegmentInner(ref, deps, options) {
|
|
|
2570
2580
|
throw err;
|
|
2571
2581
|
}
|
|
2572
2582
|
}
|
|
2583
|
+
var PUBLISH_GEN0_ATTEMPTS = 5;
|
|
2584
|
+
async function publishGenZero(registry, ref, known, wrappedDeks, wrote) {
|
|
2585
|
+
if (!wrote && wrappedDeks !== void 0) return false;
|
|
2586
|
+
let current = known;
|
|
2587
|
+
for (let attempt = 0; attempt < PUBLISH_GEN0_ATTEMPTS; attempt += 1) {
|
|
2588
|
+
try {
|
|
2589
|
+
if (current === null) {
|
|
2590
|
+
await registry.create(ref, {
|
|
2591
|
+
currentGen: 0,
|
|
2592
|
+
dirtyChunkCount: 0,
|
|
2593
|
+
status: "active",
|
|
2594
|
+
wrappedDeks
|
|
2595
|
+
});
|
|
2596
|
+
} else if (current.status === "destroyed") {
|
|
2597
|
+
return false;
|
|
2598
|
+
} else if (current.currentGen === null) {
|
|
2599
|
+
await registry.compareAndSwap(ref, current.token, {
|
|
2600
|
+
currentGen: 0,
|
|
2601
|
+
dirtyChunkCount: 0,
|
|
2602
|
+
...wrappedDeks === void 0 ? {} : { wrappedDeks }
|
|
2603
|
+
});
|
|
2604
|
+
} else {
|
|
2605
|
+
return current.currentGen === 0 && sameWrappings(current.wrappedDeks, wrappedDeks);
|
|
2606
|
+
}
|
|
2607
|
+
return true;
|
|
2608
|
+
} catch (err) {
|
|
2609
|
+
if (!isWriteConflictError(err)) throw err;
|
|
2610
|
+
current = await registry.get(ref);
|
|
2611
|
+
}
|
|
2612
|
+
}
|
|
2613
|
+
return false;
|
|
2614
|
+
}
|
|
2615
|
+
function sameWrappings(a, b) {
|
|
2616
|
+
if (a === void 0 || b === void 0) return a === b;
|
|
2617
|
+
if (a.length !== b.length) return false;
|
|
2618
|
+
return a.every((w, i) => w.keyId === b[i]?.keyId && w.wrapped === b[i]?.wrapped);
|
|
2619
|
+
}
|
|
2573
2620
|
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
2574
2621
|
const keep = Math.max(0, options.keep ?? 1);
|
|
2575
2622
|
const record = await deps.registry.get(ref);
|
|
@@ -2577,7 +2624,12 @@ async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
|
2577
2624
|
const current = record.currentGen;
|
|
2578
2625
|
const gens = [];
|
|
2579
2626
|
for await (const key of deps.cold.list(ref)) gens.push(key.generation);
|
|
2580
|
-
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : (
|
|
2627
|
+
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : current === null ? (
|
|
2628
|
+
// No Cold pointer yet, so "below current" selects nothing and there is nothing safe to infer: an
|
|
2629
|
+
// object here is either a bootstrap about to publish gen 0 or an orphan we cannot tell apart from it.
|
|
2630
|
+
// Deleting would race that publish into a dangling pointer. RECONCILE collects it on the next cycle.
|
|
2631
|
+
[]
|
|
2632
|
+
) : (
|
|
2581
2633
|
// Delete generations below current, except the newest `keep` of them (the grace window).
|
|
2582
2634
|
gens.filter((g) => g < current).sort((a, b) => b - a).slice(keep)
|
|
2583
2635
|
);
|
|
@@ -3960,8 +4012,309 @@ async function eraseWarm(warm, ref, op = "destroySegment") {
|
|
|
3960
4012
|
return deleted;
|
|
3961
4013
|
}
|
|
3962
4014
|
|
|
3963
|
-
// src/core/
|
|
4015
|
+
// src/core/retention.ts
|
|
4016
|
+
var EXPIRES_AT = "expiresAt";
|
|
4017
|
+
var RETENTION_CAS_ATTEMPTS = 5;
|
|
4018
|
+
var MIN_EXPIRES_AT_MS = 1e12;
|
|
4019
|
+
function validateRetentionPolicy(policy) {
|
|
4020
|
+
const { expiresAt } = policy;
|
|
4021
|
+
if (typeof expiresAt !== "number" || !Number.isInteger(expiresAt)) {
|
|
4022
|
+
throw new ValidationError(
|
|
4023
|
+
`retention.expiresAt must be an integer epoch-ms; got ${String(expiresAt)}`
|
|
4024
|
+
);
|
|
4025
|
+
}
|
|
4026
|
+
if (expiresAt < MIN_EXPIRES_AT_MS) {
|
|
4027
|
+
throw new ValidationError(
|
|
4028
|
+
`retention.expiresAt (${expiresAt}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds, which would read as "already expired" and retire the segment on the next sweep. Use \`Date.now() + days * 86_400_000\`. To retire a segment now, call \`dropSegment\`.`
|
|
4029
|
+
);
|
|
4030
|
+
}
|
|
4031
|
+
}
|
|
4032
|
+
function readRetentionPolicy(meta) {
|
|
4033
|
+
if (meta === void 0) return null;
|
|
4034
|
+
if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "invalid";
|
|
4035
|
+
if (!(EXPIRES_AT in meta)) return null;
|
|
4036
|
+
const raw = meta[EXPIRES_AT];
|
|
4037
|
+
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < MIN_EXPIRES_AT_MS)
|
|
4038
|
+
return "invalid";
|
|
4039
|
+
return { expiresAt: raw };
|
|
4040
|
+
}
|
|
4041
|
+
async function getSegmentRetention(ref, deps) {
|
|
4042
|
+
const record = await deps.registry.get(ref);
|
|
4043
|
+
if (record === null || record.status === "destroyed") return null;
|
|
4044
|
+
return readRetentionPolicy(record.retention);
|
|
4045
|
+
}
|
|
4046
|
+
async function setSegmentRetention(ref, deps, policy) {
|
|
4047
|
+
validateRetentionPolicy(policy);
|
|
4048
|
+
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
4049
|
+
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
4050
|
+
const record = await deps.registry.get(ref);
|
|
4051
|
+
try {
|
|
4052
|
+
if (record === null) {
|
|
4053
|
+
await deps.registry.create(ref, {
|
|
4054
|
+
currentGen: null,
|
|
4055
|
+
retention: { [EXPIRES_AT]: policy.expiresAt }
|
|
4056
|
+
});
|
|
4057
|
+
return { ...base, expiresAt: policy.expiresAt, createdRow: true };
|
|
4058
|
+
}
|
|
4059
|
+
assertNotDestroyed(record, ref);
|
|
4060
|
+
await deps.registry.compareAndSwap(ref, record.token, {
|
|
4061
|
+
retention: { ...record.retention, [EXPIRES_AT]: policy.expiresAt }
|
|
4062
|
+
});
|
|
4063
|
+
return { ...base, expiresAt: policy.expiresAt, createdRow: false };
|
|
4064
|
+
} catch (err) {
|
|
4065
|
+
if (!isWriteConflictError(err)) throw err;
|
|
4066
|
+
}
|
|
4067
|
+
}
|
|
4068
|
+
throw new WriteConflictError(
|
|
4069
|
+
`setRetention: contention writing the retention policy for segment "${ref.segment}"`
|
|
4070
|
+
);
|
|
4071
|
+
}
|
|
4072
|
+
async function clearSegmentRetention(ref, deps) {
|
|
4073
|
+
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
4074
|
+
const record = await deps.registry.get(ref);
|
|
4075
|
+
if (record === null) return false;
|
|
4076
|
+
if (record.status === "destroyed") return false;
|
|
4077
|
+
if (record.retention === void 0 || !(EXPIRES_AT in record.retention)) return false;
|
|
4078
|
+
const rest = { ...record.retention };
|
|
4079
|
+
delete rest[EXPIRES_AT];
|
|
4080
|
+
try {
|
|
4081
|
+
await deps.registry.compareAndSwap(ref, record.token, {
|
|
4082
|
+
retention: Object.keys(rest).length === 0 ? void 0 : rest
|
|
4083
|
+
});
|
|
4084
|
+
return true;
|
|
4085
|
+
} catch (err) {
|
|
4086
|
+
if (!isWriteConflictError(err)) throw err;
|
|
4087
|
+
}
|
|
4088
|
+
}
|
|
4089
|
+
throw new WriteConflictError(
|
|
4090
|
+
`clearRetention: contention clearing the retention policy for segment "${ref.segment}"`
|
|
4091
|
+
);
|
|
4092
|
+
}
|
|
4093
|
+
function assertNotDestroyed(record, ref) {
|
|
4094
|
+
if (record.status === "destroyed") {
|
|
4095
|
+
throw new ValidationError(
|
|
4096
|
+
`segment "${ref.segment}" is destroyed (crypto-shredded) \u2014 refusing to set a retention policy on a tombstone; there is nothing left to retire. Use a new segment.`
|
|
4097
|
+
);
|
|
4098
|
+
}
|
|
4099
|
+
}
|
|
4100
|
+
|
|
4101
|
+
// src/core/registry-scan.ts
|
|
3964
4102
|
var DEFAULT_MAX_SCAN_SEGMENTS = 25e4;
|
|
4103
|
+
function validateMaxScanSegments(value, op) {
|
|
4104
|
+
if (!Number.isFinite(value) || value < 1) {
|
|
4105
|
+
throw new ValidationError(`${op}: maxScanSegments must be a finite number >= 1; got ${value}`);
|
|
4106
|
+
}
|
|
4107
|
+
}
|
|
4108
|
+
async function drainRegistry(registry, options) {
|
|
4109
|
+
const { maxScanSegments, op } = options;
|
|
4110
|
+
validateMaxScanSegments(maxScanSegments, op);
|
|
4111
|
+
const rows = [];
|
|
4112
|
+
for await (const rec of registry.list(options.namespace)) {
|
|
4113
|
+
if (rows.length >= maxScanSegments) {
|
|
4114
|
+
throw new BudgetExceededError(
|
|
4115
|
+
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Narrow it with \`namespace\`, or raise \`maxScanSegments\` if the fleet really is that large and the memory is available (a record is a few hundred bytes resident).`
|
|
4116
|
+
);
|
|
4117
|
+
}
|
|
4118
|
+
rows.push(rec);
|
|
4119
|
+
}
|
|
4120
|
+
return rows;
|
|
4121
|
+
}
|
|
4122
|
+
|
|
4123
|
+
// src/core/retention-sweep.ts
|
|
4124
|
+
var DEFAULT_RETIRE_LIMIT = 100;
|
|
4125
|
+
var DEFAULT_TOMBSTONE_GRACE_MS = 864e5;
|
|
4126
|
+
async function retireExpired(deps, options) {
|
|
4127
|
+
const now = options.now;
|
|
4128
|
+
if (!Number.isFinite(now)) {
|
|
4129
|
+
throw new ValidationError(
|
|
4130
|
+
`retireExpired: \`now\` must be a finite epoch-ms; got ${String(now)}`
|
|
4131
|
+
);
|
|
4132
|
+
}
|
|
4133
|
+
if (now < MIN_EXPIRES_AT_MS) {
|
|
4134
|
+
throw new ValidationError(
|
|
4135
|
+
`retireExpired: \`now\` (${now}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds. A sweep with a wrong-units clock treats every policy as expired.`
|
|
4136
|
+
);
|
|
4137
|
+
}
|
|
4138
|
+
const limit = options.limit ?? DEFAULT_RETIRE_LIMIT;
|
|
4139
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
4140
|
+
throw new ValidationError(`retireExpired: \`limit\` must be a positive integer; got ${limit}`);
|
|
4141
|
+
}
|
|
4142
|
+
const purgeTombstones = options.purgeTombstones ?? true;
|
|
4143
|
+
const grace = options.tombstoneGraceMs ?? DEFAULT_TOMBSTONE_GRACE_MS;
|
|
4144
|
+
if (!Number.isFinite(grace) || grace < 0) {
|
|
4145
|
+
throw new ValidationError(
|
|
4146
|
+
`retireExpired: \`tombstoneGraceMs\` must be a non-negative finite number; got ${String(grace)}`
|
|
4147
|
+
);
|
|
4148
|
+
}
|
|
4149
|
+
const maxScanSegments = options.maxScanSegments ?? DEFAULT_MAX_SCAN_SEGMENTS;
|
|
4150
|
+
if (!Number.isFinite(maxScanSegments) || maxScanSegments < 1) {
|
|
4151
|
+
throw new ValidationError(
|
|
4152
|
+
`retireExpired: \`maxScanSegments\` must be a finite number >= 1; got ${String(maxScanSegments)}`
|
|
4153
|
+
);
|
|
4154
|
+
}
|
|
4155
|
+
const dryRun = options.dryRun === true;
|
|
4156
|
+
const rows = await drainRegistry(deps.registry, {
|
|
4157
|
+
namespace: options.namespace,
|
|
4158
|
+
maxScanSegments,
|
|
4159
|
+
op: "retireExpired"
|
|
4160
|
+
});
|
|
4161
|
+
const entries = [];
|
|
4162
|
+
let eligible = 0;
|
|
4163
|
+
let retired = 0;
|
|
4164
|
+
let wouldRetire = 0;
|
|
4165
|
+
let tombstonesPurged = 0;
|
|
4166
|
+
let limited = false;
|
|
4167
|
+
let attempted = 0;
|
|
4168
|
+
for (const rec of rows) {
|
|
4169
|
+
const ref = { namespace: rec.namespace, segment: rec.segment };
|
|
4170
|
+
const base = { segment: rec.segment, namespace: rec.namespace };
|
|
4171
|
+
const policy = readRetentionPolicy(rec.retention);
|
|
4172
|
+
if (rec.status === "destroyed") {
|
|
4173
|
+
if (!purgeTombstones) continue;
|
|
4174
|
+
const retiredAt = retirementStamp(rec.retention);
|
|
4175
|
+
if (retiredAt === null) continue;
|
|
4176
|
+
if (now - retiredAt < grace) continue;
|
|
4177
|
+
if (attempted >= limit) {
|
|
4178
|
+
limited = true;
|
|
4179
|
+
break;
|
|
4180
|
+
}
|
|
4181
|
+
attempted += 1;
|
|
4182
|
+
try {
|
|
4183
|
+
if (!await isFullyReclaimed(deps, ref)) {
|
|
4184
|
+
if (!dryRun) await gcOrphanGenerations(ref, deps).catch(() => void 0);
|
|
4185
|
+
if (!await isFullyReclaimed(deps, ref)) {
|
|
4186
|
+
entries.push({ ...base, action: "skipped", reason: "tombstone-not-empty" });
|
|
4187
|
+
continue;
|
|
4188
|
+
}
|
|
4189
|
+
}
|
|
4190
|
+
if (!dryRun) await deps.registry.delete(ref);
|
|
4191
|
+
if (!dryRun) tombstonesPurged += 1;
|
|
4192
|
+
entries.push({
|
|
4193
|
+
...base,
|
|
4194
|
+
action: dryRun ? "would-purge-tombstone" : "purged-tombstone"
|
|
4195
|
+
});
|
|
4196
|
+
} catch (err) {
|
|
4197
|
+
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4198
|
+
}
|
|
4199
|
+
continue;
|
|
4200
|
+
}
|
|
4201
|
+
if (policy === null) continue;
|
|
4202
|
+
if (policy === "invalid") {
|
|
4203
|
+
entries.push({ ...base, action: "skipped", reason: "invalid-policy" });
|
|
4204
|
+
continue;
|
|
4205
|
+
}
|
|
4206
|
+
if (policy.expiresAt > now) continue;
|
|
4207
|
+
eligible += 1;
|
|
4208
|
+
if (attempted >= limit) {
|
|
4209
|
+
limited = true;
|
|
4210
|
+
break;
|
|
4211
|
+
}
|
|
4212
|
+
try {
|
|
4213
|
+
const live = await deps.registry.get(ref);
|
|
4214
|
+
const livePolicy = live === null ? null : readRetentionPolicy(live.retention);
|
|
4215
|
+
if (live === null || live.status === "destroyed" || livePolicy === null || livePolicy === "invalid" || livePolicy.expiresAt > now) {
|
|
4216
|
+
entries.push({ ...base, action: "skipped", reason: "policy-changed" });
|
|
4217
|
+
continue;
|
|
4218
|
+
}
|
|
4219
|
+
attempted += 1;
|
|
4220
|
+
const result = await dropSegment(ref, deps, {
|
|
4221
|
+
confirmSegment: rec.segment,
|
|
4222
|
+
dryRun,
|
|
4223
|
+
audit: options.audit
|
|
4224
|
+
});
|
|
4225
|
+
if (dryRun) {
|
|
4226
|
+
wouldRetire += 1;
|
|
4227
|
+
entries.push({ ...base, action: "would-retire", expiresAt: livePolicy.expiresAt, result });
|
|
4228
|
+
continue;
|
|
4229
|
+
}
|
|
4230
|
+
if (!result.dropped) {
|
|
4231
|
+
entries.push({
|
|
4232
|
+
...base,
|
|
4233
|
+
action: "skipped",
|
|
4234
|
+
reason: `failed: drop reported ${result.reason}`
|
|
4235
|
+
});
|
|
4236
|
+
continue;
|
|
4237
|
+
}
|
|
4238
|
+
retired += 1;
|
|
4239
|
+
entries.push({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
|
|
4240
|
+
if (result.warmRowsDeleted === 0 && result.generationsDeleted.length === 0) {
|
|
4241
|
+
await deps.registry.delete(ref).catch(() => void 0);
|
|
4242
|
+
continue;
|
|
4243
|
+
}
|
|
4244
|
+
await stampRetirement(deps.registry, ref, now).catch(() => void 0);
|
|
4245
|
+
} catch (err) {
|
|
4246
|
+
const after = await deps.registry.get(ref).catch(() => null);
|
|
4247
|
+
if (after?.status === "destroyed") {
|
|
4248
|
+
await stampRetirement(deps.registry, ref, now).catch(() => void 0);
|
|
4249
|
+
retired += 1;
|
|
4250
|
+
entries.push({
|
|
4251
|
+
...base,
|
|
4252
|
+
action: "retired",
|
|
4253
|
+
expiresAt: policy.expiresAt,
|
|
4254
|
+
result: {
|
|
4255
|
+
...base,
|
|
4256
|
+
dropped: true,
|
|
4257
|
+
warmRowsDeleted: 0,
|
|
4258
|
+
generationsDeleted: [],
|
|
4259
|
+
generationsRemaining: [],
|
|
4260
|
+
cryptoShredded: false,
|
|
4261
|
+
reason: void 0
|
|
4262
|
+
},
|
|
4263
|
+
fault: failureReason(err)
|
|
4264
|
+
});
|
|
4265
|
+
continue;
|
|
4266
|
+
}
|
|
4267
|
+
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4268
|
+
}
|
|
4269
|
+
}
|
|
4270
|
+
return {
|
|
4271
|
+
scanned: rows.length,
|
|
4272
|
+
eligible,
|
|
4273
|
+
retired,
|
|
4274
|
+
wouldRetire,
|
|
4275
|
+
tombstonesPurged,
|
|
4276
|
+
limited,
|
|
4277
|
+
dryRun,
|
|
4278
|
+
entries
|
|
4279
|
+
};
|
|
4280
|
+
}
|
|
4281
|
+
var RETIRED_AT = "retiredBySweepAt";
|
|
4282
|
+
function retirementStamp(meta) {
|
|
4283
|
+
if (meta === void 0 || meta === null || typeof meta !== "object" || Array.isArray(meta)) {
|
|
4284
|
+
return null;
|
|
4285
|
+
}
|
|
4286
|
+
const raw = meta[RETIRED_AT];
|
|
4287
|
+
return typeof raw === "number" && Number.isInteger(raw) && raw >= MIN_EXPIRES_AT_MS ? raw : null;
|
|
4288
|
+
}
|
|
4289
|
+
async function stampRetirement(registry, ref, now) {
|
|
4290
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4291
|
+
const rec = await registry.get(ref);
|
|
4292
|
+
if (rec === null || rec.status !== "destroyed") return;
|
|
4293
|
+
try {
|
|
4294
|
+
await registry.compareAndSwap(ref, rec.token, {
|
|
4295
|
+
retention: { ...rec.retention, [RETIRED_AT]: now }
|
|
4296
|
+
});
|
|
4297
|
+
return;
|
|
4298
|
+
} catch (err) {
|
|
4299
|
+
if (!isWriteConflictError(err)) throw err;
|
|
4300
|
+
}
|
|
4301
|
+
}
|
|
4302
|
+
}
|
|
4303
|
+
async function isFullyReclaimed(deps, ref) {
|
|
4304
|
+
for await (const key of deps.cold.list(ref)) {
|
|
4305
|
+
return false;
|
|
4306
|
+
}
|
|
4307
|
+
for await (const row of deps.warm.listChunks(ref)) {
|
|
4308
|
+
return false;
|
|
4309
|
+
}
|
|
4310
|
+
return true;
|
|
4311
|
+
}
|
|
4312
|
+
function failureReason(err) {
|
|
4313
|
+
if (isWriteConflictError(err)) return "failed: contended";
|
|
4314
|
+
return `failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
4315
|
+
}
|
|
4316
|
+
|
|
4317
|
+
// src/core/consistency.ts
|
|
3965
4318
|
var DEFAULT_CHECK_CONCURRENCY = 8;
|
|
3966
4319
|
async function listGenerations2(cold, ref) {
|
|
3967
4320
|
const present = /* @__PURE__ */ new Set();
|
|
@@ -3974,26 +4327,18 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
3974
4327
|
throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
3975
4328
|
}
|
|
3976
4329
|
const maxScanSegments = options.maxScanSegments ?? DEFAULT_MAX_SCAN_SEGMENTS;
|
|
3977
|
-
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
}
|
|
3982
|
-
const recs = [];
|
|
3983
|
-
for await (const rec of deps.registry.list(options.namespace)) {
|
|
3984
|
-
recs.push(rec);
|
|
3985
|
-
if (recs.length > maxScanSegments) {
|
|
3986
|
-
throw new BudgetExceededError(
|
|
3987
|
-
`checkConsistency would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Narrow it with \`namespace\`, or raise \`maxScanSegments\` if the fleet really is that large and the memory is available.`
|
|
3988
|
-
);
|
|
3989
|
-
}
|
|
3990
|
-
}
|
|
4330
|
+
const recs = await drainRegistry(deps.registry, {
|
|
4331
|
+
namespace: options.namespace,
|
|
4332
|
+
maxScanSegments,
|
|
4333
|
+
op: "checkConsistency"
|
|
4334
|
+
});
|
|
3991
4335
|
const results = await mapWithConcurrency(recs, concurrency, async (rec) => {
|
|
3992
4336
|
if (rec.status === "destroyed") return { kind: "ok" };
|
|
3993
4337
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
3994
4338
|
try {
|
|
3995
4339
|
const live = await deps.registry.get(ref);
|
|
3996
4340
|
if (!live || live.status === "destroyed") return { kind: "ok" };
|
|
4341
|
+
if (live.currentGen === null) return { kind: "ok" };
|
|
3997
4342
|
const present = await listGenerations2(deps.cold, ref);
|
|
3998
4343
|
if (present.has(live.currentGen)) return { kind: "ok" };
|
|
3999
4344
|
return {
|
|
@@ -4209,6 +4554,6 @@ var RetryingRegistryDriver = class {
|
|
|
4209
4554
|
}
|
|
4210
4555
|
};
|
|
4211
4556
|
|
|
4212
|
-
export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isTransient, joinId, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, withRetry, writeCrbmGeneration };
|
|
4557
|
+
export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETIRE_LIMIT, DEFAULT_RETRY_POLICY, DEFAULT_TOMBSTONE_GRACE_MS, DEFAULT_WRITE_CONCURRENCY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MIN_EXPIRES_AT_MS, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, clearSegmentRetention, collectWithinBudget, compactSegment, destroySegment, drainRegistry, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, getSegmentRetention, groundedReport, isTransient, joinId, publishGeneration, readRetentionPolicy, resolveBudget, resolvePerOpBudget, retireExpired, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, setSegmentRetention, splitId, validateCompactionOptions, validateMaxScanSegments, withRetry, writeCrbmGeneration };
|
|
4213
4558
|
//# sourceMappingURL=index.js.map
|
|
4214
4559
|
//# sourceMappingURL=index.js.map
|