@cloudbitmaps/core 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-5OMC5U7U.js → chunk-5W3XS2IU.js} +292 -22
- package/dist/chunk-5W3XS2IU.js.map +7 -0
- package/dist/core/audit.d.ts +6 -0
- package/dist/core/codec.d.ts +33 -1
- package/dist/core/consistency.d.ts +21 -2
- package/dist/core/cost.d.ts +91 -10
- package/dist/core/crbm/format.d.ts +31 -1
- package/dist/core/crbm/reader.d.ts +44 -2
- package/dist/core/crbm/writer.d.ts +32 -3
- package/dist/core/crbm-storage-source.d.ts +203 -22
- package/dist/core/crypto.d.ts +11 -7
- package/dist/core/determinism.d.ts +7 -0
- package/dist/core/engine.d.ts +17 -5
- package/dist/core/erase-id.d.ts +20 -0
- package/dist/core/erasure.d.ts +8 -4
- package/dist/core/generation-gc.d.ts +102 -4
- package/dist/core/load-input.d.ts +55 -0
- package/dist/core/load.d.ts +65 -10
- package/dist/core/metadata.d.ts +42 -0
- package/dist/core/pinned-storage-source.d.ts +2 -1
- package/dist/core/ports.d.ts +119 -13
- package/dist/core/registry-scan.d.ts +6 -0
- package/dist/core/retention-sweep.d.ts +47 -2
- package/dist/core/retry.d.ts +14 -0
- package/dist/core/rollback.d.ts +34 -4
- package/dist/core/summary.d.ts +61 -0
- package/dist/core/token.d.ts +36 -0
- package/dist/core/validate.d.ts +2 -0
- package/dist/driver-kit.js +39 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/entropy.d.ts +5 -0
- package/dist/drivers/_shared/object-registry.d.ts +36 -6
- package/dist/drivers/_shared/registry.d.ts +47 -19
- package/dist/drivers/localfs/registry.d.ts +10 -0
- package/dist/drivers/memory.d.ts +9 -3
- package/dist/drivers/retry/retrying-chunk-source.d.ts +2 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +1794 -461
- package/dist/index.js.map +4 -4
- package/dist/testing/fuzz-core.d.ts +6 -5
- package/package.json +1 -1
- package/dist/chunk-5OMC5U7U.js.map +0 -7
package/dist/driver-kit.js
CHANGED
|
@@ -11,20 +11,24 @@ import {
|
|
|
11
11
|
decodeNameFromKey,
|
|
12
12
|
encodeNameForKey,
|
|
13
13
|
encodeNameForPath,
|
|
14
|
+
entropyIsAvailable,
|
|
15
|
+
incarnationOf,
|
|
14
16
|
isNotFoundError,
|
|
15
17
|
isValidationError,
|
|
16
18
|
isWriteConflictError,
|
|
17
19
|
mapWithConcurrency,
|
|
18
20
|
namespaceKeyPart,
|
|
19
21
|
namespacePathPart,
|
|
22
|
+
newIncarnationToken,
|
|
23
|
+
nextRegistryToken,
|
|
20
24
|
parseRegistryEnvelope,
|
|
21
25
|
recordFromNew,
|
|
22
|
-
registryCounterOf,
|
|
23
26
|
serializeRegistryEnvelope,
|
|
24
27
|
validateNewRegistryRecord,
|
|
25
28
|
validateRegistryPatch,
|
|
26
|
-
validateSegmentRef
|
|
27
|
-
|
|
29
|
+
validateSegmentRef,
|
|
30
|
+
webCryptoEntropy
|
|
31
|
+
} from "./chunk-5W3XS2IU.js";
|
|
28
32
|
|
|
29
33
|
// src/drivers/_shared/object-registry-keys.ts
|
|
30
34
|
var REGISTRY_SUFFIX = ".reg";
|
|
@@ -108,47 +112,59 @@ var ObjectVersionRaced = class extends Error {
|
|
|
108
112
|
}
|
|
109
113
|
};
|
|
110
114
|
var ObjectStoreRegistry = class {
|
|
111
|
-
|
|
115
|
+
/**
|
|
116
|
+
* `entropy` draws the random parts of every token: a new row's incarnation id, and each write's own part. It
|
|
117
|
+
* defaults to the platform's Web Crypto. Inject one only to make a test replayable: a seeded source in production
|
|
118
|
+
* gives every process the same ids.
|
|
119
|
+
*/
|
|
120
|
+
constructor(store, prefix, now, entropy = webCryptoEntropy) {
|
|
112
121
|
this.store = store;
|
|
113
122
|
this.prefix = prefix;
|
|
114
123
|
this.now = now;
|
|
124
|
+
this.entropy = entropy;
|
|
115
125
|
}
|
|
116
126
|
store;
|
|
117
127
|
prefix;
|
|
118
128
|
now;
|
|
129
|
+
entropy;
|
|
119
130
|
capabilities() {
|
|
120
|
-
|
|
131
|
+
const conditionalDelete = this.removesRows();
|
|
132
|
+
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete } : { strongRead: true, canWrite: false, conditionalDelete };
|
|
133
|
+
}
|
|
134
|
+
/** Whether a delete removes a row born with an incarnation id: the store has a delete, and vouches for it. */
|
|
135
|
+
removesRows() {
|
|
136
|
+
return this.store.conditionalDelete === true && typeof this.store.delete === "function";
|
|
121
137
|
}
|
|
122
138
|
async get(ref) {
|
|
123
139
|
const current = await this.readRow(registryObjectKey(this.prefix, ref));
|
|
124
140
|
return current && !current.env.deleted ? current.env.record : null;
|
|
125
141
|
}
|
|
126
142
|
async create(ref, record) {
|
|
127
|
-
validateNewRegistryRecord(record);
|
|
143
|
+
const checked = validateNewRegistryRecord(record);
|
|
128
144
|
const key = registryObjectKey(this.prefix, ref);
|
|
129
145
|
const current = await this.readRow(key);
|
|
130
146
|
if (current !== null && !current.env.deleted) {
|
|
131
147
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
132
148
|
}
|
|
133
|
-
const token =
|
|
149
|
+
const token = newIncarnationToken(this.entropy, current?.env.record);
|
|
134
150
|
const env = {
|
|
135
151
|
deleted: false,
|
|
136
|
-
record: recordFromNew(ref,
|
|
152
|
+
record: recordFromNew(ref, checked, this.now(), token)
|
|
137
153
|
};
|
|
138
154
|
await this.putRow(key, env, current ? { version: current.version } : "absent");
|
|
139
155
|
return { token };
|
|
140
156
|
}
|
|
141
157
|
async compareAndSwap(ref, expected, patch) {
|
|
142
|
-
validateRegistryPatch(patch);
|
|
158
|
+
const checked = validateRegistryPatch(patch);
|
|
143
159
|
const key = registryObjectKey(this.prefix, ref);
|
|
144
160
|
const current = await this.readRow(key);
|
|
145
161
|
if (current === null || current.env.deleted || current.env.record.token !== expected) {
|
|
146
162
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
147
163
|
}
|
|
148
|
-
const token =
|
|
164
|
+
const token = nextRegistryToken(current.env.record, this.entropy);
|
|
149
165
|
const env = {
|
|
150
166
|
deleted: false,
|
|
151
|
-
record: applyRegistryPatch(current.env.record,
|
|
167
|
+
record: applyRegistryPatch(current.env.record, checked, this.now(), token)
|
|
152
168
|
};
|
|
153
169
|
await this.putRow(key, env, { version: current.version });
|
|
154
170
|
return { token };
|
|
@@ -187,12 +203,17 @@ var ObjectStoreRegistry = class {
|
|
|
187
203
|
} else if (current === null || current.env.deleted) {
|
|
188
204
|
return;
|
|
189
205
|
}
|
|
190
|
-
const token = String(registryCounterOf(current.env.record) + 1);
|
|
191
|
-
const env = {
|
|
192
|
-
deleted: true,
|
|
193
|
-
record: { ...current.env.record, token, updatedAt: this.now() }
|
|
194
|
-
};
|
|
195
206
|
try {
|
|
207
|
+
const remove = this.removesRows() ? this.store.delete : void 0;
|
|
208
|
+
if (remove !== void 0 && incarnationOf(current.env.record.token) !== void 0) {
|
|
209
|
+
await remove.call(this.store, key, { version: current.version });
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
const token = nextRegistryToken(current.env.record, this.entropy);
|
|
213
|
+
const env = {
|
|
214
|
+
deleted: true,
|
|
215
|
+
record: { ...current.env.record, token, updatedAt: this.now() }
|
|
216
|
+
};
|
|
196
217
|
await this.putRow(key, env, { version: current.version });
|
|
197
218
|
return;
|
|
198
219
|
} catch (err) {
|
|
@@ -200,9 +221,7 @@ var ObjectStoreRegistry = class {
|
|
|
200
221
|
throw err;
|
|
201
222
|
}
|
|
202
223
|
}
|
|
203
|
-
throw new WriteConflictError(
|
|
204
|
-
`registry delete: contention tombstoning "${ref.segment}" \u2014 retry`
|
|
205
|
-
);
|
|
224
|
+
throw new WriteConflictError(`registry delete: contention deleting "${ref.segment}" \u2014 retry`);
|
|
206
225
|
}
|
|
207
226
|
/**
|
|
208
227
|
* Read + parse a registry object, returning its envelope and version fence, or null if absent.
|
|
@@ -210,7 +229,7 @@ var ObjectStoreRegistry = class {
|
|
|
210
229
|
* A store that pins a version to read consistently ({@link ObjectVersionRaced}) loses that pin whenever a
|
|
211
230
|
* concurrent writer lands mid-read. Re-reading is the whole answer — the next read simply observes the
|
|
212
231
|
* newer version — and it is bounded so a row under permanent write saturation fails typed instead of
|
|
213
|
-
* spinning. Only a store
|
|
232
|
+
* spinning. Only a store that reads in two calls gets here; none of the shipped stores does.
|
|
214
233
|
*/
|
|
215
234
|
async readRow(objectKey) {
|
|
216
235
|
const row = await this.readRaced(objectKey);
|
package/dist/driver-kit.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/drivers/_shared/object-registry-keys.ts", "../src/drivers/_shared/object-registry.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * The registry object layout, shared by every object-store registry (S3, GCS, Azure Blob).\n *\n * One tiny JSON object per segment at `<prefix>registry/<ns>/<segment>.reg`, **registry-first** \u2014 separate\n * from the namespace-first storage layout (`<prefix><ns>/segments/\u2026`) \u2014 so discovery is a single list over\n * `registry/` for all namespaces, or `registry/<ns>/` for one, never entangled with the `.crbm` payloads.\n *\n * It lives here rather than in a driver because all three object stores encode names the same way\n * (`encodeNameForKey`, `namespaceKeyPart`) and build byte-identical keys. Three copies of this would be\n * three chances for one cloud's layout to drift from the others \u2014 and a drifted key is not a visible bug, it\n * is a segment that quietly cannot be found by a store pointed at the same bucket.\n */\nimport { ValidationError } from '@/core/errors';\nimport { validateSegmentRef } from '@/core/validate';\nimport type { SegmentRef } from '@/core/ports';\nimport { DEFAULT_NAMESPACE, decodeNameFromKey, encodeNameForKey, namespaceKeyPart } from './keys';\n\nconst REGISTRY_SUFFIX = '.reg';\n\n/** Normalize an optional caller prefix to either `''` or `trimmed/` (no leading/trailing slashes). */\nexport function prefixPart(prefix: string | undefined): string {\n if (prefix === undefined) return '';\n const trimmed = prefix.replace(/^\\/+|\\/+$/g, '');\n return trimmed === '' ? '' : `${trimmed}/`;\n}\n\n/**\n * Validate a caller-supplied key prefix (trusted config, but make it a real containment boundary): reject\n * control characters and `.`/`..` path segments so a prefix cannot traverse out of its intended space.\n * Returns it unchanged \u2014 the key builders normalize slashes.\n *\n * The traversal check covers three spellings, because the tools that walk these buckets do. A bare `..` is\n * the obvious one. A **backslash** segment matters on ADLS Gen2 and to Windows-side tooling, which treat it\n * as a separator this code otherwise would not. And a percent-encoded `..` matters because `gsutil`,\n * `s3fs`, `gcsfuse` and `azcopy` decode a key on their way to a local path \u2014 the same reasoning\n * `name-codec.ts` already applies to segment names, which a prefix has no reason to be exempt from.\n */\nexport function normalizeObjectPrefix(prefix: string | undefined): string | undefined {\n if (prefix === undefined) return undefined;\n for (const ch of prefix) {\n const code = ch.charCodeAt(0);\n // C0 plus DEL: a bare `< 0x20` lets U+007F through, and it is as unprintable as the rest.\n if (code < 0x20 || code === 0x7f) {\n throw new ValidationError('prefix must not contain control characters');\n }\n }\n if (prefix.includes('\\\\')) {\n throw new ValidationError(\n `prefix must not contain backslashes (a path separator on some backends): ${JSON.stringify(prefix)}`,\n );\n }\n // Check the decoded spelling too, so an encoded `..` cannot slip past the literal comparison.\n let decoded: string;\n try {\n decoded = decodeURIComponent(prefix);\n } catch {\n decoded = prefix; // not valid percent-encoding; the literal check still applies\n }\n for (const source of decoded === prefix ? [prefix] : [prefix, decoded]) {\n for (const segment of source.split('/')) {\n if (segment === '.' || segment === '..') {\n throw new ValidationError(\n `prefix must not contain \".\" or \"..\" path segments: ${JSON.stringify(prefix)}`,\n );\n }\n }\n }\n return prefix;\n}\n\n/** The key prefix under which every registry object lives: `<prefix>registry/`. */\nexport function registryPrefix(prefix: string | undefined): string {\n return `${prefixPart(prefix)}registry/`;\n}\n\n/** The full key of one segment's registry object: `<prefix>registry/<ns>/<segment>.reg`. */\nexport function registryObjectKey(prefix: string | undefined, ref: SegmentRef): string {\n validateSegmentRef(ref);\n return `${registryPrefix(prefix)}${namespaceKeyPart(ref.namespace)}/${encodeNameForKey(ref.segment)}${REGISTRY_SUFFIX}`;\n}\n\n/** The list prefix for discovery: registry-wide, or scoped to one namespace. */\nexport function registryListPrefix(prefix: string | undefined, namespace?: string): string {\n const base = registryPrefix(prefix);\n return namespace === undefined ? base : `${base}${namespaceKeyPart(namespace)}/`;\n}\n\n/**\n * Parse a `<prefix>registry/<ns>/<segment>.reg` key back to its {@link SegmentRef}, or `null` if it does not\n * match \u2014 a stray or foreign object under the prefix, or one whose parsed ref fails the round-trip check.\n * `_default` maps back to the absent namespace. A name is percent-encoded on the way in, so no encoded name\n * can contain `/` and the split stays unambiguous whatever the caller named their segment.\n */\nexport function parseRegistryKey(prefix: string | undefined, objectKey: string): SegmentRef | null {\n const base = registryPrefix(prefix);\n if (!objectKey.startsWith(base) || !objectKey.endsWith(REGISTRY_SUFFIX)) return null;\n const rest = objectKey.slice(base.length, objectKey.length - REGISTRY_SUFFIX.length);\n const slash = rest.indexOf('/');\n if (slash < 0) return null;\n const nsPart = rest.slice(0, slash);\n const encodedSegment = rest.slice(slash + 1);\n if (encodedSegment.length === 0 || encodedSegment.includes('/')) return null;\n const segment = decodeNameFromKey(encodedSegment);\n const namespace = nsPart === DEFAULT_NAMESPACE ? undefined : decodeNameFromKey(nsPart);\n // The encoding must ROUND-TRIP, not merely decode: a foreign object placed under our prefix could spell a\n // name two ways, and reporting both would hand a sweep one segment under two identities.\n if (encodeNameForKey(segment) !== encodedSegment) return null;\n if (namespace !== undefined && encodeNameForKey(namespace) !== nsPart) return null;\n const ref: SegmentRef = { segment, namespace };\n try {\n validateSegmentRef(ref); // reject a hostile/foreign key that isn't a valid ref\n } catch {\n return null;\n }\n return ref;\n}\n", "/**\n * `ObjectStoreRegistry` \u2014 an {@link IRegistryDriver} over any object store with conditional writes.\n *\n * Lets a **read-mostly deployment run on one bucket alone** \u2014 storage `.crbm` generations plus the registry in\n * the same place, with no separate database. One tiny JSON object per segment at\n * `<prefix>registry/<ns>/<segment>.reg` holding the `{ deleted, record }` envelope (the same shape LocalFs\n * persists). The OCC token is a monotonic counter, advanced on every mutation and even across a `delete`\n * (which **tombstones** rather than removes the object) so a deleted-then-recreated row never re-issues an\n * old token (ABA-safe) \u2014 identical semantics to the LocalFs registry, so it passes the same conformance suite.\n *\n * **The atomic swap is offloaded to the store's conditional writes.** `create` writes only if absent (or\n * over a tombstone under its version), and `compareAndSwap`/`delete` write only if the object still carries\n * the version we read \u2014 so a concurrent writer between our read and our write loses, and the loss surfaces\n * as {@link WriteConflictError}. No in-process lock is needed (unlike LocalFs): the precondition fences\n * writers *across processes*.\n *\n * **Why this is shared rather than written per cloud.** Every object store worth using has the same two\n * primitives under different names \u2014 S3 `If-None-Match: *` / `If-Match: <etag>`, GCS `ifGenerationMatch: 0`\n * / `ifGenerationMatch: <generation>`, Azure `If-None-Match: *` / `If-Match: <etag>` \u2014 so the only thing\n * that differs is three I/O calls, which is what {@link ObjectRegistryStore} abstracts. The parts that are\n * genuinely hard (the ABA-safe counter, the tombstone, the bounded delete retry, the round-trip key check)\n * exist once. Three copies would be three chances for one cloud to drift, and a drifted registry is not a\n * visible bug \u2014 it is a lost `currentGen` swap.\n *\n * **Deployment requirements** (a backend that violates these silently corrupts the registry):\n * - The store **must honor the conditional write**. One that accepts the precondition header and ignores it\n * degrades compare-and-swap to last-write-wins \u2192 lost `currentGen` swaps.\n * - Reads must be **strongly consistent** \u2014 true of S3 (since 2020), GCS and Azure Blob \u2014 since the registry\n * advertises `strongRead` and generation resolution depends on it.\n * - **Do not apply a lifecycle-expiration rule to the `registry/` prefix that expires a current version.**\n * `delete` tombstones (keeps the object with an advanced counter) for ABA-safety; expiring a tombstone would let a recreate reset the\n * token to 0 and re-issue a stale one.\n *\n * **`list()` is fail-closed, and one bad object stops it for everyone.** An object under the `registry/`\n * prefix whose key parses but whose body does not aborts the whole enumeration \u2014 every namespace, not just\n * the affected segment \u2014 and it stays that way until an operator removes the object. The error names the\n * offending key.\n *\n * That is the deliberate choice, and it is the *safer* one rather than the more convenient one. Skipping the\n * unreadable row would keep discovery running, but `list()` is what tells orphan generation collection which\n * segments exist: a row missing from the enumeration makes its storage `.crbm` generations look unreferenced,\n * and the next GC pass would delete them. Silently skipping turns a parse error into data loss. Refusing to\n * enumerate keeps every sweep off a set it cannot vouch for (invariant 5 \u2014 bytes from the tier are\n * untrusted). **Recovery:** read the named object, and either restore a valid envelope or delete it; nothing\n * else is damaged, and discovery resumes on its own.\n */\nimport {\n IntegrityError,\n TransientError,\n ValidationError,\n WriteConflictError,\n isWriteConflictError,\n} from '@/core/errors';\nimport type {\n IRegistryDriver,\n NewRegistryRecord,\n RegCaps,\n RegistryPatch,\n RegistryRecord,\n SegmentRef,\n Token,\n} from '@/core/ports';\nimport { mapWithConcurrency } from '@/core/concurrency';\nimport {\n applyRegistryPatch,\n parseRegistryEnvelope,\n recordFromNew,\n registryCounterOf,\n serializeRegistryEnvelope,\n validateNewRegistryRecord,\n validateRegistryPatch,\n type RegistryEnvelope,\n} from './registry';\nimport { parseRegistryKey, registryListPrefix, registryObjectKey } from './object-registry-keys';\n\n/** Defensive cap on a single registry object read from storage, before allocation (rows are tiny; ~1 KB). */\nexport const MAX_ROW_BYTES = 1 * 1024 * 1024;\n/** Bounded retry for `delete`'s read\u2192tombstone under cross-process contention (converges; then fails typed). */\nconst MAX_DELETE_ATTEMPTS = 8;\n/** Bounded re-read when a store's pinned version is overwritten mid-read (see {@link ObjectVersionRaced}). */\nconst MAX_READ_ATTEMPTS = 8;\n/** In-flight reads per `list()` page \u2014 turns the serial N+1 into one list plus bounded parallel reads. */\nconst LIST_READ_CONCURRENCY = 16;\n\n/**\n * Raised by a store when the version it pinned for a read vanished before it could fetch the bytes \u2014\n * a concurrent writer overwrote or deleted the object in between.\n *\n * This is deliberately **neither** of the two things it superficially resembles. It is not absence: the row\n * is very likely still live, just one version further on. And it is not a write conflict: the caller may be\n * a read-only `get()`, which has nothing to conflict with. Either is an easy misreading for a store that reads a\n * row in two calls, as the Azure Blob store does, which is why the signal is part of the port rather than\n * left to each store's judgement. {@link ObjectStoreRegistry} answers it the only way that is correct for every\n * caller: it reads again.\n *\n * Stores whose read is atomic (S3 serves bytes and `ETag` from one `GetObject`, GCS bytes and `generation` from one GET) can never raise it.\n */\nexport class ObjectVersionRaced extends Error {\n constructor(objectKey: string) {\n super(`registry object was overwritten mid-read: ${objectKey}`);\n this.name = 'ObjectVersionRaced';\n }\n}\n\n/** One object as read, with the opaque version that fences a later conditional write. */\nexport interface ObjectRow {\n readonly bytes: Uint8Array;\n /** The store's version fence \u2014 an S3/Azure ETag, a GCS generation. Opaque to this class. */\n readonly version: string;\n}\n\n/**\n * The three I/O calls an object store must provide to host a registry. Everything else \u2014 the OCC counter,\n * tombstones, the delete retry, key parsing \u2014 is implemented once above.\n */\nexport interface ObjectRegistryStore {\n /** A label for error messages, e.g. `GCS` or `Azure Blob`. */\n readonly label: string;\n /**\n * Read one object, or `null` when it is absent.\n *\n * **`null` means the object does not exist** \u2014 nothing weaker. A store that reads in two round trips\n * (metadata for the version fence, then the bytes pinned to it) can find that version already gone; it\n * MUST report that with {@link ObjectVersionRaced} rather than `null` or {@link WriteConflictError}, and\n * the caller re-reads. Retryable faults throw {@link TransientError}.\n *\n * `bytes` and `version` MUST describe the **same** observation of the object: `version` is the fence a\n * later conditional write is conditioned on, so a pair drawn from two different versions would either\n * fail a write that should have succeeded or, worse, pass one that should not.\n */\n read(key: string): Promise<ObjectRow | null>;\n /**\n * Write one object under a precondition: `'absent'` means create-only, a version means\n * compare-and-swap against exactly that version. A lost race MUST throw {@link WriteConflictError}.\n *\n * A store must not replay this write after a lost response without telling the replay apart: the replay meets\n * the row it just wrote and its precondition fails, which would read as a lost race. Send it once, or store a\n * write id with the row and treat a conflict whose stored row carries it as success. Retryable faults throw\n * {@link TransientError}.\n */\n write(key: string, body: Uint8Array, expect: 'absent' | { version: string }): Promise<void>;\n /** Every object key under `prefix`, paginated internally. */\n listKeys(prefix: string): AsyncIterable<string>;\n}\n\nexport class ObjectStoreRegistry implements IRegistryDriver {\n constructor(\n private readonly store: ObjectRegistryStore,\n private readonly prefix: string | undefined,\n private readonly now: () => number,\n ) {}\n\n capabilities(): RegCaps {\n return { strongRead: true };\n }\n\n async get(ref: SegmentRef): Promise<RegistryRecord | null> {\n const current = await this.readRow(registryObjectKey(this.prefix, ref));\n return current && !current.env.deleted ? current.env.record : null;\n }\n\n async create(ref: SegmentRef, record: NewRegistryRecord): Promise<{ token: Token }> {\n validateNewRegistryRecord(record);\n const key = registryObjectKey(this.prefix, ref);\n const current = await this.readRow(key);\n if (current !== null && !current.env.deleted) {\n throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);\n }\n // Advance across a tombstone so a recreate never re-issues an old token (ABA-safe).\n const token = String(current ? registryCounterOf(current.env.record) + 1 : 0);\n const env: RegistryEnvelope = {\n deleted: false,\n record: recordFromNew(ref, record, this.now(), token),\n };\n // Create-only when truly absent; overwrite the tombstone under its version when recreating. Either way\n // a concurrent create loses the precondition and surfaces as a conflict.\n await this.putRow(key, env, current ? { version: current.version } : 'absent');\n return { token };\n }\n\n async compareAndSwap(\n ref: SegmentRef,\n expected: Token,\n patch: RegistryPatch,\n ): Promise<{ token: Token }> {\n validateRegistryPatch(patch);\n const key = registryObjectKey(this.prefix, ref);\n const current = await this.readRow(key);\n if (current === null || current.env.deleted || current.env.record.token !== expected) {\n throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);\n }\n const token = String(registryCounterOf(current.env.record) + 1);\n const env: RegistryEnvelope = {\n deleted: false,\n record: applyRegistryPatch(current.env.record, patch, this.now(), token),\n };\n // The version we read fences a concurrent writer between that read and this write.\n await this.putRow(key, env, { version: current.version });\n return { token };\n }\n\n async *list(namespace?: string): AsyncIterable<RegistryRecord> {\n const prefix = registryListPrefix(this.prefix, namespace);\n // One list plus bounded parallel reads. A serial await-in-loop made this O(N) sequential round trips.\n let page: string[] = [];\n const flush = async function* (\n this: ObjectStoreRegistry,\n keys: string[],\n ): AsyncIterable<RegistryRecord> {\n const rows = await mapWithConcurrency(keys, LIST_READ_CONCURRENCY, (key) =>\n this.readRow(key),\n );\n for (const current of rows) {\n if (current && !current.env.deleted) yield current.env.record;\n }\n }.bind(this);\n\n for await (const key of this.store.listKeys(prefix)) {\n if (parseRegistryKey(this.prefix, key) === null) continue; // stray or foreign object\n page.push(key);\n if (page.length >= LIST_READ_CONCURRENCY) {\n yield* flush(page);\n page = [];\n }\n }\n if (page.length > 0) yield* flush(page);\n }\n\n async delete(ref: SegmentRef, expected?: Token): Promise<void> {\n const key = registryObjectKey(this.prefix, ref);\n // Tombstone (advance the counter) rather than remove the object \u2014 keeps the token monotonic for\n // ABA-safety. Retry the read\u2192tombstone on a cross-process race; it converges, then fails typed rather\n // than silently leaving the row live.\n for (let attempt = 0; attempt < MAX_DELETE_ATTEMPTS; attempt++) {\n const current = await this.readRow(key);\n if (expected !== undefined) {\n // Fenced: the row must still carry the token the caller read. Re-checked on every attempt, so a row\n // that was swapped, or deleted and re-created, between two attempts is refused rather than tombstoned.\n if (current === null || current.env.deleted || current.env.record.token !== expected) {\n throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);\n }\n } else if (current === null || current.env.deleted) {\n return; // idempotent \u2014 already gone\n }\n const token = String(registryCounterOf(current.env.record) + 1);\n const env: RegistryEnvelope = {\n deleted: true,\n record: { ...current.env.record, token, updatedAt: this.now() },\n };\n try {\n await this.putRow(key, env, { version: current.version });\n return;\n } catch (err) {\n if (isWriteConflictError(err)) continue; // raced \u2014 re-read and re-tombstone\n throw err;\n }\n }\n throw new WriteConflictError(\n `registry delete: contention tombstoning \"${ref.segment}\" \u2014 retry`,\n );\n }\n\n /**\n * Read + parse a registry object, returning its envelope and version fence, or null if absent.\n *\n * A store that pins a version to read consistently ({@link ObjectVersionRaced}) loses that pin whenever a\n * concurrent writer lands mid-read. Re-reading is the whole answer \u2014 the next read simply observes the\n * newer version \u2014 and it is bounded so a row under permanent write saturation fails typed instead of\n * spinning. Only a store racing itself gets here; S3 never does.\n */\n private async readRow(\n objectKey: string,\n ): Promise<{ env: RegistryEnvelope; version: string } | null> {\n const row = await this.readRaced(objectKey);\n if (row === null) return null;\n if (row.bytes.length > MAX_ROW_BYTES) {\n throw new IntegrityError(\n `registry object ${row.bytes.length}B exceeds cap ${MAX_ROW_BYTES}B`,\n );\n }\n if (row.version.length === 0) {\n // The version is the OCC fence for CAS; a backend that omits it cannot be used safely as a registry.\n throw new IntegrityError(\n `registry object has no version fence (needed for compare-and-swap): ${objectKey}`,\n );\n }\n const env = parseRegistryEnvelope(new TextDecoder().decode(row.bytes), objectKey);\n return { env, version: row.version };\n }\n\n /** {@link ObjectRegistryStore.read}, retrying the bounded number of times a mid-read overwrite allows. */\n private async readRaced(objectKey: string): Promise<ObjectRow | null> {\n for (let attempt = 0; attempt < MAX_READ_ATTEMPTS; attempt++) {\n try {\n return await this.store.read(objectKey);\n } catch (err) {\n if (!(err instanceof ObjectVersionRaced)) throw err;\n }\n }\n throw new TransientError(\n `registry read: ${this.store.label} object \"${objectKey}\" was overwritten on every one of ` +\n `${MAX_READ_ATTEMPTS} attempts \u2014 retry`,\n );\n }\n\n /** Conditional write of an envelope. The store maps a lost precondition to {@link WriteConflictError}. */\n private async putRow(\n objectKey: string,\n env: RegistryEnvelope,\n expect: 'absent' | { version: string },\n ): Promise<void> {\n const body = new TextEncoder().encode(serializeRegistryEnvelope(env));\n if (body.length > MAX_ROW_BYTES) {\n // Belt-and-braces: the governance/DEK fields are already capped by validate*, so this cannot fire for\n // valid input \u2014 but never write a row that would later be unreadable.\n throw new ValidationError(`registry object ${body.length}B exceeds cap ${MAX_ROW_BYTES}B`);\n }\n await this.store.write(objectKey, body, expect);\n }\n}\n"],
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["/**\n * The registry object layout, shared by every object-store registry (S3, GCS, Azure Blob).\n *\n * One tiny JSON object per segment at `<prefix>registry/<ns>/<segment>.reg`, **registry-first** \u2014 separate\n * from the namespace-first storage layout (`<prefix><ns>/segments/\u2026`) \u2014 so discovery is a single list over\n * `registry/` for all namespaces, or `registry/<ns>/` for one, never entangled with the `.crbm` payloads.\n *\n * It lives here rather than in a driver because all three object stores encode names the same way\n * (`encodeNameForKey`, `namespaceKeyPart`) and build byte-identical keys. Three copies of this would be\n * three chances for one cloud's layout to drift from the others \u2014 and a drifted key is not a visible bug, it\n * is a segment that quietly cannot be found by a store pointed at the same bucket.\n */\nimport { ValidationError } from '@/core/errors';\nimport { validateSegmentRef } from '@/core/validate';\nimport type { SegmentRef } from '@/core/ports';\nimport { DEFAULT_NAMESPACE, decodeNameFromKey, encodeNameForKey, namespaceKeyPart } from './keys';\n\nconst REGISTRY_SUFFIX = '.reg';\n\n/** Normalize an optional caller prefix to either `''` or `trimmed/` (no leading/trailing slashes). */\nexport function prefixPart(prefix: string | undefined): string {\n if (prefix === undefined) return '';\n const trimmed = prefix.replace(/^\\/+|\\/+$/g, '');\n return trimmed === '' ? '' : `${trimmed}/`;\n}\n\n/**\n * Validate a caller-supplied key prefix (trusted config, but make it a real containment boundary): reject\n * control characters and `.`/`..` path segments so a prefix cannot traverse out of its intended space.\n * Returns it unchanged \u2014 the key builders normalize slashes.\n *\n * The traversal check covers three spellings, because the tools that walk these buckets do. A bare `..` is\n * the obvious one. A **backslash** segment matters on ADLS Gen2 and to Windows-side tooling, which treat it\n * as a separator this code otherwise would not. And a percent-encoded `..` matters because `gsutil`,\n * `s3fs`, `gcsfuse` and `azcopy` decode a key on their way to a local path \u2014 the same reasoning\n * `name-codec.ts` already applies to segment names, which a prefix has no reason to be exempt from.\n */\nexport function normalizeObjectPrefix(prefix: string | undefined): string | undefined {\n if (prefix === undefined) return undefined;\n for (const ch of prefix) {\n const code = ch.charCodeAt(0);\n // C0 plus DEL: a bare `< 0x20` lets U+007F through, and it is as unprintable as the rest.\n if (code < 0x20 || code === 0x7f) {\n throw new ValidationError('prefix must not contain control characters');\n }\n }\n if (prefix.includes('\\\\')) {\n throw new ValidationError(\n `prefix must not contain backslashes (a path separator on some backends): ${JSON.stringify(prefix)}`,\n );\n }\n // Check the decoded spelling too, so an encoded `..` cannot slip past the literal comparison.\n let decoded: string;\n try {\n decoded = decodeURIComponent(prefix);\n } catch {\n decoded = prefix; // not valid percent-encoding; the literal check still applies\n }\n for (const source of decoded === prefix ? [prefix] : [prefix, decoded]) {\n for (const segment of source.split('/')) {\n if (segment === '.' || segment === '..') {\n throw new ValidationError(\n `prefix must not contain \".\" or \"..\" path segments: ${JSON.stringify(prefix)}`,\n );\n }\n }\n }\n return prefix;\n}\n\n/** The key prefix under which every registry object lives: `<prefix>registry/`. */\nexport function registryPrefix(prefix: string | undefined): string {\n return `${prefixPart(prefix)}registry/`;\n}\n\n/** The full key of one segment's registry object: `<prefix>registry/<ns>/<segment>.reg`. */\nexport function registryObjectKey(prefix: string | undefined, ref: SegmentRef): string {\n validateSegmentRef(ref);\n return `${registryPrefix(prefix)}${namespaceKeyPart(ref.namespace)}/${encodeNameForKey(ref.segment)}${REGISTRY_SUFFIX}`;\n}\n\n/** The list prefix for discovery: registry-wide, or scoped to one namespace. */\nexport function registryListPrefix(prefix: string | undefined, namespace?: string): string {\n const base = registryPrefix(prefix);\n return namespace === undefined ? base : `${base}${namespaceKeyPart(namespace)}/`;\n}\n\n/**\n * Parse a `<prefix>registry/<ns>/<segment>.reg` key back to its {@link SegmentRef}, or `null` if it does not\n * match \u2014 a stray or foreign object under the prefix, or one whose parsed ref fails the round-trip check.\n * `_default` maps back to the absent namespace. A name is percent-encoded on the way in, so no encoded name\n * can contain `/` and the split stays unambiguous whatever the caller named their segment.\n */\nexport function parseRegistryKey(prefix: string | undefined, objectKey: string): SegmentRef | null {\n const base = registryPrefix(prefix);\n if (!objectKey.startsWith(base) || !objectKey.endsWith(REGISTRY_SUFFIX)) return null;\n const rest = objectKey.slice(base.length, objectKey.length - REGISTRY_SUFFIX.length);\n const slash = rest.indexOf('/');\n if (slash < 0) return null;\n const nsPart = rest.slice(0, slash);\n const encodedSegment = rest.slice(slash + 1);\n if (encodedSegment.length === 0 || encodedSegment.includes('/')) return null;\n const segment = decodeNameFromKey(encodedSegment);\n const namespace = nsPart === DEFAULT_NAMESPACE ? undefined : decodeNameFromKey(nsPart);\n // The encoding must ROUND-TRIP, not merely decode: a foreign object placed under our prefix could spell a\n // name two ways, and reporting both would hand a sweep one segment under two identities.\n if (encodeNameForKey(segment) !== encodedSegment) return null;\n if (namespace !== undefined && encodeNameForKey(namespace) !== nsPart) return null;\n const ref: SegmentRef = { segment, namespace };\n try {\n validateSegmentRef(ref); // reject a hostile/foreign key that isn't a valid ref\n } catch {\n return null;\n }\n return ref;\n}\n", "/**\n * `ObjectStoreRegistry` \u2014 an {@link IRegistryDriver} over any object store with conditional writes.\n *\n * Lets a **read-mostly deployment run on one bucket alone** \u2014 storage `.crbm` generations plus the registry in\n * the same place, with no separate database. One tiny JSON object per segment at\n * `<prefix>registry/<ns>/<segment>.reg` holding the `{ deleted, record }` envelope (the same shape LocalFs\n * persists). The OCC token is a random incarnation id drawn when the row is created, a counter advanced on every\n * mutation, and a random part drawn for every write, so with overwhelming probability a deleted-then-recreated row, or\n * one restored from a backup, never re-issues an old token (ABA-safe) \u2014 identical semantics to the LocalFs registry, so it passes the same\n * conformance suite.\n *\n * **`delete` removes a row for good where the store vouches for a conditional delete** (`If-Match: <etag>`,\n * `ifGenerationMatch: <generation>`), and only a row whose token carries an incarnation id: a re-create draws a new\n * incarnation, so nothing of the old row is needed to keep its tokens apart, and a full `list()` no longer reads a\n * row for every name it ever held. Every other row is **tombstoned**: the object stays, its counter advanced, and a\n * re-create carries the counter on. A row a release before 0.12 wrote has a bare decimal token and is always\n * tombstoned, since a process still on that release, re-creating the name over nothing, would start its counter at 0\n * again and issue the old row's tokens.\n *\n * **The atomic swap is offloaded to the store's conditional writes.** `create` writes only if absent (or\n * over a tombstone under its version), and `compareAndSwap`/`delete` write (or delete) only if the object still\n * carries the version we read \u2014 so a concurrent writer between our read and our write loses, and the loss surfaces\n * as {@link WriteConflictError}. No in-process lock is needed (unlike LocalFs): the precondition fences\n * writers *across processes*.\n *\n * **Why this is shared rather than written per cloud.** Every object store worth using has the same two\n * primitives under different names \u2014 S3 `If-None-Match: *` / `If-Match: <etag>`, GCS `ifGenerationMatch: 0`\n * / `ifGenerationMatch: <generation>`, Azure `If-None-Match: *` / `If-Match: <etag>` \u2014 so the only thing\n * that differs is a few I/O calls, which is what {@link ObjectRegistryStore} abstracts. The parts that are\n * genuinely hard (the ABA-safe token, the delete and its tombstone, the bounded delete retry, the round-trip key check)\n * exist once. Three copies would be three chances for one cloud to drift, and a drifted registry is not a\n * visible bug \u2014 it is a lost `currentGen` swap.\n *\n * **Deployment requirements** (a backend that violates these silently corrupts the registry):\n * - The store **must honor the conditional write**. One that accepts the precondition header and ignores it\n * degrades compare-and-swap to last-write-wins \u2192 lost `currentGen` swaps.\n * - Reads must be **strongly consistent** \u2014 true of S3 (since 2020), GCS and Azure Blob \u2014 since the registry\n * advertises `strongRead` and generation resolution depends on it.\n * - **Do not apply a lifecycle-expiration rule to the `registry/` prefix that expires a current version.**\n * Expiring a live row loses its pointer. A tombstone `delete` left keeps the counter for a re-create; one expired\n * by a rule is no worse than a row removed for good, since a recreate draws a fresh incarnation and its tokens are\n * new but for a collision of probability 2^-128 per pair of incarnations, but a rule cannot tell a tombstone from a\n * live row.\n * - **A store's conditional delete must be honoured by its backend** before the store sets `conditionalDelete`.\n * One that ignores the precondition lets two sweepers and a re-create delete a live row.\n *\n * **`list()` is fail-closed, and one bad object stops it for everyone.** An object under the `registry/`\n * prefix whose key parses but whose body does not aborts the whole enumeration \u2014 every namespace, not just\n * the affected segment \u2014 and it stays that way until an operator removes the object. The error names the\n * offending key.\n *\n * That is the deliberate choice, and it is the *safer* one rather than the more convenient one. Skipping the\n * unreadable row would keep discovery running, but `list()` is what tells orphan generation collection which\n * segments exist: a row missing from the enumeration makes its storage `.crbm` generations look unreferenced,\n * and the next GC pass would delete them. Silently skipping turns a parse error into data loss. Refusing to\n * enumerate keeps every sweep off a set it cannot vouch for (invariant 5 \u2014 bytes from the tier are\n * untrusted). **Recovery:** read the named object, and either restore a valid envelope or delete it; nothing\n * else is damaged, and discovery resumes on its own.\n */\nimport {\n IntegrityError,\n TransientError,\n ValidationError,\n WriteConflictError,\n isWriteConflictError,\n} from '@/core/errors';\nimport type {\n IRegistryDriver,\n NewRegistryRecord,\n RegCaps,\n RegistryPatch,\n RegistryRecord,\n SegmentRef,\n Token,\n} from '@/core/ports';\nimport type { Entropy } from '@/core/determinism';\nimport { mapWithConcurrency } from '@/core/concurrency';\nimport {\n applyRegistryPatch,\n incarnationOf,\n newIncarnationToken,\n nextRegistryToken,\n parseRegistryEnvelope,\n recordFromNew,\n serializeRegistryEnvelope,\n validateNewRegistryRecord,\n validateRegistryPatch,\n type RegistryEnvelope,\n} from './registry';\nimport { entropyIsAvailable, webCryptoEntropy } from './entropy';\nimport { parseRegistryKey, registryListPrefix, registryObjectKey } from './object-registry-keys';\n\n/** Defensive cap on a single registry object read from storage, before allocation (rows are tiny; ~1 KB). */\nexport const MAX_ROW_BYTES = 1 * 1024 * 1024;\n/** Bounded retry for `delete`'s read then delete or tombstone, under cross-process contention (then fails typed). */\nconst MAX_DELETE_ATTEMPTS = 8;\n/** Bounded re-read when a store's pinned version is overwritten mid-read (see {@link ObjectVersionRaced}). */\nconst MAX_READ_ATTEMPTS = 8;\n/** In-flight reads per `list()` page \u2014 turns the serial N+1 into one list plus bounded parallel reads. */\nconst LIST_READ_CONCURRENCY = 16;\n\n/**\n * Raised by a store when the version it pinned for a read vanished before it could fetch the bytes \u2014\n * a concurrent writer overwrote or deleted the object in between.\n *\n * This is deliberately **neither** of the two things it superficially resembles. It is not absence: the row\n * is very likely still live, just one version further on. And it is not a write conflict: the caller may be\n * a read-only `get()`, which has nothing to conflict with. Either is an easy misreading for a store that reads a\n * row in two calls, metadata and then bytes pinned to it, which is why the signal is part of the port rather than\n * left to each store's judgement. {@link ObjectStoreRegistry} answers it the only way that is correct for every\n * caller: it reads again.\n *\n * Stores whose read is atomic can never raise it, and every shipped store's is: S3 serves bytes and `ETag` from one\n * `GetObject`, GCS bytes and `generation` from one GET, and Azure Blob bytes and `ETag` from one GET.\n */\nexport class ObjectVersionRaced extends Error {\n constructor(objectKey: string) {\n super(`registry object was overwritten mid-read: ${objectKey}`);\n this.name = 'ObjectVersionRaced';\n }\n}\n\n/** One object as read, with the opaque version that fences a later conditional write. */\nexport interface ObjectRow {\n readonly bytes: Uint8Array;\n /** The store's version fence \u2014 an S3/Azure ETag, a GCS generation. Opaque to this class. */\n readonly version: string;\n}\n\n/**\n * The I/O calls an object store must provide to host a registry: three, and an optional conditional delete. Everything\n * else \u2014 the OCC token, tombstones, the delete retry, key parsing \u2014 is implemented once above.\n */\nexport interface ObjectRegistryStore {\n /** A label for error messages, e.g. `GCS` or `Azure Blob`. */\n readonly label: string;\n /**\n * Read one object, or `null` when it is absent.\n *\n * **`null` means the object does not exist** \u2014 nothing weaker. A store that reads in two round trips\n * (metadata for the version fence, then the bytes pinned to it) can find that version already gone; it\n * MUST report that with {@link ObjectVersionRaced} rather than `null` or {@link WriteConflictError}, and\n * the caller re-reads. Retryable faults throw {@link TransientError}.\n *\n * `bytes` and `version` MUST describe the **same** observation of the object: `version` is the fence a\n * later conditional write is conditioned on, so a pair drawn from two different versions would either\n * fail a write that should have succeeded or, worse, pass one that should not.\n */\n read(key: string): Promise<ObjectRow | null>;\n /**\n * Write one object under a precondition: `'absent'` means create-only, a version means\n * compare-and-swap against exactly that version. A lost race MUST throw {@link WriteConflictError}.\n *\n * A store must not replay this write after a lost response without telling the replay apart: the replay meets\n * the row it just wrote and its precondition fails, which would read as a lost race. Send it once, or store a\n * write id with the row and treat a conflict whose stored row carries it as success. Retryable faults throw\n * {@link TransientError}.\n */\n write(key: string, body: Uint8Array, expect: 'absent' | { version: string }): Promise<void>;\n /** Every object key under `prefix`, paginated internally. */\n listKeys(prefix: string): AsyncIterable<string>;\n /**\n * Optional: delete one object, and only while it is still at exactly `expect.version`, by a precondition the\n * backend applies (`If-Match: <etag>`, `ifGenerationMatch: <generation>`). An object that was overwritten, or is\n * gone, MUST throw {@link WriteConflictError} and delete nothing. Retryable faults throw {@link TransientError}.\n *\n * Sending it again after a lost response is safe, unlike a write: the precondition names one version, so a second\n * copy can remove nothing the first was not allowed to. A copy that meets its own landed delete fails the\n * precondition and reports a conflict; the registry re-reads, and finds the row gone.\n *\n * The registry calls it only when {@link conditionalDelete} is `true`.\n */\n delete?(key: string, expect: { version: string }): Promise<void>;\n /**\n * Whether {@link delete} may be relied on: `true` only where the backend is known to apply the precondition. A\n * backend that accepts the header and ignores it would let a delete remove a row written after the delete read it,\n * a live one included. Absent or `false`, the registry never calls `delete` and tombstones every row instead.\n */\n readonly conditionalDelete?: boolean;\n}\n\nexport class ObjectStoreRegistry implements IRegistryDriver {\n /**\n * `entropy` draws the random parts of every token: a new row's incarnation id, and each write's own part. It\n * defaults to the platform's Web Crypto. Inject one only to make a test replayable: a seeded source in production\n * gives every process the same ids.\n */\n constructor(\n private readonly store: ObjectRegistryStore,\n private readonly prefix: string | undefined,\n private readonly now: () => number,\n private readonly entropy: Entropy = webCryptoEntropy,\n ) {}\n\n capabilities(): RegCaps {\n const conditionalDelete = this.removesRows();\n return entropyIsAvailable(this.entropy)\n ? { strongRead: true, conditionalDelete }\n : { strongRead: true, canWrite: false, conditionalDelete };\n }\n\n /** Whether a delete removes a row born with an incarnation id: the store has a delete, and vouches for it. */\n private removesRows(): boolean {\n return this.store.conditionalDelete === true && typeof this.store.delete === 'function';\n }\n\n async get(ref: SegmentRef): Promise<RegistryRecord | null> {\n const current = await this.readRow(registryObjectKey(this.prefix, ref));\n return current && !current.env.deleted ? current.env.record : null;\n }\n\n async create(ref: SegmentRef, record: NewRegistryRecord): Promise<{ token: Token }> {\n const checked = validateNewRegistryRecord(record);\n const key = registryObjectKey(this.prefix, ref);\n const current = await this.readRow(key);\n if (current !== null && !current.env.deleted) {\n throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);\n }\n // A new incarnation, whose counter continues across a tombstone, so with overwhelming probability a recreate never re-issues an old token.\n const token = newIncarnationToken(this.entropy, current?.env.record);\n const env: RegistryEnvelope = {\n deleted: false,\n record: recordFromNew(ref, checked, this.now(), token),\n };\n // Create-only when truly absent; overwrite the tombstone under its version when recreating. Either way\n // a concurrent create loses the precondition and surfaces as a conflict.\n await this.putRow(key, env, current ? { version: current.version } : 'absent');\n return { token };\n }\n\n async compareAndSwap(\n ref: SegmentRef,\n expected: Token,\n patch: RegistryPatch,\n ): Promise<{ token: Token }> {\n const checked = validateRegistryPatch(patch);\n const key = registryObjectKey(this.prefix, ref);\n const current = await this.readRow(key);\n if (current === null || current.env.deleted || current.env.record.token !== expected) {\n throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);\n }\n const token = nextRegistryToken(current.env.record, this.entropy);\n const env: RegistryEnvelope = {\n deleted: false,\n record: applyRegistryPatch(current.env.record, checked, this.now(), token),\n };\n // The version we read fences a concurrent writer between that read and this write.\n await this.putRow(key, env, { version: current.version });\n return { token };\n }\n\n async *list(namespace?: string): AsyncIterable<RegistryRecord> {\n const prefix = registryListPrefix(this.prefix, namespace);\n // One list plus bounded parallel reads. A serial await-in-loop made this O(N) sequential round trips.\n let page: string[] = [];\n const flush = async function* (\n this: ObjectStoreRegistry,\n keys: string[],\n ): AsyncIterable<RegistryRecord> {\n const rows = await mapWithConcurrency(keys, LIST_READ_CONCURRENCY, (key) =>\n this.readRow(key),\n );\n for (const current of rows) {\n if (current && !current.env.deleted) yield current.env.record;\n }\n }.bind(this);\n\n for await (const key of this.store.listKeys(prefix)) {\n if (parseRegistryKey(this.prefix, key) === null) continue; // stray or foreign object\n page.push(key);\n if (page.length >= LIST_READ_CONCURRENCY) {\n yield* flush(page);\n page = [];\n }\n }\n if (page.length > 0) yield* flush(page);\n }\n\n async delete(ref: SegmentRef, expected?: Token): Promise<void> {\n const key = registryObjectKey(this.prefix, ref);\n // A row born with an incarnation id is removed from the store, by a delete the store applies only while the\n // object is still the version read here: a later create draws a new incarnation, so nothing of this row is needed\n // to keep its tokens apart. Any other row is tombstoned (the counter advanced, the object kept), so a re-create\n // carries the counter on. Retry the read and the write on a cross-process race; it converges, then fails typed\n // rather than silently leaving the row live.\n for (let attempt = 0; attempt < MAX_DELETE_ATTEMPTS; attempt++) {\n const current = await this.readRow(key);\n if (expected !== undefined) {\n // Fenced: the row must still carry the token the caller read. Re-checked on every attempt, so a row\n // that was swapped, or deleted and re-created, between two attempts is refused rather than tombstoned.\n if (current === null || current.env.deleted || current.env.record.token !== expected) {\n throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);\n }\n } else if (current === null || current.env.deleted) {\n return; // idempotent \u2014 already gone\n }\n try {\n const remove = this.removesRows() ? this.store.delete : undefined;\n if (remove !== undefined && incarnationOf(current.env.record.token) !== undefined) {\n // The version read above is the row the caller's token names, so this removes that row and no later one.\n await remove.call(this.store, key, { version: current.version });\n return;\n }\n const token = nextRegistryToken(current.env.record, this.entropy);\n const env: RegistryEnvelope = {\n deleted: true,\n record: { ...current.env.record, token, updatedAt: this.now() },\n };\n await this.putRow(key, env, { version: current.version });\n return;\n } catch (err) {\n if (isWriteConflictError(err)) continue; // raced \u2014 re-read and try again\n throw err;\n }\n }\n throw new WriteConflictError(`registry delete: contention deleting \"${ref.segment}\" \u2014 retry`);\n }\n\n /**\n * Read + parse a registry object, returning its envelope and version fence, or null if absent.\n *\n * A store that pins a version to read consistently ({@link ObjectVersionRaced}) loses that pin whenever a\n * concurrent writer lands mid-read. Re-reading is the whole answer \u2014 the next read simply observes the\n * newer version \u2014 and it is bounded so a row under permanent write saturation fails typed instead of\n * spinning. Only a store that reads in two calls gets here; none of the shipped stores does.\n */\n private async readRow(\n objectKey: string,\n ): Promise<{ env: RegistryEnvelope; version: string } | null> {\n const row = await this.readRaced(objectKey);\n if (row === null) return null;\n if (row.bytes.length > MAX_ROW_BYTES) {\n throw new IntegrityError(\n `registry object ${row.bytes.length}B exceeds cap ${MAX_ROW_BYTES}B`,\n );\n }\n if (row.version.length === 0) {\n // The version is the OCC fence for CAS; a backend that omits it cannot be used safely as a registry.\n throw new IntegrityError(\n `registry object has no version fence (needed for compare-and-swap): ${objectKey}`,\n );\n }\n const env = parseRegistryEnvelope(new TextDecoder().decode(row.bytes), objectKey);\n return { env, version: row.version };\n }\n\n /** {@link ObjectRegistryStore.read}, retrying the bounded number of times a mid-read overwrite allows. */\n private async readRaced(objectKey: string): Promise<ObjectRow | null> {\n for (let attempt = 0; attempt < MAX_READ_ATTEMPTS; attempt++) {\n try {\n return await this.store.read(objectKey);\n } catch (err) {\n if (!(err instanceof ObjectVersionRaced)) throw err;\n }\n }\n throw new TransientError(\n `registry read: ${this.store.label} object \"${objectKey}\" was overwritten on every one of ` +\n `${MAX_READ_ATTEMPTS} attempts \u2014 retry`,\n );\n }\n\n /** Conditional write of an envelope. The store maps a lost precondition to {@link WriteConflictError}. */\n private async putRow(\n objectKey: string,\n env: RegistryEnvelope,\n expect: 'absent' | { version: string },\n ): Promise<void> {\n const body = new TextEncoder().encode(serializeRegistryEnvelope(env));\n if (body.length > MAX_ROW_BYTES) {\n // Belt-and-braces: the governance/DEK fields are already capped by validate*, so this cannot fire for\n // valid input \u2014 but never write a row that would later be unreadable.\n throw new ValidationError(`registry object ${body.length}B exceeds cap ${MAX_ROW_BYTES}B`);\n }\n await this.store.write(objectKey, body, expect);\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiBA,IAAM,kBAAkB;AAGjB,SAAS,WAAW,QAAoC;AAC7D,MAAI,WAAW,OAAW,QAAO;AACjC,QAAM,UAAU,OAAO,QAAQ,cAAc,EAAE;AAC/C,SAAO,YAAY,KAAK,KAAK,GAAG,OAAO;AACzC;AAaO,SAAS,sBAAsB,QAAgD;AACpF,MAAI,WAAW,OAAW,QAAO;AACjC,aAAW,MAAM,QAAQ;AACvB,UAAM,OAAO,GAAG,WAAW,CAAC;AAE5B,QAAI,OAAO,MAAQ,SAAS,KAAM;AAChC,YAAM,IAAI,gBAAgB,4CAA4C;AAAA,IACxE;AAAA,EACF;AACA,MAAI,OAAO,SAAS,IAAI,GAAG;AACzB,UAAM,IAAI;AAAA,MACR,4EAA4E,KAAK,UAAU,MAAM,CAAC;AAAA,IACpG;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,cAAU,mBAAmB,MAAM;AAAA,EACrC,QAAQ;AACN,cAAU;AAAA,EACZ;AACA,aAAW,UAAU,YAAY,SAAS,CAAC,MAAM,IAAI,CAAC,QAAQ,OAAO,GAAG;AACtE,eAAW,WAAW,OAAO,MAAM,GAAG,GAAG;AACvC,UAAI,YAAY,OAAO,YAAY,MAAM;AACvC,cAAM,IAAI;AAAA,UACR,sDAAsD,KAAK,UAAU,MAAM,CAAC;AAAA,QAC9E;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAGO,SAAS,eAAe,QAAoC;AACjE,SAAO,GAAG,WAAW,MAAM,CAAC;AAC9B;AAGO,SAAS,kBAAkB,QAA4B,KAAyB;AACrF,qBAAmB,GAAG;AACtB,SAAO,GAAG,eAAe,MAAM,CAAC,GAAG,iBAAiB,IAAI,SAAS,CAAC,IAAI,iBAAiB,IAAI,OAAO,CAAC,GAAG,eAAe;AACvH;AAGO,SAAS,mBAAmB,QAA4B,WAA4B;AACzF,QAAM,OAAO,eAAe,MAAM;AAClC,SAAO,cAAc,SAAY,OAAO,GAAG,IAAI,GAAG,iBAAiB,SAAS,CAAC;AAC/E;AAQO,SAAS,iBAAiB,QAA4B,WAAsC;AACjG,QAAM,OAAO,eAAe,MAAM;AAClC,MAAI,CAAC,UAAU,WAAW,IAAI,KAAK,CAAC,UAAU,SAAS,eAAe,EAAG,QAAO;AAChF,QAAM,OAAO,UAAU,MAAM,KAAK,QAAQ,UAAU,SAAS,gBAAgB,MAAM;AACnF,QAAM,QAAQ,KAAK,QAAQ,GAAG;AAC9B,MAAI,QAAQ,EAAG,QAAO;AACtB,QAAM,SAAS,KAAK,MAAM,GAAG,KAAK;AAClC,QAAM,iBAAiB,KAAK,MAAM,QAAQ,CAAC;AAC3C,MAAI,eAAe,WAAW,KAAK,eAAe,SAAS,GAAG,EAAG,QAAO;AACxE,QAAM,UAAU,kBAAkB,cAAc;AAChD,QAAM,YAAY,WAAW,oBAAoB,SAAY,kBAAkB,MAAM;AAGrF,MAAI,iBAAiB,OAAO,MAAM,eAAgB,QAAO;AACzD,MAAI,cAAc,UAAa,iBAAiB,SAAS,MAAM,OAAQ,QAAO;AAC9E,QAAM,MAAkB,EAAE,SAAS,UAAU;AAC7C,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,SAAO;AACT;;;ACtBO,IAAM,gBAAgB,IAAI,OAAO;AAExC,IAAM,sBAAsB;AAE5B,IAAM,oBAAoB;AAE1B,IAAM,wBAAwB;AAgBvB,IAAM,qBAAN,cAAiC,MAAM;AAAA,EAC5C,YAAY,WAAmB;AAC7B,UAAM,6CAA6C,SAAS,EAAE;AAC9D,SAAK,OAAO;AAAA,EACd;AACF;AA6DO,IAAM,sBAAN,MAAqD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAM1D,YACmB,OACA,QACA,KACA,UAAmB,kBACpC;AAJiB;AACA;AACA;AACA;AAAA,EAChB;AAAA,EAJgB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAGnB,eAAwB;AACtB,UAAM,oBAAoB,KAAK,YAAY;AAC3C,WAAO,mBAAmB,KAAK,OAAO,IAClC,EAAE,YAAY,MAAM,kBAAkB,IACtC,EAAE,YAAY,MAAM,UAAU,OAAO,kBAAkB;AAAA,EAC7D;AAAA;AAAA,EAGQ,cAAuB;AAC7B,WAAO,KAAK,MAAM,sBAAsB,QAAQ,OAAO,KAAK,MAAM,WAAW;AAAA,EAC/E;AAAA,EAEA,MAAM,IAAI,KAAiD;AACzD,UAAM,UAAU,MAAM,KAAK,QAAQ,kBAAkB,KAAK,QAAQ,GAAG,CAAC;AACtE,WAAO,WAAW,CAAC,QAAQ,IAAI,UAAU,QAAQ,IAAI,SAAS;AAAA,EAChE;AAAA,EAEA,MAAM,OAAO,KAAiB,QAAsD;AAClF,UAAM,UAAU,0BAA0B,MAAM;AAChD,UAAM,MAAM,kBAAkB,KAAK,QAAQ,GAAG;AAC9C,UAAM,UAAU,MAAM,KAAK,QAAQ,GAAG;AACtC,QAAI,YAAY,QAAQ,CAAC,QAAQ,IAAI,SAAS;AAC5C,YAAM,IAAI,mBAAmB,2CAA2C,IAAI,OAAO,EAAE;AAAA,IACvF;AAEA,UAAM,QAAQ,oBAAoB,KAAK,SAAS,SAAS,IAAI,MAAM;AACnE,UAAM,MAAwB;AAAA,MAC5B,SAAS;AAAA,MACT,QAAQ,cAAc,KAAK,SAAS,KAAK,IAAI,GAAG,KAAK;AAAA,IACvD;AAGA,UAAM,KAAK,OAAO,KAAK,KAAK,UAAU,EAAE,SAAS,QAAQ,QAAQ,IAAI,QAAQ;AAC7E,WAAO,EAAE,MAAM;AAAA,EACjB;AAAA,EAEA,MAAM,eACJ,KACA,UACA,OAC2B;AAC3B,UAAM,UAAU,sBAAsB,KAAK;AAC3C,UAAM,MAAM,kBAAkB,KAAK,QAAQ,GAAG;AAC9C,UAAM,UAAU,MAAM,KAAK,QAAQ,GAAG;AACtC,QAAI,YAAY,QAAQ,QAAQ,IAAI,WAAW,QAAQ,IAAI,OAAO,UAAU,UAAU;AACpF,YAAM,IAAI,mBAAmB,uCAAuC,IAAI,OAAO,EAAE;AAAA,IACnF;AACA,UAAM,QAAQ,kBAAkB,QAAQ,IAAI,QAAQ,KAAK,OAAO;AAChE,UAAM,MAAwB;AAAA,MAC5B,SAAS;AAAA,MACT,QAAQ,mBAAmB,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,GAAG,KAAK;AAAA,IAC3E;AAEA,UAAM,KAAK,OAAO,KAAK,KAAK,EAAE,SAAS,QAAQ,QAAQ,CAAC;AACxD,WAAO,EAAE,MAAM;AAAA,EACjB;AAAA,EAEA,OAAO,KAAK,WAAmD;AAC7D,UAAM,SAAS,mBAAmB,KAAK,QAAQ,SAAS;AAExD,QAAI,OAAiB,CAAC;AACtB,UAAM,QAAQ,iBAEZ,MAC+B;AAC/B,YAAM,OAAO,MAAM;AAAA,QAAmB;AAAA,QAAM;AAAA,QAAuB,CAAC,QAClE,KAAK,QAAQ,GAAG;AAAA,MAClB;AACA,iBAAW,WAAW,MAAM;AAC1B,YAAI,WAAW,CAAC,QAAQ,IAAI,QAAS,OAAM,QAAQ,IAAI;AAAA,MACzD;AAAA,IACF,EAAE,KAAK,IAAI;AAEX,qBAAiB,OAAO,KAAK,MAAM,SAAS,MAAM,GAAG;AACnD,UAAI,iBAAiB,KAAK,QAAQ,GAAG,MAAM,KAAM;AACjD,WAAK,KAAK,GAAG;AACb,UAAI,KAAK,UAAU,uBAAuB;AACxC,eAAO,MAAM,IAAI;AACjB,eAAO,CAAC;AAAA,MACV;AAAA,IACF;AACA,QAAI,KAAK,SAAS,EAAG,QAAO,MAAM,IAAI;AAAA,EACxC;AAAA,EAEA,MAAM,OAAO,KAAiB,UAAiC;AAC7D,UAAM,MAAM,kBAAkB,KAAK,QAAQ,GAAG;AAM9C,aAAS,UAAU,GAAG,UAAU,qBAAqB,WAAW;AAC9D,YAAM,UAAU,MAAM,KAAK,QAAQ,GAAG;AACtC,UAAI,aAAa,QAAW;AAG1B,YAAI,YAAY,QAAQ,QAAQ,IAAI,WAAW,QAAQ,IAAI,OAAO,UAAU,UAAU;AACpF,gBAAM,IAAI,mBAAmB,uCAAuC,IAAI,OAAO,EAAE;AAAA,QACnF;AAAA,MACF,WAAW,YAAY,QAAQ,QAAQ,IAAI,SAAS;AAClD;AAAA,MACF;AACA,UAAI;AACF,cAAM,SAAS,KAAK,YAAY,IAAI,KAAK,MAAM,SAAS;AACxD,YAAI,WAAW,UAAa,cAAc,QAAQ,IAAI,OAAO,KAAK,MAAM,QAAW;AAEjF,gBAAM,OAAO,KAAK,KAAK,OAAO,KAAK,EAAE,SAAS,QAAQ,QAAQ,CAAC;AAC/D;AAAA,QACF;AACA,cAAM,QAAQ,kBAAkB,QAAQ,IAAI,QAAQ,KAAK,OAAO;AAChE,cAAM,MAAwB;AAAA,UAC5B,SAAS;AAAA,UACT,QAAQ,EAAE,GAAG,QAAQ,IAAI,QAAQ,OAAO,WAAW,KAAK,IAAI,EAAE;AAAA,QAChE;AACA,cAAM,KAAK,OAAO,KAAK,KAAK,EAAE,SAAS,QAAQ,QAAQ,CAAC;AACxD;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,qBAAqB,GAAG,EAAG;AAC/B,cAAM;AAAA,MACR;AAAA,IACF;AACA,UAAM,IAAI,mBAAmB,yCAAyC,IAAI,OAAO,gBAAW;AAAA,EAC9F;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAc,QACZ,WAC4D;AAC5D,UAAM,MAAM,MAAM,KAAK,UAAU,SAAS;AAC1C,QAAI,QAAQ,KAAM,QAAO;AACzB,QAAI,IAAI,MAAM,SAAS,eAAe;AACpC,YAAM,IAAI;AAAA,QACR,mBAAmB,IAAI,MAAM,MAAM,iBAAiB,aAAa;AAAA,MACnE;AAAA,IACF;AACA,QAAI,IAAI,QAAQ,WAAW,GAAG;AAE5B,YAAM,IAAI;AAAA,QACR,uEAAuE,SAAS;AAAA,MAClF;AAAA,IACF;AACA,UAAM,MAAM,sBAAsB,IAAI,YAAY,EAAE,OAAO,IAAI,KAAK,GAAG,SAAS;AAChF,WAAO,EAAE,KAAK,SAAS,IAAI,QAAQ;AAAA,EACrC;AAAA;AAAA,EAGA,MAAc,UAAU,WAA8C;AACpE,aAAS,UAAU,GAAG,UAAU,mBAAmB,WAAW;AAC5D,UAAI;AACF,eAAO,MAAM,KAAK,MAAM,KAAK,SAAS;AAAA,MACxC,SAAS,KAAK;AACZ,YAAI,EAAE,eAAe,oBAAqB,OAAM;AAAA,MAClD;AAAA,IACF;AACA,UAAM,IAAI;AAAA,MACR,kBAAkB,KAAK,MAAM,KAAK,YAAY,SAAS,qCAClD,iBAAiB;AAAA,IACxB;AAAA,EACF;AAAA;AAAA,EAGA,MAAc,OACZ,WACA,KACA,QACe;AACf,UAAM,OAAO,IAAI,YAAY,EAAE,OAAO,0BAA0B,GAAG,CAAC;AACpE,QAAI,KAAK,SAAS,eAAe;AAG/B,YAAM,IAAI,gBAAgB,mBAAmB,KAAK,MAAM,iBAAiB,aAAa,GAAG;AAAA,IAC3F;AACA,UAAM,KAAK,MAAM,MAAM,WAAW,MAAM,MAAM;AAAA,EAChD;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { Entropy } from '../../core/determinism.js';
|
|
2
|
+
/** Draws from the platform's Web Crypto; `UnsupportedError` on a runtime without it. */
|
|
3
|
+
export declare const webCryptoEntropy: Entropy;
|
|
4
|
+
/** Whether `entropy` can draw here: an injected source always, the default only where Web Crypto exists. */
|
|
5
|
+
export declare function entropyIsAvailable(entropy: Entropy): boolean;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { IRegistryDriver, NewRegistryRecord, RegCaps, RegistryPatch, RegistryRecord, SegmentRef, Token } from '../../core/ports.js';
|
|
2
|
+
import type { Entropy } from '../../core/determinism.js';
|
|
2
3
|
/** Defensive cap on a single registry object read from storage, before allocation (rows are tiny; ~1 KB). */
|
|
3
4
|
export declare const MAX_ROW_BYTES: number;
|
|
4
5
|
/**
|
|
@@ -8,11 +9,12 @@ export declare const MAX_ROW_BYTES: number;
|
|
|
8
9
|
* This is deliberately **neither** of the two things it superficially resembles. It is not absence: the row
|
|
9
10
|
* is very likely still live, just one version further on. And it is not a write conflict: the caller may be
|
|
10
11
|
* a read-only `get()`, which has nothing to conflict with. Either is an easy misreading for a store that reads a
|
|
11
|
-
* row in two calls,
|
|
12
|
+
* row in two calls, metadata and then bytes pinned to it, which is why the signal is part of the port rather than
|
|
12
13
|
* left to each store's judgement. {@link ObjectStoreRegistry} answers it the only way that is correct for every
|
|
13
14
|
* caller: it reads again.
|
|
14
15
|
*
|
|
15
|
-
* Stores whose read is atomic
|
|
16
|
+
* Stores whose read is atomic can never raise it, and every shipped store's is: S3 serves bytes and `ETag` from one
|
|
17
|
+
* `GetObject`, GCS bytes and `generation` from one GET, and Azure Blob bytes and `ETag` from one GET.
|
|
16
18
|
*/
|
|
17
19
|
export declare class ObjectVersionRaced extends Error {
|
|
18
20
|
constructor(objectKey: string);
|
|
@@ -24,8 +26,8 @@ export interface ObjectRow {
|
|
|
24
26
|
readonly version: string;
|
|
25
27
|
}
|
|
26
28
|
/**
|
|
27
|
-
* The
|
|
28
|
-
* tombstones, the delete retry, key parsing — is implemented once above.
|
|
29
|
+
* The I/O calls an object store must provide to host a registry: three, and an optional conditional delete. Everything
|
|
30
|
+
* else — the OCC token, tombstones, the delete retry, key parsing — is implemented once above.
|
|
29
31
|
*/
|
|
30
32
|
export interface ObjectRegistryStore {
|
|
31
33
|
/** A label for error messages, e.g. `GCS` or `Azure Blob`. */
|
|
@@ -57,13 +59,41 @@ export interface ObjectRegistryStore {
|
|
|
57
59
|
}): Promise<void>;
|
|
58
60
|
/** Every object key under `prefix`, paginated internally. */
|
|
59
61
|
listKeys(prefix: string): AsyncIterable<string>;
|
|
62
|
+
/**
|
|
63
|
+
* Optional: delete one object, and only while it is still at exactly `expect.version`, by a precondition the
|
|
64
|
+
* backend applies (`If-Match: <etag>`, `ifGenerationMatch: <generation>`). An object that was overwritten, or is
|
|
65
|
+
* gone, MUST throw {@link WriteConflictError} and delete nothing. Retryable faults throw {@link TransientError}.
|
|
66
|
+
*
|
|
67
|
+
* Sending it again after a lost response is safe, unlike a write: the precondition names one version, so a second
|
|
68
|
+
* copy can remove nothing the first was not allowed to. A copy that meets its own landed delete fails the
|
|
69
|
+
* precondition and reports a conflict; the registry re-reads, and finds the row gone.
|
|
70
|
+
*
|
|
71
|
+
* The registry calls it only when {@link conditionalDelete} is `true`.
|
|
72
|
+
*/
|
|
73
|
+
delete?(key: string, expect: {
|
|
74
|
+
version: string;
|
|
75
|
+
}): Promise<void>;
|
|
76
|
+
/**
|
|
77
|
+
* Whether {@link delete} may be relied on: `true` only where the backend is known to apply the precondition. A
|
|
78
|
+
* backend that accepts the header and ignores it would let a delete remove a row written after the delete read it,
|
|
79
|
+
* a live one included. Absent or `false`, the registry never calls `delete` and tombstones every row instead.
|
|
80
|
+
*/
|
|
81
|
+
readonly conditionalDelete?: boolean;
|
|
60
82
|
}
|
|
61
83
|
export declare class ObjectStoreRegistry implements IRegistryDriver {
|
|
62
84
|
private readonly store;
|
|
63
85
|
private readonly prefix;
|
|
64
86
|
private readonly now;
|
|
65
|
-
|
|
87
|
+
private readonly entropy;
|
|
88
|
+
/**
|
|
89
|
+
* `entropy` draws the random parts of every token: a new row's incarnation id, and each write's own part. It
|
|
90
|
+
* defaults to the platform's Web Crypto. Inject one only to make a test replayable: a seeded source in production
|
|
91
|
+
* gives every process the same ids.
|
|
92
|
+
*/
|
|
93
|
+
constructor(store: ObjectRegistryStore, prefix: string | undefined, now: () => number, entropy?: Entropy);
|
|
66
94
|
capabilities(): RegCaps;
|
|
95
|
+
/** Whether a delete removes a row born with an incarnation id: the store has a delete, and vouches for it. */
|
|
96
|
+
private removesRows;
|
|
67
97
|
get(ref: SegmentRef): Promise<RegistryRecord | null>;
|
|
68
98
|
create(ref: SegmentRef, record: NewRegistryRecord): Promise<{
|
|
69
99
|
token: Token;
|
|
@@ -79,7 +109,7 @@ export declare class ObjectStoreRegistry implements IRegistryDriver {
|
|
|
79
109
|
* A store that pins a version to read consistently ({@link ObjectVersionRaced}) loses that pin whenever a
|
|
80
110
|
* concurrent writer lands mid-read. Re-reading is the whole answer — the next read simply observes the
|
|
81
111
|
* newer version — and it is bounded so a row under permanent write saturation fails typed instead of
|
|
82
|
-
* spinning. Only a store
|
|
112
|
+
* spinning. Only a store that reads in two calls gets here; none of the shipped stores does.
|
|
83
113
|
*/
|
|
84
114
|
private readRow;
|
|
85
115
|
/** {@link ObjectRegistryStore.read}, retrying the bounded number of times a mid-read overwrite allows. */
|
|
@@ -1,17 +1,27 @@
|
|
|
1
|
+
import type { Entropy } from '../../core/determinism.js';
|
|
2
|
+
import { incarnationOf } from '../../core/token.js';
|
|
1
3
|
import type { NewRegistryRecord, RegistryPatch, RegistryRecord, SegmentRef, Token } from '../../core/ports.js';
|
|
2
4
|
/**
|
|
3
5
|
* The fields a stored record may carry: {@link RegistryRecord}'s, and nothing else. A field no reader resolves
|
|
4
6
|
* through is refused on read-back like any other corruption (invariant 5), rather than ignored.
|
|
5
7
|
*/
|
|
6
|
-
export declare const RECORD_FIELDS: readonly ["namespace", "segment", "currentGen", "wrappedDeks", "keyId", "status", "retention", "residency", "createdAt", "updatedAt", "token"];
|
|
7
|
-
/**
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
export declare const RECORD_FIELDS: readonly ["namespace", "segment", "currentGen", "wrappedDeks", "keyId", "status", "retention", "residency", "summary", "createdAt", "updatedAt", "token"];
|
|
9
|
+
/**
|
|
10
|
+
* Validate the caller-settable fields at `create`, and return the record to store: the caller's, with its summary
|
|
11
|
+
* replaced by the checked, frozen copy {@link validateSummary} builds.
|
|
12
|
+
*/
|
|
13
|
+
export declare function validateNewRegistryRecord(rec: NewRegistryRecord): NewRegistryRecord;
|
|
14
|
+
/**
|
|
15
|
+
* Validate the caller-settable fields in a `compareAndSwap` patch, and return the patch to apply: the caller's, with
|
|
16
|
+
* its summary replaced by the checked, frozen copy. That a given `summary` names the row's resulting `currentGen`,
|
|
17
|
+
* and agrees with its resulting keys, needs the stored row, so {@link applyRegistryPatch} checks both.
|
|
18
|
+
*/
|
|
19
|
+
export declare function validateRegistryPatch(patch: RegistryPatch): RegistryPatch;
|
|
11
20
|
/**
|
|
12
21
|
* The persisted envelope for a registry row: the record plus a tombstone flag. A **deleted** row keeps its
|
|
13
|
-
* record (with an advanced counter) rather than being removed, so the
|
|
14
|
-
*
|
|
22
|
+
* record (with an advanced counter) rather than being removed, so a re-create carries the counter on and the two
|
|
23
|
+
* incarnations' tokens are apart by construction, as well as by their random parts — ABA-safety. Shared by the
|
|
24
|
+
* persistent drivers (LocalFs file, S3 object).
|
|
15
25
|
*
|
|
16
26
|
* `schemaVersion` is a wire-only concern — it stamps the persisted bytes, not the in-memory domain object —
|
|
17
27
|
* so it lives on the serialize/parse boundary ({@link serializeRegistryEnvelope} /
|
|
@@ -27,22 +37,35 @@ export interface RegistryEnvelope {
|
|
|
27
37
|
* with this so a reader can fail-closed on a future, incompatible layout instead of misparsing it. Bump only
|
|
28
38
|
* on a backward-incompatible change. Policy: a **higher** stamp than this build knows → `UnsupportedError`
|
|
29
39
|
* (fail-closed); an **absent** or malformed stamp → `IntegrityError`, since every row this build writes has one.
|
|
40
|
+
*
|
|
41
|
+
* Schema 2 adds the record's `summary` and the incarnation-form token. Every row this build writes is stamped 2,
|
|
42
|
+
* whatever it holds, and a build that reads only schema 1 refuses it, so a fleet cannot go back once one has been
|
|
43
|
+
* written.
|
|
30
44
|
*/
|
|
31
|
-
export declare const REGISTRY_SCHEMA_VERSION =
|
|
45
|
+
export declare const REGISTRY_SCHEMA_VERSION = 2;
|
|
32
46
|
/**
|
|
33
|
-
* Validate a persisted registry row's `schemaVersion` (untrusted bytes, invariant 5): an absent or
|
|
34
|
-
* → `IntegrityError`; a version newer than this build → `UnsupportedError` (fail-closed rather than
|
|
35
|
-
* format we don't understand).
|
|
47
|
+
* Validate a persisted registry row's `schemaVersion` (untrusted bytes, invariant 5) and return it: an absent or
|
|
48
|
+
* malformed value → `IntegrityError`; a version newer than this build → `UnsupportedError` (fail-closed rather than
|
|
49
|
+
* misread a format we don't understand).
|
|
36
50
|
*/
|
|
37
|
-
export declare function assertRegistrySchemaVersion(raw: unknown, ctx: string):
|
|
51
|
+
export declare function assertRegistrySchemaVersion(raw: unknown, ctx: string): number;
|
|
38
52
|
/** Serialize a registry envelope for persistence, stamping the current {@link REGISTRY_SCHEMA_VERSION}. */
|
|
39
53
|
export declare function serializeRegistryEnvelope(env: RegistryEnvelope): string;
|
|
54
|
+
export { incarnationOf };
|
|
55
|
+
/** A fresh incarnation id from `entropy`: 128 bits as 32 lowercase hex digits. */
|
|
56
|
+
export declare function drawIncarnation(entropy: Entropy): string;
|
|
57
|
+
/** A fresh write part from `entropy`: 64 bits as 16 lowercase hex digits. */
|
|
58
|
+
export declare function drawWrite(entropy: Entropy): string;
|
|
59
|
+
/**
|
|
60
|
+
* The token for the write after `record`'s: the same incarnation, if it has one, the counter one on, and a fresh write
|
|
61
|
+
* part. For a compare-and-swap or a tombstone.
|
|
62
|
+
*/
|
|
63
|
+
export declare function nextRegistryToken(record: RegistryRecord, entropy: Entropy): Token;
|
|
40
64
|
/**
|
|
41
|
-
* The
|
|
42
|
-
*
|
|
43
|
-
* tampering — reject it (invariant 5) rather than let `Number()` coerce it and silently break monotonicity.
|
|
65
|
+
* The token of a new row: a fresh incarnation and write part from `entropy`, and a counter that continues
|
|
66
|
+
* `tombstone`'s when the row is created over one, else starts at 0.
|
|
44
67
|
*/
|
|
45
|
-
export declare function
|
|
68
|
+
export declare function newIncarnationToken(entropy: Entropy, tombstone: RegistryRecord | undefined): Token;
|
|
46
69
|
/**
|
|
47
70
|
* Parse + structurally validate a persisted `{ deleted, record }` envelope from stored bytes. A published row
|
|
48
71
|
* is always whole (atomic write), so a parse failure or a missing/mistyped field means corruption/tampering —
|
|
@@ -54,13 +77,18 @@ export declare function recordFromNew(ref: SegmentRef, rec: NewRegistryRecord, n
|
|
|
54
77
|
/**
|
|
55
78
|
* Fail-fast structural check on a record read back from a persistent tier (untrusted bytes, invariant 5):
|
|
56
79
|
* a published row is always whole, so a missing/mistyped required field means corruption/tampering — reject
|
|
57
|
-
* it rather than silently report "absent".
|
|
58
|
-
*
|
|
80
|
+
* it rather than silently report "absent". A row of an older `schemaVersion` may carry only that schema's
|
|
81
|
+
* fields. `token` is checked by the caller.
|
|
59
82
|
*/
|
|
60
|
-
export declare function assertStoredRecordShape(r: Record<string, unknown>, ctx: string): void;
|
|
83
|
+
export declare function assertStoredRecordShape(r: Record<string, unknown>, ctx: string, schemaVersion?: number): void;
|
|
61
84
|
/**
|
|
62
85
|
* Apply a patch to an existing record, returning a new one with a fresh `updatedAt` + `token` (identity and
|
|
63
86
|
* `createdAt` are preserved). Optional fields use `'k' in patch` so a patch can *clear* them (set to
|
|
64
87
|
* `undefined`, e.g. dropping `keyId` on crypto-shred); required fields use `??` (they always have a value).
|
|
88
|
+
*
|
|
89
|
+
* `summary` describes the current generation, so it follows the pointer and the keys: a patch that moves
|
|
90
|
+
* `currentGen`, or changes `wrappedDeks` so the summary's shape no longer agrees, without mentioning `summary` drops
|
|
91
|
+
* the old one; and a summary the patch gives must name the resulting `currentGen` and agree with the resulting keys
|
|
92
|
+
* (`ValidationError`, before anything is written).
|
|
65
93
|
*/
|
|
66
94
|
export declare function applyRegistryPatch(prev: RegistryRecord, patch: RegistryPatch, now: number, token: Token): RegistryRecord;
|
|
@@ -1,14 +1,22 @@
|
|
|
1
|
+
import type { Entropy } from '../../core/determinism.js';
|
|
1
2
|
import type { IRegistryDriver, NewRegistryRecord, RegCaps, RegistryPatch, RegistryRecord, SegmentRef, Token } from '../../core/ports.js';
|
|
2
3
|
/** Entries in the process-wide row-lock map; the map is empty whenever no registry operation is in flight. */
|
|
3
4
|
export declare function localFsRowLockCount(): number;
|
|
4
5
|
export interface LocalFsRegistryDriverOptions {
|
|
5
6
|
/** Injected clock for `createdAt`/`updatedAt`; defaults to `Date.now`. */
|
|
6
7
|
readonly now?: () => number;
|
|
8
|
+
/** Draws every token's random parts; defaults to Web Crypto. Inject one only to make a test replayable. */
|
|
9
|
+
readonly entropy?: Entropy;
|
|
7
10
|
}
|
|
8
11
|
export declare class LocalFsRegistryDriver implements IRegistryDriver {
|
|
9
12
|
private readonly root;
|
|
10
13
|
private readonly now;
|
|
14
|
+
private readonly entropy;
|
|
11
15
|
constructor(root: string, options?: LocalFsRegistryDriverOptions);
|
|
16
|
+
/**
|
|
17
|
+
* `conditionalDelete`: a delete reads the row, checks it, and unlinks it under the row's lock, which every writer on
|
|
18
|
+
* the root in this process takes, so no write can land between the check and the unlink.
|
|
19
|
+
*/
|
|
12
20
|
capabilities(): RegCaps;
|
|
13
21
|
get(ref: SegmentRef): Promise<RegistryRecord | null>;
|
|
14
22
|
create(ref: SegmentRef, record: NewRegistryRecord): Promise<{
|
|
@@ -19,6 +27,8 @@ export declare class LocalFsRegistryDriver implements IRegistryDriver {
|
|
|
19
27
|
}>;
|
|
20
28
|
list(namespace?: string): AsyncIterable<RegistryRecord>;
|
|
21
29
|
delete(ref: SegmentRef, expected?: Token): Promise<void>;
|
|
30
|
+
/** Remove a row's file and make the removal durable (fsync the directory). */
|
|
31
|
+
private unlinkRow;
|
|
22
32
|
/** Namespaces to scan: just the one requested, or every namespace dir under the root. */
|
|
23
33
|
private namespaceDirs;
|
|
24
34
|
private readRow;
|
package/dist/drivers/memory.d.ts
CHANGED
|
@@ -1,20 +1,26 @@
|
|
|
1
1
|
import type { BlobSink } from '../core/blob.js';
|
|
2
|
+
import type { Entropy } from '../core/determinism.js';
|
|
2
3
|
import type { StorageCaps, GenKey, IStorageDriver, IRegistryDriver, NewRegistryRecord, RegCaps, RegistryPatch, RegistryRecord, SegmentRef, Token } from '../core/ports.js';
|
|
3
4
|
export interface MemoryRegistryDriverOptions {
|
|
4
5
|
/** Injected clock for `createdAt`/`updatedAt`; defaults to `Date.now` (drivers may use ambient time). */
|
|
5
6
|
readonly now?: () => number;
|
|
7
|
+
/** Draws every token's random parts; defaults to Web Crypto. Inject one only to make a test replayable. */
|
|
8
|
+
readonly entropy?: Entropy;
|
|
6
9
|
}
|
|
7
10
|
/**
|
|
8
|
-
* In-memory {@link IRegistryDriver} — one record per segment under OCC. The token
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
+
* In-memory {@link IRegistryDriver} — one record per segment under OCC. The token has the form every shipped driver
|
|
12
|
+
* gives it, an incarnation id drawn at create, a counter and a per-write random part, but its counter is global to the
|
|
13
|
+
* driver: monotonic and never reused, so a record recreated after `delete` never meets an earlier token, by
|
|
14
|
+
* construction, even though `delete` removes the row physically.
|
|
11
15
|
*/
|
|
12
16
|
export declare class MemoryRegistryDriver implements IRegistryDriver {
|
|
13
17
|
private readonly rows;
|
|
14
18
|
private readonly now;
|
|
19
|
+
private readonly entropy;
|
|
15
20
|
private seq;
|
|
16
21
|
constructor(options?: MemoryRegistryDriverOptions);
|
|
17
22
|
private nextToken;
|
|
23
|
+
/** `conditionalDelete`: a delete checks the token and removes the row in one step, with no await between. */
|
|
18
24
|
capabilities(): RegCaps;
|
|
19
25
|
get(ref: SegmentRef): Promise<RegistryRecord | null>;
|
|
20
26
|
create(ref: SegmentRef, record: NewRegistryRecord): Promise<{
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
*/
|
|
22
22
|
import type { Clock, Rng } from '../../core/determinism.js';
|
|
23
23
|
import type { RetryPolicy } from '../../core/retry.js';
|
|
24
|
-
import type { ChunkRef, StorageChunkSource, SegmentRef, SegmentSize } from '../../core/ports.js';
|
|
24
|
+
import type { ChunkRef, GenerationSummary, StorageChunkSource, SegmentRef, SegmentSize } from '../../core/ports.js';
|
|
25
25
|
export interface RetryingOptions {
|
|
26
26
|
readonly clock: Clock;
|
|
27
27
|
readonly rng: Rng;
|
|
@@ -44,6 +44,7 @@ export declare class RetryingStorageChunkSource implements StorageChunkSource {
|
|
|
44
44
|
/** Present only when the inner source supports it — so capability detection stays honest. */
|
|
45
45
|
readonly sizeOf?: (ref: SegmentRef) => Promise<SegmentSize | null>;
|
|
46
46
|
readonly cardinalities?: (ref: SegmentRef) => Promise<ReadonlyMap<number, number> | null>;
|
|
47
|
+
readonly summary?: (ref: SegmentRef) => Promise<GenerationSummary | null>;
|
|
47
48
|
readonly currentGeneration?: (ref: SegmentRef) => Promise<number | null>;
|
|
48
49
|
readonly invalidate?: (ref: SegmentRef) => void;
|
|
49
50
|
readonly exists?: (ref: SegmentRef) => Promise<boolean>;
|
package/dist/index.d.ts
CHANGED
|
@@ -37,13 +37,14 @@ export { PinnedStorageChunkSource } from './core/pinned-storage-source.js';
|
|
|
37
37
|
export type { PinnedAt } from './core/pinned-storage-source.js';
|
|
38
38
|
export { loadSegment } from './core/load.js';
|
|
39
39
|
export type { LoadDeps, LoadOptions, LoadGuard, LoadResult, LoadRefusal } from './core/load.js';
|
|
40
|
+
export type { LoadInput, PortableBitmap } from './core/load-input.js';
|
|
40
41
|
export { listGenerations, rollbackSegment } from './core/rollback.js';
|
|
41
42
|
export type { GenerationListDeps, GenerationEntry, RollbackResult } from './core/rollback.js';
|
|
42
43
|
export { eraseIdFromSegment } from './core/erase-id.js';
|
|
43
44
|
export type { EraseIdDeps, EraseIdResult } from './core/erase-id.js';
|
|
44
|
-
export type { CodecInterface, CodecBitmap } from './core/codec.js';
|
|
45
|
-
export type { Clock, Rng } from './core/determinism.js';
|
|
46
|
-
export type { StorageChunkSource, IStorageDriver, StorageCaps, ChunkRef, SegmentRef, GenKey, IRegistryDriver, StorageBackend, RegistryRecord, NewRegistryRecord, RegistryPatch, RegistryStatus, RegCaps, GovernanceMeta, SegmentSize, } from './core/ports.js';
|
|
45
|
+
export type { CodecInterface, CodecBitmap, EncodedChunk } from './core/codec.js';
|
|
46
|
+
export type { Clock, Entropy, Rng } from './core/determinism.js';
|
|
47
|
+
export type { StorageChunkSource, IStorageDriver, StorageCaps, ChunkRef, SegmentRef, GenKey, IRegistryDriver, StorageBackend, RegistryRecord, NewRegistryRecord, RegistryPatch, RegistryStatus, RegistrySummary, ClearRegistrySummary, SealedRegistrySummary, GenerationMetadata, GenerationSummary, RegCaps, GovernanceMeta, SegmentSize, } from './core/ports.js';
|
|
47
48
|
export { CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, } from './core/errors.js';
|
|
48
49
|
export type { Aead, AeadSealed, IKeystore, WrappedDek, CrbmCrypto } from './core/crypto.js';
|
|
49
50
|
export { aadFor } from './core/crypto.js';
|