@bjornpagen/bumbledb-log 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -15
- package/package.json +3 -2
- package/src/braids.ts +5 -8
- package/src/bytes.ts +2 -2
- package/src/chain.ts +16 -17
- package/src/codec.ts +21 -23
- package/src/descriptor.ts +245 -276
- package/src/errors.ts +8 -6
- package/src/index.ts +10 -8
- package/src/keys.ts +69 -15
- package/src/manifest.ts +11 -7
- package/src/replica.ts +42 -44
- package/src/store-s3.ts +222 -0
- package/src/store.ts +76 -31
- package/src/value.ts +16 -16
- package/src/writer.ts +37 -34
package/src/store-s3.ts
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `s3Store`: the five verbs over one S3-compatible target. The official
|
|
3
|
+
* `@aws-sdk/client-s3` client signs and talks; this module maps HTTP
|
|
4
|
+
* outcomes onto the sums. One storage class for every key. The vendor
|
|
5
|
+
* ETag header is the opaque token, carried verbatim. R2 rides region
|
|
6
|
+
* `auto` and a required endpoint.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"
|
|
10
|
+
import * as errors from "@superbuilders/errors"
|
|
11
|
+
import { wrapStore } from "#errors.ts"
|
|
12
|
+
import type { StoreKey } from "#keys.ts"
|
|
13
|
+
import type { Etag, ObjectStore } from "#store.ts"
|
|
14
|
+
|
|
15
|
+
interface StaticKeys {
|
|
16
|
+
readonly accessKeyId: string
|
|
17
|
+
readonly secretAccessKey: string
|
|
18
|
+
readonly sessionToken?: string
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Static keys, or a caller-owned refresh the store invokes before each signed request. */
|
|
22
|
+
type S3Credentials = StaticKeys | (() => StaticKeys | Promise<StaticKeys>)
|
|
23
|
+
|
|
24
|
+
interface S3Config {
|
|
25
|
+
readonly endpoint?: string
|
|
26
|
+
readonly region: string
|
|
27
|
+
readonly bucket: string
|
|
28
|
+
readonly credentials: S3Credentials
|
|
29
|
+
readonly prefix?: string
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function asEtag(raw: string): Etag {
|
|
33
|
+
return raw as Etag
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function normalizePrefix(raw: string | undefined): string {
|
|
37
|
+
if (raw === undefined || raw.length === 0) {
|
|
38
|
+
return ""
|
|
39
|
+
}
|
|
40
|
+
const trimmed = raw.replace(/^\/+|\/+$/g, "")
|
|
41
|
+
if (trimmed.length === 0 || trimmed.split("/").some((seg) => seg.length === 0)) {
|
|
42
|
+
throw errors.new(`store prefix is not a slash path: ${raw}`)
|
|
43
|
+
}
|
|
44
|
+
return trimmed
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function joinPrefix(prefix: string, key: string): string {
|
|
48
|
+
return prefix.length === 0 ? key : `${prefix}/${key}`
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function resolveKeys(credentials: S3Credentials): Promise<StaticKeys> {
|
|
52
|
+
if (typeof credentials === "function") {
|
|
53
|
+
return await credentials()
|
|
54
|
+
}
|
|
55
|
+
return credentials
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function httpStatus(error: unknown): number | undefined {
|
|
59
|
+
if (error === null || typeof error !== "object" || !("$metadata" in error)) {
|
|
60
|
+
return undefined
|
|
61
|
+
}
|
|
62
|
+
const metadata = error.$metadata
|
|
63
|
+
if (metadata === null || typeof metadata !== "object" || !("httpStatusCode" in metadata)) {
|
|
64
|
+
return undefined
|
|
65
|
+
}
|
|
66
|
+
const status = metadata.httpStatusCode
|
|
67
|
+
return typeof status === "number" ? status : undefined
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function asError(error: unknown): Error {
|
|
71
|
+
if (error instanceof Error) {
|
|
72
|
+
return error
|
|
73
|
+
}
|
|
74
|
+
return errors.new(String(error))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function etagOf(raw: string | undefined, op: string, key: StoreKey): Etag {
|
|
78
|
+
if (raw === undefined || raw.length === 0) {
|
|
79
|
+
throw wrapStore(errors.new("vendor omitted ETag"), `${op} ${key}`)
|
|
80
|
+
}
|
|
81
|
+
return asEtag(raw)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function s3Store(config: S3Config): ObjectStore {
|
|
85
|
+
if (config.region === "auto" && config.endpoint === undefined) {
|
|
86
|
+
throw errors.new("region auto needs an endpoint")
|
|
87
|
+
}
|
|
88
|
+
const prefix = normalizePrefix(config.prefix)
|
|
89
|
+
const client = new S3Client({
|
|
90
|
+
region: config.region,
|
|
91
|
+
credentials: async function credentials() {
|
|
92
|
+
const keys = await resolveKeys(config.credentials)
|
|
93
|
+
return {
|
|
94
|
+
accessKeyId: keys.accessKeyId,
|
|
95
|
+
secretAccessKey: keys.secretAccessKey,
|
|
96
|
+
...(keys.sessionToken === undefined ? {} : { sessionToken: keys.sessionToken })
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
requestChecksumCalculation: "WHEN_REQUIRED",
|
|
100
|
+
responseChecksumValidation: "WHEN_REQUIRED",
|
|
101
|
+
...(config.endpoint === undefined ? {} : { endpoint: config.endpoint, forcePathStyle: true })
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
function objectKey(key: StoreKey): string {
|
|
105
|
+
return joinPrefix(prefix, key)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
async get(key) {
|
|
110
|
+
const ran = await errors.try(
|
|
111
|
+
client.send(
|
|
112
|
+
new GetObjectCommand({
|
|
113
|
+
Bucket: config.bucket,
|
|
114
|
+
Key: objectKey(key)
|
|
115
|
+
})
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
if (ran.error) {
|
|
119
|
+
if (httpStatus(ran.error) === 404) {
|
|
120
|
+
return null
|
|
121
|
+
}
|
|
122
|
+
throw wrapStore(asError(ran.error), `get ${key}`)
|
|
123
|
+
}
|
|
124
|
+
if (ran.data.Body === undefined) {
|
|
125
|
+
throw wrapStore(errors.new("vendor omitted body"), `get ${key}`)
|
|
126
|
+
}
|
|
127
|
+
return {
|
|
128
|
+
bytes: await ran.data.Body.transformToByteArray(),
|
|
129
|
+
etag: etagOf(ran.data.ETag, "get", key)
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
|
|
133
|
+
async getIfChanged(key, etag) {
|
|
134
|
+
const ran = await errors.try(
|
|
135
|
+
client.send(
|
|
136
|
+
new GetObjectCommand({
|
|
137
|
+
Bucket: config.bucket,
|
|
138
|
+
Key: objectKey(key),
|
|
139
|
+
IfNoneMatch: etag
|
|
140
|
+
})
|
|
141
|
+
)
|
|
142
|
+
)
|
|
143
|
+
if (ran.error) {
|
|
144
|
+
if (httpStatus(ran.error) === 304) {
|
|
145
|
+
return { tag: "unchanged" }
|
|
146
|
+
}
|
|
147
|
+
throw wrapStore(asError(ran.error), `getIfChanged ${key}`)
|
|
148
|
+
}
|
|
149
|
+
if (ran.data.Body === undefined) {
|
|
150
|
+
throw wrapStore(errors.new("vendor omitted body"), `getIfChanged ${key}`)
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
tag: "changed",
|
|
154
|
+
fetched: {
|
|
155
|
+
bytes: await ran.data.Body.transformToByteArray(),
|
|
156
|
+
etag: etagOf(ran.data.ETag, "getIfChanged", key)
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
|
|
161
|
+
async putCreate(key, bytes) {
|
|
162
|
+
const ran = await errors.try(
|
|
163
|
+
client.send(
|
|
164
|
+
new PutObjectCommand({
|
|
165
|
+
Bucket: config.bucket,
|
|
166
|
+
Key: objectKey(key),
|
|
167
|
+
Body: bytes,
|
|
168
|
+
IfNoneMatch: "*"
|
|
169
|
+
})
|
|
170
|
+
)
|
|
171
|
+
)
|
|
172
|
+
if (ran.error) {
|
|
173
|
+
if (httpStatus(ran.error) === 412) {
|
|
174
|
+
return { tag: "exists" }
|
|
175
|
+
}
|
|
176
|
+
throw wrapStore(asError(ran.error), `putCreate ${key}`)
|
|
177
|
+
}
|
|
178
|
+
return { tag: "created", etag: etagOf(ran.data.ETag, "putCreate", key) }
|
|
179
|
+
},
|
|
180
|
+
|
|
181
|
+
async putSwap(key, bytes, etag) {
|
|
182
|
+
const ran = await errors.try(
|
|
183
|
+
client.send(
|
|
184
|
+
new PutObjectCommand({
|
|
185
|
+
Bucket: config.bucket,
|
|
186
|
+
Key: objectKey(key),
|
|
187
|
+
Body: bytes,
|
|
188
|
+
IfMatch: etag
|
|
189
|
+
})
|
|
190
|
+
)
|
|
191
|
+
)
|
|
192
|
+
if (ran.error) {
|
|
193
|
+
const status = httpStatus(ran.error)
|
|
194
|
+
if (status === 412 || status === 404) {
|
|
195
|
+
return { tag: "moved" }
|
|
196
|
+
}
|
|
197
|
+
throw wrapStore(asError(ran.error), `putSwap ${key}`)
|
|
198
|
+
}
|
|
199
|
+
return { tag: "swapped", etag: etagOf(ran.data.ETag, "putSwap", key) }
|
|
200
|
+
},
|
|
201
|
+
|
|
202
|
+
async delete(key) {
|
|
203
|
+
const ran = await errors.try(
|
|
204
|
+
client.send(
|
|
205
|
+
new DeleteObjectCommand({
|
|
206
|
+
Bucket: config.bucket,
|
|
207
|
+
Key: objectKey(key)
|
|
208
|
+
})
|
|
209
|
+
)
|
|
210
|
+
)
|
|
211
|
+
if (ran.error) {
|
|
212
|
+
if (httpStatus(ran.error) === 404) {
|
|
213
|
+
return
|
|
214
|
+
}
|
|
215
|
+
throw wrapStore(asError(ran.error), `delete ${key}`)
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export type { S3Config, S3Credentials, StaticKeys }
|
|
222
|
+
export { joinPrefix, s3Store }
|
package/src/store.ts
CHANGED
|
@@ -21,53 +21,45 @@ import { internalBlake3 } from "@bjornpagen/bumbledb"
|
|
|
21
21
|
import * as errors from "@superbuilders/errors"
|
|
22
22
|
import { toHex } from "#bytes.ts"
|
|
23
23
|
import { wrapStore } from "#errors.ts"
|
|
24
|
+
import type { StoreKey } from "#keys.ts"
|
|
25
|
+
import { LOCK_SUFFIX } from "#keys.ts"
|
|
26
|
+
|
|
27
|
+
declare const etagBrand: unique symbol
|
|
28
|
+
type Etag = string & { readonly [etagBrand]: typeof etagBrand }
|
|
29
|
+
|
|
30
|
+
function etag(raw: string): Etag {
|
|
31
|
+
return raw as Etag
|
|
32
|
+
}
|
|
24
33
|
|
|
25
34
|
interface Fetched {
|
|
26
35
|
readonly bytes: Uint8Array
|
|
27
|
-
readonly etag:
|
|
36
|
+
readonly etag: Etag
|
|
28
37
|
}
|
|
29
38
|
|
|
30
39
|
type Poll = { readonly tag: "unchanged" } | { readonly tag: "changed"; readonly fetched: Fetched }
|
|
31
40
|
|
|
32
|
-
type Create = { readonly tag: "created"; readonly etag:
|
|
41
|
+
type Create = { readonly tag: "created"; readonly etag: Etag } | { readonly tag: "exists" }
|
|
33
42
|
|
|
34
|
-
type Swap = { readonly tag: "swapped"; readonly etag:
|
|
43
|
+
type Swap = { readonly tag: "swapped"; readonly etag: Etag } | { readonly tag: "moved" }
|
|
35
44
|
|
|
36
45
|
interface ObjectStore {
|
|
37
46
|
/** GET; null on 404. */
|
|
38
|
-
get(key:
|
|
47
|
+
get(key: StoreKey): Promise<Fetched | null>
|
|
39
48
|
/** GET with If-None-Match — the cheap manifest poll. */
|
|
40
|
-
getIfChanged(key:
|
|
49
|
+
getIfChanged(key: StoreKey, etag: Etag): Promise<Poll>
|
|
41
50
|
/** PUT with If-None-Match: * — the log-slot arbitration primitive. */
|
|
42
|
-
putCreate(key:
|
|
51
|
+
putCreate(key: StoreKey, bytes: Uint8Array): Promise<Create>
|
|
43
52
|
/** PUT with If-Match — the manifest CAS primitive. */
|
|
44
|
-
putSwap(key:
|
|
53
|
+
putSwap(key: StoreKey, bytes: Uint8Array, etag: Etag): Promise<Swap>
|
|
45
54
|
/** DELETE, unconditional — the gc verb's tool. */
|
|
46
|
-
delete(key:
|
|
55
|
+
delete(key: StoreKey): Promise<void>
|
|
47
56
|
}
|
|
48
57
|
|
|
49
|
-
/** Suffix of the per-key pid-lockfile that serializes `putSwap`. */
|
|
50
|
-
const LOCK_SUFFIX = ".lock"
|
|
51
|
-
|
|
52
58
|
/** Ceiling of the jittered wait between probes of a live-held lock. */
|
|
53
59
|
const LOCK_RETRY_MS = 10
|
|
54
60
|
|
|
55
|
-
function contentEtag(bytes: Uint8Array):
|
|
56
|
-
return toHex(new Uint8Array(internalBlake3(bytes)))
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
function checkKey(key: string): void {
|
|
60
|
-
if (key.length === 0 || key.startsWith("/") || key.endsWith("/")) {
|
|
61
|
-
throw errors.new(`store key is not a slash path: ${key}`)
|
|
62
|
-
}
|
|
63
|
-
for (const segment of key.split("/")) {
|
|
64
|
-
if (segment.length === 0 || segment === "." || segment === "..") {
|
|
65
|
-
throw errors.new(`store key segment is illegal: ${key}`)
|
|
66
|
-
}
|
|
67
|
-
if (segment.endsWith(LOCK_SUFFIX)) {
|
|
68
|
-
throw errors.new(`store key collides with the lockfile suffix: ${key}`)
|
|
69
|
-
}
|
|
70
|
-
}
|
|
61
|
+
function contentEtag(bytes: Uint8Array): Etag {
|
|
62
|
+
return etag(toHex(new Uint8Array(internalBlake3(bytes))))
|
|
71
63
|
}
|
|
72
64
|
|
|
73
65
|
function codeOf(error: Error): string | undefined {
|
|
@@ -173,8 +165,7 @@ async function releaseLock(lockPath: string): Promise<void> {
|
|
|
173
165
|
function fsStore(root: string): ObjectStore {
|
|
174
166
|
const rootPath = path.resolve(root)
|
|
175
167
|
|
|
176
|
-
function objectPath(key:
|
|
177
|
-
checkKey(key)
|
|
168
|
+
function objectPath(key: StoreKey): string {
|
|
178
169
|
return path.join(rootPath, ...key.split("/"))
|
|
179
170
|
}
|
|
180
171
|
|
|
@@ -288,5 +279,59 @@ function fsStore(root: string): ObjectStore {
|
|
|
288
279
|
}
|
|
289
280
|
}
|
|
290
281
|
|
|
291
|
-
|
|
292
|
-
|
|
282
|
+
/**
|
|
283
|
+
* The five verbs over one in-process Map. Single-process only: tests
|
|
284
|
+
* and ephemeral dev inside this process, no persistence, no
|
|
285
|
+
* cross-process claim, no configuration. Third `Etag` producer beside
|
|
286
|
+
* `fsStore` and `s3Store`: blake3 of the content, `fsStore`'s mint,
|
|
287
|
+
* carried as the same opaque brand.
|
|
288
|
+
*/
|
|
289
|
+
function memStore(): ObjectStore {
|
|
290
|
+
const objects = new Map<StoreKey, Fetched>()
|
|
291
|
+
return {
|
|
292
|
+
async get(key) {
|
|
293
|
+
return objects.get(key) ?? null
|
|
294
|
+
},
|
|
295
|
+
|
|
296
|
+
async getIfChanged(key, etag) {
|
|
297
|
+
const current = objects.get(key)
|
|
298
|
+
if (current === undefined) {
|
|
299
|
+
throw wrapStore(errors.new("poll target absent"), `getIfChanged ${key}`)
|
|
300
|
+
}
|
|
301
|
+
if (current.etag === etag) {
|
|
302
|
+
return { tag: "unchanged" }
|
|
303
|
+
}
|
|
304
|
+
return { tag: "changed", fetched: current }
|
|
305
|
+
},
|
|
306
|
+
|
|
307
|
+
async putCreate(key, bytes) {
|
|
308
|
+
if (objects.has(key)) {
|
|
309
|
+
return { tag: "exists" }
|
|
310
|
+
}
|
|
311
|
+
const copy = new Uint8Array(bytes)
|
|
312
|
+
const tag = contentEtag(copy)
|
|
313
|
+
objects.set(key, { bytes: copy, etag: tag })
|
|
314
|
+
return { tag: "created", etag: tag }
|
|
315
|
+
},
|
|
316
|
+
|
|
317
|
+
async putSwap(key, bytes, etag) {
|
|
318
|
+
const current = objects.get(key)
|
|
319
|
+
if (current === undefined || current.etag !== etag) {
|
|
320
|
+
return { tag: "moved" }
|
|
321
|
+
}
|
|
322
|
+
const copy = new Uint8Array(bytes)
|
|
323
|
+
const tag = contentEtag(copy)
|
|
324
|
+
objects.set(key, { bytes: copy, etag: tag })
|
|
325
|
+
return { tag: "swapped", etag: tag }
|
|
326
|
+
},
|
|
327
|
+
|
|
328
|
+
async delete(key) {
|
|
329
|
+
objects.delete(key)
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
export type { S3Config, S3Credentials } from "#store-s3.ts"
|
|
335
|
+
export { s3Store } from "#store-s3.ts"
|
|
336
|
+
export type { Create, Etag, Fetched, ObjectStore, Poll, Swap }
|
|
337
|
+
export { etag, fsStore, memStore }
|
package/src/value.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The driver's raw-value vocabulary and its two canonical encodings:
|
|
3
|
-
* the command codec's tagged little-endian form (
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
3
|
+
* the command codec's tagged little-endian form (the tag table the
|
|
4
|
+
* batch wire already declared), and the engine's big-endian
|
|
5
|
+
* order-preserving literal form the fingerprint mirror reproduces.
|
|
6
|
+
* Values here are always RAW — strings as UTF-8, never intern ids —
|
|
7
|
+
* so a key hashed from a raw value cannot depend on a catalog.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import type { ValueTypeSpec } from "@bjornpagen/bumbledb"
|
|
@@ -12,12 +12,12 @@ import * as errors from "@superbuilders/errors"
|
|
|
12
12
|
import type { ByteReader, ByteWriter } from "#bytes.ts"
|
|
13
13
|
import { bytesEqual, I64_MAX, I64_MIN, U64_MAX, utf8Encoder, utf8StrictDecoder } from "#bytes.ts"
|
|
14
14
|
|
|
15
|
-
interface
|
|
15
|
+
interface Interval {
|
|
16
16
|
readonly start: bigint
|
|
17
17
|
readonly end: bigint
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
type
|
|
20
|
+
type Value = boolean | bigint | string | Uint8Array | Interval
|
|
21
21
|
|
|
22
22
|
/** 20's tag table: the codec's own numbering, normative to the byte. */
|
|
23
23
|
const TAG = {
|
|
@@ -30,7 +30,7 @@ const TAG = {
|
|
|
30
30
|
fixedInterval: 6
|
|
31
31
|
} as const
|
|
32
32
|
|
|
33
|
-
function isInterval(value:
|
|
33
|
+
function isInterval(value: Value): value is Interval {
|
|
34
34
|
return typeof value === "object" && !(value instanceof Uint8Array)
|
|
35
35
|
}
|
|
36
36
|
|
|
@@ -51,7 +51,7 @@ function wireTagOf(type: ValueTypeSpec): number {
|
|
|
51
51
|
}
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
function checkAgainst(context: string, type: ValueTypeSpec, value:
|
|
54
|
+
function checkAgainst(context: string, type: ValueTypeSpec, value: Value): void {
|
|
55
55
|
switch (type.kind) {
|
|
56
56
|
case "bool": {
|
|
57
57
|
if (typeof value !== "boolean") {
|
|
@@ -101,7 +101,7 @@ function checkAgainst(context: string, type: ValueTypeSpec, value: LogValue): vo
|
|
|
101
101
|
}
|
|
102
102
|
|
|
103
103
|
/** The codec's tagged form: `tag u8` + payload, at the field's layout. */
|
|
104
|
-
function writeTagged(out: ByteWriter, type: ValueTypeSpec, value:
|
|
104
|
+
function writeTagged(out: ByteWriter, type: ValueTypeSpec, value: Value): void {
|
|
105
105
|
switch (type.kind) {
|
|
106
106
|
case "bool": {
|
|
107
107
|
out.u8(TAG.bool)
|
|
@@ -131,7 +131,7 @@ function writeTagged(out: ByteWriter, type: ValueTypeSpec, value: LogValue): voi
|
|
|
131
131
|
return
|
|
132
132
|
}
|
|
133
133
|
case "interval": {
|
|
134
|
-
const interval = value as
|
|
134
|
+
const interval = value as Interval
|
|
135
135
|
if (type.width === undefined) {
|
|
136
136
|
out.u8(TAG.interval)
|
|
137
137
|
if (type.element === "u64") {
|
|
@@ -165,7 +165,7 @@ interface TaggedRefusal {
|
|
|
165
165
|
}
|
|
166
166
|
|
|
167
167
|
/** Full parse at the layout's type; every illegal byte is a typed refusal. */
|
|
168
|
-
function readTagged(reader: ByteReader, type: ValueTypeSpec, refusal: TaggedRefusal):
|
|
168
|
+
function readTagged(reader: ByteReader, type: ValueTypeSpec, refusal: TaggedRefusal): Value {
|
|
169
169
|
const expected = wireTagOf(type)
|
|
170
170
|
const tag = reader.u8("value tag")
|
|
171
171
|
if (tag !== expected) {
|
|
@@ -215,7 +215,7 @@ function readTagged(reader: ByteReader, type: ValueTypeSpec, refusal: TaggedRefu
|
|
|
215
215
|
}
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
-
function valuesEqual(a:
|
|
218
|
+
function valuesEqual(a: Value, b: Value): boolean {
|
|
219
219
|
if (typeof a === "boolean" || typeof a === "bigint" || typeof a === "string") {
|
|
220
220
|
return a === b
|
|
221
221
|
}
|
|
@@ -234,7 +234,7 @@ function valuesEqual(a: LogValue, b: LogValue): boolean {
|
|
|
234
234
|
* the fingerprint's `put_literal` length-prefixes them separately, and
|
|
235
235
|
* closed ground axioms with string columns are the mirror's recorded gap.
|
|
236
236
|
*/
|
|
237
|
-
function writeCanonicalLiteral(out: ByteWriter, type: ValueTypeSpec, value:
|
|
237
|
+
function writeCanonicalLiteral(out: ByteWriter, type: ValueTypeSpec, value: Value): void {
|
|
238
238
|
switch (type.kind) {
|
|
239
239
|
case "bool": {
|
|
240
240
|
out.u8(value === true ? 1 : 0)
|
|
@@ -260,7 +260,7 @@ function writeCanonicalLiteral(out: ByteWriter, type: ValueTypeSpec, value: LogV
|
|
|
260
260
|
return
|
|
261
261
|
}
|
|
262
262
|
case "interval": {
|
|
263
|
-
const interval = value as
|
|
263
|
+
const interval = value as Interval
|
|
264
264
|
if (type.width !== undefined) {
|
|
265
265
|
if (type.element === "u64") {
|
|
266
266
|
out.u64be(interval.start)
|
|
@@ -281,5 +281,5 @@ function writeCanonicalLiteral(out: ByteWriter, type: ValueTypeSpec, value: LogV
|
|
|
281
281
|
}
|
|
282
282
|
}
|
|
283
283
|
|
|
284
|
-
export type {
|
|
284
|
+
export type { Interval, TaggedRefusal, Value }
|
|
285
285
|
export { checkAgainst, isInterval, readTagged, TAG, valuesEqual, wireTagOf, writeCanonicalLiteral, writeTagged }
|
package/src/writer.ts
CHANGED
|
@@ -14,11 +14,12 @@ import * as crypto from "node:crypto"
|
|
|
14
14
|
import type { Fact, FreshKeys, MemberRelation, SchemaRelations, Violation } from "@bjornpagen/bumbledb"
|
|
15
15
|
import * as errors from "@superbuilders/errors"
|
|
16
16
|
import { bytesEqual, utf8Encoder, utf8StrictDecoder } from "#bytes.ts"
|
|
17
|
-
import type {
|
|
17
|
+
import type { Op } from "#codec.ts"
|
|
18
18
|
import { decodeBatch, encodeBatch } from "#codec.ts"
|
|
19
|
-
import type { RelationInfo } from "#descriptor.ts"
|
|
19
|
+
import type { Braid, RelationInfo } from "#descriptor.ts"
|
|
20
20
|
import { ErrSpanningCommit, throwContention } from "#errors.ts"
|
|
21
|
-
import {
|
|
21
|
+
import type { Generation } from "#keys.ts"
|
|
22
|
+
import { generation, idsKey, logKey } from "#keys.ts"
|
|
22
23
|
import type { Core, Replica } from "#replica.ts"
|
|
23
24
|
import {
|
|
24
25
|
applyOps,
|
|
@@ -33,7 +34,7 @@ import {
|
|
|
33
34
|
readdressPending,
|
|
34
35
|
withGate
|
|
35
36
|
} from "#replica.ts"
|
|
36
|
-
import type {
|
|
37
|
+
import type { Value } from "#value.ts"
|
|
37
38
|
import { checkAgainst } from "#value.ts"
|
|
38
39
|
|
|
39
40
|
/** 10 owns the width: one CAS amortizes counter traffic 4096× below slot traffic. */
|
|
@@ -48,15 +49,20 @@ type Commit<Rels extends SchemaRelations, R> =
|
|
|
48
49
|
| {
|
|
49
50
|
readonly tag: "accepted"
|
|
50
51
|
readonly value: R
|
|
51
|
-
readonly braid:
|
|
52
|
-
readonly generation:
|
|
52
|
+
readonly braid: Braid
|
|
53
|
+
readonly generation: Generation
|
|
53
54
|
readonly durability: Durability
|
|
54
55
|
}
|
|
55
56
|
| { readonly tag: "rejected"; readonly violations: readonly Violation<Rels>[] }
|
|
56
57
|
|
|
57
58
|
type BraidOutcome<Rels extends SchemaRelations> =
|
|
58
|
-
| {
|
|
59
|
-
|
|
59
|
+
| {
|
|
60
|
+
readonly tag: "accepted"
|
|
61
|
+
readonly braid: Braid
|
|
62
|
+
readonly generation: Generation
|
|
63
|
+
readonly durability: Durability
|
|
64
|
+
}
|
|
65
|
+
| { readonly tag: "rejected"; readonly braid: Braid; readonly violations: readonly Violation<Rels>[] }
|
|
60
66
|
|
|
61
67
|
interface CommitSplit<Rels extends SchemaRelations, R> {
|
|
62
68
|
readonly value: R
|
|
@@ -68,7 +74,7 @@ interface CommitSplit<Rels extends SchemaRelations, R> {
|
|
|
68
74
|
* drawing on the id lease (10) — reservations never appear in the log;
|
|
69
75
|
* the resulting inserts carry concrete values. Pure and synchronous.
|
|
70
76
|
*/
|
|
71
|
-
interface
|
|
77
|
+
interface Batch<Rels extends SchemaRelations> {
|
|
72
78
|
insert<Rel extends MemberRelation<Rels>>(relation: Rel, facts: Iterable<Fact<Rel>>): void
|
|
73
79
|
delete<Rel extends MemberRelation<Rels>>(relation: Rel, facts: Iterable<Fact<Rel>>): void
|
|
74
80
|
reserve<Rel extends MemberRelation<Rels>>(
|
|
@@ -79,8 +85,8 @@ interface LogBatch<Rels extends SchemaRelations> {
|
|
|
79
85
|
}
|
|
80
86
|
|
|
81
87
|
interface Writer<Rels extends SchemaRelations> {
|
|
82
|
-
commit<R>(body: (batch:
|
|
83
|
-
commitSplit<R>(body: (batch:
|
|
88
|
+
commit<R>(body: (batch: Batch<Rels>) => R): Promise<Commit<Rels, R>>
|
|
89
|
+
commitSplit<R>(body: (batch: Batch<Rels>) => R): Promise<CommitSplit<Rels, R>>
|
|
84
90
|
}
|
|
85
91
|
|
|
86
92
|
interface LeaseRange {
|
|
@@ -162,20 +168,20 @@ function pushRange(state: WriterState, relation: number, field: number, next: bi
|
|
|
162
168
|
}
|
|
163
169
|
|
|
164
170
|
interface Recording {
|
|
165
|
-
readonly ops:
|
|
171
|
+
readonly ops: Op[]
|
|
166
172
|
}
|
|
167
173
|
|
|
168
174
|
function lowerFact<Rels extends SchemaRelations>(
|
|
169
175
|
core: Core<Rels>,
|
|
170
176
|
info: RelationInfo,
|
|
171
177
|
fact: Record<string, unknown>
|
|
172
|
-
):
|
|
178
|
+
): Value[] {
|
|
173
179
|
return info.fields.map(function lowerCell(field) {
|
|
174
180
|
const raw = fact[field.name]
|
|
175
181
|
if (raw === undefined) {
|
|
176
182
|
throw errors.new(`relation ${info.name}: fact is missing field ${field.name}`)
|
|
177
183
|
}
|
|
178
|
-
let value:
|
|
184
|
+
let value: Value
|
|
179
185
|
if (field.closedRef !== undefined) {
|
|
180
186
|
if (typeof raw !== "string") {
|
|
181
187
|
throw errors.new(`relation ${info.name} field ${field.name}: expected a ${field.closedRef} handle name`)
|
|
@@ -193,7 +199,7 @@ function lowerFact<Rels extends SchemaRelations>(
|
|
|
193
199
|
}
|
|
194
200
|
value = { start: interval.start, end: interval.end }
|
|
195
201
|
} else {
|
|
196
|
-
value = raw as
|
|
202
|
+
value = raw as Value
|
|
197
203
|
}
|
|
198
204
|
checkAgainst(`relation ${info.name} field ${field.name}`, field.type, value)
|
|
199
205
|
return value
|
|
@@ -203,7 +209,7 @@ function lowerFact<Rels extends SchemaRelations>(
|
|
|
203
209
|
function recorderOf<Rels extends SchemaRelations>(
|
|
204
210
|
core: Core<Rels>,
|
|
205
211
|
state: WriterState
|
|
206
|
-
): { batch:
|
|
212
|
+
): { batch: Batch<Rels>; recording: Recording } {
|
|
207
213
|
const recording: Recording = { ops: [] }
|
|
208
214
|
function infoOf(name: string): RelationInfo {
|
|
209
215
|
const info = core.descriptor.relationByName.get(name)
|
|
@@ -217,7 +223,7 @@ function recorderOf<Rels extends SchemaRelations>(
|
|
|
217
223
|
}
|
|
218
224
|
function record(op: "insert" | "delete", relationName: string, facts: Iterable<unknown>): void {
|
|
219
225
|
const info = infoOf(relationName)
|
|
220
|
-
const rows:
|
|
226
|
+
const rows: Value[][] = []
|
|
221
227
|
for (const fact of facts) {
|
|
222
228
|
if (typeof fact !== "object" || fact === null) {
|
|
223
229
|
throw errors.new(`relation ${relationName}: a fact is not an object`)
|
|
@@ -226,7 +232,7 @@ function recorderOf<Rels extends SchemaRelations>(
|
|
|
226
232
|
}
|
|
227
233
|
recording.ops.push({ op, relation: relationName, rows })
|
|
228
234
|
}
|
|
229
|
-
const batch:
|
|
235
|
+
const batch: Batch<Rels> = {
|
|
230
236
|
insert(relation, facts) {
|
|
231
237
|
record("insert", relation.name, facts)
|
|
232
238
|
},
|
|
@@ -254,8 +260,8 @@ function recorderOf<Rels extends SchemaRelations>(
|
|
|
254
260
|
async function recordWithLeases<Rels extends SchemaRelations, R>(
|
|
255
261
|
core: Core<Rels>,
|
|
256
262
|
state: WriterState,
|
|
257
|
-
body: (batch:
|
|
258
|
-
): Promise<{ value: R; ops:
|
|
263
|
+
body: (batch: Batch<Rels>) => R
|
|
264
|
+
): Promise<{ value: R; ops: Op[] }> {
|
|
259
265
|
for (let attempt = 0; attempt < 16; attempt++) {
|
|
260
266
|
const { batch, recording } = recorderOf(core, state)
|
|
261
267
|
const ran = errors.trySync(function runBody() {
|
|
@@ -273,11 +279,8 @@ async function recordWithLeases<Rels extends SchemaRelations, R>(
|
|
|
273
279
|
throw errors.new("the id-lease pool could not satisfy the recording after 16 refills")
|
|
274
280
|
}
|
|
275
281
|
|
|
276
|
-
function braidsTouched<Rels extends SchemaRelations>(
|
|
277
|
-
|
|
278
|
-
ops: readonly BatchOp[]
|
|
279
|
-
): Map<string, BatchOp[]> {
|
|
280
|
-
const partitioned = new Map<string, BatchOp[]>()
|
|
282
|
+
function braidsTouched<Rels extends SchemaRelations>(core: Core<Rels>, ops: readonly Op[]): Map<Braid, Op[]> {
|
|
283
|
+
const partitioned = new Map<Braid, Op[]>()
|
|
281
284
|
for (const op of ops) {
|
|
282
285
|
const info = core.descriptor.relationByName.get(op.relation)
|
|
283
286
|
const braid = info === undefined ? undefined : core.descriptor.braidOfRelation.get(info.id)
|
|
@@ -298,10 +301,10 @@ function braidsTouched<Rels extends SchemaRelations>(
|
|
|
298
301
|
* the hot statement and carries the offending facts' raw values; an
|
|
299
302
|
* accepted-but-outraced terminal loss carries the racing tip. */
|
|
300
303
|
function screamContention<Rels extends SchemaRelations>(
|
|
301
|
-
braid:
|
|
304
|
+
braid: Braid,
|
|
302
305
|
rejudged:
|
|
303
306
|
| { readonly tag: "rejected"; readonly violations: readonly Violation<Rels>[] }
|
|
304
|
-
| { readonly tag: "outraced"; readonly tip:
|
|
307
|
+
| { readonly tag: "outraced"; readonly tip: Generation }
|
|
305
308
|
): never {
|
|
306
309
|
if (rejudged.tag === "rejected") {
|
|
307
310
|
const violation = rejudged.violations[0]
|
|
@@ -339,7 +342,7 @@ function screamContention<Rels extends SchemaRelations>(
|
|
|
339
342
|
async function publishPending<Rels extends SchemaRelations>(
|
|
340
343
|
core: Core<Rels>,
|
|
341
344
|
state: WriterState,
|
|
342
|
-
ops: readonly
|
|
345
|
+
ops: readonly Op[]
|
|
343
346
|
): Promise<Commit<Rels, undefined>> {
|
|
344
347
|
let losses = 0
|
|
345
348
|
for (;;) {
|
|
@@ -400,8 +403,8 @@ async function publishPending<Rels extends SchemaRelations>(
|
|
|
400
403
|
async function disciplineCommit<Rels extends SchemaRelations>(
|
|
401
404
|
core: Core<Rels>,
|
|
402
405
|
state: WriterState,
|
|
403
|
-
braid:
|
|
404
|
-
ops: readonly
|
|
406
|
+
braid: Braid,
|
|
407
|
+
ops: readonly Op[]
|
|
405
408
|
): Promise<Commit<Rels, undefined>> {
|
|
406
409
|
const entry = chainEntry(core, braid)
|
|
407
410
|
const timestamp = maxBigint(BigInt(Date.now()), entry.ts)
|
|
@@ -410,14 +413,14 @@ async function disciplineCommit<Rels extends SchemaRelations>(
|
|
|
410
413
|
{
|
|
411
414
|
fingerprint: core.descriptor.fingerprint,
|
|
412
415
|
braid,
|
|
413
|
-
braidGen: entry.g + 1n,
|
|
416
|
+
braidGen: generation(entry.g + 1n),
|
|
414
417
|
prev: entry.prev,
|
|
415
418
|
writer: state.writerId,
|
|
416
419
|
timestamp
|
|
417
420
|
},
|
|
418
421
|
ops
|
|
419
422
|
)
|
|
420
|
-
core.pending = { braid, gen: entry.g + 1n, bytes }
|
|
423
|
+
core.pending = { braid, gen: generation(entry.g + 1n), bytes }
|
|
421
424
|
core.pendingOps = ops
|
|
422
425
|
core.pendingApplied = false
|
|
423
426
|
await persistSidecar(core)
|
|
@@ -465,7 +468,7 @@ function openWriter<Rels extends SchemaRelations>(replica: Replica<Rels>): Write
|
|
|
465
468
|
if (partitioned.size > 1) {
|
|
466
469
|
throw errors.wrap(ErrSpanningCommit, `the recorded ops span braids ${[...partitioned.keys()].join(", ")}`)
|
|
467
470
|
}
|
|
468
|
-
const [braid, ops] = [...partitioned.entries()][0] as [
|
|
471
|
+
const [braid, ops] = [...partitioned.entries()][0] as [Braid, Op[]]
|
|
469
472
|
const outcome = await disciplineCommit(core, state, braid, ops)
|
|
470
473
|
if (outcome.tag === "rejected") {
|
|
471
474
|
return outcome
|
|
@@ -502,5 +505,5 @@ function openWriter<Rels extends SchemaRelations>(replica: Replica<Rels>): Write
|
|
|
502
505
|
}
|
|
503
506
|
}
|
|
504
507
|
|
|
505
|
-
export type { BraidOutcome, Commit, CommitSplit, Durability,
|
|
508
|
+
export type { Batch, BraidOutcome, Commit, CommitSplit, Durability, Writer }
|
|
506
509
|
export { openWriter }
|