@fireproof/core 0.19.99 → 0.19.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{chunk-OFGPKRCM.js → chunk-3EB3ENHT.js} +54 -25
- package/chunk-3EB3ENHT.js.map +1 -0
- package/chunk-HQ7D3PEU.js +61 -0
- package/chunk-HQ7D3PEU.js.map +1 -0
- package/chunk-PZ5AY32C.js +10 -0
- package/deno-filesystem-Q2IJ7YDR.js +57 -0
- package/deno-filesystem-Q2IJ7YDR.js.map +1 -0
- package/deno.json +3 -1
- package/{gateway-5FCWPX5W.js → gateway-GK5QZ6KP.js} +13 -12
- package/gateway-GK5QZ6KP.js.map +1 -0
- package/{gateway-H7UD6TNB.js → gateway-TQTGDRCN.js} +9 -8
- package/gateway-TQTGDRCN.js.map +1 -0
- package/index.cjs +2232 -1781
- package/index.cjs.map +1 -1
- package/index.d.cts +261 -117
- package/index.d.ts +261 -117
- package/index.global.js +12776 -11829
- package/index.global.js.map +1 -1
- package/index.js +1936 -1579
- package/index.js.map +1 -1
- package/{key-bag-file-WADZBHYG.js → key-bag-file-VOSSK46F.js} +4 -3
- package/{key-bag-file-WADZBHYG.js.map → key-bag-file-VOSSK46F.js.map} +1 -1
- package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-AXTQOSMC.js} +4 -3
- package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-AXTQOSMC.js.map} +1 -1
- package/key-bag-memory-LWE6ARPX.js +29 -0
- package/key-bag-memory-LWE6ARPX.js.map +1 -0
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/{node-filesystem-INX4ZTHE.js → node-filesystem-CFRXFSO7.js} +6 -9
- package/node-filesystem-CFRXFSO7.js.map +1 -0
- package/package.json +4 -2
- package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +129 -0
- package/tests/blockstore/keyed-crypto.test.ts +63 -227
- package/tests/blockstore/loader.test.ts +19 -11
- package/tests/blockstore/store.test.ts +23 -19
- package/tests/blockstore/transaction.test.ts +12 -12
- package/tests/fireproof/all-gateway.test.ts +201 -193
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +324 -316
- package/tests/fireproof/crdt.test.ts +67 -16
- package/tests/fireproof/database.test.ts +183 -21
- package/tests/fireproof/fireproof.test.ts +83 -74
- package/tests/fireproof/hello.test.ts +18 -14
- package/tests/fireproof/indexer.test.ts +53 -43
- package/tests/fireproof/utils.test.ts +18 -6
- package/tests/gateway/file/loader-config.test.ts +303 -0
- package/tests/gateway/indexdb/loader-config.test.ts +75 -0
- package/tests/helpers.ts +27 -9
- package/tests/react/useFireproof.test.tsx +1 -1
- package/{utils-QO2HIWGI.js → utils-STA2C35G.js} +4 -3
- package/utils-STA2C35G.js.map +1 -0
- package/chunk-OFGPKRCM.js.map +0 -1
- package/chunk-WS3YRPIA.js +0 -75
- package/chunk-WS3YRPIA.js.map +0 -1
- package/gateway-5FCWPX5W.js.map +0 -1
- package/gateway-H7UD6TNB.js.map +0 -1
- package/mem-filesystem-YPPJV7Q2.js +0 -41
- package/mem-filesystem-YPPJV7Q2.js.map +0 -1
- package/node-filesystem-INX4ZTHE.js.map +0 -1
- package/tests/fireproof/config.test.ts +0 -172
- /package/{utils-QO2HIWGI.js.map → chunk-PZ5AY32C.js.map} +0 -0
- /package/tests/fireproof/{fireproof.test.fixture.ts → fireproof.fixture.ts} +0 -0
package/index.js
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
import {
|
2
|
+
INDEXDB_VERSION
|
3
|
+
} from "./chunk-PB4BKL4O.js";
|
1
4
|
import {
|
2
5
|
FILESTORE_VERSION
|
3
6
|
} from "./chunk-7EWIAXTM.js";
|
@@ -6,28 +9,30 @@ import {
|
|
6
9
|
getFileSystem,
|
7
10
|
getPath,
|
8
11
|
toArrayBuffer
|
9
|
-
} from "./chunk-
|
10
|
-
import {
|
11
|
-
INDEXDB_VERSION
|
12
|
-
} from "./chunk-PB4BKL4O.js";
|
12
|
+
} from "./chunk-HQ7D3PEU.js";
|
13
13
|
import {
|
14
14
|
NotFoundError,
|
15
|
+
PARAM,
|
15
16
|
Result,
|
16
17
|
UInt8ArrayEqual,
|
17
|
-
__export,
|
18
|
-
dataDir,
|
19
18
|
ensureLogger,
|
20
19
|
ensureSuperLog,
|
21
20
|
ensureSuperThis,
|
22
21
|
exceptionWrapper,
|
22
|
+
falsyToUndef,
|
23
23
|
getKey,
|
24
24
|
getName,
|
25
25
|
getStore,
|
26
|
-
|
27
|
-
|
26
|
+
isFalsy,
|
27
|
+
isNotFoundError,
|
28
|
+
throwFalsy
|
29
|
+
} from "./chunk-3EB3ENHT.js";
|
30
|
+
import {
|
31
|
+
__export
|
32
|
+
} from "./chunk-PZ5AY32C.js";
|
28
33
|
|
29
34
|
// src/database.ts
|
30
|
-
import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
|
35
|
+
import { BuildURI as BuildURI2, KeyedResolvOnce as KeyedResolvOnce3, ResolveOnce as ResolveOnce6, URI as URI8 } from "@adviser/cement";
|
31
36
|
|
32
37
|
// src/write-queue.ts
|
33
38
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -72,85 +77,6 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
72
77
|
// src/crdt.ts
|
73
78
|
import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
|
74
79
|
|
75
|
-
// src/runtime/wait-pr-multiformats/block.ts
|
76
|
-
var block_exports = {};
|
77
|
-
__export(block_exports, {
|
78
|
-
Block: () => Block,
|
79
|
-
create: () => create,
|
80
|
-
createUnsafe: () => createUnsafe,
|
81
|
-
decode: () => decode,
|
82
|
-
encode: () => encode
|
83
|
-
});
|
84
|
-
import { bytes as binary, CID } from "multiformats";
|
85
|
-
import { Block as mfBlock } from "multiformats/block";
|
86
|
-
var Block = mfBlock;
|
87
|
-
async function decode({
|
88
|
-
bytes,
|
89
|
-
codec: codec3,
|
90
|
-
hasher: hasher7
|
91
|
-
}) {
|
92
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
93
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
94
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
95
|
-
const hash = await hasher7.digest(bytes);
|
96
|
-
const cid = CID.create(1, codec3.code, hash);
|
97
|
-
return new mfBlock({ value, bytes, cid });
|
98
|
-
}
|
99
|
-
async function encode({
|
100
|
-
value,
|
101
|
-
codec: codec3,
|
102
|
-
hasher: hasher7
|
103
|
-
}) {
|
104
|
-
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
105
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
106
|
-
const bytes = await Promise.resolve(codec3.encode(value));
|
107
|
-
const hash = await hasher7.digest(bytes);
|
108
|
-
const cid = CID.create(1, codec3.code, hash);
|
109
|
-
return new mfBlock({ value, bytes, cid });
|
110
|
-
}
|
111
|
-
async function create({
|
112
|
-
bytes,
|
113
|
-
cid,
|
114
|
-
hasher: hasher7,
|
115
|
-
codec: codec3
|
116
|
-
}) {
|
117
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
118
|
-
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
119
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
120
|
-
const hash = await hasher7.digest(bytes);
|
121
|
-
if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
|
122
|
-
throw new Error("CID hash does not match bytes");
|
123
|
-
}
|
124
|
-
return createUnsafe({
|
125
|
-
bytes,
|
126
|
-
cid,
|
127
|
-
value,
|
128
|
-
codec: codec3
|
129
|
-
});
|
130
|
-
}
|
131
|
-
async function createUnsafe({
|
132
|
-
bytes,
|
133
|
-
cid,
|
134
|
-
value: maybeValue,
|
135
|
-
codec: codec3
|
136
|
-
}) {
|
137
|
-
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
138
|
-
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
139
|
-
return new Block({
|
140
|
-
cid,
|
141
|
-
bytes,
|
142
|
-
value
|
143
|
-
});
|
144
|
-
}
|
145
|
-
|
146
|
-
// src/crdt-helpers.ts
|
147
|
-
import { parse as parse3 } from "multiformats/link";
|
148
|
-
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
149
|
-
import * as codec from "@ipld/dag-cbor";
|
150
|
-
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
151
|
-
import { EventFetcher, vis } from "@web3-storage/pail/clock";
|
152
|
-
import * as Batch from "@web3-storage/pail/crdt/batch";
|
153
|
-
|
154
80
|
// src/blockstore/index.ts
|
155
81
|
var blockstore_exports = {};
|
156
82
|
__export(blockstore_exports, {
|
@@ -162,8 +88,11 @@ __export(blockstore_exports, {
|
|
162
88
|
FragmentGateway: () => FragmentGateway,
|
163
89
|
Loader: () => Loader,
|
164
90
|
addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
|
165
|
-
|
166
|
-
|
91
|
+
ensureStoreEnDeFile: () => ensureStoreEnDeFile,
|
92
|
+
fileGatewayFactoryItem: () => fileGatewayFactoryItem,
|
93
|
+
getDefaultURI: () => getDefaultURI,
|
94
|
+
getGatewayFactoryItem: () => getGatewayFactoryItem,
|
95
|
+
getStartedGateway: () => getStartedGateway,
|
167
96
|
parseCarFile: () => parseCarFile,
|
168
97
|
registerStoreProtocol: () => registerStoreProtocol,
|
169
98
|
setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
|
@@ -178,7 +107,7 @@ function toCIDBlock(block) {
|
|
178
107
|
}
|
179
108
|
|
180
109
|
// src/blockstore/store-factory.ts
|
181
|
-
import { KeyedResolvOnce as KeyedResolvOnce2,
|
110
|
+
import { KeyedResolvOnce as KeyedResolvOnce2, Result as Result7 } from "@adviser/cement";
|
182
111
|
|
183
112
|
// src/runtime/files.ts
|
184
113
|
var files_exports = {};
|
@@ -252,329 +181,437 @@ var UnixFSFileBuilder = class {
|
|
252
181
|
};
|
253
182
|
|
254
183
|
// src/blockstore/store.ts
|
255
|
-
import pLimit2 from "p-limit";
|
256
184
|
import { format as format2, parse as parse2 } from "@ipld/dag-json";
|
257
|
-
import {
|
258
|
-
|
259
|
-
// src/types.ts
|
260
|
-
function isFalsy(value) {
|
261
|
-
return value === false && value === null && value === void 0;
|
262
|
-
}
|
263
|
-
function throwFalsy(value) {
|
264
|
-
if (isFalsy(value)) {
|
265
|
-
throw new Error("value is Falsy");
|
266
|
-
}
|
267
|
-
return value;
|
268
|
-
}
|
269
|
-
function falsyToUndef(value) {
|
270
|
-
if (isFalsy(value)) {
|
271
|
-
return void 0;
|
272
|
-
}
|
273
|
-
return value;
|
274
|
-
}
|
275
|
-
|
276
|
-
// src/blockstore/loader.ts
|
277
|
-
import pLimit from "p-limit";
|
278
|
-
import { CarReader } from "@ipld/car";
|
279
|
-
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
185
|
+
import { ResolveOnce as ResolveOnce3, Result as Result5, exception2Result } from "@adviser/cement";
|
280
186
|
|
281
|
-
// src/blockstore/
|
282
|
-
import {
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
|
289
|
-
const fpvalue = dec.value;
|
290
|
-
if (fpvalue && !fpvalue.fp) {
|
291
|
-
throw logger.Error().Msg("missing fp").AsError();
|
187
|
+
// src/blockstore/commit-queue.ts
|
188
|
+
import { Future } from "@adviser/cement";
|
189
|
+
var CommitQueue = class {
|
190
|
+
constructor() {
|
191
|
+
this.queue = [];
|
192
|
+
this.processing = false;
|
193
|
+
this._waitIdleItems = /* @__PURE__ */ new Set();
|
292
194
|
}
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
// src/blockstore/transaction.ts
|
297
|
-
import { MemoryBlockstore } from "@web3-storage/pail/block";
|
298
|
-
import { toCryptoRuntime } from "@adviser/cement";
|
299
|
-
var CarTransaction = class extends MemoryBlockstore {
|
300
|
-
constructor(parent, opts = { add: true, noLoader: false }) {
|
301
|
-
super();
|
302
|
-
if (opts.add) {
|
303
|
-
parent.transactions.add(this);
|
195
|
+
waitIdle() {
|
196
|
+
if (this.queue.length === 0 && !this.processing) {
|
197
|
+
return Promise.resolve();
|
304
198
|
}
|
305
|
-
|
199
|
+
const fn = new Future();
|
200
|
+
this._waitIdleItems.add(fn);
|
201
|
+
return fn.asPromise();
|
306
202
|
}
|
307
|
-
async
|
308
|
-
return
|
203
|
+
async enqueue(fn) {
|
204
|
+
return new Promise((resolve, reject) => {
|
205
|
+
const queueFn = async () => {
|
206
|
+
try {
|
207
|
+
resolve(await fn());
|
208
|
+
} catch (e) {
|
209
|
+
reject(e);
|
210
|
+
} finally {
|
211
|
+
this.processing = false;
|
212
|
+
this.processNext();
|
213
|
+
}
|
214
|
+
};
|
215
|
+
this.queue.push(queueFn);
|
216
|
+
if (!this.processing) {
|
217
|
+
this.processNext();
|
218
|
+
}
|
219
|
+
});
|
309
220
|
}
|
310
|
-
|
311
|
-
|
221
|
+
processNext() {
|
222
|
+
if (this.queue.length > 0 && !this.processing) {
|
223
|
+
this.processing = true;
|
224
|
+
const queueFn = this.queue.shift();
|
225
|
+
if (queueFn) {
|
226
|
+
queueFn().finally(() => {
|
227
|
+
});
|
228
|
+
}
|
229
|
+
}
|
230
|
+
if (this.queue.length === 0 && !this.processing) {
|
231
|
+
const toResolve = Array.from(this._waitIdleItems);
|
232
|
+
this._waitIdleItems.clear();
|
233
|
+
toResolve.map((fn) => fn.resolve());
|
234
|
+
}
|
312
235
|
}
|
313
236
|
};
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
237
|
+
|
238
|
+
// src/runtime/keyed-crypto.ts
|
239
|
+
var keyed_crypto_exports = {};
|
240
|
+
__export(keyed_crypto_exports, {
|
241
|
+
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
242
|
+
keyedCryptoFactory: () => keyedCryptoFactory
|
243
|
+
});
|
244
|
+
import { base58btc } from "multiformats/bases/base58";
|
245
|
+
import { sha256 as hasher } from "multiformats/hashes/sha2";
|
246
|
+
import * as CBOR from "cborg";
|
247
|
+
var generateIV = {
|
248
|
+
random: {
|
318
249
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
319
|
-
|
320
|
-
return
|
250
|
+
calc: async (ko, crypto, data) => {
|
251
|
+
return crypto.randomBytes(ko.ivLength);
|
321
252
|
},
|
322
253
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
323
|
-
|
324
|
-
return
|
254
|
+
verify: async (ko, crypto, iv, data) => {
|
255
|
+
return true;
|
256
|
+
}
|
257
|
+
},
|
258
|
+
hash: {
|
259
|
+
calc: async (ko, crypto, data) => {
|
260
|
+
const hash = await hasher.digest(data);
|
261
|
+
const hashBytes = new Uint8Array(hash.bytes);
|
262
|
+
const hashArray = new Uint8Array(ko.ivLength);
|
263
|
+
for (let i = 0; i < hashBytes.length; i++) {
|
264
|
+
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
265
|
+
}
|
266
|
+
return hashArray;
|
325
267
|
},
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
threshold: 1e3 * 1e3,
|
330
|
-
...opts,
|
331
|
-
logger,
|
332
|
-
keyBag: opts.keyBag || {},
|
333
|
-
crypto: toCryptoRuntime(opts.crypto),
|
334
|
-
store,
|
335
|
-
storeRuntime: toStoreRuntime(store, sthis)
|
336
|
-
};
|
337
|
-
}
|
338
|
-
function blockstoreFactory(sthis, opts) {
|
339
|
-
if (opts.name) {
|
340
|
-
return new EncryptedBlockstore(sthis, opts);
|
341
|
-
} else {
|
342
|
-
return new BaseBlockstore(opts);
|
268
|
+
verify: async function(ko, crypto, iv, data) {
|
269
|
+
return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
270
|
+
}
|
343
271
|
}
|
272
|
+
};
|
273
|
+
function getGenerateIVFn(url, opts) {
|
274
|
+
const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
|
275
|
+
return generateIV[ivhash] || generateIV["hash"];
|
344
276
|
}
|
345
|
-
var
|
346
|
-
constructor(
|
347
|
-
this.
|
348
|
-
this.
|
349
|
-
this.
|
350
|
-
this.
|
351
|
-
|
352
|
-
// ready: Promise<void>;
|
353
|
-
ready() {
|
354
|
-
return Promise.resolve();
|
355
|
-
}
|
356
|
-
async close() {
|
277
|
+
var BlockIvKeyIdCodec = class {
|
278
|
+
constructor(ko, iv, opts) {
|
279
|
+
this.code = 3147065;
|
280
|
+
this.name = "Fireproof@encrypted-block:aes-gcm";
|
281
|
+
this.ko = ko;
|
282
|
+
this.iv = iv;
|
283
|
+
this.opts = opts || {};
|
357
284
|
}
|
358
|
-
async
|
285
|
+
async encode(data) {
|
286
|
+
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
287
|
+
const { iv } = this.ko.algo(calcIv);
|
288
|
+
const fprt = await this.ko.fingerPrint();
|
289
|
+
const keyId = base58btc.decode(fprt);
|
290
|
+
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
291
|
+
return CBOR.encode({
|
292
|
+
iv,
|
293
|
+
keyId,
|
294
|
+
data: await this.ko._encrypt({ iv, bytes: data })
|
295
|
+
});
|
359
296
|
}
|
360
|
-
async
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
297
|
+
async decode(abytes) {
|
298
|
+
let bytes;
|
299
|
+
if (abytes instanceof Uint8Array) {
|
300
|
+
bytes = abytes;
|
301
|
+
} else {
|
302
|
+
bytes = new Uint8Array(abytes);
|
365
303
|
}
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
// TransactionMeta
|
372
|
-
async transaction(fn, _opts) {
|
373
|
-
const t = new CarTransaction(this, _opts);
|
374
|
-
const done = await fn(t);
|
375
|
-
this.lastTxMeta = done;
|
376
|
-
return { t, meta: done };
|
377
|
-
}
|
378
|
-
async *entries() {
|
379
|
-
const seen = /* @__PURE__ */ new Set();
|
380
|
-
for (const t of this.transactions) {
|
381
|
-
for await (const blk of t.entries()) {
|
382
|
-
if (seen.has(blk.cid.toString())) continue;
|
383
|
-
seen.add(blk.cid.toString());
|
384
|
-
yield blk;
|
385
|
-
}
|
304
|
+
const { iv, keyId, data } = CBOR.decode(bytes);
|
305
|
+
const fprt = await this.ko.fingerPrint();
|
306
|
+
this.ko.logger.Debug().Str("fp", base58btc.encode(keyId)).Msg("decode");
|
307
|
+
if (base58btc.encode(keyId) !== fprt) {
|
308
|
+
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
|
386
309
|
}
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
constructor(sthis, ebOpts) {
|
391
|
-
super(ebOpts);
|
392
|
-
this.compacting = false;
|
393
|
-
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
|
394
|
-
const { name } = ebOpts;
|
395
|
-
if (!name) {
|
396
|
-
throw this.logger.Error().Msg("name required").AsError();
|
310
|
+
const result = await this.ko._decrypt({ iv, bytes: data });
|
311
|
+
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
312
|
+
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
397
313
|
}
|
398
|
-
|
399
|
-
this.loader = new Loader(this.name, ebOpts, sthis);
|
314
|
+
return result;
|
400
315
|
}
|
401
|
-
|
402
|
-
|
316
|
+
};
|
317
|
+
var keyedCrypto = class {
|
318
|
+
constructor(url, key, cyopt, sthis) {
|
319
|
+
this.ivLength = 12;
|
320
|
+
this.isEncrypting = true;
|
321
|
+
this.logger = ensureLogger(sthis, "keyedCrypto");
|
322
|
+
this.crypto = cyopt;
|
323
|
+
this.key = key;
|
324
|
+
this.url = url;
|
403
325
|
}
|
404
|
-
|
405
|
-
return this.
|
326
|
+
fingerPrint() {
|
327
|
+
return Promise.resolve(this.key.fingerPrint);
|
406
328
|
}
|
407
|
-
|
408
|
-
return this
|
329
|
+
codec(iv, opts) {
|
330
|
+
return new BlockIvKeyIdCodec(this, iv, opts);
|
409
331
|
}
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
}
|
416
|
-
return falsyToUndef(await this.loader.getBlock(cid));
|
332
|
+
algo(iv) {
|
333
|
+
return {
|
334
|
+
name: "AES-GCM",
|
335
|
+
iv: iv || this.crypto.randomBytes(this.ivLength),
|
336
|
+
tagLength: 128
|
337
|
+
};
|
417
338
|
}
|
418
|
-
async
|
419
|
-
|
420
|
-
|
421
|
-
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
422
|
-
setTimeout(() => void this.compact(), 10);
|
423
|
-
}
|
424
|
-
if (cars) {
|
425
|
-
this.transactions.delete(t);
|
426
|
-
return { meta: done, cars, t };
|
427
|
-
}
|
428
|
-
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
339
|
+
async _decrypt(data) {
|
340
|
+
this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
|
341
|
+
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
429
342
|
}
|
430
|
-
async
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
car
|
435
|
-
/*, isPublic */
|
436
|
-
);
|
437
|
-
const block = await reader.get(cid);
|
438
|
-
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
439
|
-
return block.bytes;
|
343
|
+
async _encrypt(data) {
|
344
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
|
345
|
+
const a = this.algo(data.iv);
|
346
|
+
return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
|
440
347
|
}
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
if (!compactFn || this.compacting) return;
|
447
|
-
const blockLog = new CompactionFetcher(this);
|
448
|
-
this.compacting = true;
|
449
|
-
const meta = await compactFn(blockLog);
|
450
|
-
await this.loader?.commit(blockLog.loggedBlocks, meta, {
|
451
|
-
compact: true,
|
452
|
-
noLoader: true
|
453
|
-
});
|
454
|
-
this.compacting = false;
|
348
|
+
};
|
349
|
+
var nullCodec = class {
|
350
|
+
constructor() {
|
351
|
+
this.code = 0;
|
352
|
+
this.name = "Fireproof@unencrypted-block";
|
455
353
|
}
|
456
|
-
|
457
|
-
|
458
|
-
throw logger.Error().Msg("no loader").AsError();
|
459
|
-
}
|
460
|
-
if (!this.lastTxMeta) {
|
461
|
-
throw logger.Error().Msg("no lastTxMeta").AsError();
|
462
|
-
}
|
463
|
-
for await (const blk of this.loader.entries(false)) {
|
464
|
-
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
465
|
-
}
|
466
|
-
for (const t of this.transactions) {
|
467
|
-
for await (const blk of t.entries()) {
|
468
|
-
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
469
|
-
}
|
470
|
-
}
|
471
|
-
return this.lastTxMeta;
|
354
|
+
encode(data) {
|
355
|
+
return data;
|
472
356
|
}
|
473
|
-
|
474
|
-
|
475
|
-
yield blk;
|
476
|
-
}
|
357
|
+
decode(data) {
|
358
|
+
return data;
|
477
359
|
}
|
478
360
|
};
|
479
|
-
var
|
480
|
-
constructor(
|
481
|
-
this.
|
482
|
-
this.
|
361
|
+
var noCrypto = class {
|
362
|
+
constructor(url, cyrt, sthis) {
|
363
|
+
this.ivLength = 0;
|
364
|
+
this.code = 0;
|
365
|
+
this.name = "Fireproof@unencrypted-block";
|
366
|
+
this.isEncrypting = false;
|
367
|
+
this._fingerPrint = "noCrypto:" + Math.random();
|
368
|
+
this.logger = ensureLogger(sthis, "noCrypto");
|
369
|
+
this.crypto = cyrt;
|
370
|
+
this.url = url;
|
483
371
|
}
|
484
|
-
|
485
|
-
|
486
|
-
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
487
|
-
return falsyToUndef(block);
|
372
|
+
fingerPrint() {
|
373
|
+
return Promise.resolve(this._fingerPrint);
|
488
374
|
}
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
import { Future } from "@adviser/cement";
|
493
|
-
var CommitQueue = class {
|
494
|
-
constructor() {
|
495
|
-
this.queue = [];
|
496
|
-
this.processing = false;
|
497
|
-
this._waitIdleItems = /* @__PURE__ */ new Set();
|
375
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
376
|
+
codec(iv) {
|
377
|
+
return new nullCodec();
|
498
378
|
}
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
379
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
380
|
+
algo(iv) {
|
381
|
+
return {
|
382
|
+
name: "noCrypto",
|
383
|
+
iv: new Uint8Array(),
|
384
|
+
tagLength: 0
|
385
|
+
};
|
506
386
|
}
|
507
|
-
|
508
|
-
|
509
|
-
const queueFn = async () => {
|
510
|
-
try {
|
511
|
-
resolve(await fn());
|
512
|
-
} catch (e) {
|
513
|
-
reject(e);
|
514
|
-
} finally {
|
515
|
-
this.processing = false;
|
516
|
-
this.processNext();
|
517
|
-
}
|
518
|
-
};
|
519
|
-
this.queue.push(queueFn);
|
520
|
-
if (!this.processing) {
|
521
|
-
this.processNext();
|
522
|
-
}
|
523
|
-
});
|
387
|
+
_decrypt() {
|
388
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
524
389
|
}
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
390
|
+
_encrypt() {
|
391
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
392
|
+
}
|
393
|
+
};
|
394
|
+
async function keyedCryptoFactory(url, kb, sthis) {
|
395
|
+
const storekey = url.getParam("storekey" /* STORE_KEY */);
|
396
|
+
if (storekey && storekey !== "insecure") {
|
397
|
+
let rkey = await kb.getNamedKey(storekey, true);
|
398
|
+
if (rkey.isErr()) {
|
399
|
+
try {
|
400
|
+
rkey = await kb.toKeyWithFingerPrint(storekey);
|
401
|
+
} catch (e) {
|
402
|
+
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
532
403
|
}
|
533
404
|
}
|
534
|
-
|
535
|
-
const toResolve = Array.from(this._waitIdleItems);
|
536
|
-
this._waitIdleItems.clear();
|
537
|
-
toResolve.map((fn) => fn.resolve());
|
538
|
-
}
|
405
|
+
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
539
406
|
}
|
540
|
-
|
407
|
+
return new noCrypto(url, kb.rt.crypto, sthis);
|
408
|
+
}
|
541
409
|
|
542
|
-
// src/
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
ResolveOnce,
|
552
|
-
ResolveSeq,
|
553
|
-
Result as Result2,
|
554
|
-
runtimeFn,
|
555
|
-
toCryptoRuntime as toCryptoRuntime2,
|
556
|
-
URI
|
557
|
-
} from "@adviser/cement";
|
558
|
-
import { base58btc } from "multiformats/bases/base58";
|
559
|
-
var KeyBag = class {
|
560
|
-
constructor(rt) {
|
561
|
-
this.rt = rt;
|
562
|
-
this._warnOnce = new ResolveOnce();
|
563
|
-
this._seq = new ResolveSeq();
|
564
|
-
this.logger = ensureLogger(rt.sthis, "KeyBag");
|
565
|
-
this.logger.Debug().Msg("KeyBag created");
|
410
|
+
// src/blockstore/fragment-gateway.ts
|
411
|
+
import { Result as Result2 } from "@adviser/cement";
|
412
|
+
import { base58btc as base58btc2 } from "multiformats/bases/base58";
|
413
|
+
import { encode as encode2, decode as decode2 } from "cborg";
|
414
|
+
function getFragSize(url) {
|
415
|
+
const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
|
416
|
+
let ret = 0;
|
417
|
+
if (fragSize) {
|
418
|
+
ret = parseInt(fragSize);
|
566
419
|
}
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
420
|
+
if (isNaN(ret) || ret <= 0) {
|
421
|
+
ret = 0;
|
422
|
+
}
|
423
|
+
return ret;
|
424
|
+
}
|
425
|
+
async function getFrags(url, innerGW, headerSize, logger) {
|
426
|
+
const fragSize = getFragSize(url);
|
427
|
+
if (!fragSize) {
|
428
|
+
const res = await innerGW.get(url);
|
429
|
+
if (res.isErr()) {
|
430
|
+
return [res];
|
431
|
+
}
|
432
|
+
const data = res.unwrap();
|
433
|
+
return [
|
434
|
+
Result2.Ok({
|
435
|
+
fid: new Uint8Array(0),
|
436
|
+
ofs: 0,
|
437
|
+
len: data.length,
|
438
|
+
data
|
439
|
+
})
|
440
|
+
];
|
441
|
+
}
|
442
|
+
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
443
|
+
if (firstRaw.isErr()) {
|
444
|
+
return [firstRaw];
|
445
|
+
}
|
446
|
+
const firstFragment = decode2(firstRaw.unwrap());
|
447
|
+
const blockSize = firstFragment.data.length;
|
448
|
+
const ops = [Promise.resolve(Result2.Ok(firstFragment))];
|
449
|
+
const fidStr = base58btc2.encode(firstFragment.fid);
|
450
|
+
const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
|
451
|
+
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
452
|
+
ops.push(
|
453
|
+
(async (furl, ofs2) => {
|
454
|
+
const raw2 = await innerGW.get(furl);
|
455
|
+
if (raw2.isErr()) {
|
456
|
+
return raw2;
|
457
|
+
}
|
458
|
+
const fragment = decode2(raw2.unwrap());
|
459
|
+
if (base58btc2.encode(fragment.fid) !== fidStr) {
|
460
|
+
return Result2.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
461
|
+
}
|
462
|
+
if (fragment.ofs !== ofs2) {
|
463
|
+
return Result2.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
464
|
+
}
|
465
|
+
return Result2.Ok(fragment);
|
466
|
+
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
467
|
+
);
|
468
|
+
}
|
469
|
+
return Promise.all(ops);
|
470
|
+
}
|
471
|
+
var FragmentGateway = class {
|
472
|
+
constructor(sthis, innerGW) {
|
473
|
+
this.fidLength = 4;
|
474
|
+
this.headerSize = 32;
|
475
|
+
this.sthis = ensureSuperLog(sthis, "FragmentGateway");
|
476
|
+
this.logger = this.sthis.logger;
|
477
|
+
this.innerGW = innerGW;
|
478
|
+
}
|
479
|
+
slicer(url, body) {
|
480
|
+
const fragSize = getFragSize(url);
|
481
|
+
if (!fragSize) {
|
482
|
+
return [this.innerGW.put(url, body)];
|
483
|
+
}
|
484
|
+
const blocksize = fragSize - this.headerSize;
|
485
|
+
if (blocksize <= 0) {
|
486
|
+
throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
|
487
|
+
}
|
488
|
+
const ops = [];
|
489
|
+
const fid = this.sthis.nextId(this.fidLength);
|
490
|
+
const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
|
491
|
+
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
492
|
+
const block = encode2({
|
493
|
+
fid: fid.bin,
|
494
|
+
ofs,
|
495
|
+
len: body.length,
|
496
|
+
data: body.slice(ofs, ofs + blocksize)
|
497
|
+
});
|
498
|
+
if (block.length > fragSize) {
|
499
|
+
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
500
|
+
}
|
501
|
+
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
502
|
+
}
|
503
|
+
return ops;
|
504
|
+
}
|
505
|
+
buildUrl(baseUrl, key) {
|
506
|
+
return this.innerGW.buildUrl(baseUrl, key);
|
507
|
+
}
|
508
|
+
async destroy(iurl) {
|
509
|
+
return this.innerGW.destroy(iurl);
|
510
|
+
}
|
511
|
+
async start(url) {
|
512
|
+
this.headerSize = encode2({
|
513
|
+
fid: this.sthis.nextId(this.fidLength).bin,
|
514
|
+
ofs: 1024 * 1024,
|
515
|
+
// 32bit
|
516
|
+
len: 16 * 1024 * 1024,
|
517
|
+
// 32bit
|
518
|
+
data: new Uint8Array(1024)
|
519
|
+
}).length - 1024;
|
520
|
+
return this.innerGW.start(url);
|
521
|
+
}
|
522
|
+
async close(url) {
|
523
|
+
return this.innerGW.close(url);
|
524
|
+
}
|
525
|
+
async put(url, body) {
|
526
|
+
await Promise.all(this.slicer(url, body));
|
527
|
+
return Result2.Ok(void 0);
|
528
|
+
}
|
529
|
+
async get(url) {
|
530
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
531
|
+
let buffer = void 0;
|
532
|
+
for (const rfrag of rfrags) {
|
533
|
+
if (rfrag.isErr()) {
|
534
|
+
return Result2.Err(rfrag.Err());
|
535
|
+
}
|
536
|
+
const frag = rfrag.Ok();
|
537
|
+
buffer = buffer || new Uint8Array(frag.len);
|
538
|
+
buffer.set(frag.data, frag.ofs);
|
539
|
+
}
|
540
|
+
return Result2.Ok(buffer || new Uint8Array(0));
|
541
|
+
}
|
542
|
+
async subscribe(url, callback) {
|
543
|
+
if (this.innerGW.subscribe) {
|
544
|
+
return this.innerGW.subscribe(url, callback);
|
545
|
+
} else {
|
546
|
+
return Result2.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
|
547
|
+
}
|
548
|
+
}
|
549
|
+
async delete(url) {
|
550
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
551
|
+
for (const rfrag of rfrags) {
|
552
|
+
if (rfrag.isErr()) {
|
553
|
+
return Result2.Err(rfrag.Err());
|
554
|
+
}
|
555
|
+
const frag = rfrag.Ok();
|
556
|
+
let fragUrl;
|
557
|
+
if (rfrags.length > 1) {
|
558
|
+
const fidStr = base58btc2.encode(frag.fid);
|
559
|
+
fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
|
560
|
+
} else {
|
561
|
+
fragUrl = url;
|
562
|
+
}
|
563
|
+
await this.innerGW.delete(fragUrl);
|
564
|
+
}
|
565
|
+
return Result2.Ok(void 0);
|
566
|
+
}
|
567
|
+
};
|
568
|
+
|
569
|
+
// src/blockstore/meta-key-helper.ts
|
570
|
+
import { format, parse } from "@ipld/dag-json";
|
571
|
+
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
572
|
+
import { CID } from "multiformats";
|
573
|
+
import { base64pad } from "multiformats/bases/base64";
|
574
|
+
import { Result as Result4 } from "@adviser/cement";
|
575
|
+
|
576
|
+
// src/runtime/key-bag.ts
|
577
|
+
var key_bag_exports = {};
|
578
|
+
__export(key_bag_exports, {
|
579
|
+
KeyBag: () => KeyBag,
|
580
|
+
defaultKeyBagOpts: () => defaultKeyBagOpts,
|
581
|
+
getKeyBag: () => getKeyBag,
|
582
|
+
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
583
|
+
});
|
584
|
+
import {
|
585
|
+
KeyedResolvOnce,
|
586
|
+
ResolveOnce,
|
587
|
+
ResolveSeq,
|
588
|
+
Result as Result3,
|
589
|
+
runtimeFn,
|
590
|
+
toCryptoRuntime,
|
591
|
+
URI as URI2
|
592
|
+
} from "@adviser/cement";
|
593
|
+
import { base58btc as base58btc3 } from "multiformats/bases/base58";
|
594
|
+
var KeyBag = class {
|
595
|
+
constructor(rt) {
|
596
|
+
this.rt = rt;
|
597
|
+
this._warnOnce = new ResolveOnce();
|
598
|
+
this._seq = new ResolveSeq();
|
599
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag", {
|
600
|
+
id: rt.id()
|
601
|
+
});
|
602
|
+
this.logger.Debug().Msg("KeyBag created");
|
603
|
+
}
|
604
|
+
async subtleKey(key) {
|
605
|
+
const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
|
606
|
+
if (extractable) {
|
607
|
+
this._warnOnce.once(
|
608
|
+
() => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
|
572
609
|
);
|
573
610
|
}
|
574
611
|
return await this.rt.crypto.importKey(
|
575
612
|
"raw",
|
576
613
|
// raw or jwk
|
577
|
-
|
614
|
+
base58btc3.decode(key),
|
578
615
|
// hexStringToUint8Array(key), // raw data
|
579
616
|
"AES-GCM",
|
580
617
|
extractable,
|
@@ -582,9 +619,9 @@ var KeyBag = class {
|
|
582
619
|
);
|
583
620
|
}
|
584
621
|
async ensureKeyFromUrl(url, keyFactory) {
|
585
|
-
const storeKey = url.getParam("storekey");
|
622
|
+
const storeKey = url.getParam("storekey" /* STORE_KEY */);
|
586
623
|
if (storeKey === "insecure") {
|
587
|
-
return
|
624
|
+
return Result3.Ok(url);
|
588
625
|
}
|
589
626
|
if (!storeKey) {
|
590
627
|
const keyName = `@${keyFactory()}@`;
|
@@ -592,8 +629,8 @@ var KeyBag = class {
|
|
592
629
|
if (ret.isErr()) {
|
593
630
|
return ret;
|
594
631
|
}
|
595
|
-
const urb = url.build().setParam("storekey"
|
596
|
-
return
|
632
|
+
const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
|
633
|
+
return Result3.Ok(urb.URI());
|
597
634
|
}
|
598
635
|
if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
|
599
636
|
const ret = await this.getNamedKey(storeKey);
|
@@ -601,15 +638,15 @@ var KeyBag = class {
|
|
601
638
|
return ret;
|
602
639
|
}
|
603
640
|
}
|
604
|
-
return
|
641
|
+
return Result3.Ok(url);
|
605
642
|
}
|
606
643
|
async toKeyWithFingerPrint(keyStr) {
|
607
|
-
const material =
|
644
|
+
const material = base58btc3.decode(keyStr);
|
608
645
|
const key = await this.subtleKey(keyStr);
|
609
646
|
const fpr = await this.rt.crypto.digestSHA256(material);
|
610
|
-
return
|
647
|
+
return Result3.Ok({
|
611
648
|
key,
|
612
|
-
fingerPrint:
|
649
|
+
fingerPrint: base58btc3.encode(new Uint8Array(fpr))
|
613
650
|
});
|
614
651
|
}
|
615
652
|
async setNamedKey(name, key) {
|
@@ -632,13 +669,13 @@ var KeyBag = class {
|
|
632
669
|
return ret;
|
633
670
|
}
|
634
671
|
const named = ret.Ok();
|
635
|
-
return
|
672
|
+
return Result3.Ok({
|
636
673
|
...named,
|
637
674
|
extract: async () => {
|
638
675
|
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
639
676
|
return {
|
640
677
|
key: ext,
|
641
|
-
keyStr:
|
678
|
+
keyStr: base58btc3.encode(ext)
|
642
679
|
};
|
643
680
|
}
|
644
681
|
});
|
@@ -655,9 +692,9 @@ var KeyBag = class {
|
|
655
692
|
}
|
656
693
|
if (failIfNotFound) {
|
657
694
|
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
658
|
-
return
|
695
|
+
return Result3.Err(new Error(`Key not found: ${name}`));
|
659
696
|
}
|
660
|
-
const ret = await this._setNamedKey(name,
|
697
|
+
const ret = await this._setNamedKey(name, base58btc3.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
661
698
|
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
662
699
|
return ret;
|
663
700
|
});
|
@@ -668,14 +705,14 @@ var keyBagProviderFactories = new Map(
|
|
668
705
|
{
|
669
706
|
protocol: "file:",
|
670
707
|
factory: async (url, sthis) => {
|
671
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-
|
708
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
|
672
709
|
return new KeyBagProviderFile(url, sthis);
|
673
710
|
}
|
674
711
|
},
|
675
712
|
{
|
676
713
|
protocol: "indexdb:",
|
677
714
|
factory: async (url, sthis) => {
|
678
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-
|
715
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
|
679
716
|
return new KeyBagProviderIndexDB(url, sthis);
|
680
717
|
}
|
681
718
|
}
|
@@ -689,44 +726,63 @@ function registerKeyBagProviderFactory(item) {
|
|
689
726
|
});
|
690
727
|
}
|
691
728
|
function defaultKeyBagOpts(sthis, kbo) {
|
729
|
+
kbo = kbo || {};
|
692
730
|
if (kbo.keyRuntime) {
|
693
731
|
return kbo.keyRuntime;
|
694
732
|
}
|
695
733
|
const logger = ensureLogger(sthis, "KeyBag");
|
696
734
|
let url;
|
697
735
|
if (kbo.url) {
|
698
|
-
url =
|
736
|
+
url = URI2.from(kbo.url);
|
699
737
|
logger.Debug().Url(url).Msg("from opts");
|
700
738
|
} else {
|
701
739
|
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
702
740
|
if (runtimeFn().isBrowser) {
|
703
|
-
url =
|
741
|
+
url = URI2.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
704
742
|
} else {
|
705
743
|
if (!bagFnameOrUrl) {
|
706
744
|
const home = sthis.env.get("HOME");
|
707
745
|
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
708
|
-
url =
|
746
|
+
url = URI2.from(`file://${bagFnameOrUrl}`);
|
709
747
|
} else {
|
710
|
-
url =
|
748
|
+
url = URI2.from(bagFnameOrUrl);
|
711
749
|
}
|
712
750
|
}
|
713
751
|
logger.Debug().Url(url).Msg("from env");
|
714
752
|
}
|
715
|
-
|
716
|
-
|
717
|
-
|
753
|
+
let keyProviderFactory;
|
754
|
+
switch (url.protocol) {
|
755
|
+
case "file:":
|
756
|
+
keyProviderFactory = async () => {
|
757
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
|
758
|
+
return new KeyBagProviderFile(url, sthis);
|
759
|
+
};
|
760
|
+
break;
|
761
|
+
case "indexdb:":
|
762
|
+
keyProviderFactory = async () => {
|
763
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
|
764
|
+
return new KeyBagProviderIndexDB(url, sthis);
|
765
|
+
};
|
766
|
+
break;
|
767
|
+
case "memory:":
|
768
|
+
keyProviderFactory = async () => {
|
769
|
+
const { KeyBagProviderMemory } = await import("./key-bag-memory-LWE6ARPX.js");
|
770
|
+
return new KeyBagProviderMemory(url, sthis);
|
771
|
+
};
|
772
|
+
break;
|
773
|
+
default:
|
774
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
718
775
|
}
|
719
|
-
const getBag = async () => kitem.factory(url, sthis);
|
720
776
|
if (url.hasParam("masterkey")) {
|
721
777
|
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
722
778
|
}
|
723
779
|
return {
|
724
780
|
url,
|
725
|
-
crypto: kbo.crypto ||
|
781
|
+
crypto: kbo.crypto || toCryptoRuntime({}),
|
726
782
|
sthis,
|
727
783
|
logger,
|
728
784
|
keyLength: kbo.keyLength || 16,
|
729
|
-
getBag,
|
785
|
+
getBag: keyProviderFactory,
|
730
786
|
id: () => {
|
731
787
|
return url.toString();
|
732
788
|
}
|
@@ -739,932 +795,938 @@ async function getKeyBag(sthis, kbo = {}) {
|
|
739
795
|
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
740
796
|
}
|
741
797
|
|
742
|
-
// src/blockstore/
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
const headerSize = CBW.headerLength({ roots });
|
749
|
-
size += headerSize;
|
750
|
-
for (const { cid, bytes } of t.entries()) {
|
751
|
-
size += CBW.blockLength({ cid, bytes });
|
752
|
-
}
|
753
|
-
const buffer = new Uint8Array(size);
|
754
|
-
const writer = CBW.createWriter(buffer, { headerSize });
|
755
|
-
for (const r of roots) {
|
756
|
-
writer.addRoot(r);
|
757
|
-
}
|
758
|
-
for (const { cid, bytes } of t.entries()) {
|
759
|
-
writer.write({ cid, bytes });
|
798
|
+
// src/blockstore/meta-key-helper.ts
|
799
|
+
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
800
|
+
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
801
|
+
if (!crdtEntries.length) {
|
802
|
+
sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
|
803
|
+
return [];
|
760
804
|
}
|
761
|
-
|
762
|
-
|
763
|
-
|
764
|
-
async function createCarFile(encoder, cid, t) {
|
765
|
-
return encodeCarFile([cid], t, encoder);
|
766
|
-
}
|
767
|
-
async function commitFiles(fileStore, walStore, t, done) {
|
768
|
-
const { files: roots } = makeFileCarHeader(done);
|
769
|
-
const cids = [];
|
770
|
-
const codec3 = (await fileStore.keyedCrypto()).codec();
|
771
|
-
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
772
|
-
for (const car of cars) {
|
773
|
-
const { cid, bytes } = car;
|
774
|
-
await fileStore.save({ cid, bytes });
|
775
|
-
await walStore.enqueueFile(
|
776
|
-
cid
|
777
|
-
/*, !!opts.public*/
|
778
|
-
);
|
779
|
-
cids.push(cid);
|
805
|
+
if (!crdtEntries.map) {
|
806
|
+
sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
|
807
|
+
return [];
|
780
808
|
}
|
781
|
-
return
|
809
|
+
return Promise.all(
|
810
|
+
crdtEntries.map(async (crdtEntry) => {
|
811
|
+
const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
|
812
|
+
const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
|
813
|
+
return {
|
814
|
+
eventCid: eventBlock.cid,
|
815
|
+
parents: crdtEntry.parents,
|
816
|
+
dbMeta
|
817
|
+
};
|
818
|
+
})
|
819
|
+
);
|
782
820
|
}
|
783
|
-
function
|
784
|
-
|
785
|
-
|
786
|
-
|
787
|
-
|
821
|
+
async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
|
822
|
+
try {
|
823
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
|
824
|
+
const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
|
825
|
+
if (keyInfo.length) {
|
826
|
+
const dbMeta = keyInfo[0].dbMeta;
|
827
|
+
if (dbMeta.key) {
|
828
|
+
const kb = await getKeyBag(sthis);
|
829
|
+
const keyName = getStoreKeyName(uri);
|
830
|
+
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
831
|
+
if (res.isErr()) {
|
832
|
+
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
833
|
+
throw res.Err();
|
834
|
+
}
|
835
|
+
}
|
836
|
+
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
837
|
+
return Result4.Ok(dbMeta);
|
788
838
|
}
|
839
|
+
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
840
|
+
return Result4.Ok(void 0);
|
841
|
+
} catch (error) {
|
842
|
+
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
843
|
+
return Result4.Err(error);
|
789
844
|
}
|
790
|
-
return { ...result, files };
|
791
845
|
}
|
792
|
-
async function
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
cids.push(cid);
|
846
|
+
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
847
|
+
try {
|
848
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
849
|
+
const keyName = getStoreKeyName(uri);
|
850
|
+
const kb = await getKeyBag(sthis);
|
851
|
+
const res = await kb.getNamedExtractableKey(keyName, true);
|
852
|
+
if (res.isErr()) {
|
853
|
+
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
854
|
+
throw res.Err();
|
855
|
+
}
|
856
|
+
const keyData = await res.Ok().extract();
|
857
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
858
|
+
const { dbMeta, parents } = dbMetas[0];
|
859
|
+
const parentLinks = parents.map((p) => CID.parse(p));
|
860
|
+
dbMeta.key = keyData.keyStr;
|
861
|
+
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
862
|
+
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
863
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
864
|
+
return Result4.Ok(encoded);
|
865
|
+
} catch (error) {
|
866
|
+
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
867
|
+
return Result4.Err(error);
|
815
868
|
}
|
816
|
-
const newDbMeta = { cars: cids };
|
817
|
-
await params.WALStore.enqueue(newDbMeta, opts);
|
818
|
-
await params.metaStore.save(newDbMeta);
|
819
|
-
return { cgrp: cids, header: fp };
|
820
869
|
}
|
821
|
-
|
822
|
-
const
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
827
|
-
let cidRootBlock = rootBlock;
|
828
|
-
for (const { cid, bytes } of t.entries()) {
|
829
|
-
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
830
|
-
if (newsize >= threshold) {
|
831
|
-
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
832
|
-
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
833
|
-
clonedt.putSync(cid, bytes);
|
834
|
-
cidRootBlock = { cid, bytes };
|
835
|
-
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
836
|
-
} else {
|
837
|
-
clonedt.putSync(cid, bytes);
|
838
|
-
}
|
870
|
+
function getStoreKeyName(url) {
|
871
|
+
const storeKeyName = [url.getParam("localName") || url.getParam("name")];
|
872
|
+
const idx = url.getParam("index");
|
873
|
+
if (idx) {
|
874
|
+
storeKeyName.push(idx);
|
839
875
|
}
|
840
|
-
|
841
|
-
return
|
876
|
+
storeKeyName.push("data");
|
877
|
+
return `@${storeKeyName.join(":")}@`;
|
878
|
+
}
|
879
|
+
async function createDbMetaEventBlock(sthis, dbMeta, parents) {
|
880
|
+
const event = await EventBlock.create(
|
881
|
+
{
|
882
|
+
dbMeta: sthis.txt.encode(format(dbMeta))
|
883
|
+
},
|
884
|
+
parents
|
885
|
+
);
|
886
|
+
return event;
|
887
|
+
}
|
888
|
+
async function encodeEventsWithParents(sthis, events, parents) {
|
889
|
+
const crdtEntries = events.map((event) => {
|
890
|
+
const base64String = base64pad.encode(event.bytes);
|
891
|
+
return {
|
892
|
+
cid: event.cid.toString(),
|
893
|
+
data: base64String,
|
894
|
+
parents: parents.map((p) => p.toString())
|
895
|
+
};
|
896
|
+
});
|
897
|
+
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
842
898
|
}
|
843
899
|
|
844
|
-
// src/blockstore/
|
845
|
-
import
|
846
|
-
|
847
|
-
// src/blockstore/task-manager.ts
|
848
|
-
var TaskManager = class {
|
849
|
-
constructor(sthis, callback) {
|
850
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
851
|
-
this.queue = [];
|
852
|
-
this.isProcessing = false;
|
853
|
-
this.logger = ensureLogger(sthis, "TaskManager");
|
854
|
-
this.callback = callback;
|
855
|
-
}
|
856
|
-
async handleEvent(cid, parents, dbMeta) {
|
857
|
-
for (const parent of parents) {
|
858
|
-
this.eventsWeHandled.add(parent.toString());
|
859
|
-
}
|
860
|
-
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
861
|
-
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
862
|
-
void this.processQueue();
|
863
|
-
}
|
864
|
-
async processQueue() {
|
865
|
-
if (this.isProcessing) return;
|
866
|
-
this.isProcessing = true;
|
867
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
868
|
-
const first = filteredQueue[0];
|
869
|
-
if (!first) {
|
870
|
-
return;
|
871
|
-
}
|
872
|
-
try {
|
873
|
-
await this.callback(first.dbMeta);
|
874
|
-
this.eventsWeHandled.add(first.cid);
|
875
|
-
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
876
|
-
} catch (err) {
|
877
|
-
if (first.retries++ > 3) {
|
878
|
-
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
879
|
-
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
880
|
-
}
|
881
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
882
|
-
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
883
|
-
} finally {
|
884
|
-
this.isProcessing = false;
|
885
|
-
if (this.queue.length > 0) {
|
886
|
-
void this.processQueue();
|
887
|
-
}
|
888
|
-
}
|
889
|
-
}
|
890
|
-
};
|
900
|
+
// src/blockstore/store.ts
|
901
|
+
import pRetry from "p-retry";
|
902
|
+
import pMap from "p-map";
|
891
903
|
|
892
904
|
// src/blockstore/loader.ts
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
905
|
+
import pLimit from "p-limit";
|
906
|
+
import { CarReader } from "@ipld/car";
|
907
|
+
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
908
|
+
|
909
|
+
// src/runtime/wait-pr-multiformats/block.ts
|
910
|
+
var block_exports = {};
|
911
|
+
__export(block_exports, {
|
912
|
+
Block: () => Block,
|
913
|
+
create: () => create,
|
914
|
+
createUnsafe: () => createUnsafe,
|
915
|
+
decode: () => decode3,
|
916
|
+
encode: () => encode3
|
917
|
+
});
|
918
|
+
import { bytes as binary, CID as CID2 } from "multiformats";
|
919
|
+
import { Block as mfBlock } from "multiformats/block";
|
920
|
+
var Block = mfBlock;
|
921
|
+
async function decode3({
|
922
|
+
bytes,
|
923
|
+
codec: codec3,
|
924
|
+
hasher: hasher7
|
925
|
+
}) {
|
926
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
927
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
928
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
929
|
+
const hash = await hasher7.digest(bytes);
|
930
|
+
const cid = CID2.create(1, codec3.code, hash);
|
931
|
+
return new mfBlock({ value, bytes, cid });
|
897
932
|
}
|
898
|
-
function
|
899
|
-
|
900
|
-
|
901
|
-
|
902
|
-
|
903
|
-
|
904
|
-
|
933
|
+
async function encode3({
|
934
|
+
value,
|
935
|
+
codec: codec3,
|
936
|
+
hasher: hasher7
|
937
|
+
}) {
|
938
|
+
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
939
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
940
|
+
const bytes = await Promise.resolve(codec3.encode(value));
|
941
|
+
const hash = await hasher7.digest(bytes);
|
942
|
+
const cid = CID2.create(1, codec3.code, hash);
|
943
|
+
return new mfBlock({ value, bytes, cid });
|
905
944
|
}
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
|
916
|
-
|
917
|
-
|
918
|
-
this.name = name;
|
919
|
-
this.sthis = sthis;
|
920
|
-
this.ebOpts = defaultedBlockstoreRuntime(
|
921
|
-
sthis,
|
922
|
-
{
|
923
|
-
...ebOpts,
|
924
|
-
name
|
925
|
-
},
|
926
|
-
"Loader"
|
927
|
-
);
|
928
|
-
this.logger = this.ebOpts.logger;
|
929
|
-
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
930
|
-
await this.handleDbMetasFromStore([dbMeta]);
|
931
|
-
});
|
932
|
-
}
|
933
|
-
// readonly id = uuidv4();
|
934
|
-
async keyBag() {
|
935
|
-
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
936
|
-
}
|
937
|
-
async carStore() {
|
938
|
-
return this.ebOpts.storeRuntime.makeDataStore(this);
|
939
|
-
}
|
940
|
-
async fileStore() {
|
941
|
-
return this.ebOpts.storeRuntime.makeDataStore(this);
|
942
|
-
}
|
943
|
-
async WALStore() {
|
944
|
-
return this.ebOpts.storeRuntime.makeWALStore(this);
|
945
|
-
}
|
946
|
-
async metaStore() {
|
947
|
-
return this.ebOpts.storeRuntime.makeMetaStore(this);
|
948
|
-
}
|
949
|
-
async ready() {
|
950
|
-
return this.onceReady.once(async () => {
|
951
|
-
const metas = await (await this.metaStore()).load();
|
952
|
-
if (this.ebOpts.meta) {
|
953
|
-
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
954
|
-
} else if (metas) {
|
955
|
-
await this.handleDbMetasFromStore(metas);
|
956
|
-
}
|
957
|
-
});
|
958
|
-
}
|
959
|
-
async close() {
|
960
|
-
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
961
|
-
await Promise.all(toClose.map((store) => store.close()));
|
962
|
-
}
|
963
|
-
async destroy() {
|
964
|
-
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
965
|
-
await Promise.all(toDestroy.map((store) => store.destroy()));
|
966
|
-
}
|
967
|
-
// async snapToCar(carCid: AnyLink | string) {
|
968
|
-
// await this.ready
|
969
|
-
// if (typeof carCid === 'string') {
|
970
|
-
// carCid = CID.parse(carCid)
|
971
|
-
// }
|
972
|
-
// const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
|
973
|
-
// this.carLog = [carCid, ...carHeader.cars]
|
974
|
-
// await this.getMoreReaders(carHeader.cars)
|
975
|
-
// await this._applyCarHeader(carHeader, true)
|
976
|
-
// }
|
977
|
-
async handleDbMetasFromStore(metas) {
|
978
|
-
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
979
|
-
for (const meta of metas) {
|
980
|
-
await this.writeLimit(async () => {
|
981
|
-
await this.mergeDbMetaIntoClock(meta);
|
982
|
-
});
|
983
|
-
}
|
984
|
-
}
|
985
|
-
async mergeDbMetaIntoClock(meta) {
|
986
|
-
if (this.isCompacting) {
|
987
|
-
throw this.logger.Error().Msg("cannot merge while compacting").AsError();
|
988
|
-
}
|
989
|
-
if (this.seenMeta.has(meta.cars.toString())) return;
|
990
|
-
this.seenMeta.add(meta.cars.toString());
|
991
|
-
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
992
|
-
return;
|
993
|
-
}
|
994
|
-
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
995
|
-
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
996
|
-
await this.getMoreReaders(carHeader.cars.flat());
|
997
|
-
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
998
|
-
await this.ebOpts.applyMeta?.(carHeader.meta);
|
999
|
-
}
|
1000
|
-
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1001
|
-
// const { key } = meta;
|
1002
|
-
// if (key) {
|
1003
|
-
// await this.setKey(key);
|
1004
|
-
// }
|
1005
|
-
// }
|
1006
|
-
async loadCarHeaderFromMeta({ cars: cids }) {
|
1007
|
-
const reader = await this.loadCar(cids[0]);
|
1008
|
-
return await parseCarFile(reader, this.logger);
|
1009
|
-
}
|
1010
|
-
// async _getKey(): Promise<string | undefined> {
|
1011
|
-
// if (this.key) return this.key;
|
1012
|
-
// // generate a random key
|
1013
|
-
// if (!this.ebOpts.public) {
|
1014
|
-
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1015
|
-
// }
|
1016
|
-
// return this.key || undefined;
|
1017
|
-
// }
|
1018
|
-
async commitFiles(t, done) {
|
1019
|
-
await this.ready();
|
1020
|
-
const fstore = await this.fileStore();
|
1021
|
-
const wstore = await this.WALStore();
|
1022
|
-
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1023
|
-
}
|
1024
|
-
async loadFileCar(cid) {
|
1025
|
-
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
1026
|
-
}
|
1027
|
-
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
1028
|
-
await this.ready();
|
1029
|
-
const fstore = await this.fileStore();
|
1030
|
-
const params = {
|
1031
|
-
encoder: (await fstore.keyedCrypto()).codec(),
|
1032
|
-
carLog: this.carLog,
|
1033
|
-
carStore: fstore,
|
1034
|
-
WALStore: await this.WALStore(),
|
1035
|
-
metaStore: await this.metaStore(),
|
1036
|
-
threshold: this.ebOpts.threshold
|
1037
|
-
};
|
1038
|
-
return this.commitQueue.enqueue(async () => {
|
1039
|
-
await this.cacheTransaction(t);
|
1040
|
-
const ret = await commit(params, t, done, opts);
|
1041
|
-
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1042
|
-
return ret.cgrp;
|
1043
|
-
});
|
1044
|
-
}
|
1045
|
-
async updateCarLog(cids, fp, compact) {
|
1046
|
-
if (compact) {
|
1047
|
-
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1048
|
-
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1049
|
-
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1050
|
-
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1051
|
-
} else {
|
1052
|
-
this.carLog.unshift(cids);
|
1053
|
-
}
|
1054
|
-
}
|
1055
|
-
async cacheTransaction(t) {
|
1056
|
-
for await (const block of t.entries()) {
|
1057
|
-
const sBlock = block.cid.toString();
|
1058
|
-
if (!this.getBlockCache.has(sBlock)) {
|
1059
|
-
this.getBlockCache.set(sBlock, block);
|
1060
|
-
}
|
1061
|
-
}
|
1062
|
-
}
|
1063
|
-
async cacheCarReader(carCidStr, reader) {
|
1064
|
-
if (this.processedCars.has(carCidStr)) return;
|
1065
|
-
this.processedCars.add(carCidStr);
|
1066
|
-
for await (const block of reader.blocks()) {
|
1067
|
-
const sBlock = block.cid.toString();
|
1068
|
-
if (!this.getBlockCache.has(sBlock)) {
|
1069
|
-
this.getBlockCache.set(sBlock, block);
|
1070
|
-
}
|
1071
|
-
}
|
1072
|
-
}
|
1073
|
-
async removeCidsForCompact(cid) {
|
1074
|
-
const carHeader = await this.loadCarHeaderFromMeta({
|
1075
|
-
cars: [cid]
|
1076
|
-
});
|
1077
|
-
for (const cids of carHeader.compact) {
|
1078
|
-
for (const cid2 of cids) {
|
1079
|
-
await (await this.carStore()).remove(cid2);
|
1080
|
-
}
|
1081
|
-
}
|
1082
|
-
}
|
1083
|
-
// async flushCars() {
|
1084
|
-
// await this.ready
|
1085
|
-
// // for each cid in car log, make a dbMeta
|
1086
|
-
// for (const cid of this.carLog) {
|
1087
|
-
// const dbMeta = { car: cid, key: this.key || null } as DbMeta
|
1088
|
-
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
1089
|
-
// }
|
1090
|
-
// }
|
1091
|
-
async *entries(cache2 = true) {
|
1092
|
-
await this.ready();
|
1093
|
-
if (cache2) {
|
1094
|
-
for (const [, block] of this.getBlockCache) {
|
1095
|
-
yield block;
|
1096
|
-
}
|
1097
|
-
} else {
|
1098
|
-
for (const [, block] of this.getBlockCache) {
|
1099
|
-
yield block;
|
1100
|
-
}
|
1101
|
-
for (const cids of this.carLog) {
|
1102
|
-
for (const cid of cids) {
|
1103
|
-
const reader = await this.loadCar(cid);
|
1104
|
-
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
1105
|
-
for await (const block of reader.blocks()) {
|
1106
|
-
const sCid = block.cid.toString();
|
1107
|
-
if (!this.getBlockCache.has(sCid)) {
|
1108
|
-
yield block;
|
1109
|
-
}
|
1110
|
-
}
|
1111
|
-
}
|
1112
|
-
}
|
1113
|
-
}
|
1114
|
-
}
|
1115
|
-
async getBlock(cid) {
|
1116
|
-
await this.ready();
|
1117
|
-
const sCid = cid.toString();
|
1118
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1119
|
-
const getCarCid = async (carCid) => {
|
1120
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1121
|
-
const reader = await this.loadCar(carCid);
|
1122
|
-
if (!reader) {
|
1123
|
-
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
1124
|
-
}
|
1125
|
-
await this.cacheCarReader(carCid.toString(), reader).catch(() => {
|
1126
|
-
return;
|
1127
|
-
});
|
1128
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1129
|
-
throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
|
1130
|
-
};
|
1131
|
-
const getCompactCarCids = async (carCid) => {
|
1132
|
-
const reader = await this.loadCar(carCid);
|
1133
|
-
if (!reader) {
|
1134
|
-
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
1135
|
-
}
|
1136
|
-
const header = await parseCarFile(reader, this.logger);
|
1137
|
-
const compacts = header.compact;
|
1138
|
-
let got2;
|
1139
|
-
const batchSize2 = 5;
|
1140
|
-
for (let i = 0; i < compacts.length; i += batchSize2) {
|
1141
|
-
const promises = [];
|
1142
|
-
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
1143
|
-
for (const cid2 of compacts[j]) {
|
1144
|
-
promises.push(getCarCid(cid2));
|
1145
|
-
}
|
1146
|
-
}
|
1147
|
-
try {
|
1148
|
-
got2 = await Promise.any(promises);
|
1149
|
-
} catch {
|
1150
|
-
}
|
1151
|
-
if (got2) break;
|
1152
|
-
}
|
1153
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1154
|
-
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
1155
|
-
};
|
1156
|
-
let got;
|
1157
|
-
const batchSize = 5;
|
1158
|
-
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
1159
|
-
const batch = this.carLog.slice(i, i + batchSize);
|
1160
|
-
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
1161
|
-
try {
|
1162
|
-
got = await Promise.any(promises);
|
1163
|
-
} catch {
|
1164
|
-
}
|
1165
|
-
if (got) break;
|
1166
|
-
}
|
1167
|
-
if (!got) {
|
1168
|
-
try {
|
1169
|
-
got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
|
1170
|
-
} catch {
|
1171
|
-
}
|
1172
|
-
}
|
1173
|
-
return got;
|
945
|
+
async function create({
|
946
|
+
bytes,
|
947
|
+
cid,
|
948
|
+
hasher: hasher7,
|
949
|
+
codec: codec3
|
950
|
+
}) {
|
951
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
952
|
+
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
953
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
954
|
+
const hash = await hasher7.digest(bytes);
|
955
|
+
if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
|
956
|
+
throw new Error("CID hash does not match bytes");
|
1174
957
|
}
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
958
|
+
return createUnsafe({
|
959
|
+
bytes,
|
960
|
+
cid,
|
961
|
+
value,
|
962
|
+
codec: codec3
|
963
|
+
});
|
964
|
+
}
|
965
|
+
async function createUnsafe({
|
966
|
+
bytes,
|
967
|
+
cid,
|
968
|
+
value: maybeValue,
|
969
|
+
codec: codec3
|
970
|
+
}) {
|
971
|
+
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
972
|
+
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
973
|
+
return new Block({
|
974
|
+
cid,
|
975
|
+
bytes,
|
976
|
+
value
|
977
|
+
});
|
978
|
+
}
|
979
|
+
|
980
|
+
// src/blockstore/loader-helpers.ts
|
981
|
+
import { sha256 as hasher2 } from "multiformats/hashes/sha2";
|
982
|
+
import * as dagCodec from "@ipld/dag-cbor";
|
983
|
+
async function parseCarFile(reader, logger) {
|
984
|
+
const roots = await reader.getRoots();
|
985
|
+
const header = await reader.get(roots[0]);
|
986
|
+
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
987
|
+
const dec = await decode3({ bytes: header.bytes, hasher: hasher2, codec: dagCodec });
|
988
|
+
const fpvalue = dec.value;
|
989
|
+
if (fpvalue && !fpvalue.fp) {
|
990
|
+
throw logger.Error().Msg("missing fp").AsError();
|
1181
991
|
}
|
1182
|
-
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
1188
|
-
|
1189
|
-
|
1190
|
-
|
1191
|
-
|
1192
|
-
|
1193
|
-
if (remoteCar) {
|
1194
|
-
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1195
|
-
await local.save(remoteCar);
|
1196
|
-
loadedCar = remoteCar;
|
1197
|
-
activeStore = remote;
|
1198
|
-
}
|
1199
|
-
} else {
|
1200
|
-
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1201
|
-
}
|
1202
|
-
}
|
1203
|
-
if (!loadedCar) {
|
1204
|
-
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
992
|
+
return fpvalue.fp;
|
993
|
+
}
|
994
|
+
|
995
|
+
// src/blockstore/transaction.ts
|
996
|
+
import { MemoryBlockstore } from "@web3-storage/pail/block";
|
997
|
+
import { toCryptoRuntime as toCryptoRuntime2 } from "@adviser/cement";
|
998
|
+
var CarTransaction = class extends MemoryBlockstore {
|
999
|
+
constructor(parent, opts = { add: true, noLoader: false }) {
|
1000
|
+
super();
|
1001
|
+
if (opts.add) {
|
1002
|
+
parent.transactions.add(this);
|
1205
1003
|
}
|
1206
|
-
|
1207
|
-
const rawReader = await CarReader.fromBytes(bytes.value);
|
1208
|
-
const readerP = Promise.resolve(rawReader);
|
1209
|
-
const cachedReaderP = readerP.then(async (reader) => {
|
1210
|
-
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
1211
|
-
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
1212
|
-
return;
|
1213
|
-
});
|
1214
|
-
return reader;
|
1215
|
-
});
|
1216
|
-
this.carReaders.set(cidsString, cachedReaderP);
|
1217
|
-
return readerP;
|
1004
|
+
this.parent = parent;
|
1218
1005
|
}
|
1219
|
-
|
1220
|
-
|
1221
|
-
const cidsString = cid.toString();
|
1222
|
-
let dacr = this.carReaders.get(cidsString);
|
1223
|
-
if (!dacr) {
|
1224
|
-
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
1225
|
-
this.carReaders.set(cidsString, dacr);
|
1226
|
-
}
|
1227
|
-
return dacr;
|
1006
|
+
async get(cid) {
|
1007
|
+
return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
|
1228
1008
|
}
|
1229
|
-
async
|
1230
|
-
|
1231
|
-
const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
|
1232
|
-
await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
|
1009
|
+
async superGet(cid) {
|
1010
|
+
return super.get(cid);
|
1233
1011
|
}
|
1234
1012
|
};
|
1235
|
-
|
1236
|
-
|
1237
|
-
|
1238
|
-
__export(keyed_crypto_exports, {
|
1239
|
-
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
1240
|
-
keyedCryptoFactory: () => keyedCryptoFactory
|
1241
|
-
});
|
1242
|
-
import { base58btc as base58btc2 } from "multiformats/bases/base58";
|
1243
|
-
import { sha256 as hasher4 } from "multiformats/hashes/sha2";
|
1244
|
-
import * as CBOR from "cborg";
|
1245
|
-
var generateIV = {
|
1246
|
-
random: {
|
1013
|
+
function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
|
1014
|
+
const logger = ensureLogger(sthis, component, ctx);
|
1015
|
+
return {
|
1247
1016
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1248
|
-
|
1249
|
-
return
|
1017
|
+
applyMeta: (meta, snap) => {
|
1018
|
+
return Promise.resolve();
|
1250
1019
|
},
|
1251
1020
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1252
|
-
|
1253
|
-
return
|
1021
|
+
compact: async (blocks) => {
|
1022
|
+
return {};
|
1023
|
+
},
|
1024
|
+
autoCompact: 100,
|
1025
|
+
public: false,
|
1026
|
+
// name: undefined,
|
1027
|
+
threshold: 1e3 * 1e3,
|
1028
|
+
...opts,
|
1029
|
+
logger,
|
1030
|
+
keyBag: opts.keyBag || {},
|
1031
|
+
crypto: toCryptoRuntime2(opts.crypto),
|
1032
|
+
storeUrls: opts.storeUrls,
|
1033
|
+
// storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
|
1034
|
+
// store,
|
1035
|
+
storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
|
1036
|
+
};
|
1037
|
+
}
|
1038
|
+
function blockstoreFactory(sthis, opts) {
|
1039
|
+
return new EncryptedBlockstore(sthis, opts);
|
1040
|
+
}
|
1041
|
+
var BaseBlockstore = class {
|
1042
|
+
constructor(ebOpts) {
|
1043
|
+
this.transactions = /* @__PURE__ */ new Set();
|
1044
|
+
this.sthis = ensureSuperThis(ebOpts);
|
1045
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
1046
|
+
this.sthis = ensureSuperThis(ebOpts);
|
1047
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
1048
|
+
this.logger = this.ebOpts.logger;
|
1049
|
+
}
|
1050
|
+
// readonly name?: string;
|
1051
|
+
// ready: Promise<void>;
|
1052
|
+
ready() {
|
1053
|
+
return Promise.resolve();
|
1054
|
+
}
|
1055
|
+
async close() {
|
1056
|
+
}
|
1057
|
+
async destroy() {
|
1058
|
+
}
|
1059
|
+
async compact() {
|
1060
|
+
}
|
1061
|
+
async get(cid) {
|
1062
|
+
if (!cid) throw this.logger.Error().Msg("required cid").AsError();
|
1063
|
+
for (const f of this.transactions) {
|
1064
|
+
const v = await f.superGet(cid);
|
1065
|
+
if (v) return v;
|
1254
1066
|
}
|
1255
|
-
}
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
|
1067
|
+
}
|
1068
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1069
|
+
async put(cid, block) {
|
1070
|
+
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
1071
|
+
}
|
1072
|
+
// TransactionMeta
|
1073
|
+
async transaction(fn, _opts) {
|
1074
|
+
this.logger.Debug().Msg("enter transaction");
|
1075
|
+
const t = new CarTransaction(this, _opts);
|
1076
|
+
this.logger.Debug().Msg("post CarTransaction");
|
1077
|
+
const done = await fn(t);
|
1078
|
+
this.logger.Debug().Msg("post fn");
|
1079
|
+
this.lastTxMeta = done;
|
1080
|
+
return { t, meta: done };
|
1081
|
+
}
|
1082
|
+
openTransaction(opts = { add: true, noLoader: false }) {
|
1083
|
+
return new CarTransaction(this, opts);
|
1084
|
+
}
|
1085
|
+
async commitTransaction(t, done, opts) {
|
1086
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
|
1087
|
+
const cars = await this.loader?.commit(t, done, opts);
|
1088
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
1089
|
+
setTimeout(() => void this.compact(), 10);
|
1090
|
+
}
|
1091
|
+
if (cars) {
|
1092
|
+
this.transactions.delete(t);
|
1093
|
+
return { meta: done, cars, t };
|
1094
|
+
}
|
1095
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1096
|
+
}
|
1097
|
+
async *entries() {
|
1098
|
+
const seen = /* @__PURE__ */ new Set();
|
1099
|
+
for (const t of this.transactions) {
|
1100
|
+
for await (const blk of t.entries()) {
|
1101
|
+
if (seen.has(blk.cid.toString())) continue;
|
1102
|
+
seen.add(blk.cid.toString());
|
1103
|
+
yield blk;
|
1263
1104
|
}
|
1264
|
-
return hashArray;
|
1265
|
-
},
|
1266
|
-
verify: async function(ko, crypto, iv, data) {
|
1267
|
-
return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
1268
1105
|
}
|
1269
1106
|
}
|
1270
1107
|
};
|
1271
|
-
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1275
|
-
|
1276
|
-
|
1277
|
-
|
1278
|
-
this.
|
1279
|
-
this.ko = ko;
|
1280
|
-
this.iv = iv;
|
1281
|
-
this.opts = opts || {};
|
1108
|
+
var EncryptedBlockstore = class extends BaseBlockstore {
|
1109
|
+
constructor(sthis, ebOpts) {
|
1110
|
+
super(ebOpts);
|
1111
|
+
this.compacting = false;
|
1112
|
+
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
|
1113
|
+
this: 1
|
1114
|
+
});
|
1115
|
+
this.loader = new Loader(sthis, ebOpts);
|
1282
1116
|
}
|
1283
|
-
|
1284
|
-
|
1285
|
-
|
1286
|
-
|
1287
|
-
|
1288
|
-
|
1289
|
-
|
1290
|
-
|
1291
|
-
|
1292
|
-
|
1117
|
+
ready() {
|
1118
|
+
return this.loader.ready();
|
1119
|
+
}
|
1120
|
+
close() {
|
1121
|
+
return this.loader.close();
|
1122
|
+
}
|
1123
|
+
destroy() {
|
1124
|
+
return this.loader.destroy();
|
1125
|
+
}
|
1126
|
+
async get(cid) {
|
1127
|
+
const got = await super.get(cid);
|
1128
|
+
if (got) return got;
|
1129
|
+
if (!this.loader) {
|
1130
|
+
return;
|
1131
|
+
}
|
1132
|
+
return falsyToUndef(await this.loader.getBlock(cid));
|
1133
|
+
}
|
1134
|
+
async transaction(fn, opts = { noLoader: false }) {
|
1135
|
+
this.logger.Debug().Msg("enter transaction");
|
1136
|
+
const { t, meta: done } = await super.transaction(fn);
|
1137
|
+
this.logger.Debug().Msg("post super.transaction");
|
1138
|
+
const cars = await this.loader.commit(t, done, opts);
|
1139
|
+
this.logger.Debug().Msg("post this.loader.commit");
|
1140
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
1141
|
+
setTimeout(() => void this.compact(), 10);
|
1142
|
+
}
|
1143
|
+
if (cars) {
|
1144
|
+
this.transactions.delete(t);
|
1145
|
+
return { meta: done, cars, t };
|
1146
|
+
}
|
1147
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1148
|
+
}
|
1149
|
+
async getFile(car, cid) {
|
1150
|
+
await this.ready();
|
1151
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
1152
|
+
const reader = await this.loader.loadFileCar(
|
1153
|
+
car
|
1154
|
+
/*, isPublic */
|
1155
|
+
);
|
1156
|
+
const block = await reader.get(cid);
|
1157
|
+
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
1158
|
+
return block.bytes;
|
1159
|
+
}
|
1160
|
+
async compact() {
|
1161
|
+
await this.ready();
|
1162
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
|
1163
|
+
if (this.loader.carLog.length < 2) return;
|
1164
|
+
const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
|
1165
|
+
if (!compactFn || this.compacting) return;
|
1166
|
+
const blockLog = new CompactionFetcher(this);
|
1167
|
+
this.compacting = true;
|
1168
|
+
const meta = await compactFn(blockLog);
|
1169
|
+
await this.loader?.commit(blockLog.loggedBlocks, meta, {
|
1170
|
+
compact: true,
|
1171
|
+
noLoader: true
|
1293
1172
|
});
|
1173
|
+
this.compacting = false;
|
1294
1174
|
}
|
1295
|
-
async
|
1296
|
-
|
1297
|
-
|
1298
|
-
bytes = abytes;
|
1299
|
-
} else {
|
1300
|
-
bytes = new Uint8Array(abytes);
|
1175
|
+
async defaultCompact(blocks, logger) {
|
1176
|
+
if (!this.loader) {
|
1177
|
+
throw logger.Error().Msg("no loader").AsError();
|
1301
1178
|
}
|
1302
|
-
|
1303
|
-
|
1304
|
-
this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
|
1305
|
-
if (base58btc2.encode(keyId) !== fprt) {
|
1306
|
-
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
|
1179
|
+
if (!this.lastTxMeta) {
|
1180
|
+
throw logger.Error().Msg("no lastTxMeta").AsError();
|
1307
1181
|
}
|
1308
|
-
const
|
1309
|
-
|
1310
|
-
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
1182
|
+
for await (const blk of this.loader.entries(false)) {
|
1183
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
1311
1184
|
}
|
1312
|
-
|
1313
|
-
|
1314
|
-
|
1315
|
-
|
1316
|
-
|
1317
|
-
this.
|
1318
|
-
this.isEncrypting = true;
|
1319
|
-
this.logger = ensureLogger(sthis, "keyedCrypto");
|
1320
|
-
this.crypto = cyopt;
|
1321
|
-
this.key = key;
|
1322
|
-
this.url = url;
|
1323
|
-
}
|
1324
|
-
fingerPrint() {
|
1325
|
-
return Promise.resolve(this.key.fingerPrint);
|
1326
|
-
}
|
1327
|
-
codec(iv, opts) {
|
1328
|
-
return new BlockIvKeyIdCodec(this, iv, opts);
|
1185
|
+
for (const t of this.transactions) {
|
1186
|
+
for await (const blk of t.entries()) {
|
1187
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
1188
|
+
}
|
1189
|
+
}
|
1190
|
+
return this.lastTxMeta;
|
1329
1191
|
}
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
|
1334
|
-
tagLength: 128
|
1335
|
-
};
|
1192
|
+
async *entries() {
|
1193
|
+
for await (const blk of this.loader.entries()) {
|
1194
|
+
yield blk;
|
1195
|
+
}
|
1336
1196
|
}
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1197
|
+
};
|
1198
|
+
var CompactionFetcher = class {
|
1199
|
+
constructor(blocks) {
|
1200
|
+
this.blockstore = blocks;
|
1201
|
+
this.loggedBlocks = new CarTransaction(blocks);
|
1340
1202
|
}
|
1341
|
-
async
|
1342
|
-
|
1343
|
-
|
1344
|
-
return
|
1203
|
+
async get(cid) {
|
1204
|
+
const block = await this.blockstore.get(cid);
|
1205
|
+
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
1206
|
+
return falsyToUndef(block);
|
1345
1207
|
}
|
1346
1208
|
};
|
1347
|
-
|
1348
|
-
|
1349
|
-
|
1350
|
-
|
1209
|
+
|
1210
|
+
// src/blockstore/commitor.ts
|
1211
|
+
import * as CBW from "@ipld/car/buffer-writer";
|
1212
|
+
import { sha256 as hasher3 } from "multiformats/hashes/sha2";
|
1213
|
+
import * as dagCodec2 from "@ipld/dag-cbor";
|
1214
|
+
async function encodeCarFile(roots, t, codec3) {
|
1215
|
+
let size = 0;
|
1216
|
+
const headerSize = CBW.headerLength({ roots });
|
1217
|
+
size += headerSize;
|
1218
|
+
for (const { cid, bytes } of t.entries()) {
|
1219
|
+
size += CBW.blockLength({ cid, bytes });
|
1351
1220
|
}
|
1352
|
-
|
1353
|
-
|
1221
|
+
const buffer = new Uint8Array(size);
|
1222
|
+
const writer = CBW.createWriter(buffer, { headerSize });
|
1223
|
+
for (const r of roots) {
|
1224
|
+
writer.addRoot(r);
|
1354
1225
|
}
|
1355
|
-
|
1356
|
-
|
1226
|
+
for (const { cid, bytes } of t.entries()) {
|
1227
|
+
writer.write({ cid, bytes });
|
1357
1228
|
}
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
|
1364
|
-
|
1365
|
-
|
1366
|
-
|
1367
|
-
|
1368
|
-
|
1229
|
+
writer.close();
|
1230
|
+
return await encode3({ value: writer.bytes, hasher: hasher3, codec: codec3 });
|
1231
|
+
}
|
1232
|
+
async function createCarFile(encoder, cid, t) {
|
1233
|
+
return encodeCarFile([cid], t, encoder);
|
1234
|
+
}
|
1235
|
+
async function commitFiles(fileStore, walStore, t, done) {
|
1236
|
+
const { files: roots } = makeFileCarHeader(done);
|
1237
|
+
const cids = [];
|
1238
|
+
const codec3 = (await fileStore.keyedCrypto()).codec();
|
1239
|
+
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
1240
|
+
for (const car of cars) {
|
1241
|
+
const { cid, bytes } = car;
|
1242
|
+
await fileStore.save({ cid, bytes });
|
1243
|
+
await walStore.enqueueFile(
|
1244
|
+
cid
|
1245
|
+
/*, !!opts.public*/
|
1246
|
+
);
|
1247
|
+
cids.push(cid);
|
1369
1248
|
}
|
1370
|
-
|
1371
|
-
|
1249
|
+
return cids;
|
1250
|
+
}
|
1251
|
+
function makeFileCarHeader(result) {
|
1252
|
+
const files = [];
|
1253
|
+
for (const [, meta] of Object.entries(result.files || {})) {
|
1254
|
+
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
1255
|
+
files.push(meta.cid);
|
1256
|
+
}
|
1372
1257
|
}
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1258
|
+
return { ...result, files };
|
1259
|
+
}
|
1260
|
+
async function prepareCarFilesFiles(encoder, roots, t) {
|
1261
|
+
return [await encodeCarFile(roots, t, encoder)];
|
1262
|
+
}
|
1263
|
+
function makeCarHeader(meta, cars, compact = false) {
|
1264
|
+
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
1265
|
+
return { ...coreHeader, meta };
|
1266
|
+
}
|
1267
|
+
async function encodeCarHeader(fp) {
|
1268
|
+
return await encode3({
|
1269
|
+
value: { fp },
|
1270
|
+
hasher: hasher3,
|
1271
|
+
codec: dagCodec2
|
1272
|
+
});
|
1273
|
+
}
|
1274
|
+
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
1275
|
+
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
1276
|
+
const rootBlock = await encodeCarHeader(fp);
|
1277
|
+
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
1278
|
+
const cids = [];
|
1279
|
+
for (const car of cars) {
|
1280
|
+
const { cid, bytes } = car;
|
1281
|
+
await params.carStore.save({ cid, bytes });
|
1282
|
+
cids.push(cid);
|
1376
1283
|
}
|
1377
|
-
|
1378
|
-
|
1379
|
-
|
1380
|
-
|
1381
|
-
|
1382
|
-
|
1383
|
-
|
1284
|
+
const newDbMeta = { cars: cids };
|
1285
|
+
await params.WALStore.enqueue(newDbMeta, opts);
|
1286
|
+
await params.metaStore.save(newDbMeta);
|
1287
|
+
return { cgrp: cids, header: fp };
|
1288
|
+
}
|
1289
|
+
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
1290
|
+
const carFiles = [];
|
1291
|
+
threshold = threshold || 128e3 * 8;
|
1292
|
+
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1293
|
+
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1294
|
+
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
1295
|
+
let cidRootBlock = rootBlock;
|
1296
|
+
for (const { cid, bytes } of t.entries()) {
|
1297
|
+
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1298
|
+
if (newsize >= threshold) {
|
1299
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1300
|
+
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1301
|
+
clonedt.putSync(cid, bytes);
|
1302
|
+
cidRootBlock = { cid, bytes };
|
1303
|
+
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1304
|
+
} else {
|
1305
|
+
clonedt.putSync(cid, bytes);
|
1306
|
+
}
|
1384
1307
|
}
|
1385
|
-
|
1386
|
-
|
1308
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1309
|
+
return carFiles;
|
1310
|
+
}
|
1311
|
+
|
1312
|
+
// src/blockstore/loader.ts
|
1313
|
+
import { sha256 as hasher4 } from "multiformats/hashes/sha2";
|
1314
|
+
|
1315
|
+
// src/blockstore/task-manager.ts
|
1316
|
+
var TaskManager = class {
|
1317
|
+
constructor(sthis, callback) {
|
1318
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
1319
|
+
this.queue = [];
|
1320
|
+
this.isProcessing = false;
|
1321
|
+
this.logger = ensureLogger(sthis, "TaskManager");
|
1322
|
+
this.callback = callback;
|
1387
1323
|
}
|
1388
|
-
|
1389
|
-
|
1324
|
+
async handleEvent(cid, parents, dbMeta) {
|
1325
|
+
for (const parent of parents) {
|
1326
|
+
this.eventsWeHandled.add(parent.toString());
|
1327
|
+
}
|
1328
|
+
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
1329
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
1330
|
+
void this.processQueue();
|
1390
1331
|
}
|
1391
|
-
|
1392
|
-
|
1393
|
-
|
1394
|
-
|
1395
|
-
|
1396
|
-
if (
|
1397
|
-
|
1398
|
-
|
1399
|
-
|
1400
|
-
|
1332
|
+
async processQueue() {
|
1333
|
+
if (this.isProcessing) return;
|
1334
|
+
this.isProcessing = true;
|
1335
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1336
|
+
const first = filteredQueue[0];
|
1337
|
+
if (!first) {
|
1338
|
+
return;
|
1339
|
+
}
|
1340
|
+
try {
|
1341
|
+
await this.callback(first.dbMeta);
|
1342
|
+
this.eventsWeHandled.add(first.cid);
|
1343
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1344
|
+
} catch (err) {
|
1345
|
+
if (first.retries++ > 3) {
|
1346
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
1347
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
1348
|
+
}
|
1349
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
1350
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
1351
|
+
} finally {
|
1352
|
+
this.isProcessing = false;
|
1353
|
+
if (this.queue.length > 0) {
|
1354
|
+
void this.processQueue();
|
1401
1355
|
}
|
1402
1356
|
}
|
1403
|
-
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
1404
1357
|
}
|
1405
|
-
|
1406
|
-
}
|
1358
|
+
};
|
1407
1359
|
|
1408
|
-
// src/blockstore/
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1419
|
-
ret = 0;
|
1360
|
+
// src/blockstore/loader.ts
|
1361
|
+
function carLogIncludesGroup(list, cids) {
|
1362
|
+
return list.some((arr) => {
|
1363
|
+
return arr.toString() === cids.toString();
|
1364
|
+
});
|
1365
|
+
}
|
1366
|
+
function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
1367
|
+
const byString = /* @__PURE__ */ new Map();
|
1368
|
+
for (const cid of list) {
|
1369
|
+
if (remove.has(cid.toString())) continue;
|
1370
|
+
byString.set(cid.toString(), cid);
|
1420
1371
|
}
|
1421
|
-
return
|
1372
|
+
return [...byString.values()];
|
1422
1373
|
}
|
1423
|
-
|
1424
|
-
|
1425
|
-
|
1426
|
-
|
1427
|
-
|
1428
|
-
|
1429
|
-
|
1430
|
-
|
1431
|
-
|
1432
|
-
|
1433
|
-
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1374
|
+
var Loader = class {
|
1375
|
+
constructor(sthis, ebOpts) {
|
1376
|
+
this.commitQueue = new CommitQueue();
|
1377
|
+
this.isCompacting = false;
|
1378
|
+
this.carReaders = /* @__PURE__ */ new Map();
|
1379
|
+
this.seenCompacted = /* @__PURE__ */ new Set();
|
1380
|
+
this.processedCars = /* @__PURE__ */ new Set();
|
1381
|
+
this.carLog = [];
|
1382
|
+
this.getBlockCache = /* @__PURE__ */ new Map();
|
1383
|
+
this.seenMeta = /* @__PURE__ */ new Set();
|
1384
|
+
this.writeLimit = pLimit(1);
|
1385
|
+
this._carStore = new ResolveOnce2();
|
1386
|
+
this._fileStore = new ResolveOnce2();
|
1387
|
+
this._WALStore = new ResolveOnce2();
|
1388
|
+
this._metaStore = new ResolveOnce2();
|
1389
|
+
this.onceReady = new ResolveOnce2();
|
1390
|
+
this.sthis = sthis;
|
1391
|
+
this.ebOpts = defaultedBlockstoreRuntime(
|
1392
|
+
sthis,
|
1393
|
+
{
|
1394
|
+
...ebOpts
|
1395
|
+
// name,
|
1396
|
+
},
|
1397
|
+
"Loader"
|
1398
|
+
);
|
1399
|
+
this.logger = this.ebOpts.logger;
|
1400
|
+
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
1401
|
+
await this.handleDbMetasFromStore([dbMeta]);
|
1402
|
+
});
|
1403
|
+
}
|
1404
|
+
async carStore() {
|
1405
|
+
return this._carStore.once(
|
1406
|
+
async () => this.ebOpts.storeRuntime.makeDataStore({
|
1407
|
+
sthis: this.sthis,
|
1408
|
+
url: this.ebOpts.storeUrls.data,
|
1409
|
+
keybag: await this.keyBag()
|
1437
1410
|
})
|
1438
|
-
|
1411
|
+
);
|
1439
1412
|
}
|
1440
|
-
|
1441
|
-
|
1442
|
-
|
1413
|
+
async fileStore() {
|
1414
|
+
return this._fileStore.once(
|
1415
|
+
async () => this.ebOpts.storeRuntime.makeDataStore({
|
1416
|
+
sthis: this.sthis,
|
1417
|
+
url: this.ebOpts.storeUrls.file,
|
1418
|
+
keybag: await this.keyBag()
|
1419
|
+
})
|
1420
|
+
);
|
1443
1421
|
}
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1448
|
-
|
1449
|
-
|
1450
|
-
|
1451
|
-
(async (furl, ofs2) => {
|
1452
|
-
const raw2 = await innerGW.get(furl);
|
1453
|
-
if (raw2.isErr()) {
|
1454
|
-
return raw2;
|
1455
|
-
}
|
1456
|
-
const fragment = decode3(raw2.unwrap());
|
1457
|
-
if (base58btc3.encode(fragment.fid) !== fidStr) {
|
1458
|
-
return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
1459
|
-
}
|
1460
|
-
if (fragment.ofs !== ofs2) {
|
1461
|
-
return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
1462
|
-
}
|
1463
|
-
return Result3.Ok(fragment);
|
1464
|
-
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
1422
|
+
async WALStore() {
|
1423
|
+
return this._WALStore.once(
|
1424
|
+
async () => this.ebOpts.storeRuntime.makeWALStore({
|
1425
|
+
sthis: this.sthis,
|
1426
|
+
url: this.ebOpts.storeUrls.wal,
|
1427
|
+
keybag: await this.keyBag()
|
1428
|
+
})
|
1465
1429
|
);
|
1466
1430
|
}
|
1467
|
-
|
1468
|
-
|
1469
|
-
|
1470
|
-
|
1471
|
-
|
1472
|
-
|
1473
|
-
|
1474
|
-
|
1475
|
-
this.innerGW = innerGW;
|
1431
|
+
async metaStore() {
|
1432
|
+
return this._metaStore.once(
|
1433
|
+
async () => this.ebOpts.storeRuntime.makeMetaStore({
|
1434
|
+
sthis: this.sthis,
|
1435
|
+
url: this.ebOpts.storeUrls.meta,
|
1436
|
+
keybag: await this.keyBag()
|
1437
|
+
})
|
1438
|
+
);
|
1476
1439
|
}
|
1477
|
-
|
1478
|
-
|
1479
|
-
|
1480
|
-
|
1440
|
+
keyBag() {
|
1441
|
+
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
1442
|
+
}
|
1443
|
+
async ready() {
|
1444
|
+
return this.onceReady.once(async () => {
|
1445
|
+
const metas = await (await this.metaStore()).load();
|
1446
|
+
if (this.ebOpts.meta) {
|
1447
|
+
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
1448
|
+
} else if (metas) {
|
1449
|
+
await this.handleDbMetasFromStore(metas);
|
1450
|
+
}
|
1451
|
+
});
|
1452
|
+
}
|
1453
|
+
async close() {
|
1454
|
+
await this.commitQueue.waitIdle();
|
1455
|
+
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1456
|
+
await Promise.all(toClose.map((store) => store.close()));
|
1457
|
+
}
|
1458
|
+
async destroy() {
|
1459
|
+
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1460
|
+
await Promise.all(toDestroy.map((store) => store.destroy()));
|
1461
|
+
}
|
1462
|
+
// async snapToCar(carCid: AnyLink | string) {
|
1463
|
+
// await this.ready
|
1464
|
+
// if (typeof carCid === 'string') {
|
1465
|
+
// carCid = CID.parse(carCid)
|
1466
|
+
// }
|
1467
|
+
// const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
|
1468
|
+
// this.carLog = [carCid, ...carHeader.cars]
|
1469
|
+
// await this.getMoreReaders(carHeader.cars)
|
1470
|
+
// await this._applyCarHeader(carHeader, true)
|
1471
|
+
// }
|
1472
|
+
async handleDbMetasFromStore(metas) {
|
1473
|
+
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
1474
|
+
for (const meta of metas) {
|
1475
|
+
await this.writeLimit(async () => {
|
1476
|
+
await this.mergeDbMetaIntoClock(meta);
|
1477
|
+
});
|
1481
1478
|
}
|
1482
|
-
|
1483
|
-
|
1484
|
-
|
1479
|
+
}
|
1480
|
+
async mergeDbMetaIntoClock(meta) {
|
1481
|
+
if (this.isCompacting) {
|
1482
|
+
throw this.logger.Error().Msg("cannot merge while compacting").AsError();
|
1485
1483
|
}
|
1486
|
-
|
1487
|
-
|
1488
|
-
|
1489
|
-
|
1490
|
-
const block = encode3({
|
1491
|
-
fid: fid.bin,
|
1492
|
-
ofs,
|
1493
|
-
len: body.length,
|
1494
|
-
data: body.slice(ofs, ofs + blocksize)
|
1495
|
-
});
|
1496
|
-
if (block.length > fragSize) {
|
1497
|
-
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
1498
|
-
}
|
1499
|
-
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
1484
|
+
if (this.seenMeta.has(meta.cars.toString())) return;
|
1485
|
+
this.seenMeta.add(meta.cars.toString());
|
1486
|
+
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
1487
|
+
return;
|
1500
1488
|
}
|
1501
|
-
|
1489
|
+
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
1490
|
+
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1491
|
+
await this.getMoreReaders(carHeader.cars.flat());
|
1492
|
+
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
1493
|
+
await this.ebOpts.applyMeta?.(carHeader.meta);
|
1502
1494
|
}
|
1503
|
-
|
1504
|
-
|
1495
|
+
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1496
|
+
// const { key } = meta;
|
1497
|
+
// if (key) {
|
1498
|
+
// await this.setKey(key);
|
1499
|
+
// }
|
1500
|
+
// }
|
1501
|
+
async loadCarHeaderFromMeta({ cars: cids }) {
|
1502
|
+
const reader = await this.loadCar(cids[0]);
|
1503
|
+
return await parseCarFile(reader, this.logger);
|
1505
1504
|
}
|
1506
|
-
async
|
1507
|
-
|
1505
|
+
// async _getKey(): Promise<string | undefined> {
|
1506
|
+
// if (this.key) return this.key;
|
1507
|
+
// // generate a random key
|
1508
|
+
// if (!this.ebOpts.public) {
|
1509
|
+
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1510
|
+
// }
|
1511
|
+
// return this.key || undefined;
|
1512
|
+
// }
|
1513
|
+
async commitFiles(t, done) {
|
1514
|
+
await this.ready();
|
1515
|
+
const fstore = await this.fileStore();
|
1516
|
+
const wstore = await this.WALStore();
|
1517
|
+
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1508
1518
|
}
|
1509
|
-
async
|
1510
|
-
this.
|
1511
|
-
fid: this.sthis.nextId(this.fidLength).bin,
|
1512
|
-
ofs: 1024 * 1024,
|
1513
|
-
// 32bit
|
1514
|
-
len: 16 * 1024 * 1024,
|
1515
|
-
// 32bit
|
1516
|
-
data: new Uint8Array(1024)
|
1517
|
-
}).length - 1024;
|
1518
|
-
return this.innerGW.start(url);
|
1519
|
+
async loadFileCar(cid) {
|
1520
|
+
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
1519
1521
|
}
|
1520
|
-
async
|
1521
|
-
|
1522
|
+
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
1523
|
+
await this.ready();
|
1524
|
+
const carStore = await this.carStore();
|
1525
|
+
const params = {
|
1526
|
+
encoder: (await carStore.keyedCrypto()).codec(),
|
1527
|
+
carLog: this.carLog,
|
1528
|
+
carStore,
|
1529
|
+
WALStore: await this.WALStore(),
|
1530
|
+
metaStore: await this.metaStore(),
|
1531
|
+
threshold: this.ebOpts.threshold
|
1532
|
+
};
|
1533
|
+
return this.commitQueue.enqueue(async () => {
|
1534
|
+
await this.cacheTransaction(t);
|
1535
|
+
const ret = await commit(params, t, done, opts);
|
1536
|
+
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1537
|
+
return ret.cgrp;
|
1538
|
+
});
|
1522
1539
|
}
|
1523
|
-
async
|
1524
|
-
|
1525
|
-
|
1540
|
+
async updateCarLog(cids, fp, compact) {
|
1541
|
+
if (compact) {
|
1542
|
+
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1543
|
+
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1544
|
+
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1545
|
+
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1546
|
+
} else {
|
1547
|
+
this.carLog.unshift(cids);
|
1548
|
+
}
|
1549
|
+
}
|
1550
|
+
async cacheTransaction(t) {
|
1551
|
+
for await (const block of t.entries()) {
|
1552
|
+
const sBlock = block.cid.toString();
|
1553
|
+
if (!this.getBlockCache.has(sBlock)) {
|
1554
|
+
this.getBlockCache.set(sBlock, block);
|
1555
|
+
}
|
1556
|
+
}
|
1557
|
+
}
|
1558
|
+
async cacheCarReader(carCidStr, reader) {
|
1559
|
+
if (this.processedCars.has(carCidStr)) return;
|
1560
|
+
this.processedCars.add(carCidStr);
|
1561
|
+
for await (const block of reader.blocks()) {
|
1562
|
+
const sBlock = block.cid.toString();
|
1563
|
+
if (!this.getBlockCache.has(sBlock)) {
|
1564
|
+
this.getBlockCache.set(sBlock, block);
|
1565
|
+
}
|
1566
|
+
}
|
1526
1567
|
}
|
1527
|
-
async
|
1528
|
-
const
|
1529
|
-
|
1530
|
-
|
1531
|
-
|
1532
|
-
|
1568
|
+
async removeCidsForCompact(cid) {
|
1569
|
+
const carHeader = await this.loadCarHeaderFromMeta({
|
1570
|
+
cars: [cid]
|
1571
|
+
});
|
1572
|
+
for (const cids of carHeader.compact) {
|
1573
|
+
for (const cid2 of cids) {
|
1574
|
+
await (await this.carStore()).remove(cid2);
|
1533
1575
|
}
|
1534
|
-
const frag = rfrag.Ok();
|
1535
|
-
buffer = buffer || new Uint8Array(frag.len);
|
1536
|
-
buffer.set(frag.data, frag.ofs);
|
1537
1576
|
}
|
1538
|
-
return Result3.Ok(buffer || new Uint8Array(0));
|
1539
1577
|
}
|
1540
|
-
async
|
1541
|
-
|
1542
|
-
|
1578
|
+
// async flushCars() {
|
1579
|
+
// await this.ready
|
1580
|
+
// // for each cid in car log, make a dbMeta
|
1581
|
+
// for (const cid of this.carLog) {
|
1582
|
+
// const dbMeta = { car: cid, key: this.key || null } as DbMeta
|
1583
|
+
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
1584
|
+
// }
|
1585
|
+
// }
|
1586
|
+
async *entries(cache2 = true) {
|
1587
|
+
await this.ready();
|
1588
|
+
if (cache2) {
|
1589
|
+
for (const [, block] of this.getBlockCache) {
|
1590
|
+
yield block;
|
1591
|
+
}
|
1543
1592
|
} else {
|
1544
|
-
|
1593
|
+
for (const [, block] of this.getBlockCache) {
|
1594
|
+
yield block;
|
1595
|
+
}
|
1596
|
+
for (const cids of this.carLog) {
|
1597
|
+
for (const cid of cids) {
|
1598
|
+
const reader = await this.loadCar(cid);
|
1599
|
+
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
1600
|
+
for await (const block of reader.blocks()) {
|
1601
|
+
const sCid = block.cid.toString();
|
1602
|
+
if (!this.getBlockCache.has(sCid)) {
|
1603
|
+
yield block;
|
1604
|
+
}
|
1605
|
+
}
|
1606
|
+
}
|
1607
|
+
}
|
1545
1608
|
}
|
1546
1609
|
}
|
1547
|
-
async
|
1548
|
-
|
1549
|
-
|
1550
|
-
|
1551
|
-
|
1610
|
+
async getBlock(cid) {
|
1611
|
+
await this.ready();
|
1612
|
+
const sCid = cid.toString();
|
1613
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1614
|
+
const getCarCid = async (carCid) => {
|
1615
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1616
|
+
const reader = await this.loadCar(carCid);
|
1617
|
+
if (!reader) {
|
1618
|
+
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
1552
1619
|
}
|
1553
|
-
|
1554
|
-
|
1555
|
-
|
1556
|
-
|
1620
|
+
await this.cacheCarReader(carCid.toString(), reader).catch(() => {
|
1621
|
+
return;
|
1622
|
+
});
|
1623
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1624
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
|
1625
|
+
};
|
1626
|
+
const getCompactCarCids = async (carCid) => {
|
1627
|
+
const reader = await this.loadCar(carCid);
|
1628
|
+
if (!reader) {
|
1629
|
+
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
1630
|
+
}
|
1631
|
+
const header = await parseCarFile(reader, this.logger);
|
1632
|
+
const compacts = header.compact;
|
1633
|
+
let got2;
|
1634
|
+
const batchSize2 = 5;
|
1635
|
+
for (let i = 0; i < compacts.length; i += batchSize2) {
|
1636
|
+
const promises = [];
|
1637
|
+
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
1638
|
+
for (const cid2 of compacts[j]) {
|
1639
|
+
promises.push(getCarCid(cid2));
|
1640
|
+
}
|
1641
|
+
}
|
1642
|
+
try {
|
1643
|
+
got2 = await Promise.any(promises);
|
1644
|
+
} catch {
|
1645
|
+
}
|
1646
|
+
if (got2) break;
|
1647
|
+
}
|
1648
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1649
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
1650
|
+
};
|
1651
|
+
let got;
|
1652
|
+
const batchSize = 5;
|
1653
|
+
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
1654
|
+
const batch = this.carLog.slice(i, i + batchSize);
|
1655
|
+
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
1656
|
+
try {
|
1657
|
+
got = await Promise.any(promises);
|
1658
|
+
} catch {
|
1659
|
+
}
|
1660
|
+
if (got) break;
|
1557
1661
|
}
|
1558
|
-
|
1559
|
-
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1563
|
-
|
1564
|
-
|
1565
|
-
import { CID as CID2 } from "multiformats";
|
1566
|
-
import { base64pad } from "multiformats/bases/base64";
|
1567
|
-
import { Result as Result4 } from "@adviser/cement";
|
1568
|
-
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
1569
|
-
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
1570
|
-
if (!crdtEntries.length) {
|
1571
|
-
sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
|
1572
|
-
return [];
|
1662
|
+
if (!got) {
|
1663
|
+
try {
|
1664
|
+
got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
|
1665
|
+
} catch {
|
1666
|
+
}
|
1667
|
+
}
|
1668
|
+
return got;
|
1573
1669
|
}
|
1574
|
-
|
1575
|
-
|
1576
|
-
|
1670
|
+
async loadCar(cid) {
|
1671
|
+
if (!this.carStore) {
|
1672
|
+
throw this.logger.Error().Msg("car store not initialized").AsError();
|
1673
|
+
}
|
1674
|
+
const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
|
1675
|
+
return loaded;
|
1577
1676
|
}
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
|
1582
|
-
|
1583
|
-
|
1584
|
-
|
1585
|
-
|
1586
|
-
|
1587
|
-
|
1588
|
-
|
1589
|
-
|
1590
|
-
|
1591
|
-
|
1592
|
-
|
1593
|
-
|
1594
|
-
if (keyInfo.length) {
|
1595
|
-
const dbMeta = keyInfo[0].dbMeta;
|
1596
|
-
if (dbMeta.key) {
|
1597
|
-
const kb = await getKeyBag(sthis);
|
1598
|
-
const keyName = getStoreKeyName(uri);
|
1599
|
-
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
1600
|
-
if (res.isErr()) {
|
1601
|
-
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
1602
|
-
throw res.Err();
|
1677
|
+
async makeDecoderAndCarReader(cid, local, remote) {
|
1678
|
+
const cidsString = cid.toString();
|
1679
|
+
let loadedCar = void 0;
|
1680
|
+
let activeStore = local;
|
1681
|
+
try {
|
1682
|
+
this.logger.Debug().Str("cid", cidsString).Msg("loading car");
|
1683
|
+
loadedCar = await local.load(cid);
|
1684
|
+
this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
|
1685
|
+
} catch (e) {
|
1686
|
+
if (remote) {
|
1687
|
+
const remoteCar = await remote.load(cid);
|
1688
|
+
if (remoteCar) {
|
1689
|
+
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1690
|
+
await local.save(remoteCar);
|
1691
|
+
loadedCar = remoteCar;
|
1692
|
+
activeStore = remote;
|
1603
1693
|
}
|
1694
|
+
} else {
|
1695
|
+
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1604
1696
|
}
|
1605
|
-
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
1606
|
-
return Result4.Ok(dbMeta);
|
1607
1697
|
}
|
1608
|
-
|
1609
|
-
|
1610
|
-
} catch (error) {
|
1611
|
-
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
1612
|
-
return Result4.Err(error);
|
1613
|
-
}
|
1614
|
-
}
|
1615
|
-
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
1616
|
-
try {
|
1617
|
-
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
1618
|
-
const keyName = getStoreKeyName(uri);
|
1619
|
-
const kb = await getKeyBag(sthis);
|
1620
|
-
const res = await kb.getNamedExtractableKey(keyName, true);
|
1621
|
-
if (res.isErr()) {
|
1622
|
-
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
1623
|
-
throw res.Err();
|
1698
|
+
if (!loadedCar) {
|
1699
|
+
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
1624
1700
|
}
|
1625
|
-
const
|
1626
|
-
const
|
1627
|
-
const
|
1628
|
-
const
|
1629
|
-
|
1630
|
-
|
1631
|
-
|
1632
|
-
|
1633
|
-
|
1634
|
-
|
1635
|
-
|
1636
|
-
return
|
1701
|
+
const bytes = await decode3({ bytes: loadedCar.bytes, hasher: hasher4, codec: (await activeStore.keyedCrypto()).codec() });
|
1702
|
+
const rawReader = await CarReader.fromBytes(bytes.value);
|
1703
|
+
const readerP = Promise.resolve(rawReader);
|
1704
|
+
const cachedReaderP = readerP.then(async (reader) => {
|
1705
|
+
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
1706
|
+
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
1707
|
+
return;
|
1708
|
+
});
|
1709
|
+
return reader;
|
1710
|
+
});
|
1711
|
+
this.carReaders.set(cidsString, cachedReaderP);
|
1712
|
+
return readerP;
|
1637
1713
|
}
|
1638
|
-
|
1639
|
-
|
1640
|
-
|
1641
|
-
|
1642
|
-
|
1643
|
-
|
1714
|
+
//What if instead it returns an Array of CarHeader
|
1715
|
+
async storesLoadCar(cid, local, remote) {
|
1716
|
+
const cidsString = cid.toString();
|
1717
|
+
let dacr = this.carReaders.get(cidsString);
|
1718
|
+
if (!dacr) {
|
1719
|
+
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
1720
|
+
this.carReaders.set(cidsString, dacr);
|
1721
|
+
}
|
1722
|
+
return dacr;
|
1644
1723
|
}
|
1645
|
-
|
1646
|
-
|
1647
|
-
|
1648
|
-
|
1649
|
-
|
1650
|
-
|
1651
|
-
dbMeta: sthis.txt.encode(format(dbMeta))
|
1652
|
-
},
|
1653
|
-
parents
|
1654
|
-
);
|
1655
|
-
return event;
|
1656
|
-
}
|
1657
|
-
async function encodeEventsWithParents(sthis, events, parents) {
|
1658
|
-
const crdtEntries = events.map((event) => {
|
1659
|
-
const base64String = base64pad.encode(event.bytes);
|
1660
|
-
return {
|
1661
|
-
cid: event.cid.toString(),
|
1662
|
-
data: base64String,
|
1663
|
-
parents: parents.map((p) => p.toString())
|
1664
|
-
};
|
1665
|
-
});
|
1666
|
-
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
1667
|
-
}
|
1724
|
+
async getMoreReaders(cids) {
|
1725
|
+
const limit = pLimit(5);
|
1726
|
+
const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
|
1727
|
+
await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
|
1728
|
+
}
|
1729
|
+
};
|
1668
1730
|
|
1669
1731
|
// src/blockstore/store.ts
|
1670
1732
|
function guardVersion(url) {
|
@@ -1674,16 +1736,21 @@ function guardVersion(url) {
|
|
1674
1736
|
return Result5.Ok(url);
|
1675
1737
|
}
|
1676
1738
|
var BaseStoreImpl = class {
|
1677
|
-
constructor(
|
1739
|
+
constructor(sthis, url, opts, logger) {
|
1678
1740
|
this._onStarted = [];
|
1679
1741
|
this._onClosed = [];
|
1680
|
-
this.name = name;
|
1681
1742
|
this._url = url;
|
1682
1743
|
this.keybag = opts.keybag;
|
1744
|
+
this.loader = opts.loader;
|
1683
1745
|
this.sthis = sthis;
|
1684
|
-
|
1746
|
+
const name = this._url.getParam("name" /* NAME */);
|
1747
|
+
if (!name) {
|
1748
|
+
throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
|
1749
|
+
}
|
1750
|
+
this.name = name;
|
1751
|
+
this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
|
1752
|
+
this.realGateway = opts.gateway;
|
1685
1753
|
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
1686
|
-
this.loader = opts.loader;
|
1687
1754
|
}
|
1688
1755
|
url() {
|
1689
1756
|
return this._url;
|
@@ -1698,20 +1765,20 @@ var BaseStoreImpl = class {
|
|
1698
1765
|
return;
|
1699
1766
|
}
|
1700
1767
|
async keyedCrypto() {
|
1701
|
-
return keyedCryptoFactory(this._url,
|
1768
|
+
return keyedCryptoFactory(this._url, this.keybag, this.sthis);
|
1702
1769
|
}
|
1703
1770
|
async start() {
|
1704
1771
|
this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
|
1705
|
-
this._url = this._url.build().setParam("store"
|
1772
|
+
this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
|
1706
1773
|
const res = await this.gateway.start(this._url);
|
1707
1774
|
if (res.isErr()) {
|
1708
1775
|
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
1709
1776
|
return res;
|
1710
1777
|
}
|
1711
1778
|
this._url = res.Ok();
|
1712
|
-
const kb = await this.keybag
|
1779
|
+
const kb = await this.keybag;
|
1713
1780
|
const skRes = await kb.ensureKeyFromUrl(this._url, () => {
|
1714
|
-
const idx = this._url.getParam("index");
|
1781
|
+
const idx = this._url.getParam("index" /* INDEX */);
|
1715
1782
|
const storeKeyName = [this.name];
|
1716
1783
|
if (idx) {
|
1717
1784
|
storeKeyName.push(idx);
|
@@ -1744,8 +1811,8 @@ var BaseStoreImpl = class {
|
|
1744
1811
|
};
|
1745
1812
|
var MetaStoreImpl = class extends BaseStoreImpl {
|
1746
1813
|
// remote: boolean;
|
1747
|
-
constructor(sthis,
|
1748
|
-
super(
|
1814
|
+
constructor(sthis, url, opts) {
|
1815
|
+
super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
|
1749
1816
|
this.storeType = "meta";
|
1750
1817
|
this.subscribers = /* @__PURE__ */ new Map();
|
1751
1818
|
this.parents = [];
|
@@ -1816,13 +1883,21 @@ var MetaStoreImpl = class extends BaseStoreImpl {
|
|
1816
1883
|
return Result5.Ok(void 0);
|
1817
1884
|
}
|
1818
1885
|
async destroy() {
|
1886
|
+
this.logger.Debug().Msg("destroy");
|
1819
1887
|
return this.gateway.destroy(this.url());
|
1820
1888
|
}
|
1821
1889
|
};
|
1822
1890
|
var DataStoreImpl = class extends BaseStoreImpl {
|
1823
1891
|
// readonly tag: string = "car-base";
|
1824
|
-
constructor(sthis,
|
1825
|
-
super(
|
1892
|
+
constructor(sthis, url, opts) {
|
1893
|
+
super(
|
1894
|
+
sthis,
|
1895
|
+
url,
|
1896
|
+
{
|
1897
|
+
...opts
|
1898
|
+
},
|
1899
|
+
ensureLogger(sthis, "DataStoreImpl")
|
1900
|
+
);
|
1826
1901
|
this.storeType = "data";
|
1827
1902
|
}
|
1828
1903
|
async load(cid) {
|
@@ -1863,23 +1938,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
|
|
1863
1938
|
return Result5.Ok(void 0);
|
1864
1939
|
}
|
1865
1940
|
destroy() {
|
1941
|
+
this.logger.Debug().Msg("destroy");
|
1866
1942
|
return this.gateway.destroy(this.url());
|
1867
1943
|
}
|
1868
1944
|
};
|
1869
1945
|
var WALStoreImpl = class extends BaseStoreImpl {
|
1870
|
-
constructor(
|
1871
|
-
super(
|
1946
|
+
constructor(sthis, url, opts) {
|
1947
|
+
super(
|
1948
|
+
sthis,
|
1949
|
+
url,
|
1950
|
+
{
|
1951
|
+
...opts
|
1952
|
+
},
|
1953
|
+
ensureLogger(sthis, "WALStoreImpl")
|
1954
|
+
);
|
1872
1955
|
this.storeType = "wal";
|
1956
|
+
// readonly tag: string = "rwal-base";
|
1957
|
+
// readonly loader: Loadable;
|
1873
1958
|
this._ready = new ResolveOnce3();
|
1874
1959
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
1875
1960
|
this.processing = void 0;
|
1876
1961
|
this.processQueue = new CommitQueue();
|
1877
|
-
this.loader = loader;
|
1878
1962
|
}
|
1879
1963
|
async ready() {
|
1880
1964
|
return this._ready.once(async () => {
|
1881
1965
|
const walState = await this.load().catch((e) => {
|
1882
|
-
this.logger.Error().
|
1966
|
+
this.logger.Error().Err(e).Msg("error loading wal");
|
1883
1967
|
return void 0;
|
1884
1968
|
});
|
1885
1969
|
if (!walState) {
|
@@ -1912,7 +1996,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1912
1996
|
}
|
1913
1997
|
async process() {
|
1914
1998
|
await this.ready();
|
1915
|
-
if (!this.loader
|
1999
|
+
if (!this.loader?.remoteCarStore) return;
|
1916
2000
|
await this.processQueue.enqueue(async () => {
|
1917
2001
|
try {
|
1918
2002
|
await this._doProcess();
|
@@ -1925,73 +2009,88 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1925
2009
|
});
|
1926
2010
|
}
|
1927
2011
|
async _doProcess() {
|
2012
|
+
if (!this.loader) return;
|
1928
2013
|
if (!this.loader.remoteCarStore) return;
|
1929
|
-
const
|
1930
|
-
|
1931
|
-
|
1932
|
-
|
1933
|
-
|
1934
|
-
|
1935
|
-
|
1936
|
-
|
1937
|
-
|
1938
|
-
|
1939
|
-
|
1940
|
-
|
1941
|
-
|
1942
|
-
|
1943
|
-
|
1944
|
-
|
2014
|
+
const operations = [...this.walState.operations];
|
2015
|
+
const noLoaderOps = [...this.walState.noLoaderOps];
|
2016
|
+
const fileOperations = [...this.walState.fileOperations];
|
2017
|
+
if (operations.length + noLoaderOps.length + fileOperations.length === 0) return;
|
2018
|
+
const concurrencyLimit = 3;
|
2019
|
+
const retryableUpload = (fn, description) => pRetry(fn, {
|
2020
|
+
retries: 5,
|
2021
|
+
onFailedAttempt: (error) => {
|
2022
|
+
this.logger.Warn().Msg(`Attempt ${error.attemptNumber} failed for ${description}. There are ${error.retriesLeft} retries left.`);
|
2023
|
+
}
|
2024
|
+
});
|
2025
|
+
try {
|
2026
|
+
await pMap(
|
2027
|
+
noLoaderOps,
|
2028
|
+
async (dbMeta) => {
|
2029
|
+
await retryableUpload(async () => {
|
2030
|
+
if (!this.loader) return;
|
2031
|
+
for (const cid of dbMeta.cars) {
|
2032
|
+
const car = await (await this.loader.carStore()).load(cid);
|
2033
|
+
if (!car) {
|
2034
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
|
2035
|
+
throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
|
2036
|
+
}
|
2037
|
+
} else {
|
2038
|
+
await throwFalsy(this.loader.remoteCarStore).save(car);
|
2039
|
+
}
|
1945
2040
|
}
|
1946
2041
|
this.walState.noLoaderOps = this.walState.noLoaderOps.filter((op) => op !== dbMeta);
|
1947
|
-
}
|
1948
|
-
}
|
1949
|
-
|
1950
|
-
|
1951
|
-
|
1952
|
-
|
1953
|
-
|
1954
|
-
|
1955
|
-
if (!
|
1956
|
-
|
1957
|
-
|
1958
|
-
|
1959
|
-
|
2042
|
+
}, `noLoaderOp with dbMeta.cars=${dbMeta.cars.toString()}`);
|
2043
|
+
},
|
2044
|
+
{ concurrency: concurrencyLimit }
|
2045
|
+
);
|
2046
|
+
await pMap(
|
2047
|
+
operations,
|
2048
|
+
async (dbMeta) => {
|
2049
|
+
await retryableUpload(async () => {
|
2050
|
+
if (!this.loader) return;
|
2051
|
+
for (const cid of dbMeta.cars) {
|
2052
|
+
const car = await (await this.loader.carStore()).load(cid);
|
2053
|
+
if (!car) {
|
2054
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
|
2055
|
+
throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
|
2056
|
+
}
|
2057
|
+
} else {
|
2058
|
+
await throwFalsy(this.loader.remoteCarStore).save(car);
|
2059
|
+
}
|
1960
2060
|
}
|
1961
|
-
|
1962
|
-
|
1963
|
-
}
|
1964
|
-
|
1965
|
-
|
1966
|
-
|
1967
|
-
|
1968
|
-
|
1969
|
-
|
1970
|
-
|
1971
|
-
|
2061
|
+
this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
|
2062
|
+
}, `operation with dbMeta.cars=${dbMeta.cars.toString()}`);
|
2063
|
+
},
|
2064
|
+
{ concurrency: concurrencyLimit }
|
2065
|
+
);
|
2066
|
+
await pMap(
|
2067
|
+
fileOperations,
|
2068
|
+
async ({ cid: fileCid, public: publicFile }) => {
|
2069
|
+
await retryableUpload(async () => {
|
2070
|
+
if (!this.loader) return;
|
2071
|
+
const fileBlock = await (await this.loader.fileStore()).load(fileCid);
|
2072
|
+
if (!fileBlock) {
|
2073
|
+
throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
|
2074
|
+
}
|
2075
|
+
await this.loader.remoteFileStore?.save(fileBlock, { public: publicFile });
|
1972
2076
|
this.walState.fileOperations = this.walState.fileOperations.filter((op) => op.cid !== fileCid);
|
1973
|
-
});
|
1974
|
-
|
1975
|
-
}
|
1976
|
-
|
1977
|
-
|
1978
|
-
const
|
1979
|
-
|
1980
|
-
|
1981
|
-
|
1982
|
-
}
|
1983
|
-
|
1984
|
-
|
1985
|
-
|
1986
|
-
|
1987
|
-
|
1988
|
-
|
1989
|
-
|
1990
|
-
} finally {
|
1991
|
-
await this.save(this.walState);
|
1992
|
-
}
|
1993
|
-
})();
|
1994
|
-
await rmlp;
|
2077
|
+
}, `fileOperation with cid=${fileCid.toString()}`);
|
2078
|
+
},
|
2079
|
+
{ concurrency: concurrencyLimit }
|
2080
|
+
);
|
2081
|
+
if (operations.length) {
|
2082
|
+
const lastOp = operations[operations.length - 1];
|
2083
|
+
await retryableUpload(async () => {
|
2084
|
+
if (!this.loader) return;
|
2085
|
+
await this.loader.remoteMetaStore?.save(lastOp);
|
2086
|
+
}, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
|
2087
|
+
}
|
2088
|
+
} catch (error) {
|
2089
|
+
this.logger.Error().Any("error", error).Msg("Processing failed");
|
2090
|
+
return;
|
2091
|
+
} finally {
|
2092
|
+
await this.save(this.walState);
|
2093
|
+
}
|
1995
2094
|
}
|
1996
2095
|
async load() {
|
1997
2096
|
this.logger.Debug().Msg("loading");
|
@@ -2008,6 +2107,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2008
2107
|
}
|
2009
2108
|
try {
|
2010
2109
|
return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
|
2110
|
+
return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
|
2011
2111
|
} catch (e) {
|
2012
2112
|
throw this.logger.Error().Err(e).Msg("error parse").AsError();
|
2013
2113
|
}
|
@@ -2034,72 +2134,92 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2034
2134
|
return Result5.Ok(void 0);
|
2035
2135
|
}
|
2036
2136
|
destroy() {
|
2137
|
+
this.logger.Debug().Msg("destroy");
|
2037
2138
|
return this.gateway.destroy(this.url());
|
2038
2139
|
}
|
2039
2140
|
};
|
2040
2141
|
|
2041
|
-
// src/blockstore/store-
|
2042
|
-
|
2043
|
-
|
2044
|
-
|
2142
|
+
// src/blockstore/register-store-protocol.ts
|
2143
|
+
import { BuildURI, runtimeFn as runtimeFn2 } from "@adviser/cement";
|
2144
|
+
|
2145
|
+
// src/runtime/gateways/memory/gateway.ts
|
2146
|
+
import { Result as Result6 } from "@adviser/cement";
|
2147
|
+
|
2148
|
+
// src/runtime/gateways/memory/version.ts
|
2149
|
+
var MEMORY_VERSION = "v0.19-memory";
|
2150
|
+
|
2151
|
+
// src/runtime/gateways/memory/gateway.ts
|
2152
|
+
var MemoryGateway = class {
|
2153
|
+
constructor(memorys) {
|
2154
|
+
this.memorys = memorys;
|
2045
2155
|
}
|
2046
|
-
|
2047
|
-
|
2048
|
-
function ensureName(name, url) {
|
2049
|
-
if (!url.hasParam("name")) {
|
2050
|
-
return url.build().setParam("name", name).URI();
|
2156
|
+
buildUrl(baseUrl, key) {
|
2157
|
+
return Promise.resolve(Result6.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
|
2051
2158
|
}
|
2052
|
-
|
2053
|
-
|
2159
|
+
start(baseUrl) {
|
2160
|
+
return Promise.resolve(Result6.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
|
2161
|
+
}
|
2162
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2163
|
+
close(baseUrl) {
|
2164
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2165
|
+
}
|
2166
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2167
|
+
destroy(baseUrl) {
|
2168
|
+
this.memorys.clear();
|
2169
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2170
|
+
}
|
2171
|
+
put(url, body) {
|
2172
|
+
this.memorys.set(url.toString(), body);
|
2173
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2174
|
+
}
|
2175
|
+
// get could return a NotFoundError if the key is not found
|
2176
|
+
get(url) {
|
2177
|
+
const x = this.memorys.get(url.toString());
|
2178
|
+
if (x === void 0) {
|
2179
|
+
return Promise.resolve(Result6.Err(new NotFoundError("not found")));
|
2180
|
+
}
|
2181
|
+
return Promise.resolve(Result6.Ok(x));
|
2182
|
+
}
|
2183
|
+
delete(url) {
|
2184
|
+
this.memorys.delete(url.toString());
|
2185
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2186
|
+
}
|
2187
|
+
};
|
2188
|
+
var MemoryTestGateway = class {
|
2189
|
+
constructor(memorys) {
|
2190
|
+
this.memorys = memorys;
|
2191
|
+
}
|
2192
|
+
async get(url, key) {
|
2193
|
+
return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
|
2194
|
+
}
|
2195
|
+
};
|
2196
|
+
|
2197
|
+
// src/blockstore/register-store-protocol.ts
|
2054
2198
|
var storeFactory = /* @__PURE__ */ new Map();
|
2055
|
-
function
|
2056
|
-
|
2057
|
-
|
2058
|
-
|
2059
|
-
if (obuItem && obuItem.overrideBaseURL) {
|
2060
|
-
obuUrl = URI5.from(obuItem.overrideBaseURL);
|
2061
|
-
}
|
2062
|
-
const ret = ensureIsIndex(
|
2063
|
-
URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
2064
|
-
storeOpts.isIndex
|
2065
|
-
);
|
2066
|
-
return ret;
|
2067
|
-
}
|
2068
|
-
var onceGateway = new KeyedResolvOnce2();
|
2069
|
-
async function getGatewayFromURL(url, sthis) {
|
2070
|
-
return onceGateway.get(url.toString()).once(async () => {
|
2071
|
-
const item = storeFactory.get(url.protocol);
|
2072
|
-
if (item) {
|
2073
|
-
const ret = {
|
2074
|
-
gateway: await item.gateway(sthis),
|
2075
|
-
test: await item.test(sthis)
|
2076
|
-
};
|
2077
|
-
const res = await ret.gateway.start(url);
|
2078
|
-
if (res.isErr()) {
|
2079
|
-
sthis.logger.Error().Result("start", res).Msg("start failed");
|
2080
|
-
return void 0;
|
2081
|
-
}
|
2082
|
-
return ret;
|
2199
|
+
function getDefaultURI(sthis, protocol) {
|
2200
|
+
if (protocol) {
|
2201
|
+
if (!protocol.endsWith(":")) {
|
2202
|
+
protocol += ":";
|
2083
2203
|
}
|
2084
|
-
|
2085
|
-
|
2086
|
-
|
2204
|
+
const gfi = storeFactory.get(protocol);
|
2205
|
+
if (gfi) {
|
2206
|
+
return gfi.defaultURI(sthis);
|
2207
|
+
}
|
2208
|
+
}
|
2209
|
+
const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
|
2210
|
+
if (!found) {
|
2211
|
+
throw sthis.logger.Error().Msg("no default found").AsError();
|
2212
|
+
}
|
2213
|
+
return found.defaultURI(sthis);
|
2087
2214
|
}
|
2088
2215
|
function registerStoreProtocol(item) {
|
2089
2216
|
let protocol = item.protocol;
|
2090
2217
|
if (!protocol.endsWith(":")) {
|
2091
2218
|
protocol += ":";
|
2092
2219
|
}
|
2093
|
-
if (
|
2094
|
-
if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
|
2095
|
-
throw new Error(`we need a logger here`);
|
2096
|
-
return () => {
|
2097
|
-
};
|
2098
|
-
}
|
2099
|
-
}
|
2100
|
-
if (item.overrideBaseURL) {
|
2220
|
+
if (item.isDefault) {
|
2101
2221
|
Array.from(storeFactory.values()).forEach((items) => {
|
2102
|
-
items.
|
2222
|
+
items.isDefault = false;
|
2103
2223
|
});
|
2104
2224
|
}
|
2105
2225
|
storeFactory.set(protocol, item);
|
@@ -2107,134 +2227,183 @@ function registerStoreProtocol(item) {
|
|
2107
2227
|
storeFactory.delete(protocol);
|
2108
2228
|
};
|
2109
2229
|
}
|
2110
|
-
|
2111
|
-
|
2112
|
-
|
2113
|
-
|
2114
|
-
return
|
2115
|
-
|
2116
|
-
|
2117
|
-
|
2230
|
+
function getGatewayFactoryItem(protocol) {
|
2231
|
+
return storeFactory.get(protocol);
|
2232
|
+
}
|
2233
|
+
function fileGatewayFactoryItem() {
|
2234
|
+
return {
|
2235
|
+
protocol: "file:",
|
2236
|
+
isDefault: true,
|
2237
|
+
defaultURI: (sthis) => {
|
2238
|
+
return BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
|
2239
|
+
},
|
2240
|
+
gateway: async (sthis) => {
|
2241
|
+
const { FileGateway } = await import("./gateway-GK5QZ6KP.js");
|
2242
|
+
return new FileGateway(sthis);
|
2243
|
+
},
|
2244
|
+
test: async (sthis) => {
|
2245
|
+
const { FileTestStore } = await import("./gateway-GK5QZ6KP.js");
|
2246
|
+
return new FileTestStore(sthis);
|
2247
|
+
}
|
2248
|
+
};
|
2249
|
+
}
|
2250
|
+
if (runtimeFn2().isBrowser) {
|
2251
|
+
registerStoreProtocol({
|
2252
|
+
protocol: "indexdb:",
|
2253
|
+
isDefault: true,
|
2254
|
+
defaultURI: () => {
|
2255
|
+
return BuildURI.from("indexdb://").pathname("fp").URI();
|
2256
|
+
},
|
2257
|
+
gateway: async (logger) => {
|
2258
|
+
const { IndexDBGateway } = await import("./gateway-TQTGDRCN.js");
|
2259
|
+
return new IndexDBGateway(logger);
|
2260
|
+
},
|
2261
|
+
test: async (logger) => {
|
2262
|
+
const { IndexDBTestStore } = await import("./gateway-TQTGDRCN.js");
|
2263
|
+
return new IndexDBTestStore(logger);
|
2118
2264
|
}
|
2119
|
-
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2120
|
-
gateway: gateway.gateway,
|
2121
|
-
keybag: () => getKeyBag(loader.sthis, {
|
2122
|
-
...loader.ebOpts.keyBag
|
2123
|
-
})
|
2124
|
-
});
|
2125
|
-
return store;
|
2126
2265
|
});
|
2266
|
+
} else {
|
2267
|
+
registerStoreProtocol(fileGatewayFactoryItem());
|
2127
2268
|
}
|
2128
|
-
var
|
2129
|
-
|
2130
|
-
|
2131
|
-
|
2132
|
-
|
2133
|
-
|
2134
|
-
|
2135
|
-
|
2136
|
-
|
2137
|
-
|
2138
|
-
|
2139
|
-
|
2140
|
-
|
2141
|
-
|
2142
|
-
|
2143
|
-
|
2144
|
-
|
2269
|
+
var memory = /* @__PURE__ */ new Map();
|
2270
|
+
registerStoreProtocol({
|
2271
|
+
protocol: "memory:",
|
2272
|
+
isDefault: false,
|
2273
|
+
defaultURI: () => {
|
2274
|
+
return BuildURI.from("memory://").pathname("ram").URI();
|
2275
|
+
},
|
2276
|
+
gateway: async () => {
|
2277
|
+
return new MemoryGateway(memory);
|
2278
|
+
},
|
2279
|
+
test: async () => {
|
2280
|
+
return new MemoryTestGateway(memory);
|
2281
|
+
}
|
2282
|
+
});
|
2283
|
+
|
2284
|
+
// src/blockstore/store-factory.ts
|
2285
|
+
var onceGateway = new KeyedResolvOnce2();
|
2286
|
+
var gatewayInstances = new KeyedResolvOnce2();
|
2287
|
+
async function getStartedGateway(sthis, url) {
|
2288
|
+
return onceGateway.get(url.toString()).once(async () => {
|
2289
|
+
const item = getGatewayFactoryItem(url.protocol);
|
2290
|
+
if (item) {
|
2291
|
+
const ret = {
|
2292
|
+
url,
|
2293
|
+
...await gatewayInstances.get(url.protocol).once(async () => ({
|
2294
|
+
gateway: await item.gateway(sthis),
|
2295
|
+
test: await item.test(sthis)
|
2296
|
+
}))
|
2297
|
+
};
|
2298
|
+
const res = await ret.gateway.start(url);
|
2299
|
+
if (res.isErr()) {
|
2300
|
+
return Result7.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
|
2301
|
+
}
|
2302
|
+
ret.url = res.Ok();
|
2303
|
+
return Result7.Ok(ret);
|
2304
|
+
}
|
2305
|
+
return Result7.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
|
2145
2306
|
});
|
2146
2307
|
}
|
2147
|
-
|
2148
|
-
|
2149
|
-
const
|
2150
|
-
|
2151
|
-
|
2152
|
-
|
2153
|
-
|
2154
|
-
|
2155
|
-
|
2156
|
-
|
2157
|
-
|
2158
|
-
|
2159
|
-
|
2160
|
-
|
2161
|
-
|
2162
|
-
|
2163
|
-
|
2308
|
+
async function dataStoreFactory(sfi) {
|
2309
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
|
2310
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
2311
|
+
if (rgateway.isErr()) {
|
2312
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
2313
|
+
}
|
2314
|
+
const gateway = rgateway.Ok();
|
2315
|
+
const store = new DataStoreImpl(sfi.sthis, gateway.url, {
|
2316
|
+
gateway: gateway.gateway,
|
2317
|
+
keybag: sfi.keybag
|
2318
|
+
});
|
2319
|
+
return store;
|
2320
|
+
}
|
2321
|
+
async function metaStoreFactory(sfi) {
|
2322
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
|
2323
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
2324
|
+
if (rgateway.isErr()) {
|
2325
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
2326
|
+
}
|
2327
|
+
const gateway = rgateway.Ok();
|
2328
|
+
const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
|
2329
|
+
gateway: gateway.gateway,
|
2330
|
+
keybag: sfi.keybag
|
2331
|
+
});
|
2332
|
+
return store;
|
2333
|
+
}
|
2334
|
+
async function WALStoreFactory(sfi) {
|
2335
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
|
2336
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
2337
|
+
if (rgateway.isErr()) {
|
2338
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
2339
|
+
}
|
2340
|
+
const gateway = rgateway.Ok();
|
2341
|
+
const store = new WALStoreImpl(sfi.sthis, gateway.url, {
|
2342
|
+
gateway: gateway.gateway,
|
2343
|
+
keybag: sfi.keybag
|
2164
2344
|
});
|
2345
|
+
return store;
|
2165
2346
|
}
|
2166
2347
|
async function testStoreFactory(url, sthis) {
|
2167
|
-
|
2168
|
-
|
2169
|
-
if (!gateway) {
|
2348
|
+
const rgateway = await getStartedGateway(sthis, url);
|
2349
|
+
if (!rgateway) {
|
2170
2350
|
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2171
2351
|
}
|
2172
|
-
return
|
2352
|
+
return rgateway.Ok().test;
|
2173
2353
|
}
|
2174
|
-
async function ensureStart(store
|
2354
|
+
async function ensureStart(store) {
|
2175
2355
|
const ret = await store.start();
|
2176
2356
|
if (ret.isErr()) {
|
2177
|
-
throw logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2357
|
+
throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2178
2358
|
}
|
2179
|
-
logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2359
|
+
store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2180
2360
|
return store;
|
2181
2361
|
}
|
2182
|
-
function
|
2183
|
-
|
2362
|
+
function ensureStoreEnDeFile(ende) {
|
2363
|
+
ende = ende || {};
|
2184
2364
|
return {
|
2185
|
-
|
2186
|
-
|
2187
|
-
|
2188
|
-
|
2189
|
-
|
2190
|
-
|
2191
|
-
|
2192
|
-
|
2193
|
-
|
2194
|
-
|
2195
|
-
|
2196
|
-
|
2197
|
-
|
2198
|
-
|
2365
|
+
encodeFile: ende.encodeFile || encodeFile,
|
2366
|
+
decodeFile: ende.decodeFile || decodeFile
|
2367
|
+
};
|
2368
|
+
}
|
2369
|
+
function toStoreRuntime(sthis, endeOpts = {}) {
|
2370
|
+
return {
|
2371
|
+
makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
|
2372
|
+
// async (loader: Loadable) => {
|
2373
|
+
// logger
|
2374
|
+
// .Debug()
|
2375
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
|
2376
|
+
// .Msg("makeMetaStore");
|
2377
|
+
// return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
|
2378
|
+
// },
|
2379
|
+
makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
|
2380
|
+
// async (loader: Loadable) => {
|
2381
|
+
// logger
|
2382
|
+
// .Debug()
|
2383
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
|
2384
|
+
// .Msg("makeDataStore");
|
2385
|
+
// return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
|
2386
|
+
// },
|
2387
|
+
makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
|
2388
|
+
// async (loader: Loadable) => {
|
2389
|
+
// logger
|
2390
|
+
// .Debug()
|
2391
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
|
2392
|
+
// .Msg("makeRemoteWAL");
|
2393
|
+
// return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
|
2394
|
+
// },
|
2395
|
+
...ensureStoreEnDeFile(endeOpts)
|
2199
2396
|
};
|
2200
2397
|
}
|
2201
|
-
registerStoreProtocol({
|
2202
|
-
protocol: "file:",
|
2203
|
-
gateway: async (sthis) => {
|
2204
|
-
const { FileGateway } = await import("./gateway-5FCWPX5W.js");
|
2205
|
-
return new FileGateway(sthis);
|
2206
|
-
},
|
2207
|
-
test: async (sthis) => {
|
2208
|
-
const { FileTestStore } = await import("./gateway-5FCWPX5W.js");
|
2209
|
-
return new FileTestStore(sthis);
|
2210
|
-
}
|
2211
|
-
});
|
2212
|
-
registerStoreProtocol({
|
2213
|
-
protocol: "indexdb:",
|
2214
|
-
gateway: async (sthis) => {
|
2215
|
-
const { IndexDBGateway } = await import("./gateway-H7UD6TNB.js");
|
2216
|
-
return new IndexDBGateway(sthis);
|
2217
|
-
},
|
2218
|
-
test: async (sthis) => {
|
2219
|
-
const { IndexDBTestStore } = await import("./gateway-H7UD6TNB.js");
|
2220
|
-
return new IndexDBTestStore(sthis);
|
2221
|
-
}
|
2222
|
-
});
|
2223
2398
|
|
2224
2399
|
// src/blockstore/store-remote.ts
|
2225
2400
|
async function RemoteDataStore(sthis, name, url, opts) {
|
2226
|
-
const ds = new DataStoreImpl(sthis,
|
2401
|
+
const ds = new DataStoreImpl(sthis, url, opts);
|
2227
2402
|
await ds.start();
|
2228
2403
|
return ds;
|
2229
2404
|
}
|
2230
2405
|
async function RemoteMetaStore(sthis, name, url, opts) {
|
2231
|
-
const ms = new MetaStoreImpl(
|
2232
|
-
sthis,
|
2233
|
-
name,
|
2234
|
-
url,
|
2235
|
-
opts
|
2236
|
-
/* , true*/
|
2237
|
-
);
|
2406
|
+
const ms = new MetaStoreImpl(sthis, url, opts);
|
2238
2407
|
await ms.start();
|
2239
2408
|
return ms;
|
2240
2409
|
}
|
@@ -2259,14 +2428,17 @@ var ConnectionBase = class {
|
|
2259
2428
|
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
2260
2429
|
this.loader = loader;
|
2261
2430
|
await this.onConnect();
|
2262
|
-
const metaUrl = this.url.build().defParam("store"
|
2263
|
-
const
|
2264
|
-
if (
|
2431
|
+
const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
|
2432
|
+
const rgateway = await getStartedGateway(loader.sthis, metaUrl);
|
2433
|
+
if (rgateway.isErr())
|
2434
|
+
throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
2435
|
+
const name = metaUrl.toString();
|
2265
2436
|
const dbName = metaUrl.getParam("name");
|
2266
|
-
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X:
|
2267
|
-
const
|
2437
|
+
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
|
2438
|
+
const gateway = rgateway.Ok();
|
2439
|
+
const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
|
2268
2440
|
gateway: gateway.gateway,
|
2269
|
-
keybag:
|
2441
|
+
keybag: await loader.keyBag(),
|
2270
2442
|
loader
|
2271
2443
|
});
|
2272
2444
|
this.loader.remoteMetaStore = remote;
|
@@ -2279,14 +2451,15 @@ var ConnectionBase = class {
|
|
2279
2451
|
async connectStorage_X({ loader }) {
|
2280
2452
|
if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
|
2281
2453
|
this.loader = loader;
|
2282
|
-
const dataUrl = this.url.build().defParam("store"
|
2283
|
-
const
|
2284
|
-
if (
|
2454
|
+
const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
|
2455
|
+
const rgateway = await getStartedGateway(loader.sthis, dataUrl);
|
2456
|
+
if (rgateway.isErr())
|
2457
|
+
throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
2285
2458
|
const name = dataUrl.getParam("name");
|
2286
2459
|
if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
|
2287
2460
|
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
2288
|
-
gateway:
|
2289
|
-
keybag:
|
2461
|
+
gateway: rgateway.Ok().gateway,
|
2462
|
+
keybag: await loader.keyBag()
|
2290
2463
|
});
|
2291
2464
|
loader.remoteFileStore = loader.remoteCarStore;
|
2292
2465
|
}
|
@@ -2326,6 +2499,12 @@ var ConnectionBase = class {
|
|
2326
2499
|
};
|
2327
2500
|
|
2328
2501
|
// src/crdt-helpers.ts
|
2502
|
+
import { parse as parse3 } from "multiformats/link";
|
2503
|
+
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
2504
|
+
import * as codec from "@ipld/dag-cbor";
|
2505
|
+
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
2506
|
+
import { EventFetcher, vis } from "@web3-storage/pail/clock";
|
2507
|
+
import * as Batch from "@web3-storage/pail/crdt/batch";
|
2329
2508
|
function time(tag) {
|
2330
2509
|
}
|
2331
2510
|
function timeEnd(tag) {
|
@@ -2373,7 +2552,7 @@ async function writeDocContent(store, blocks, update, logger) {
|
|
2373
2552
|
await processFiles(store, blocks, update.value, logger);
|
2374
2553
|
value = { doc: update.value };
|
2375
2554
|
}
|
2376
|
-
const block = await
|
2555
|
+
const block = await encode3({ value, hasher: hasher5, codec });
|
2377
2556
|
blocks.putSync(block.cid, block.bytes);
|
2378
2557
|
return block.cid;
|
2379
2558
|
}
|
@@ -2464,7 +2643,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
2464
2643
|
async function getValueFromLink(blocks, link, logger) {
|
2465
2644
|
const block = await blocks.get(link);
|
2466
2645
|
if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
|
2467
|
-
const { value } = await
|
2646
|
+
const { value } = await decode3({ bytes: block.bytes, hasher: hasher5, codec });
|
2468
2647
|
const cvalue = {
|
2469
2648
|
...value,
|
2470
2649
|
cid: link
|
@@ -2585,7 +2764,7 @@ async function doCompact(blockLog, head, logger) {
|
|
2585
2764
|
async function getBlock(blocks, cidString) {
|
2586
2765
|
const block = await blocks.get(parse3(cidString));
|
2587
2766
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
2588
|
-
const { cid, value } = await
|
2767
|
+
const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: hasher5 });
|
2589
2768
|
return new Block({ cid, value, bytes: block.bytes });
|
2590
2769
|
}
|
2591
2770
|
|
@@ -2643,7 +2822,8 @@ function makeProllyGetBlock(blocks) {
|
|
2643
2822
|
return create({ cid, bytes, hasher: hasher6, codec: codec2 });
|
2644
2823
|
};
|
2645
2824
|
}
|
2646
|
-
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
2825
|
+
async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
|
2826
|
+
logger.Debug().Msg("enter bulkIndex");
|
2647
2827
|
if (!indexEntries.length) return inIndex;
|
2648
2828
|
if (!inIndex.root) {
|
2649
2829
|
if (!inIndex.cid) {
|
@@ -2660,18 +2840,22 @@ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
|
2660
2840
|
returnNode = node;
|
2661
2841
|
}
|
2662
2842
|
if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
|
2843
|
+
logger.Debug().Msg("exit !root bulkIndex");
|
2663
2844
|
return { root: returnNode, cid: returnRootBlock.cid };
|
2664
2845
|
} else {
|
2665
2846
|
inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
|
2666
2847
|
}
|
2667
2848
|
}
|
2849
|
+
logger.Debug().Msg("pre bulk bulkIndex");
|
2668
2850
|
const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
|
2669
2851
|
if (root3) {
|
2852
|
+
logger.Debug().Msg("pre root put bulkIndex");
|
2670
2853
|
for await (const block of newBlocks) {
|
2671
2854
|
await tblocks.put(block.cid, block.bytes);
|
2672
2855
|
}
|
2673
2856
|
return { root: root3, cid: (await root3.block).cid };
|
2674
2857
|
} else {
|
2858
|
+
logger.Debug().Msg("pre !root bulkIndex");
|
2675
2859
|
return { root: void 0, cid: void 0 };
|
2676
2860
|
}
|
2677
2861
|
}
|
@@ -2711,17 +2895,17 @@ function encodeKey(key) {
|
|
2711
2895
|
}
|
2712
2896
|
|
2713
2897
|
// src/indexer.ts
|
2714
|
-
function index(
|
2715
|
-
if (mapFn && meta) throw
|
2716
|
-
if (mapFn && mapFn.constructor.name !== "Function") throw
|
2717
|
-
if (
|
2718
|
-
const idx =
|
2898
|
+
function index(refDb, name, mapFn, meta) {
|
2899
|
+
if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
2900
|
+
if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
|
2901
|
+
if (refDb.crdt.indexers.has(name)) {
|
2902
|
+
const idx = refDb.crdt.indexers.get(name);
|
2719
2903
|
idx.applyMapFn(name, mapFn, meta);
|
2720
2904
|
} else {
|
2721
|
-
const idx = new Index(sthis,
|
2722
|
-
|
2905
|
+
const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
|
2906
|
+
refDb.crdt.indexers.set(name, idx);
|
2723
2907
|
}
|
2724
|
-
return
|
2908
|
+
return refDb.crdt.indexers.get(name);
|
2725
2909
|
}
|
2726
2910
|
var Index = class {
|
2727
2911
|
constructor(sthis, crdt, name, mapFn, meta) {
|
@@ -2740,18 +2924,9 @@ var Index = class {
|
|
2740
2924
|
return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
|
2741
2925
|
});
|
2742
2926
|
}
|
2743
|
-
close() {
|
2744
|
-
return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
|
2745
|
-
});
|
2746
|
-
}
|
2747
|
-
destroy() {
|
2748
|
-
return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
|
2749
|
-
});
|
2750
|
-
}
|
2751
2927
|
applyMapFn(name, mapFn, meta) {
|
2752
2928
|
if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
2753
2929
|
if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
|
2754
|
-
this.name = name;
|
2755
2930
|
try {
|
2756
2931
|
if (meta) {
|
2757
2932
|
if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
|
@@ -2799,9 +2974,13 @@ var Index = class {
|
|
2799
2974
|
}
|
2800
2975
|
}
|
2801
2976
|
async query(opts = {}) {
|
2977
|
+
this.logger.Debug().Msg("enter query");
|
2802
2978
|
await this.ready();
|
2979
|
+
this.logger.Debug().Msg("post ready query");
|
2803
2980
|
await this._updateIndex();
|
2981
|
+
this.logger.Debug().Msg("post _updateIndex query");
|
2804
2982
|
await this._hydrateIndex();
|
2983
|
+
this.logger.Debug().Msg("post _hydrateIndex query");
|
2805
2984
|
if (!this.byKey.root) {
|
2806
2985
|
return await applyQuery(this.crdt, { result: [] }, opts);
|
2807
2986
|
}
|
@@ -2857,13 +3036,16 @@ var Index = class {
|
|
2857
3036
|
}
|
2858
3037
|
async _updateIndex() {
|
2859
3038
|
await this.ready();
|
3039
|
+
this.logger.Debug().Msg("enter _updateIndex");
|
2860
3040
|
if (this.initError) throw this.initError;
|
2861
3041
|
if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
|
2862
3042
|
let result, head;
|
2863
3043
|
if (!this.indexHead || this.indexHead.length === 0) {
|
2864
3044
|
({ result, head } = await this.crdt.allDocs());
|
3045
|
+
this.logger.Debug().Msg("enter crdt.allDocs");
|
2865
3046
|
} else {
|
2866
3047
|
({ result, head } = await this.crdt.changes(this.indexHead));
|
3048
|
+
this.logger.Debug().Msg("enter crdt.changes");
|
2867
3049
|
}
|
2868
3050
|
if (result.length === 0) {
|
2869
3051
|
this.indexHead = head;
|
@@ -2896,9 +3078,22 @@ var Index = class {
|
|
2896
3078
|
if (result.length === 0) {
|
2897
3079
|
return indexerMeta;
|
2898
3080
|
}
|
3081
|
+
this.logger.Debug().Msg("pre this.blockstore.transaction");
|
2899
3082
|
const { meta } = await this.blockstore.transaction(async (tblocks) => {
|
2900
|
-
this.byId = await bulkIndex(
|
2901
|
-
|
3083
|
+
this.byId = await bulkIndex(
|
3084
|
+
this.logger,
|
3085
|
+
tblocks,
|
3086
|
+
this.byId,
|
3087
|
+
removeIdIndexEntries.concat(byIdIndexEntries),
|
3088
|
+
byIdOpts
|
3089
|
+
);
|
3090
|
+
this.byKey = await bulkIndex(
|
3091
|
+
this.logger,
|
3092
|
+
tblocks,
|
3093
|
+
this.byKey,
|
3094
|
+
staleKeyIndexEntries.concat(indexEntries),
|
3095
|
+
byKeyOpts
|
3096
|
+
);
|
2902
3097
|
this.indexHead = head;
|
2903
3098
|
if (this.byId.cid && this.byKey.cid) {
|
2904
3099
|
const idxMeta = {
|
@@ -2910,8 +3105,10 @@ var Index = class {
|
|
2910
3105
|
};
|
2911
3106
|
indexerMeta.indexes?.set(this.name, idxMeta);
|
2912
3107
|
}
|
3108
|
+
this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
|
2913
3109
|
return indexerMeta;
|
2914
3110
|
});
|
3111
|
+
this.logger.Debug().Msg("post this.blockstore.transaction");
|
2915
3112
|
return meta;
|
2916
3113
|
}
|
2917
3114
|
};
|
@@ -3039,21 +3236,23 @@ var CRDTClock = class {
|
|
3039
3236
|
throw this.logger.Error().Msg("missing blockstore").AsError();
|
3040
3237
|
}
|
3041
3238
|
await validateBlocks(this.logger, newHead, this.blockstore);
|
3042
|
-
|
3043
|
-
|
3044
|
-
|
3045
|
-
|
3046
|
-
|
3047
|
-
|
3048
|
-
|
3049
|
-
|
3050
|
-
|
3051
|
-
|
3052
|
-
|
3053
|
-
|
3054
|
-
|
3055
|
-
|
3056
|
-
|
3239
|
+
if (!this.transaction) {
|
3240
|
+
this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
|
3241
|
+
}
|
3242
|
+
const tblocks = this.transaction;
|
3243
|
+
const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
|
3244
|
+
const result = await root2(tblocks, advancedHead);
|
3245
|
+
for (const { cid, bytes } of [
|
3246
|
+
...result.additions
|
3247
|
+
// ...result.removals
|
3248
|
+
]) {
|
3249
|
+
tblocks.putSync(cid, bytes);
|
3250
|
+
}
|
3251
|
+
if (!noLoader) {
|
3252
|
+
await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
|
3253
|
+
this.transaction = void 0;
|
3254
|
+
}
|
3255
|
+
this.setHead(advancedHead);
|
3057
3256
|
}
|
3058
3257
|
};
|
3059
3258
|
function sortClockHead(clockHead) {
|
@@ -3085,15 +3284,13 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
3085
3284
|
|
3086
3285
|
// src/crdt.ts
|
3087
3286
|
var CRDT = class {
|
3088
|
-
constructor(sthis,
|
3287
|
+
constructor(sthis, opts) {
|
3089
3288
|
this.indexers = /* @__PURE__ */ new Map();
|
3090
3289
|
this.onceReady = new ResolveOnce5();
|
3091
3290
|
this.sthis = sthis;
|
3092
|
-
this.name = name;
|
3093
3291
|
this.logger = ensureLogger(sthis, "CRDT");
|
3094
3292
|
this.opts = opts;
|
3095
3293
|
this.blockstore = blockstoreFactory(sthis, {
|
3096
|
-
name,
|
3097
3294
|
applyMeta: async (meta) => {
|
3098
3295
|
const crdtMeta = meta;
|
3099
3296
|
if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
|
@@ -3103,23 +3300,27 @@ var CRDT = class {
|
|
3103
3300
|
await doCompact(blocks, this.clock.head, this.logger);
|
3104
3301
|
return { head: this.clock.head };
|
3105
3302
|
},
|
3106
|
-
autoCompact: this.opts.autoCompact || 100,
|
3107
|
-
|
3108
|
-
|
3109
|
-
|
3110
|
-
|
3303
|
+
// autoCompact: this.opts.autoCompact || 100,
|
3304
|
+
storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
|
3305
|
+
storeUrls: this.opts.storeUrls.data,
|
3306
|
+
keyBag: this.opts.keyBag,
|
3307
|
+
// public: this.opts.public,
|
3308
|
+
meta: this.opts.meta
|
3309
|
+
// threshold: this.opts.threshold,
|
3111
3310
|
});
|
3112
3311
|
this.indexBlockstore = blockstoreFactory(sthis, {
|
3113
|
-
name,
|
3312
|
+
// name: opts.name,
|
3114
3313
|
applyMeta: async (meta) => {
|
3115
3314
|
const idxCarMeta = meta;
|
3116
3315
|
if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
|
3117
|
-
for (const [
|
3118
|
-
index(
|
3316
|
+
for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
|
3317
|
+
index({ crdt: this }, name, void 0, idx);
|
3119
3318
|
}
|
3120
3319
|
},
|
3121
|
-
|
3122
|
-
|
3320
|
+
storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
|
3321
|
+
storeUrls: this.opts.storeUrls.idx,
|
3322
|
+
keyBag: this.opts.keyBag
|
3323
|
+
// public: this.opts.public,
|
3123
3324
|
});
|
3124
3325
|
this.clock = new CRDTClock(this.blockstore);
|
3125
3326
|
this.clock.onZoom(() => {
|
@@ -3201,51 +3402,164 @@ var CRDT = class {
|
|
3201
3402
|
};
|
3202
3403
|
|
3203
3404
|
// src/database.ts
|
3204
|
-
var
|
3205
|
-
|
3206
|
-
|
3405
|
+
var databases = new KeyedResolvOnce3();
|
3406
|
+
function toSortedArray(set) {
|
3407
|
+
if (!set) return [];
|
3408
|
+
return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
|
3409
|
+
}
|
3410
|
+
function keyConfigOpts(sthis, name, opts) {
|
3411
|
+
return JSON.stringify(
|
3412
|
+
toSortedArray({
|
3413
|
+
name,
|
3414
|
+
stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
|
3415
|
+
})
|
3416
|
+
);
|
3417
|
+
}
|
3418
|
+
function isDatabase(db) {
|
3419
|
+
return db instanceof DatabaseImpl || db instanceof DatabaseShell;
|
3420
|
+
}
|
3421
|
+
function DatabaseFactory(name, opts) {
|
3422
|
+
const sthis = ensureSuperThis(opts);
|
3423
|
+
return new DatabaseShell(
|
3424
|
+
databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
|
3425
|
+
const db = new DatabaseImpl(sthis, {
|
3426
|
+
name,
|
3427
|
+
meta: opts?.meta,
|
3428
|
+
keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
|
3429
|
+
storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
|
3430
|
+
storeEnDe: {
|
3431
|
+
encodeFile,
|
3432
|
+
decodeFile,
|
3433
|
+
...opts?.storeEnDe
|
3434
|
+
}
|
3435
|
+
});
|
3436
|
+
db.onClosed(() => {
|
3437
|
+
databases.unget(key);
|
3438
|
+
});
|
3439
|
+
return db;
|
3440
|
+
})
|
3441
|
+
);
|
3442
|
+
}
|
3443
|
+
var DatabaseShell = class {
|
3444
|
+
constructor(ref) {
|
3445
|
+
this.ref = ref;
|
3446
|
+
ref.addShell(this);
|
3447
|
+
}
|
3448
|
+
get id() {
|
3449
|
+
return this.ref.id;
|
3450
|
+
}
|
3451
|
+
get logger() {
|
3452
|
+
return this.ref.logger;
|
3453
|
+
}
|
3454
|
+
get sthis() {
|
3455
|
+
return this.ref.sthis;
|
3456
|
+
}
|
3457
|
+
get crdt() {
|
3458
|
+
return this.ref.crdt;
|
3459
|
+
}
|
3460
|
+
name() {
|
3461
|
+
return this.ref.name();
|
3462
|
+
}
|
3463
|
+
onClosed(fn) {
|
3464
|
+
return this.ref.onClosed(fn);
|
3465
|
+
}
|
3466
|
+
close() {
|
3467
|
+
return this.ref.shellClose(this);
|
3468
|
+
}
|
3469
|
+
destroy() {
|
3470
|
+
return this.ref.destroy();
|
3471
|
+
}
|
3472
|
+
ready() {
|
3473
|
+
return this.ref.ready();
|
3474
|
+
}
|
3475
|
+
get(id) {
|
3476
|
+
return this.ref.get(id);
|
3477
|
+
}
|
3478
|
+
put(doc) {
|
3479
|
+
return this.ref.put(doc);
|
3480
|
+
}
|
3481
|
+
del(id) {
|
3482
|
+
return this.ref.del(id);
|
3483
|
+
}
|
3484
|
+
changes(since, opts) {
|
3485
|
+
return this.ref.changes(since, opts);
|
3486
|
+
}
|
3487
|
+
allDocs(opts) {
|
3488
|
+
return this.ref.allDocs(opts);
|
3489
|
+
}
|
3490
|
+
allDocuments() {
|
3491
|
+
return this.ref.allDocuments();
|
3492
|
+
}
|
3493
|
+
subscribe(listener, updates) {
|
3494
|
+
return this.ref.subscribe(listener, updates);
|
3495
|
+
}
|
3496
|
+
query(field, opts) {
|
3497
|
+
return this.ref.query(field, opts);
|
3498
|
+
}
|
3499
|
+
compact() {
|
3500
|
+
return this.ref.compact();
|
3501
|
+
}
|
3502
|
+
};
|
3503
|
+
var DatabaseImpl = class {
|
3504
|
+
constructor(sthis, opts) {
|
3207
3505
|
this._listening = false;
|
3208
3506
|
this._listeners = /* @__PURE__ */ new Set();
|
3209
3507
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
3508
|
+
// readonly blockstore: BaseBlockstore;
|
3509
|
+
this.shells = /* @__PURE__ */ new Set();
|
3510
|
+
this._onClosedFns = /* @__PURE__ */ new Set();
|
3210
3511
|
this._ready = new ResolveOnce6();
|
3211
|
-
this.
|
3212
|
-
this.
|
3213
|
-
this.
|
3512
|
+
this.opts = opts;
|
3513
|
+
this.sthis = sthis;
|
3514
|
+
this.id = sthis.timeOrderedNextId().str;
|
3214
3515
|
this.logger = ensureLogger(this.sthis, "Database");
|
3215
|
-
this.
|
3216
|
-
this.blockstore = this._crdt.blockstore;
|
3516
|
+
this.crdt = new CRDT(this.sthis, this.opts);
|
3217
3517
|
this._writeQueue = writeQueue(async (updates) => {
|
3218
|
-
return await this.
|
3518
|
+
return await this.crdt.bulk(updates);
|
3219
3519
|
});
|
3220
|
-
this.
|
3520
|
+
this.crdt.clock.onTock(() => {
|
3221
3521
|
this._no_update_notify();
|
3222
3522
|
});
|
3223
3523
|
}
|
3224
|
-
|
3225
|
-
this.
|
3524
|
+
addShell(shell) {
|
3525
|
+
this.shells.add(shell);
|
3526
|
+
}
|
3527
|
+
onClosed(fn) {
|
3528
|
+
this._onClosedFns.add(fn);
|
3226
3529
|
}
|
3227
3530
|
async close() {
|
3228
|
-
|
3229
|
-
|
3230
|
-
|
3531
|
+
throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
|
3532
|
+
}
|
3533
|
+
async shellClose(db) {
|
3534
|
+
if (!this.shells.has(db)) {
|
3535
|
+
throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
|
3536
|
+
}
|
3537
|
+
this.shells.delete(db);
|
3538
|
+
if (this.shells.size === 0) {
|
3539
|
+
await this.ready();
|
3540
|
+
await this.crdt.close();
|
3541
|
+
this._onClosedFns.forEach((fn) => fn());
|
3542
|
+
}
|
3231
3543
|
}
|
3232
3544
|
async destroy() {
|
3233
3545
|
await this.ready();
|
3234
|
-
await this.
|
3235
|
-
await this.blockstore.destroy();
|
3546
|
+
await this.crdt.destroy();
|
3236
3547
|
}
|
3237
3548
|
async ready() {
|
3238
|
-
|
3549
|
+
const ret = await this._ready.once(async () => {
|
3239
3550
|
await this.sthis.start();
|
3240
|
-
await this.
|
3241
|
-
await this.blockstore.ready();
|
3551
|
+
await this.crdt.ready();
|
3242
3552
|
});
|
3553
|
+
return ret;
|
3554
|
+
}
|
3555
|
+
name() {
|
3556
|
+
return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
|
3243
3557
|
}
|
3244
3558
|
async get(id) {
|
3245
|
-
if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
|
3559
|
+
if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
|
3246
3560
|
await this.ready();
|
3247
3561
|
this.logger.Debug().Str("id", id).Msg("get");
|
3248
|
-
const got = await this.
|
3562
|
+
const got = await this.crdt.get(id).catch((e) => {
|
3249
3563
|
throw new NotFoundError(`Not found: ${id} - ${e.message}`);
|
3250
3564
|
});
|
3251
3565
|
if (!got) throw new NotFoundError(`Not found: ${id}`);
|
@@ -3264,34 +3578,34 @@ var Database = class {
|
|
3264
3578
|
_id: docId
|
3265
3579
|
}
|
3266
3580
|
});
|
3267
|
-
return { id: docId, clock: result?.head, name: this.name };
|
3581
|
+
return { id: docId, clock: result?.head, name: this.name() };
|
3268
3582
|
}
|
3269
3583
|
async del(id) {
|
3270
3584
|
await this.ready();
|
3271
3585
|
this.logger.Debug().Str("id", id).Msg("del");
|
3272
3586
|
const result = await this._writeQueue.push({ id, del: true });
|
3273
|
-
return { id, clock: result?.head, name: this.name };
|
3587
|
+
return { id, clock: result?.head, name: this.name() };
|
3274
3588
|
}
|
3275
3589
|
async changes(since = [], opts = {}) {
|
3276
3590
|
await this.ready();
|
3277
3591
|
this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
|
3278
|
-
const { result, head } = await this.
|
3592
|
+
const { result, head } = await this.crdt.changes(since, opts);
|
3279
3593
|
const rows = result.map(({ id: key, value, del, clock }) => ({
|
3280
3594
|
key,
|
3281
3595
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
|
3282
3596
|
clock
|
3283
3597
|
}));
|
3284
|
-
return { rows, clock: head, name: this.name };
|
3598
|
+
return { rows, clock: head, name: this.name() };
|
3285
3599
|
}
|
3286
3600
|
async allDocs(opts = {}) {
|
3287
3601
|
await this.ready();
|
3288
3602
|
this.logger.Debug().Msg("allDocs");
|
3289
|
-
const { result, head } = await this.
|
3603
|
+
const { result, head } = await this.crdt.allDocs();
|
3290
3604
|
const rows = result.map(({ id: key, value, del }) => ({
|
3291
3605
|
key,
|
3292
3606
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
|
3293
3607
|
}));
|
3294
|
-
return { rows, clock: head, name: this.name };
|
3608
|
+
return { rows, clock: head, name: this.name() };
|
3295
3609
|
}
|
3296
3610
|
async allDocuments() {
|
3297
3611
|
return this.allDocs();
|
@@ -3301,7 +3615,7 @@ var Database = class {
|
|
3301
3615
|
if (updates) {
|
3302
3616
|
if (!this._listening) {
|
3303
3617
|
this._listening = true;
|
3304
|
-
this.
|
3618
|
+
this.crdt.clock.onTick((updates2) => {
|
3305
3619
|
void this._notify(updates2);
|
3306
3620
|
});
|
3307
3621
|
}
|
@@ -3320,13 +3634,13 @@ var Database = class {
|
|
3320
3634
|
async query(field, opts = {}) {
|
3321
3635
|
await this.ready();
|
3322
3636
|
this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
|
3323
|
-
const _crdt = this.
|
3324
|
-
const idx = typeof field === "string" ? index(
|
3637
|
+
const _crdt = this.crdt;
|
3638
|
+
const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
|
3325
3639
|
return await idx.query(opts);
|
3326
3640
|
}
|
3327
3641
|
async compact() {
|
3328
3642
|
await this.ready();
|
3329
|
-
await this.
|
3643
|
+
await this.crdt.compact();
|
3330
3644
|
}
|
3331
3645
|
async _notify(updates) {
|
3332
3646
|
await this.ready();
|
@@ -3350,23 +3664,62 @@ var Database = class {
|
|
3350
3664
|
}
|
3351
3665
|
}
|
3352
3666
|
};
|
3353
|
-
function
|
3354
|
-
|
3355
|
-
|
3667
|
+
function defaultURI(sthis, curi, uri, store, ctx) {
|
3668
|
+
ctx = ctx || {};
|
3669
|
+
const ret = (curi ? URI8.from(curi) : uri).build().setParam("store" /* STORE */, store);
|
3670
|
+
if (!ret.hasParam("name" /* NAME */)) {
|
3671
|
+
const name = sthis.pathOps.basename(ret.URI().pathname);
|
3672
|
+
if (!name) {
|
3673
|
+
throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
|
3674
|
+
}
|
3675
|
+
ret.setParam("name" /* NAME */, name);
|
3676
|
+
}
|
3677
|
+
if (ctx.idx) {
|
3678
|
+
ret.defParam("index" /* INDEX */, "idx");
|
3679
|
+
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
|
3680
|
+
} else {
|
3681
|
+
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
|
3682
|
+
}
|
3683
|
+
if (store === "data") {
|
3684
|
+
if (ctx.file) {
|
3685
|
+
} else {
|
3686
|
+
ret.defParam("suffix" /* SUFFIX */, ".car");
|
3687
|
+
}
|
3688
|
+
}
|
3689
|
+
return ret.URI();
|
3356
3690
|
}
|
3357
|
-
function
|
3358
|
-
|
3359
|
-
|
3360
|
-
|
3361
|
-
|
3362
|
-
|
3363
|
-
|
3364
|
-
|
3365
|
-
|
3366
|
-
db = new Database(name, opts);
|
3367
|
-
Database.databases.set(key, db);
|
3691
|
+
function toStoreURIRuntime(sthis, name, sopts) {
|
3692
|
+
sopts = sopts || {};
|
3693
|
+
if (!sopts.base) {
|
3694
|
+
const fp_env = sthis.env.get("FP_STORAGE_URL");
|
3695
|
+
if (fp_env) {
|
3696
|
+
sopts = { ...sopts, base: BuildURI2.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
|
3697
|
+
} else {
|
3698
|
+
sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
|
3699
|
+
}
|
3368
3700
|
}
|
3369
|
-
|
3701
|
+
const bbase = BuildURI2.from(sopts.base);
|
3702
|
+
if (name) {
|
3703
|
+
bbase.setParam("name" /* NAME */, name);
|
3704
|
+
}
|
3705
|
+
const base = bbase.URI();
|
3706
|
+
return {
|
3707
|
+
idx: {
|
3708
|
+
data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
|
3709
|
+
file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
|
3710
|
+
meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
|
3711
|
+
wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
|
3712
|
+
},
|
3713
|
+
data: {
|
3714
|
+
data: defaultURI(sthis, sopts.data?.data, base, "data"),
|
3715
|
+
file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
|
3716
|
+
meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
|
3717
|
+
wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
|
3718
|
+
}
|
3719
|
+
};
|
3720
|
+
}
|
3721
|
+
function fireproof(name, opts) {
|
3722
|
+
return DatabaseFactory(name, opts);
|
3370
3723
|
}
|
3371
3724
|
function makeName(fnString) {
|
3372
3725
|
const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
|
@@ -3397,7 +3750,7 @@ __export(runtime_exports, {
|
|
3397
3750
|
kb: () => key_bag_exports,
|
3398
3751
|
kc: () => keyed_crypto_exports,
|
3399
3752
|
mf: () => wait_pr_multiformats_exports,
|
3400
|
-
runtimeFn: () =>
|
3753
|
+
runtimeFn: () => runtimeFn3,
|
3401
3754
|
toArrayBuffer: () => toArrayBuffer
|
3402
3755
|
});
|
3403
3756
|
|
@@ -3412,23 +3765,24 @@ __export(wait_pr_multiformats_exports, {
|
|
3412
3765
|
var codec_interface_exports = {};
|
3413
3766
|
|
3414
3767
|
// src/runtime/index.ts
|
3415
|
-
import { runtimeFn as
|
3768
|
+
import { runtimeFn as runtimeFn3 } from "@adviser/cement";
|
3416
3769
|
|
3417
3770
|
// src/version.ts
|
3418
3771
|
var PACKAGE_VERSION = Object.keys({
|
3419
|
-
"0.19.
|
3772
|
+
"0.19.101": "xxxx"
|
3420
3773
|
})[0];
|
3421
3774
|
export {
|
3422
3775
|
CRDT,
|
3423
|
-
|
3776
|
+
DatabaseFactory,
|
3777
|
+
DatabaseShell,
|
3424
3778
|
Index,
|
3425
3779
|
NotFoundError,
|
3426
3780
|
PACKAGE_VERSION,
|
3781
|
+
PARAM,
|
3427
3782
|
Result,
|
3428
3783
|
UInt8ArrayEqual,
|
3429
3784
|
blockstore_exports as blockstore,
|
3430
3785
|
blockstore_exports as bs,
|
3431
|
-
dataDir,
|
3432
3786
|
ensureLogger,
|
3433
3787
|
ensureSuperLog,
|
3434
3788
|
ensureSuperThis,
|
@@ -3439,10 +3793,13 @@ export {
|
|
3439
3793
|
getName,
|
3440
3794
|
getStore,
|
3441
3795
|
index,
|
3796
|
+
isDatabase,
|
3442
3797
|
isFalsy,
|
3443
3798
|
isNotFoundError,
|
3799
|
+
keyConfigOpts,
|
3444
3800
|
runtime_exports as rt,
|
3445
3801
|
runtime_exports as runtime,
|
3446
|
-
throwFalsy
|
3802
|
+
throwFalsy,
|
3803
|
+
toStoreURIRuntime
|
3447
3804
|
};
|
3448
3805
|
//# sourceMappingURL=index.js.map
|