@fireproof/core 0.19.100 → 0.19.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{chunk-OFGPKRCM.js → chunk-3EB3ENHT.js} +54 -25
- package/chunk-3EB3ENHT.js.map +1 -0
- package/chunk-HQ7D3PEU.js +61 -0
- package/chunk-HQ7D3PEU.js.map +1 -0
- package/chunk-PZ5AY32C.js +10 -0
- package/deno-filesystem-Q2IJ7YDR.js +57 -0
- package/deno-filesystem-Q2IJ7YDR.js.map +1 -0
- package/{gateway-5FCWPX5W.js → gateway-GK5QZ6KP.js} +13 -12
- package/gateway-GK5QZ6KP.js.map +1 -0
- package/{gateway-H7UD6TNB.js → gateway-TQTGDRCN.js} +9 -8
- package/gateway-TQTGDRCN.js.map +1 -0
- package/index.cjs +2158 -1718
- package/index.cjs.map +1 -1
- package/index.d.cts +261 -117
- package/index.d.ts +261 -117
- package/index.global.js +11834 -11354
- package/index.global.js.map +1 -1
- package/index.js +1865 -1519
- package/index.js.map +1 -1
- package/{key-bag-file-WADZBHYG.js → key-bag-file-VOSSK46F.js} +4 -3
- package/{key-bag-file-WADZBHYG.js.map → key-bag-file-VOSSK46F.js.map} +1 -1
- package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-AXTQOSMC.js} +4 -3
- package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-AXTQOSMC.js.map} +1 -1
- package/key-bag-memory-LWE6ARPX.js +29 -0
- package/key-bag-memory-LWE6ARPX.js.map +1 -0
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/{node-filesystem-INX4ZTHE.js → node-filesystem-CFRXFSO7.js} +6 -9
- package/node-filesystem-CFRXFSO7.js.map +1 -0
- package/package.json +1 -1
- package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +129 -0
- package/tests/blockstore/keyed-crypto.test.ts +63 -227
- package/tests/blockstore/loader.test.ts +19 -11
- package/tests/blockstore/store.test.ts +23 -19
- package/tests/blockstore/transaction.test.ts +12 -12
- package/tests/fireproof/all-gateway.test.ts +201 -193
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +324 -316
- package/tests/fireproof/crdt.test.ts +67 -16
- package/tests/fireproof/database.test.ts +183 -21
- package/tests/fireproof/fireproof.test.ts +83 -74
- package/tests/fireproof/hello.test.ts +18 -14
- package/tests/fireproof/indexer.test.ts +53 -43
- package/tests/fireproof/utils.test.ts +18 -6
- package/tests/gateway/file/loader-config.test.ts +303 -0
- package/tests/gateway/indexdb/loader-config.test.ts +75 -0
- package/tests/helpers.ts +27 -9
- package/tests/react/useFireproof.test.tsx +1 -1
- package/{utils-QO2HIWGI.js → utils-STA2C35G.js} +4 -3
- package/utils-STA2C35G.js.map +1 -0
- package/chunk-OFGPKRCM.js.map +0 -1
- package/chunk-WS3YRPIA.js +0 -75
- package/chunk-WS3YRPIA.js.map +0 -1
- package/gateway-5FCWPX5W.js.map +0 -1
- package/gateway-H7UD6TNB.js.map +0 -1
- package/mem-filesystem-YPPJV7Q2.js +0 -41
- package/mem-filesystem-YPPJV7Q2.js.map +0 -1
- package/node-filesystem-INX4ZTHE.js.map +0 -1
- package/tests/fireproof/config.test.ts +0 -172
- /package/{utils-QO2HIWGI.js.map → chunk-PZ5AY32C.js.map} +0 -0
- /package/tests/fireproof/{fireproof.test.fixture.ts → fireproof.fixture.ts} +0 -0
package/index.js
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
import {
|
2
|
+
INDEXDB_VERSION
|
3
|
+
} from "./chunk-PB4BKL4O.js";
|
1
4
|
import {
|
2
5
|
FILESTORE_VERSION
|
3
6
|
} from "./chunk-7EWIAXTM.js";
|
@@ -6,28 +9,30 @@ import {
|
|
6
9
|
getFileSystem,
|
7
10
|
getPath,
|
8
11
|
toArrayBuffer
|
9
|
-
} from "./chunk-
|
10
|
-
import {
|
11
|
-
INDEXDB_VERSION
|
12
|
-
} from "./chunk-PB4BKL4O.js";
|
12
|
+
} from "./chunk-HQ7D3PEU.js";
|
13
13
|
import {
|
14
14
|
NotFoundError,
|
15
|
+
PARAM,
|
15
16
|
Result,
|
16
17
|
UInt8ArrayEqual,
|
17
|
-
__export,
|
18
|
-
dataDir,
|
19
18
|
ensureLogger,
|
20
19
|
ensureSuperLog,
|
21
20
|
ensureSuperThis,
|
22
21
|
exceptionWrapper,
|
22
|
+
falsyToUndef,
|
23
23
|
getKey,
|
24
24
|
getName,
|
25
25
|
getStore,
|
26
|
-
|
27
|
-
|
26
|
+
isFalsy,
|
27
|
+
isNotFoundError,
|
28
|
+
throwFalsy
|
29
|
+
} from "./chunk-3EB3ENHT.js";
|
30
|
+
import {
|
31
|
+
__export
|
32
|
+
} from "./chunk-PZ5AY32C.js";
|
28
33
|
|
29
34
|
// src/database.ts
|
30
|
-
import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
|
35
|
+
import { BuildURI as BuildURI2, KeyedResolvOnce as KeyedResolvOnce3, ResolveOnce as ResolveOnce6, URI as URI8 } from "@adviser/cement";
|
31
36
|
|
32
37
|
// src/write-queue.ts
|
33
38
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -72,85 +77,6 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
72
77
|
// src/crdt.ts
|
73
78
|
import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
|
74
79
|
|
75
|
-
// src/runtime/wait-pr-multiformats/block.ts
|
76
|
-
var block_exports = {};
|
77
|
-
__export(block_exports, {
|
78
|
-
Block: () => Block,
|
79
|
-
create: () => create,
|
80
|
-
createUnsafe: () => createUnsafe,
|
81
|
-
decode: () => decode,
|
82
|
-
encode: () => encode
|
83
|
-
});
|
84
|
-
import { bytes as binary, CID } from "multiformats";
|
85
|
-
import { Block as mfBlock } from "multiformats/block";
|
86
|
-
var Block = mfBlock;
|
87
|
-
async function decode({
|
88
|
-
bytes,
|
89
|
-
codec: codec3,
|
90
|
-
hasher: hasher7
|
91
|
-
}) {
|
92
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
93
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
94
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
95
|
-
const hash = await hasher7.digest(bytes);
|
96
|
-
const cid = CID.create(1, codec3.code, hash);
|
97
|
-
return new mfBlock({ value, bytes, cid });
|
98
|
-
}
|
99
|
-
async function encode({
|
100
|
-
value,
|
101
|
-
codec: codec3,
|
102
|
-
hasher: hasher7
|
103
|
-
}) {
|
104
|
-
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
105
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
106
|
-
const bytes = await Promise.resolve(codec3.encode(value));
|
107
|
-
const hash = await hasher7.digest(bytes);
|
108
|
-
const cid = CID.create(1, codec3.code, hash);
|
109
|
-
return new mfBlock({ value, bytes, cid });
|
110
|
-
}
|
111
|
-
async function create({
|
112
|
-
bytes,
|
113
|
-
cid,
|
114
|
-
hasher: hasher7,
|
115
|
-
codec: codec3
|
116
|
-
}) {
|
117
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
118
|
-
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
119
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
120
|
-
const hash = await hasher7.digest(bytes);
|
121
|
-
if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
|
122
|
-
throw new Error("CID hash does not match bytes");
|
123
|
-
}
|
124
|
-
return createUnsafe({
|
125
|
-
bytes,
|
126
|
-
cid,
|
127
|
-
value,
|
128
|
-
codec: codec3
|
129
|
-
});
|
130
|
-
}
|
131
|
-
async function createUnsafe({
|
132
|
-
bytes,
|
133
|
-
cid,
|
134
|
-
value: maybeValue,
|
135
|
-
codec: codec3
|
136
|
-
}) {
|
137
|
-
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
138
|
-
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
139
|
-
return new Block({
|
140
|
-
cid,
|
141
|
-
bytes,
|
142
|
-
value
|
143
|
-
});
|
144
|
-
}
|
145
|
-
|
146
|
-
// src/crdt-helpers.ts
|
147
|
-
import { parse as parse3 } from "multiformats/link";
|
148
|
-
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
149
|
-
import * as codec from "@ipld/dag-cbor";
|
150
|
-
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
151
|
-
import { EventFetcher, vis } from "@web3-storage/pail/clock";
|
152
|
-
import * as Batch from "@web3-storage/pail/crdt/batch";
|
153
|
-
|
154
80
|
// src/blockstore/index.ts
|
155
81
|
var blockstore_exports = {};
|
156
82
|
__export(blockstore_exports, {
|
@@ -162,8 +88,11 @@ __export(blockstore_exports, {
|
|
162
88
|
FragmentGateway: () => FragmentGateway,
|
163
89
|
Loader: () => Loader,
|
164
90
|
addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
|
165
|
-
|
166
|
-
|
91
|
+
ensureStoreEnDeFile: () => ensureStoreEnDeFile,
|
92
|
+
fileGatewayFactoryItem: () => fileGatewayFactoryItem,
|
93
|
+
getDefaultURI: () => getDefaultURI,
|
94
|
+
getGatewayFactoryItem: () => getGatewayFactoryItem,
|
95
|
+
getStartedGateway: () => getStartedGateway,
|
167
96
|
parseCarFile: () => parseCarFile,
|
168
97
|
registerStoreProtocol: () => registerStoreProtocol,
|
169
98
|
setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
|
@@ -178,7 +107,7 @@ function toCIDBlock(block) {
|
|
178
107
|
}
|
179
108
|
|
180
109
|
// src/blockstore/store-factory.ts
|
181
|
-
import { KeyedResolvOnce as KeyedResolvOnce2,
|
110
|
+
import { KeyedResolvOnce as KeyedResolvOnce2, Result as Result7 } from "@adviser/cement";
|
182
111
|
|
183
112
|
// src/runtime/files.ts
|
184
113
|
var files_exports = {};
|
@@ -253,327 +182,436 @@ var UnixFSFileBuilder = class {
|
|
253
182
|
|
254
183
|
// src/blockstore/store.ts
|
255
184
|
import { format as format2, parse as parse2 } from "@ipld/dag-json";
|
256
|
-
import {
|
257
|
-
|
258
|
-
// src/types.ts
|
259
|
-
function isFalsy(value) {
|
260
|
-
return value === false && value === null && value === void 0;
|
261
|
-
}
|
262
|
-
function throwFalsy(value) {
|
263
|
-
if (isFalsy(value)) {
|
264
|
-
throw new Error("value is Falsy");
|
265
|
-
}
|
266
|
-
return value;
|
267
|
-
}
|
268
|
-
function falsyToUndef(value) {
|
269
|
-
if (isFalsy(value)) {
|
270
|
-
return void 0;
|
271
|
-
}
|
272
|
-
return value;
|
273
|
-
}
|
274
|
-
|
275
|
-
// src/blockstore/loader.ts
|
276
|
-
import pLimit from "p-limit";
|
277
|
-
import { CarReader } from "@ipld/car";
|
278
|
-
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
185
|
+
import { ResolveOnce as ResolveOnce3, Result as Result5, exception2Result } from "@adviser/cement";
|
279
186
|
|
280
|
-
// src/blockstore/
|
281
|
-
import {
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
|
288
|
-
const fpvalue = dec.value;
|
289
|
-
if (fpvalue && !fpvalue.fp) {
|
290
|
-
throw logger.Error().Msg("missing fp").AsError();
|
187
|
+
// src/blockstore/commit-queue.ts
|
188
|
+
import { Future } from "@adviser/cement";
|
189
|
+
var CommitQueue = class {
|
190
|
+
constructor() {
|
191
|
+
this.queue = [];
|
192
|
+
this.processing = false;
|
193
|
+
this._waitIdleItems = /* @__PURE__ */ new Set();
|
291
194
|
}
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
// src/blockstore/transaction.ts
|
296
|
-
import { MemoryBlockstore } from "@web3-storage/pail/block";
|
297
|
-
import { toCryptoRuntime } from "@adviser/cement";
|
298
|
-
var CarTransaction = class extends MemoryBlockstore {
|
299
|
-
constructor(parent, opts = { add: true, noLoader: false }) {
|
300
|
-
super();
|
301
|
-
if (opts.add) {
|
302
|
-
parent.transactions.add(this);
|
195
|
+
waitIdle() {
|
196
|
+
if (this.queue.length === 0 && !this.processing) {
|
197
|
+
return Promise.resolve();
|
303
198
|
}
|
304
|
-
|
199
|
+
const fn = new Future();
|
200
|
+
this._waitIdleItems.add(fn);
|
201
|
+
return fn.asPromise();
|
305
202
|
}
|
306
|
-
async
|
307
|
-
return
|
203
|
+
async enqueue(fn) {
|
204
|
+
return new Promise((resolve, reject) => {
|
205
|
+
const queueFn = async () => {
|
206
|
+
try {
|
207
|
+
resolve(await fn());
|
208
|
+
} catch (e) {
|
209
|
+
reject(e);
|
210
|
+
} finally {
|
211
|
+
this.processing = false;
|
212
|
+
this.processNext();
|
213
|
+
}
|
214
|
+
};
|
215
|
+
this.queue.push(queueFn);
|
216
|
+
if (!this.processing) {
|
217
|
+
this.processNext();
|
218
|
+
}
|
219
|
+
});
|
308
220
|
}
|
309
|
-
|
310
|
-
|
221
|
+
processNext() {
|
222
|
+
if (this.queue.length > 0 && !this.processing) {
|
223
|
+
this.processing = true;
|
224
|
+
const queueFn = this.queue.shift();
|
225
|
+
if (queueFn) {
|
226
|
+
queueFn().finally(() => {
|
227
|
+
});
|
228
|
+
}
|
229
|
+
}
|
230
|
+
if (this.queue.length === 0 && !this.processing) {
|
231
|
+
const toResolve = Array.from(this._waitIdleItems);
|
232
|
+
this._waitIdleItems.clear();
|
233
|
+
toResolve.map((fn) => fn.resolve());
|
234
|
+
}
|
311
235
|
}
|
312
236
|
};
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
237
|
+
|
238
|
+
// src/runtime/keyed-crypto.ts
|
239
|
+
var keyed_crypto_exports = {};
|
240
|
+
__export(keyed_crypto_exports, {
|
241
|
+
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
242
|
+
keyedCryptoFactory: () => keyedCryptoFactory
|
243
|
+
});
|
244
|
+
import { base58btc } from "multiformats/bases/base58";
|
245
|
+
import { sha256 as hasher } from "multiformats/hashes/sha2";
|
246
|
+
import * as CBOR from "cborg";
|
247
|
+
var generateIV = {
|
248
|
+
random: {
|
317
249
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
318
|
-
|
319
|
-
return
|
250
|
+
calc: async (ko, crypto, data) => {
|
251
|
+
return crypto.randomBytes(ko.ivLength);
|
320
252
|
},
|
321
253
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
322
|
-
|
323
|
-
return
|
254
|
+
verify: async (ko, crypto, iv, data) => {
|
255
|
+
return true;
|
256
|
+
}
|
257
|
+
},
|
258
|
+
hash: {
|
259
|
+
calc: async (ko, crypto, data) => {
|
260
|
+
const hash = await hasher.digest(data);
|
261
|
+
const hashBytes = new Uint8Array(hash.bytes);
|
262
|
+
const hashArray = new Uint8Array(ko.ivLength);
|
263
|
+
for (let i = 0; i < hashBytes.length; i++) {
|
264
|
+
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
265
|
+
}
|
266
|
+
return hashArray;
|
324
267
|
},
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
threshold: 1e3 * 1e3,
|
329
|
-
...opts,
|
330
|
-
logger,
|
331
|
-
keyBag: opts.keyBag || {},
|
332
|
-
crypto: toCryptoRuntime(opts.crypto),
|
333
|
-
store,
|
334
|
-
storeRuntime: toStoreRuntime(store, sthis)
|
335
|
-
};
|
336
|
-
}
|
337
|
-
function blockstoreFactory(sthis, opts) {
|
338
|
-
if (opts.name) {
|
339
|
-
return new EncryptedBlockstore(sthis, opts);
|
340
|
-
} else {
|
341
|
-
return new BaseBlockstore(opts);
|
268
|
+
verify: async function(ko, crypto, iv, data) {
|
269
|
+
return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
270
|
+
}
|
342
271
|
}
|
272
|
+
};
|
273
|
+
function getGenerateIVFn(url, opts) {
|
274
|
+
const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
|
275
|
+
return generateIV[ivhash] || generateIV["hash"];
|
343
276
|
}
|
344
|
-
var
|
345
|
-
constructor(
|
346
|
-
this.
|
347
|
-
this.
|
348
|
-
this.
|
349
|
-
this.
|
350
|
-
|
351
|
-
// ready: Promise<void>;
|
352
|
-
ready() {
|
353
|
-
return Promise.resolve();
|
354
|
-
}
|
355
|
-
async close() {
|
277
|
+
var BlockIvKeyIdCodec = class {
|
278
|
+
constructor(ko, iv, opts) {
|
279
|
+
this.code = 3147065;
|
280
|
+
this.name = "Fireproof@encrypted-block:aes-gcm";
|
281
|
+
this.ko = ko;
|
282
|
+
this.iv = iv;
|
283
|
+
this.opts = opts || {};
|
356
284
|
}
|
357
|
-
async
|
285
|
+
async encode(data) {
|
286
|
+
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
287
|
+
const { iv } = this.ko.algo(calcIv);
|
288
|
+
const fprt = await this.ko.fingerPrint();
|
289
|
+
const keyId = base58btc.decode(fprt);
|
290
|
+
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
291
|
+
return CBOR.encode({
|
292
|
+
iv,
|
293
|
+
keyId,
|
294
|
+
data: await this.ko._encrypt({ iv, bytes: data })
|
295
|
+
});
|
358
296
|
}
|
359
|
-
async
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
297
|
+
async decode(abytes) {
|
298
|
+
let bytes;
|
299
|
+
if (abytes instanceof Uint8Array) {
|
300
|
+
bytes = abytes;
|
301
|
+
} else {
|
302
|
+
bytes = new Uint8Array(abytes);
|
364
303
|
}
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
// TransactionMeta
|
371
|
-
async transaction(fn, _opts) {
|
372
|
-
const t = new CarTransaction(this, _opts);
|
373
|
-
const done = await fn(t);
|
374
|
-
this.lastTxMeta = done;
|
375
|
-
return { t, meta: done };
|
376
|
-
}
|
377
|
-
async *entries() {
|
378
|
-
const seen = /* @__PURE__ */ new Set();
|
379
|
-
for (const t of this.transactions) {
|
380
|
-
for await (const blk of t.entries()) {
|
381
|
-
if (seen.has(blk.cid.toString())) continue;
|
382
|
-
seen.add(blk.cid.toString());
|
383
|
-
yield blk;
|
384
|
-
}
|
304
|
+
const { iv, keyId, data } = CBOR.decode(bytes);
|
305
|
+
const fprt = await this.ko.fingerPrint();
|
306
|
+
this.ko.logger.Debug().Str("fp", base58btc.encode(keyId)).Msg("decode");
|
307
|
+
if (base58btc.encode(keyId) !== fprt) {
|
308
|
+
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
|
385
309
|
}
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
constructor(sthis, ebOpts) {
|
390
|
-
super(ebOpts);
|
391
|
-
this.compacting = false;
|
392
|
-
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
|
393
|
-
const { name } = ebOpts;
|
394
|
-
if (!name) {
|
395
|
-
throw this.logger.Error().Msg("name required").AsError();
|
310
|
+
const result = await this.ko._decrypt({ iv, bytes: data });
|
311
|
+
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
312
|
+
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
396
313
|
}
|
397
|
-
|
398
|
-
this.loader = new Loader(this.name, ebOpts, sthis);
|
314
|
+
return result;
|
399
315
|
}
|
400
|
-
|
401
|
-
|
316
|
+
};
|
317
|
+
var keyedCrypto = class {
|
318
|
+
constructor(url, key, cyopt, sthis) {
|
319
|
+
this.ivLength = 12;
|
320
|
+
this.isEncrypting = true;
|
321
|
+
this.logger = ensureLogger(sthis, "keyedCrypto");
|
322
|
+
this.crypto = cyopt;
|
323
|
+
this.key = key;
|
324
|
+
this.url = url;
|
402
325
|
}
|
403
|
-
|
404
|
-
return this.
|
326
|
+
fingerPrint() {
|
327
|
+
return Promise.resolve(this.key.fingerPrint);
|
405
328
|
}
|
406
|
-
|
407
|
-
return this
|
329
|
+
codec(iv, opts) {
|
330
|
+
return new BlockIvKeyIdCodec(this, iv, opts);
|
408
331
|
}
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
}
|
415
|
-
return falsyToUndef(await this.loader.getBlock(cid));
|
332
|
+
algo(iv) {
|
333
|
+
return {
|
334
|
+
name: "AES-GCM",
|
335
|
+
iv: iv || this.crypto.randomBytes(this.ivLength),
|
336
|
+
tagLength: 128
|
337
|
+
};
|
416
338
|
}
|
417
|
-
async
|
418
|
-
|
419
|
-
|
420
|
-
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
421
|
-
setTimeout(() => void this.compact(), 10);
|
422
|
-
}
|
423
|
-
if (cars) {
|
424
|
-
this.transactions.delete(t);
|
425
|
-
return { meta: done, cars, t };
|
426
|
-
}
|
427
|
-
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
339
|
+
async _decrypt(data) {
|
340
|
+
this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
|
341
|
+
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
428
342
|
}
|
429
|
-
async
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
car
|
434
|
-
/*, isPublic */
|
435
|
-
);
|
436
|
-
const block = await reader.get(cid);
|
437
|
-
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
438
|
-
return block.bytes;
|
343
|
+
async _encrypt(data) {
|
344
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
|
345
|
+
const a = this.algo(data.iv);
|
346
|
+
return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
|
439
347
|
}
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
if (!compactFn || this.compacting) return;
|
446
|
-
const blockLog = new CompactionFetcher(this);
|
447
|
-
this.compacting = true;
|
448
|
-
const meta = await compactFn(blockLog);
|
449
|
-
await this.loader?.commit(blockLog.loggedBlocks, meta, {
|
450
|
-
compact: true,
|
451
|
-
noLoader: true
|
452
|
-
});
|
453
|
-
this.compacting = false;
|
348
|
+
};
|
349
|
+
var nullCodec = class {
|
350
|
+
constructor() {
|
351
|
+
this.code = 0;
|
352
|
+
this.name = "Fireproof@unencrypted-block";
|
454
353
|
}
|
455
|
-
|
456
|
-
|
457
|
-
throw logger.Error().Msg("no loader").AsError();
|
458
|
-
}
|
459
|
-
if (!this.lastTxMeta) {
|
460
|
-
throw logger.Error().Msg("no lastTxMeta").AsError();
|
461
|
-
}
|
462
|
-
for await (const blk of this.loader.entries(false)) {
|
463
|
-
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
464
|
-
}
|
465
|
-
for (const t of this.transactions) {
|
466
|
-
for await (const blk of t.entries()) {
|
467
|
-
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
468
|
-
}
|
469
|
-
}
|
470
|
-
return this.lastTxMeta;
|
354
|
+
encode(data) {
|
355
|
+
return data;
|
471
356
|
}
|
472
|
-
|
473
|
-
|
474
|
-
yield blk;
|
475
|
-
}
|
357
|
+
decode(data) {
|
358
|
+
return data;
|
476
359
|
}
|
477
360
|
};
|
478
|
-
var
|
479
|
-
constructor(
|
480
|
-
this.
|
481
|
-
this.
|
361
|
+
var noCrypto = class {
|
362
|
+
constructor(url, cyrt, sthis) {
|
363
|
+
this.ivLength = 0;
|
364
|
+
this.code = 0;
|
365
|
+
this.name = "Fireproof@unencrypted-block";
|
366
|
+
this.isEncrypting = false;
|
367
|
+
this._fingerPrint = "noCrypto:" + Math.random();
|
368
|
+
this.logger = ensureLogger(sthis, "noCrypto");
|
369
|
+
this.crypto = cyrt;
|
370
|
+
this.url = url;
|
482
371
|
}
|
483
|
-
|
484
|
-
|
485
|
-
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
486
|
-
return falsyToUndef(block);
|
372
|
+
fingerPrint() {
|
373
|
+
return Promise.resolve(this._fingerPrint);
|
487
374
|
}
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
import { Future } from "@adviser/cement";
|
492
|
-
var CommitQueue = class {
|
493
|
-
constructor() {
|
494
|
-
this.queue = [];
|
495
|
-
this.processing = false;
|
496
|
-
this._waitIdleItems = /* @__PURE__ */ new Set();
|
375
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
376
|
+
codec(iv) {
|
377
|
+
return new nullCodec();
|
497
378
|
}
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
379
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
380
|
+
algo(iv) {
|
381
|
+
return {
|
382
|
+
name: "noCrypto",
|
383
|
+
iv: new Uint8Array(),
|
384
|
+
tagLength: 0
|
385
|
+
};
|
505
386
|
}
|
506
|
-
|
507
|
-
|
508
|
-
const queueFn = async () => {
|
509
|
-
try {
|
510
|
-
resolve(await fn());
|
511
|
-
} catch (e) {
|
512
|
-
reject(e);
|
513
|
-
} finally {
|
514
|
-
this.processing = false;
|
515
|
-
this.processNext();
|
516
|
-
}
|
517
|
-
};
|
518
|
-
this.queue.push(queueFn);
|
519
|
-
if (!this.processing) {
|
520
|
-
this.processNext();
|
521
|
-
}
|
522
|
-
});
|
387
|
+
_decrypt() {
|
388
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
523
389
|
}
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
390
|
+
_encrypt() {
|
391
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
392
|
+
}
|
393
|
+
};
|
394
|
+
async function keyedCryptoFactory(url, kb, sthis) {
|
395
|
+
const storekey = url.getParam("storekey" /* STORE_KEY */);
|
396
|
+
if (storekey && storekey !== "insecure") {
|
397
|
+
let rkey = await kb.getNamedKey(storekey, true);
|
398
|
+
if (rkey.isErr()) {
|
399
|
+
try {
|
400
|
+
rkey = await kb.toKeyWithFingerPrint(storekey);
|
401
|
+
} catch (e) {
|
402
|
+
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
531
403
|
}
|
532
404
|
}
|
533
|
-
|
534
|
-
const toResolve = Array.from(this._waitIdleItems);
|
535
|
-
this._waitIdleItems.clear();
|
536
|
-
toResolve.map((fn) => fn.resolve());
|
537
|
-
}
|
405
|
+
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
538
406
|
}
|
539
|
-
|
407
|
+
return new noCrypto(url, kb.rt.crypto, sthis);
|
408
|
+
}
|
540
409
|
|
541
|
-
// src/
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
ResolveOnce,
|
551
|
-
ResolveSeq,
|
552
|
-
Result as Result2,
|
553
|
-
runtimeFn,
|
554
|
-
toCryptoRuntime as toCryptoRuntime2,
|
555
|
-
URI
|
556
|
-
} from "@adviser/cement";
|
557
|
-
import { base58btc } from "multiformats/bases/base58";
|
558
|
-
var KeyBag = class {
|
559
|
-
constructor(rt) {
|
560
|
-
this.rt = rt;
|
561
|
-
this._warnOnce = new ResolveOnce();
|
562
|
-
this._seq = new ResolveSeq();
|
563
|
-
this.logger = ensureLogger(rt.sthis, "KeyBag");
|
564
|
-
this.logger.Debug().Msg("KeyBag created");
|
410
|
+
// src/blockstore/fragment-gateway.ts
|
411
|
+
import { Result as Result2 } from "@adviser/cement";
|
412
|
+
import { base58btc as base58btc2 } from "multiformats/bases/base58";
|
413
|
+
import { encode as encode2, decode as decode2 } from "cborg";
|
414
|
+
function getFragSize(url) {
|
415
|
+
const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
|
416
|
+
let ret = 0;
|
417
|
+
if (fragSize) {
|
418
|
+
ret = parseInt(fragSize);
|
565
419
|
}
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
420
|
+
if (isNaN(ret) || ret <= 0) {
|
421
|
+
ret = 0;
|
422
|
+
}
|
423
|
+
return ret;
|
424
|
+
}
|
425
|
+
async function getFrags(url, innerGW, headerSize, logger) {
|
426
|
+
const fragSize = getFragSize(url);
|
427
|
+
if (!fragSize) {
|
428
|
+
const res = await innerGW.get(url);
|
429
|
+
if (res.isErr()) {
|
430
|
+
return [res];
|
431
|
+
}
|
432
|
+
const data = res.unwrap();
|
433
|
+
return [
|
434
|
+
Result2.Ok({
|
435
|
+
fid: new Uint8Array(0),
|
436
|
+
ofs: 0,
|
437
|
+
len: data.length,
|
438
|
+
data
|
439
|
+
})
|
440
|
+
];
|
441
|
+
}
|
442
|
+
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
443
|
+
if (firstRaw.isErr()) {
|
444
|
+
return [firstRaw];
|
445
|
+
}
|
446
|
+
const firstFragment = decode2(firstRaw.unwrap());
|
447
|
+
const blockSize = firstFragment.data.length;
|
448
|
+
const ops = [Promise.resolve(Result2.Ok(firstFragment))];
|
449
|
+
const fidStr = base58btc2.encode(firstFragment.fid);
|
450
|
+
const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
|
451
|
+
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
452
|
+
ops.push(
|
453
|
+
(async (furl, ofs2) => {
|
454
|
+
const raw2 = await innerGW.get(furl);
|
455
|
+
if (raw2.isErr()) {
|
456
|
+
return raw2;
|
457
|
+
}
|
458
|
+
const fragment = decode2(raw2.unwrap());
|
459
|
+
if (base58btc2.encode(fragment.fid) !== fidStr) {
|
460
|
+
return Result2.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
461
|
+
}
|
462
|
+
if (fragment.ofs !== ofs2) {
|
463
|
+
return Result2.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
464
|
+
}
|
465
|
+
return Result2.Ok(fragment);
|
466
|
+
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
467
|
+
);
|
468
|
+
}
|
469
|
+
return Promise.all(ops);
|
470
|
+
}
|
471
|
+
var FragmentGateway = class {
|
472
|
+
constructor(sthis, innerGW) {
|
473
|
+
this.fidLength = 4;
|
474
|
+
this.headerSize = 32;
|
475
|
+
this.sthis = ensureSuperLog(sthis, "FragmentGateway");
|
476
|
+
this.logger = this.sthis.logger;
|
477
|
+
this.innerGW = innerGW;
|
478
|
+
}
|
479
|
+
slicer(url, body) {
|
480
|
+
const fragSize = getFragSize(url);
|
481
|
+
if (!fragSize) {
|
482
|
+
return [this.innerGW.put(url, body)];
|
483
|
+
}
|
484
|
+
const blocksize = fragSize - this.headerSize;
|
485
|
+
if (blocksize <= 0) {
|
486
|
+
throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
|
487
|
+
}
|
488
|
+
const ops = [];
|
489
|
+
const fid = this.sthis.nextId(this.fidLength);
|
490
|
+
const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
|
491
|
+
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
492
|
+
const block = encode2({
|
493
|
+
fid: fid.bin,
|
494
|
+
ofs,
|
495
|
+
len: body.length,
|
496
|
+
data: body.slice(ofs, ofs + blocksize)
|
497
|
+
});
|
498
|
+
if (block.length > fragSize) {
|
499
|
+
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
500
|
+
}
|
501
|
+
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
502
|
+
}
|
503
|
+
return ops;
|
504
|
+
}
|
505
|
+
buildUrl(baseUrl, key) {
|
506
|
+
return this.innerGW.buildUrl(baseUrl, key);
|
507
|
+
}
|
508
|
+
async destroy(iurl) {
|
509
|
+
return this.innerGW.destroy(iurl);
|
510
|
+
}
|
511
|
+
async start(url) {
|
512
|
+
this.headerSize = encode2({
|
513
|
+
fid: this.sthis.nextId(this.fidLength).bin,
|
514
|
+
ofs: 1024 * 1024,
|
515
|
+
// 32bit
|
516
|
+
len: 16 * 1024 * 1024,
|
517
|
+
// 32bit
|
518
|
+
data: new Uint8Array(1024)
|
519
|
+
}).length - 1024;
|
520
|
+
return this.innerGW.start(url);
|
521
|
+
}
|
522
|
+
async close(url) {
|
523
|
+
return this.innerGW.close(url);
|
524
|
+
}
|
525
|
+
async put(url, body) {
|
526
|
+
await Promise.all(this.slicer(url, body));
|
527
|
+
return Result2.Ok(void 0);
|
528
|
+
}
|
529
|
+
async get(url) {
|
530
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
531
|
+
let buffer = void 0;
|
532
|
+
for (const rfrag of rfrags) {
|
533
|
+
if (rfrag.isErr()) {
|
534
|
+
return Result2.Err(rfrag.Err());
|
535
|
+
}
|
536
|
+
const frag = rfrag.Ok();
|
537
|
+
buffer = buffer || new Uint8Array(frag.len);
|
538
|
+
buffer.set(frag.data, frag.ofs);
|
539
|
+
}
|
540
|
+
return Result2.Ok(buffer || new Uint8Array(0));
|
541
|
+
}
|
542
|
+
async subscribe(url, callback) {
|
543
|
+
if (this.innerGW.subscribe) {
|
544
|
+
return this.innerGW.subscribe(url, callback);
|
545
|
+
} else {
|
546
|
+
return Result2.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
|
547
|
+
}
|
548
|
+
}
|
549
|
+
async delete(url) {
|
550
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
551
|
+
for (const rfrag of rfrags) {
|
552
|
+
if (rfrag.isErr()) {
|
553
|
+
return Result2.Err(rfrag.Err());
|
554
|
+
}
|
555
|
+
const frag = rfrag.Ok();
|
556
|
+
let fragUrl;
|
557
|
+
if (rfrags.length > 1) {
|
558
|
+
const fidStr = base58btc2.encode(frag.fid);
|
559
|
+
fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
|
560
|
+
} else {
|
561
|
+
fragUrl = url;
|
562
|
+
}
|
563
|
+
await this.innerGW.delete(fragUrl);
|
564
|
+
}
|
565
|
+
return Result2.Ok(void 0);
|
566
|
+
}
|
567
|
+
};
|
568
|
+
|
569
|
+
// src/blockstore/meta-key-helper.ts
|
570
|
+
import { format, parse } from "@ipld/dag-json";
|
571
|
+
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
572
|
+
import { CID } from "multiformats";
|
573
|
+
import { base64pad } from "multiformats/bases/base64";
|
574
|
+
import { Result as Result4 } from "@adviser/cement";
|
575
|
+
|
576
|
+
// src/runtime/key-bag.ts
|
577
|
+
var key_bag_exports = {};
|
578
|
+
__export(key_bag_exports, {
|
579
|
+
KeyBag: () => KeyBag,
|
580
|
+
defaultKeyBagOpts: () => defaultKeyBagOpts,
|
581
|
+
getKeyBag: () => getKeyBag,
|
582
|
+
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
583
|
+
});
|
584
|
+
import {
|
585
|
+
KeyedResolvOnce,
|
586
|
+
ResolveOnce,
|
587
|
+
ResolveSeq,
|
588
|
+
Result as Result3,
|
589
|
+
runtimeFn,
|
590
|
+
toCryptoRuntime,
|
591
|
+
URI as URI2
|
592
|
+
} from "@adviser/cement";
|
593
|
+
import { base58btc as base58btc3 } from "multiformats/bases/base58";
|
594
|
+
var KeyBag = class {
|
595
|
+
constructor(rt) {
|
596
|
+
this.rt = rt;
|
597
|
+
this._warnOnce = new ResolveOnce();
|
598
|
+
this._seq = new ResolveSeq();
|
599
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag", {
|
600
|
+
id: rt.id()
|
601
|
+
});
|
602
|
+
this.logger.Debug().Msg("KeyBag created");
|
603
|
+
}
|
604
|
+
async subtleKey(key) {
|
605
|
+
const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
|
606
|
+
if (extractable) {
|
607
|
+
this._warnOnce.once(
|
608
|
+
() => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
|
571
609
|
);
|
572
610
|
}
|
573
611
|
return await this.rt.crypto.importKey(
|
574
612
|
"raw",
|
575
613
|
// raw or jwk
|
576
|
-
|
614
|
+
base58btc3.decode(key),
|
577
615
|
// hexStringToUint8Array(key), // raw data
|
578
616
|
"AES-GCM",
|
579
617
|
extractable,
|
@@ -581,9 +619,9 @@ var KeyBag = class {
|
|
581
619
|
);
|
582
620
|
}
|
583
621
|
async ensureKeyFromUrl(url, keyFactory) {
|
584
|
-
const storeKey = url.getParam("storekey");
|
622
|
+
const storeKey = url.getParam("storekey" /* STORE_KEY */);
|
585
623
|
if (storeKey === "insecure") {
|
586
|
-
return
|
624
|
+
return Result3.Ok(url);
|
587
625
|
}
|
588
626
|
if (!storeKey) {
|
589
627
|
const keyName = `@${keyFactory()}@`;
|
@@ -591,8 +629,8 @@ var KeyBag = class {
|
|
591
629
|
if (ret.isErr()) {
|
592
630
|
return ret;
|
593
631
|
}
|
594
|
-
const urb = url.build().setParam("storekey"
|
595
|
-
return
|
632
|
+
const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
|
633
|
+
return Result3.Ok(urb.URI());
|
596
634
|
}
|
597
635
|
if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
|
598
636
|
const ret = await this.getNamedKey(storeKey);
|
@@ -600,15 +638,15 @@ var KeyBag = class {
|
|
600
638
|
return ret;
|
601
639
|
}
|
602
640
|
}
|
603
|
-
return
|
641
|
+
return Result3.Ok(url);
|
604
642
|
}
|
605
643
|
async toKeyWithFingerPrint(keyStr) {
|
606
|
-
const material =
|
644
|
+
const material = base58btc3.decode(keyStr);
|
607
645
|
const key = await this.subtleKey(keyStr);
|
608
646
|
const fpr = await this.rt.crypto.digestSHA256(material);
|
609
|
-
return
|
647
|
+
return Result3.Ok({
|
610
648
|
key,
|
611
|
-
fingerPrint:
|
649
|
+
fingerPrint: base58btc3.encode(new Uint8Array(fpr))
|
612
650
|
});
|
613
651
|
}
|
614
652
|
async setNamedKey(name, key) {
|
@@ -631,13 +669,13 @@ var KeyBag = class {
|
|
631
669
|
return ret;
|
632
670
|
}
|
633
671
|
const named = ret.Ok();
|
634
|
-
return
|
672
|
+
return Result3.Ok({
|
635
673
|
...named,
|
636
674
|
extract: async () => {
|
637
675
|
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
638
676
|
return {
|
639
677
|
key: ext,
|
640
|
-
keyStr:
|
678
|
+
keyStr: base58btc3.encode(ext)
|
641
679
|
};
|
642
680
|
}
|
643
681
|
});
|
@@ -654,9 +692,9 @@ var KeyBag = class {
|
|
654
692
|
}
|
655
693
|
if (failIfNotFound) {
|
656
694
|
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
657
|
-
return
|
695
|
+
return Result3.Err(new Error(`Key not found: ${name}`));
|
658
696
|
}
|
659
|
-
const ret = await this._setNamedKey(name,
|
697
|
+
const ret = await this._setNamedKey(name, base58btc3.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
660
698
|
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
661
699
|
return ret;
|
662
700
|
});
|
@@ -667,14 +705,14 @@ var keyBagProviderFactories = new Map(
|
|
667
705
|
{
|
668
706
|
protocol: "file:",
|
669
707
|
factory: async (url, sthis) => {
|
670
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-
|
708
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
|
671
709
|
return new KeyBagProviderFile(url, sthis);
|
672
710
|
}
|
673
711
|
},
|
674
712
|
{
|
675
713
|
protocol: "indexdb:",
|
676
714
|
factory: async (url, sthis) => {
|
677
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-
|
715
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
|
678
716
|
return new KeyBagProviderIndexDB(url, sthis);
|
679
717
|
}
|
680
718
|
}
|
@@ -688,44 +726,63 @@ function registerKeyBagProviderFactory(item) {
|
|
688
726
|
});
|
689
727
|
}
|
690
728
|
function defaultKeyBagOpts(sthis, kbo) {
|
729
|
+
kbo = kbo || {};
|
691
730
|
if (kbo.keyRuntime) {
|
692
731
|
return kbo.keyRuntime;
|
693
732
|
}
|
694
733
|
const logger = ensureLogger(sthis, "KeyBag");
|
695
734
|
let url;
|
696
735
|
if (kbo.url) {
|
697
|
-
url =
|
736
|
+
url = URI2.from(kbo.url);
|
698
737
|
logger.Debug().Url(url).Msg("from opts");
|
699
738
|
} else {
|
700
739
|
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
701
740
|
if (runtimeFn().isBrowser) {
|
702
|
-
url =
|
741
|
+
url = URI2.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
703
742
|
} else {
|
704
743
|
if (!bagFnameOrUrl) {
|
705
744
|
const home = sthis.env.get("HOME");
|
706
745
|
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
707
|
-
url =
|
746
|
+
url = URI2.from(`file://${bagFnameOrUrl}`);
|
708
747
|
} else {
|
709
|
-
url =
|
748
|
+
url = URI2.from(bagFnameOrUrl);
|
710
749
|
}
|
711
750
|
}
|
712
751
|
logger.Debug().Url(url).Msg("from env");
|
713
752
|
}
|
714
|
-
|
715
|
-
|
716
|
-
|
753
|
+
let keyProviderFactory;
|
754
|
+
switch (url.protocol) {
|
755
|
+
case "file:":
|
756
|
+
keyProviderFactory = async () => {
|
757
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
|
758
|
+
return new KeyBagProviderFile(url, sthis);
|
759
|
+
};
|
760
|
+
break;
|
761
|
+
case "indexdb:":
|
762
|
+
keyProviderFactory = async () => {
|
763
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
|
764
|
+
return new KeyBagProviderIndexDB(url, sthis);
|
765
|
+
};
|
766
|
+
break;
|
767
|
+
case "memory:":
|
768
|
+
keyProviderFactory = async () => {
|
769
|
+
const { KeyBagProviderMemory } = await import("./key-bag-memory-LWE6ARPX.js");
|
770
|
+
return new KeyBagProviderMemory(url, sthis);
|
771
|
+
};
|
772
|
+
break;
|
773
|
+
default:
|
774
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
717
775
|
}
|
718
|
-
const getBag = async () => kitem.factory(url, sthis);
|
719
776
|
if (url.hasParam("masterkey")) {
|
720
777
|
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
721
778
|
}
|
722
779
|
return {
|
723
780
|
url,
|
724
|
-
crypto: kbo.crypto ||
|
781
|
+
crypto: kbo.crypto || toCryptoRuntime({}),
|
725
782
|
sthis,
|
726
783
|
logger,
|
727
784
|
keyLength: kbo.keyLength || 16,
|
728
|
-
getBag,
|
785
|
+
getBag: keyProviderFactory,
|
729
786
|
id: () => {
|
730
787
|
return url.toString();
|
731
788
|
}
|
@@ -738,936 +795,940 @@ async function getKeyBag(sthis, kbo = {}) {
|
|
738
795
|
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
739
796
|
}
|
740
797
|
|
741
|
-
// src/blockstore/
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
const headerSize = CBW.headerLength({ roots });
|
748
|
-
size += headerSize;
|
749
|
-
for (const { cid, bytes } of t.entries()) {
|
750
|
-
size += CBW.blockLength({ cid, bytes });
|
751
|
-
}
|
752
|
-
const buffer = new Uint8Array(size);
|
753
|
-
const writer = CBW.createWriter(buffer, { headerSize });
|
754
|
-
for (const r of roots) {
|
755
|
-
writer.addRoot(r);
|
756
|
-
}
|
757
|
-
for (const { cid, bytes } of t.entries()) {
|
758
|
-
writer.write({ cid, bytes });
|
798
|
+
// src/blockstore/meta-key-helper.ts
|
799
|
+
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
800
|
+
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
801
|
+
if (!crdtEntries.length) {
|
802
|
+
sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
|
803
|
+
return [];
|
759
804
|
}
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
async function createCarFile(encoder, cid, t) {
|
764
|
-
return encodeCarFile([cid], t, encoder);
|
765
|
-
}
|
766
|
-
async function commitFiles(fileStore, walStore, t, done) {
|
767
|
-
const { files: roots } = makeFileCarHeader(done);
|
768
|
-
const cids = [];
|
769
|
-
const codec3 = (await fileStore.keyedCrypto()).codec();
|
770
|
-
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
771
|
-
for (const car of cars) {
|
772
|
-
const { cid, bytes } = car;
|
773
|
-
await fileStore.save({ cid, bytes });
|
774
|
-
await walStore.enqueueFile(
|
775
|
-
cid
|
776
|
-
/*, !!opts.public*/
|
777
|
-
);
|
778
|
-
cids.push(cid);
|
805
|
+
if (!crdtEntries.map) {
|
806
|
+
sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
|
807
|
+
return [];
|
779
808
|
}
|
780
|
-
return
|
809
|
+
return Promise.all(
|
810
|
+
crdtEntries.map(async (crdtEntry) => {
|
811
|
+
const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
|
812
|
+
const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
|
813
|
+
return {
|
814
|
+
eventCid: eventBlock.cid,
|
815
|
+
parents: crdtEntry.parents,
|
816
|
+
dbMeta
|
817
|
+
};
|
818
|
+
})
|
819
|
+
);
|
781
820
|
}
|
782
|
-
function
|
783
|
-
|
784
|
-
|
785
|
-
|
786
|
-
|
821
|
+
async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
|
822
|
+
try {
|
823
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
|
824
|
+
const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
|
825
|
+
if (keyInfo.length) {
|
826
|
+
const dbMeta = keyInfo[0].dbMeta;
|
827
|
+
if (dbMeta.key) {
|
828
|
+
const kb = await getKeyBag(sthis);
|
829
|
+
const keyName = getStoreKeyName(uri);
|
830
|
+
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
831
|
+
if (res.isErr()) {
|
832
|
+
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
833
|
+
throw res.Err();
|
834
|
+
}
|
835
|
+
}
|
836
|
+
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
837
|
+
return Result4.Ok(dbMeta);
|
787
838
|
}
|
839
|
+
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
840
|
+
return Result4.Ok(void 0);
|
841
|
+
} catch (error) {
|
842
|
+
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
843
|
+
return Result4.Err(error);
|
788
844
|
}
|
789
|
-
return { ...result, files };
|
790
845
|
}
|
791
|
-
async function
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
cids.push(cid);
|
846
|
+
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
847
|
+
try {
|
848
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
849
|
+
const keyName = getStoreKeyName(uri);
|
850
|
+
const kb = await getKeyBag(sthis);
|
851
|
+
const res = await kb.getNamedExtractableKey(keyName, true);
|
852
|
+
if (res.isErr()) {
|
853
|
+
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
854
|
+
throw res.Err();
|
855
|
+
}
|
856
|
+
const keyData = await res.Ok().extract();
|
857
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
858
|
+
const { dbMeta, parents } = dbMetas[0];
|
859
|
+
const parentLinks = parents.map((p) => CID.parse(p));
|
860
|
+
dbMeta.key = keyData.keyStr;
|
861
|
+
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
862
|
+
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
863
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
864
|
+
return Result4.Ok(encoded);
|
865
|
+
} catch (error) {
|
866
|
+
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
867
|
+
return Result4.Err(error);
|
814
868
|
}
|
815
|
-
const newDbMeta = { cars: cids };
|
816
|
-
await params.WALStore.enqueue(newDbMeta, opts);
|
817
|
-
await params.metaStore.save(newDbMeta);
|
818
|
-
return { cgrp: cids, header: fp };
|
819
869
|
}
|
820
|
-
|
821
|
-
const
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
826
|
-
let cidRootBlock = rootBlock;
|
827
|
-
for (const { cid, bytes } of t.entries()) {
|
828
|
-
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
829
|
-
if (newsize >= threshold) {
|
830
|
-
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
831
|
-
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
832
|
-
clonedt.putSync(cid, bytes);
|
833
|
-
cidRootBlock = { cid, bytes };
|
834
|
-
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
835
|
-
} else {
|
836
|
-
clonedt.putSync(cid, bytes);
|
837
|
-
}
|
870
|
+
function getStoreKeyName(url) {
|
871
|
+
const storeKeyName = [url.getParam("localName") || url.getParam("name")];
|
872
|
+
const idx = url.getParam("index");
|
873
|
+
if (idx) {
|
874
|
+
storeKeyName.push(idx);
|
838
875
|
}
|
839
|
-
|
840
|
-
return
|
876
|
+
storeKeyName.push("data");
|
877
|
+
return `@${storeKeyName.join(":")}@`;
|
878
|
+
}
|
879
|
+
async function createDbMetaEventBlock(sthis, dbMeta, parents) {
|
880
|
+
const event = await EventBlock.create(
|
881
|
+
{
|
882
|
+
dbMeta: sthis.txt.encode(format(dbMeta))
|
883
|
+
},
|
884
|
+
parents
|
885
|
+
);
|
886
|
+
return event;
|
887
|
+
}
|
888
|
+
async function encodeEventsWithParents(sthis, events, parents) {
|
889
|
+
const crdtEntries = events.map((event) => {
|
890
|
+
const base64String = base64pad.encode(event.bytes);
|
891
|
+
return {
|
892
|
+
cid: event.cid.toString(),
|
893
|
+
data: base64String,
|
894
|
+
parents: parents.map((p) => p.toString())
|
895
|
+
};
|
896
|
+
});
|
897
|
+
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
841
898
|
}
|
842
899
|
|
843
|
-
// src/blockstore/
|
844
|
-
import
|
845
|
-
|
846
|
-
// src/blockstore/task-manager.ts
|
847
|
-
var TaskManager = class {
|
848
|
-
constructor(sthis, callback) {
|
849
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
850
|
-
this.queue = [];
|
851
|
-
this.isProcessing = false;
|
852
|
-
this.logger = ensureLogger(sthis, "TaskManager");
|
853
|
-
this.callback = callback;
|
854
|
-
}
|
855
|
-
async handleEvent(cid, parents, dbMeta) {
|
856
|
-
for (const parent of parents) {
|
857
|
-
this.eventsWeHandled.add(parent.toString());
|
858
|
-
}
|
859
|
-
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
860
|
-
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
861
|
-
void this.processQueue();
|
862
|
-
}
|
863
|
-
async processQueue() {
|
864
|
-
if (this.isProcessing) return;
|
865
|
-
this.isProcessing = true;
|
866
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
867
|
-
const first = filteredQueue[0];
|
868
|
-
if (!first) {
|
869
|
-
return;
|
870
|
-
}
|
871
|
-
try {
|
872
|
-
await this.callback(first.dbMeta);
|
873
|
-
this.eventsWeHandled.add(first.cid);
|
874
|
-
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
875
|
-
} catch (err) {
|
876
|
-
if (first.retries++ > 3) {
|
877
|
-
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
878
|
-
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
879
|
-
}
|
880
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
881
|
-
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
882
|
-
} finally {
|
883
|
-
this.isProcessing = false;
|
884
|
-
if (this.queue.length > 0) {
|
885
|
-
void this.processQueue();
|
886
|
-
}
|
887
|
-
}
|
888
|
-
}
|
889
|
-
};
|
900
|
+
// src/blockstore/store.ts
|
901
|
+
import pRetry from "p-retry";
|
902
|
+
import pMap from "p-map";
|
890
903
|
|
891
904
|
// src/blockstore/loader.ts
|
892
|
-
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
899
|
-
|
900
|
-
|
901
|
-
|
902
|
-
|
903
|
-
|
905
|
+
import pLimit from "p-limit";
|
906
|
+
import { CarReader } from "@ipld/car";
|
907
|
+
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
908
|
+
|
909
|
+
// src/runtime/wait-pr-multiformats/block.ts
|
910
|
+
var block_exports = {};
|
911
|
+
__export(block_exports, {
|
912
|
+
Block: () => Block,
|
913
|
+
create: () => create,
|
914
|
+
createUnsafe: () => createUnsafe,
|
915
|
+
decode: () => decode3,
|
916
|
+
encode: () => encode3
|
917
|
+
});
|
918
|
+
import { bytes as binary, CID as CID2 } from "multiformats";
|
919
|
+
import { Block as mfBlock } from "multiformats/block";
|
920
|
+
var Block = mfBlock;
|
921
|
+
async function decode3({
|
922
|
+
bytes,
|
923
|
+
codec: codec3,
|
924
|
+
hasher: hasher7
|
925
|
+
}) {
|
926
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
927
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
928
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
929
|
+
const hash = await hasher7.digest(bytes);
|
930
|
+
const cid = CID2.create(1, codec3.code, hash);
|
931
|
+
return new mfBlock({ value, bytes, cid });
|
904
932
|
}
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
|
916
|
-
|
917
|
-
|
918
|
-
|
919
|
-
|
920
|
-
|
921
|
-
|
922
|
-
|
923
|
-
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
await this.handleDbMetasFromStore([dbMeta]);
|
930
|
-
});
|
931
|
-
}
|
932
|
-
// readonly id = uuidv4();
|
933
|
-
async keyBag() {
|
934
|
-
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
935
|
-
}
|
936
|
-
async carStore() {
|
937
|
-
return this.ebOpts.storeRuntime.makeDataStore(this);
|
938
|
-
}
|
939
|
-
async fileStore() {
|
940
|
-
return this.ebOpts.storeRuntime.makeDataStore(this);
|
941
|
-
}
|
942
|
-
async WALStore() {
|
943
|
-
return this.ebOpts.storeRuntime.makeWALStore(this);
|
944
|
-
}
|
945
|
-
async metaStore() {
|
946
|
-
return this.ebOpts.storeRuntime.makeMetaStore(this);
|
947
|
-
}
|
948
|
-
async ready() {
|
949
|
-
return this.onceReady.once(async () => {
|
950
|
-
const metas = await (await this.metaStore()).load();
|
951
|
-
if (this.ebOpts.meta) {
|
952
|
-
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
953
|
-
} else if (metas) {
|
954
|
-
await this.handleDbMetasFromStore(metas);
|
955
|
-
}
|
956
|
-
});
|
957
|
-
}
|
958
|
-
async close() {
|
959
|
-
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
960
|
-
await Promise.all(toClose.map((store) => store.close()));
|
961
|
-
}
|
962
|
-
async destroy() {
|
963
|
-
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
964
|
-
await Promise.all(toDestroy.map((store) => store.destroy()));
|
965
|
-
}
|
966
|
-
// async snapToCar(carCid: AnyLink | string) {
|
967
|
-
// await this.ready
|
968
|
-
// if (typeof carCid === 'string') {
|
969
|
-
// carCid = CID.parse(carCid)
|
970
|
-
// }
|
971
|
-
// const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
|
972
|
-
// this.carLog = [carCid, ...carHeader.cars]
|
973
|
-
// await this.getMoreReaders(carHeader.cars)
|
974
|
-
// await this._applyCarHeader(carHeader, true)
|
975
|
-
// }
|
976
|
-
async handleDbMetasFromStore(metas) {
|
977
|
-
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
978
|
-
for (const meta of metas) {
|
979
|
-
await this.writeLimit(async () => {
|
980
|
-
await this.mergeDbMetaIntoClock(meta);
|
981
|
-
});
|
982
|
-
}
|
983
|
-
}
|
984
|
-
async mergeDbMetaIntoClock(meta) {
|
985
|
-
if (this.isCompacting) {
|
986
|
-
throw this.logger.Error().Msg("cannot merge while compacting").AsError();
|
987
|
-
}
|
988
|
-
if (this.seenMeta.has(meta.cars.toString())) return;
|
989
|
-
this.seenMeta.add(meta.cars.toString());
|
990
|
-
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
991
|
-
return;
|
992
|
-
}
|
993
|
-
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
994
|
-
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
995
|
-
await this.getMoreReaders(carHeader.cars.flat());
|
996
|
-
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
997
|
-
await this.ebOpts.applyMeta?.(carHeader.meta);
|
998
|
-
}
|
999
|
-
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1000
|
-
// const { key } = meta;
|
1001
|
-
// if (key) {
|
1002
|
-
// await this.setKey(key);
|
1003
|
-
// }
|
1004
|
-
// }
|
1005
|
-
async loadCarHeaderFromMeta({ cars: cids }) {
|
1006
|
-
const reader = await this.loadCar(cids[0]);
|
1007
|
-
return await parseCarFile(reader, this.logger);
|
1008
|
-
}
|
1009
|
-
// async _getKey(): Promise<string | undefined> {
|
1010
|
-
// if (this.key) return this.key;
|
1011
|
-
// // generate a random key
|
1012
|
-
// if (!this.ebOpts.public) {
|
1013
|
-
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1014
|
-
// }
|
1015
|
-
// return this.key || undefined;
|
1016
|
-
// }
|
1017
|
-
async commitFiles(t, done) {
|
1018
|
-
await this.ready();
|
1019
|
-
const fstore = await this.fileStore();
|
1020
|
-
const wstore = await this.WALStore();
|
1021
|
-
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1022
|
-
}
|
1023
|
-
async loadFileCar(cid) {
|
1024
|
-
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
1025
|
-
}
|
1026
|
-
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
1027
|
-
await this.ready();
|
1028
|
-
const fstore = await this.fileStore();
|
1029
|
-
const params = {
|
1030
|
-
encoder: (await fstore.keyedCrypto()).codec(),
|
1031
|
-
carLog: this.carLog,
|
1032
|
-
carStore: fstore,
|
1033
|
-
WALStore: await this.WALStore(),
|
1034
|
-
metaStore: await this.metaStore(),
|
1035
|
-
threshold: this.ebOpts.threshold
|
1036
|
-
};
|
1037
|
-
return this.commitQueue.enqueue(async () => {
|
1038
|
-
await this.cacheTransaction(t);
|
1039
|
-
const ret = await commit(params, t, done, opts);
|
1040
|
-
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1041
|
-
return ret.cgrp;
|
1042
|
-
});
|
1043
|
-
}
|
1044
|
-
async updateCarLog(cids, fp, compact) {
|
1045
|
-
if (compact) {
|
1046
|
-
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1047
|
-
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1048
|
-
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1049
|
-
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1050
|
-
} else {
|
1051
|
-
this.carLog.unshift(cids);
|
1052
|
-
}
|
1053
|
-
}
|
1054
|
-
async cacheTransaction(t) {
|
1055
|
-
for await (const block of t.entries()) {
|
1056
|
-
const sBlock = block.cid.toString();
|
1057
|
-
if (!this.getBlockCache.has(sBlock)) {
|
1058
|
-
this.getBlockCache.set(sBlock, block);
|
1059
|
-
}
|
1060
|
-
}
|
1061
|
-
}
|
1062
|
-
async cacheCarReader(carCidStr, reader) {
|
1063
|
-
if (this.processedCars.has(carCidStr)) return;
|
1064
|
-
this.processedCars.add(carCidStr);
|
1065
|
-
for await (const block of reader.blocks()) {
|
1066
|
-
const sBlock = block.cid.toString();
|
1067
|
-
if (!this.getBlockCache.has(sBlock)) {
|
1068
|
-
this.getBlockCache.set(sBlock, block);
|
1069
|
-
}
|
1070
|
-
}
|
1071
|
-
}
|
1072
|
-
async removeCidsForCompact(cid) {
|
1073
|
-
const carHeader = await this.loadCarHeaderFromMeta({
|
1074
|
-
cars: [cid]
|
1075
|
-
});
|
1076
|
-
for (const cids of carHeader.compact) {
|
1077
|
-
for (const cid2 of cids) {
|
1078
|
-
await (await this.carStore()).remove(cid2);
|
1079
|
-
}
|
1080
|
-
}
|
1081
|
-
}
|
1082
|
-
// async flushCars() {
|
1083
|
-
// await this.ready
|
1084
|
-
// // for each cid in car log, make a dbMeta
|
1085
|
-
// for (const cid of this.carLog) {
|
1086
|
-
// const dbMeta = { car: cid, key: this.key || null } as DbMeta
|
1087
|
-
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
1088
|
-
// }
|
1089
|
-
// }
|
1090
|
-
async *entries(cache2 = true) {
|
1091
|
-
await this.ready();
|
1092
|
-
if (cache2) {
|
1093
|
-
for (const [, block] of this.getBlockCache) {
|
1094
|
-
yield block;
|
1095
|
-
}
|
1096
|
-
} else {
|
1097
|
-
for (const [, block] of this.getBlockCache) {
|
1098
|
-
yield block;
|
1099
|
-
}
|
1100
|
-
for (const cids of this.carLog) {
|
1101
|
-
for (const cid of cids) {
|
1102
|
-
const reader = await this.loadCar(cid);
|
1103
|
-
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
1104
|
-
for await (const block of reader.blocks()) {
|
1105
|
-
const sCid = block.cid.toString();
|
1106
|
-
if (!this.getBlockCache.has(sCid)) {
|
1107
|
-
yield block;
|
1108
|
-
}
|
1109
|
-
}
|
1110
|
-
}
|
1111
|
-
}
|
1112
|
-
}
|
1113
|
-
}
|
1114
|
-
async getBlock(cid) {
|
1115
|
-
await this.ready();
|
1116
|
-
const sCid = cid.toString();
|
1117
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1118
|
-
const getCarCid = async (carCid) => {
|
1119
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1120
|
-
const reader = await this.loadCar(carCid);
|
1121
|
-
if (!reader) {
|
1122
|
-
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
1123
|
-
}
|
1124
|
-
await this.cacheCarReader(carCid.toString(), reader).catch(() => {
|
1125
|
-
return;
|
1126
|
-
});
|
1127
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1128
|
-
throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
|
1129
|
-
};
|
1130
|
-
const getCompactCarCids = async (carCid) => {
|
1131
|
-
const reader = await this.loadCar(carCid);
|
1132
|
-
if (!reader) {
|
1133
|
-
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
1134
|
-
}
|
1135
|
-
const header = await parseCarFile(reader, this.logger);
|
1136
|
-
const compacts = header.compact;
|
1137
|
-
let got2;
|
1138
|
-
const batchSize2 = 5;
|
1139
|
-
for (let i = 0; i < compacts.length; i += batchSize2) {
|
1140
|
-
const promises = [];
|
1141
|
-
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
1142
|
-
for (const cid2 of compacts[j]) {
|
1143
|
-
promises.push(getCarCid(cid2));
|
1144
|
-
}
|
1145
|
-
}
|
1146
|
-
try {
|
1147
|
-
got2 = await Promise.any(promises);
|
1148
|
-
} catch {
|
1149
|
-
}
|
1150
|
-
if (got2) break;
|
1151
|
-
}
|
1152
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1153
|
-
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
1154
|
-
};
|
1155
|
-
let got;
|
1156
|
-
const batchSize = 5;
|
1157
|
-
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
1158
|
-
const batch = this.carLog.slice(i, i + batchSize);
|
1159
|
-
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
1160
|
-
try {
|
1161
|
-
got = await Promise.any(promises);
|
1162
|
-
} catch {
|
1163
|
-
}
|
1164
|
-
if (got) break;
|
1165
|
-
}
|
1166
|
-
if (!got) {
|
1167
|
-
try {
|
1168
|
-
got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
|
1169
|
-
} catch {
|
1170
|
-
}
|
1171
|
-
}
|
1172
|
-
return got;
|
933
|
+
async function encode3({
|
934
|
+
value,
|
935
|
+
codec: codec3,
|
936
|
+
hasher: hasher7
|
937
|
+
}) {
|
938
|
+
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
939
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
940
|
+
const bytes = await Promise.resolve(codec3.encode(value));
|
941
|
+
const hash = await hasher7.digest(bytes);
|
942
|
+
const cid = CID2.create(1, codec3.code, hash);
|
943
|
+
return new mfBlock({ value, bytes, cid });
|
944
|
+
}
|
945
|
+
async function create({
|
946
|
+
bytes,
|
947
|
+
cid,
|
948
|
+
hasher: hasher7,
|
949
|
+
codec: codec3
|
950
|
+
}) {
|
951
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
952
|
+
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
953
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
954
|
+
const hash = await hasher7.digest(bytes);
|
955
|
+
if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
|
956
|
+
throw new Error("CID hash does not match bytes");
|
1173
957
|
}
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
958
|
+
return createUnsafe({
|
959
|
+
bytes,
|
960
|
+
cid,
|
961
|
+
value,
|
962
|
+
codec: codec3
|
963
|
+
});
|
964
|
+
}
|
965
|
+
async function createUnsafe({
|
966
|
+
bytes,
|
967
|
+
cid,
|
968
|
+
value: maybeValue,
|
969
|
+
codec: codec3
|
970
|
+
}) {
|
971
|
+
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
972
|
+
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
973
|
+
return new Block({
|
974
|
+
cid,
|
975
|
+
bytes,
|
976
|
+
value
|
977
|
+
});
|
978
|
+
}
|
979
|
+
|
980
|
+
// src/blockstore/loader-helpers.ts
|
981
|
+
import { sha256 as hasher2 } from "multiformats/hashes/sha2";
|
982
|
+
import * as dagCodec from "@ipld/dag-cbor";
|
983
|
+
async function parseCarFile(reader, logger) {
|
984
|
+
const roots = await reader.getRoots();
|
985
|
+
const header = await reader.get(roots[0]);
|
986
|
+
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
987
|
+
const dec = await decode3({ bytes: header.bytes, hasher: hasher2, codec: dagCodec });
|
988
|
+
const fpvalue = dec.value;
|
989
|
+
if (fpvalue && !fpvalue.fp) {
|
990
|
+
throw logger.Error().Msg("missing fp").AsError();
|
1180
991
|
}
|
1181
|
-
|
1182
|
-
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
1188
|
-
|
1189
|
-
|
1190
|
-
|
1191
|
-
|
1192
|
-
if (remoteCar) {
|
1193
|
-
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1194
|
-
await local.save(remoteCar);
|
1195
|
-
loadedCar = remoteCar;
|
1196
|
-
activeStore = remote;
|
1197
|
-
}
|
1198
|
-
} else {
|
1199
|
-
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1200
|
-
}
|
1201
|
-
}
|
1202
|
-
if (!loadedCar) {
|
1203
|
-
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
992
|
+
return fpvalue.fp;
|
993
|
+
}
|
994
|
+
|
995
|
+
// src/blockstore/transaction.ts
|
996
|
+
import { MemoryBlockstore } from "@web3-storage/pail/block";
|
997
|
+
import { toCryptoRuntime as toCryptoRuntime2 } from "@adviser/cement";
|
998
|
+
var CarTransaction = class extends MemoryBlockstore {
|
999
|
+
constructor(parent, opts = { add: true, noLoader: false }) {
|
1000
|
+
super();
|
1001
|
+
if (opts.add) {
|
1002
|
+
parent.transactions.add(this);
|
1204
1003
|
}
|
1205
|
-
|
1206
|
-
const rawReader = await CarReader.fromBytes(bytes.value);
|
1207
|
-
const readerP = Promise.resolve(rawReader);
|
1208
|
-
const cachedReaderP = readerP.then(async (reader) => {
|
1209
|
-
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
1210
|
-
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
1211
|
-
return;
|
1212
|
-
});
|
1213
|
-
return reader;
|
1214
|
-
});
|
1215
|
-
this.carReaders.set(cidsString, cachedReaderP);
|
1216
|
-
return readerP;
|
1004
|
+
this.parent = parent;
|
1217
1005
|
}
|
1218
|
-
|
1219
|
-
|
1220
|
-
const cidsString = cid.toString();
|
1221
|
-
let dacr = this.carReaders.get(cidsString);
|
1222
|
-
if (!dacr) {
|
1223
|
-
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
1224
|
-
this.carReaders.set(cidsString, dacr);
|
1225
|
-
}
|
1226
|
-
return dacr;
|
1006
|
+
async get(cid) {
|
1007
|
+
return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
|
1227
1008
|
}
|
1228
|
-
async
|
1229
|
-
|
1230
|
-
const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
|
1231
|
-
await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
|
1009
|
+
async superGet(cid) {
|
1010
|
+
return super.get(cid);
|
1232
1011
|
}
|
1233
1012
|
};
|
1234
|
-
|
1235
|
-
|
1236
|
-
|
1237
|
-
__export(keyed_crypto_exports, {
|
1238
|
-
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
1239
|
-
keyedCryptoFactory: () => keyedCryptoFactory
|
1240
|
-
});
|
1241
|
-
import { base58btc as base58btc2 } from "multiformats/bases/base58";
|
1242
|
-
import { sha256 as hasher4 } from "multiformats/hashes/sha2";
|
1243
|
-
import * as CBOR from "cborg";
|
1244
|
-
var generateIV = {
|
1245
|
-
random: {
|
1013
|
+
function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
|
1014
|
+
const logger = ensureLogger(sthis, component, ctx);
|
1015
|
+
return {
|
1246
1016
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1247
|
-
|
1248
|
-
return
|
1017
|
+
applyMeta: (meta, snap) => {
|
1018
|
+
return Promise.resolve();
|
1249
1019
|
},
|
1250
1020
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1251
|
-
|
1252
|
-
return
|
1021
|
+
compact: async (blocks) => {
|
1022
|
+
return {};
|
1023
|
+
},
|
1024
|
+
autoCompact: 100,
|
1025
|
+
public: false,
|
1026
|
+
// name: undefined,
|
1027
|
+
threshold: 1e3 * 1e3,
|
1028
|
+
...opts,
|
1029
|
+
logger,
|
1030
|
+
keyBag: opts.keyBag || {},
|
1031
|
+
crypto: toCryptoRuntime2(opts.crypto),
|
1032
|
+
storeUrls: opts.storeUrls,
|
1033
|
+
// storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
|
1034
|
+
// store,
|
1035
|
+
storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
|
1036
|
+
};
|
1037
|
+
}
|
1038
|
+
function blockstoreFactory(sthis, opts) {
|
1039
|
+
return new EncryptedBlockstore(sthis, opts);
|
1040
|
+
}
|
1041
|
+
var BaseBlockstore = class {
|
1042
|
+
constructor(ebOpts) {
|
1043
|
+
this.transactions = /* @__PURE__ */ new Set();
|
1044
|
+
this.sthis = ensureSuperThis(ebOpts);
|
1045
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
1046
|
+
this.sthis = ensureSuperThis(ebOpts);
|
1047
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
1048
|
+
this.logger = this.ebOpts.logger;
|
1049
|
+
}
|
1050
|
+
// readonly name?: string;
|
1051
|
+
// ready: Promise<void>;
|
1052
|
+
ready() {
|
1053
|
+
return Promise.resolve();
|
1054
|
+
}
|
1055
|
+
async close() {
|
1056
|
+
}
|
1057
|
+
async destroy() {
|
1058
|
+
}
|
1059
|
+
async compact() {
|
1060
|
+
}
|
1061
|
+
async get(cid) {
|
1062
|
+
if (!cid) throw this.logger.Error().Msg("required cid").AsError();
|
1063
|
+
for (const f of this.transactions) {
|
1064
|
+
const v = await f.superGet(cid);
|
1065
|
+
if (v) return v;
|
1253
1066
|
}
|
1254
|
-
}
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1067
|
+
}
|
1068
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1069
|
+
async put(cid, block) {
|
1070
|
+
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
1071
|
+
}
|
1072
|
+
// TransactionMeta
|
1073
|
+
async transaction(fn, _opts) {
|
1074
|
+
this.logger.Debug().Msg("enter transaction");
|
1075
|
+
const t = new CarTransaction(this, _opts);
|
1076
|
+
this.logger.Debug().Msg("post CarTransaction");
|
1077
|
+
const done = await fn(t);
|
1078
|
+
this.logger.Debug().Msg("post fn");
|
1079
|
+
this.lastTxMeta = done;
|
1080
|
+
return { t, meta: done };
|
1081
|
+
}
|
1082
|
+
openTransaction(opts = { add: true, noLoader: false }) {
|
1083
|
+
return new CarTransaction(this, opts);
|
1084
|
+
}
|
1085
|
+
async commitTransaction(t, done, opts) {
|
1086
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
|
1087
|
+
const cars = await this.loader?.commit(t, done, opts);
|
1088
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
1089
|
+
setTimeout(() => void this.compact(), 10);
|
1090
|
+
}
|
1091
|
+
if (cars) {
|
1092
|
+
this.transactions.delete(t);
|
1093
|
+
return { meta: done, cars, t };
|
1094
|
+
}
|
1095
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1096
|
+
}
|
1097
|
+
async *entries() {
|
1098
|
+
const seen = /* @__PURE__ */ new Set();
|
1099
|
+
for (const t of this.transactions) {
|
1100
|
+
for await (const blk of t.entries()) {
|
1101
|
+
if (seen.has(blk.cid.toString())) continue;
|
1102
|
+
seen.add(blk.cid.toString());
|
1103
|
+
yield blk;
|
1262
1104
|
}
|
1263
|
-
return hashArray;
|
1264
|
-
},
|
1265
|
-
verify: async function(ko, crypto, iv, data) {
|
1266
|
-
return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
1267
1105
|
}
|
1268
1106
|
}
|
1269
1107
|
};
|
1270
|
-
|
1271
|
-
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1275
|
-
|
1276
|
-
this.code = 3147065;
|
1277
|
-
this.name = "Fireproof@encrypted-block:aes-gcm";
|
1278
|
-
this.ko = ko;
|
1279
|
-
this.iv = iv;
|
1280
|
-
this.opts = opts || {};
|
1281
|
-
}
|
1282
|
-
async encode(data) {
|
1283
|
-
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
1284
|
-
const { iv } = this.ko.algo(calcIv);
|
1285
|
-
const fprt = await this.ko.fingerPrint();
|
1286
|
-
const keyId = base58btc2.decode(fprt);
|
1287
|
-
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
1288
|
-
return CBOR.encode({
|
1289
|
-
iv,
|
1290
|
-
keyId,
|
1291
|
-
data: await this.ko._encrypt({ iv, bytes: data })
|
1108
|
+
var EncryptedBlockstore = class extends BaseBlockstore {
|
1109
|
+
constructor(sthis, ebOpts) {
|
1110
|
+
super(ebOpts);
|
1111
|
+
this.compacting = false;
|
1112
|
+
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
|
1113
|
+
this: 1
|
1292
1114
|
});
|
1115
|
+
this.loader = new Loader(sthis, ebOpts);
|
1293
1116
|
}
|
1294
|
-
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1298
|
-
|
1299
|
-
|
1117
|
+
ready() {
|
1118
|
+
return this.loader.ready();
|
1119
|
+
}
|
1120
|
+
close() {
|
1121
|
+
return this.loader.close();
|
1122
|
+
}
|
1123
|
+
destroy() {
|
1124
|
+
return this.loader.destroy();
|
1125
|
+
}
|
1126
|
+
async get(cid) {
|
1127
|
+
const got = await super.get(cid);
|
1128
|
+
if (got) return got;
|
1129
|
+
if (!this.loader) {
|
1130
|
+
return;
|
1300
1131
|
}
|
1301
|
-
|
1302
|
-
|
1303
|
-
|
1304
|
-
|
1305
|
-
|
1132
|
+
return falsyToUndef(await this.loader.getBlock(cid));
|
1133
|
+
}
|
1134
|
+
async transaction(fn, opts = { noLoader: false }) {
|
1135
|
+
this.logger.Debug().Msg("enter transaction");
|
1136
|
+
const { t, meta: done } = await super.transaction(fn);
|
1137
|
+
this.logger.Debug().Msg("post super.transaction");
|
1138
|
+
const cars = await this.loader.commit(t, done, opts);
|
1139
|
+
this.logger.Debug().Msg("post this.loader.commit");
|
1140
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
1141
|
+
setTimeout(() => void this.compact(), 10);
|
1306
1142
|
}
|
1307
|
-
|
1308
|
-
|
1309
|
-
|
1143
|
+
if (cars) {
|
1144
|
+
this.transactions.delete(t);
|
1145
|
+
return { meta: done, cars, t };
|
1310
1146
|
}
|
1311
|
-
|
1147
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1312
1148
|
}
|
1313
|
-
|
1314
|
-
|
1315
|
-
|
1316
|
-
|
1317
|
-
|
1318
|
-
|
1319
|
-
|
1320
|
-
|
1321
|
-
this.
|
1149
|
+
async getFile(car, cid) {
|
1150
|
+
await this.ready();
|
1151
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
1152
|
+
const reader = await this.loader.loadFileCar(
|
1153
|
+
car
|
1154
|
+
/*, isPublic */
|
1155
|
+
);
|
1156
|
+
const block = await reader.get(cid);
|
1157
|
+
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
1158
|
+
return block.bytes;
|
1322
1159
|
}
|
1323
|
-
|
1324
|
-
|
1160
|
+
async compact() {
|
1161
|
+
await this.ready();
|
1162
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
|
1163
|
+
if (this.loader.carLog.length < 2) return;
|
1164
|
+
const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
|
1165
|
+
if (!compactFn || this.compacting) return;
|
1166
|
+
const blockLog = new CompactionFetcher(this);
|
1167
|
+
this.compacting = true;
|
1168
|
+
const meta = await compactFn(blockLog);
|
1169
|
+
await this.loader?.commit(blockLog.loggedBlocks, meta, {
|
1170
|
+
compact: true,
|
1171
|
+
noLoader: true
|
1172
|
+
});
|
1173
|
+
this.compacting = false;
|
1325
1174
|
}
|
1326
|
-
|
1327
|
-
|
1175
|
+
async defaultCompact(blocks, logger) {
|
1176
|
+
if (!this.loader) {
|
1177
|
+
throw logger.Error().Msg("no loader").AsError();
|
1178
|
+
}
|
1179
|
+
if (!this.lastTxMeta) {
|
1180
|
+
throw logger.Error().Msg("no lastTxMeta").AsError();
|
1181
|
+
}
|
1182
|
+
for await (const blk of this.loader.entries(false)) {
|
1183
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
1184
|
+
}
|
1185
|
+
for (const t of this.transactions) {
|
1186
|
+
for await (const blk of t.entries()) {
|
1187
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
1188
|
+
}
|
1189
|
+
}
|
1190
|
+
return this.lastTxMeta;
|
1328
1191
|
}
|
1329
|
-
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
tagLength: 128
|
1334
|
-
};
|
1192
|
+
async *entries() {
|
1193
|
+
for await (const blk of this.loader.entries()) {
|
1194
|
+
yield blk;
|
1195
|
+
}
|
1335
1196
|
}
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1197
|
+
};
|
1198
|
+
var CompactionFetcher = class {
|
1199
|
+
constructor(blocks) {
|
1200
|
+
this.blockstore = blocks;
|
1201
|
+
this.loggedBlocks = new CarTransaction(blocks);
|
1339
1202
|
}
|
1340
|
-
async
|
1341
|
-
|
1342
|
-
|
1343
|
-
return
|
1203
|
+
async get(cid) {
|
1204
|
+
const block = await this.blockstore.get(cid);
|
1205
|
+
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
1206
|
+
return falsyToUndef(block);
|
1344
1207
|
}
|
1345
1208
|
};
|
1346
|
-
|
1347
|
-
|
1348
|
-
|
1349
|
-
|
1209
|
+
|
1210
|
+
// src/blockstore/commitor.ts
|
1211
|
+
import * as CBW from "@ipld/car/buffer-writer";
|
1212
|
+
import { sha256 as hasher3 } from "multiformats/hashes/sha2";
|
1213
|
+
import * as dagCodec2 from "@ipld/dag-cbor";
|
1214
|
+
async function encodeCarFile(roots, t, codec3) {
|
1215
|
+
let size = 0;
|
1216
|
+
const headerSize = CBW.headerLength({ roots });
|
1217
|
+
size += headerSize;
|
1218
|
+
for (const { cid, bytes } of t.entries()) {
|
1219
|
+
size += CBW.blockLength({ cid, bytes });
|
1350
1220
|
}
|
1351
|
-
|
1352
|
-
|
1221
|
+
const buffer = new Uint8Array(size);
|
1222
|
+
const writer = CBW.createWriter(buffer, { headerSize });
|
1223
|
+
for (const r of roots) {
|
1224
|
+
writer.addRoot(r);
|
1353
1225
|
}
|
1354
|
-
|
1355
|
-
|
1226
|
+
for (const { cid, bytes } of t.entries()) {
|
1227
|
+
writer.write({ cid, bytes });
|
1356
1228
|
}
|
1357
|
-
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
|
1364
|
-
|
1365
|
-
|
1366
|
-
|
1367
|
-
|
1229
|
+
writer.close();
|
1230
|
+
return await encode3({ value: writer.bytes, hasher: hasher3, codec: codec3 });
|
1231
|
+
}
|
1232
|
+
async function createCarFile(encoder, cid, t) {
|
1233
|
+
return encodeCarFile([cid], t, encoder);
|
1234
|
+
}
|
1235
|
+
async function commitFiles(fileStore, walStore, t, done) {
|
1236
|
+
const { files: roots } = makeFileCarHeader(done);
|
1237
|
+
const cids = [];
|
1238
|
+
const codec3 = (await fileStore.keyedCrypto()).codec();
|
1239
|
+
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
1240
|
+
for (const car of cars) {
|
1241
|
+
const { cid, bytes } = car;
|
1242
|
+
await fileStore.save({ cid, bytes });
|
1243
|
+
await walStore.enqueueFile(
|
1244
|
+
cid
|
1245
|
+
/*, !!opts.public*/
|
1246
|
+
);
|
1247
|
+
cids.push(cid);
|
1368
1248
|
}
|
1369
|
-
|
1370
|
-
|
1249
|
+
return cids;
|
1250
|
+
}
|
1251
|
+
function makeFileCarHeader(result) {
|
1252
|
+
const files = [];
|
1253
|
+
for (const [, meta] of Object.entries(result.files || {})) {
|
1254
|
+
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
1255
|
+
files.push(meta.cid);
|
1256
|
+
}
|
1371
1257
|
}
|
1372
|
-
|
1373
|
-
|
1374
|
-
|
1258
|
+
return { ...result, files };
|
1259
|
+
}
|
1260
|
+
async function prepareCarFilesFiles(encoder, roots, t) {
|
1261
|
+
return [await encodeCarFile(roots, t, encoder)];
|
1262
|
+
}
|
1263
|
+
function makeCarHeader(meta, cars, compact = false) {
|
1264
|
+
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
1265
|
+
return { ...coreHeader, meta };
|
1266
|
+
}
|
1267
|
+
async function encodeCarHeader(fp) {
|
1268
|
+
return await encode3({
|
1269
|
+
value: { fp },
|
1270
|
+
hasher: hasher3,
|
1271
|
+
codec: dagCodec2
|
1272
|
+
});
|
1273
|
+
}
|
1274
|
+
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
1275
|
+
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
1276
|
+
const rootBlock = await encodeCarHeader(fp);
|
1277
|
+
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
1278
|
+
const cids = [];
|
1279
|
+
for (const car of cars) {
|
1280
|
+
const { cid, bytes } = car;
|
1281
|
+
await params.carStore.save({ cid, bytes });
|
1282
|
+
cids.push(cid);
|
1375
1283
|
}
|
1376
|
-
|
1377
|
-
|
1378
|
-
|
1379
|
-
|
1380
|
-
|
1381
|
-
|
1382
|
-
|
1284
|
+
const newDbMeta = { cars: cids };
|
1285
|
+
await params.WALStore.enqueue(newDbMeta, opts);
|
1286
|
+
await params.metaStore.save(newDbMeta);
|
1287
|
+
return { cgrp: cids, header: fp };
|
1288
|
+
}
|
1289
|
+
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
1290
|
+
const carFiles = [];
|
1291
|
+
threshold = threshold || 128e3 * 8;
|
1292
|
+
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1293
|
+
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1294
|
+
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
1295
|
+
let cidRootBlock = rootBlock;
|
1296
|
+
for (const { cid, bytes } of t.entries()) {
|
1297
|
+
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1298
|
+
if (newsize >= threshold) {
|
1299
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1300
|
+
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1301
|
+
clonedt.putSync(cid, bytes);
|
1302
|
+
cidRootBlock = { cid, bytes };
|
1303
|
+
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1304
|
+
} else {
|
1305
|
+
clonedt.putSync(cid, bytes);
|
1306
|
+
}
|
1383
1307
|
}
|
1384
|
-
|
1385
|
-
|
1308
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1309
|
+
return carFiles;
|
1310
|
+
}
|
1311
|
+
|
1312
|
+
// src/blockstore/loader.ts
|
1313
|
+
import { sha256 as hasher4 } from "multiformats/hashes/sha2";
|
1314
|
+
|
1315
|
+
// src/blockstore/task-manager.ts
|
1316
|
+
var TaskManager = class {
|
1317
|
+
constructor(sthis, callback) {
|
1318
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
1319
|
+
this.queue = [];
|
1320
|
+
this.isProcessing = false;
|
1321
|
+
this.logger = ensureLogger(sthis, "TaskManager");
|
1322
|
+
this.callback = callback;
|
1386
1323
|
}
|
1387
|
-
|
1388
|
-
|
1324
|
+
async handleEvent(cid, parents, dbMeta) {
|
1325
|
+
for (const parent of parents) {
|
1326
|
+
this.eventsWeHandled.add(parent.toString());
|
1327
|
+
}
|
1328
|
+
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
1329
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
1330
|
+
void this.processQueue();
|
1389
1331
|
}
|
1390
|
-
|
1391
|
-
|
1392
|
-
|
1393
|
-
|
1394
|
-
|
1395
|
-
if (
|
1396
|
-
|
1397
|
-
|
1398
|
-
|
1399
|
-
|
1332
|
+
async processQueue() {
|
1333
|
+
if (this.isProcessing) return;
|
1334
|
+
this.isProcessing = true;
|
1335
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1336
|
+
const first = filteredQueue[0];
|
1337
|
+
if (!first) {
|
1338
|
+
return;
|
1339
|
+
}
|
1340
|
+
try {
|
1341
|
+
await this.callback(first.dbMeta);
|
1342
|
+
this.eventsWeHandled.add(first.cid);
|
1343
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1344
|
+
} catch (err) {
|
1345
|
+
if (first.retries++ > 3) {
|
1346
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
1347
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
1348
|
+
}
|
1349
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
1350
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
1351
|
+
} finally {
|
1352
|
+
this.isProcessing = false;
|
1353
|
+
if (this.queue.length > 0) {
|
1354
|
+
void this.processQueue();
|
1400
1355
|
}
|
1401
1356
|
}
|
1402
|
-
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
1403
1357
|
}
|
1404
|
-
|
1405
|
-
}
|
1358
|
+
};
|
1406
1359
|
|
1407
|
-
// src/blockstore/
|
1408
|
-
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
|
1416
|
-
|
1417
|
-
|
1418
|
-
ret = 0;
|
1360
|
+
// src/blockstore/loader.ts
|
1361
|
+
function carLogIncludesGroup(list, cids) {
|
1362
|
+
return list.some((arr) => {
|
1363
|
+
return arr.toString() === cids.toString();
|
1364
|
+
});
|
1365
|
+
}
|
1366
|
+
function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
1367
|
+
const byString = /* @__PURE__ */ new Map();
|
1368
|
+
for (const cid of list) {
|
1369
|
+
if (remove.has(cid.toString())) continue;
|
1370
|
+
byString.set(cid.toString(), cid);
|
1419
1371
|
}
|
1420
|
-
return
|
1372
|
+
return [...byString.values()];
|
1421
1373
|
}
|
1422
|
-
|
1423
|
-
|
1424
|
-
|
1425
|
-
|
1426
|
-
|
1427
|
-
|
1428
|
-
|
1429
|
-
|
1430
|
-
|
1431
|
-
|
1432
|
-
|
1433
|
-
|
1434
|
-
|
1435
|
-
|
1374
|
+
var Loader = class {
|
1375
|
+
constructor(sthis, ebOpts) {
|
1376
|
+
this.commitQueue = new CommitQueue();
|
1377
|
+
this.isCompacting = false;
|
1378
|
+
this.carReaders = /* @__PURE__ */ new Map();
|
1379
|
+
this.seenCompacted = /* @__PURE__ */ new Set();
|
1380
|
+
this.processedCars = /* @__PURE__ */ new Set();
|
1381
|
+
this.carLog = [];
|
1382
|
+
this.getBlockCache = /* @__PURE__ */ new Map();
|
1383
|
+
this.seenMeta = /* @__PURE__ */ new Set();
|
1384
|
+
this.writeLimit = pLimit(1);
|
1385
|
+
this._carStore = new ResolveOnce2();
|
1386
|
+
this._fileStore = new ResolveOnce2();
|
1387
|
+
this._WALStore = new ResolveOnce2();
|
1388
|
+
this._metaStore = new ResolveOnce2();
|
1389
|
+
this.onceReady = new ResolveOnce2();
|
1390
|
+
this.sthis = sthis;
|
1391
|
+
this.ebOpts = defaultedBlockstoreRuntime(
|
1392
|
+
sthis,
|
1393
|
+
{
|
1394
|
+
...ebOpts
|
1395
|
+
// name,
|
1396
|
+
},
|
1397
|
+
"Loader"
|
1398
|
+
);
|
1399
|
+
this.logger = this.ebOpts.logger;
|
1400
|
+
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
1401
|
+
await this.handleDbMetasFromStore([dbMeta]);
|
1402
|
+
});
|
1403
|
+
}
|
1404
|
+
async carStore() {
|
1405
|
+
return this._carStore.once(
|
1406
|
+
async () => this.ebOpts.storeRuntime.makeDataStore({
|
1407
|
+
sthis: this.sthis,
|
1408
|
+
url: this.ebOpts.storeUrls.data,
|
1409
|
+
keybag: await this.keyBag()
|
1436
1410
|
})
|
1437
|
-
|
1411
|
+
);
|
1438
1412
|
}
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1413
|
+
async fileStore() {
|
1414
|
+
return this._fileStore.once(
|
1415
|
+
async () => this.ebOpts.storeRuntime.makeDataStore({
|
1416
|
+
sthis: this.sthis,
|
1417
|
+
url: this.ebOpts.storeUrls.file,
|
1418
|
+
keybag: await this.keyBag()
|
1419
|
+
})
|
1420
|
+
);
|
1442
1421
|
}
|
1443
|
-
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1448
|
-
|
1449
|
-
|
1450
|
-
(async (furl, ofs2) => {
|
1451
|
-
const raw2 = await innerGW.get(furl);
|
1452
|
-
if (raw2.isErr()) {
|
1453
|
-
return raw2;
|
1454
|
-
}
|
1455
|
-
const fragment = decode3(raw2.unwrap());
|
1456
|
-
if (base58btc3.encode(fragment.fid) !== fidStr) {
|
1457
|
-
return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
1458
|
-
}
|
1459
|
-
if (fragment.ofs !== ofs2) {
|
1460
|
-
return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
1461
|
-
}
|
1462
|
-
return Result3.Ok(fragment);
|
1463
|
-
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
1422
|
+
async WALStore() {
|
1423
|
+
return this._WALStore.once(
|
1424
|
+
async () => this.ebOpts.storeRuntime.makeWALStore({
|
1425
|
+
sthis: this.sthis,
|
1426
|
+
url: this.ebOpts.storeUrls.wal,
|
1427
|
+
keybag: await this.keyBag()
|
1428
|
+
})
|
1464
1429
|
);
|
1465
1430
|
}
|
1466
|
-
|
1467
|
-
|
1468
|
-
|
1469
|
-
|
1470
|
-
|
1471
|
-
|
1472
|
-
|
1473
|
-
|
1474
|
-
this.innerGW = innerGW;
|
1431
|
+
async metaStore() {
|
1432
|
+
return this._metaStore.once(
|
1433
|
+
async () => this.ebOpts.storeRuntime.makeMetaStore({
|
1434
|
+
sthis: this.sthis,
|
1435
|
+
url: this.ebOpts.storeUrls.meta,
|
1436
|
+
keybag: await this.keyBag()
|
1437
|
+
})
|
1438
|
+
);
|
1475
1439
|
}
|
1476
|
-
|
1477
|
-
|
1478
|
-
|
1479
|
-
|
1440
|
+
keyBag() {
|
1441
|
+
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
1442
|
+
}
|
1443
|
+
async ready() {
|
1444
|
+
return this.onceReady.once(async () => {
|
1445
|
+
const metas = await (await this.metaStore()).load();
|
1446
|
+
if (this.ebOpts.meta) {
|
1447
|
+
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
1448
|
+
} else if (metas) {
|
1449
|
+
await this.handleDbMetasFromStore(metas);
|
1450
|
+
}
|
1451
|
+
});
|
1452
|
+
}
|
1453
|
+
async close() {
|
1454
|
+
await this.commitQueue.waitIdle();
|
1455
|
+
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1456
|
+
await Promise.all(toClose.map((store) => store.close()));
|
1457
|
+
}
|
1458
|
+
async destroy() {
|
1459
|
+
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1460
|
+
await Promise.all(toDestroy.map((store) => store.destroy()));
|
1461
|
+
}
|
1462
|
+
// async snapToCar(carCid: AnyLink | string) {
|
1463
|
+
// await this.ready
|
1464
|
+
// if (typeof carCid === 'string') {
|
1465
|
+
// carCid = CID.parse(carCid)
|
1466
|
+
// }
|
1467
|
+
// const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
|
1468
|
+
// this.carLog = [carCid, ...carHeader.cars]
|
1469
|
+
// await this.getMoreReaders(carHeader.cars)
|
1470
|
+
// await this._applyCarHeader(carHeader, true)
|
1471
|
+
// }
|
1472
|
+
async handleDbMetasFromStore(metas) {
|
1473
|
+
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
1474
|
+
for (const meta of metas) {
|
1475
|
+
await this.writeLimit(async () => {
|
1476
|
+
await this.mergeDbMetaIntoClock(meta);
|
1477
|
+
});
|
1480
1478
|
}
|
1481
|
-
|
1482
|
-
|
1483
|
-
|
1479
|
+
}
|
1480
|
+
async mergeDbMetaIntoClock(meta) {
|
1481
|
+
if (this.isCompacting) {
|
1482
|
+
throw this.logger.Error().Msg("cannot merge while compacting").AsError();
|
1484
1483
|
}
|
1485
|
-
|
1486
|
-
|
1487
|
-
|
1488
|
-
|
1489
|
-
const block = encode3({
|
1490
|
-
fid: fid.bin,
|
1491
|
-
ofs,
|
1492
|
-
len: body.length,
|
1493
|
-
data: body.slice(ofs, ofs + blocksize)
|
1494
|
-
});
|
1495
|
-
if (block.length > fragSize) {
|
1496
|
-
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
1497
|
-
}
|
1498
|
-
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
1484
|
+
if (this.seenMeta.has(meta.cars.toString())) return;
|
1485
|
+
this.seenMeta.add(meta.cars.toString());
|
1486
|
+
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
1487
|
+
return;
|
1499
1488
|
}
|
1500
|
-
|
1489
|
+
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
1490
|
+
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1491
|
+
await this.getMoreReaders(carHeader.cars.flat());
|
1492
|
+
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
1493
|
+
await this.ebOpts.applyMeta?.(carHeader.meta);
|
1501
1494
|
}
|
1502
|
-
|
1503
|
-
|
1495
|
+
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1496
|
+
// const { key } = meta;
|
1497
|
+
// if (key) {
|
1498
|
+
// await this.setKey(key);
|
1499
|
+
// }
|
1500
|
+
// }
|
1501
|
+
async loadCarHeaderFromMeta({ cars: cids }) {
|
1502
|
+
const reader = await this.loadCar(cids[0]);
|
1503
|
+
return await parseCarFile(reader, this.logger);
|
1504
1504
|
}
|
1505
|
-
async
|
1506
|
-
|
1505
|
+
// async _getKey(): Promise<string | undefined> {
|
1506
|
+
// if (this.key) return this.key;
|
1507
|
+
// // generate a random key
|
1508
|
+
// if (!this.ebOpts.public) {
|
1509
|
+
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1510
|
+
// }
|
1511
|
+
// return this.key || undefined;
|
1512
|
+
// }
|
1513
|
+
async commitFiles(t, done) {
|
1514
|
+
await this.ready();
|
1515
|
+
const fstore = await this.fileStore();
|
1516
|
+
const wstore = await this.WALStore();
|
1517
|
+
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1507
1518
|
}
|
1508
|
-
async
|
1509
|
-
this.
|
1510
|
-
fid: this.sthis.nextId(this.fidLength).bin,
|
1511
|
-
ofs: 1024 * 1024,
|
1512
|
-
// 32bit
|
1513
|
-
len: 16 * 1024 * 1024,
|
1514
|
-
// 32bit
|
1515
|
-
data: new Uint8Array(1024)
|
1516
|
-
}).length - 1024;
|
1517
|
-
return this.innerGW.start(url);
|
1519
|
+
async loadFileCar(cid) {
|
1520
|
+
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
1518
1521
|
}
|
1519
|
-
async
|
1520
|
-
|
1522
|
+
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
1523
|
+
await this.ready();
|
1524
|
+
const carStore = await this.carStore();
|
1525
|
+
const params = {
|
1526
|
+
encoder: (await carStore.keyedCrypto()).codec(),
|
1527
|
+
carLog: this.carLog,
|
1528
|
+
carStore,
|
1529
|
+
WALStore: await this.WALStore(),
|
1530
|
+
metaStore: await this.metaStore(),
|
1531
|
+
threshold: this.ebOpts.threshold
|
1532
|
+
};
|
1533
|
+
return this.commitQueue.enqueue(async () => {
|
1534
|
+
await this.cacheTransaction(t);
|
1535
|
+
const ret = await commit(params, t, done, opts);
|
1536
|
+
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1537
|
+
return ret.cgrp;
|
1538
|
+
});
|
1539
|
+
}
|
1540
|
+
async updateCarLog(cids, fp, compact) {
|
1541
|
+
if (compact) {
|
1542
|
+
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1543
|
+
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1544
|
+
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1545
|
+
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1546
|
+
} else {
|
1547
|
+
this.carLog.unshift(cids);
|
1548
|
+
}
|
1549
|
+
}
|
1550
|
+
async cacheTransaction(t) {
|
1551
|
+
for await (const block of t.entries()) {
|
1552
|
+
const sBlock = block.cid.toString();
|
1553
|
+
if (!this.getBlockCache.has(sBlock)) {
|
1554
|
+
this.getBlockCache.set(sBlock, block);
|
1555
|
+
}
|
1556
|
+
}
|
1521
1557
|
}
|
1522
|
-
async
|
1523
|
-
|
1524
|
-
|
1558
|
+
async cacheCarReader(carCidStr, reader) {
|
1559
|
+
if (this.processedCars.has(carCidStr)) return;
|
1560
|
+
this.processedCars.add(carCidStr);
|
1561
|
+
for await (const block of reader.blocks()) {
|
1562
|
+
const sBlock = block.cid.toString();
|
1563
|
+
if (!this.getBlockCache.has(sBlock)) {
|
1564
|
+
this.getBlockCache.set(sBlock, block);
|
1565
|
+
}
|
1566
|
+
}
|
1525
1567
|
}
|
1526
|
-
async
|
1527
|
-
const
|
1528
|
-
|
1529
|
-
|
1530
|
-
|
1531
|
-
|
1568
|
+
async removeCidsForCompact(cid) {
|
1569
|
+
const carHeader = await this.loadCarHeaderFromMeta({
|
1570
|
+
cars: [cid]
|
1571
|
+
});
|
1572
|
+
for (const cids of carHeader.compact) {
|
1573
|
+
for (const cid2 of cids) {
|
1574
|
+
await (await this.carStore()).remove(cid2);
|
1532
1575
|
}
|
1533
|
-
const frag = rfrag.Ok();
|
1534
|
-
buffer = buffer || new Uint8Array(frag.len);
|
1535
|
-
buffer.set(frag.data, frag.ofs);
|
1536
1576
|
}
|
1537
|
-
return Result3.Ok(buffer || new Uint8Array(0));
|
1538
1577
|
}
|
1539
|
-
async
|
1540
|
-
|
1541
|
-
|
1578
|
+
// async flushCars() {
|
1579
|
+
// await this.ready
|
1580
|
+
// // for each cid in car log, make a dbMeta
|
1581
|
+
// for (const cid of this.carLog) {
|
1582
|
+
// const dbMeta = { car: cid, key: this.key || null } as DbMeta
|
1583
|
+
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
1584
|
+
// }
|
1585
|
+
// }
|
1586
|
+
async *entries(cache2 = true) {
|
1587
|
+
await this.ready();
|
1588
|
+
if (cache2) {
|
1589
|
+
for (const [, block] of this.getBlockCache) {
|
1590
|
+
yield block;
|
1591
|
+
}
|
1542
1592
|
} else {
|
1543
|
-
|
1593
|
+
for (const [, block] of this.getBlockCache) {
|
1594
|
+
yield block;
|
1595
|
+
}
|
1596
|
+
for (const cids of this.carLog) {
|
1597
|
+
for (const cid of cids) {
|
1598
|
+
const reader = await this.loadCar(cid);
|
1599
|
+
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
1600
|
+
for await (const block of reader.blocks()) {
|
1601
|
+
const sCid = block.cid.toString();
|
1602
|
+
if (!this.getBlockCache.has(sCid)) {
|
1603
|
+
yield block;
|
1604
|
+
}
|
1605
|
+
}
|
1606
|
+
}
|
1607
|
+
}
|
1544
1608
|
}
|
1545
1609
|
}
|
1546
|
-
async
|
1547
|
-
|
1548
|
-
|
1549
|
-
|
1550
|
-
|
1610
|
+
async getBlock(cid) {
|
1611
|
+
await this.ready();
|
1612
|
+
const sCid = cid.toString();
|
1613
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1614
|
+
const getCarCid = async (carCid) => {
|
1615
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1616
|
+
const reader = await this.loadCar(carCid);
|
1617
|
+
if (!reader) {
|
1618
|
+
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
1551
1619
|
}
|
1552
|
-
|
1553
|
-
|
1554
|
-
|
1555
|
-
|
1620
|
+
await this.cacheCarReader(carCid.toString(), reader).catch(() => {
|
1621
|
+
return;
|
1622
|
+
});
|
1623
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1624
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
|
1625
|
+
};
|
1626
|
+
const getCompactCarCids = async (carCid) => {
|
1627
|
+
const reader = await this.loadCar(carCid);
|
1628
|
+
if (!reader) {
|
1629
|
+
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
1630
|
+
}
|
1631
|
+
const header = await parseCarFile(reader, this.logger);
|
1632
|
+
const compacts = header.compact;
|
1633
|
+
let got2;
|
1634
|
+
const batchSize2 = 5;
|
1635
|
+
for (let i = 0; i < compacts.length; i += batchSize2) {
|
1636
|
+
const promises = [];
|
1637
|
+
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
1638
|
+
for (const cid2 of compacts[j]) {
|
1639
|
+
promises.push(getCarCid(cid2));
|
1640
|
+
}
|
1641
|
+
}
|
1642
|
+
try {
|
1643
|
+
got2 = await Promise.any(promises);
|
1644
|
+
} catch {
|
1645
|
+
}
|
1646
|
+
if (got2) break;
|
1647
|
+
}
|
1648
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1649
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
1650
|
+
};
|
1651
|
+
let got;
|
1652
|
+
const batchSize = 5;
|
1653
|
+
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
1654
|
+
const batch = this.carLog.slice(i, i + batchSize);
|
1655
|
+
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
1656
|
+
try {
|
1657
|
+
got = await Promise.any(promises);
|
1658
|
+
} catch {
|
1659
|
+
}
|
1660
|
+
if (got) break;
|
1556
1661
|
}
|
1557
|
-
|
1558
|
-
|
1559
|
-
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1563
|
-
|
1564
|
-
import { CID as CID2 } from "multiformats";
|
1565
|
-
import { base64pad } from "multiformats/bases/base64";
|
1566
|
-
import { Result as Result4 } from "@adviser/cement";
|
1567
|
-
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
1568
|
-
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
1569
|
-
if (!crdtEntries.length) {
|
1570
|
-
sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
|
1571
|
-
return [];
|
1662
|
+
if (!got) {
|
1663
|
+
try {
|
1664
|
+
got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
|
1665
|
+
} catch {
|
1666
|
+
}
|
1667
|
+
}
|
1668
|
+
return got;
|
1572
1669
|
}
|
1573
|
-
|
1574
|
-
|
1575
|
-
|
1670
|
+
async loadCar(cid) {
|
1671
|
+
if (!this.carStore) {
|
1672
|
+
throw this.logger.Error().Msg("car store not initialized").AsError();
|
1673
|
+
}
|
1674
|
+
const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
|
1675
|
+
return loaded;
|
1576
1676
|
}
|
1577
|
-
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
|
1582
|
-
|
1583
|
-
|
1584
|
-
|
1585
|
-
|
1586
|
-
|
1587
|
-
|
1588
|
-
|
1589
|
-
|
1590
|
-
|
1591
|
-
|
1592
|
-
|
1593
|
-
if (keyInfo.length) {
|
1594
|
-
const dbMeta = keyInfo[0].dbMeta;
|
1595
|
-
if (dbMeta.key) {
|
1596
|
-
const kb = await getKeyBag(sthis);
|
1597
|
-
const keyName = getStoreKeyName(uri);
|
1598
|
-
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
1599
|
-
if (res.isErr()) {
|
1600
|
-
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
1601
|
-
throw res.Err();
|
1677
|
+
async makeDecoderAndCarReader(cid, local, remote) {
|
1678
|
+
const cidsString = cid.toString();
|
1679
|
+
let loadedCar = void 0;
|
1680
|
+
let activeStore = local;
|
1681
|
+
try {
|
1682
|
+
this.logger.Debug().Str("cid", cidsString).Msg("loading car");
|
1683
|
+
loadedCar = await local.load(cid);
|
1684
|
+
this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
|
1685
|
+
} catch (e) {
|
1686
|
+
if (remote) {
|
1687
|
+
const remoteCar = await remote.load(cid);
|
1688
|
+
if (remoteCar) {
|
1689
|
+
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1690
|
+
await local.save(remoteCar);
|
1691
|
+
loadedCar = remoteCar;
|
1692
|
+
activeStore = remote;
|
1602
1693
|
}
|
1694
|
+
} else {
|
1695
|
+
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1603
1696
|
}
|
1604
|
-
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
1605
|
-
return Result4.Ok(dbMeta);
|
1606
1697
|
}
|
1607
|
-
|
1608
|
-
|
1609
|
-
} catch (error) {
|
1610
|
-
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
1611
|
-
return Result4.Err(error);
|
1612
|
-
}
|
1613
|
-
}
|
1614
|
-
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
1615
|
-
try {
|
1616
|
-
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
1617
|
-
const keyName = getStoreKeyName(uri);
|
1618
|
-
const kb = await getKeyBag(sthis);
|
1619
|
-
const res = await kb.getNamedExtractableKey(keyName, true);
|
1620
|
-
if (res.isErr()) {
|
1621
|
-
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
1622
|
-
throw res.Err();
|
1698
|
+
if (!loadedCar) {
|
1699
|
+
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
1623
1700
|
}
|
1624
|
-
const
|
1625
|
-
const
|
1626
|
-
const
|
1627
|
-
const
|
1628
|
-
|
1629
|
-
|
1630
|
-
|
1631
|
-
|
1632
|
-
|
1633
|
-
|
1634
|
-
|
1635
|
-
return
|
1636
|
-
}
|
1637
|
-
|
1638
|
-
|
1639
|
-
|
1640
|
-
|
1641
|
-
|
1642
|
-
|
1643
|
-
|
1644
|
-
|
1645
|
-
|
1646
|
-
}
|
1647
|
-
async
|
1648
|
-
|
1649
|
-
|
1650
|
-
|
1651
|
-
|
1652
|
-
|
1653
|
-
);
|
1654
|
-
return event;
|
1655
|
-
}
|
1656
|
-
async function encodeEventsWithParents(sthis, events, parents) {
|
1657
|
-
const crdtEntries = events.map((event) => {
|
1658
|
-
const base64String = base64pad.encode(event.bytes);
|
1659
|
-
return {
|
1660
|
-
cid: event.cid.toString(),
|
1661
|
-
data: base64String,
|
1662
|
-
parents: parents.map((p) => p.toString())
|
1663
|
-
};
|
1664
|
-
});
|
1665
|
-
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
1666
|
-
}
|
1701
|
+
const bytes = await decode3({ bytes: loadedCar.bytes, hasher: hasher4, codec: (await activeStore.keyedCrypto()).codec() });
|
1702
|
+
const rawReader = await CarReader.fromBytes(bytes.value);
|
1703
|
+
const readerP = Promise.resolve(rawReader);
|
1704
|
+
const cachedReaderP = readerP.then(async (reader) => {
|
1705
|
+
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
1706
|
+
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
1707
|
+
return;
|
1708
|
+
});
|
1709
|
+
return reader;
|
1710
|
+
});
|
1711
|
+
this.carReaders.set(cidsString, cachedReaderP);
|
1712
|
+
return readerP;
|
1713
|
+
}
|
1714
|
+
//What if instead it returns an Array of CarHeader
|
1715
|
+
async storesLoadCar(cid, local, remote) {
|
1716
|
+
const cidsString = cid.toString();
|
1717
|
+
let dacr = this.carReaders.get(cidsString);
|
1718
|
+
if (!dacr) {
|
1719
|
+
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
1720
|
+
this.carReaders.set(cidsString, dacr);
|
1721
|
+
}
|
1722
|
+
return dacr;
|
1723
|
+
}
|
1724
|
+
async getMoreReaders(cids) {
|
1725
|
+
const limit = pLimit(5);
|
1726
|
+
const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
|
1727
|
+
await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
|
1728
|
+
}
|
1729
|
+
};
|
1667
1730
|
|
1668
1731
|
// src/blockstore/store.ts
|
1669
|
-
import pRetry from "p-retry";
|
1670
|
-
import pMap from "p-map";
|
1671
1732
|
function guardVersion(url) {
|
1672
1733
|
if (!url.hasParam("version")) {
|
1673
1734
|
return Result5.Err(`missing version: ${url.toString()}`);
|
@@ -1675,16 +1736,21 @@ function guardVersion(url) {
|
|
1675
1736
|
return Result5.Ok(url);
|
1676
1737
|
}
|
1677
1738
|
var BaseStoreImpl = class {
|
1678
|
-
constructor(
|
1739
|
+
constructor(sthis, url, opts, logger) {
|
1679
1740
|
this._onStarted = [];
|
1680
1741
|
this._onClosed = [];
|
1681
|
-
this.name = name;
|
1682
1742
|
this._url = url;
|
1683
1743
|
this.keybag = opts.keybag;
|
1744
|
+
this.loader = opts.loader;
|
1684
1745
|
this.sthis = sthis;
|
1685
|
-
|
1746
|
+
const name = this._url.getParam("name" /* NAME */);
|
1747
|
+
if (!name) {
|
1748
|
+
throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
|
1749
|
+
}
|
1750
|
+
this.name = name;
|
1751
|
+
this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
|
1752
|
+
this.realGateway = opts.gateway;
|
1686
1753
|
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
1687
|
-
this.loader = opts.loader;
|
1688
1754
|
}
|
1689
1755
|
url() {
|
1690
1756
|
return this._url;
|
@@ -1699,20 +1765,20 @@ var BaseStoreImpl = class {
|
|
1699
1765
|
return;
|
1700
1766
|
}
|
1701
1767
|
async keyedCrypto() {
|
1702
|
-
return keyedCryptoFactory(this._url,
|
1768
|
+
return keyedCryptoFactory(this._url, this.keybag, this.sthis);
|
1703
1769
|
}
|
1704
1770
|
async start() {
|
1705
1771
|
this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
|
1706
|
-
this._url = this._url.build().setParam("store"
|
1772
|
+
this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
|
1707
1773
|
const res = await this.gateway.start(this._url);
|
1708
1774
|
if (res.isErr()) {
|
1709
1775
|
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
1710
1776
|
return res;
|
1711
1777
|
}
|
1712
1778
|
this._url = res.Ok();
|
1713
|
-
const kb = await this.keybag
|
1779
|
+
const kb = await this.keybag;
|
1714
1780
|
const skRes = await kb.ensureKeyFromUrl(this._url, () => {
|
1715
|
-
const idx = this._url.getParam("index");
|
1781
|
+
const idx = this._url.getParam("index" /* INDEX */);
|
1716
1782
|
const storeKeyName = [this.name];
|
1717
1783
|
if (idx) {
|
1718
1784
|
storeKeyName.push(idx);
|
@@ -1745,8 +1811,8 @@ var BaseStoreImpl = class {
|
|
1745
1811
|
};
|
1746
1812
|
var MetaStoreImpl = class extends BaseStoreImpl {
|
1747
1813
|
// remote: boolean;
|
1748
|
-
constructor(sthis,
|
1749
|
-
super(
|
1814
|
+
constructor(sthis, url, opts) {
|
1815
|
+
super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
|
1750
1816
|
this.storeType = "meta";
|
1751
1817
|
this.subscribers = /* @__PURE__ */ new Map();
|
1752
1818
|
this.parents = [];
|
@@ -1817,13 +1883,21 @@ var MetaStoreImpl = class extends BaseStoreImpl {
|
|
1817
1883
|
return Result5.Ok(void 0);
|
1818
1884
|
}
|
1819
1885
|
async destroy() {
|
1886
|
+
this.logger.Debug().Msg("destroy");
|
1820
1887
|
return this.gateway.destroy(this.url());
|
1821
1888
|
}
|
1822
1889
|
};
|
1823
1890
|
var DataStoreImpl = class extends BaseStoreImpl {
|
1824
1891
|
// readonly tag: string = "car-base";
|
1825
|
-
constructor(sthis,
|
1826
|
-
super(
|
1892
|
+
constructor(sthis, url, opts) {
|
1893
|
+
super(
|
1894
|
+
sthis,
|
1895
|
+
url,
|
1896
|
+
{
|
1897
|
+
...opts
|
1898
|
+
},
|
1899
|
+
ensureLogger(sthis, "DataStoreImpl")
|
1900
|
+
);
|
1827
1901
|
this.storeType = "data";
|
1828
1902
|
}
|
1829
1903
|
async load(cid) {
|
@@ -1864,23 +1938,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
|
|
1864
1938
|
return Result5.Ok(void 0);
|
1865
1939
|
}
|
1866
1940
|
destroy() {
|
1941
|
+
this.logger.Debug().Msg("destroy");
|
1867
1942
|
return this.gateway.destroy(this.url());
|
1868
1943
|
}
|
1869
1944
|
};
|
1870
1945
|
var WALStoreImpl = class extends BaseStoreImpl {
|
1871
|
-
constructor(
|
1872
|
-
super(
|
1946
|
+
constructor(sthis, url, opts) {
|
1947
|
+
super(
|
1948
|
+
sthis,
|
1949
|
+
url,
|
1950
|
+
{
|
1951
|
+
...opts
|
1952
|
+
},
|
1953
|
+
ensureLogger(sthis, "WALStoreImpl")
|
1954
|
+
);
|
1873
1955
|
this.storeType = "wal";
|
1956
|
+
// readonly tag: string = "rwal-base";
|
1957
|
+
// readonly loader: Loadable;
|
1874
1958
|
this._ready = new ResolveOnce3();
|
1875
1959
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
1876
1960
|
this.processing = void 0;
|
1877
1961
|
this.processQueue = new CommitQueue();
|
1878
|
-
this.loader = loader;
|
1879
1962
|
}
|
1880
1963
|
async ready() {
|
1881
1964
|
return this._ready.once(async () => {
|
1882
1965
|
const walState = await this.load().catch((e) => {
|
1883
|
-
this.logger.Error().
|
1966
|
+
this.logger.Error().Err(e).Msg("error loading wal");
|
1884
1967
|
return void 0;
|
1885
1968
|
});
|
1886
1969
|
if (!walState) {
|
@@ -1913,7 +1996,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1913
1996
|
}
|
1914
1997
|
async process() {
|
1915
1998
|
await this.ready();
|
1916
|
-
if (!this.loader
|
1999
|
+
if (!this.loader?.remoteCarStore) return;
|
1917
2000
|
await this.processQueue.enqueue(async () => {
|
1918
2001
|
try {
|
1919
2002
|
await this._doProcess();
|
@@ -1926,6 +2009,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1926
2009
|
});
|
1927
2010
|
}
|
1928
2011
|
async _doProcess() {
|
2012
|
+
if (!this.loader) return;
|
1929
2013
|
if (!this.loader.remoteCarStore) return;
|
1930
2014
|
const operations = [...this.walState.operations];
|
1931
2015
|
const noLoaderOps = [...this.walState.noLoaderOps];
|
@@ -1943,6 +2027,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1943
2027
|
noLoaderOps,
|
1944
2028
|
async (dbMeta) => {
|
1945
2029
|
await retryableUpload(async () => {
|
2030
|
+
if (!this.loader) return;
|
1946
2031
|
for (const cid of dbMeta.cars) {
|
1947
2032
|
const car = await (await this.loader.carStore()).load(cid);
|
1948
2033
|
if (!car) {
|
@@ -1962,6 +2047,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1962
2047
|
operations,
|
1963
2048
|
async (dbMeta) => {
|
1964
2049
|
await retryableUpload(async () => {
|
2050
|
+
if (!this.loader) return;
|
1965
2051
|
for (const cid of dbMeta.cars) {
|
1966
2052
|
const car = await (await this.loader.carStore()).load(cid);
|
1967
2053
|
if (!car) {
|
@@ -1981,6 +2067,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1981
2067
|
fileOperations,
|
1982
2068
|
async ({ cid: fileCid, public: publicFile }) => {
|
1983
2069
|
await retryableUpload(async () => {
|
2070
|
+
if (!this.loader) return;
|
1984
2071
|
const fileBlock = await (await this.loader.fileStore()).load(fileCid);
|
1985
2072
|
if (!fileBlock) {
|
1986
2073
|
throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
|
@@ -1994,6 +2081,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1994
2081
|
if (operations.length) {
|
1995
2082
|
const lastOp = operations[operations.length - 1];
|
1996
2083
|
await retryableUpload(async () => {
|
2084
|
+
if (!this.loader) return;
|
1997
2085
|
await this.loader.remoteMetaStore?.save(lastOp);
|
1998
2086
|
}, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
|
1999
2087
|
}
|
@@ -2019,6 +2107,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2019
2107
|
}
|
2020
2108
|
try {
|
2021
2109
|
return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
|
2110
|
+
return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
|
2022
2111
|
} catch (e) {
|
2023
2112
|
throw this.logger.Error().Err(e).Msg("error parse").AsError();
|
2024
2113
|
}
|
@@ -2045,72 +2134,92 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2045
2134
|
return Result5.Ok(void 0);
|
2046
2135
|
}
|
2047
2136
|
destroy() {
|
2137
|
+
this.logger.Debug().Msg("destroy");
|
2048
2138
|
return this.gateway.destroy(this.url());
|
2049
2139
|
}
|
2050
2140
|
};
|
2051
2141
|
|
2052
|
-
// src/blockstore/store-
|
2053
|
-
|
2054
|
-
|
2055
|
-
|
2142
|
+
// src/blockstore/register-store-protocol.ts
|
2143
|
+
import { BuildURI, runtimeFn as runtimeFn2 } from "@adviser/cement";
|
2144
|
+
|
2145
|
+
// src/runtime/gateways/memory/gateway.ts
|
2146
|
+
import { Result as Result6 } from "@adviser/cement";
|
2147
|
+
|
2148
|
+
// src/runtime/gateways/memory/version.ts
|
2149
|
+
var MEMORY_VERSION = "v0.19-memory";
|
2150
|
+
|
2151
|
+
// src/runtime/gateways/memory/gateway.ts
|
2152
|
+
var MemoryGateway = class {
|
2153
|
+
constructor(memorys) {
|
2154
|
+
this.memorys = memorys;
|
2056
2155
|
}
|
2057
|
-
|
2058
|
-
|
2059
|
-
function ensureName(name, url) {
|
2060
|
-
if (!url.hasParam("name")) {
|
2061
|
-
return url.build().setParam("name", name).URI();
|
2156
|
+
buildUrl(baseUrl, key) {
|
2157
|
+
return Promise.resolve(Result6.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
|
2062
2158
|
}
|
2063
|
-
|
2064
|
-
|
2159
|
+
start(baseUrl) {
|
2160
|
+
return Promise.resolve(Result6.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
|
2161
|
+
}
|
2162
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2163
|
+
close(baseUrl) {
|
2164
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2165
|
+
}
|
2166
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2167
|
+
destroy(baseUrl) {
|
2168
|
+
this.memorys.clear();
|
2169
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2170
|
+
}
|
2171
|
+
put(url, body) {
|
2172
|
+
this.memorys.set(url.toString(), body);
|
2173
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2174
|
+
}
|
2175
|
+
// get could return a NotFoundError if the key is not found
|
2176
|
+
get(url) {
|
2177
|
+
const x = this.memorys.get(url.toString());
|
2178
|
+
if (x === void 0) {
|
2179
|
+
return Promise.resolve(Result6.Err(new NotFoundError("not found")));
|
2180
|
+
}
|
2181
|
+
return Promise.resolve(Result6.Ok(x));
|
2182
|
+
}
|
2183
|
+
delete(url) {
|
2184
|
+
this.memorys.delete(url.toString());
|
2185
|
+
return Promise.resolve(Result6.Ok(void 0));
|
2186
|
+
}
|
2187
|
+
};
|
2188
|
+
var MemoryTestGateway = class {
|
2189
|
+
constructor(memorys) {
|
2190
|
+
this.memorys = memorys;
|
2191
|
+
}
|
2192
|
+
async get(url, key) {
|
2193
|
+
return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
|
2194
|
+
}
|
2195
|
+
};
|
2196
|
+
|
2197
|
+
// src/blockstore/register-store-protocol.ts
|
2065
2198
|
var storeFactory = /* @__PURE__ */ new Map();
|
2066
|
-
function
|
2067
|
-
|
2068
|
-
|
2069
|
-
|
2070
|
-
if (obuItem && obuItem.overrideBaseURL) {
|
2071
|
-
obuUrl = URI5.from(obuItem.overrideBaseURL);
|
2072
|
-
}
|
2073
|
-
const ret = ensureIsIndex(
|
2074
|
-
URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
2075
|
-
storeOpts.isIndex
|
2076
|
-
);
|
2077
|
-
return ret;
|
2078
|
-
}
|
2079
|
-
var onceGateway = new KeyedResolvOnce2();
|
2080
|
-
async function getGatewayFromURL(url, sthis) {
|
2081
|
-
return onceGateway.get(url.toString()).once(async () => {
|
2082
|
-
const item = storeFactory.get(url.protocol);
|
2083
|
-
if (item) {
|
2084
|
-
const ret = {
|
2085
|
-
gateway: await item.gateway(sthis),
|
2086
|
-
test: await item.test(sthis)
|
2087
|
-
};
|
2088
|
-
const res = await ret.gateway.start(url);
|
2089
|
-
if (res.isErr()) {
|
2090
|
-
sthis.logger.Error().Result("start", res).Msg("start failed");
|
2091
|
-
return void 0;
|
2092
|
-
}
|
2093
|
-
return ret;
|
2199
|
+
function getDefaultURI(sthis, protocol) {
|
2200
|
+
if (protocol) {
|
2201
|
+
if (!protocol.endsWith(":")) {
|
2202
|
+
protocol += ":";
|
2094
2203
|
}
|
2095
|
-
|
2096
|
-
|
2097
|
-
|
2204
|
+
const gfi = storeFactory.get(protocol);
|
2205
|
+
if (gfi) {
|
2206
|
+
return gfi.defaultURI(sthis);
|
2207
|
+
}
|
2208
|
+
}
|
2209
|
+
const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
|
2210
|
+
if (!found) {
|
2211
|
+
throw sthis.logger.Error().Msg("no default found").AsError();
|
2212
|
+
}
|
2213
|
+
return found.defaultURI(sthis);
|
2098
2214
|
}
|
2099
2215
|
function registerStoreProtocol(item) {
|
2100
2216
|
let protocol = item.protocol;
|
2101
2217
|
if (!protocol.endsWith(":")) {
|
2102
2218
|
protocol += ":";
|
2103
2219
|
}
|
2104
|
-
if (
|
2105
|
-
if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
|
2106
|
-
throw new Error(`we need a logger here`);
|
2107
|
-
return () => {
|
2108
|
-
};
|
2109
|
-
}
|
2110
|
-
}
|
2111
|
-
if (item.overrideBaseURL) {
|
2220
|
+
if (item.isDefault) {
|
2112
2221
|
Array.from(storeFactory.values()).forEach((items) => {
|
2113
|
-
items.
|
2222
|
+
items.isDefault = false;
|
2114
2223
|
});
|
2115
2224
|
}
|
2116
2225
|
storeFactory.set(protocol, item);
|
@@ -2118,134 +2227,183 @@ function registerStoreProtocol(item) {
|
|
2118
2227
|
storeFactory.delete(protocol);
|
2119
2228
|
};
|
2120
2229
|
}
|
2121
|
-
|
2122
|
-
|
2123
|
-
|
2124
|
-
|
2125
|
-
return
|
2126
|
-
|
2127
|
-
|
2128
|
-
|
2230
|
+
function getGatewayFactoryItem(protocol) {
|
2231
|
+
return storeFactory.get(protocol);
|
2232
|
+
}
|
2233
|
+
function fileGatewayFactoryItem() {
|
2234
|
+
return {
|
2235
|
+
protocol: "file:",
|
2236
|
+
isDefault: true,
|
2237
|
+
defaultURI: (sthis) => {
|
2238
|
+
return BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
|
2239
|
+
},
|
2240
|
+
gateway: async (sthis) => {
|
2241
|
+
const { FileGateway } = await import("./gateway-GK5QZ6KP.js");
|
2242
|
+
return new FileGateway(sthis);
|
2243
|
+
},
|
2244
|
+
test: async (sthis) => {
|
2245
|
+
const { FileTestStore } = await import("./gateway-GK5QZ6KP.js");
|
2246
|
+
return new FileTestStore(sthis);
|
2247
|
+
}
|
2248
|
+
};
|
2249
|
+
}
|
2250
|
+
if (runtimeFn2().isBrowser) {
|
2251
|
+
registerStoreProtocol({
|
2252
|
+
protocol: "indexdb:",
|
2253
|
+
isDefault: true,
|
2254
|
+
defaultURI: () => {
|
2255
|
+
return BuildURI.from("indexdb://").pathname("fp").URI();
|
2256
|
+
},
|
2257
|
+
gateway: async (logger) => {
|
2258
|
+
const { IndexDBGateway } = await import("./gateway-TQTGDRCN.js");
|
2259
|
+
return new IndexDBGateway(logger);
|
2260
|
+
},
|
2261
|
+
test: async (logger) => {
|
2262
|
+
const { IndexDBTestStore } = await import("./gateway-TQTGDRCN.js");
|
2263
|
+
return new IndexDBTestStore(logger);
|
2129
2264
|
}
|
2130
|
-
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2131
|
-
gateway: gateway.gateway,
|
2132
|
-
keybag: () => getKeyBag(loader.sthis, {
|
2133
|
-
...loader.ebOpts.keyBag
|
2134
|
-
})
|
2135
|
-
});
|
2136
|
-
return store;
|
2137
2265
|
});
|
2266
|
+
} else {
|
2267
|
+
registerStoreProtocol(fileGatewayFactoryItem());
|
2138
2268
|
}
|
2139
|
-
var
|
2140
|
-
|
2141
|
-
|
2142
|
-
|
2143
|
-
|
2144
|
-
|
2145
|
-
|
2146
|
-
|
2147
|
-
|
2148
|
-
|
2149
|
-
|
2150
|
-
|
2151
|
-
|
2152
|
-
|
2153
|
-
|
2154
|
-
|
2155
|
-
|
2269
|
+
var memory = /* @__PURE__ */ new Map();
|
2270
|
+
registerStoreProtocol({
|
2271
|
+
protocol: "memory:",
|
2272
|
+
isDefault: false,
|
2273
|
+
defaultURI: () => {
|
2274
|
+
return BuildURI.from("memory://").pathname("ram").URI();
|
2275
|
+
},
|
2276
|
+
gateway: async () => {
|
2277
|
+
return new MemoryGateway(memory);
|
2278
|
+
},
|
2279
|
+
test: async () => {
|
2280
|
+
return new MemoryTestGateway(memory);
|
2281
|
+
}
|
2282
|
+
});
|
2283
|
+
|
2284
|
+
// src/blockstore/store-factory.ts
|
2285
|
+
var onceGateway = new KeyedResolvOnce2();
|
2286
|
+
var gatewayInstances = new KeyedResolvOnce2();
|
2287
|
+
async function getStartedGateway(sthis, url) {
|
2288
|
+
return onceGateway.get(url.toString()).once(async () => {
|
2289
|
+
const item = getGatewayFactoryItem(url.protocol);
|
2290
|
+
if (item) {
|
2291
|
+
const ret = {
|
2292
|
+
url,
|
2293
|
+
...await gatewayInstances.get(url.protocol).once(async () => ({
|
2294
|
+
gateway: await item.gateway(sthis),
|
2295
|
+
test: await item.test(sthis)
|
2296
|
+
}))
|
2297
|
+
};
|
2298
|
+
const res = await ret.gateway.start(url);
|
2299
|
+
if (res.isErr()) {
|
2300
|
+
return Result7.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
|
2301
|
+
}
|
2302
|
+
ret.url = res.Ok();
|
2303
|
+
return Result7.Ok(ret);
|
2304
|
+
}
|
2305
|
+
return Result7.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
|
2156
2306
|
});
|
2157
2307
|
}
|
2158
|
-
|
2159
|
-
|
2160
|
-
const
|
2161
|
-
|
2162
|
-
|
2163
|
-
|
2164
|
-
|
2165
|
-
|
2166
|
-
|
2167
|
-
|
2168
|
-
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
|
2173
|
-
|
2174
|
-
|
2308
|
+
async function dataStoreFactory(sfi) {
|
2309
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
|
2310
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
2311
|
+
if (rgateway.isErr()) {
|
2312
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
2313
|
+
}
|
2314
|
+
const gateway = rgateway.Ok();
|
2315
|
+
const store = new DataStoreImpl(sfi.sthis, gateway.url, {
|
2316
|
+
gateway: gateway.gateway,
|
2317
|
+
keybag: sfi.keybag
|
2318
|
+
});
|
2319
|
+
return store;
|
2320
|
+
}
|
2321
|
+
async function metaStoreFactory(sfi) {
|
2322
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
|
2323
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
2324
|
+
if (rgateway.isErr()) {
|
2325
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
2326
|
+
}
|
2327
|
+
const gateway = rgateway.Ok();
|
2328
|
+
const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
|
2329
|
+
gateway: gateway.gateway,
|
2330
|
+
keybag: sfi.keybag
|
2331
|
+
});
|
2332
|
+
return store;
|
2333
|
+
}
|
2334
|
+
async function WALStoreFactory(sfi) {
|
2335
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
|
2336
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
2337
|
+
if (rgateway.isErr()) {
|
2338
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
2339
|
+
}
|
2340
|
+
const gateway = rgateway.Ok();
|
2341
|
+
const store = new WALStoreImpl(sfi.sthis, gateway.url, {
|
2342
|
+
gateway: gateway.gateway,
|
2343
|
+
keybag: sfi.keybag
|
2175
2344
|
});
|
2345
|
+
return store;
|
2176
2346
|
}
|
2177
2347
|
async function testStoreFactory(url, sthis) {
|
2178
|
-
|
2179
|
-
|
2180
|
-
if (!gateway) {
|
2348
|
+
const rgateway = await getStartedGateway(sthis, url);
|
2349
|
+
if (!rgateway) {
|
2181
2350
|
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2182
2351
|
}
|
2183
|
-
return
|
2352
|
+
return rgateway.Ok().test;
|
2184
2353
|
}
|
2185
|
-
async function ensureStart(store
|
2354
|
+
async function ensureStart(store) {
|
2186
2355
|
const ret = await store.start();
|
2187
2356
|
if (ret.isErr()) {
|
2188
|
-
throw logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2357
|
+
throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2189
2358
|
}
|
2190
|
-
logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2359
|
+
store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2191
2360
|
return store;
|
2192
2361
|
}
|
2193
|
-
function
|
2194
|
-
|
2362
|
+
function ensureStoreEnDeFile(ende) {
|
2363
|
+
ende = ende || {};
|
2195
2364
|
return {
|
2196
|
-
|
2197
|
-
|
2198
|
-
|
2199
|
-
|
2200
|
-
|
2201
|
-
|
2202
|
-
|
2203
|
-
|
2204
|
-
|
2205
|
-
|
2206
|
-
|
2207
|
-
|
2208
|
-
|
2209
|
-
|
2365
|
+
encodeFile: ende.encodeFile || encodeFile,
|
2366
|
+
decodeFile: ende.decodeFile || decodeFile
|
2367
|
+
};
|
2368
|
+
}
|
2369
|
+
function toStoreRuntime(sthis, endeOpts = {}) {
|
2370
|
+
return {
|
2371
|
+
makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
|
2372
|
+
// async (loader: Loadable) => {
|
2373
|
+
// logger
|
2374
|
+
// .Debug()
|
2375
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
|
2376
|
+
// .Msg("makeMetaStore");
|
2377
|
+
// return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
|
2378
|
+
// },
|
2379
|
+
makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
|
2380
|
+
// async (loader: Loadable) => {
|
2381
|
+
// logger
|
2382
|
+
// .Debug()
|
2383
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
|
2384
|
+
// .Msg("makeDataStore");
|
2385
|
+
// return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
|
2386
|
+
// },
|
2387
|
+
makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
|
2388
|
+
// async (loader: Loadable) => {
|
2389
|
+
// logger
|
2390
|
+
// .Debug()
|
2391
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
|
2392
|
+
// .Msg("makeRemoteWAL");
|
2393
|
+
// return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
|
2394
|
+
// },
|
2395
|
+
...ensureStoreEnDeFile(endeOpts)
|
2210
2396
|
};
|
2211
2397
|
}
|
2212
|
-
registerStoreProtocol({
|
2213
|
-
protocol: "file:",
|
2214
|
-
gateway: async (sthis) => {
|
2215
|
-
const { FileGateway } = await import("./gateway-5FCWPX5W.js");
|
2216
|
-
return new FileGateway(sthis);
|
2217
|
-
},
|
2218
|
-
test: async (sthis) => {
|
2219
|
-
const { FileTestStore } = await import("./gateway-5FCWPX5W.js");
|
2220
|
-
return new FileTestStore(sthis);
|
2221
|
-
}
|
2222
|
-
});
|
2223
|
-
registerStoreProtocol({
|
2224
|
-
protocol: "indexdb:",
|
2225
|
-
gateway: async (sthis) => {
|
2226
|
-
const { IndexDBGateway } = await import("./gateway-H7UD6TNB.js");
|
2227
|
-
return new IndexDBGateway(sthis);
|
2228
|
-
},
|
2229
|
-
test: async (sthis) => {
|
2230
|
-
const { IndexDBTestStore } = await import("./gateway-H7UD6TNB.js");
|
2231
|
-
return new IndexDBTestStore(sthis);
|
2232
|
-
}
|
2233
|
-
});
|
2234
2398
|
|
2235
2399
|
// src/blockstore/store-remote.ts
|
2236
2400
|
async function RemoteDataStore(sthis, name, url, opts) {
|
2237
|
-
const ds = new DataStoreImpl(sthis,
|
2401
|
+
const ds = new DataStoreImpl(sthis, url, opts);
|
2238
2402
|
await ds.start();
|
2239
2403
|
return ds;
|
2240
2404
|
}
|
2241
2405
|
async function RemoteMetaStore(sthis, name, url, opts) {
|
2242
|
-
const ms = new MetaStoreImpl(
|
2243
|
-
sthis,
|
2244
|
-
name,
|
2245
|
-
url,
|
2246
|
-
opts
|
2247
|
-
/* , true*/
|
2248
|
-
);
|
2406
|
+
const ms = new MetaStoreImpl(sthis, url, opts);
|
2249
2407
|
await ms.start();
|
2250
2408
|
return ms;
|
2251
2409
|
}
|
@@ -2270,14 +2428,17 @@ var ConnectionBase = class {
|
|
2270
2428
|
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
2271
2429
|
this.loader = loader;
|
2272
2430
|
await this.onConnect();
|
2273
|
-
const metaUrl = this.url.build().defParam("store"
|
2274
|
-
const
|
2275
|
-
if (
|
2431
|
+
const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
|
2432
|
+
const rgateway = await getStartedGateway(loader.sthis, metaUrl);
|
2433
|
+
if (rgateway.isErr())
|
2434
|
+
throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
2435
|
+
const name = metaUrl.toString();
|
2276
2436
|
const dbName = metaUrl.getParam("name");
|
2277
|
-
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X:
|
2278
|
-
const
|
2437
|
+
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
|
2438
|
+
const gateway = rgateway.Ok();
|
2439
|
+
const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
|
2279
2440
|
gateway: gateway.gateway,
|
2280
|
-
keybag:
|
2441
|
+
keybag: await loader.keyBag(),
|
2281
2442
|
loader
|
2282
2443
|
});
|
2283
2444
|
this.loader.remoteMetaStore = remote;
|
@@ -2290,14 +2451,15 @@ var ConnectionBase = class {
|
|
2290
2451
|
async connectStorage_X({ loader }) {
|
2291
2452
|
if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
|
2292
2453
|
this.loader = loader;
|
2293
|
-
const dataUrl = this.url.build().defParam("store"
|
2294
|
-
const
|
2295
|
-
if (
|
2454
|
+
const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
|
2455
|
+
const rgateway = await getStartedGateway(loader.sthis, dataUrl);
|
2456
|
+
if (rgateway.isErr())
|
2457
|
+
throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
2296
2458
|
const name = dataUrl.getParam("name");
|
2297
2459
|
if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
|
2298
2460
|
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
2299
|
-
gateway:
|
2300
|
-
keybag:
|
2461
|
+
gateway: rgateway.Ok().gateway,
|
2462
|
+
keybag: await loader.keyBag()
|
2301
2463
|
});
|
2302
2464
|
loader.remoteFileStore = loader.remoteCarStore;
|
2303
2465
|
}
|
@@ -2337,6 +2499,12 @@ var ConnectionBase = class {
|
|
2337
2499
|
};
|
2338
2500
|
|
2339
2501
|
// src/crdt-helpers.ts
|
2502
|
+
import { parse as parse3 } from "multiformats/link";
|
2503
|
+
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
2504
|
+
import * as codec from "@ipld/dag-cbor";
|
2505
|
+
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
2506
|
+
import { EventFetcher, vis } from "@web3-storage/pail/clock";
|
2507
|
+
import * as Batch from "@web3-storage/pail/crdt/batch";
|
2340
2508
|
function time(tag) {
|
2341
2509
|
}
|
2342
2510
|
function timeEnd(tag) {
|
@@ -2384,7 +2552,7 @@ async function writeDocContent(store, blocks, update, logger) {
|
|
2384
2552
|
await processFiles(store, blocks, update.value, logger);
|
2385
2553
|
value = { doc: update.value };
|
2386
2554
|
}
|
2387
|
-
const block = await
|
2555
|
+
const block = await encode3({ value, hasher: hasher5, codec });
|
2388
2556
|
blocks.putSync(block.cid, block.bytes);
|
2389
2557
|
return block.cid;
|
2390
2558
|
}
|
@@ -2475,7 +2643,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
2475
2643
|
async function getValueFromLink(blocks, link, logger) {
|
2476
2644
|
const block = await blocks.get(link);
|
2477
2645
|
if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
|
2478
|
-
const { value } = await
|
2646
|
+
const { value } = await decode3({ bytes: block.bytes, hasher: hasher5, codec });
|
2479
2647
|
const cvalue = {
|
2480
2648
|
...value,
|
2481
2649
|
cid: link
|
@@ -2596,7 +2764,7 @@ async function doCompact(blockLog, head, logger) {
|
|
2596
2764
|
async function getBlock(blocks, cidString) {
|
2597
2765
|
const block = await blocks.get(parse3(cidString));
|
2598
2766
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
2599
|
-
const { cid, value } = await
|
2767
|
+
const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: hasher5 });
|
2600
2768
|
return new Block({ cid, value, bytes: block.bytes });
|
2601
2769
|
}
|
2602
2770
|
|
@@ -2654,7 +2822,8 @@ function makeProllyGetBlock(blocks) {
|
|
2654
2822
|
return create({ cid, bytes, hasher: hasher6, codec: codec2 });
|
2655
2823
|
};
|
2656
2824
|
}
|
2657
|
-
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
2825
|
+
async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
|
2826
|
+
logger.Debug().Msg("enter bulkIndex");
|
2658
2827
|
if (!indexEntries.length) return inIndex;
|
2659
2828
|
if (!inIndex.root) {
|
2660
2829
|
if (!inIndex.cid) {
|
@@ -2671,18 +2840,22 @@ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
|
2671
2840
|
returnNode = node;
|
2672
2841
|
}
|
2673
2842
|
if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
|
2843
|
+
logger.Debug().Msg("exit !root bulkIndex");
|
2674
2844
|
return { root: returnNode, cid: returnRootBlock.cid };
|
2675
2845
|
} else {
|
2676
2846
|
inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
|
2677
2847
|
}
|
2678
2848
|
}
|
2849
|
+
logger.Debug().Msg("pre bulk bulkIndex");
|
2679
2850
|
const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
|
2680
2851
|
if (root3) {
|
2852
|
+
logger.Debug().Msg("pre root put bulkIndex");
|
2681
2853
|
for await (const block of newBlocks) {
|
2682
2854
|
await tblocks.put(block.cid, block.bytes);
|
2683
2855
|
}
|
2684
2856
|
return { root: root3, cid: (await root3.block).cid };
|
2685
2857
|
} else {
|
2858
|
+
logger.Debug().Msg("pre !root bulkIndex");
|
2686
2859
|
return { root: void 0, cid: void 0 };
|
2687
2860
|
}
|
2688
2861
|
}
|
@@ -2722,17 +2895,17 @@ function encodeKey(key) {
|
|
2722
2895
|
}
|
2723
2896
|
|
2724
2897
|
// src/indexer.ts
|
2725
|
-
function index(
|
2726
|
-
if (mapFn && meta) throw
|
2727
|
-
if (mapFn && mapFn.constructor.name !== "Function") throw
|
2728
|
-
if (
|
2729
|
-
const idx =
|
2898
|
+
function index(refDb, name, mapFn, meta) {
|
2899
|
+
if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
2900
|
+
if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
|
2901
|
+
if (refDb.crdt.indexers.has(name)) {
|
2902
|
+
const idx = refDb.crdt.indexers.get(name);
|
2730
2903
|
idx.applyMapFn(name, mapFn, meta);
|
2731
2904
|
} else {
|
2732
|
-
const idx = new Index(sthis,
|
2733
|
-
|
2905
|
+
const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
|
2906
|
+
refDb.crdt.indexers.set(name, idx);
|
2734
2907
|
}
|
2735
|
-
return
|
2908
|
+
return refDb.crdt.indexers.get(name);
|
2736
2909
|
}
|
2737
2910
|
var Index = class {
|
2738
2911
|
constructor(sthis, crdt, name, mapFn, meta) {
|
@@ -2751,18 +2924,9 @@ var Index = class {
|
|
2751
2924
|
return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
|
2752
2925
|
});
|
2753
2926
|
}
|
2754
|
-
close() {
|
2755
|
-
return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
|
2756
|
-
});
|
2757
|
-
}
|
2758
|
-
destroy() {
|
2759
|
-
return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
|
2760
|
-
});
|
2761
|
-
}
|
2762
2927
|
applyMapFn(name, mapFn, meta) {
|
2763
2928
|
if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
2764
2929
|
if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
|
2765
|
-
this.name = name;
|
2766
2930
|
try {
|
2767
2931
|
if (meta) {
|
2768
2932
|
if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
|
@@ -2810,9 +2974,13 @@ var Index = class {
|
|
2810
2974
|
}
|
2811
2975
|
}
|
2812
2976
|
async query(opts = {}) {
|
2977
|
+
this.logger.Debug().Msg("enter query");
|
2813
2978
|
await this.ready();
|
2979
|
+
this.logger.Debug().Msg("post ready query");
|
2814
2980
|
await this._updateIndex();
|
2981
|
+
this.logger.Debug().Msg("post _updateIndex query");
|
2815
2982
|
await this._hydrateIndex();
|
2983
|
+
this.logger.Debug().Msg("post _hydrateIndex query");
|
2816
2984
|
if (!this.byKey.root) {
|
2817
2985
|
return await applyQuery(this.crdt, { result: [] }, opts);
|
2818
2986
|
}
|
@@ -2868,13 +3036,16 @@ var Index = class {
|
|
2868
3036
|
}
|
2869
3037
|
async _updateIndex() {
|
2870
3038
|
await this.ready();
|
3039
|
+
this.logger.Debug().Msg("enter _updateIndex");
|
2871
3040
|
if (this.initError) throw this.initError;
|
2872
3041
|
if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
|
2873
3042
|
let result, head;
|
2874
3043
|
if (!this.indexHead || this.indexHead.length === 0) {
|
2875
3044
|
({ result, head } = await this.crdt.allDocs());
|
3045
|
+
this.logger.Debug().Msg("enter crdt.allDocs");
|
2876
3046
|
} else {
|
2877
3047
|
({ result, head } = await this.crdt.changes(this.indexHead));
|
3048
|
+
this.logger.Debug().Msg("enter crdt.changes");
|
2878
3049
|
}
|
2879
3050
|
if (result.length === 0) {
|
2880
3051
|
this.indexHead = head;
|
@@ -2907,9 +3078,22 @@ var Index = class {
|
|
2907
3078
|
if (result.length === 0) {
|
2908
3079
|
return indexerMeta;
|
2909
3080
|
}
|
3081
|
+
this.logger.Debug().Msg("pre this.blockstore.transaction");
|
2910
3082
|
const { meta } = await this.blockstore.transaction(async (tblocks) => {
|
2911
|
-
this.byId = await bulkIndex(
|
2912
|
-
|
3083
|
+
this.byId = await bulkIndex(
|
3084
|
+
this.logger,
|
3085
|
+
tblocks,
|
3086
|
+
this.byId,
|
3087
|
+
removeIdIndexEntries.concat(byIdIndexEntries),
|
3088
|
+
byIdOpts
|
3089
|
+
);
|
3090
|
+
this.byKey = await bulkIndex(
|
3091
|
+
this.logger,
|
3092
|
+
tblocks,
|
3093
|
+
this.byKey,
|
3094
|
+
staleKeyIndexEntries.concat(indexEntries),
|
3095
|
+
byKeyOpts
|
3096
|
+
);
|
2913
3097
|
this.indexHead = head;
|
2914
3098
|
if (this.byId.cid && this.byKey.cid) {
|
2915
3099
|
const idxMeta = {
|
@@ -2921,8 +3105,10 @@ var Index = class {
|
|
2921
3105
|
};
|
2922
3106
|
indexerMeta.indexes?.set(this.name, idxMeta);
|
2923
3107
|
}
|
3108
|
+
this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
|
2924
3109
|
return indexerMeta;
|
2925
3110
|
});
|
3111
|
+
this.logger.Debug().Msg("post this.blockstore.transaction");
|
2926
3112
|
return meta;
|
2927
3113
|
}
|
2928
3114
|
};
|
@@ -3050,21 +3236,23 @@ var CRDTClock = class {
|
|
3050
3236
|
throw this.logger.Error().Msg("missing blockstore").AsError();
|
3051
3237
|
}
|
3052
3238
|
await validateBlocks(this.logger, newHead, this.blockstore);
|
3053
|
-
|
3054
|
-
|
3055
|
-
|
3056
|
-
|
3057
|
-
|
3058
|
-
|
3059
|
-
|
3060
|
-
|
3061
|
-
|
3062
|
-
|
3063
|
-
|
3064
|
-
|
3065
|
-
|
3066
|
-
|
3067
|
-
|
3239
|
+
if (!this.transaction) {
|
3240
|
+
this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
|
3241
|
+
}
|
3242
|
+
const tblocks = this.transaction;
|
3243
|
+
const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
|
3244
|
+
const result = await root2(tblocks, advancedHead);
|
3245
|
+
for (const { cid, bytes } of [
|
3246
|
+
...result.additions
|
3247
|
+
// ...result.removals
|
3248
|
+
]) {
|
3249
|
+
tblocks.putSync(cid, bytes);
|
3250
|
+
}
|
3251
|
+
if (!noLoader) {
|
3252
|
+
await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
|
3253
|
+
this.transaction = void 0;
|
3254
|
+
}
|
3255
|
+
this.setHead(advancedHead);
|
3068
3256
|
}
|
3069
3257
|
};
|
3070
3258
|
function sortClockHead(clockHead) {
|
@@ -3096,15 +3284,13 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
3096
3284
|
|
3097
3285
|
// src/crdt.ts
|
3098
3286
|
var CRDT = class {
|
3099
|
-
constructor(sthis,
|
3287
|
+
constructor(sthis, opts) {
|
3100
3288
|
this.indexers = /* @__PURE__ */ new Map();
|
3101
3289
|
this.onceReady = new ResolveOnce5();
|
3102
3290
|
this.sthis = sthis;
|
3103
|
-
this.name = name;
|
3104
3291
|
this.logger = ensureLogger(sthis, "CRDT");
|
3105
3292
|
this.opts = opts;
|
3106
3293
|
this.blockstore = blockstoreFactory(sthis, {
|
3107
|
-
name,
|
3108
3294
|
applyMeta: async (meta) => {
|
3109
3295
|
const crdtMeta = meta;
|
3110
3296
|
if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
|
@@ -3114,23 +3300,27 @@ var CRDT = class {
|
|
3114
3300
|
await doCompact(blocks, this.clock.head, this.logger);
|
3115
3301
|
return { head: this.clock.head };
|
3116
3302
|
},
|
3117
|
-
autoCompact: this.opts.autoCompact || 100,
|
3118
|
-
|
3119
|
-
|
3120
|
-
|
3121
|
-
|
3303
|
+
// autoCompact: this.opts.autoCompact || 100,
|
3304
|
+
storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
|
3305
|
+
storeUrls: this.opts.storeUrls.data,
|
3306
|
+
keyBag: this.opts.keyBag,
|
3307
|
+
// public: this.opts.public,
|
3308
|
+
meta: this.opts.meta
|
3309
|
+
// threshold: this.opts.threshold,
|
3122
3310
|
});
|
3123
3311
|
this.indexBlockstore = blockstoreFactory(sthis, {
|
3124
|
-
name,
|
3312
|
+
// name: opts.name,
|
3125
3313
|
applyMeta: async (meta) => {
|
3126
3314
|
const idxCarMeta = meta;
|
3127
3315
|
if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
|
3128
|
-
for (const [
|
3129
|
-
index(
|
3316
|
+
for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
|
3317
|
+
index({ crdt: this }, name, void 0, idx);
|
3130
3318
|
}
|
3131
3319
|
},
|
3132
|
-
|
3133
|
-
|
3320
|
+
storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
|
3321
|
+
storeUrls: this.opts.storeUrls.idx,
|
3322
|
+
keyBag: this.opts.keyBag
|
3323
|
+
// public: this.opts.public,
|
3134
3324
|
});
|
3135
3325
|
this.clock = new CRDTClock(this.blockstore);
|
3136
3326
|
this.clock.onZoom(() => {
|
@@ -3212,51 +3402,164 @@ var CRDT = class {
|
|
3212
3402
|
};
|
3213
3403
|
|
3214
3404
|
// src/database.ts
|
3215
|
-
var
|
3216
|
-
|
3217
|
-
|
3405
|
+
var databases = new KeyedResolvOnce3();
|
3406
|
+
function toSortedArray(set) {
|
3407
|
+
if (!set) return [];
|
3408
|
+
return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
|
3409
|
+
}
|
3410
|
+
function keyConfigOpts(sthis, name, opts) {
|
3411
|
+
return JSON.stringify(
|
3412
|
+
toSortedArray({
|
3413
|
+
name,
|
3414
|
+
stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
|
3415
|
+
})
|
3416
|
+
);
|
3417
|
+
}
|
3418
|
+
function isDatabase(db) {
|
3419
|
+
return db instanceof DatabaseImpl || db instanceof DatabaseShell;
|
3420
|
+
}
|
3421
|
+
function DatabaseFactory(name, opts) {
|
3422
|
+
const sthis = ensureSuperThis(opts);
|
3423
|
+
return new DatabaseShell(
|
3424
|
+
databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
|
3425
|
+
const db = new DatabaseImpl(sthis, {
|
3426
|
+
name,
|
3427
|
+
meta: opts?.meta,
|
3428
|
+
keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
|
3429
|
+
storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
|
3430
|
+
storeEnDe: {
|
3431
|
+
encodeFile,
|
3432
|
+
decodeFile,
|
3433
|
+
...opts?.storeEnDe
|
3434
|
+
}
|
3435
|
+
});
|
3436
|
+
db.onClosed(() => {
|
3437
|
+
databases.unget(key);
|
3438
|
+
});
|
3439
|
+
return db;
|
3440
|
+
})
|
3441
|
+
);
|
3442
|
+
}
|
3443
|
+
var DatabaseShell = class {
|
3444
|
+
constructor(ref) {
|
3445
|
+
this.ref = ref;
|
3446
|
+
ref.addShell(this);
|
3447
|
+
}
|
3448
|
+
get id() {
|
3449
|
+
return this.ref.id;
|
3450
|
+
}
|
3451
|
+
get logger() {
|
3452
|
+
return this.ref.logger;
|
3453
|
+
}
|
3454
|
+
get sthis() {
|
3455
|
+
return this.ref.sthis;
|
3456
|
+
}
|
3457
|
+
get crdt() {
|
3458
|
+
return this.ref.crdt;
|
3459
|
+
}
|
3460
|
+
name() {
|
3461
|
+
return this.ref.name();
|
3462
|
+
}
|
3463
|
+
onClosed(fn) {
|
3464
|
+
return this.ref.onClosed(fn);
|
3465
|
+
}
|
3466
|
+
close() {
|
3467
|
+
return this.ref.shellClose(this);
|
3468
|
+
}
|
3469
|
+
destroy() {
|
3470
|
+
return this.ref.destroy();
|
3471
|
+
}
|
3472
|
+
ready() {
|
3473
|
+
return this.ref.ready();
|
3474
|
+
}
|
3475
|
+
get(id) {
|
3476
|
+
return this.ref.get(id);
|
3477
|
+
}
|
3478
|
+
put(doc) {
|
3479
|
+
return this.ref.put(doc);
|
3480
|
+
}
|
3481
|
+
del(id) {
|
3482
|
+
return this.ref.del(id);
|
3483
|
+
}
|
3484
|
+
changes(since, opts) {
|
3485
|
+
return this.ref.changes(since, opts);
|
3486
|
+
}
|
3487
|
+
allDocs(opts) {
|
3488
|
+
return this.ref.allDocs(opts);
|
3489
|
+
}
|
3490
|
+
allDocuments() {
|
3491
|
+
return this.ref.allDocuments();
|
3492
|
+
}
|
3493
|
+
subscribe(listener, updates) {
|
3494
|
+
return this.ref.subscribe(listener, updates);
|
3495
|
+
}
|
3496
|
+
query(field, opts) {
|
3497
|
+
return this.ref.query(field, opts);
|
3498
|
+
}
|
3499
|
+
compact() {
|
3500
|
+
return this.ref.compact();
|
3501
|
+
}
|
3502
|
+
};
|
3503
|
+
var DatabaseImpl = class {
|
3504
|
+
constructor(sthis, opts) {
|
3218
3505
|
this._listening = false;
|
3219
3506
|
this._listeners = /* @__PURE__ */ new Set();
|
3220
3507
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
3508
|
+
// readonly blockstore: BaseBlockstore;
|
3509
|
+
this.shells = /* @__PURE__ */ new Set();
|
3510
|
+
this._onClosedFns = /* @__PURE__ */ new Set();
|
3221
3511
|
this._ready = new ResolveOnce6();
|
3222
|
-
this.
|
3223
|
-
this.
|
3224
|
-
this.
|
3512
|
+
this.opts = opts;
|
3513
|
+
this.sthis = sthis;
|
3514
|
+
this.id = sthis.timeOrderedNextId().str;
|
3225
3515
|
this.logger = ensureLogger(this.sthis, "Database");
|
3226
|
-
this.
|
3227
|
-
this.blockstore = this._crdt.blockstore;
|
3516
|
+
this.crdt = new CRDT(this.sthis, this.opts);
|
3228
3517
|
this._writeQueue = writeQueue(async (updates) => {
|
3229
|
-
return await this.
|
3518
|
+
return await this.crdt.bulk(updates);
|
3230
3519
|
});
|
3231
|
-
this.
|
3520
|
+
this.crdt.clock.onTock(() => {
|
3232
3521
|
this._no_update_notify();
|
3233
3522
|
});
|
3234
3523
|
}
|
3235
|
-
|
3236
|
-
this.
|
3524
|
+
addShell(shell) {
|
3525
|
+
this.shells.add(shell);
|
3526
|
+
}
|
3527
|
+
onClosed(fn) {
|
3528
|
+
this._onClosedFns.add(fn);
|
3237
3529
|
}
|
3238
3530
|
async close() {
|
3239
|
-
|
3240
|
-
|
3241
|
-
|
3531
|
+
throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
|
3532
|
+
}
|
3533
|
+
async shellClose(db) {
|
3534
|
+
if (!this.shells.has(db)) {
|
3535
|
+
throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
|
3536
|
+
}
|
3537
|
+
this.shells.delete(db);
|
3538
|
+
if (this.shells.size === 0) {
|
3539
|
+
await this.ready();
|
3540
|
+
await this.crdt.close();
|
3541
|
+
this._onClosedFns.forEach((fn) => fn());
|
3542
|
+
}
|
3242
3543
|
}
|
3243
3544
|
async destroy() {
|
3244
3545
|
await this.ready();
|
3245
|
-
await this.
|
3246
|
-
await this.blockstore.destroy();
|
3546
|
+
await this.crdt.destroy();
|
3247
3547
|
}
|
3248
3548
|
async ready() {
|
3249
|
-
|
3549
|
+
const ret = await this._ready.once(async () => {
|
3250
3550
|
await this.sthis.start();
|
3251
|
-
await this.
|
3252
|
-
await this.blockstore.ready();
|
3551
|
+
await this.crdt.ready();
|
3253
3552
|
});
|
3553
|
+
return ret;
|
3554
|
+
}
|
3555
|
+
name() {
|
3556
|
+
return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
|
3254
3557
|
}
|
3255
3558
|
async get(id) {
|
3256
|
-
if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
|
3559
|
+
if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
|
3257
3560
|
await this.ready();
|
3258
3561
|
this.logger.Debug().Str("id", id).Msg("get");
|
3259
|
-
const got = await this.
|
3562
|
+
const got = await this.crdt.get(id).catch((e) => {
|
3260
3563
|
throw new NotFoundError(`Not found: ${id} - ${e.message}`);
|
3261
3564
|
});
|
3262
3565
|
if (!got) throw new NotFoundError(`Not found: ${id}`);
|
@@ -3275,34 +3578,34 @@ var Database = class {
|
|
3275
3578
|
_id: docId
|
3276
3579
|
}
|
3277
3580
|
});
|
3278
|
-
return { id: docId, clock: result?.head, name: this.name };
|
3581
|
+
return { id: docId, clock: result?.head, name: this.name() };
|
3279
3582
|
}
|
3280
3583
|
async del(id) {
|
3281
3584
|
await this.ready();
|
3282
3585
|
this.logger.Debug().Str("id", id).Msg("del");
|
3283
3586
|
const result = await this._writeQueue.push({ id, del: true });
|
3284
|
-
return { id, clock: result?.head, name: this.name };
|
3587
|
+
return { id, clock: result?.head, name: this.name() };
|
3285
3588
|
}
|
3286
3589
|
async changes(since = [], opts = {}) {
|
3287
3590
|
await this.ready();
|
3288
3591
|
this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
|
3289
|
-
const { result, head } = await this.
|
3592
|
+
const { result, head } = await this.crdt.changes(since, opts);
|
3290
3593
|
const rows = result.map(({ id: key, value, del, clock }) => ({
|
3291
3594
|
key,
|
3292
3595
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
|
3293
3596
|
clock
|
3294
3597
|
}));
|
3295
|
-
return { rows, clock: head, name: this.name };
|
3598
|
+
return { rows, clock: head, name: this.name() };
|
3296
3599
|
}
|
3297
3600
|
async allDocs(opts = {}) {
|
3298
3601
|
await this.ready();
|
3299
3602
|
this.logger.Debug().Msg("allDocs");
|
3300
|
-
const { result, head } = await this.
|
3603
|
+
const { result, head } = await this.crdt.allDocs();
|
3301
3604
|
const rows = result.map(({ id: key, value, del }) => ({
|
3302
3605
|
key,
|
3303
3606
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
|
3304
3607
|
}));
|
3305
|
-
return { rows, clock: head, name: this.name };
|
3608
|
+
return { rows, clock: head, name: this.name() };
|
3306
3609
|
}
|
3307
3610
|
async allDocuments() {
|
3308
3611
|
return this.allDocs();
|
@@ -3312,7 +3615,7 @@ var Database = class {
|
|
3312
3615
|
if (updates) {
|
3313
3616
|
if (!this._listening) {
|
3314
3617
|
this._listening = true;
|
3315
|
-
this.
|
3618
|
+
this.crdt.clock.onTick((updates2) => {
|
3316
3619
|
void this._notify(updates2);
|
3317
3620
|
});
|
3318
3621
|
}
|
@@ -3331,13 +3634,13 @@ var Database = class {
|
|
3331
3634
|
async query(field, opts = {}) {
|
3332
3635
|
await this.ready();
|
3333
3636
|
this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
|
3334
|
-
const _crdt = this.
|
3335
|
-
const idx = typeof field === "string" ? index(
|
3637
|
+
const _crdt = this.crdt;
|
3638
|
+
const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
|
3336
3639
|
return await idx.query(opts);
|
3337
3640
|
}
|
3338
3641
|
async compact() {
|
3339
3642
|
await this.ready();
|
3340
|
-
await this.
|
3643
|
+
await this.crdt.compact();
|
3341
3644
|
}
|
3342
3645
|
async _notify(updates) {
|
3343
3646
|
await this.ready();
|
@@ -3361,23 +3664,62 @@ var Database = class {
|
|
3361
3664
|
}
|
3362
3665
|
}
|
3363
3666
|
};
|
3364
|
-
function
|
3365
|
-
|
3366
|
-
|
3667
|
+
function defaultURI(sthis, curi, uri, store, ctx) {
|
3668
|
+
ctx = ctx || {};
|
3669
|
+
const ret = (curi ? URI8.from(curi) : uri).build().setParam("store" /* STORE */, store);
|
3670
|
+
if (!ret.hasParam("name" /* NAME */)) {
|
3671
|
+
const name = sthis.pathOps.basename(ret.URI().pathname);
|
3672
|
+
if (!name) {
|
3673
|
+
throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
|
3674
|
+
}
|
3675
|
+
ret.setParam("name" /* NAME */, name);
|
3676
|
+
}
|
3677
|
+
if (ctx.idx) {
|
3678
|
+
ret.defParam("index" /* INDEX */, "idx");
|
3679
|
+
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
|
3680
|
+
} else {
|
3681
|
+
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
|
3682
|
+
}
|
3683
|
+
if (store === "data") {
|
3684
|
+
if (ctx.file) {
|
3685
|
+
} else {
|
3686
|
+
ret.defParam("suffix" /* SUFFIX */, ".car");
|
3687
|
+
}
|
3688
|
+
}
|
3689
|
+
return ret.URI();
|
3367
3690
|
}
|
3368
|
-
function
|
3369
|
-
|
3370
|
-
|
3371
|
-
|
3372
|
-
|
3373
|
-
|
3374
|
-
|
3375
|
-
|
3376
|
-
|
3377
|
-
db = new Database(name, opts);
|
3378
|
-
Database.databases.set(key, db);
|
3691
|
+
function toStoreURIRuntime(sthis, name, sopts) {
|
3692
|
+
sopts = sopts || {};
|
3693
|
+
if (!sopts.base) {
|
3694
|
+
const fp_env = sthis.env.get("FP_STORAGE_URL");
|
3695
|
+
if (fp_env) {
|
3696
|
+
sopts = { ...sopts, base: BuildURI2.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
|
3697
|
+
} else {
|
3698
|
+
sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
|
3699
|
+
}
|
3379
3700
|
}
|
3380
|
-
|
3701
|
+
const bbase = BuildURI2.from(sopts.base);
|
3702
|
+
if (name) {
|
3703
|
+
bbase.setParam("name" /* NAME */, name);
|
3704
|
+
}
|
3705
|
+
const base = bbase.URI();
|
3706
|
+
return {
|
3707
|
+
idx: {
|
3708
|
+
data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
|
3709
|
+
file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
|
3710
|
+
meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
|
3711
|
+
wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
|
3712
|
+
},
|
3713
|
+
data: {
|
3714
|
+
data: defaultURI(sthis, sopts.data?.data, base, "data"),
|
3715
|
+
file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
|
3716
|
+
meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
|
3717
|
+
wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
|
3718
|
+
}
|
3719
|
+
};
|
3720
|
+
}
|
3721
|
+
function fireproof(name, opts) {
|
3722
|
+
return DatabaseFactory(name, opts);
|
3381
3723
|
}
|
3382
3724
|
function makeName(fnString) {
|
3383
3725
|
const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
|
@@ -3408,7 +3750,7 @@ __export(runtime_exports, {
|
|
3408
3750
|
kb: () => key_bag_exports,
|
3409
3751
|
kc: () => keyed_crypto_exports,
|
3410
3752
|
mf: () => wait_pr_multiformats_exports,
|
3411
|
-
runtimeFn: () =>
|
3753
|
+
runtimeFn: () => runtimeFn3,
|
3412
3754
|
toArrayBuffer: () => toArrayBuffer
|
3413
3755
|
});
|
3414
3756
|
|
@@ -3423,23 +3765,24 @@ __export(wait_pr_multiformats_exports, {
|
|
3423
3765
|
var codec_interface_exports = {};
|
3424
3766
|
|
3425
3767
|
// src/runtime/index.ts
|
3426
|
-
import { runtimeFn as
|
3768
|
+
import { runtimeFn as runtimeFn3 } from "@adviser/cement";
|
3427
3769
|
|
3428
3770
|
// src/version.ts
|
3429
3771
|
var PACKAGE_VERSION = Object.keys({
|
3430
|
-
"0.19.
|
3772
|
+
"0.19.101": "xxxx"
|
3431
3773
|
})[0];
|
3432
3774
|
export {
|
3433
3775
|
CRDT,
|
3434
|
-
|
3776
|
+
DatabaseFactory,
|
3777
|
+
DatabaseShell,
|
3435
3778
|
Index,
|
3436
3779
|
NotFoundError,
|
3437
3780
|
PACKAGE_VERSION,
|
3781
|
+
PARAM,
|
3438
3782
|
Result,
|
3439
3783
|
UInt8ArrayEqual,
|
3440
3784
|
blockstore_exports as blockstore,
|
3441
3785
|
blockstore_exports as bs,
|
3442
|
-
dataDir,
|
3443
3786
|
ensureLogger,
|
3444
3787
|
ensureSuperLog,
|
3445
3788
|
ensureSuperThis,
|
@@ -3450,10 +3793,13 @@ export {
|
|
3450
3793
|
getName,
|
3451
3794
|
getStore,
|
3452
3795
|
index,
|
3796
|
+
isDatabase,
|
3453
3797
|
isFalsy,
|
3454
3798
|
isNotFoundError,
|
3799
|
+
keyConfigOpts,
|
3455
3800
|
runtime_exports as rt,
|
3456
3801
|
runtime_exports as runtime,
|
3457
|
-
throwFalsy
|
3802
|
+
throwFalsy,
|
3803
|
+
toStoreURIRuntime
|
3458
3804
|
};
|
3459
3805
|
//# sourceMappingURL=index.js.map
|