@fireproof/core 0.19.101 → 0.19.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{chunk-3EB3ENHT.js → chunk-OFGPKRCM.js} +25 -54
- package/chunk-OFGPKRCM.js.map +1 -0
- package/chunk-WS3YRPIA.js +75 -0
- package/chunk-WS3YRPIA.js.map +1 -0
- package/{gateway-GK5QZ6KP.js → gateway-5FCWPX5W.js} +12 -13
- package/gateway-5FCWPX5W.js.map +1 -0
- package/{gateway-TQTGDRCN.js → gateway-H7UD6TNB.js} +8 -9
- package/gateway-H7UD6TNB.js.map +1 -0
- package/index.cjs +1571 -1992
- package/index.cjs.map +1 -1
- package/index.d.cts +117 -257
- package/index.d.ts +117 -257
- package/index.global.js +12280 -12741
- package/index.global.js.map +1 -1
- package/index.js +1463 -1790
- package/index.js.map +1 -1
- package/{key-bag-file-VOSSK46F.js → key-bag-file-WADZBHYG.js} +3 -4
- package/{key-bag-file-VOSSK46F.js.map → key-bag-file-WADZBHYG.js.map} +1 -1
- package/{key-bag-indexdb-AXTQOSMC.js → key-bag-indexdb-PGVAI3FJ.js} +3 -4
- package/{key-bag-indexdb-AXTQOSMC.js.map → key-bag-indexdb-PGVAI3FJ.js.map} +1 -1
- package/mem-filesystem-YPPJV7Q2.js +41 -0
- package/mem-filesystem-YPPJV7Q2.js.map +1 -0
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/{node-filesystem-CFRXFSO7.js → node-filesystem-INX4ZTHE.js} +9 -6
- package/node-filesystem-INX4ZTHE.js.map +1 -0
- package/package.json +1 -1
- package/tests/blockstore/keyed-crypto.test.ts +227 -63
- package/tests/blockstore/loader.test.ts +11 -19
- package/tests/blockstore/store.test.ts +19 -23
- package/tests/blockstore/transaction.test.ts +12 -12
- package/tests/fireproof/all-gateway.test.ts +193 -201
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +316 -324
- package/tests/fireproof/config.test.ts +172 -0
- package/tests/fireproof/crdt.test.ts +16 -67
- package/tests/fireproof/database.test.ts +21 -183
- package/tests/fireproof/fireproof.test.ts +74 -83
- package/tests/fireproof/hello.test.ts +14 -18
- package/tests/fireproof/indexer.test.ts +43 -53
- package/tests/fireproof/utils.test.ts +6 -18
- package/tests/helpers.ts +9 -27
- package/tests/react/useFireproof.test.tsx +1 -1
- package/{utils-STA2C35G.js → utils-QO2HIWGI.js} +3 -4
- package/chunk-3EB3ENHT.js.map +0 -1
- package/chunk-HQ7D3PEU.js +0 -61
- package/chunk-HQ7D3PEU.js.map +0 -1
- package/chunk-PZ5AY32C.js +0 -10
- package/deno-filesystem-Q2IJ7YDR.js +0 -57
- package/deno-filesystem-Q2IJ7YDR.js.map +0 -1
- package/gateway-GK5QZ6KP.js.map +0 -1
- package/gateway-TQTGDRCN.js.map +0 -1
- package/key-bag-memory-LWE6ARPX.js +0 -29
- package/key-bag-memory-LWE6ARPX.js.map +0 -1
- package/node-filesystem-CFRXFSO7.js.map +0 -1
- package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +0 -129
- package/tests/gateway/file/loader-config.test.ts +0 -303
- package/tests/gateway/indexdb/loader-config.test.ts +0 -75
- package/utils-STA2C35G.js.map +0 -1
- /package/tests/fireproof/{fireproof.fixture.ts → fireproof.test.fixture.ts} +0 -0
- /package/{chunk-PZ5AY32C.js.map → utils-QO2HIWGI.js.map} +0 -0
package/index.js
CHANGED
@@ -1,6 +1,3 @@
|
|
1
|
-
import {
|
2
|
-
INDEXDB_VERSION
|
3
|
-
} from "./chunk-PB4BKL4O.js";
|
4
1
|
import {
|
5
2
|
FILESTORE_VERSION
|
6
3
|
} from "./chunk-7EWIAXTM.js";
|
@@ -9,30 +6,28 @@ import {
|
|
9
6
|
getFileSystem,
|
10
7
|
getPath,
|
11
8
|
toArrayBuffer
|
12
|
-
} from "./chunk-
|
9
|
+
} from "./chunk-WS3YRPIA.js";
|
10
|
+
import {
|
11
|
+
INDEXDB_VERSION
|
12
|
+
} from "./chunk-PB4BKL4O.js";
|
13
13
|
import {
|
14
14
|
NotFoundError,
|
15
|
-
PARAM,
|
16
15
|
Result,
|
17
16
|
UInt8ArrayEqual,
|
17
|
+
__export,
|
18
|
+
dataDir,
|
18
19
|
ensureLogger,
|
19
20
|
ensureSuperLog,
|
20
21
|
ensureSuperThis,
|
21
22
|
exceptionWrapper,
|
22
|
-
falsyToUndef,
|
23
23
|
getKey,
|
24
24
|
getName,
|
25
25
|
getStore,
|
26
|
-
|
27
|
-
|
28
|
-
throwFalsy
|
29
|
-
} from "./chunk-3EB3ENHT.js";
|
30
|
-
import {
|
31
|
-
__export
|
32
|
-
} from "./chunk-PZ5AY32C.js";
|
26
|
+
isNotFoundError
|
27
|
+
} from "./chunk-OFGPKRCM.js";
|
33
28
|
|
34
29
|
// src/database.ts
|
35
|
-
import {
|
30
|
+
import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
|
36
31
|
|
37
32
|
// src/write-queue.ts
|
38
33
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -77,6 +72,85 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
77
72
|
// src/crdt.ts
|
78
73
|
import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
|
79
74
|
|
75
|
+
// src/runtime/wait-pr-multiformats/block.ts
|
76
|
+
var block_exports = {};
|
77
|
+
__export(block_exports, {
|
78
|
+
Block: () => Block,
|
79
|
+
create: () => create,
|
80
|
+
createUnsafe: () => createUnsafe,
|
81
|
+
decode: () => decode,
|
82
|
+
encode: () => encode
|
83
|
+
});
|
84
|
+
import { bytes as binary, CID } from "multiformats";
|
85
|
+
import { Block as mfBlock } from "multiformats/block";
|
86
|
+
var Block = mfBlock;
|
87
|
+
async function decode({
|
88
|
+
bytes,
|
89
|
+
codec: codec3,
|
90
|
+
hasher: hasher7
|
91
|
+
}) {
|
92
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
93
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
94
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
95
|
+
const hash = await hasher7.digest(bytes);
|
96
|
+
const cid = CID.create(1, codec3.code, hash);
|
97
|
+
return new mfBlock({ value, bytes, cid });
|
98
|
+
}
|
99
|
+
async function encode({
|
100
|
+
value,
|
101
|
+
codec: codec3,
|
102
|
+
hasher: hasher7
|
103
|
+
}) {
|
104
|
+
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
105
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
106
|
+
const bytes = await Promise.resolve(codec3.encode(value));
|
107
|
+
const hash = await hasher7.digest(bytes);
|
108
|
+
const cid = CID.create(1, codec3.code, hash);
|
109
|
+
return new mfBlock({ value, bytes, cid });
|
110
|
+
}
|
111
|
+
async function create({
|
112
|
+
bytes,
|
113
|
+
cid,
|
114
|
+
hasher: hasher7,
|
115
|
+
codec: codec3
|
116
|
+
}) {
|
117
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
118
|
+
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
119
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
120
|
+
const hash = await hasher7.digest(bytes);
|
121
|
+
if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
|
122
|
+
throw new Error("CID hash does not match bytes");
|
123
|
+
}
|
124
|
+
return createUnsafe({
|
125
|
+
bytes,
|
126
|
+
cid,
|
127
|
+
value,
|
128
|
+
codec: codec3
|
129
|
+
});
|
130
|
+
}
|
131
|
+
async function createUnsafe({
|
132
|
+
bytes,
|
133
|
+
cid,
|
134
|
+
value: maybeValue,
|
135
|
+
codec: codec3
|
136
|
+
}) {
|
137
|
+
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
138
|
+
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
139
|
+
return new Block({
|
140
|
+
cid,
|
141
|
+
bytes,
|
142
|
+
value
|
143
|
+
});
|
144
|
+
}
|
145
|
+
|
146
|
+
// src/crdt-helpers.ts
|
147
|
+
import { parse as parse3 } from "multiformats/link";
|
148
|
+
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
149
|
+
import * as codec from "@ipld/dag-cbor";
|
150
|
+
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
151
|
+
import { EventFetcher, vis } from "@web3-storage/pail/clock";
|
152
|
+
import * as Batch from "@web3-storage/pail/crdt/batch";
|
153
|
+
|
80
154
|
// src/blockstore/index.ts
|
81
155
|
var blockstore_exports = {};
|
82
156
|
__export(blockstore_exports, {
|
@@ -88,11 +162,8 @@ __export(blockstore_exports, {
|
|
88
162
|
FragmentGateway: () => FragmentGateway,
|
89
163
|
Loader: () => Loader,
|
90
164
|
addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
|
91
|
-
|
92
|
-
|
93
|
-
getDefaultURI: () => getDefaultURI,
|
94
|
-
getGatewayFactoryItem: () => getGatewayFactoryItem,
|
95
|
-
getStartedGateway: () => getStartedGateway,
|
165
|
+
ensureStart: () => ensureStart,
|
166
|
+
getGatewayFromURL: () => getGatewayFromURL,
|
96
167
|
parseCarFile: () => parseCarFile,
|
97
168
|
registerStoreProtocol: () => registerStoreProtocol,
|
98
169
|
setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
|
@@ -107,7 +178,7 @@ function toCIDBlock(block) {
|
|
107
178
|
}
|
108
179
|
|
109
180
|
// src/blockstore/store-factory.ts
|
110
|
-
import { KeyedResolvOnce as KeyedResolvOnce2,
|
181
|
+
import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
|
111
182
|
|
112
183
|
// src/runtime/files.ts
|
113
184
|
var files_exports = {};
|
@@ -182,402 +253,312 @@ var UnixFSFileBuilder = class {
|
|
182
253
|
|
183
254
|
// src/blockstore/store.ts
|
184
255
|
import { format as format2, parse as parse2 } from "@ipld/dag-json";
|
185
|
-
import { ResolveOnce as ResolveOnce3, Result as Result5
|
256
|
+
import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result5 } from "@adviser/cement";
|
186
257
|
|
187
|
-
// src/
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
258
|
+
// src/types.ts
|
259
|
+
function isFalsy(value) {
|
260
|
+
return value === false && value === null && value === void 0;
|
261
|
+
}
|
262
|
+
function throwFalsy(value) {
|
263
|
+
if (isFalsy(value)) {
|
264
|
+
throw new Error("value is Falsy");
|
194
265
|
}
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
this._waitIdleItems.add(fn);
|
201
|
-
return fn.asPromise();
|
266
|
+
return value;
|
267
|
+
}
|
268
|
+
function falsyToUndef(value) {
|
269
|
+
if (isFalsy(value)) {
|
270
|
+
return void 0;
|
202
271
|
}
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
272
|
+
return value;
|
273
|
+
}
|
274
|
+
|
275
|
+
// src/blockstore/loader.ts
|
276
|
+
import pLimit from "p-limit";
|
277
|
+
import { CarReader } from "@ipld/car";
|
278
|
+
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
279
|
+
|
280
|
+
// src/blockstore/loader-helpers.ts
|
281
|
+
import { sha256 as hasher } from "multiformats/hashes/sha2";
|
282
|
+
import * as dagCodec from "@ipld/dag-cbor";
|
283
|
+
async function parseCarFile(reader, logger) {
|
284
|
+
const roots = await reader.getRoots();
|
285
|
+
const header = await reader.get(roots[0]);
|
286
|
+
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
287
|
+
const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
|
288
|
+
const fpvalue = dec.value;
|
289
|
+
if (fpvalue && !fpvalue.fp) {
|
290
|
+
throw logger.Error().Msg("missing fp").AsError();
|
220
291
|
}
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
if (
|
231
|
-
|
232
|
-
this._waitIdleItems.clear();
|
233
|
-
toResolve.map((fn) => fn.resolve());
|
292
|
+
return fpvalue.fp;
|
293
|
+
}
|
294
|
+
|
295
|
+
// src/blockstore/transaction.ts
|
296
|
+
import { MemoryBlockstore } from "@web3-storage/pail/block";
|
297
|
+
import { toCryptoRuntime } from "@adviser/cement";
|
298
|
+
var CarTransaction = class extends MemoryBlockstore {
|
299
|
+
constructor(parent, opts = { add: true, noLoader: false }) {
|
300
|
+
super();
|
301
|
+
if (opts.add) {
|
302
|
+
parent.transactions.add(this);
|
234
303
|
}
|
304
|
+
this.parent = parent;
|
305
|
+
}
|
306
|
+
async get(cid) {
|
307
|
+
return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
|
308
|
+
}
|
309
|
+
async superGet(cid) {
|
310
|
+
return super.get(cid);
|
235
311
|
}
|
236
312
|
};
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
242
|
-
keyedCryptoFactory: () => keyedCryptoFactory
|
243
|
-
});
|
244
|
-
import { base58btc } from "multiformats/bases/base58";
|
245
|
-
import { sha256 as hasher } from "multiformats/hashes/sha2";
|
246
|
-
import * as CBOR from "cborg";
|
247
|
-
var generateIV = {
|
248
|
-
random: {
|
313
|
+
function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
|
314
|
+
const logger = ensureLogger(sthis, component, ctx);
|
315
|
+
const store = opts.store || {};
|
316
|
+
return {
|
249
317
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
250
|
-
|
251
|
-
return
|
318
|
+
applyMeta: (meta, snap) => {
|
319
|
+
return Promise.resolve();
|
252
320
|
},
|
253
321
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
254
|
-
|
255
|
-
return
|
256
|
-
}
|
257
|
-
},
|
258
|
-
hash: {
|
259
|
-
calc: async (ko, crypto, data) => {
|
260
|
-
const hash = await hasher.digest(data);
|
261
|
-
const hashBytes = new Uint8Array(hash.bytes);
|
262
|
-
const hashArray = new Uint8Array(ko.ivLength);
|
263
|
-
for (let i = 0; i < hashBytes.length; i++) {
|
264
|
-
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
265
|
-
}
|
266
|
-
return hashArray;
|
322
|
+
compact: async (blocks) => {
|
323
|
+
return {};
|
267
324
|
},
|
268
|
-
|
269
|
-
|
270
|
-
|
325
|
+
autoCompact: 100,
|
326
|
+
public: false,
|
327
|
+
name: void 0,
|
328
|
+
threshold: 1e3 * 1e3,
|
329
|
+
...opts,
|
330
|
+
logger,
|
331
|
+
keyBag: opts.keyBag || {},
|
332
|
+
crypto: toCryptoRuntime(opts.crypto),
|
333
|
+
store,
|
334
|
+
storeRuntime: toStoreRuntime(store, sthis)
|
335
|
+
};
|
336
|
+
}
|
337
|
+
function blockstoreFactory(sthis, opts) {
|
338
|
+
if (opts.name) {
|
339
|
+
return new EncryptedBlockstore(sthis, opts);
|
340
|
+
} else {
|
341
|
+
return new BaseBlockstore(opts);
|
271
342
|
}
|
272
|
-
};
|
273
|
-
function getGenerateIVFn(url, opts) {
|
274
|
-
const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
|
275
|
-
return generateIV[ivhash] || generateIV["hash"];
|
276
343
|
}
|
277
|
-
var
|
278
|
-
constructor(
|
279
|
-
this.
|
280
|
-
this.
|
281
|
-
this.
|
282
|
-
this.
|
283
|
-
this.opts = opts || {};
|
344
|
+
var BaseBlockstore = class {
|
345
|
+
constructor(ebOpts = {}) {
|
346
|
+
this.transactions = /* @__PURE__ */ new Set();
|
347
|
+
this.sthis = ensureSuperThis(ebOpts);
|
348
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
349
|
+
this.logger = this.ebOpts.logger;
|
284
350
|
}
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
const fprt = await this.ko.fingerPrint();
|
289
|
-
const keyId = base58btc.decode(fprt);
|
290
|
-
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
291
|
-
return CBOR.encode({
|
292
|
-
iv,
|
293
|
-
keyId,
|
294
|
-
data: await this.ko._encrypt({ iv, bytes: data })
|
295
|
-
});
|
351
|
+
// ready: Promise<void>;
|
352
|
+
ready() {
|
353
|
+
return Promise.resolve();
|
296
354
|
}
|
297
|
-
async
|
298
|
-
let bytes;
|
299
|
-
if (abytes instanceof Uint8Array) {
|
300
|
-
bytes = abytes;
|
301
|
-
} else {
|
302
|
-
bytes = new Uint8Array(abytes);
|
303
|
-
}
|
304
|
-
const { iv, keyId, data } = CBOR.decode(bytes);
|
305
|
-
const fprt = await this.ko.fingerPrint();
|
306
|
-
this.ko.logger.Debug().Str("fp", base58btc.encode(keyId)).Msg("decode");
|
307
|
-
if (base58btc.encode(keyId) !== fprt) {
|
308
|
-
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
|
309
|
-
}
|
310
|
-
const result = await this.ko._decrypt({ iv, bytes: data });
|
311
|
-
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
312
|
-
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
313
|
-
}
|
314
|
-
return result;
|
355
|
+
async close() {
|
315
356
|
}
|
316
|
-
|
317
|
-
var keyedCrypto = class {
|
318
|
-
constructor(url, key, cyopt, sthis) {
|
319
|
-
this.ivLength = 12;
|
320
|
-
this.isEncrypting = true;
|
321
|
-
this.logger = ensureLogger(sthis, "keyedCrypto");
|
322
|
-
this.crypto = cyopt;
|
323
|
-
this.key = key;
|
324
|
-
this.url = url;
|
325
|
-
}
|
326
|
-
fingerPrint() {
|
327
|
-
return Promise.resolve(this.key.fingerPrint);
|
357
|
+
async destroy() {
|
328
358
|
}
|
329
|
-
|
330
|
-
return new BlockIvKeyIdCodec(this, iv, opts);
|
359
|
+
async compact() {
|
331
360
|
}
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
}
|
361
|
+
async get(cid) {
|
362
|
+
if (!cid) throw this.logger.Error().Msg("required cid").AsError();
|
363
|
+
for (const f of this.transactions) {
|
364
|
+
const v = await f.superGet(cid);
|
365
|
+
if (v) return v;
|
366
|
+
}
|
338
367
|
}
|
339
|
-
|
340
|
-
|
341
|
-
|
368
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
369
|
+
async put(cid, block) {
|
370
|
+
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
342
371
|
}
|
343
|
-
|
344
|
-
|
345
|
-
const
|
346
|
-
|
372
|
+
// TransactionMeta
|
373
|
+
async transaction(fn, _opts) {
|
374
|
+
const t = new CarTransaction(this, _opts);
|
375
|
+
const done = await fn(t);
|
376
|
+
this.lastTxMeta = done;
|
377
|
+
return { t, meta: done };
|
347
378
|
}
|
348
|
-
}
|
349
|
-
|
350
|
-
constructor() {
|
351
|
-
this.code = 0;
|
352
|
-
this.name = "Fireproof@unencrypted-block";
|
379
|
+
openTransaction(opts = { add: true, noLoader: false }) {
|
380
|
+
return new CarTransaction(this, opts);
|
353
381
|
}
|
354
|
-
|
355
|
-
|
382
|
+
async commitTransaction(t, done, opts) {
|
383
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
|
384
|
+
const cars = await this.loader?.commit(t, done, opts);
|
385
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
386
|
+
setTimeout(() => void this.compact(), 10);
|
387
|
+
}
|
388
|
+
if (cars) {
|
389
|
+
this.transactions.delete(t);
|
390
|
+
return { meta: done, cars, t };
|
391
|
+
}
|
392
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
356
393
|
}
|
357
|
-
|
358
|
-
|
394
|
+
async *entries() {
|
395
|
+
const seen = /* @__PURE__ */ new Set();
|
396
|
+
for (const t of this.transactions) {
|
397
|
+
for await (const blk of t.entries()) {
|
398
|
+
if (seen.has(blk.cid.toString())) continue;
|
399
|
+
seen.add(blk.cid.toString());
|
400
|
+
yield blk;
|
401
|
+
}
|
402
|
+
}
|
359
403
|
}
|
360
404
|
};
|
361
|
-
var
|
362
|
-
constructor(
|
363
|
-
|
364
|
-
this.
|
365
|
-
this.
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
this.
|
371
|
-
|
372
|
-
fingerPrint() {
|
373
|
-
return Promise.resolve(this._fingerPrint);
|
374
|
-
}
|
375
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
376
|
-
codec(iv) {
|
377
|
-
return new nullCodec();
|
405
|
+
var EncryptedBlockstore = class extends BaseBlockstore {
|
406
|
+
constructor(sthis, ebOpts) {
|
407
|
+
super(ebOpts);
|
408
|
+
this.compacting = false;
|
409
|
+
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
|
410
|
+
const { name } = ebOpts;
|
411
|
+
if (!name) {
|
412
|
+
throw this.logger.Error().Msg("name required").AsError();
|
413
|
+
}
|
414
|
+
this.name = name;
|
415
|
+
this.loader = new Loader(this.name, ebOpts, sthis);
|
378
416
|
}
|
379
|
-
|
380
|
-
|
381
|
-
return {
|
382
|
-
name: "noCrypto",
|
383
|
-
iv: new Uint8Array(),
|
384
|
-
tagLength: 0
|
385
|
-
};
|
417
|
+
ready() {
|
418
|
+
return this.loader.ready();
|
386
419
|
}
|
387
|
-
|
388
|
-
|
420
|
+
close() {
|
421
|
+
return this.loader.close();
|
389
422
|
}
|
390
|
-
|
391
|
-
|
423
|
+
destroy() {
|
424
|
+
return this.loader.destroy();
|
392
425
|
}
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
if (rkey.isErr()) {
|
399
|
-
try {
|
400
|
-
rkey = await kb.toKeyWithFingerPrint(storekey);
|
401
|
-
} catch (e) {
|
402
|
-
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
403
|
-
}
|
426
|
+
async get(cid) {
|
427
|
+
const got = await super.get(cid);
|
428
|
+
if (got) return got;
|
429
|
+
if (!this.loader) {
|
430
|
+
return;
|
404
431
|
}
|
405
|
-
return
|
406
|
-
}
|
407
|
-
return new noCrypto(url, kb.rt.crypto, sthis);
|
408
|
-
}
|
409
|
-
|
410
|
-
// src/blockstore/fragment-gateway.ts
|
411
|
-
import { Result as Result2 } from "@adviser/cement";
|
412
|
-
import { base58btc as base58btc2 } from "multiformats/bases/base58";
|
413
|
-
import { encode as encode2, decode as decode2 } from "cborg";
|
414
|
-
function getFragSize(url) {
|
415
|
-
const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
|
416
|
-
let ret = 0;
|
417
|
-
if (fragSize) {
|
418
|
-
ret = parseInt(fragSize);
|
419
|
-
}
|
420
|
-
if (isNaN(ret) || ret <= 0) {
|
421
|
-
ret = 0;
|
432
|
+
return falsyToUndef(await this.loader.getBlock(cid));
|
422
433
|
}
|
423
|
-
|
424
|
-
}
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
const res = await innerGW.get(url);
|
429
|
-
if (res.isErr()) {
|
430
|
-
return [res];
|
434
|
+
async transaction(fn, opts = { noLoader: false }) {
|
435
|
+
const { t, meta: done } = await super.transaction(fn);
|
436
|
+
const cars = await this.loader.commit(t, done, opts);
|
437
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
438
|
+
setTimeout(() => void this.compact(), 10);
|
431
439
|
}
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
len: data.length,
|
438
|
-
data
|
439
|
-
})
|
440
|
-
];
|
441
|
-
}
|
442
|
-
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
443
|
-
if (firstRaw.isErr()) {
|
444
|
-
return [firstRaw];
|
440
|
+
if (cars) {
|
441
|
+
this.transactions.delete(t);
|
442
|
+
return { meta: done, cars, t };
|
443
|
+
}
|
444
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
445
445
|
}
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
ops.push(
|
453
|
-
(async (furl, ofs2) => {
|
454
|
-
const raw2 = await innerGW.get(furl);
|
455
|
-
if (raw2.isErr()) {
|
456
|
-
return raw2;
|
457
|
-
}
|
458
|
-
const fragment = decode2(raw2.unwrap());
|
459
|
-
if (base58btc2.encode(fragment.fid) !== fidStr) {
|
460
|
-
return Result2.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
461
|
-
}
|
462
|
-
if (fragment.ofs !== ofs2) {
|
463
|
-
return Result2.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
464
|
-
}
|
465
|
-
return Result2.Ok(fragment);
|
466
|
-
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
446
|
+
async getFile(car, cid) {
|
447
|
+
await this.ready();
|
448
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
449
|
+
const reader = await this.loader.loadFileCar(
|
450
|
+
car
|
451
|
+
/*, isPublic */
|
467
452
|
);
|
453
|
+
const block = await reader.get(cid);
|
454
|
+
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
455
|
+
return block.bytes;
|
468
456
|
}
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
this.
|
474
|
-
this.
|
475
|
-
|
476
|
-
this.
|
477
|
-
|
457
|
+
async compact() {
|
458
|
+
await this.ready();
|
459
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
|
460
|
+
if (this.loader.carLog.length < 2) return;
|
461
|
+
const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
|
462
|
+
if (!compactFn || this.compacting) return;
|
463
|
+
const blockLog = new CompactionFetcher(this);
|
464
|
+
this.compacting = true;
|
465
|
+
const meta = await compactFn(blockLog);
|
466
|
+
await this.loader?.commit(blockLog.loggedBlocks, meta, {
|
467
|
+
compact: true,
|
468
|
+
noLoader: true
|
469
|
+
});
|
470
|
+
this.compacting = false;
|
478
471
|
}
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
return [this.innerGW.put(url, body)];
|
472
|
+
async defaultCompact(blocks, logger) {
|
473
|
+
if (!this.loader) {
|
474
|
+
throw logger.Error().Msg("no loader").AsError();
|
483
475
|
}
|
484
|
-
|
485
|
-
|
486
|
-
throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
|
476
|
+
if (!this.lastTxMeta) {
|
477
|
+
throw logger.Error().Msg("no lastTxMeta").AsError();
|
487
478
|
}
|
488
|
-
const
|
489
|
-
|
490
|
-
|
491
|
-
for (
|
492
|
-
const
|
493
|
-
|
494
|
-
ofs,
|
495
|
-
len: body.length,
|
496
|
-
data: body.slice(ofs, ofs + blocksize)
|
497
|
-
});
|
498
|
-
if (block.length > fragSize) {
|
499
|
-
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
479
|
+
for await (const blk of this.loader.entries(false)) {
|
480
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
481
|
+
}
|
482
|
+
for (const t of this.transactions) {
|
483
|
+
for await (const blk of t.entries()) {
|
484
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
500
485
|
}
|
501
|
-
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
502
486
|
}
|
503
|
-
return
|
504
|
-
}
|
505
|
-
buildUrl(baseUrl, key) {
|
506
|
-
return this.innerGW.buildUrl(baseUrl, key);
|
487
|
+
return this.lastTxMeta;
|
507
488
|
}
|
508
|
-
async
|
509
|
-
|
489
|
+
async *entries() {
|
490
|
+
for await (const blk of this.loader.entries()) {
|
491
|
+
yield blk;
|
492
|
+
}
|
510
493
|
}
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
len: 16 * 1024 * 1024,
|
517
|
-
// 32bit
|
518
|
-
data: new Uint8Array(1024)
|
519
|
-
}).length - 1024;
|
520
|
-
return this.innerGW.start(url);
|
521
|
-
}
|
522
|
-
async close(url) {
|
523
|
-
return this.innerGW.close(url);
|
494
|
+
};
|
495
|
+
var CompactionFetcher = class {
|
496
|
+
constructor(blocks) {
|
497
|
+
this.blockstore = blocks;
|
498
|
+
this.loggedBlocks = new CarTransaction(blocks);
|
524
499
|
}
|
525
|
-
async
|
526
|
-
await
|
527
|
-
|
500
|
+
async get(cid) {
|
501
|
+
const block = await this.blockstore.get(cid);
|
502
|
+
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
503
|
+
return falsyToUndef(block);
|
528
504
|
}
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
buffer.set(frag.data, frag.ofs);
|
539
|
-
}
|
540
|
-
return Result2.Ok(buffer || new Uint8Array(0));
|
505
|
+
};
|
506
|
+
|
507
|
+
// src/blockstore/commit-queue.ts
|
508
|
+
import { Future } from "@adviser/cement";
|
509
|
+
var CommitQueue = class {
|
510
|
+
constructor() {
|
511
|
+
this.queue = [];
|
512
|
+
this.processing = false;
|
513
|
+
this._waitIdleItems = /* @__PURE__ */ new Set();
|
541
514
|
}
|
542
|
-
|
543
|
-
if (this.
|
544
|
-
return
|
545
|
-
} else {
|
546
|
-
return Result2.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
|
515
|
+
waitIdle() {
|
516
|
+
if (this.queue.length === 0 && !this.processing) {
|
517
|
+
return Promise.resolve();
|
547
518
|
}
|
519
|
+
const fn = new Future();
|
520
|
+
this._waitIdleItems.add(fn);
|
521
|
+
return fn.asPromise();
|
548
522
|
}
|
549
|
-
async
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
523
|
+
async enqueue(fn) {
|
524
|
+
return new Promise((resolve, reject) => {
|
525
|
+
const queueFn = async () => {
|
526
|
+
try {
|
527
|
+
resolve(await fn());
|
528
|
+
} catch (e) {
|
529
|
+
reject(e);
|
530
|
+
} finally {
|
531
|
+
this.processing = false;
|
532
|
+
this.processNext();
|
533
|
+
}
|
534
|
+
};
|
535
|
+
this.queue.push(queueFn);
|
536
|
+
if (!this.processing) {
|
537
|
+
this.processNext();
|
554
538
|
}
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
539
|
+
});
|
540
|
+
}
|
541
|
+
processNext() {
|
542
|
+
if (this.queue.length > 0 && !this.processing) {
|
543
|
+
this.processing = true;
|
544
|
+
const queueFn = this.queue.shift();
|
545
|
+
if (queueFn) {
|
546
|
+
queueFn().finally(() => {
|
547
|
+
});
|
562
548
|
}
|
563
|
-
await this.innerGW.delete(fragUrl);
|
564
549
|
}
|
565
|
-
|
550
|
+
if (this.queue.length === 0 && !this.processing) {
|
551
|
+
const toResolve = Array.from(this._waitIdleItems);
|
552
|
+
this._waitIdleItems.clear();
|
553
|
+
toResolve.map((fn) => fn.resolve());
|
554
|
+
}
|
566
555
|
}
|
567
556
|
};
|
568
557
|
|
569
|
-
// src/blockstore/meta-key-helper.ts
|
570
|
-
import { format, parse } from "@ipld/dag-json";
|
571
|
-
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
572
|
-
import { CID } from "multiformats";
|
573
|
-
import { base64pad } from "multiformats/bases/base64";
|
574
|
-
import { Result as Result4 } from "@adviser/cement";
|
575
|
-
|
576
558
|
// src/runtime/key-bag.ts
|
577
559
|
var key_bag_exports = {};
|
578
560
|
__export(key_bag_exports, {
|
579
561
|
KeyBag: () => KeyBag,
|
580
|
-
defaultKeyBagOpts: () => defaultKeyBagOpts,
|
581
562
|
getKeyBag: () => getKeyBag,
|
582
563
|
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
583
564
|
});
|
@@ -585,20 +566,18 @@ import {
|
|
585
566
|
KeyedResolvOnce,
|
586
567
|
ResolveOnce,
|
587
568
|
ResolveSeq,
|
588
|
-
Result as
|
569
|
+
Result as Result2,
|
589
570
|
runtimeFn,
|
590
|
-
toCryptoRuntime,
|
591
|
-
URI
|
571
|
+
toCryptoRuntime as toCryptoRuntime2,
|
572
|
+
URI
|
592
573
|
} from "@adviser/cement";
|
593
|
-
import { base58btc
|
574
|
+
import { base58btc } from "multiformats/bases/base58";
|
594
575
|
var KeyBag = class {
|
595
576
|
constructor(rt) {
|
596
577
|
this.rt = rt;
|
597
578
|
this._warnOnce = new ResolveOnce();
|
598
579
|
this._seq = new ResolveSeq();
|
599
|
-
this.logger = ensureLogger(rt.sthis, "KeyBag"
|
600
|
-
id: rt.id()
|
601
|
-
});
|
580
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag");
|
602
581
|
this.logger.Debug().Msg("KeyBag created");
|
603
582
|
}
|
604
583
|
async subtleKey(key) {
|
@@ -611,7 +590,7 @@ var KeyBag = class {
|
|
611
590
|
return await this.rt.crypto.importKey(
|
612
591
|
"raw",
|
613
592
|
// raw or jwk
|
614
|
-
|
593
|
+
base58btc.decode(key),
|
615
594
|
// hexStringToUint8Array(key), // raw data
|
616
595
|
"AES-GCM",
|
617
596
|
extractable,
|
@@ -619,9 +598,9 @@ var KeyBag = class {
|
|
619
598
|
);
|
620
599
|
}
|
621
600
|
async ensureKeyFromUrl(url, keyFactory) {
|
622
|
-
const storeKey = url.getParam("storekey"
|
601
|
+
const storeKey = url.getParam("storekey");
|
623
602
|
if (storeKey === "insecure") {
|
624
|
-
return
|
603
|
+
return Result2.Ok(url);
|
625
604
|
}
|
626
605
|
if (!storeKey) {
|
627
606
|
const keyName = `@${keyFactory()}@`;
|
@@ -629,8 +608,8 @@ var KeyBag = class {
|
|
629
608
|
if (ret.isErr()) {
|
630
609
|
return ret;
|
631
610
|
}
|
632
|
-
const urb = url.build().setParam("storekey"
|
633
|
-
return
|
611
|
+
const urb = url.build().setParam("storekey", keyName);
|
612
|
+
return Result2.Ok(urb.URI());
|
634
613
|
}
|
635
614
|
if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
|
636
615
|
const ret = await this.getNamedKey(storeKey);
|
@@ -638,15 +617,15 @@ var KeyBag = class {
|
|
638
617
|
return ret;
|
639
618
|
}
|
640
619
|
}
|
641
|
-
return
|
620
|
+
return Result2.Ok(url);
|
642
621
|
}
|
643
622
|
async toKeyWithFingerPrint(keyStr) {
|
644
|
-
const material =
|
623
|
+
const material = base58btc.decode(keyStr);
|
645
624
|
const key = await this.subtleKey(keyStr);
|
646
625
|
const fpr = await this.rt.crypto.digestSHA256(material);
|
647
|
-
return
|
626
|
+
return Result2.Ok({
|
648
627
|
key,
|
649
|
-
fingerPrint:
|
628
|
+
fingerPrint: base58btc.encode(new Uint8Array(fpr))
|
650
629
|
});
|
651
630
|
}
|
652
631
|
async setNamedKey(name, key) {
|
@@ -669,13 +648,13 @@ var KeyBag = class {
|
|
669
648
|
return ret;
|
670
649
|
}
|
671
650
|
const named = ret.Ok();
|
672
|
-
return
|
651
|
+
return Result2.Ok({
|
673
652
|
...named,
|
674
653
|
extract: async () => {
|
675
654
|
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
676
655
|
return {
|
677
656
|
key: ext,
|
678
|
-
keyStr:
|
657
|
+
keyStr: base58btc.encode(ext)
|
679
658
|
};
|
680
659
|
}
|
681
660
|
});
|
@@ -692,9 +671,9 @@ var KeyBag = class {
|
|
692
671
|
}
|
693
672
|
if (failIfNotFound) {
|
694
673
|
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
695
|
-
return
|
674
|
+
return Result2.Err(new Error(`Key not found: ${name}`));
|
696
675
|
}
|
697
|
-
const ret = await this._setNamedKey(name,
|
676
|
+
const ret = await this._setNamedKey(name, base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
698
677
|
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
699
678
|
return ret;
|
700
679
|
});
|
@@ -705,14 +684,14 @@ var keyBagProviderFactories = new Map(
|
|
705
684
|
{
|
706
685
|
protocol: "file:",
|
707
686
|
factory: async (url, sthis) => {
|
708
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-
|
687
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-WADZBHYG.js");
|
709
688
|
return new KeyBagProviderFile(url, sthis);
|
710
689
|
}
|
711
690
|
},
|
712
691
|
{
|
713
692
|
protocol: "indexdb:",
|
714
693
|
factory: async (url, sthis) => {
|
715
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-
|
694
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-PGVAI3FJ.js");
|
716
695
|
return new KeyBagProviderIndexDB(url, sthis);
|
717
696
|
}
|
718
697
|
}
|
@@ -726,63 +705,44 @@ function registerKeyBagProviderFactory(item) {
|
|
726
705
|
});
|
727
706
|
}
|
728
707
|
function defaultKeyBagOpts(sthis, kbo) {
|
729
|
-
kbo = kbo || {};
|
730
708
|
if (kbo.keyRuntime) {
|
731
709
|
return kbo.keyRuntime;
|
732
710
|
}
|
733
711
|
const logger = ensureLogger(sthis, "KeyBag");
|
734
712
|
let url;
|
735
713
|
if (kbo.url) {
|
736
|
-
url =
|
714
|
+
url = URI.from(kbo.url);
|
737
715
|
logger.Debug().Url(url).Msg("from opts");
|
738
716
|
} else {
|
739
717
|
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
740
718
|
if (runtimeFn().isBrowser) {
|
741
|
-
url =
|
719
|
+
url = URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
742
720
|
} else {
|
743
721
|
if (!bagFnameOrUrl) {
|
744
722
|
const home = sthis.env.get("HOME");
|
745
723
|
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
746
|
-
url =
|
724
|
+
url = URI.from(`file://${bagFnameOrUrl}`);
|
747
725
|
} else {
|
748
|
-
url =
|
726
|
+
url = URI.from(bagFnameOrUrl);
|
749
727
|
}
|
750
728
|
}
|
751
729
|
logger.Debug().Url(url).Msg("from env");
|
752
730
|
}
|
753
|
-
|
754
|
-
|
755
|
-
|
756
|
-
keyProviderFactory = async () => {
|
757
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
|
758
|
-
return new KeyBagProviderFile(url, sthis);
|
759
|
-
};
|
760
|
-
break;
|
761
|
-
case "indexdb:":
|
762
|
-
keyProviderFactory = async () => {
|
763
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
|
764
|
-
return new KeyBagProviderIndexDB(url, sthis);
|
765
|
-
};
|
766
|
-
break;
|
767
|
-
case "memory:":
|
768
|
-
keyProviderFactory = async () => {
|
769
|
-
const { KeyBagProviderMemory } = await import("./key-bag-memory-LWE6ARPX.js");
|
770
|
-
return new KeyBagProviderMemory(url, sthis);
|
771
|
-
};
|
772
|
-
break;
|
773
|
-
default:
|
774
|
-
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
731
|
+
const kitem = keyBagProviderFactories.get(url.protocol);
|
732
|
+
if (!kitem) {
|
733
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
775
734
|
}
|
735
|
+
const getBag = async () => kitem.factory(url, sthis);
|
776
736
|
if (url.hasParam("masterkey")) {
|
777
737
|
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
778
738
|
}
|
779
739
|
return {
|
780
740
|
url,
|
781
|
-
crypto: kbo.crypto ||
|
741
|
+
crypto: kbo.crypto || toCryptoRuntime2({}),
|
782
742
|
sthis,
|
783
743
|
logger,
|
784
744
|
keyLength: kbo.keyLength || 16,
|
785
|
-
getBag
|
745
|
+
getBag,
|
786
746
|
id: () => {
|
787
747
|
return url.toString();
|
788
748
|
}
|
@@ -795,940 +755,936 @@ async function getKeyBag(sthis, kbo = {}) {
|
|
795
755
|
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
796
756
|
}
|
797
757
|
|
798
|
-
// src/blockstore/
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
758
|
+
// src/blockstore/commitor.ts
|
759
|
+
import * as CBW from "@ipld/car/buffer-writer";
|
760
|
+
import { sha256 as hasher2 } from "multiformats/hashes/sha2";
|
761
|
+
import * as dagCodec2 from "@ipld/dag-cbor";
|
762
|
+
async function encodeCarFile(roots, t, codec3) {
|
763
|
+
let size = 0;
|
764
|
+
const headerSize = CBW.headerLength({ roots });
|
765
|
+
size += headerSize;
|
766
|
+
for (const { cid, bytes } of t.entries()) {
|
767
|
+
size += CBW.blockLength({ cid, bytes });
|
804
768
|
}
|
805
|
-
|
806
|
-
|
807
|
-
|
769
|
+
const buffer = new Uint8Array(size);
|
770
|
+
const writer = CBW.createWriter(buffer, { headerSize });
|
771
|
+
for (const r of roots) {
|
772
|
+
writer.addRoot(r);
|
808
773
|
}
|
809
|
-
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
eventCid: eventBlock.cid,
|
815
|
-
parents: crdtEntry.parents,
|
816
|
-
dbMeta
|
817
|
-
};
|
818
|
-
})
|
819
|
-
);
|
774
|
+
for (const { cid, bytes } of t.entries()) {
|
775
|
+
writer.write({ cid, bytes });
|
776
|
+
}
|
777
|
+
writer.close();
|
778
|
+
return await encode({ value: writer.bytes, hasher: hasher2, codec: codec3 });
|
820
779
|
}
|
821
|
-
async function
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
|
837
|
-
return Result4.Ok(dbMeta);
|
838
|
-
}
|
839
|
-
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
840
|
-
return Result4.Ok(void 0);
|
841
|
-
} catch (error) {
|
842
|
-
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
843
|
-
return Result4.Err(error);
|
780
|
+
async function createCarFile(encoder, cid, t) {
|
781
|
+
return encodeCarFile([cid], t, encoder);
|
782
|
+
}
|
783
|
+
async function commitFiles(fileStore, walStore, t, done) {
|
784
|
+
const { files: roots } = makeFileCarHeader(done);
|
785
|
+
const cids = [];
|
786
|
+
const codec3 = (await fileStore.keyedCrypto()).codec();
|
787
|
+
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
788
|
+
for (const car of cars) {
|
789
|
+
const { cid, bytes } = car;
|
790
|
+
await fileStore.save({ cid, bytes });
|
791
|
+
await walStore.enqueueFile(
|
792
|
+
cid
|
793
|
+
/*, !!opts.public*/
|
794
|
+
);
|
795
|
+
cids.push(cid);
|
844
796
|
}
|
797
|
+
return cids;
|
845
798
|
}
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
const res = await kb.getNamedExtractableKey(keyName, true);
|
852
|
-
if (res.isErr()) {
|
853
|
-
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
854
|
-
throw res.Err();
|
799
|
+
function makeFileCarHeader(result) {
|
800
|
+
const files = [];
|
801
|
+
for (const [, meta] of Object.entries(result.files || {})) {
|
802
|
+
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
803
|
+
files.push(meta.cid);
|
855
804
|
}
|
856
|
-
const keyData = await res.Ok().extract();
|
857
|
-
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
858
|
-
const { dbMeta, parents } = dbMetas[0];
|
859
|
-
const parentLinks = parents.map((p) => CID.parse(p));
|
860
|
-
dbMeta.key = keyData.keyStr;
|
861
|
-
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
862
|
-
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
863
|
-
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
864
|
-
return Result4.Ok(encoded);
|
865
|
-
} catch (error) {
|
866
|
-
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
867
|
-
return Result4.Err(error);
|
868
805
|
}
|
806
|
+
return { ...result, files };
|
869
807
|
}
|
870
|
-
function
|
871
|
-
|
872
|
-
const idx = url.getParam("index");
|
873
|
-
if (idx) {
|
874
|
-
storeKeyName.push(idx);
|
875
|
-
}
|
876
|
-
storeKeyName.push("data");
|
877
|
-
return `@${storeKeyName.join(":")}@`;
|
808
|
+
async function prepareCarFilesFiles(encoder, roots, t) {
|
809
|
+
return [await encodeCarFile(roots, t, encoder)];
|
878
810
|
}
|
879
|
-
|
880
|
-
const
|
881
|
-
|
882
|
-
dbMeta: sthis.txt.encode(format(dbMeta))
|
883
|
-
},
|
884
|
-
parents
|
885
|
-
);
|
886
|
-
return event;
|
811
|
+
function makeCarHeader(meta, cars, compact = false) {
|
812
|
+
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
813
|
+
return { ...coreHeader, meta };
|
887
814
|
}
|
888
|
-
async function
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
data: base64String,
|
894
|
-
parents: parents.map((p) => p.toString())
|
895
|
-
};
|
815
|
+
async function encodeCarHeader(fp) {
|
816
|
+
return await encode({
|
817
|
+
value: { fp },
|
818
|
+
hasher: hasher2,
|
819
|
+
codec: dagCodec2
|
896
820
|
});
|
897
|
-
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
898
821
|
}
|
899
|
-
|
900
|
-
|
901
|
-
|
902
|
-
|
903
|
-
|
904
|
-
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
// src/runtime/wait-pr-multiformats/block.ts
|
910
|
-
var block_exports = {};
|
911
|
-
__export(block_exports, {
|
912
|
-
Block: () => Block,
|
913
|
-
create: () => create,
|
914
|
-
createUnsafe: () => createUnsafe,
|
915
|
-
decode: () => decode3,
|
916
|
-
encode: () => encode3
|
917
|
-
});
|
918
|
-
import { bytes as binary, CID as CID2 } from "multiformats";
|
919
|
-
import { Block as mfBlock } from "multiformats/block";
|
920
|
-
var Block = mfBlock;
|
921
|
-
async function decode3({
|
922
|
-
bytes,
|
923
|
-
codec: codec3,
|
924
|
-
hasher: hasher7
|
925
|
-
}) {
|
926
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
927
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
928
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
929
|
-
const hash = await hasher7.digest(bytes);
|
930
|
-
const cid = CID2.create(1, codec3.code, hash);
|
931
|
-
return new mfBlock({ value, bytes, cid });
|
932
|
-
}
|
933
|
-
async function encode3({
|
934
|
-
value,
|
935
|
-
codec: codec3,
|
936
|
-
hasher: hasher7
|
937
|
-
}) {
|
938
|
-
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
939
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
940
|
-
const bytes = await Promise.resolve(codec3.encode(value));
|
941
|
-
const hash = await hasher7.digest(bytes);
|
942
|
-
const cid = CID2.create(1, codec3.code, hash);
|
943
|
-
return new mfBlock({ value, bytes, cid });
|
944
|
-
}
|
945
|
-
async function create({
|
946
|
-
bytes,
|
947
|
-
cid,
|
948
|
-
hasher: hasher7,
|
949
|
-
codec: codec3
|
950
|
-
}) {
|
951
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
952
|
-
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
953
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
954
|
-
const hash = await hasher7.digest(bytes);
|
955
|
-
if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
|
956
|
-
throw new Error("CID hash does not match bytes");
|
822
|
+
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
823
|
+
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
824
|
+
const rootBlock = await encodeCarHeader(fp);
|
825
|
+
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
826
|
+
const cids = [];
|
827
|
+
for (const car of cars) {
|
828
|
+
const { cid, bytes } = car;
|
829
|
+
await params.carStore.save({ cid, bytes });
|
830
|
+
cids.push(cid);
|
957
831
|
}
|
958
|
-
|
959
|
-
|
960
|
-
|
961
|
-
|
962
|
-
codec: codec3
|
963
|
-
});
|
964
|
-
}
|
965
|
-
async function createUnsafe({
|
966
|
-
bytes,
|
967
|
-
cid,
|
968
|
-
value: maybeValue,
|
969
|
-
codec: codec3
|
970
|
-
}) {
|
971
|
-
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
972
|
-
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
973
|
-
return new Block({
|
974
|
-
cid,
|
975
|
-
bytes,
|
976
|
-
value
|
977
|
-
});
|
832
|
+
const newDbMeta = { cars: cids };
|
833
|
+
await params.WALStore.enqueue(newDbMeta, opts);
|
834
|
+
await params.metaStore.save(newDbMeta);
|
835
|
+
return { cgrp: cids, header: fp };
|
978
836
|
}
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
|
983
|
-
|
984
|
-
|
985
|
-
|
986
|
-
|
987
|
-
|
988
|
-
|
989
|
-
|
990
|
-
|
837
|
+
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
838
|
+
const carFiles = [];
|
839
|
+
threshold = threshold || 128e3 * 8;
|
840
|
+
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
841
|
+
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
842
|
+
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
843
|
+
let cidRootBlock = rootBlock;
|
844
|
+
for (const { cid, bytes } of t.entries()) {
|
845
|
+
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
846
|
+
if (newsize >= threshold) {
|
847
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
848
|
+
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
849
|
+
clonedt.putSync(cid, bytes);
|
850
|
+
cidRootBlock = { cid, bytes };
|
851
|
+
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
852
|
+
} else {
|
853
|
+
clonedt.putSync(cid, bytes);
|
854
|
+
}
|
991
855
|
}
|
992
|
-
|
856
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
857
|
+
return carFiles;
|
993
858
|
}
|
994
859
|
|
995
|
-
// src/blockstore/
|
996
|
-
import {
|
997
|
-
|
998
|
-
|
999
|
-
|
1000
|
-
|
1001
|
-
|
1002
|
-
|
1003
|
-
|
1004
|
-
this.
|
860
|
+
// src/blockstore/loader.ts
|
861
|
+
import { sha256 as hasher3 } from "multiformats/hashes/sha2";
|
862
|
+
|
863
|
+
// src/blockstore/task-manager.ts
|
864
|
+
var TaskManager = class {
|
865
|
+
constructor(sthis, callback) {
|
866
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
867
|
+
this.queue = [];
|
868
|
+
this.isProcessing = false;
|
869
|
+
this.logger = ensureLogger(sthis, "TaskManager");
|
870
|
+
this.callback = callback;
|
1005
871
|
}
|
1006
|
-
async
|
1007
|
-
|
872
|
+
async handleEvent(cid, parents, dbMeta) {
|
873
|
+
for (const parent of parents) {
|
874
|
+
this.eventsWeHandled.add(parent.toString());
|
875
|
+
}
|
876
|
+
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
877
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
878
|
+
void this.processQueue();
|
1008
879
|
}
|
1009
|
-
async
|
1010
|
-
|
880
|
+
async processQueue() {
|
881
|
+
if (this.isProcessing) return;
|
882
|
+
this.isProcessing = true;
|
883
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
884
|
+
const first = filteredQueue[0];
|
885
|
+
if (!first) {
|
886
|
+
return;
|
887
|
+
}
|
888
|
+
try {
|
889
|
+
await this.callback(first.dbMeta);
|
890
|
+
this.eventsWeHandled.add(first.cid);
|
891
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
892
|
+
} catch (err) {
|
893
|
+
if (first.retries++ > 3) {
|
894
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
895
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
896
|
+
}
|
897
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
898
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
899
|
+
} finally {
|
900
|
+
this.isProcessing = false;
|
901
|
+
if (this.queue.length > 0) {
|
902
|
+
void this.processQueue();
|
903
|
+
}
|
904
|
+
}
|
1011
905
|
}
|
1012
906
|
};
|
1013
|
-
|
1014
|
-
|
1015
|
-
|
1016
|
-
|
1017
|
-
|
1018
|
-
|
1019
|
-
},
|
1020
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1021
|
-
compact: async (blocks) => {
|
1022
|
-
return {};
|
1023
|
-
},
|
1024
|
-
autoCompact: 100,
|
1025
|
-
public: false,
|
1026
|
-
// name: undefined,
|
1027
|
-
threshold: 1e3 * 1e3,
|
1028
|
-
...opts,
|
1029
|
-
logger,
|
1030
|
-
keyBag: opts.keyBag || {},
|
1031
|
-
crypto: toCryptoRuntime2(opts.crypto),
|
1032
|
-
storeUrls: opts.storeUrls,
|
1033
|
-
// storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
|
1034
|
-
// store,
|
1035
|
-
storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
|
1036
|
-
};
|
907
|
+
|
908
|
+
// src/blockstore/loader.ts
|
909
|
+
function carLogIncludesGroup(list, cids) {
|
910
|
+
return list.some((arr) => {
|
911
|
+
return arr.toString() === cids.toString();
|
912
|
+
});
|
1037
913
|
}
|
1038
|
-
function
|
1039
|
-
|
914
|
+
function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
915
|
+
const byString = /* @__PURE__ */ new Map();
|
916
|
+
for (const cid of list) {
|
917
|
+
if (remove.has(cid.toString())) continue;
|
918
|
+
byString.set(cid.toString(), cid);
|
919
|
+
}
|
920
|
+
return [...byString.values()];
|
1040
921
|
}
|
1041
|
-
var
|
1042
|
-
constructor(ebOpts) {
|
1043
|
-
this.
|
1044
|
-
this.
|
1045
|
-
this.
|
1046
|
-
this.
|
1047
|
-
this.
|
922
|
+
var Loader = class {
|
923
|
+
constructor(name, ebOpts, sthis) {
|
924
|
+
this.commitQueue = new CommitQueue();
|
925
|
+
this.isCompacting = false;
|
926
|
+
this.carReaders = /* @__PURE__ */ new Map();
|
927
|
+
this.seenCompacted = /* @__PURE__ */ new Set();
|
928
|
+
this.processedCars = /* @__PURE__ */ new Set();
|
929
|
+
this.carLog = [];
|
930
|
+
this.getBlockCache = /* @__PURE__ */ new Map();
|
931
|
+
this.seenMeta = /* @__PURE__ */ new Set();
|
932
|
+
this.writeLimit = pLimit(1);
|
933
|
+
this.onceReady = new ResolveOnce2();
|
934
|
+
this.name = name;
|
935
|
+
this.sthis = sthis;
|
936
|
+
this.ebOpts = defaultedBlockstoreRuntime(
|
937
|
+
sthis,
|
938
|
+
{
|
939
|
+
...ebOpts,
|
940
|
+
name
|
941
|
+
},
|
942
|
+
"Loader"
|
943
|
+
);
|
1048
944
|
this.logger = this.ebOpts.logger;
|
945
|
+
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
946
|
+
await this.handleDbMetasFromStore([dbMeta]);
|
947
|
+
});
|
1049
948
|
}
|
1050
|
-
// readonly
|
1051
|
-
|
1052
|
-
|
1053
|
-
return Promise.resolve();
|
1054
|
-
}
|
1055
|
-
async close() {
|
949
|
+
// readonly id = uuidv4();
|
950
|
+
async keyBag() {
|
951
|
+
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
1056
952
|
}
|
1057
|
-
async
|
953
|
+
async carStore() {
|
954
|
+
return this.ebOpts.storeRuntime.makeDataStore(this);
|
1058
955
|
}
|
1059
|
-
async
|
956
|
+
async fileStore() {
|
957
|
+
return this.ebOpts.storeRuntime.makeDataStore(this);
|
1060
958
|
}
|
1061
|
-
async
|
1062
|
-
|
1063
|
-
for (const f of this.transactions) {
|
1064
|
-
const v = await f.superGet(cid);
|
1065
|
-
if (v) return v;
|
1066
|
-
}
|
959
|
+
async WALStore() {
|
960
|
+
return this.ebOpts.storeRuntime.makeWALStore(this);
|
1067
961
|
}
|
1068
|
-
|
1069
|
-
|
1070
|
-
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
962
|
+
async metaStore() {
|
963
|
+
return this.ebOpts.storeRuntime.makeMetaStore(this);
|
1071
964
|
}
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1080
|
-
|
965
|
+
async ready() {
|
966
|
+
return this.onceReady.once(async () => {
|
967
|
+
const metas = await (await this.metaStore()).load();
|
968
|
+
if (this.ebOpts.meta) {
|
969
|
+
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
970
|
+
} else if (metas) {
|
971
|
+
await this.handleDbMetasFromStore(metas);
|
972
|
+
}
|
973
|
+
});
|
1081
974
|
}
|
1082
|
-
|
1083
|
-
|
975
|
+
async close() {
|
976
|
+
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
977
|
+
await Promise.all(toClose.map((store) => store.close()));
|
1084
978
|
}
|
1085
|
-
async
|
1086
|
-
|
1087
|
-
|
1088
|
-
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
1089
|
-
setTimeout(() => void this.compact(), 10);
|
1090
|
-
}
|
1091
|
-
if (cars) {
|
1092
|
-
this.transactions.delete(t);
|
1093
|
-
return { meta: done, cars, t };
|
1094
|
-
}
|
1095
|
-
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
979
|
+
async destroy() {
|
980
|
+
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
981
|
+
await Promise.all(toDestroy.map((store) => store.destroy()));
|
1096
982
|
}
|
1097
|
-
async
|
1098
|
-
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
983
|
+
// async snapToCar(carCid: AnyLink | string) {
|
984
|
+
// await this.ready
|
985
|
+
// if (typeof carCid === 'string') {
|
986
|
+
// carCid = CID.parse(carCid)
|
987
|
+
// }
|
988
|
+
// const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
|
989
|
+
// this.carLog = [carCid, ...carHeader.cars]
|
990
|
+
// await this.getMoreReaders(carHeader.cars)
|
991
|
+
// await this._applyCarHeader(carHeader, true)
|
992
|
+
// }
|
993
|
+
async handleDbMetasFromStore(metas) {
|
994
|
+
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
995
|
+
for (const meta of metas) {
|
996
|
+
await this.writeLimit(async () => {
|
997
|
+
await this.mergeDbMetaIntoClock(meta);
|
998
|
+
});
|
1105
999
|
}
|
1106
1000
|
}
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
this.
|
1112
|
-
this.
|
1113
|
-
|
1114
|
-
});
|
1115
|
-
this.loader = new Loader(sthis, ebOpts);
|
1116
|
-
}
|
1117
|
-
ready() {
|
1118
|
-
return this.loader.ready();
|
1119
|
-
}
|
1120
|
-
close() {
|
1121
|
-
return this.loader.close();
|
1122
|
-
}
|
1123
|
-
destroy() {
|
1124
|
-
return this.loader.destroy();
|
1125
|
-
}
|
1126
|
-
async get(cid) {
|
1127
|
-
const got = await super.get(cid);
|
1128
|
-
if (got) return got;
|
1129
|
-
if (!this.loader) {
|
1001
|
+
async mergeDbMetaIntoClock(meta) {
|
1002
|
+
if (this.isCompacting) {
|
1003
|
+
throw this.logger.Error().Msg("cannot merge while compacting").AsError();
|
1004
|
+
}
|
1005
|
+
if (this.seenMeta.has(meta.cars.toString())) return;
|
1006
|
+
this.seenMeta.add(meta.cars.toString());
|
1007
|
+
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
1130
1008
|
return;
|
1131
1009
|
}
|
1132
|
-
|
1010
|
+
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
1011
|
+
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1012
|
+
await this.getMoreReaders(carHeader.cars.flat());
|
1013
|
+
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
1014
|
+
await this.ebOpts.applyMeta?.(carHeader.meta);
|
1133
1015
|
}
|
1134
|
-
async
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
if (cars) {
|
1144
|
-
this.transactions.delete(t);
|
1145
|
-
return { meta: done, cars, t };
|
1146
|
-
}
|
1147
|
-
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1016
|
+
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1017
|
+
// const { key } = meta;
|
1018
|
+
// if (key) {
|
1019
|
+
// await this.setKey(key);
|
1020
|
+
// }
|
1021
|
+
// }
|
1022
|
+
async loadCarHeaderFromMeta({ cars: cids }) {
|
1023
|
+
const reader = await this.loadCar(cids[0]);
|
1024
|
+
return await parseCarFile(reader, this.logger);
|
1148
1025
|
}
|
1149
|
-
async
|
1026
|
+
// async _getKey(): Promise<string | undefined> {
|
1027
|
+
// if (this.key) return this.key;
|
1028
|
+
// // generate a random key
|
1029
|
+
// if (!this.ebOpts.public) {
|
1030
|
+
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1031
|
+
// }
|
1032
|
+
// return this.key || undefined;
|
1033
|
+
// }
|
1034
|
+
async commitFiles(t, done) {
|
1150
1035
|
await this.ready();
|
1151
|
-
|
1152
|
-
const
|
1153
|
-
|
1154
|
-
/*, isPublic */
|
1155
|
-
);
|
1156
|
-
const block = await reader.get(cid);
|
1157
|
-
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
1158
|
-
return block.bytes;
|
1036
|
+
const fstore = await this.fileStore();
|
1037
|
+
const wstore = await this.WALStore();
|
1038
|
+
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1159
1039
|
}
|
1160
|
-
async
|
1040
|
+
async loadFileCar(cid) {
|
1041
|
+
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
1042
|
+
}
|
1043
|
+
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
1161
1044
|
await this.ready();
|
1162
|
-
|
1163
|
-
|
1164
|
-
|
1165
|
-
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1045
|
+
const fstore = await this.fileStore();
|
1046
|
+
const params = {
|
1047
|
+
encoder: (await fstore.keyedCrypto()).codec(),
|
1048
|
+
carLog: this.carLog,
|
1049
|
+
carStore: fstore,
|
1050
|
+
WALStore: await this.WALStore(),
|
1051
|
+
metaStore: await this.metaStore(),
|
1052
|
+
threshold: this.ebOpts.threshold
|
1053
|
+
};
|
1054
|
+
return this.commitQueue.enqueue(async () => {
|
1055
|
+
await this.cacheTransaction(t);
|
1056
|
+
const ret = await commit(params, t, done, opts);
|
1057
|
+
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1058
|
+
return ret.cgrp;
|
1172
1059
|
});
|
1173
|
-
this.compacting = false;
|
1174
1060
|
}
|
1175
|
-
async
|
1176
|
-
if (
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1181
|
-
}
|
1182
|
-
|
1183
|
-
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
1061
|
+
async updateCarLog(cids, fp, compact) {
|
1062
|
+
if (compact) {
|
1063
|
+
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1064
|
+
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1065
|
+
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1066
|
+
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1067
|
+
} else {
|
1068
|
+
this.carLog.unshift(cids);
|
1184
1069
|
}
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
1070
|
+
}
|
1071
|
+
async cacheTransaction(t) {
|
1072
|
+
for await (const block of t.entries()) {
|
1073
|
+
const sBlock = block.cid.toString();
|
1074
|
+
if (!this.getBlockCache.has(sBlock)) {
|
1075
|
+
this.getBlockCache.set(sBlock, block);
|
1188
1076
|
}
|
1189
1077
|
}
|
1190
|
-
return this.lastTxMeta;
|
1191
1078
|
}
|
1192
|
-
async
|
1193
|
-
|
1194
|
-
|
1079
|
+
async cacheCarReader(carCidStr, reader) {
|
1080
|
+
if (this.processedCars.has(carCidStr)) return;
|
1081
|
+
this.processedCars.add(carCidStr);
|
1082
|
+
for await (const block of reader.blocks()) {
|
1083
|
+
const sBlock = block.cid.toString();
|
1084
|
+
if (!this.getBlockCache.has(sBlock)) {
|
1085
|
+
this.getBlockCache.set(sBlock, block);
|
1086
|
+
}
|
1195
1087
|
}
|
1196
1088
|
}
|
1197
|
-
|
1198
|
-
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
1206
|
-
return falsyToUndef(block);
|
1207
|
-
}
|
1208
|
-
};
|
1209
|
-
|
1210
|
-
// src/blockstore/commitor.ts
|
1211
|
-
import * as CBW from "@ipld/car/buffer-writer";
|
1212
|
-
import { sha256 as hasher3 } from "multiformats/hashes/sha2";
|
1213
|
-
import * as dagCodec2 from "@ipld/dag-cbor";
|
1214
|
-
async function encodeCarFile(roots, t, codec3) {
|
1215
|
-
let size = 0;
|
1216
|
-
const headerSize = CBW.headerLength({ roots });
|
1217
|
-
size += headerSize;
|
1218
|
-
for (const { cid, bytes } of t.entries()) {
|
1219
|
-
size += CBW.blockLength({ cid, bytes });
|
1220
|
-
}
|
1221
|
-
const buffer = new Uint8Array(size);
|
1222
|
-
const writer = CBW.createWriter(buffer, { headerSize });
|
1223
|
-
for (const r of roots) {
|
1224
|
-
writer.addRoot(r);
|
1225
|
-
}
|
1226
|
-
for (const { cid, bytes } of t.entries()) {
|
1227
|
-
writer.write({ cid, bytes });
|
1228
|
-
}
|
1229
|
-
writer.close();
|
1230
|
-
return await encode3({ value: writer.bytes, hasher: hasher3, codec: codec3 });
|
1231
|
-
}
|
1232
|
-
async function createCarFile(encoder, cid, t) {
|
1233
|
-
return encodeCarFile([cid], t, encoder);
|
1234
|
-
}
|
1235
|
-
async function commitFiles(fileStore, walStore, t, done) {
|
1236
|
-
const { files: roots } = makeFileCarHeader(done);
|
1237
|
-
const cids = [];
|
1238
|
-
const codec3 = (await fileStore.keyedCrypto()).codec();
|
1239
|
-
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
1240
|
-
for (const car of cars) {
|
1241
|
-
const { cid, bytes } = car;
|
1242
|
-
await fileStore.save({ cid, bytes });
|
1243
|
-
await walStore.enqueueFile(
|
1244
|
-
cid
|
1245
|
-
/*, !!opts.public*/
|
1246
|
-
);
|
1247
|
-
cids.push(cid);
|
1248
|
-
}
|
1249
|
-
return cids;
|
1250
|
-
}
|
1251
|
-
function makeFileCarHeader(result) {
|
1252
|
-
const files = [];
|
1253
|
-
for (const [, meta] of Object.entries(result.files || {})) {
|
1254
|
-
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
1255
|
-
files.push(meta.cid);
|
1089
|
+
async removeCidsForCompact(cid) {
|
1090
|
+
const carHeader = await this.loadCarHeaderFromMeta({
|
1091
|
+
cars: [cid]
|
1092
|
+
});
|
1093
|
+
for (const cids of carHeader.compact) {
|
1094
|
+
for (const cid2 of cids) {
|
1095
|
+
await (await this.carStore()).remove(cid2);
|
1096
|
+
}
|
1256
1097
|
}
|
1257
1098
|
}
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
}
|
1263
|
-
|
1264
|
-
|
1265
|
-
|
1266
|
-
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1270
|
-
|
1271
|
-
|
1272
|
-
});
|
1273
|
-
}
|
1274
|
-
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
1275
|
-
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
1276
|
-
const rootBlock = await encodeCarHeader(fp);
|
1277
|
-
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
1278
|
-
const cids = [];
|
1279
|
-
for (const car of cars) {
|
1280
|
-
const { cid, bytes } = car;
|
1281
|
-
await params.carStore.save({ cid, bytes });
|
1282
|
-
cids.push(cid);
|
1283
|
-
}
|
1284
|
-
const newDbMeta = { cars: cids };
|
1285
|
-
await params.WALStore.enqueue(newDbMeta, opts);
|
1286
|
-
await params.metaStore.save(newDbMeta);
|
1287
|
-
return { cgrp: cids, header: fp };
|
1288
|
-
}
|
1289
|
-
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
1290
|
-
const carFiles = [];
|
1291
|
-
threshold = threshold || 128e3 * 8;
|
1292
|
-
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1293
|
-
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1294
|
-
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
1295
|
-
let cidRootBlock = rootBlock;
|
1296
|
-
for (const { cid, bytes } of t.entries()) {
|
1297
|
-
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1298
|
-
if (newsize >= threshold) {
|
1299
|
-
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1300
|
-
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1301
|
-
clonedt.putSync(cid, bytes);
|
1302
|
-
cidRootBlock = { cid, bytes };
|
1303
|
-
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1099
|
+
// async flushCars() {
|
1100
|
+
// await this.ready
|
1101
|
+
// // for each cid in car log, make a dbMeta
|
1102
|
+
// for (const cid of this.carLog) {
|
1103
|
+
// const dbMeta = { car: cid, key: this.key || null } as DbMeta
|
1104
|
+
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
1105
|
+
// }
|
1106
|
+
// }
|
1107
|
+
async *entries(cache2 = true) {
|
1108
|
+
await this.ready();
|
1109
|
+
if (cache2) {
|
1110
|
+
for (const [, block] of this.getBlockCache) {
|
1111
|
+
yield block;
|
1112
|
+
}
|
1304
1113
|
} else {
|
1305
|
-
|
1114
|
+
for (const [, block] of this.getBlockCache) {
|
1115
|
+
yield block;
|
1116
|
+
}
|
1117
|
+
for (const cids of this.carLog) {
|
1118
|
+
for (const cid of cids) {
|
1119
|
+
const reader = await this.loadCar(cid);
|
1120
|
+
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
1121
|
+
for await (const block of reader.blocks()) {
|
1122
|
+
const sCid = block.cid.toString();
|
1123
|
+
if (!this.getBlockCache.has(sCid)) {
|
1124
|
+
yield block;
|
1125
|
+
}
|
1126
|
+
}
|
1127
|
+
}
|
1128
|
+
}
|
1306
1129
|
}
|
1307
1130
|
}
|
1308
|
-
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1312
|
-
|
1313
|
-
|
1314
|
-
|
1315
|
-
|
1316
|
-
|
1317
|
-
|
1318
|
-
|
1319
|
-
|
1320
|
-
|
1321
|
-
|
1322
|
-
|
1323
|
-
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1131
|
+
async getBlock(cid) {
|
1132
|
+
await this.ready();
|
1133
|
+
const sCid = cid.toString();
|
1134
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1135
|
+
const getCarCid = async (carCid) => {
|
1136
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1137
|
+
const reader = await this.loadCar(carCid);
|
1138
|
+
if (!reader) {
|
1139
|
+
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
1140
|
+
}
|
1141
|
+
await this.cacheCarReader(carCid.toString(), reader).catch(() => {
|
1142
|
+
return;
|
1143
|
+
});
|
1144
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1145
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
|
1146
|
+
};
|
1147
|
+
const getCompactCarCids = async (carCid) => {
|
1148
|
+
const reader = await this.loadCar(carCid);
|
1149
|
+
if (!reader) {
|
1150
|
+
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
1151
|
+
}
|
1152
|
+
const header = await parseCarFile(reader, this.logger);
|
1153
|
+
const compacts = header.compact;
|
1154
|
+
let got2;
|
1155
|
+
const batchSize2 = 5;
|
1156
|
+
for (let i = 0; i < compacts.length; i += batchSize2) {
|
1157
|
+
const promises = [];
|
1158
|
+
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
1159
|
+
for (const cid2 of compacts[j]) {
|
1160
|
+
promises.push(getCarCid(cid2));
|
1161
|
+
}
|
1162
|
+
}
|
1163
|
+
try {
|
1164
|
+
got2 = await Promise.any(promises);
|
1165
|
+
} catch {
|
1166
|
+
}
|
1167
|
+
if (got2) break;
|
1168
|
+
}
|
1169
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1170
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
1171
|
+
};
|
1172
|
+
let got;
|
1173
|
+
const batchSize = 5;
|
1174
|
+
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
1175
|
+
const batch = this.carLog.slice(i, i + batchSize);
|
1176
|
+
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
1177
|
+
try {
|
1178
|
+
got = await Promise.any(promises);
|
1179
|
+
} catch {
|
1180
|
+
}
|
1181
|
+
if (got) break;
|
1327
1182
|
}
|
1328
|
-
|
1329
|
-
|
1330
|
-
|
1183
|
+
if (!got) {
|
1184
|
+
try {
|
1185
|
+
got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
|
1186
|
+
} catch {
|
1187
|
+
}
|
1188
|
+
}
|
1189
|
+
return got;
|
1331
1190
|
}
|
1332
|
-
async
|
1333
|
-
if (this.
|
1334
|
-
|
1335
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1336
|
-
const first = filteredQueue[0];
|
1337
|
-
if (!first) {
|
1338
|
-
return;
|
1191
|
+
async loadCar(cid) {
|
1192
|
+
if (!this.carStore) {
|
1193
|
+
throw this.logger.Error().Msg("car store not initialized").AsError();
|
1339
1194
|
}
|
1195
|
+
const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
|
1196
|
+
return loaded;
|
1197
|
+
}
|
1198
|
+
async makeDecoderAndCarReader(cid, local, remote) {
|
1199
|
+
const cidsString = cid.toString();
|
1200
|
+
let loadedCar = void 0;
|
1201
|
+
let activeStore = local;
|
1340
1202
|
try {
|
1341
|
-
|
1342
|
-
|
1343
|
-
this.
|
1344
|
-
} catch (
|
1345
|
-
if (
|
1346
|
-
|
1347
|
-
|
1348
|
-
|
1349
|
-
|
1350
|
-
|
1351
|
-
|
1352
|
-
|
1353
|
-
|
1354
|
-
|
1203
|
+
this.logger.Debug().Str("cid", cidsString).Msg("loading car");
|
1204
|
+
loadedCar = await local.load(cid);
|
1205
|
+
this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
|
1206
|
+
} catch (e) {
|
1207
|
+
if (remote) {
|
1208
|
+
const remoteCar = await remote.load(cid);
|
1209
|
+
if (remoteCar) {
|
1210
|
+
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1211
|
+
await local.save(remoteCar);
|
1212
|
+
loadedCar = remoteCar;
|
1213
|
+
activeStore = remote;
|
1214
|
+
}
|
1215
|
+
} else {
|
1216
|
+
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1355
1217
|
}
|
1356
1218
|
}
|
1219
|
+
if (!loadedCar) {
|
1220
|
+
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
1221
|
+
}
|
1222
|
+
const bytes = await decode({ bytes: loadedCar.bytes, hasher: hasher3, codec: (await activeStore.keyedCrypto()).codec() });
|
1223
|
+
const rawReader = await CarReader.fromBytes(bytes.value);
|
1224
|
+
const readerP = Promise.resolve(rawReader);
|
1225
|
+
const cachedReaderP = readerP.then(async (reader) => {
|
1226
|
+
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
1227
|
+
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
1228
|
+
return;
|
1229
|
+
});
|
1230
|
+
return reader;
|
1231
|
+
});
|
1232
|
+
this.carReaders.set(cidsString, cachedReaderP);
|
1233
|
+
return readerP;
|
1234
|
+
}
|
1235
|
+
//What if instead it returns an Array of CarHeader
|
1236
|
+
async storesLoadCar(cid, local, remote) {
|
1237
|
+
const cidsString = cid.toString();
|
1238
|
+
let dacr = this.carReaders.get(cidsString);
|
1239
|
+
if (!dacr) {
|
1240
|
+
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
1241
|
+
this.carReaders.set(cidsString, dacr);
|
1242
|
+
}
|
1243
|
+
return dacr;
|
1244
|
+
}
|
1245
|
+
async getMoreReaders(cids) {
|
1246
|
+
const limit = pLimit(5);
|
1247
|
+
const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
|
1248
|
+
await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
|
1357
1249
|
}
|
1358
1250
|
};
|
1359
1251
|
|
1360
|
-
// src/
|
1361
|
-
|
1362
|
-
|
1363
|
-
|
1364
|
-
|
1365
|
-
}
|
1366
|
-
|
1367
|
-
|
1368
|
-
|
1369
|
-
|
1370
|
-
|
1252
|
+
// src/runtime/keyed-crypto.ts
|
1253
|
+
var keyed_crypto_exports = {};
|
1254
|
+
__export(keyed_crypto_exports, {
|
1255
|
+
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
1256
|
+
keyedCryptoFactory: () => keyedCryptoFactory
|
1257
|
+
});
|
1258
|
+
import { base58btc as base58btc2 } from "multiformats/bases/base58";
|
1259
|
+
import { sha256 as hasher4 } from "multiformats/hashes/sha2";
|
1260
|
+
import * as CBOR from "cborg";
|
1261
|
+
var generateIV = {
|
1262
|
+
random: {
|
1263
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1264
|
+
calc: async (ko, crypto, data) => {
|
1265
|
+
return crypto.randomBytes(ko.ivLength);
|
1266
|
+
},
|
1267
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1268
|
+
verify: async (ko, crypto, iv, data) => {
|
1269
|
+
return true;
|
1270
|
+
}
|
1271
|
+
},
|
1272
|
+
hash: {
|
1273
|
+
calc: async (ko, crypto, data) => {
|
1274
|
+
const hash = await hasher4.digest(data);
|
1275
|
+
const hashBytes = new Uint8Array(hash.bytes);
|
1276
|
+
const hashArray = new Uint8Array(ko.ivLength);
|
1277
|
+
for (let i = 0; i < hashBytes.length; i++) {
|
1278
|
+
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
1279
|
+
}
|
1280
|
+
return hashArray;
|
1281
|
+
},
|
1282
|
+
verify: async function(ko, crypto, iv, data) {
|
1283
|
+
return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
1284
|
+
}
|
1371
1285
|
}
|
1372
|
-
|
1286
|
+
};
|
1287
|
+
function getGenerateIVFn(url, opts) {
|
1288
|
+
const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
|
1289
|
+
return generateIV[ivhash] || generateIV["hash"];
|
1373
1290
|
}
|
1374
|
-
var
|
1375
|
-
constructor(
|
1376
|
-
this.
|
1377
|
-
this.
|
1378
|
-
this.
|
1379
|
-
this.
|
1380
|
-
this.
|
1381
|
-
|
1382
|
-
|
1383
|
-
this.
|
1384
|
-
|
1385
|
-
|
1386
|
-
|
1387
|
-
this.
|
1388
|
-
|
1389
|
-
|
1390
|
-
|
1391
|
-
|
1392
|
-
sthis,
|
1393
|
-
{
|
1394
|
-
...ebOpts
|
1395
|
-
// name,
|
1396
|
-
},
|
1397
|
-
"Loader"
|
1398
|
-
);
|
1399
|
-
this.logger = this.ebOpts.logger;
|
1400
|
-
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
1401
|
-
await this.handleDbMetasFromStore([dbMeta]);
|
1291
|
+
var BlockIvKeyIdCodec = class {
|
1292
|
+
constructor(ko, iv, opts) {
|
1293
|
+
this.code = 3147065;
|
1294
|
+
this.name = "Fireproof@encrypted-block:aes-gcm";
|
1295
|
+
this.ko = ko;
|
1296
|
+
this.iv = iv;
|
1297
|
+
this.opts = opts || {};
|
1298
|
+
}
|
1299
|
+
async encode(data) {
|
1300
|
+
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
1301
|
+
const { iv } = this.ko.algo(calcIv);
|
1302
|
+
const fprt = await this.ko.fingerPrint();
|
1303
|
+
const keyId = base58btc2.decode(fprt);
|
1304
|
+
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
1305
|
+
return CBOR.encode({
|
1306
|
+
iv,
|
1307
|
+
keyId,
|
1308
|
+
data: await this.ko._encrypt({ iv, bytes: data })
|
1402
1309
|
});
|
1403
1310
|
}
|
1404
|
-
async
|
1405
|
-
|
1406
|
-
|
1407
|
-
|
1408
|
-
|
1409
|
-
|
1410
|
-
|
1411
|
-
);
|
1311
|
+
async decode(abytes) {
|
1312
|
+
let bytes;
|
1313
|
+
if (abytes instanceof Uint8Array) {
|
1314
|
+
bytes = abytes;
|
1315
|
+
} else {
|
1316
|
+
bytes = new Uint8Array(abytes);
|
1317
|
+
}
|
1318
|
+
const { iv, keyId, data } = CBOR.decode(bytes);
|
1319
|
+
const fprt = await this.ko.fingerPrint();
|
1320
|
+
this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
|
1321
|
+
if (base58btc2.encode(keyId) !== fprt) {
|
1322
|
+
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
|
1323
|
+
}
|
1324
|
+
const result = await this.ko._decrypt({ iv, bytes: data });
|
1325
|
+
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
1326
|
+
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
1327
|
+
}
|
1328
|
+
return result;
|
1329
|
+
}
|
1330
|
+
};
|
1331
|
+
var keyedCrypto = class {
|
1332
|
+
constructor(url, key, cyopt, sthis) {
|
1333
|
+
this.ivLength = 12;
|
1334
|
+
this.isEncrypting = true;
|
1335
|
+
this.logger = ensureLogger(sthis, "keyedCrypto");
|
1336
|
+
this.crypto = cyopt;
|
1337
|
+
this.key = key;
|
1338
|
+
this.url = url;
|
1339
|
+
}
|
1340
|
+
fingerPrint() {
|
1341
|
+
return Promise.resolve(this.key.fingerPrint);
|
1412
1342
|
}
|
1413
|
-
|
1414
|
-
return this
|
1415
|
-
async () => this.ebOpts.storeRuntime.makeDataStore({
|
1416
|
-
sthis: this.sthis,
|
1417
|
-
url: this.ebOpts.storeUrls.file,
|
1418
|
-
keybag: await this.keyBag()
|
1419
|
-
})
|
1420
|
-
);
|
1343
|
+
codec(iv, opts) {
|
1344
|
+
return new BlockIvKeyIdCodec(this, iv, opts);
|
1421
1345
|
}
|
1422
|
-
|
1423
|
-
return
|
1424
|
-
|
1425
|
-
|
1426
|
-
|
1427
|
-
|
1428
|
-
})
|
1429
|
-
);
|
1346
|
+
algo(iv) {
|
1347
|
+
return {
|
1348
|
+
name: "AES-GCM",
|
1349
|
+
iv: iv || this.crypto.randomBytes(this.ivLength),
|
1350
|
+
tagLength: 128
|
1351
|
+
};
|
1430
1352
|
}
|
1431
|
-
async
|
1432
|
-
|
1433
|
-
|
1434
|
-
sthis: this.sthis,
|
1435
|
-
url: this.ebOpts.storeUrls.meta,
|
1436
|
-
keybag: await this.keyBag()
|
1437
|
-
})
|
1438
|
-
);
|
1353
|
+
async _decrypt(data) {
|
1354
|
+
this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
|
1355
|
+
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
1439
1356
|
}
|
1440
|
-
|
1441
|
-
|
1357
|
+
async _encrypt(data) {
|
1358
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
|
1359
|
+
const a = this.algo(data.iv);
|
1360
|
+
return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
|
1442
1361
|
}
|
1443
|
-
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1448
|
-
} else if (metas) {
|
1449
|
-
await this.handleDbMetasFromStore(metas);
|
1450
|
-
}
|
1451
|
-
});
|
1362
|
+
};
|
1363
|
+
var nullCodec = class {
|
1364
|
+
constructor() {
|
1365
|
+
this.code = 0;
|
1366
|
+
this.name = "Fireproof@unencrypted-block";
|
1452
1367
|
}
|
1453
|
-
|
1454
|
-
|
1455
|
-
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1456
|
-
await Promise.all(toClose.map((store) => store.close()));
|
1368
|
+
encode(data) {
|
1369
|
+
return data;
|
1457
1370
|
}
|
1458
|
-
|
1459
|
-
|
1460
|
-
await Promise.all(toDestroy.map((store) => store.destroy()));
|
1371
|
+
decode(data) {
|
1372
|
+
return data;
|
1461
1373
|
}
|
1462
|
-
|
1463
|
-
|
1464
|
-
|
1465
|
-
|
1466
|
-
|
1467
|
-
|
1468
|
-
|
1469
|
-
|
1470
|
-
|
1471
|
-
|
1472
|
-
|
1473
|
-
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
1474
|
-
for (const meta of metas) {
|
1475
|
-
await this.writeLimit(async () => {
|
1476
|
-
await this.mergeDbMetaIntoClock(meta);
|
1477
|
-
});
|
1478
|
-
}
|
1374
|
+
};
|
1375
|
+
var noCrypto = class {
|
1376
|
+
constructor(url, cyrt, sthis) {
|
1377
|
+
this.ivLength = 0;
|
1378
|
+
this.code = 0;
|
1379
|
+
this.name = "Fireproof@unencrypted-block";
|
1380
|
+
this.isEncrypting = false;
|
1381
|
+
this._fingerPrint = "noCrypto:" + Math.random();
|
1382
|
+
this.logger = ensureLogger(sthis, "noCrypto");
|
1383
|
+
this.crypto = cyrt;
|
1384
|
+
this.url = url;
|
1479
1385
|
}
|
1480
|
-
|
1481
|
-
|
1482
|
-
throw this.logger.Error().Msg("cannot merge while compacting").AsError();
|
1483
|
-
}
|
1484
|
-
if (this.seenMeta.has(meta.cars.toString())) return;
|
1485
|
-
this.seenMeta.add(meta.cars.toString());
|
1486
|
-
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
1487
|
-
return;
|
1488
|
-
}
|
1489
|
-
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
1490
|
-
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1491
|
-
await this.getMoreReaders(carHeader.cars.flat());
|
1492
|
-
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
1493
|
-
await this.ebOpts.applyMeta?.(carHeader.meta);
|
1386
|
+
fingerPrint() {
|
1387
|
+
return Promise.resolve(this._fingerPrint);
|
1494
1388
|
}
|
1495
|
-
//
|
1496
|
-
|
1497
|
-
|
1498
|
-
// await this.setKey(key);
|
1499
|
-
// }
|
1500
|
-
// }
|
1501
|
-
async loadCarHeaderFromMeta({ cars: cids }) {
|
1502
|
-
const reader = await this.loadCar(cids[0]);
|
1503
|
-
return await parseCarFile(reader, this.logger);
|
1389
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1390
|
+
codec(iv) {
|
1391
|
+
return new nullCodec();
|
1504
1392
|
}
|
1505
|
-
//
|
1506
|
-
|
1507
|
-
|
1508
|
-
|
1509
|
-
|
1510
|
-
|
1511
|
-
|
1512
|
-
// }
|
1513
|
-
async commitFiles(t, done) {
|
1514
|
-
await this.ready();
|
1515
|
-
const fstore = await this.fileStore();
|
1516
|
-
const wstore = await this.WALStore();
|
1517
|
-
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1393
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1394
|
+
algo(iv) {
|
1395
|
+
return {
|
1396
|
+
name: "noCrypto",
|
1397
|
+
iv: new Uint8Array(),
|
1398
|
+
tagLength: 0
|
1399
|
+
};
|
1518
1400
|
}
|
1519
|
-
|
1520
|
-
|
1401
|
+
_decrypt() {
|
1402
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
1521
1403
|
}
|
1522
|
-
|
1523
|
-
|
1524
|
-
const carStore = await this.carStore();
|
1525
|
-
const params = {
|
1526
|
-
encoder: (await carStore.keyedCrypto()).codec(),
|
1527
|
-
carLog: this.carLog,
|
1528
|
-
carStore,
|
1529
|
-
WALStore: await this.WALStore(),
|
1530
|
-
metaStore: await this.metaStore(),
|
1531
|
-
threshold: this.ebOpts.threshold
|
1532
|
-
};
|
1533
|
-
return this.commitQueue.enqueue(async () => {
|
1534
|
-
await this.cacheTransaction(t);
|
1535
|
-
const ret = await commit(params, t, done, opts);
|
1536
|
-
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1537
|
-
return ret.cgrp;
|
1538
|
-
});
|
1404
|
+
_encrypt() {
|
1405
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
1539
1406
|
}
|
1540
|
-
|
1541
|
-
|
1542
|
-
|
1543
|
-
|
1544
|
-
|
1545
|
-
|
1546
|
-
|
1547
|
-
|
1407
|
+
};
|
1408
|
+
async function keyedCryptoFactory(url, kb, sthis) {
|
1409
|
+
const storekey = url.getParam("storekey");
|
1410
|
+
if (storekey && storekey !== "insecure") {
|
1411
|
+
let rkey = await kb.getNamedKey(storekey, true);
|
1412
|
+
if (rkey.isErr()) {
|
1413
|
+
try {
|
1414
|
+
rkey = await kb.toKeyWithFingerPrint(storekey);
|
1415
|
+
} catch (e) {
|
1416
|
+
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
1417
|
+
}
|
1548
1418
|
}
|
1419
|
+
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
1549
1420
|
}
|
1550
|
-
|
1551
|
-
|
1552
|
-
|
1553
|
-
|
1554
|
-
|
1555
|
-
|
1421
|
+
return new noCrypto(url, kb.rt.crypto, sthis);
|
1422
|
+
}
|
1423
|
+
|
1424
|
+
// src/blockstore/fragment-gateway.ts
|
1425
|
+
import { Result as Result3 } from "@adviser/cement";
|
1426
|
+
import { base58btc as base58btc3 } from "multiformats/bases/base58";
|
1427
|
+
import { encode as encode3, decode as decode3 } from "cborg";
|
1428
|
+
function getFragSize(url) {
|
1429
|
+
const fragSize = url.getParam("fragSize");
|
1430
|
+
let ret = 0;
|
1431
|
+
if (fragSize) {
|
1432
|
+
ret = parseInt(fragSize);
|
1433
|
+
}
|
1434
|
+
if (isNaN(ret) || ret <= 0) {
|
1435
|
+
ret = 0;
|
1436
|
+
}
|
1437
|
+
return ret;
|
1438
|
+
}
|
1439
|
+
async function getFrags(url, innerGW, headerSize, logger) {
|
1440
|
+
const fragSize = getFragSize(url);
|
1441
|
+
if (!fragSize) {
|
1442
|
+
const res = await innerGW.get(url);
|
1443
|
+
if (res.isErr()) {
|
1444
|
+
return [res];
|
1556
1445
|
}
|
1446
|
+
const data = res.unwrap();
|
1447
|
+
return [
|
1448
|
+
Result3.Ok({
|
1449
|
+
fid: new Uint8Array(0),
|
1450
|
+
ofs: 0,
|
1451
|
+
len: data.length,
|
1452
|
+
data
|
1453
|
+
})
|
1454
|
+
];
|
1557
1455
|
}
|
1558
|
-
|
1559
|
-
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1563
|
-
|
1564
|
-
|
1456
|
+
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
1457
|
+
if (firstRaw.isErr()) {
|
1458
|
+
return [firstRaw];
|
1459
|
+
}
|
1460
|
+
const firstFragment = decode3(firstRaw.unwrap());
|
1461
|
+
const blockSize = firstFragment.data.length;
|
1462
|
+
const ops = [Promise.resolve(Result3.Ok(firstFragment))];
|
1463
|
+
const fidStr = base58btc3.encode(firstFragment.fid);
|
1464
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
|
1465
|
+
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
1466
|
+
ops.push(
|
1467
|
+
(async (furl, ofs2) => {
|
1468
|
+
const raw2 = await innerGW.get(furl);
|
1469
|
+
if (raw2.isErr()) {
|
1470
|
+
return raw2;
|
1471
|
+
}
|
1472
|
+
const fragment = decode3(raw2.unwrap());
|
1473
|
+
if (base58btc3.encode(fragment.fid) !== fidStr) {
|
1474
|
+
return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
1475
|
+
}
|
1476
|
+
if (fragment.ofs !== ofs2) {
|
1477
|
+
return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
1478
|
+
}
|
1479
|
+
return Result3.Ok(fragment);
|
1480
|
+
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
1481
|
+
);
|
1482
|
+
}
|
1483
|
+
return Promise.all(ops);
|
1484
|
+
}
|
1485
|
+
var FragmentGateway = class {
|
1486
|
+
constructor(sthis, innerGW) {
|
1487
|
+
this.fidLength = 4;
|
1488
|
+
this.headerSize = 32;
|
1489
|
+
this.sthis = ensureSuperLog(sthis, "FragmentGateway");
|
1490
|
+
this.logger = this.sthis.logger;
|
1491
|
+
this.innerGW = innerGW;
|
1492
|
+
}
|
1493
|
+
slicer(url, body) {
|
1494
|
+
const fragSize = getFragSize(url);
|
1495
|
+
if (!fragSize) {
|
1496
|
+
return [this.innerGW.put(url, body)];
|
1497
|
+
}
|
1498
|
+
const blocksize = fragSize - this.headerSize;
|
1499
|
+
if (blocksize <= 0) {
|
1500
|
+
throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
|
1501
|
+
}
|
1502
|
+
const ops = [];
|
1503
|
+
const fid = this.sthis.nextId(this.fidLength);
|
1504
|
+
const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
|
1505
|
+
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
1506
|
+
const block = encode3({
|
1507
|
+
fid: fid.bin,
|
1508
|
+
ofs,
|
1509
|
+
len: body.length,
|
1510
|
+
data: body.slice(ofs, ofs + blocksize)
|
1511
|
+
});
|
1512
|
+
if (block.length > fragSize) {
|
1513
|
+
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
1565
1514
|
}
|
1515
|
+
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
1566
1516
|
}
|
1517
|
+
return ops;
|
1567
1518
|
}
|
1568
|
-
|
1569
|
-
|
1570
|
-
|
1571
|
-
|
1572
|
-
|
1573
|
-
|
1574
|
-
|
1519
|
+
buildUrl(baseUrl, key) {
|
1520
|
+
return this.innerGW.buildUrl(baseUrl, key);
|
1521
|
+
}
|
1522
|
+
async destroy(iurl) {
|
1523
|
+
return this.innerGW.destroy(iurl);
|
1524
|
+
}
|
1525
|
+
async start(url) {
|
1526
|
+
this.headerSize = encode3({
|
1527
|
+
fid: this.sthis.nextId(this.fidLength).bin,
|
1528
|
+
ofs: 1024 * 1024,
|
1529
|
+
// 32bit
|
1530
|
+
len: 16 * 1024 * 1024,
|
1531
|
+
// 32bit
|
1532
|
+
data: new Uint8Array(1024)
|
1533
|
+
}).length - 1024;
|
1534
|
+
return this.innerGW.start(url);
|
1535
|
+
}
|
1536
|
+
async close(url) {
|
1537
|
+
return this.innerGW.close(url);
|
1538
|
+
}
|
1539
|
+
async put(url, body) {
|
1540
|
+
await Promise.all(this.slicer(url, body));
|
1541
|
+
return Result3.Ok(void 0);
|
1542
|
+
}
|
1543
|
+
async get(url) {
|
1544
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1545
|
+
let buffer = void 0;
|
1546
|
+
for (const rfrag of rfrags) {
|
1547
|
+
if (rfrag.isErr()) {
|
1548
|
+
return Result3.Err(rfrag.Err());
|
1575
1549
|
}
|
1550
|
+
const frag = rfrag.Ok();
|
1551
|
+
buffer = buffer || new Uint8Array(frag.len);
|
1552
|
+
buffer.set(frag.data, frag.ofs);
|
1576
1553
|
}
|
1554
|
+
return Result3.Ok(buffer || new Uint8Array(0));
|
1577
1555
|
}
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
// for (const cid of this.carLog) {
|
1582
|
-
// const dbMeta = { car: cid, key: this.key || null } as DbMeta
|
1583
|
-
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
1584
|
-
// }
|
1585
|
-
// }
|
1586
|
-
async *entries(cache2 = true) {
|
1587
|
-
await this.ready();
|
1588
|
-
if (cache2) {
|
1589
|
-
for (const [, block] of this.getBlockCache) {
|
1590
|
-
yield block;
|
1591
|
-
}
|
1556
|
+
async subscribe(url, callback) {
|
1557
|
+
if (this.innerGW.subscribe) {
|
1558
|
+
return this.innerGW.subscribe(url, callback);
|
1592
1559
|
} else {
|
1593
|
-
|
1594
|
-
yield block;
|
1595
|
-
}
|
1596
|
-
for (const cids of this.carLog) {
|
1597
|
-
for (const cid of cids) {
|
1598
|
-
const reader = await this.loadCar(cid);
|
1599
|
-
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
1600
|
-
for await (const block of reader.blocks()) {
|
1601
|
-
const sCid = block.cid.toString();
|
1602
|
-
if (!this.getBlockCache.has(sCid)) {
|
1603
|
-
yield block;
|
1604
|
-
}
|
1605
|
-
}
|
1606
|
-
}
|
1607
|
-
}
|
1560
|
+
return Result3.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
|
1608
1561
|
}
|
1609
1562
|
}
|
1610
|
-
async
|
1611
|
-
await this.
|
1612
|
-
const
|
1613
|
-
|
1614
|
-
|
1615
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1616
|
-
const reader = await this.loadCar(carCid);
|
1617
|
-
if (!reader) {
|
1618
|
-
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
1619
|
-
}
|
1620
|
-
await this.cacheCarReader(carCid.toString(), reader).catch(() => {
|
1621
|
-
return;
|
1622
|
-
});
|
1623
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1624
|
-
throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
|
1625
|
-
};
|
1626
|
-
const getCompactCarCids = async (carCid) => {
|
1627
|
-
const reader = await this.loadCar(carCid);
|
1628
|
-
if (!reader) {
|
1629
|
-
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
1630
|
-
}
|
1631
|
-
const header = await parseCarFile(reader, this.logger);
|
1632
|
-
const compacts = header.compact;
|
1633
|
-
let got2;
|
1634
|
-
const batchSize2 = 5;
|
1635
|
-
for (let i = 0; i < compacts.length; i += batchSize2) {
|
1636
|
-
const promises = [];
|
1637
|
-
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
1638
|
-
for (const cid2 of compacts[j]) {
|
1639
|
-
promises.push(getCarCid(cid2));
|
1640
|
-
}
|
1641
|
-
}
|
1642
|
-
try {
|
1643
|
-
got2 = await Promise.any(promises);
|
1644
|
-
} catch {
|
1645
|
-
}
|
1646
|
-
if (got2) break;
|
1647
|
-
}
|
1648
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
1649
|
-
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
1650
|
-
};
|
1651
|
-
let got;
|
1652
|
-
const batchSize = 5;
|
1653
|
-
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
1654
|
-
const batch = this.carLog.slice(i, i + batchSize);
|
1655
|
-
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
1656
|
-
try {
|
1657
|
-
got = await Promise.any(promises);
|
1658
|
-
} catch {
|
1659
|
-
}
|
1660
|
-
if (got) break;
|
1661
|
-
}
|
1662
|
-
if (!got) {
|
1663
|
-
try {
|
1664
|
-
got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
|
1665
|
-
} catch {
|
1563
|
+
async delete(url) {
|
1564
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1565
|
+
for (const rfrag of rfrags) {
|
1566
|
+
if (rfrag.isErr()) {
|
1567
|
+
return Result3.Err(rfrag.Err());
|
1666
1568
|
}
|
1569
|
+
const frag = rfrag.Ok();
|
1570
|
+
const fidStr = base58btc3.encode(frag.fid);
|
1571
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
|
1572
|
+
await this.innerGW.delete(fragUrl);
|
1667
1573
|
}
|
1668
|
-
return
|
1574
|
+
return Result3.Ok(void 0);
|
1669
1575
|
}
|
1670
|
-
|
1671
|
-
|
1672
|
-
|
1673
|
-
|
1674
|
-
|
1675
|
-
|
1576
|
+
};
|
1577
|
+
|
1578
|
+
// src/blockstore/meta-key-helper.ts
|
1579
|
+
import { format, parse } from "@ipld/dag-json";
|
1580
|
+
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
1581
|
+
import { CID as CID2 } from "multiformats";
|
1582
|
+
import { base64pad } from "multiformats/bases/base64";
|
1583
|
+
import { Result as Result4 } from "@adviser/cement";
|
1584
|
+
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
1585
|
+
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
1586
|
+
if (!crdtEntries.length) {
|
1587
|
+
sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
|
1588
|
+
return [];
|
1676
1589
|
}
|
1677
|
-
|
1678
|
-
|
1679
|
-
|
1680
|
-
|
1681
|
-
|
1682
|
-
|
1683
|
-
|
1684
|
-
|
1685
|
-
|
1686
|
-
|
1687
|
-
|
1688
|
-
|
1689
|
-
|
1690
|
-
|
1691
|
-
|
1692
|
-
|
1590
|
+
if (!crdtEntries.map) {
|
1591
|
+
sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
|
1592
|
+
return [];
|
1593
|
+
}
|
1594
|
+
return Promise.all(
|
1595
|
+
crdtEntries.map(async (crdtEntry) => {
|
1596
|
+
const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
|
1597
|
+
const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
|
1598
|
+
return {
|
1599
|
+
eventCid: eventBlock.cid,
|
1600
|
+
parents: crdtEntry.parents,
|
1601
|
+
dbMeta
|
1602
|
+
};
|
1603
|
+
})
|
1604
|
+
);
|
1605
|
+
}
|
1606
|
+
async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
|
1607
|
+
try {
|
1608
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
|
1609
|
+
const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
|
1610
|
+
if (keyInfo.length) {
|
1611
|
+
const dbMeta = keyInfo[0].dbMeta;
|
1612
|
+
if (dbMeta.key) {
|
1613
|
+
const kb = await getKeyBag(sthis);
|
1614
|
+
const keyName = getStoreKeyName(uri);
|
1615
|
+
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
1616
|
+
if (res.isErr()) {
|
1617
|
+
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
1618
|
+
throw res.Err();
|
1693
1619
|
}
|
1694
|
-
} else {
|
1695
|
-
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1696
1620
|
}
|
1621
|
+
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
1622
|
+
return Result4.Ok(dbMeta);
|
1697
1623
|
}
|
1698
|
-
|
1699
|
-
|
1700
|
-
|
1701
|
-
|
1702
|
-
|
1703
|
-
const readerP = Promise.resolve(rawReader);
|
1704
|
-
const cachedReaderP = readerP.then(async (reader) => {
|
1705
|
-
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
1706
|
-
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
1707
|
-
return;
|
1708
|
-
});
|
1709
|
-
return reader;
|
1710
|
-
});
|
1711
|
-
this.carReaders.set(cidsString, cachedReaderP);
|
1712
|
-
return readerP;
|
1624
|
+
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
1625
|
+
return Result4.Ok(void 0);
|
1626
|
+
} catch (error) {
|
1627
|
+
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
1628
|
+
return Result4.Err(error);
|
1713
1629
|
}
|
1714
|
-
|
1715
|
-
|
1716
|
-
|
1717
|
-
|
1718
|
-
|
1719
|
-
|
1720
|
-
|
1630
|
+
}
|
1631
|
+
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
1632
|
+
try {
|
1633
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
1634
|
+
const keyName = getStoreKeyName(uri);
|
1635
|
+
const kb = await getKeyBag(sthis);
|
1636
|
+
const res = await kb.getNamedExtractableKey(keyName, true);
|
1637
|
+
if (res.isErr()) {
|
1638
|
+
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
1639
|
+
throw res.Err();
|
1721
1640
|
}
|
1722
|
-
|
1723
|
-
|
1724
|
-
|
1725
|
-
const
|
1726
|
-
|
1727
|
-
await Promise.all(
|
1641
|
+
const keyData = await res.Ok().extract();
|
1642
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
1643
|
+
const { dbMeta, parents } = dbMetas[0];
|
1644
|
+
const parentLinks = parents.map((p) => CID2.parse(p));
|
1645
|
+
dbMeta.key = keyData.keyStr;
|
1646
|
+
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
1647
|
+
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
1648
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
1649
|
+
return Result4.Ok(encoded);
|
1650
|
+
} catch (error) {
|
1651
|
+
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
1652
|
+
return Result4.Err(error);
|
1728
1653
|
}
|
1729
|
-
}
|
1654
|
+
}
|
1655
|
+
function getStoreKeyName(url) {
|
1656
|
+
const storeKeyName = [url.getParam("localName") || url.getParam("name")];
|
1657
|
+
const idx = url.getParam("index");
|
1658
|
+
if (idx) {
|
1659
|
+
storeKeyName.push(idx);
|
1660
|
+
}
|
1661
|
+
storeKeyName.push("data");
|
1662
|
+
return `@${storeKeyName.join(":")}@`;
|
1663
|
+
}
|
1664
|
+
async function createDbMetaEventBlock(sthis, dbMeta, parents) {
|
1665
|
+
const event = await EventBlock.create(
|
1666
|
+
{
|
1667
|
+
dbMeta: sthis.txt.encode(format(dbMeta))
|
1668
|
+
},
|
1669
|
+
parents
|
1670
|
+
);
|
1671
|
+
return event;
|
1672
|
+
}
|
1673
|
+
async function encodeEventsWithParents(sthis, events, parents) {
|
1674
|
+
const crdtEntries = events.map((event) => {
|
1675
|
+
const base64String = base64pad.encode(event.bytes);
|
1676
|
+
return {
|
1677
|
+
cid: event.cid.toString(),
|
1678
|
+
data: base64String,
|
1679
|
+
parents: parents.map((p) => p.toString())
|
1680
|
+
};
|
1681
|
+
});
|
1682
|
+
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
1683
|
+
}
|
1730
1684
|
|
1731
1685
|
// src/blockstore/store.ts
|
1686
|
+
import pRetry from "p-retry";
|
1687
|
+
import pMap from "p-map";
|
1732
1688
|
function guardVersion(url) {
|
1733
1689
|
if (!url.hasParam("version")) {
|
1734
1690
|
return Result5.Err(`missing version: ${url.toString()}`);
|
@@ -1736,21 +1692,16 @@ function guardVersion(url) {
|
|
1736
1692
|
return Result5.Ok(url);
|
1737
1693
|
}
|
1738
1694
|
var BaseStoreImpl = class {
|
1739
|
-
constructor(
|
1695
|
+
constructor(name, url, opts, sthis, logger) {
|
1740
1696
|
this._onStarted = [];
|
1741
1697
|
this._onClosed = [];
|
1698
|
+
this.name = name;
|
1742
1699
|
this._url = url;
|
1743
1700
|
this.keybag = opts.keybag;
|
1744
|
-
this.loader = opts.loader;
|
1745
1701
|
this.sthis = sthis;
|
1746
|
-
|
1747
|
-
if (!name) {
|
1748
|
-
throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
|
1749
|
-
}
|
1750
|
-
this.name = name;
|
1751
|
-
this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
|
1752
|
-
this.realGateway = opts.gateway;
|
1702
|
+
this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
|
1753
1703
|
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
1704
|
+
this.loader = opts.loader;
|
1754
1705
|
}
|
1755
1706
|
url() {
|
1756
1707
|
return this._url;
|
@@ -1765,20 +1716,20 @@ var BaseStoreImpl = class {
|
|
1765
1716
|
return;
|
1766
1717
|
}
|
1767
1718
|
async keyedCrypto() {
|
1768
|
-
return keyedCryptoFactory(this._url, this.keybag, this.sthis);
|
1719
|
+
return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
|
1769
1720
|
}
|
1770
1721
|
async start() {
|
1771
1722
|
this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
|
1772
|
-
this._url = this._url.build().setParam("store"
|
1723
|
+
this._url = this._url.build().setParam("store", this.storeType).URI();
|
1773
1724
|
const res = await this.gateway.start(this._url);
|
1774
1725
|
if (res.isErr()) {
|
1775
1726
|
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
1776
1727
|
return res;
|
1777
1728
|
}
|
1778
1729
|
this._url = res.Ok();
|
1779
|
-
const kb = await this.keybag;
|
1730
|
+
const kb = await this.keybag();
|
1780
1731
|
const skRes = await kb.ensureKeyFromUrl(this._url, () => {
|
1781
|
-
const idx = this._url.getParam("index"
|
1732
|
+
const idx = this._url.getParam("index");
|
1782
1733
|
const storeKeyName = [this.name];
|
1783
1734
|
if (idx) {
|
1784
1735
|
storeKeyName.push(idx);
|
@@ -1811,8 +1762,8 @@ var BaseStoreImpl = class {
|
|
1811
1762
|
};
|
1812
1763
|
var MetaStoreImpl = class extends BaseStoreImpl {
|
1813
1764
|
// remote: boolean;
|
1814
|
-
constructor(sthis, url, opts) {
|
1815
|
-
super(
|
1765
|
+
constructor(sthis, name, url, opts) {
|
1766
|
+
super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
|
1816
1767
|
this.storeType = "meta";
|
1817
1768
|
this.subscribers = /* @__PURE__ */ new Map();
|
1818
1769
|
this.parents = [];
|
@@ -1883,21 +1834,13 @@ var MetaStoreImpl = class extends BaseStoreImpl {
|
|
1883
1834
|
return Result5.Ok(void 0);
|
1884
1835
|
}
|
1885
1836
|
async destroy() {
|
1886
|
-
this.logger.Debug().Msg("destroy");
|
1887
1837
|
return this.gateway.destroy(this.url());
|
1888
1838
|
}
|
1889
1839
|
};
|
1890
1840
|
var DataStoreImpl = class extends BaseStoreImpl {
|
1891
1841
|
// readonly tag: string = "car-base";
|
1892
|
-
constructor(sthis, url, opts) {
|
1893
|
-
super(
|
1894
|
-
sthis,
|
1895
|
-
url,
|
1896
|
-
{
|
1897
|
-
...opts
|
1898
|
-
},
|
1899
|
-
ensureLogger(sthis, "DataStoreImpl")
|
1900
|
-
);
|
1842
|
+
constructor(sthis, name, url, opts) {
|
1843
|
+
super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
|
1901
1844
|
this.storeType = "data";
|
1902
1845
|
}
|
1903
1846
|
async load(cid) {
|
@@ -1938,32 +1881,23 @@ var DataStoreImpl = class extends BaseStoreImpl {
|
|
1938
1881
|
return Result5.Ok(void 0);
|
1939
1882
|
}
|
1940
1883
|
destroy() {
|
1941
|
-
this.logger.Debug().Msg("destroy");
|
1942
1884
|
return this.gateway.destroy(this.url());
|
1943
1885
|
}
|
1944
1886
|
};
|
1945
1887
|
var WALStoreImpl = class extends BaseStoreImpl {
|
1946
|
-
constructor(
|
1947
|
-
super(
|
1948
|
-
sthis,
|
1949
|
-
url,
|
1950
|
-
{
|
1951
|
-
...opts
|
1952
|
-
},
|
1953
|
-
ensureLogger(sthis, "WALStoreImpl")
|
1954
|
-
);
|
1888
|
+
constructor(loader, url, opts) {
|
1889
|
+
super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
|
1955
1890
|
this.storeType = "wal";
|
1956
|
-
// readonly tag: string = "rwal-base";
|
1957
|
-
// readonly loader: Loadable;
|
1958
1891
|
this._ready = new ResolveOnce3();
|
1959
1892
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
1960
1893
|
this.processing = void 0;
|
1961
1894
|
this.processQueue = new CommitQueue();
|
1895
|
+
this.loader = loader;
|
1962
1896
|
}
|
1963
1897
|
async ready() {
|
1964
1898
|
return this._ready.once(async () => {
|
1965
1899
|
const walState = await this.load().catch((e) => {
|
1966
|
-
this.logger.Error().
|
1900
|
+
this.logger.Error().Any("error", e).Msg("error loading wal");
|
1967
1901
|
return void 0;
|
1968
1902
|
});
|
1969
1903
|
if (!walState) {
|
@@ -1996,7 +1930,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1996
1930
|
}
|
1997
1931
|
async process() {
|
1998
1932
|
await this.ready();
|
1999
|
-
if (!this.loader
|
1933
|
+
if (!this.loader.remoteCarStore) return;
|
2000
1934
|
await this.processQueue.enqueue(async () => {
|
2001
1935
|
try {
|
2002
1936
|
await this._doProcess();
|
@@ -2009,7 +1943,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2009
1943
|
});
|
2010
1944
|
}
|
2011
1945
|
async _doProcess() {
|
2012
|
-
if (!this.loader) return;
|
2013
1946
|
if (!this.loader.remoteCarStore) return;
|
2014
1947
|
const operations = [...this.walState.operations];
|
2015
1948
|
const noLoaderOps = [...this.walState.noLoaderOps];
|
@@ -2027,7 +1960,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2027
1960
|
noLoaderOps,
|
2028
1961
|
async (dbMeta) => {
|
2029
1962
|
await retryableUpload(async () => {
|
2030
|
-
if (!this.loader) return;
|
2031
1963
|
for (const cid of dbMeta.cars) {
|
2032
1964
|
const car = await (await this.loader.carStore()).load(cid);
|
2033
1965
|
if (!car) {
|
@@ -2047,7 +1979,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2047
1979
|
operations,
|
2048
1980
|
async (dbMeta) => {
|
2049
1981
|
await retryableUpload(async () => {
|
2050
|
-
if (!this.loader) return;
|
2051
1982
|
for (const cid of dbMeta.cars) {
|
2052
1983
|
const car = await (await this.loader.carStore()).load(cid);
|
2053
1984
|
if (!car) {
|
@@ -2067,7 +1998,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2067
1998
|
fileOperations,
|
2068
1999
|
async ({ cid: fileCid, public: publicFile }) => {
|
2069
2000
|
await retryableUpload(async () => {
|
2070
|
-
if (!this.loader) return;
|
2071
2001
|
const fileBlock = await (await this.loader.fileStore()).load(fileCid);
|
2072
2002
|
if (!fileBlock) {
|
2073
2003
|
throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
|
@@ -2081,7 +2011,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2081
2011
|
if (operations.length) {
|
2082
2012
|
const lastOp = operations[operations.length - 1];
|
2083
2013
|
await retryableUpload(async () => {
|
2084
|
-
if (!this.loader) return;
|
2085
2014
|
await this.loader.remoteMetaStore?.save(lastOp);
|
2086
2015
|
}, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
|
2087
2016
|
}
|
@@ -2107,7 +2036,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2107
2036
|
}
|
2108
2037
|
try {
|
2109
2038
|
return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
|
2110
|
-
return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
|
2111
2039
|
} catch (e) {
|
2112
2040
|
throw this.logger.Error().Err(e).Msg("error parse").AsError();
|
2113
2041
|
}
|
@@ -2134,92 +2062,72 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2134
2062
|
return Result5.Ok(void 0);
|
2135
2063
|
}
|
2136
2064
|
destroy() {
|
2137
|
-
this.logger.Debug().Msg("destroy");
|
2138
2065
|
return this.gateway.destroy(this.url());
|
2139
2066
|
}
|
2140
2067
|
};
|
2141
2068
|
|
2142
|
-
// src/blockstore/
|
2143
|
-
|
2144
|
-
|
2145
|
-
|
2146
|
-
import { Result as Result6 } from "@adviser/cement";
|
2147
|
-
|
2148
|
-
// src/runtime/gateways/memory/version.ts
|
2149
|
-
var MEMORY_VERSION = "v0.19-memory";
|
2150
|
-
|
2151
|
-
// src/runtime/gateways/memory/gateway.ts
|
2152
|
-
var MemoryGateway = class {
|
2153
|
-
constructor(memorys) {
|
2154
|
-
this.memorys = memorys;
|
2155
|
-
}
|
2156
|
-
buildUrl(baseUrl, key) {
|
2157
|
-
return Promise.resolve(Result6.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
|
2158
|
-
}
|
2159
|
-
start(baseUrl) {
|
2160
|
-
return Promise.resolve(Result6.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
|
2161
|
-
}
|
2162
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2163
|
-
close(baseUrl) {
|
2164
|
-
return Promise.resolve(Result6.Ok(void 0));
|
2165
|
-
}
|
2166
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2167
|
-
destroy(baseUrl) {
|
2168
|
-
this.memorys.clear();
|
2169
|
-
return Promise.resolve(Result6.Ok(void 0));
|
2170
|
-
}
|
2171
|
-
put(url, body) {
|
2172
|
-
this.memorys.set(url.toString(), body);
|
2173
|
-
return Promise.resolve(Result6.Ok(void 0));
|
2174
|
-
}
|
2175
|
-
// get could return a NotFoundError if the key is not found
|
2176
|
-
get(url) {
|
2177
|
-
const x = this.memorys.get(url.toString());
|
2178
|
-
if (x === void 0) {
|
2179
|
-
return Promise.resolve(Result6.Err(new NotFoundError("not found")));
|
2180
|
-
}
|
2181
|
-
return Promise.resolve(Result6.Ok(x));
|
2182
|
-
}
|
2183
|
-
delete(url) {
|
2184
|
-
this.memorys.delete(url.toString());
|
2185
|
-
return Promise.resolve(Result6.Ok(void 0));
|
2186
|
-
}
|
2187
|
-
};
|
2188
|
-
var MemoryTestGateway = class {
|
2189
|
-
constructor(memorys) {
|
2190
|
-
this.memorys = memorys;
|
2069
|
+
// src/blockstore/store-factory.ts
|
2070
|
+
function ensureIsIndex(url, isIndex) {
|
2071
|
+
if (isIndex) {
|
2072
|
+
return url.build().setParam("index", isIndex).URI();
|
2191
2073
|
}
|
2192
|
-
|
2193
|
-
|
2074
|
+
return url.build().delParam("index").URI();
|
2075
|
+
}
|
2076
|
+
function ensureName(name, url) {
|
2077
|
+
if (!url.hasParam("name")) {
|
2078
|
+
return url.build().setParam("name", name).URI();
|
2194
2079
|
}
|
2195
|
-
|
2196
|
-
|
2197
|
-
// src/blockstore/register-store-protocol.ts
|
2080
|
+
return url;
|
2081
|
+
}
|
2198
2082
|
var storeFactory = /* @__PURE__ */ new Map();
|
2199
|
-
function
|
2200
|
-
|
2201
|
-
|
2202
|
-
|
2203
|
-
|
2204
|
-
|
2205
|
-
|
2206
|
-
|
2083
|
+
function buildURL(optURL, loader) {
|
2084
|
+
const storeOpts = loader.ebOpts.store;
|
2085
|
+
const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
|
2086
|
+
let obuUrl;
|
2087
|
+
if (obuItem && obuItem.overrideBaseURL) {
|
2088
|
+
obuUrl = URI5.from(obuItem.overrideBaseURL);
|
2089
|
+
}
|
2090
|
+
const ret = ensureIsIndex(
|
2091
|
+
URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
2092
|
+
storeOpts.isIndex
|
2093
|
+
);
|
2094
|
+
return ret;
|
2095
|
+
}
|
2096
|
+
var onceGateway = new KeyedResolvOnce2();
|
2097
|
+
async function getGatewayFromURL(url, sthis) {
|
2098
|
+
return onceGateway.get(url.toString()).once(async () => {
|
2099
|
+
const item = storeFactory.get(url.protocol);
|
2100
|
+
if (item) {
|
2101
|
+
const ret = {
|
2102
|
+
gateway: await item.gateway(sthis),
|
2103
|
+
test: await item.test(sthis)
|
2104
|
+
};
|
2105
|
+
const res = await ret.gateway.start(url);
|
2106
|
+
if (res.isErr()) {
|
2107
|
+
sthis.logger.Error().Result("start", res).Msg("start failed");
|
2108
|
+
return void 0;
|
2109
|
+
}
|
2110
|
+
return ret;
|
2207
2111
|
}
|
2208
|
-
|
2209
|
-
|
2210
|
-
|
2211
|
-
throw sthis.logger.Error().Msg("no default found").AsError();
|
2212
|
-
}
|
2213
|
-
return found.defaultURI(sthis);
|
2112
|
+
sthis.logger.Warn().Url(url).Msg("unsupported protocol");
|
2113
|
+
return void 0;
|
2114
|
+
});
|
2214
2115
|
}
|
2215
2116
|
function registerStoreProtocol(item) {
|
2216
2117
|
let protocol = item.protocol;
|
2217
2118
|
if (!protocol.endsWith(":")) {
|
2218
2119
|
protocol += ":";
|
2219
2120
|
}
|
2220
|
-
if (
|
2121
|
+
if (storeFactory.has(protocol)) {
|
2122
|
+
if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
|
2123
|
+
throw new Error(`we need a logger here`);
|
2124
|
+
return () => {
|
2125
|
+
};
|
2126
|
+
}
|
2127
|
+
}
|
2128
|
+
if (item.overrideBaseURL) {
|
2221
2129
|
Array.from(storeFactory.values()).forEach((items) => {
|
2222
|
-
items.
|
2130
|
+
items.overrideBaseURL = void 0;
|
2223
2131
|
});
|
2224
2132
|
}
|
2225
2133
|
storeFactory.set(protocol, item);
|
@@ -2227,183 +2135,134 @@ function registerStoreProtocol(item) {
|
|
2227
2135
|
storeFactory.delete(protocol);
|
2228
2136
|
};
|
2229
2137
|
}
|
2230
|
-
|
2231
|
-
|
2232
|
-
|
2233
|
-
|
2234
|
-
return {
|
2235
|
-
|
2236
|
-
|
2237
|
-
|
2238
|
-
return BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
|
2239
|
-
},
|
2240
|
-
gateway: async (sthis) => {
|
2241
|
-
const { FileGateway } = await import("./gateway-GK5QZ6KP.js");
|
2242
|
-
return new FileGateway(sthis);
|
2243
|
-
},
|
2244
|
-
test: async (sthis) => {
|
2245
|
-
const { FileTestStore } = await import("./gateway-GK5QZ6KP.js");
|
2246
|
-
return new FileTestStore(sthis);
|
2247
|
-
}
|
2248
|
-
};
|
2249
|
-
}
|
2250
|
-
if (runtimeFn2().isBrowser) {
|
2251
|
-
registerStoreProtocol({
|
2252
|
-
protocol: "indexdb:",
|
2253
|
-
isDefault: true,
|
2254
|
-
defaultURI: () => {
|
2255
|
-
return BuildURI.from("indexdb://").pathname("fp").URI();
|
2256
|
-
},
|
2257
|
-
gateway: async (logger) => {
|
2258
|
-
const { IndexDBGateway } = await import("./gateway-TQTGDRCN.js");
|
2259
|
-
return new IndexDBGateway(logger);
|
2260
|
-
},
|
2261
|
-
test: async (logger) => {
|
2262
|
-
const { IndexDBTestStore } = await import("./gateway-TQTGDRCN.js");
|
2263
|
-
return new IndexDBTestStore(logger);
|
2264
|
-
}
|
2265
|
-
});
|
2266
|
-
} else {
|
2267
|
-
registerStoreProtocol(fileGatewayFactoryItem());
|
2268
|
-
}
|
2269
|
-
var memory = /* @__PURE__ */ new Map();
|
2270
|
-
registerStoreProtocol({
|
2271
|
-
protocol: "memory:",
|
2272
|
-
isDefault: false,
|
2273
|
-
defaultURI: () => {
|
2274
|
-
return BuildURI.from("memory://").pathname("ram").URI();
|
2275
|
-
},
|
2276
|
-
gateway: async () => {
|
2277
|
-
return new MemoryGateway(memory);
|
2278
|
-
},
|
2279
|
-
test: async () => {
|
2280
|
-
return new MemoryTestGateway(memory);
|
2281
|
-
}
|
2282
|
-
});
|
2283
|
-
|
2284
|
-
// src/blockstore/store-factory.ts
|
2285
|
-
var onceGateway = new KeyedResolvOnce2();
|
2286
|
-
var gatewayInstances = new KeyedResolvOnce2();
|
2287
|
-
async function getStartedGateway(sthis, url) {
|
2288
|
-
return onceGateway.get(url.toString()).once(async () => {
|
2289
|
-
const item = getGatewayFactoryItem(url.protocol);
|
2290
|
-
if (item) {
|
2291
|
-
const ret = {
|
2292
|
-
url,
|
2293
|
-
...await gatewayInstances.get(url.protocol).once(async () => ({
|
2294
|
-
gateway: await item.gateway(sthis),
|
2295
|
-
test: await item.test(sthis)
|
2296
|
-
}))
|
2297
|
-
};
|
2298
|
-
const res = await ret.gateway.start(url);
|
2299
|
-
if (res.isErr()) {
|
2300
|
-
return Result7.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
|
2301
|
-
}
|
2302
|
-
ret.url = res.Ok();
|
2303
|
-
return Result7.Ok(ret);
|
2138
|
+
var onceDataStoreFactory = new KeyedResolvOnce2();
|
2139
|
+
async function dataStoreFactory(loader) {
|
2140
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
|
2141
|
+
const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
|
2142
|
+
return onceDataStoreFactory.get(url.toString()).once(async () => {
|
2143
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2144
|
+
if (!gateway) {
|
2145
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2304
2146
|
}
|
2305
|
-
|
2306
|
-
|
2307
|
-
|
2308
|
-
|
2309
|
-
|
2310
|
-
|
2311
|
-
|
2312
|
-
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
2313
|
-
}
|
2314
|
-
const gateway = rgateway.Ok();
|
2315
|
-
const store = new DataStoreImpl(sfi.sthis, gateway.url, {
|
2316
|
-
gateway: gateway.gateway,
|
2317
|
-
keybag: sfi.keybag
|
2147
|
+
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2148
|
+
gateway: gateway.gateway,
|
2149
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2150
|
+
...loader.ebOpts.keyBag
|
2151
|
+
})
|
2152
|
+
});
|
2153
|
+
return store;
|
2318
2154
|
});
|
2319
|
-
return store;
|
2320
2155
|
}
|
2321
|
-
|
2322
|
-
|
2323
|
-
const
|
2324
|
-
|
2325
|
-
|
2326
|
-
|
2327
|
-
|
2328
|
-
|
2329
|
-
|
2330
|
-
|
2156
|
+
var onceMetaStoreFactory = new KeyedResolvOnce2();
|
2157
|
+
async function metaStoreFactory(loader) {
|
2158
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
|
2159
|
+
const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
|
2160
|
+
return onceMetaStoreFactory.get(url.toString()).once(async () => {
|
2161
|
+
sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
|
2162
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2163
|
+
if (!gateway) {
|
2164
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2165
|
+
}
|
2166
|
+
const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
|
2167
|
+
gateway: gateway.gateway,
|
2168
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2169
|
+
...loader.ebOpts.keyBag
|
2170
|
+
})
|
2171
|
+
});
|
2172
|
+
return store;
|
2331
2173
|
});
|
2332
|
-
return store;
|
2333
2174
|
}
|
2334
|
-
|
2335
|
-
|
2336
|
-
const
|
2337
|
-
|
2338
|
-
|
2339
|
-
|
2340
|
-
|
2341
|
-
|
2342
|
-
|
2343
|
-
|
2175
|
+
var onceRemoteWalFactory = new KeyedResolvOnce2();
|
2176
|
+
async function remoteWalFactory(loader) {
|
2177
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
|
2178
|
+
const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
|
2179
|
+
return onceRemoteWalFactory.get(url.toString()).once(async () => {
|
2180
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2181
|
+
if (!gateway) {
|
2182
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2183
|
+
}
|
2184
|
+
sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
|
2185
|
+
const store = new WALStoreImpl(loader, url, {
|
2186
|
+
gateway: gateway.gateway,
|
2187
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2188
|
+
...loader.ebOpts.keyBag
|
2189
|
+
})
|
2190
|
+
});
|
2191
|
+
return store;
|
2344
2192
|
});
|
2345
|
-
return store;
|
2346
2193
|
}
|
2347
2194
|
async function testStoreFactory(url, sthis) {
|
2348
|
-
|
2349
|
-
|
2195
|
+
sthis = ensureSuperLog(sthis, "testStoreFactory");
|
2196
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2197
|
+
if (!gateway) {
|
2350
2198
|
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2351
2199
|
}
|
2352
|
-
return
|
2200
|
+
return gateway.test;
|
2353
2201
|
}
|
2354
|
-
async function ensureStart(store) {
|
2202
|
+
async function ensureStart(store, logger) {
|
2355
2203
|
const ret = await store.start();
|
2356
2204
|
if (ret.isErr()) {
|
2357
|
-
throw
|
2205
|
+
throw logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2358
2206
|
}
|
2359
|
-
|
2207
|
+
logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2360
2208
|
return store;
|
2361
2209
|
}
|
2362
|
-
function
|
2363
|
-
|
2364
|
-
return {
|
2365
|
-
encodeFile: ende.encodeFile || encodeFile,
|
2366
|
-
decodeFile: ende.decodeFile || decodeFile
|
2367
|
-
};
|
2368
|
-
}
|
2369
|
-
function toStoreRuntime(sthis, endeOpts = {}) {
|
2210
|
+
function toStoreRuntime(opts, sthis) {
|
2211
|
+
const logger = ensureLogger(sthis, "toStoreRuntime", {});
|
2370
2212
|
return {
|
2371
|
-
makeMetaStore: async (
|
2372
|
-
|
2373
|
-
|
2374
|
-
|
2375
|
-
|
2376
|
-
|
2377
|
-
|
2378
|
-
|
2379
|
-
|
2380
|
-
|
2381
|
-
|
2382
|
-
|
2383
|
-
|
2384
|
-
|
2385
|
-
// return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
|
2386
|
-
// },
|
2387
|
-
makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
|
2388
|
-
// async (loader: Loadable) => {
|
2389
|
-
// logger
|
2390
|
-
// .Debug()
|
2391
|
-
// .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
|
2392
|
-
// .Msg("makeRemoteWAL");
|
2393
|
-
// return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
|
2394
|
-
// },
|
2395
|
-
...ensureStoreEnDeFile(endeOpts)
|
2213
|
+
makeMetaStore: async (loader) => {
|
2214
|
+
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
|
2215
|
+
return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
|
2216
|
+
},
|
2217
|
+
makeDataStore: async (loader) => {
|
2218
|
+
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
|
2219
|
+
return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
|
2220
|
+
},
|
2221
|
+
makeWALStore: async (loader) => {
|
2222
|
+
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
|
2223
|
+
return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
|
2224
|
+
},
|
2225
|
+
encodeFile: opts.encodeFile || encodeFile,
|
2226
|
+
decodeFile: opts.decodeFile || decodeFile
|
2396
2227
|
};
|
2397
2228
|
}
|
2229
|
+
registerStoreProtocol({
|
2230
|
+
protocol: "file:",
|
2231
|
+
gateway: async (sthis) => {
|
2232
|
+
const { FileGateway } = await import("./gateway-5FCWPX5W.js");
|
2233
|
+
return new FileGateway(sthis);
|
2234
|
+
},
|
2235
|
+
test: async (sthis) => {
|
2236
|
+
const { FileTestStore } = await import("./gateway-5FCWPX5W.js");
|
2237
|
+
return new FileTestStore(sthis);
|
2238
|
+
}
|
2239
|
+
});
|
2240
|
+
registerStoreProtocol({
|
2241
|
+
protocol: "indexdb:",
|
2242
|
+
gateway: async (sthis) => {
|
2243
|
+
const { IndexDBGateway } = await import("./gateway-H7UD6TNB.js");
|
2244
|
+
return new IndexDBGateway(sthis);
|
2245
|
+
},
|
2246
|
+
test: async (sthis) => {
|
2247
|
+
const { IndexDBTestStore } = await import("./gateway-H7UD6TNB.js");
|
2248
|
+
return new IndexDBTestStore(sthis);
|
2249
|
+
}
|
2250
|
+
});
|
2398
2251
|
|
2399
2252
|
// src/blockstore/store-remote.ts
|
2400
2253
|
async function RemoteDataStore(sthis, name, url, opts) {
|
2401
|
-
const ds = new DataStoreImpl(sthis, url, opts);
|
2254
|
+
const ds = new DataStoreImpl(sthis, name, url, opts);
|
2402
2255
|
await ds.start();
|
2403
2256
|
return ds;
|
2404
2257
|
}
|
2405
2258
|
async function RemoteMetaStore(sthis, name, url, opts) {
|
2406
|
-
const ms = new MetaStoreImpl(
|
2259
|
+
const ms = new MetaStoreImpl(
|
2260
|
+
sthis,
|
2261
|
+
name,
|
2262
|
+
url,
|
2263
|
+
opts
|
2264
|
+
/* , true*/
|
2265
|
+
);
|
2407
2266
|
await ms.start();
|
2408
2267
|
return ms;
|
2409
2268
|
}
|
@@ -2428,17 +2287,14 @@ var ConnectionBase = class {
|
|
2428
2287
|
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
2429
2288
|
this.loader = loader;
|
2430
2289
|
await this.onConnect();
|
2431
|
-
const metaUrl = this.url.build().defParam("store"
|
2432
|
-
const
|
2433
|
-
if (
|
2434
|
-
throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
2435
|
-
const name = metaUrl.toString();
|
2290
|
+
const metaUrl = this.url.build().defParam("store", "meta").URI();
|
2291
|
+
const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
|
2292
|
+
if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
2436
2293
|
const dbName = metaUrl.getParam("name");
|
2437
|
-
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X:
|
2438
|
-
const
|
2439
|
-
const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
|
2294
|
+
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
|
2295
|
+
const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
|
2440
2296
|
gateway: gateway.gateway,
|
2441
|
-
keybag:
|
2297
|
+
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
|
2442
2298
|
loader
|
2443
2299
|
});
|
2444
2300
|
this.loader.remoteMetaStore = remote;
|
@@ -2451,15 +2307,14 @@ var ConnectionBase = class {
|
|
2451
2307
|
async connectStorage_X({ loader }) {
|
2452
2308
|
if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
|
2453
2309
|
this.loader = loader;
|
2454
|
-
const dataUrl = this.url.build().defParam("store"
|
2455
|
-
const
|
2456
|
-
if (
|
2457
|
-
throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
2310
|
+
const dataUrl = this.url.build().defParam("store", "data").URI();
|
2311
|
+
const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
|
2312
|
+
if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
2458
2313
|
const name = dataUrl.getParam("name");
|
2459
2314
|
if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
|
2460
2315
|
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
2461
|
-
gateway:
|
2462
|
-
keybag:
|
2316
|
+
gateway: gateway.gateway,
|
2317
|
+
keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
|
2463
2318
|
});
|
2464
2319
|
loader.remoteFileStore = loader.remoteCarStore;
|
2465
2320
|
}
|
@@ -2499,12 +2354,6 @@ var ConnectionBase = class {
|
|
2499
2354
|
};
|
2500
2355
|
|
2501
2356
|
// src/crdt-helpers.ts
|
2502
|
-
import { parse as parse3 } from "multiformats/link";
|
2503
|
-
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
2504
|
-
import * as codec from "@ipld/dag-cbor";
|
2505
|
-
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
2506
|
-
import { EventFetcher, vis } from "@web3-storage/pail/clock";
|
2507
|
-
import * as Batch from "@web3-storage/pail/crdt/batch";
|
2508
2357
|
function time(tag) {
|
2509
2358
|
}
|
2510
2359
|
function timeEnd(tag) {
|
@@ -2552,7 +2401,7 @@ async function writeDocContent(store, blocks, update, logger) {
|
|
2552
2401
|
await processFiles(store, blocks, update.value, logger);
|
2553
2402
|
value = { doc: update.value };
|
2554
2403
|
}
|
2555
|
-
const block = await
|
2404
|
+
const block = await encode({ value, hasher: hasher5, codec });
|
2556
2405
|
blocks.putSync(block.cid, block.bytes);
|
2557
2406
|
return block.cid;
|
2558
2407
|
}
|
@@ -2643,7 +2492,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
2643
2492
|
async function getValueFromLink(blocks, link, logger) {
|
2644
2493
|
const block = await blocks.get(link);
|
2645
2494
|
if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
|
2646
|
-
const { value } = await
|
2495
|
+
const { value } = await decode({ bytes: block.bytes, hasher: hasher5, codec });
|
2647
2496
|
const cvalue = {
|
2648
2497
|
...value,
|
2649
2498
|
cid: link
|
@@ -2764,7 +2613,7 @@ async function doCompact(blockLog, head, logger) {
|
|
2764
2613
|
async function getBlock(blocks, cidString) {
|
2765
2614
|
const block = await blocks.get(parse3(cidString));
|
2766
2615
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
2767
|
-
const { cid, value } = await
|
2616
|
+
const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
|
2768
2617
|
return new Block({ cid, value, bytes: block.bytes });
|
2769
2618
|
}
|
2770
2619
|
|
@@ -2822,8 +2671,7 @@ function makeProllyGetBlock(blocks) {
|
|
2822
2671
|
return create({ cid, bytes, hasher: hasher6, codec: codec2 });
|
2823
2672
|
};
|
2824
2673
|
}
|
2825
|
-
async function bulkIndex(
|
2826
|
-
logger.Debug().Msg("enter bulkIndex");
|
2674
|
+
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
2827
2675
|
if (!indexEntries.length) return inIndex;
|
2828
2676
|
if (!inIndex.root) {
|
2829
2677
|
if (!inIndex.cid) {
|
@@ -2840,22 +2688,18 @@ async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
|
|
2840
2688
|
returnNode = node;
|
2841
2689
|
}
|
2842
2690
|
if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
|
2843
|
-
logger.Debug().Msg("exit !root bulkIndex");
|
2844
2691
|
return { root: returnNode, cid: returnRootBlock.cid };
|
2845
2692
|
} else {
|
2846
2693
|
inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
|
2847
2694
|
}
|
2848
2695
|
}
|
2849
|
-
logger.Debug().Msg("pre bulk bulkIndex");
|
2850
2696
|
const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
|
2851
2697
|
if (root3) {
|
2852
|
-
logger.Debug().Msg("pre root put bulkIndex");
|
2853
2698
|
for await (const block of newBlocks) {
|
2854
2699
|
await tblocks.put(block.cid, block.bytes);
|
2855
2700
|
}
|
2856
2701
|
return { root: root3, cid: (await root3.block).cid };
|
2857
2702
|
} else {
|
2858
|
-
logger.Debug().Msg("pre !root bulkIndex");
|
2859
2703
|
return { root: void 0, cid: void 0 };
|
2860
2704
|
}
|
2861
2705
|
}
|
@@ -2895,17 +2739,17 @@ function encodeKey(key) {
|
|
2895
2739
|
}
|
2896
2740
|
|
2897
2741
|
// src/indexer.ts
|
2898
|
-
function index(
|
2899
|
-
if (mapFn && meta) throw
|
2900
|
-
if (mapFn && mapFn.constructor.name !== "Function") throw
|
2901
|
-
if (
|
2902
|
-
const idx =
|
2742
|
+
function index(sthis, { _crdt }, name, mapFn, meta) {
|
2743
|
+
if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
2744
|
+
if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
|
2745
|
+
if (_crdt.indexers.has(name)) {
|
2746
|
+
const idx = _crdt.indexers.get(name);
|
2903
2747
|
idx.applyMapFn(name, mapFn, meta);
|
2904
2748
|
} else {
|
2905
|
-
const idx = new Index(
|
2906
|
-
|
2749
|
+
const idx = new Index(sthis, _crdt, name, mapFn, meta);
|
2750
|
+
_crdt.indexers.set(name, idx);
|
2907
2751
|
}
|
2908
|
-
return
|
2752
|
+
return _crdt.indexers.get(name);
|
2909
2753
|
}
|
2910
2754
|
var Index = class {
|
2911
2755
|
constructor(sthis, crdt, name, mapFn, meta) {
|
@@ -2924,9 +2768,18 @@ var Index = class {
|
|
2924
2768
|
return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
|
2925
2769
|
});
|
2926
2770
|
}
|
2771
|
+
close() {
|
2772
|
+
return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
|
2773
|
+
});
|
2774
|
+
}
|
2775
|
+
destroy() {
|
2776
|
+
return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
|
2777
|
+
});
|
2778
|
+
}
|
2927
2779
|
applyMapFn(name, mapFn, meta) {
|
2928
2780
|
if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
2929
2781
|
if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
|
2782
|
+
this.name = name;
|
2930
2783
|
try {
|
2931
2784
|
if (meta) {
|
2932
2785
|
if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
|
@@ -2974,13 +2827,9 @@ var Index = class {
|
|
2974
2827
|
}
|
2975
2828
|
}
|
2976
2829
|
async query(opts = {}) {
|
2977
|
-
this.logger.Debug().Msg("enter query");
|
2978
2830
|
await this.ready();
|
2979
|
-
this.logger.Debug().Msg("post ready query");
|
2980
2831
|
await this._updateIndex();
|
2981
|
-
this.logger.Debug().Msg("post _updateIndex query");
|
2982
2832
|
await this._hydrateIndex();
|
2983
|
-
this.logger.Debug().Msg("post _hydrateIndex query");
|
2984
2833
|
if (!this.byKey.root) {
|
2985
2834
|
return await applyQuery(this.crdt, { result: [] }, opts);
|
2986
2835
|
}
|
@@ -3036,16 +2885,13 @@ var Index = class {
|
|
3036
2885
|
}
|
3037
2886
|
async _updateIndex() {
|
3038
2887
|
await this.ready();
|
3039
|
-
this.logger.Debug().Msg("enter _updateIndex");
|
3040
2888
|
if (this.initError) throw this.initError;
|
3041
2889
|
if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
|
3042
2890
|
let result, head;
|
3043
2891
|
if (!this.indexHead || this.indexHead.length === 0) {
|
3044
2892
|
({ result, head } = await this.crdt.allDocs());
|
3045
|
-
this.logger.Debug().Msg("enter crdt.allDocs");
|
3046
2893
|
} else {
|
3047
2894
|
({ result, head } = await this.crdt.changes(this.indexHead));
|
3048
|
-
this.logger.Debug().Msg("enter crdt.changes");
|
3049
2895
|
}
|
3050
2896
|
if (result.length === 0) {
|
3051
2897
|
this.indexHead = head;
|
@@ -3078,22 +2924,9 @@ var Index = class {
|
|
3078
2924
|
if (result.length === 0) {
|
3079
2925
|
return indexerMeta;
|
3080
2926
|
}
|
3081
|
-
this.logger.Debug().Msg("pre this.blockstore.transaction");
|
3082
2927
|
const { meta } = await this.blockstore.transaction(async (tblocks) => {
|
3083
|
-
this.byId = await bulkIndex(
|
3084
|
-
|
3085
|
-
tblocks,
|
3086
|
-
this.byId,
|
3087
|
-
removeIdIndexEntries.concat(byIdIndexEntries),
|
3088
|
-
byIdOpts
|
3089
|
-
);
|
3090
|
-
this.byKey = await bulkIndex(
|
3091
|
-
this.logger,
|
3092
|
-
tblocks,
|
3093
|
-
this.byKey,
|
3094
|
-
staleKeyIndexEntries.concat(indexEntries),
|
3095
|
-
byKeyOpts
|
3096
|
-
);
|
2928
|
+
this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
|
2929
|
+
this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
|
3097
2930
|
this.indexHead = head;
|
3098
2931
|
if (this.byId.cid && this.byKey.cid) {
|
3099
2932
|
const idxMeta = {
|
@@ -3105,10 +2938,8 @@ var Index = class {
|
|
3105
2938
|
};
|
3106
2939
|
indexerMeta.indexes?.set(this.name, idxMeta);
|
3107
2940
|
}
|
3108
|
-
this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
|
3109
2941
|
return indexerMeta;
|
3110
2942
|
});
|
3111
|
-
this.logger.Debug().Msg("post this.blockstore.transaction");
|
3112
2943
|
return meta;
|
3113
2944
|
}
|
3114
2945
|
};
|
@@ -3284,13 +3115,15 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
3284
3115
|
|
3285
3116
|
// src/crdt.ts
|
3286
3117
|
var CRDT = class {
|
3287
|
-
constructor(sthis, opts) {
|
3118
|
+
constructor(sthis, name, opts = {}) {
|
3288
3119
|
this.indexers = /* @__PURE__ */ new Map();
|
3289
3120
|
this.onceReady = new ResolveOnce5();
|
3290
3121
|
this.sthis = sthis;
|
3122
|
+
this.name = name;
|
3291
3123
|
this.logger = ensureLogger(sthis, "CRDT");
|
3292
3124
|
this.opts = opts;
|
3293
3125
|
this.blockstore = blockstoreFactory(sthis, {
|
3126
|
+
name,
|
3294
3127
|
applyMeta: async (meta) => {
|
3295
3128
|
const crdtMeta = meta;
|
3296
3129
|
if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
|
@@ -3300,27 +3133,23 @@ var CRDT = class {
|
|
3300
3133
|
await doCompact(blocks, this.clock.head, this.logger);
|
3301
3134
|
return { head: this.clock.head };
|
3302
3135
|
},
|
3303
|
-
|
3304
|
-
|
3305
|
-
|
3306
|
-
|
3307
|
-
|
3308
|
-
meta: this.opts.meta
|
3309
|
-
// threshold: this.opts.threshold,
|
3136
|
+
autoCompact: this.opts.autoCompact || 100,
|
3137
|
+
store: { ...this.opts.store, isIndex: void 0 },
|
3138
|
+
public: this.opts.public,
|
3139
|
+
meta: this.opts.meta,
|
3140
|
+
threshold: this.opts.threshold
|
3310
3141
|
});
|
3311
3142
|
this.indexBlockstore = blockstoreFactory(sthis, {
|
3312
|
-
|
3143
|
+
name,
|
3313
3144
|
applyMeta: async (meta) => {
|
3314
3145
|
const idxCarMeta = meta;
|
3315
3146
|
if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
|
3316
|
-
for (const [
|
3317
|
-
index({
|
3147
|
+
for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
|
3148
|
+
index(this.sthis, { _crdt: this }, name2, void 0, idx);
|
3318
3149
|
}
|
3319
3150
|
},
|
3320
|
-
|
3321
|
-
|
3322
|
-
keyBag: this.opts.keyBag
|
3323
|
-
// public: this.opts.public,
|
3151
|
+
store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
|
3152
|
+
public: this.opts.public
|
3324
3153
|
});
|
3325
3154
|
this.clock = new CRDTClock(this.blockstore);
|
3326
3155
|
this.clock.onZoom(() => {
|
@@ -3402,164 +3231,51 @@ var CRDT = class {
|
|
3402
3231
|
};
|
3403
3232
|
|
3404
3233
|
// src/database.ts
|
3405
|
-
var
|
3406
|
-
|
3407
|
-
|
3408
|
-
return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
|
3409
|
-
}
|
3410
|
-
function keyConfigOpts(sthis, name, opts) {
|
3411
|
-
return JSON.stringify(
|
3412
|
-
toSortedArray({
|
3413
|
-
name,
|
3414
|
-
stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
|
3415
|
-
})
|
3416
|
-
);
|
3417
|
-
}
|
3418
|
-
function isDatabase(db) {
|
3419
|
-
return db instanceof DatabaseImpl || db instanceof DatabaseShell;
|
3420
|
-
}
|
3421
|
-
function DatabaseFactory(name, opts) {
|
3422
|
-
const sthis = ensureSuperThis(opts);
|
3423
|
-
return new DatabaseShell(
|
3424
|
-
databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
|
3425
|
-
const db = new DatabaseImpl(sthis, {
|
3426
|
-
name,
|
3427
|
-
meta: opts?.meta,
|
3428
|
-
keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
|
3429
|
-
storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
|
3430
|
-
storeEnDe: {
|
3431
|
-
encodeFile,
|
3432
|
-
decodeFile,
|
3433
|
-
...opts?.storeEnDe
|
3434
|
-
}
|
3435
|
-
});
|
3436
|
-
db.onClosed(() => {
|
3437
|
-
databases.unget(key);
|
3438
|
-
});
|
3439
|
-
return db;
|
3440
|
-
})
|
3441
|
-
);
|
3442
|
-
}
|
3443
|
-
var DatabaseShell = class {
|
3444
|
-
constructor(ref) {
|
3445
|
-
this.ref = ref;
|
3446
|
-
ref.addShell(this);
|
3447
|
-
}
|
3448
|
-
get id() {
|
3449
|
-
return this.ref.id;
|
3450
|
-
}
|
3451
|
-
get logger() {
|
3452
|
-
return this.ref.logger;
|
3453
|
-
}
|
3454
|
-
get sthis() {
|
3455
|
-
return this.ref.sthis;
|
3456
|
-
}
|
3457
|
-
get crdt() {
|
3458
|
-
return this.ref.crdt;
|
3459
|
-
}
|
3460
|
-
name() {
|
3461
|
-
return this.ref.name();
|
3462
|
-
}
|
3463
|
-
onClosed(fn) {
|
3464
|
-
return this.ref.onClosed(fn);
|
3465
|
-
}
|
3466
|
-
close() {
|
3467
|
-
return this.ref.shellClose(this);
|
3468
|
-
}
|
3469
|
-
destroy() {
|
3470
|
-
return this.ref.destroy();
|
3471
|
-
}
|
3472
|
-
ready() {
|
3473
|
-
return this.ref.ready();
|
3474
|
-
}
|
3475
|
-
get(id) {
|
3476
|
-
return this.ref.get(id);
|
3477
|
-
}
|
3478
|
-
put(doc) {
|
3479
|
-
return this.ref.put(doc);
|
3480
|
-
}
|
3481
|
-
del(id) {
|
3482
|
-
return this.ref.del(id);
|
3483
|
-
}
|
3484
|
-
changes(since, opts) {
|
3485
|
-
return this.ref.changes(since, opts);
|
3486
|
-
}
|
3487
|
-
allDocs(opts) {
|
3488
|
-
return this.ref.allDocs(opts);
|
3489
|
-
}
|
3490
|
-
allDocuments() {
|
3491
|
-
return this.ref.allDocuments();
|
3492
|
-
}
|
3493
|
-
subscribe(listener, updates) {
|
3494
|
-
return this.ref.subscribe(listener, updates);
|
3495
|
-
}
|
3496
|
-
query(field, opts) {
|
3497
|
-
return this.ref.query(field, opts);
|
3498
|
-
}
|
3499
|
-
compact() {
|
3500
|
-
return this.ref.compact();
|
3501
|
-
}
|
3502
|
-
};
|
3503
|
-
var DatabaseImpl = class {
|
3504
|
-
constructor(sthis, opts) {
|
3234
|
+
var Database = class {
|
3235
|
+
constructor(name, opts) {
|
3236
|
+
this.opts = {};
|
3505
3237
|
this._listening = false;
|
3506
3238
|
this._listeners = /* @__PURE__ */ new Set();
|
3507
3239
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
3508
|
-
// readonly blockstore: BaseBlockstore;
|
3509
|
-
this.shells = /* @__PURE__ */ new Set();
|
3510
|
-
this._onClosedFns = /* @__PURE__ */ new Set();
|
3511
3240
|
this._ready = new ResolveOnce6();
|
3512
|
-
this.
|
3513
|
-
this.
|
3514
|
-
this.
|
3241
|
+
this.name = name;
|
3242
|
+
this.opts = opts || this.opts;
|
3243
|
+
this.sthis = ensureSuperThis(this.opts);
|
3515
3244
|
this.logger = ensureLogger(this.sthis, "Database");
|
3516
|
-
this.
|
3245
|
+
this._crdt = new CRDT(this.sthis, name, this.opts);
|
3246
|
+
this.blockstore = this._crdt.blockstore;
|
3517
3247
|
this._writeQueue = writeQueue(async (updates) => {
|
3518
|
-
return await this.
|
3248
|
+
return await this._crdt.bulk(updates);
|
3519
3249
|
});
|
3520
|
-
this.
|
3250
|
+
this._crdt.clock.onTock(() => {
|
3521
3251
|
this._no_update_notify();
|
3522
3252
|
});
|
3523
3253
|
}
|
3524
|
-
|
3525
|
-
this.
|
3526
|
-
}
|
3527
|
-
onClosed(fn) {
|
3528
|
-
this._onClosedFns.add(fn);
|
3254
|
+
static {
|
3255
|
+
this.databases = /* @__PURE__ */ new Map();
|
3529
3256
|
}
|
3530
3257
|
async close() {
|
3531
|
-
|
3532
|
-
|
3533
|
-
|
3534
|
-
if (!this.shells.has(db)) {
|
3535
|
-
throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
|
3536
|
-
}
|
3537
|
-
this.shells.delete(db);
|
3538
|
-
if (this.shells.size === 0) {
|
3539
|
-
await this.ready();
|
3540
|
-
await this.crdt.close();
|
3541
|
-
this._onClosedFns.forEach((fn) => fn());
|
3542
|
-
}
|
3258
|
+
await this.ready();
|
3259
|
+
await this._crdt.close();
|
3260
|
+
await this.blockstore.close();
|
3543
3261
|
}
|
3544
3262
|
async destroy() {
|
3545
3263
|
await this.ready();
|
3546
|
-
await this.
|
3264
|
+
await this._crdt.destroy();
|
3265
|
+
await this.blockstore.destroy();
|
3547
3266
|
}
|
3548
3267
|
async ready() {
|
3549
|
-
|
3268
|
+
return this._ready.once(async () => {
|
3550
3269
|
await this.sthis.start();
|
3551
|
-
await this.
|
3270
|
+
await this._crdt.ready();
|
3271
|
+
await this.blockstore.ready();
|
3552
3272
|
});
|
3553
|
-
return ret;
|
3554
|
-
}
|
3555
|
-
name() {
|
3556
|
-
return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
|
3557
3273
|
}
|
3558
3274
|
async get(id) {
|
3559
|
-
if (!id) throw this.logger.Error().Str("db", this.name
|
3275
|
+
if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
|
3560
3276
|
await this.ready();
|
3561
3277
|
this.logger.Debug().Str("id", id).Msg("get");
|
3562
|
-
const got = await this.
|
3278
|
+
const got = await this._crdt.get(id).catch((e) => {
|
3563
3279
|
throw new NotFoundError(`Not found: ${id} - ${e.message}`);
|
3564
3280
|
});
|
3565
3281
|
if (!got) throw new NotFoundError(`Not found: ${id}`);
|
@@ -3578,34 +3294,34 @@ var DatabaseImpl = class {
|
|
3578
3294
|
_id: docId
|
3579
3295
|
}
|
3580
3296
|
});
|
3581
|
-
return { id: docId, clock: result?.head, name: this.name
|
3297
|
+
return { id: docId, clock: result?.head, name: this.name };
|
3582
3298
|
}
|
3583
3299
|
async del(id) {
|
3584
3300
|
await this.ready();
|
3585
3301
|
this.logger.Debug().Str("id", id).Msg("del");
|
3586
3302
|
const result = await this._writeQueue.push({ id, del: true });
|
3587
|
-
return { id, clock: result?.head, name: this.name
|
3303
|
+
return { id, clock: result?.head, name: this.name };
|
3588
3304
|
}
|
3589
3305
|
async changes(since = [], opts = {}) {
|
3590
3306
|
await this.ready();
|
3591
3307
|
this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
|
3592
|
-
const { result, head } = await this.
|
3308
|
+
const { result, head } = await this._crdt.changes(since, opts);
|
3593
3309
|
const rows = result.map(({ id: key, value, del, clock }) => ({
|
3594
3310
|
key,
|
3595
3311
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
|
3596
3312
|
clock
|
3597
3313
|
}));
|
3598
|
-
return { rows, clock: head, name: this.name
|
3314
|
+
return { rows, clock: head, name: this.name };
|
3599
3315
|
}
|
3600
3316
|
async allDocs(opts = {}) {
|
3601
3317
|
await this.ready();
|
3602
3318
|
this.logger.Debug().Msg("allDocs");
|
3603
|
-
const { result, head } = await this.
|
3319
|
+
const { result, head } = await this._crdt.allDocs();
|
3604
3320
|
const rows = result.map(({ id: key, value, del }) => ({
|
3605
3321
|
key,
|
3606
3322
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
|
3607
3323
|
}));
|
3608
|
-
return { rows, clock: head, name: this.name
|
3324
|
+
return { rows, clock: head, name: this.name };
|
3609
3325
|
}
|
3610
3326
|
async allDocuments() {
|
3611
3327
|
return this.allDocs();
|
@@ -3615,7 +3331,7 @@ var DatabaseImpl = class {
|
|
3615
3331
|
if (updates) {
|
3616
3332
|
if (!this._listening) {
|
3617
3333
|
this._listening = true;
|
3618
|
-
this.
|
3334
|
+
this._crdt.clock.onTick((updates2) => {
|
3619
3335
|
void this._notify(updates2);
|
3620
3336
|
});
|
3621
3337
|
}
|
@@ -3634,13 +3350,13 @@ var DatabaseImpl = class {
|
|
3634
3350
|
async query(field, opts = {}) {
|
3635
3351
|
await this.ready();
|
3636
3352
|
this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
|
3637
|
-
const _crdt = this.
|
3638
|
-
const idx = typeof field === "string" ? index({
|
3353
|
+
const _crdt = this._crdt;
|
3354
|
+
const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
|
3639
3355
|
return await idx.query(opts);
|
3640
3356
|
}
|
3641
3357
|
async compact() {
|
3642
3358
|
await this.ready();
|
3643
|
-
await this.
|
3359
|
+
await this._crdt.compact();
|
3644
3360
|
}
|
3645
3361
|
async _notify(updates) {
|
3646
3362
|
await this.ready();
|
@@ -3664,62 +3380,23 @@ var DatabaseImpl = class {
|
|
3664
3380
|
}
|
3665
3381
|
}
|
3666
3382
|
};
|
3667
|
-
function
|
3668
|
-
|
3669
|
-
|
3670
|
-
if (!ret.hasParam("name" /* NAME */)) {
|
3671
|
-
const name = sthis.pathOps.basename(ret.URI().pathname);
|
3672
|
-
if (!name) {
|
3673
|
-
throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
|
3674
|
-
}
|
3675
|
-
ret.setParam("name" /* NAME */, name);
|
3676
|
-
}
|
3677
|
-
if (ctx.idx) {
|
3678
|
-
ret.defParam("index" /* INDEX */, "idx");
|
3679
|
-
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
|
3680
|
-
} else {
|
3681
|
-
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
|
3682
|
-
}
|
3683
|
-
if (store === "data") {
|
3684
|
-
if (ctx.file) {
|
3685
|
-
} else {
|
3686
|
-
ret.defParam("suffix" /* SUFFIX */, ".car");
|
3687
|
-
}
|
3688
|
-
}
|
3689
|
-
return ret.URI();
|
3690
|
-
}
|
3691
|
-
function toStoreURIRuntime(sthis, name, sopts) {
|
3692
|
-
sopts = sopts || {};
|
3693
|
-
if (!sopts.base) {
|
3694
|
-
const fp_env = sthis.env.get("FP_STORAGE_URL");
|
3695
|
-
if (fp_env) {
|
3696
|
-
sopts = { ...sopts, base: BuildURI2.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
|
3697
|
-
} else {
|
3698
|
-
sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
|
3699
|
-
}
|
3700
|
-
}
|
3701
|
-
const bbase = BuildURI2.from(sopts.base);
|
3702
|
-
if (name) {
|
3703
|
-
bbase.setParam("name" /* NAME */, name);
|
3704
|
-
}
|
3705
|
-
const base = bbase.URI();
|
3706
|
-
return {
|
3707
|
-
idx: {
|
3708
|
-
data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
|
3709
|
-
file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
|
3710
|
-
meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
|
3711
|
-
wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
|
3712
|
-
},
|
3713
|
-
data: {
|
3714
|
-
data: defaultURI(sthis, sopts.data?.data, base, "data"),
|
3715
|
-
file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
|
3716
|
-
meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
|
3717
|
-
wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
|
3718
|
-
}
|
3719
|
-
};
|
3383
|
+
function toSortedArray(set) {
|
3384
|
+
if (!set) return [];
|
3385
|
+
return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
|
3720
3386
|
}
|
3721
3387
|
function fireproof(name, opts) {
|
3722
|
-
|
3388
|
+
const key = JSON.stringify(
|
3389
|
+
toSortedArray({
|
3390
|
+
name,
|
3391
|
+
stores: toSortedArray(opts?.store?.stores)
|
3392
|
+
})
|
3393
|
+
);
|
3394
|
+
let db = Database.databases.get(key);
|
3395
|
+
if (!db) {
|
3396
|
+
db = new Database(name, opts);
|
3397
|
+
Database.databases.set(key, db);
|
3398
|
+
}
|
3399
|
+
return db;
|
3723
3400
|
}
|
3724
3401
|
function makeName(fnString) {
|
3725
3402
|
const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
|
@@ -3750,7 +3427,7 @@ __export(runtime_exports, {
|
|
3750
3427
|
kb: () => key_bag_exports,
|
3751
3428
|
kc: () => keyed_crypto_exports,
|
3752
3429
|
mf: () => wait_pr_multiformats_exports,
|
3753
|
-
runtimeFn: () =>
|
3430
|
+
runtimeFn: () => runtimeFn2,
|
3754
3431
|
toArrayBuffer: () => toArrayBuffer
|
3755
3432
|
});
|
3756
3433
|
|
@@ -3765,24 +3442,23 @@ __export(wait_pr_multiformats_exports, {
|
|
3765
3442
|
var codec_interface_exports = {};
|
3766
3443
|
|
3767
3444
|
// src/runtime/index.ts
|
3768
|
-
import { runtimeFn as
|
3445
|
+
import { runtimeFn as runtimeFn2 } from "@adviser/cement";
|
3769
3446
|
|
3770
3447
|
// src/version.ts
|
3771
3448
|
var PACKAGE_VERSION = Object.keys({
|
3772
|
-
"0.19.
|
3449
|
+
"0.19.103": "xxxx"
|
3773
3450
|
})[0];
|
3774
3451
|
export {
|
3775
3452
|
CRDT,
|
3776
|
-
|
3777
|
-
DatabaseShell,
|
3453
|
+
Database,
|
3778
3454
|
Index,
|
3779
3455
|
NotFoundError,
|
3780
3456
|
PACKAGE_VERSION,
|
3781
|
-
PARAM,
|
3782
3457
|
Result,
|
3783
3458
|
UInt8ArrayEqual,
|
3784
3459
|
blockstore_exports as blockstore,
|
3785
3460
|
blockstore_exports as bs,
|
3461
|
+
dataDir,
|
3786
3462
|
ensureLogger,
|
3787
3463
|
ensureSuperLog,
|
3788
3464
|
ensureSuperThis,
|
@@ -3793,13 +3469,10 @@ export {
|
|
3793
3469
|
getName,
|
3794
3470
|
getStore,
|
3795
3471
|
index,
|
3796
|
-
isDatabase,
|
3797
3472
|
isFalsy,
|
3798
3473
|
isNotFoundError,
|
3799
|
-
keyConfigOpts,
|
3800
3474
|
runtime_exports as rt,
|
3801
3475
|
runtime_exports as runtime,
|
3802
|
-
throwFalsy
|
3803
|
-
toStoreURIRuntime
|
3476
|
+
throwFalsy
|
3804
3477
|
};
|
3805
3478
|
//# sourceMappingURL=index.js.map
|