@fireproof/core 0.19.9-dev-frag → 0.19.11-dev-dryrun
Sign up to get free protection for your applications and to get access to all the features.
- package/README.md +27 -0
- package/{chunk-YS4GL6OK.js → chunk-MAK4D54P.js} +20 -6
- package/chunk-MAK4D54P.js.map +1 -0
- package/{chunk-JO5AVWG7.js → chunk-XINRLWR3.js} +14 -6
- package/chunk-XINRLWR3.js.map +1 -0
- package/deno.json +20 -0
- package/{gateway-IZRHJWPE.js → gateway-7OM6OSYK.js} +3 -4
- package/gateway-7OM6OSYK.js.map +1 -0
- package/{gateway-YSNUK2L3.js → gateway-VWWKLHUI.js} +4 -5
- package/{gateway-YSNUK2L3.js.map → gateway-VWWKLHUI.js.map} +1 -1
- package/index.cjs +549 -372
- package/index.cjs.map +1 -1
- package/index.d.cts +75 -51
- package/index.d.ts +75 -51
- package/index.global.js +21461 -15114
- package/index.global.js.map +1 -1
- package/index.js +465 -313
- package/index.js.map +1 -1
- package/{key-bag-file-NMEBFSPM.js → key-bag-file-DFMW6ZM6.js} +3 -3
- package/{key-bag-indexdb-X5V6GNBZ.js → key-bag-indexdb-R2RWGSQ4.js} +3 -3
- package/key-bag-indexdb-R2RWGSQ4.js.map +1 -0
- package/{mem-filesystem-B6C6QOIP.js → mem-filesystem-BZQZLUR6.js} +3 -3
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/{node-filesystem-5JLBSHKQ.js → node-filesystem-7YZR3POJ.js} +8 -4
- package/node-filesystem-7YZR3POJ.js.map +1 -0
- package/package.json +8 -5
- package/tests/blockstore/keyed-crypto.test.ts +34 -4
- package/tests/blockstore/store.test.ts +18 -13
- package/tests/fireproof/all-gateway.test.ts +395 -0
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.car +0 -0
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.js +316 -0
- package/tests/fireproof/convert_uint8.py +27 -0
- package/tests/fireproof/fireproof.test.ts +18 -18
- package/tests/fireproof/hello.test.ts +33 -19
- package/tests/helpers.ts +7 -4
- package/{utils-IZPK4QS7.js → utils-AISQB3PB.js} +3 -3
- package/chunk-JO5AVWG7.js.map +0 -1
- package/chunk-YS4GL6OK.js.map +0 -1
- package/gateway-IZRHJWPE.js.map +0 -1
- package/key-bag-indexdb-X5V6GNBZ.js.map +0 -1
- package/node-filesystem-5JLBSHKQ.js.map +0 -1
- /package/{key-bag-file-NMEBFSPM.js.map → key-bag-file-DFMW6ZM6.js.map} +0 -0
- /package/{mem-filesystem-B6C6QOIP.js.map → mem-filesystem-BZQZLUR6.js.map} +0 -0
- /package/{utils-IZPK4QS7.js.map → utils-AISQB3PB.js.map} +0 -0
package/index.js
CHANGED
@@ -6,7 +6,7 @@ import {
|
|
6
6
|
getFileSystem,
|
7
7
|
getPath,
|
8
8
|
toArrayBuffer
|
9
|
-
} from "./chunk-
|
9
|
+
} from "./chunk-XINRLWR3.js";
|
10
10
|
import {
|
11
11
|
INDEXDB_VERSION
|
12
12
|
} from "./chunk-PB4BKL4O.js";
|
@@ -19,16 +19,15 @@ import {
|
|
19
19
|
ensureLogger,
|
20
20
|
ensureSuperLog,
|
21
21
|
ensureSuperThis,
|
22
|
-
exception2Result,
|
23
22
|
exceptionWrapper,
|
24
23
|
getKey,
|
25
24
|
getName,
|
26
25
|
getStore,
|
27
26
|
isNotFoundError
|
28
|
-
} from "./chunk-
|
27
|
+
} from "./chunk-MAK4D54P.js";
|
29
28
|
|
30
29
|
// src/database.ts
|
31
|
-
import { ResolveOnce as
|
30
|
+
import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
|
32
31
|
|
33
32
|
// src/write-queue.ts
|
34
33
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -71,7 +70,7 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
71
70
|
}
|
72
71
|
|
73
72
|
// src/crdt.ts
|
74
|
-
import { ResolveOnce as
|
73
|
+
import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
|
75
74
|
|
76
75
|
// src/runtime/wait-pr-multiformats/block.ts
|
77
76
|
var block_exports = {};
|
@@ -145,7 +144,7 @@ async function createUnsafe({
|
|
145
144
|
}
|
146
145
|
|
147
146
|
// src/crdt-helpers.ts
|
148
|
-
import { parse as
|
147
|
+
import { parse as parse3 } from "multiformats/link";
|
149
148
|
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
150
149
|
import * as codec from "@ipld/dag-cbor";
|
151
150
|
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
@@ -162,10 +161,12 @@ __export(blockstore_exports, {
|
|
162
161
|
EncryptedBlockstore: () => EncryptedBlockstore,
|
163
162
|
FragmentGateway: () => FragmentGateway,
|
164
163
|
Loader: () => Loader,
|
164
|
+
addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
|
165
165
|
ensureStart: () => ensureStart,
|
166
166
|
getGatewayFromURL: () => getGatewayFromURL,
|
167
167
|
parseCarFile: () => parseCarFile,
|
168
168
|
registerStoreProtocol: () => registerStoreProtocol,
|
169
|
+
setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
|
169
170
|
testStoreFactory: () => testStoreFactory,
|
170
171
|
toCIDBlock: () => toCIDBlock,
|
171
172
|
toStoreRuntime: () => toStoreRuntime
|
@@ -177,7 +178,7 @@ function toCIDBlock(block) {
|
|
177
178
|
}
|
178
179
|
|
179
180
|
// src/blockstore/store-factory.ts
|
180
|
-
import { KeyedResolvOnce as KeyedResolvOnce2, URI as
|
181
|
+
import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
|
181
182
|
|
182
183
|
// src/runtime/files.ts
|
183
184
|
var files_exports = {};
|
@@ -253,7 +254,7 @@ var UnixFSFileBuilder = class {
|
|
253
254
|
// src/blockstore/store.ts
|
254
255
|
import pLimit2 from "p-limit";
|
255
256
|
import { format, parse } from "@ipld/dag-json";
|
256
|
-
import { ResolveOnce as
|
257
|
+
import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result4 } from "@adviser/cement";
|
257
258
|
|
258
259
|
// src/types.ts
|
259
260
|
function isFalsy(value) {
|
@@ -275,7 +276,7 @@ function falsyToUndef(value) {
|
|
275
276
|
// src/blockstore/loader.ts
|
276
277
|
import pLimit from "p-limit";
|
277
278
|
import { CarReader } from "@ipld/car";
|
278
|
-
import { ResolveOnce } from "@adviser/cement";
|
279
|
+
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
279
280
|
|
280
281
|
// src/blockstore/loader-helpers.ts
|
281
282
|
import { sha256 as hasher } from "multiformats/hashes/sha2";
|
@@ -547,6 +548,7 @@ __export(key_bag_exports, {
|
|
547
548
|
});
|
548
549
|
import {
|
549
550
|
KeyedResolvOnce,
|
551
|
+
ResolveOnce,
|
550
552
|
ResolveSeq,
|
551
553
|
Result as Result2,
|
552
554
|
runtimeFn,
|
@@ -557,21 +559,25 @@ import { base58btc } from "multiformats/bases/base58";
|
|
557
559
|
var KeyBag = class {
|
558
560
|
constructor(rt) {
|
559
561
|
this.rt = rt;
|
562
|
+
this._warnOnce = new ResolveOnce();
|
560
563
|
this._seq = new ResolveSeq();
|
561
|
-
this.logger = ensureLogger(rt.sthis, "KeyBag"
|
562
|
-
id: rt.id()
|
563
|
-
});
|
564
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag");
|
564
565
|
this.logger.Debug().Msg("KeyBag created");
|
565
566
|
}
|
566
567
|
async subtleKey(key) {
|
568
|
+
const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
|
569
|
+
if (extractable) {
|
570
|
+
this._warnOnce.once(
|
571
|
+
() => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
|
572
|
+
);
|
573
|
+
}
|
567
574
|
return await this.rt.crypto.importKey(
|
568
575
|
"raw",
|
569
576
|
// raw or jwk
|
570
577
|
base58btc.decode(key),
|
571
578
|
// hexStringToUint8Array(key), // raw data
|
572
579
|
"AES-GCM",
|
573
|
-
|
574
|
-
// extractable
|
580
|
+
extractable,
|
575
581
|
["encrypt", "decrypt"]
|
576
582
|
);
|
577
583
|
}
|
@@ -620,6 +626,23 @@ var KeyBag = class {
|
|
620
626
|
await bag.set(name, item);
|
621
627
|
return await this.toKeyWithFingerPrint(item.key);
|
622
628
|
}
|
629
|
+
async getNamedExtractableKey(name, failIfNotFound = false) {
|
630
|
+
const ret = await this.getNamedKey(name, failIfNotFound);
|
631
|
+
if (ret.isErr()) {
|
632
|
+
return ret;
|
633
|
+
}
|
634
|
+
const named = ret.Ok();
|
635
|
+
return Result2.Ok({
|
636
|
+
...named,
|
637
|
+
extract: async () => {
|
638
|
+
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
639
|
+
return {
|
640
|
+
key: ext,
|
641
|
+
keyStr: base58btc.encode(ext)
|
642
|
+
};
|
643
|
+
}
|
644
|
+
});
|
645
|
+
}
|
623
646
|
async getNamedKey(name, failIfNotFound = false) {
|
624
647
|
const id = this.rt.sthis.nextId(4).str;
|
625
648
|
return this._seq.add(async () => {
|
@@ -645,14 +668,14 @@ var keyBagProviderFactories = new Map(
|
|
645
668
|
{
|
646
669
|
protocol: "file:",
|
647
670
|
factory: async (url, sthis) => {
|
648
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-
|
671
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-DFMW6ZM6.js");
|
649
672
|
return new KeyBagProviderFile(url, sthis);
|
650
673
|
}
|
651
674
|
},
|
652
675
|
{
|
653
676
|
protocol: "indexdb:",
|
654
677
|
factory: async (url, sthis) => {
|
655
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-
|
678
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-R2RWGSQ4.js");
|
656
679
|
return new KeyBagProviderIndexDB(url, sthis);
|
657
680
|
}
|
658
681
|
}
|
@@ -689,23 +712,11 @@ function defaultKeyBagOpts(sthis, kbo) {
|
|
689
712
|
}
|
690
713
|
logger.Debug().Url(url).Msg("from env");
|
691
714
|
}
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
keyProviderFactory = async () => {
|
696
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-NMEBFSPM.js");
|
697
|
-
return new KeyBagProviderFile(url, sthis);
|
698
|
-
};
|
699
|
-
break;
|
700
|
-
case "indexdb:":
|
701
|
-
keyProviderFactory = async () => {
|
702
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-X5V6GNBZ.js");
|
703
|
-
return new KeyBagProviderIndexDB(url, sthis);
|
704
|
-
};
|
705
|
-
break;
|
706
|
-
default:
|
707
|
-
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
715
|
+
const kitem = keyBagProviderFactories.get(url.protocol);
|
716
|
+
if (!kitem) {
|
717
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
708
718
|
}
|
719
|
+
const getBag = async () => kitem.factory(url, sthis);
|
709
720
|
if (url.hasParam("masterkey")) {
|
710
721
|
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
711
722
|
}
|
@@ -715,7 +726,7 @@ function defaultKeyBagOpts(sthis, kbo) {
|
|
715
726
|
sthis,
|
716
727
|
logger,
|
717
728
|
keyLength: kbo.keyLength || 16,
|
718
|
-
getBag
|
729
|
+
getBag,
|
719
730
|
id: () => {
|
720
731
|
return url.toString();
|
721
732
|
}
|
@@ -832,6 +843,53 @@ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
|
832
843
|
|
833
844
|
// src/blockstore/loader.ts
|
834
845
|
import { sha256 as hasher3 } from "multiformats/hashes/sha2";
|
846
|
+
|
847
|
+
// src/blockstore/task-manager.ts
|
848
|
+
var TaskManager = class {
|
849
|
+
constructor(sthis, callback) {
|
850
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
851
|
+
this.queue = [];
|
852
|
+
this.isProcessing = false;
|
853
|
+
this.logger = ensureLogger(sthis, "TaskManager");
|
854
|
+
this.callback = callback;
|
855
|
+
}
|
856
|
+
async handleEvent(cid, parents, dbMeta) {
|
857
|
+
for (const parent of parents) {
|
858
|
+
this.eventsWeHandled.add(parent.toString());
|
859
|
+
}
|
860
|
+
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
861
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
862
|
+
void this.processQueue();
|
863
|
+
}
|
864
|
+
async processQueue() {
|
865
|
+
if (this.isProcessing) return;
|
866
|
+
this.isProcessing = true;
|
867
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
868
|
+
const first = filteredQueue[0];
|
869
|
+
if (!first) {
|
870
|
+
return;
|
871
|
+
}
|
872
|
+
try {
|
873
|
+
await this.callback(first.dbMeta);
|
874
|
+
this.eventsWeHandled.add(first.cid);
|
875
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
876
|
+
} catch (err) {
|
877
|
+
if (first.retries++ > 3) {
|
878
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
879
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
880
|
+
}
|
881
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
882
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
883
|
+
} finally {
|
884
|
+
this.isProcessing = false;
|
885
|
+
if (this.queue.length > 0) {
|
886
|
+
void this.processQueue();
|
887
|
+
}
|
888
|
+
}
|
889
|
+
}
|
890
|
+
};
|
891
|
+
|
892
|
+
// src/blockstore/loader.ts
|
835
893
|
function carLogIncludesGroup(list, cids) {
|
836
894
|
return list.some((arr) => {
|
837
895
|
return arr.toString() === cids.toString();
|
@@ -856,7 +914,7 @@ var Loader = class {
|
|
856
914
|
this.getBlockCache = /* @__PURE__ */ new Map();
|
857
915
|
this.seenMeta = /* @__PURE__ */ new Set();
|
858
916
|
this.writeLimit = pLimit(1);
|
859
|
-
this.onceReady = new
|
917
|
+
this.onceReady = new ResolveOnce2();
|
860
918
|
this.name = name;
|
861
919
|
this.sthis = sthis;
|
862
920
|
this.ebOpts = defaultedBlockstoreRuntime(
|
@@ -868,6 +926,9 @@ var Loader = class {
|
|
868
926
|
"Loader"
|
869
927
|
);
|
870
928
|
this.logger = this.ebOpts.logger;
|
929
|
+
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
930
|
+
await this.handleDbMetasFromStore([dbMeta]);
|
931
|
+
});
|
871
932
|
}
|
872
933
|
// readonly id = uuidv4();
|
873
934
|
async keyBag() {
|
@@ -887,8 +948,10 @@ var Loader = class {
|
|
887
948
|
}
|
888
949
|
async ready() {
|
889
950
|
return this.onceReady.once(async () => {
|
890
|
-
const metas =
|
891
|
-
if (
|
951
|
+
const metas = await (await this.metaStore()).load();
|
952
|
+
if (this.ebOpts.meta) {
|
953
|
+
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
954
|
+
} else if (metas) {
|
892
955
|
await this.handleDbMetasFromStore(metas);
|
893
956
|
}
|
894
957
|
});
|
@@ -1193,7 +1256,7 @@ var generateIV = {
|
|
1193
1256
|
calc: async (ko, crypto, data) => {
|
1194
1257
|
const hash = await hasher4.digest(data);
|
1195
1258
|
const hashBytes = new Uint8Array(hash.bytes);
|
1196
|
-
const hashArray = new Uint8Array(ko.ivLength
|
1259
|
+
const hashArray = new Uint8Array(ko.ivLength);
|
1197
1260
|
for (let i = 0; i < hashBytes.length; i++) {
|
1198
1261
|
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
1199
1262
|
}
|
@@ -1271,7 +1334,7 @@ var keyedCrypto = class {
|
|
1271
1334
|
};
|
1272
1335
|
}
|
1273
1336
|
async _decrypt(data) {
|
1274
|
-
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("decrypting");
|
1337
|
+
this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
|
1275
1338
|
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
1276
1339
|
}
|
1277
1340
|
async _encrypt(data) {
|
@@ -1342,6 +1405,7 @@ async function keyedCryptoFactory(url, kb, sthis) {
|
|
1342
1405
|
}
|
1343
1406
|
|
1344
1407
|
// src/blockstore/fragment-gateway.ts
|
1408
|
+
import { Result as Result3 } from "@adviser/cement";
|
1345
1409
|
import { base58btc as base58btc3 } from "multiformats/bases/base58";
|
1346
1410
|
import { encode as encode3, decode as decode3 } from "cborg";
|
1347
1411
|
function getFragSize(url) {
|
@@ -1364,7 +1428,7 @@ async function getFrags(url, innerGW, headerSize, logger) {
|
|
1364
1428
|
}
|
1365
1429
|
const data = res.unwrap();
|
1366
1430
|
return [
|
1367
|
-
|
1431
|
+
Result3.Ok({
|
1368
1432
|
fid: new Uint8Array(0),
|
1369
1433
|
ofs: 0,
|
1370
1434
|
len: data.length,
|
@@ -1378,7 +1442,7 @@ async function getFrags(url, innerGW, headerSize, logger) {
|
|
1378
1442
|
}
|
1379
1443
|
const firstFragment = decode3(firstRaw.unwrap());
|
1380
1444
|
const blockSize = firstFragment.data.length;
|
1381
|
-
const ops = [Promise.resolve(
|
1445
|
+
const ops = [Promise.resolve(Result3.Ok(firstFragment))];
|
1382
1446
|
const fidStr = base58btc3.encode(firstFragment.fid);
|
1383
1447
|
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
|
1384
1448
|
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
@@ -1390,12 +1454,12 @@ async function getFrags(url, innerGW, headerSize, logger) {
|
|
1390
1454
|
}
|
1391
1455
|
const fragment = decode3(raw2.unwrap());
|
1392
1456
|
if (base58btc3.encode(fragment.fid) !== fidStr) {
|
1393
|
-
return
|
1457
|
+
return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
1394
1458
|
}
|
1395
1459
|
if (fragment.ofs !== ofs2) {
|
1396
|
-
return
|
1460
|
+
return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
1397
1461
|
}
|
1398
|
-
return
|
1462
|
+
return Result3.Ok(fragment);
|
1399
1463
|
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
1400
1464
|
);
|
1401
1465
|
}
|
@@ -1457,42 +1521,68 @@ var FragmentGateway = class {
|
|
1457
1521
|
}
|
1458
1522
|
async put(url, body) {
|
1459
1523
|
await Promise.all(this.slicer(url, body));
|
1460
|
-
return
|
1524
|
+
return Result3.Ok(void 0);
|
1461
1525
|
}
|
1462
1526
|
async get(url) {
|
1463
1527
|
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1464
1528
|
let buffer = void 0;
|
1465
1529
|
for (const rfrag of rfrags) {
|
1466
1530
|
if (rfrag.isErr()) {
|
1467
|
-
return
|
1531
|
+
return Result3.Err(rfrag.Err());
|
1468
1532
|
}
|
1469
1533
|
const frag = rfrag.Ok();
|
1470
1534
|
buffer = buffer || new Uint8Array(frag.len);
|
1471
1535
|
buffer.set(frag.data, frag.ofs);
|
1472
1536
|
}
|
1473
|
-
return
|
1537
|
+
return Result3.Ok(buffer || new Uint8Array(0));
|
1538
|
+
}
|
1539
|
+
async subscribe(url, callback) {
|
1540
|
+
if (this.innerGW.subscribe) {
|
1541
|
+
return this.innerGW.subscribe(url, callback);
|
1542
|
+
} else {
|
1543
|
+
let lastData = void 0;
|
1544
|
+
let interval = 100;
|
1545
|
+
const fetchData = async () => {
|
1546
|
+
const result = await this.innerGW.get(url);
|
1547
|
+
if (result.isOk()) {
|
1548
|
+
const data = result.Ok();
|
1549
|
+
if (!lastData || !data.every((value, index2) => lastData && value === lastData[index2])) {
|
1550
|
+
lastData = data;
|
1551
|
+
callback(data);
|
1552
|
+
interval = 100;
|
1553
|
+
} else {
|
1554
|
+
interval *= 2;
|
1555
|
+
}
|
1556
|
+
}
|
1557
|
+
timeoutId = setTimeout(fetchData, interval);
|
1558
|
+
};
|
1559
|
+
let timeoutId = setTimeout(fetchData, interval);
|
1560
|
+
return Result3.Ok(() => {
|
1561
|
+
clearTimeout(timeoutId);
|
1562
|
+
});
|
1563
|
+
}
|
1474
1564
|
}
|
1475
1565
|
async delete(url) {
|
1476
1566
|
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1477
1567
|
for (const rfrag of rfrags) {
|
1478
1568
|
if (rfrag.isErr()) {
|
1479
|
-
return
|
1569
|
+
return Result3.Err(rfrag.Err());
|
1480
1570
|
}
|
1481
1571
|
const frag = rfrag.Ok();
|
1482
1572
|
const fidStr = base58btc3.encode(frag.fid);
|
1483
1573
|
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
|
1484
1574
|
await this.innerGW.delete(fragUrl);
|
1485
1575
|
}
|
1486
|
-
return
|
1576
|
+
return Result3.Ok(void 0);
|
1487
1577
|
}
|
1488
1578
|
};
|
1489
1579
|
|
1490
1580
|
// src/blockstore/store.ts
|
1491
1581
|
function guardVersion(url) {
|
1492
1582
|
if (!url.hasParam("version")) {
|
1493
|
-
return
|
1583
|
+
return Result4.Err(`missing version: ${url.toString()}`);
|
1494
1584
|
}
|
1495
|
-
return
|
1585
|
+
return Result4.Ok(url);
|
1496
1586
|
}
|
1497
1587
|
var BaseStoreImpl = class {
|
1498
1588
|
constructor(name, url, opts, sthis, logger) {
|
@@ -1504,6 +1594,7 @@ var BaseStoreImpl = class {
|
|
1504
1594
|
this.sthis = sthis;
|
1505
1595
|
this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
|
1506
1596
|
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
1597
|
+
this.loader = opts.loader;
|
1507
1598
|
}
|
1508
1599
|
url() {
|
1509
1600
|
return this._url;
|
@@ -1559,102 +1650,6 @@ var BaseStoreImpl = class {
|
|
1559
1650
|
return version;
|
1560
1651
|
}
|
1561
1652
|
};
|
1562
|
-
var MetaStoreImpl = class extends BaseStoreImpl {
|
1563
|
-
constructor(sthis, name, url, opts) {
|
1564
|
-
super(
|
1565
|
-
name,
|
1566
|
-
url,
|
1567
|
-
{
|
1568
|
-
...opts
|
1569
|
-
},
|
1570
|
-
sthis,
|
1571
|
-
ensureLogger(sthis, "MetaStoreImpl")
|
1572
|
-
);
|
1573
|
-
this.storeType = "meta";
|
1574
|
-
this.subscribers = /* @__PURE__ */ new Map();
|
1575
|
-
}
|
1576
|
-
onLoad(branch, loadHandler) {
|
1577
|
-
const subscribers = this.subscribers.get(branch) || [];
|
1578
|
-
subscribers.push(loadHandler);
|
1579
|
-
this.subscribers.set(branch, subscribers);
|
1580
|
-
return () => {
|
1581
|
-
const subscribers2 = this.subscribers.get(branch) || [];
|
1582
|
-
const idx = subscribers2.indexOf(loadHandler);
|
1583
|
-
if (idx > -1) subscribers2.splice(idx, 1);
|
1584
|
-
};
|
1585
|
-
}
|
1586
|
-
makeHeader({ cars }) {
|
1587
|
-
const toEncode = { cars };
|
1588
|
-
return format(toEncode);
|
1589
|
-
}
|
1590
|
-
parseHeader(headerData) {
|
1591
|
-
const got = parse(headerData);
|
1592
|
-
return got;
|
1593
|
-
}
|
1594
|
-
async handleSubscribers(dbMetas, branch) {
|
1595
|
-
try {
|
1596
|
-
const subscribers = this.subscribers.get(branch) || [];
|
1597
|
-
await Promise.all(subscribers.map((subscriber) => subscriber(dbMetas)));
|
1598
|
-
} catch (e) {
|
1599
|
-
this.logger.Error().Err(e).Msg("handleSubscribers").AsError();
|
1600
|
-
}
|
1601
|
-
}
|
1602
|
-
async handleByteHeads(byteHeads, branch = "main") {
|
1603
|
-
let dbMetas;
|
1604
|
-
try {
|
1605
|
-
dbMetas = this.dbMetasForByteHeads(byteHeads);
|
1606
|
-
} catch (e) {
|
1607
|
-
throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
|
1608
|
-
}
|
1609
|
-
await this.handleSubscribers(dbMetas, branch);
|
1610
|
-
return dbMetas;
|
1611
|
-
}
|
1612
|
-
dbMetasForByteHeads(byteHeads) {
|
1613
|
-
return byteHeads.map((bytes) => {
|
1614
|
-
const txt = this.sthis.txt.decode(bytes);
|
1615
|
-
return this.parseHeader(txt);
|
1616
|
-
});
|
1617
|
-
}
|
1618
|
-
async load(branch) {
|
1619
|
-
branch = branch || "main";
|
1620
|
-
this.logger.Debug().Str("branch", branch).Msg("loading");
|
1621
|
-
const url = await this.gateway.buildUrl(this.url(), branch);
|
1622
|
-
if (url.isErr()) {
|
1623
|
-
throw this.logger.Error().Result("buidUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
1624
|
-
}
|
1625
|
-
const bytes = await this.gateway.get(url.Ok());
|
1626
|
-
if (bytes.isErr()) {
|
1627
|
-
if (isNotFoundError(bytes)) {
|
1628
|
-
return void 0;
|
1629
|
-
}
|
1630
|
-
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
1631
|
-
}
|
1632
|
-
return this.handleByteHeads([bytes.Ok()], branch);
|
1633
|
-
}
|
1634
|
-
async save(meta, branch) {
|
1635
|
-
branch = branch || "main";
|
1636
|
-
this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
|
1637
|
-
const bytes = this.makeHeader(meta);
|
1638
|
-
const url = await this.gateway.buildUrl(this.url(), branch);
|
1639
|
-
if (url.isErr()) {
|
1640
|
-
throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
1641
|
-
}
|
1642
|
-
const res = await this.gateway.put(url.Ok(), this.sthis.txt.encode(bytes));
|
1643
|
-
if (res.isErr()) {
|
1644
|
-
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
1645
|
-
}
|
1646
|
-
await this.handleSubscribers([meta], branch);
|
1647
|
-
return res;
|
1648
|
-
}
|
1649
|
-
async close() {
|
1650
|
-
await this.gateway.close(this.url());
|
1651
|
-
this._onClosed.forEach((fn) => fn());
|
1652
|
-
return Result3.Ok(void 0);
|
1653
|
-
}
|
1654
|
-
async destroy() {
|
1655
|
-
return this.gateway.destroy(this.url());
|
1656
|
-
}
|
1657
|
-
};
|
1658
1653
|
var DataStoreImpl = class extends BaseStoreImpl {
|
1659
1654
|
// readonly tag: string = "car-base";
|
1660
1655
|
constructor(sthis, name, url, opts) {
|
@@ -1704,7 +1699,7 @@ var DataStoreImpl = class extends BaseStoreImpl {
|
|
1704
1699
|
async close() {
|
1705
1700
|
await this.gateway.close(this.url());
|
1706
1701
|
this._onClosed.forEach((fn) => fn());
|
1707
|
-
return
|
1702
|
+
return Result4.Ok(void 0);
|
1708
1703
|
}
|
1709
1704
|
destroy() {
|
1710
1705
|
return this.gateway.destroy(this.url());
|
@@ -1722,7 +1717,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1722
1717
|
ensureLogger(loader.sthis, "WALStoreImpl")
|
1723
1718
|
);
|
1724
1719
|
this.storeType = "wal";
|
1725
|
-
this._ready = new
|
1720
|
+
this._ready = new ResolveOnce3();
|
1726
1721
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
1727
1722
|
this.processing = void 0;
|
1728
1723
|
this.processQueue = new CommitQueue();
|
@@ -1878,13 +1873,261 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1878
1873
|
async close() {
|
1879
1874
|
await this.gateway.close(this.url());
|
1880
1875
|
this._onClosed.forEach((fn) => fn());
|
1881
|
-
return
|
1876
|
+
return Result4.Ok(void 0);
|
1882
1877
|
}
|
1883
1878
|
destroy() {
|
1884
1879
|
return this.gateway.destroy(this.url());
|
1885
1880
|
}
|
1886
1881
|
};
|
1887
1882
|
|
1883
|
+
// src/blockstore/store-meta.ts
|
1884
|
+
import { format as format2, parse as parse2 } from "@ipld/dag-json";
|
1885
|
+
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
1886
|
+
import { CID as CID2 } from "multiformats";
|
1887
|
+
import { Result as Result5 } from "@adviser/cement";
|
1888
|
+
|
1889
|
+
// src/runtime/index.ts
|
1890
|
+
var runtime_exports = {};
|
1891
|
+
__export(runtime_exports, {
|
1892
|
+
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
1893
|
+
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
1894
|
+
files: () => files_exports,
|
1895
|
+
getFileName: () => getFileName,
|
1896
|
+
getFileSystem: () => getFileSystem,
|
1897
|
+
getPath: () => getPath,
|
1898
|
+
kb: () => key_bag_exports,
|
1899
|
+
kc: () => keyed_crypto_exports,
|
1900
|
+
mf: () => wait_pr_multiformats_exports,
|
1901
|
+
runtimeFn: () => runtimeFn2,
|
1902
|
+
toArrayBuffer: () => toArrayBuffer
|
1903
|
+
});
|
1904
|
+
|
1905
|
+
// src/runtime/wait-pr-multiformats/index.ts
|
1906
|
+
var wait_pr_multiformats_exports = {};
|
1907
|
+
__export(wait_pr_multiformats_exports, {
|
1908
|
+
block: () => block_exports,
|
1909
|
+
codec: () => codec_interface_exports
|
1910
|
+
});
|
1911
|
+
|
1912
|
+
// src/runtime/wait-pr-multiformats/codec-interface.ts
|
1913
|
+
var codec_interface_exports = {};
|
1914
|
+
|
1915
|
+
// src/runtime/index.ts
|
1916
|
+
import { runtimeFn as runtimeFn2 } from "@adviser/cement";
|
1917
|
+
|
1918
|
+
// src/blockstore/store-meta.ts
|
1919
|
+
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
1920
|
+
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
1921
|
+
return Promise.all(
|
1922
|
+
crdtEntries.map(async (crdtEntry) => {
|
1923
|
+
const eventBlock = await decodeEventBlock(decodeFromBase64(crdtEntry.data));
|
1924
|
+
const dbMeta = parse2(sthis.txt.decode(eventBlock.value.data.dbMeta));
|
1925
|
+
return {
|
1926
|
+
eventCid: eventBlock.cid,
|
1927
|
+
parents: crdtEntry.parents,
|
1928
|
+
dbMeta
|
1929
|
+
};
|
1930
|
+
})
|
1931
|
+
);
|
1932
|
+
}
|
1933
|
+
async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
|
1934
|
+
try {
|
1935
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
|
1936
|
+
const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
|
1937
|
+
if (keyInfo.length) {
|
1938
|
+
const dbMeta = keyInfo[0].dbMeta;
|
1939
|
+
if (dbMeta.key) {
|
1940
|
+
const kb = await key_bag_exports.getKeyBag(sthis);
|
1941
|
+
const keyName = getStoreKeyName(uri);
|
1942
|
+
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
1943
|
+
if (res.isErr()) {
|
1944
|
+
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
1945
|
+
throw res.Err();
|
1946
|
+
}
|
1947
|
+
}
|
1948
|
+
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
1949
|
+
return Result5.Ok(dbMeta);
|
1950
|
+
}
|
1951
|
+
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
1952
|
+
return Result5.Ok(void 0);
|
1953
|
+
} catch (error) {
|
1954
|
+
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
1955
|
+
return Result5.Err(error);
|
1956
|
+
}
|
1957
|
+
}
|
1958
|
+
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
1959
|
+
try {
|
1960
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
1961
|
+
const keyName = getStoreKeyName(uri);
|
1962
|
+
const kb = await key_bag_exports.getKeyBag(sthis);
|
1963
|
+
const res = await kb.getNamedExtractableKey(keyName, true);
|
1964
|
+
if (res.isErr()) {
|
1965
|
+
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
1966
|
+
throw res.Err();
|
1967
|
+
}
|
1968
|
+
const keyData = await res.Ok().extract();
|
1969
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
1970
|
+
const { dbMeta, parents } = dbMetas[0];
|
1971
|
+
const parentLinks = parents.map((p) => CID2.parse(p));
|
1972
|
+
dbMeta.key = keyData.keyStr;
|
1973
|
+
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
1974
|
+
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
1975
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
1976
|
+
return Result5.Ok(encoded);
|
1977
|
+
} catch (error) {
|
1978
|
+
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
1979
|
+
return Result5.Err(error);
|
1980
|
+
}
|
1981
|
+
}
|
1982
|
+
function getStoreKeyName(url) {
|
1983
|
+
const storeKeyName = [url.getParam("localName") || url.getParam("name")];
|
1984
|
+
const idx = url.getParam("index");
|
1985
|
+
if (idx) {
|
1986
|
+
storeKeyName.push(idx);
|
1987
|
+
}
|
1988
|
+
storeKeyName.push("data");
|
1989
|
+
return `@${storeKeyName.join(":")}@`;
|
1990
|
+
}
|
1991
|
+
async function createDbMetaEventBlock(sthis, dbMeta, parents) {
|
1992
|
+
const event = await EventBlock.create(
|
1993
|
+
{
|
1994
|
+
dbMeta: sthis.txt.encode(format2(dbMeta))
|
1995
|
+
},
|
1996
|
+
parents
|
1997
|
+
);
|
1998
|
+
return event;
|
1999
|
+
}
|
2000
|
+
async function encodeEventsWithParents(sthis, events, parents) {
|
2001
|
+
const crdtEntries = events.map((event) => {
|
2002
|
+
const base64String = encodeToBase64(event.bytes);
|
2003
|
+
return {
|
2004
|
+
cid: event.cid.toString(),
|
2005
|
+
data: base64String,
|
2006
|
+
parents: parents.map((p) => p.toString())
|
2007
|
+
};
|
2008
|
+
});
|
2009
|
+
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
2010
|
+
}
|
2011
|
+
var MetaStoreImpl = class extends BaseStoreImpl {
|
2012
|
+
constructor(sthis, name, url, opts, remote) {
|
2013
|
+
super(
|
2014
|
+
name,
|
2015
|
+
url,
|
2016
|
+
{
|
2017
|
+
...opts
|
2018
|
+
},
|
2019
|
+
sthis,
|
2020
|
+
ensureLogger(sthis, "MetaStoreImpl")
|
2021
|
+
);
|
2022
|
+
this.storeType = "meta";
|
2023
|
+
this.subscribers = /* @__PURE__ */ new Map();
|
2024
|
+
this.parents = [];
|
2025
|
+
if (remote && opts.gateway.subscribe) {
|
2026
|
+
this.onStarted(async () => {
|
2027
|
+
this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
|
2028
|
+
opts.gateway.subscribe?.(this.url(), async (message) => {
|
2029
|
+
this.logger.Debug().Msg("Received message from gateway");
|
2030
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
|
2031
|
+
await Promise.all(
|
2032
|
+
dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
|
2033
|
+
);
|
2034
|
+
});
|
2035
|
+
});
|
2036
|
+
}
|
2037
|
+
}
|
2038
|
+
async handleByteHeads(byteHeads) {
|
2039
|
+
return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
|
2040
|
+
}
|
2041
|
+
async load() {
|
2042
|
+
const branch = "main";
|
2043
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2044
|
+
if (url.isErr()) {
|
2045
|
+
throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2046
|
+
}
|
2047
|
+
const bytes = await this.gateway.get(url.Ok());
|
2048
|
+
if (bytes.isErr()) {
|
2049
|
+
if (isNotFoundError(bytes)) {
|
2050
|
+
return void 0;
|
2051
|
+
}
|
2052
|
+
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
2053
|
+
}
|
2054
|
+
const dbMetas = await this.handleByteHeads(bytes.Ok());
|
2055
|
+
await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
|
2056
|
+
const cids = dbMetas.map((m) => m.eventCid);
|
2057
|
+
const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
|
2058
|
+
this.parents = Array.from(uniqueParentsMap.values());
|
2059
|
+
return dbMetas.map((m) => m.dbMeta);
|
2060
|
+
}
|
2061
|
+
async save(meta, branch) {
|
2062
|
+
branch = branch || "main";
|
2063
|
+
this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
|
2064
|
+
const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
|
2065
|
+
const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
|
2066
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2067
|
+
if (url.isErr()) {
|
2068
|
+
throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2069
|
+
}
|
2070
|
+
const res = await this.gateway.put(url.Ok(), bytes);
|
2071
|
+
if (res.isErr()) {
|
2072
|
+
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
2073
|
+
}
|
2074
|
+
await this.loader?.handleDbMetasFromStore([meta]);
|
2075
|
+
this.parents = [event.cid];
|
2076
|
+
return res;
|
2077
|
+
}
|
2078
|
+
async close() {
|
2079
|
+
await this.gateway.close(this.url());
|
2080
|
+
this._onClosed.forEach((fn) => fn());
|
2081
|
+
return Result5.Ok(void 0);
|
2082
|
+
}
|
2083
|
+
async destroy() {
|
2084
|
+
return this.gateway.destroy(this.url());
|
2085
|
+
}
|
2086
|
+
};
|
2087
|
+
function encodeToBase64(bytes) {
|
2088
|
+
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
2089
|
+
let base64 = "";
|
2090
|
+
let i;
|
2091
|
+
for (i = 0; i < bytes.length - 2; i += 3) {
|
2092
|
+
base64 += chars[bytes[i] >> 2];
|
2093
|
+
base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
|
2094
|
+
base64 += chars[(bytes[i + 1] & 15) << 2 | bytes[i + 2] >> 6];
|
2095
|
+
base64 += chars[bytes[i + 2] & 63];
|
2096
|
+
}
|
2097
|
+
if (i < bytes.length) {
|
2098
|
+
base64 += chars[bytes[i] >> 2];
|
2099
|
+
if (i === bytes.length - 1) {
|
2100
|
+
base64 += chars[(bytes[i] & 3) << 4];
|
2101
|
+
base64 += "==";
|
2102
|
+
} else {
|
2103
|
+
base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
|
2104
|
+
base64 += chars[(bytes[i + 1] & 15) << 2];
|
2105
|
+
base64 += "=";
|
2106
|
+
}
|
2107
|
+
}
|
2108
|
+
return base64;
|
2109
|
+
}
|
2110
|
+
function decodeFromBase64(base64) {
|
2111
|
+
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
2112
|
+
const bytes = new Uint8Array(base64.length * 3 / 4);
|
2113
|
+
let i;
|
2114
|
+
let j = 0;
|
2115
|
+
for (i = 0; i < base64.length; i += 4) {
|
2116
|
+
const a = chars.indexOf(base64[i]);
|
2117
|
+
const b = chars.indexOf(base64[i + 1]);
|
2118
|
+
const c = chars.indexOf(base64[i + 2]);
|
2119
|
+
const d = chars.indexOf(base64[i + 3]);
|
2120
|
+
bytes[j++] = a << 2 | b >> 4;
|
2121
|
+
if (base64[i + 2] !== "=") {
|
2122
|
+
bytes[j++] = (b & 15) << 4 | c >> 2;
|
2123
|
+
}
|
2124
|
+
if (base64[i + 3] !== "=") {
|
2125
|
+
bytes[j++] = (c & 3) << 6 | d;
|
2126
|
+
}
|
2127
|
+
}
|
2128
|
+
return bytes.slice(0, j);
|
2129
|
+
}
|
2130
|
+
|
1888
2131
|
// src/blockstore/store-factory.ts
|
1889
2132
|
function ensureIsIndex(url, isIndex) {
|
1890
2133
|
if (isIndex) {
|
@@ -1904,10 +2147,10 @@ function buildURL(optURL, loader) {
|
|
1904
2147
|
const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
|
1905
2148
|
let obuUrl;
|
1906
2149
|
if (obuItem && obuItem.overrideBaseURL) {
|
1907
|
-
obuUrl =
|
2150
|
+
obuUrl = URI5.from(obuItem.overrideBaseURL);
|
1908
2151
|
}
|
1909
2152
|
const ret = ensureIsIndex(
|
1910
|
-
|
2153
|
+
URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
1911
2154
|
storeOpts.isIndex
|
1912
2155
|
);
|
1913
2156
|
return ret;
|
@@ -1954,52 +2197,61 @@ function registerStoreProtocol(item) {
|
|
1954
2197
|
storeFactory.delete(protocol);
|
1955
2198
|
};
|
1956
2199
|
}
|
2200
|
+
var onceDataStoreFactory = new KeyedResolvOnce2();
|
1957
2201
|
async function dataStoreFactory(loader) {
|
1958
2202
|
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
|
1959
2203
|
const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
|
1960
|
-
|
1961
|
-
|
1962
|
-
|
1963
|
-
|
1964
|
-
|
1965
|
-
|
1966
|
-
|
1967
|
-
|
1968
|
-
|
2204
|
+
return onceDataStoreFactory.get(url.toString()).once(async () => {
|
2205
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2206
|
+
if (!gateway) {
|
2207
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2208
|
+
}
|
2209
|
+
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2210
|
+
gateway: gateway.gateway,
|
2211
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2212
|
+
...loader.ebOpts.keyBag
|
2213
|
+
})
|
2214
|
+
});
|
2215
|
+
return store;
|
1969
2216
|
});
|
1970
|
-
return store;
|
1971
2217
|
}
|
2218
|
+
var onceMetaStoreFactory = new KeyedResolvOnce2();
|
1972
2219
|
async function metaStoreFactory(loader) {
|
1973
2220
|
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
|
1974
2221
|
const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
|
1975
|
-
|
1976
|
-
|
1977
|
-
|
1978
|
-
|
1979
|
-
|
1980
|
-
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
2222
|
+
return onceMetaStoreFactory.get(url.toString()).once(async () => {
|
2223
|
+
sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
|
2224
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2225
|
+
if (!gateway) {
|
2226
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2227
|
+
}
|
2228
|
+
const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
|
2229
|
+
gateway: gateway.gateway,
|
2230
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2231
|
+
...loader.ebOpts.keyBag
|
2232
|
+
})
|
2233
|
+
});
|
2234
|
+
return store;
|
1985
2235
|
});
|
1986
|
-
return store;
|
1987
2236
|
}
|
2237
|
+
var onceRemoteWalFactory = new KeyedResolvOnce2();
|
1988
2238
|
async function remoteWalFactory(loader) {
|
1989
2239
|
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
|
1990
2240
|
const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
|
1991
|
-
|
1992
|
-
|
1993
|
-
|
1994
|
-
|
1995
|
-
|
1996
|
-
|
1997
|
-
|
1998
|
-
|
1999
|
-
|
2000
|
-
|
2241
|
+
return onceRemoteWalFactory.get(url.toString()).once(async () => {
|
2242
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2243
|
+
if (!gateway) {
|
2244
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2245
|
+
}
|
2246
|
+
sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
|
2247
|
+
const store = new WALStoreImpl(loader, url, {
|
2248
|
+
gateway: gateway.gateway,
|
2249
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2250
|
+
...loader.ebOpts.keyBag
|
2251
|
+
})
|
2252
|
+
});
|
2253
|
+
return store;
|
2001
2254
|
});
|
2002
|
-
return store;
|
2003
2255
|
}
|
2004
2256
|
async function testStoreFactory(url, sthis) {
|
2005
2257
|
sthis = ensureSuperLog(sthis, "testStoreFactory");
|
@@ -2038,78 +2290,27 @@ function toStoreRuntime(opts, sthis) {
|
|
2038
2290
|
}
|
2039
2291
|
registerStoreProtocol({
|
2040
2292
|
protocol: "file:",
|
2041
|
-
gateway: async (
|
2042
|
-
const { FileGateway } = await import("./gateway-
|
2043
|
-
return new FileGateway(
|
2293
|
+
gateway: async (sthis) => {
|
2294
|
+
const { FileGateway } = await import("./gateway-VWWKLHUI.js");
|
2295
|
+
return new FileGateway(sthis);
|
2044
2296
|
},
|
2045
|
-
test: async (
|
2046
|
-
const { FileTestStore } = await import("./gateway-
|
2047
|
-
return new FileTestStore(
|
2297
|
+
test: async (sthis) => {
|
2298
|
+
const { FileTestStore } = await import("./gateway-VWWKLHUI.js");
|
2299
|
+
return new FileTestStore(sthis);
|
2048
2300
|
}
|
2049
2301
|
});
|
2050
2302
|
registerStoreProtocol({
|
2051
2303
|
protocol: "indexdb:",
|
2052
|
-
gateway: async (
|
2053
|
-
const { IndexDBGateway } = await import("./gateway-
|
2054
|
-
return new IndexDBGateway(
|
2304
|
+
gateway: async (sthis) => {
|
2305
|
+
const { IndexDBGateway } = await import("./gateway-7OM6OSYK.js");
|
2306
|
+
return new IndexDBGateway(sthis);
|
2055
2307
|
},
|
2056
|
-
test: async (
|
2057
|
-
const { IndexDBTestStore } = await import("./gateway-
|
2058
|
-
return new IndexDBTestStore(
|
2308
|
+
test: async (sthis) => {
|
2309
|
+
const { IndexDBTestStore } = await import("./gateway-7OM6OSYK.js");
|
2310
|
+
return new IndexDBTestStore(sthis);
|
2059
2311
|
}
|
2060
2312
|
});
|
2061
2313
|
|
2062
|
-
// src/blockstore/connection-base.ts
|
2063
|
-
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
2064
|
-
import { MemoryBlockstore as MemoryBlockstore2 } from "@web3-storage/pail/block";
|
2065
|
-
|
2066
|
-
// src/blockstore/task-manager.ts
|
2067
|
-
var TaskManager = class {
|
2068
|
-
constructor(loader) {
|
2069
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
2070
|
-
this.queue = [];
|
2071
|
-
this.isProcessing = false;
|
2072
|
-
this.loader = loader;
|
2073
|
-
this.logger = ensureLogger(loader.sthis, "TaskManager");
|
2074
|
-
}
|
2075
|
-
async handleEvent(eventBlock) {
|
2076
|
-
const cid = eventBlock.cid.toString();
|
2077
|
-
const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
|
2078
|
-
for (const parent of parents) {
|
2079
|
-
this.eventsWeHandled.add(parent);
|
2080
|
-
}
|
2081
|
-
this.queue.push({ cid, eventBlock, retries: 0 });
|
2082
|
-
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
2083
|
-
void this.processQueue();
|
2084
|
-
}
|
2085
|
-
async processQueue() {
|
2086
|
-
if (this.isProcessing) return;
|
2087
|
-
this.isProcessing = true;
|
2088
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2089
|
-
const first = filteredQueue[0];
|
2090
|
-
if (!first) {
|
2091
|
-
return;
|
2092
|
-
}
|
2093
|
-
try {
|
2094
|
-
this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
|
2095
|
-
this.eventsWeHandled.add(first.cid);
|
2096
|
-
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2097
|
-
} catch (err) {
|
2098
|
-
if (first.retries++ > 3) {
|
2099
|
-
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
2100
|
-
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
2101
|
-
}
|
2102
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
2103
|
-
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
2104
|
-
} finally {
|
2105
|
-
this.isProcessing = false;
|
2106
|
-
if (this.queue.length > 0) {
|
2107
|
-
void this.processQueue();
|
2108
|
-
}
|
2109
|
-
}
|
2110
|
-
}
|
2111
|
-
};
|
2112
|
-
|
2113
2314
|
// src/blockstore/store-remote.ts
|
2114
2315
|
async function RemoteDataStore(sthis, name, url, opts) {
|
2115
2316
|
const ds = new DataStoreImpl(sthis, name, url, opts);
|
@@ -2117,7 +2318,7 @@ async function RemoteDataStore(sthis, name, url, opts) {
|
|
2117
2318
|
return ds;
|
2118
2319
|
}
|
2119
2320
|
async function RemoteMetaStore(sthis, name, url, opts) {
|
2120
|
-
const ms = new MetaStoreImpl(sthis, name, url, opts);
|
2321
|
+
const ms = new MetaStoreImpl(sthis, name, url, opts, true);
|
2121
2322
|
await ms.start();
|
2122
2323
|
return ms;
|
2123
2324
|
}
|
@@ -2125,16 +2326,12 @@ async function RemoteMetaStore(sthis, name, url, opts) {
|
|
2125
2326
|
// src/blockstore/connection-base.ts
|
2126
2327
|
var ConnectionBase = class {
|
2127
2328
|
constructor(url, logger) {
|
2128
|
-
// readonly ready: Promise<unknown>;
|
2129
|
-
// todo move to LRU blockstore https://github.com/web3-storage/w3clock/blob/main/src/worker/block.js
|
2130
|
-
this.eventBlocks = new MemoryBlockstore2();
|
2131
|
-
this.parents = [];
|
2132
2329
|
this.loaded = Promise.resolve();
|
2133
2330
|
this.logger = logger;
|
2134
2331
|
this.url = url;
|
2135
2332
|
}
|
2136
2333
|
async refresh() {
|
2137
|
-
await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load(
|
2334
|
+
await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
|
2138
2335
|
await (await throwFalsy(this.loader).WALStore()).process();
|
2139
2336
|
}
|
2140
2337
|
async connect_X({ loader }) {
|
@@ -2145,26 +2342,20 @@ var ConnectionBase = class {
|
|
2145
2342
|
async connectMeta_X({ loader }) {
|
2146
2343
|
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
2147
2344
|
this.loader = loader;
|
2148
|
-
this.taskManager = new TaskManager(loader);
|
2149
2345
|
await this.onConnect();
|
2150
2346
|
const metaUrl = this.url.build().defParam("store", "meta").URI();
|
2151
2347
|
const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
|
2152
2348
|
if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
2153
|
-
const
|
2154
|
-
|
2349
|
+
const dbName = metaUrl.getParam("name");
|
2350
|
+
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
|
2351
|
+
const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
|
2155
2352
|
gateway: gateway.gateway,
|
2156
|
-
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag)
|
2157
|
-
|
2158
|
-
remote.onLoad("main", async (metas) => {
|
2159
|
-
if (metas) {
|
2160
|
-
this.logger.Debug().Any("metas", metas).Bool("loader", this.loader).Msg("connectMeta_X: handleDbMetasFromStore pre");
|
2161
|
-
await throwFalsy(this.loader).handleDbMetasFromStore(metas);
|
2162
|
-
this.logger.Debug().Any("metas", metas).Msg("connectMeta_X: handleDbMetasFromStore post");
|
2163
|
-
}
|
2353
|
+
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
|
2354
|
+
loader
|
2164
2355
|
});
|
2165
2356
|
this.loader.remoteMetaStore = remote;
|
2166
2357
|
this.loaded = this.loader.ready().then(async () => {
|
2167
|
-
remote.load(
|
2358
|
+
remote.load().then(async () => {
|
2168
2359
|
(await throwFalsy(this.loader).WALStore()).process();
|
2169
2360
|
});
|
2170
2361
|
});
|
@@ -2175,28 +2366,14 @@ var ConnectionBase = class {
|
|
2175
2366
|
const dataUrl = this.url.build().defParam("store", "data").URI();
|
2176
2367
|
const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
|
2177
2368
|
if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
2178
|
-
const name = dataUrl.
|
2369
|
+
const name = dataUrl.getParam("name");
|
2370
|
+
if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
|
2179
2371
|
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
2180
2372
|
gateway: gateway.gateway,
|
2181
2373
|
keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
|
2182
2374
|
});
|
2183
2375
|
loader.remoteFileStore = loader.remoteCarStore;
|
2184
2376
|
}
|
2185
|
-
async createEventBlock(bytes) {
|
2186
|
-
const data = {
|
2187
|
-
dbMeta: bytes
|
2188
|
-
};
|
2189
|
-
const event = await EventBlock.create(
|
2190
|
-
data,
|
2191
|
-
this.parents
|
2192
|
-
);
|
2193
|
-
await this.eventBlocks.put(event.cid, event.bytes);
|
2194
|
-
return event;
|
2195
|
-
}
|
2196
|
-
async decodeEventBlock(bytes) {
|
2197
|
-
const event = await decodeEventBlock(bytes);
|
2198
|
-
return event;
|
2199
|
-
}
|
2200
2377
|
// move this stuff to connect
|
2201
2378
|
// async getDashboardURL(compact = true) {
|
2202
2379
|
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
@@ -2490,7 +2667,7 @@ async function doCompact(blockLog, head, logger) {
|
|
2490
2667
|
isCompacting = false;
|
2491
2668
|
}
|
2492
2669
|
async function getBlock(blocks, cidString) {
|
2493
|
-
const block = await blocks.get(
|
2670
|
+
const block = await blocks.get(parse3(cidString));
|
2494
2671
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
2495
2672
|
const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
|
2496
2673
|
return new Block({ cid, value, bytes: block.bytes });
|
@@ -2826,7 +3003,7 @@ var Index = class {
|
|
2826
3003
|
// src/crdt-clock.ts
|
2827
3004
|
import { advance } from "@web3-storage/pail/clock";
|
2828
3005
|
import { root as root2 } from "@web3-storage/pail/crdt";
|
2829
|
-
import { ResolveOnce as
|
3006
|
+
import { ResolveOnce as ResolveOnce4 } from "@adviser/cement";
|
2830
3007
|
|
2831
3008
|
// src/apply-head-queue.ts
|
2832
3009
|
function applyHeadQueue(worker, logger) {
|
@@ -2883,7 +3060,7 @@ var CRDTClock = class {
|
|
2883
3060
|
this.zoomers = /* @__PURE__ */ new Set();
|
2884
3061
|
this.watchers = /* @__PURE__ */ new Set();
|
2885
3062
|
this.emptyWatchers = /* @__PURE__ */ new Set();
|
2886
|
-
this._ready = new
|
3063
|
+
this._ready = new ResolveOnce4();
|
2887
3064
|
this.blockstore = blockstore;
|
2888
3065
|
this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
|
2889
3066
|
this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
|
@@ -2994,7 +3171,7 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
2994
3171
|
var CRDT = class {
|
2995
3172
|
constructor(sthis, name, opts = {}) {
|
2996
3173
|
this.indexers = /* @__PURE__ */ new Map();
|
2997
|
-
this.onceReady = new
|
3174
|
+
this.onceReady = new ResolveOnce5();
|
2998
3175
|
this.sthis = sthis;
|
2999
3176
|
this.name = name;
|
3000
3177
|
this.logger = ensureLogger(sthis, "CRDT");
|
@@ -3060,7 +3237,8 @@ var CRDT = class {
|
|
3060
3237
|
try {
|
3061
3238
|
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
3062
3239
|
} catch (e) {
|
3063
|
-
|
3240
|
+
const ee = e;
|
3241
|
+
throw this.logger.Error().Err(e).Msg(`CRDT is not ready: ${ee.stack}`).AsError();
|
3064
3242
|
}
|
3065
3243
|
});
|
3066
3244
|
}
|
@@ -3114,7 +3292,7 @@ var Database = class {
|
|
3114
3292
|
this._listening = false;
|
3115
3293
|
this._listeners = /* @__PURE__ */ new Set();
|
3116
3294
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
3117
|
-
this._ready = new
|
3295
|
+
this._ready = new ResolveOnce6();
|
3118
3296
|
this.name = name;
|
3119
3297
|
this.opts = opts || this.opts;
|
3120
3298
|
this.sthis = ensureSuperThis(this.opts);
|
@@ -3292,34 +3470,9 @@ function makeName(fnString) {
|
|
3292
3470
|
}
|
3293
3471
|
}
|
3294
3472
|
|
3295
|
-
// src/runtime/index.ts
|
3296
|
-
var runtime_exports = {};
|
3297
|
-
__export(runtime_exports, {
|
3298
|
-
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
3299
|
-
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
3300
|
-
files: () => files_exports,
|
3301
|
-
getFileName: () => getFileName,
|
3302
|
-
getFileSystem: () => getFileSystem,
|
3303
|
-
getPath: () => getPath,
|
3304
|
-
kb: () => key_bag_exports,
|
3305
|
-
kc: () => keyed_crypto_exports,
|
3306
|
-
mf: () => wait_pr_multiformats_exports,
|
3307
|
-
toArrayBuffer: () => toArrayBuffer
|
3308
|
-
});
|
3309
|
-
|
3310
|
-
// src/runtime/wait-pr-multiformats/index.ts
|
3311
|
-
var wait_pr_multiformats_exports = {};
|
3312
|
-
__export(wait_pr_multiformats_exports, {
|
3313
|
-
block: () => block_exports,
|
3314
|
-
codec: () => codec_interface_exports
|
3315
|
-
});
|
3316
|
-
|
3317
|
-
// src/runtime/wait-pr-multiformats/codec-interface.ts
|
3318
|
-
var codec_interface_exports = {};
|
3319
|
-
|
3320
3473
|
// src/version.ts
|
3321
3474
|
var PACKAGE_VERSION = Object.keys({
|
3322
|
-
"0.19.
|
3475
|
+
"0.19.11-dev-dryrun": "xxxx"
|
3323
3476
|
})[0];
|
3324
3477
|
export {
|
3325
3478
|
CRDT,
|
@@ -3335,7 +3488,6 @@ export {
|
|
3335
3488
|
ensureLogger,
|
3336
3489
|
ensureSuperLog,
|
3337
3490
|
ensureSuperThis,
|
3338
|
-
exception2Result,
|
3339
3491
|
exceptionWrapper,
|
3340
3492
|
falsyToUndef,
|
3341
3493
|
fireproof,
|