@fireproof/core 0.19.9-dev-frag → 0.19.11-dev-dryrun
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/{chunk-YS4GL6OK.js → chunk-MAK4D54P.js} +20 -6
- package/chunk-MAK4D54P.js.map +1 -0
- package/{chunk-JO5AVWG7.js → chunk-XINRLWR3.js} +14 -6
- package/chunk-XINRLWR3.js.map +1 -0
- package/deno.json +20 -0
- package/{gateway-IZRHJWPE.js → gateway-7OM6OSYK.js} +3 -4
- package/gateway-7OM6OSYK.js.map +1 -0
- package/{gateway-YSNUK2L3.js → gateway-VWWKLHUI.js} +4 -5
- package/{gateway-YSNUK2L3.js.map → gateway-VWWKLHUI.js.map} +1 -1
- package/index.cjs +549 -372
- package/index.cjs.map +1 -1
- package/index.d.cts +75 -51
- package/index.d.ts +75 -51
- package/index.global.js +21461 -15114
- package/index.global.js.map +1 -1
- package/index.js +465 -313
- package/index.js.map +1 -1
- package/{key-bag-file-NMEBFSPM.js → key-bag-file-DFMW6ZM6.js} +3 -3
- package/{key-bag-indexdb-X5V6GNBZ.js → key-bag-indexdb-R2RWGSQ4.js} +3 -3
- package/key-bag-indexdb-R2RWGSQ4.js.map +1 -0
- package/{mem-filesystem-B6C6QOIP.js → mem-filesystem-BZQZLUR6.js} +3 -3
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/{node-filesystem-5JLBSHKQ.js → node-filesystem-7YZR3POJ.js} +8 -4
- package/node-filesystem-7YZR3POJ.js.map +1 -0
- package/package.json +8 -5
- package/tests/blockstore/keyed-crypto.test.ts +34 -4
- package/tests/blockstore/store.test.ts +18 -13
- package/tests/fireproof/all-gateway.test.ts +395 -0
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.car +0 -0
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.js +316 -0
- package/tests/fireproof/convert_uint8.py +27 -0
- package/tests/fireproof/fireproof.test.ts +18 -18
- package/tests/fireproof/hello.test.ts +33 -19
- package/tests/helpers.ts +7 -4
- package/{utils-IZPK4QS7.js → utils-AISQB3PB.js} +3 -3
- package/chunk-JO5AVWG7.js.map +0 -1
- package/chunk-YS4GL6OK.js.map +0 -1
- package/gateway-IZRHJWPE.js.map +0 -1
- package/key-bag-indexdb-X5V6GNBZ.js.map +0 -1
- package/node-filesystem-5JLBSHKQ.js.map +0 -1
- /package/{key-bag-file-NMEBFSPM.js.map → key-bag-file-DFMW6ZM6.js.map} +0 -0
- /package/{mem-filesystem-B6C6QOIP.js.map → mem-filesystem-BZQZLUR6.js.map} +0 -0
- /package/{utils-IZPK4QS7.js.map → utils-AISQB3PB.js.map} +0 -0
package/index.js
CHANGED
@@ -6,7 +6,7 @@ import {
|
|
6
6
|
getFileSystem,
|
7
7
|
getPath,
|
8
8
|
toArrayBuffer
|
9
|
-
} from "./chunk-
|
9
|
+
} from "./chunk-XINRLWR3.js";
|
10
10
|
import {
|
11
11
|
INDEXDB_VERSION
|
12
12
|
} from "./chunk-PB4BKL4O.js";
|
@@ -19,16 +19,15 @@ import {
|
|
19
19
|
ensureLogger,
|
20
20
|
ensureSuperLog,
|
21
21
|
ensureSuperThis,
|
22
|
-
exception2Result,
|
23
22
|
exceptionWrapper,
|
24
23
|
getKey,
|
25
24
|
getName,
|
26
25
|
getStore,
|
27
26
|
isNotFoundError
|
28
|
-
} from "./chunk-
|
27
|
+
} from "./chunk-MAK4D54P.js";
|
29
28
|
|
30
29
|
// src/database.ts
|
31
|
-
import { ResolveOnce as
|
30
|
+
import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
|
32
31
|
|
33
32
|
// src/write-queue.ts
|
34
33
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -71,7 +70,7 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
71
70
|
}
|
72
71
|
|
73
72
|
// src/crdt.ts
|
74
|
-
import { ResolveOnce as
|
73
|
+
import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
|
75
74
|
|
76
75
|
// src/runtime/wait-pr-multiformats/block.ts
|
77
76
|
var block_exports = {};
|
@@ -145,7 +144,7 @@ async function createUnsafe({
|
|
145
144
|
}
|
146
145
|
|
147
146
|
// src/crdt-helpers.ts
|
148
|
-
import { parse as
|
147
|
+
import { parse as parse3 } from "multiformats/link";
|
149
148
|
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
150
149
|
import * as codec from "@ipld/dag-cbor";
|
151
150
|
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
@@ -162,10 +161,12 @@ __export(blockstore_exports, {
|
|
162
161
|
EncryptedBlockstore: () => EncryptedBlockstore,
|
163
162
|
FragmentGateway: () => FragmentGateway,
|
164
163
|
Loader: () => Loader,
|
164
|
+
addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
|
165
165
|
ensureStart: () => ensureStart,
|
166
166
|
getGatewayFromURL: () => getGatewayFromURL,
|
167
167
|
parseCarFile: () => parseCarFile,
|
168
168
|
registerStoreProtocol: () => registerStoreProtocol,
|
169
|
+
setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
|
169
170
|
testStoreFactory: () => testStoreFactory,
|
170
171
|
toCIDBlock: () => toCIDBlock,
|
171
172
|
toStoreRuntime: () => toStoreRuntime
|
@@ -177,7 +178,7 @@ function toCIDBlock(block) {
|
|
177
178
|
}
|
178
179
|
|
179
180
|
// src/blockstore/store-factory.ts
|
180
|
-
import { KeyedResolvOnce as KeyedResolvOnce2, URI as
|
181
|
+
import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
|
181
182
|
|
182
183
|
// src/runtime/files.ts
|
183
184
|
var files_exports = {};
|
@@ -253,7 +254,7 @@ var UnixFSFileBuilder = class {
|
|
253
254
|
// src/blockstore/store.ts
|
254
255
|
import pLimit2 from "p-limit";
|
255
256
|
import { format, parse } from "@ipld/dag-json";
|
256
|
-
import { ResolveOnce as
|
257
|
+
import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result4 } from "@adviser/cement";
|
257
258
|
|
258
259
|
// src/types.ts
|
259
260
|
function isFalsy(value) {
|
@@ -275,7 +276,7 @@ function falsyToUndef(value) {
|
|
275
276
|
// src/blockstore/loader.ts
|
276
277
|
import pLimit from "p-limit";
|
277
278
|
import { CarReader } from "@ipld/car";
|
278
|
-
import { ResolveOnce } from "@adviser/cement";
|
279
|
+
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
279
280
|
|
280
281
|
// src/blockstore/loader-helpers.ts
|
281
282
|
import { sha256 as hasher } from "multiformats/hashes/sha2";
|
@@ -547,6 +548,7 @@ __export(key_bag_exports, {
|
|
547
548
|
});
|
548
549
|
import {
|
549
550
|
KeyedResolvOnce,
|
551
|
+
ResolveOnce,
|
550
552
|
ResolveSeq,
|
551
553
|
Result as Result2,
|
552
554
|
runtimeFn,
|
@@ -557,21 +559,25 @@ import { base58btc } from "multiformats/bases/base58";
|
|
557
559
|
var KeyBag = class {
|
558
560
|
constructor(rt) {
|
559
561
|
this.rt = rt;
|
562
|
+
this._warnOnce = new ResolveOnce();
|
560
563
|
this._seq = new ResolveSeq();
|
561
|
-
this.logger = ensureLogger(rt.sthis, "KeyBag"
|
562
|
-
id: rt.id()
|
563
|
-
});
|
564
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag");
|
564
565
|
this.logger.Debug().Msg("KeyBag created");
|
565
566
|
}
|
566
567
|
async subtleKey(key) {
|
568
|
+
const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
|
569
|
+
if (extractable) {
|
570
|
+
this._warnOnce.once(
|
571
|
+
() => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
|
572
|
+
);
|
573
|
+
}
|
567
574
|
return await this.rt.crypto.importKey(
|
568
575
|
"raw",
|
569
576
|
// raw or jwk
|
570
577
|
base58btc.decode(key),
|
571
578
|
// hexStringToUint8Array(key), // raw data
|
572
579
|
"AES-GCM",
|
573
|
-
|
574
|
-
// extractable
|
580
|
+
extractable,
|
575
581
|
["encrypt", "decrypt"]
|
576
582
|
);
|
577
583
|
}
|
@@ -620,6 +626,23 @@ var KeyBag = class {
|
|
620
626
|
await bag.set(name, item);
|
621
627
|
return await this.toKeyWithFingerPrint(item.key);
|
622
628
|
}
|
629
|
+
async getNamedExtractableKey(name, failIfNotFound = false) {
|
630
|
+
const ret = await this.getNamedKey(name, failIfNotFound);
|
631
|
+
if (ret.isErr()) {
|
632
|
+
return ret;
|
633
|
+
}
|
634
|
+
const named = ret.Ok();
|
635
|
+
return Result2.Ok({
|
636
|
+
...named,
|
637
|
+
extract: async () => {
|
638
|
+
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
639
|
+
return {
|
640
|
+
key: ext,
|
641
|
+
keyStr: base58btc.encode(ext)
|
642
|
+
};
|
643
|
+
}
|
644
|
+
});
|
645
|
+
}
|
623
646
|
async getNamedKey(name, failIfNotFound = false) {
|
624
647
|
const id = this.rt.sthis.nextId(4).str;
|
625
648
|
return this._seq.add(async () => {
|
@@ -645,14 +668,14 @@ var keyBagProviderFactories = new Map(
|
|
645
668
|
{
|
646
669
|
protocol: "file:",
|
647
670
|
factory: async (url, sthis) => {
|
648
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-
|
671
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-DFMW6ZM6.js");
|
649
672
|
return new KeyBagProviderFile(url, sthis);
|
650
673
|
}
|
651
674
|
},
|
652
675
|
{
|
653
676
|
protocol: "indexdb:",
|
654
677
|
factory: async (url, sthis) => {
|
655
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-
|
678
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-R2RWGSQ4.js");
|
656
679
|
return new KeyBagProviderIndexDB(url, sthis);
|
657
680
|
}
|
658
681
|
}
|
@@ -689,23 +712,11 @@ function defaultKeyBagOpts(sthis, kbo) {
|
|
689
712
|
}
|
690
713
|
logger.Debug().Url(url).Msg("from env");
|
691
714
|
}
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
keyProviderFactory = async () => {
|
696
|
-
const { KeyBagProviderFile } = await import("./key-bag-file-NMEBFSPM.js");
|
697
|
-
return new KeyBagProviderFile(url, sthis);
|
698
|
-
};
|
699
|
-
break;
|
700
|
-
case "indexdb:":
|
701
|
-
keyProviderFactory = async () => {
|
702
|
-
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-X5V6GNBZ.js");
|
703
|
-
return new KeyBagProviderIndexDB(url, sthis);
|
704
|
-
};
|
705
|
-
break;
|
706
|
-
default:
|
707
|
-
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
715
|
+
const kitem = keyBagProviderFactories.get(url.protocol);
|
716
|
+
if (!kitem) {
|
717
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
708
718
|
}
|
719
|
+
const getBag = async () => kitem.factory(url, sthis);
|
709
720
|
if (url.hasParam("masterkey")) {
|
710
721
|
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
711
722
|
}
|
@@ -715,7 +726,7 @@ function defaultKeyBagOpts(sthis, kbo) {
|
|
715
726
|
sthis,
|
716
727
|
logger,
|
717
728
|
keyLength: kbo.keyLength || 16,
|
718
|
-
getBag
|
729
|
+
getBag,
|
719
730
|
id: () => {
|
720
731
|
return url.toString();
|
721
732
|
}
|
@@ -832,6 +843,53 @@ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
|
832
843
|
|
833
844
|
// src/blockstore/loader.ts
|
834
845
|
import { sha256 as hasher3 } from "multiformats/hashes/sha2";
|
846
|
+
|
847
|
+
// src/blockstore/task-manager.ts
|
848
|
+
var TaskManager = class {
|
849
|
+
constructor(sthis, callback) {
|
850
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
851
|
+
this.queue = [];
|
852
|
+
this.isProcessing = false;
|
853
|
+
this.logger = ensureLogger(sthis, "TaskManager");
|
854
|
+
this.callback = callback;
|
855
|
+
}
|
856
|
+
async handleEvent(cid, parents, dbMeta) {
|
857
|
+
for (const parent of parents) {
|
858
|
+
this.eventsWeHandled.add(parent.toString());
|
859
|
+
}
|
860
|
+
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
861
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
862
|
+
void this.processQueue();
|
863
|
+
}
|
864
|
+
async processQueue() {
|
865
|
+
if (this.isProcessing) return;
|
866
|
+
this.isProcessing = true;
|
867
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
868
|
+
const first = filteredQueue[0];
|
869
|
+
if (!first) {
|
870
|
+
return;
|
871
|
+
}
|
872
|
+
try {
|
873
|
+
await this.callback(first.dbMeta);
|
874
|
+
this.eventsWeHandled.add(first.cid);
|
875
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
876
|
+
} catch (err) {
|
877
|
+
if (first.retries++ > 3) {
|
878
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
879
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
880
|
+
}
|
881
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
882
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
883
|
+
} finally {
|
884
|
+
this.isProcessing = false;
|
885
|
+
if (this.queue.length > 0) {
|
886
|
+
void this.processQueue();
|
887
|
+
}
|
888
|
+
}
|
889
|
+
}
|
890
|
+
};
|
891
|
+
|
892
|
+
// src/blockstore/loader.ts
|
835
893
|
function carLogIncludesGroup(list, cids) {
|
836
894
|
return list.some((arr) => {
|
837
895
|
return arr.toString() === cids.toString();
|
@@ -856,7 +914,7 @@ var Loader = class {
|
|
856
914
|
this.getBlockCache = /* @__PURE__ */ new Map();
|
857
915
|
this.seenMeta = /* @__PURE__ */ new Set();
|
858
916
|
this.writeLimit = pLimit(1);
|
859
|
-
this.onceReady = new
|
917
|
+
this.onceReady = new ResolveOnce2();
|
860
918
|
this.name = name;
|
861
919
|
this.sthis = sthis;
|
862
920
|
this.ebOpts = defaultedBlockstoreRuntime(
|
@@ -868,6 +926,9 @@ var Loader = class {
|
|
868
926
|
"Loader"
|
869
927
|
);
|
870
928
|
this.logger = this.ebOpts.logger;
|
929
|
+
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
930
|
+
await this.handleDbMetasFromStore([dbMeta]);
|
931
|
+
});
|
871
932
|
}
|
872
933
|
// readonly id = uuidv4();
|
873
934
|
async keyBag() {
|
@@ -887,8 +948,10 @@ var Loader = class {
|
|
887
948
|
}
|
888
949
|
async ready() {
|
889
950
|
return this.onceReady.once(async () => {
|
890
|
-
const metas =
|
891
|
-
if (
|
951
|
+
const metas = await (await this.metaStore()).load();
|
952
|
+
if (this.ebOpts.meta) {
|
953
|
+
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
954
|
+
} else if (metas) {
|
892
955
|
await this.handleDbMetasFromStore(metas);
|
893
956
|
}
|
894
957
|
});
|
@@ -1193,7 +1256,7 @@ var generateIV = {
|
|
1193
1256
|
calc: async (ko, crypto, data) => {
|
1194
1257
|
const hash = await hasher4.digest(data);
|
1195
1258
|
const hashBytes = new Uint8Array(hash.bytes);
|
1196
|
-
const hashArray = new Uint8Array(ko.ivLength
|
1259
|
+
const hashArray = new Uint8Array(ko.ivLength);
|
1197
1260
|
for (let i = 0; i < hashBytes.length; i++) {
|
1198
1261
|
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
1199
1262
|
}
|
@@ -1271,7 +1334,7 @@ var keyedCrypto = class {
|
|
1271
1334
|
};
|
1272
1335
|
}
|
1273
1336
|
async _decrypt(data) {
|
1274
|
-
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("decrypting");
|
1337
|
+
this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
|
1275
1338
|
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
1276
1339
|
}
|
1277
1340
|
async _encrypt(data) {
|
@@ -1342,6 +1405,7 @@ async function keyedCryptoFactory(url, kb, sthis) {
|
|
1342
1405
|
}
|
1343
1406
|
|
1344
1407
|
// src/blockstore/fragment-gateway.ts
|
1408
|
+
import { Result as Result3 } from "@adviser/cement";
|
1345
1409
|
import { base58btc as base58btc3 } from "multiformats/bases/base58";
|
1346
1410
|
import { encode as encode3, decode as decode3 } from "cborg";
|
1347
1411
|
function getFragSize(url) {
|
@@ -1364,7 +1428,7 @@ async function getFrags(url, innerGW, headerSize, logger) {
|
|
1364
1428
|
}
|
1365
1429
|
const data = res.unwrap();
|
1366
1430
|
return [
|
1367
|
-
|
1431
|
+
Result3.Ok({
|
1368
1432
|
fid: new Uint8Array(0),
|
1369
1433
|
ofs: 0,
|
1370
1434
|
len: data.length,
|
@@ -1378,7 +1442,7 @@ async function getFrags(url, innerGW, headerSize, logger) {
|
|
1378
1442
|
}
|
1379
1443
|
const firstFragment = decode3(firstRaw.unwrap());
|
1380
1444
|
const blockSize = firstFragment.data.length;
|
1381
|
-
const ops = [Promise.resolve(
|
1445
|
+
const ops = [Promise.resolve(Result3.Ok(firstFragment))];
|
1382
1446
|
const fidStr = base58btc3.encode(firstFragment.fid);
|
1383
1447
|
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
|
1384
1448
|
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
@@ -1390,12 +1454,12 @@ async function getFrags(url, innerGW, headerSize, logger) {
|
|
1390
1454
|
}
|
1391
1455
|
const fragment = decode3(raw2.unwrap());
|
1392
1456
|
if (base58btc3.encode(fragment.fid) !== fidStr) {
|
1393
|
-
return
|
1457
|
+
return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
1394
1458
|
}
|
1395
1459
|
if (fragment.ofs !== ofs2) {
|
1396
|
-
return
|
1460
|
+
return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
1397
1461
|
}
|
1398
|
-
return
|
1462
|
+
return Result3.Ok(fragment);
|
1399
1463
|
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
1400
1464
|
);
|
1401
1465
|
}
|
@@ -1457,42 +1521,68 @@ var FragmentGateway = class {
|
|
1457
1521
|
}
|
1458
1522
|
async put(url, body) {
|
1459
1523
|
await Promise.all(this.slicer(url, body));
|
1460
|
-
return
|
1524
|
+
return Result3.Ok(void 0);
|
1461
1525
|
}
|
1462
1526
|
async get(url) {
|
1463
1527
|
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1464
1528
|
let buffer = void 0;
|
1465
1529
|
for (const rfrag of rfrags) {
|
1466
1530
|
if (rfrag.isErr()) {
|
1467
|
-
return
|
1531
|
+
return Result3.Err(rfrag.Err());
|
1468
1532
|
}
|
1469
1533
|
const frag = rfrag.Ok();
|
1470
1534
|
buffer = buffer || new Uint8Array(frag.len);
|
1471
1535
|
buffer.set(frag.data, frag.ofs);
|
1472
1536
|
}
|
1473
|
-
return
|
1537
|
+
return Result3.Ok(buffer || new Uint8Array(0));
|
1538
|
+
}
|
1539
|
+
async subscribe(url, callback) {
|
1540
|
+
if (this.innerGW.subscribe) {
|
1541
|
+
return this.innerGW.subscribe(url, callback);
|
1542
|
+
} else {
|
1543
|
+
let lastData = void 0;
|
1544
|
+
let interval = 100;
|
1545
|
+
const fetchData = async () => {
|
1546
|
+
const result = await this.innerGW.get(url);
|
1547
|
+
if (result.isOk()) {
|
1548
|
+
const data = result.Ok();
|
1549
|
+
if (!lastData || !data.every((value, index2) => lastData && value === lastData[index2])) {
|
1550
|
+
lastData = data;
|
1551
|
+
callback(data);
|
1552
|
+
interval = 100;
|
1553
|
+
} else {
|
1554
|
+
interval *= 2;
|
1555
|
+
}
|
1556
|
+
}
|
1557
|
+
timeoutId = setTimeout(fetchData, interval);
|
1558
|
+
};
|
1559
|
+
let timeoutId = setTimeout(fetchData, interval);
|
1560
|
+
return Result3.Ok(() => {
|
1561
|
+
clearTimeout(timeoutId);
|
1562
|
+
});
|
1563
|
+
}
|
1474
1564
|
}
|
1475
1565
|
async delete(url) {
|
1476
1566
|
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1477
1567
|
for (const rfrag of rfrags) {
|
1478
1568
|
if (rfrag.isErr()) {
|
1479
|
-
return
|
1569
|
+
return Result3.Err(rfrag.Err());
|
1480
1570
|
}
|
1481
1571
|
const frag = rfrag.Ok();
|
1482
1572
|
const fidStr = base58btc3.encode(frag.fid);
|
1483
1573
|
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
|
1484
1574
|
await this.innerGW.delete(fragUrl);
|
1485
1575
|
}
|
1486
|
-
return
|
1576
|
+
return Result3.Ok(void 0);
|
1487
1577
|
}
|
1488
1578
|
};
|
1489
1579
|
|
1490
1580
|
// src/blockstore/store.ts
|
1491
1581
|
function guardVersion(url) {
|
1492
1582
|
if (!url.hasParam("version")) {
|
1493
|
-
return
|
1583
|
+
return Result4.Err(`missing version: ${url.toString()}`);
|
1494
1584
|
}
|
1495
|
-
return
|
1585
|
+
return Result4.Ok(url);
|
1496
1586
|
}
|
1497
1587
|
var BaseStoreImpl = class {
|
1498
1588
|
constructor(name, url, opts, sthis, logger) {
|
@@ -1504,6 +1594,7 @@ var BaseStoreImpl = class {
|
|
1504
1594
|
this.sthis = sthis;
|
1505
1595
|
this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
|
1506
1596
|
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
1597
|
+
this.loader = opts.loader;
|
1507
1598
|
}
|
1508
1599
|
url() {
|
1509
1600
|
return this._url;
|
@@ -1559,102 +1650,6 @@ var BaseStoreImpl = class {
|
|
1559
1650
|
return version;
|
1560
1651
|
}
|
1561
1652
|
};
|
1562
|
-
var MetaStoreImpl = class extends BaseStoreImpl {
|
1563
|
-
constructor(sthis, name, url, opts) {
|
1564
|
-
super(
|
1565
|
-
name,
|
1566
|
-
url,
|
1567
|
-
{
|
1568
|
-
...opts
|
1569
|
-
},
|
1570
|
-
sthis,
|
1571
|
-
ensureLogger(sthis, "MetaStoreImpl")
|
1572
|
-
);
|
1573
|
-
this.storeType = "meta";
|
1574
|
-
this.subscribers = /* @__PURE__ */ new Map();
|
1575
|
-
}
|
1576
|
-
onLoad(branch, loadHandler) {
|
1577
|
-
const subscribers = this.subscribers.get(branch) || [];
|
1578
|
-
subscribers.push(loadHandler);
|
1579
|
-
this.subscribers.set(branch, subscribers);
|
1580
|
-
return () => {
|
1581
|
-
const subscribers2 = this.subscribers.get(branch) || [];
|
1582
|
-
const idx = subscribers2.indexOf(loadHandler);
|
1583
|
-
if (idx > -1) subscribers2.splice(idx, 1);
|
1584
|
-
};
|
1585
|
-
}
|
1586
|
-
makeHeader({ cars }) {
|
1587
|
-
const toEncode = { cars };
|
1588
|
-
return format(toEncode);
|
1589
|
-
}
|
1590
|
-
parseHeader(headerData) {
|
1591
|
-
const got = parse(headerData);
|
1592
|
-
return got;
|
1593
|
-
}
|
1594
|
-
async handleSubscribers(dbMetas, branch) {
|
1595
|
-
try {
|
1596
|
-
const subscribers = this.subscribers.get(branch) || [];
|
1597
|
-
await Promise.all(subscribers.map((subscriber) => subscriber(dbMetas)));
|
1598
|
-
} catch (e) {
|
1599
|
-
this.logger.Error().Err(e).Msg("handleSubscribers").AsError();
|
1600
|
-
}
|
1601
|
-
}
|
1602
|
-
async handleByteHeads(byteHeads, branch = "main") {
|
1603
|
-
let dbMetas;
|
1604
|
-
try {
|
1605
|
-
dbMetas = this.dbMetasForByteHeads(byteHeads);
|
1606
|
-
} catch (e) {
|
1607
|
-
throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
|
1608
|
-
}
|
1609
|
-
await this.handleSubscribers(dbMetas, branch);
|
1610
|
-
return dbMetas;
|
1611
|
-
}
|
1612
|
-
dbMetasForByteHeads(byteHeads) {
|
1613
|
-
return byteHeads.map((bytes) => {
|
1614
|
-
const txt = this.sthis.txt.decode(bytes);
|
1615
|
-
return this.parseHeader(txt);
|
1616
|
-
});
|
1617
|
-
}
|
1618
|
-
async load(branch) {
|
1619
|
-
branch = branch || "main";
|
1620
|
-
this.logger.Debug().Str("branch", branch).Msg("loading");
|
1621
|
-
const url = await this.gateway.buildUrl(this.url(), branch);
|
1622
|
-
if (url.isErr()) {
|
1623
|
-
throw this.logger.Error().Result("buidUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
1624
|
-
}
|
1625
|
-
const bytes = await this.gateway.get(url.Ok());
|
1626
|
-
if (bytes.isErr()) {
|
1627
|
-
if (isNotFoundError(bytes)) {
|
1628
|
-
return void 0;
|
1629
|
-
}
|
1630
|
-
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
1631
|
-
}
|
1632
|
-
return this.handleByteHeads([bytes.Ok()], branch);
|
1633
|
-
}
|
1634
|
-
async save(meta, branch) {
|
1635
|
-
branch = branch || "main";
|
1636
|
-
this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
|
1637
|
-
const bytes = this.makeHeader(meta);
|
1638
|
-
const url = await this.gateway.buildUrl(this.url(), branch);
|
1639
|
-
if (url.isErr()) {
|
1640
|
-
throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
1641
|
-
}
|
1642
|
-
const res = await this.gateway.put(url.Ok(), this.sthis.txt.encode(bytes));
|
1643
|
-
if (res.isErr()) {
|
1644
|
-
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
1645
|
-
}
|
1646
|
-
await this.handleSubscribers([meta], branch);
|
1647
|
-
return res;
|
1648
|
-
}
|
1649
|
-
async close() {
|
1650
|
-
await this.gateway.close(this.url());
|
1651
|
-
this._onClosed.forEach((fn) => fn());
|
1652
|
-
return Result3.Ok(void 0);
|
1653
|
-
}
|
1654
|
-
async destroy() {
|
1655
|
-
return this.gateway.destroy(this.url());
|
1656
|
-
}
|
1657
|
-
};
|
1658
1653
|
var DataStoreImpl = class extends BaseStoreImpl {
|
1659
1654
|
// readonly tag: string = "car-base";
|
1660
1655
|
constructor(sthis, name, url, opts) {
|
@@ -1704,7 +1699,7 @@ var DataStoreImpl = class extends BaseStoreImpl {
|
|
1704
1699
|
async close() {
|
1705
1700
|
await this.gateway.close(this.url());
|
1706
1701
|
this._onClosed.forEach((fn) => fn());
|
1707
|
-
return
|
1702
|
+
return Result4.Ok(void 0);
|
1708
1703
|
}
|
1709
1704
|
destroy() {
|
1710
1705
|
return this.gateway.destroy(this.url());
|
@@ -1722,7 +1717,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1722
1717
|
ensureLogger(loader.sthis, "WALStoreImpl")
|
1723
1718
|
);
|
1724
1719
|
this.storeType = "wal";
|
1725
|
-
this._ready = new
|
1720
|
+
this._ready = new ResolveOnce3();
|
1726
1721
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
1727
1722
|
this.processing = void 0;
|
1728
1723
|
this.processQueue = new CommitQueue();
|
@@ -1878,13 +1873,261 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
1878
1873
|
async close() {
|
1879
1874
|
await this.gateway.close(this.url());
|
1880
1875
|
this._onClosed.forEach((fn) => fn());
|
1881
|
-
return
|
1876
|
+
return Result4.Ok(void 0);
|
1882
1877
|
}
|
1883
1878
|
destroy() {
|
1884
1879
|
return this.gateway.destroy(this.url());
|
1885
1880
|
}
|
1886
1881
|
};
|
1887
1882
|
|
1883
|
+
// src/blockstore/store-meta.ts
|
1884
|
+
import { format as format2, parse as parse2 } from "@ipld/dag-json";
|
1885
|
+
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
1886
|
+
import { CID as CID2 } from "multiformats";
|
1887
|
+
import { Result as Result5 } from "@adviser/cement";
|
1888
|
+
|
1889
|
+
// src/runtime/index.ts
|
1890
|
+
var runtime_exports = {};
|
1891
|
+
__export(runtime_exports, {
|
1892
|
+
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
1893
|
+
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
1894
|
+
files: () => files_exports,
|
1895
|
+
getFileName: () => getFileName,
|
1896
|
+
getFileSystem: () => getFileSystem,
|
1897
|
+
getPath: () => getPath,
|
1898
|
+
kb: () => key_bag_exports,
|
1899
|
+
kc: () => keyed_crypto_exports,
|
1900
|
+
mf: () => wait_pr_multiformats_exports,
|
1901
|
+
runtimeFn: () => runtimeFn2,
|
1902
|
+
toArrayBuffer: () => toArrayBuffer
|
1903
|
+
});
|
1904
|
+
|
1905
|
+
// src/runtime/wait-pr-multiformats/index.ts
|
1906
|
+
var wait_pr_multiformats_exports = {};
|
1907
|
+
__export(wait_pr_multiformats_exports, {
|
1908
|
+
block: () => block_exports,
|
1909
|
+
codec: () => codec_interface_exports
|
1910
|
+
});
|
1911
|
+
|
1912
|
+
// src/runtime/wait-pr-multiformats/codec-interface.ts
|
1913
|
+
var codec_interface_exports = {};
|
1914
|
+
|
1915
|
+
// src/runtime/index.ts
|
1916
|
+
import { runtimeFn as runtimeFn2 } from "@adviser/cement";
|
1917
|
+
|
1918
|
+
// src/blockstore/store-meta.ts
|
1919
|
+
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
1920
|
+
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
1921
|
+
return Promise.all(
|
1922
|
+
crdtEntries.map(async (crdtEntry) => {
|
1923
|
+
const eventBlock = await decodeEventBlock(decodeFromBase64(crdtEntry.data));
|
1924
|
+
const dbMeta = parse2(sthis.txt.decode(eventBlock.value.data.dbMeta));
|
1925
|
+
return {
|
1926
|
+
eventCid: eventBlock.cid,
|
1927
|
+
parents: crdtEntry.parents,
|
1928
|
+
dbMeta
|
1929
|
+
};
|
1930
|
+
})
|
1931
|
+
);
|
1932
|
+
}
|
1933
|
+
async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
|
1934
|
+
try {
|
1935
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
|
1936
|
+
const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
|
1937
|
+
if (keyInfo.length) {
|
1938
|
+
const dbMeta = keyInfo[0].dbMeta;
|
1939
|
+
if (dbMeta.key) {
|
1940
|
+
const kb = await key_bag_exports.getKeyBag(sthis);
|
1941
|
+
const keyName = getStoreKeyName(uri);
|
1942
|
+
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
1943
|
+
if (res.isErr()) {
|
1944
|
+
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
1945
|
+
throw res.Err();
|
1946
|
+
}
|
1947
|
+
}
|
1948
|
+
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
1949
|
+
return Result5.Ok(dbMeta);
|
1950
|
+
}
|
1951
|
+
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
1952
|
+
return Result5.Ok(void 0);
|
1953
|
+
} catch (error) {
|
1954
|
+
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
1955
|
+
return Result5.Err(error);
|
1956
|
+
}
|
1957
|
+
}
|
1958
|
+
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
1959
|
+
try {
|
1960
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
1961
|
+
const keyName = getStoreKeyName(uri);
|
1962
|
+
const kb = await key_bag_exports.getKeyBag(sthis);
|
1963
|
+
const res = await kb.getNamedExtractableKey(keyName, true);
|
1964
|
+
if (res.isErr()) {
|
1965
|
+
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
1966
|
+
throw res.Err();
|
1967
|
+
}
|
1968
|
+
const keyData = await res.Ok().extract();
|
1969
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
1970
|
+
const { dbMeta, parents } = dbMetas[0];
|
1971
|
+
const parentLinks = parents.map((p) => CID2.parse(p));
|
1972
|
+
dbMeta.key = keyData.keyStr;
|
1973
|
+
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
1974
|
+
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
1975
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
1976
|
+
return Result5.Ok(encoded);
|
1977
|
+
} catch (error) {
|
1978
|
+
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
1979
|
+
return Result5.Err(error);
|
1980
|
+
}
|
1981
|
+
}
|
1982
|
+
function getStoreKeyName(url) {
|
1983
|
+
const storeKeyName = [url.getParam("localName") || url.getParam("name")];
|
1984
|
+
const idx = url.getParam("index");
|
1985
|
+
if (idx) {
|
1986
|
+
storeKeyName.push(idx);
|
1987
|
+
}
|
1988
|
+
storeKeyName.push("data");
|
1989
|
+
return `@${storeKeyName.join(":")}@`;
|
1990
|
+
}
|
1991
|
+
async function createDbMetaEventBlock(sthis, dbMeta, parents) {
|
1992
|
+
const event = await EventBlock.create(
|
1993
|
+
{
|
1994
|
+
dbMeta: sthis.txt.encode(format2(dbMeta))
|
1995
|
+
},
|
1996
|
+
parents
|
1997
|
+
);
|
1998
|
+
return event;
|
1999
|
+
}
|
2000
|
+
async function encodeEventsWithParents(sthis, events, parents) {
|
2001
|
+
const crdtEntries = events.map((event) => {
|
2002
|
+
const base64String = encodeToBase64(event.bytes);
|
2003
|
+
return {
|
2004
|
+
cid: event.cid.toString(),
|
2005
|
+
data: base64String,
|
2006
|
+
parents: parents.map((p) => p.toString())
|
2007
|
+
};
|
2008
|
+
});
|
2009
|
+
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
2010
|
+
}
|
2011
|
+
var MetaStoreImpl = class extends BaseStoreImpl {
|
2012
|
+
constructor(sthis, name, url, opts, remote) {
|
2013
|
+
super(
|
2014
|
+
name,
|
2015
|
+
url,
|
2016
|
+
{
|
2017
|
+
...opts
|
2018
|
+
},
|
2019
|
+
sthis,
|
2020
|
+
ensureLogger(sthis, "MetaStoreImpl")
|
2021
|
+
);
|
2022
|
+
this.storeType = "meta";
|
2023
|
+
this.subscribers = /* @__PURE__ */ new Map();
|
2024
|
+
this.parents = [];
|
2025
|
+
if (remote && opts.gateway.subscribe) {
|
2026
|
+
this.onStarted(async () => {
|
2027
|
+
this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
|
2028
|
+
opts.gateway.subscribe?.(this.url(), async (message) => {
|
2029
|
+
this.logger.Debug().Msg("Received message from gateway");
|
2030
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
|
2031
|
+
await Promise.all(
|
2032
|
+
dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
|
2033
|
+
);
|
2034
|
+
});
|
2035
|
+
});
|
2036
|
+
}
|
2037
|
+
}
|
2038
|
+
async handleByteHeads(byteHeads) {
|
2039
|
+
return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
|
2040
|
+
}
|
2041
|
+
async load() {
|
2042
|
+
const branch = "main";
|
2043
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2044
|
+
if (url.isErr()) {
|
2045
|
+
throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2046
|
+
}
|
2047
|
+
const bytes = await this.gateway.get(url.Ok());
|
2048
|
+
if (bytes.isErr()) {
|
2049
|
+
if (isNotFoundError(bytes)) {
|
2050
|
+
return void 0;
|
2051
|
+
}
|
2052
|
+
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
2053
|
+
}
|
2054
|
+
const dbMetas = await this.handleByteHeads(bytes.Ok());
|
2055
|
+
await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
|
2056
|
+
const cids = dbMetas.map((m) => m.eventCid);
|
2057
|
+
const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
|
2058
|
+
this.parents = Array.from(uniqueParentsMap.values());
|
2059
|
+
return dbMetas.map((m) => m.dbMeta);
|
2060
|
+
}
|
2061
|
+
async save(meta, branch) {
|
2062
|
+
branch = branch || "main";
|
2063
|
+
this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
|
2064
|
+
const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
|
2065
|
+
const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
|
2066
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2067
|
+
if (url.isErr()) {
|
2068
|
+
throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2069
|
+
}
|
2070
|
+
const res = await this.gateway.put(url.Ok(), bytes);
|
2071
|
+
if (res.isErr()) {
|
2072
|
+
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
2073
|
+
}
|
2074
|
+
await this.loader?.handleDbMetasFromStore([meta]);
|
2075
|
+
this.parents = [event.cid];
|
2076
|
+
return res;
|
2077
|
+
}
|
2078
|
+
async close() {
|
2079
|
+
await this.gateway.close(this.url());
|
2080
|
+
this._onClosed.forEach((fn) => fn());
|
2081
|
+
return Result5.Ok(void 0);
|
2082
|
+
}
|
2083
|
+
async destroy() {
|
2084
|
+
return this.gateway.destroy(this.url());
|
2085
|
+
}
|
2086
|
+
};
|
2087
|
+
function encodeToBase64(bytes) {
|
2088
|
+
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
2089
|
+
let base64 = "";
|
2090
|
+
let i;
|
2091
|
+
for (i = 0; i < bytes.length - 2; i += 3) {
|
2092
|
+
base64 += chars[bytes[i] >> 2];
|
2093
|
+
base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
|
2094
|
+
base64 += chars[(bytes[i + 1] & 15) << 2 | bytes[i + 2] >> 6];
|
2095
|
+
base64 += chars[bytes[i + 2] & 63];
|
2096
|
+
}
|
2097
|
+
if (i < bytes.length) {
|
2098
|
+
base64 += chars[bytes[i] >> 2];
|
2099
|
+
if (i === bytes.length - 1) {
|
2100
|
+
base64 += chars[(bytes[i] & 3) << 4];
|
2101
|
+
base64 += "==";
|
2102
|
+
} else {
|
2103
|
+
base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
|
2104
|
+
base64 += chars[(bytes[i + 1] & 15) << 2];
|
2105
|
+
base64 += "=";
|
2106
|
+
}
|
2107
|
+
}
|
2108
|
+
return base64;
|
2109
|
+
}
|
2110
|
+
function decodeFromBase64(base64) {
|
2111
|
+
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
2112
|
+
const bytes = new Uint8Array(base64.length * 3 / 4);
|
2113
|
+
let i;
|
2114
|
+
let j = 0;
|
2115
|
+
for (i = 0; i < base64.length; i += 4) {
|
2116
|
+
const a = chars.indexOf(base64[i]);
|
2117
|
+
const b = chars.indexOf(base64[i + 1]);
|
2118
|
+
const c = chars.indexOf(base64[i + 2]);
|
2119
|
+
const d = chars.indexOf(base64[i + 3]);
|
2120
|
+
bytes[j++] = a << 2 | b >> 4;
|
2121
|
+
if (base64[i + 2] !== "=") {
|
2122
|
+
bytes[j++] = (b & 15) << 4 | c >> 2;
|
2123
|
+
}
|
2124
|
+
if (base64[i + 3] !== "=") {
|
2125
|
+
bytes[j++] = (c & 3) << 6 | d;
|
2126
|
+
}
|
2127
|
+
}
|
2128
|
+
return bytes.slice(0, j);
|
2129
|
+
}
|
2130
|
+
|
1888
2131
|
// src/blockstore/store-factory.ts
|
1889
2132
|
function ensureIsIndex(url, isIndex) {
|
1890
2133
|
if (isIndex) {
|
@@ -1904,10 +2147,10 @@ function buildURL(optURL, loader) {
|
|
1904
2147
|
const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
|
1905
2148
|
let obuUrl;
|
1906
2149
|
if (obuItem && obuItem.overrideBaseURL) {
|
1907
|
-
obuUrl =
|
2150
|
+
obuUrl = URI5.from(obuItem.overrideBaseURL);
|
1908
2151
|
}
|
1909
2152
|
const ret = ensureIsIndex(
|
1910
|
-
|
2153
|
+
URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
1911
2154
|
storeOpts.isIndex
|
1912
2155
|
);
|
1913
2156
|
return ret;
|
@@ -1954,52 +2197,61 @@ function registerStoreProtocol(item) {
|
|
1954
2197
|
storeFactory.delete(protocol);
|
1955
2198
|
};
|
1956
2199
|
}
|
2200
|
+
var onceDataStoreFactory = new KeyedResolvOnce2();
|
1957
2201
|
async function dataStoreFactory(loader) {
|
1958
2202
|
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
|
1959
2203
|
const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
|
1960
|
-
|
1961
|
-
|
1962
|
-
|
1963
|
-
|
1964
|
-
|
1965
|
-
|
1966
|
-
|
1967
|
-
|
1968
|
-
|
2204
|
+
return onceDataStoreFactory.get(url.toString()).once(async () => {
|
2205
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2206
|
+
if (!gateway) {
|
2207
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2208
|
+
}
|
2209
|
+
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2210
|
+
gateway: gateway.gateway,
|
2211
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2212
|
+
...loader.ebOpts.keyBag
|
2213
|
+
})
|
2214
|
+
});
|
2215
|
+
return store;
|
1969
2216
|
});
|
1970
|
-
return store;
|
1971
2217
|
}
|
2218
|
+
var onceMetaStoreFactory = new KeyedResolvOnce2();
|
1972
2219
|
async function metaStoreFactory(loader) {
|
1973
2220
|
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
|
1974
2221
|
const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
|
1975
|
-
|
1976
|
-
|
1977
|
-
|
1978
|
-
|
1979
|
-
|
1980
|
-
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
2222
|
+
return onceMetaStoreFactory.get(url.toString()).once(async () => {
|
2223
|
+
sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
|
2224
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2225
|
+
if (!gateway) {
|
2226
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2227
|
+
}
|
2228
|
+
const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
|
2229
|
+
gateway: gateway.gateway,
|
2230
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2231
|
+
...loader.ebOpts.keyBag
|
2232
|
+
})
|
2233
|
+
});
|
2234
|
+
return store;
|
1985
2235
|
});
|
1986
|
-
return store;
|
1987
2236
|
}
|
2237
|
+
var onceRemoteWalFactory = new KeyedResolvOnce2();
|
1988
2238
|
async function remoteWalFactory(loader) {
|
1989
2239
|
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
|
1990
2240
|
const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
|
1991
|
-
|
1992
|
-
|
1993
|
-
|
1994
|
-
|
1995
|
-
|
1996
|
-
|
1997
|
-
|
1998
|
-
|
1999
|
-
|
2000
|
-
|
2241
|
+
return onceRemoteWalFactory.get(url.toString()).once(async () => {
|
2242
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2243
|
+
if (!gateway) {
|
2244
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2245
|
+
}
|
2246
|
+
sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
|
2247
|
+
const store = new WALStoreImpl(loader, url, {
|
2248
|
+
gateway: gateway.gateway,
|
2249
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2250
|
+
...loader.ebOpts.keyBag
|
2251
|
+
})
|
2252
|
+
});
|
2253
|
+
return store;
|
2001
2254
|
});
|
2002
|
-
return store;
|
2003
2255
|
}
|
2004
2256
|
async function testStoreFactory(url, sthis) {
|
2005
2257
|
sthis = ensureSuperLog(sthis, "testStoreFactory");
|
@@ -2038,78 +2290,27 @@ function toStoreRuntime(opts, sthis) {
|
|
2038
2290
|
}
|
2039
2291
|
registerStoreProtocol({
|
2040
2292
|
protocol: "file:",
|
2041
|
-
gateway: async (
|
2042
|
-
const { FileGateway } = await import("./gateway-
|
2043
|
-
return new FileGateway(
|
2293
|
+
gateway: async (sthis) => {
|
2294
|
+
const { FileGateway } = await import("./gateway-VWWKLHUI.js");
|
2295
|
+
return new FileGateway(sthis);
|
2044
2296
|
},
|
2045
|
-
test: async (
|
2046
|
-
const { FileTestStore } = await import("./gateway-
|
2047
|
-
return new FileTestStore(
|
2297
|
+
test: async (sthis) => {
|
2298
|
+
const { FileTestStore } = await import("./gateway-VWWKLHUI.js");
|
2299
|
+
return new FileTestStore(sthis);
|
2048
2300
|
}
|
2049
2301
|
});
|
2050
2302
|
registerStoreProtocol({
|
2051
2303
|
protocol: "indexdb:",
|
2052
|
-
gateway: async (
|
2053
|
-
const { IndexDBGateway } = await import("./gateway-
|
2054
|
-
return new IndexDBGateway(
|
2304
|
+
gateway: async (sthis) => {
|
2305
|
+
const { IndexDBGateway } = await import("./gateway-7OM6OSYK.js");
|
2306
|
+
return new IndexDBGateway(sthis);
|
2055
2307
|
},
|
2056
|
-
test: async (
|
2057
|
-
const { IndexDBTestStore } = await import("./gateway-
|
2058
|
-
return new IndexDBTestStore(
|
2308
|
+
test: async (sthis) => {
|
2309
|
+
const { IndexDBTestStore } = await import("./gateway-7OM6OSYK.js");
|
2310
|
+
return new IndexDBTestStore(sthis);
|
2059
2311
|
}
|
2060
2312
|
});
|
2061
2313
|
|
2062
|
-
// src/blockstore/connection-base.ts
|
2063
|
-
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
2064
|
-
import { MemoryBlockstore as MemoryBlockstore2 } from "@web3-storage/pail/block";
|
2065
|
-
|
2066
|
-
// src/blockstore/task-manager.ts
|
2067
|
-
var TaskManager = class {
|
2068
|
-
constructor(loader) {
|
2069
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
2070
|
-
this.queue = [];
|
2071
|
-
this.isProcessing = false;
|
2072
|
-
this.loader = loader;
|
2073
|
-
this.logger = ensureLogger(loader.sthis, "TaskManager");
|
2074
|
-
}
|
2075
|
-
async handleEvent(eventBlock) {
|
2076
|
-
const cid = eventBlock.cid.toString();
|
2077
|
-
const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
|
2078
|
-
for (const parent of parents) {
|
2079
|
-
this.eventsWeHandled.add(parent);
|
2080
|
-
}
|
2081
|
-
this.queue.push({ cid, eventBlock, retries: 0 });
|
2082
|
-
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
2083
|
-
void this.processQueue();
|
2084
|
-
}
|
2085
|
-
async processQueue() {
|
2086
|
-
if (this.isProcessing) return;
|
2087
|
-
this.isProcessing = true;
|
2088
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2089
|
-
const first = filteredQueue[0];
|
2090
|
-
if (!first) {
|
2091
|
-
return;
|
2092
|
-
}
|
2093
|
-
try {
|
2094
|
-
this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
|
2095
|
-
this.eventsWeHandled.add(first.cid);
|
2096
|
-
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2097
|
-
} catch (err) {
|
2098
|
-
if (first.retries++ > 3) {
|
2099
|
-
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
2100
|
-
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
2101
|
-
}
|
2102
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
2103
|
-
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
2104
|
-
} finally {
|
2105
|
-
this.isProcessing = false;
|
2106
|
-
if (this.queue.length > 0) {
|
2107
|
-
void this.processQueue();
|
2108
|
-
}
|
2109
|
-
}
|
2110
|
-
}
|
2111
|
-
};
|
2112
|
-
|
2113
2314
|
// src/blockstore/store-remote.ts
|
2114
2315
|
async function RemoteDataStore(sthis, name, url, opts) {
|
2115
2316
|
const ds = new DataStoreImpl(sthis, name, url, opts);
|
@@ -2117,7 +2318,7 @@ async function RemoteDataStore(sthis, name, url, opts) {
|
|
2117
2318
|
return ds;
|
2118
2319
|
}
|
2119
2320
|
async function RemoteMetaStore(sthis, name, url, opts) {
|
2120
|
-
const ms = new MetaStoreImpl(sthis, name, url, opts);
|
2321
|
+
const ms = new MetaStoreImpl(sthis, name, url, opts, true);
|
2121
2322
|
await ms.start();
|
2122
2323
|
return ms;
|
2123
2324
|
}
|
@@ -2125,16 +2326,12 @@ async function RemoteMetaStore(sthis, name, url, opts) {
|
|
2125
2326
|
// src/blockstore/connection-base.ts
|
2126
2327
|
var ConnectionBase = class {
|
2127
2328
|
constructor(url, logger) {
|
2128
|
-
// readonly ready: Promise<unknown>;
|
2129
|
-
// todo move to LRU blockstore https://github.com/web3-storage/w3clock/blob/main/src/worker/block.js
|
2130
|
-
this.eventBlocks = new MemoryBlockstore2();
|
2131
|
-
this.parents = [];
|
2132
2329
|
this.loaded = Promise.resolve();
|
2133
2330
|
this.logger = logger;
|
2134
2331
|
this.url = url;
|
2135
2332
|
}
|
2136
2333
|
async refresh() {
|
2137
|
-
await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load(
|
2334
|
+
await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
|
2138
2335
|
await (await throwFalsy(this.loader).WALStore()).process();
|
2139
2336
|
}
|
2140
2337
|
async connect_X({ loader }) {
|
@@ -2145,26 +2342,20 @@ var ConnectionBase = class {
|
|
2145
2342
|
async connectMeta_X({ loader }) {
|
2146
2343
|
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
2147
2344
|
this.loader = loader;
|
2148
|
-
this.taskManager = new TaskManager(loader);
|
2149
2345
|
await this.onConnect();
|
2150
2346
|
const metaUrl = this.url.build().defParam("store", "meta").URI();
|
2151
2347
|
const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
|
2152
2348
|
if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
2153
|
-
const
|
2154
|
-
|
2349
|
+
const dbName = metaUrl.getParam("name");
|
2350
|
+
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
|
2351
|
+
const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
|
2155
2352
|
gateway: gateway.gateway,
|
2156
|
-
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag)
|
2157
|
-
|
2158
|
-
remote.onLoad("main", async (metas) => {
|
2159
|
-
if (metas) {
|
2160
|
-
this.logger.Debug().Any("metas", metas).Bool("loader", this.loader).Msg("connectMeta_X: handleDbMetasFromStore pre");
|
2161
|
-
await throwFalsy(this.loader).handleDbMetasFromStore(metas);
|
2162
|
-
this.logger.Debug().Any("metas", metas).Msg("connectMeta_X: handleDbMetasFromStore post");
|
2163
|
-
}
|
2353
|
+
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
|
2354
|
+
loader
|
2164
2355
|
});
|
2165
2356
|
this.loader.remoteMetaStore = remote;
|
2166
2357
|
this.loaded = this.loader.ready().then(async () => {
|
2167
|
-
remote.load(
|
2358
|
+
remote.load().then(async () => {
|
2168
2359
|
(await throwFalsy(this.loader).WALStore()).process();
|
2169
2360
|
});
|
2170
2361
|
});
|
@@ -2175,28 +2366,14 @@ var ConnectionBase = class {
|
|
2175
2366
|
const dataUrl = this.url.build().defParam("store", "data").URI();
|
2176
2367
|
const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
|
2177
2368
|
if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
2178
|
-
const name = dataUrl.
|
2369
|
+
const name = dataUrl.getParam("name");
|
2370
|
+
if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
|
2179
2371
|
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
2180
2372
|
gateway: gateway.gateway,
|
2181
2373
|
keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
|
2182
2374
|
});
|
2183
2375
|
loader.remoteFileStore = loader.remoteCarStore;
|
2184
2376
|
}
|
2185
|
-
async createEventBlock(bytes) {
|
2186
|
-
const data = {
|
2187
|
-
dbMeta: bytes
|
2188
|
-
};
|
2189
|
-
const event = await EventBlock.create(
|
2190
|
-
data,
|
2191
|
-
this.parents
|
2192
|
-
);
|
2193
|
-
await this.eventBlocks.put(event.cid, event.bytes);
|
2194
|
-
return event;
|
2195
|
-
}
|
2196
|
-
async decodeEventBlock(bytes) {
|
2197
|
-
const event = await decodeEventBlock(bytes);
|
2198
|
-
return event;
|
2199
|
-
}
|
2200
2377
|
// move this stuff to connect
|
2201
2378
|
// async getDashboardURL(compact = true) {
|
2202
2379
|
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
@@ -2490,7 +2667,7 @@ async function doCompact(blockLog, head, logger) {
|
|
2490
2667
|
isCompacting = false;
|
2491
2668
|
}
|
2492
2669
|
async function getBlock(blocks, cidString) {
|
2493
|
-
const block = await blocks.get(
|
2670
|
+
const block = await blocks.get(parse3(cidString));
|
2494
2671
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
2495
2672
|
const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
|
2496
2673
|
return new Block({ cid, value, bytes: block.bytes });
|
@@ -2826,7 +3003,7 @@ var Index = class {
|
|
2826
3003
|
// src/crdt-clock.ts
|
2827
3004
|
import { advance } from "@web3-storage/pail/clock";
|
2828
3005
|
import { root as root2 } from "@web3-storage/pail/crdt";
|
2829
|
-
import { ResolveOnce as
|
3006
|
+
import { ResolveOnce as ResolveOnce4 } from "@adviser/cement";
|
2830
3007
|
|
2831
3008
|
// src/apply-head-queue.ts
|
2832
3009
|
function applyHeadQueue(worker, logger) {
|
@@ -2883,7 +3060,7 @@ var CRDTClock = class {
|
|
2883
3060
|
this.zoomers = /* @__PURE__ */ new Set();
|
2884
3061
|
this.watchers = /* @__PURE__ */ new Set();
|
2885
3062
|
this.emptyWatchers = /* @__PURE__ */ new Set();
|
2886
|
-
this._ready = new
|
3063
|
+
this._ready = new ResolveOnce4();
|
2887
3064
|
this.blockstore = blockstore;
|
2888
3065
|
this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
|
2889
3066
|
this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
|
@@ -2994,7 +3171,7 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
2994
3171
|
var CRDT = class {
|
2995
3172
|
constructor(sthis, name, opts = {}) {
|
2996
3173
|
this.indexers = /* @__PURE__ */ new Map();
|
2997
|
-
this.onceReady = new
|
3174
|
+
this.onceReady = new ResolveOnce5();
|
2998
3175
|
this.sthis = sthis;
|
2999
3176
|
this.name = name;
|
3000
3177
|
this.logger = ensureLogger(sthis, "CRDT");
|
@@ -3060,7 +3237,8 @@ var CRDT = class {
|
|
3060
3237
|
try {
|
3061
3238
|
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
3062
3239
|
} catch (e) {
|
3063
|
-
|
3240
|
+
const ee = e;
|
3241
|
+
throw this.logger.Error().Err(e).Msg(`CRDT is not ready: ${ee.stack}`).AsError();
|
3064
3242
|
}
|
3065
3243
|
});
|
3066
3244
|
}
|
@@ -3114,7 +3292,7 @@ var Database = class {
|
|
3114
3292
|
this._listening = false;
|
3115
3293
|
this._listeners = /* @__PURE__ */ new Set();
|
3116
3294
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
3117
|
-
this._ready = new
|
3295
|
+
this._ready = new ResolveOnce6();
|
3118
3296
|
this.name = name;
|
3119
3297
|
this.opts = opts || this.opts;
|
3120
3298
|
this.sthis = ensureSuperThis(this.opts);
|
@@ -3292,34 +3470,9 @@ function makeName(fnString) {
|
|
3292
3470
|
}
|
3293
3471
|
}
|
3294
3472
|
|
3295
|
-
// src/runtime/index.ts
|
3296
|
-
var runtime_exports = {};
|
3297
|
-
__export(runtime_exports, {
|
3298
|
-
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
3299
|
-
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
3300
|
-
files: () => files_exports,
|
3301
|
-
getFileName: () => getFileName,
|
3302
|
-
getFileSystem: () => getFileSystem,
|
3303
|
-
getPath: () => getPath,
|
3304
|
-
kb: () => key_bag_exports,
|
3305
|
-
kc: () => keyed_crypto_exports,
|
3306
|
-
mf: () => wait_pr_multiformats_exports,
|
3307
|
-
toArrayBuffer: () => toArrayBuffer
|
3308
|
-
});
|
3309
|
-
|
3310
|
-
// src/runtime/wait-pr-multiformats/index.ts
|
3311
|
-
var wait_pr_multiformats_exports = {};
|
3312
|
-
__export(wait_pr_multiformats_exports, {
|
3313
|
-
block: () => block_exports,
|
3314
|
-
codec: () => codec_interface_exports
|
3315
|
-
});
|
3316
|
-
|
3317
|
-
// src/runtime/wait-pr-multiformats/codec-interface.ts
|
3318
|
-
var codec_interface_exports = {};
|
3319
|
-
|
3320
3473
|
// src/version.ts
|
3321
3474
|
var PACKAGE_VERSION = Object.keys({
|
3322
|
-
"0.19.
|
3475
|
+
"0.19.11-dev-dryrun": "xxxx"
|
3323
3476
|
})[0];
|
3324
3477
|
export {
|
3325
3478
|
CRDT,
|
@@ -3335,7 +3488,6 @@ export {
|
|
3335
3488
|
ensureLogger,
|
3336
3489
|
ensureSuperLog,
|
3337
3490
|
ensureSuperThis,
|
3338
|
-
exception2Result,
|
3339
3491
|
exceptionWrapper,
|
3340
3492
|
falsyToUndef,
|
3341
3493
|
fireproof,
|