@atproto/repo 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/car.d.ts +23 -0
- package/dist/car.d.ts.map +1 -0
- package/dist/car.js +217 -0
- package/dist/car.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mst/mst.d.ts +2 -2
- package/dist/mst/mst.d.ts.map +1 -1
- package/dist/mst/mst.js +3 -3
- package/dist/mst/mst.js.map +1 -1
- package/dist/repo.d.ts +1 -1
- package/dist/repo.d.ts.map +1 -1
- package/dist/repo.js +2 -2
- package/dist/repo.js.map +1 -1
- package/dist/sync/consumer.d.ts.map +1 -1
- package/dist/sync/consumer.js +5 -4
- package/dist/sync/consumer.js.map +1 -1
- package/dist/sync/provider.d.ts.map +1 -1
- package/dist/sync/provider.js +27 -22
- package/dist/sync/provider.js.map +1 -1
- package/dist/types.d.ts +11 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/util.d.ts +2 -28
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +1 -80
- package/dist/util.js.map +1 -1
- package/package.json +5 -5
- package/src/car.ts +219 -0
- package/src/index.ts +1 -0
- package/src/mst/mst.ts +4 -4
- package/src/repo.ts +2 -1
- package/src/sync/consumer.ts +5 -4
- package/src/sync/provider.ts +38 -27
- package/src/util.ts +1 -114
- package/tests/car-file-fixtures.json +28 -0
- package/tests/car.test.ts +58 -0
- package/tests/sync.test.ts +7 -7
- package/tsconfig.build.tsbuildinfo +1 -1
- package/tsconfig.tests.tsbuildinfo +1 -1
- package/tests/util.test.ts +0 -21
package/dist/sync/consumer.js
CHANGED
|
@@ -24,6 +24,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
26
|
exports.RepoVerificationError = exports.verifyRecords = exports.verifyProofs = exports.verifyDiff = exports.verifyDiffCar = exports.verifyRepo = exports.verifyRepoCar = void 0;
|
|
27
|
+
const car_1 = require("../car");
|
|
27
28
|
const data_diff_1 = require("../data-diff");
|
|
28
29
|
const mst_1 = require("../mst");
|
|
29
30
|
const readable_repo_1 = require("../readable-repo");
|
|
@@ -31,7 +32,7 @@ const storage_1 = require("../storage");
|
|
|
31
32
|
const types_1 = require("../types");
|
|
32
33
|
const util = __importStar(require("../util"));
|
|
33
34
|
const verifyRepoCar = async (carBytes, did, signingKey) => {
|
|
34
|
-
const car = await
|
|
35
|
+
const car = await (0, car_1.readCarWithRoot)(carBytes);
|
|
35
36
|
return (0, exports.verifyRepo)(car.blocks, car.root, did, signingKey);
|
|
36
37
|
};
|
|
37
38
|
exports.verifyRepoCar = verifyRepoCar;
|
|
@@ -45,7 +46,7 @@ const verifyRepo = async (blocks, head, did, signingKey, opts) => {
|
|
|
45
46
|
};
|
|
46
47
|
exports.verifyRepo = verifyRepo;
|
|
47
48
|
const verifyDiffCar = async (repo, carBytes, did, signingKey, opts) => {
|
|
48
|
-
const car = await
|
|
49
|
+
const car = await (0, car_1.readCarWithRoot)(carBytes);
|
|
49
50
|
return (0, exports.verifyDiff)(repo, car.blocks, car.root, did, signingKey, opts);
|
|
50
51
|
};
|
|
51
52
|
exports.verifyDiffCar = verifyDiffCar;
|
|
@@ -104,7 +105,7 @@ const verifyRepoRoot = async (storage, head, did, signingKey) => {
|
|
|
104
105
|
return repo;
|
|
105
106
|
};
|
|
106
107
|
const verifyProofs = async (proofs, claims, did, didKey) => {
|
|
107
|
-
const car = await
|
|
108
|
+
const car = await (0, car_1.readCarWithRoot)(proofs);
|
|
108
109
|
const blockstore = new storage_1.MemoryBlockstore(car.blocks);
|
|
109
110
|
const commit = await blockstore.readObj(car.root, types_1.def.commit);
|
|
110
111
|
if (commit.did !== did) {
|
|
@@ -141,7 +142,7 @@ const verifyProofs = async (proofs, claims, did, didKey) => {
|
|
|
141
142
|
};
|
|
142
143
|
exports.verifyProofs = verifyProofs;
|
|
143
144
|
const verifyRecords = async (proofs, did, signingKey) => {
|
|
144
|
-
const car = await
|
|
145
|
+
const car = await (0, car_1.readCarWithRoot)(proofs);
|
|
145
146
|
const blockstore = new storage_1.MemoryBlockstore(car.blocks);
|
|
146
147
|
const commit = await blockstore.readObj(car.root, types_1.def.commit);
|
|
147
148
|
if (commit.did !== did) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consumer.js","sourceRoot":"","sources":["../../src/sync/consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,4CAAuC;AACvC,gCAA4B;AAC5B,oDAA+C;AAC/C,wCAA8E;AAC9E,oCAMiB;AACjB,8CAA+B;AAExB,MAAM,aAAa,GAAG,KAAK,EAChC,QAAoB,EACpB,GAAY,EACZ,UAAmB,EACI,EAAE;IACzB,MAAM,GAAG,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"consumer.js","sourceRoot":"","sources":["../../src/sync/consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,gCAAwC;AACxC,4CAAuC;AACvC,gCAA4B;AAC5B,oDAA+C;AAC/C,wCAA8E;AAC9E,oCAMiB;AACjB,8CAA+B;AAExB,MAAM,aAAa,GAAG,KAAK,EAChC,QAAoB,EACpB,GAAY,EACZ,UAAmB,EACI,EAAE;IACzB,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAe,EAAC,QAAQ,CAAC,CAAA;IAC3C,OAAO,IAAA,kBAAU,EAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,UAAU,CAAC,CAAA;AAC1D,CAAC,CAAA;AAPY,QAAA,aAAa,iBAOzB;AAEM,MAAM,UAAU,GAAG,KAAK,EAC7B,MAAgB,EAChB,IAAS,EACT,GAAY,EACZ,UAAmB,EACnB,IAAiC,EACV,EAAE;IACzB,MAAM,IAAI,GAAG,MAAM,IAAA,kBAAU,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;IACxE,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC/C,OAAO;QACL,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAA;AACH,CAAC,CAAA;AAbY,QAAA,UAAU,cAatB;AAEM,MAAM,aAAa,GAAG,KAAK,EAChC,IAAyB,EACzB,QAAoB,EACpB,GAAY,EACZ,UAAmB,EACnB,IAAiC,EACV,EAAE;IACzB,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAe,EAAC,QAAQ,CAAC,CAAA;IAC3C,OAAO,IAAA,kBAAU,EAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;AACtE,CAAC,CAAA;AATY,QAAA,aAAa,iBASzB;AAEM,MAAM,UAAU,GAAG,KAAK,EAC7B,IAAyB,EACzB,YAAsB,EACtB,UAAe,EACf,GAAY,EACZ,UAAmB,EACnB,IAAiC,EACV,EAAE;IACzB,MAAM,EAAE,YAAY,GAAG,IAAI,EAAE,GAAG,IAAI,IAAI,EAAE,CAAA;IAC1C,MAAM,aAAa,GAAG,IAAI,0BAAgB,CAAC,YAAY,CAAC,CAAA;IACxD,MAAM,aAAa,GAAG,IAAI;QACxB,CAAC,CAAC,IAAI,qBAAW,CAAC,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC;QAC9C,CAAC,CAAC,aAAa,CAAA;IACjB,MAAM,OAAO,GAAG,MAAM,cAAc,CAClC,aAAa,EACb,UAAU,EACV,GAAG,EACH,UAAU,CACX,CAAA;IACD,MAAM,IAAI,GAAG,MAAM,oBAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,CAAA;IAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAA;IACnC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAA;IAC9D,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAA;IAC3D,CAAC;IACD,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;IACpC,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACrD,yCAAyC;IACzC,IAAI,IAAI,EAAE,CAAC;QACT,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAC7B,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC;IACH,CAAC;IACD,OAAO;QACL,MAAM;QACN,MAAM,EAAE;YACN,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG;YACvB,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI;YACvB,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,IAAI;YAC/B,SAAS;YACT,cAAc,EAAE,SAAS;YACzB,WAAW;SACZ;KACF,CAAA;AACH,CAAC,CAAA;AAjDY,QAAA,UAAU,cAiDtB;AAED,sDAAsD;AACtD,MAAM,cAAc,GAAG,KAAK,EAC1B,OAA2B,EAC3B,IAAS,EACT,GAAY,EACZ,UAAmB,EACI,EAAE;IACzB,MAAM,IAAI,GAAG,MAAM,4BAAY,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IACnD,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;QAC1C,MAAM,IAAI,qBAAqB,CAAC,qBAAqB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IAClE,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;QACpE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,qBAAqB,CAC7B,gCAAgC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CACtD,CAAA;QACH,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAEM,MAAM,YAAY,GAAG,KAAK,EAC/B,MAAkB,EAClB,MAAwB,EACxB,GAAW,EACX,MAAc,EACyD,EAAE;IACzE,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAe,EAAC,MAAM,CAAC,CAAA;IACzC,MAAM,UAAU,GAAG,IAAI,0BAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACnD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;IAC7D,IAAI,MAAM,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,qBAAqB,CAAC,qBAAqB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,qBAAqB,CAC7B,gCAAgC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CACtD,CAAA;IACH,CAAC;IACD,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;IAC7C,MAAM,QAAQ,GAAqB,EAAE,CAAA;IACrC,MAAM,UAAU,GAAqB,EAAE,CAAA;IACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,GAAG,CACzB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CACjD,CAAA;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,WAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACtE,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YACvB,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACxB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAA;AACjC,CAAC,CAAA;AAzCY,QAAA,YAAY,gBAyCxB;AAEM,MAAM,aAAa,GAAG,KAAK,EAChC,MAAkB,EAClB,GAAW,EACX,UAAkB,EACM,EAAE;IAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAe,EAAC,MAAM,CAAC,CAAA;IACzC,MAAM,UAAU,GAAG,IAAI,0BAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACnD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;IAC7D,IAAI,MAAM,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,qBAAqB,CAAC,qBAAqB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,qBAAqB,CAC7B,gCAAgC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CACtD,CAAA;IACH,CAAC;IACD,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;IAE7C,MAAM,OAAO,GAAkB,EAAE,CAAA;IACjC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAA;IAC1C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC7D,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC;gBACX,UAAU;gBACV,IAAI;gBACJ,MAAM;aACP,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAjCY,QAAA,aAAa,iBAiCzB;AAED,MAAa,qBAAsB,SAAQ,KAAK;CAAG;AAAnD,sDAAmD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/sync/provider.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/sync/provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAKtC,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAO,MAAM,UAAU,CAAA;AAM1C,eAAO,MAAM,WAAW,YACb,WAAW,aACT,GAAG,KACb,aAAa,CAAC,UAAU,CAE1B,CAAA;AAcD,eAAO,MAAM,UAAU,YACZ,kBAAkB,aAChB,GAAG,SACP,UAAU,EAAE,KAClB,aAAa,CAAC,UAAU,CAK1B,CAAA"}
|
package/dist/sync/provider.js
CHANGED
|
@@ -24,6 +24,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
26
|
exports.getRecords = exports.getFullRepo = void 0;
|
|
27
|
+
const car_1 = require("../car");
|
|
27
28
|
const cid_set_1 = require("../cid-set");
|
|
28
29
|
const error_1 = require("../error");
|
|
29
30
|
const mst_1 = require("../mst");
|
|
@@ -32,33 +33,37 @@ const util = __importStar(require("../util"));
|
|
|
32
33
|
// Full Repo
|
|
33
34
|
// -------------
|
|
34
35
|
const getFullRepo = (storage, commitCid) => {
|
|
35
|
-
return
|
|
36
|
-
const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
|
|
37
|
-
await car.put({ cid: commitCid, bytes: commit.bytes });
|
|
38
|
-
const mst = mst_1.MST.load(storage, commit.obj.data);
|
|
39
|
-
await mst.writeToCarStream(car);
|
|
40
|
-
});
|
|
36
|
+
return (0, car_1.writeCarStream)(commitCid, iterateFullRepo(storage, commitCid));
|
|
41
37
|
};
|
|
42
38
|
exports.getFullRepo = getFullRepo;
|
|
39
|
+
async function* iterateFullRepo(storage, commitCid) {
|
|
40
|
+
const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
|
|
41
|
+
yield { cid: commitCid, bytes: commit.bytes };
|
|
42
|
+
const mst = mst_1.MST.load(storage, commit.obj.data);
|
|
43
|
+
for await (const block of mst.carBlockStream()) {
|
|
44
|
+
yield block;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
43
47
|
// Narrow slices
|
|
44
48
|
// -------------
|
|
45
49
|
const getRecords = (storage, commitCid, paths) => {
|
|
46
|
-
return
|
|
47
|
-
const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
|
|
48
|
-
await car.put({ cid: commitCid, bytes: commit.bytes });
|
|
49
|
-
const mst = mst_1.MST.load(storage, commit.obj.data);
|
|
50
|
-
const cidsForPaths = await Promise.all(paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))));
|
|
51
|
-
const allCids = cidsForPaths.reduce((acc, cur) => {
|
|
52
|
-
return acc.addSet(new cid_set_1.CidSet(cur));
|
|
53
|
-
}, new cid_set_1.CidSet());
|
|
54
|
-
const found = await storage.getBlocks(allCids.toList());
|
|
55
|
-
if (found.missing.length > 0) {
|
|
56
|
-
throw new error_1.MissingBlocksError('writeRecordsToCarStream', found.missing);
|
|
57
|
-
}
|
|
58
|
-
for (const block of found.blocks.entries()) {
|
|
59
|
-
await car.put(block);
|
|
60
|
-
}
|
|
61
|
-
});
|
|
50
|
+
return (0, car_1.writeCarStream)(commitCid, iterateRecordBlocks(storage, commitCid, paths));
|
|
62
51
|
};
|
|
63
52
|
exports.getRecords = getRecords;
|
|
53
|
+
async function* iterateRecordBlocks(storage, commitCid, paths) {
|
|
54
|
+
const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
|
|
55
|
+
yield { cid: commitCid, bytes: commit.bytes };
|
|
56
|
+
const mst = mst_1.MST.load(storage, commit.obj.data);
|
|
57
|
+
const cidsForPaths = await Promise.all(paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))));
|
|
58
|
+
const allCids = cidsForPaths.reduce((acc, cur) => {
|
|
59
|
+
return acc.addSet(new cid_set_1.CidSet(cur));
|
|
60
|
+
}, new cid_set_1.CidSet());
|
|
61
|
+
const found = await storage.getBlocks(allCids.toList());
|
|
62
|
+
if (found.missing.length > 0) {
|
|
63
|
+
throw new error_1.MissingBlocksError('writeRecordsToCarStream', found.missing);
|
|
64
|
+
}
|
|
65
|
+
for (const block of found.blocks.entries()) {
|
|
66
|
+
yield block;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
64
69
|
//# sourceMappingURL=provider.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/sync/provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/sync/provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,gCAAuC;AACvC,wCAAmC;AACnC,oCAA6C;AAC7C,gCAA4B;AAE5B,oCAA0C;AAC1C,8CAA+B;AAE/B,YAAY;AACZ,gBAAgB;AAET,MAAM,WAAW,GAAG,CACzB,OAAoB,EACpB,SAAc,EACa,EAAE;IAC7B,OAAO,IAAA,oBAAc,EAAC,SAAS,EAAE,eAAe,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAA;AACvE,CAAC,CAAA;AALY,QAAA,WAAW,eAKvB;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAAoB,EAAE,SAAc;IAClE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;IACnE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAA;IAC7C,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,gBAAgB;AAChB,gBAAgB;AAET,MAAM,UAAU,GAAG,CACxB,OAA2B,EAC3B,SAAc,EACd,KAAmB,EACQ,EAAE;IAC7B,OAAO,IAAA,oBAAc,EACnB,SAAS,EACT,mBAAmB,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAC/C,CAAA;AACH,CAAC,CAAA;AATY,QAAA,UAAU,cAStB;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAA2B,EAC3B,SAAc,EACd,KAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;IACnE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAA;IAC7C,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAC5E,CAAA;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC/C,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,gBAAM,CAAC,GAAG,CAAC,CAAC,CAAA;IACpC,CAAC,EAAE,IAAI,gBAAM,EAAE,CAAC,CAAA;IAChB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,0BAAkB,CAAC,yBAAyB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IACxE,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -204,6 +204,16 @@ export declare const schema: {
|
|
|
204
204
|
prev?: any;
|
|
205
205
|
}>]>;
|
|
206
206
|
cid: z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>;
|
|
207
|
+
carHeader: z.ZodObject<{
|
|
208
|
+
version: z.ZodLiteral<1>;
|
|
209
|
+
roots: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>, "many">;
|
|
210
|
+
}, "strip", z.ZodTypeAny, {
|
|
211
|
+
version: 1;
|
|
212
|
+
roots: CID[];
|
|
213
|
+
}, {
|
|
214
|
+
version: 1;
|
|
215
|
+
roots: any[];
|
|
216
|
+
}>;
|
|
207
217
|
bytes: z.ZodType<Uint8Array, z.ZodTypeDef, Uint8Array>;
|
|
208
218
|
string: z.ZodString;
|
|
209
219
|
array: z.ZodArray<z.ZodUnknown, "many">;
|
|
@@ -283,6 +293,7 @@ export declare const def: {
|
|
|
283
293
|
}>]>;
|
|
284
294
|
};
|
|
285
295
|
cid: import("@atproto/common-web/dist/check").Def<CID>;
|
|
296
|
+
carHeader: import("@atproto/common-web/dist/check").Def<import("@atproto/common").CarHeader>;
|
|
286
297
|
bytes: import("@atproto/common-web/dist/check").Def<Uint8Array>;
|
|
287
298
|
string: import("@atproto/common-web/dist/check").Def<string>;
|
|
288
299
|
map: import("@atproto/common-web/dist/check").Def<Record<string, unknown>>;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAKlC,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,KAAK,CAAA;CAAE,CAAA;AAE7E,QAAA,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;EAOV,CAAA;AACF,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAA;AAE3C,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAA;AAE3D,QAAA,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAGnB,CAAA;AACF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAA;AAE7D,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAKlC,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,KAAK,CAAA;CAAE,CAAA;AAE7E,QAAA,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;EAOV,CAAA;AACF,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAA;AAE3C,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAA;AAE3D,QAAA,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAGnB,CAAA;AACF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAA;AAE7D,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;SA/BP,EAAG,UAAU,CAAC,EAAE,UAAU,CAAC,EAAE,MAAM;eAAsC,EAAG,SACvF;iBAAmB,EAAG,UACrB;eACgB,EAAG,QAAQ,CAAC,EAAE,UAAU,CAAE,EAAC,UAAU,CAAC,EAAE,MAAM;gBAGzD,EAAG,UACT;;;;;;;WAKD,EACA,OAAO,aAAY,EAAG,UAAU;YAEnB,EAAG,SAAS;WACtB,EAAI,QAAO,CAAC,EAAE,UACT;SACN,EAAG,SAAS,CAAC,EAAE,SACX,EAAC,EAAG,UAAU;aACf,EAAG,UAAU;CAgBjB,CAAA;AAED,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAUf,CAAA;AAKD,oBAAY,aAAa;IACvB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,cAAc,CAAA;AAE5E,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,GAAG,CAAA;IACT,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,mBAAmB,GAC3B,oBAAoB,GACpB,oBAAoB,GACpB,oBAAoB,CAAA;AAExB,MAAM,MAAM,QAAQ,GAAG,mBAAmB,EAAE,EAAE,CAAA;AAK9C,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,GAAG,CAAA;IACR,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,IAAI,EAAE,GAAG,GAAG,IAAI,CAAA;IAChB,SAAS,EAAE,QAAQ,CAAA;IACnB,cAAc,EAAE,QAAQ,CAAA;IACxB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAED,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG;IACpC,GAAG,EAAE,aAAa,EAAE,CAAA;CACrB,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;AAC3D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAE7D,MAAM,MAAM,iBAAiB,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,CAAA;AAC/D,MAAM,MAAM,0BAA0B,GAAG,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAA;AAC1E,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAA;AAE7E,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;AAEnD,MAAM,MAAM,UAAU,GAAG;IACvB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,GAAG,IAAI,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,GAAG,IAAI,CAAA;CAC1B,CAAA;AAKD,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,mBAAmB,EAAE,CAAA;IAC7B,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC/B,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,QAAQ,GAAG;IACrB,GAAG,EAAE,GAAG,CAAA;IACR,KAAK,EAAE,UAAU,CAAA;CAClB,CAAA"}
|
package/dist/util.d.ts
CHANGED
|
@@ -1,33 +1,7 @@
|
|
|
1
|
-
import { Readable } from 'node:stream';
|
|
2
|
-
import { CarBlockIterator } from '@ipld/car/iterator';
|
|
3
|
-
import { BlockWriter } from '@ipld/car/writer';
|
|
4
|
-
import { CID } from 'multiformats/cid';
|
|
5
1
|
import { Keypair } from '@atproto/crypto';
|
|
6
2
|
import { LexValue, RepoRecord } from '@atproto/lexicon';
|
|
7
|
-
import { BlockMap } from './block-map';
|
|
8
3
|
import { DataDiff } from './data-diff';
|
|
9
|
-
import {
|
|
10
|
-
export declare function verifyIncomingCarBlocks(car: AsyncIterable<CarBlock>): AsyncIterable<CarBlock>;
|
|
11
|
-
export declare function writeCarStream(root: CID | null, fn: (car: BlockWriter) => Promise<void>): Readable;
|
|
12
|
-
export declare function writeCar(root: CID | null, fn: (car: BlockWriter) => Promise<void>): AsyncIterable<Uint8Array>;
|
|
13
|
-
export declare const blocksToCarStream: (root: CID | null, blocks: BlockMap) => AsyncIterable<Uint8Array>;
|
|
14
|
-
export declare const blocksToCarFile: (root: CID | null, blocks: BlockMap) => Promise<Uint8Array>;
|
|
15
|
-
export declare const carToBlocks: (car: CarBlockIterator) => Promise<{
|
|
16
|
-
roots: CID[];
|
|
17
|
-
blocks: BlockMap;
|
|
18
|
-
}>;
|
|
19
|
-
export declare const readCar: (bytes: Uint8Array) => Promise<{
|
|
20
|
-
roots: CID[];
|
|
21
|
-
blocks: BlockMap;
|
|
22
|
-
}>;
|
|
23
|
-
export declare const readCarStream: (stream: AsyncIterable<Uint8Array>) => Promise<{
|
|
24
|
-
roots: CID[];
|
|
25
|
-
blocks: BlockMap;
|
|
26
|
-
}>;
|
|
27
|
-
export declare const readCarWithRoot: (bytes: Uint8Array) => Promise<{
|
|
28
|
-
root: CID;
|
|
29
|
-
blocks: BlockMap;
|
|
30
|
-
}>;
|
|
4
|
+
import { Commit, LegacyV2Commit, RecordCreateDescript, RecordPath, RecordWriteDescript, UnsignedCommit } from './types';
|
|
31
5
|
export declare const diffToWriteDescripts: (diff: DataDiff) => Promise<RecordWriteDescript[]>;
|
|
32
6
|
export declare const ensureCreates: (descripts: RecordWriteDescript[]) => RecordCreateDescript[];
|
|
33
7
|
export declare const parseDataKey: (key: string) => RecordPath;
|
|
@@ -37,6 +11,6 @@ export declare const signCommit: (unsigned: UnsignedCommit, keypair: Keypair) =>
|
|
|
37
11
|
export declare const verifyCommitSig: (commit: Commit, didKey: string) => Promise<boolean>;
|
|
38
12
|
export declare const cborToLex: (val: Uint8Array) => LexValue;
|
|
39
13
|
export declare const cborToLexRecord: (val: Uint8Array) => RepoRecord;
|
|
40
|
-
export declare const cidForRecord: (val: LexValue) => Promise<CID>;
|
|
14
|
+
export declare const cidForRecord: (val: LexValue) => Promise<import("multiformats/cid").CID>;
|
|
41
15
|
export declare const ensureV3Commit: (commit: LegacyV2Commit | Commit) => Commit;
|
|
42
16
|
//# sourceMappingURL=util.d.ts.map
|
package/dist/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAwB,MAAM,kBAAkB,CAAA;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EACL,MAAM,EACN,cAAc,EACd,oBAAoB,EAEpB,UAAU,EAEV,mBAAmB,EACnB,cAAc,EAEf,MAAM,SAAS,CAAA;AAEhB,eAAO,MAAM,oBAAoB,SACzB,QAAQ,KACb,OAAO,CAAC,mBAAmB,EAAE,CA+B/B,CAAA;AAED,eAAO,MAAM,aAAa,cACb,mBAAmB,EAAE,KAC/B,oBAAoB,EAUtB,CAAA;AAED,eAAO,MAAM,YAAY,QAAS,MAAM,KAAG,UAI1C,CAAA;AAED,eAAO,MAAM,aAAa,eAAgB,MAAM,QAAQ,MAAM,KAAG,MAEhE,CAAA;AAED,eAAO,MAAM,SAAS,MAAO,MAAM,KAAK,MAAM,KAAG,OAEhD,CAAA;AAED,eAAO,MAAM,UAAU,aACX,cAAc,WACf,OAAO,KACf,OAAO,CAAC,MAAM,CAOhB,CAAA;AAED,eAAO,MAAM,eAAe,WAClB,MAAM,UACN,MAAM,KACb,OAAO,CAAC,OAAO,CAIjB,CAAA;AAED,eAAO,MAAM,SAAS,QAAS,UAAU,KAAG,QAE3C,CAAA;AAED,eAAO,MAAM,eAAe,QAAS,UAAU,KAAG,UAMjD,CAAA;AAED,eAAO,MAAM,YAAY,QAAe,QAAQ,4CAE/C,CAAA;AAED,eAAO,MAAM,cAAc,WAAY,cAAc,GAAG,MAAM,KAAG,MAUhE,CAAA"}
|
package/dist/util.js
CHANGED
|
@@ -23,91 +23,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.ensureV3Commit = exports.cidForRecord = exports.cborToLexRecord = exports.cborToLex = exports.verifyCommitSig = exports.signCommit = exports.metaEqual = exports.formatDataKey = exports.parseDataKey = exports.ensureCreates = exports.diffToWriteDescripts =
|
|
27
|
-
exports.verifyIncomingCarBlocks = verifyIncomingCarBlocks;
|
|
28
|
-
exports.writeCarStream = writeCarStream;
|
|
29
|
-
exports.writeCar = writeCar;
|
|
30
|
-
const promises_1 = require("node:timers/promises");
|
|
31
|
-
const iterator_1 = require("@ipld/car/iterator");
|
|
32
|
-
const writer_1 = require("@ipld/car/writer");
|
|
26
|
+
exports.ensureV3Commit = exports.cidForRecord = exports.cborToLexRecord = exports.cborToLex = exports.verifyCommitSig = exports.signCommit = exports.metaEqual = exports.formatDataKey = exports.parseDataKey = exports.ensureCreates = exports.diffToWriteDescripts = void 0;
|
|
33
27
|
const cbor = __importStar(require("@ipld/dag-cbor"));
|
|
34
28
|
const common_1 = require("@atproto/common");
|
|
35
29
|
const crypto = __importStar(require("@atproto/crypto"));
|
|
36
30
|
const lexicon_1 = require("@atproto/lexicon");
|
|
37
|
-
const block_map_1 = require("./block-map");
|
|
38
31
|
const types_1 = require("./types");
|
|
39
|
-
async function* verifyIncomingCarBlocks(car) {
|
|
40
|
-
for await (const block of car) {
|
|
41
|
-
await (0, common_1.verifyCidForBytes)(block.cid, block.bytes);
|
|
42
|
-
yield block;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
// we have to turn the car writer output into a stream in order to properly handle errors
|
|
46
|
-
function writeCarStream(root, fn) {
|
|
47
|
-
const { writer, out } = root !== null ? writer_1.CarWriter.create(root) : writer_1.CarWriter.create();
|
|
48
|
-
const stream = (0, common_1.byteIterableToStream)(out);
|
|
49
|
-
fn(writer)
|
|
50
|
-
.catch((err) => {
|
|
51
|
-
stream.destroy(err);
|
|
52
|
-
})
|
|
53
|
-
.finally(() => writer.close());
|
|
54
|
-
return stream;
|
|
55
|
-
}
|
|
56
|
-
async function* writeCar(root, fn) {
|
|
57
|
-
const stream = writeCarStream(root, fn);
|
|
58
|
-
for await (const chunk of stream) {
|
|
59
|
-
yield chunk;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
const blocksToCarStream = (root, blocks) => {
|
|
63
|
-
return writeCar(root, async (writer) => {
|
|
64
|
-
for (const entry of blocks.entries()) {
|
|
65
|
-
await writer.put(entry);
|
|
66
|
-
}
|
|
67
|
-
});
|
|
68
|
-
};
|
|
69
|
-
exports.blocksToCarStream = blocksToCarStream;
|
|
70
|
-
const blocksToCarFile = (root, blocks) => {
|
|
71
|
-
const carStream = (0, exports.blocksToCarStream)(root, blocks);
|
|
72
|
-
return (0, common_1.streamToBuffer)(carStream);
|
|
73
|
-
};
|
|
74
|
-
exports.blocksToCarFile = blocksToCarFile;
|
|
75
|
-
const carToBlocks = async (car) => {
|
|
76
|
-
const roots = await car.getRoots();
|
|
77
|
-
const blocks = new block_map_1.BlockMap();
|
|
78
|
-
for await (const block of verifyIncomingCarBlocks(car)) {
|
|
79
|
-
blocks.set(block.cid, block.bytes);
|
|
80
|
-
// break up otherwise "synchronous" work in car parsing
|
|
81
|
-
await (0, promises_1.setImmediate)();
|
|
82
|
-
}
|
|
83
|
-
return {
|
|
84
|
-
roots,
|
|
85
|
-
blocks,
|
|
86
|
-
};
|
|
87
|
-
};
|
|
88
|
-
exports.carToBlocks = carToBlocks;
|
|
89
|
-
const readCar = async (bytes) => {
|
|
90
|
-
const car = await iterator_1.CarBlockIterator.fromBytes(bytes);
|
|
91
|
-
return (0, exports.carToBlocks)(car);
|
|
92
|
-
};
|
|
93
|
-
exports.readCar = readCar;
|
|
94
|
-
const readCarStream = async (stream) => {
|
|
95
|
-
const car = await iterator_1.CarBlockIterator.fromIterable(stream);
|
|
96
|
-
return (0, exports.carToBlocks)(car);
|
|
97
|
-
};
|
|
98
|
-
exports.readCarStream = readCarStream;
|
|
99
|
-
const readCarWithRoot = async (bytes) => {
|
|
100
|
-
const { roots, blocks } = await (0, exports.readCar)(bytes);
|
|
101
|
-
if (roots.length !== 1) {
|
|
102
|
-
throw new Error(`Expected one root, got ${roots.length}`);
|
|
103
|
-
}
|
|
104
|
-
const root = roots[0];
|
|
105
|
-
return {
|
|
106
|
-
root,
|
|
107
|
-
blocks,
|
|
108
|
-
};
|
|
109
|
-
};
|
|
110
|
-
exports.readCarWithRoot = readCarWithRoot;
|
|
111
32
|
const diffToWriteDescripts = (diff) => {
|
|
112
33
|
return Promise.all([
|
|
113
34
|
...diff.addList().map(async (add) => {
|
package/dist/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qDAAsC;AACtC,4CAA4E;AAC5E,wDAAyC;AAEzC,8CAA6E;AAE7E,mCAUgB;AAET,MAAM,oBAAoB,GAAG,CAClC,IAAc,EACkB,EAAE;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC;QACjB,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAClC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACrC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,IAAI;aACS,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/B,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAA;AAjCY,QAAA,oBAAoB,wBAiChC;AAEM,MAAM,aAAa,GAAG,CAC3B,SAAgC,EACR,EAAE;IAC1B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,KAAK,qBAAa,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAZY,QAAA,aAAa,iBAYzB;AAEM,MAAM,YAAY,GAAG,CAAC,GAAW,EAAc,EAAE;IACtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAA;IACrE,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;AACjD,CAAC,CAAA;AAJY,QAAA,YAAY,gBAIxB;AAEM,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,IAAY,EAAU,EAAE;IACxE,OAAO,UAAU,GAAG,GAAG,GAAG,IAAI,CAAA;AAChC,CAAC,CAAA;AAFY,QAAA,aAAa,iBAEzB;AAEM,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS,EAAW,EAAE;IACzD,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAA;AACnD,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,UAAU,GAAG,KAAK,EAC7B,QAAwB,EACxB,OAAgB,EACC,EAAE;IACnB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACvC,OAAO;QACL,GAAG,QAAQ;QACX,GAAG;KACJ,CAAA;AACH,CAAC,CAAA;AAVY,QAAA,UAAU,cAUtB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,MAAc,EACd,MAAc,EACI,EAAE;IACpB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAA;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACjC,OAAO,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;AACrD,CAAC,CAAA;AAPY,QAAA,eAAe,mBAO3B;AAEM,MAAM,SAAS,GAAG,CAAC,GAAe,EAAY,EAAE;IACrD,OAAO,IAAA,mBAAS,EAAC,IAAA,mBAAU,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,eAAe,GAAG,CAAC,GAAe,EAAc,EAAE;IAC7D,MAAM,MAAM,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,CAAA;IAC7B,IAAI,CAAC,cAAK,CAAC,EAAE,CAAC,MAAM,EAAE,eAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,YAAY,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;IAClD,OAAO,IAAA,mBAAU,EAAC,IAAA,mBAAS,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,YAAY,gBAExB;AAEM,MAAM,cAAc,GAAG,CAAC,MAA+B,EAAU,EAAE;IACxE,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,MAAM,CAAA;IACf,CAAC;SAAM,CAAC;QACN,OAAO;YACL,GAAG,MAAM;YACT,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,YAAG,CAAC,OAAO,EAAE;SACjC,CAAA;IACH,CAAC;AACH,CAAC,CAAA;AAVY,QAAA,cAAc,kBAU1B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atproto/repo",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "atproto repo and MST implementation",
|
|
6
6
|
"keywords": [
|
|
@@ -19,15 +19,15 @@
|
|
|
19
19
|
"main": "dist/index.js",
|
|
20
20
|
"types": "dist/index.d.ts",
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@ipld/car": "^3.2.3",
|
|
23
22
|
"@ipld/dag-cbor": "^7.0.0",
|
|
24
23
|
"multiformats": "^9.9.0",
|
|
25
24
|
"uint8arrays": "3.0.0",
|
|
25
|
+
"varint": "^6.0.0",
|
|
26
26
|
"zod": "^3.23.8",
|
|
27
|
-
"@atproto/common": "^0.4.
|
|
28
|
-
"@atproto/common-web": "^0.4.
|
|
27
|
+
"@atproto/common": "^0.4.10",
|
|
28
|
+
"@atproto/common-web": "^0.4.1",
|
|
29
29
|
"@atproto/crypto": "^0.4.4",
|
|
30
|
-
"@atproto/lexicon": "^0.4.
|
|
30
|
+
"@atproto/lexicon": "^0.4.10"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"jest": "^28.1.2",
|
package/src/car.ts
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import * as cbor from '@ipld/dag-cbor'
|
|
2
|
+
import { CID } from 'multiformats/cid'
|
|
3
|
+
import * as ui8 from 'uint8arrays'
|
|
4
|
+
import * as varint from 'varint'
|
|
5
|
+
import {
|
|
6
|
+
check,
|
|
7
|
+
parseCidFromBytes,
|
|
8
|
+
schema,
|
|
9
|
+
streamToBuffer,
|
|
10
|
+
verifyCidForBytes,
|
|
11
|
+
} from '@atproto/common'
|
|
12
|
+
import { BlockMap } from './block-map'
|
|
13
|
+
import { CarBlock } from './types'
|
|
14
|
+
|
|
15
|
+
export async function* writeCarStream(
|
|
16
|
+
root: CID | null,
|
|
17
|
+
blocks: AsyncIterable<CarBlock>,
|
|
18
|
+
): AsyncIterable<Uint8Array> {
|
|
19
|
+
const header = new Uint8Array(
|
|
20
|
+
cbor.encode({
|
|
21
|
+
version: 1,
|
|
22
|
+
roots: root ? [root] : [],
|
|
23
|
+
}),
|
|
24
|
+
)
|
|
25
|
+
yield new Uint8Array(varint.encode(header.byteLength))
|
|
26
|
+
yield header
|
|
27
|
+
for await (const block of blocks) {
|
|
28
|
+
yield new Uint8Array(
|
|
29
|
+
varint.encode(block.cid.bytes.byteLength + block.bytes.byteLength),
|
|
30
|
+
)
|
|
31
|
+
yield block.cid.bytes
|
|
32
|
+
yield block.bytes
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export const blocksToCarFile = (
|
|
37
|
+
root: CID | null,
|
|
38
|
+
blocks: BlockMap,
|
|
39
|
+
): Promise<Uint8Array> => {
|
|
40
|
+
const carStream = blocksToCarStream(root, blocks)
|
|
41
|
+
return streamToBuffer(carStream)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export const blocksToCarStream = (
|
|
45
|
+
root: CID | null,
|
|
46
|
+
blocks: BlockMap,
|
|
47
|
+
): AsyncIterable<Uint8Array> => {
|
|
48
|
+
return writeCarStream(root, iterateBlocks(blocks))
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function* iterateBlocks(blocks: BlockMap) {
|
|
52
|
+
for (const entry of blocks.entries()) {
|
|
53
|
+
yield { cid: entry.cid, bytes: entry.bytes }
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const readCar = async (
|
|
58
|
+
bytes: Uint8Array,
|
|
59
|
+
): Promise<{ roots: CID[]; blocks: BlockMap }> => {
|
|
60
|
+
const { roots, blocks } = await readCarStream([bytes])
|
|
61
|
+
const blockMap = new BlockMap()
|
|
62
|
+
for await (const block of blocks) {
|
|
63
|
+
blockMap.set(block.cid, block.bytes)
|
|
64
|
+
}
|
|
65
|
+
return { roots, blocks: blockMap }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export const readCarWithRoot = async (
|
|
69
|
+
bytes: Uint8Array,
|
|
70
|
+
): Promise<{ root: CID; blocks: BlockMap }> => {
|
|
71
|
+
const { roots, blocks } = await readCar(bytes)
|
|
72
|
+
if (roots.length !== 1) {
|
|
73
|
+
throw new Error(`Expected one root, got ${roots.length}`)
|
|
74
|
+
}
|
|
75
|
+
const root = roots[0]
|
|
76
|
+
return {
|
|
77
|
+
root,
|
|
78
|
+
blocks,
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
export type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {
|
|
82
|
+
dump: () => Promise<void>
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export const readCarStream = async (
|
|
86
|
+
car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
|
|
87
|
+
): Promise<{
|
|
88
|
+
roots: CID[]
|
|
89
|
+
blocks: CarBlockIterable
|
|
90
|
+
}> => {
|
|
91
|
+
const reader = new BufferedReader(car)
|
|
92
|
+
try {
|
|
93
|
+
const headerSize = await reader.readVarint()
|
|
94
|
+
if (headerSize === null) {
|
|
95
|
+
throw new Error('Could not parse CAR header')
|
|
96
|
+
}
|
|
97
|
+
const headerBytes = await reader.read(headerSize)
|
|
98
|
+
const header = cbor.decode(headerBytes)
|
|
99
|
+
if (!check.is(header, schema.carHeader)) {
|
|
100
|
+
throw new Error('Could not parse CAR header')
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
roots: header.roots,
|
|
104
|
+
blocks: readCarBlocksIter(reader),
|
|
105
|
+
}
|
|
106
|
+
} catch (err) {
|
|
107
|
+
await reader.close()
|
|
108
|
+
throw err
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const readCarBlocksIter = (reader: BufferedReader) => {
|
|
113
|
+
const iter = readCarBlocksIterGenerator(reader) as CarBlockIterable
|
|
114
|
+
|
|
115
|
+
iter.dump = async () => {
|
|
116
|
+
// try/finally to ensure that reader.close is called even if blocks.return throws.
|
|
117
|
+
try {
|
|
118
|
+
// Prevent the iterator from being started after this method is called.
|
|
119
|
+
await iter.return()
|
|
120
|
+
} finally {
|
|
121
|
+
// @NOTE the "finally" block of the async generator won't be called
|
|
122
|
+
// if the iteration was never started so we need to manually close here.
|
|
123
|
+
await reader.close()
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return iter
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function* readCarBlocksIterGenerator(
|
|
131
|
+
reader: BufferedReader,
|
|
132
|
+
): AsyncIterable<CarBlock> {
|
|
133
|
+
try {
|
|
134
|
+
while (!reader.isDone) {
|
|
135
|
+
const blockSize = await reader.readVarint()
|
|
136
|
+
if (blockSize === null) {
|
|
137
|
+
break
|
|
138
|
+
}
|
|
139
|
+
const blockBytes = await reader.read(blockSize)
|
|
140
|
+
const cid = parseCidFromBytes(blockBytes.slice(0, 36))
|
|
141
|
+
const bytes = blockBytes.slice(36)
|
|
142
|
+
yield { cid, bytes }
|
|
143
|
+
}
|
|
144
|
+
} finally {
|
|
145
|
+
await reader.close()
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export async function* verifyIncomingCarBlocks(
|
|
150
|
+
car: AsyncIterable<CarBlock>,
|
|
151
|
+
): AsyncIterable<CarBlock> {
|
|
152
|
+
for await (const block of car) {
|
|
153
|
+
await verifyCidForBytes(block.cid, block.bytes)
|
|
154
|
+
yield block
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
class BufferedReader {
|
|
159
|
+
buffer: Uint8Array = new Uint8Array()
|
|
160
|
+
iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>
|
|
161
|
+
isDone = false
|
|
162
|
+
|
|
163
|
+
constructor(stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) {
|
|
164
|
+
this.iterator =
|
|
165
|
+
Symbol.asyncIterator in stream
|
|
166
|
+
? stream[Symbol.asyncIterator]()
|
|
167
|
+
: stream[Symbol.iterator]()
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async read(bytesToRead: number): Promise<Uint8Array> {
|
|
171
|
+
await this.readUntilBuffered(bytesToRead)
|
|
172
|
+
const value = this.buffer.slice(0, bytesToRead)
|
|
173
|
+
this.buffer = this.buffer.slice(bytesToRead)
|
|
174
|
+
return value
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async readVarint(): Promise<number | null> {
|
|
178
|
+
let done = false
|
|
179
|
+
const bytes: Uint8Array[] = []
|
|
180
|
+
while (!done) {
|
|
181
|
+
const byte = await this.read(1)
|
|
182
|
+
if (byte.byteLength === 0) {
|
|
183
|
+
if (bytes.length > 0) {
|
|
184
|
+
throw new Error('could not parse varint')
|
|
185
|
+
} else {
|
|
186
|
+
return null
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
bytes.push(byte)
|
|
190
|
+
if (byte[0] < 128) {
|
|
191
|
+
done = true
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const concatted = ui8.concat(bytes)
|
|
195
|
+
return varint.decode(concatted)
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
private async readUntilBuffered(bytesToRead: number) {
|
|
199
|
+
if (this.isDone) {
|
|
200
|
+
return
|
|
201
|
+
}
|
|
202
|
+
while (this.buffer.length < bytesToRead) {
|
|
203
|
+
const next = await this.iterator.next()
|
|
204
|
+
if (next.done) {
|
|
205
|
+
this.isDone = true
|
|
206
|
+
return
|
|
207
|
+
}
|
|
208
|
+
this.buffer = ui8.concat([this.buffer, next.value])
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
async close(): Promise<void> {
|
|
213
|
+
if (!this.isDone && this.iterator.return) {
|
|
214
|
+
await this.iterator.return()
|
|
215
|
+
}
|
|
216
|
+
this.isDone = true
|
|
217
|
+
this.buffer = new Uint8Array()
|
|
218
|
+
}
|
|
219
|
+
}
|