@atproto/repo 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/car.d.ts +10 -4
- package/dist/car.d.ts.map +1 -1
- package/dist/car.js +25 -21
- package/dist/car.js.map +1 -1
- package/package.json +2 -2
- package/src/car.ts +35 -20
- package/tests/car.test.ts +47 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @atproto/repo
|
|
2
2
|
|
|
3
|
+
## 0.8.6
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#4069](https://github.com/bluesky-social/atproto/pull/4069) [`331a356ce`](https://github.com/bluesky-social/atproto/commit/331a356ce27ff1d0b24747b0c16f3b54b07a0a12) Thanks [@devinivy](https://github.com/devinivy)! - Verify CID to content mapping within CARs unless explicitly skipped.
|
|
8
|
+
|
|
9
|
+
- Updated dependencies [[`331a356ce`](https://github.com/bluesky-social/atproto/commit/331a356ce27ff1d0b24747b0c16f3b54b07a0a12)]:
|
|
10
|
+
- @atproto/lexicon@0.4.13
|
|
11
|
+
|
|
12
|
+
## 0.8.5
|
|
13
|
+
|
|
14
|
+
### Patch Changes
|
|
15
|
+
|
|
16
|
+
- Updated dependencies [[`8ef976d38`](https://github.com/bluesky-social/atproto/commit/8ef976d3852df4bfa376e515e131cc0810a42f20)]:
|
|
17
|
+
- @atproto/lexicon@0.4.12
|
|
18
|
+
|
|
3
19
|
## 0.8.4
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
package/dist/car.d.ts
CHANGED
|
@@ -4,20 +4,26 @@ import { CarBlock } from './types';
|
|
|
4
4
|
export declare function writeCarStream(root: CID | null, blocks: AsyncIterable<CarBlock>): AsyncIterable<Uint8Array>;
|
|
5
5
|
export declare const blocksToCarFile: (root: CID | null, blocks: BlockMap) => Promise<Uint8Array>;
|
|
6
6
|
export declare const blocksToCarStream: (root: CID | null, blocks: BlockMap) => AsyncIterable<Uint8Array>;
|
|
7
|
-
export
|
|
7
|
+
export type ReadCarOptions = {
|
|
8
|
+
/**
|
|
9
|
+
* When true, does not verify CID-to-content mapping within CAR.
|
|
10
|
+
*/
|
|
11
|
+
skipCidVerification?: boolean;
|
|
12
|
+
};
|
|
13
|
+
export declare const readCar: (bytes: Uint8Array, opts?: ReadCarOptions) => Promise<{
|
|
8
14
|
roots: CID[];
|
|
9
15
|
blocks: BlockMap;
|
|
10
16
|
}>;
|
|
11
|
-
export declare const readCarWithRoot: (bytes: Uint8Array) => Promise<{
|
|
17
|
+
export declare const readCarWithRoot: (bytes: Uint8Array, opts?: ReadCarOptions) => Promise<{
|
|
12
18
|
root: CID;
|
|
13
19
|
blocks: BlockMap;
|
|
14
20
|
}>;
|
|
15
21
|
export type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {
|
|
16
22
|
dump: () => Promise<void>;
|
|
17
23
|
};
|
|
18
|
-
export declare const readCarStream: (car: Iterable<Uint8Array> | AsyncIterable<Uint8Array
|
|
24
|
+
export declare const readCarStream: (car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>, opts?: ReadCarOptions) => Promise<{
|
|
19
25
|
roots: CID[];
|
|
20
26
|
blocks: CarBlockIterable;
|
|
21
27
|
}>;
|
|
22
|
-
export declare function verifyIncomingCarBlocks(car: AsyncIterable<CarBlock>):
|
|
28
|
+
export declare function verifyIncomingCarBlocks(car: AsyncIterable<CarBlock>): AsyncGenerator<CarBlock, void, unknown>;
|
|
23
29
|
//# sourceMappingURL=car.d.ts.map
|
package/dist/car.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAUtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAElC,wBAAuB,cAAc,CACnC,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC9B,aAAa,CAAC,UAAU,CAAC,CAgB3B;AAED,eAAO,MAAM,eAAe,GAC1B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,OAAO,CAAC,UAAU,CAGpB,CAAA;AAED,eAAO,MAAM,iBAAiB,GAC5B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,aAAa,CAAC,UAAU,CAE1B,CAAA;AAQD,eAAO,MAAM,OAAO,GAClB,OAAO,UAAU,
|
|
1
|
+
{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAUtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAElC,wBAAuB,cAAc,CACnC,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC9B,aAAa,CAAC,UAAU,CAAC,CAgB3B;AAED,eAAO,MAAM,eAAe,GAC1B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,OAAO,CAAC,UAAU,CAGpB,CAAA;AAED,eAAO,MAAM,iBAAiB,GAC5B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,aAAa,CAAC,UAAU,CAE1B,CAAA;AAQD,MAAM,MAAM,cAAc,GAAG;IAC3B;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;CAC9B,CAAA;AAED,eAAO,MAAM,OAAO,GAClB,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAO5C,CAAA;AAED,eAAO,MAAM,eAAe,GAC1B,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAUzC,CAAA;AACD,MAAM,MAAM,gBAAgB,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG;IACvE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1B,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,KAAK,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACrD,OAAO,cAAc,KACpB,OAAO,CAAC;IACT,KAAK,EAAE,GAAG,EAAE,CAAA;IACZ,MAAM,EAAE,gBAAgB,CAAA;CACzB,CAoBA,CAAA;AA4CD,wBAAuB,uBAAuB,CAC5C,GAAG,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC3B,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAKzC"}
|
package/dist/car.js
CHANGED
|
@@ -68,8 +68,8 @@ async function* iterateBlocks(blocks) {
|
|
|
68
68
|
yield { cid: entry.cid, bytes: entry.bytes };
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
-
const readCar = async (bytes) => {
|
|
72
|
-
const { roots, blocks } = await (0, exports.readCarStream)([bytes]);
|
|
71
|
+
const readCar = async (bytes, opts) => {
|
|
72
|
+
const { roots, blocks } = await (0, exports.readCarStream)([bytes], opts);
|
|
73
73
|
const blockMap = new block_map_1.BlockMap();
|
|
74
74
|
for await (const block of blocks) {
|
|
75
75
|
blockMap.set(block.cid, block.bytes);
|
|
@@ -77,8 +77,8 @@ const readCar = async (bytes) => {
|
|
|
77
77
|
return { roots, blocks: blockMap };
|
|
78
78
|
};
|
|
79
79
|
exports.readCar = readCar;
|
|
80
|
-
const readCarWithRoot = async (bytes) => {
|
|
81
|
-
const { roots, blocks } = await (0, exports.readCar)(bytes);
|
|
80
|
+
const readCarWithRoot = async (bytes, opts) => {
|
|
81
|
+
const { roots, blocks } = await (0, exports.readCar)(bytes, opts);
|
|
82
82
|
if (roots.length !== 1) {
|
|
83
83
|
throw new Error(`Expected one root, got ${roots.length}`);
|
|
84
84
|
}
|
|
@@ -89,7 +89,7 @@ const readCarWithRoot = async (bytes) => {
|
|
|
89
89
|
};
|
|
90
90
|
};
|
|
91
91
|
exports.readCarWithRoot = readCarWithRoot;
|
|
92
|
-
const readCarStream = async (car) => {
|
|
92
|
+
const readCarStream = async (car, opts) => {
|
|
93
93
|
const reader = new BufferedReader(car);
|
|
94
94
|
try {
|
|
95
95
|
const headerSize = await reader.readVarint();
|
|
@@ -103,7 +103,7 @@ const readCarStream = async (car) => {
|
|
|
103
103
|
}
|
|
104
104
|
return {
|
|
105
105
|
roots: header.roots,
|
|
106
|
-
blocks: readCarBlocksIter(reader),
|
|
106
|
+
blocks: readCarBlocksIter(reader, opts),
|
|
107
107
|
};
|
|
108
108
|
}
|
|
109
109
|
catch (err) {
|
|
@@ -112,21 +112,25 @@ const readCarStream = async (car) => {
|
|
|
112
112
|
}
|
|
113
113
|
};
|
|
114
114
|
exports.readCarStream = readCarStream;
|
|
115
|
-
const readCarBlocksIter = (reader) => {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
115
|
+
const readCarBlocksIter = (reader, opts) => {
|
|
116
|
+
let generator = readCarBlocksIterGenerator(reader);
|
|
117
|
+
if (!opts?.skipCidVerification) {
|
|
118
|
+
generator = verifyIncomingCarBlocks(generator);
|
|
119
|
+
}
|
|
120
|
+
return Object.assign(generator, {
|
|
121
|
+
async dump() {
|
|
122
|
+
// try/finally to ensure that reader.close is called even if blocks.return throws.
|
|
123
|
+
try {
|
|
124
|
+
// Prevent the iterator from being started after this method is called.
|
|
125
|
+
await generator.return();
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
// @NOTE the "finally" block of the async generator won't be called
|
|
129
|
+
// if the iteration was never started so we need to manually close here.
|
|
130
|
+
await reader.close();
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
});
|
|
130
134
|
};
|
|
131
135
|
async function* readCarBlocksIterGenerator(reader) {
|
|
132
136
|
try {
|
package/dist/car.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"car.js","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcA,wCAmBC;
|
|
1
|
+
{"version":3,"file":"car.js","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcA,wCAmBC;AAkID,0DAOC;AA1KD,qDAAsC;AAEtC,iDAAkC;AAClC,+CAAgC;AAChC,4CAMwB;AACxB,2CAAsC;AAG/B,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,IAAgB,EAChB,MAA+B;IAE/B,MAAM,MAAM,GAAG,IAAI,UAAU,CAC3B,IAAI,CAAC,MAAM,CAAC;QACV,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;KAC1B,CAAC,CACH,CAAA;IACD,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,UAAU,CAClB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CACnE,CAAA;QACD,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAA;QACrB,MAAM,KAAK,CAAC,KAAK,CAAA;IACnB,CAAC;AACH,CAAC;AAEM,MAAM,eAAe,GAAG,CAC7B,IAAgB,EAChB,MAAgB,EACK,EAAE;IACvB,MAAM,SAAS,GAAG,IAAA,yBAAiB,EAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACjD,OAAO,IAAA,uBAAc,EAAC,SAAS,CAAC,CAAA;AAClC,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,iBAAiB,GAAG,CAC/B,IAAgB,EAChB,MAAgB,EACW,EAAE;IAC7B,OAAO,cAAc,CAAC,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;AACpD,CAAC,CAAA;AALY,QAAA,iBAAiB,qBAK7B;AAED,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,MAAgB;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAA;IAC9C,CAAC;AACH,CAAC;AASM,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAiB,EACjB,IAAqB,EACwB,EAAE;IAC/C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,qBAAa,EAAC,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAA;IAC5D,MAAM,QAAQ,GAAG,IAAI,oBAAQ,EAAE,CAAA;IAC/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACtC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACpC,CAAC,CAAA;AAVY,QAAA,OAAO,WAUnB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,KAAiB,EACjB,IAAqB,EACqB,EAAE;IAC5C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,eAAO,EAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,OAAO;QACL,IAAI;QACJ,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAbY,QAAA,eAAe,mBAa3B;AAKM,MAAM,aAAa,GAAG,KAAK,EAChC,GAAqD,EACrD,IAAqB,EAIpB,EAAE;IACH,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,CAAA;IACtC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAA;QAC5C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QACvC,IAAI,CAAC,cAAK,CAAC,EAAE,CAAC,MAAM,EAAE,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QACD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC;SACxC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AA1BY,QAAA,aAAa,iBA0BzB;AAED,MAAM,iBAAiB,GAAG,CACxB,MAAsB,EACtB,IAAqB,EACH,EAAE;IACpB,IAAI,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAA;IAClD,IAAI,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,SAAS,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE;QAC9B,KAAK,CAAC,IAAI;YACR,kFAAkF;YAClF,IAAI,CAAC;gBACH,uEAAuE;gBACvE,MAAM,SAAS,CAAC,MAAM,EAAE,CAAA;YAC1B,CAAC;oBAAS,CAAC;gBACT,mEAAmE;gBACnE,wEAAwE;gBACxE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACtB,CAAC;QACH,CAAC;KACF,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,KAAK,SAAS,CAAC,CAAC,0BAA0B,CACxC,MAAsB;IAEtB,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAA;YAC3C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,MAAK;YACP,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,IAAA,0BAAiB,EAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YACzD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YACrC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;QACtB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,GAA4B;IAE5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,MAAM,IAAA,0BAAiB,EAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,cAAc;IAKlB,YAAY,MAAwD;QAJpE;;;;mBAAqB,IAAI,UAAU,EAAE;WAAA;QACrC;;;;;WAA0D;QAC1D;;;;mBAAS,KAAK;WAAA;QAGZ,IAAI,CAAC,QAAQ;YACX,MAAM,CAAC,aAAa,IAAI,MAAM;gBAC5B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;gBAChC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QAClD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;QAC/C,OAAO,KAAK,CAAA;IACd,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,GAAG,KAAK,CAAA;QAChB,MAAM,KAAK,GAAiB,EAAE,CAAA;QAC9B,OAAO,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;gBAC1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;gBAC3C,CAAC;qBAAM,CAAC;oBACN,OAAO,IAAI,CAAA;gBACb,CAAC;YACH,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;gBAClB,IAAI,GAAG,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QACD,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACjC,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAM;QACR,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;gBAClB,OAAM;YACR,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QACrD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC9B,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,EAAE,CAAA;IAChC,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atproto/repo",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.6",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "atproto repo and MST implementation",
|
|
6
6
|
"keywords": [
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"@atproto/common": "^0.4.11",
|
|
28
28
|
"@atproto/common-web": "^0.4.2",
|
|
29
29
|
"@atproto/crypto": "^0.4.4",
|
|
30
|
-
"@atproto/lexicon": "^0.4.
|
|
30
|
+
"@atproto/lexicon": "^0.4.13"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"jest": "^28.1.2",
|
package/src/car.ts
CHANGED
|
@@ -54,10 +54,18 @@ async function* iterateBlocks(blocks: BlockMap) {
|
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
export type ReadCarOptions = {
|
|
58
|
+
/**
|
|
59
|
+
* When true, does not verify CID-to-content mapping within CAR.
|
|
60
|
+
*/
|
|
61
|
+
skipCidVerification?: boolean
|
|
62
|
+
}
|
|
63
|
+
|
|
57
64
|
export const readCar = async (
|
|
58
65
|
bytes: Uint8Array,
|
|
66
|
+
opts?: ReadCarOptions,
|
|
59
67
|
): Promise<{ roots: CID[]; blocks: BlockMap }> => {
|
|
60
|
-
const { roots, blocks } = await readCarStream([bytes])
|
|
68
|
+
const { roots, blocks } = await readCarStream([bytes], opts)
|
|
61
69
|
const blockMap = new BlockMap()
|
|
62
70
|
for await (const block of blocks) {
|
|
63
71
|
blockMap.set(block.cid, block.bytes)
|
|
@@ -67,8 +75,9 @@ export const readCar = async (
|
|
|
67
75
|
|
|
68
76
|
export const readCarWithRoot = async (
|
|
69
77
|
bytes: Uint8Array,
|
|
78
|
+
opts?: ReadCarOptions,
|
|
70
79
|
): Promise<{ root: CID; blocks: BlockMap }> => {
|
|
71
|
-
const { roots, blocks } = await readCar(bytes)
|
|
80
|
+
const { roots, blocks } = await readCar(bytes, opts)
|
|
72
81
|
if (roots.length !== 1) {
|
|
73
82
|
throw new Error(`Expected one root, got ${roots.length}`)
|
|
74
83
|
}
|
|
@@ -84,6 +93,7 @@ export type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {
|
|
|
84
93
|
|
|
85
94
|
export const readCarStream = async (
|
|
86
95
|
car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
|
|
96
|
+
opts?: ReadCarOptions,
|
|
87
97
|
): Promise<{
|
|
88
98
|
roots: CID[]
|
|
89
99
|
blocks: CarBlockIterable
|
|
@@ -101,7 +111,7 @@ export const readCarStream = async (
|
|
|
101
111
|
}
|
|
102
112
|
return {
|
|
103
113
|
roots: header.roots,
|
|
104
|
-
blocks: readCarBlocksIter(reader),
|
|
114
|
+
blocks: readCarBlocksIter(reader, opts),
|
|
105
115
|
}
|
|
106
116
|
} catch (err) {
|
|
107
117
|
await reader.close()
|
|
@@ -109,27 +119,32 @@ export const readCarStream = async (
|
|
|
109
119
|
}
|
|
110
120
|
}
|
|
111
121
|
|
|
112
|
-
const readCarBlocksIter = (
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
await iter.return()
|
|
120
|
-
} finally {
|
|
121
|
-
// @NOTE the "finally" block of the async generator won't be called
|
|
122
|
-
// if the iteration was never started so we need to manually close here.
|
|
123
|
-
await reader.close()
|
|
124
|
-
}
|
|
122
|
+
const readCarBlocksIter = (
|
|
123
|
+
reader: BufferedReader,
|
|
124
|
+
opts?: ReadCarOptions,
|
|
125
|
+
): CarBlockIterable => {
|
|
126
|
+
let generator = readCarBlocksIterGenerator(reader)
|
|
127
|
+
if (!opts?.skipCidVerification) {
|
|
128
|
+
generator = verifyIncomingCarBlocks(generator)
|
|
125
129
|
}
|
|
126
|
-
|
|
127
|
-
|
|
130
|
+
return Object.assign(generator, {
|
|
131
|
+
async dump() {
|
|
132
|
+
// try/finally to ensure that reader.close is called even if blocks.return throws.
|
|
133
|
+
try {
|
|
134
|
+
// Prevent the iterator from being started after this method is called.
|
|
135
|
+
await generator.return()
|
|
136
|
+
} finally {
|
|
137
|
+
// @NOTE the "finally" block of the async generator won't be called
|
|
138
|
+
// if the iteration was never started so we need to manually close here.
|
|
139
|
+
await reader.close()
|
|
140
|
+
}
|
|
141
|
+
},
|
|
142
|
+
})
|
|
128
143
|
}
|
|
129
144
|
|
|
130
145
|
async function* readCarBlocksIterGenerator(
|
|
131
146
|
reader: BufferedReader,
|
|
132
|
-
):
|
|
147
|
+
): AsyncGenerator<CarBlock, void, unknown> {
|
|
133
148
|
try {
|
|
134
149
|
while (!reader.isDone) {
|
|
135
150
|
const blockSize = await reader.readVarint()
|
|
@@ -148,7 +163,7 @@ async function* readCarBlocksIterGenerator(
|
|
|
148
163
|
|
|
149
164
|
export async function* verifyIncomingCarBlocks(
|
|
150
165
|
car: AsyncIterable<CarBlock>,
|
|
151
|
-
):
|
|
166
|
+
): AsyncGenerator<CarBlock, void, unknown> {
|
|
152
167
|
for await (const block of car) {
|
|
153
168
|
await verifyCidForBytes(block.cid, block.bytes)
|
|
154
169
|
yield block
|
package/tests/car.test.ts
CHANGED
|
@@ -55,4 +55,51 @@ describe('car', () => {
|
|
|
55
55
|
}
|
|
56
56
|
await expect(iterate).rejects.toThrow('Oops!')
|
|
57
57
|
})
|
|
58
|
+
|
|
59
|
+
it('verifies CIDs', async () => {
|
|
60
|
+
const block0 = await dataToCborBlock({ block: 0 })
|
|
61
|
+
const block1 = await dataToCborBlock({ block: 1 })
|
|
62
|
+
const block2 = await dataToCborBlock({ block: 2 })
|
|
63
|
+
const block3 = await dataToCborBlock({ block: 3 })
|
|
64
|
+
const badBlock = await dataToCborBlock({ block: 'bad' })
|
|
65
|
+
const blockIter = async function* () {
|
|
66
|
+
yield block0
|
|
67
|
+
yield block1
|
|
68
|
+
yield block2
|
|
69
|
+
yield { cid: block3.cid, bytes: badBlock.bytes }
|
|
70
|
+
}
|
|
71
|
+
const flush = async function (iter: AsyncIterable<unknown>) {
|
|
72
|
+
for await (const _ of iter) {
|
|
73
|
+
// no-op
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const badCar = await readCarStream(writeCarStream(block0.cid, blockIter()))
|
|
77
|
+
await expect(flush(badCar.blocks)).rejects.toThrow(
|
|
78
|
+
'Not a valid CID for bytes',
|
|
79
|
+
)
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
it('skips CID verification', async () => {
|
|
83
|
+
const block0 = await dataToCborBlock({ block: 0 })
|
|
84
|
+
const block1 = await dataToCborBlock({ block: 1 })
|
|
85
|
+
const block2 = await dataToCborBlock({ block: 2 })
|
|
86
|
+
const block3 = await dataToCborBlock({ block: 3 })
|
|
87
|
+
const badBlock = await dataToCborBlock({ block: 'bad' })
|
|
88
|
+
const blockIter = async function* () {
|
|
89
|
+
yield block0
|
|
90
|
+
yield block1
|
|
91
|
+
yield block2
|
|
92
|
+
yield { cid: block3.cid, bytes: badBlock.bytes }
|
|
93
|
+
}
|
|
94
|
+
const flush = async function (iter: AsyncIterable<unknown>) {
|
|
95
|
+
for await (const _ of iter) {
|
|
96
|
+
// no-op
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const badCar = await readCarStream(
|
|
100
|
+
writeCarStream(block0.cid, blockIter()),
|
|
101
|
+
{ skipCidVerification: true },
|
|
102
|
+
)
|
|
103
|
+
await expect(flush(badCar.blocks)).resolves.toBeUndefined()
|
|
104
|
+
})
|
|
58
105
|
})
|