@atproto/repo 0.8.9 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/car.d.ts +10 -0
- package/dist/car.d.ts.map +1 -1
- package/dist/car.js +69 -27
- package/dist/car.js.map +1 -1
- package/package.json +1 -1
- package/src/car.ts +70 -28
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @atproto/repo
|
|
2
2
|
|
|
3
|
+
## 0.8.10
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#4223](https://github.com/bluesky-social/atproto/pull/4223) [`8dd77bad2`](https://github.com/bluesky-social/atproto/commit/8dd77bad2fdee20e39d3787198d960c19d8df3d0) Thanks [@dholms](https://github.com/dholms)! - Ensure that reading CAR files is actually done asynchronously
|
|
8
|
+
|
|
3
9
|
## 0.8.9
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
package/dist/car.d.ts
CHANGED
|
@@ -25,5 +25,15 @@ export declare const readCarStream: (car: Iterable<Uint8Array> | AsyncIterable<U
|
|
|
25
25
|
roots: CID[];
|
|
26
26
|
blocks: CarBlockIterable;
|
|
27
27
|
}>;
|
|
28
|
+
export declare const readCarReader: (reader: BytesReader, opts?: ReadCarOptions) => Promise<{
|
|
29
|
+
roots: CID[];
|
|
30
|
+
blocks: CarBlockIterable;
|
|
31
|
+
}>;
|
|
28
32
|
export declare function verifyIncomingCarBlocks(car: AsyncIterable<CarBlock>): AsyncGenerator<CarBlock, void, unknown>;
|
|
33
|
+
interface BytesReader {
|
|
34
|
+
isDone: boolean;
|
|
35
|
+
read(bytesToRead: number): Promise<Uint8Array>;
|
|
36
|
+
close(): Promise<void>;
|
|
37
|
+
}
|
|
38
|
+
export {};
|
|
29
39
|
//# sourceMappingURL=car.d.ts.map
|
package/dist/car.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAUtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAElC,wBAAuB,cAAc,CACnC,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC9B,aAAa,CAAC,UAAU,CAAC,CAgB3B;AAED,eAAO,MAAM,eAAe,GAC1B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,OAAO,CAAC,UAAU,CAGpB,CAAA;AAED,eAAO,MAAM,iBAAiB,GAC5B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,aAAa,CAAC,UAAU,CAE1B,CAAA;AAQD,MAAM,MAAM,cAAc,GAAG;IAC3B;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;CAC9B,CAAA;AAED,eAAO,MAAM,OAAO,GAClB,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAO5C,CAAA;AAED,eAAO,MAAM,eAAe,GAC1B,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAUzC,CAAA;AACD,MAAM,MAAM,gBAAgB,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG;IACvE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1B,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,KAAK,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACrD,OAAO,cAAc,KACpB,OAAO,CAAC;IACT,KAAK,EAAE,GAAG,EAAE,CAAA;IACZ,MAAM,EAAE,gBAAgB,CAAA;CACzB,CAEA,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,QAAQ,WAAW,EACnB,OAAO,cAAc,KACpB,OAAO,CAAC;IACT,KAAK,EAAE,GAAG,EAAE,CAAA;IACZ,MAAM,EAAE,gBAAgB,CAAA;CACzB,CAmBA,CAAA;AAoDD,wBAAuB,uBAAuB,CAC5C,GAAG,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC3B,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAKzC;AAuBD,UAAU,WAAW;IACnB,MAAM,EAAE,OAAO,CAAA;IACf,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IAC9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB"}
|
package/dist/car.js
CHANGED
|
@@ -33,9 +33,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.readCarStream = exports.readCarWithRoot = exports.readCar = exports.blocksToCarStream = exports.blocksToCarFile = void 0;
|
|
36
|
+
exports.readCarReader = exports.readCarStream = exports.readCarWithRoot = exports.readCar = exports.blocksToCarStream = exports.blocksToCarFile = void 0;
|
|
37
37
|
exports.writeCarStream = writeCarStream;
|
|
38
38
|
exports.verifyIncomingCarBlocks = verifyIncomingCarBlocks;
|
|
39
|
+
const promises_1 = require("node:timers/promises");
|
|
39
40
|
const cbor = __importStar(require("@ipld/dag-cbor"));
|
|
40
41
|
const ui8 = __importStar(require("uint8arrays"));
|
|
41
42
|
const varint = __importStar(require("varint"));
|
|
@@ -69,7 +70,7 @@ async function* iterateBlocks(blocks) {
|
|
|
69
70
|
}
|
|
70
71
|
}
|
|
71
72
|
const readCar = async (bytes, opts) => {
|
|
72
|
-
const { roots, blocks } = await (0, exports.
|
|
73
|
+
const { roots, blocks } = await (0, exports.readCarReader)(new Ui8Reader(bytes), opts);
|
|
73
74
|
const blockMap = new block_map_1.BlockMap();
|
|
74
75
|
for await (const block of blocks) {
|
|
75
76
|
blockMap.set(block.cid, block.bytes);
|
|
@@ -90,9 +91,12 @@ const readCarWithRoot = async (bytes, opts) => {
|
|
|
90
91
|
};
|
|
91
92
|
exports.readCarWithRoot = readCarWithRoot;
|
|
92
93
|
const readCarStream = async (car, opts) => {
|
|
93
|
-
|
|
94
|
+
return (0, exports.readCarReader)(new BufferedReader(car), opts);
|
|
95
|
+
};
|
|
96
|
+
exports.readCarStream = readCarStream;
|
|
97
|
+
const readCarReader = async (reader, opts) => {
|
|
94
98
|
try {
|
|
95
|
-
const headerSize = await
|
|
99
|
+
const headerSize = await readVarint(reader);
|
|
96
100
|
if (headerSize === null) {
|
|
97
101
|
throw new Error('Could not parse CAR header');
|
|
98
102
|
}
|
|
@@ -111,7 +115,7 @@ const readCarStream = async (car, opts) => {
|
|
|
111
115
|
throw err;
|
|
112
116
|
}
|
|
113
117
|
};
|
|
114
|
-
exports.
|
|
118
|
+
exports.readCarReader = readCarReader;
|
|
115
119
|
const readCarBlocksIter = (reader, opts) => {
|
|
116
120
|
let generator = readCarBlocksIterGenerator(reader);
|
|
117
121
|
if (!opts?.skipCidVerification) {
|
|
@@ -133,9 +137,10 @@ const readCarBlocksIter = (reader, opts) => {
|
|
|
133
137
|
});
|
|
134
138
|
};
|
|
135
139
|
async function* readCarBlocksIterGenerator(reader) {
|
|
140
|
+
let blocks = 0;
|
|
136
141
|
try {
|
|
137
142
|
while (!reader.isDone) {
|
|
138
|
-
const blockSize = await
|
|
143
|
+
const blockSize = await readVarint(reader);
|
|
139
144
|
if (blockSize === null) {
|
|
140
145
|
break;
|
|
141
146
|
}
|
|
@@ -143,6 +148,12 @@ async function* readCarBlocksIterGenerator(reader) {
|
|
|
143
148
|
const cid = (0, common_1.parseCidFromBytes)(blockBytes.subarray(0, 36));
|
|
144
149
|
const bytes = blockBytes.subarray(36);
|
|
145
150
|
yield { cid, bytes };
|
|
151
|
+
// yield to the event loop every 25 blocks
|
|
152
|
+
// in the case the incoming CAR is synchronous, this can end up jamming up the thread
|
|
153
|
+
blocks++;
|
|
154
|
+
if (blocks % 25 === 0) {
|
|
155
|
+
await (0, promises_1.setImmediate)();
|
|
156
|
+
}
|
|
146
157
|
}
|
|
147
158
|
}
|
|
148
159
|
finally {
|
|
@@ -155,6 +166,58 @@ async function* verifyIncomingCarBlocks(car) {
|
|
|
155
166
|
yield block;
|
|
156
167
|
}
|
|
157
168
|
}
|
|
169
|
+
const readVarint = async (reader) => {
|
|
170
|
+
let done = false;
|
|
171
|
+
const bytes = [];
|
|
172
|
+
while (!done) {
|
|
173
|
+
const byte = await reader.read(1);
|
|
174
|
+
if (byte.byteLength === 0) {
|
|
175
|
+
if (bytes.length > 0) {
|
|
176
|
+
throw new Error('could not parse varint');
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
bytes.push(byte);
|
|
183
|
+
if (byte[0] < 128) {
|
|
184
|
+
done = true;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const concatted = ui8.concat(bytes);
|
|
188
|
+
return varint.decode(concatted);
|
|
189
|
+
};
|
|
190
|
+
class Ui8Reader {
|
|
191
|
+
constructor(bytes) {
|
|
192
|
+
Object.defineProperty(this, "bytes", {
|
|
193
|
+
enumerable: true,
|
|
194
|
+
configurable: true,
|
|
195
|
+
writable: true,
|
|
196
|
+
value: bytes
|
|
197
|
+
});
|
|
198
|
+
Object.defineProperty(this, "idx", {
|
|
199
|
+
enumerable: true,
|
|
200
|
+
configurable: true,
|
|
201
|
+
writable: true,
|
|
202
|
+
value: 0
|
|
203
|
+
});
|
|
204
|
+
Object.defineProperty(this, "isDone", {
|
|
205
|
+
enumerable: true,
|
|
206
|
+
configurable: true,
|
|
207
|
+
writable: true,
|
|
208
|
+
value: false
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
async read(bytesToRead) {
|
|
212
|
+
const value = this.bytes.subarray(this.idx, this.idx + bytesToRead);
|
|
213
|
+
this.idx += bytesToRead;
|
|
214
|
+
if (this.idx >= this.bytes.length) {
|
|
215
|
+
this.isDone = true;
|
|
216
|
+
}
|
|
217
|
+
return value;
|
|
218
|
+
}
|
|
219
|
+
async close() { }
|
|
220
|
+
}
|
|
158
221
|
class BufferedReader {
|
|
159
222
|
constructor(stream) {
|
|
160
223
|
Object.defineProperty(this, "buffer", {
|
|
@@ -186,27 +249,6 @@ class BufferedReader {
|
|
|
186
249
|
this.buffer = this.buffer.subarray(bytesToRead);
|
|
187
250
|
return value;
|
|
188
251
|
}
|
|
189
|
-
async readVarint() {
|
|
190
|
-
let done = false;
|
|
191
|
-
const bytes = [];
|
|
192
|
-
while (!done) {
|
|
193
|
-
const byte = await this.read(1);
|
|
194
|
-
if (byte.byteLength === 0) {
|
|
195
|
-
if (bytes.length > 0) {
|
|
196
|
-
throw new Error('could not parse varint');
|
|
197
|
-
}
|
|
198
|
-
else {
|
|
199
|
-
return null;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
bytes.push(byte);
|
|
203
|
-
if (byte[0] < 128) {
|
|
204
|
-
done = true;
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
const concatted = ui8.concat(bytes);
|
|
208
|
-
return varint.decode(concatted);
|
|
209
|
-
}
|
|
210
252
|
async readUntilBuffered(bytesToRead) {
|
|
211
253
|
if (this.isDone) {
|
|
212
254
|
return;
|
package/dist/car.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"car.js","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"car.js","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAeA,wCAmBC;AAmJD,0DAOC;AA5LD,mDAAmD;AACnD,qDAAsC;AAEtC,iDAAkC;AAClC,+CAAgC;AAChC,4CAMwB;AACxB,2CAAsC;AAG/B,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,IAAgB,EAChB,MAA+B;IAE/B,MAAM,MAAM,GAAG,IAAI,UAAU,CAC3B,IAAI,CAAC,MAAM,CAAC;QACV,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;KAC1B,CAAC,CACH,CAAA;IACD,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,UAAU,CAClB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CACnE,CAAA;QACD,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAA;QACrB,MAAM,KAAK,CAAC,KAAK,CAAA;IACnB,CAAC;AACH,CAAC;AAEM,MAAM,eAAe,GAAG,CAC7B,IAAgB,EAChB,MAAgB,EACK,EAAE;IACvB,MAAM,SAAS,GAAG,IAAA,yBAAiB,EAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACjD,OAAO,IAAA,uBAAc,EAAC,SAAS,CAAC,CAAA;AAClC,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,iBAAiB,GAAG,CAC/B,IAAgB,EAChB,MAAgB,EACW,EAAE;IAC7B,OAAO,cAAc,CAAC,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;AACpD,CAAC,CAAA;AALY,QAAA,iBAAiB,qBAK7B;AAED,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,MAAgB;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAA;IAC9C,CAAC;AACH,CAAC;AASM,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAiB,EACjB,IAAqB,EACwB,EAAE;IAC/C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,qBAAa,EAAC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,IAAI,oBAAQ,EAAE,CAAA;IAC/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACtC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACpC,CAAC,CAAA;AAVY,QAAA,OAAO,WAUnB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,KAAiB,EACjB,IAAqB,EACqB,EAAE;IAC5C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,eAAO,EAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,OAAO;QACL,IAAI;QACJ,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAbY,QAAA,eAAe,mBAa3B;AAKM,MAAM,aAAa,GAAG,KAAK,EAChC,GAAqD,EACrD,IAAqB,EAIpB,EAAE;IACH,OAAO,IAAA,qBAAa,EAAC,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;AACrD,CAAC,CAAA;AARY,QAAA,aAAa,iBAQzB;AAEM,MAAM,aAAa,GAAG,KAAK,EAChC,MAAmB,EACnB,IAAqB,EAIpB,EAAE;IACH,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;QAC3C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QACvC,IAAI,CAAC,cAAK,CAAC,EAAE,CAAC,MAAM,EAAE,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QACD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC;SACxC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAzBY,QAAA,aAAa,iBAyBzB;AAED,MAAM,iBAAiB,GAAG,CACxB,MAAmB,EACnB,IAAqB,EACH,EAAE;IACpB,IAAI,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAA;IAClD,IAAI,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,SAAS,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE;QAC9B,KAAK,CAAC,IAAI;YACR,kFAAkF;YAClF,IAAI,CAAC;gBACH,uEAAuE;gBACvE,MAAM,SAAS,CAAC,MAAM,EAAE,CAAA;YAC1B,CAAC;oBAAS,CAAC;gBACT,mEAAmE;gBACnE,wEAAwE;gBACxE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACtB,CAAC;QACH,CAAC;KACF,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,KAAK,SAAS,CAAC,CAAC,0BAA0B,CACxC,MAAmB;IAEnB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;YAC1C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,MAAK;YACP,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,IAAA,0BAAiB,EAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YACzD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YACrC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;YAEpB,0CAA0C;YAC1C,qFAAqF;YACrF,MAAM,EAAE,CAAA;YACR,IAAI,MAAM,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAA,uBAAY,GAAE,CAAA;YACtB,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,GAA4B;IAE5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,MAAM,IAAA,0BAAiB,EAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,GAAG,KAAK,EAAE,MAAmB,EAA0B,EAAE;IACvE,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,OAAO,CAAC,IAAI,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjC,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YAC3C,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;YAClB,IAAI,GAAG,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACnC,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;AACjC,CAAC,CAAA;AAQD,MAAM,SAAS;IAIb,YAAmB,KAAiB;QAAxB;;;;mBAAO,KAAK;WAAY;QAHpC;;;;mBAAM,CAAC;WAAA;QACP;;;;mBAAS,KAAK;WAAA;IAEyB,CAAC;IAExC,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,GAAG,WAAW,CAAC,CAAA;QACnE,IAAI,CAAC,GAAG,IAAI,WAAW,CAAA;QACvB,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QACpB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,KAAK,CAAC,KAAK,KAAmB,CAAC;CAChC;AAED,MAAM,cAAc;IAKlB,YAAY,MAAwD;QAJpE;;;;mBAAqB,IAAI,UAAU,EAAE;WAAA;QACrC;;;;;WAA0D;QAC1D;;;;mBAAS,KAAK;WAAA;QAGZ,IAAI,CAAC,QAAQ;YACX,MAAM,CAAC,aAAa,IAAI,MAAM;gBAC5B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;gBAChC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QAClD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;QAC/C,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAM;QACR,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;gBAClB,OAAM;YACR,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QACrD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC9B,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,EAAE,CAAA;IAChC,CAAC;CACF"}
|
package/package.json
CHANGED
package/src/car.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { setImmediate } from 'node:timers/promises'
|
|
1
2
|
import * as cbor from '@ipld/dag-cbor'
|
|
2
3
|
import { CID } from 'multiformats/cid'
|
|
3
4
|
import * as ui8 from 'uint8arrays'
|
|
@@ -65,7 +66,7 @@ export const readCar = async (
|
|
|
65
66
|
bytes: Uint8Array,
|
|
66
67
|
opts?: ReadCarOptions,
|
|
67
68
|
): Promise<{ roots: CID[]; blocks: BlockMap }> => {
|
|
68
|
-
const { roots, blocks } = await
|
|
69
|
+
const { roots, blocks } = await readCarReader(new Ui8Reader(bytes), opts)
|
|
69
70
|
const blockMap = new BlockMap()
|
|
70
71
|
for await (const block of blocks) {
|
|
71
72
|
blockMap.set(block.cid, block.bytes)
|
|
@@ -98,9 +99,18 @@ export const readCarStream = async (
|
|
|
98
99
|
roots: CID[]
|
|
99
100
|
blocks: CarBlockIterable
|
|
100
101
|
}> => {
|
|
101
|
-
|
|
102
|
+
return readCarReader(new BufferedReader(car), opts)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export const readCarReader = async (
|
|
106
|
+
reader: BytesReader,
|
|
107
|
+
opts?: ReadCarOptions,
|
|
108
|
+
): Promise<{
|
|
109
|
+
roots: CID[]
|
|
110
|
+
blocks: CarBlockIterable
|
|
111
|
+
}> => {
|
|
102
112
|
try {
|
|
103
|
-
const headerSize = await
|
|
113
|
+
const headerSize = await readVarint(reader)
|
|
104
114
|
if (headerSize === null) {
|
|
105
115
|
throw new Error('Could not parse CAR header')
|
|
106
116
|
}
|
|
@@ -120,7 +130,7 @@ export const readCarStream = async (
|
|
|
120
130
|
}
|
|
121
131
|
|
|
122
132
|
const readCarBlocksIter = (
|
|
123
|
-
reader:
|
|
133
|
+
reader: BytesReader,
|
|
124
134
|
opts?: ReadCarOptions,
|
|
125
135
|
): CarBlockIterable => {
|
|
126
136
|
let generator = readCarBlocksIterGenerator(reader)
|
|
@@ -143,11 +153,12 @@ const readCarBlocksIter = (
|
|
|
143
153
|
}
|
|
144
154
|
|
|
145
155
|
async function* readCarBlocksIterGenerator(
|
|
146
|
-
reader:
|
|
156
|
+
reader: BytesReader,
|
|
147
157
|
): AsyncGenerator<CarBlock, void, unknown> {
|
|
158
|
+
let blocks = 0
|
|
148
159
|
try {
|
|
149
160
|
while (!reader.isDone) {
|
|
150
|
-
const blockSize = await
|
|
161
|
+
const blockSize = await readVarint(reader)
|
|
151
162
|
if (blockSize === null) {
|
|
152
163
|
break
|
|
153
164
|
}
|
|
@@ -155,6 +166,13 @@ async function* readCarBlocksIterGenerator(
|
|
|
155
166
|
const cid = parseCidFromBytes(blockBytes.subarray(0, 36))
|
|
156
167
|
const bytes = blockBytes.subarray(36)
|
|
157
168
|
yield { cid, bytes }
|
|
169
|
+
|
|
170
|
+
// yield to the event loop every 25 blocks
|
|
171
|
+
// in the case the incoming CAR is synchronous, this can end up jamming up the thread
|
|
172
|
+
blocks++
|
|
173
|
+
if (blocks % 25 === 0) {
|
|
174
|
+
await setImmediate()
|
|
175
|
+
}
|
|
158
176
|
}
|
|
159
177
|
} finally {
|
|
160
178
|
await reader.close()
|
|
@@ -170,7 +188,52 @@ export async function* verifyIncomingCarBlocks(
|
|
|
170
188
|
}
|
|
171
189
|
}
|
|
172
190
|
|
|
173
|
-
|
|
191
|
+
const readVarint = async (reader: BytesReader): Promise<number | null> => {
|
|
192
|
+
let done = false
|
|
193
|
+
const bytes: Uint8Array[] = []
|
|
194
|
+
while (!done) {
|
|
195
|
+
const byte = await reader.read(1)
|
|
196
|
+
if (byte.byteLength === 0) {
|
|
197
|
+
if (bytes.length > 0) {
|
|
198
|
+
throw new Error('could not parse varint')
|
|
199
|
+
} else {
|
|
200
|
+
return null
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
bytes.push(byte)
|
|
204
|
+
if (byte[0] < 128) {
|
|
205
|
+
done = true
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
const concatted = ui8.concat(bytes)
|
|
209
|
+
return varint.decode(concatted)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
interface BytesReader {
|
|
213
|
+
isDone: boolean
|
|
214
|
+
read(bytesToRead: number): Promise<Uint8Array>
|
|
215
|
+
close(): Promise<void>
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
class Ui8Reader implements BytesReader {
|
|
219
|
+
idx = 0
|
|
220
|
+
isDone = false
|
|
221
|
+
|
|
222
|
+
constructor(public bytes: Uint8Array) {}
|
|
223
|
+
|
|
224
|
+
async read(bytesToRead: number): Promise<Uint8Array> {
|
|
225
|
+
const value = this.bytes.subarray(this.idx, this.idx + bytesToRead)
|
|
226
|
+
this.idx += bytesToRead
|
|
227
|
+
if (this.idx >= this.bytes.length) {
|
|
228
|
+
this.isDone = true
|
|
229
|
+
}
|
|
230
|
+
return value
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async close(): Promise<void> {}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
class BufferedReader implements BytesReader {
|
|
174
237
|
buffer: Uint8Array = new Uint8Array()
|
|
175
238
|
iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>
|
|
176
239
|
isDone = false
|
|
@@ -189,27 +252,6 @@ class BufferedReader {
|
|
|
189
252
|
return value
|
|
190
253
|
}
|
|
191
254
|
|
|
192
|
-
async readVarint(): Promise<number | null> {
|
|
193
|
-
let done = false
|
|
194
|
-
const bytes: Uint8Array[] = []
|
|
195
|
-
while (!done) {
|
|
196
|
-
const byte = await this.read(1)
|
|
197
|
-
if (byte.byteLength === 0) {
|
|
198
|
-
if (bytes.length > 0) {
|
|
199
|
-
throw new Error('could not parse varint')
|
|
200
|
-
} else {
|
|
201
|
-
return null
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
bytes.push(byte)
|
|
205
|
-
if (byte[0] < 128) {
|
|
206
|
-
done = true
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
const concatted = ui8.concat(bytes)
|
|
210
|
-
return varint.decode(concatted)
|
|
211
|
-
}
|
|
212
|
-
|
|
213
255
|
private async readUntilBuffered(bytesToRead: number) {
|
|
214
256
|
if (this.isDone) {
|
|
215
257
|
return
|