@ipld/car 4.1.6 → 5.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +12 -4
  3. package/dist/index.min.js +5 -0
  4. package/{types → dist/src}/api.d.ts +19 -19
  5. package/dist/src/api.d.ts.map +1 -0
  6. package/{types/lib → dist/src}/buffer-writer.d.ts +10 -10
  7. package/{types/lib → dist/src}/buffer-writer.d.ts.map +1 -1
  8. package/dist/src/coding.d.ts +41 -0
  9. package/dist/src/coding.d.ts.map +1 -0
  10. package/{types/lib → dist/src}/decoder.d.ts +3 -3
  11. package/{types/lib → dist/src}/decoder.d.ts.map +1 -1
  12. package/{types/lib → dist/src}/encoder.d.ts +2 -2
  13. package/{types/lib → dist/src}/encoder.d.ts.map +1 -1
  14. package/{types/lib → dist/src}/header-validator.d.ts +0 -0
  15. package/dist/src/header-validator.d.ts.map +1 -0
  16. package/dist/src/index-browser.d.ts +8 -0
  17. package/dist/src/index-browser.d.ts.map +1 -0
  18. package/dist/src/index.d.ts +9 -0
  19. package/dist/src/index.d.ts.map +1 -0
  20. package/{types/lib → dist/src}/indexed-reader-browser.d.ts +0 -0
  21. package/dist/src/indexed-reader-browser.d.ts.map +1 -0
  22. package/{types/lib → dist/src}/indexed-reader.d.ts +13 -13
  23. package/dist/src/indexed-reader.d.ts.map +1 -0
  24. package/{types/lib → dist/src}/indexer.d.ts +9 -9
  25. package/{types/lib → dist/src}/indexer.d.ts.map +1 -1
  26. package/{types/lib → dist/src}/iterator-channel.d.ts +0 -0
  27. package/{types/lib → dist/src}/iterator-channel.d.ts.map +1 -1
  28. package/{types/lib → dist/src}/iterator.d.ts +8 -8
  29. package/{types/lib → dist/src}/iterator.d.ts.map +1 -1
  30. package/{types/lib → dist/src}/reader-browser.d.ts +10 -10
  31. package/{types/lib → dist/src}/reader-browser.d.ts.map +1 -1
  32. package/{types/lib → dist/src}/reader.d.ts +5 -5
  33. package/{types/lib → dist/src}/reader.d.ts.map +1 -1
  34. package/{types/lib → dist/src}/writer-browser.d.ts +10 -10
  35. package/{types/lib → dist/src}/writer-browser.d.ts.map +1 -1
  36. package/{types/lib → dist/src}/writer.d.ts +4 -4
  37. package/{types/lib → dist/src}/writer.d.ts.map +1 -1
  38. package/package.json +102 -209
  39. package/{api.ts → src/api.ts} +14 -14
  40. package/{lib → src}/buffer-writer.js +5 -5
  41. package/src/coding.ts +56 -0
  42. package/{lib → src}/decoder.js +7 -7
  43. package/{lib → src}/encoder.js +1 -1
  44. package/src/header-validator.js +74 -0
  45. package/{lib → src}/header.ipldsch +0 -0
  46. package/src/index-browser.js +14 -0
  47. package/src/index.js +16 -0
  48. package/{lib → src}/indexed-reader-browser.js +0 -0
  49. package/{lib → src}/indexed-reader.js +9 -9
  50. package/{lib → src}/indexer.js +4 -4
  51. package/{lib → src}/iterator-channel.js +0 -0
  52. package/{lib → src}/iterator.js +4 -4
  53. package/{lib → src}/reader-browser.js +7 -7
  54. package/{lib → src}/reader.js +5 -5
  55. package/{lib → src}/writer-browser.js +7 -7
  56. package/{lib → src}/writer.js +3 -3
  57. package/LICENSE-APACHE +0 -13
  58. package/LICENSE-MIT +0 -7
  59. package/buffer-writer +0 -1
  60. package/car-browser.js +0 -14
  61. package/car.js +0 -16
  62. package/cjs/browser-test/common.js +0 -301
  63. package/cjs/browser-test/node-test-file-streams.js +0 -71
  64. package/cjs/browser-test/node-test-indexed-reader.js +0 -51
  65. package/cjs/browser-test/node-test-large.js +0 -135
  66. package/cjs/browser-test/node-test-raw.js +0 -68
  67. package/cjs/browser-test/node-test-updateroots.js +0 -78
  68. package/cjs/browser-test/test-buffer-writer.js +0 -330
  69. package/cjs/browser-test/test-errors.js +0 -90
  70. package/cjs/browser-test/test-indexer.js +0 -80
  71. package/cjs/browser-test/test-interface.js +0 -21
  72. package/cjs/browser-test/test-iterator.js +0 -83
  73. package/cjs/browser-test/test-reader.js +0 -278
  74. package/cjs/browser-test/test-writer.js +0 -286
  75. package/cjs/browser-test/verify-store-reader.js +0 -142
  76. package/cjs/car-browser.js +0 -18
  77. package/cjs/car.js +0 -20
  78. package/cjs/lib/buffer-writer.js +0 -164
  79. package/cjs/lib/decoder.js +0 -301
  80. package/cjs/lib/encoder.js +0 -44
  81. package/cjs/lib/header-validator.js +0 -29
  82. package/cjs/lib/indexed-reader-browser.js +0 -13
  83. package/cjs/lib/indexed-reader.js +0 -90
  84. package/cjs/lib/indexer.js +0 -41
  85. package/cjs/lib/iterator-channel.js +0 -77
  86. package/cjs/lib/iterator.js +0 -97
  87. package/cjs/lib/reader-browser.js +0 -62
  88. package/cjs/lib/reader.js +0 -38
  89. package/cjs/lib/writer-browser.js +0 -118
  90. package/cjs/lib/writer.js +0 -51
  91. package/cjs/node-test/common.js +0 -301
  92. package/cjs/node-test/node-test-file-streams.js +0 -71
  93. package/cjs/node-test/node-test-indexed-reader.js +0 -51
  94. package/cjs/node-test/node-test-large.js +0 -135
  95. package/cjs/node-test/node-test-raw.js +0 -68
  96. package/cjs/node-test/node-test-updateroots.js +0 -78
  97. package/cjs/node-test/test-buffer-writer.js +0 -330
  98. package/cjs/node-test/test-errors.js +0 -90
  99. package/cjs/node-test/test-indexer.js +0 -80
  100. package/cjs/node-test/test-interface.js +0 -21
  101. package/cjs/node-test/test-iterator.js +0 -83
  102. package/cjs/node-test/test-reader.js +0 -278
  103. package/cjs/node-test/test-writer.js +0 -286
  104. package/cjs/node-test/verify-store-reader.js +0 -142
  105. package/decoder +0 -1
  106. package/esm/browser-test/common.js +0 -274
  107. package/esm/browser-test/node-test-file-streams.js +0 -76
  108. package/esm/browser-test/node-test-indexed-reader.js +0 -53
  109. package/esm/browser-test/node-test-large.js +0 -109
  110. package/esm/browser-test/node-test-raw.js +0 -63
  111. package/esm/browser-test/node-test-updateroots.js +0 -76
  112. package/esm/browser-test/test-buffer-writer.js +0 -311
  113. package/esm/browser-test/test-errors.js +0 -91
  114. package/esm/browser-test/test-indexer.js +0 -85
  115. package/esm/browser-test/test-interface.js +0 -24
  116. package/esm/browser-test/test-iterator.js +0 -91
  117. package/esm/browser-test/test-reader.js +0 -270
  118. package/esm/browser-test/test-writer.js +0 -297
  119. package/esm/browser-test/verify-store-reader.js +0 -122
  120. package/esm/car-browser.js +0 -16
  121. package/esm/car.js +0 -18
  122. package/esm/lib/buffer-writer.js +0 -129
  123. package/esm/lib/decoder.js +0 -265
  124. package/esm/lib/encoder.js +0 -35
  125. package/esm/lib/header-validator.js +0 -23
  126. package/esm/lib/indexed-reader-browser.js +0 -6
  127. package/esm/lib/indexed-reader.js +0 -78
  128. package/esm/lib/indexer.js +0 -38
  129. package/esm/lib/iterator-channel.js +0 -71
  130. package/esm/lib/iterator.js +0 -92
  131. package/esm/lib/reader-browser.js +0 -57
  132. package/esm/lib/reader.js +0 -26
  133. package/esm/lib/writer-browser.js +0 -115
  134. package/esm/lib/writer.js +0 -42
  135. package/esm/node-test/common.js +0 -274
  136. package/esm/node-test/node-test-file-streams.js +0 -76
  137. package/esm/node-test/node-test-indexed-reader.js +0 -53
  138. package/esm/node-test/node-test-large.js +0 -109
  139. package/esm/node-test/node-test-raw.js +0 -63
  140. package/esm/node-test/node-test-updateroots.js +0 -76
  141. package/esm/node-test/test-buffer-writer.js +0 -311
  142. package/esm/node-test/test-errors.js +0 -91
  143. package/esm/node-test/test-indexer.js +0 -85
  144. package/esm/node-test/test-interface.js +0 -24
  145. package/esm/node-test/test-iterator.js +0 -91
  146. package/esm/node-test/test-reader.js +0 -270
  147. package/esm/node-test/test-writer.js +0 -297
  148. package/esm/node-test/verify-store-reader.js +0 -122
  149. package/esm/package.json +0 -9
  150. package/examples/car-to-fixture.js +0 -66
  151. package/examples/dump-car.js +0 -58
  152. package/examples/dump-index.js +0 -24
  153. package/examples/package.json +0 -27
  154. package/examples/round-trip.js +0 -45
  155. package/examples/test-examples.js +0 -115
  156. package/examples/verify-car.js +0 -87
  157. package/index.js +0 -1
  158. package/indexed-reader +0 -1
  159. package/indexer +0 -1
  160. package/iterator +0 -1
  161. package/lib/coding.ts +0 -56
  162. package/lib/header-validator.js +0 -33
  163. package/reader +0 -1
  164. package/test/_fixtures_to_js.mjs +0 -24
  165. package/test/common.js +0 -210
  166. package/test/go.car +0 -0
  167. package/test/go.carv2 +0 -0
  168. package/test/node-test-file-streams.js +0 -84
  169. package/test/node-test-indexed-reader.js +0 -45
  170. package/test/node-test-large.js +0 -112
  171. package/test/node-test-raw.js +0 -83
  172. package/test/node-test-updateroots.js +0 -89
  173. package/test/test-buffer-writer.js +0 -256
  174. package/test/test-errors.js +0 -98
  175. package/test/test-indexer.js +0 -87
  176. package/test/test-interface.js +0 -24
  177. package/test/test-iterator.js +0 -74
  178. package/test/test-reader.js +0 -245
  179. package/test/test-writer.js +0 -336
  180. package/test/verify-store-reader.js +0 -191
  181. package/tsconfig.json +0 -48
  182. package/types/api.d.ts.map +0 -1
  183. package/types/car-browser.d.ts +0 -8
  184. package/types/car-browser.d.ts.map +0 -1
  185. package/types/car.d.ts +0 -9
  186. package/types/car.d.ts.map +0 -1
  187. package/types/lib/coding.d.ts +0 -41
  188. package/types/lib/coding.d.ts.map +0 -1
  189. package/types/lib/header-validator.d.ts.map +0 -1
  190. package/types/lib/indexed-reader-browser.d.ts.map +0 -1
  191. package/types/lib/indexed-reader.d.ts.map +0 -1
  192. package/types/test/_fixtures_to_js.d.mts +0 -3
  193. package/types/test/_fixtures_to_js.d.mts.map +0 -1
  194. package/types/test/common.d.ts +0 -73
  195. package/types/test/common.d.ts.map +0 -1
  196. package/types/test/fixtures-expectations.d.ts +0 -63
  197. package/types/test/fixtures-expectations.d.ts.map +0 -1
  198. package/types/test/fixtures.d.ts +0 -3
  199. package/types/test/fixtures.d.ts.map +0 -1
  200. package/types/test/node-test-file-streams.d.ts +0 -3
  201. package/types/test/node-test-file-streams.d.ts.map +0 -1
  202. package/types/test/node-test-indexed-reader.d.ts +0 -2
  203. package/types/test/node-test-indexed-reader.d.ts.map +0 -1
  204. package/types/test/node-test-large.d.ts +0 -2
  205. package/types/test/node-test-large.d.ts.map +0 -1
  206. package/types/test/node-test-raw.d.ts +0 -2
  207. package/types/test/node-test-raw.d.ts.map +0 -1
  208. package/types/test/node-test-updateroots.d.ts +0 -2
  209. package/types/test/node-test-updateroots.d.ts.map +0 -1
  210. package/types/test/test-buffer-writer.d.ts +0 -2
  211. package/types/test/test-buffer-writer.d.ts.map +0 -1
  212. package/types/test/test-errors.d.ts +0 -2
  213. package/types/test/test-errors.d.ts.map +0 -1
  214. package/types/test/test-indexer.d.ts +0 -2
  215. package/types/test/test-indexer.d.ts.map +0 -1
  216. package/types/test/test-interface.d.ts +0 -2
  217. package/types/test/test-interface.d.ts.map +0 -1
  218. package/types/test/test-iterator.d.ts +0 -2
  219. package/types/test/test-iterator.d.ts.map +0 -1
  220. package/types/test/test-reader.d.ts +0 -2
  221. package/types/test/test-reader.d.ts.map +0 -1
  222. package/types/test/test-writer.d.ts +0 -2
  223. package/types/test/test-writer.d.ts.map +0 -1
  224. package/types/test/verify-store-reader.d.ts +0 -29
  225. package/types/test/verify-store-reader.d.ts.map +0 -1
  226. package/writer +0 -1
@@ -1,53 +0,0 @@
1
- import path from 'path';
2
- import { fileURLToPath } from 'url';
3
- import { CarIndexedReader } from '../lib/indexed-reader.js';
4
- import {
5
- assert,
6
- goCarIndex
7
- } from './common.js';
8
- import {
9
- verifyRoots,
10
- verifyHas,
11
- verifyGet,
12
- verifyBlocks,
13
- verifyCids
14
- } from './verify-store-reader.js';
15
- const __filename = fileURLToPath(import.meta.url);
16
- const __dirname = path.dirname(__filename);
17
- describe('CarIndexedReader fromFile()', () => {
18
- it('complete', async () => {
19
- const reader = await CarIndexedReader.fromFile(path.join(__dirname, 'go.car'));
20
- await verifyRoots(reader);
21
- await verifyHas(reader);
22
- await verifyGet(reader);
23
- await verifyBlocks(reader.blocks(), true);
24
- await verifyCids(reader.cids(), true);
25
- let i = 0;
26
- for await (const block of reader.blocks()) {
27
- assert.strictEqual(block.cid.toString(), goCarIndex[i++].cid.toString());
28
- }
29
- i = 0;
30
- for await (const cid of reader.cids()) {
31
- assert.strictEqual(cid.toString(), goCarIndex[i++].cid.toString());
32
- }
33
- assert.strictEqual(reader.version, 1);
34
- await reader.close();
35
- });
36
- it('bad argument', async () => {
37
- for (const arg of [
38
- true,
39
- false,
40
- null,
41
- undefined,
42
- Uint8Array.from([
43
- 1,
44
- 2,
45
- 3
46
- ]),
47
- 100,
48
- { obj: 'nope' }
49
- ]) {
50
- await assert.isRejected(CarIndexedReader.fromFile(arg));
51
- }
52
- });
53
- });
@@ -1,109 +0,0 @@
1
- import fs from 'fs';
2
- import { Readable } from 'stream';
3
- import { garbage } from 'ipld-garbage';
4
- import varint from 'varint';
5
- import * as dagCbor from '@ipld/dag-cbor';
6
- import { sha256 } from 'multiformats/hashes/sha2';
7
- import { CID } from 'multiformats/cid';
8
- import {
9
- CarWriter,
10
- CarIndexer,
11
- CarReader,
12
- CarIndexedReader
13
- } from '../car.js';
14
- import { assert } from './common.js';
15
- describe('Large CAR', () => {
16
- const objects = [];
17
- const cids = [];
18
- const expectedIndex = [];
19
- it('create, no roots', async () => {
20
- const {writer, out} = CarWriter.create([]);
21
- Readable.from(out).pipe(fs.createWriteStream('./test.car'));
22
- let offset = dagCbor.encode({
23
- version: 1,
24
- roots: []
25
- }).length;
26
- offset += varint.encode(offset).length;
27
- for (let i = 0; i < 500; i++) {
28
- const obj = garbage(1000);
29
- objects.push(obj);
30
- const bytes = dagCbor.encode(obj);
31
- const hash = await sha256.digest(bytes);
32
- const cid = CID.create(1, dagCbor.code, hash);
33
- cids.push(cid.toString());
34
- const blockLength = bytes.length;
35
- let length = cid.bytes.length + blockLength;
36
- const lengthLength = varint.encode(length).length;
37
- length += lengthLength;
38
- const blockOffset = offset + lengthLength + cid.bytes.length;
39
- expectedIndex.push({
40
- cid,
41
- offset,
42
- length,
43
- blockOffset,
44
- blockLength
45
- });
46
- offset += length;
47
- await writer.put({
48
- cid,
49
- bytes
50
- });
51
- }
52
- await writer.close();
53
- });
54
- it('CarIndexer.fromIterable', async () => {
55
- const indexer = await CarIndexer.fromIterable(fs.createReadStream('./test.car'));
56
- assert.deepStrictEqual(await indexer.getRoots(), []);
57
- let i = 0;
58
- for await (const blockIndex of indexer) {
59
- assert.deepStrictEqual(blockIndex, expectedIndex[i]);
60
- i++;
61
- }
62
- });
63
- it('CarIndexer.fromBytes', async () => {
64
- const indexer = await CarIndexer.fromBytes(await fs.promises.readFile('./test.car'));
65
- assert.deepStrictEqual(await indexer.getRoots(), []);
66
- let i = 0;
67
- for await (const blockIndex of indexer) {
68
- assert.deepStrictEqual(blockIndex, expectedIndex[i]);
69
- i++;
70
- }
71
- });
72
- it('CarReader.fromBytes', async () => {
73
- const reader = await CarReader.fromBytes(await fs.promises.readFile('./test.car'));
74
- assert.deepStrictEqual(await reader.getRoots(), []);
75
- let i = 0;
76
- for await (const {cid, bytes} of reader.blocks()) {
77
- assert.strictEqual(cid.toString(), cids[i], `cid #${ i } ${ cid } <> ${ cids[i] }`);
78
- const obj = dagCbor.decode(bytes);
79
- assert.deepStrictEqual(obj, objects[i], `object #${ i }`);
80
- i++;
81
- }
82
- });
83
- it('CarReader.fromIterable', async () => {
84
- const reader = await CarReader.fromIterable(fs.createReadStream('./test.car'));
85
- assert.deepStrictEqual(await reader.getRoots(), []);
86
- let i = 0;
87
- for await (const {cid, bytes} of reader.blocks()) {
88
- assert.strictEqual(cid.toString(), cids[i], `cid #${ i } ${ cid } <> ${ cids[i] }`);
89
- const obj = dagCbor.decode(bytes);
90
- assert.deepStrictEqual(obj, objects[i], `object #${ i }`);
91
- i++;
92
- }
93
- });
94
- it('CarIndexedReader.fromFile', async () => {
95
- const reader = await CarIndexedReader.fromFile('./test.car');
96
- assert.deepStrictEqual(await reader.getRoots(), []);
97
- let i = 0;
98
- for await (const {cid, bytes} of reader.blocks()) {
99
- assert.strictEqual(cid.toString(), cids[i], `cid #${ i } ${ cid } <> ${ cids[i] }`);
100
- const obj = dagCbor.decode(bytes);
101
- assert.deepStrictEqual(obj, objects[i], `object #${ i }`);
102
- i++;
103
- }
104
- });
105
- after(async () => {
106
- return fs.promises.unlink('./test.car').catch(() => {
107
- });
108
- });
109
- });
@@ -1,63 +0,0 @@
1
- import path from 'path';
2
- import fs from 'fs';
3
- import { promisify } from 'util';
4
- import { fileURLToPath } from 'url';
5
- import { bytes } from 'multiformats';
6
- import { CarReader } from '../car.js';
7
- import {
8
- assert,
9
- makeData,
10
- goCarIndex
11
- } from './common.js';
12
- const fsopen = promisify(fs.open);
13
- const fsclose = promisify(fs.close);
14
- const {toHex} = bytes;
15
- const __filename = fileURLToPath(import.meta.url);
16
- const __dirname = path.dirname(__filename);
17
- describe('CarReader.readRaw', () => {
18
- let allBlocksFlattened;
19
- before(async () => {
20
- const data = await makeData();
21
- allBlocksFlattened = data.allBlocksFlattened;
22
- });
23
- async function verifyRead(fd) {
24
- const expectedBlocks = allBlocksFlattened.slice();
25
- const expectedCids = [];
26
- for (const {cid} of expectedBlocks) {
27
- expectedCids.push(cid.toString());
28
- }
29
- for (const blockIndex of goCarIndex) {
30
- const {cid, bytes} = await CarReader.readRaw(fd, blockIndex);
31
- const index = expectedCids.indexOf(cid.toString());
32
- assert.ok(index >= 0, 'got expected block');
33
- assert.strictEqual(toHex(expectedBlocks[index].bytes), toHex(bytes), 'got expected block content');
34
- expectedBlocks.splice(index, 1);
35
- expectedCids.splice(index, 1);
36
- }
37
- assert.strictEqual(expectedBlocks.length, 0, 'got all expected blocks');
38
- }
39
- it('read raw using index (fd)', async () => {
40
- const fd = await fsopen(path.join(__dirname, 'go.car'), 'r');
41
- await verifyRead(fd);
42
- await fsclose(fd);
43
- });
44
- it('read raw using index (FileHandle)', async () => {
45
- const fd = await fs.promises.open(path.join(__dirname, 'go.car'), 'r');
46
- await verifyRead(fd);
47
- await fd.close();
48
- });
49
- it('errors', async () => {
50
- await assert.isRejected(CarReader.readRaw(true, goCarIndex[0]), {
51
- name: 'TypeError',
52
- message: 'Bad fd'
53
- });
54
- const badBlock = Object.assign({}, goCarIndex[goCarIndex.length - 1]);
55
- badBlock.blockLength += 10;
56
- const fd = await fsopen(path.join(__dirname, 'go.car'), 'r');
57
- await assert.isRejected(CarReader.readRaw(fd, badBlock), {
58
- name: 'Error',
59
- message: `Failed to read entire block (${ badBlock.blockLength - 10 } instead of ${ badBlock.blockLength })`
60
- });
61
- await fsclose(fd);
62
- });
63
- });
@@ -1,76 +0,0 @@
1
- import path from 'path';
2
- import { fileURLToPath } from 'url';
3
- import fs from 'fs';
4
- import { promisify } from 'util';
5
- import { CID } from 'multiformats/cid';
6
- import { CarReader } from '../lib/reader.js';
7
- import { CarWriter } from '../lib/writer.js';
8
- import {
9
- verifyRoots,
10
- verifyHas,
11
- verifyGet,
12
- verifyBlocks,
13
- verifyCids
14
- } from './verify-store-reader.js';
15
- import { assert } from './common.js';
16
- const fsopen = promisify(fs.open);
17
- const fsclose = promisify(fs.close);
18
- const __filename = fileURLToPath(import.meta.url);
19
- const __dirname = path.dirname(__filename);
20
- const goCarPath = path.join(__dirname, 'go.car');
21
- const tmpCarPath = path.join(__dirname, 'tmp.car');
22
- const newRoots = [
23
- CID.parse('bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq'),
24
- CID.parse('bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4')
25
- ];
26
- async function verify() {
27
- const reader = await CarReader.fromIterable(fs.createReadStream(tmpCarPath));
28
- await assert.isRejected(verifyRoots(reader));
29
- assert.deepEqual(await reader.getRoots(), newRoots);
30
- await verifyHas(reader);
31
- await verifyGet(reader);
32
- await verifyBlocks(reader.blocks(), true);
33
- await verifyCids(reader.cids(), true);
34
- }
35
- describe('Node CarWriter.updateHeader()', () => {
36
- before(async () => {
37
- try {
38
- await fs.promises.unlink(tmpCarPath);
39
- } catch (e) {
40
- }
41
- });
42
- beforeEach(async () => {
43
- await fs.promises.copyFile(goCarPath, tmpCarPath);
44
- });
45
- afterEach(async () => {
46
- await fs.promises.unlink(tmpCarPath);
47
- });
48
- it('update roots (fd)', async () => {
49
- const fd = await fsopen(tmpCarPath, 'r+');
50
- await CarWriter.updateRootsInFile(fd, newRoots);
51
- await fsclose(fd);
52
- await verify();
53
- });
54
- it('update roots (FileHandle)', async () => {
55
- const fd = await fs.promises.open(tmpCarPath, 'r+');
56
- await CarWriter.updateRootsInFile(fd, newRoots);
57
- await fd.close();
58
- await verify();
59
- });
60
- it('error: bad fd', async () => {
61
- await assert.isRejected(CarWriter.updateRootsInFile(true, newRoots), {
62
- name: 'TypeError',
63
- message: 'Bad fd'
64
- });
65
- });
66
- it('error: wrong header size', async () => {
67
- const fd = await fs.promises.open(tmpCarPath, 'r+');
68
- await assert.isRejected(CarWriter.updateRootsInFile(fd, [
69
- ...newRoots,
70
- newRoots[0]
71
- ]), /can only overwrite a header of the same length/);
72
- await assert.isRejected(CarWriter.updateRootsInFile(fd, [newRoots[0]]), /can only overwrite a header of the same length/);
73
- await assert.isRejected(CarWriter.updateRootsInFile(fd, []), /can only overwrite a header of the same length/);
74
- await fd.close();
75
- });
76
- });
@@ -1,311 +0,0 @@
1
- import * as CarBufferWriter from '../lib/buffer-writer.js';
2
- import { CarReader } from '../lib/reader.js';
3
- import { createHeader } from '../lib/encoder.js';
4
- import { assert } from './common.js';
5
- import {
6
- CID,
7
- varint
8
- } from 'multiformats';
9
- import * as CBOR from '@ipld/dag-cbor';
10
- import {
11
- sha256,
12
- sha512
13
- } from 'multiformats/hashes/sha2';
14
- import { identity } from 'multiformats/hashes/identity';
15
- import * as Raw from 'multiformats/codecs/raw';
16
- import * as Block from 'multiformats/block';
17
- describe('CarBufferWriter', () => {
18
- const cid = CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu');
19
- describe('calculateHeaderLength', async () => {
20
- for (const count of [
21
- 0,
22
- 1,
23
- 10,
24
- 18,
25
- 24,
26
- 48,
27
- 124,
28
- 255,
29
- 258,
30
- 65536 - 1,
31
- 65536
32
- ]) {
33
- it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
34
- const roots = new Array(count).fill(cid);
35
- const sizes = new Array(count).fill(cid.bytes.byteLength);
36
- assert.deepEqual(CarBufferWriter.calculateHeaderLength(sizes), createHeader(roots).byteLength);
37
- });
38
- it(`calculateHeaderLength(new Array(${ count }).fill(36))`, () => {
39
- const roots = new Array(count).fill(cid);
40
- const rootLengths = roots.map(c => c.bytes.byteLength);
41
- assert.deepEqual(CarBufferWriter.calculateHeaderLength(rootLengths), createHeader(roots).byteLength);
42
- });
43
- }
44
- it('estimate on large CIDs', () => {
45
- const largeCID = CID.parse(`bafkqbbac${ 'a'.repeat(416) }`);
46
- assert.equal(CarBufferWriter.calculateHeaderLength([
47
- cid.bytes.byteLength,
48
- largeCID.bytes.byteLength
49
- ]), createHeader([
50
- cid,
51
- largeCID
52
- ]).byteLength);
53
- });
54
- it('estimate on large CIDs 2', () => {
55
- const largeCID = CID.createV1(Raw.code, identity.digest(new Uint8Array(512).fill(1)));
56
- assert.equal(CarBufferWriter.calculateHeaderLength([
57
- cid.bytes.byteLength,
58
- largeCID.bytes.byteLength
59
- ]), createHeader([
60
- cid,
61
- largeCID
62
- ]).byteLength);
63
- });
64
- });
65
- describe('writer', () => {
66
- it('estimate header and write blocks', async () => {
67
- const headerSize = CarBufferWriter.estimateHeaderLength(1);
68
- const dataSize = 256;
69
- const buffer = new ArrayBuffer(headerSize + dataSize);
70
- const writer = CarBufferWriter.createWriter(buffer, { headerSize });
71
- const b1 = await Block.encode({
72
- value: { hello: 'world' },
73
- codec: CBOR,
74
- hasher: sha256
75
- });
76
- writer.write(b1);
77
- const b2 = await Block.encode({
78
- value: { bye: 'world' },
79
- codec: CBOR,
80
- hasher: sha256
81
- });
82
- writer.write(b2);
83
- writer.addRoot(b1.cid);
84
- const bytes = writer.close();
85
- const reader = await CarReader.fromBytes(bytes);
86
- assert.deepEqual(await reader.getRoots(), [b1.cid]);
87
- assert.deepEqual(reader._blocks, [
88
- {
89
- cid: b1.cid,
90
- bytes: b1.bytes
91
- },
92
- {
93
- cid: b2.cid,
94
- bytes: b2.bytes
95
- }
96
- ]);
97
- });
98
- it('overestimate header', async () => {
99
- const headerSize = CarBufferWriter.estimateHeaderLength(2);
100
- const dataSize = 256;
101
- const buffer = new ArrayBuffer(headerSize + dataSize);
102
- const writer = CarBufferWriter.createWriter(buffer, { headerSize });
103
- const b1 = await Block.encode({
104
- value: { hello: 'world' },
105
- codec: CBOR,
106
- hasher: sha256
107
- });
108
- writer.write(b1);
109
- const b2 = await Block.encode({
110
- value: { bye: 'world' },
111
- codec: CBOR,
112
- hasher: sha256
113
- });
114
- writer.write(b2);
115
- writer.addRoot(b1.cid);
116
- assert.throws(() => writer.close(), /Header size was overestimate/);
117
- const bytes = writer.close({ resize: true });
118
- const reader = await CarReader.fromBytes(bytes);
119
- assert.deepEqual(await reader.getRoots(), [b1.cid]);
120
- assert.deepEqual(reader._blocks, [
121
- {
122
- cid: b1.cid,
123
- bytes: b1.bytes
124
- },
125
- {
126
- cid: b2.cid,
127
- bytes: b2.bytes
128
- }
129
- ]);
130
- });
131
- it('underestimate header', async () => {
132
- const headerSize = CarBufferWriter.estimateHeaderLength(2);
133
- const dataSize = 300;
134
- const buffer = new ArrayBuffer(headerSize + dataSize);
135
- const writer = CarBufferWriter.createWriter(buffer, { headerSize });
136
- const b1 = await Block.encode({
137
- value: { hello: 'world' },
138
- codec: CBOR,
139
- hasher: sha256
140
- });
141
- writer.write(b1);
142
- writer.addRoot(b1.cid);
143
- const b2 = await Block.encode({
144
- value: { bye: 'world' },
145
- codec: CBOR,
146
- hasher: sha512
147
- });
148
- writer.write(b2);
149
- assert.throws(() => writer.addRoot(b2.cid), /has no capacity/);
150
- writer.addRoot(b2.cid, { resize: true });
151
- const bytes = writer.close();
152
- const reader = await CarReader.fromBytes(bytes);
153
- assert.deepEqual(await reader.getRoots(), [
154
- b1.cid,
155
- b2.cid
156
- ]);
157
- assert.deepEqual(reader._blocks, [
158
- {
159
- cid: b1.cid,
160
- bytes: b1.bytes
161
- },
162
- {
163
- cid: b2.cid,
164
- bytes: b2.bytes
165
- }
166
- ]);
167
- });
168
- });
169
- it('has no space for the root', async () => {
170
- const headerSize = CarBufferWriter.estimateHeaderLength(1);
171
- const dataSize = 100;
172
- const buffer = new ArrayBuffer(headerSize + dataSize);
173
- const writer = CarBufferWriter.createWriter(buffer, { headerSize });
174
- const b1 = await Block.encode({
175
- value: { hello: 'world' },
176
- codec: CBOR,
177
- hasher: sha256
178
- });
179
- writer.write(b1);
180
- writer.addRoot(b1.cid);
181
- const b2 = await Block.encode({
182
- value: { bye: 'world' },
183
- codec: CBOR,
184
- hasher: sha256
185
- });
186
- writer.write(b2);
187
- assert.throws(() => writer.addRoot(b2.cid), /Buffer has no capacity for a new root/);
188
- assert.throws(() => writer.addRoot(b2.cid, { resize: true }), /Buffer has no capacity for a new root/);
189
- const bytes = writer.close();
190
- const reader = await CarReader.fromBytes(bytes);
191
- assert.deepEqual(await reader.getRoots(), [b1.cid]);
192
- assert.deepEqual(reader._blocks, [
193
- {
194
- cid: b1.cid,
195
- bytes: b1.bytes
196
- },
197
- {
198
- cid: b2.cid,
199
- bytes: b2.bytes
200
- }
201
- ]);
202
- });
203
- it('has no space for the block', async () => {
204
- const headerSize = CarBufferWriter.estimateHeaderLength(1);
205
- const dataSize = 58;
206
- const buffer = new ArrayBuffer(headerSize + dataSize);
207
- const writer = CarBufferWriter.createWriter(buffer, { headerSize });
208
- const b1 = await Block.encode({
209
- value: { hello: 'world' },
210
- codec: CBOR,
211
- hasher: sha256
212
- });
213
- writer.write(b1);
214
- writer.addRoot(b1.cid);
215
- const b2 = await Block.encode({
216
- value: { bye: 'world' },
217
- codec: CBOR,
218
- hasher: sha256
219
- });
220
- assert.throws(() => writer.write(b2), /Buffer has no capacity for this block/);
221
- const bytes = writer.close();
222
- const reader = await CarReader.fromBytes(bytes);
223
- assert.deepEqual(await reader.getRoots(), [b1.cid]);
224
- assert.deepEqual(reader._blocks, [{
225
- cid: b1.cid,
226
- bytes: b1.bytes
227
- }]);
228
- });
229
- it('provide roots', async () => {
230
- const b1 = await Block.encode({
231
- value: { hello: 'world' },
232
- codec: CBOR,
233
- hasher: sha256
234
- });
235
- const b2 = await Block.encode({
236
- value: { bye: 'world' },
237
- codec: CBOR,
238
- hasher: sha512
239
- });
240
- const buffer = new ArrayBuffer(300);
241
- const writer = CarBufferWriter.createWriter(buffer, {
242
- roots: [
243
- b1.cid,
244
- b2.cid
245
- ]
246
- });
247
- writer.write(b1);
248
- writer.write(b2);
249
- const bytes = writer.close();
250
- const reader = await CarReader.fromBytes(bytes);
251
- assert.deepEqual(await reader.getRoots(), [
252
- b1.cid,
253
- b2.cid
254
- ]);
255
- assert.deepEqual(reader._blocks, [
256
- {
257
- cid: b1.cid,
258
- bytes: b1.bytes
259
- },
260
- {
261
- cid: b2.cid,
262
- bytes: b2.bytes
263
- }
264
- ]);
265
- });
266
- it('provide large CID root', async () => {
267
- const bytes = new Uint8Array(512).fill(1);
268
- const b1 = await Block.encode({
269
- value: { hello: 'world' },
270
- codec: CBOR,
271
- hasher: sha256
272
- });
273
- const b2 = {
274
- cid: CID.createV1(Raw.code, identity.digest(bytes)),
275
- bytes
276
- };
277
- const headerSize = CBOR.encode({
278
- version: 1,
279
- roots: [
280
- b1.cid,
281
- b2.cid
282
- ]
283
- }).byteLength;
284
- const bodySize = CarBufferWriter.blockLength(b1) + CarBufferWriter.blockLength(b2);
285
- const varintSize = varint.encodingLength(headerSize);
286
- const writer = CarBufferWriter.createWriter(new ArrayBuffer(varintSize + headerSize + bodySize), {
287
- roots: [
288
- b1.cid,
289
- b2.cid
290
- ]
291
- });
292
- writer.write(b1);
293
- writer.write(b2);
294
- const car = writer.close();
295
- const reader = await CarReader.fromBytes(car);
296
- assert.deepEqual(await reader.getRoots(), [
297
- b1.cid,
298
- b2.cid
299
- ]);
300
- assert.deepEqual(reader._blocks, [
301
- {
302
- cid: b1.cid,
303
- bytes: b1.bytes
304
- },
305
- {
306
- cid: b2.cid,
307
- bytes: b2.bytes
308
- }
309
- ]);
310
- });
311
- });
@@ -1,91 +0,0 @@
1
- import { bytes } from 'multiformats';
2
- import { encode as cbEncode } from '@ipld/dag-cbor';
3
- import { encode as vEncode } from 'varint';
4
- import { CarReader } from '../lib/reader.js';
5
- import {
6
- carBytes,
7
- assert,
8
- goCarV2Bytes
9
- } from './common.js';
10
- function makeHeader(block) {
11
- const u = cbEncode(block);
12
- const l = vEncode(u.length);
13
- const u2 = new Uint8Array(u.length + l.length);
14
- u2.set(l, 0);
15
- u2.set(u, l.length);
16
- return u2;
17
- }
18
- describe('Misc errors', () => {
19
- const buf = carBytes.slice();
20
- it('decode errors', async () => {
21
- const buf2 = new Uint8Array(buf.length);
22
- buf2.set(buf, 0);
23
- buf2[101] = 0;
24
- await assert.isRejected(CarReader.fromBytes(buf2), {
25
- name: 'Error',
26
- message: 'Unexpected CID version (0)'
27
- });
28
- });
29
- it('bad version', async () => {
30
- const buf2 = bytes.fromHex('0aa16776657273696f6e03');
31
- assert.strictEqual(bytes.toHex(makeHeader({ version: 3 })), '0aa16776657273696f6e03');
32
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 3');
33
- });
34
- describe('bad header', async () => {
35
- it('sanity check', async () => {
36
- const buf2 = makeHeader({
37
- version: 1,
38
- roots: []
39
- });
40
- await assert.isFulfilled(CarReader.fromBytes(buf2));
41
- });
42
- it('no \'version\' array', async () => {
43
- const buf2 = makeHeader({ roots: [] });
44
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
45
- });
46
- it('bad \'version\' type', async () => {
47
- const buf2 = makeHeader({
48
- version: '1',
49
- roots: []
50
- });
51
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
52
- });
53
- it('no \'roots\' array', async () => {
54
- const buf2 = makeHeader({ version: 1 });
55
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
56
- });
57
- it('bad \'roots\' type', async () => {
58
- const buf2 = makeHeader({
59
- version: 1,
60
- roots: {}
61
- });
62
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
63
- });
64
- it('extraneous properties', async () => {
65
- const buf2 = makeHeader({
66
- version: 1,
67
- roots: [],
68
- blip: true
69
- });
70
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
71
- });
72
- it('not an object', async () => {
73
- const buf2 = makeHeader([
74
- 1,
75
- []
76
- ]);
77
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
78
- });
79
- it('not an object', async () => {
80
- const buf2 = makeHeader(null);
81
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format');
82
- });
83
- it('recursive v2 header', async () => {
84
- const v2Header = goCarV2Bytes.slice(0, 51);
85
- const buf2 = new Uint8Array(51 * 2);
86
- buf2.set(v2Header, 0);
87
- buf2.set(v2Header, 51);
88
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2 (expected 1)');
89
- });
90
- });
91
- });