@ipld/car 4.1.6 → 5.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +12 -4
  3. package/dist/index.min.js +5 -0
  4. package/{types → dist/src}/api.d.ts +19 -19
  5. package/dist/src/api.d.ts.map +1 -0
  6. package/{types/lib → dist/src}/buffer-writer.d.ts +10 -10
  7. package/{types/lib → dist/src}/buffer-writer.d.ts.map +1 -1
  8. package/dist/src/coding.d.ts +41 -0
  9. package/dist/src/coding.d.ts.map +1 -0
  10. package/{types/lib → dist/src}/decoder.d.ts +3 -3
  11. package/{types/lib → dist/src}/decoder.d.ts.map +1 -1
  12. package/{types/lib → dist/src}/encoder.d.ts +2 -2
  13. package/{types/lib → dist/src}/encoder.d.ts.map +1 -1
  14. package/{types/lib → dist/src}/header-validator.d.ts +0 -0
  15. package/dist/src/header-validator.d.ts.map +1 -0
  16. package/dist/src/index-browser.d.ts +8 -0
  17. package/dist/src/index-browser.d.ts.map +1 -0
  18. package/dist/src/index.d.ts +9 -0
  19. package/dist/src/index.d.ts.map +1 -0
  20. package/{types/lib → dist/src}/indexed-reader-browser.d.ts +0 -0
  21. package/dist/src/indexed-reader-browser.d.ts.map +1 -0
  22. package/{types/lib → dist/src}/indexed-reader.d.ts +13 -13
  23. package/dist/src/indexed-reader.d.ts.map +1 -0
  24. package/{types/lib → dist/src}/indexer.d.ts +9 -9
  25. package/{types/lib → dist/src}/indexer.d.ts.map +1 -1
  26. package/{types/lib → dist/src}/iterator-channel.d.ts +0 -0
  27. package/{types/lib → dist/src}/iterator-channel.d.ts.map +1 -1
  28. package/{types/lib → dist/src}/iterator.d.ts +8 -8
  29. package/{types/lib → dist/src}/iterator.d.ts.map +1 -1
  30. package/{types/lib → dist/src}/reader-browser.d.ts +10 -10
  31. package/{types/lib → dist/src}/reader-browser.d.ts.map +1 -1
  32. package/{types/lib → dist/src}/reader.d.ts +5 -5
  33. package/{types/lib → dist/src}/reader.d.ts.map +1 -1
  34. package/{types/lib → dist/src}/writer-browser.d.ts +10 -10
  35. package/{types/lib → dist/src}/writer-browser.d.ts.map +1 -1
  36. package/{types/lib → dist/src}/writer.d.ts +4 -4
  37. package/{types/lib → dist/src}/writer.d.ts.map +1 -1
  38. package/package.json +102 -209
  39. package/{api.ts → src/api.ts} +14 -14
  40. package/{lib → src}/buffer-writer.js +5 -5
  41. package/src/coding.ts +56 -0
  42. package/{lib → src}/decoder.js +7 -7
  43. package/{lib → src}/encoder.js +1 -1
  44. package/src/header-validator.js +74 -0
  45. package/{lib → src}/header.ipldsch +0 -0
  46. package/src/index-browser.js +14 -0
  47. package/src/index.js +16 -0
  48. package/{lib → src}/indexed-reader-browser.js +0 -0
  49. package/{lib → src}/indexed-reader.js +9 -9
  50. package/{lib → src}/indexer.js +4 -4
  51. package/{lib → src}/iterator-channel.js +0 -0
  52. package/{lib → src}/iterator.js +4 -4
  53. package/{lib → src}/reader-browser.js +7 -7
  54. package/{lib → src}/reader.js +5 -5
  55. package/{lib → src}/writer-browser.js +7 -7
  56. package/{lib → src}/writer.js +3 -3
  57. package/LICENSE-APACHE +0 -13
  58. package/LICENSE-MIT +0 -7
  59. package/buffer-writer +0 -1
  60. package/car-browser.js +0 -14
  61. package/car.js +0 -16
  62. package/cjs/browser-test/common.js +0 -301
  63. package/cjs/browser-test/node-test-file-streams.js +0 -71
  64. package/cjs/browser-test/node-test-indexed-reader.js +0 -51
  65. package/cjs/browser-test/node-test-large.js +0 -135
  66. package/cjs/browser-test/node-test-raw.js +0 -68
  67. package/cjs/browser-test/node-test-updateroots.js +0 -78
  68. package/cjs/browser-test/test-buffer-writer.js +0 -330
  69. package/cjs/browser-test/test-errors.js +0 -90
  70. package/cjs/browser-test/test-indexer.js +0 -80
  71. package/cjs/browser-test/test-interface.js +0 -21
  72. package/cjs/browser-test/test-iterator.js +0 -83
  73. package/cjs/browser-test/test-reader.js +0 -278
  74. package/cjs/browser-test/test-writer.js +0 -286
  75. package/cjs/browser-test/verify-store-reader.js +0 -142
  76. package/cjs/car-browser.js +0 -18
  77. package/cjs/car.js +0 -20
  78. package/cjs/lib/buffer-writer.js +0 -164
  79. package/cjs/lib/decoder.js +0 -301
  80. package/cjs/lib/encoder.js +0 -44
  81. package/cjs/lib/header-validator.js +0 -29
  82. package/cjs/lib/indexed-reader-browser.js +0 -13
  83. package/cjs/lib/indexed-reader.js +0 -90
  84. package/cjs/lib/indexer.js +0 -41
  85. package/cjs/lib/iterator-channel.js +0 -77
  86. package/cjs/lib/iterator.js +0 -97
  87. package/cjs/lib/reader-browser.js +0 -62
  88. package/cjs/lib/reader.js +0 -38
  89. package/cjs/lib/writer-browser.js +0 -118
  90. package/cjs/lib/writer.js +0 -51
  91. package/cjs/node-test/common.js +0 -301
  92. package/cjs/node-test/node-test-file-streams.js +0 -71
  93. package/cjs/node-test/node-test-indexed-reader.js +0 -51
  94. package/cjs/node-test/node-test-large.js +0 -135
  95. package/cjs/node-test/node-test-raw.js +0 -68
  96. package/cjs/node-test/node-test-updateroots.js +0 -78
  97. package/cjs/node-test/test-buffer-writer.js +0 -330
  98. package/cjs/node-test/test-errors.js +0 -90
  99. package/cjs/node-test/test-indexer.js +0 -80
  100. package/cjs/node-test/test-interface.js +0 -21
  101. package/cjs/node-test/test-iterator.js +0 -83
  102. package/cjs/node-test/test-reader.js +0 -278
  103. package/cjs/node-test/test-writer.js +0 -286
  104. package/cjs/node-test/verify-store-reader.js +0 -142
  105. package/decoder +0 -1
  106. package/esm/browser-test/common.js +0 -274
  107. package/esm/browser-test/node-test-file-streams.js +0 -76
  108. package/esm/browser-test/node-test-indexed-reader.js +0 -53
  109. package/esm/browser-test/node-test-large.js +0 -109
  110. package/esm/browser-test/node-test-raw.js +0 -63
  111. package/esm/browser-test/node-test-updateroots.js +0 -76
  112. package/esm/browser-test/test-buffer-writer.js +0 -311
  113. package/esm/browser-test/test-errors.js +0 -91
  114. package/esm/browser-test/test-indexer.js +0 -85
  115. package/esm/browser-test/test-interface.js +0 -24
  116. package/esm/browser-test/test-iterator.js +0 -91
  117. package/esm/browser-test/test-reader.js +0 -270
  118. package/esm/browser-test/test-writer.js +0 -297
  119. package/esm/browser-test/verify-store-reader.js +0 -122
  120. package/esm/car-browser.js +0 -16
  121. package/esm/car.js +0 -18
  122. package/esm/lib/buffer-writer.js +0 -129
  123. package/esm/lib/decoder.js +0 -265
  124. package/esm/lib/encoder.js +0 -35
  125. package/esm/lib/header-validator.js +0 -23
  126. package/esm/lib/indexed-reader-browser.js +0 -6
  127. package/esm/lib/indexed-reader.js +0 -78
  128. package/esm/lib/indexer.js +0 -38
  129. package/esm/lib/iterator-channel.js +0 -71
  130. package/esm/lib/iterator.js +0 -92
  131. package/esm/lib/reader-browser.js +0 -57
  132. package/esm/lib/reader.js +0 -26
  133. package/esm/lib/writer-browser.js +0 -115
  134. package/esm/lib/writer.js +0 -42
  135. package/esm/node-test/common.js +0 -274
  136. package/esm/node-test/node-test-file-streams.js +0 -76
  137. package/esm/node-test/node-test-indexed-reader.js +0 -53
  138. package/esm/node-test/node-test-large.js +0 -109
  139. package/esm/node-test/node-test-raw.js +0 -63
  140. package/esm/node-test/node-test-updateroots.js +0 -76
  141. package/esm/node-test/test-buffer-writer.js +0 -311
  142. package/esm/node-test/test-errors.js +0 -91
  143. package/esm/node-test/test-indexer.js +0 -85
  144. package/esm/node-test/test-interface.js +0 -24
  145. package/esm/node-test/test-iterator.js +0 -91
  146. package/esm/node-test/test-reader.js +0 -270
  147. package/esm/node-test/test-writer.js +0 -297
  148. package/esm/node-test/verify-store-reader.js +0 -122
  149. package/esm/package.json +0 -9
  150. package/examples/car-to-fixture.js +0 -66
  151. package/examples/dump-car.js +0 -58
  152. package/examples/dump-index.js +0 -24
  153. package/examples/package.json +0 -27
  154. package/examples/round-trip.js +0 -45
  155. package/examples/test-examples.js +0 -115
  156. package/examples/verify-car.js +0 -87
  157. package/index.js +0 -1
  158. package/indexed-reader +0 -1
  159. package/indexer +0 -1
  160. package/iterator +0 -1
  161. package/lib/coding.ts +0 -56
  162. package/lib/header-validator.js +0 -33
  163. package/reader +0 -1
  164. package/test/_fixtures_to_js.mjs +0 -24
  165. package/test/common.js +0 -210
  166. package/test/go.car +0 -0
  167. package/test/go.carv2 +0 -0
  168. package/test/node-test-file-streams.js +0 -84
  169. package/test/node-test-indexed-reader.js +0 -45
  170. package/test/node-test-large.js +0 -112
  171. package/test/node-test-raw.js +0 -83
  172. package/test/node-test-updateroots.js +0 -89
  173. package/test/test-buffer-writer.js +0 -256
  174. package/test/test-errors.js +0 -98
  175. package/test/test-indexer.js +0 -87
  176. package/test/test-interface.js +0 -24
  177. package/test/test-iterator.js +0 -74
  178. package/test/test-reader.js +0 -245
  179. package/test/test-writer.js +0 -336
  180. package/test/verify-store-reader.js +0 -191
  181. package/tsconfig.json +0 -48
  182. package/types/api.d.ts.map +0 -1
  183. package/types/car-browser.d.ts +0 -8
  184. package/types/car-browser.d.ts.map +0 -1
  185. package/types/car.d.ts +0 -9
  186. package/types/car.d.ts.map +0 -1
  187. package/types/lib/coding.d.ts +0 -41
  188. package/types/lib/coding.d.ts.map +0 -1
  189. package/types/lib/header-validator.d.ts.map +0 -1
  190. package/types/lib/indexed-reader-browser.d.ts.map +0 -1
  191. package/types/lib/indexed-reader.d.ts.map +0 -1
  192. package/types/test/_fixtures_to_js.d.mts +0 -3
  193. package/types/test/_fixtures_to_js.d.mts.map +0 -1
  194. package/types/test/common.d.ts +0 -73
  195. package/types/test/common.d.ts.map +0 -1
  196. package/types/test/fixtures-expectations.d.ts +0 -63
  197. package/types/test/fixtures-expectations.d.ts.map +0 -1
  198. package/types/test/fixtures.d.ts +0 -3
  199. package/types/test/fixtures.d.ts.map +0 -1
  200. package/types/test/node-test-file-streams.d.ts +0 -3
  201. package/types/test/node-test-file-streams.d.ts.map +0 -1
  202. package/types/test/node-test-indexed-reader.d.ts +0 -2
  203. package/types/test/node-test-indexed-reader.d.ts.map +0 -1
  204. package/types/test/node-test-large.d.ts +0 -2
  205. package/types/test/node-test-large.d.ts.map +0 -1
  206. package/types/test/node-test-raw.d.ts +0 -2
  207. package/types/test/node-test-raw.d.ts.map +0 -1
  208. package/types/test/node-test-updateroots.d.ts +0 -2
  209. package/types/test/node-test-updateroots.d.ts.map +0 -1
  210. package/types/test/test-buffer-writer.d.ts +0 -2
  211. package/types/test/test-buffer-writer.d.ts.map +0 -1
  212. package/types/test/test-errors.d.ts +0 -2
  213. package/types/test/test-errors.d.ts.map +0 -1
  214. package/types/test/test-indexer.d.ts +0 -2
  215. package/types/test/test-indexer.d.ts.map +0 -1
  216. package/types/test/test-interface.d.ts +0 -2
  217. package/types/test/test-interface.d.ts.map +0 -1
  218. package/types/test/test-iterator.d.ts +0 -2
  219. package/types/test/test-iterator.d.ts.map +0 -1
  220. package/types/test/test-reader.d.ts +0 -2
  221. package/types/test/test-reader.d.ts.map +0 -1
  222. package/types/test/test-writer.d.ts +0 -2
  223. package/types/test/test-writer.d.ts.map +0 -1
  224. package/types/test/verify-store-reader.d.ts +0 -29
  225. package/types/test/verify-store-reader.d.ts.map +0 -1
  226. package/writer +0 -1
@@ -1,45 +0,0 @@
1
- /* eslint-env mocha */
2
-
3
- import path from 'path'
4
- import { fileURLToPath } from 'url'
5
- import { CarIndexedReader } from '@ipld/car/indexed-reader'
6
- import { assert, goCarIndex } from './common.js'
7
- import {
8
- verifyRoots,
9
- verifyHas,
10
- verifyGet,
11
- verifyBlocks,
12
- verifyCids
13
- } from './verify-store-reader.js'
14
-
15
- const __filename = fileURLToPath(import.meta.url)
16
- const __dirname = path.dirname(__filename)
17
-
18
- describe('CarIndexedReader fromFile()', () => {
19
- it('complete', async () => {
20
- const reader = await CarIndexedReader.fromFile(path.join(__dirname, 'go.car'))
21
- await verifyRoots(reader)
22
- await verifyHas(reader)
23
- await verifyGet(reader)
24
- await verifyBlocks(reader.blocks(), true)
25
- await verifyCids(reader.cids(), true)
26
- // now verify the ordering is correct
27
- let i = 0
28
- for await (const block of reader.blocks()) {
29
- assert.strictEqual(block.cid.toString(), goCarIndex[i++].cid.toString())
30
- }
31
- i = 0
32
- for await (const cid of reader.cids()) {
33
- assert.strictEqual(cid.toString(), goCarIndex[i++].cid.toString())
34
- }
35
- assert.strictEqual(reader.version, 1)
36
- await reader.close()
37
- })
38
-
39
- it('bad argument', async () => {
40
- for (const arg of [true, false, null, undefined, Uint8Array.from([1, 2, 3]), 100, { obj: 'nope' }]) {
41
- // @ts-ignore
42
- await assert.isRejected(CarIndexedReader.fromFile(arg))
43
- }
44
- })
45
- })
@@ -1,112 +0,0 @@
1
- /* eslint-env mocha */
2
-
3
- // this test is not intended to run in the browser
4
-
5
- import fs from 'fs'
6
- import { Readable } from 'stream'
7
- import { garbage } from 'ipld-garbage'
8
- import varint from 'varint'
9
- import * as dagCbor from '@ipld/dag-cbor'
10
- import { sha256 } from 'multiformats/hashes/sha2'
11
- import { CID } from 'multiformats/cid'
12
- import { CarWriter, CarIndexer, CarReader, CarIndexedReader } from '@ipld/car'
13
- import { assert } from './common.js'
14
-
15
- /** @typedef {import('../api').BlockIndex} BlockIndex */
16
-
17
- describe('Large CAR', () => {
18
- /** @type {any[]} */
19
- const objects = []
20
- /** @type {string[]} */
21
- const cids = []
22
- /** @type {BlockIndex[]} */
23
- const expectedIndex = []
24
-
25
- it('create, no roots', async () => {
26
- const { writer, out } = CarWriter.create([])
27
- Readable.from(out).pipe(fs.createWriteStream('./test.car'))
28
-
29
- // offset starts at header length
30
- let offset = dagCbor.encode({ version: 1, roots: [] }).length
31
- offset += varint.encode(offset).length
32
-
33
- for (let i = 0; i < 500; i++) {
34
- const obj = garbage(1000)
35
- objects.push(obj)
36
- const bytes = dagCbor.encode(obj)
37
- const hash = await sha256.digest(bytes)
38
- const cid = CID.create(1, dagCbor.code, hash)
39
- cids.push(cid.toString())
40
- const blockLength = bytes.length
41
- let length = cid.bytes.length + blockLength
42
- const lengthLength = varint.encode(length).length
43
- length += lengthLength
44
- const blockOffset = offset + lengthLength + cid.bytes.length
45
- expectedIndex.push({ cid, offset, length, blockOffset, blockLength })
46
- offset += length
47
- await writer.put({ cid, bytes })
48
- }
49
-
50
- await writer.close()
51
- })
52
-
53
- it('CarIndexer.fromIterable', async () => {
54
- const indexer = await CarIndexer.fromIterable(fs.createReadStream('./test.car'))
55
- assert.deepStrictEqual(await indexer.getRoots(), [])
56
- let i = 0
57
- for await (const blockIndex of indexer) {
58
- assert.deepStrictEqual(blockIndex, expectedIndex[i])
59
- i++
60
- }
61
- })
62
-
63
- it('CarIndexer.fromBytes', async () => {
64
- const indexer = await CarIndexer.fromBytes(await fs.promises.readFile('./test.car'))
65
- assert.deepStrictEqual(await indexer.getRoots(), [])
66
- let i = 0
67
- for await (const blockIndex of indexer) {
68
- assert.deepStrictEqual(blockIndex, expectedIndex[i])
69
- i++
70
- }
71
- })
72
-
73
- it('CarReader.fromBytes', async () => {
74
- const reader = await CarReader.fromBytes(await fs.promises.readFile('./test.car'))
75
- assert.deepStrictEqual(await reader.getRoots(), [])
76
- let i = 0
77
- for await (const { cid, bytes } of reader.blocks()) {
78
- assert.strictEqual(cid.toString(), cids[i], `cid #${i} ${cid} <> ${cids[i]}`)
79
- const obj = dagCbor.decode(bytes)
80
- assert.deepStrictEqual(obj, objects[i], `object #${i}`)
81
- i++
82
- }
83
- })
84
-
85
- it('CarReader.fromIterable', async () => {
86
- const reader = await CarReader.fromIterable(fs.createReadStream('./test.car'))
87
- assert.deepStrictEqual(await reader.getRoots(), [])
88
- let i = 0
89
- for await (const { cid, bytes } of reader.blocks()) {
90
- assert.strictEqual(cid.toString(), cids[i], `cid #${i} ${cid} <> ${cids[i]}`)
91
- const obj = dagCbor.decode(bytes)
92
- assert.deepStrictEqual(obj, objects[i], `object #${i}`)
93
- i++
94
- }
95
- })
96
-
97
- it('CarIndexedReader.fromFile', async () => {
98
- const reader = await CarIndexedReader.fromFile('./test.car')
99
- assert.deepStrictEqual(await reader.getRoots(), [])
100
- let i = 0
101
- for await (const { cid, bytes } of reader.blocks()) {
102
- assert.strictEqual(cid.toString(), cids[i], `cid #${i} ${cid} <> ${cids[i]}`)
103
- const obj = dagCbor.decode(bytes)
104
- assert.deepStrictEqual(obj, objects[i], `object #${i}`)
105
- i++
106
- }
107
- })
108
-
109
- after(async () => {
110
- return fs.promises.unlink('./test.car').catch(() => {})
111
- })
112
- })
@@ -1,83 +0,0 @@
1
- /* eslint-env mocha */
2
-
3
- // this test is not intended to run in the browser
4
-
5
- import path from 'path'
6
- import fs from 'fs'
7
- import { promisify } from 'util'
8
- import { fileURLToPath } from 'url'
9
- import { bytes } from 'multiformats'
10
- import { CarReader } from '@ipld/car'
11
- import { assert, makeData, goCarIndex } from './common.js'
12
-
13
- /** @typedef {import('../api').Block} Block */
14
-
15
- const fsopen = promisify(fs.open)
16
- const fsclose = promisify(fs.close)
17
-
18
- const { toHex } = bytes
19
- const __filename = fileURLToPath(import.meta.url)
20
- const __dirname = path.dirname(__filename)
21
-
22
- describe('CarReader.readRaw', () => {
23
- /** @type {Block[]} */
24
- let allBlocksFlattened
25
-
26
- before(async () => {
27
- const data = await makeData()
28
- allBlocksFlattened = data.allBlocksFlattened
29
- })
30
-
31
- /**
32
- * @param {fs.promises.FileHandle | number} fd
33
- */
34
- async function verifyRead (fd) {
35
- const expectedBlocks = allBlocksFlattened.slice()
36
- const expectedCids = []
37
- for (const { cid } of expectedBlocks) {
38
- expectedCids.push(cid.toString())
39
- }
40
-
41
- for (const blockIndex of goCarIndex) {
42
- const { cid, bytes } = await CarReader.readRaw(fd, blockIndex)
43
- const index = expectedCids.indexOf(cid.toString())
44
- assert.ok(index >= 0, 'got expected block')
45
- assert.strictEqual(
46
- toHex(expectedBlocks[index].bytes),
47
- toHex(bytes),
48
- 'got expected block content')
49
- expectedBlocks.splice(index, 1)
50
- expectedCids.splice(index, 1)
51
- }
52
- assert.strictEqual(expectedBlocks.length, 0, 'got all expected blocks')
53
- }
54
-
55
- it('read raw using index (fd)', async () => {
56
- const fd = await fsopen(path.join(__dirname, 'go.car'), 'r')
57
- await verifyRead(fd)
58
- await fsclose(fd)
59
- })
60
-
61
- it('read raw using index (FileHandle)', async () => {
62
- const fd = await fs.promises.open(path.join(__dirname, 'go.car'), 'r')
63
- await verifyRead(fd)
64
- await fd.close()
65
- })
66
-
67
- it('errors', async () => {
68
- // @ts-ignore
69
- await assert.isRejected(CarReader.readRaw(true, goCarIndex[0]), {
70
- name: 'TypeError',
71
- message: 'Bad fd'
72
- })
73
-
74
- const badBlock = Object.assign({}, goCarIndex[goCarIndex.length - 1])
75
- badBlock.blockLength += 10
76
- const fd = await fsopen(path.join(__dirname, 'go.car'), 'r')
77
- await assert.isRejected(CarReader.readRaw(fd, badBlock), {
78
- name: 'Error',
79
- message: `Failed to read entire block (${badBlock.blockLength - 10} instead of ${badBlock.blockLength})`
80
- })
81
- await fsclose(fd)
82
- })
83
- })
@@ -1,89 +0,0 @@
1
- /* eslint-env mocha */
2
-
3
- // this test is not intended to run in the browser
4
-
5
- import path from 'path'
6
- import { fileURLToPath } from 'url'
7
- import fs from 'fs'
8
- import { promisify } from 'util'
9
- import { CID } from 'multiformats/cid'
10
- import { CarReader } from '@ipld/car/reader'
11
- import { CarWriter } from '@ipld/car/writer'
12
- import {
13
- verifyRoots,
14
- verifyHas,
15
- verifyGet,
16
- verifyBlocks,
17
- verifyCids
18
- } from './verify-store-reader.js'
19
- import { assert } from './common.js'
20
-
21
- const fsopen = promisify(fs.open)
22
- const fsclose = promisify(fs.close)
23
-
24
- const __filename = fileURLToPath(import.meta.url)
25
- const __dirname = path.dirname(__filename)
26
-
27
- const goCarPath = path.join(__dirname, 'go.car')
28
- const tmpCarPath = path.join(__dirname, 'tmp.car')
29
-
30
- const newRoots = [
31
- CID.parse('bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq'),
32
- CID.parse('bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4')
33
- ]
34
-
35
- async function verify () {
36
- const reader = await CarReader.fromIterable(fs.createReadStream(tmpCarPath))
37
- await assert.isRejected(verifyRoots(reader)) // whoa, different roots? like magic
38
- assert.deepEqual(await reader.getRoots(), newRoots)
39
- await verifyHas(reader)
40
- await verifyGet(reader)
41
- await verifyBlocks(reader.blocks(), true)
42
- await verifyCids(reader.cids(), true)
43
- }
44
-
45
- describe('Node CarWriter.updateHeader()', () => {
46
- before(async () => {
47
- try {
48
- await fs.promises.unlink(tmpCarPath)
49
- } catch (e) {}
50
- })
51
-
52
- beforeEach(async () => {
53
- await fs.promises.copyFile(goCarPath, tmpCarPath)
54
- })
55
-
56
- afterEach(async () => {
57
- await fs.promises.unlink(tmpCarPath)
58
- })
59
-
60
- it('update roots (fd)', async () => {
61
- const fd = await fsopen(tmpCarPath, 'r+')
62
- await CarWriter.updateRootsInFile(fd, newRoots)
63
- await fsclose(fd)
64
- await verify()
65
- })
66
-
67
- it('update roots (FileHandle)', async () => {
68
- const fd = await fs.promises.open(tmpCarPath, 'r+')
69
- await CarWriter.updateRootsInFile(fd, newRoots)
70
- await fd.close()
71
- await verify()
72
- })
73
-
74
- it('error: bad fd', async () => {
75
- // @ts-ignore
76
- await assert.isRejected(CarWriter.updateRootsInFile(true, newRoots), {
77
- name: 'TypeError',
78
- message: 'Bad fd'
79
- })
80
- })
81
-
82
- it('error: wrong header size', async () => {
83
- const fd = await fs.promises.open(tmpCarPath, 'r+')
84
- await assert.isRejected(CarWriter.updateRootsInFile(fd, [...newRoots, newRoots[0]]), /can only overwrite a header of the same length/)
85
- await assert.isRejected(CarWriter.updateRootsInFile(fd, [newRoots[0]]), /can only overwrite a header of the same length/)
86
- await assert.isRejected(CarWriter.updateRootsInFile(fd, []), /can only overwrite a header of the same length/)
87
- await fd.close()
88
- })
89
- })
@@ -1,256 +0,0 @@
1
- /* eslint-env mocha */
2
-
3
- import * as CarBufferWriter from '@ipld/car/buffer-writer'
4
- import { CarReader } from '@ipld/car/reader'
5
- import { createHeader } from '../lib/encoder.js'
6
- import { assert } from './common.js'
7
- import { CID, varint } from 'multiformats'
8
- import * as CBOR from '@ipld/dag-cbor'
9
- import { sha256, sha512 } from 'multiformats/hashes/sha2'
10
- import { identity } from 'multiformats/hashes/identity'
11
- import * as Raw from 'multiformats/codecs/raw'
12
- import * as Block from 'multiformats/block'
13
-
14
- describe('CarBufferWriter', () => {
15
- const cid = CID.parse('bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu')
16
- describe('calculateHeaderLength', async () => {
17
- for (const count of [0, 1, 10, 18, 24, 48, 124, 255, 258, 65536 - 1, 65536]) {
18
- it(`calculateHeaderLength(new Array(${count}).fill(36))`, () => {
19
- const roots = new Array(count).fill(cid)
20
- const sizes = new Array(count).fill(cid.bytes.byteLength)
21
- assert.deepEqual(
22
- CarBufferWriter.calculateHeaderLength(sizes),
23
- createHeader(roots).byteLength
24
- )
25
- })
26
- it(`calculateHeaderLength(new Array(${count}).fill(36))`, () => {
27
- const roots = new Array(count).fill(cid)
28
- const rootLengths = roots.map((c) => c.bytes.byteLength)
29
- assert.deepEqual(CarBufferWriter.calculateHeaderLength(rootLengths), createHeader(roots).byteLength)
30
- })
31
- }
32
- it('estimate on large CIDs', () => {
33
- const largeCID = CID.parse(`bafkqbbac${'a'.repeat(416)}`)
34
- assert.equal(
35
- CarBufferWriter.calculateHeaderLength([
36
- cid.bytes.byteLength,
37
- largeCID.bytes.byteLength
38
- ]),
39
- createHeader([
40
- cid,
41
- largeCID
42
- ]).byteLength
43
- )
44
- })
45
-
46
- it('estimate on large CIDs 2', () => {
47
- const largeCID = CID.createV1(Raw.code, identity.digest(new Uint8Array(512).fill(1)))
48
- assert.equal(
49
- CarBufferWriter.calculateHeaderLength([
50
- cid.bytes.byteLength,
51
- largeCID.bytes.byteLength
52
- ]),
53
- createHeader([cid, largeCID]).byteLength
54
- )
55
- })
56
- })
57
-
58
- describe('writer', () => {
59
- it('estimate header and write blocks', async () => {
60
- const headerSize = CarBufferWriter.estimateHeaderLength(1)
61
- const dataSize = 256
62
- const buffer = new ArrayBuffer(headerSize + dataSize)
63
- const writer = CarBufferWriter.createWriter(buffer, { headerSize })
64
- const b1 = await Block.encode({
65
- value: { hello: 'world' },
66
- codec: CBOR,
67
- hasher: sha256
68
- })
69
-
70
- writer.write(b1)
71
-
72
- const b2 = await Block.encode({
73
- value: { bye: 'world' },
74
- codec: CBOR,
75
- hasher: sha256
76
- })
77
- writer.write(b2)
78
-
79
- writer.addRoot(b1.cid)
80
- const bytes = writer.close()
81
-
82
- const reader = await CarReader.fromBytes(bytes)
83
- assert.deepEqual(await reader.getRoots(), [b1.cid])
84
- assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
85
- })
86
-
87
- it('overestimate header', async () => {
88
- const headerSize = CarBufferWriter.estimateHeaderLength(2)
89
- const dataSize = 256
90
- const buffer = new ArrayBuffer(headerSize + dataSize)
91
- const writer = CarBufferWriter.createWriter(buffer, { headerSize })
92
- const b1 = await Block.encode({
93
- value: { hello: 'world' },
94
- codec: CBOR,
95
- hasher: sha256
96
- })
97
-
98
- writer.write(b1)
99
-
100
- const b2 = await Block.encode({
101
- value: { bye: 'world' },
102
- codec: CBOR,
103
- hasher: sha256
104
- })
105
- writer.write(b2)
106
-
107
- writer.addRoot(b1.cid)
108
- assert.throws(() => writer.close(), /Header size was overestimate/)
109
- const bytes = writer.close({ resize: true })
110
-
111
- const reader = await CarReader.fromBytes(bytes)
112
- assert.deepEqual(await reader.getRoots(), [b1.cid])
113
- assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
114
- })
115
-
116
- it('underestimate header', async () => {
117
- const headerSize = CarBufferWriter.estimateHeaderLength(2)
118
- const dataSize = 300
119
- const buffer = new ArrayBuffer(headerSize + dataSize)
120
- const writer = CarBufferWriter.createWriter(buffer, { headerSize })
121
- const b1 = await Block.encode({
122
- value: { hello: 'world' },
123
- codec: CBOR,
124
- hasher: sha256
125
- })
126
-
127
- writer.write(b1)
128
- writer.addRoot(b1.cid)
129
-
130
- const b2 = await Block.encode({
131
- value: { bye: 'world' },
132
- codec: CBOR,
133
- hasher: sha512
134
- })
135
- writer.write(b2)
136
- assert.throws(() => writer.addRoot(b2.cid), /has no capacity/)
137
- writer.addRoot(b2.cid, { resize: true })
138
-
139
- const bytes = writer.close()
140
-
141
- const reader = await CarReader.fromBytes(bytes)
142
- assert.deepEqual(await reader.getRoots(), [b1.cid, b2.cid])
143
- assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
144
- })
145
- })
146
-
147
- it('has no space for the root', async () => {
148
- const headerSize = CarBufferWriter.estimateHeaderLength(1)
149
- const dataSize = 100
150
- const buffer = new ArrayBuffer(headerSize + dataSize)
151
- const writer = CarBufferWriter.createWriter(buffer, { headerSize })
152
- const b1 = await Block.encode({
153
- value: { hello: 'world' },
154
- codec: CBOR,
155
- hasher: sha256
156
- })
157
-
158
- writer.write(b1)
159
- writer.addRoot(b1.cid)
160
-
161
- const b2 = await Block.encode({
162
- value: { bye: 'world' },
163
- codec: CBOR,
164
- hasher: sha256
165
- })
166
- writer.write(b2)
167
- assert.throws(() => writer.addRoot(b2.cid), /Buffer has no capacity for a new root/)
168
- assert.throws(() => writer.addRoot(b2.cid, { resize: true }), /Buffer has no capacity for a new root/)
169
-
170
- const bytes = writer.close()
171
-
172
- const reader = await CarReader.fromBytes(bytes)
173
- assert.deepEqual(await reader.getRoots(), [b1.cid])
174
- assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
175
- })
176
-
177
- it('has no space for the block', async () => {
178
- const headerSize = CarBufferWriter.estimateHeaderLength(1)
179
- const dataSize = 58
180
- const buffer = new ArrayBuffer(headerSize + dataSize)
181
- const writer = CarBufferWriter.createWriter(buffer, { headerSize })
182
- const b1 = await Block.encode({
183
- value: { hello: 'world' },
184
- codec: CBOR,
185
- hasher: sha256
186
- })
187
-
188
- writer.write(b1)
189
- writer.addRoot(b1.cid)
190
-
191
- const b2 = await Block.encode({
192
- value: { bye: 'world' },
193
- codec: CBOR,
194
- hasher: sha256
195
- })
196
- assert.throws(() => writer.write(b2), /Buffer has no capacity for this block/)
197
-
198
- const bytes = writer.close()
199
-
200
- const reader = await CarReader.fromBytes(bytes)
201
- assert.deepEqual(await reader.getRoots(), [b1.cid])
202
- assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }])
203
- })
204
-
205
- it('provide roots', async () => {
206
- const b1 = await Block.encode({
207
- value: { hello: 'world' },
208
- codec: CBOR,
209
- hasher: sha256
210
- })
211
- const b2 = await Block.encode({
212
- value: { bye: 'world' },
213
- codec: CBOR,
214
- hasher: sha512
215
- })
216
-
217
- const buffer = new ArrayBuffer(300)
218
- const writer = CarBufferWriter.createWriter(buffer, { roots: [b1.cid, b2.cid] })
219
-
220
- writer.write(b1)
221
- writer.write(b2)
222
-
223
- const bytes = writer.close()
224
-
225
- const reader = await CarReader.fromBytes(bytes)
226
- assert.deepEqual(await reader.getRoots(), [b1.cid, b2.cid])
227
- assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
228
- })
229
-
230
- it('provide large CID root', async () => {
231
- const bytes = new Uint8Array(512).fill(1)
232
- const b1 = await Block.encode({
233
- value: { hello: 'world' },
234
- codec: CBOR,
235
- hasher: sha256
236
- })
237
-
238
- const b2 = {
239
- cid: CID.createV1(Raw.code, identity.digest(bytes)),
240
- bytes
241
- }
242
-
243
- const headerSize = CBOR.encode({ version: 1, roots: [b1.cid, b2.cid] }).byteLength
244
- const bodySize = CarBufferWriter.blockLength(b1) + CarBufferWriter.blockLength(b2)
245
- const varintSize = varint.encodingLength(headerSize)
246
-
247
- const writer = CarBufferWriter.createWriter(new ArrayBuffer(varintSize + headerSize + bodySize), { roots: [b1.cid, b2.cid] })
248
-
249
- writer.write(b1)
250
- writer.write(b2)
251
- const car = writer.close()
252
- const reader = await CarReader.fromBytes(car)
253
- assert.deepEqual(await reader.getRoots(), [b1.cid, b2.cid])
254
- assert.deepEqual(reader._blocks, [{ cid: b1.cid, bytes: b1.bytes }, { cid: b2.cid, bytes: b2.bytes }])
255
- })
256
- })
@@ -1,98 +0,0 @@
1
- /* eslint-env mocha */
2
- import { bytes } from 'multiformats'
3
- import { encode as cbEncode } from '@ipld/dag-cbor'
4
- import { encode as vEncode } from 'varint'
5
- import { CarReader } from '@ipld/car/reader'
6
- import { carBytes, assert, goCarV2Bytes } from './common.js'
7
-
8
- /**
9
- * @param {any} block
10
- * @returns {Uint8Array}
11
- */
12
- function makeHeader (block) {
13
- const u = cbEncode(block)
14
- const l = vEncode(u.length)
15
- const u2 = new Uint8Array(u.length + l.length)
16
- u2.set(l, 0)
17
- u2.set(u, l.length)
18
- return u2
19
- }
20
-
21
- describe('Misc errors', () => {
22
- const buf = carBytes.slice()
23
-
24
- it('decode errors', async () => {
25
- // cid v0
26
- const buf2 = new Uint8Array(buf.length)
27
- buf2.set(buf, 0)
28
- buf2[101] = 0 // first block's CID
29
- await assert.isRejected(CarReader.fromBytes(buf2), {
30
- name: 'Error',
31
- message: 'Unexpected CID version (0)'
32
- })
33
- })
34
-
35
- it('bad version', async () => {
36
- // quick sanity check that makeHeader() works properly!
37
- const buf2 = bytes.fromHex('0aa16776657273696f6e03')
38
- assert.strictEqual(bytes.toHex(makeHeader({ version: 3 })), '0aa16776657273696f6e03')
39
- // {version:3}
40
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 3')
41
- })
42
-
43
- describe('bad header', async () => {
44
- it('sanity check', async () => {
45
- // sanity check, this should be fine
46
- const buf2 = makeHeader({ version: 1, roots: [] })
47
- await assert.isFulfilled(CarReader.fromBytes(buf2))
48
- })
49
-
50
- it('no \'version\' array', async () => {
51
- const buf2 = makeHeader({ roots: [] })
52
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
53
- })
54
-
55
- it('bad \'version\' type', async () => {
56
- const buf2 = makeHeader({ version: '1', roots: [] })
57
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
58
- })
59
-
60
- it('no \'roots\' array', async () => {
61
- const buf2 = makeHeader({ version: 1 })
62
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
63
- })
64
-
65
- it('bad \'roots\' type', async () => {
66
- const buf2 = makeHeader({ version: 1, roots: {} })
67
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
68
- })
69
-
70
- it('extraneous properties', async () => {
71
- const buf2 = makeHeader({ version: 1, roots: [], blip: true })
72
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
73
- })
74
-
75
- it('not an object', async () => {
76
- const buf2 = makeHeader([1, []])
77
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
78
- })
79
-
80
- it('not an object', async () => {
81
- const buf2 = makeHeader(null)
82
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR header format')
83
- })
84
-
85
- it('recursive v2 header', async () => {
86
- // first 51 bytes are the carv2 header:
87
- // 11b prefix, 16b characteristics, 8b data offset, 8b data size, 8b index offset
88
- const v2Header = goCarV2Bytes.slice(0, 51)
89
- // parser should expect to get a carv1 header at the data offset, but it uses the same
90
- // code to check the carv2 header so let's make sure it doesn't allow recursive carv2
91
- // headers
92
- const buf2 = new Uint8Array(51 * 2)
93
- buf2.set(v2Header, 0)
94
- buf2.set(v2Header, 51)
95
- await assert.isRejected(CarReader.fromBytes(buf2), Error, 'Invalid CAR version: 2 (expected 1)')
96
- })
97
- })
98
- })