@atproto/repo 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/car.d.ts +23 -0
- package/dist/car.d.ts.map +1 -0
- package/dist/car.js +217 -0
- package/dist/car.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mst/mst.d.ts +2 -2
- package/dist/mst/mst.d.ts.map +1 -1
- package/dist/mst/mst.js +3 -3
- package/dist/mst/mst.js.map +1 -1
- package/dist/repo.d.ts +1 -1
- package/dist/repo.d.ts.map +1 -1
- package/dist/repo.js +2 -2
- package/dist/repo.js.map +1 -1
- package/dist/sync/consumer.d.ts.map +1 -1
- package/dist/sync/consumer.js +5 -4
- package/dist/sync/consumer.js.map +1 -1
- package/dist/sync/provider.d.ts.map +1 -1
- package/dist/sync/provider.js +27 -22
- package/dist/sync/provider.js.map +1 -1
- package/dist/types.d.ts +11 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/util.d.ts +2 -28
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +1 -80
- package/dist/util.js.map +1 -1
- package/package.json +5 -5
- package/src/car.ts +219 -0
- package/src/index.ts +1 -0
- package/src/mst/mst.ts +4 -4
- package/src/repo.ts +2 -1
- package/src/sync/consumer.ts +5 -4
- package/src/sync/provider.ts +38 -27
- package/src/util.ts +1 -114
- package/tests/car-file-fixtures.json +28 -0
- package/tests/car.test.ts +58 -0
- package/tests/sync.test.ts +7 -7
- package/tsconfig.build.tsbuildinfo +1 -1
- package/tsconfig.tests.tsbuildinfo +1 -1
- package/tests/util.test.ts +0 -21
package/src/mst/mst.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { BlockWriter } from '@ipld/car/writer'
|
|
2
1
|
import { CID } from 'multiformats'
|
|
3
2
|
import { z } from 'zod'
|
|
4
3
|
import { cidForCbor, dataToCborBlock, schema as common } from '@atproto/common'
|
|
@@ -7,6 +6,7 @@ import { CidSet } from '../cid-set'
|
|
|
7
6
|
import { MissingBlockError, MissingBlocksError } from '../error'
|
|
8
7
|
import * as parse from '../parse'
|
|
9
8
|
import { ReadableBlockstore } from '../storage'
|
|
9
|
+
import { CarBlock } from '../types'
|
|
10
10
|
import * as util from './util'
|
|
11
11
|
|
|
12
12
|
/**
|
|
@@ -726,7 +726,7 @@ export class MST {
|
|
|
726
726
|
|
|
727
727
|
// Sync Protocol
|
|
728
728
|
|
|
729
|
-
async
|
|
729
|
+
async *carBlockStream(): AsyncIterable<CarBlock> {
|
|
730
730
|
const leaves = new CidSet()
|
|
731
731
|
let toFetch = new CidSet()
|
|
732
732
|
toFetch.add(await this.getPointer())
|
|
@@ -742,7 +742,7 @@ export class MST {
|
|
|
742
742
|
cid,
|
|
743
743
|
nodeDataDef,
|
|
744
744
|
)
|
|
745
|
-
|
|
745
|
+
yield { cid, bytes: found.bytes }
|
|
746
746
|
const entries = await util.deserializeNodeData(this.storage, found.obj)
|
|
747
747
|
|
|
748
748
|
for (const entry of entries) {
|
|
@@ -761,7 +761,7 @@ export class MST {
|
|
|
761
761
|
}
|
|
762
762
|
|
|
763
763
|
for (const leaf of leafData.blocks.entries()) {
|
|
764
|
-
|
|
764
|
+
yield leaf
|
|
765
765
|
}
|
|
766
766
|
}
|
|
767
767
|
|
package/src/repo.ts
CHANGED
|
@@ -39,6 +39,7 @@ export class Repo extends ReadableRepo {
|
|
|
39
39
|
did: string,
|
|
40
40
|
keypair: crypto.Keypair,
|
|
41
41
|
initialWrites: RecordCreateOp[] = [],
|
|
42
|
+
revOverride?: string,
|
|
42
43
|
): Promise<CommitData> {
|
|
43
44
|
const newBlocks = new BlockMap()
|
|
44
45
|
|
|
@@ -52,7 +53,7 @@ export class Repo extends ReadableRepo {
|
|
|
52
53
|
const diff = await DataDiff.of(data, null)
|
|
53
54
|
newBlocks.addMap(diff.newMstBlocks)
|
|
54
55
|
|
|
55
|
-
const rev = TID.nextStr()
|
|
56
|
+
const rev = revOverride ?? TID.nextStr()
|
|
56
57
|
const commit = await util.signCommit(
|
|
57
58
|
{
|
|
58
59
|
did,
|
package/src/sync/consumer.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CID } from 'multiformats/cid'
|
|
2
2
|
import { BlockMap } from '../block-map'
|
|
3
|
+
import { readCarWithRoot } from '../car'
|
|
3
4
|
import { DataDiff } from '../data-diff'
|
|
4
5
|
import { MST } from '../mst'
|
|
5
6
|
import { ReadableRepo } from '../readable-repo'
|
|
@@ -18,7 +19,7 @@ export const verifyRepoCar = async (
|
|
|
18
19
|
did?: string,
|
|
19
20
|
signingKey?: string,
|
|
20
21
|
): Promise<VerifiedRepo> => {
|
|
21
|
-
const car = await
|
|
22
|
+
const car = await readCarWithRoot(carBytes)
|
|
22
23
|
return verifyRepo(car.blocks, car.root, did, signingKey)
|
|
23
24
|
}
|
|
24
25
|
|
|
@@ -44,7 +45,7 @@ export const verifyDiffCar = async (
|
|
|
44
45
|
signingKey?: string,
|
|
45
46
|
opts?: { ensureLeaves?: boolean },
|
|
46
47
|
): Promise<VerifiedDiff> => {
|
|
47
|
-
const car = await
|
|
48
|
+
const car = await readCarWithRoot(carBytes)
|
|
48
49
|
return verifyDiff(repo, car.blocks, car.root, did, signingKey, opts)
|
|
49
50
|
}
|
|
50
51
|
|
|
@@ -127,7 +128,7 @@ export const verifyProofs = async (
|
|
|
127
128
|
did: string,
|
|
128
129
|
didKey: string,
|
|
129
130
|
): Promise<{ verified: RecordCidClaim[]; unverified: RecordCidClaim[] }> => {
|
|
130
|
-
const car = await
|
|
131
|
+
const car = await readCarWithRoot(proofs)
|
|
131
132
|
const blockstore = new MemoryBlockstore(car.blocks)
|
|
132
133
|
const commit = await blockstore.readObj(car.root, def.commit)
|
|
133
134
|
if (commit.did !== did) {
|
|
@@ -169,7 +170,7 @@ export const verifyRecords = async (
|
|
|
169
170
|
did: string,
|
|
170
171
|
signingKey: string,
|
|
171
172
|
): Promise<RecordClaim[]> => {
|
|
172
|
-
const car = await
|
|
173
|
+
const car = await readCarWithRoot(proofs)
|
|
173
174
|
const blockstore = new MemoryBlockstore(car.blocks)
|
|
174
175
|
const commit = await blockstore.readObj(car.root, def.commit)
|
|
175
176
|
if (commit.did !== did) {
|
package/src/sync/provider.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { BlockWriter } from '@ipld/car/writer'
|
|
2
1
|
import { CID } from 'multiformats/cid'
|
|
2
|
+
import { writeCarStream } from '../car'
|
|
3
3
|
import { CidSet } from '../cid-set'
|
|
4
4
|
import { MissingBlocksError } from '../error'
|
|
5
5
|
import { MST } from '../mst'
|
|
@@ -14,12 +14,16 @@ export const getFullRepo = (
|
|
|
14
14
|
storage: RepoStorage,
|
|
15
15
|
commitCid: CID,
|
|
16
16
|
): AsyncIterable<Uint8Array> => {
|
|
17
|
-
return
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
17
|
+
return writeCarStream(commitCid, iterateFullRepo(storage, commitCid))
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
|
|
21
|
+
const commit = await storage.readObjAndBytes(commitCid, def.commit)
|
|
22
|
+
yield { cid: commitCid, bytes: commit.bytes }
|
|
23
|
+
const mst = MST.load(storage, commit.obj.data)
|
|
24
|
+
for await (const block of mst.carBlockStream()) {
|
|
25
|
+
yield block
|
|
26
|
+
}
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
// Narrow slices
|
|
@@ -30,24 +34,31 @@ export const getRecords = (
|
|
|
30
34
|
commitCid: CID,
|
|
31
35
|
paths: RecordPath[],
|
|
32
36
|
): AsyncIterable<Uint8Array> => {
|
|
33
|
-
return
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
})
|
|
37
|
+
return writeCarStream(
|
|
38
|
+
commitCid,
|
|
39
|
+
iterateRecordBlocks(storage, commitCid, paths),
|
|
40
|
+
)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function* iterateRecordBlocks(
|
|
44
|
+
storage: ReadableBlockstore,
|
|
45
|
+
commitCid: CID,
|
|
46
|
+
paths: RecordPath[],
|
|
47
|
+
) {
|
|
48
|
+
const commit = await storage.readObjAndBytes(commitCid, def.commit)
|
|
49
|
+
yield { cid: commitCid, bytes: commit.bytes }
|
|
50
|
+
const mst = MST.load(storage, commit.obj.data)
|
|
51
|
+
const cidsForPaths = await Promise.all(
|
|
52
|
+
paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))),
|
|
53
|
+
)
|
|
54
|
+
const allCids = cidsForPaths.reduce((acc, cur) => {
|
|
55
|
+
return acc.addSet(new CidSet(cur))
|
|
56
|
+
}, new CidSet())
|
|
57
|
+
const found = await storage.getBlocks(allCids.toList())
|
|
58
|
+
if (found.missing.length > 0) {
|
|
59
|
+
throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
|
|
60
|
+
}
|
|
61
|
+
for (const block of found.blocks.entries()) {
|
|
62
|
+
yield block
|
|
63
|
+
}
|
|
53
64
|
}
|
package/src/util.ts
CHANGED
|
@@ -1,26 +1,10 @@
|
|
|
1
|
-
import { Readable } from 'node:stream'
|
|
2
|
-
import { setImmediate } from 'node:timers/promises'
|
|
3
|
-
import { CarBlockIterator } from '@ipld/car/iterator'
|
|
4
|
-
import { BlockWriter, CarWriter } from '@ipld/car/writer'
|
|
5
1
|
import * as cbor from '@ipld/dag-cbor'
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
TID,
|
|
9
|
-
byteIterableToStream,
|
|
10
|
-
cborDecode,
|
|
11
|
-
check,
|
|
12
|
-
cidForCbor,
|
|
13
|
-
schema,
|
|
14
|
-
streamToBuffer,
|
|
15
|
-
verifyCidForBytes,
|
|
16
|
-
} from '@atproto/common'
|
|
2
|
+
import { TID, cborDecode, check, cidForCbor, schema } from '@atproto/common'
|
|
17
3
|
import * as crypto from '@atproto/crypto'
|
|
18
4
|
import { Keypair } from '@atproto/crypto'
|
|
19
5
|
import { LexValue, RepoRecord, ipldToLex, lexToIpld } from '@atproto/lexicon'
|
|
20
|
-
import { BlockMap } from './block-map'
|
|
21
6
|
import { DataDiff } from './data-diff'
|
|
22
7
|
import {
|
|
23
|
-
CarBlock,
|
|
24
8
|
Commit,
|
|
25
9
|
LegacyV2Commit,
|
|
26
10
|
RecordCreateDescript,
|
|
@@ -32,103 +16,6 @@ import {
|
|
|
32
16
|
WriteOpAction,
|
|
33
17
|
} from './types'
|
|
34
18
|
|
|
35
|
-
export async function* verifyIncomingCarBlocks(
|
|
36
|
-
car: AsyncIterable<CarBlock>,
|
|
37
|
-
): AsyncIterable<CarBlock> {
|
|
38
|
-
for await (const block of car) {
|
|
39
|
-
await verifyCidForBytes(block.cid, block.bytes)
|
|
40
|
-
yield block
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// we have to turn the car writer output into a stream in order to properly handle errors
|
|
45
|
-
export function writeCarStream(
|
|
46
|
-
root: CID | null,
|
|
47
|
-
fn: (car: BlockWriter) => Promise<void>,
|
|
48
|
-
): Readable {
|
|
49
|
-
const { writer, out } =
|
|
50
|
-
root !== null ? CarWriter.create(root) : CarWriter.create()
|
|
51
|
-
|
|
52
|
-
const stream = byteIterableToStream(out)
|
|
53
|
-
fn(writer)
|
|
54
|
-
.catch((err) => {
|
|
55
|
-
stream.destroy(err)
|
|
56
|
-
})
|
|
57
|
-
.finally(() => writer.close())
|
|
58
|
-
return stream
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
export async function* writeCar(
|
|
62
|
-
root: CID | null,
|
|
63
|
-
fn: (car: BlockWriter) => Promise<void>,
|
|
64
|
-
): AsyncIterable<Uint8Array> {
|
|
65
|
-
const stream = writeCarStream(root, fn)
|
|
66
|
-
for await (const chunk of stream) {
|
|
67
|
-
yield chunk
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export const blocksToCarStream = (
|
|
72
|
-
root: CID | null,
|
|
73
|
-
blocks: BlockMap,
|
|
74
|
-
): AsyncIterable<Uint8Array> => {
|
|
75
|
-
return writeCar(root, async (writer) => {
|
|
76
|
-
for (const entry of blocks.entries()) {
|
|
77
|
-
await writer.put(entry)
|
|
78
|
-
}
|
|
79
|
-
})
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
export const blocksToCarFile = (
|
|
83
|
-
root: CID | null,
|
|
84
|
-
blocks: BlockMap,
|
|
85
|
-
): Promise<Uint8Array> => {
|
|
86
|
-
const carStream = blocksToCarStream(root, blocks)
|
|
87
|
-
return streamToBuffer(carStream)
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
export const carToBlocks = async (
|
|
91
|
-
car: CarBlockIterator,
|
|
92
|
-
): Promise<{ roots: CID[]; blocks: BlockMap }> => {
|
|
93
|
-
const roots = await car.getRoots()
|
|
94
|
-
const blocks = new BlockMap()
|
|
95
|
-
for await (const block of verifyIncomingCarBlocks(car)) {
|
|
96
|
-
blocks.set(block.cid, block.bytes)
|
|
97
|
-
// break up otherwise "synchronous" work in car parsing
|
|
98
|
-
await setImmediate()
|
|
99
|
-
}
|
|
100
|
-
return {
|
|
101
|
-
roots,
|
|
102
|
-
blocks,
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
export const readCar = async (
|
|
107
|
-
bytes: Uint8Array,
|
|
108
|
-
): Promise<{ roots: CID[]; blocks: BlockMap }> => {
|
|
109
|
-
const car = await CarBlockIterator.fromBytes(bytes)
|
|
110
|
-
return carToBlocks(car)
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
export const readCarStream = async (stream: AsyncIterable<Uint8Array>) => {
|
|
114
|
-
const car = await CarBlockIterator.fromIterable(stream)
|
|
115
|
-
return carToBlocks(car)
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
export const readCarWithRoot = async (
|
|
119
|
-
bytes: Uint8Array,
|
|
120
|
-
): Promise<{ root: CID; blocks: BlockMap }> => {
|
|
121
|
-
const { roots, blocks } = await readCar(bytes)
|
|
122
|
-
if (roots.length !== 1) {
|
|
123
|
-
throw new Error(`Expected one root, got ${roots.length}`)
|
|
124
|
-
}
|
|
125
|
-
const root = roots[0]
|
|
126
|
-
return {
|
|
127
|
-
root,
|
|
128
|
-
blocks,
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
19
|
export const diffToWriteDescripts = (
|
|
133
20
|
diff: DataDiff,
|
|
134
21
|
): Promise<RecordWriteDescript[]> => {
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"root": "bafyreiapldaco7m23c7qzc4w42r7kxmcswm64nkindtuh4vwztrpoe7m5m",
|
|
4
|
+
"blocks": [
|
|
5
|
+
{
|
|
6
|
+
"cid": "bafyreiapldaco7m23c7qzc4w42r7kxmcswm64nkindtuh4vwztrpoe7m5m",
|
|
7
|
+
"bytes": "oWR0ZXN0ZHJvb3Q"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"cid": "bafyreieteuyxvbvjbvuhsuo54qx6r3tnjxtp3ub6kb66rdjml3murxjcsy",
|
|
11
|
+
"bytes": "oWR0ZXN0WQEAM32TVA//xgHS9Gtukp0vw6whQ+TnlwF9czt5A7Dxz/URSbryc9Pdw7HESX+jC2oPI/6rwKbhSml2kxJo4MaUeIg/HWI9ixxALw5gIF/I0JC3ejXVAu1Pw6bI9RWa5TgIvAnSow0pJ6jbaaWHlxCpqqHCNHUYbIC14D9k+RK3yS0h2g+O+gRUETQt8t4jOKxEhD037cYEuJCD+fWzFoLkEkrPdUWeqlFQxGt6bflCYjZFgiZKFUo72afR3XM16+jOlhOl+EtuqFcijYJ6MIB53qI8P8HMC2RVH0Mv8UYWLcWatl+CNLykEjesnMar5CvZT8j4w5EyEiS09iD4r6bljg"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"cid": "bafyreiahvdwcywzm35id7hajal5l73p4bnoyaowgckeatd6sbxsacoo5jq",
|
|
15
|
+
"bytes": "oWR0ZXN0WID81CSnkdKi0OBexCStL5gex6Fkehtg7R9Jaww7LFuJC/6Or9Cdb58I+6T9Kjqhf1bf5t5WLbTGt8HOlf1Ysl3wqfdxdxnRZYjyng8YjEmH2Twkc//moluskzywxfOwhRXsXI7SYt36OUkS8AKDzmhTijOG2XWSa2QvU3iSSjP8zw"
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"cid": "bafyreictawwxwjuto6csptbtktbbgwkrryqoakgyb3qovakgiqjzmvzi6a",
|
|
19
|
+
"bytes": "oWR0ZXN0RBnEJSE"
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"cid": "bafyreierfkr6cwv2ux5hk6hp5qh5fhgzyr4yicad5m3pyu3ndaatyqucxu",
|
|
23
|
+
"bytes": "oWR0ZXN0WQH09xLXHen1UTIPEap18egWK6OiRVQ4oLBqjfBVQGdiSsmsIn8uXDM6wp63kCrYWLvXoI0dREVW6+RUjoQIbmDhiKbEUTuLbcFiJnwD5gSlkXdUEwO4UeLfvcAsDJXel5lLaOlOeRTfA3Wf+rEEpr/ccCuwOATBPZBjPrneFVQu8UdsvTu5W144tGxp/ptBsMBqM3yLYkYYHNRrjcAI7iFiszsfKzo28GyM199Jko11mcVhNQ8KHZS72jbsWW/HyfJsL1M/dn6sDCfIaro66mTLRddSQaheQL4NBW5FEgLUBO2VDuBT9fVIl2IwQcijZAOakjLkS1sY2SyUmdsicqGRalnOrVlC5iWQcwHDXLzm9GWz/vfuoF+jzWsdpo6cjT5MIN8uXpzoZRSjH1+UXFxCzFhXkDwL/xJUq8u/0OFGp0mzDc7RLO3gC7X/ENaVPtz/wQaZ3q00EeHvDuiaxHdKIesf/+IkbqS1XjKtaHemB511MFiVf7l2OcqsUU12A5VMreqWPwbAHLgFRDPWiLS0D7yEF3KJX1IupxBmidQMT+SlmCp7FZMdJeHJ3pzpbQv+EoKSXja7it2D2uYsMcTn2DGhlVYsCcQd8PVNKZmPXuC7D82N4sh8p2XVW4LbsDfNTOrgHLX7zRX31VNa47w5Uc03Wuk"
|
|
24
|
+
}
|
|
25
|
+
],
|
|
26
|
+
"car": "OqJlcm9vdHOB2CpYJQABcRIgD1jAJ32a2L8Mi5bmo/VdgpWZ7jVIaOdD8rbM4vcT7OtndmVyc2lvbgEvAXESIA9YwCd9mti/DIuW5qP1XYKVme41SGjnQ/K2zOL3E+zroWR0ZXN0ZHJvb3StAgFxEiCTJTF6hqkNaHlR3eQv6O5tTeb90D5QfeiNLF7ZSN0ilqFkdGVzdFkBADN9k1QP/8YB0vRrbpKdL8OsIUPk55cBfXM7eQOw8c/1EUm68nPT3cOxxEl/owtqDyP+q8Cm4UppdpMSaODGlHiIPx1iPYscQC8OYCBfyNCQt3o11QLtT8OmyPUVmuU4CLwJ0qMNKSeo22mlh5cQqaqhwjR1GGyAteA/ZPkSt8ktIdoPjvoEVBE0LfLeIzisRIQ9N+3GBLiQg/n1sxaC5BJKz3VFnqpRUMRrem35QmI2RYImShVKO9mn0d1zNevozpYTpfhLbqhXIo2CejCAed6iPD/BzAtkVR9DL/FGFi3FmrZfgjS8pBI3rJzGq+Qr2U/I+MORMhIktPYg+K+m5Y6sAQFxEiAHqOwsWyzfUD+cCQL6v+38C12AOsYSiAmP0g3kATndTKFkdGVzdFiA/NQkp5HSotDgXsQkrS+YHsehZHobYO0fSWsMOyxbiQv+jq/QnW+fCPuk/So6oX9W3+beVi20xrfBzpX9WLJd8Kn3cXcZ0WWI8p4PGIxJh9k8JHP/5qJbrJM8sMXzsIUV7FyO0mLd+jlJEvACg85oU4ozhtl1kmtkL1N4kkoz/M8vAXESIFMFrXsmk3eFJ8wzVMITWVGOIOAo2A7g6oFGRBOWVyjwoWR0ZXN0RBnEJSGhBAFxEiCRKqPhWrql+nV47+wP0pzZxHmECAPrNvxTbRgBPEKCvaFkdGVzdFkB9PcS1x3p9VEyDxGqdfHoFiujokVUOKCwao3wVUBnYkrJrCJ/LlwzOsKet5Aq2Fi716CNHURFVuvkVI6ECG5g4YimxFE7i23BYiZ8A+YEpZF3VBMDuFHi373ALAyV3peZS2jpTnkU3wN1n/qxBKa/3HArsDgEwT2QYz653hVULvFHbL07uVteOLRsaf6bQbDAajN8i2JGGBzUa43ACO4hYrM7Hys6NvBsjNffSZKNdZnFYTUPCh2Uu9o27Flvx8nybC9TP3Z+rAwnyGq6Oupky0XXUkGoXkC+DQVuRRIC1ATtlQ7gU/X1SJdiMEHIo2QDmpIy5EtbGNkslJnbInKhkWpZzq1ZQuYlkHMBw1y85vRls/737qBfo81rHaaOnI0+TCDfLl6c6GUUox9flFxcQsxYV5A8C/8SVKvLv9DhRqdJsw3O0Szt4Au1/xDWlT7c/8EGmd6tNBHh7w7omsR3SiHrH//iJG6ktV4yrWh3pgeddTBYlX+5djnKrFFNdgOVTK3qlj8GwBy4BUQz1oi0tA+8hBdyiV9SLqcQZonUDE/kpZgqexWTHSXhyd6c6W0L/hKCkl42u4rdg9rmLDHE59gxoZVWLAnEHfD1TSmZj17guw/NjeLIfKdl1VuC27A3zUzq4By1+80V99VTWuO8OVHNN1rp"
|
|
27
|
+
}
|
|
28
|
+
]
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { CID } from 'multiformats/cid'
|
|
2
|
+
import * as ui8 from 'uint8arrays'
|
|
3
|
+
import { dataToCborBlock, streamToBytes, wait } from '@atproto/common'
|
|
4
|
+
import { CarBlock, readCarStream, writeCarStream } from '../src'
|
|
5
|
+
import fixtures from './car-file-fixtures.json'
|
|
6
|
+
|
|
7
|
+
describe('car', () => {
|
|
8
|
+
for (const fixture of fixtures) {
|
|
9
|
+
it('correctly writes car files', async () => {
|
|
10
|
+
const root = CID.parse(fixture.root)
|
|
11
|
+
async function* blockIter() {
|
|
12
|
+
for (const block of fixture.blocks) {
|
|
13
|
+
const cid = CID.parse(block.cid)
|
|
14
|
+
const bytes = ui8.fromString(block.bytes, 'base64')
|
|
15
|
+
yield { cid, bytes }
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
const carStream = writeCarStream(root, blockIter())
|
|
19
|
+
const car = await streamToBytes(carStream)
|
|
20
|
+
const carB64 = ui8.toString(car, 'base64')
|
|
21
|
+
expect(carB64).toEqual(fixture.car)
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
it('correctly reads carfiles', async () => {
|
|
25
|
+
const carStream = [ui8.fromString(fixture.car, 'base64')]
|
|
26
|
+
const { roots, blocks } = await readCarStream(carStream)
|
|
27
|
+
expect(roots.length).toBe(1)
|
|
28
|
+
expect(roots[0].toString()).toEqual(fixture.root)
|
|
29
|
+
const carBlocks: CarBlock[] = []
|
|
30
|
+
for await (const block of blocks) {
|
|
31
|
+
carBlocks.push(block)
|
|
32
|
+
}
|
|
33
|
+
expect(carBlocks.length).toEqual(fixture.blocks.length)
|
|
34
|
+
for (let i = 0; i < carBlocks.length; i++) {
|
|
35
|
+
expect(carBlocks[i].cid.toString()).toEqual(fixture.blocks[i].cid)
|
|
36
|
+
expect(ui8.toString(carBlocks[i].bytes, 'base64')).toEqual(
|
|
37
|
+
fixture.blocks[i].bytes,
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
})
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
it('writeCar propagates errors', async () => {
|
|
44
|
+
const iterate = async () => {
|
|
45
|
+
async function* blockIterator() {
|
|
46
|
+
await wait(1)
|
|
47
|
+
const block = await dataToCborBlock({ test: 1 })
|
|
48
|
+
yield block
|
|
49
|
+
throw new Error('Oops!')
|
|
50
|
+
}
|
|
51
|
+
const iter = writeCarStream(null, blockIterator())
|
|
52
|
+
for await (const _bytes of iter) {
|
|
53
|
+
// no-op
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
await expect(iterate).rejects.toThrow('Oops!')
|
|
57
|
+
})
|
|
58
|
+
})
|
package/tests/sync.test.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { CarReader } from '@ipld/car/reader'
|
|
2
1
|
import { streamToBuffer } from '@atproto/common'
|
|
3
2
|
import * as crypto from '@atproto/crypto'
|
|
4
3
|
import {
|
|
@@ -7,6 +6,7 @@ import {
|
|
|
7
6
|
RepoContents,
|
|
8
7
|
RepoVerificationError,
|
|
9
8
|
getAndParseRecord,
|
|
9
|
+
readCar,
|
|
10
10
|
readCarWithRoot,
|
|
11
11
|
} from '../src'
|
|
12
12
|
import { MemoryBlockstore } from '../src/storage'
|
|
@@ -55,14 +55,14 @@ describe('Repo Sync', () => {
|
|
|
55
55
|
|
|
56
56
|
it('does not sync duplicate blocks', async () => {
|
|
57
57
|
const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
|
|
58
|
-
const car = await
|
|
58
|
+
const car = await readCar(carBytes)
|
|
59
59
|
const cids = new CidSet()
|
|
60
|
-
|
|
61
|
-
if (cids.has(
|
|
62
|
-
throw new Error(`duplicate block: :${
|
|
60
|
+
car.blocks.forEach((_, cid) => {
|
|
61
|
+
if (cids.has(cid)) {
|
|
62
|
+
throw new Error(`duplicate block: :${cid.toString()}`)
|
|
63
63
|
}
|
|
64
|
-
cids.add(
|
|
65
|
-
}
|
|
64
|
+
cids.add(cid)
|
|
65
|
+
})
|
|
66
66
|
})
|
|
67
67
|
|
|
68
68
|
it('syncs a repo that is behind', async () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"root":["./src/block-map.ts","./src/cid-set.ts","./src/data-diff.ts","./src/error.ts","./src/index.ts","./src/logger.ts","./src/parse.ts","./src/readable-repo.ts","./src/repo.ts","./src/types.ts","./src/util.ts","./src/mst/diff.ts","./src/mst/index.ts","./src/mst/mst.ts","./src/mst/util.ts","./src/mst/walker.ts","./src/storage/index.ts","./src/storage/memory-blockstore.ts","./src/storage/readable-blockstore.ts","./src/storage/sync-storage.ts","./src/storage/types.ts","./src/sync/consumer.ts","./src/sync/index.ts","./src/sync/provider.ts"],"version":"5.6.3"}
|
|
1
|
+
{"root":["./src/block-map.ts","./src/car.ts","./src/cid-set.ts","./src/data-diff.ts","./src/error.ts","./src/index.ts","./src/logger.ts","./src/parse.ts","./src/readable-repo.ts","./src/repo.ts","./src/types.ts","./src/util.ts","./src/mst/diff.ts","./src/mst/index.ts","./src/mst/mst.ts","./src/mst/util.ts","./src/mst/walker.ts","./src/storage/index.ts","./src/storage/memory-blockstore.ts","./src/storage/readable-blockstore.ts","./src/storage/sync-storage.ts","./src/storage/types.ts","./src/sync/consumer.ts","./src/sync/index.ts","./src/sync/provider.ts"],"version":"5.6.3"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"root":["./tests/_keys.ts","./tests/_util.ts","./tests/
|
|
1
|
+
{"root":["./tests/_keys.ts","./tests/_util.ts","./tests/car.test.ts","./tests/commit-data.test.ts","./tests/commit-proofs.test.ts","./tests/covering-proofs.test.ts","./tests/mst.test.ts","./tests/proofs.test.ts","./tests/repo.test.ts","./tests/sync.test.ts"],"version":"5.6.3"}
|
package/tests/util.test.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { dataToCborBlock, wait } from '@atproto/common'
|
|
2
|
-
import { writeCar } from '../src'
|
|
3
|
-
|
|
4
|
-
describe('Utils', () => {
|
|
5
|
-
describe('writeCar()', () => {
|
|
6
|
-
it('propagates errors', async () => {
|
|
7
|
-
const iterate = async () => {
|
|
8
|
-
const iter = writeCar(null, async (car) => {
|
|
9
|
-
await wait(1)
|
|
10
|
-
const block = await dataToCborBlock({ test: 1 })
|
|
11
|
-
await car.put(block)
|
|
12
|
-
throw new Error('Oops!')
|
|
13
|
-
})
|
|
14
|
-
for await (const _bytes of iter) {
|
|
15
|
-
// no-op
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
await expect(iterate).rejects.toThrow('Oops!')
|
|
19
|
-
})
|
|
20
|
-
})
|
|
21
|
-
})
|