@atproto/repo 0.8.13 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/block-map.d.ts +15 -16
- package/dist/block-map.d.ts.map +1 -1
- package/dist/block-map.js +9 -43
- package/dist/block-map.js.map +1 -1
- package/dist/car.d.ts +8 -8
- package/dist/car.d.ts.map +1 -1
- package/dist/car.js +18 -15
- package/dist/car.js.map +1 -1
- package/dist/cid-set.d.ts +8 -7
- package/dist/cid-set.d.ts.map +1 -1
- package/dist/cid-set.js +11 -4
- package/dist/cid-set.js.map +1 -1
- package/dist/data-diff.d.ts +10 -10
- package/dist/data-diff.d.ts.map +1 -1
- package/dist/data-diff.js.map +1 -1
- package/dist/error.d.ts +10 -10
- package/dist/error.d.ts.map +1 -1
- package/dist/error.js.map +1 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +1 -0
- package/dist/logger.js.map +1 -1
- package/dist/mst/mst.d.ts +29 -29
- package/dist/mst/mst.d.ts.map +1 -1
- package/dist/mst/mst.js +12 -12
- package/dist/mst/mst.js.map +1 -1
- package/dist/mst/util.d.ts +2 -2
- package/dist/mst/util.d.ts.map +1 -1
- package/dist/mst/util.js +19 -39
- package/dist/mst/util.js.map +1 -1
- package/dist/parse.d.ts +6 -7
- package/dist/parse.d.ts.map +1 -1
- package/dist/parse.js +2 -2
- package/dist/parse.js.map +1 -1
- package/dist/readable-repo.d.ts +6 -7
- package/dist/readable-repo.d.ts.map +1 -1
- package/dist/readable-repo.js.map +1 -1
- package/dist/repo.d.ts +4 -4
- package/dist/repo.d.ts.map +1 -1
- package/dist/repo.js +11 -9
- package/dist/repo.js.map +1 -1
- package/dist/storage/memory-blockstore.d.ts +9 -9
- package/dist/storage/memory-blockstore.d.ts.map +1 -1
- package/dist/storage/memory-blockstore.js.map +1 -1
- package/dist/storage/readable-blockstore.d.ts +11 -12
- package/dist/storage/readable-blockstore.d.ts.map +1 -1
- package/dist/storage/readable-blockstore.js +2 -35
- package/dist/storage/readable-blockstore.js.map +1 -1
- package/dist/storage/sync-storage.d.ts +5 -5
- package/dist/storage/sync-storage.d.ts.map +1 -1
- package/dist/storage/sync-storage.js.map +1 -1
- package/dist/storage/types.d.ts +25 -26
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/types.js.map +1 -1
- package/dist/sync/consumer.d.ts +3 -3
- package/dist/sync/consumer.d.ts.map +1 -1
- package/dist/sync/consumer.js +1 -1
- package/dist/sync/consumer.js.map +1 -1
- package/dist/sync/provider.d.ts +3 -3
- package/dist/sync/provider.d.ts.map +1 -1
- package/dist/sync/provider.js.map +1 -1
- package/dist/types.d.ts +127 -100
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +53 -12
- package/dist/types.js.map +1 -1
- package/dist/util.d.ts +8 -3
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +33 -16
- package/dist/util.js.map +1 -1
- package/package.json +6 -7
- package/src/block-map.ts +31 -26
- package/src/car.ts +23 -28
- package/src/cid-set.ts +16 -10
- package/src/data-diff.ts +10 -10
- package/src/error.ts +6 -6
- package/src/logger.ts +1 -0
- package/src/mst/mst.ts +24 -26
- package/src/mst/util.ts +24 -9
- package/src/parse.ts +8 -8
- package/src/readable-repo.ts +6 -7
- package/src/repo.ts +13 -10
- package/src/storage/memory-blockstore.ts +8 -8
- package/src/storage/readable-blockstore.ts +12 -13
- package/src/storage/sync-storage.ts +4 -4
- package/src/storage/types.ts +25 -25
- package/src/sync/consumer.ts +5 -5
- package/src/sync/provider.ts +10 -7
- package/src/types.ts +82 -41
- package/src/util.ts +31 -14
- package/tests/_util.ts +28 -12
- package/tests/car.test.ts +31 -11
- package/tests/commit-proofs.test.ts +3 -3
- package/tests/covering-proofs.test.ts +6 -6
- package/tests/mst.test.ts +18 -19
- package/tests/proofs.test.ts +8 -6
- package/tests/repo.test.ts +1 -1
- package/tests/sync.test.ts +3 -6
- package/bench/mst.bench.ts +0 -165
- package/bench/repo.bench.ts +0 -48
package/src/mst/mst.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { CID } from 'multiformats'
|
|
2
1
|
import { z } from 'zod'
|
|
3
|
-
import {
|
|
2
|
+
import { cidForLex, encode } from '@atproto/lex-cbor'
|
|
3
|
+
import { Cid, cidForCbor } from '@atproto/lex-data'
|
|
4
4
|
import { BlockMap } from '../block-map'
|
|
5
5
|
import { CidSet } from '../cid-set'
|
|
6
6
|
import { MissingBlockError, MissingBlocksError } from '../error'
|
|
7
7
|
import * as parse from '../parse'
|
|
8
8
|
import { ReadableBlockstore } from '../storage'
|
|
9
|
-
import { CarBlock } from '../types'
|
|
9
|
+
import { CarBlock, schema } from '../types'
|
|
10
10
|
import * as util from './util'
|
|
11
11
|
|
|
12
12
|
/**
|
|
@@ -16,7 +16,7 @@ import * as util from './util'
|
|
|
16
16
|
* Keys are laid out in alphabetic order.
|
|
17
17
|
* The key insight of an MST is that each key is hashed and starting 0s are counted
|
|
18
18
|
* to determine which layer it falls on (5 zeros for ~32 fanout).
|
|
19
|
-
* This is a merkle tree, so each subtree is referred to by it's hash (
|
|
19
|
+
* This is a merkle tree, so each subtree is referred to by it's hash (Cid).
|
|
20
20
|
* When a leaf is changed, ever tree on the path to that leaf is changed as well,
|
|
21
21
|
* thereby updating the root hash.
|
|
22
22
|
*
|
|
@@ -42,11 +42,11 @@ import * as util from './util'
|
|
|
42
42
|
* Then the first will be described as `prefix: 0, key: 'bsky/posts/abcdefg'`,
|
|
43
43
|
* and the second will be described as `prefix: 16, key: 'hi'.`
|
|
44
44
|
*/
|
|
45
|
-
const subTreePointer = z.nullable(
|
|
45
|
+
const subTreePointer = z.nullable(schema.cid)
|
|
46
46
|
const treeEntry = z.object({
|
|
47
47
|
p: z.number(), // prefix count of ascii chars that this key shares with the prev key
|
|
48
|
-
k:
|
|
49
|
-
v:
|
|
48
|
+
k: schema.bytes, // the rest of the key outside the shared prefix
|
|
49
|
+
v: schema.cid, // value
|
|
50
50
|
t: subTreePointer, // next subtree (to the right of leaf)
|
|
51
51
|
})
|
|
52
52
|
const nodeData = z.object({
|
|
@@ -70,12 +70,12 @@ export class MST {
|
|
|
70
70
|
storage: ReadableBlockstore
|
|
71
71
|
entries: NodeEntry[] | null
|
|
72
72
|
layer: number | null
|
|
73
|
-
pointer:
|
|
73
|
+
pointer: Cid
|
|
74
74
|
outdatedPointer = false
|
|
75
75
|
|
|
76
76
|
constructor(
|
|
77
77
|
storage: ReadableBlockstore,
|
|
78
|
-
pointer:
|
|
78
|
+
pointer: Cid,
|
|
79
79
|
entries: NodeEntry[] | null,
|
|
80
80
|
layer: number | null,
|
|
81
81
|
) {
|
|
@@ -102,14 +102,14 @@ export class MST {
|
|
|
102
102
|
): Promise<MST> {
|
|
103
103
|
const { layer = null } = opts || {}
|
|
104
104
|
const entries = await util.deserializeNodeData(storage, data, opts)
|
|
105
|
-
const pointer = await
|
|
105
|
+
const pointer = await cidForLex(data)
|
|
106
106
|
return new MST(storage, pointer, entries, layer)
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
// this is really a *lazy* load, doesn't actually touch storage
|
|
110
110
|
static load(
|
|
111
111
|
storage: ReadableBlockstore,
|
|
112
|
-
cid:
|
|
112
|
+
cid: Cid,
|
|
113
113
|
opts?: Partial<MstOpts>,
|
|
114
114
|
): MST {
|
|
115
115
|
const { layer = null } = opts || {}
|
|
@@ -145,12 +145,12 @@ export class MST {
|
|
|
145
145
|
|
|
146
146
|
return this.entries
|
|
147
147
|
}
|
|
148
|
-
throw new Error('No entries or
|
|
148
|
+
throw new Error('No entries or Cid provided')
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
// We don't hash the node on every mutation for performance reasons
|
|
152
152
|
// Instead we keep track of whether the pointer is outdated and only (recursively) calculate when needed
|
|
153
|
-
async getPointer(): Promise<
|
|
153
|
+
async getPointer(): Promise<Cid> {
|
|
154
154
|
if (!this.outdatedPointer) return this.pointer
|
|
155
155
|
const { cid } = await this.serialize()
|
|
156
156
|
this.pointer = cid
|
|
@@ -158,7 +158,7 @@ export class MST {
|
|
|
158
158
|
return this.pointer
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
-
async serialize(): Promise<{ cid:
|
|
161
|
+
async serialize(): Promise<{ cid: Cid; bytes: Uint8Array }> {
|
|
162
162
|
let entries = await this.getEntries()
|
|
163
163
|
const outdated = entries.filter(
|
|
164
164
|
(e) => e.isTree() && e.outdatedPointer,
|
|
@@ -168,11 +168,9 @@ export class MST {
|
|
|
168
168
|
entries = await this.getEntries()
|
|
169
169
|
}
|
|
170
170
|
const data = util.serializeNodeData(entries)
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
bytes: block.bytes,
|
|
175
|
-
}
|
|
171
|
+
const bytes = encode(data)
|
|
172
|
+
const cid = await cidForCbor(bytes)
|
|
173
|
+
return { cid, bytes }
|
|
176
174
|
}
|
|
177
175
|
|
|
178
176
|
// In most cases, we get the layer of a node from a hint on creation
|
|
@@ -208,7 +206,7 @@ export class MST {
|
|
|
208
206
|
// -------------------
|
|
209
207
|
|
|
210
208
|
// Return the necessary blocks to persist the MST to repo storage
|
|
211
|
-
async getUnstoredBlocks(): Promise<{ root:
|
|
209
|
+
async getUnstoredBlocks(): Promise<{ root: Cid; blocks: BlockMap }> {
|
|
212
210
|
const blocks = new BlockMap()
|
|
213
211
|
const pointer = await this.getPointer()
|
|
214
212
|
const alreadyHas = await this.storage.has(pointer)
|
|
@@ -227,7 +225,7 @@ export class MST {
|
|
|
227
225
|
|
|
228
226
|
// Adds a new leaf for the given key/value pair
|
|
229
227
|
// Throws if a leaf with that key already exists
|
|
230
|
-
async add(key: string, value:
|
|
228
|
+
async add(key: string, value: Cid, knownZeros?: number): Promise<MST> {
|
|
231
229
|
util.ensureValidMstKey(key)
|
|
232
230
|
const keyZeros = knownZeros ?? (await util.leadingZerosOnHash(key))
|
|
233
231
|
const layer = await this.getLayer()
|
|
@@ -297,7 +295,7 @@ export class MST {
|
|
|
297
295
|
}
|
|
298
296
|
|
|
299
297
|
// Gets the value at the given key
|
|
300
|
-
async get(key: string): Promise<
|
|
298
|
+
async get(key: string): Promise<Cid | null> {
|
|
301
299
|
const index = await this.findGtOrEqualLeafIndex(key)
|
|
302
300
|
const found = await this.atIndex(index)
|
|
303
301
|
if (found && found.isLeaf() && found.key === key) {
|
|
@@ -312,7 +310,7 @@ export class MST {
|
|
|
312
310
|
|
|
313
311
|
// Edits the value at the given key
|
|
314
312
|
// Throws if the given key does not exist
|
|
315
|
-
async update(key: string, value:
|
|
313
|
+
async update(key: string, value: Cid): Promise<MST> {
|
|
316
314
|
util.ensureValidMstKey(key)
|
|
317
315
|
const index = await this.findGtOrEqualLeafIndex(key)
|
|
318
316
|
const found = await this.atIndex(index)
|
|
@@ -765,8 +763,8 @@ export class MST {
|
|
|
765
763
|
}
|
|
766
764
|
}
|
|
767
765
|
|
|
768
|
-
async cidsForPath(key: string): Promise<
|
|
769
|
-
const cids:
|
|
766
|
+
async cidsForPath(key: string): Promise<Cid[]> {
|
|
767
|
+
const cids: Cid[] = [await this.getPointer()]
|
|
770
768
|
const index = await this.findGtOrEqualLeafIndex(key)
|
|
771
769
|
const found = await this.atIndex(index)
|
|
772
770
|
if (found && found.isLeaf() && found.key === key) {
|
|
@@ -873,7 +871,7 @@ export class MST {
|
|
|
873
871
|
export class Leaf {
|
|
874
872
|
constructor(
|
|
875
873
|
public key: string,
|
|
876
|
-
public value:
|
|
874
|
+
public value: Cid,
|
|
877
875
|
) {}
|
|
878
876
|
|
|
879
877
|
isTree(): this is MST {
|
package/src/mst/util.ts
CHANGED
|
@@ -1,10 +1,25 @@
|
|
|
1
|
-
import { CID } from 'multiformats'
|
|
2
|
-
import * as uint8arrays from 'uint8arrays'
|
|
3
|
-
import { cidForCbor } from '@atproto/common'
|
|
4
1
|
import { sha256 } from '@atproto/crypto'
|
|
2
|
+
import { cidForLex } from '@atproto/lex-cbor'
|
|
3
|
+
import { Cid } from '@atproto/lex-data'
|
|
5
4
|
import { ReadableBlockstore } from '../storage'
|
|
6
5
|
import { Leaf, MST, MstOpts, NodeData, NodeEntry } from './mst'
|
|
7
6
|
|
|
7
|
+
function toAscii(bytes: Uint8Array): string {
|
|
8
|
+
let string = ''
|
|
9
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
10
|
+
string += String.fromCharCode(bytes[i])
|
|
11
|
+
}
|
|
12
|
+
return string
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function fromAscii(str: string): Uint8Array {
|
|
16
|
+
const bytes = new Uint8Array(str.length)
|
|
17
|
+
for (let i = 0; i < str.length; i++) {
|
|
18
|
+
bytes[i] = str.charCodeAt(i)
|
|
19
|
+
}
|
|
20
|
+
return bytes
|
|
21
|
+
}
|
|
22
|
+
|
|
8
23
|
export const leadingZerosOnHash = async (key: string | Uint8Array) => {
|
|
9
24
|
const hash = await sha256(key)
|
|
10
25
|
let leadingZeros = 0
|
|
@@ -46,8 +61,8 @@ export const deserializeNodeData = async (
|
|
|
46
61
|
}
|
|
47
62
|
let lastKey = ''
|
|
48
63
|
for (const entry of data.e) {
|
|
49
|
-
const keyStr =
|
|
50
|
-
const key = lastKey.slice(0, entry.p)
|
|
64
|
+
const keyStr = toAscii(entry.k)
|
|
65
|
+
const key = `${lastKey.slice(0, entry.p)}${keyStr}`
|
|
51
66
|
ensureValidMstKey(key)
|
|
52
67
|
entries.push(new Leaf(key, entry.v))
|
|
53
68
|
lastKey = key
|
|
@@ -80,7 +95,7 @@ export const serializeNodeData = (entries: NodeEntry[]): NodeData => {
|
|
|
80
95
|
throw new Error('Not a valid node: two subtrees next to each other')
|
|
81
96
|
}
|
|
82
97
|
i++
|
|
83
|
-
let subtree:
|
|
98
|
+
let subtree: Cid | null = null
|
|
84
99
|
if (next?.isTree()) {
|
|
85
100
|
subtree = next.pointer
|
|
86
101
|
i++
|
|
@@ -89,7 +104,7 @@ export const serializeNodeData = (entries: NodeEntry[]): NodeData => {
|
|
|
89
104
|
const prefixLen = countPrefixLen(lastKey, leaf.key)
|
|
90
105
|
data.e.push({
|
|
91
106
|
p: prefixLen,
|
|
92
|
-
k:
|
|
107
|
+
k: fromAscii(leaf.key.slice(prefixLen)),
|
|
93
108
|
v: leaf.value,
|
|
94
109
|
t: subtree,
|
|
95
110
|
})
|
|
@@ -109,9 +124,9 @@ export const countPrefixLen = (a: string, b: string): number => {
|
|
|
109
124
|
return i
|
|
110
125
|
}
|
|
111
126
|
|
|
112
|
-
export const cidForEntries = async (entries: NodeEntry[]): Promise<
|
|
127
|
+
export const cidForEntries = async (entries: NodeEntry[]): Promise<Cid> => {
|
|
113
128
|
const data = serializeNodeData(entries)
|
|
114
|
-
return
|
|
129
|
+
return cidForLex(data)
|
|
115
130
|
}
|
|
116
131
|
|
|
117
132
|
export const isValidMstKey = (str: string): boolean => {
|
package/src/parse.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { check } from '@atproto/common-web'
|
|
2
|
+
import { decode } from '@atproto/lex-cbor'
|
|
3
|
+
import { Cid, LexMap } from '@atproto/lex-data'
|
|
4
4
|
import { BlockMap } from './block-map'
|
|
5
5
|
import { MissingBlockError, UnexpectedObjectError } from './error'
|
|
6
6
|
import { cborToLexRecord } from './util'
|
|
7
7
|
|
|
8
8
|
export const getAndParseRecord = async (
|
|
9
9
|
blocks: BlockMap,
|
|
10
|
-
cid:
|
|
11
|
-
): Promise<{ record:
|
|
10
|
+
cid: Cid,
|
|
11
|
+
): Promise<{ record: LexMap; bytes: Uint8Array }> => {
|
|
12
12
|
const bytes = blocks.get(cid)
|
|
13
13
|
if (!bytes) {
|
|
14
14
|
throw new MissingBlockError(cid, 'record')
|
|
@@ -19,7 +19,7 @@ export const getAndParseRecord = async (
|
|
|
19
19
|
|
|
20
20
|
export const getAndParseByDef = async <T>(
|
|
21
21
|
blocks: BlockMap,
|
|
22
|
-
cid:
|
|
22
|
+
cid: Cid,
|
|
23
23
|
def: check.Def<T>,
|
|
24
24
|
): Promise<{ obj: T; bytes: Uint8Array }> => {
|
|
25
25
|
const bytes = blocks.get(cid)
|
|
@@ -31,10 +31,10 @@ export const getAndParseByDef = async <T>(
|
|
|
31
31
|
|
|
32
32
|
export const parseObjByDef = <T>(
|
|
33
33
|
bytes: Uint8Array,
|
|
34
|
-
cid:
|
|
34
|
+
cid: Cid,
|
|
35
35
|
def: check.Def<T>,
|
|
36
36
|
): { obj: T; bytes: Uint8Array } => {
|
|
37
|
-
const obj =
|
|
37
|
+
const obj = decode(bytes)
|
|
38
38
|
const res = def.schema.safeParse(obj)
|
|
39
39
|
if (res.success) {
|
|
40
40
|
return { obj: res.data, bytes }
|
package/src/readable-repo.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { RepoRecord } from '@atproto/lexicon'
|
|
1
|
+
import { Cid, LexMap } from '@atproto/lex-data'
|
|
3
2
|
import { MissingBlocksError } from './error'
|
|
4
3
|
import log from './logger'
|
|
5
4
|
import { MST } from './mst'
|
|
@@ -12,14 +11,14 @@ type Params = {
|
|
|
12
11
|
storage: ReadableBlockstore
|
|
13
12
|
data: MST
|
|
14
13
|
commit: Commit
|
|
15
|
-
cid:
|
|
14
|
+
cid: Cid
|
|
16
15
|
}
|
|
17
16
|
|
|
18
17
|
export class ReadableRepo {
|
|
19
18
|
storage: ReadableBlockstore
|
|
20
19
|
data: MST
|
|
21
20
|
commit: Commit
|
|
22
|
-
cid:
|
|
21
|
+
cid: Cid
|
|
23
22
|
|
|
24
23
|
constructor(params: Params) {
|
|
25
24
|
this.storage = params.storage
|
|
@@ -28,7 +27,7 @@ export class ReadableRepo {
|
|
|
28
27
|
this.cid = params.cid
|
|
29
28
|
}
|
|
30
29
|
|
|
31
|
-
static async load(storage: ReadableBlockstore, commitCid:
|
|
30
|
+
static async load(storage: ReadableBlockstore, commitCid: Cid) {
|
|
32
31
|
const commit = await storage.readObj(commitCid, def.versionedCommit)
|
|
33
32
|
const data = await MST.load(storage, commit.data)
|
|
34
33
|
log.info({ did: commit.did }, 'loaded repo for')
|
|
@@ -51,8 +50,8 @@ export class ReadableRepo {
|
|
|
51
50
|
async *walkRecords(from?: string): AsyncIterable<{
|
|
52
51
|
collection: string
|
|
53
52
|
rkey: string
|
|
54
|
-
cid:
|
|
55
|
-
record:
|
|
53
|
+
cid: Cid
|
|
54
|
+
record: LexMap
|
|
56
55
|
}> {
|
|
57
56
|
for await (const leaf of this.data.walkLeavesFrom(from ?? '')) {
|
|
58
57
|
const { collection, rkey } = util.parseDataKey(leaf.key)
|
package/src/repo.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { TID, dataToCborBlock } from '@atproto/common'
|
|
1
|
+
import { TID } from '@atproto/common-web'
|
|
3
2
|
import * as crypto from '@atproto/crypto'
|
|
4
|
-
import {
|
|
3
|
+
import { encode } from '@atproto/lex-cbor'
|
|
4
|
+
import { Cid, cidForCbor } from '@atproto/lex-data'
|
|
5
5
|
import { BlockMap } from './block-map'
|
|
6
6
|
import { CidSet } from './cid-set'
|
|
7
7
|
import { DataDiff } from './data-diff'
|
|
@@ -23,7 +23,7 @@ type Params = {
|
|
|
23
23
|
storage: RepoStorage
|
|
24
24
|
data: MST
|
|
25
25
|
commit: Commit
|
|
26
|
-
cid:
|
|
26
|
+
cid: Cid
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
export class Repo extends ReadableRepo {
|
|
@@ -99,7 +99,7 @@ export class Repo extends ReadableRepo {
|
|
|
99
99
|
return Repo.createFromCommit(storage, commit)
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
-
static async load(storage: RepoStorage, cid?:
|
|
102
|
+
static async load(storage: RepoStorage, cid?: Cid) {
|
|
103
103
|
const commitCid = cid || (await storage.getRoot())
|
|
104
104
|
if (!commitCid) {
|
|
105
105
|
throw new Error('No cid provided and none in storage')
|
|
@@ -169,15 +169,18 @@ export class Repo extends ReadableRepo {
|
|
|
169
169
|
},
|
|
170
170
|
keypair,
|
|
171
171
|
)
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
172
|
+
|
|
173
|
+
const commitBytes = encode(commit)
|
|
174
|
+
const commitCid = await cidForCbor(commitBytes)
|
|
175
|
+
|
|
176
|
+
if (!commitCid.equals(this.cid)) {
|
|
177
|
+
newBlocks.set(commitCid, commitBytes)
|
|
178
|
+
relevantBlocks.set(commitCid, commitBytes)
|
|
176
179
|
removedCids.add(this.cid)
|
|
177
180
|
}
|
|
178
181
|
|
|
179
182
|
return {
|
|
180
|
-
cid:
|
|
183
|
+
cid: commitCid,
|
|
181
184
|
rev,
|
|
182
185
|
since: this.commit.rev,
|
|
183
186
|
prev: this.cid,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Cid } from '@atproto/lex-data'
|
|
2
2
|
import { BlockMap } from '../block-map'
|
|
3
3
|
import { CommitData } from '../types'
|
|
4
4
|
import { ReadableBlockstore } from './readable-blockstore'
|
|
@@ -9,7 +9,7 @@ export class MemoryBlockstore
|
|
|
9
9
|
implements RepoStorage
|
|
10
10
|
{
|
|
11
11
|
blocks: BlockMap
|
|
12
|
-
root:
|
|
12
|
+
root: Cid | null = null
|
|
13
13
|
rev: string | null = null
|
|
14
14
|
|
|
15
15
|
constructor(blocks?: BlockMap) {
|
|
@@ -20,23 +20,23 @@ export class MemoryBlockstore
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
async getRoot(): Promise<
|
|
23
|
+
async getRoot(): Promise<Cid | null> {
|
|
24
24
|
return this.root
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
async getBytes(cid:
|
|
27
|
+
async getBytes(cid: Cid): Promise<Uint8Array | null> {
|
|
28
28
|
return this.blocks.get(cid) || null
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
async has(cid:
|
|
31
|
+
async has(cid: Cid): Promise<boolean> {
|
|
32
32
|
return this.blocks.has(cid)
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
async getBlocks(cids:
|
|
35
|
+
async getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }> {
|
|
36
36
|
return this.blocks.getMany(cids)
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
async putBlock(cid:
|
|
39
|
+
async putBlock(cid: Cid, block: Uint8Array): Promise<void> {
|
|
40
40
|
this.blocks.set(cid, block)
|
|
41
41
|
}
|
|
42
42
|
|
|
@@ -44,7 +44,7 @@ export class MemoryBlockstore
|
|
|
44
44
|
this.blocks.addMap(blocks)
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
async updateRoot(cid:
|
|
47
|
+
async updateRoot(cid: Cid, rev: string): Promise<void> {
|
|
48
48
|
this.root = cid
|
|
49
49
|
this.rev = rev
|
|
50
50
|
}
|
|
@@ -1,27 +1,26 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { RepoRecord } from '@atproto/lexicon'
|
|
1
|
+
import { check } from '@atproto/common-web'
|
|
2
|
+
import { Cid, LexMap } from '@atproto/lex-data'
|
|
4
3
|
import { BlockMap } from '../block-map'
|
|
5
4
|
import { MissingBlockError } from '../error'
|
|
6
|
-
import
|
|
5
|
+
import { parseObjByDef } from '../parse'
|
|
7
6
|
import { cborToLexRecord } from '../util'
|
|
8
7
|
|
|
9
8
|
export abstract class ReadableBlockstore {
|
|
10
|
-
abstract getBytes(cid:
|
|
11
|
-
abstract has(cid:
|
|
12
|
-
abstract getBlocks(cids:
|
|
9
|
+
abstract getBytes(cid: Cid): Promise<Uint8Array | null>
|
|
10
|
+
abstract has(cid: Cid): Promise<boolean>
|
|
11
|
+
abstract getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }>
|
|
13
12
|
|
|
14
13
|
async attemptRead<T>(
|
|
15
|
-
cid:
|
|
14
|
+
cid: Cid,
|
|
16
15
|
def: check.Def<T>,
|
|
17
16
|
): Promise<{ obj: T; bytes: Uint8Array } | null> {
|
|
18
17
|
const bytes = await this.getBytes(cid)
|
|
19
18
|
if (!bytes) return null
|
|
20
|
-
return
|
|
19
|
+
return parseObjByDef(bytes, cid, def)
|
|
21
20
|
}
|
|
22
21
|
|
|
23
22
|
async readObjAndBytes<T>(
|
|
24
|
-
cid:
|
|
23
|
+
cid: Cid,
|
|
25
24
|
def: check.Def<T>,
|
|
26
25
|
): Promise<{ obj: T; bytes: Uint8Array }> {
|
|
27
26
|
const read = await this.attemptRead(cid, def)
|
|
@@ -31,12 +30,12 @@ export abstract class ReadableBlockstore {
|
|
|
31
30
|
return read
|
|
32
31
|
}
|
|
33
32
|
|
|
34
|
-
async readObj<T>(cid:
|
|
33
|
+
async readObj<T>(cid: Cid, def: check.Def<T>): Promise<T> {
|
|
35
34
|
const obj = await this.readObjAndBytes(cid, def)
|
|
36
35
|
return obj.obj
|
|
37
36
|
}
|
|
38
37
|
|
|
39
|
-
async attemptReadRecord(cid:
|
|
38
|
+
async attemptReadRecord(cid: Cid): Promise<LexMap | null> {
|
|
40
39
|
try {
|
|
41
40
|
return await this.readRecord(cid)
|
|
42
41
|
} catch {
|
|
@@ -44,7 +43,7 @@ export abstract class ReadableBlockstore {
|
|
|
44
43
|
}
|
|
45
44
|
}
|
|
46
45
|
|
|
47
|
-
async readRecord(cid:
|
|
46
|
+
async readRecord(cid: Cid): Promise<LexMap> {
|
|
48
47
|
const bytes = await this.getBytes(cid)
|
|
49
48
|
if (!bytes) {
|
|
50
49
|
throw new MissingBlockError(cid)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Cid } from '@atproto/lex-data'
|
|
2
2
|
import { BlockMap } from '../block-map'
|
|
3
3
|
import { ReadableBlockstore } from './readable-blockstore'
|
|
4
4
|
|
|
@@ -10,13 +10,13 @@ export class SyncStorage extends ReadableBlockstore {
|
|
|
10
10
|
super()
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
async getBytes(cid:
|
|
13
|
+
async getBytes(cid: Cid): Promise<Uint8Array | null> {
|
|
14
14
|
const got = await this.staged.getBytes(cid)
|
|
15
15
|
if (got) return got
|
|
16
16
|
return this.saved.getBytes(cid)
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
async getBlocks(cids:
|
|
19
|
+
async getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }> {
|
|
20
20
|
const fromStaged = await this.staged.getBlocks(cids)
|
|
21
21
|
const fromSaved = await this.saved.getBlocks(fromStaged.missing)
|
|
22
22
|
const blocks = fromStaged.blocks
|
|
@@ -27,7 +27,7 @@ export class SyncStorage extends ReadableBlockstore {
|
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
async has(cid:
|
|
30
|
+
async has(cid: Cid): Promise<boolean> {
|
|
31
31
|
return (await this.staged.has(cid)) || (await this.saved.has(cid))
|
|
32
32
|
}
|
|
33
33
|
}
|
package/src/storage/types.ts
CHANGED
|
@@ -1,47 +1,47 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { RepoRecord } from '@atproto/lexicon'
|
|
1
|
+
import type { Readable } from 'node:stream'
|
|
2
|
+
import { check } from '@atproto/common-web'
|
|
3
|
+
import { Cid, LexMap } from '@atproto/lex-data'
|
|
5
4
|
import { BlockMap } from '../block-map'
|
|
6
5
|
import { CommitData } from '../types'
|
|
7
6
|
|
|
8
7
|
export interface RepoStorage {
|
|
9
8
|
// Writable
|
|
10
|
-
getRoot(): Promise<
|
|
11
|
-
putBlock(cid:
|
|
9
|
+
getRoot(): Promise<Cid | null>
|
|
10
|
+
putBlock(cid: Cid, block: Uint8Array, rev: string): Promise<void>
|
|
12
11
|
putMany(blocks: BlockMap, rev: string): Promise<void>
|
|
13
|
-
updateRoot(cid:
|
|
12
|
+
updateRoot(cid: Cid, rev: string): Promise<void>
|
|
14
13
|
applyCommit(commit: CommitData)
|
|
15
14
|
|
|
16
15
|
// Readable
|
|
17
|
-
getBytes(cid:
|
|
18
|
-
has(cid:
|
|
19
|
-
getBlocks(cids:
|
|
16
|
+
getBytes(cid: Cid): Promise<Uint8Array | null>
|
|
17
|
+
has(cid: Cid): Promise<boolean>
|
|
18
|
+
getBlocks(cids: Cid[]): Promise<{ blocks: BlockMap; missing: Cid[] }>
|
|
20
19
|
attemptRead<T>(
|
|
21
|
-
cid:
|
|
20
|
+
cid: Cid,
|
|
22
21
|
def: check.Def<T>,
|
|
23
22
|
): Promise<{ obj: T; bytes: Uint8Array } | null>
|
|
24
23
|
readObjAndBytes<T>(
|
|
25
|
-
cid:
|
|
24
|
+
cid: Cid,
|
|
26
25
|
def: check.Def<T>,
|
|
27
26
|
): Promise<{ obj: T; bytes: Uint8Array }>
|
|
28
|
-
readObj<T>(cid:
|
|
29
|
-
attemptReadRecord(cid:
|
|
30
|
-
readRecord(cid:
|
|
27
|
+
readObj<T>(cid: Cid, def: check.Def<T>): Promise<T>
|
|
28
|
+
attemptReadRecord(cid: Cid): Promise<LexMap | null>
|
|
29
|
+
readRecord(cid: Cid): Promise<LexMap>
|
|
31
30
|
}
|
|
32
31
|
|
|
32
|
+
// @TODO make this less node-js specific by using AsyncIterable<Uint8Array> instead of Readable
|
|
33
33
|
export interface BlobStore {
|
|
34
|
-
putTemp(bytes: Uint8Array |
|
|
35
|
-
makePermanent(key: string, cid:
|
|
36
|
-
putPermanent(cid:
|
|
37
|
-
quarantine(cid:
|
|
38
|
-
unquarantine(cid:
|
|
39
|
-
getBytes(cid:
|
|
40
|
-
getStream(cid:
|
|
34
|
+
putTemp(bytes: Uint8Array | Readable): Promise<string>
|
|
35
|
+
makePermanent(key: string, cid: Cid): Promise<void>
|
|
36
|
+
putPermanent(cid: Cid, bytes: Uint8Array | Readable): Promise<void>
|
|
37
|
+
quarantine(cid: Cid): Promise<void>
|
|
38
|
+
unquarantine(cid: Cid): Promise<void>
|
|
39
|
+
getBytes(cid: Cid): Promise<Uint8Array>
|
|
40
|
+
getStream(cid: Cid): Promise<Readable>
|
|
41
41
|
hasTemp(key: string): Promise<boolean>
|
|
42
|
-
hasStored(cid:
|
|
43
|
-
delete(cid:
|
|
44
|
-
deleteMany(cid:
|
|
42
|
+
hasStored(cid: Cid): Promise<boolean>
|
|
43
|
+
delete(cid: Cid): Promise<void>
|
|
44
|
+
deleteMany(cid: Cid[]): Promise<void>
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
export class BlobNotFoundError extends Error {}
|
package/src/sync/consumer.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Cid } from '@atproto/lex-data'
|
|
2
2
|
import { BlockMap } from '../block-map'
|
|
3
3
|
import { readCarWithRoot } from '../car'
|
|
4
4
|
import { DataDiff } from '../data-diff'
|
|
@@ -25,7 +25,7 @@ export const verifyRepoCar = async (
|
|
|
25
25
|
|
|
26
26
|
export const verifyRepo = async (
|
|
27
27
|
blocks: BlockMap,
|
|
28
|
-
head:
|
|
28
|
+
head: Cid,
|
|
29
29
|
did?: string,
|
|
30
30
|
signingKey?: string,
|
|
31
31
|
opts?: { ensureLeaves?: boolean },
|
|
@@ -52,7 +52,7 @@ export const verifyDiffCar = async (
|
|
|
52
52
|
export const verifyDiff = async (
|
|
53
53
|
repo: ReadableRepo | null,
|
|
54
54
|
updateBlocks: BlockMap,
|
|
55
|
-
updateRoot:
|
|
55
|
+
updateRoot: Cid,
|
|
56
56
|
did?: string,
|
|
57
57
|
signingKey?: string,
|
|
58
58
|
opts?: { ensureLeaves?: boolean },
|
|
@@ -103,7 +103,7 @@ export const verifyDiff = async (
|
|
|
103
103
|
// @NOTE only verifies the root, not the repo contents
|
|
104
104
|
const verifyRepoRoot = async (
|
|
105
105
|
storage: ReadableBlockstore,
|
|
106
|
-
head:
|
|
106
|
+
head: Cid,
|
|
107
107
|
did?: string,
|
|
108
108
|
signingKey?: string,
|
|
109
109
|
): Promise<ReadableRepo> => {
|
|
@@ -155,7 +155,7 @@ export const verifyProofs = async (
|
|
|
155
155
|
unverified.push(claim)
|
|
156
156
|
}
|
|
157
157
|
} else {
|
|
158
|
-
if (claim.cid
|
|
158
|
+
if (found?.equals(claim.cid)) {
|
|
159
159
|
verified.push(claim)
|
|
160
160
|
} else {
|
|
161
161
|
unverified.push(claim)
|