@atcute/car 2.0.3 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/atproto-repo.d.ts +65 -6
- package/dist/atproto-repo.js +73 -18
- package/dist/atproto-repo.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/reader.d.ts +4 -7
- package/dist/reader.js.map +1 -1
- package/dist/utilities/car.d.ts +17 -0
- package/dist/utilities/sync-byte-reader.js +1 -1
- package/dist/utilities/sync-byte-reader.js.map +1 -1
- package/dist/utilities/sync-car-reader.d.ts +6 -9
- package/dist/utilities/sync-car-reader.js +41 -28
- package/dist/utilities/sync-car-reader.js.map +1 -1
- package/lib/atproto-repo.ts +85 -25
- package/lib/index.ts +2 -2
- package/lib/reader.ts +5 -2
- package/lib/utilities/car.ts +22 -0
- package/lib/utilities/sync-byte-reader.ts +1 -1
- package/lib/utilities/sync-car-reader.ts +57 -38
- package/package.json +3 -3
package/dist/atproto-repo.d.ts
CHANGED
|
@@ -1,17 +1,63 @@
|
|
|
1
1
|
import * as CBOR from '@atcute/cbor';
|
|
2
2
|
import * as CID from '@atcute/cid';
|
|
3
|
+
import { type CarEntry } from './reader.js';
|
|
4
|
+
export type BlockMap = Map<string, CarEntry>;
|
|
3
5
|
export declare class RepoEntry {
|
|
6
|
+
/** The collection this record belongs to */
|
|
4
7
|
readonly collection: string;
|
|
8
|
+
/** Record key */
|
|
5
9
|
readonly rkey: string;
|
|
10
|
+
/** CID of this record */
|
|
6
11
|
readonly cid: CID.CidLink;
|
|
7
12
|
private blockmap;
|
|
8
|
-
constructor(
|
|
13
|
+
constructor(
|
|
14
|
+
/** The collection this record belongs to */
|
|
15
|
+
collection: string,
|
|
16
|
+
/** Record key */
|
|
17
|
+
rkey: string,
|
|
18
|
+
/** CID of this record */
|
|
19
|
+
cid: CID.CidLink, blockmap: BlockMap);
|
|
20
|
+
/**
|
|
21
|
+
* returns the associated CarEntry for this record
|
|
22
|
+
*/
|
|
23
|
+
get carEntry(): CarEntry;
|
|
24
|
+
/**
|
|
25
|
+
* returns the raw contents of this record
|
|
26
|
+
*/
|
|
9
27
|
get bytes(): Uint8Array;
|
|
28
|
+
/**
|
|
29
|
+
* returns the decoded contents of this record
|
|
30
|
+
*/
|
|
10
31
|
get record(): unknown;
|
|
11
32
|
}
|
|
12
33
|
export declare function iterateAtpRepo(buf: Uint8Array): Generator<RepoEntry>;
|
|
34
|
+
/**
|
|
35
|
+
* collects entries from a CAR archive into a mapping of CID string -> actual bytes
|
|
36
|
+
* @param iterator a generator that yields objects with a `cid` and `bytes` property
|
|
37
|
+
* @returns a mapping of CID string -> actual bytes
|
|
38
|
+
*/
|
|
39
|
+
export declare function collectBlock(iterator: Generator<CarEntry>): BlockMap;
|
|
40
|
+
/**
|
|
41
|
+
* reads a block from the blockmap and validates it against the provided validation function
|
|
42
|
+
* @param map a mapping of CID string -> actual bytes
|
|
43
|
+
* @param link a CID link to read
|
|
44
|
+
* @param validate a validation function to validate the decoded data
|
|
45
|
+
* @returns the decoded and validated data
|
|
46
|
+
*/
|
|
47
|
+
export declare function readBlock<T>(map: BlockMap, link: CID.CidLink, validate: (value: unknown) => value is T): T;
|
|
48
|
+
/** node entry object */
|
|
49
|
+
export interface NodeEntry {
|
|
50
|
+
key: string;
|
|
51
|
+
cid: CID.CidLink;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
|
|
55
|
+
* @param map a mapping of CID string -> actual bytes
|
|
56
|
+
* @param pointer a CID link to the root of the MST
|
|
57
|
+
* @returns a generator that yields the entries of the MST
|
|
58
|
+
*/
|
|
13
59
|
export declare function walkMstEntries(map: BlockMap, pointer: CID.CidLink): Generator<NodeEntry>;
|
|
14
|
-
|
|
60
|
+
/** commit object */
|
|
15
61
|
export interface Commit {
|
|
16
62
|
version: 3;
|
|
17
63
|
did: string;
|
|
@@ -20,7 +66,13 @@ export interface Commit {
|
|
|
20
66
|
prev: CID.CidLink | null;
|
|
21
67
|
sig: CBOR.Bytes;
|
|
22
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* checks if a value is a valid commit object
|
|
71
|
+
* @param value the value to check
|
|
72
|
+
* @returns true if the value is a valid commit object, false otherwise
|
|
73
|
+
*/
|
|
23
74
|
export declare const isCommit: (value: unknown) => value is Commit;
|
|
75
|
+
/** mst tree entry object */
|
|
24
76
|
export interface TreeEntry {
|
|
25
77
|
/** count of bytes shared with previous TreeEntry in this Node (if any) */
|
|
26
78
|
p: number;
|
|
@@ -31,15 +83,22 @@ export interface TreeEntry {
|
|
|
31
83
|
/** next subtree (to the right of leaf) */
|
|
32
84
|
t: CID.CidLink | null;
|
|
33
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* checks if a value is a valid mst tree entry object
|
|
88
|
+
* @param value the value to check
|
|
89
|
+
* @returns true if the value is a valid mst tree entry object, false otherwise
|
|
90
|
+
*/
|
|
34
91
|
export declare const isTreeEntry: (value: unknown) => value is TreeEntry;
|
|
92
|
+
/** mst node object */
|
|
35
93
|
export interface MstNode {
|
|
36
94
|
/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
|
|
37
95
|
l: CID.CidLink | null;
|
|
38
96
|
/** ordered list of TreeEntry objects */
|
|
39
97
|
e: TreeEntry[];
|
|
40
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* checks if a value is a valid mst node object
|
|
101
|
+
* @param value the value to check
|
|
102
|
+
* @returns true if the value is a valid mst node object, false otherwise
|
|
103
|
+
*/
|
|
41
104
|
export declare const isMstNode: (value: unknown) => value is MstNode;
|
|
42
|
-
export interface NodeEntry {
|
|
43
|
-
key: string;
|
|
44
|
-
cid: CID.CidLink;
|
|
45
|
-
}
|
package/dist/atproto-repo.js
CHANGED
|
@@ -7,47 +7,87 @@ export class RepoEntry {
|
|
|
7
7
|
rkey;
|
|
8
8
|
cid;
|
|
9
9
|
blockmap;
|
|
10
|
-
constructor(
|
|
10
|
+
constructor(
|
|
11
|
+
/** The collection this record belongs to */
|
|
12
|
+
collection,
|
|
13
|
+
/** Record key */
|
|
14
|
+
rkey,
|
|
15
|
+
/** CID of this record */
|
|
16
|
+
cid, blockmap) {
|
|
11
17
|
this.collection = collection;
|
|
12
18
|
this.rkey = rkey;
|
|
13
19
|
this.cid = cid;
|
|
14
20
|
this.blockmap = blockmap;
|
|
15
21
|
}
|
|
16
|
-
|
|
22
|
+
/**
|
|
23
|
+
* returns the associated CarEntry for this record
|
|
24
|
+
*/
|
|
25
|
+
get carEntry() {
|
|
17
26
|
const cid = this.cid.$link;
|
|
18
|
-
const
|
|
19
|
-
assert(
|
|
20
|
-
return
|
|
27
|
+
const entry = this.blockmap.get(cid);
|
|
28
|
+
assert(entry != null, `cid not found in blockmap; cid=${cid}`);
|
|
29
|
+
return entry;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* returns the raw contents of this record
|
|
33
|
+
*/
|
|
34
|
+
get bytes() {
|
|
35
|
+
return this.carEntry.bytes;
|
|
21
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* returns the decoded contents of this record
|
|
39
|
+
*/
|
|
22
40
|
get record() {
|
|
23
41
|
return CBOR.decode(this.bytes);
|
|
24
42
|
}
|
|
25
43
|
}
|
|
26
44
|
export function* iterateAtpRepo(buf) {
|
|
27
|
-
const {
|
|
45
|
+
const { header, iterate } = readCar(buf);
|
|
46
|
+
const roots = header.data.roots;
|
|
28
47
|
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
blockmap.set(CID.toString(entry.cid), entry.bytes);
|
|
33
|
-
}
|
|
34
|
-
// Read the head, then walk through the MST tree from there.
|
|
35
|
-
const commit = readObject(blockmap, roots[0], isCommit);
|
|
48
|
+
const blockmap = collectBlock(iterate());
|
|
49
|
+
assert(blockmap.size > 0, `expected at least 1 block in the archive; got=${blockmap.size}`);
|
|
50
|
+
const commit = readBlock(blockmap, roots[0], isCommit);
|
|
36
51
|
for (const { key, cid } of walkMstEntries(blockmap, commit.data)) {
|
|
37
52
|
const [collection, rkey] = key.split('/');
|
|
38
53
|
yield new RepoEntry(collection, rkey, cid, blockmap);
|
|
39
54
|
}
|
|
40
55
|
}
|
|
41
|
-
|
|
56
|
+
/**
|
|
57
|
+
* collects entries from a CAR archive into a mapping of CID string -> actual bytes
|
|
58
|
+
* @param iterator a generator that yields objects with a `cid` and `bytes` property
|
|
59
|
+
* @returns a mapping of CID string -> actual bytes
|
|
60
|
+
*/
|
|
61
|
+
export function collectBlock(iterator) {
|
|
62
|
+
const blockmap = new Map();
|
|
63
|
+
for (const entry of iterator) {
|
|
64
|
+
blockmap.set(CID.toString(entry.cid), entry);
|
|
65
|
+
}
|
|
66
|
+
return blockmap;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* reads a block from the blockmap and validates it against the provided validation function
|
|
70
|
+
* @param map a mapping of CID string -> actual bytes
|
|
71
|
+
* @param link a CID link to read
|
|
72
|
+
* @param validate a validation function to validate the decoded data
|
|
73
|
+
* @returns the decoded and validated data
|
|
74
|
+
*/
|
|
75
|
+
export function readBlock(map, link, validate) {
|
|
42
76
|
const cid = link.$link;
|
|
43
|
-
const
|
|
44
|
-
assert(
|
|
45
|
-
const data = CBOR.decode(bytes);
|
|
77
|
+
const entry = map.get(cid);
|
|
78
|
+
assert(entry != null, `cid not found in blockmap; cid=${cid}`);
|
|
79
|
+
const data = CBOR.decode(entry.bytes);
|
|
46
80
|
assert(validate(data), `validation failed for cid=${cid}`);
|
|
47
81
|
return data;
|
|
48
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
|
|
85
|
+
* @param map a mapping of CID string -> actual bytes
|
|
86
|
+
* @param pointer a CID link to the root of the MST
|
|
87
|
+
* @returns a generator that yields the entries of the MST
|
|
88
|
+
*/
|
|
49
89
|
export function* walkMstEntries(map, pointer) {
|
|
50
|
-
const data =
|
|
90
|
+
const data = readBlock(map, pointer, isMstNode);
|
|
51
91
|
const entries = data.e;
|
|
52
92
|
let lastKey = '';
|
|
53
93
|
if (data.l !== null) {
|
|
@@ -87,6 +127,11 @@ const isBytes = (value) => {
|
|
|
87
127
|
}
|
|
88
128
|
return '$bytes' in value && typeof value.$bytes === 'string';
|
|
89
129
|
};
|
|
130
|
+
/**
|
|
131
|
+
* checks if a value is a valid commit object
|
|
132
|
+
* @param value the value to check
|
|
133
|
+
* @returns true if the value is a valid commit object, false otherwise
|
|
134
|
+
*/
|
|
90
135
|
export const isCommit = (value) => {
|
|
91
136
|
if (value === null || typeof value !== 'object') {
|
|
92
137
|
return false;
|
|
@@ -99,6 +144,11 @@ export const isCommit = (value) => {
|
|
|
99
144
|
(obj.prev === null || isCidLink(obj.prev)) &&
|
|
100
145
|
isBytes(obj.sig));
|
|
101
146
|
};
|
|
147
|
+
/**
|
|
148
|
+
* checks if a value is a valid mst tree entry object
|
|
149
|
+
* @param value the value to check
|
|
150
|
+
* @returns true if the value is a valid mst tree entry object, false otherwise
|
|
151
|
+
*/
|
|
102
152
|
export const isTreeEntry = (value) => {
|
|
103
153
|
if (value === null || typeof value !== 'object') {
|
|
104
154
|
return false;
|
|
@@ -106,6 +156,11 @@ export const isTreeEntry = (value) => {
|
|
|
106
156
|
const obj = value;
|
|
107
157
|
return (typeof obj.p === 'number' && isBytes(obj.k) && isCidLink(obj.v) && (obj.t === null || isCidLink(obj.t)));
|
|
108
158
|
};
|
|
159
|
+
/**
|
|
160
|
+
* checks if a value is a valid mst node object
|
|
161
|
+
* @param value the value to check
|
|
162
|
+
* @returns true if the value is a valid mst node object, false otherwise
|
|
163
|
+
*/
|
|
109
164
|
export const isMstNode = (value) => {
|
|
110
165
|
if (value === null || typeof value !== 'object') {
|
|
111
166
|
return false;
|
package/dist/atproto-repo.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"atproto-repo.js","sourceRoot":"","sources":["../lib/atproto-repo.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,OAAO,
|
|
1
|
+
{"version":3,"file":"atproto-repo.js","sourceRoot":"","sources":["../lib/atproto-repo.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,OAAO,EAAiB,MAAM,aAAa,CAAC;AAErD,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAIlC,MAAM,OAAO,SAAS;IAGJ;IAEA;IAEA;IACR;IAPT;IACC,4CAA4C;IAC5B,UAAkB;IAClC,iBAAiB;IACD,IAAY;IAC5B,yBAAyB;IACT,GAAgB,EACxB,QAAkB;QALV,eAAU,GAAV,UAAU,CAAQ;QAElB,SAAI,GAAJ,IAAI,CAAQ;QAEZ,QAAG,GAAH,GAAG,CAAa;QACxB,aAAQ,GAAR,QAAQ,CAAU;IACxB,CAAC;IAEJ;;OAEG;IACH,IAAI,QAAQ;QACX,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;QAE3B,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,kCAAkC,GAAG,EAAE,CAAC,CAAC;QAE/D,OAAO,KAAK,CAAC;IACd,CAAC;IAED;;OAEG;IACH,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CACD;AAED,MAAM,SAAS,CAAC,CAAC,cAAc,CAAC,GAAe;IAC9C,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;IAEhC,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,gDAAgD,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAE3F,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,iDAAiD,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5F,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvD,KAAK,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAClE,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE1C,MAAM,IAAI,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;IACtD,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,QAA6B;IACzD,MAAM,QAAQ,GAAa,IAAI,GAAG,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAI,GAAa,EAAE,IAAiB,EAAE,QAAwC;IACtG,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC;IAEvB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,kCAAkC,GAAG,EAAE,CAAC,CAAC;IAE/D,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,6BAA6B,GAAG,EAAE,CAAC,CAAC;IAE3D,OAAO,IAAI,CAAC;AACb,CAAC;AAQD;;;;;GAKG;AACH,MAAM,SAAS,CAAC,CAAC,cAAc,CAAC,GAAa,EAAE,OAAoB;IAClE,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IAEvB,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAEzB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAEhD,OAAO,GAAG,GAAG,CAAC;QAEd,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QAEjC,IAAI,KAAK,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACtB,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;AACF,CAAC;AAED,SAAS,MAAM,CAAC,SAAkB,EAAE,OAAe;IAClD,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;AACF,CAAC;AAED,MAAM,SAAS,GAAG,CAAC,KAAc,EAAwB,EAAE;IAC1D,IAAI,KAAK,YAAY,GAAG,CAAC,cAAc,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,OAAO,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,KAAc,EAAuB,EAAE;IACvD,IAAI,KAAK,YAAY,IAAI,CAAC,YAAY,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC;AAC9D,CAAC,CAAC;AAYF;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAmB,EAAE;IAC3D,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,GAAG,GAAG,KAAgC,CAAC;IAE7C,OAAO,CACN,GAAG,CAAC,OAAO,KAAK,CAAC;QACjB,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ;QAC3B,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QACnB,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ;QAC3B,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAChB,CAAC;AACH,CAAC,CAAC;AAcF;;;;GAIG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAsB,EAAE;IACjE,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,GAAG,GAAG,KAAgC,CAAC;IAE7C,OAAO,CACN,OAAO,GAAG,CAAC,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CACvG,CAAC;AACH,CAAC,CAAC;AAUF;;;;GAIG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAc,EAAoB,EAAE;IAC7D,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,GAAG,GAAG,KAAgC,CAAC;IAE7C,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AACjG,CAAC,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
1
|
+
export * from './atproto-repo.js';
|
|
2
|
+
export * from './reader.js';
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
1
|
+
export * from './atproto-repo.js';
|
|
2
|
+
export * from './reader.js';
|
|
3
3
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,aAAa,CAAC"}
|
package/dist/reader.d.ts
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
bytes: Uint8Array;
|
|
6
|
-
}>;
|
|
7
|
-
};
|
|
1
|
+
import { type SyncCarReader } from './utilities/sync-car-reader.js';
|
|
2
|
+
export type { CarEntry, CarHeader, CarV1Header } from './utilities/car.js';
|
|
3
|
+
export type { SyncCarReader } from './utilities/sync-car-reader.js';
|
|
4
|
+
export declare const readCar: (buffer: Uint8Array) => SyncCarReader;
|
package/dist/reader.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../lib/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../lib/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,eAAe,EAAsB,MAAM,gCAAgC,CAAC;AAKrF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAkB,EAAiB,EAAE;IAC5D,MAAM,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACzC,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC,CAAC"}
|
package/dist/utilities/car.d.ts
CHANGED
|
@@ -4,3 +4,20 @@ export interface CarV1Header {
|
|
|
4
4
|
roots: CID.CidLink[];
|
|
5
5
|
}
|
|
6
6
|
export declare const isCarV1Header: (value: unknown) => value is CarV1Header;
|
|
7
|
+
export interface CarHeader {
|
|
8
|
+
headerStart: number;
|
|
9
|
+
headerEnd: number;
|
|
10
|
+
data: CarV1Header;
|
|
11
|
+
dataStart: number;
|
|
12
|
+
dataEnd: number;
|
|
13
|
+
}
|
|
14
|
+
export interface CarEntry {
|
|
15
|
+
entryStart: number;
|
|
16
|
+
entryEnd: number;
|
|
17
|
+
cid: CID.Cid;
|
|
18
|
+
cidStart: number;
|
|
19
|
+
cidEnd: number;
|
|
20
|
+
bytes: Uint8Array;
|
|
21
|
+
bytesStart: number;
|
|
22
|
+
bytesEnd: number;
|
|
23
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-byte-reader.js","sourceRoot":"","sources":["../../lib/utilities/sync-byte-reader.ts"],"names":[],"mappings":"AAOA,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAe,EAAkB,EAAE;IACpE,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,OAAO;QACN,IAAI,GAAG;YACN,OAAO,GAAG,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,IAAI;YACR,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC7B,MAAM,IAAI,UAAU,CAAC,wBAAwB,CAAC,CAAC;YAChD,CAAC;YAED,GAAG,IAAI,IAAI,CAAC;QACb,CAAC;QACD,IAAI,CAAC,IAAI;YACR,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"sync-byte-reader.js","sourceRoot":"","sources":["../../lib/utilities/sync-byte-reader.ts"],"names":[],"mappings":"AAOA,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAe,EAAkB,EAAE;IACpE,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,OAAO;QACN,IAAI,GAAG;YACN,OAAO,GAAG,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,IAAI;YACR,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC7B,MAAM,IAAI,UAAU,CAAC,wBAAwB,CAAC,CAAC;YAChD,CAAC;YAED,GAAG,IAAI,IAAI,CAAC;QACb,CAAC;QACD,IAAI,CAAC,IAAI;YACR,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,CAAC,IAAI,EAAE,IAAI;YACjB,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC7B,MAAM,IAAI,UAAU,CAAC,wBAAwB,CAAC,CAAC;YAChD,CAAC;YAED,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;YAC5C,IAAI,IAAI,EAAE,CAAC;gBACV,GAAG,IAAI,IAAI,CAAC;YACb,CAAC;YAED,OAAO,KAAK,CAAC;QACd,CAAC;KACD,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
import
|
|
2
|
-
import * as CID from '@atcute/cid';
|
|
1
|
+
import { type CarEntry, type CarHeader } from './car.js';
|
|
3
2
|
import type { SyncByteReader } from './sync-byte-reader.js';
|
|
4
|
-
export
|
|
5
|
-
|
|
6
|
-
iterate(): Generator<
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
}>;
|
|
10
|
-
};
|
|
3
|
+
export interface SyncCarReader {
|
|
4
|
+
header: CarHeader;
|
|
5
|
+
iterate(): Generator<CarEntry>;
|
|
6
|
+
}
|
|
7
|
+
export declare const createCarReader: (reader: SyncByteReader) => SyncCarReader;
|
|
@@ -12,65 +12,78 @@ const readVarint = (reader, size) => {
|
|
|
12
12
|
return int;
|
|
13
13
|
};
|
|
14
14
|
const readHeader = (reader) => {
|
|
15
|
+
const headerStart = reader.pos;
|
|
15
16
|
const length = readVarint(reader, 8);
|
|
16
17
|
if (length === 0) {
|
|
17
18
|
throw new RangeError(`invalid car header; length=0`);
|
|
18
19
|
}
|
|
20
|
+
const dataStart = reader.pos;
|
|
19
21
|
const rawHeader = reader.exactly(length, true);
|
|
20
|
-
const
|
|
21
|
-
if (!isCarV1Header(
|
|
22
|
+
const data = CBOR.decode(rawHeader);
|
|
23
|
+
if (!isCarV1Header(data)) {
|
|
22
24
|
throw new TypeError(`expected a car v1 archive`);
|
|
23
25
|
}
|
|
24
|
-
|
|
26
|
+
const dataEnd = reader.pos;
|
|
27
|
+
const headerEnd = dataEnd;
|
|
28
|
+
return { data, headerStart, headerEnd, dataStart, dataEnd };
|
|
25
29
|
};
|
|
26
30
|
const readCid = (reader) => {
|
|
27
|
-
const head = reader.
|
|
31
|
+
const head = reader.exactly(4, false);
|
|
28
32
|
const version = head[0];
|
|
29
33
|
const codec = head[1];
|
|
30
|
-
const
|
|
34
|
+
const digestType = head[2];
|
|
35
|
+
const digestSize = head[3];
|
|
31
36
|
if (version !== CID.CID_VERSION) {
|
|
32
37
|
throw new RangeError(`incorrect cid version (got v${version})`);
|
|
33
38
|
}
|
|
34
39
|
if (codec !== CID.CODEC_DCBOR && codec !== CID.CODEC_RAW) {
|
|
35
40
|
throw new RangeError(`incorrect cid codec (got 0x${codec.toString(16)})`);
|
|
36
41
|
}
|
|
37
|
-
if (
|
|
38
|
-
throw new RangeError(`incorrect cid hash type (got 0x${
|
|
42
|
+
if (digestType !== CID.HASH_SHA256) {
|
|
43
|
+
throw new RangeError(`incorrect cid hash type (got 0x${digestType.toString(16)})`);
|
|
39
44
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
45
|
+
if (digestSize !== 32 && digestSize !== 0) {
|
|
46
|
+
throw new RangeError(`incorrect cid digest size (got ${digestSize})`);
|
|
47
|
+
}
|
|
48
|
+
const bytes = reader.exactly(4 + digestSize, true);
|
|
49
|
+
const digest = bytes.subarray(4, 4 + digestSize);
|
|
43
50
|
const cid = {
|
|
44
51
|
version: version,
|
|
45
52
|
codec: codec,
|
|
46
53
|
digest: {
|
|
47
|
-
codec:
|
|
54
|
+
codec: digestType,
|
|
48
55
|
contents: digest,
|
|
49
56
|
},
|
|
50
57
|
bytes: bytes,
|
|
51
58
|
};
|
|
52
59
|
return cid;
|
|
53
60
|
};
|
|
54
|
-
const readBlockHeader = (reader) => {
|
|
55
|
-
const start = reader.pos;
|
|
56
|
-
let size = readVarint(reader, 8);
|
|
57
|
-
if (size === 0) {
|
|
58
|
-
throw new Error(`invalid car section; length=0`);
|
|
59
|
-
}
|
|
60
|
-
size += reader.pos - start;
|
|
61
|
-
const cid = readCid(reader);
|
|
62
|
-
const blockSize = size - (reader.pos - start);
|
|
63
|
-
return { cid, blockSize };
|
|
64
|
-
};
|
|
65
61
|
export const createCarReader = (reader) => {
|
|
66
|
-
const
|
|
62
|
+
const header = readHeader(reader);
|
|
67
63
|
return {
|
|
68
|
-
|
|
64
|
+
header,
|
|
69
65
|
*iterate() {
|
|
70
|
-
while (reader.upto(8).length > 0) {
|
|
71
|
-
const
|
|
72
|
-
const
|
|
73
|
-
|
|
66
|
+
while (reader.upto(8 + 36).length > 0) {
|
|
67
|
+
const entryStart = reader.pos;
|
|
68
|
+
const entrySize = readVarint(reader, 8);
|
|
69
|
+
const cidStart = reader.pos;
|
|
70
|
+
const cid = readCid(reader);
|
|
71
|
+
const bytesStart = reader.pos;
|
|
72
|
+
const bytesSize = entrySize - (bytesStart - cidStart);
|
|
73
|
+
const bytes = reader.exactly(bytesSize, true);
|
|
74
|
+
const cidEnd = bytesStart;
|
|
75
|
+
const bytesEnd = reader.pos;
|
|
76
|
+
const entryEnd = bytesEnd;
|
|
77
|
+
yield {
|
|
78
|
+
cid,
|
|
79
|
+
bytes,
|
|
80
|
+
entryStart,
|
|
81
|
+
entryEnd,
|
|
82
|
+
cidStart,
|
|
83
|
+
cidEnd,
|
|
84
|
+
bytesStart,
|
|
85
|
+
bytesEnd,
|
|
86
|
+
};
|
|
74
87
|
}
|
|
75
88
|
},
|
|
76
89
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-car-reader.js","sourceRoot":"","sources":["../../lib/utilities/sync-car-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,KAAK,MAAM,MAAM,gBAAgB,CAAC;AAEzC,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"sync-car-reader.js","sourceRoot":"","sources":["../../lib/utilities/sync-car-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,KAAK,MAAM,MAAM,gBAAgB,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAiC,MAAM,UAAU,CAAC;AAQxE,MAAM,UAAU,GAAG,CAAC,MAAsB,EAAE,IAAY,EAAU,EAAE;IACnE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,UAAU,CAAC,wBAAwB,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAElB,OAAO,GAAG,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,MAAsB,EAAa,EAAE;IACxD,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC;IAC/B,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,UAAU,CAAC,8BAA8B,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;IAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE/C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,SAAS,CAAC,2BAA2B,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC;IAC3B,MAAM,SAAS,GAAG,OAAO,CAAC;IAE1B,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AAC7D,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,MAAsB,EAAW,EAAE;IACnD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAE3B,IAAI,OAAO,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,IAAI,UAAU,CAAC,+BAA+B,OAAO,GAAG,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,KAAK,KAAK,GAAG,CAAC,WAAW,IAAI,KAAK,KAAK,GAAG,CAAC,SAAS,EAAE,CAAC;QAC1D,MAAM,IAAI,UAAU,CAAC,8BAA8B,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,UAAU,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,IAAI,UAAU,CAAC,kCAAkC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,IAAI,UAAU,KAAK,EAAE,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,UAAU,CAAC,kCAAkC,UAAU,GAAG,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,UAAU,EAAE,IAAI,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;IAEjD,MAAM,GAAG,GAAY;QACpB,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE,KAAK;QACZ,MAAM,EAAE;YACP,KAAK,EAAE,UAAU;YACjB,QAAQ,EAAE,MAAM;SAChB;QACD,KAAK,EAAE,KAAK;KACZ,CAAC;IAEF,OAAO,GAAG,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAsB,EAAiB,EAAE;IACxE,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAElC,OAAO;QACN,MAAM;QACN,CAAC,OAAO;YACP,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC;gBAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAExC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;gBAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gBAE5B,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC;gBAC9B,MAAM,SAAS,GAAG,SAAS,GAAG,CAAC,UAAU,GAAG,QAAQ,CAAC,CAAC;gBACtD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBAE9C,MAAM,MAAM,GAAG,UAAU,CAAC;gBAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;gBAC5B,MAAM,QAAQ,GAAG,QAAQ,CAAC;gBAE1B,MAAM;oBACL,GAAG;oBACH,KAAK;oBAEL,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,MAAM;oBACN,UAAU;oBACV,QAAQ;iBACR,CAAC;YACH,CAAC;QACF,CAAC;KACD,CAAC;AACH,CAAC,CAAC"}
|
package/lib/atproto-repo.ts
CHANGED
|
@@ -1,44 +1,60 @@
|
|
|
1
1
|
import * as CBOR from '@atcute/cbor';
|
|
2
2
|
import * as CID from '@atcute/cid';
|
|
3
3
|
|
|
4
|
-
import { readCar } from './reader.js';
|
|
4
|
+
import { readCar, type CarEntry } from './reader.js';
|
|
5
5
|
|
|
6
6
|
const decoder = new TextDecoder();
|
|
7
7
|
|
|
8
|
+
export type BlockMap = Map<string, CarEntry>;
|
|
9
|
+
|
|
8
10
|
export class RepoEntry {
|
|
9
11
|
constructor(
|
|
12
|
+
/** The collection this record belongs to */
|
|
10
13
|
public readonly collection: string,
|
|
14
|
+
/** Record key */
|
|
11
15
|
public readonly rkey: string,
|
|
16
|
+
/** CID of this record */
|
|
12
17
|
public readonly cid: CID.CidLink,
|
|
13
18
|
private blockmap: BlockMap,
|
|
14
19
|
) {}
|
|
15
20
|
|
|
16
|
-
|
|
21
|
+
/**
|
|
22
|
+
* returns the associated CarEntry for this record
|
|
23
|
+
*/
|
|
24
|
+
get carEntry(): CarEntry {
|
|
17
25
|
const cid = this.cid.$link;
|
|
18
26
|
|
|
19
|
-
const
|
|
20
|
-
assert(
|
|
27
|
+
const entry = this.blockmap.get(cid);
|
|
28
|
+
assert(entry != null, `cid not found in blockmap; cid=${cid}`);
|
|
29
|
+
|
|
30
|
+
return entry;
|
|
31
|
+
}
|
|
21
32
|
|
|
22
|
-
|
|
33
|
+
/**
|
|
34
|
+
* returns the raw contents of this record
|
|
35
|
+
*/
|
|
36
|
+
get bytes(): Uint8Array {
|
|
37
|
+
return this.carEntry.bytes;
|
|
23
38
|
}
|
|
24
39
|
|
|
40
|
+
/**
|
|
41
|
+
* returns the decoded contents of this record
|
|
42
|
+
*/
|
|
25
43
|
get record(): unknown {
|
|
26
44
|
return CBOR.decode(this.bytes);
|
|
27
45
|
}
|
|
28
46
|
}
|
|
29
47
|
|
|
30
48
|
export function* iterateAtpRepo(buf: Uint8Array): Generator<RepoEntry> {
|
|
31
|
-
const {
|
|
49
|
+
const { header, iterate } = readCar(buf);
|
|
50
|
+
const roots = header.data.roots;
|
|
51
|
+
|
|
32
52
|
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
|
|
33
53
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
for (const entry of iterate()) {
|
|
37
|
-
blockmap.set(CID.toString(entry.cid), entry.bytes);
|
|
38
|
-
}
|
|
54
|
+
const blockmap = collectBlock(iterate());
|
|
55
|
+
assert(blockmap.size > 0, `expected at least 1 block in the archive; got=${blockmap.size}`);
|
|
39
56
|
|
|
40
|
-
|
|
41
|
-
const commit = readObject(blockmap, roots[0], isCommit);
|
|
57
|
+
const commit = readBlock(blockmap, roots[0], isCommit);
|
|
42
58
|
for (const { key, cid } of walkMstEntries(blockmap, commit.data)) {
|
|
43
59
|
const [collection, rkey] = key.split('/');
|
|
44
60
|
|
|
@@ -46,20 +62,53 @@ export function* iterateAtpRepo(buf: Uint8Array): Generator<RepoEntry> {
|
|
|
46
62
|
}
|
|
47
63
|
}
|
|
48
64
|
|
|
49
|
-
|
|
65
|
+
/**
|
|
66
|
+
* collects entries from a CAR archive into a mapping of CID string -> actual bytes
|
|
67
|
+
* @param iterator a generator that yields objects with a `cid` and `bytes` property
|
|
68
|
+
* @returns a mapping of CID string -> actual bytes
|
|
69
|
+
*/
|
|
70
|
+
export function collectBlock(iterator: Generator<CarEntry>): BlockMap {
|
|
71
|
+
const blockmap: BlockMap = new Map();
|
|
72
|
+
for (const entry of iterator) {
|
|
73
|
+
blockmap.set(CID.toString(entry.cid), entry);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return blockmap;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* reads a block from the blockmap and validates it against the provided validation function
|
|
81
|
+
* @param map a mapping of CID string -> actual bytes
|
|
82
|
+
* @param link a CID link to read
|
|
83
|
+
* @param validate a validation function to validate the decoded data
|
|
84
|
+
* @returns the decoded and validated data
|
|
85
|
+
*/
|
|
86
|
+
export function readBlock<T>(map: BlockMap, link: CID.CidLink, validate: (value: unknown) => value is T): T {
|
|
50
87
|
const cid = link.$link;
|
|
51
88
|
|
|
52
|
-
const
|
|
53
|
-
assert(
|
|
89
|
+
const entry = map.get(cid);
|
|
90
|
+
assert(entry != null, `cid not found in blockmap; cid=${cid}`);
|
|
54
91
|
|
|
55
|
-
const data = CBOR.decode(bytes);
|
|
92
|
+
const data = CBOR.decode(entry.bytes);
|
|
56
93
|
assert(validate(data), `validation failed for cid=${cid}`);
|
|
57
94
|
|
|
58
95
|
return data;
|
|
59
96
|
}
|
|
60
97
|
|
|
98
|
+
/** node entry object */
|
|
99
|
+
export interface NodeEntry {
|
|
100
|
+
key: string;
|
|
101
|
+
cid: CID.CidLink;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
|
|
106
|
+
* @param map a mapping of CID string -> actual bytes
|
|
107
|
+
* @param pointer a CID link to the root of the MST
|
|
108
|
+
* @returns a generator that yields the entries of the MST
|
|
109
|
+
*/
|
|
61
110
|
export function* walkMstEntries(map: BlockMap, pointer: CID.CidLink): Generator<NodeEntry> {
|
|
62
|
-
const data =
|
|
111
|
+
const data = readBlock(map, pointer, isMstNode);
|
|
63
112
|
const entries = data.e;
|
|
64
113
|
|
|
65
114
|
let lastKey = '';
|
|
@@ -90,8 +139,6 @@ function assert(condition: boolean, message: string): asserts condition {
|
|
|
90
139
|
}
|
|
91
140
|
}
|
|
92
141
|
|
|
93
|
-
export type BlockMap = Map<string, Uint8Array>;
|
|
94
|
-
|
|
95
142
|
const isCidLink = (value: unknown): value is CID.CidLink => {
|
|
96
143
|
if (value instanceof CID.CidLinkWrapper) {
|
|
97
144
|
return true;
|
|
@@ -116,6 +163,7 @@ const isBytes = (value: unknown): value is CBOR.Bytes => {
|
|
|
116
163
|
return '$bytes' in value && typeof value.$bytes === 'string';
|
|
117
164
|
};
|
|
118
165
|
|
|
166
|
+
/** commit object */
|
|
119
167
|
export interface Commit {
|
|
120
168
|
version: 3;
|
|
121
169
|
did: string;
|
|
@@ -125,6 +173,11 @@ export interface Commit {
|
|
|
125
173
|
sig: CBOR.Bytes;
|
|
126
174
|
}
|
|
127
175
|
|
|
176
|
+
/**
|
|
177
|
+
* checks if a value is a valid commit object
|
|
178
|
+
* @param value the value to check
|
|
179
|
+
* @returns true if the value is a valid commit object, false otherwise
|
|
180
|
+
*/
|
|
128
181
|
export const isCommit = (value: unknown): value is Commit => {
|
|
129
182
|
if (value === null || typeof value !== 'object') {
|
|
130
183
|
return false;
|
|
@@ -142,6 +195,7 @@ export const isCommit = (value: unknown): value is Commit => {
|
|
|
142
195
|
);
|
|
143
196
|
};
|
|
144
197
|
|
|
198
|
+
/** mst tree entry object */
|
|
145
199
|
export interface TreeEntry {
|
|
146
200
|
/** count of bytes shared with previous TreeEntry in this Node (if any) */
|
|
147
201
|
p: number;
|
|
@@ -153,6 +207,11 @@ export interface TreeEntry {
|
|
|
153
207
|
t: CID.CidLink | null;
|
|
154
208
|
}
|
|
155
209
|
|
|
210
|
+
/**
|
|
211
|
+
* checks if a value is a valid mst tree entry object
|
|
212
|
+
* @param value the value to check
|
|
213
|
+
* @returns true if the value is a valid mst tree entry object, false otherwise
|
|
214
|
+
*/
|
|
156
215
|
export const isTreeEntry = (value: unknown): value is TreeEntry => {
|
|
157
216
|
if (value === null || typeof value !== 'object') {
|
|
158
217
|
return false;
|
|
@@ -165,6 +224,7 @@ export const isTreeEntry = (value: unknown): value is TreeEntry => {
|
|
|
165
224
|
);
|
|
166
225
|
};
|
|
167
226
|
|
|
227
|
+
/** mst node object */
|
|
168
228
|
export interface MstNode {
|
|
169
229
|
/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
|
|
170
230
|
l: CID.CidLink | null;
|
|
@@ -172,6 +232,11 @@ export interface MstNode {
|
|
|
172
232
|
e: TreeEntry[];
|
|
173
233
|
}
|
|
174
234
|
|
|
235
|
+
/**
|
|
236
|
+
* checks if a value is a valid mst node object
|
|
237
|
+
* @param value the value to check
|
|
238
|
+
* @returns true if the value is a valid mst node object, false otherwise
|
|
239
|
+
*/
|
|
175
240
|
export const isMstNode = (value: unknown): value is MstNode => {
|
|
176
241
|
if (value === null || typeof value !== 'object') {
|
|
177
242
|
return false;
|
|
@@ -181,8 +246,3 @@ export const isMstNode = (value: unknown): value is MstNode => {
|
|
|
181
246
|
|
|
182
247
|
return (obj.l === null || isCidLink(obj.l)) && Array.isArray(obj.e) && obj.e.every(isTreeEntry);
|
|
183
248
|
};
|
|
184
|
-
|
|
185
|
-
export interface NodeEntry {
|
|
186
|
-
key: string;
|
|
187
|
-
cid: CID.CidLink;
|
|
188
|
-
}
|
package/lib/index.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
1
|
+
export * from './atproto-repo.js';
|
|
2
|
+
export * from './reader.js';
|
package/lib/reader.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { createUint8Reader } from './utilities/sync-byte-reader.js';
|
|
2
|
-
import { createCarReader } from './utilities/sync-car-reader.js';
|
|
2
|
+
import { createCarReader, type SyncCarReader } from './utilities/sync-car-reader.js';
|
|
3
3
|
|
|
4
|
-
export
|
|
4
|
+
export type { CarEntry, CarHeader, CarV1Header } from './utilities/car.js';
|
|
5
|
+
export type { SyncCarReader } from './utilities/sync-car-reader.js';
|
|
6
|
+
|
|
7
|
+
export const readCar = (buffer: Uint8Array): SyncCarReader => {
|
|
5
8
|
const reader = createUint8Reader(buffer);
|
|
6
9
|
return createCarReader(reader);
|
|
7
10
|
};
|
package/lib/utilities/car.ts
CHANGED
|
@@ -14,3 +14,25 @@ export const isCarV1Header = (value: unknown): value is CarV1Header => {
|
|
|
14
14
|
const { version, roots } = value as CarV1Header;
|
|
15
15
|
return version === 1 && Array.isArray(roots) && roots.every((root) => root instanceof CBOR.CidLinkWrapper);
|
|
16
16
|
};
|
|
17
|
+
|
|
18
|
+
export interface CarHeader {
|
|
19
|
+
headerStart: number;
|
|
20
|
+
headerEnd: number;
|
|
21
|
+
|
|
22
|
+
data: CarV1Header;
|
|
23
|
+
dataStart: number;
|
|
24
|
+
dataEnd: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface CarEntry {
|
|
28
|
+
entryStart: number;
|
|
29
|
+
entryEnd: number;
|
|
30
|
+
|
|
31
|
+
cid: CID.Cid;
|
|
32
|
+
cidStart: number;
|
|
33
|
+
cidEnd: number;
|
|
34
|
+
|
|
35
|
+
bytes: Uint8Array;
|
|
36
|
+
bytesStart: number;
|
|
37
|
+
bytesEnd: number;
|
|
38
|
+
}
|
|
@@ -21,7 +21,7 @@ export const createUint8Reader = (buf: Uint8Array): SyncByteReader => {
|
|
|
21
21
|
pos += size;
|
|
22
22
|
},
|
|
23
23
|
upto(size) {
|
|
24
|
-
return buf.subarray(pos, pos +
|
|
24
|
+
return buf.subarray(pos, pos + size);
|
|
25
25
|
},
|
|
26
26
|
exactly(size, seek) {
|
|
27
27
|
if (size > buf.length - pos) {
|
|
@@ -2,9 +2,14 @@ import * as CBOR from '@atcute/cbor';
|
|
|
2
2
|
import * as CID from '@atcute/cid';
|
|
3
3
|
import * as varint from '@atcute/varint';
|
|
4
4
|
|
|
5
|
-
import { isCarV1Header, type
|
|
5
|
+
import { isCarV1Header, type CarEntry, type CarHeader } from './car.js';
|
|
6
6
|
import type { SyncByteReader } from './sync-byte-reader.js';
|
|
7
7
|
|
|
8
|
+
export interface SyncCarReader {
|
|
9
|
+
header: CarHeader;
|
|
10
|
+
iterate(): Generator<CarEntry>;
|
|
11
|
+
}
|
|
12
|
+
|
|
8
13
|
const readVarint = (reader: SyncByteReader, size: number): number => {
|
|
9
14
|
const buf = reader.upto(size);
|
|
10
15
|
if (buf.length === 0) {
|
|
@@ -17,27 +22,34 @@ const readVarint = (reader: SyncByteReader, size: number): number => {
|
|
|
17
22
|
return int;
|
|
18
23
|
};
|
|
19
24
|
|
|
20
|
-
const readHeader = (reader: SyncByteReader):
|
|
25
|
+
const readHeader = (reader: SyncByteReader): CarHeader => {
|
|
26
|
+
const headerStart = reader.pos;
|
|
21
27
|
const length = readVarint(reader, 8);
|
|
22
28
|
if (length === 0) {
|
|
23
29
|
throw new RangeError(`invalid car header; length=0`);
|
|
24
30
|
}
|
|
25
31
|
|
|
32
|
+
const dataStart = reader.pos;
|
|
26
33
|
const rawHeader = reader.exactly(length, true);
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
|
|
35
|
+
const data = CBOR.decode(rawHeader);
|
|
36
|
+
if (!isCarV1Header(data)) {
|
|
29
37
|
throw new TypeError(`expected a car v1 archive`);
|
|
30
38
|
}
|
|
31
39
|
|
|
32
|
-
|
|
40
|
+
const dataEnd = reader.pos;
|
|
41
|
+
const headerEnd = dataEnd;
|
|
42
|
+
|
|
43
|
+
return { data, headerStart, headerEnd, dataStart, dataEnd };
|
|
33
44
|
};
|
|
34
45
|
|
|
35
46
|
const readCid = (reader: SyncByteReader): CID.Cid => {
|
|
36
|
-
const head = reader.
|
|
47
|
+
const head = reader.exactly(4, false);
|
|
37
48
|
|
|
38
49
|
const version = head[0];
|
|
39
50
|
const codec = head[1];
|
|
40
|
-
const
|
|
51
|
+
const digestType = head[2];
|
|
52
|
+
const digestSize = head[3];
|
|
41
53
|
|
|
42
54
|
if (version !== CID.CID_VERSION) {
|
|
43
55
|
throw new RangeError(`incorrect cid version (got v${version})`);
|
|
@@ -47,20 +59,22 @@ const readCid = (reader: SyncByteReader): CID.Cid => {
|
|
|
47
59
|
throw new RangeError(`incorrect cid codec (got 0x${codec.toString(16)})`);
|
|
48
60
|
}
|
|
49
61
|
|
|
50
|
-
if (
|
|
51
|
-
throw new RangeError(`incorrect cid hash type (got 0x${
|
|
62
|
+
if (digestType !== CID.HASH_SHA256) {
|
|
63
|
+
throw new RangeError(`incorrect cid hash type (got 0x${digestType.toString(16)})`);
|
|
52
64
|
}
|
|
53
65
|
|
|
54
|
-
|
|
66
|
+
if (digestSize !== 32 && digestSize !== 0) {
|
|
67
|
+
throw new RangeError(`incorrect cid digest size (got ${digestSize})`);
|
|
68
|
+
}
|
|
55
69
|
|
|
56
|
-
const bytes = reader.exactly(
|
|
57
|
-
const digest = bytes.subarray(
|
|
70
|
+
const bytes = reader.exactly(4 + digestSize, true);
|
|
71
|
+
const digest = bytes.subarray(4, 4 + digestSize);
|
|
58
72
|
|
|
59
73
|
const cid: CID.Cid = {
|
|
60
74
|
version: version,
|
|
61
75
|
codec: codec,
|
|
62
76
|
digest: {
|
|
63
|
-
codec:
|
|
77
|
+
codec: digestType,
|
|
64
78
|
contents: digest,
|
|
65
79
|
},
|
|
66
80
|
bytes: bytes,
|
|
@@ -69,33 +83,38 @@ const readCid = (reader: SyncByteReader): CID.Cid => {
|
|
|
69
83
|
return cid;
|
|
70
84
|
};
|
|
71
85
|
|
|
72
|
-
const
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
let size = readVarint(reader, 8);
|
|
76
|
-
if (size === 0) {
|
|
77
|
-
throw new Error(`invalid car section; length=0`);
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
size += reader.pos - start;
|
|
81
|
-
|
|
82
|
-
const cid = readCid(reader);
|
|
83
|
-
const blockSize = size - (reader.pos - start);
|
|
84
|
-
|
|
85
|
-
return { cid, blockSize };
|
|
86
|
-
};
|
|
87
|
-
|
|
88
|
-
export const createCarReader = (reader: SyncByteReader) => {
|
|
89
|
-
const { roots } = readHeader(reader);
|
|
86
|
+
export const createCarReader = (reader: SyncByteReader): SyncCarReader => {
|
|
87
|
+
const header = readHeader(reader);
|
|
90
88
|
|
|
91
89
|
return {
|
|
92
|
-
|
|
93
|
-
*iterate()
|
|
94
|
-
while (reader.upto(8).length > 0) {
|
|
95
|
-
const
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
90
|
+
header,
|
|
91
|
+
*iterate() {
|
|
92
|
+
while (reader.upto(8 + 36).length > 0) {
|
|
93
|
+
const entryStart = reader.pos;
|
|
94
|
+
const entrySize = readVarint(reader, 8);
|
|
95
|
+
|
|
96
|
+
const cidStart = reader.pos;
|
|
97
|
+
const cid = readCid(reader);
|
|
98
|
+
|
|
99
|
+
const bytesStart = reader.pos;
|
|
100
|
+
const bytesSize = entrySize - (bytesStart - cidStart);
|
|
101
|
+
const bytes = reader.exactly(bytesSize, true);
|
|
102
|
+
|
|
103
|
+
const cidEnd = bytesStart;
|
|
104
|
+
const bytesEnd = reader.pos;
|
|
105
|
+
const entryEnd = bytesEnd;
|
|
106
|
+
|
|
107
|
+
yield {
|
|
108
|
+
cid,
|
|
109
|
+
bytes,
|
|
110
|
+
|
|
111
|
+
entryStart,
|
|
112
|
+
entryEnd,
|
|
113
|
+
cidStart,
|
|
114
|
+
cidEnd,
|
|
115
|
+
bytesStart,
|
|
116
|
+
bytesEnd,
|
|
117
|
+
};
|
|
99
118
|
}
|
|
100
119
|
},
|
|
101
120
|
};
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@atcute/car",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "3.0.0",
|
|
5
5
|
"description": "lightweight DASL CAR and atproto repository decoder for AT Protocol.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"atproto",
|
|
@@ -28,9 +28,9 @@
|
|
|
28
28
|
"@atcute/multibase": "^1.1.2"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@atcute/
|
|
31
|
+
"@atcute/cid": "^2.2.0",
|
|
32
32
|
"@atcute/varint": "^1.0.2",
|
|
33
|
-
"@atcute/
|
|
33
|
+
"@atcute/cbor": "^2.2.0"
|
|
34
34
|
},
|
|
35
35
|
"scripts": {
|
|
36
36
|
"build": "tsc --project tsconfig.build.json",
|