@atcute/car 3.1.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -5,9 +5,17 @@ library for AT Protocol.
|
|
|
5
5
|
|
|
6
6
|
[dasl-car]: https://dasl.ing/car.html
|
|
7
7
|
|
|
8
|
+
## usage
|
|
9
|
+
|
|
10
|
+
### streaming usage
|
|
11
|
+
|
|
8
12
|
```ts
|
|
9
13
|
import { CarReader, RepoReader } from '@atcute/car/v4';
|
|
10
14
|
|
|
15
|
+
const stream = new ReadableStream({
|
|
16
|
+
/* ... */
|
|
17
|
+
});
|
|
18
|
+
|
|
11
19
|
// read AT Protocol repository exports
|
|
12
20
|
{
|
|
13
21
|
await using repo = RepoReader.fromStream(stream);
|
|
@@ -33,3 +41,51 @@ import { CarReader, RepoReader } from '@atcute/car/v4';
|
|
|
33
41
|
}
|
|
34
42
|
}
|
|
35
43
|
```
|
|
44
|
+
|
|
45
|
+
### streaming usage (for runtimes without `await using` yet)
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
const repo = RepoReader.fromStream(stream);
|
|
49
|
+
|
|
50
|
+
try {
|
|
51
|
+
for await (const entry of repo) {
|
|
52
|
+
entry;
|
|
53
|
+
// ^? RepoEntry
|
|
54
|
+
}
|
|
55
|
+
} finally {
|
|
56
|
+
await repo.dispose();
|
|
57
|
+
}
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### sync usage
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
const buffer = Uint8Array.from([
|
|
64
|
+
/* ... */
|
|
65
|
+
]);
|
|
66
|
+
|
|
67
|
+
// read AT Protocol repository exports
|
|
68
|
+
{
|
|
69
|
+
const repo = RepoReader.fromUint8Array(buffer);
|
|
70
|
+
|
|
71
|
+
for (const entry of repo) {
|
|
72
|
+
entry;
|
|
73
|
+
// ^? RepoEntry { collection: 'app.bsky.feed.post', rkey: '3lprcc55bb222', ... }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
repo.missingBlocks;
|
|
77
|
+
// ^? []
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// read generic CAR archives
|
|
81
|
+
{
|
|
82
|
+
const car = CarReader.fromUint8Array(buffer);
|
|
83
|
+
|
|
84
|
+
const roots = car.roots;
|
|
85
|
+
|
|
86
|
+
for (const entry of car) {
|
|
87
|
+
entry;
|
|
88
|
+
// ^? CarEntry { cid: CidLink {}, bytes: Uint8Array {}, ... }
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
```
|
|
@@ -46,91 +46,97 @@ export const fromStream = (stream) => {
|
|
|
46
46
|
return this.dispose();
|
|
47
47
|
},
|
|
48
48
|
async *[Symbol.asyncIterator]() {
|
|
49
|
-
await using car = CarReader.fromStream(stream);
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
strays.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
else {
|
|
60
|
-
pending.set(cid, meta);
|
|
61
|
-
}
|
|
62
|
-
};
|
|
63
|
-
{
|
|
64
|
-
const roots = await car.roots();
|
|
65
|
-
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
|
|
66
|
-
const rootCid = roots[0].$link;
|
|
67
|
-
request(rootCid, { t: 0 });
|
|
68
|
-
}
|
|
69
|
-
for await (const entry of car) {
|
|
70
|
-
const cid = CID.toString(entry.cid);
|
|
71
|
-
{
|
|
72
|
-
const meta = pending.get(cid);
|
|
73
|
-
if (meta !== undefined) {
|
|
74
|
-
pending.delete(cid);
|
|
49
|
+
// await using car = CarReader.fromStream(stream);
|
|
50
|
+
const car = CarReader.fromStream(stream);
|
|
51
|
+
try {
|
|
52
|
+
const pending = new Map();
|
|
53
|
+
const strays = new Map();
|
|
54
|
+
const queue = new Queue();
|
|
55
|
+
const request = (cid, meta) => {
|
|
56
|
+
const entry = strays.get(cid);
|
|
57
|
+
if (entry !== undefined) {
|
|
58
|
+
strays.delete(cid);
|
|
75
59
|
queue.enqueue({ c: cid, e: entry, m: meta });
|
|
76
60
|
}
|
|
77
61
|
else {
|
|
78
|
-
|
|
62
|
+
pending.set(cid, meta);
|
|
79
63
|
}
|
|
64
|
+
};
|
|
65
|
+
{
|
|
66
|
+
const roots = await car.roots();
|
|
67
|
+
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
|
|
68
|
+
const rootCid = roots[0].$link;
|
|
69
|
+
request(rootCid, { t: 0 });
|
|
80
70
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
request(commit.data.$link, { t: 1 });
|
|
89
|
-
break;
|
|
71
|
+
for await (const entry of car) {
|
|
72
|
+
const cid = CID.toString(entry.cid);
|
|
73
|
+
{
|
|
74
|
+
const meta = pending.get(cid);
|
|
75
|
+
if (meta !== undefined) {
|
|
76
|
+
pending.delete(cid);
|
|
77
|
+
queue.enqueue({ c: cid, e: entry, m: meta });
|
|
90
78
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
79
|
+
else {
|
|
80
|
+
strays.set(cid, entry);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
let task;
|
|
84
|
+
while ((task = queue.dequeue())) {
|
|
85
|
+
const { c: cid, e: entry, m: meta } = task;
|
|
86
|
+
switch (meta.t) {
|
|
87
|
+
case 0: {
|
|
88
|
+
const commit = CBOR.decode(entry.bytes);
|
|
89
|
+
assert(isCommit(commit), `expected commit block; cid=${cid}`);
|
|
90
|
+
request(commit.data.$link, { t: 1 });
|
|
91
|
+
break;
|
|
99
92
|
}
|
|
100
|
-
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
const
|
|
104
|
-
const
|
|
105
|
-
lastKey =
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
request(next.$link, { t: 1 });
|
|
93
|
+
case 1: {
|
|
94
|
+
const node = CBOR.decode(entry.bytes);
|
|
95
|
+
assert(isMstNode(node), `expected mst node block; cid=${cid}`);
|
|
96
|
+
const entries = node.e;
|
|
97
|
+
const left = node.l;
|
|
98
|
+
let lastKey = '';
|
|
99
|
+
if (left !== null) {
|
|
100
|
+
request(left.$link, meta);
|
|
109
101
|
}
|
|
102
|
+
for (let i = 0, il = entries.length; i < il; i++) {
|
|
103
|
+
const entry = entries[i];
|
|
104
|
+
const next = entry.t;
|
|
105
|
+
const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
|
|
106
|
+
const key = lastKey.slice(0, entry.p) + key_str;
|
|
107
|
+
lastKey = key;
|
|
108
|
+
request(entry.v.$link, { t: 2, k: key });
|
|
109
|
+
if (next !== null) {
|
|
110
|
+
request(next.$link, { t: 1 });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
case 2: {
|
|
116
|
+
const [collection, rkey] = meta.k.split('/');
|
|
117
|
+
yield new RepoEntry(collection, rkey, CID.toCidLink(entry.cid), entry);
|
|
118
|
+
break;
|
|
110
119
|
}
|
|
111
|
-
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
missingBlocks = Array.from(pending, ([cid, meta]) => {
|
|
124
|
+
switch (meta.t) {
|
|
125
|
+
case 0: {
|
|
126
|
+
return { cid, type: 'commit' };
|
|
127
|
+
}
|
|
128
|
+
case 1: {
|
|
129
|
+
return { cid, type: 'mst-node' };
|
|
112
130
|
}
|
|
113
131
|
case 2: {
|
|
114
|
-
|
|
115
|
-
yield new RepoEntry(collection, rkey, CID.toCidLink(entry.cid), entry);
|
|
116
|
-
break;
|
|
132
|
+
return { cid, type: 'record', key: meta.k };
|
|
117
133
|
}
|
|
118
134
|
}
|
|
119
|
-
}
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
await car.dispose();
|
|
120
139
|
}
|
|
121
|
-
missingBlocks = Array.from(pending, ([cid, meta]) => {
|
|
122
|
-
switch (meta.t) {
|
|
123
|
-
case 0: {
|
|
124
|
-
return { cid, type: 'commit' };
|
|
125
|
-
}
|
|
126
|
-
case 1: {
|
|
127
|
-
return { cid, type: 'mst-node' };
|
|
128
|
-
}
|
|
129
|
-
case 2: {
|
|
130
|
-
return { cid, type: 'record', key: meta.k };
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
});
|
|
134
140
|
},
|
|
135
141
|
};
|
|
136
142
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream-repo-reader.js","sourceRoot":"","sources":["../../../lib/v4/repo-reader/stream-repo-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,OAAO,KAAK,SAAS,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAsCvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAgD,EAAE;IACnF,MAAM,SAAS,GAAG,IAAI,eAAe,EAA0B,CAAC;IAChE,IAAI,IAAoC,CAAC;IAEzC,OAAO;QACN,QAAQ,EAAE,IAAI,cAAc,CAAC;YAC5B,KAAK,CAAC,KAAK,CAAC,UAAU;gBACrB,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBAEtC,IAAI,CAAC;oBACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;wBAChC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC3B,CAAC;oBAED,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAErB,UAAU,CAAC,KAAK,EAAE,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;YACF,CAAC;YACD,KAAK,CAAC,MAAM;gBACX,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACtB,CAAC;YACF,CAAC;SACD,CAAC;QACF,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC5B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAkC,EAAsB,EAAE;IACpF,IAAI,aAAa,GAAwB,EAAE,CAAC;IAE5C,OAAO;QACN,IAAI,aAAa;YAChB,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,KAAK,CAAC,OAAO;YACZ,uBAAuB;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,YAAY,CAAC;YACpB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;QACD,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAC5B,
|
|
1
|
+
{"version":3,"file":"stream-repo-reader.js","sourceRoot":"","sources":["../../../lib/v4/repo-reader/stream-repo-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,OAAO,KAAK,SAAS,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAsCvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAgD,EAAE;IACnF,MAAM,SAAS,GAAG,IAAI,eAAe,EAA0B,CAAC;IAChE,IAAI,IAAoC,CAAC;IAEzC,OAAO;QACN,QAAQ,EAAE,IAAI,cAAc,CAAC;YAC5B,KAAK,CAAC,KAAK,CAAC,UAAU;gBACrB,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBAEtC,IAAI,CAAC;oBACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;wBAChC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC3B,CAAC;oBAED,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAErB,UAAU,CAAC,KAAK,EAAE,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;YACF,CAAC;YACD,KAAK,CAAC,MAAM;gBACX,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACtB,CAAC;YACF,CAAC;SACD,CAAC;QACF,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC5B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAkC,EAAsB,EAAE;IACpF,IAAI,aAAa,GAAwB,EAAE,CAAC;IAE5C,OAAO;QACN,IAAI,aAAa;YAChB,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,KAAK,CAAC,OAAO;YACZ,uBAAuB;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,YAAY,CAAC;YACpB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;QACD,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAC5B,kDAAkD;YAClD,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAEzC,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;gBAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;gBAErD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAQ,CAAC;gBAEhC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,IAAe,EAAQ,EAAE;oBACtD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAE9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBACnB,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC9C,CAAC;yBAAM,CAAC;wBACP,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBACxB,CAAC;gBACF,CAAC,CAAC;gBAEF,CAAC;oBACA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;oBAChC,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,gDAAgD,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;oBAE3F,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC/B,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAEpC,CAAC;wBACA,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBAE9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;4BACxB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;4BACpB,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC9C,CAAC;6BAAM,CAAC;4BACP,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBACxB,CAAC;oBACF,CAAC;oBAED,IAAI,IAAsB,CAAC;oBAC3B,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;wBACjC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;wBAE3C,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;4BAChB,KAAK,CAAC,CAAC,CAAC,CAAC;gCACR,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gCACxC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,8BAA8B,GAAG,EAAE,CAAC,CAAC;gCAE9D,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gCACrC,MAAM;4BACP,CAAC;4BACD,KAAK,CAAC,CAAC,CAAC,CAAC;gCACR,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gCACtC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,gCAAgC,GAAG,EAAE,CAAC,CAAC;gCAE/D,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;gCACvB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;gCAEpB,IAAI,OAAO,GAAG,EAAE,CAAC;gCAEjB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oCACnB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gCAC3B,CAAC;gCAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oCAClD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;oCACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC;oCAErB,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oCACxD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;oCAEhD,OAAO,GAAG,GAAG,CAAC;oCAEd,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oCAEzC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wCACnB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oCAC/B,CAAC;gCACF,CAAC;gCAED,MAAM;4BACP,CAAC;4BACD,KAAK,CAAC,CAAC,CAAC,CAAC;gCACR,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gCAE7C,MAAM,IAAI,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;gCACvE,MAAM;4BACP,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAqB,EAAE;oBACtE,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;wBAChB,KAAK,CAAC,CAAC,CAAC,CAAC;4BACR,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;wBAChC,CAAC;wBACD,KAAK,CAAC,CAAC,CAAC,CAAC;4BACR,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;wBAClC,CAAC;wBACD,KAAK,CAAC,CAAC,CAAC,CAAC;4BACR,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;wBAC7C,CAAC;oBACF,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACV,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;YACrB,CAAC;QACF,CAAC;KACD,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -93,112 +93,117 @@ export const fromStream = (stream: ReadableStream<Uint8Array>): StreamedRepoRead
|
|
|
93
93
|
return this.dispose();
|
|
94
94
|
},
|
|
95
95
|
async *[Symbol.asyncIterator]() {
|
|
96
|
-
await using car = CarReader.fromStream(stream);
|
|
96
|
+
// await using car = CarReader.fromStream(stream);
|
|
97
|
+
const car = CarReader.fromStream(stream);
|
|
97
98
|
|
|
98
|
-
|
|
99
|
-
|
|
99
|
+
try {
|
|
100
|
+
const pending = new Map<string, EntryMeta>();
|
|
101
|
+
const strays = new Map<string, CarReader.CarEntry>();
|
|
100
102
|
|
|
101
|
-
|
|
103
|
+
const queue = new Queue<Task>();
|
|
102
104
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
+
const request = (cid: string, meta: EntryMeta): void => {
|
|
106
|
+
const entry = strays.get(cid);
|
|
105
107
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
queue.enqueue({ c: cid, e: entry, m: meta });
|
|
109
|
-
} else {
|
|
110
|
-
pending.set(cid, meta);
|
|
111
|
-
}
|
|
112
|
-
};
|
|
113
|
-
|
|
114
|
-
{
|
|
115
|
-
const roots = await car.roots();
|
|
116
|
-
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
|
|
117
|
-
|
|
118
|
-
const rootCid = roots[0].$link;
|
|
119
|
-
request(rootCid, { t: 0 });
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
for await (const entry of car) {
|
|
123
|
-
const cid = CID.toString(entry.cid);
|
|
124
|
-
|
|
125
|
-
{
|
|
126
|
-
const meta = pending.get(cid);
|
|
127
|
-
|
|
128
|
-
if (meta !== undefined) {
|
|
129
|
-
pending.delete(cid);
|
|
108
|
+
if (entry !== undefined) {
|
|
109
|
+
strays.delete(cid);
|
|
130
110
|
queue.enqueue({ c: cid, e: entry, m: meta });
|
|
131
111
|
} else {
|
|
132
|
-
|
|
112
|
+
pending.set(cid, meta);
|
|
133
113
|
}
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
{
|
|
117
|
+
const roots = await car.roots();
|
|
118
|
+
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
|
|
119
|
+
|
|
120
|
+
const rootCid = roots[0].$link;
|
|
121
|
+
request(rootCid, { t: 0 });
|
|
134
122
|
}
|
|
135
123
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
const { c: cid, e: entry, m: meta } = task;
|
|
124
|
+
for await (const entry of car) {
|
|
125
|
+
const cid = CID.toString(entry.cid);
|
|
139
126
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
const commit = CBOR.decode(entry.bytes);
|
|
143
|
-
assert(isCommit(commit), `expected commit block; cid=${cid}`);
|
|
127
|
+
{
|
|
128
|
+
const meta = pending.get(cid);
|
|
144
129
|
|
|
145
|
-
|
|
146
|
-
|
|
130
|
+
if (meta !== undefined) {
|
|
131
|
+
pending.delete(cid);
|
|
132
|
+
queue.enqueue({ c: cid, e: entry, m: meta });
|
|
133
|
+
} else {
|
|
134
|
+
strays.set(cid, entry);
|
|
147
135
|
}
|
|
148
|
-
|
|
149
|
-
const node = CBOR.decode(entry.bytes);
|
|
150
|
-
assert(isMstNode(node), `expected mst node block; cid=${cid}`);
|
|
136
|
+
}
|
|
151
137
|
|
|
152
|
-
|
|
153
|
-
|
|
138
|
+
let task: Task | undefined;
|
|
139
|
+
while ((task = queue.dequeue())) {
|
|
140
|
+
const { c: cid, e: entry, m: meta } = task;
|
|
154
141
|
|
|
155
|
-
|
|
142
|
+
switch (meta.t) {
|
|
143
|
+
case 0: {
|
|
144
|
+
const commit = CBOR.decode(entry.bytes);
|
|
145
|
+
assert(isCommit(commit), `expected commit block; cid=${cid}`);
|
|
156
146
|
|
|
157
|
-
|
|
158
|
-
|
|
147
|
+
request(commit.data.$link, { t: 1 });
|
|
148
|
+
break;
|
|
159
149
|
}
|
|
150
|
+
case 1: {
|
|
151
|
+
const node = CBOR.decode(entry.bytes);
|
|
152
|
+
assert(isMstNode(node), `expected mst node block; cid=${cid}`);
|
|
153
|
+
|
|
154
|
+
const entries = node.e;
|
|
155
|
+
const left = node.l;
|
|
156
|
+
|
|
157
|
+
let lastKey = '';
|
|
158
|
+
|
|
159
|
+
if (left !== null) {
|
|
160
|
+
request(left.$link, meta);
|
|
161
|
+
}
|
|
160
162
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
163
|
+
for (let i = 0, il = entries.length; i < il; i++) {
|
|
164
|
+
const entry = entries[i];
|
|
165
|
+
const next = entry.t;
|
|
164
166
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
+
const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
|
|
168
|
+
const key = lastKey.slice(0, entry.p) + key_str;
|
|
167
169
|
|
|
168
|
-
|
|
170
|
+
lastKey = key;
|
|
169
171
|
|
|
170
|
-
|
|
172
|
+
request(entry.v.$link, { t: 2, k: key });
|
|
171
173
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
+
if (next !== null) {
|
|
175
|
+
request(next.$link, { t: 1 });
|
|
176
|
+
}
|
|
174
177
|
}
|
|
175
|
-
}
|
|
176
178
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
179
|
+
break;
|
|
180
|
+
}
|
|
181
|
+
case 2: {
|
|
182
|
+
const [collection, rkey] = meta.k.split('/');
|
|
181
183
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
+
yield new RepoEntry(collection, rkey, CID.toCidLink(entry.cid), entry);
|
|
185
|
+
break;
|
|
186
|
+
}
|
|
184
187
|
}
|
|
185
188
|
}
|
|
186
189
|
}
|
|
187
|
-
}
|
|
188
190
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
191
|
+
missingBlocks = Array.from(pending, ([cid, meta]): MissingBlockEntry => {
|
|
192
|
+
switch (meta.t) {
|
|
193
|
+
case 0: {
|
|
194
|
+
return { cid, type: 'commit' };
|
|
195
|
+
}
|
|
196
|
+
case 1: {
|
|
197
|
+
return { cid, type: 'mst-node' };
|
|
198
|
+
}
|
|
199
|
+
case 2: {
|
|
200
|
+
return { cid, type: 'record', key: meta.k };
|
|
201
|
+
}
|
|
199
202
|
}
|
|
200
|
-
}
|
|
201
|
-
}
|
|
203
|
+
});
|
|
204
|
+
} finally {
|
|
205
|
+
await car.dispose();
|
|
206
|
+
}
|
|
202
207
|
},
|
|
203
208
|
};
|
|
204
209
|
};
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@atcute/car",
|
|
4
|
-
"version": "3.1.
|
|
4
|
+
"version": "3.1.1",
|
|
5
5
|
"description": "lightweight DASL CAR and atproto repository decoder for AT Protocol.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"atproto",
|
|
@@ -33,9 +33,9 @@
|
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"yocto-queue": "^1.2.1",
|
|
35
35
|
"@atcute/cbor": "^2.2.4",
|
|
36
|
+
"@atcute/cid": "^2.2.3",
|
|
36
37
|
"@atcute/uint8array": "^1.0.2",
|
|
37
|
-
"@atcute/varint": "^1.0.2"
|
|
38
|
-
"@atcute/cid": "^2.2.3"
|
|
38
|
+
"@atcute/varint": "^1.0.2"
|
|
39
39
|
},
|
|
40
40
|
"scripts": {
|
|
41
41
|
"build": "tsc --project tsconfig.build.json",
|