@peerbit/blocks 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/esm/block.js +4 -4
- package/lib/esm/index.d.ts +1 -1
- package/lib/esm/index.js +1 -1
- package/lib/esm/index.js.map +1 -1
- package/lib/esm/level.js +3 -3
- package/lib/esm/level.js.map +1 -1
- package/lib/esm/libp2p.d.ts +3 -0
- package/lib/esm/libp2p.js +26 -14
- package/lib/esm/libp2p.js.map +1 -1
- package/package.json +7 -7
- package/src/block.ts +4 -4
- package/src/index.ts +1 -1
- package/src/level.ts +3 -3
- package/src/libp2p.ts +40 -17
package/lib/esm/block.js
CHANGED
|
@@ -9,11 +9,11 @@ const defaultBase = base58btc;
|
|
|
9
9
|
export const defaultHasher = sha256;
|
|
10
10
|
export const codecCodes = {
|
|
11
11
|
[dagCbor.code]: dagCbor,
|
|
12
|
-
[raw.code]: raw
|
|
12
|
+
[raw.code]: raw
|
|
13
13
|
};
|
|
14
14
|
export const codecMap = {
|
|
15
15
|
raw: raw,
|
|
16
|
-
"dag-cbor": dagCbor
|
|
16
|
+
"dag-cbor": dagCbor
|
|
17
17
|
};
|
|
18
18
|
export const cidifyString = (str) => {
|
|
19
19
|
if (!str) {
|
|
@@ -36,7 +36,7 @@ export const checkDecodeBlock = async (expectedCID, bytes, options) => {
|
|
|
36
36
|
const block = await Block.decode({
|
|
37
37
|
bytes: bytes,
|
|
38
38
|
codec,
|
|
39
|
-
hasher: options?.hasher || defaultHasher
|
|
39
|
+
hasher: options?.hasher || defaultHasher
|
|
40
40
|
});
|
|
41
41
|
if (!block.cid.equals(cidObject)) {
|
|
42
42
|
throw new Error("CID does not match");
|
|
@@ -82,7 +82,7 @@ export const createBlock = async (value, format, options) => {
|
|
|
82
82
|
const block = await Block.encode({
|
|
83
83
|
value,
|
|
84
84
|
codec,
|
|
85
|
-
hasher: options?.hasher || defaultHasher
|
|
85
|
+
hasher: options?.hasher || defaultHasher
|
|
86
86
|
});
|
|
87
87
|
return block;
|
|
88
88
|
};
|
package/lib/esm/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { cidifyString, stringifyCid, createBlock, getBlockValue
|
|
1
|
+
export { cidifyString, stringifyCid, createBlock, getBlockValue } from "./block.js";
|
|
2
2
|
export { DirectBlock } from "./libp2p.js";
|
|
3
3
|
export * from "./store.js";
|
|
4
4
|
export * from "./level.js";
|
package/lib/esm/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { cidifyString, stringifyCid, createBlock, getBlockValue
|
|
1
|
+
export { cidifyString, stringifyCid, createBlock, getBlockValue } from "./block.js";
|
|
2
2
|
export { DirectBlock } from "./libp2p.js";
|
|
3
3
|
export * from "./store.js";
|
|
4
4
|
export * from "./level.js";
|
package/lib/esm/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,aAAa,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,aAAa,EACb,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC"}
|
package/lib/esm/level.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cidifyString, codecCodes, createBlock, defaultHasher, stringifyCid
|
|
1
|
+
import { cidifyString, codecCodes, createBlock, defaultHasher, stringifyCid } from "./block.js";
|
|
2
2
|
import * as Block from "multiformats/block";
|
|
3
3
|
import { MemoryLevel } from "memory-level";
|
|
4
4
|
import { waitFor } from "@peerbit/time";
|
|
@@ -22,7 +22,7 @@ export class LevelBlockStore {
|
|
|
22
22
|
const block = await Block.decode({
|
|
23
23
|
bytes,
|
|
24
24
|
codec,
|
|
25
|
-
hasher: options?.hasher || defaultHasher
|
|
25
|
+
hasher: options?.hasher || defaultHasher
|
|
26
26
|
});
|
|
27
27
|
return block.bytes;
|
|
28
28
|
}
|
|
@@ -56,7 +56,7 @@ export class LevelBlockStore {
|
|
|
56
56
|
timeout: 10 * 1000,
|
|
57
57
|
stopperCallback: (fn) => {
|
|
58
58
|
this._onClose = fn;
|
|
59
|
-
}
|
|
59
|
+
}
|
|
60
60
|
});
|
|
61
61
|
await this._opening;
|
|
62
62
|
}
|
package/lib/esm/level.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"level.js","sourceRoot":"","sources":["../../src/level.ts"],"names":[],"mappings":"AACA,OAAO,EACN,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EACb,YAAY,
|
|
1
|
+
{"version":3,"file":"level.js","sourceRoot":"","sources":["../../src/level.ts"],"names":[],"mappings":"AACA,OAAO,EACN,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EACb,YAAY,EACZ,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,KAAK,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,SAA+B,MAAM,qBAAqB,CAAC;AAKlE,MAAM,OAAO,eAAe;IACnB,MAAM,CAAY;IAClB,QAAQ,CAAe;IACvB,OAAO,GAAG,KAAK,CAAC;IAChB,QAAQ,CAA0B;IAC1C,YACC,KAA6C,EAC7C,OAA0B;QAE1B,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,GAAG,CACR,GAAW,EACX,OAKC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,EAAE;gBACX,OAAO,SAAS,CAAC;aACjB;YACD,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;gBAChC,KAAK;gBACL,KAAK;gBACL,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa;aACxC,CAAC,CAAC;YACH,OAAQ,KAAgD,CAAC,KAAK,CAAC;SAC/D;QAAC,OAAO,KAAU,EAAE;YACpB,IACC,OAAO,KAAK,EAAE,IAAI,KAAK,QAAQ;gBAC/B,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAC7C;gBACD,OAAO,SAAS,CAAC;aACjB;YACD,MAAM,KAAK,CAAC;SACZ;IACF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAiB;QAC1B,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;QAC3B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,GAAG,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,EAAE,CAAC,GAAW;QACnB,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACpB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAEzB,IAAI;YACH,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,MAAM,EAAE;gBAC9D,aAAa,EAAE,GAAG;gBAClB,OAAO,EAAE,EAAE,GAAG,IAAI;gBAClB,eAAe,EAAE,CAAC,EAAE,EAAE,EAAE;oBACvB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;gBACpB,CAAC;aACD,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC;SACpB;gBAAS;YACT,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;SAC1B;IACF,CAAC;IAED,KAAK,CAAC,IAAI;QACT,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,IAAI;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,MAAM;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,IAA4B;QACzC,OAAO,CAAC,mDAAmD;IAC5D,CAAC;CACD;AAED,MAAM,OAAO,qBAAsB,SAAQ,eAAe;IACzD,YAAY,OAA0B;QACrC,KAAK,CAAC,IAAI,WAAW,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAC5D,CAAC;CACD"}
|
package/lib/esm/libp2p.d.ts
CHANGED
|
@@ -18,18 +18,21 @@ export declare class DirectBlock extends DirectStream implements IBlocks {
|
|
|
18
18
|
private _localStore;
|
|
19
19
|
private _responseHandler?;
|
|
20
20
|
private _resolvers;
|
|
21
|
+
private _loadFetchQueue;
|
|
21
22
|
private _readFromPeersPromises;
|
|
22
23
|
_open: boolean;
|
|
23
24
|
constructor(components: DirectBlockComponents, options?: {
|
|
24
25
|
directory?: string;
|
|
25
26
|
canRelayMessage?: boolean;
|
|
26
27
|
localTimeout?: number;
|
|
28
|
+
messageProcessingConcurrency?: number;
|
|
27
29
|
});
|
|
28
30
|
put(bytes: Uint8Array): Promise<string>;
|
|
29
31
|
has(cid: string): Promise<boolean>;
|
|
30
32
|
get(cid: string, options?: GetOptions | undefined): Promise<Uint8Array | undefined>;
|
|
31
33
|
rm(cid: string): Promise<void>;
|
|
32
34
|
start(): Promise<void>;
|
|
35
|
+
private handleFetchRequest;
|
|
33
36
|
private _readFromPeers;
|
|
34
37
|
stop(): Promise<void>;
|
|
35
38
|
get status(): import("./store.js").StoreStatus;
|
package/lib/esm/libp2p.js
CHANGED
|
@@ -7,11 +7,12 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
9
|
};
|
|
10
|
-
import { stringifyCid, cidifyString, codecCodes, checkDecodeBlock
|
|
10
|
+
import { stringifyCid, cidifyString, codecCodes, checkDecodeBlock } from "./block.js";
|
|
11
11
|
import { variant, field, serialize, deserialize } from "@dao-xyz/borsh";
|
|
12
12
|
import { DirectStream } from "@peerbit/stream";
|
|
13
13
|
import { LevelBlockStore, MemoryLevelBlockStore } from "./level.js";
|
|
14
14
|
import { Level } from "level";
|
|
15
|
+
import PQueue from "p-queue";
|
|
15
16
|
export class BlockMessage {
|
|
16
17
|
}
|
|
17
18
|
export let BlockRequest = class BlockRequest extends BlockMessage {
|
|
@@ -54,16 +55,20 @@ export class DirectBlock extends DirectStream {
|
|
|
54
55
|
_localStore;
|
|
55
56
|
_responseHandler;
|
|
56
57
|
_resolvers;
|
|
58
|
+
_loadFetchQueue;
|
|
57
59
|
_readFromPeersPromises;
|
|
58
60
|
_open = false;
|
|
59
61
|
constructor(components, options) {
|
|
60
62
|
super(components, ["direct-block/1.0.0"], {
|
|
61
63
|
emitSelf: false,
|
|
62
64
|
signaturePolicy: "StrictNoSign",
|
|
63
|
-
messageProcessingConcurrency: 10,
|
|
64
|
-
canRelayMessage: options?.canRelayMessage ?? true
|
|
65
|
+
messageProcessingConcurrency: options?.messageProcessingConcurrency || 10,
|
|
66
|
+
canRelayMessage: options?.canRelayMessage ?? true
|
|
65
67
|
});
|
|
66
68
|
const localTimeout = options?.localTimeout || 1000;
|
|
69
|
+
this._loadFetchQueue = new PQueue({
|
|
70
|
+
concurrency: options?.messageProcessingConcurrency || 10
|
|
71
|
+
});
|
|
67
72
|
this._localStore =
|
|
68
73
|
options?.directory != null
|
|
69
74
|
? new LevelBlockStore(new Level(options.directory))
|
|
@@ -78,15 +83,7 @@ export class DirectBlock extends DirectStream {
|
|
|
78
83
|
try {
|
|
79
84
|
const decoded = deserialize(message.data, BlockMessage);
|
|
80
85
|
if (decoded instanceof BlockRequest && this._localStore) {
|
|
81
|
-
|
|
82
|
-
const bytes = await this._localStore.get(cid, {
|
|
83
|
-
timeout: localTimeout,
|
|
84
|
-
});
|
|
85
|
-
if (!bytes) {
|
|
86
|
-
return;
|
|
87
|
-
}
|
|
88
|
-
const response = serialize(new BlockResponse(cid, bytes));
|
|
89
|
-
await this.publish(response);
|
|
86
|
+
this._loadFetchQueue.add(() => this.handleFetchRequest(decoded, localTimeout));
|
|
90
87
|
}
|
|
91
88
|
else if (decoded instanceof BlockResponse) {
|
|
92
89
|
// TODO make sure we are not storing too much bytes in ram (like filter large blocks)
|
|
@@ -131,6 +128,17 @@ export class DirectBlock extends DirectStream {
|
|
|
131
128
|
this.addEventListener("data", this._responseHandler);
|
|
132
129
|
this._open = true;
|
|
133
130
|
}
|
|
131
|
+
async handleFetchRequest(request, localTimeout) {
|
|
132
|
+
const cid = stringifyCid(request.cid);
|
|
133
|
+
const bytes = await this._localStore.get(cid, {
|
|
134
|
+
timeout: localTimeout
|
|
135
|
+
});
|
|
136
|
+
if (!bytes) {
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const response = serialize(new BlockResponse(cid, bytes));
|
|
140
|
+
await this.publish(response);
|
|
141
|
+
}
|
|
134
142
|
async _readFromPeers(cidString, cidObject, options = {}) {
|
|
135
143
|
const codec = codecCodes[cidObject.code];
|
|
136
144
|
let promise = this._readFromPeersPromises.get(cidString);
|
|
@@ -142,7 +150,7 @@ export class DirectBlock extends DirectStream {
|
|
|
142
150
|
this._resolvers.set(cidString, async (bytes) => {
|
|
143
151
|
const value = await checkDecodeBlock(cidObject, bytes, {
|
|
144
152
|
codec,
|
|
145
|
-
hasher: options?.hasher
|
|
153
|
+
hasher: options?.hasher
|
|
146
154
|
});
|
|
147
155
|
clearTimeout(timeoutCallback);
|
|
148
156
|
this._resolvers.delete(cidString); // TODO concurrency might not work as expected here
|
|
@@ -169,8 +177,12 @@ export class DirectBlock extends DirectStream {
|
|
|
169
177
|
}
|
|
170
178
|
}
|
|
171
179
|
async stop() {
|
|
172
|
-
|
|
180
|
+
// Dont listen for more incoming messages
|
|
173
181
|
this.removeEventListener("data", this._responseHandler);
|
|
182
|
+
// Wait for processing request
|
|
183
|
+
this._loadFetchQueue.clear();
|
|
184
|
+
await this._loadFetchQueue.onIdle(); // wait for pending
|
|
185
|
+
await super.stop();
|
|
174
186
|
await this._localStore?.stop();
|
|
175
187
|
this._readFromPeersPromises.clear();
|
|
176
188
|
this._resolvers.clear();
|
package/lib/esm/libp2p.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"libp2p.js","sourceRoot":"","sources":["../../src/libp2p.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EACN,YAAY,EACZ,YAAY,EACZ,UAAU,EACV,gBAAgB,
|
|
1
|
+
{"version":3,"file":"libp2p.js","sourceRoot":"","sources":["../../src/libp2p.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EACN,YAAY,EACZ,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAGxE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAK/C,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACpE,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,MAAM,OAAO,YAAY;CAAG;AAGrB,WAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,YAAY;IAE7C,GAAG,CAAS;IAEZ,YAAY,GAAW;QACtB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IAChB,CAAC;CACD,CAAA;AANA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;;yCACd;AAFA,YAAY;IADxB,OAAO,CAAC,CAAC,CAAC;;GACE,YAAY,CAQxB;AAGM,WAAM,aAAa,GAAnB,MAAM,aAAc,SAAQ,YAAY;IAE9C,GAAG,CAAS;IAGZ,KAAK,CAAa;IAElB,YAAY,GAAW,EAAE,KAAiB;QACzC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACpB,CAAC;CACD,CAAA;AAVA;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;;0CACd;AAGZ;IADC,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;8BACrB,UAAU;4CAAC;AALN,aAAa;IADzB,OAAO,CAAC,CAAC,CAAC;6CAQsB,UAAU;GAP9B,aAAa,CAYzB;AAGD,MAAM,OAAO,WAAY,SAAQ,YAAY;IACpC,WAAW,CAAa;IACxB,gBAAgB,CAA0C;IAC1D,UAAU,CAA0C;IACpD,eAAe,CAAS;IACxB,sBAAsB,CAG5B;IACF,KAAK,GAAG,KAAK,CAAC;IAEd,YACC,UAAiC,EACjC,OAKC;QAED,KAAK,CAAC,UAAU,EAAE,CAAC,oBAAoB,CAAC,EAAE;YACzC,QAAQ,EAAE,KAAK;YACf,eAAe,EAAE,cAAc;YAC/B,4BAA4B,EAAE,OAAO,EAAE,4BAA4B,IAAI,EAAE;YACzE,eAAe,EAAE,OAAO,EAAE,eAAe,IAAI,IAAI;SACjD,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,eAAe,GAAG,IAAI,MAAM,CAAC;YACjC,WAAW,EAAE,OAAO,EAAE,4BAA4B,IAAI,EAAE;SACxD,CAAC,CAAC;QACH,IAAI,CAAC,WAAW;YACf,OAAO,EAAE,SAAS,IAAI,IAAI;gBACzB,CAAC,CAAC,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBACnD,CAAC,CAAC,IAAI,qBAAqB,EAAE,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,sBAAsB,GAAG,IAAI,GAAG,EAAE,CAAC;QACxC,IAAI,CAAC,gBAAgB,GAAG,KAAK,EAAE,GAA6B,EAAE,EAAE;YAC/D,IAAI,CAAC,GAAG,EAAE;gBACT,OAAO;aACP;YACD,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC;YAC3B,IAAI;gBACH,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;gBACxD,IAAI,OAAO,YAAY,YAAY,IAAI,IAAI,CAAC,WAAW,EAAE;oBACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAC7B,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,YAAY,CAAC,CAC9C,CAAC;iBACF;qBAAM,IAAI,OAAO,YAAY,aAAa,EAAE;oBAC5C,qFAAqF;oBAErF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;iBAClD;aACD;YAAC,OAAO,KAAK,EAAE;gBACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;gBAC/D,OAAO,CAAC,0BAA0B;aAClC;QACF,CAAC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAiB;QAC1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;SACvC;QACD,OAAO,IAAI,CAAC,WAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IACD,KAAK,CAAC,GAAG,CACR,GAAW,EACX,OAAgC;QAEhC,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,KAAK,GAAG,IAAI,CAAC,WAAW;YAC3B,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC;YAC1C,CAAC,CAAC,SAAS,CAAC;QAEb,IAAI,CAAC,KAAK,EAAE;YACX,0BAA0B;YAC1B,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAC3D,IAAI,OAAO,EAAE,SAAS,IAAI,KAAK,EAAE;gBAChC,IAAI,CAAC,WAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aAC7B;SACD;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,KAAK,CAAC,EAAE,CAAC,GAAW;QACnB,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,MAAM,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC;QAChC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,gBAAiB,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC/B,OAAqB,EACrB,YAAoB;QAEpB,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE;YAC7C,OAAO,EAAE,YAAY;SACrB,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,EAAE;YACX,OAAO;SACP;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,cAAc,CAC3B,SAAiB,EACjB,SAAc,EACd,UAA8C,EAAE;QAEhD,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO,EAAE;YACb,OAAO,GAAG,IAAI,OAAO,CACpB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnB,MAAM,eAAe,GAAG,UAAU,CACjC,GAAG,EAAE;oBACJ,OAAO,CAAC,SAAS,CAAC,CAAC;gBACpB,CAAC,EACD,OAAO,CAAC,OAAO,IAAI,EAAE,GAAG,IAAI,CAC5B,CAAC;gBAEF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,KAAiB,EAAE,EAAE;oBAC1D,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE;wBACtD,KAAK;wBACL,MAAM,EAAE,OAAO,EAAE,MAAM;qBACvB,CAAC,CAAC;oBAEH,YAAY,CAAC,eAAe,CAAC,CAAC;oBAC9B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,mDAAmD;oBACtF,OAAO,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC,CAAC,CAAC;YACJ,CAAC,CACD,CAAC;YACF,MAAM,OAAO,GAAG,CAAC,EAAoB,EAAE,EAAE,CACxC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAClE,MAAM,OAAO,EAAE,CAAC;YAEhB,MAAM,iBAAiB,GAAG,CAAC,CAA6B,EAAE,EAAE;gBAC3D,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5B,CAAC,CAAC;YAEF,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAEpD,4EAA4E;YAC5E,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;YAE3D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;YAC7B,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAE9C,4DAA4D;YAC5D,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;YAC9D,OAAO,MAAM,EAAE,KAAK,CAAC;SACrB;aAAM;YACN,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;YAC7B,OAAO,MAAM,EAAE,KAAK,CAAC;SACrB;IACF,CAAC;IAED,KAAK,CAAC,IAAI;QACT,yCAAyC;QACzC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAExD,8BAA8B;QAC9B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,mBAAmB;QACxD,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QAEnB,wFAAwF;IACzF,CAAC;IAED,IAAI,MAAM;QACT,IAAI,IAAI,CAAC,KAAK,EAAE;YACf,OAAO,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC;SAClD;aAAM;YACN,OAAO,QAAQ,CAAC;SAChB;IACF,CAAC;CACD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@peerbit/blocks",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.12",
|
|
4
4
|
"description": "Block store streaming",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -49,18 +49,18 @@
|
|
|
49
49
|
"dao.xyz"
|
|
50
50
|
],
|
|
51
51
|
"devDependencies": {
|
|
52
|
-
"@peerbit/libp2p-test-utils": "1.0.
|
|
52
|
+
"@peerbit/libp2p-test-utils": "1.0.4"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
55
|
"@dao-xyz/borsh": "^5.1.5",
|
|
56
56
|
"@ipld/dag-cbor": "^9.0.2",
|
|
57
|
-
"@peerbit/blocks-interface": "1.0.
|
|
58
|
-
"@peerbit/crypto": "1.0.
|
|
59
|
-
"@peerbit/lazy-level": "1.
|
|
60
|
-
"@peerbit/stream": "1.0.
|
|
57
|
+
"@peerbit/blocks-interface": "1.0.8",
|
|
58
|
+
"@peerbit/crypto": "1.0.6",
|
|
59
|
+
"@peerbit/lazy-level": "1.1.1",
|
|
60
|
+
"@peerbit/stream": "1.0.10",
|
|
61
61
|
"abstract-level": "^1.0.3",
|
|
62
62
|
"libp2p": "^0.46.6",
|
|
63
63
|
"memory-level": "^1.0.0"
|
|
64
64
|
},
|
|
65
|
-
"gitHead": "
|
|
65
|
+
"gitHead": "6442481a66eec1bc13d06b3663f93643c247d141"
|
|
66
66
|
}
|
package/src/block.ts
CHANGED
|
@@ -14,11 +14,11 @@ export const defaultHasher = sha256;
|
|
|
14
14
|
|
|
15
15
|
export const codecCodes = {
|
|
16
16
|
[dagCbor.code]: dagCbor,
|
|
17
|
-
[raw.code]: raw
|
|
17
|
+
[raw.code]: raw
|
|
18
18
|
};
|
|
19
19
|
export const codecMap = {
|
|
20
20
|
raw: raw,
|
|
21
|
-
"dag-cbor": dagCbor
|
|
21
|
+
"dag-cbor": dagCbor
|
|
22
22
|
};
|
|
23
23
|
|
|
24
24
|
export const cidifyString = (str: string): CID => {
|
|
@@ -51,7 +51,7 @@ export const checkDecodeBlock = async (
|
|
|
51
51
|
const block = await Block.decode({
|
|
52
52
|
bytes: bytes,
|
|
53
53
|
codec,
|
|
54
|
-
hasher: options?.hasher || defaultHasher
|
|
54
|
+
hasher: options?.hasher || defaultHasher
|
|
55
55
|
});
|
|
56
56
|
if (!block.cid.equals(cidObject)) {
|
|
57
57
|
throw new Error("CID does not match");
|
|
@@ -108,7 +108,7 @@ export const createBlock = async (
|
|
|
108
108
|
const block = await Block.encode({
|
|
109
109
|
value,
|
|
110
110
|
codec,
|
|
111
|
-
hasher: options?.hasher || defaultHasher
|
|
111
|
+
hasher: options?.hasher || defaultHasher
|
|
112
112
|
});
|
|
113
113
|
return block as Block.Block<any, any, any, any>;
|
|
114
114
|
};
|
package/src/index.ts
CHANGED
package/src/level.ts
CHANGED
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
codecCodes,
|
|
5
5
|
createBlock,
|
|
6
6
|
defaultHasher,
|
|
7
|
-
stringifyCid
|
|
7
|
+
stringifyCid
|
|
8
8
|
} from "./block.js";
|
|
9
9
|
import * as Block from "multiformats/block";
|
|
10
10
|
import { AbstractLevel } from "abstract-level";
|
|
@@ -46,7 +46,7 @@ export class LevelBlockStore implements Blocks {
|
|
|
46
46
|
const block = await Block.decode({
|
|
47
47
|
bytes,
|
|
48
48
|
codec,
|
|
49
|
-
hasher: options?.hasher || defaultHasher
|
|
49
|
+
hasher: options?.hasher || defaultHasher
|
|
50
50
|
});
|
|
51
51
|
return (block as Block.Block<Uint8Array, any, any, any>).bytes;
|
|
52
52
|
} catch (error: any) {
|
|
@@ -86,7 +86,7 @@ export class LevelBlockStore implements Blocks {
|
|
|
86
86
|
timeout: 10 * 1000,
|
|
87
87
|
stopperCallback: (fn) => {
|
|
88
88
|
this._onClose = fn;
|
|
89
|
-
}
|
|
89
|
+
}
|
|
90
90
|
});
|
|
91
91
|
await this._opening;
|
|
92
92
|
} finally {
|
package/src/libp2p.ts
CHANGED
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
stringifyCid,
|
|
5
5
|
cidifyString,
|
|
6
6
|
codecCodes,
|
|
7
|
-
checkDecodeBlock
|
|
7
|
+
checkDecodeBlock
|
|
8
8
|
} from "./block.js";
|
|
9
9
|
import { variant, field, serialize, deserialize } from "@dao-xyz/borsh";
|
|
10
10
|
import { CID } from "multiformats/cid";
|
|
@@ -17,6 +17,7 @@ import { DirectStreamComponents } from "@peerbit/stream";
|
|
|
17
17
|
import { LevelBlockStore, MemoryLevelBlockStore } from "./level.js";
|
|
18
18
|
import { Level } from "level";
|
|
19
19
|
import { GetOptions, PutOptions } from "@peerbit/blocks-interface";
|
|
20
|
+
import PQueue from "p-queue";
|
|
20
21
|
|
|
21
22
|
export class BlockMessage {}
|
|
22
23
|
|
|
@@ -51,6 +52,7 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
51
52
|
private _localStore: BlockStore;
|
|
52
53
|
private _responseHandler?: (evt: CustomEvent<DataMessage>) => any;
|
|
53
54
|
private _resolvers: Map<string, (data: Uint8Array) => void>;
|
|
55
|
+
private _loadFetchQueue: PQueue;
|
|
54
56
|
private _readFromPeersPromises: Map<
|
|
55
57
|
string,
|
|
56
58
|
Promise<Block.Block<any, any, any, 1> | undefined> | undefined
|
|
@@ -63,16 +65,20 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
63
65
|
directory?: string;
|
|
64
66
|
canRelayMessage?: boolean;
|
|
65
67
|
localTimeout?: number;
|
|
68
|
+
messageProcessingConcurrency?: number;
|
|
66
69
|
}
|
|
67
70
|
) {
|
|
68
71
|
super(components, ["direct-block/1.0.0"], {
|
|
69
72
|
emitSelf: false,
|
|
70
73
|
signaturePolicy: "StrictNoSign",
|
|
71
|
-
messageProcessingConcurrency: 10,
|
|
72
|
-
canRelayMessage: options?.canRelayMessage ?? true
|
|
74
|
+
messageProcessingConcurrency: options?.messageProcessingConcurrency || 10,
|
|
75
|
+
canRelayMessage: options?.canRelayMessage ?? true
|
|
73
76
|
});
|
|
74
77
|
|
|
75
78
|
const localTimeout = options?.localTimeout || 1000;
|
|
79
|
+
this._loadFetchQueue = new PQueue({
|
|
80
|
+
concurrency: options?.messageProcessingConcurrency || 10
|
|
81
|
+
});
|
|
76
82
|
this._localStore =
|
|
77
83
|
options?.directory != null
|
|
78
84
|
? new LevelBlockStore(new Level(options.directory))
|
|
@@ -87,15 +93,9 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
87
93
|
try {
|
|
88
94
|
const decoded = deserialize(message.data, BlockMessage);
|
|
89
95
|
if (decoded instanceof BlockRequest && this._localStore) {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
});
|
|
94
|
-
if (!bytes) {
|
|
95
|
-
return;
|
|
96
|
-
}
|
|
97
|
-
const response = serialize(new BlockResponse(cid, bytes));
|
|
98
|
-
await this.publish(response);
|
|
96
|
+
this._loadFetchQueue.add(() =>
|
|
97
|
+
this.handleFetchRequest(decoded, localTimeout)
|
|
98
|
+
);
|
|
99
99
|
} else if (decoded instanceof BlockResponse) {
|
|
100
100
|
// TODO make sure we are not storing too much bytes in ram (like filter large blocks)
|
|
101
101
|
|
|
@@ -148,6 +148,21 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
148
148
|
this._open = true;
|
|
149
149
|
}
|
|
150
150
|
|
|
151
|
+
private async handleFetchRequest(
|
|
152
|
+
request: BlockRequest,
|
|
153
|
+
localTimeout: number
|
|
154
|
+
) {
|
|
155
|
+
const cid = stringifyCid(request.cid);
|
|
156
|
+
const bytes = await this._localStore.get(cid, {
|
|
157
|
+
timeout: localTimeout
|
|
158
|
+
});
|
|
159
|
+
if (!bytes) {
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const response = serialize(new BlockResponse(cid, bytes));
|
|
163
|
+
await this.publish(response);
|
|
164
|
+
}
|
|
165
|
+
|
|
151
166
|
private async _readFromPeers(
|
|
152
167
|
cidString: string,
|
|
153
168
|
cidObject: CID,
|
|
@@ -158,14 +173,17 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
158
173
|
if (!promise) {
|
|
159
174
|
promise = new Promise<Block.Block<any, any, any, 1> | undefined>(
|
|
160
175
|
(resolve, reject) => {
|
|
161
|
-
const timeoutCallback = setTimeout(
|
|
162
|
-
|
|
163
|
-
|
|
176
|
+
const timeoutCallback = setTimeout(
|
|
177
|
+
() => {
|
|
178
|
+
resolve(undefined);
|
|
179
|
+
},
|
|
180
|
+
options.timeout || 30 * 1000
|
|
181
|
+
);
|
|
164
182
|
|
|
165
183
|
this._resolvers.set(cidString, async (bytes: Uint8Array) => {
|
|
166
184
|
const value = await checkDecodeBlock(cidObject, bytes, {
|
|
167
185
|
codec,
|
|
168
|
-
hasher: options?.hasher
|
|
186
|
+
hasher: options?.hasher
|
|
169
187
|
});
|
|
170
188
|
|
|
171
189
|
clearTimeout(timeoutCallback);
|
|
@@ -200,8 +218,13 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
200
218
|
}
|
|
201
219
|
|
|
202
220
|
async stop(): Promise<void> {
|
|
203
|
-
|
|
221
|
+
// Dont listen for more incoming messages
|
|
204
222
|
this.removeEventListener("data", this._responseHandler);
|
|
223
|
+
|
|
224
|
+
// Wait for processing request
|
|
225
|
+
this._loadFetchQueue.clear();
|
|
226
|
+
await this._loadFetchQueue.onIdle(); // wait for pending
|
|
227
|
+
await super.stop();
|
|
205
228
|
await this._localStore?.stop();
|
|
206
229
|
this._readFromPeersPromises.clear();
|
|
207
230
|
this._resolvers.clear();
|