@peerbit/blocks 4.1.9 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/any-store.d.ts +2 -0
- package/dist/benchmark/any-store.d.ts.map +1 -0
- package/dist/benchmark/any-store.js +112 -0
- package/dist/benchmark/any-store.js.map +1 -0
- package/dist/src/any-blockstore.d.ts +23 -9
- package/dist/src/any-blockstore.d.ts.map +1 -1
- package/dist/src/any-blockstore.js +161 -18
- package/dist/src/any-blockstore.js.map +1 -1
- package/dist/src/interface.d.ts +19 -1
- package/dist/src/interface.d.ts.map +1 -1
- package/dist/src/libp2p.d.ts +28 -2
- package/dist/src/libp2p.d.ts.map +1 -1
- package/dist/src/libp2p.js +90 -9
- package/dist/src/libp2p.js.map +1 -1
- package/dist/src/remote.d.ts +48 -4
- package/dist/src/remote.d.ts.map +1 -1
- package/dist/src/remote.js +197 -19
- package/dist/src/remote.js.map +1 -1
- package/package.json +11 -9
- package/src/any-blockstore.ts +244 -28
- package/src/interface.ts +23 -1
- package/src/libp2p.ts +133 -14
- package/src/remote.ts +261 -21
package/src/libp2p.ts
CHANGED
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
import { deserialize, serialize } from "@dao-xyz/borsh";
|
|
2
2
|
import { createStore } from "@peerbit/any-store";
|
|
3
|
+
import type { AnyStore } from "@peerbit/any-store-interface";
|
|
3
4
|
import type { GetOptions, Blocks as IBlocks } from "@peerbit/blocks-interface";
|
|
4
5
|
import { getPublicKeyFromPeerId, type PublicSignKey } from "@peerbit/crypto";
|
|
5
6
|
import { DirectStream } from "@peerbit/stream";
|
|
6
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
type DirectStreamComponents,
|
|
9
|
+
type RustCoreStream,
|
|
10
|
+
} from "@peerbit/stream";
|
|
7
11
|
import {
|
|
8
12
|
createRequestTransportContext,
|
|
9
13
|
type DataMessage,
|
|
10
14
|
type WaitForPeersFn,
|
|
11
15
|
} from "@peerbit/stream-interface";
|
|
16
|
+
import type { Block } from "multiformats/block";
|
|
12
17
|
import { AnyBlockStore } from "./any-blockstore.js";
|
|
13
|
-
import { BlockMessage, RemoteBlocks } from "./remote.js";
|
|
18
|
+
import { BlockMessage, BlockRequest, BlockResponse, RemoteBlocks } from "./remote.js";
|
|
14
19
|
|
|
15
20
|
export type DirectBlockComponents = DirectStreamComponents;
|
|
16
21
|
|
|
@@ -23,6 +28,7 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
23
28
|
components: DirectBlockComponents,
|
|
24
29
|
options?: {
|
|
25
30
|
directory?: string;
|
|
31
|
+
localStore?: AnyStore;
|
|
26
32
|
canRelayMessage?: boolean;
|
|
27
33
|
localTimeout?: number;
|
|
28
34
|
messageProcessingConcurrency?: number;
|
|
@@ -47,8 +53,20 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
47
53
|
maxProvidersPerCid?: number;
|
|
48
54
|
};
|
|
49
55
|
requeryOnReachable?: number;
|
|
56
|
+
/**
|
|
57
|
+
* Run the block-exchange protocol on the native DirectStream core
|
|
58
|
+
* (`@peerbit/network-rust`): codec, provider resolution and caches
|
|
59
|
+
* execute in wasm, and natively stored blocks are served without
|
|
60
|
+
* surfacing their bytes to JS. Defaults to the same rust-core mode
|
|
61
|
+
* as the underlying DirectStream.
|
|
62
|
+
*/
|
|
63
|
+
rustCore?: RustCoreStream | false;
|
|
50
64
|
},
|
|
51
65
|
) {
|
|
66
|
+
if (options?.directory && options.localStore) {
|
|
67
|
+
throw new Error("DirectBlock options cannot include both directory and localStore");
|
|
68
|
+
}
|
|
69
|
+
|
|
52
70
|
super(components, ["/peerbit/direct-block/1.0.0"], {
|
|
53
71
|
messageProcessingConcurrency: options?.messageProcessingConcurrency || 10,
|
|
54
72
|
canRelayMessage: options?.canRelayMessage ?? true,
|
|
@@ -56,9 +74,29 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
56
74
|
dialer: false,
|
|
57
75
|
pruner: false,
|
|
58
76
|
},
|
|
77
|
+
rustCore: options?.rustCore,
|
|
59
78
|
});
|
|
60
79
|
|
|
80
|
+
const blockExchange = this.rustCore?.blockExchange;
|
|
61
81
|
const defaultResolveProviders = () => {
|
|
82
|
+
if (blockExchange) {
|
|
83
|
+
const negotiated = [...this.peers.keys()];
|
|
84
|
+
const connected: string[] = [];
|
|
85
|
+
for (const conn of this.components.connectionManager.getConnections()) {
|
|
86
|
+
try {
|
|
87
|
+
connected.push(
|
|
88
|
+
getPublicKeyFromPeerId(conn.remotePeer).hashcode(),
|
|
89
|
+
);
|
|
90
|
+
} catch {
|
|
91
|
+
// ignore unexpected key types
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return blockExchange.defaultProviderCandidates(
|
|
95
|
+
negotiated,
|
|
96
|
+
connected,
|
|
97
|
+
this.publicKeyHash,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
62
100
|
const out: string[] = [];
|
|
63
101
|
const push = (hash?: string) => {
|
|
64
102
|
if (!hash) return;
|
|
@@ -89,8 +127,11 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
89
127
|
return out;
|
|
90
128
|
};
|
|
91
129
|
this.remoteBlocks = new RemoteBlocks({
|
|
92
|
-
local: new AnyBlockStore(
|
|
93
|
-
|
|
130
|
+
local: new AnyBlockStore(
|
|
131
|
+
options?.localStore ?? createStore(options?.directory),
|
|
132
|
+
),
|
|
133
|
+
publish: (message, options) =>
|
|
134
|
+
this.publish(this.encodeBlockMessage(message), options),
|
|
94
135
|
localTimeout: options?.localTimeout || 1000,
|
|
95
136
|
messageProcessingConcurrency: options?.messageProcessingConcurrency || 10,
|
|
96
137
|
waitFor: this.waitFor.bind(this) as WaitForPeersFn,
|
|
@@ -101,30 +142,97 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
101
142
|
onPut: options?.onPut,
|
|
102
143
|
providerCache: options?.providerCache,
|
|
103
144
|
requeryOnReachable: options?.requeryOnReachable,
|
|
145
|
+
rust: blockExchange
|
|
146
|
+
? {
|
|
147
|
+
exchange: blockExchange,
|
|
148
|
+
publishRaw: (payload, options) => this.publish(payload, options),
|
|
149
|
+
}
|
|
150
|
+
: undefined,
|
|
104
151
|
});
|
|
105
152
|
|
|
106
153
|
this.onDataFn = (data: CustomEvent<DataMessage>) => {
|
|
107
|
-
data.detail?.data?.length &&
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
)
|
|
154
|
+
if (!(data.detail?.data?.length && data.detail.data.length > 0)) {
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
let message: BlockMessage;
|
|
158
|
+
try {
|
|
159
|
+
message = this.decodeBlockMessage(data.detail.data!);
|
|
160
|
+
} catch {
|
|
161
|
+
// Drop undecodable frames instead of letting the throw escape
|
|
162
|
+
// the 'data' listener (an uncaughtException in Node). The TS
|
|
163
|
+
// borsh decoder tolerates malformed frames lossily and fails
|
|
164
|
+
// later inside handleFetchRequest, which is caught; the native
|
|
165
|
+
// decoder throws on e.g. non-UTF-8 cid bytes, so match the
|
|
166
|
+
// tolerant behaviour here.
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
this.remoteBlocks.onMessage(message, {
|
|
170
|
+
from: data.detail.header.signatures?.publicKeys[0]?.hashcode(),
|
|
171
|
+
transport: createRequestTransportContext(data.detail),
|
|
172
|
+
});
|
|
116
173
|
};
|
|
117
174
|
this.onPeerConnectedFn = (evt: CustomEvent<PublicSignKey>) =>
|
|
118
175
|
this.remoteBlocks.onReachable(evt.detail);
|
|
119
176
|
}
|
|
120
177
|
|
|
121
|
-
|
|
178
|
+
private encodeBlockMessage(message: BlockRequest | BlockResponse): Uint8Array {
|
|
179
|
+
const blockExchange = this.rustCore?.blockExchange;
|
|
180
|
+
if (blockExchange) {
|
|
181
|
+
if (message instanceof BlockRequest) {
|
|
182
|
+
return blockExchange.encodeBlockRequest(message.cid);
|
|
183
|
+
}
|
|
184
|
+
if (message instanceof BlockResponse) {
|
|
185
|
+
return blockExchange.encodeBlockResponse(message.cid, message.bytes);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return serialize(message);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private decodeBlockMessage(bytes: Uint8Array): BlockMessage {
|
|
192
|
+
const blockExchange = this.rustCore?.blockExchange;
|
|
193
|
+
if (blockExchange) {
|
|
194
|
+
const decoded = blockExchange.decodeBlockMessage(bytes);
|
|
195
|
+
return decoded.type === "request"
|
|
196
|
+
? new BlockRequest(decoded.cid)
|
|
197
|
+
: new BlockResponse(decoded.cid, decoded.bytes);
|
|
198
|
+
}
|
|
199
|
+
return deserialize(bytes, BlockMessage);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
getNativeLogBlockStoreHandle(): unknown {
|
|
203
|
+
return this.remoteBlocks.getNativeLogBlockStoreHandle();
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async put(
|
|
207
|
+
bytes: Uint8Array | { block: Block<any, any, any, any>; cid: string },
|
|
208
|
+
): Promise<string> {
|
|
122
209
|
return this.remoteBlocks.put(bytes);
|
|
123
210
|
}
|
|
124
211
|
|
|
212
|
+
async putMany(
|
|
213
|
+
blocks: Array<
|
|
214
|
+
Uint8Array | { block: Block<any, any, any, any>; cid: string }
|
|
215
|
+
>,
|
|
216
|
+
): Promise<string[]> {
|
|
217
|
+
return this.remoteBlocks.putMany(blocks);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async putKnown(cid: string, bytes: Uint8Array): Promise<string> {
|
|
221
|
+
return this.remoteBlocks.putKnown(cid, bytes);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async putKnownMany(
|
|
225
|
+
blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
|
|
226
|
+
): Promise<string[]> {
|
|
227
|
+
return this.remoteBlocks.putKnownMany(blocks);
|
|
228
|
+
}
|
|
229
|
+
|
|
125
230
|
async has(cid: string) {
|
|
126
231
|
return this.remoteBlocks.has(cid);
|
|
127
232
|
}
|
|
233
|
+
async hasMany(cids: string[]): Promise<boolean[]> {
|
|
234
|
+
return this.remoteBlocks.hasMany(cids);
|
|
235
|
+
}
|
|
128
236
|
async get(
|
|
129
237
|
cid: string,
|
|
130
238
|
options?: GetOptions | undefined,
|
|
@@ -132,6 +240,13 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
132
240
|
return this.remoteBlocks.get(cid, options);
|
|
133
241
|
}
|
|
134
242
|
|
|
243
|
+
async getMany(
|
|
244
|
+
cids: string[],
|
|
245
|
+
options?: GetOptions | undefined,
|
|
246
|
+
): Promise<Array<Uint8Array | undefined>> {
|
|
247
|
+
return this.remoteBlocks.getMany(cids, options);
|
|
248
|
+
}
|
|
249
|
+
|
|
135
250
|
hintProviders(cid: string, providers: string[]) {
|
|
136
251
|
this.remoteBlocks.hintProviders(cid, providers);
|
|
137
252
|
}
|
|
@@ -140,6 +255,10 @@ export class DirectBlock extends DirectStream implements IBlocks {
|
|
|
140
255
|
return this.remoteBlocks.rm(cid);
|
|
141
256
|
}
|
|
142
257
|
|
|
258
|
+
async rmMany(cids: string[]) {
|
|
259
|
+
return this.remoteBlocks.rmMany(cids);
|
|
260
|
+
}
|
|
261
|
+
|
|
143
262
|
async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
|
|
144
263
|
for await (const [key, value] of this.remoteBlocks.iterator()) {
|
|
145
264
|
yield [key, value];
|
package/src/remote.ts
CHANGED
|
@@ -11,7 +11,12 @@ import {
|
|
|
11
11
|
import { Cache } from "@peerbit/cache";
|
|
12
12
|
import { PublicSignKey } from "@peerbit/crypto";
|
|
13
13
|
import { logger as loggerFn } from "@peerbit/logger";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
type PublishOptions,
|
|
16
|
+
type RustBlockExchange,
|
|
17
|
+
type RustBlockProviderCache,
|
|
18
|
+
dontThrowIfDeliveryError,
|
|
19
|
+
} from "@peerbit/stream";
|
|
15
20
|
import {
|
|
16
21
|
type RequestTransportContext,
|
|
17
22
|
type PeerRefs,
|
|
@@ -24,7 +29,6 @@ import { AbortError } from "@peerbit/time";
|
|
|
24
29
|
import { CID } from "multiformats";
|
|
25
30
|
import { type Block } from "multiformats/block";
|
|
26
31
|
import PQueue from "p-queue";
|
|
27
|
-
import { AnyBlockStore } from "./any-blockstore.js";
|
|
28
32
|
import type { BlockStore } from "./interface.js";
|
|
29
33
|
|
|
30
34
|
export const logger = loggerFn("peerbit:transport:blocks");
|
|
@@ -72,6 +76,17 @@ type RemoteReadOptions = Exclude<GetOptions["remote"], boolean | undefined> & {
|
|
|
72
76
|
hasher?: any;
|
|
73
77
|
};
|
|
74
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Shared shape of the TS eager-block cache (`Cache<Uint8Array>`) and the
|
|
81
|
+
* native-backed one (`RustEagerBlockCache`).
|
|
82
|
+
*/
|
|
83
|
+
type EagerBlockCache = {
|
|
84
|
+
add(cid: string, bytes: Uint8Array): void;
|
|
85
|
+
get(cid: string): Uint8Array | null | undefined;
|
|
86
|
+
del(cid: string): unknown;
|
|
87
|
+
clear(): void;
|
|
88
|
+
};
|
|
89
|
+
|
|
75
90
|
export class RemoteBlocks implements IBlocks {
|
|
76
91
|
localStore: BlockStore;
|
|
77
92
|
|
|
@@ -80,14 +95,18 @@ export class RemoteBlocks implements IBlocks {
|
|
|
80
95
|
context?: BlockMessageContext,
|
|
81
96
|
) => any;
|
|
82
97
|
private _resolvers: Map<string, (data: Uint8Array) => Promise<void>>;
|
|
83
|
-
private _blockCache?:
|
|
98
|
+
private _blockCache?: EagerBlockCache;
|
|
84
99
|
private _providerCache?: Cache<string[]>;
|
|
100
|
+
private _rustProviderCache?: RustBlockProviderCache;
|
|
101
|
+
private readonly rustExchange?: RustBlockExchange;
|
|
85
102
|
private readonly publicKeyHash: string;
|
|
86
103
|
private readonly maxProviderHintsPerCid: number;
|
|
87
104
|
private readonly maxRequeryOnReachable: number;
|
|
88
105
|
|
|
89
106
|
private _loadFetchQueue: PQueue;
|
|
90
107
|
private _readFromPeersPromises: Map<string, InFlightRead>;
|
|
108
|
+
private _deferredStoredNotificationCids?: Set<string>;
|
|
109
|
+
private _deferredStoredNotificationTimer?: ReturnType<typeof setTimeout>;
|
|
91
110
|
_open = false;
|
|
92
111
|
private _events: TypedEventEmitter<{
|
|
93
112
|
"peer:reachable": CustomEvent<PublicSignKey>;
|
|
@@ -97,7 +116,7 @@ export class RemoteBlocks implements IBlocks {
|
|
|
97
116
|
|
|
98
117
|
constructor(
|
|
99
118
|
readonly options: {
|
|
100
|
-
local:
|
|
119
|
+
local: BlockStore;
|
|
101
120
|
localTimeout?: number;
|
|
102
121
|
messageProcessingConcurrency?: number;
|
|
103
122
|
publicKey: PublicSignKey;
|
|
@@ -158,25 +177,40 @@ export class RemoteBlocks implements IBlocks {
|
|
|
158
177
|
options: PublishOptions,
|
|
159
178
|
) => Promise<Uint8Array | undefined | void>;
|
|
160
179
|
waitFor: WaitForPeersFn;
|
|
180
|
+
/**
|
|
181
|
+
* Native block-exchange components (rust-core mode). When set, the
|
|
182
|
+
* provider caches/decisions and eager-block bookkeeping run in the
|
|
183
|
+
* native core; `publishRaw` additionally enables serving natively
|
|
184
|
+
* stored blocks as wasm-serialized payloads that never surface the
|
|
185
|
+
* block bytes to JS.
|
|
186
|
+
*/
|
|
187
|
+
rust?: {
|
|
188
|
+
exchange: RustBlockExchange;
|
|
189
|
+
publishRaw?: (
|
|
190
|
+
payload: Uint8Array,
|
|
191
|
+
options: PublishOptions,
|
|
192
|
+
) => Promise<Uint8Array | undefined | void>;
|
|
193
|
+
};
|
|
161
194
|
},
|
|
162
195
|
) {
|
|
163
196
|
const localTimeout = options?.localTimeout || 1000;
|
|
164
197
|
this.publicKeyHash = options.publicKey.hashcode();
|
|
198
|
+
this.rustExchange = options.rust?.exchange;
|
|
165
199
|
this._loadFetchQueue = new PQueue({
|
|
166
200
|
concurrency: options?.messageProcessingConcurrency || 10,
|
|
167
201
|
});
|
|
168
202
|
this.localStore = options?.local;
|
|
169
203
|
this._resolvers = new Map();
|
|
170
204
|
this._readFromPeersPromises = new Map();
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
})
|
|
179
|
-
|
|
205
|
+
if (options?.eagerBlocks) {
|
|
206
|
+
const eagerBlocksMax =
|
|
207
|
+
typeof options.eagerBlocks === "boolean"
|
|
208
|
+
? 1e3
|
|
209
|
+
: (options.eagerBlocks.cacheSize ?? 1e3);
|
|
210
|
+
this._blockCache = this.rustExchange
|
|
211
|
+
? this.rustExchange.createEagerCache({ max: eagerBlocksMax, ttl: 1e4 })
|
|
212
|
+
: new Cache<Uint8Array>({ max: eagerBlocksMax, ttl: 1e4 });
|
|
213
|
+
}
|
|
180
214
|
type ProviderCacheOptions = {
|
|
181
215
|
maxEntries?: number;
|
|
182
216
|
ttlMs?: number;
|
|
@@ -188,12 +222,21 @@ export class RemoteBlocks implements IBlocks {
|
|
|
188
222
|
: typeof options.providerCache === "object"
|
|
189
223
|
? options.providerCache
|
|
190
224
|
: {};
|
|
191
|
-
|
|
192
|
-
|
|
225
|
+
if (providerCache) {
|
|
226
|
+
if (this.rustExchange) {
|
|
227
|
+
this._rustProviderCache = this.rustExchange.createProviderCache({
|
|
228
|
+
me: this.publicKeyHash,
|
|
229
|
+
maxEntries: providerCache.maxEntries ?? 2048,
|
|
230
|
+
ttlMs: providerCache.ttlMs ?? 10 * 60 * 1000,
|
|
231
|
+
maxProvidersPerCid: providerCache.maxProvidersPerCid ?? 8,
|
|
232
|
+
});
|
|
233
|
+
} else {
|
|
234
|
+
this._providerCache = new Cache<string[]>({
|
|
193
235
|
max: providerCache.maxEntries ?? 2048,
|
|
194
236
|
ttl: providerCache.ttlMs ?? 10 * 60 * 1000,
|
|
195
|
-
})
|
|
196
|
-
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
}
|
|
197
240
|
this.maxProviderHintsPerCid = providerCache?.maxProvidersPerCid ?? 8;
|
|
198
241
|
this.maxRequeryOnReachable = options.requeryOnReachable ?? 4;
|
|
199
242
|
|
|
@@ -234,11 +277,22 @@ export class RemoteBlocks implements IBlocks {
|
|
|
234
277
|
};
|
|
235
278
|
}
|
|
236
279
|
|
|
280
|
+
getNativeLogBlockStoreHandle(): unknown {
|
|
281
|
+
return this.localStore.getNativeLogBlockStoreHandle?.();
|
|
282
|
+
}
|
|
283
|
+
|
|
237
284
|
private normalizeProviderHints(
|
|
238
285
|
providers: string[] | undefined,
|
|
239
286
|
limit = this.maxProviderHintsPerCid || 8,
|
|
240
287
|
): string[] {
|
|
241
288
|
if (!providers || providers.length === 0) return [];
|
|
289
|
+
if (this.rustExchange) {
|
|
290
|
+
return this.rustExchange.normalizeProviderHints(
|
|
291
|
+
providers,
|
|
292
|
+
this.publicKeyHash,
|
|
293
|
+
limit,
|
|
294
|
+
);
|
|
295
|
+
}
|
|
242
296
|
const out: string[] = [];
|
|
243
297
|
for (const p of providers) {
|
|
244
298
|
if (!p) continue;
|
|
@@ -252,6 +306,10 @@ export class RemoteBlocks implements IBlocks {
|
|
|
252
306
|
}
|
|
253
307
|
|
|
254
308
|
private rememberProvider(cidString: string, providerHash: string) {
|
|
309
|
+
if (this._rustProviderCache) {
|
|
310
|
+
this._rustProviderCache.rememberProvider(cidString, providerHash);
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
255
313
|
if (!this._providerCache) return;
|
|
256
314
|
if (!providerHash || providerHash === this.publicKeyHash) return;
|
|
257
315
|
const current = this._providerCache.get(cidString) ?? [];
|
|
@@ -266,13 +324,30 @@ export class RemoteBlocks implements IBlocks {
|
|
|
266
324
|
}
|
|
267
325
|
|
|
268
326
|
private rememberProviderHints(cidString: string, providers: string[]) {
|
|
327
|
+
if (this._rustProviderCache) {
|
|
328
|
+
this._rustProviderCache.rememberHints(cidString, providers);
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
269
331
|
if (!this._providerCache) return;
|
|
270
332
|
const normalized = this.normalizeProviderHints(providers);
|
|
271
333
|
if (normalized.length === 0) return;
|
|
272
334
|
this._providerCache.add(cidString, normalized);
|
|
273
335
|
}
|
|
274
336
|
|
|
337
|
+
private getCachedProviders(cidString: string): string[] | undefined {
|
|
338
|
+
return this._rustProviderCache
|
|
339
|
+
? this._rustProviderCache.get(cidString)
|
|
340
|
+
: (this._providerCache?.get(cidString) ?? undefined);
|
|
341
|
+
}
|
|
342
|
+
|
|
275
343
|
private pickRequestBatch(providers: string[], attempt: number): string[] {
|
|
344
|
+
if (this.rustExchange) {
|
|
345
|
+
return this.rustExchange.pickRequestBatch(
|
|
346
|
+
providers,
|
|
347
|
+
this.publicKeyHash,
|
|
348
|
+
attempt,
|
|
349
|
+
);
|
|
350
|
+
}
|
|
276
351
|
if (providers.length <= 1) {
|
|
277
352
|
return providers;
|
|
278
353
|
}
|
|
@@ -293,7 +368,7 @@ export class RemoteBlocks implements IBlocks {
|
|
|
293
368
|
// 1. cached providers (from previous reads)
|
|
294
369
|
// 2. resolveProviders hook (e.g. program-level replicators, DHT, tracker)
|
|
295
370
|
const cached = this.normalizeProviderHints(
|
|
296
|
-
this.
|
|
371
|
+
this.getCachedProviders(cidString),
|
|
297
372
|
);
|
|
298
373
|
if (!this.options.resolveProviders) return cached;
|
|
299
374
|
if (cached.length > 0 && !options?.refresh) return cached;
|
|
@@ -316,18 +391,147 @@ export class RemoteBlocks implements IBlocks {
|
|
|
316
391
|
throw new Error("Local store not set");
|
|
317
392
|
}
|
|
318
393
|
const cid = await this.localStore!.put(bytes);
|
|
394
|
+
await this.notifyPut(cid);
|
|
395
|
+
return cid;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
async putMany(
|
|
399
|
+
blocks: Array<Uint8Array | { block: Block<any, any, any, any>; cid: string }>,
|
|
400
|
+
): Promise<string[]> {
|
|
401
|
+
if (!this.localStore) {
|
|
402
|
+
throw new Error("Local store not set");
|
|
403
|
+
}
|
|
404
|
+
const cids = await this.localStore.putMany(blocks);
|
|
405
|
+
await this.notifyPuts(cids);
|
|
406
|
+
return cids;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
async putKnown(cid: string, bytes: Uint8Array): Promise<string> {
|
|
410
|
+
if (!this.localStore) {
|
|
411
|
+
throw new Error("Local store not set");
|
|
412
|
+
}
|
|
413
|
+
const storedCid = await this.localStore.putKnown(cid, bytes);
|
|
414
|
+
await this.notifyPut(storedCid);
|
|
415
|
+
return storedCid;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
async putKnownMany(
|
|
419
|
+
blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
|
|
420
|
+
): Promise<string[]> {
|
|
421
|
+
if (!this.localStore) {
|
|
422
|
+
throw new Error("Local store not set");
|
|
423
|
+
}
|
|
424
|
+
const cids = await this.localStore.putKnownMany(blocks);
|
|
425
|
+
await this.notifyPuts(cids);
|
|
426
|
+
return cids;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
hasNotifyStoredHook(): boolean {
|
|
430
|
+
return !!this.options.onPut;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
notifyStored(cid: string): Promise<void> | void {
|
|
434
|
+
return this.notifyPut(cid);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
notifyStoredMany(cids: string[]): Promise<void> | void {
|
|
438
|
+
return this.notifyPuts(cids);
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
notifyStoredDeferred(cid: string): void {
|
|
442
|
+
this.notifyStoredManyDeferred([cid]);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
notifyStoredManyDeferred(cids: string[]): void {
|
|
446
|
+
if (!this.options.onPut || cids.length === 0) {
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
let pending = this._deferredStoredNotificationCids;
|
|
450
|
+
if (!pending) {
|
|
451
|
+
pending = new Set();
|
|
452
|
+
this._deferredStoredNotificationCids = pending;
|
|
453
|
+
}
|
|
454
|
+
for (const cid of cids) {
|
|
455
|
+
if (cid) {
|
|
456
|
+
pending.add(cid);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
if (pending.size === 0 || this._deferredStoredNotificationTimer) {
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
this._deferredStoredNotificationTimer = setTimeout(() => {
|
|
463
|
+
this._deferredStoredNotificationTimer = undefined;
|
|
464
|
+
const flushed = this.flushDeferredStoredNotifications();
|
|
465
|
+
if (flushed && typeof flushed.catch === "function") {
|
|
466
|
+
flushed.catch((): void => undefined);
|
|
467
|
+
}
|
|
468
|
+
}, 0);
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
private flushDeferredStoredNotifications(): Promise<void> | void {
|
|
472
|
+
const pending = this._deferredStoredNotificationCids;
|
|
473
|
+
if (!pending || pending.size === 0) {
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
this._deferredStoredNotificationCids = undefined;
|
|
477
|
+
const waits: Promise<void>[] = [];
|
|
478
|
+
for (const cid of pending) {
|
|
479
|
+
try {
|
|
480
|
+
const result = this.notifyStored(cid);
|
|
481
|
+
if (result && typeof result.then === "function") {
|
|
482
|
+
waits.push(result);
|
|
483
|
+
}
|
|
484
|
+
} catch {
|
|
485
|
+
// ignore best-effort hooks
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
if (waits.length > 0) {
|
|
489
|
+
return Promise.all(waits).then((): void => undefined);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
private notifyPut(cid: string): Promise<void> | void {
|
|
494
|
+
const onPut = this.options.onPut;
|
|
495
|
+
if (!onPut) {
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
319
498
|
try {
|
|
320
|
-
|
|
499
|
+
const result = onPut(cid);
|
|
500
|
+
if (result && typeof result.then === "function") {
|
|
501
|
+
return result.catch((): void => undefined);
|
|
502
|
+
}
|
|
321
503
|
} catch {
|
|
322
504
|
// ignore best-effort hooks
|
|
323
505
|
}
|
|
324
|
-
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
private notifyPuts(cids: string[]): Promise<void> | void {
|
|
509
|
+
const onPut = this.options.onPut;
|
|
510
|
+
if (!onPut || cids.length === 0) {
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
if (cids.length === 1) {
|
|
514
|
+
return this.notifyPut(cids[0]!);
|
|
515
|
+
}
|
|
516
|
+
return Promise.all(
|
|
517
|
+
cids.map(async (cid) => {
|
|
518
|
+
try {
|
|
519
|
+
await onPut(cid);
|
|
520
|
+
} catch {
|
|
521
|
+
// ignore best-effort hooks
|
|
522
|
+
}
|
|
523
|
+
}),
|
|
524
|
+
).then((): void => undefined);
|
|
325
525
|
}
|
|
326
526
|
|
|
327
527
|
async has(cid: string) {
|
|
328
528
|
return this.localStore.has(cid);
|
|
329
529
|
}
|
|
330
530
|
|
|
531
|
+
async hasMany(cids: string[]): Promise<boolean[]> {
|
|
532
|
+
return this.localStore.hasMany(cids);
|
|
533
|
+
}
|
|
534
|
+
|
|
331
535
|
async get(
|
|
332
536
|
cid: string,
|
|
333
537
|
options?: GetOptions | undefined,
|
|
@@ -351,6 +555,16 @@ export class RemoteBlocks implements IBlocks {
|
|
|
351
555
|
return value;
|
|
352
556
|
}
|
|
353
557
|
|
|
558
|
+
async getMany(
|
|
559
|
+
cids: string[],
|
|
560
|
+
options?: GetOptions | undefined,
|
|
561
|
+
): Promise<Array<Uint8Array | undefined>> {
|
|
562
|
+
if (!options?.remote) {
|
|
563
|
+
return this.localStore.getMany(cids, options);
|
|
564
|
+
}
|
|
565
|
+
return Promise.all(cids.map((cid) => this.get(cid, options)));
|
|
566
|
+
}
|
|
567
|
+
|
|
354
568
|
hintProviders(cid: string, providers: string[]) {
|
|
355
569
|
const cidString = stringifyCid(cid);
|
|
356
570
|
this.rememberProviderHints(cidString, providers);
|
|
@@ -363,6 +577,10 @@ export class RemoteBlocks implements IBlocks {
|
|
|
363
577
|
await this.localStore?.rm(cid);
|
|
364
578
|
}
|
|
365
579
|
|
|
580
|
+
async rmMany(cids: string[]) {
|
|
581
|
+
await this.localStore?.rmMany(cids);
|
|
582
|
+
}
|
|
583
|
+
|
|
366
584
|
async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
|
|
367
585
|
for await (const [key, value] of this.localStore.iterator()) {
|
|
368
586
|
yield [key, value];
|
|
@@ -396,6 +614,22 @@ export class RemoteBlocks implements IBlocks {
|
|
|
396
614
|
return;
|
|
397
615
|
}
|
|
398
616
|
const cid = stringifyCid(request.cid);
|
|
617
|
+
const publishRaw = this.options.rust?.publishRaw;
|
|
618
|
+
if (publishRaw) {
|
|
619
|
+
// Native-store-served response: the borsh BlockResponse payload is
|
|
620
|
+
// serialized inside wasm straight from the native log block store,
|
|
621
|
+
// so the block bytes never materialize as a JS value.
|
|
622
|
+
const payload = this.localStore.getBlockResponsePayload?.(cid);
|
|
623
|
+
if (payload) {
|
|
624
|
+
const responsePublishOptions = context?.transport
|
|
625
|
+
? context.transport.withResponseOptions({ to: [from] })
|
|
626
|
+
: { to: [from] };
|
|
627
|
+
await publishRaw(payload, responsePublishOptions).catch(
|
|
628
|
+
dontThrowIfDeliveryError,
|
|
629
|
+
);
|
|
630
|
+
return;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
399
633
|
let bytes = await this.localStore.get(cid, {
|
|
400
634
|
remote: {
|
|
401
635
|
timeout: localTimeout,
|
|
@@ -459,7 +693,7 @@ export class RemoteBlocks implements IBlocks {
|
|
|
459
693
|
cidObject: CID,
|
|
460
694
|
options: RemoteReadOptions = {},
|
|
461
695
|
): Promise<Uint8Array | undefined> {
|
|
462
|
-
const codec =
|
|
696
|
+
const codec = codecCodes[cidObject.code as keyof typeof codecCodes];
|
|
463
697
|
|
|
464
698
|
const tryDecode = async (bytes: Uint8Array) => {
|
|
465
699
|
const value = await checkDecodeBlock(cidObject, bytes, {
|
|
@@ -756,6 +990,11 @@ export class RemoteBlocks implements IBlocks {
|
|
|
756
990
|
|
|
757
991
|
// Wait for processing request
|
|
758
992
|
this.closeController.abort();
|
|
993
|
+
if (this._deferredStoredNotificationTimer) {
|
|
994
|
+
clearTimeout(this._deferredStoredNotificationTimer);
|
|
995
|
+
this._deferredStoredNotificationTimer = undefined;
|
|
996
|
+
}
|
|
997
|
+
await this.flushDeferredStoredNotifications();
|
|
759
998
|
this._loadFetchQueue.clear();
|
|
760
999
|
await this._loadFetchQueue.onIdle(); // wait for pending
|
|
761
1000
|
await this.localStore?.stop();
|
|
@@ -763,6 +1002,7 @@ export class RemoteBlocks implements IBlocks {
|
|
|
763
1002
|
this._resolvers.clear();
|
|
764
1003
|
this._blockCache?.clear();
|
|
765
1004
|
this._providerCache?.clear();
|
|
1005
|
+
this._rustProviderCache?.clear();
|
|
766
1006
|
this._open = false;
|
|
767
1007
|
// we dont cleanup subscription because we dont know if someone else is sbuscribing also
|
|
768
1008
|
}
|