@libp2p/kad-dht 10.0.14 → 10.0.15-0b4a2ee79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -5
- package/dist/index.min.js +20 -20
- package/dist/src/content-fetching/index.d.ts.map +1 -1
- package/dist/src/content-fetching/index.js +1 -2
- package/dist/src/content-fetching/index.js.map +1 -1
- package/dist/src/content-routing/index.d.ts.map +1 -1
- package/dist/src/content-routing/index.js +3 -6
- package/dist/src/content-routing/index.js.map +1 -1
- package/dist/src/dual-kad-dht.d.ts +1 -0
- package/dist/src/dual-kad-dht.d.ts.map +1 -1
- package/dist/src/dual-kad-dht.js +5 -5
- package/dist/src/dual-kad-dht.js.map +1 -1
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/kad-dht.d.ts.map +1 -1
- package/dist/src/kad-dht.js +3 -4
- package/dist/src/kad-dht.js.map +1 -1
- package/dist/src/message/index.d.ts +2 -0
- package/dist/src/message/index.d.ts.map +1 -1
- package/dist/src/message/index.js +7 -2
- package/dist/src/message/index.js.map +1 -1
- package/dist/src/network.d.ts +3 -4
- package/dist/src/network.d.ts.map +1 -1
- package/dist/src/network.js +11 -25
- package/dist/src/network.js.map +1 -1
- package/dist/src/peer-routing/index.d.ts +2 -1
- package/dist/src/peer-routing/index.d.ts.map +1 -1
- package/dist/src/peer-routing/index.js +15 -17
- package/dist/src/peer-routing/index.js.map +1 -1
- package/dist/src/providers.d.ts +4 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +17 -17
- package/dist/src/providers.js.map +1 -1
- package/dist/src/query/manager.d.ts +5 -2
- package/dist/src/query/manager.d.ts.map +1 -1
- package/dist/src/query/manager.js +13 -12
- package/dist/src/query/manager.js.map +1 -1
- package/dist/src/query/query-path.d.ts +1 -1
- package/dist/src/query/query-path.d.ts.map +1 -1
- package/dist/src/query/query-path.js +2 -59
- package/dist/src/query/query-path.js.map +1 -1
- package/dist/src/query/utils.d.ts +7 -0
- package/dist/src/query/utils.d.ts.map +1 -0
- package/dist/src/query/utils.js +53 -0
- package/dist/src/query/utils.js.map +1 -0
- package/dist/src/query-self.d.ts +3 -1
- package/dist/src/query-self.d.ts.map +1 -1
- package/dist/src/query-self.js +4 -5
- package/dist/src/query-self.js.map +1 -1
- package/dist/src/routing-table/index.d.ts +2 -0
- package/dist/src/routing-table/index.d.ts.map +1 -1
- package/dist/src/routing-table/index.js +1 -2
- package/dist/src/routing-table/index.js.map +1 -1
- package/dist/src/routing-table/refresh.d.ts +5 -1
- package/dist/src/routing-table/refresh.d.ts.map +1 -1
- package/dist/src/routing-table/refresh.js +2 -3
- package/dist/src/routing-table/refresh.js.map +1 -1
- package/dist/src/rpc/handlers/add-provider.d.ts +6 -1
- package/dist/src/rpc/handlers/add-provider.d.ts.map +1 -1
- package/dist/src/rpc/handlers/add-provider.js +9 -10
- package/dist/src/rpc/handlers/add-provider.js.map +1 -1
- package/dist/src/rpc/handlers/find-node.d.ts +5 -1
- package/dist/src/rpc/handlers/find-node.d.ts.map +1 -1
- package/dist/src/rpc/handlers/find-node.js +11 -10
- package/dist/src/rpc/handlers/find-node.js.map +1 -1
- package/dist/src/rpc/handlers/get-providers.d.ts +4 -1
- package/dist/src/rpc/handlers/get-providers.d.ts.map +1 -1
- package/dist/src/rpc/handlers/get-providers.js +8 -9
- package/dist/src/rpc/handlers/get-providers.js.map +1 -1
- package/dist/src/rpc/handlers/get-value.d.ts +5 -1
- package/dist/src/rpc/handlers/get-value.d.ts.map +1 -1
- package/dist/src/rpc/handlers/get-value.js +16 -15
- package/dist/src/rpc/handlers/get-value.js.map +1 -1
- package/dist/src/rpc/handlers/ping.d.ts +6 -0
- package/dist/src/rpc/handlers/ping.d.ts.map +1 -1
- package/dist/src/rpc/handlers/ping.js +5 -3
- package/dist/src/rpc/handlers/ping.js.map +1 -1
- package/dist/src/rpc/handlers/put-value.d.ts +3 -1
- package/dist/src/rpc/handlers/put-value.d.ts.map +1 -1
- package/dist/src/rpc/handlers/put-value.js +2 -3
- package/dist/src/rpc/handlers/put-value.js.map +1 -1
- package/dist/src/rpc/index.d.ts.map +1 -1
- package/dist/src/rpc/index.js +3 -4
- package/dist/src/rpc/index.js.map +1 -1
- package/dist/src/topology-listener.d.ts.map +1 -1
- package/dist/src/topology-listener.js +1 -2
- package/dist/src/topology-listener.js.map +1 -1
- package/package.json +23 -22
- package/src/content-fetching/index.ts +2 -4
- package/src/content-routing/index.ts +4 -7
- package/src/dual-kad-dht.ts +6 -6
- package/src/index.ts +2 -1
- package/src/kad-dht.ts +4 -4
- package/src/message/index.ts +9 -2
- package/src/network.ts +14 -44
- package/src/peer-routing/index.ts +17 -19
- package/src/providers.ts +19 -18
- package/src/query/manager.ts +18 -16
- package/src/query/query-path.ts +3 -72
- package/src/query/utils.ts +65 -0
- package/src/query-self.ts +6 -5
- package/src/routing-table/index.ts +3 -3
- package/src/routing-table/refresh.ts +7 -4
- package/src/rpc/handlers/add-provider.ts +13 -10
- package/src/rpc/handlers/find-node.ts +13 -11
- package/src/rpc/handlers/get-providers.ts +10 -10
- package/src/rpc/handlers/get-value.ts +18 -17
- package/src/rpc/handlers/ping.ts +11 -3
- package/src/rpc/handlers/put-value.ts +4 -3
- package/src/rpc/index.ts +4 -4
- package/src/topology-listener.ts +2 -3
- package/dist/typedoc-urls.json +0 -55
package/src/network.ts
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
|
-
import { CodeError } from '@libp2p/interface/errors'
|
|
2
1
|
import { TypedEventEmitter, CustomEvent } from '@libp2p/interface/events'
|
|
3
|
-
import {
|
|
4
|
-
import { abortableDuplex } from 'abortable-iterator'
|
|
5
|
-
import drain from 'it-drain'
|
|
6
|
-
import first from 'it-first'
|
|
7
|
-
import * as lp from 'it-length-prefixed'
|
|
8
|
-
import { pipe } from 'it-pipe'
|
|
2
|
+
import { pbStream } from 'it-protobuf-stream'
|
|
9
3
|
import { Message } from './message/index.js'
|
|
10
4
|
import {
|
|
11
5
|
dialPeerEvent,
|
|
@@ -14,14 +8,11 @@ import {
|
|
|
14
8
|
queryErrorEvent
|
|
15
9
|
} from './query/events.js'
|
|
16
10
|
import type { KadDHTComponents, QueryEvent, QueryOptions } from './index.js'
|
|
17
|
-
import type { AbortOptions } from '@libp2p/interface'
|
|
11
|
+
import type { AbortOptions, Logger } from '@libp2p/interface'
|
|
18
12
|
import type { Stream } from '@libp2p/interface/connection'
|
|
19
13
|
import type { PeerId } from '@libp2p/interface/peer-id'
|
|
20
14
|
import type { PeerInfo } from '@libp2p/interface/peer-info'
|
|
21
15
|
import type { Startable } from '@libp2p/interface/startable'
|
|
22
|
-
import type { Logger } from '@libp2p/logger'
|
|
23
|
-
import type { Duplex, Source } from 'it-stream-types'
|
|
24
|
-
import type { Uint8ArrayList } from 'uint8arraylist'
|
|
25
16
|
|
|
26
17
|
export interface NetworkInit {
|
|
27
18
|
protocol: string
|
|
@@ -49,7 +40,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
|
|
|
49
40
|
|
|
50
41
|
const { protocol, lan } = init
|
|
51
42
|
this.components = components
|
|
52
|
-
this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:network`)
|
|
43
|
+
this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:network`)
|
|
53
44
|
this.running = false
|
|
54
45
|
this.protocol = protocol
|
|
55
46
|
}
|
|
@@ -97,7 +88,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
|
|
|
97
88
|
const connection = await this.components.connectionManager.openConnection(to, options)
|
|
98
89
|
const stream = await connection.newStream(this.protocol, options)
|
|
99
90
|
|
|
100
|
-
const response = await this._writeReadMessage(stream, msg
|
|
91
|
+
const response = await this._writeReadMessage(stream, msg, options)
|
|
101
92
|
|
|
102
93
|
yield peerResponseEvent({
|
|
103
94
|
from: to,
|
|
@@ -133,7 +124,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
|
|
|
133
124
|
const connection = await this.components.connectionManager.openConnection(to, options)
|
|
134
125
|
const stream = await connection.newStream(this.protocol, options)
|
|
135
126
|
|
|
136
|
-
await this._writeMessage(stream, msg
|
|
127
|
+
await this._writeMessage(stream, msg, options)
|
|
137
128
|
|
|
138
129
|
yield peerResponseEvent({ from: to, messageType: msg.type }, options)
|
|
139
130
|
} catch (err: any) {
|
|
@@ -148,17 +139,10 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
|
|
|
148
139
|
/**
|
|
149
140
|
* Write a message to the given stream
|
|
150
141
|
*/
|
|
151
|
-
async _writeMessage (stream:
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
await pipe(
|
|
157
|
-
[msg],
|
|
158
|
-
(source) => lp.encode(source),
|
|
159
|
-
stream,
|
|
160
|
-
drain
|
|
161
|
-
)
|
|
142
|
+
async _writeMessage (stream: Stream, msg: Message, options: AbortOptions): Promise<void> {
|
|
143
|
+
const pb = pbStream(stream)
|
|
144
|
+
await pb.write(msg, Message, options)
|
|
145
|
+
await pb.unwrap().close(options)
|
|
162
146
|
}
|
|
163
147
|
|
|
164
148
|
/**
|
|
@@ -166,28 +150,14 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
|
|
|
166
150
|
* If no response is received after the specified timeout
|
|
167
151
|
* this will error out.
|
|
168
152
|
*/
|
|
169
|
-
async _writeReadMessage (stream:
|
|
170
|
-
|
|
171
|
-
stream = abortableDuplex(stream, options.signal)
|
|
172
|
-
}
|
|
153
|
+
async _writeReadMessage (stream: Stream, msg: Message, options: AbortOptions): Promise<Message> {
|
|
154
|
+
const pb = pbStream(stream)
|
|
173
155
|
|
|
174
|
-
|
|
175
|
-
[msg],
|
|
176
|
-
(source) => lp.encode(source),
|
|
177
|
-
stream,
|
|
178
|
-
(source) => lp.decode(source),
|
|
179
|
-
async source => {
|
|
180
|
-
const buf = await first(source)
|
|
156
|
+
await pb.write(msg, Message, options)
|
|
181
157
|
|
|
182
|
-
|
|
183
|
-
return buf
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
throw new CodeError('No message received', 'ERR_NO_MESSAGE_RECEIVED')
|
|
187
|
-
}
|
|
188
|
-
)
|
|
158
|
+
const message = await pb.read(Message, options)
|
|
189
159
|
|
|
190
|
-
|
|
160
|
+
await pb.unwrap().close(options)
|
|
191
161
|
|
|
192
162
|
// tell any listeners about new peers we've seen
|
|
193
163
|
message.closerPeers.forEach(peerData => {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { keys } from '@libp2p/crypto'
|
|
2
2
|
import { CodeError } from '@libp2p/interface/errors'
|
|
3
|
-
import { logger } from '@libp2p/logger'
|
|
4
3
|
import { peerIdFromKeys } from '@libp2p/peer-id'
|
|
5
4
|
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
|
|
6
5
|
import { Message, MESSAGE_TYPE } from '../message/index.js'
|
|
@@ -18,10 +17,10 @@ import type { Network } from '../network.js'
|
|
|
18
17
|
import type { QueryManager, QueryOptions } from '../query/manager.js'
|
|
19
18
|
import type { QueryFunc } from '../query/types.js'
|
|
20
19
|
import type { RoutingTable } from '../routing-table/index.js'
|
|
21
|
-
import type { AbortOptions } from '@libp2p/interface'
|
|
20
|
+
import type { AbortOptions, Logger } from '@libp2p/interface'
|
|
22
21
|
import type { PeerId } from '@libp2p/interface/peer-id'
|
|
23
22
|
import type { PeerInfo } from '@libp2p/interface/peer-info'
|
|
24
|
-
import type {
|
|
23
|
+
import type { PeerStore } from '@libp2p/interface/src/peer-store/index.js'
|
|
25
24
|
|
|
26
25
|
export interface PeerRoutingInit {
|
|
27
26
|
routingTable: RoutingTable
|
|
@@ -32,22 +31,24 @@ export interface PeerRoutingInit {
|
|
|
32
31
|
}
|
|
33
32
|
|
|
34
33
|
export class PeerRouting {
|
|
35
|
-
private readonly components: KadDHTComponents
|
|
36
34
|
private readonly log: Logger
|
|
37
35
|
private readonly routingTable: RoutingTable
|
|
38
36
|
private readonly network: Network
|
|
39
37
|
private readonly validators: Validators
|
|
40
38
|
private readonly queryManager: QueryManager
|
|
39
|
+
private readonly peerStore: PeerStore
|
|
40
|
+
private readonly peerId: PeerId
|
|
41
41
|
|
|
42
42
|
constructor (components: KadDHTComponents, init: PeerRoutingInit) {
|
|
43
43
|
const { routingTable, network, validators, queryManager, lan } = init
|
|
44
44
|
|
|
45
|
-
this.components = components
|
|
46
45
|
this.routingTable = routingTable
|
|
47
46
|
this.network = network
|
|
48
47
|
this.validators = validators
|
|
49
48
|
this.queryManager = queryManager
|
|
50
|
-
this.
|
|
49
|
+
this.peerStore = components.peerStore
|
|
50
|
+
this.peerId = components.peerId
|
|
51
|
+
this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:peer-routing`)
|
|
51
52
|
}
|
|
52
53
|
|
|
53
54
|
/**
|
|
@@ -62,7 +63,7 @@ export class PeerRouting {
|
|
|
62
63
|
this.log('findPeerLocal found %p in routing table', peer)
|
|
63
64
|
|
|
64
65
|
try {
|
|
65
|
-
peerData = await this.
|
|
66
|
+
peerData = await this.peerStore.get(p)
|
|
66
67
|
} catch (err: any) {
|
|
67
68
|
if (err.code !== 'ERR_NOT_FOUND') {
|
|
68
69
|
throw err
|
|
@@ -72,7 +73,7 @@ export class PeerRouting {
|
|
|
72
73
|
|
|
73
74
|
if (peerData == null) {
|
|
74
75
|
try {
|
|
75
|
-
peerData = await this.
|
|
76
|
+
peerData = await this.peerStore.get(peer)
|
|
76
77
|
} catch (err: any) {
|
|
77
78
|
if (err.code !== 'ERR_NOT_FOUND') {
|
|
78
79
|
throw err
|
|
@@ -85,8 +86,7 @@ export class PeerRouting {
|
|
|
85
86
|
|
|
86
87
|
return {
|
|
87
88
|
id: peerData.id,
|
|
88
|
-
multiaddrs: peerData.addresses.map((address) => address.multiaddr)
|
|
89
|
-
protocols: []
|
|
89
|
+
multiaddrs: peerData.addresses.map((address) => address.multiaddr)
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
|
|
@@ -142,7 +142,7 @@ export class PeerRouting {
|
|
|
142
142
|
if (pi != null) {
|
|
143
143
|
this.log('found local')
|
|
144
144
|
yield finalPeerEvent({
|
|
145
|
-
from: this.
|
|
145
|
+
from: this.peerId,
|
|
146
146
|
peer: pi
|
|
147
147
|
}, options)
|
|
148
148
|
return
|
|
@@ -181,7 +181,7 @@ export class PeerRouting {
|
|
|
181
181
|
}
|
|
182
182
|
|
|
183
183
|
if (!foundPeer) {
|
|
184
|
-
yield queryErrorEvent({ from: this.
|
|
184
|
+
yield queryErrorEvent({ from: this.peerId, error: new CodeError('Not found', 'ERR_NOT_FOUND') }, options)
|
|
185
185
|
}
|
|
186
186
|
}
|
|
187
187
|
|
|
@@ -220,14 +220,13 @@ export class PeerRouting {
|
|
|
220
220
|
|
|
221
221
|
for (const peerId of peers.peers) {
|
|
222
222
|
try {
|
|
223
|
-
const peer = await this.
|
|
223
|
+
const peer = await this.peerStore.get(peerId)
|
|
224
224
|
|
|
225
225
|
yield finalPeerEvent({
|
|
226
|
-
from: this.
|
|
226
|
+
from: this.peerId,
|
|
227
227
|
peer: {
|
|
228
228
|
id: peerId,
|
|
229
|
-
multiaddrs: peer.addresses.map(({ multiaddr }) => multiaddr)
|
|
230
|
-
protocols: peer.protocols
|
|
229
|
+
multiaddrs: peer.addresses.map(({ multiaddr }) => multiaddr)
|
|
231
230
|
}
|
|
232
231
|
}, options)
|
|
233
232
|
} catch (err: any) {
|
|
@@ -292,12 +291,11 @@ export class PeerRouting {
|
|
|
292
291
|
}
|
|
293
292
|
|
|
294
293
|
try {
|
|
295
|
-
const peer = await this.
|
|
294
|
+
const peer = await this.peerStore.get(peerId)
|
|
296
295
|
|
|
297
296
|
output.push({
|
|
298
297
|
id: peerId,
|
|
299
|
-
multiaddrs: peer.addresses.map(({ multiaddr }) => multiaddr)
|
|
300
|
-
protocols: peer.protocols
|
|
298
|
+
multiaddrs: peer.addresses.map(({ multiaddr }) => multiaddr)
|
|
301
299
|
})
|
|
302
300
|
} catch (err: any) {
|
|
303
301
|
if (err.code !== 'ERR_NOT_FOUND') {
|
package/src/providers.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { logger } from '@libp2p/logger'
|
|
2
1
|
import { peerIdFromString } from '@libp2p/peer-id'
|
|
3
2
|
import cache from 'hashlru'
|
|
4
3
|
import { Key } from 'interface-datastore/key'
|
|
@@ -11,13 +10,12 @@ import {
|
|
|
11
10
|
PROVIDERS_LRU_CACHE_SIZE,
|
|
12
11
|
PROVIDER_KEY_PREFIX
|
|
13
12
|
} from './constants.js'
|
|
13
|
+
import type { ComponentLogger, Logger } from '@libp2p/interface'
|
|
14
14
|
import type { PeerId } from '@libp2p/interface/peer-id'
|
|
15
15
|
import type { Startable } from '@libp2p/interface/startable'
|
|
16
16
|
import type { Datastore } from 'interface-datastore'
|
|
17
17
|
import type { CID } from 'multiformats'
|
|
18
18
|
|
|
19
|
-
const log = logger('libp2p:kad-dht:providers')
|
|
20
|
-
|
|
21
19
|
export interface ProvidersInit {
|
|
22
20
|
cacheSize?: number
|
|
23
21
|
/**
|
|
@@ -32,6 +30,7 @@ export interface ProvidersInit {
|
|
|
32
30
|
|
|
33
31
|
export interface ProvidersComponents {
|
|
34
32
|
datastore: Datastore
|
|
33
|
+
logger: ComponentLogger
|
|
35
34
|
}
|
|
36
35
|
|
|
37
36
|
/**
|
|
@@ -47,7 +46,8 @@ export interface ProvidersComponents {
|
|
|
47
46
|
* access is fast there is an LRU cache in front of that.
|
|
48
47
|
*/
|
|
49
48
|
export class Providers implements Startable {
|
|
50
|
-
private readonly
|
|
49
|
+
private readonly log: Logger
|
|
50
|
+
private readonly datastore: Datastore
|
|
51
51
|
private readonly cache: ReturnType<typeof cache>
|
|
52
52
|
private readonly cleanupInterval: number
|
|
53
53
|
private readonly provideValidity: number
|
|
@@ -58,7 +58,8 @@ export class Providers implements Startable {
|
|
|
58
58
|
constructor (components: ProvidersComponents, init: ProvidersInit = {}) {
|
|
59
59
|
const { cacheSize, cleanupInterval, provideValidity } = init
|
|
60
60
|
|
|
61
|
-
this.
|
|
61
|
+
this.log = components.logger.forComponent('libp2p:kad-dht:providers')
|
|
62
|
+
this.datastore = components.datastore
|
|
62
63
|
this.cleanupInterval = cleanupInterval ?? PROVIDERS_CLEANUP_INTERVAL
|
|
63
64
|
this.provideValidity = provideValidity ?? PROVIDERS_VALIDITY
|
|
64
65
|
this.cache = cache(cacheSize ?? PROVIDERS_LRU_CACHE_SIZE)
|
|
@@ -83,7 +84,7 @@ export class Providers implements Startable {
|
|
|
83
84
|
this.cleaner = setInterval(
|
|
84
85
|
() => {
|
|
85
86
|
this._cleanup().catch(err => {
|
|
86
|
-
log.error(err)
|
|
87
|
+
this.log.error(err)
|
|
87
88
|
})
|
|
88
89
|
},
|
|
89
90
|
this.cleanupInterval
|
|
@@ -112,10 +113,10 @@ export class Providers implements Startable {
|
|
|
112
113
|
let count = 0
|
|
113
114
|
let deleteCount = 0
|
|
114
115
|
const deleted = new Map<string, Set<string>>()
|
|
115
|
-
const batch = this.
|
|
116
|
+
const batch = this.datastore.batch()
|
|
116
117
|
|
|
117
118
|
// Get all provider entries from the datastore
|
|
118
|
-
const query = this.
|
|
119
|
+
const query = this.datastore.query({ prefix: PROVIDER_KEY_PREFIX })
|
|
119
120
|
|
|
120
121
|
for await (const entry of query) {
|
|
121
122
|
try {
|
|
@@ -126,7 +127,7 @@ export class Providers implements Startable {
|
|
|
126
127
|
const delta = now - time
|
|
127
128
|
const expired = delta > this.provideValidity
|
|
128
129
|
|
|
129
|
-
log('comparing: %d - %d = %d > %d %s', now, time, delta, this.provideValidity, expired ? '(expired)' : '')
|
|
130
|
+
this.log('comparing: %d - %d = %d > %d %s', now, time, delta, this.provideValidity, expired ? '(expired)' : '')
|
|
130
131
|
|
|
131
132
|
if (expired) {
|
|
132
133
|
deleteCount++
|
|
@@ -137,16 +138,16 @@ export class Providers implements Startable {
|
|
|
137
138
|
}
|
|
138
139
|
count++
|
|
139
140
|
} catch (err: any) {
|
|
140
|
-
log.error(err.message)
|
|
141
|
+
this.log.error(err.message)
|
|
141
142
|
}
|
|
142
143
|
}
|
|
143
144
|
|
|
144
145
|
// Commit the deletes to the datastore
|
|
145
146
|
if (deleted.size > 0) {
|
|
146
|
-
log('deleting %d / %d entries', deleteCount, count)
|
|
147
|
+
this.log('deleting %d / %d entries', deleteCount, count)
|
|
147
148
|
await batch.commit()
|
|
148
149
|
} else {
|
|
149
|
-
log('nothing to delete')
|
|
150
|
+
this.log('nothing to delete')
|
|
150
151
|
}
|
|
151
152
|
|
|
152
153
|
// Clear expired entries from the cache
|
|
@@ -167,7 +168,7 @@ export class Providers implements Startable {
|
|
|
167
168
|
}
|
|
168
169
|
}
|
|
169
170
|
|
|
170
|
-
log('Cleanup successful (%dms)', Date.now() - start)
|
|
171
|
+
this.log('Cleanup successful (%dms)', Date.now() - start)
|
|
171
172
|
})
|
|
172
173
|
}
|
|
173
174
|
|
|
@@ -179,7 +180,7 @@ export class Providers implements Startable {
|
|
|
179
180
|
let provs: Map<string, Date> = this.cache.get(cacheKey)
|
|
180
181
|
|
|
181
182
|
if (provs == null) {
|
|
182
|
-
provs = await loadProviders(this.
|
|
183
|
+
provs = await loadProviders(this.datastore, cid)
|
|
183
184
|
this.cache.set(cacheKey, provs)
|
|
184
185
|
}
|
|
185
186
|
|
|
@@ -191,17 +192,17 @@ export class Providers implements Startable {
|
|
|
191
192
|
*/
|
|
192
193
|
async addProvider (cid: CID, provider: PeerId): Promise<void> {
|
|
193
194
|
await this.syncQueue.add(async () => {
|
|
194
|
-
log('%p provides %s', provider, cid)
|
|
195
|
+
this.log('%p provides %s', provider, cid)
|
|
195
196
|
const provs = await this._getProvidersMap(cid)
|
|
196
197
|
|
|
197
|
-
log('loaded %s provs', provs.size)
|
|
198
|
+
this.log('loaded %s provs', provs.size)
|
|
198
199
|
const now = new Date()
|
|
199
200
|
provs.set(provider.toString(), now)
|
|
200
201
|
|
|
201
202
|
const dsKey = makeProviderKey(cid)
|
|
202
203
|
this.cache.set(dsKey, provs)
|
|
203
204
|
|
|
204
|
-
await writeProviderEntry(this.
|
|
205
|
+
await writeProviderEntry(this.datastore, cid, provider, now)
|
|
205
206
|
})
|
|
206
207
|
}
|
|
207
208
|
|
|
@@ -210,7 +211,7 @@ export class Providers implements Startable {
|
|
|
210
211
|
*/
|
|
211
212
|
async getProviders (cid: CID): Promise<PeerId[]> {
|
|
212
213
|
return this.syncQueue.add(async () => {
|
|
213
|
-
log('get providers for %s', cid)
|
|
214
|
+
this.log('get providers for %s', cid)
|
|
214
215
|
const provs = await this._getProvidersMap(cid)
|
|
215
216
|
|
|
216
217
|
return [...provs.keys()].map(peerIdStr => {
|
package/src/query/manager.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { AbortError } from '@libp2p/interface/errors'
|
|
2
2
|
import { TypedEventEmitter, CustomEvent, setMaxListeners } from '@libp2p/interface/events'
|
|
3
|
-
import { logger } from '@libp2p/logger'
|
|
4
3
|
import { PeerSet } from '@libp2p/peer-collections'
|
|
5
4
|
import { anySignal } from 'any-signal'
|
|
6
5
|
import merge from 'it-merge'
|
|
@@ -13,6 +12,7 @@ import { queryPath } from './query-path.js'
|
|
|
13
12
|
import type { QueryFunc } from './types.js'
|
|
14
13
|
import type { QueryEvent, QueryOptions as RootQueryOptions } from '../index.js'
|
|
15
14
|
import type { RoutingTable } from '../routing-table/index.js'
|
|
15
|
+
import type { ComponentLogger } from '@libp2p/interface'
|
|
16
16
|
import type { Metric, Metrics } from '@libp2p/interface/metrics'
|
|
17
17
|
import type { PeerId } from '@libp2p/interface/peer-id'
|
|
18
18
|
import type { Startable } from '@libp2p/interface/startable'
|
|
@@ -33,6 +33,7 @@ export interface QueryManagerInit {
|
|
|
33
33
|
export interface QueryManagerComponents {
|
|
34
34
|
peerId: PeerId
|
|
35
35
|
metrics?: Metrics
|
|
36
|
+
logger: ComponentLogger
|
|
36
37
|
}
|
|
37
38
|
|
|
38
39
|
export interface QueryOptions extends RootQueryOptions {
|
|
@@ -44,25 +45,24 @@ export interface QueryOptions extends RootQueryOptions {
|
|
|
44
45
|
* Keeps track of all running queries
|
|
45
46
|
*/
|
|
46
47
|
export class QueryManager implements Startable {
|
|
47
|
-
private readonly components: QueryManagerComponents
|
|
48
48
|
private readonly lan: boolean
|
|
49
49
|
public disjointPaths: number
|
|
50
50
|
private readonly alpha: number
|
|
51
51
|
private readonly shutDownController: AbortController
|
|
52
52
|
private running: boolean
|
|
53
53
|
private queries: number
|
|
54
|
-
private
|
|
54
|
+
private readonly logger: ComponentLogger
|
|
55
|
+
private readonly peerId: PeerId
|
|
56
|
+
private readonly routingTable: RoutingTable
|
|
57
|
+
private initialQuerySelfHasRun?: DeferredPromise<void>
|
|
58
|
+
private readonly metrics?: {
|
|
55
59
|
runningQueries: Metric
|
|
56
60
|
queryTime: Metric
|
|
57
61
|
}
|
|
58
62
|
|
|
59
|
-
private readonly routingTable: RoutingTable
|
|
60
|
-
private initialQuerySelfHasRun?: DeferredPromise<void>
|
|
61
|
-
|
|
62
63
|
constructor (components: QueryManagerComponents, init: QueryManagerInit) {
|
|
63
64
|
const { lan = false, disjointPaths = K, alpha = ALPHA } = init
|
|
64
65
|
|
|
65
|
-
this.components = components
|
|
66
66
|
this.disjointPaths = disjointPaths ?? K
|
|
67
67
|
this.running = false
|
|
68
68
|
this.alpha = alpha ?? ALPHA
|
|
@@ -70,6 +70,15 @@ export class QueryManager implements Startable {
|
|
|
70
70
|
this.queries = 0
|
|
71
71
|
this.initialQuerySelfHasRun = init.initialQuerySelfHasRun
|
|
72
72
|
this.routingTable = init.routingTable
|
|
73
|
+
this.logger = components.logger
|
|
74
|
+
this.peerId = components.peerId
|
|
75
|
+
|
|
76
|
+
if (components.metrics != null) {
|
|
77
|
+
this.metrics = {
|
|
78
|
+
runningQueries: components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_running_queries`),
|
|
79
|
+
queryTime: components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_query_time_seconds`)
|
|
80
|
+
}
|
|
81
|
+
}
|
|
73
82
|
|
|
74
83
|
// allow us to stop queries on shut down
|
|
75
84
|
this.shutDownController = new AbortController()
|
|
@@ -86,13 +95,6 @@ export class QueryManager implements Startable {
|
|
|
86
95
|
*/
|
|
87
96
|
async start (): Promise<void> {
|
|
88
97
|
this.running = true
|
|
89
|
-
|
|
90
|
-
if (this.components.metrics != null && this.metrics == null) {
|
|
91
|
-
this.metrics = {
|
|
92
|
-
runningQueries: this.components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_running_queries`),
|
|
93
|
-
queryTime: this.components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_query_time_seconds`)
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
98
|
}
|
|
97
99
|
|
|
98
100
|
/**
|
|
@@ -131,7 +133,7 @@ export class QueryManager implements Startable {
|
|
|
131
133
|
// so make sure we don't make a lot of noise in the logs
|
|
132
134
|
setMaxListeners(Infinity, signal)
|
|
133
135
|
|
|
134
|
-
const log = logger(`libp2p:kad-dht:${this.lan ? 'lan' : 'wan'}:query:` + uint8ArrayToString(key, 'base58btc'))
|
|
136
|
+
const log = this.logger.forComponent(`libp2p:kad-dht:${this.lan ? 'lan' : 'wan'}:query:` + uint8ArrayToString(key, 'base58btc'))
|
|
135
137
|
|
|
136
138
|
// query a subset of peers up to `kBucketSize / 2` in length
|
|
137
139
|
const startTime = Date.now()
|
|
@@ -174,7 +176,7 @@ export class QueryManager implements Startable {
|
|
|
174
176
|
return queryPath({
|
|
175
177
|
key,
|
|
176
178
|
startingPeer: peer,
|
|
177
|
-
ourPeerId: this.
|
|
179
|
+
ourPeerId: this.peerId,
|
|
178
180
|
signal,
|
|
179
181
|
query: queryFunc,
|
|
180
182
|
pathIndex: index,
|
package/src/query/query-path.ts
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
|
-
import { CodeError } from '@libp2p/interface/errors'
|
|
2
1
|
import { anySignal } from 'any-signal'
|
|
3
|
-
import defer from 'p-defer'
|
|
4
2
|
import Queue from 'p-queue'
|
|
5
3
|
import { toString } from 'uint8arrays/to-string'
|
|
6
4
|
import { xor } from 'uint8arrays/xor'
|
|
7
5
|
import { convertPeerId, convertBuffer } from '../utils.js'
|
|
8
6
|
import { queryErrorEvent } from './events.js'
|
|
7
|
+
import { queueToGenerator } from './utils.js'
|
|
9
8
|
import type { CleanUpEvents } from './manager.js'
|
|
10
9
|
import type { QueryEvent, QueryOptions } from '../index.js'
|
|
11
10
|
import type { QueryFunc } from '../query/types.js'
|
|
11
|
+
import type { Logger } from '@libp2p/interface'
|
|
12
12
|
import type { TypedEventTarget } from '@libp2p/interface/events'
|
|
13
13
|
import type { PeerId } from '@libp2p/interface/peer-id'
|
|
14
|
-
import type { Logger } from '@libp2p/logger'
|
|
15
14
|
import type { PeerSet } from '@libp2p/peer-collections'
|
|
16
15
|
|
|
17
16
|
const MAX_XOR = BigInt('0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
|
|
@@ -182,73 +181,5 @@ export async function * queryPath (options: QueryPathOptions): AsyncGenerator<Qu
|
|
|
182
181
|
queryPeer(startingPeer, await convertPeerId(startingPeer))
|
|
183
182
|
|
|
184
183
|
// yield results as they come in
|
|
185
|
-
yield *
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
async function * toGenerator (queue: Queue, signal: AbortSignal, cleanUp: TypedEventTarget<CleanUpEvents>, log: Logger): AsyncGenerator<QueryEvent, void, undefined> {
|
|
189
|
-
let deferred = defer()
|
|
190
|
-
let running = true
|
|
191
|
-
const results: QueryEvent[] = []
|
|
192
|
-
|
|
193
|
-
const cleanup = (): void => {
|
|
194
|
-
if (!running) {
|
|
195
|
-
return
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
log('clean up queue, results %d, queue size %d, pending tasks %d', results.length, queue.size, queue.pending)
|
|
199
|
-
|
|
200
|
-
running = false
|
|
201
|
-
queue.clear()
|
|
202
|
-
results.splice(0, results.length)
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
queue.on('completed', result => {
|
|
206
|
-
results.push(result)
|
|
207
|
-
deferred.resolve()
|
|
208
|
-
})
|
|
209
|
-
queue.on('error', err => {
|
|
210
|
-
log('queue error', err)
|
|
211
|
-
cleanup()
|
|
212
|
-
deferred.reject(err)
|
|
213
|
-
})
|
|
214
|
-
queue.on('idle', () => {
|
|
215
|
-
log('queue idle')
|
|
216
|
-
running = false
|
|
217
|
-
deferred.resolve()
|
|
218
|
-
})
|
|
219
|
-
|
|
220
|
-
// clear the queue and throw if the query is aborted
|
|
221
|
-
signal.addEventListener('abort', () => {
|
|
222
|
-
log('abort queue')
|
|
223
|
-
const wasRunning = running
|
|
224
|
-
cleanup()
|
|
225
|
-
|
|
226
|
-
if (wasRunning) {
|
|
227
|
-
deferred.reject(new CodeError('Query aborted', 'ERR_QUERY_ABORTED'))
|
|
228
|
-
}
|
|
229
|
-
})
|
|
230
|
-
|
|
231
|
-
// the user broke out of the loop early, ensure we resolve the deferred result
|
|
232
|
-
// promise and clear the queue of any remaining jobs
|
|
233
|
-
cleanUp.addEventListener('cleanup', () => {
|
|
234
|
-
cleanup()
|
|
235
|
-
deferred.resolve()
|
|
236
|
-
})
|
|
237
|
-
|
|
238
|
-
while (running) { // eslint-disable-line no-unmodified-loop-condition
|
|
239
|
-
await deferred.promise
|
|
240
|
-
deferred = defer()
|
|
241
|
-
|
|
242
|
-
// yield all available results
|
|
243
|
-
while (results.length > 0) {
|
|
244
|
-
const result = results.shift()
|
|
245
|
-
|
|
246
|
-
if (result != null) {
|
|
247
|
-
yield result
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
// yield any remaining results
|
|
253
|
-
yield * results
|
|
184
|
+
yield * queueToGenerator(queue, signal, cleanUp, log)
|
|
254
185
|
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { CodeError } from '@libp2p/interface/errors'
|
|
2
|
+
import { pushable } from 'it-pushable'
|
|
3
|
+
import type { CleanUpEvents } from './manager.js'
|
|
4
|
+
import type { QueryEvent } from '../index.js'
|
|
5
|
+
import type { Logger } from '@libp2p/interface'
|
|
6
|
+
import type { TypedEventTarget } from '@libp2p/interface/events'
|
|
7
|
+
import type Queue from 'p-queue'
|
|
8
|
+
|
|
9
|
+
export async function * queueToGenerator (queue: Queue, signal: AbortSignal, cleanUp: TypedEventTarget<CleanUpEvents>, log: Logger): AsyncGenerator<QueryEvent, void, undefined> {
|
|
10
|
+
const stream = pushable<QueryEvent>({
|
|
11
|
+
objectMode: true
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
const cleanup = (err?: Error): void => {
|
|
15
|
+
log('clean up queue, results %d, queue size %d, pending tasks %d', stream.readableLength, queue.size, queue.pending)
|
|
16
|
+
queue.clear()
|
|
17
|
+
stream.end(err)
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const onQueueJobComplete = (result: QueryEvent): void => {
|
|
21
|
+
if (result != null) {
|
|
22
|
+
stream.push(result)
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const onQueueError = (err: Error): void => {
|
|
27
|
+
log('queue error', err)
|
|
28
|
+
cleanup(err)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const onQueueIdle = (): void => {
|
|
32
|
+
log('queue idle')
|
|
33
|
+
cleanup()
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// clear the queue and throw if the query is aborted
|
|
37
|
+
const onSignalAbort = (): void => {
|
|
38
|
+
log('abort queue')
|
|
39
|
+
cleanup(new CodeError('Query aborted', 'ERR_QUERY_ABORTED'))
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// the user broke out of the loop early, ensure we resolve the deferred result
|
|
43
|
+
// promise and clear the queue of any remaining jobs
|
|
44
|
+
const onCleanUp = (): void => {
|
|
45
|
+
cleanup()
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// add listeners
|
|
49
|
+
queue.on('completed', onQueueJobComplete)
|
|
50
|
+
queue.on('error', onQueueError)
|
|
51
|
+
queue.on('idle', onQueueIdle)
|
|
52
|
+
signal.addEventListener('abort', onSignalAbort)
|
|
53
|
+
cleanUp.addEventListener('cleanup', onCleanUp)
|
|
54
|
+
|
|
55
|
+
try {
|
|
56
|
+
yield * stream
|
|
57
|
+
} finally {
|
|
58
|
+
// remove listeners
|
|
59
|
+
queue.removeListener('completed', onQueueJobComplete)
|
|
60
|
+
queue.removeListener('error', onQueueError)
|
|
61
|
+
queue.removeListener('idle', onQueueIdle)
|
|
62
|
+
signal.removeEventListener('abort', onSignalAbort)
|
|
63
|
+
cleanUp.removeEventListener('cleanup', onCleanUp)
|
|
64
|
+
}
|
|
65
|
+
}
|
package/src/query-self.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { setMaxListeners } from '@libp2p/interface/events'
|
|
2
|
-
import { logger, type Logger } from '@libp2p/logger'
|
|
3
2
|
import { anySignal } from 'any-signal'
|
|
4
3
|
import length from 'it-length'
|
|
5
4
|
import { pipe } from 'it-pipe'
|
|
@@ -9,6 +8,7 @@ import { pEvent } from 'p-event'
|
|
|
9
8
|
import { QUERY_SELF_INTERVAL, QUERY_SELF_TIMEOUT, K, QUERY_SELF_INITIAL_INTERVAL } from './constants.js'
|
|
10
9
|
import type { PeerRouting } from './peer-routing/index.js'
|
|
11
10
|
import type { RoutingTable } from './routing-table/index.js'
|
|
11
|
+
import type { ComponentLogger, Logger } from '@libp2p/interface'
|
|
12
12
|
import type { PeerId } from '@libp2p/interface/peer-id'
|
|
13
13
|
import type { Startable } from '@libp2p/interface/startable'
|
|
14
14
|
import type { DeferredPromise } from 'p-defer'
|
|
@@ -26,6 +26,7 @@ export interface QuerySelfInit {
|
|
|
26
26
|
|
|
27
27
|
export interface QuerySelfComponents {
|
|
28
28
|
peerId: PeerId
|
|
29
|
+
logger: ComponentLogger
|
|
29
30
|
}
|
|
30
31
|
|
|
31
32
|
/**
|
|
@@ -33,7 +34,7 @@ export interface QuerySelfComponents {
|
|
|
33
34
|
*/
|
|
34
35
|
export class QuerySelf implements Startable {
|
|
35
36
|
private readonly log: Logger
|
|
36
|
-
private readonly
|
|
37
|
+
private readonly peerId: PeerId
|
|
37
38
|
private readonly peerRouting: PeerRouting
|
|
38
39
|
private readonly routingTable: RoutingTable
|
|
39
40
|
private readonly count: number
|
|
@@ -49,8 +50,8 @@ export class QuerySelf implements Startable {
|
|
|
49
50
|
constructor (components: QuerySelfComponents, init: QuerySelfInit) {
|
|
50
51
|
const { peerRouting, lan, count, interval, queryTimeout, routingTable } = init
|
|
51
52
|
|
|
52
|
-
this.
|
|
53
|
-
this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:query-self`)
|
|
53
|
+
this.peerId = components.peerId
|
|
54
|
+
this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:query-self`)
|
|
54
55
|
this.started = false
|
|
55
56
|
this.peerRouting = peerRouting
|
|
56
57
|
this.routingTable = routingTable
|
|
@@ -125,7 +126,7 @@ export class QuerySelf implements Startable {
|
|
|
125
126
|
const start = Date.now()
|
|
126
127
|
|
|
127
128
|
const found = await pipe(
|
|
128
|
-
this.peerRouting.getClosestPeers(this.
|
|
129
|
+
this.peerRouting.getClosestPeers(this.peerId.toBytes(), {
|
|
129
130
|
signal,
|
|
130
131
|
isSelfQuery: true
|
|
131
132
|
}),
|