@libp2p/kad-dht 14.0.2 → 14.1.0-665769021
Sign up to get free protection for your applications and to get access to all the features.
- package/README.md +36 -0
- package/dist/index.min.js +2 -2
- package/dist/src/constants.d.ts +4 -3
- package/dist/src/constants.d.ts.map +1 -1
- package/dist/src/constants.js +8 -3
- package/dist/src/constants.js.map +1 -1
- package/dist/src/content-fetching/index.d.ts +1 -0
- package/dist/src/content-fetching/index.d.ts.map +1 -1
- package/dist/src/content-fetching/index.js +5 -3
- package/dist/src/content-fetching/index.js.map +1 -1
- package/dist/src/index.d.ts +107 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +36 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/kad-dht.d.ts +14 -1
- package/dist/src/kad-dht.d.ts.map +1 -1
- package/dist/src/kad-dht.js +76 -27
- package/dist/src/kad-dht.js.map +1 -1
- package/dist/src/network.d.ts +1 -0
- package/dist/src/network.d.ts.map +1 -1
- package/dist/src/network.js +4 -5
- package/dist/src/network.js.map +1 -1
- package/dist/src/peer-routing/index.d.ts.map +1 -1
- package/dist/src/peer-routing/index.js +9 -10
- package/dist/src/peer-routing/index.js.map +1 -1
- package/dist/src/providers.d.ts +21 -50
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +51 -192
- package/dist/src/providers.js.map +1 -1
- package/dist/src/query/manager.d.ts +1 -1
- package/dist/src/query/manager.d.ts.map +1 -1
- package/dist/src/query/manager.js +8 -19
- package/dist/src/query/manager.js.map +1 -1
- package/dist/src/query-self.d.ts +3 -1
- package/dist/src/query-self.d.ts.map +1 -1
- package/dist/src/query-self.js +17 -16
- package/dist/src/query-self.js.map +1 -1
- package/dist/src/reprovider.d.ts +63 -0
- package/dist/src/reprovider.d.ts.map +1 -0
- package/dist/src/reprovider.js +161 -0
- package/dist/src/reprovider.js.map +1 -0
- package/dist/src/routing-table/closest-peers.d.ts +1 -0
- package/dist/src/routing-table/closest-peers.d.ts.map +1 -1
- package/dist/src/routing-table/closest-peers.js +7 -0
- package/dist/src/routing-table/closest-peers.js.map +1 -1
- package/dist/src/routing-table/index.d.ts +1 -0
- package/dist/src/routing-table/index.d.ts.map +1 -1
- package/dist/src/routing-table/index.js +14 -11
- package/dist/src/routing-table/index.js.map +1 -1
- package/dist/src/rpc/handlers/get-value.d.ts +1 -0
- package/dist/src/rpc/handlers/get-value.d.ts.map +1 -1
- package/dist/src/rpc/handlers/get-value.js +3 -1
- package/dist/src/rpc/handlers/get-value.js.map +1 -1
- package/dist/src/rpc/handlers/put-value.d.ts +1 -0
- package/dist/src/rpc/handlers/put-value.d.ts.map +1 -1
- package/dist/src/rpc/handlers/put-value.js +3 -1
- package/dist/src/rpc/handlers/put-value.js.map +1 -1
- package/dist/src/rpc/index.d.ts +1 -0
- package/dist/src/rpc/index.d.ts.map +1 -1
- package/dist/src/rpc/index.js +9 -10
- package/dist/src/rpc/index.js.map +1 -1
- package/dist/src/utils.d.ts +20 -1
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +87 -4
- package/dist/src/utils.js.map +1 -1
- package/package.json +12 -14
- package/src/constants.ts +11 -5
- package/src/content-fetching/index.ts +5 -3
- package/src/index.ts +116 -2
- package/src/kad-dht.ts +94 -41
- package/src/network.ts +5 -5
- package/src/peer-routing/index.ts +9 -11
- package/src/providers.ts +57 -244
- package/src/query/manager.ts +12 -28
- package/src/query-self.ts +20 -17
- package/src/reprovider.ts +226 -0
- package/src/routing-table/closest-peers.ts +9 -0
- package/src/routing-table/index.ts +16 -11
- package/src/rpc/handlers/get-value.ts +3 -1
- package/src/rpc/handlers/put-value.ts +3 -1
- package/src/rpc/index.ts +10 -10
- package/src/utils.ts +102 -4
- package/dist/typedoc-urls.json +0 -56
@@ -0,0 +1,226 @@
|
|
1
|
+
import { TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
|
2
|
+
import { AdaptiveTimeout } from '@libp2p/utils/adaptive-timeout'
|
3
|
+
import { Queue } from '@libp2p/utils/queue'
|
4
|
+
import drain from 'it-drain'
|
5
|
+
import { PROVIDERS_VALIDITY, REPROVIDE_CONCURRENCY, REPROVIDE_INTERVAL, REPROVIDE_MAX_QUEUE_SIZE, REPROVIDE_THRESHOLD } from './constants.js'
|
6
|
+
import { parseProviderKey, readProviderTime, timeOperationMethod } from './utils.js'
|
7
|
+
import type { ContentRouting } from './content-routing/index.js'
|
8
|
+
import type { OperationMetrics } from './kad-dht.js'
|
9
|
+
import type { AbortOptions, ComponentLogger, Logger, Metrics, PeerId } from '@libp2p/interface'
|
10
|
+
import type { AddressManager } from '@libp2p/interface-internal'
|
11
|
+
import type { AdaptiveTimeoutInit } from '@libp2p/utils/adaptive-timeout'
|
12
|
+
import type { Datastore } from 'interface-datastore'
|
13
|
+
import type { Mortice } from 'mortice'
|
14
|
+
import type { CID } from 'multiformats/cid'
|
15
|
+
|
16
|
+
export interface ReproviderComponents {
|
17
|
+
peerId: PeerId
|
18
|
+
datastore: Datastore
|
19
|
+
logger: ComponentLogger
|
20
|
+
addressManager: AddressManager
|
21
|
+
metrics?: Metrics
|
22
|
+
}
|
23
|
+
|
24
|
+
export interface ReproviderInit {
|
25
|
+
logPrefix: string
|
26
|
+
metricsPrefix: string
|
27
|
+
datastorePrefix: string
|
28
|
+
contentRouting: ContentRouting
|
29
|
+
lock: Mortice
|
30
|
+
operationMetrics: OperationMetrics
|
31
|
+
concurrency?: number
|
32
|
+
maxQueueSize?: number
|
33
|
+
threshold?: number
|
34
|
+
validity?: number
|
35
|
+
interval?: number
|
36
|
+
timeout?: Omit<AdaptiveTimeoutInit, 'metricsName' | 'metrics'>
|
37
|
+
}
|
38
|
+
|
39
|
+
interface QueueJobOptions extends AbortOptions {
|
40
|
+
cid: CID
|
41
|
+
}
|
42
|
+
|
43
|
+
interface ReprovideEvents {
|
44
|
+
'reprovide:start': CustomEvent
|
45
|
+
'reprovide:end': CustomEvent
|
46
|
+
}
|
47
|
+
|
48
|
+
export class Reprovider extends TypedEventEmitter<ReprovideEvents> {
|
49
|
+
private readonly log: Logger
|
50
|
+
private readonly reprovideQueue: Queue<void, QueueJobOptions>
|
51
|
+
private readonly maxQueueSize: number
|
52
|
+
private readonly datastore: Datastore
|
53
|
+
private timeout?: ReturnType<typeof setTimeout>
|
54
|
+
private readonly reprovideTimeout: AdaptiveTimeout
|
55
|
+
private running: boolean
|
56
|
+
private shutdownController?: AbortController
|
57
|
+
private readonly reprovideThreshold: number
|
58
|
+
private readonly contentRouting: ContentRouting
|
59
|
+
private readonly datastorePrefix: string
|
60
|
+
private readonly addressManager: AddressManager
|
61
|
+
private readonly validity: number
|
62
|
+
private readonly interval: number
|
63
|
+
private readonly lock: Mortice
|
64
|
+
private readonly peerId: PeerId
|
65
|
+
|
66
|
+
constructor (components: ReproviderComponents, init: ReproviderInit) {
|
67
|
+
super()
|
68
|
+
|
69
|
+
this.log = components.logger.forComponent(`${init.logPrefix}:reprovider`)
|
70
|
+
this.peerId = components.peerId
|
71
|
+
this.reprovideQueue = new Queue({
|
72
|
+
concurrency: init.concurrency ?? REPROVIDE_CONCURRENCY,
|
73
|
+
metrics: components.metrics,
|
74
|
+
metricName: `${init.metricsPrefix}_reprovide_queue`
|
75
|
+
})
|
76
|
+
this.reprovideTimeout = new AdaptiveTimeout({
|
77
|
+
...(init.timeout ?? {}),
|
78
|
+
metrics: components.metrics,
|
79
|
+
metricName: `${init.metricsPrefix}_reprovide_timeout_milliseconds`
|
80
|
+
})
|
81
|
+
this.datastore = components.datastore
|
82
|
+
this.addressManager = components.addressManager
|
83
|
+
this.datastorePrefix = `/${init.datastorePrefix}/provider`
|
84
|
+
this.reprovideThreshold = init.threshold ?? REPROVIDE_THRESHOLD
|
85
|
+
this.maxQueueSize = init.maxQueueSize ?? REPROVIDE_MAX_QUEUE_SIZE
|
86
|
+
this.validity = init.validity ?? PROVIDERS_VALIDITY
|
87
|
+
this.interval = init.interval ?? REPROVIDE_INTERVAL
|
88
|
+
this.contentRouting = init.contentRouting
|
89
|
+
this.lock = init.lock
|
90
|
+
this.running = false
|
91
|
+
|
92
|
+
this.reprovide = timeOperationMethod(this.reprovide.bind(this), init.operationMetrics, 'PROVIDE')
|
93
|
+
}
|
94
|
+
|
95
|
+
start (): void {
|
96
|
+
if (this.running) {
|
97
|
+
return
|
98
|
+
}
|
99
|
+
|
100
|
+
this.running = true
|
101
|
+
|
102
|
+
this.shutdownController = new AbortController()
|
103
|
+
setMaxListeners(Infinity, this.shutdownController.signal)
|
104
|
+
|
105
|
+
this.timeout = setTimeout(() => {
|
106
|
+
this.cleanUp().catch(err => {
|
107
|
+
this.log.error('error running reprovide/cleanup - %e', err)
|
108
|
+
})
|
109
|
+
}, this.interval)
|
110
|
+
}
|
111
|
+
|
112
|
+
stop (): void {
|
113
|
+
this.running = false
|
114
|
+
this.reprovideQueue.clear()
|
115
|
+
clearTimeout(this.timeout)
|
116
|
+
this.shutdownController?.abort()
|
117
|
+
}
|
118
|
+
|
119
|
+
/**
|
120
|
+
* Check all provider records. Delete them if they have expired, reprovide
|
121
|
+
* them if the provider is us and the expiry is within the reprovide window.
|
122
|
+
*/
|
123
|
+
private async cleanUp (): Promise<void> {
|
124
|
+
const release = await this.lock.writeLock()
|
125
|
+
|
126
|
+
try {
|
127
|
+
this.safeDispatchEvent('reprovide:start')
|
128
|
+
|
129
|
+
// Get all provider entries from the datastore
|
130
|
+
for await (const entry of this.datastore.query({
|
131
|
+
prefix: this.datastorePrefix
|
132
|
+
})) {
|
133
|
+
try {
|
134
|
+
// Add a delete to the batch for each expired entry
|
135
|
+
const { cid, peerId } = parseProviderKey(entry.key)
|
136
|
+
const created = readProviderTime(entry.value).getTime()
|
137
|
+
const expires = created + this.validity
|
138
|
+
const now = Date.now()
|
139
|
+
const expired = now > expires
|
140
|
+
|
141
|
+
this.log.trace('comparing: %d < %d = %s %s', created, now - this.validity, expired, expired ? '(expired)' : '')
|
142
|
+
|
143
|
+
// delete the record if it has expired
|
144
|
+
if (expired) {
|
145
|
+
await this.datastore.delete(entry.key)
|
146
|
+
}
|
147
|
+
|
148
|
+
// if the provider is us and we are within the reprovide threshold,
|
149
|
+
// reprovide the record
|
150
|
+
if (this.peerId.equals(peerId) && (now - expires) < this.reprovideThreshold) {
|
151
|
+
this.queueReprovide(cid)
|
152
|
+
.catch(err => {
|
153
|
+
this.log.error('could not reprovide %c - %e', cid, err)
|
154
|
+
})
|
155
|
+
}
|
156
|
+
} catch (err: any) {
|
157
|
+
this.log.error('error processing datastore key %s - %e', entry.key, err.message)
|
158
|
+
}
|
159
|
+
}
|
160
|
+
|
161
|
+
this.log('reprovide/cleanup successful')
|
162
|
+
} finally {
|
163
|
+
release()
|
164
|
+
this.safeDispatchEvent('reprovide:end')
|
165
|
+
|
166
|
+
if (this.running) {
|
167
|
+
this.timeout = setTimeout(() => {
|
168
|
+
this.cleanUp().catch(err => {
|
169
|
+
this.log.error('error running re-provide - %e', err)
|
170
|
+
})
|
171
|
+
}, this.interval)
|
172
|
+
}
|
173
|
+
}
|
174
|
+
}
|
175
|
+
|
176
|
+
private async queueReprovide (cid: CID): Promise<void> {
|
177
|
+
if (!this.running) {
|
178
|
+
return
|
179
|
+
}
|
180
|
+
|
181
|
+
this.log.trace('waiting for queue capacity before adding %c to re-provide queue', cid)
|
182
|
+
await this.reprovideQueue.onSizeLessThan(this.maxQueueSize)
|
183
|
+
|
184
|
+
const existingJob = this.reprovideQueue.queue.find(job => job.options.cid.equals(cid))
|
185
|
+
|
186
|
+
if (existingJob != null) {
|
187
|
+
this.log.trace('not adding %c to re-provide queue - already in queue', cid)
|
188
|
+
return existingJob.join()
|
189
|
+
}
|
190
|
+
|
191
|
+
this.log.trace('adding %c to re-provide queue', cid)
|
192
|
+
|
193
|
+
this.reprovideQueue.add(async (options): Promise<void> => {
|
194
|
+
options.signal?.throwIfAborted()
|
195
|
+
|
196
|
+
if (!this.running) {
|
197
|
+
return
|
198
|
+
}
|
199
|
+
|
200
|
+
this.log.trace('re-providing %c', cid)
|
201
|
+
|
202
|
+
// use adaptive timeout
|
203
|
+
const signal = this.reprovideTimeout.getTimeoutSignal(options)
|
204
|
+
|
205
|
+
try {
|
206
|
+
// reprovide
|
207
|
+
await this.reprovide(options.cid, options)
|
208
|
+
} finally {
|
209
|
+
this.reprovideTimeout.cleanUp(signal)
|
210
|
+
}
|
211
|
+
|
212
|
+
this.log.trace('re-provided %c', cid)
|
213
|
+
}, {
|
214
|
+
signal: this.shutdownController?.signal,
|
215
|
+
cid
|
216
|
+
})
|
217
|
+
.catch(err => {
|
218
|
+
this.log.error('could not re-provide key %c - %e', cid, err)
|
219
|
+
})
|
220
|
+
}
|
221
|
+
|
222
|
+
private async reprovide (cid: CID, options?: AbortOptions): Promise<void> {
|
223
|
+
// reprovide
|
224
|
+
await drain(this.contentRouting.provide(cid, this.addressManager.getAddresses(), options))
|
225
|
+
}
|
226
|
+
}
|
@@ -42,6 +42,7 @@ export class ClosestPeers implements Startable {
|
|
42
42
|
private readonly closeTagName: string
|
43
43
|
private readonly closeTagValue: number
|
44
44
|
private readonly log: Logger
|
45
|
+
private running: boolean
|
45
46
|
|
46
47
|
constructor (components: ClosestPeersComponents, init: ClosestPeersInit) {
|
47
48
|
this.components = components
|
@@ -54,9 +55,16 @@ export class ClosestPeers implements Startable {
|
|
54
55
|
|
55
56
|
this.closestPeers = new PeerSet()
|
56
57
|
this.onPeerPing = this.onPeerPing.bind(this)
|
58
|
+
this.running = false
|
57
59
|
}
|
58
60
|
|
59
61
|
async start (): Promise<void> {
|
62
|
+
if (this.running) {
|
63
|
+
return
|
64
|
+
}
|
65
|
+
|
66
|
+
this.running = true
|
67
|
+
|
60
68
|
const targetKadId = await convertPeerId(this.components.peerId)
|
61
69
|
this.newPeers = new PeerDistanceList(targetKadId, this.peerSetSize)
|
62
70
|
this.routingTable.addEventListener('peer:ping', this.onPeerPing)
|
@@ -70,6 +78,7 @@ export class ClosestPeers implements Startable {
|
|
70
78
|
}
|
71
79
|
|
72
80
|
stop (): void {
|
81
|
+
this.running = false
|
73
82
|
this.routingTable.removeEventListener('peer:ping', this.onPeerPing)
|
74
83
|
clearTimeout(this.timeout)
|
75
84
|
}
|
@@ -30,6 +30,7 @@ export const POPULATE_FROM_DATASTORE_LIMIT = 1000
|
|
30
30
|
|
31
31
|
export interface RoutingTableInit {
|
32
32
|
logPrefix: string
|
33
|
+
metricsPrefix: string
|
33
34
|
protocol: string
|
34
35
|
prefixLength?: number
|
35
36
|
splitThreshold?: number
|
@@ -116,26 +117,26 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
|
|
116
117
|
|
117
118
|
this.pingOldContactQueue = new PeerQueue({
|
118
119
|
concurrency: init.pingOldContactConcurrency ?? PING_OLD_CONTACT_CONCURRENCY,
|
119
|
-
metricName: `${init.
|
120
|
+
metricName: `${init.metricsPrefix}_ping_old_contact_queue`,
|
120
121
|
metrics: this.components.metrics,
|
121
122
|
maxSize: init.pingOldContactMaxQueueSize ?? PING_OLD_CONTACT_MAX_QUEUE_SIZE
|
122
123
|
})
|
123
124
|
this.pingOldContactTimeout = new AdaptiveTimeout({
|
124
125
|
...(init.pingOldContactTimeout ?? {}),
|
125
126
|
metrics: this.components.metrics,
|
126
|
-
metricName: `${init.
|
127
|
+
metricName: `${init.metricsPrefix}_routing_table_ping_old_contact_time_milliseconds`
|
127
128
|
})
|
128
129
|
|
129
130
|
this.pingNewContactQueue = new PeerQueue({
|
130
131
|
concurrency: init.pingNewContactConcurrency ?? PING_NEW_CONTACT_CONCURRENCY,
|
131
|
-
metricName: `${init.
|
132
|
+
metricName: `${init.metricsPrefix}_ping_new_contact_queue`,
|
132
133
|
metrics: this.components.metrics,
|
133
134
|
maxSize: init.pingNewContactMaxQueueSize ?? PING_NEW_CONTACT_MAX_QUEUE_SIZE
|
134
135
|
})
|
135
136
|
this.pingNewContactTimeout = new AdaptiveTimeout({
|
136
137
|
...(init.pingNewContactTimeout ?? {}),
|
137
138
|
metrics: this.components.metrics,
|
138
|
-
metricName: `${init.
|
139
|
+
metricName: `${init.metricsPrefix}_routing_table_ping_new_contact_time_milliseconds`
|
139
140
|
})
|
140
141
|
|
141
142
|
this.kb = new KBucket({
|
@@ -161,13 +162,13 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
|
|
161
162
|
|
162
163
|
if (this.components.metrics != null) {
|
163
164
|
this.metrics = {
|
164
|
-
routingTableSize: this.components.metrics.registerMetric(`${init.
|
165
|
-
routingTableKadBucketTotal: this.components.metrics.registerMetric(`${init.
|
166
|
-
routingTableKadBucketAverageOccupancy: this.components.metrics.registerMetric(`${init.
|
167
|
-
routingTableKadBucketMinOccupancy: this.components.metrics.registerMetric(`${init.
|
168
|
-
routingTableKadBucketMaxOccupancy: this.components.metrics.registerMetric(`${init.
|
169
|
-
routingTableKadBucketMaxDepth: this.components.metrics.registerMetric(`${init.
|
170
|
-
kadBucketEvents: this.components.metrics.registerCounterGroup(`${init.
|
165
|
+
routingTableSize: this.components.metrics.registerMetric(`${init.metricsPrefix}_routing_table_size`),
|
166
|
+
routingTableKadBucketTotal: this.components.metrics.registerMetric(`${init.metricsPrefix}_routing_table_kad_bucket_total`),
|
167
|
+
routingTableKadBucketAverageOccupancy: this.components.metrics.registerMetric(`${init.metricsPrefix}_routing_table_kad_bucket_average_occupancy`),
|
168
|
+
routingTableKadBucketMinOccupancy: this.components.metrics.registerMetric(`${init.metricsPrefix}_routing_table_kad_bucket_min_occupancy`),
|
169
|
+
routingTableKadBucketMaxOccupancy: this.components.metrics.registerMetric(`${init.metricsPrefix}_routing_table_kad_bucket_max_occupancy`),
|
170
|
+
routingTableKadBucketMaxDepth: this.components.metrics.registerMetric(`${init.metricsPrefix}_routing_table_kad_bucket_max_depth`),
|
171
|
+
kadBucketEvents: this.components.metrics.registerCounterGroup(`${init.metricsPrefix}_kad_bucket_events_total`)
|
171
172
|
}
|
172
173
|
}
|
173
174
|
}
|
@@ -177,6 +178,10 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
|
|
177
178
|
}
|
178
179
|
|
179
180
|
async start (): Promise<void> {
|
181
|
+
if (this.running) {
|
182
|
+
return
|
183
|
+
}
|
184
|
+
|
180
185
|
this.running = true
|
181
186
|
|
182
187
|
await start(this.closestPeerTagger)
|
@@ -28,9 +28,11 @@ export class GetValueHandler implements DHTMessageHandler {
|
|
28
28
|
private readonly datastore: Datastore
|
29
29
|
private readonly peerRouting: PeerRouting
|
30
30
|
private readonly log: Logger
|
31
|
+
private readonly datastorePrefix: string
|
31
32
|
|
32
33
|
constructor (components: GetValueHandlerComponents, init: GetValueHandlerInit) {
|
33
34
|
this.log = components.logger.forComponent(`${init.logPrefix}:rpc:handlers:get-value`)
|
35
|
+
this.datastorePrefix = `/${init.logPrefix.replaceAll(':', '/')}/record`
|
34
36
|
this.peerStore = components.peerStore
|
35
37
|
this.datastore = components.datastore
|
36
38
|
this.peerRouting = init.peerRouting
|
@@ -108,7 +110,7 @@ export class GetValueHandler implements DHTMessageHandler {
|
|
108
110
|
*/
|
109
111
|
async _checkLocalDatastore (key: Uint8Array): Promise<Libp2pRecord | undefined> {
|
110
112
|
this.log('checkLocalDatastore looking for %b', key)
|
111
|
-
const dsKey = bufferToRecordKey(key)
|
113
|
+
const dsKey = bufferToRecordKey(this.datastorePrefix, key)
|
112
114
|
|
113
115
|
// Fetch value from ds
|
114
116
|
let rawRecord
|
@@ -22,12 +22,14 @@ export class PutValueHandler implements DHTMessageHandler {
|
|
22
22
|
private readonly components: PutValueHandlerComponents
|
23
23
|
private readonly validators: Validators
|
24
24
|
private readonly log: Logger
|
25
|
+
private readonly datastorePrefix: string
|
25
26
|
|
26
27
|
constructor (components: PutValueHandlerComponents, init: PutValueHandlerInit) {
|
27
28
|
const { validators } = init
|
28
29
|
|
29
30
|
this.components = components
|
30
31
|
this.log = components.logger.forComponent(`${init.logPrefix}:rpc:handlers:put-value`)
|
32
|
+
this.datastorePrefix = `/${init.logPrefix.replaceAll(':', '/')}/record`
|
31
33
|
this.validators = validators
|
32
34
|
}
|
33
35
|
|
@@ -48,7 +50,7 @@ export class PutValueHandler implements DHTMessageHandler {
|
|
48
50
|
await verifyRecord(this.validators, deserializedRecord)
|
49
51
|
|
50
52
|
deserializedRecord.timeReceived = new Date()
|
51
|
-
const recordKey = bufferToRecordKey(deserializedRecord.key)
|
53
|
+
const recordKey = bufferToRecordKey(this.datastorePrefix, deserializedRecord.key)
|
52
54
|
await this.components.datastore.put(recordKey, deserializedRecord.serialize().subarray())
|
53
55
|
this.log('put record for %b into datastore under key %k', key, recordKey)
|
54
56
|
} catch (err: any) {
|
package/src/rpc/index.ts
CHANGED
@@ -24,6 +24,7 @@ export interface RPCInit {
|
|
24
24
|
peerRouting: PeerRouting
|
25
25
|
validators: Validators
|
26
26
|
logPrefix: string
|
27
|
+
metricsPrefix: string
|
27
28
|
peerInfoMapper: PeerInfoMapper
|
28
29
|
}
|
29
30
|
|
@@ -41,21 +42,20 @@ export class RPC {
|
|
41
42
|
}
|
42
43
|
|
43
44
|
constructor (components: RPCComponents, init: RPCInit) {
|
44
|
-
const { providers, peerRouting, validators, logPrefix, peerInfoMapper } = init
|
45
45
|
this.metrics = {
|
46
|
-
operations: components.metrics?.registerCounterGroup(`${
|
47
|
-
errors: components.metrics?.registerCounterGroup(`${
|
46
|
+
operations: components.metrics?.registerCounterGroup(`${init.metricsPrefix}_inbound_rpc_requests_total`),
|
47
|
+
errors: components.metrics?.registerCounterGroup(`${init.metricsPrefix}_inbound_rpc_errors_total`)
|
48
48
|
}
|
49
49
|
|
50
|
-
this.log = components.logger.forComponent(`${logPrefix}:rpc`)
|
50
|
+
this.log = components.logger.forComponent(`${init.logPrefix}:rpc`)
|
51
51
|
this.routingTable = init.routingTable
|
52
52
|
this.handlers = {
|
53
|
-
[MessageType.GET_VALUE.toString()]: new GetValueHandler(components,
|
54
|
-
[MessageType.PUT_VALUE.toString()]: new PutValueHandler(components,
|
55
|
-
[MessageType.FIND_NODE.toString()]: new FindNodeHandler(components,
|
56
|
-
[MessageType.ADD_PROVIDER.toString()]: new AddProviderHandler(components,
|
57
|
-
[MessageType.GET_PROVIDERS.toString()]: new GetProvidersHandler(components,
|
58
|
-
[MessageType.PING.toString()]: new PingHandler(components,
|
53
|
+
[MessageType.GET_VALUE.toString()]: new GetValueHandler(components, init),
|
54
|
+
[MessageType.PUT_VALUE.toString()]: new PutValueHandler(components, init),
|
55
|
+
[MessageType.FIND_NODE.toString()]: new FindNodeHandler(components, init),
|
56
|
+
[MessageType.ADD_PROVIDER.toString()]: new AddProviderHandler(components, init),
|
57
|
+
[MessageType.GET_PROVIDERS.toString()]: new GetProvidersHandler(components, init),
|
58
|
+
[MessageType.PING.toString()]: new PingHandler(components, init)
|
59
59
|
}
|
60
60
|
}
|
61
61
|
|
package/src/utils.ts
CHANGED
@@ -1,13 +1,16 @@
|
|
1
|
-
import { peerIdFromMultihash } from '@libp2p/peer-id'
|
1
|
+
import { peerIdFromMultihash, peerIdFromString } from '@libp2p/peer-id'
|
2
2
|
import { Libp2pRecord } from '@libp2p/record'
|
3
3
|
import { isPrivateIp } from '@libp2p/utils/private-ip'
|
4
4
|
import { Key } from 'interface-datastore/key'
|
5
|
+
import { CID } from 'multiformats/cid'
|
6
|
+
import * as raw from 'multiformats/codecs/raw'
|
5
7
|
import * as Digest from 'multiformats/hashes/digest'
|
6
8
|
import { sha256 } from 'multiformats/hashes/sha2'
|
9
|
+
import * as varint from 'uint8-varint'
|
7
10
|
import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
|
8
11
|
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
|
9
12
|
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
|
10
|
-
import {
|
13
|
+
import type { Operation, OperationMetrics } from './kad-dht.js'
|
11
14
|
import type { PeerId, PeerInfo } from '@libp2p/interface'
|
12
15
|
import type { Multiaddr } from '@multiformats/multiaddr'
|
13
16
|
|
@@ -110,8 +113,8 @@ export function bufferToKey (buf: Uint8Array): Key {
|
|
110
113
|
/**
|
111
114
|
* Convert a Uint8Array to their SHA2-256 hash
|
112
115
|
*/
|
113
|
-
export function bufferToRecordKey (buf: Uint8Array): Key {
|
114
|
-
return new Key(`${
|
116
|
+
export function bufferToRecordKey (prefix: string, buf: Uint8Array): Key {
|
117
|
+
return new Key(`${prefix}/${uint8ArrayToString(buf, 'base32')}`, false)
|
115
118
|
}
|
116
119
|
|
117
120
|
/**
|
@@ -189,3 +192,98 @@ export function multiaddrIsPublic (multiaddr: Multiaddr): boolean {
|
|
189
192
|
|
190
193
|
return false
|
191
194
|
}
|
195
|
+
|
196
|
+
/**
|
197
|
+
* Parse the CID and provider peer id from the key
|
198
|
+
*/
|
199
|
+
export function parseProviderKey (key: Key): { cid: CID, peerId: PeerId } {
|
200
|
+
const parts = key.toString().split('/')
|
201
|
+
const peerIdStr = parts.pop()
|
202
|
+
const cidStr = parts.pop()
|
203
|
+
|
204
|
+
if (peerIdStr == null || cidStr == null) {
|
205
|
+
throw new Error(`incorrectly formatted provider entry key in datastore: ${key.toString()}`)
|
206
|
+
}
|
207
|
+
|
208
|
+
return {
|
209
|
+
cid: CID.createV1(raw.code, Digest.decode(uint8ArrayFromString(cidStr, 'base32'))),
|
210
|
+
peerId: peerIdFromString(peerIdStr)
|
211
|
+
}
|
212
|
+
}
|
213
|
+
|
214
|
+
/**
|
215
|
+
* Encode the given key its matching datastore key
|
216
|
+
*/
|
217
|
+
export function toProviderKey (prefix: string, cid: CID | string, peerId?: PeerId): Key {
|
218
|
+
const cidStr = typeof cid === 'string' ? cid : uint8ArrayToString(cid.multihash.bytes, 'base32')
|
219
|
+
|
220
|
+
const parts = [
|
221
|
+
prefix,
|
222
|
+
cidStr
|
223
|
+
]
|
224
|
+
|
225
|
+
if (peerId != null) {
|
226
|
+
parts.push(peerId.toString())
|
227
|
+
}
|
228
|
+
|
229
|
+
return new Key(parts.join('/'))
|
230
|
+
}
|
231
|
+
|
232
|
+
export function readProviderTime (buf: Uint8Array): Date {
|
233
|
+
return new Date(varint.decode(buf))
|
234
|
+
}
|
235
|
+
|
236
|
+
/**
|
237
|
+
* Wraps the passed generator function with timing metrics
|
238
|
+
*/
|
239
|
+
export function timeOperationGenerator (fn: (...args: any[]) => AsyncGenerator<any>, operationMetrics: OperationMetrics, type: Operation): (...args: any[]) => AsyncGenerator<any> {
|
240
|
+
return async function * (...args: any[]): AsyncGenerator<any> {
|
241
|
+
const stopSuccessTimer = operationMetrics.queryTime?.timer(type)
|
242
|
+
const stopErrorTimer = operationMetrics.errorTime?.timer(type)
|
243
|
+
let errored = false
|
244
|
+
|
245
|
+
try {
|
246
|
+
operationMetrics.queries?.increment({ [type]: true })
|
247
|
+
|
248
|
+
yield * fn(...args)
|
249
|
+
} catch (err) {
|
250
|
+
errored = true
|
251
|
+
stopErrorTimer?.()
|
252
|
+
operationMetrics.errors?.increment({ [type]: true })
|
253
|
+
|
254
|
+
throw err
|
255
|
+
} finally {
|
256
|
+
operationMetrics.queries?.decrement({ [type]: true })
|
257
|
+
|
258
|
+
if (!errored) {
|
259
|
+
stopSuccessTimer?.()
|
260
|
+
}
|
261
|
+
}
|
262
|
+
}
|
263
|
+
}
|
264
|
+
|
265
|
+
export function timeOperationMethod (fn: (...args: any[]) => Promise<any>, operationMetrics: OperationMetrics, type: Operation): (...args: any[]) => Promise<any> {
|
266
|
+
return async function (...args: any[]): Promise<any> {
|
267
|
+
const stopSuccessTimer = operationMetrics?.queryTime?.timer(type)
|
268
|
+
const stopErrorTimer = operationMetrics?.errorTime?.timer(type)
|
269
|
+
let errored = false
|
270
|
+
|
271
|
+
try {
|
272
|
+
operationMetrics.queries?.increment({ [type]: true })
|
273
|
+
|
274
|
+
return await fn(...args)
|
275
|
+
} catch (err) {
|
276
|
+
errored = true
|
277
|
+
stopErrorTimer?.()
|
278
|
+
operationMetrics.errors?.increment({ [type]: true })
|
279
|
+
|
280
|
+
throw err
|
281
|
+
} finally {
|
282
|
+
operationMetrics.queries?.decrement({ [type]: true })
|
283
|
+
|
284
|
+
if (!errored) {
|
285
|
+
stopSuccessTimer?.()
|
286
|
+
}
|
287
|
+
}
|
288
|
+
}
|
289
|
+
}
|
package/dist/typedoc-urls.json
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"codec": "https://libp2p.github.io/js-libp2p/functions/_libp2p_kad_dht.MessageType.codec.html",
|
3
|
-
"EventTypes": "https://libp2p.github.io/js-libp2p/enums/_libp2p_kad_dht.EventTypes.html",
|
4
|
-
".:EventTypes": "https://libp2p.github.io/js-libp2p/enums/_libp2p_kad_dht.EventTypes.html",
|
5
|
-
"MessageType": "https://libp2p.github.io/js-libp2p/enums/_libp2p_kad_dht.MessageType-1.html",
|
6
|
-
"AddPeerEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.AddPeerEvent.html",
|
7
|
-
".:AddPeerEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.AddPeerEvent.html",
|
8
|
-
"DHTRecord": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.DHTRecord.html",
|
9
|
-
".:DHTRecord": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.DHTRecord.html",
|
10
|
-
"DialPeerEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.DialPeerEvent.html",
|
11
|
-
".:DialPeerEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.DialPeerEvent.html",
|
12
|
-
"FinalPeerEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.FinalPeerEvent.html",
|
13
|
-
".:FinalPeerEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.FinalPeerEvent.html",
|
14
|
-
"KadDHT": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.KadDHT.html",
|
15
|
-
".:KadDHT": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.KadDHT.html",
|
16
|
-
"KadDHTComponents": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.KadDHTComponents.html",
|
17
|
-
".:KadDHTComponents": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.KadDHTComponents.html",
|
18
|
-
"KadDHTInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.KadDHTInit.html",
|
19
|
-
".:KadDHTInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.KadDHTInit.html",
|
20
|
-
"PeerInfoMapper": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.PeerInfoMapper.html",
|
21
|
-
".:PeerInfoMapper": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.PeerInfoMapper.html",
|
22
|
-
"PeerResponseEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.PeerResponseEvent.html",
|
23
|
-
".:PeerResponseEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.PeerResponseEvent.html",
|
24
|
-
"ProviderEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.ProviderEvent.html",
|
25
|
-
".:ProviderEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.ProviderEvent.html",
|
26
|
-
"ProvidersInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.ProvidersInit.html",
|
27
|
-
"QueryErrorEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.QueryErrorEvent.html",
|
28
|
-
".:QueryErrorEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.QueryErrorEvent.html",
|
29
|
-
"RoutingTable": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.RoutingTable.html",
|
30
|
-
".:RoutingTable": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.RoutingTable.html",
|
31
|
-
"SelectFn": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.SelectFn.html",
|
32
|
-
".:SelectFn": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.SelectFn.html",
|
33
|
-
"SendQueryEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.SendQueryEvent.html",
|
34
|
-
".:SendQueryEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.SendQueryEvent.html",
|
35
|
-
"SingleKadDHT": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.SingleKadDHT.html",
|
36
|
-
".:SingleKadDHT": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.SingleKadDHT.html",
|
37
|
-
"ValidateFn": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.ValidateFn.html",
|
38
|
-
".:ValidateFn": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.ValidateFn.html",
|
39
|
-
"ValueEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.ValueEvent.html",
|
40
|
-
".:ValueEvent": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_kad_dht.ValueEvent.html",
|
41
|
-
"DHTProgressEvents": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.DHTProgressEvents.html",
|
42
|
-
".:DHTProgressEvents": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.DHTProgressEvents.html",
|
43
|
-
"MessageName": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.MessageName.html",
|
44
|
-
".:MessageName": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.MessageName.html",
|
45
|
-
"QueryEvent": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.QueryEvent.html",
|
46
|
-
".:QueryEvent": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.QueryEvent.html",
|
47
|
-
"Selectors": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.Selectors.html",
|
48
|
-
".:Selectors": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.Selectors.html",
|
49
|
-
"Validators": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.Validators.html",
|
50
|
-
".:Validators": "https://libp2p.github.io/js-libp2p/types/_libp2p_kad_dht.Validators.html",
|
51
|
-
"kadDHT": "https://libp2p.github.io/js-libp2p/functions/_libp2p_kad_dht.kadDHT-1.html",
|
52
|
-
".:kadDHT": "https://libp2p.github.io/js-libp2p/functions/_libp2p_kad_dht.kadDHT-1.html",
|
53
|
-
"passthroughMapper": "https://libp2p.github.io/js-libp2p/functions/_libp2p_kad_dht.passthroughMapper.html",
|
54
|
-
"removePrivateAddressesMapper": "https://libp2p.github.io/js-libp2p/functions/_libp2p_kad_dht.removePrivateAddressesMapper.html",
|
55
|
-
"removePublicAddressesMapper": "https://libp2p.github.io/js-libp2p/functions/_libp2p_kad_dht.removePublicAddressesMapper.html"
|
56
|
-
}
|