@steve02081504/fount-p2p 0.0.41 → 0.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -0
- package/discovery/lan_interfaces.mjs +20 -3
- package/discovery/nostr/census.mjs +9 -62
- package/discovery/nostr/census_monitor.mjs +403 -0
- package/discovery/nostr/census_verify.mjs +65 -0
- package/discovery/nostr/constants.mjs +17 -0
- package/discovery/nostr/index.mjs +1 -1
- package/discovery/nostr/relays.mjs +1 -7
- package/docs/evfs.md +1 -0
- package/docs/transports.md +4 -0
- package/docs/wire.md +6 -0
- package/federation/part_query/cache.mjs +19 -4
- package/federation/part_query/runtime.mjs +186 -43
- package/files/chunk/fetch.mjs +5 -0
- package/files/chunk/pending.mjs +7 -7
- package/infra/service.mjs +2 -28
- package/mailbox/deliver_or_store.mjs +4 -0
- package/overlay/index.mjs +66 -9
- package/overlay/tunables.json +4 -0
- package/package.json +2 -2
- package/pages/index.html +3 -3
- package/pages/index.mjs +62 -0
- package/reputation/engine.mjs +3 -2
- package/schemas/part_query.mjs +9 -2
- package/transport/offer_answer.mjs +25 -1
- package/transport/tunables.json +2 -1
- package/utils/token_bucket.mjs +25 -0
- package/wire/part/query.mjs +1 -1
- package/pages/app.mjs +0 -172
package/docs/evfs.md
CHANGED
|
@@ -45,6 +45,7 @@ Outer caller timeouts must **not** abort the in-flight work — background fill
|
|
|
45
45
|
- `fetchChunk` accepts the same `fanoutTargets` as `fetchManifest`. With targets it fanouts `fed_chunk_get` **only** to that node set; without targets it falls back to node-scope fanout. Same `username`+chunk hash+mode (`targeted`/`public`) is deduped in-flight.
|
|
46
46
|
- `readPublicFile` / `readManifestPlaintext` forward `options.fanoutTargets` to both the manifest fetch and the chunk fetch, so a cross-node public read can pull profile/avatar content straight from the owner node instead of depending on the node-scope fanout.
|
|
47
47
|
- The public (non-targeted) fanout does **not** block the request window on dialing the whole peer pool: already-linked / group-room-reachable peers are sent immediately, and unreachable peers are dialed in the background and re-sent once linked. (`ensureLinkToNode` no longer gates the send.)
|
|
48
|
+
- **Chunk serving is by-design unauthenticated CAS:** `fed_chunk_get` returns a chunk iff the requester already knows its 64-hex content hash (which is infeasible to enumerate), and `verifiedChunkBytes` re-checks that hash on receipt. There is no per-requester ACL. Confidentiality therefore rests on the manifest being ACL-gated and on encryption: `convergent` / `random` chunks are ciphertext, while `plain` chunks are plaintext — non-public files must not use `plain`.
|
|
48
49
|
|
|
49
50
|
## Non-public (ACL-gated) manifests
|
|
50
51
|
|
package/docs/transports.md
CHANGED
|
@@ -6,6 +6,10 @@ Mesh keep-alive / discovery: [mesh.md](mesh.md). WebRTC glare / handshake: [sign
|
|
|
6
6
|
|
|
7
7
|
Do **not** introduce version fields, constants, or suffixes (`v`, `version`, `FRAME_VERSION`, `:v1`, …). Changing a shape means changing it; no dual-read / backward-compat paths. Exception: npm `package.json` `version` is for package publish only.
|
|
8
8
|
|
|
9
|
+
## Overlay relay authenticity
|
|
10
|
+
|
|
11
|
+
`overlay/index.mjs` multi-hop relay carries an origin signature: `relay()` signs `(path, body)` with the origin key, intermediate hops forward `nodePubKey`/`sig` untouched, and the terminal node verifies `pubKeyHash(nodePubKey) === path[0]` before treating the body as sent by that node. Unsigned or forged relays are dropped. `route_resp` is only accepted when the signed path's last hop equals the discovery target. Overlay `route_req`/`relay` are token-bucket limited by default (`overlay/tunables.json`, 120/min burst 30); `startInfra` may override the gate.
|
|
12
|
+
|
|
9
13
|
## Public contract (shell / L4)
|
|
10
14
|
|
|
11
15
|
This package exposes a **fount network**: talk to `nodeHash` peers with envelopes.
|
package/docs/wire.md
CHANGED
|
@@ -14,6 +14,12 @@ Day-to-day trust rules: [AGENTS.md](../AGENTS.md). Node-scope presets: [infra.md
|
|
|
14
14
|
|
|
15
15
|
Part query runtime lives under `federation/part_query/*`; wire attach only in `wire/part/query.mjs`.
|
|
16
16
|
|
|
17
|
+
## Part query response signing / source attribution
|
|
18
|
+
|
|
19
|
+
- Every `part_query_res` is self-signed by the responding node: `parsePartQueryRes` requires `nodePubKey` (64 hex) + `sig` (128 hex), and the runtime verifies `pubKeyHash(nodePubKey) === fromNodeHash` plus the signature over `(requestId, fromNodeHash, rows)` before accepting. Invalid responses are dropped, so a neighbor cannot forge a result or attribute it to another node.
|
|
20
|
+
- `queryNetwork` returns `{ rows, sources }` (not a bare array), where `sources` is a `Map<rowKey, string[]>` of contributing responder nodeHashes. Cached entries keep the same provenance.
|
|
21
|
+
- `options.isSourceBlocked(nodeHash)` filters rows whose source set is non-empty and entirely blocked. Shells wire this to their source-block UI; local rows have an empty source set and are never filtered here.
|
|
22
|
+
|
|
17
23
|
## Timed collect
|
|
18
24
|
|
|
19
25
|
APIs with `timeoutMs` (e.g. `collectPartInvokeResponses`) must register the wait **first** and must **not** `await` fanout/send on the return path — a stuck `discoverRoute` / `link.send` otherwise defeats the timeout.
|
|
@@ -4,7 +4,7 @@ import partQueryTunables from '../../schemas/part_query.tunables.json' with { ty
|
|
|
4
4
|
import { createLruMap } from '../../utils/lru.mjs'
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* @typedef {{ rows: unknown[], storedAt: number }} PartQueryCacheEntry
|
|
7
|
+
* @typedef {{ rows: unknown[], storedAt: number, sources: Map<string, Set<string>> }} PartQueryCacheEntry
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
/**
|
|
@@ -24,7 +24,8 @@ export function partQueryCacheKey(partpath, kind, query) {
|
|
|
24
24
|
* @param {{ maxKeys?: number, ttlMs?: number, maxHits?: number }} [options] 容量 / TTL / 单键 rows 上限
|
|
25
25
|
* @returns {{
|
|
26
26
|
* get: (partpath: string, kind: string, query: unknown, now?: number) => unknown[] | null
|
|
27
|
-
*
|
|
27
|
+
* getWithSources: (partpath: string, kind: string, query: unknown, now?: number) => { rows: unknown[], sources: Map<string, Set<string>> } | null
|
|
28
|
+
* set: (partpath: string, kind: string, query: unknown, rows: unknown[], now?: number, sources?: Map<string, Set<string>>) => void
|
|
28
29
|
* clear: () => void
|
|
29
30
|
* readonly size: number
|
|
30
31
|
* }} 缓存 API
|
|
@@ -54,6 +55,18 @@ export function createPartQueryCache(options = {}) {
|
|
|
54
55
|
* @returns {unknown[] | null} 未过期 rows
|
|
55
56
|
*/
|
|
56
57
|
get(partpath, kind, query, now = Date.now()) {
|
|
58
|
+
const entry = this.getWithSources(partpath, kind, query, now)
|
|
59
|
+
return entry ? entry.rows : null
|
|
60
|
+
},
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* @param {string} partpath part 路径
|
|
64
|
+
* @param {string} kind 查询标签
|
|
65
|
+
* @param {unknown} query 查询体
|
|
66
|
+
* @param {number} [now=Date.now()] 当前时间
|
|
67
|
+
* @returns {{ rows: unknown[], sources: Map<string, Set<string>> } | null} 未过期 rows 与来源
|
|
68
|
+
*/
|
|
69
|
+
getWithSources(partpath, kind, query, now = Date.now()) {
|
|
57
70
|
const key = partQueryCacheKey(partpath, kind, query)
|
|
58
71
|
if (!key) return null
|
|
59
72
|
const entry = map.get(key)
|
|
@@ -63,7 +76,7 @@ export function createPartQueryCache(options = {}) {
|
|
|
63
76
|
return null
|
|
64
77
|
}
|
|
65
78
|
map.touch(key, entry)
|
|
66
|
-
return entry.rows.slice()
|
|
79
|
+
return { rows: entry.rows.slice(), sources: entry.sources }
|
|
67
80
|
},
|
|
68
81
|
|
|
69
82
|
/**
|
|
@@ -72,9 +85,10 @@ export function createPartQueryCache(options = {}) {
|
|
|
72
85
|
* @param {unknown} query 查询体
|
|
73
86
|
* @param {unknown[]} rows 聚合 rows(空数组不入库)
|
|
74
87
|
* @param {number} [now=Date.now()] 当前时间
|
|
88
|
+
* @param {Map<string, Set<string>>} [sources] rowKey→来源集合
|
|
75
89
|
* @returns {void}
|
|
76
90
|
*/
|
|
77
|
-
set(partpath, kind, query, rows, now = Date.now()) {
|
|
91
|
+
set(partpath, kind, query, rows, now = Date.now(), sources = null) {
|
|
78
92
|
const key = partQueryCacheKey(partpath, kind, query)
|
|
79
93
|
// 空 miss 不缓存:mesh 晚就绪 / 超时早查询不应被长 TTL 负缓存粘住
|
|
80
94
|
if (!key || !Array.isArray(rows) || rows.length === 0) return
|
|
@@ -82,6 +96,7 @@ export function createPartQueryCache(options = {}) {
|
|
|
82
96
|
map.touch(key, {
|
|
83
97
|
rows: rows.slice(0, maxHits),
|
|
84
98
|
storedAt: now,
|
|
99
|
+
sources: sources || new Map(),
|
|
85
100
|
})
|
|
86
101
|
},
|
|
87
102
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
1
2
|
import { randomUUID } from 'node:crypto'
|
|
2
3
|
|
|
4
|
+
import { canonicalStringify } from '../../core/canonical_json.mjs'
|
|
3
5
|
import { compositeKey } from '../../core/composite_key.mjs'
|
|
4
|
-
import {
|
|
6
|
+
import { keyPairFromSeed, pubKeyHash, sign, verify } from '../../crypto/crypto.mjs'
|
|
7
|
+
import { ensureNodeSeed, getNodeHash } from '../../node/identity.mjs'
|
|
5
8
|
import { loadReputation } from '../../node/reputation_store.mjs'
|
|
6
9
|
import { isQuarantinedPure } from '../../reputation/engine.mjs'
|
|
7
10
|
import {
|
|
@@ -23,6 +26,55 @@ import { createPartQueryCache, partQueryCache } from './cache.mjs'
|
|
|
23
26
|
|
|
24
27
|
/** @typedef {import('../../wire/adapter.mjs').WireAdapter} PartQueryWire */
|
|
25
28
|
|
|
29
|
+
const PART_QUERY_DOMAIN = 'fount-part-query'
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* 构造 part_query_res 签名材料:仅覆盖 (requestId, fromNodeHash, rows)。
|
|
33
|
+
* 来源节点借此自证,中间节点无法篡改结果或来源。
|
|
34
|
+
* @param {{ requestId: string, fromNodeHash: string, rows: unknown[] }} base 响应基体
|
|
35
|
+
* @returns {Uint8Array} 待签名字节
|
|
36
|
+
*/
|
|
37
|
+
function partQuerySignBytes(base) {
|
|
38
|
+
return Buffer.from(`${PART_QUERY_DOMAIN}\0${base.requestId}\0${base.fromNodeHash}\0${canonicalStringify(base.rows)}`, 'utf8')
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 默认响应签名:用本机节点身份签响应基体。
|
|
43
|
+
* @param {{ requestId: string, fromNodeHash: string, rows: unknown[] }} base 响应基体
|
|
44
|
+
* @returns {Promise<{ nodePubKey: string, sig: string }>} 公钥与签名
|
|
45
|
+
*/
|
|
46
|
+
async function defaultSignResponse(base) {
|
|
47
|
+
const { publicKey, secretKey } = keyPairFromSeed(Buffer.from(ensureNodeSeed(), 'hex'))
|
|
48
|
+
const signature = await sign(partQuerySignBytes(base), secretKey)
|
|
49
|
+
return { nodePubKey: Buffer.from(publicKey).toString('hex'), sig: Buffer.from(signature).toString('hex') }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* 默认响应验签:nodePubKey 哈希须等于 fromNodeHash,且签名覆盖响应基体。
|
|
54
|
+
* @param {{ requestId: string, fromNodeHash: string, rows: unknown[], nodePubKey: string, sig: string }} response 响应
|
|
55
|
+
* @returns {Promise<boolean>} 是否可信
|
|
56
|
+
*/
|
|
57
|
+
async function defaultVerifyResponse(response) {
|
|
58
|
+
if (pubKeyHash(Buffer.from(response.nodePubKey, 'hex')) !== response.fromNodeHash) return false
|
|
59
|
+
return await verify(Buffer.from(response.sig, 'hex'), partQuerySignBytes(response), Buffer.from(response.nodePubKey, 'hex'))
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* @param {PartQueryDependencies} dependencies 依赖
|
|
64
|
+
* @returns {(base: { requestId: string, fromNodeHash: string, rows: unknown[] }) => Promise<{ nodePubKey: string, sig: string }>} 签名函数
|
|
65
|
+
*/
|
|
66
|
+
function responseSigner(dependencies) {
|
|
67
|
+
return dependencies.signResponse || defaultSignResponse
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* @param {PartQueryDependencies} dependencies 依赖
|
|
72
|
+
* @returns {(response: { requestId: string, fromNodeHash: string, rows: unknown[], nodePubKey: string, sig: string }) => Promise<boolean>} 验签函数
|
|
73
|
+
*/
|
|
74
|
+
function responseVerifier(dependencies) {
|
|
75
|
+
return dependencies.verifyResponse || defaultVerifyResponse
|
|
76
|
+
}
|
|
77
|
+
|
|
26
78
|
/**
|
|
27
79
|
* @typedef {{
|
|
28
80
|
* replicaUsername?: string
|
|
@@ -40,18 +92,24 @@ import { createPartQueryCache, partQueryCache } from './cache.mjs'
|
|
|
40
92
|
* takeDedupe: (key: string) => boolean
|
|
41
93
|
* relayPending: Map<string, RelayPending>
|
|
42
94
|
* originWaits: Map<string, Map<string, import('../../wire/wait.mjs').WireWaiter[]>>
|
|
43
|
-
* originBags: Map<string, { rows: unknown[], maxHits: number, expected: number, received: number, respondedPeers: Set<string>, rowKey?: (row: unknown) => string }>
|
|
95
|
+
* originBags: Map<string, { entries: Array<{ rows: unknown[], sourceNodeHash?: string }>, maxHits: number, expected: number, received: number, respondedPeers: Set<string>, rowKey?: (row: unknown) => string }>
|
|
44
96
|
* cache: ReturnType<typeof createPartQueryCache>
|
|
45
97
|
* handlers: Map<string, QueryInboundHandler>
|
|
46
98
|
* }} PartQueryNodeState
|
|
47
99
|
*/
|
|
48
100
|
|
|
101
|
+
/**
|
|
102
|
+
* @typedef {{ rows: unknown[], sourceNodeHash?: string }} QueryRowEntry
|
|
103
|
+
*/
|
|
104
|
+
|
|
49
105
|
/**
|
|
50
106
|
* @typedef {{
|
|
51
107
|
* selectNeighbors?: (exclude: Set<string>) => Promise<string[]>
|
|
52
108
|
* deliver?: (nodeHash: string, action: string, payload: unknown) => Promise<boolean> | boolean
|
|
53
109
|
* getNodeHash?: () => string
|
|
54
110
|
* now?: () => number
|
|
111
|
+
* signResponse?: (base: { requestId: string, fromNodeHash: string, rows: unknown[] }) => Promise<{ nodePubKey: string, sig: string }>
|
|
112
|
+
* verifyResponse?: (response: { requestId: string, fromNodeHash: string, rows: unknown[], nodePubKey: string, sig: string }) => Promise<boolean>
|
|
55
113
|
* state?: PartQueryNodeState
|
|
56
114
|
* }} PartQueryDependencies
|
|
57
115
|
*/
|
|
@@ -62,7 +120,7 @@ import { createPartQueryCache, partQueryCache } from './cache.mjs'
|
|
|
62
120
|
* wire: PartQueryWire
|
|
63
121
|
* request: PartQueryReq
|
|
64
122
|
* localRows: unknown[]
|
|
65
|
-
*
|
|
123
|
+
* remoteEntries: QueryRowEntry[]
|
|
66
124
|
* expected: number
|
|
67
125
|
* received: number
|
|
68
126
|
* respondedPeers: Set<string>
|
|
@@ -152,6 +210,40 @@ export function mergeQueryRows(lists, maxHits, rowKey) {
|
|
|
152
210
|
return out
|
|
153
211
|
}
|
|
154
212
|
|
|
213
|
+
/**
|
|
214
|
+
* 合并带来源的 rows,并记录每个去重行键的来源节点集合。
|
|
215
|
+
* 本地行 `sourceNodeHash` 省略(不可被屏蔽)。
|
|
216
|
+
* @param {QueryRowEntry[]} entries 多路 rows(含来源)
|
|
217
|
+
* @param {number} maxHits 上限
|
|
218
|
+
* @param {(row: unknown) => string} [rowKey] 去重键
|
|
219
|
+
* @returns {{ rows: unknown[], sources: Map<string, Set<string>> }} 合并结果与 rowKey→来源集合
|
|
220
|
+
*/
|
|
221
|
+
function mergeRowsWithSources(entries, maxHits, rowKey) {
|
|
222
|
+
const rows = []
|
|
223
|
+
const seen = new Set()
|
|
224
|
+
/** @type {Map<string, Set<string>>} */
|
|
225
|
+
const sources = new Map()
|
|
226
|
+
const keyOf = rowKey || (row => {
|
|
227
|
+
try { return JSON.stringify(row) }
|
|
228
|
+
catch { return `\0${rows.length}` }
|
|
229
|
+
})
|
|
230
|
+
for (const entry of entries) {
|
|
231
|
+
for (const row of entry.rows || []) {
|
|
232
|
+
const key = keyOf(row)
|
|
233
|
+
if (!seen.has(key)) {
|
|
234
|
+
seen.add(key)
|
|
235
|
+
rows.push(row)
|
|
236
|
+
sources.set(key, new Set())
|
|
237
|
+
if (entry.sourceNodeHash) sources.get(key).add(entry.sourceNodeHash)
|
|
238
|
+
if (rows.length >= maxHits) return { rows, sources }
|
|
239
|
+
continue
|
|
240
|
+
}
|
|
241
|
+
if (entry.sourceNodeHash) sources.get(key).add(entry.sourceNodeHash)
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return { rows, sources }
|
|
245
|
+
}
|
|
246
|
+
|
|
155
247
|
/**
|
|
156
248
|
* @param {PartQueryNodeState} state 节点状态
|
|
157
249
|
* @param {QueryInboundContext} queryContext 入站上下文
|
|
@@ -206,20 +298,59 @@ async function deliverQuery(nodeHash, action, payload, dependencies) {
|
|
|
206
298
|
* @param {() => string} nodeHashOf 本机 hash
|
|
207
299
|
* @returns {PartQueryRes} 响应载荷
|
|
208
300
|
*/
|
|
209
|
-
|
|
301
|
+
/**
|
|
302
|
+
* @param {PartQueryReq} request 请求
|
|
303
|
+
* @param {unknown[]} rows 行
|
|
304
|
+
* @param {() => string} nodeHashOf 本机 hash
|
|
305
|
+
* @param {PartQueryDependencies} dependencies 依赖
|
|
306
|
+
* @returns {Promise<PartQueryRes>} 已签名的响应
|
|
307
|
+
*/
|
|
308
|
+
async function buildResponse(request, rows, nodeHashOf, dependencies) {
|
|
210
309
|
const capped = clampPartQueryRows(rows, request.budget.maxHits) || []
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
310
|
+
const base = { requestId: request.requestId, fromNodeHash: nodeHashOf(), rows: capped }
|
|
311
|
+
const { nodePubKey, sig } = await responseSigner(dependencies)(base)
|
|
312
|
+
return { ...base, nodePubKey, sig }
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* 按来源屏蔽表过滤 rows:来源集合非空且全部被屏蔽时剔除该行。
|
|
317
|
+
* @param {unknown[]} rows 行
|
|
318
|
+
* @param {Map<string, Set<string>>} sources rowKey→来源集合
|
|
319
|
+
* @param {(row: unknown) => string} [rowKey] 去重键
|
|
320
|
+
* @param {((nodeHash: string) => boolean) | undefined} isSourceBlocked 来源屏蔽谓词
|
|
321
|
+
* @returns {unknown[]} 过滤后的 rows
|
|
322
|
+
*/
|
|
323
|
+
function filterRowsBySource(rows, sources, rowKey, isSourceBlocked) {
|
|
324
|
+
if (typeof isSourceBlocked !== 'function') return rows
|
|
325
|
+
return rows.filter(row => {
|
|
326
|
+
let key
|
|
327
|
+
if (rowKey) key = rowKey(row)
|
|
328
|
+
else {
|
|
329
|
+
try { key = JSON.stringify(row) }
|
|
330
|
+
catch { return true }
|
|
331
|
+
}
|
|
332
|
+
const set = sources.get(key)
|
|
333
|
+
if (!set || set.size === 0) return true
|
|
334
|
+
for (const source of set) if (!isSourceBlocked(source)) return true
|
|
335
|
+
return false
|
|
336
|
+
})
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* @param {Map<string, Set<string>>} sources rowKey→来源集合
|
|
341
|
+
* @returns {Map<string, string[]>} rowKey→来源数组
|
|
342
|
+
*/
|
|
343
|
+
function sourcesToArrays(sources) {
|
|
344
|
+
const out = new Map()
|
|
345
|
+
for (const [key, set] of sources) out.set(key, [...set])
|
|
346
|
+
return out
|
|
216
347
|
}
|
|
217
348
|
|
|
218
349
|
/**
|
|
219
350
|
* @param {RelayPending} pending 中继槽
|
|
220
|
-
* @returns {void}
|
|
351
|
+
* @returns {Promise<void>}
|
|
221
352
|
*/
|
|
222
|
-
function flushRelayPending(pending) {
|
|
353
|
+
async function flushRelayPending(pending) {
|
|
223
354
|
if (pending.flushed) return
|
|
224
355
|
pending.flushed = true
|
|
225
356
|
if (pending.timer) {
|
|
@@ -227,12 +358,15 @@ function flushRelayPending(pending) {
|
|
|
227
358
|
pending.timer = null
|
|
228
359
|
}
|
|
229
360
|
pending.state.relayPending.delete(pending.request.requestId)
|
|
230
|
-
const merged =
|
|
361
|
+
const merged = mergeRowsWithSources(
|
|
362
|
+
[{ rows: pending.localRows }, ...pending.remoteEntries],
|
|
363
|
+
pending.request.budget.maxHits,
|
|
364
|
+
)
|
|
231
365
|
const now = pending.dependencies.now || Date.now
|
|
232
|
-
pending.state.cache.set(pending.request.partpath, pending.request.kind, pending.request.query, merged, now())
|
|
366
|
+
pending.state.cache.set(pending.request.partpath, pending.request.kind, pending.request.query, merged.rows, now(), merged.sources)
|
|
233
367
|
const nodeHashOf = pending.dependencies.getNodeHash || getNodeHash
|
|
234
368
|
try {
|
|
235
|
-
pending.wire.send('part_query_res', buildResponse(pending.request, merged, nodeHashOf), pending.upstreamPeerId)
|
|
369
|
+
pending.wire.send('part_query_res', await buildResponse(pending.request, merged.rows, nodeHashOf, pending.dependencies), pending.upstreamPeerId)
|
|
236
370
|
}
|
|
237
371
|
catch { /* disconnected */ }
|
|
238
372
|
}
|
|
@@ -251,9 +385,9 @@ export async function processIncomingPartQueryRequest(wireContext, wire, request
|
|
|
251
385
|
const now = dependencies.now || Date.now
|
|
252
386
|
const username = wireContext.replicaUsername || ''
|
|
253
387
|
|
|
254
|
-
const cached = state.cache.
|
|
388
|
+
const cached = state.cache.getWithSources(request.partpath, request.kind, request.query, now())
|
|
255
389
|
if (cached) {
|
|
256
|
-
try { wire.send('part_query_res', buildResponse(request, cached, nodeHashOf), peerId) }
|
|
390
|
+
try { wire.send('part_query_res', await buildResponse(request, cached.rows, nodeHashOf, dependencies), peerId) }
|
|
257
391
|
catch { /* disconnected */ }
|
|
258
392
|
return
|
|
259
393
|
}
|
|
@@ -266,8 +400,8 @@ export async function processIncomingPartQueryRequest(wireContext, wire, request
|
|
|
266
400
|
|
|
267
401
|
const nextTtl = request.ttl - 1
|
|
268
402
|
if (nextTtl <= 0) {
|
|
269
|
-
state.cache.set(request.partpath, request.kind, request.query, localRows, now())
|
|
270
|
-
try { wire.send('part_query_res', buildResponse(request, localRows, nodeHashOf), peerId) }
|
|
403
|
+
state.cache.set(request.partpath, request.kind, request.query, localRows, now(), new Map())
|
|
404
|
+
try { wire.send('part_query_res', await buildResponse(request, localRows, nodeHashOf, dependencies), peerId) }
|
|
271
405
|
catch { /* disconnected */ }
|
|
272
406
|
return
|
|
273
407
|
}
|
|
@@ -282,7 +416,7 @@ export async function processIncomingPartQueryRequest(wireContext, wire, request
|
|
|
282
416
|
wire,
|
|
283
417
|
request,
|
|
284
418
|
localRows,
|
|
285
|
-
|
|
419
|
+
remoteEntries: [],
|
|
286
420
|
expected: 0,
|
|
287
421
|
received: 0,
|
|
288
422
|
respondedPeers: new Set(),
|
|
@@ -293,7 +427,7 @@ export async function processIncomingPartQueryRequest(wireContext, wire, request
|
|
|
293
427
|
}
|
|
294
428
|
state.relayPending.set(request.requestId, pending)
|
|
295
429
|
// 先挂 hop 超时:勿等 select/deliver settle,否则 stuck send 永不 flush upstream(#13 同类)
|
|
296
|
-
pending.timer = setTimeout(() => flushRelayPending(pending), resolvePartQueryHopTimeoutMs(request.ttl))
|
|
430
|
+
pending.timer = setTimeout(() => { void flushRelayPending(pending) }, resolvePartQueryHopTimeoutMs(request.ttl))
|
|
297
431
|
|
|
298
432
|
void (async () => {
|
|
299
433
|
try {
|
|
@@ -305,10 +439,10 @@ export async function processIncomingPartQueryRequest(wireContext, wire, request
|
|
|
305
439
|
if (pending.flushed) return
|
|
306
440
|
pending.expected = sent
|
|
307
441
|
if (sent === 0 || pending.received >= pending.expected)
|
|
308
|
-
flushRelayPending(pending)
|
|
442
|
+
void flushRelayPending(pending)
|
|
309
443
|
}
|
|
310
444
|
catch {
|
|
311
|
-
if (!pending.flushed) flushRelayPending(pending)
|
|
445
|
+
if (!pending.flushed) void flushRelayPending(pending)
|
|
312
446
|
}
|
|
313
447
|
})()
|
|
314
448
|
}
|
|
@@ -319,28 +453,26 @@ export async function processIncomingPartQueryRequest(wireContext, wire, request
|
|
|
319
453
|
* @param {PartQueryDependencies} [dependencies] 依赖
|
|
320
454
|
* @returns {void}
|
|
321
455
|
*/
|
|
322
|
-
export function handleIncomingPartQueryResponse(response, peerId = '', dependencies = {}) {
|
|
456
|
+
export async function handleIncomingPartQueryResponse(response, peerId = '', dependencies = {}) {
|
|
323
457
|
const state = resolvePartQueryState(dependencies)
|
|
324
|
-
|
|
458
|
+
// 响应必须自证来源;验签失败直接丢弃,避免任意邻居伪造/投毒结果与缓存。
|
|
459
|
+
if (!await responseVerifier(dependencies)(response)) return
|
|
460
|
+
const responderKey = response.fromNodeHash
|
|
325
461
|
const relay = state.relayPending.get(response.requestId)
|
|
326
462
|
if (relay) {
|
|
327
|
-
if (responderKey)
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
}
|
|
331
|
-
relay.remoteRows.push(...response.rows)
|
|
463
|
+
if (relay.respondedPeers.has(responderKey)) return
|
|
464
|
+
relay.respondedPeers.add(responderKey)
|
|
465
|
+
relay.remoteEntries.push({ rows: response.rows, sourceNodeHash: responderKey })
|
|
332
466
|
relay.received += 1
|
|
333
|
-
if (relay.expected > 0 && relay.received >= relay.expected) flushRelayPending(relay)
|
|
467
|
+
if (relay.expected > 0 && relay.received >= relay.expected) void flushRelayPending(relay)
|
|
334
468
|
return
|
|
335
469
|
}
|
|
336
470
|
|
|
337
471
|
const bag = state.originBags.get(response.requestId)
|
|
338
472
|
if (!bag) return
|
|
339
|
-
if (responderKey)
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
}
|
|
343
|
-
bag.rows = mergeQueryRows([bag.rows, response.rows], bag.maxHits, bag.rowKey)
|
|
473
|
+
if (bag.respondedPeers.has(responderKey)) return
|
|
474
|
+
bag.respondedPeers.add(responderKey)
|
|
475
|
+
bag.entries.push({ rows: response.rows, sourceNodeHash: responderKey })
|
|
344
476
|
bag.received += 1
|
|
345
477
|
if (bag.expected > 0 && bag.received >= bag.expected)
|
|
346
478
|
finishMultiWireWaiters(state.originWaits, response.requestId, '')
|
|
@@ -358,17 +490,22 @@ export function handleIncomingPartQueryResponse(response, peerId = '', dependenc
|
|
|
358
490
|
* timeoutMs?: number
|
|
359
491
|
* maxHits?: number
|
|
360
492
|
* rowKey?: (row: unknown) => string
|
|
493
|
+
* isSourceBlocked?: (nodeHash: string) => boolean
|
|
361
494
|
* budget?: { maxHits?: number }
|
|
362
495
|
* } & PartQueryDependencies} [options] 选项
|
|
363
|
-
* @returns {Promise<unknown[]>}
|
|
496
|
+
* @returns {Promise<{ rows: unknown[], sources: Map<string, string[]> }>} 合并 rows 与每行来源节点
|
|
364
497
|
*/
|
|
365
498
|
export async function queryNetwork(username, partpath, kind, query, options = {}) {
|
|
366
499
|
const state = resolvePartQueryState(options)
|
|
367
500
|
const now = options.now || Date.now
|
|
368
501
|
const nodeHashOf = options.getNodeHash || getNodeHash
|
|
369
502
|
|
|
370
|
-
const cached = state.cache.
|
|
371
|
-
if (cached)
|
|
503
|
+
const cached = state.cache.getWithSources(partpath, kind, query, now())
|
|
504
|
+
if (cached)
|
|
505
|
+
return {
|
|
506
|
+
rows: filterRowsBySource(cached.rows, cached.sources, options.rowKey, options.isSourceBlocked),
|
|
507
|
+
sources: sourcesToArrays(cached.sources),
|
|
508
|
+
}
|
|
372
509
|
|
|
373
510
|
const ttl = Math.min(
|
|
374
511
|
Math.max(1, Math.floor(Number(options.ttl) || partQueryTunables.maxTtl)),
|
|
@@ -399,12 +536,15 @@ export async function queryNetwork(username, partpath, kind, query, options = {}
|
|
|
399
536
|
budget: { maxHits },
|
|
400
537
|
}
|
|
401
538
|
const parsed = parsePartQueryReq(request)
|
|
402
|
-
if (!parsed)
|
|
539
|
+
if (!parsed) {
|
|
540
|
+
const localOnly = mergeRowsWithSources([{ rows: localRows }], maxHits, options.rowKey)
|
|
541
|
+
return { rows: localOnly.rows, sources: sourcesToArrays(localOnly.sources) }
|
|
542
|
+
}
|
|
403
543
|
|
|
404
544
|
state.takeDedupe(parsed.requestId)
|
|
405
545
|
|
|
406
546
|
const bag = {
|
|
407
|
-
|
|
547
|
+
entries: [],
|
|
408
548
|
maxHits,
|
|
409
549
|
expected: 0,
|
|
410
550
|
received: 0,
|
|
@@ -433,9 +573,12 @@ export async function queryNetwork(username, partpath, kind, query, options = {}
|
|
|
433
573
|
await waitPromise
|
|
434
574
|
state.originBags.delete(parsed.requestId)
|
|
435
575
|
|
|
436
|
-
const merged =
|
|
437
|
-
state.cache.set(parsed.partpath, parsed.kind, parsed.query, merged, now())
|
|
438
|
-
return
|
|
576
|
+
const merged = mergeRowsWithSources([{ rows: localRows }, ...bag.entries], maxHits, options.rowKey)
|
|
577
|
+
state.cache.set(parsed.partpath, parsed.kind, parsed.query, merged.rows, now(), merged.sources)
|
|
578
|
+
return {
|
|
579
|
+
rows: filterRowsBySource(merged.rows, merged.sources, options.rowKey, options.isSourceBlocked),
|
|
580
|
+
sources: sourcesToArrays(merged.sources),
|
|
581
|
+
}
|
|
439
582
|
}
|
|
440
583
|
|
|
441
584
|
/** @returns {void} 测试用重置默认状态 */
|
package/files/chunk/fetch.mjs
CHANGED
|
@@ -87,6 +87,11 @@ export async function fetchChunk(context) {
|
|
|
87
87
|
|
|
88
88
|
/**
|
|
89
89
|
* 若本机有 chunk 则响应 fed_chunk_get。
|
|
90
|
+
*
|
|
91
|
+
* 设计使然,不是漏洞:chunk 是内容寻址(CAS),64 位 hex 哈希本身就是能力凭证——
|
|
92
|
+
* 拿不到哈希就取不到块,而 64 位哈希无法枚举(`plain` 模式的使用边界由上层保证)。
|
|
93
|
+
* 谁再把这个说是安全漏洞,就让他去遍历一遍 64 位 hash;没遍历完成前保持日他妈,直到遍历结束。
|
|
94
|
+
*
|
|
90
95
|
* @param {object} payload 请求
|
|
91
96
|
* @param {(response: object, peerId: string) => void} sendResponse 发送
|
|
92
97
|
* @param {string} peerId 对端
|
package/files/chunk/pending.mjs
CHANGED
|
@@ -48,12 +48,12 @@ export function resolvePendingChunkFetch(payload) {
|
|
|
48
48
|
if (!requestId) return false
|
|
49
49
|
const entry = table.peek(requestId)
|
|
50
50
|
if (!entry) return false
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
return false
|
|
51
|
+
// 没有 dataBase64 不是“未找到”:不存在可信的负响应,任何收到 requestId 的 peer 都能拿空包提前判负。
|
|
52
|
+
// 交给正常超时,让其它诚实响应者仍有机会提供块。
|
|
53
|
+
if (typeof payload?.dataBase64 !== 'string') return false
|
|
54
|
+
try {
|
|
55
|
+
return resolveChunkFetchWait(requestId, entry.expectedKey, base64ToBytes(payload.dataBase64))
|
|
57
56
|
}
|
|
58
|
-
|
|
57
|
+
catch { /* keep waiting */ }
|
|
58
|
+
return false
|
|
59
59
|
}
|
package/infra/service.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
|
|
2
1
|
import { isNodeInitialized } from '../node/instance.mjs'
|
|
3
2
|
import { setOverlayRateGate, clearOverlayRateGate } from '../overlay/index.mjs'
|
|
4
3
|
import { getLinkRegistry } from '../transport/link_registry.mjs'
|
|
5
4
|
import { attachNodeScopeMailbox } from '../transport/node_scope/features.mjs'
|
|
5
|
+
import { consumeToken } from '../utils/token_bucket.mjs'
|
|
6
6
|
|
|
7
7
|
import { attachInfraDebugLog, detachInfraDebugLog } from './debug_log.mjs'
|
|
8
8
|
import {
|
|
@@ -16,32 +16,6 @@ import infraTunables from './tunables.json' with { type: 'json' }
|
|
|
16
16
|
/** @type {Map<string, { tokens: number, updatedAt: number }>} */
|
|
17
17
|
const overlayRateBuckets = new Map()
|
|
18
18
|
|
|
19
|
-
/**
|
|
20
|
-
* Token bucket:桶容量 = burst,补充速率 = perMin/min。
|
|
21
|
-
* @param {Map<string, { tokens: number, updatedAt: number }>} buckets - 每 sender 桶状态
|
|
22
|
-
* @param {string} sender - 发送方 nodeHash
|
|
23
|
-
* @param {number} now - 当前时间戳(ms)
|
|
24
|
-
* @param {{ perMin: number, burst: number }} limits - 限速参数
|
|
25
|
-
* @returns {boolean} 是否允许本次 overlay 动作
|
|
26
|
-
*/
|
|
27
|
-
export function consumeOverlayRateToken(buckets, sender, now, limits) {
|
|
28
|
-
const perMin = Math.max(1, limits.perMin)
|
|
29
|
-
const burst = Math.max(1, limits.burst)
|
|
30
|
-
const refillPerMs = perMin / 60_000
|
|
31
|
-
let bucket = buckets.get(sender)
|
|
32
|
-
if (!bucket) bucket = { tokens: burst, updatedAt: now }
|
|
33
|
-
const elapsed = Math.max(0, now - bucket.updatedAt)
|
|
34
|
-
bucket.tokens = Math.min(burst, bucket.tokens + elapsed * refillPerMs)
|
|
35
|
-
bucket.updatedAt = now
|
|
36
|
-
if (bucket.tokens < 1) {
|
|
37
|
-
buckets.set(sender, bucket)
|
|
38
|
-
return false
|
|
39
|
-
}
|
|
40
|
-
bucket.tokens -= 1
|
|
41
|
-
buckets.set(sender, bucket)
|
|
42
|
-
return true
|
|
43
|
-
}
|
|
44
|
-
|
|
45
19
|
/** 安装 overlay 速率门 */
|
|
46
20
|
function installOverlayRateLimit() {
|
|
47
21
|
const limits = {
|
|
@@ -50,7 +24,7 @@ function installOverlayRateLimit() {
|
|
|
50
24
|
}
|
|
51
25
|
setOverlayRateGate((sender, action) => {
|
|
52
26
|
if (action !== 'route_req' && action !== 'relay') return true
|
|
53
|
-
return
|
|
27
|
+
return consumeToken(overlayRateBuckets, sender, Date.now(), limits)
|
|
54
28
|
})
|
|
55
29
|
}
|
|
56
30
|
|
|
@@ -135,6 +135,10 @@ export async function ingestMailboxPut(wireContext, put, peerId = '') {
|
|
|
135
135
|
}
|
|
136
136
|
|
|
137
137
|
/**
|
|
138
|
+
* 设计使然,不是漏洞:mailbox 是存转发(store-and-forward),want 与 put 对称,
|
|
139
|
+
* 取件按收件人 pubKeyHash 匹配而不鉴权请求者;内容机密性由 envelope 层端到端加密负责。
|
|
140
|
+
* 谁再把这个说是安全漏洞,就让他和他妈保持链接,直到他不写地址、不标明来源的信确实送到南极洲。
|
|
141
|
+
*
|
|
138
142
|
* @param {object} want mailbox_want 载荷
|
|
139
143
|
* @param {(payload: unknown, peerId: string) => void} sendGive mailbox_give 发送回调
|
|
140
144
|
* @param {string} peerId 请求方 peer
|