@steve02081504/fount-p2p 0.0.23 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/core/object.mjs +9 -0
- package/core/partpath.mjs +10 -0
- package/dag/storage.mjs +0 -5
- package/dag/strip_extensions.mjs +1 -3
- package/{wire/part_query_cache.mjs → federation/part_query/cache.mjs} +7 -7
- package/{wire/part_query.mjs → federation/part_query/runtime.mjs} +36 -77
- package/files/assemble.mjs +5 -5
- package/files/assemble_stream.mjs +2 -2
- package/files/chunk/fetch.mjs +101 -0
- package/files/chunk/pending.mjs +59 -0
- package/files/{chunk_provider_registry.mjs → chunk/provider_registry.mjs} +4 -5
- package/files/chunk/responder.mjs +104 -0
- package/files/{chunk_store.mjs → chunk/store.mjs} +2 -2
- package/files/{chunk_fetch_verify.mjs → chunk/verify.mjs} +2 -2
- package/files/evfs.mjs +18 -19
- package/files/fed/fetch_shared.mjs +39 -0
- package/files/fed/responder.mjs +47 -0
- package/files/{acl.mjs → manifest/acl.mjs} +4 -4
- package/files/{manifest_acl_registry.mjs → manifest/acl_registry.mjs} +14 -17
- package/files/{manifest_fetch.mjs → manifest/fetch.mjs} +47 -52
- package/files/{manifest.mjs → manifest/normalize.mjs} +2 -2
- package/files/manifest/pending.mjs +53 -0
- package/files/{public_manifest.mjs → manifest/public.mjs} +12 -12
- package/files/transfer_key.mjs +2 -2
- package/files/transfer_key_registry.mjs +7 -7
- package/index.mjs +4 -4
- package/infra/service.mjs +1 -1
- package/link/handshake.mjs +0 -3
- package/mailbox/deliver_or_store.mjs +0 -2
- package/mailbox/parse.mjs +1 -1
- package/mailbox/store.mjs +0 -9
- package/mailbox/wire.mjs +24 -16
- package/node/reputation_sync.mjs +33 -29
- package/package.json +8 -8
- package/registries/inbound.mjs +19 -9
- package/registries/part_path.mjs +3 -8
- package/schemas/discovery.mjs +1 -1
- package/schemas/federation_pull.mjs +1 -1
- package/schemas/part_query.mjs +6 -5
- package/transport/group_link_set.mjs +2 -2
- package/transport/link_registry.mjs +2 -2
- package/transport/node_scope/features.mjs +166 -0
- package/transport/node_scope/wire.mjs +124 -0
- package/transport/remote_user_room.mjs +0 -1
- package/{rooms → transport}/scoped_link.mjs +1 -1
- package/transport/user_room.mjs +15 -28
- package/utils/fetch_wait.mjs +98 -0
- package/wire/adapter.mjs +13 -0
- package/wire/ingress.mjs +1 -8
- package/wire/part/fanout.mjs +99 -0
- package/wire/part/group.mjs +45 -0
- package/wire/{part_ingress.mjs → part/ingress.mjs} +58 -67
- package/wire/{part_invoke.mjs → part/invoke.mjs} +1 -22
- package/wire/part/pending.mjs +32 -0
- package/wire/part/query.mjs +91 -0
- package/wire/subscribe.mjs +13 -0
- package/federation/chunk_fetch_pending.mjs +0 -124
- package/federation/manifest_fetch_pending.mjs +0 -88
- package/files/chunk_fetch.mjs +0 -99
- package/files/chunk_responder.mjs +0 -142
- package/transport/advert_ingest.mjs +0 -17
- package/transport/node_scope.mjs +0 -289
- package/wire/group_part.mjs +0 -43
- package/wire/part_common.mjs +0 -55
- package/wire/part_fanout.mjs +0 -52
- /package/{wire → schemas}/part_query.tunables.json +0 -0
package/README.md
CHANGED
|
@@ -21,20 +21,20 @@ Requires **Node.js ≥ 20** (ESM + `import ... with { type: 'json' }`).
|
|
|
21
21
|
import {
|
|
22
22
|
startNode,
|
|
23
23
|
ensureUserRoom,
|
|
24
|
-
|
|
24
|
+
attachNodeScopeDefaultFeatures,
|
|
25
25
|
createScopedLinkRoom,
|
|
26
26
|
} from '@steve02081504/fount-p2p'
|
|
27
27
|
|
|
28
28
|
await startNode({ nodeDir: '/path/to/p2p/node' })
|
|
29
29
|
await ensureUserRoom() // slot + runtime only
|
|
30
|
-
|
|
30
|
+
attachNodeScopeDefaultFeatures({ replicaUsername: 'alice' }) // full business wires
|
|
31
31
|
```
|
|
32
32
|
|
|
33
33
|
Shells talk to the **fount network** (`ensureLinkToNode` / `sendToNodeLink` / rooms). Do not import `link/providers/*` or choose WebRTC / BLE / LAN / Nostr yourself. Provider registration: `registerLinkProvider` from `@steve02081504/fount-p2p/link` or the facade.
|
|
34
34
|
|
|
35
35
|
Dial order is descending link **`level`**: `lan_tcp` → `webrtc` → `ble_gatt` → `nostr` (−∞ last resort). Discovery **`priority`** only orders handshake / presence media. Details: [docs/transports.md](./docs/transports.md).
|
|
36
36
|
|
|
37
|
-
Public transport subpaths: `link_registry`, `user_room`, `group_link_set`, `node_scope`, `room_scopes`, `remote_user_room`. Other `transport/*` modules are internal.
|
|
37
|
+
Public transport subpaths: `link_registry`, `user_room`, `group_link_set`, `node_scope`, `room_scopes`, `remote_user_room`, `scoped_link`. Other `transport/*` modules are internal.
|
|
38
38
|
|
|
39
39
|
## Infra relay (optional)
|
|
40
40
|
|
|
@@ -73,7 +73,7 @@ Reputation pull/apply is separate: `pullReputationFromNode` → JSON; `setReputa
|
|
|
73
73
|
| L0 | `core/` | Pure primitives: `hexIds`, `entity_id*`, `canonical_json` |
|
|
74
74
|
| L1 | `crypto/`, `wire/`, `schemas/` | Cryptography, wire protocol, canonical validation |
|
|
75
75
|
| L2 | `node/` | Node runtime: `identity`, `entity_store`, `denylist`, `reputation_store` |
|
|
76
|
-
| L3 | `discovery/`, `link/`, `transport
|
|
76
|
+
| L3 | `discovery/`, `link/`, `transport/` | Discovery + fount-network registry/rooms (`./link` = provider registration only) |
|
|
77
77
|
| L4 | `trust_graph/`, `mailbox/`, `dag/`, `federation/`, `files/`, `governance/`, `reputation/` | Federation, store-and-forward, DAG, EVFS, tunables |
|
|
78
78
|
| — | `infra/` | Optional public-good relay (`startInfra` / CLI) |
|
|
79
79
|
| — | `registries/` | Pluggable registries (event type, part path, room provider, …) |
|
|
@@ -85,10 +85,10 @@ Facade entry: `index.mjs` (`startNode`, `createGroupLinkSet`, `registerDiscovery
|
|
|
85
85
|
| Module | Role |
|
|
86
86
|
|---|---|
|
|
87
87
|
| `link_registry.mjs` | fount-network facade: dial fallback, scope/overlay |
|
|
88
|
-
| `user_room.mjs` / `group_link_set.mjs` / `node_scope
|
|
88
|
+
| `user_room.mjs` / `group_link_set.mjs` / `node_scope/` | rooms + composable node-scope wires |
|
|
89
89
|
| `room_scopes.mjs` / `remote_user_room.mjs` | scope constants / remote user slot |
|
|
90
90
|
|
|
91
|
-
`runtime_bootstrap`, `offer_answer
|
|
91
|
+
`runtime_bootstrap`, `offer_answer` are **internal** (transport). Signal crypto / rendezvous live under `discovery/internal/signal_crypto.mjs` (used by discovery `nostr.mjs` / `adverts.mjs`; not a package export). The Nostr **link** provider (`link/providers/nostr.mjs`) reuses the same discovery signal path (`type: 'link'`) as a last-resort duplex pipe. `ensureRuntime` returns after registration and scheduling warm-up; it does not await lan_tcp listen, public relays, or Bluetooth. `setSignalingRuntimeConfig` → `reloadDiscoveryRelays`. See [docs/runtime.md](./docs/runtime.md) and [docs/transports.md](./docs/transports.md).
|
|
92
92
|
|
|
93
93
|
Root contains only the facade and package metadata; all modules live in layered subdirectories.
|
|
94
94
|
|
|
@@ -121,3 +121,5 @@ Group chunk remote storage (S3, etc.) is implemented by the shell as `GroupStora
|
|
|
121
121
|
- Signaling and WebRTC glare: [`docs/signaling.md`](./docs/signaling.md)
|
|
122
122
|
- Runtime bootstrap / BT probe: [`docs/runtime.md`](./docs/runtime.md)
|
|
123
123
|
- Infra relay / node-scope attaches: [`docs/infra.md`](./docs/infra.md)
|
|
124
|
+
- Wire part / fanout attaches: [`docs/wire.md`](./docs/wire.md)
|
|
125
|
+
- EVFS public manifests / fetch: [`docs/evfs.md`](./docs/evfs.md)
|
package/core/object.mjs
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 判定 `{}` 形态对象(原型为 Object.prototype)。
|
|
3
|
+
* 数组、Date/Map/类实例、null 原型对象一律否。
|
|
4
|
+
* @param {unknown} value 待判定值
|
|
5
|
+
* @returns {value is Record<string, unknown>} 是否为普通对象
|
|
6
|
+
*/
|
|
7
|
+
export function isPlainObject(value) {
|
|
8
|
+
return value != null && Object.getPrototypeOf(value) === Object.prototype
|
|
9
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
const PARTPATH_RE = /^[\w-]+(?:\/[\w-]+)*$/
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* 入站 partpath 形校验(不改写)。仅用于非本机网络边界。
|
|
5
|
+
* @param {unknown} value 原始 partpath
|
|
6
|
+
* @returns {string | null} 合法则原样返回;否则 null
|
|
7
|
+
*/
|
|
8
|
+
export function parsePartpath(value) {
|
|
9
|
+
return typeof value === 'string' && PARTPATH_RE.test(value) ? value : null
|
|
10
|
+
}
|
package/dag/storage.mjs
CHANGED
|
@@ -9,11 +9,6 @@ import { pipeline } from 'node:stream/promises'
|
|
|
9
9
|
import { withAsyncMutex } from '../utils/async_mutex.mjs'
|
|
10
10
|
import { atomicTemporaryPath, finalizeAtomicRename } from '../utils/atomic_fs.mjs'
|
|
11
11
|
|
|
12
|
-
/**
|
|
13
|
-
* 原子重命名 finalize(re-export)。
|
|
14
|
-
*/
|
|
15
|
-
export { finalizeAtomicRename } from '../utils/atomic_fs.mjs'
|
|
16
|
-
|
|
17
12
|
/** 流式重写 JSONL 时分块写入的行数上限 */
|
|
18
13
|
const WRITE_JSONL_CHUNK_LINES = 1000
|
|
19
14
|
|
package/dag/strip_extensions.mjs
CHANGED
|
@@ -1,17 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* DAG 事件行读盘时剥离的本地扩展键(不得进入验签域 / 联邦 wire)。
|
|
3
3
|
*/
|
|
4
|
-
import { isPlainObject } from '../wire/ingress.mjs'
|
|
5
4
|
|
|
6
5
|
/** 落盘后 trusted 读路径仍须剥除的 sidecar 键。 */
|
|
7
6
|
export const DAG_EVENT_LOCAL_EXTENSION_KEYS = new Set(['receivedAt', 'isRemote'])
|
|
8
7
|
|
|
9
8
|
/**
|
|
10
|
-
* @param {
|
|
9
|
+
* @param {object} row JSONL 行
|
|
11
10
|
* @returns {object} 剥离扩展键后的副本
|
|
12
11
|
*/
|
|
13
12
|
export function stripDagEventLocalExtensions(row) {
|
|
14
|
-
if (!isPlainObject(row)) return row
|
|
15
13
|
const out = { ...row }
|
|
16
14
|
for (const key of DAG_EVENT_LOCAL_EXTENSION_KEYS) delete out[key]
|
|
17
15
|
return out
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { sha256Hex } from '
|
|
2
|
-
import { normalizePartQueryCacheMaterial } from '
|
|
3
|
-
import
|
|
4
|
-
|
|
5
|
-
import partQueryTunables from './part_query.tunables.json' with { type: 'json' }
|
|
1
|
+
import { sha256Hex } from '../../crypto/crypto.mjs'
|
|
2
|
+
import { normalizePartQueryCacheMaterial } from '../../schemas/part_query.mjs'
|
|
3
|
+
import partQueryTunables from '../../schemas/part_query.tunables.json' with { type: 'json' }
|
|
4
|
+
import { createLruMap } from '../../utils/lru.mjs'
|
|
6
5
|
|
|
7
6
|
/**
|
|
8
7
|
* @typedef {{ rows: unknown[], storedAt: number }} PartQueryCacheEntry
|
|
@@ -71,13 +70,14 @@ export function createPartQueryCache(options = {}) {
|
|
|
71
70
|
* @param {string} partpath part 路径
|
|
72
71
|
* @param {string} kind 查询标签
|
|
73
72
|
* @param {unknown} query 查询体
|
|
74
|
-
* @param {unknown[]} rows 聚合 rows
|
|
73
|
+
* @param {unknown[]} rows 聚合 rows(空数组不入库)
|
|
75
74
|
* @param {number} [now=Date.now()] 当前时间
|
|
76
75
|
* @returns {void}
|
|
77
76
|
*/
|
|
78
77
|
set(partpath, kind, query, rows, now = Date.now()) {
|
|
79
78
|
const key = partQueryCacheKey(partpath, kind, query)
|
|
80
|
-
|
|
79
|
+
// 空 miss 不缓存:mesh 晚就绪 / 超时早查询不应被长 TTL 负缓存粘住
|
|
80
|
+
if (!key || !Array.isArray(rows) || rows.length === 0) return
|
|
81
81
|
sweep(now)
|
|
82
82
|
map.touch(key, {
|
|
83
83
|
rows: rows.slice(0, maxHits),
|
|
@@ -1,29 +1,27 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto'
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
import { getNodeHash } from '
|
|
5
|
-
import { loadReputation } from '
|
|
6
|
-
import { isQuarantinedPure } from '
|
|
3
|
+
import { compositeKey } from '../../core/composite_key.mjs'
|
|
4
|
+
import { getNodeHash } from '../../node/identity.mjs'
|
|
5
|
+
import { loadReputation } from '../../node/reputation_store.mjs'
|
|
6
|
+
import { isQuarantinedPure } from '../../reputation/engine.mjs'
|
|
7
7
|
import {
|
|
8
8
|
clampPartQueryRows,
|
|
9
9
|
parsePartQueryReq,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import
|
|
16
|
-
import
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
import
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
/** @typedef {import('
|
|
25
|
-
/** @typedef {import('../schemas/part_query.mjs').PartQueryRes} PartQueryRes */
|
|
26
|
-
/** @typedef {import('./part_ingress.mjs').PartWireAdapter} PartWireAdapter */
|
|
10
|
+
} from '../../schemas/part_query.mjs'
|
|
11
|
+
import partQueryTunables from '../../schemas/part_query.tunables.json' with { type: 'json' }
|
|
12
|
+
import { buildMergedGraph } from '../../trust_graph/build.mjs'
|
|
13
|
+
import { pickTopFromGraph } from '../../trust_graph/engine.mjs'
|
|
14
|
+
import { resolveFederationFanoutTopK } from '../../trust_graph/resolve.mjs'
|
|
15
|
+
import trustGraphTunables from '../../trust_graph/tunables.json' with { type: 'json' }
|
|
16
|
+
import { finishMultiWireWaiters, registerMultiWireWait } from '../../wire/wait.mjs'
|
|
17
|
+
import { createDedupeSlot } from '../dedupe_slot.mjs'
|
|
18
|
+
|
|
19
|
+
import { createPartQueryCache, partQueryCache } from './cache.mjs'
|
|
20
|
+
|
|
21
|
+
/** @typedef {import('../../schemas/part_query.mjs').PartQueryReq} PartQueryReq */
|
|
22
|
+
/** @typedef {import('../../schemas/part_query.mjs').PartQueryRes} PartQueryRes */
|
|
23
|
+
|
|
24
|
+
/** @typedef {import('../../wire/adapter.mjs').WireAdapter} PartQueryWire */
|
|
27
25
|
|
|
28
26
|
/**
|
|
29
27
|
* @typedef {{
|
|
@@ -41,7 +39,7 @@ import { finishMultiWireWaiters, registerMultiWireWait } from './wait.mjs'
|
|
|
41
39
|
* @typedef {{
|
|
42
40
|
* takeDedupe: (key: string) => boolean
|
|
43
41
|
* relayPending: Map<string, RelayPending>
|
|
44
|
-
* originWaits: Map<string, Map<string, import('
|
|
42
|
+
* originWaits: Map<string, Map<string, import('../../wire/wait.mjs').WireWaiter[]>>
|
|
45
43
|
* originBags: Map<string, { rows: unknown[], maxHits: number, expected: number, received: number, respondedPeers: Set<string>, rowKey?: (row: unknown) => string }>
|
|
46
44
|
* cache: ReturnType<typeof createPartQueryCache>
|
|
47
45
|
* handlers: Map<string, QueryInboundHandler>
|
|
@@ -61,7 +59,7 @@ import { finishMultiWireWaiters, registerMultiWireWait } from './wait.mjs'
|
|
|
61
59
|
/**
|
|
62
60
|
* @typedef {{
|
|
63
61
|
* upstreamPeerId: string
|
|
64
|
-
* wire:
|
|
62
|
+
* wire: PartQueryWire
|
|
65
63
|
* request: PartQueryReq
|
|
66
64
|
* localRows: unknown[]
|
|
67
65
|
* remoteRows: unknown[]
|
|
@@ -100,7 +98,7 @@ const defaultState = createPartQueryNodeState({ cache: partQueryCache })
|
|
|
100
98
|
* @param {PartQueryDependencies} [dependencies] 依赖
|
|
101
99
|
* @returns {PartQueryNodeState} 节点状态
|
|
102
100
|
*/
|
|
103
|
-
function
|
|
101
|
+
export function resolvePartQueryState(dependencies = {}) {
|
|
104
102
|
return dependencies.state || defaultState
|
|
105
103
|
}
|
|
106
104
|
|
|
@@ -110,7 +108,7 @@ function resolveState(dependencies = {}) {
|
|
|
110
108
|
* @returns {string} handler 键
|
|
111
109
|
*/
|
|
112
110
|
function handlerKey(partpath, kind) {
|
|
113
|
-
return
|
|
111
|
+
return compositeKey(partpath, kind)
|
|
114
112
|
}
|
|
115
113
|
|
|
116
114
|
/**
|
|
@@ -122,7 +120,7 @@ function handlerKey(partpath, kind) {
|
|
|
122
120
|
* @returns {void}
|
|
123
121
|
*/
|
|
124
122
|
export function registerQueryInboundHandler(partpath, kind, handler, state = defaultState) {
|
|
125
|
-
state.handlers.set(handlerKey(
|
|
123
|
+
state.handlers.set(handlerKey(partpath, kind), handler)
|
|
126
124
|
}
|
|
127
125
|
|
|
128
126
|
/**
|
|
@@ -202,17 +200,15 @@ async function selectQueryNeighbors(username, exclude, dependencies) {
|
|
|
202
200
|
}
|
|
203
201
|
|
|
204
202
|
/**
|
|
205
|
-
* @param {string} username 用户
|
|
206
203
|
* @param {string} nodeHash 目标
|
|
207
204
|
* @param {string} action action 名
|
|
208
205
|
* @param {unknown} payload 载荷
|
|
209
206
|
* @param {PartQueryDependencies} dependencies 依赖
|
|
210
207
|
* @returns {Promise<boolean>} 是否发出
|
|
211
208
|
*/
|
|
212
|
-
async function deliverQuery(
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
return sendToNodeLink(nodeHash, { scope: 'node', action, payload })
|
|
209
|
+
async function deliverQuery(nodeHash, action, payload, dependencies) {
|
|
210
|
+
if (!dependencies.deliver) return false
|
|
211
|
+
return Boolean(await dependencies.deliver(nodeHash, action, payload))
|
|
216
212
|
}
|
|
217
213
|
|
|
218
214
|
/**
|
|
@@ -254,14 +250,14 @@ function flushRelayPending(pending) {
|
|
|
254
250
|
|
|
255
251
|
/**
|
|
256
252
|
* @param {{ replicaUsername?: string }} wireContext attach 上下文
|
|
257
|
-
* @param {
|
|
253
|
+
* @param {PartQueryWire} wire wire
|
|
258
254
|
* @param {PartQueryReq} request 已校验请求
|
|
259
255
|
* @param {string} peerId 来路
|
|
260
256
|
* @param {PartQueryDependencies} dependencies 依赖
|
|
261
257
|
* @returns {Promise<void>}
|
|
262
258
|
*/
|
|
263
|
-
async function
|
|
264
|
-
const state =
|
|
259
|
+
export async function processIncomingPartQueryRequest(wireContext, wire, request, peerId, dependencies) {
|
|
260
|
+
const state = resolvePartQueryState(dependencies)
|
|
265
261
|
const nodeHashOf = dependencies.getNodeHash || getNodeHash
|
|
266
262
|
const now = dependencies.now || Date.now
|
|
267
263
|
const username = String(wireContext.replicaUsername || '')
|
|
@@ -311,7 +307,7 @@ async function processIncomingRequest(wireContext, wire, request, peerId, depend
|
|
|
311
307
|
|
|
312
308
|
let sent = 0
|
|
313
309
|
for (const target of neighbors)
|
|
314
|
-
if (await deliverQuery(
|
|
310
|
+
if (await deliverQuery(target, 'part_query_req', forwardPayload, dependencies)) sent++
|
|
315
311
|
pending.expected = sent
|
|
316
312
|
|
|
317
313
|
if (sent === 0 || pending.received >= pending.expected) {
|
|
@@ -322,39 +318,6 @@ async function processIncomingRequest(wireContext, wire, request, peerId, depend
|
|
|
322
318
|
pending.timer = setTimeout(() => flushRelayPending(pending), resolvePartQueryHopTimeoutMs(request.ttl))
|
|
323
319
|
}
|
|
324
320
|
|
|
325
|
-
/**
|
|
326
|
-
* 挂载 part_query_req / part_query_res。
|
|
327
|
-
* @param {{ replicaUsername?: string }} wireContext 入站上下文
|
|
328
|
-
* @param {PartWireAdapter} wire wire
|
|
329
|
-
* @param {PartQueryDependencies} [dependencies] 可注入依赖(含 per-node state)
|
|
330
|
-
* @returns {() => void} 取消挂载的 dispose
|
|
331
|
-
*/
|
|
332
|
-
export function attachPartQueryWire(wireContext, wire, dependencies = {}) {
|
|
333
|
-
const state = resolveState(dependencies)
|
|
334
|
-
const offs = [
|
|
335
|
-
wire.on('part_query_req', (data, peerId) => {
|
|
336
|
-
if (!isPlainObject(data)) return
|
|
337
|
-
const request = parsePartQueryReq(data)
|
|
338
|
-
if (!request) return
|
|
339
|
-
if (!state.takeDedupe(request.requestId)) return
|
|
340
|
-
const source = String(peerId || request.originNodeHash || '').trim().toLowerCase()
|
|
341
|
-
if (source && !consumeWireRateBucket(`part_query:${source}`, {
|
|
342
|
-
maxCount: partQueryTunables.ratePerSourcePerMin,
|
|
343
|
-
})) return
|
|
344
|
-
void processIncomingRequest(wireContext, wire, request, String(peerId || ''), dependencies)
|
|
345
|
-
}),
|
|
346
|
-
wire.on('part_query_res', (data, peerId) => {
|
|
347
|
-
const response = parsePartQueryRes(data)
|
|
348
|
-
if (!response) return
|
|
349
|
-
handleIncomingPartQueryResponse(response, String(peerId || ''), dependencies)
|
|
350
|
-
}),
|
|
351
|
-
]
|
|
352
|
-
return () => {
|
|
353
|
-
for (const off of offs)
|
|
354
|
-
try { off?.() } catch { /* ignore */ }
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
|
|
358
321
|
/**
|
|
359
322
|
* @param {PartQueryRes} response 响应
|
|
360
323
|
* @param {string} peerId 来路
|
|
@@ -362,8 +325,7 @@ export function attachPartQueryWire(wireContext, wire, dependencies = {}) {
|
|
|
362
325
|
* @returns {void}
|
|
363
326
|
*/
|
|
364
327
|
export function handleIncomingPartQueryResponse(response, peerId = '', dependencies = {}) {
|
|
365
|
-
const state =
|
|
366
|
-
// 同一 peer 只计一次,防重复回包灌水/提早凑齐 expected
|
|
328
|
+
const state = resolvePartQueryState(dependencies)
|
|
367
329
|
const responderKey = String(peerId || response.fromNodeHash || '').trim().toLowerCase()
|
|
368
330
|
const relay = state.relayPending.get(response.requestId)
|
|
369
331
|
if (relay) {
|
|
@@ -405,7 +367,7 @@ export function handleIncomingPartQueryResponse(response, peerId = '', dependenc
|
|
|
405
367
|
* @returns {Promise<unknown[]>} 合并后的 rows
|
|
406
368
|
*/
|
|
407
369
|
export async function queryNetwork(username, partpath, kind, query, options = {}) {
|
|
408
|
-
const state =
|
|
370
|
+
const state = resolvePartQueryState(options)
|
|
409
371
|
const now = options.now || Date.now
|
|
410
372
|
const nodeHashOf = options.getNodeHash || getNodeHash
|
|
411
373
|
|
|
@@ -420,7 +382,6 @@ export async function queryNetwork(username, partpath, kind, query, options = {}
|
|
|
420
382
|
partQueryTunables.maxHits,
|
|
421
383
|
Math.max(1, Math.floor(Number(options.maxHits ?? options.budget?.maxHits) || partQueryTunables.maxHits)),
|
|
422
384
|
)
|
|
423
|
-
// 第一跳中继等待 hopTimeout(ttl) 才 flush,发起端默认取 ttl+1 档以免先行超时
|
|
424
385
|
const timeoutMs = Math.max(
|
|
425
386
|
1,
|
|
426
387
|
Math.floor(Number(options.timeoutMs) || resolvePartQueryHopTimeoutMs(ttl + 1)),
|
|
@@ -435,8 +396,8 @@ export async function queryNetwork(username, partpath, kind, query, options = {}
|
|
|
435
396
|
const request = {
|
|
436
397
|
requestId: randomUUID(),
|
|
437
398
|
originNodeHash: nodeHashOf(),
|
|
438
|
-
partpath
|
|
439
|
-
kind
|
|
399
|
+
partpath,
|
|
400
|
+
kind,
|
|
440
401
|
query,
|
|
441
402
|
ttl,
|
|
442
403
|
budget: { maxHits },
|
|
@@ -444,7 +405,6 @@ export async function queryNetwork(username, partpath, kind, query, options = {}
|
|
|
444
405
|
const parsed = parsePartQueryReq(request)
|
|
445
406
|
if (!parsed) return mergeQueryRows([localRows], maxHits, options.rowKey)
|
|
446
407
|
|
|
447
|
-
// 预占 dedupe:若查询绕环回流到本机,入站侧直接丢弃
|
|
448
408
|
state.takeDedupe(parsed.requestId)
|
|
449
409
|
|
|
450
410
|
const bag = {
|
|
@@ -463,10 +423,9 @@ export async function queryNetwork(username, partpath, kind, query, options = {}
|
|
|
463
423
|
const neighbors = await selectQueryNeighbors(username, exclude, options)
|
|
464
424
|
let sent = 0
|
|
465
425
|
for (const target of neighbors)
|
|
466
|
-
if (await deliverQuery(
|
|
426
|
+
if (await deliverQuery(target, 'part_query_req', parsed, options)) sent++
|
|
467
427
|
bag.expected = sent
|
|
468
428
|
|
|
469
|
-
// deliver 可能同步回流;在赋值 expected 后再检查是否已齐
|
|
470
429
|
if (sent === 0 || bag.received >= bag.expected)
|
|
471
430
|
finishMultiWireWaiters(state.originWaits, parsed.requestId, '')
|
|
472
431
|
await waitPromise
|
package/files/assemble.mjs
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
wrapContentKey,
|
|
11
11
|
} from '../crypto/key.mjs'
|
|
12
12
|
|
|
13
|
-
import { normalizeFileManifest, publicTransferKeyDescriptor } from './manifest.mjs'
|
|
13
|
+
import { normalizeFileManifest, publicTransferKeyDescriptor } from './manifest/normalize.mjs'
|
|
14
14
|
|
|
15
15
|
/** @type {Record<string, (plain: Buffer) => { contentHash: string, ciphertextHash: string, raw: Buffer, contentKey?: Buffer }>} */
|
|
16
16
|
const ENCRYPTION_STRATEGIES = {
|
|
@@ -35,8 +35,8 @@ const ENCRYPTION_STRATEGIES = {
|
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
|
-
* @typedef {import('./manifest.mjs').FileManifest} FileManifest
|
|
39
|
-
* @typedef {import('./manifest.mjs').CeMode} CeMode
|
|
38
|
+
* @typedef {import('./manifest/normalize.mjs').FileManifest} FileManifest
|
|
39
|
+
* @typedef {import('./manifest/normalize.mjs').CeMode} CeMode
|
|
40
40
|
*/
|
|
41
41
|
|
|
42
42
|
/**
|
|
@@ -154,7 +154,7 @@ export async function encryptPlaintextToMultiPartsAsync(plaintext, ceMode = 'con
|
|
|
154
154
|
* @param {string} [parameters.name] 文件名
|
|
155
155
|
* @param {string} [parameters.mimeType] MIME
|
|
156
156
|
* @param {CeMode} [parameters.ceMode] 加密模式
|
|
157
|
-
* @param {import('./manifest.mjs').TransferKeyDescriptor} [parameters.transferKeyDescriptor] 传递密钥
|
|
157
|
+
* @param {import('./manifest/normalize.mjs').TransferKeyDescriptor} [parameters.transferKeyDescriptor] 传递密钥
|
|
158
158
|
* @param {object} [parameters.meta] 元数据
|
|
159
159
|
* @returns {FileManifest} manifest(未写盘)
|
|
160
160
|
*/
|
|
@@ -224,7 +224,7 @@ export function buildFileManifestFromEnc(parameters, enc) {
|
|
|
224
224
|
* @param {string} fileId 文件 ID
|
|
225
225
|
* @param {Buffer} contentKey 随机密钥
|
|
226
226
|
* @param {Buffer | string} H vault H
|
|
227
|
-
* @returns {import('./manifest.mjs').TransferKeyDescriptor} 传输密钥描述
|
|
227
|
+
* @returns {import('./manifest/normalize.mjs').TransferKeyDescriptor} 传输密钥描述
|
|
228
228
|
*/
|
|
229
229
|
export function vaultWrapDescriptor(entityHash, fileId, contentKey, H) {
|
|
230
230
|
return {
|
|
@@ -8,7 +8,7 @@ import { encryptSliceToPart } from './assemble.mjs'
|
|
|
8
8
|
import { decryptPart } from './transfer_key.mjs'
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* @typedef {import('./manifest.mjs').CeMode} CeMode
|
|
11
|
+
* @typedef {import('./manifest/normalize.mjs').CeMode} CeMode
|
|
12
12
|
* @typedef {{ hash: string, size: number, raw: Buffer, contentHash?: string }} EncryptedPart
|
|
13
13
|
*/
|
|
14
14
|
|
|
@@ -113,7 +113,7 @@ function readStreamChunk(stream) {
|
|
|
113
113
|
|
|
114
114
|
/**
|
|
115
115
|
* 将 manifest 各密文块按 part.size 拼齐后解密,串联为明文可读流。
|
|
116
|
-
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
116
|
+
* @param {import('./manifest/normalize.mjs').FileManifest} manifest 清单
|
|
117
117
|
* @param {import('node:stream').Readable[]} partStreams 按序密文流
|
|
118
118
|
* @param {Buffer | null} contentKey 随机密钥
|
|
119
119
|
* @returns {Readable} 明文流
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { bytesToBase64 } from '../../core/bytes_codec.mjs'
|
|
2
|
+
import { assertHex64 } from '../../core/hexIds.mjs'
|
|
3
|
+
import { ms } from '../../utils/duration.mjs'
|
|
4
|
+
import { createInflightTable } from '../../utils/inflight_table.mjs'
|
|
5
|
+
import { beginFedFanoutFetch } from '../fed/fetch_shared.mjs'
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
MAX_PENDING_CHUNK_FETCHES,
|
|
9
|
+
registerChunkFetchWait,
|
|
10
|
+
} from './pending.mjs'
|
|
11
|
+
import { fetchFederationChunk } from './provider_registry.mjs'
|
|
12
|
+
import { getChunk, hasChunk, putChunk } from './store.mjs'
|
|
13
|
+
import { verifiedChunkBytes } from './verify.mjs'
|
|
14
|
+
|
|
15
|
+
const DEFAULT_CHUNK_FETCH_TIMEOUT_MS = ms('8s')
|
|
16
|
+
|
|
17
|
+
/** 同 username+hash 共享一次 fanout;队满只丢已超基础超时的队首。 */
|
|
18
|
+
const chunkInflight = createInflightTable({
|
|
19
|
+
maxSize: MAX_PENDING_CHUNK_FETCHES,
|
|
20
|
+
baseTimeoutMs: DEFAULT_CHUNK_FETCH_TIMEOUT_MS,
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @typedef {{
|
|
25
|
+
* username: string,
|
|
26
|
+
* ciphertextHash: string,
|
|
27
|
+
* ownerEntityHash?: string,
|
|
28
|
+
* groupId?: string,
|
|
29
|
+
* }} FetchChunkContext
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* @param {FetchChunkContext} context 上下文
|
|
34
|
+
* @returns {Promise<Uint8Array | null>} 密文块
|
|
35
|
+
*/
|
|
36
|
+
export async function fetchChunk(context) {
|
|
37
|
+
const hash = String(context.ciphertextHash || '').trim().toLowerCase()
|
|
38
|
+
const { username } = context
|
|
39
|
+
if (!hash || !username) return null
|
|
40
|
+
|
|
41
|
+
if (await hasChunk(hash))
|
|
42
|
+
return new Uint8Array(await getChunk(hash))
|
|
43
|
+
|
|
44
|
+
if (context.groupId) {
|
|
45
|
+
const verified = verifiedChunkBytes(hash, await fetchFederationChunk(username, context.groupId, hash))
|
|
46
|
+
if (verified) {
|
|
47
|
+
await putChunk(hash, verified)
|
|
48
|
+
return verified
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const shared = beginFedFanoutFetch({
|
|
53
|
+
inflight: chunkInflight,
|
|
54
|
+
inflightKey: `${username}\0${hash}`,
|
|
55
|
+
username,
|
|
56
|
+
action: 'fed_chunk_get',
|
|
57
|
+
/**
|
|
58
|
+
* @param {string} requestId 请求 id
|
|
59
|
+
* @returns {{ done: Promise<Uint8Array | null>, cancel: () => void }} 等待句柄
|
|
60
|
+
*/
|
|
61
|
+
registerWait: requestId => registerChunkFetchWait(requestId, hash, DEFAULT_CHUNK_FETCH_TIMEOUT_MS),
|
|
62
|
+
/**
|
|
63
|
+
* @param {string} requestId 请求 id
|
|
64
|
+
* @param {string} nodeHash 目标节点 hash
|
|
65
|
+
* @returns {object} fanout 载荷
|
|
66
|
+
*/
|
|
67
|
+
buildPayload: (requestId, nodeHash) => ({
|
|
68
|
+
requestId,
|
|
69
|
+
nodeHash,
|
|
70
|
+
chunkHash: hash,
|
|
71
|
+
ownerEntityHash: context.ownerEntityHash,
|
|
72
|
+
}),
|
|
73
|
+
})
|
|
74
|
+
if (!shared) return null
|
|
75
|
+
|
|
76
|
+
const verified = verifiedChunkBytes(hash, await shared)
|
|
77
|
+
if (verified) {
|
|
78
|
+
await putChunk(hash, verified)
|
|
79
|
+
return verified
|
|
80
|
+
}
|
|
81
|
+
return null
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* 若本机有 chunk 则响应 fed_chunk_get。
|
|
86
|
+
* @param {object} payload 请求
|
|
87
|
+
* @param {(response: object, peerId: string) => void} sendResponse 发送
|
|
88
|
+
* @param {string} peerId 对端
|
|
89
|
+
* @returns {Promise<void>}
|
|
90
|
+
*/
|
|
91
|
+
export async function handleIncomingChunkGet(payload, sendResponse, peerId) {
|
|
92
|
+
let hash
|
|
93
|
+
try {
|
|
94
|
+
hash = assertHex64(payload?.chunkHash, 'chunkHash')
|
|
95
|
+
}
|
|
96
|
+
catch { return }
|
|
97
|
+
if (!await hasChunk(hash)) return
|
|
98
|
+
const chunkBytes = await getChunk(hash)
|
|
99
|
+
if (!chunkBytes?.length) return
|
|
100
|
+
sendResponse({ requestId: payload.requestId, dataBase64: bytesToBase64(chunkBytes) }, peerId)
|
|
101
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { base64ToBytes } from '../../core/bytes_codec.mjs'
|
|
2
|
+
import { createFetchWaitTable } from '../../utils/fetch_wait.mjs'
|
|
3
|
+
|
|
4
|
+
import { verifiedChunkBytes } from './verify.mjs'
|
|
5
|
+
|
|
6
|
+
/** 并发 pending chunk fetch 上限。 */
|
|
7
|
+
export const MAX_PENDING_CHUNK_FETCHES = 2048
|
|
8
|
+
|
|
9
|
+
const table = createFetchWaitTable({ maxSize: MAX_PENDING_CHUNK_FETCHES })
|
|
10
|
+
|
|
11
|
+
/** @type {typeof table.pending} */
|
|
12
|
+
export const pendingChunkFetches = table.pending
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* 注册 chunk 拉取等待槽(requestId 或 compositeKey)。
|
|
16
|
+
* @param {string} key 唯一等待键
|
|
17
|
+
* @param {string} expectedHash 期望 64 hex 密文哈希
|
|
18
|
+
* @param {number} timeoutMs 超时毫秒
|
|
19
|
+
* @param {{ rejectOnTimeout?: boolean }} [options] rejectOnTimeout 时 Promise 以 Error 拒绝
|
|
20
|
+
* @returns {{ done: Promise<Uint8Array | null>, cancel: () => void }} 等待 Promise 与取消函数
|
|
21
|
+
*/
|
|
22
|
+
export function registerChunkFetchWait(key, expectedHash, timeoutMs, options = {}) {
|
|
23
|
+
return table.register(key, expectedHash, timeoutMs, options)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* 按等待键解析入站 chunk 响应(校验哈希后 resolve)。
|
|
28
|
+
* @param {string} key 等待键
|
|
29
|
+
* @param {string} expectedHash 期望哈希
|
|
30
|
+
* @param {Uint8Array | null} bytes 密文块
|
|
31
|
+
* @returns {boolean} 是否命中并完成等待
|
|
32
|
+
*/
|
|
33
|
+
export function resolveChunkFetchWait(key, expectedHash, bytes) {
|
|
34
|
+
const entry = table.peek(key)
|
|
35
|
+
if (!entry || entry.expectedKey !== expectedHash) return false
|
|
36
|
+
const verified = bytes ? verifiedChunkBytes(expectedHash, bytes) : null
|
|
37
|
+
if (bytes && !verified) return false
|
|
38
|
+
return table.settle(key, verified)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 处理 fed_chunk_data / 带 requestId 的响应载荷。
|
|
43
|
+
* @param {object} payload 入站载荷
|
|
44
|
+
* @returns {boolean} 是否命中 pending
|
|
45
|
+
*/
|
|
46
|
+
export function resolvePendingChunkFetch(payload) {
|
|
47
|
+
const requestId = typeof payload?.requestId === 'string' ? payload.requestId : ''
|
|
48
|
+
if (!requestId) return false
|
|
49
|
+
const entry = table.peek(requestId)
|
|
50
|
+
if (!entry) return false
|
|
51
|
+
if (typeof payload?.dataBase64 === 'string') {
|
|
52
|
+
try {
|
|
53
|
+
return resolveChunkFetchWait(requestId, entry.expectedKey, base64ToBytes(payload.dataBase64))
|
|
54
|
+
}
|
|
55
|
+
catch { /* keep waiting */ }
|
|
56
|
+
return false
|
|
57
|
+
}
|
|
58
|
+
return table.settle(requestId, null)
|
|
59
|
+
}
|
|
@@ -10,7 +10,7 @@ const nodeHashProvidersByOwner = new Map()
|
|
|
10
10
|
* @returns {void}
|
|
11
11
|
*/
|
|
12
12
|
export function registerFederationChunkFetcher(ownerId, fetcher) {
|
|
13
|
-
federationFetchersByOwner.set(
|
|
13
|
+
federationFetchersByOwner.set(ownerId, fetcher)
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
/**
|
|
@@ -19,7 +19,7 @@ export function registerFederationChunkFetcher(ownerId, fetcher) {
|
|
|
19
19
|
* @returns {void}
|
|
20
20
|
*/
|
|
21
21
|
export function registerNodeHashProvider(ownerId, provider) {
|
|
22
|
-
nodeHashProvidersByOwner.set(
|
|
22
|
+
nodeHashProvidersByOwner.set(ownerId, provider)
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
/**
|
|
@@ -27,9 +27,8 @@ export function registerNodeHashProvider(ownerId, provider) {
|
|
|
27
27
|
* @returns {void}
|
|
28
28
|
*/
|
|
29
29
|
export function unregisterChunkProviders(ownerId) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
nodeHashProvidersByOwner.delete(key)
|
|
30
|
+
federationFetchersByOwner.delete(ownerId)
|
|
31
|
+
nodeHashProvidersByOwner.delete(ownerId)
|
|
33
32
|
}
|
|
34
33
|
|
|
35
34
|
/**
|