@steve02081504/fount-p2p 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -0
- package/core/bytes_codec.mjs +51 -0
- package/core/canonical_json.mjs +21 -0
- package/core/composite_key.mjs +91 -0
- package/core/constants.mjs +53 -0
- package/core/entity_id.mjs +53 -0
- package/core/entity_id_parse.mjs +40 -0
- package/core/evfs_logical_path.mjs +15 -0
- package/core/hexIds.mjs +73 -0
- package/core/hlc.mjs +114 -0
- package/core/random_id.mjs +9 -0
- package/crypto/channel.mjs +101 -0
- package/crypto/checkpoint_sign.mjs +67 -0
- package/crypto/crypto.mjs +158 -0
- package/crypto/key.mjs +431 -0
- package/dag/canonicalize_row.mjs +65 -0
- package/dag/event_query.mjs +15 -0
- package/dag/index.mjs +335 -0
- package/dag/storage.mjs +333 -0
- package/dag/strip_extensions.mjs +18 -0
- package/dag/tunables.json +6 -0
- package/discovery/bt.mjs +260 -0
- package/discovery/index.mjs +134 -0
- package/discovery/mdns.mjs +138 -0
- package/discovery/nostr.mjs +273 -0
- package/entity/files/acl.mjs +38 -0
- package/entity/files/evfs.mjs +189 -0
- package/entity/files/evfs_ref.mjs +37 -0
- package/entity/files/manifest_acl_registry.mjs +93 -0
- package/entity/files/replica_host_cache.mjs +46 -0
- package/entity/files/url.mjs +19 -0
- package/entity/hosting_registry.mjs +47 -0
- package/entity/localized_core.mjs +141 -0
- package/entity/logical_entity.mjs +25 -0
- package/entity/logical_entity_id_registry.mjs +40 -0
- package/entity/node_hash.mjs +15 -0
- package/entity/presentation_registry.mjs +44 -0
- package/entity/profile.mjs +256 -0
- package/entity/replica.mjs +20 -0
- package/entity/session_snapshot_registry.mjs +38 -0
- package/federation/chunk_fetch_pending.mjs +128 -0
- package/federation/chunk_fetch_scheduler.mjs +97 -0
- package/federation/dag_order_cache.mjs +124 -0
- package/federation/dedupe_slot.mjs +33 -0
- package/federation/message_rate_limit.mjs +23 -0
- package/federation/operator_key_chain.mjs +200 -0
- package/federation/topo_order_memo.mjs +44 -0
- package/federation/volatile_streams.mjs +52 -0
- package/federation/want_ids.mjs +158 -0
- package/files/assemble.mjs +220 -0
- package/files/assemble_stream.mjs +129 -0
- package/files/chunk_fetch.mjs +113 -0
- package/files/chunk_fetch_verify.mjs +23 -0
- package/files/chunk_provider_registry.mjs +70 -0
- package/files/chunk_responder.mjs +79 -0
- package/files/chunk_store.mjs +87 -0
- package/files/manifest.mjs +99 -0
- package/files/transfer_key.mjs +87 -0
- package/files/transfer_key_registry.mjs +108 -0
- package/governance/branch.mjs +230 -0
- package/governance/join_pow.mjs +165 -0
- package/governance/owner_succession_ballot.mjs +57 -0
- package/governance/tunables.json +7 -0
- package/index.mjs +39 -0
- package/link/channel_mux.mjs +222 -0
- package/link/frame.mjs +246 -0
- package/link/handshake.mjs +175 -0
- package/link/link.mjs +617 -0
- package/link/rtc.mjs +109 -0
- package/link/sdp_fingerprint.mjs +25 -0
- package/mailbox/consumer_registry.mjs +71 -0
- package/mailbox/deliver_or_store.mjs +190 -0
- package/mailbox/importance.mjs +72 -0
- package/mailbox/parse.mjs +73 -0
- package/mailbox/prune.mjs +106 -0
- package/mailbox/rate.mjs +86 -0
- package/mailbox/settings.mjs +53 -0
- package/mailbox/store.mjs +256 -0
- package/mailbox/tunables.json +12 -0
- package/mailbox/wire.mjs +40 -0
- package/node/denylist.mjs +309 -0
- package/node/entity_store.mjs +214 -0
- package/node/identity.mjs +99 -0
- package/node/instance.mjs +115 -0
- package/node/network.mjs +306 -0
- package/node/personal_block.mjs +286 -0
- package/node/reputation_store.mjs +326 -0
- package/node/retention_policy.mjs +82 -0
- package/node/signaling_config.mjs +32 -0
- package/node/storage.mjs +30 -0
- package/node/storage_plugins.mjs +71 -0
- package/node/user_paths.mjs +10 -0
- package/overlay/index.mjs +191 -0
- package/package.json +73 -0
- package/permissions/bitmask.mjs +35 -0
- package/permissions/evaluator.mjs +90 -0
- package/permissions/index.mjs +12 -0
- package/permissions/layered.mjs +39 -0
- package/registries/action.mjs +94 -0
- package/registries/event_type.mjs +75 -0
- package/registries/inbound.mjs +90 -0
- package/registries/p2p_viewer.mjs +35 -0
- package/registries/part_path.mjs +35 -0
- package/registries/room_provider.mjs +47 -0
- package/reputation/engine.mjs +458 -0
- package/reputation/math.mjs +73 -0
- package/reputation/pick_score.mjs +11 -0
- package/reputation/relay_dedupe.mjs +22 -0
- package/reputation/tunables.json +36 -0
- package/rooms/scoped_link.mjs +185 -0
- package/schemas/discovery.mjs +110 -0
- package/schemas/federation_pull.mjs +102 -0
- package/schemas/mailbox.mjs +37 -0
- package/schemas/remote_event.mjs +17 -0
- package/timeline/append_core.mjs +54 -0
- package/timeline/materialize_runner.mjs +26 -0
- package/timeline/prune.mjs +39 -0
- package/timeline/retention.mjs +54 -0
- package/timeline/retention_runner.mjs +35 -0
- package/timeline/verify_remote.mjs +25 -0
- package/transport/group_link_set.mjs +324 -0
- package/transport/ice_servers.mjs +72 -0
- package/transport/link_registry.mjs +749 -0
- package/transport/peer_identity_maps.mjs +94 -0
- package/transport/peer_pool.mjs +355 -0
- package/transport/remote_user_room.mjs +109 -0
- package/transport/room_scopes.mjs +2 -0
- package/transport/room_wire_action.mjs +19 -0
- package/transport/rtc_connection_budget.mjs +180 -0
- package/transport/rtc_mdns_filter.mjs +118 -0
- package/transport/stale_peer_log.mjs +50 -0
- package/transport/user_room.mjs +263 -0
- package/trust_graph/build.mjs +99 -0
- package/trust_graph/cache.mjs +44 -0
- package/trust_graph/engine.mjs +184 -0
- package/trust_graph/registry.mjs +38 -0
- package/trust_graph/resolve.mjs +105 -0
- package/trust_graph/send.mjs +88 -0
- package/trust_graph/tunables.json +13 -0
- package/utils/async_mutex.mjs +69 -0
- package/utils/debug_log.mjs +16 -0
- package/utils/duration.mjs +28 -0
- package/utils/json_io.mjs +57 -0
- package/utils/lru.mjs +22 -0
- package/utils/map_pool.mjs +27 -0
- package/wire/group_part.mjs +43 -0
- package/wire/ingress.mjs +58 -0
- package/wire/part_common.mjs +55 -0
- package/wire/part_fanout.mjs +71 -0
- package/wire/part_ingress.mjs +172 -0
- package/wire/part_invoke.mjs +81 -0
- package/wire/rate_bucket.mjs +49 -0
- package/wire/volatile_signature.mjs +76 -0
- package/wire/wait.mjs +127 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { createHash } from 'node:crypto'
|
|
3
|
+
import { setImmediate } from 'node:timers'
|
|
4
|
+
|
|
5
|
+
import { FEDERATION_CHUNK_MAX_BYTES } from '../core/constants.mjs'
|
|
6
|
+
import {
|
|
7
|
+
encryptConvergentPlaintext,
|
|
8
|
+
encryptRandomPlaintext,
|
|
9
|
+
wrapContentKey,
|
|
10
|
+
} from '../crypto/key.mjs'
|
|
11
|
+
|
|
12
|
+
import { normalizeFileManifest, publicTransferKeyDescriptor } from './manifest.mjs'
|
|
13
|
+
|
|
14
|
+
/** @type {Record<string, (plain: Buffer) => { contentHash: string, ciphertextHash: string, raw: Buffer, contentKey?: Buffer }>} */
|
|
15
|
+
const ENCRYPTION_STRATEGIES = {
|
|
16
|
+
/**
|
|
17
|
+
* @param {Buffer} plain 明文
|
|
18
|
+
* @returns {{ contentHash: string, ciphertextHash: string, raw: Buffer }} 分块结果
|
|
19
|
+
*/
|
|
20
|
+
plain: (plain) => {
|
|
21
|
+
const contentHash = createHash('sha256').update(plain).digest('hex')
|
|
22
|
+
return { contentHash, ciphertextHash: contentHash, raw: plain }
|
|
23
|
+
},
|
|
24
|
+
/**
|
|
25
|
+
* @param {Buffer} plain 明文
|
|
26
|
+
* @returns {{ contentHash: string, ciphertextHash: string, raw: Buffer }} 分块结果
|
|
27
|
+
*/
|
|
28
|
+
convergent: (plain) => encryptConvergentPlaintext(plain),
|
|
29
|
+
/**
|
|
30
|
+
* @param {Buffer} plain 明文
|
|
31
|
+
* @returns {{ contentHash: string, ciphertextHash: string, raw: Buffer, contentKey: Buffer }} 分块结果
|
|
32
|
+
*/
|
|
33
|
+
random: (plain) => encryptRandomPlaintext(plain),
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @typedef {import('./manifest.mjs').FileManifest} FileManifest
|
|
38
|
+
* @typedef {import('./manifest.mjs').CeMode} CeMode
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {CeMode} ceMode 模式
|
|
43
|
+
* @returns {(plain: Buffer) => { contentHash: string, ciphertextHash: string, raw: Buffer, contentKey?: Buffer }} 加密策略
|
|
44
|
+
*/
|
|
45
|
+
function encryptionStrategyFor(ceMode) {
|
|
46
|
+
const strategy = ENCRYPTION_STRATEGIES[ceMode]
|
|
47
|
+
if (!strategy) throw new Error(`unknown ceMode: ${ceMode}`)
|
|
48
|
+
return strategy
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @param {Buffer | Uint8Array} plaintext 明文
|
|
53
|
+
* @param {CeMode} ceMode 模式
|
|
54
|
+
* @returns {{ contentHash: string, parts: Array<{ hash: string, size: number, raw: Buffer }>, contentKey?: Buffer }} 加密结果
|
|
55
|
+
*/
|
|
56
|
+
export function encryptPlaintextToParts(plaintext, ceMode = 'convergent') {
|
|
57
|
+
const plain = Buffer.from(plaintext)
|
|
58
|
+
const enc = encryptionStrategyFor(ceMode)(plain)
|
|
59
|
+
return {
|
|
60
|
+
contentHash: enc.contentHash,
|
|
61
|
+
parts: [{ hash: enc.ciphertextHash, size: enc.raw.length, raw: enc.raw }],
|
|
62
|
+
contentKey: enc.contentKey,
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* 将明文拆分为多块加密(大文件)。
|
|
68
|
+
* @param {Buffer | Uint8Array} plaintext 明文
|
|
69
|
+
* @param {CeMode} ceMode 模式
|
|
70
|
+
* @returns {{ contentHash: string, parts: Array<{ hash: string, size: number, raw: Buffer }>, contentKey?: Buffer }} 分块加密结果
|
|
71
|
+
*/
|
|
72
|
+
export function encryptPlaintextToMultiParts(plaintext, ceMode = 'convergent') {
|
|
73
|
+
const plain = Buffer.from(plaintext)
|
|
74
|
+
const contentHash = createHash('sha256').update(plain).digest('hex')
|
|
75
|
+
if (plain.length <= FEDERATION_CHUNK_MAX_BYTES)
|
|
76
|
+
return encryptPlaintextToParts(plain, ceMode)
|
|
77
|
+
|
|
78
|
+
/** @type {Array<{ hash: string, size: number, raw: Buffer }>} */
|
|
79
|
+
const parts = []
|
|
80
|
+
let contentKey = null
|
|
81
|
+
for (let offset = 0; offset < plain.length; offset += FEDERATION_CHUNK_MAX_BYTES) {
|
|
82
|
+
const slice = plain.subarray(offset, offset + FEDERATION_CHUNK_MAX_BYTES)
|
|
83
|
+
const enc = encryptionStrategyFor(ceMode)(slice)
|
|
84
|
+
if (ceMode === 'random') contentKey = enc.contentKey
|
|
85
|
+
parts.push({ hash: enc.ciphertextHash, size: enc.raw.length, raw: enc.raw })
|
|
86
|
+
}
|
|
87
|
+
return { contentHash, parts, contentKey: contentKey || undefined }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* 异步多块加密,周期性让出事件循环。
|
|
92
|
+
* @param {Buffer | Uint8Array} plaintext 明文
|
|
93
|
+
* @param {CeMode} ceMode 模式
|
|
94
|
+
* @returns {Promise<{ contentHash: string, parts: Array<{ hash: string, size: number, raw: Buffer }>, contentKey?: Buffer }>} 分块加密结果
|
|
95
|
+
*/
|
|
96
|
+
export async function encryptPlaintextToMultiPartsAsync(plaintext, ceMode = 'convergent') {
|
|
97
|
+
const plain = Buffer.from(plaintext)
|
|
98
|
+
const contentHash = createHash('sha256').update(plain).digest('hex')
|
|
99
|
+
if (plain.length <= FEDERATION_CHUNK_MAX_BYTES)
|
|
100
|
+
return encryptPlaintextToParts(plain, ceMode)
|
|
101
|
+
|
|
102
|
+
/** @type {Array<{ hash: string, size: number, raw: Buffer }>} */
|
|
103
|
+
const parts = []
|
|
104
|
+
let contentKey = null
|
|
105
|
+
for (let offset = 0; offset < plain.length; offset += FEDERATION_CHUNK_MAX_BYTES) {
|
|
106
|
+
if (offset > 0) await new Promise(resolve => setImmediate(resolve))
|
|
107
|
+
const slice = plain.subarray(offset, offset + FEDERATION_CHUNK_MAX_BYTES)
|
|
108
|
+
const enc = encryptionStrategyFor(ceMode)(slice)
|
|
109
|
+
if (ceMode === 'random') contentKey = enc.contentKey
|
|
110
|
+
parts.push({ hash: enc.ciphertextHash, size: enc.raw.length, raw: enc.raw })
|
|
111
|
+
}
|
|
112
|
+
return { contentHash, parts, contentKey: contentKey || undefined }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* @param {object} params 参数
|
|
117
|
+
* @param {string} params.ownerEntityHash 128 hex
|
|
118
|
+
* @param {string} params.logicalPath EVFS 路径
|
|
119
|
+
* @param {Buffer | Uint8Array} params.plaintext 明文
|
|
120
|
+
* @param {string} [params.name] 文件名
|
|
121
|
+
* @param {string} [params.mimeType] MIME
|
|
122
|
+
* @param {CeMode} [params.ceMode] 加密模式
|
|
123
|
+
* @param {import('./manifest.mjs').TransferKeyDescriptor} [params.transferKeyDescriptor] 传递密钥
|
|
124
|
+
* @param {object} [params.meta] 元数据
|
|
125
|
+
* @returns {FileManifest} manifest(未写盘)
|
|
126
|
+
*/
|
|
127
|
+
export function buildFileManifest(params) {
|
|
128
|
+
const {
|
|
129
|
+
ownerEntityHash,
|
|
130
|
+
logicalPath,
|
|
131
|
+
plaintext,
|
|
132
|
+
name,
|
|
133
|
+
mimeType = 'application/octet-stream',
|
|
134
|
+
ceMode = 'convergent',
|
|
135
|
+
transferKeyDescriptor,
|
|
136
|
+
meta,
|
|
137
|
+
} = params
|
|
138
|
+
const enc = encryptPlaintextToParts(plaintext, ceMode)
|
|
139
|
+
const manifest = normalizeFileManifest({
|
|
140
|
+
ownerEntityHash: ownerEntityHash.toLowerCase(),
|
|
141
|
+
logicalPath: logicalPath.replace(/^\/+/, ''),
|
|
142
|
+
name: name || logicalPath.split('/').pop() || 'file',
|
|
143
|
+
mimeType,
|
|
144
|
+
size: Buffer.from(plaintext).length,
|
|
145
|
+
contentHash: enc.contentHash,
|
|
146
|
+
ceMode,
|
|
147
|
+
parts: enc.parts.map(part => ({ hash: part.hash, size: part.size })),
|
|
148
|
+
transferKeyDescriptor: transferKeyDescriptor || publicTransferKeyDescriptor(),
|
|
149
|
+
meta,
|
|
150
|
+
})
|
|
151
|
+
if (!manifest) throw new Error('invalid manifest')
|
|
152
|
+
return manifest
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* 由已加密分块构建 manifest(vault / file-master-key-wrap 等需自定义 transferKeyDescriptor)。
|
|
157
|
+
* @param {object} params 与 buildFileManifest 相同字段(不含 plaintext 重加密)
|
|
158
|
+
* @param {{ contentHash: string, parts: Array<{ hash: string, size: number, raw?: Buffer }> }} enc 加密结果
|
|
159
|
+
* @returns {FileManifest} manifest
|
|
160
|
+
*/
|
|
161
|
+
export function buildFileManifestFromEnc(params, enc) {
|
|
162
|
+
const {
|
|
163
|
+
ownerEntityHash,
|
|
164
|
+
logicalPath,
|
|
165
|
+
plaintext,
|
|
166
|
+
name,
|
|
167
|
+
mimeType = 'application/octet-stream',
|
|
168
|
+
ceMode = 'convergent',
|
|
169
|
+
transferKeyDescriptor,
|
|
170
|
+
meta,
|
|
171
|
+
} = params
|
|
172
|
+
const manifest = normalizeFileManifest({
|
|
173
|
+
ownerEntityHash: ownerEntityHash.toLowerCase(),
|
|
174
|
+
logicalPath: logicalPath.replace(/^\/+/, ''),
|
|
175
|
+
name: name || logicalPath.split('/').pop() || 'file',
|
|
176
|
+
mimeType,
|
|
177
|
+
size: Buffer.from(plaintext).length,
|
|
178
|
+
contentHash: enc.contentHash,
|
|
179
|
+
ceMode,
|
|
180
|
+
parts: enc.parts.map(part => ({ hash: part.hash, size: part.size })),
|
|
181
|
+
transferKeyDescriptor: transferKeyDescriptor || publicTransferKeyDescriptor(),
|
|
182
|
+
meta,
|
|
183
|
+
})
|
|
184
|
+
if (!manifest) throw new Error('invalid manifest')
|
|
185
|
+
return manifest
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* @param {string} groupId 群 ID
|
|
190
|
+
* @param {Buffer} contentKey 随机 contentKey
|
|
191
|
+
* @param {Buffer | string} H 群/vault 秘密
|
|
192
|
+
* @param {string} fileId 文件 ID
|
|
193
|
+
* @param {number} keyGeneration 密钥代次
|
|
194
|
+
* @returns {import('./manifest.mjs').TransferKeyDescriptor} 传输密钥描述
|
|
195
|
+
*/
|
|
196
|
+
export function fileMasterKeyWrapDescriptor(groupId, fileId, contentKey, H, keyGeneration = 0) {
|
|
197
|
+
return {
|
|
198
|
+
type: 'file-master-key-wrap',
|
|
199
|
+
groupId,
|
|
200
|
+
fileId,
|
|
201
|
+
keyGeneration,
|
|
202
|
+
wrappedKey: wrapContentKey(contentKey, H, fileId),
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* @param {string} entityHash 所有者
|
|
208
|
+
* @param {string} fileId 文件 ID
|
|
209
|
+
* @param {Buffer} contentKey 随机密钥
|
|
210
|
+
* @param {Buffer | string} H vault H
|
|
211
|
+
* @returns {import('./manifest.mjs').TransferKeyDescriptor} 传输密钥描述
|
|
212
|
+
*/
|
|
213
|
+
export function vaultWrapDescriptor(entityHash, fileId, contentKey, H) {
|
|
214
|
+
return {
|
|
215
|
+
type: 'vault-wrap',
|
|
216
|
+
entityHash: entityHash.toLowerCase(),
|
|
217
|
+
fileId,
|
|
218
|
+
wrappedKey: wrapContentKey(contentKey, H, fileId),
|
|
219
|
+
}
|
|
220
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { createHash } from 'node:crypto'
|
|
3
|
+
import { Readable } from 'node:stream'
|
|
4
|
+
|
|
5
|
+
import { FEDERATION_CHUNK_MAX_BYTES } from '../core/constants.mjs'
|
|
6
|
+
import {
|
|
7
|
+
encryptConvergentPlaintext,
|
|
8
|
+
encryptRandomPlaintext,
|
|
9
|
+
} from '../crypto/key.mjs'
|
|
10
|
+
|
|
11
|
+
import { decryptPart } from './transfer_key.mjs'
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @typedef {import('./manifest.mjs').CeMode} CeMode
|
|
15
|
+
* @typedef {{ hash: string, size: number, raw: Buffer }} EncryptedPart
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* 从可读流分块加密并流式落盘(每块经 onPart 回调)。
|
|
20
|
+
* @param {import('node:stream').Readable} readable 明文流
|
|
21
|
+
* @param {CeMode} [ceMode] 加密模式
|
|
22
|
+
* @param {(part: EncryptedPart) => Promise<void>} onPart 每块回调
|
|
23
|
+
* @param {number} [maxBytes] 最大字节
|
|
24
|
+
* @returns {Promise<{ contentHash: string, parts: Array<{ hash: string, size: number }>, contentKey?: Buffer }>} 分块结果
|
|
25
|
+
*/
|
|
26
|
+
export async function encryptReadableToParts(readable, ceMode = 'convergent', onPart, maxBytes = Infinity) {
|
|
27
|
+
const digest = createHash('sha256')
|
|
28
|
+
/** @type {Buffer[]} */
|
|
29
|
+
let pending = Buffer.alloc(0)
|
|
30
|
+
/** @type {Array<{ hash: string, size: number }>} */
|
|
31
|
+
const parts = []
|
|
32
|
+
let contentKey = null
|
|
33
|
+
let total = 0
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @param {Buffer} slice 明文块
|
|
37
|
+
* @returns {Promise<void>}
|
|
38
|
+
*/
|
|
39
|
+
const flushSlice = async (slice) => {
|
|
40
|
+
digest.update(slice)
|
|
41
|
+
const enc = ceMode === 'random'
|
|
42
|
+
? encryptRandomPlaintext(slice)
|
|
43
|
+
: encryptConvergentPlaintext(slice)
|
|
44
|
+
if (ceMode === 'random') contentKey = enc.contentKey
|
|
45
|
+
const part = { hash: enc.ciphertextHash, size: enc.raw.length, raw: enc.raw }
|
|
46
|
+
parts.push({ hash: part.hash, size: part.size })
|
|
47
|
+
await onPart(part)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
for await (const chunk of readable) {
|
|
51
|
+
const buf = Buffer.from(chunk)
|
|
52
|
+
total += buf.length
|
|
53
|
+
if (total > maxBytes)
|
|
54
|
+
throw new Error('plaintext exceeds max upload size')
|
|
55
|
+
pending = Buffer.concat([pending, buf])
|
|
56
|
+
while (pending.length >= FEDERATION_CHUNK_MAX_BYTES) {
|
|
57
|
+
const slice = pending.subarray(0, FEDERATION_CHUNK_MAX_BYTES)
|
|
58
|
+
pending = pending.subarray(FEDERATION_CHUNK_MAX_BYTES)
|
|
59
|
+
await flushSlice(slice)
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (pending.length)
|
|
64
|
+
await flushSlice(pending)
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
contentHash: digest.digest('hex'),
|
|
68
|
+
parts,
|
|
69
|
+
contentKey: contentKey || undefined,
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* 将 manifest 各密文块解密后串联为可读流。
|
|
75
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
76
|
+
* @param {import('node:stream').Readable[]} partStreams 按序密文流
|
|
77
|
+
* @param {Buffer | null} contentKey 随机密钥
|
|
78
|
+
* @returns {Readable} 明文流
|
|
79
|
+
*/
|
|
80
|
+
export function createManifestPlaintextStream(manifest, partStreams, contentKey) {
|
|
81
|
+
let partIndex = 0
|
|
82
|
+
let currentStream = null
|
|
83
|
+
const digest = createHash('sha256')
|
|
84
|
+
|
|
85
|
+
return new Readable({
|
|
86
|
+
/**
|
|
87
|
+
*
|
|
88
|
+
*/
|
|
89
|
+
async read() {
|
|
90
|
+
try {
|
|
91
|
+
while (true) {
|
|
92
|
+
if (!currentStream) {
|
|
93
|
+
if (partIndex >= partStreams.length) {
|
|
94
|
+
this.push(null)
|
|
95
|
+
return
|
|
96
|
+
}
|
|
97
|
+
currentStream = partStreams[partIndex++]
|
|
98
|
+
}
|
|
99
|
+
const chunk = currentStream.read()
|
|
100
|
+
if (chunk === null) {
|
|
101
|
+
currentStream = await new Promise((resolve, reject) => {
|
|
102
|
+
currentStream.once('end', () => resolve(null))
|
|
103
|
+
currentStream.once('error', reject)
|
|
104
|
+
currentStream.resume()
|
|
105
|
+
})
|
|
106
|
+
if (currentStream === null) {
|
|
107
|
+
currentStream = null
|
|
108
|
+
continue
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (chunk?.length) {
|
|
112
|
+
const plain = decryptPart(Buffer.from(chunk), manifest, contentKey)
|
|
113
|
+
if (!plain) {
|
|
114
|
+
this.destroy(new Error('decrypt failed'))
|
|
115
|
+
return
|
|
116
|
+
}
|
|
117
|
+
digest.update(plain)
|
|
118
|
+
if (!this.push(plain))
|
|
119
|
+
return
|
|
120
|
+
return
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
this.destroy(error instanceof Error ? error : new Error(String(error)))
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
})
|
|
129
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
import { u8ToB64 } from '../core/bytes_codec.mjs'
|
|
4
|
+
import { FEDERATION_CHUNK_FETCH_FANOUT_K } from '../core/constants.mjs'
|
|
5
|
+
import {
|
|
6
|
+
MAX_PENDING_CHUNK_FETCHES,
|
|
7
|
+
pendingChunkFetches,
|
|
8
|
+
registerChunkFetchWait,
|
|
9
|
+
} from '../federation/chunk_fetch_pending.mjs'
|
|
10
|
+
import { loadNetwork } from '../node/network.mjs'
|
|
11
|
+
import { ensureLinkToNode, listLinks } from '../transport/link_registry.mjs'
|
|
12
|
+
import { DEFAULT_TRUST_GRAPH_OWNER, requireTrustGraphProvider } from '../trust_graph/registry.mjs'
|
|
13
|
+
|
|
14
|
+
import { verifiedChunkBytes } from './chunk_fetch_verify.mjs'
|
|
15
|
+
import { fetchFederationChunk, resolveNodeHash } from './chunk_provider_registry.mjs'
|
|
16
|
+
import { getChunk, hasChunk, putChunk } from './chunk_store.mjs'
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @typedef {{
|
|
20
|
+
* username: string,
|
|
21
|
+
* ciphertextHash: string,
|
|
22
|
+
* ownerEntityHash?: string,
|
|
23
|
+
* groupId?: string,
|
|
24
|
+
* }} FetchChunkContext
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @returns {string[]} 全局 CAS miss 时应尝试拨号/发送的 nodeHash 列表
|
|
29
|
+
*/
|
|
30
|
+
function chunkFetchPeerTargets() {
|
|
31
|
+
/** @type {Set<string>} */
|
|
32
|
+
const targets = new Set()
|
|
33
|
+
for (const { nodeHash } of listLinks())
|
|
34
|
+
if (nodeHash) targets.add(String(nodeHash).toLowerCase())
|
|
35
|
+
const net = loadNetwork()
|
|
36
|
+
for (const nodeHash of [...net.trustedPeers || [], ...net.explorePeers || []])
|
|
37
|
+
if (nodeHash) targets.add(String(nodeHash).toLowerCase())
|
|
38
|
+
for (const hint of net.hints || [])
|
|
39
|
+
if (hint?.nodeHash) targets.add(String(hint.nodeHash).toLowerCase())
|
|
40
|
+
return [...targets]
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* @param {FetchChunkContext} context 上下文
|
|
45
|
+
* @returns {Promise<Uint8Array | null>} 密文块
|
|
46
|
+
*/
|
|
47
|
+
export async function fetchChunk(context) {
|
|
48
|
+
const hash = String(context.ciphertextHash || '').trim().toLowerCase()
|
|
49
|
+
const username = context.username
|
|
50
|
+
if (!hash || !username) return null
|
|
51
|
+
|
|
52
|
+
if (await hasChunk(hash))
|
|
53
|
+
return new Uint8Array(await getChunk(hash))
|
|
54
|
+
|
|
55
|
+
if (context.groupId) {
|
|
56
|
+
const u8 = await fetchFederationChunk(username, context.groupId, hash)
|
|
57
|
+
const verified = verifiedChunkBytes(hash, u8)
|
|
58
|
+
if (verified) {
|
|
59
|
+
await putChunk(hash, verified)
|
|
60
|
+
return verified
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (pendingChunkFetches.size >= MAX_PENDING_CHUNK_FETCHES) return null
|
|
65
|
+
|
|
66
|
+
const requestId = randomUUID()
|
|
67
|
+
const { done } = registerChunkFetchWait(requestId, hash, 8000)
|
|
68
|
+
const { nodeHash } = await resolveNodeHash(username)
|
|
69
|
+
const payload = {
|
|
70
|
+
requestId,
|
|
71
|
+
nodeHash,
|
|
72
|
+
chunkHash: hash,
|
|
73
|
+
ownerEntityHash: context.ownerEntityHash,
|
|
74
|
+
}
|
|
75
|
+
const graph = await requireTrustGraphProvider(DEFAULT_TRUST_GRAPH_OWNER).buildMergedGraph(username)
|
|
76
|
+
const peerTargets = chunkFetchPeerTargets()
|
|
77
|
+
await Promise.all(peerTargets.map(nodeHash => ensureLinkToNode(nodeHash).catch(() => null)))
|
|
78
|
+
const tg = requireTrustGraphProvider(DEFAULT_TRUST_GRAPH_OWNER)
|
|
79
|
+
// 已直连 / follow hint peer 可能不在 trust-graph top-K(非成员 emoji CAS / Social 预览路径)。
|
|
80
|
+
for (const nodeHash of peerTargets)
|
|
81
|
+
void tg.sendToNode(username, nodeHash, 'fed_chunk_get', payload, graph)
|
|
82
|
+
await tg.fanoutToTopNodes(
|
|
83
|
+
username,
|
|
84
|
+
'fed_chunk_get',
|
|
85
|
+
payload,
|
|
86
|
+
FEDERATION_CHUNK_FETCH_FANOUT_K,
|
|
87
|
+
)
|
|
88
|
+
const result = await done
|
|
89
|
+
const verified = verifiedChunkBytes(hash, result)
|
|
90
|
+
if (verified) {
|
|
91
|
+
await putChunk(hash, verified)
|
|
92
|
+
return verified
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return null
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* 若本机有 chunk 则响应 fed_chunk_get。
|
|
100
|
+
* @param {string} username 用户
|
|
101
|
+
* @param {object} payload 请求
|
|
102
|
+
* @param {(response: object, peerId: string) => void} sendResponse 发送
|
|
103
|
+
* @param {string} peerId 对端
|
|
104
|
+
* @returns {Promise<void>}
|
|
105
|
+
*/
|
|
106
|
+
export async function handleIncomingChunkGet(username, payload, sendResponse, peerId) {
|
|
107
|
+
const hash = String(payload?.chunkHash || '').trim().toLowerCase()
|
|
108
|
+
if (!hash) return
|
|
109
|
+
if (!await hasChunk(hash)) return
|
|
110
|
+
const chunkBytes = await getChunk(hash)
|
|
111
|
+
if (!chunkBytes?.length) return
|
|
112
|
+
sendResponse({ requestId: payload.requestId, dataB64: u8ToB64(chunkBytes) }, peerId)
|
|
113
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { isHex64 } from '../core/hexIds.mjs'
|
|
2
|
+
import { sha256Hex } from '../crypto/crypto.mjs'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @param {string} chunkHash 期望的 64 hex 密文哈希
|
|
6
|
+
* @param {Uint8Array | Buffer | null | undefined} data 块字节
|
|
7
|
+
* @returns {boolean} 是否与 hash 一致
|
|
8
|
+
*/
|
|
9
|
+
export function chunkBytesMatchHash(chunkHash, data) {
|
|
10
|
+
const hash = String(chunkHash || '').trim().toLowerCase()
|
|
11
|
+
if (!isHex64(hash) || !data?.byteLength) return false
|
|
12
|
+
return sha256Hex(data) === hash
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} chunkHash 期望哈希
|
|
17
|
+
* @param {Uint8Array | Buffer} data 块字节
|
|
18
|
+
* @returns {Uint8Array | null} 校验通过的数据;否则 null
|
|
19
|
+
*/
|
|
20
|
+
export function verifiedChunkBytes(chunkHash, data) {
|
|
21
|
+
if (!chunkBytesMatchHash(chunkHash, data)) return null
|
|
22
|
+
return data instanceof Uint8Array ? data : new Uint8Array(data)
|
|
23
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/** @type {Map<string, (username: string, groupId: string, hash: string) => Promise<Uint8Array | null>>} */
|
|
2
|
+
const federationFetchersByOwner = new Map()
|
|
3
|
+
|
|
4
|
+
/** @type {Map<string, (username: string) => Promise<{ nodeHash: string }> | { nodeHash: string }>} */
|
|
5
|
+
const nodeHashProvidersByOwner = new Map()
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {string} ownerId 注册方
|
|
9
|
+
* @param {(username: string, groupId: string, hash: string) => Promise<Uint8Array | null>} fetcher 联邦 chunk 拉取
|
|
10
|
+
* @returns {void}
|
|
11
|
+
*/
|
|
12
|
+
export function registerFederationChunkFetcher(ownerId, fetcher) {
|
|
13
|
+
federationFetchersByOwner.set(String(ownerId), fetcher)
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* @param {string} ownerId 注册方
|
|
18
|
+
* @param {(username: string) => Promise<{ nodeHash: string }> | { nodeHash: string }} provider nodeHash
|
|
19
|
+
* @returns {void}
|
|
20
|
+
*/
|
|
21
|
+
export function registerNodeHashProvider(ownerId, provider) {
|
|
22
|
+
nodeHashProvidersByOwner.set(String(ownerId), provider)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {string} ownerId 注册方
|
|
27
|
+
* @returns {void}
|
|
28
|
+
*/
|
|
29
|
+
export function unregisterChunkProviders(ownerId) {
|
|
30
|
+
const key = String(ownerId)
|
|
31
|
+
federationFetchersByOwner.delete(key)
|
|
32
|
+
nodeHashProvidersByOwner.delete(key)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** @returns {void} */
|
|
36
|
+
export function clearChunkProviderRegistry() {
|
|
37
|
+
federationFetchersByOwner.clear()
|
|
38
|
+
nodeHashProvidersByOwner.clear()
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {string} username 用户
|
|
43
|
+
* @param {string} groupId 群 ID
|
|
44
|
+
* @param {string} hash chunk 哈希
|
|
45
|
+
* @returns {Promise<Uint8Array | null>} 密文块
|
|
46
|
+
*/
|
|
47
|
+
export async function fetchFederationChunk(username, groupId, hash) {
|
|
48
|
+
for (const fetcher of federationFetchersByOwner.values())
|
|
49
|
+
try {
|
|
50
|
+
const u8 = await fetcher(username, groupId, hash)
|
|
51
|
+
if (u8?.byteLength) return u8
|
|
52
|
+
}
|
|
53
|
+
catch { /* next */ }
|
|
54
|
+
|
|
55
|
+
return null
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* @param {string} username 副本用户名 登录名
|
|
60
|
+
* @returns {Promise<{ nodeHash: string }>} 节点标识
|
|
61
|
+
*/
|
|
62
|
+
export async function resolveNodeHash(username) {
|
|
63
|
+
for (const provider of nodeHashProvidersByOwner.values())
|
|
64
|
+
try {
|
|
65
|
+
return await provider(username)
|
|
66
|
+
}
|
|
67
|
+
catch { /* next */ }
|
|
68
|
+
|
|
69
|
+
return { nodeHash: 'local' }
|
|
70
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { resolvePendingChunkFetch } from '../federation/chunk_fetch_pending.mjs'
|
|
2
|
+
|
|
3
|
+
import { handleIncomingChunkGet } from './chunk_fetch.mjs'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @param {string} username 副本用户名 用户名
|
|
7
|
+
* @param {object} data 入站 fed_chunk_get
|
|
8
|
+
* @param {string} peerId 对端 id
|
|
9
|
+
* @param {(resp: object, peerId: string) => void} sendChunkData 发送 fed_chunk_data
|
|
10
|
+
* @returns {Promise<void>}
|
|
11
|
+
*/
|
|
12
|
+
export async function handleFedChunkGetIngress(username, data, peerId, sendChunkData) {
|
|
13
|
+
await handleIncomingChunkGet(username, data, sendChunkData, peerId)
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* @param {object} data 入站 fed_chunk_data(含 requestId 时 resolve pending fetch)
|
|
18
|
+
* @returns {void}
|
|
19
|
+
*/
|
|
20
|
+
export function handleFedChunkDataIngress(data) {
|
|
21
|
+
resolvePendingChunkFetch(data)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* node scope user-room wire:注册 fed_chunk_get / fed_chunk_data。
|
|
26
|
+
* @param {string} username 副本用户名 用户名
|
|
27
|
+
* @param {{ on: (name: string, handler: (payload: unknown, peerId: string) => void) => void, send: (name: string, payload: unknown, peerId: string | null) => void }} wire action 表
|
|
28
|
+
* @returns {void}
|
|
29
|
+
*/
|
|
30
|
+
export function attachNodeScopeFedChunkResponder(username, wire) {
|
|
31
|
+
wire.on('fed_chunk_get', (data, peerId) => {
|
|
32
|
+
void handleFedChunkGetIngress(username, data, peerId, (resp, pid) => {
|
|
33
|
+
try { wire.send('fed_chunk_data', resp, pid) }
|
|
34
|
+
catch { /* disconnected */ }
|
|
35
|
+
})
|
|
36
|
+
})
|
|
37
|
+
wire.on('fed_chunk_data', handleFedChunkDataIngress)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Trystero room:注册带 requestId 的 fed_chunk_get / fed_chunk_data(TrustGraph 全局 miss)。
|
|
42
|
+
* @param {string} username 用户
|
|
43
|
+
* @param {object} room Trystero room
|
|
44
|
+
* @param {{ enqueue: (prio: number, fn: () => void) => void }} [fedOut] 出站队列
|
|
45
|
+
* @param {(roomKey: string, action: string, rtcLimits: object) => boolean} [guardGet] RTC 负载守卫
|
|
46
|
+
* @param {object} [rtcLimits] RTC 限额
|
|
47
|
+
* @param {string} [roomKey] 房间键
|
|
48
|
+
* @returns {void}
|
|
49
|
+
*/
|
|
50
|
+
export function attachTrustGraphFedChunkResponder(username, room, fedOut, guardGet, rtcLimits = {}, roomKey = '') {
|
|
51
|
+
const [sendChunkData, getChunkData] = room.makeAction('fed_chunk_data')
|
|
52
|
+
const [, getChunkGet] = room.makeAction('fed_chunk_get')
|
|
53
|
+
|
|
54
|
+
getChunkGet((data, peerId) => {
|
|
55
|
+
if (guardGet && !guardGet(roomKey, 'fed_chunk_get', rtcLimits)) return
|
|
56
|
+
void (async () => {
|
|
57
|
+
if (!data || typeof data !== 'object') return
|
|
58
|
+
if (!String(data.requestId || '')) return
|
|
59
|
+
await handleFedChunkGetIngress(username, data, peerId, (resp, pid) => {
|
|
60
|
+
/**
|
|
61
|
+
*
|
|
62
|
+
*/
|
|
63
|
+
const send = () => {
|
|
64
|
+
try { sendChunkData(resp, pid) }
|
|
65
|
+
catch (error) {
|
|
66
|
+
console.warn('federation: trust-graph chunk response failed', error)
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (fedOut) fedOut.enqueue(6, send)
|
|
70
|
+
else send()
|
|
71
|
+
})
|
|
72
|
+
})().catch(error => console.warn('federation: trust-graph chunk handler failed', error))
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
getChunkData(data => {
|
|
76
|
+
if (!data || typeof data !== 'object' || !data.requestId) return
|
|
77
|
+
handleFedChunkDataIngress(data)
|
|
78
|
+
})
|
|
79
|
+
}
|