@steve02081504/fount-p2p 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/bytes_codec.mjs +0 -27
- package/core/constants.mjs +0 -31
- package/core/hexIds.mjs +0 -13
- package/crypto/key.mjs +14 -15
- package/dag/canonicalize_row.mjs +5 -5
- package/dag/index.mjs +0 -73
- package/dag/storage.mjs +0 -22
- package/discovery/mdns.mjs +4 -4
- package/discovery/nostr.mjs +3 -3
- package/federation/chunk_fetch_pending.mjs +3 -3
- package/federation/chunk_fetch_scheduler.mjs +3 -3
- package/federation/dag_order_cache.mjs +3 -3
- package/federation/entity_key_chain.mjs +0 -9
- package/federation/topo_order_memo.mjs +3 -18
- package/files/assemble.mjs +55 -39
- package/files/assemble_stream.mjs +96 -44
- package/files/chunk_provider_registry.mjs +0 -6
- package/files/chunk_responder.mjs +1 -1
- package/files/chunk_store.mjs +1 -11
- package/files/evfs.mjs +74 -47
- package/files/manifest.mjs +11 -2
- package/files/manifest_acl_registry.mjs +0 -6
- package/files/transfer_key.mjs +21 -15
- package/files/transfer_key_registry.mjs +9 -16
- package/governance/branch.mjs +0 -11
- package/governance/join_pow.mjs +17 -17
- package/link/channel_mux.mjs +9 -7
- package/link/frame.mjs +5 -5
- package/link/handshake.mjs +17 -17
- package/link/link.mjs +33 -33
- package/mailbox/deliver_or_store.mjs +13 -13
- package/mailbox/importance.mjs +0 -33
- package/mailbox/settings.mjs +0 -9
- package/mailbox/wire.mjs +4 -4
- package/node/denylist.mjs +0 -24
- package/node/network.mjs +0 -51
- package/node/personal_block.mjs +0 -35
- package/node/reputation_store.mjs +5 -14
- package/node/retention_policy.mjs +8 -23
- package/overlay/index.mjs +6 -6
- package/package.json +1 -1
- package/registries/event_type.mjs +0 -12
- package/registries/inbound.mjs +8 -30
- package/reputation/engine.mjs +0 -11
- package/rooms/scoped_link.mjs +18 -18
- package/schemas/part_query.mjs +156 -0
- package/timeline/append_core.mjs +3 -3
- package/transport/group_link_set.mjs +30 -30
- package/transport/ice_servers.mjs +0 -8
- package/transport/link_registry.mjs +13 -13
- package/transport/peer_identity_maps.mjs +0 -58
- package/transport/peer_pool.mjs +4 -4
- package/transport/remote_user_room.mjs +0 -12
- package/transport/rtc_connection_budget.mjs +2 -0
- package/transport/user_room.mjs +13 -24
- package/trust_graph/cache.mjs +0 -13
- package/trust_graph/engine.mjs +0 -24
- package/utils/async_mutex.mjs +0 -9
- package/wire/group_part.mjs +3 -3
- package/wire/part_fanout.mjs +0 -19
- package/wire/part_ingress.mjs +14 -14
- package/wire/part_query.mjs +486 -0
- package/wire/part_query.tunables.json +14 -0
- package/wire/part_query_cache.mjs +101 -0
|
@@ -1,18 +1,15 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer'
|
|
2
|
-
import { createHash } from 'node:crypto'
|
|
2
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
3
3
|
import { Readable } from 'node:stream'
|
|
4
4
|
|
|
5
5
|
import { FEDERATION_CHUNK_MAX_BYTES } from '../core/constants.mjs'
|
|
6
|
-
import {
|
|
7
|
-
encryptConvergentPlaintext,
|
|
8
|
-
encryptRandomPlaintext,
|
|
9
|
-
} from '../crypto/key.mjs'
|
|
10
6
|
|
|
7
|
+
import { encryptSliceToPart } from './assemble.mjs'
|
|
11
8
|
import { decryptPart } from './transfer_key.mjs'
|
|
12
9
|
|
|
13
10
|
/**
|
|
14
11
|
* @typedef {import('./manifest.mjs').CeMode} CeMode
|
|
15
|
-
* @typedef {{ hash: string, size: number, raw: Buffer }} EncryptedPart
|
|
12
|
+
* @typedef {{ hash: string, size: number, raw: Buffer, contentHash?: string }} EncryptedPart
|
|
16
13
|
*/
|
|
17
14
|
|
|
18
15
|
/**
|
|
@@ -21,15 +18,15 @@ import { decryptPart } from './transfer_key.mjs'
|
|
|
21
18
|
* @param {CeMode} [ceMode] 加密模式
|
|
22
19
|
* @param {(part: EncryptedPart) => Promise<void>} onPart 每块回调
|
|
23
20
|
* @param {number} [maxBytes] 最大字节
|
|
24
|
-
* @returns {Promise<{ contentHash: string, parts: Array<{ hash: string, size: number }>, contentKey?: Buffer }>} 分块结果
|
|
21
|
+
* @returns {Promise<{ contentHash: string, parts: Array<{ hash: string, size: number, contentHash?: string }>, contentKey?: Buffer }>} 分块结果
|
|
25
22
|
*/
|
|
26
23
|
export async function encryptReadableToParts(readable, ceMode = 'convergent', onPart, maxBytes = Infinity) {
|
|
27
24
|
const digest = createHash('sha256')
|
|
28
25
|
/** @type {Buffer[]} */
|
|
29
26
|
let pending = Buffer.alloc(0)
|
|
30
|
-
/** @type {Array<{ hash: string, size: number }>} */
|
|
27
|
+
/** @type {Array<{ hash: string, size: number, contentHash?: string }>} */
|
|
31
28
|
const parts = []
|
|
32
|
-
|
|
29
|
+
const contentKey = ceMode === 'random' ? randomBytes(32) : null
|
|
33
30
|
let total = 0
|
|
34
31
|
|
|
35
32
|
/**
|
|
@@ -38,12 +35,11 @@ export async function encryptReadableToParts(readable, ceMode = 'convergent', on
|
|
|
38
35
|
*/
|
|
39
36
|
const flushSlice = async (slice) => {
|
|
40
37
|
digest.update(slice)
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
if (
|
|
45
|
-
|
|
46
|
-
parts.push({ hash: part.hash, size: part.size })
|
|
38
|
+
const part = encryptSliceToPart(slice, ceMode, contentKey)
|
|
39
|
+
/** @type {{ hash: string, size: number, contentHash?: string }} */
|
|
40
|
+
const meta = { hash: part.hash, size: part.size }
|
|
41
|
+
if (part.contentHash) meta.contentHash = part.contentHash
|
|
42
|
+
parts.push(meta)
|
|
47
43
|
await onPart(part)
|
|
48
44
|
}
|
|
49
45
|
|
|
@@ -71,55 +67,111 @@ export async function encryptReadableToParts(readable, ceMode = 'convergent', on
|
|
|
71
67
|
}
|
|
72
68
|
|
|
73
69
|
/**
|
|
74
|
-
*
|
|
70
|
+
* @param {import('node:stream').Readable} stream 密文流
|
|
71
|
+
* @returns {Promise<Buffer | null>} 下一可读块;流结束为 null
|
|
72
|
+
*/
|
|
73
|
+
function readStreamChunk(stream) {
|
|
74
|
+
const chunk = stream.read()
|
|
75
|
+
if (chunk) return Promise.resolve(Buffer.from(chunk))
|
|
76
|
+
if (stream.readableEnded) return Promise.resolve(null)
|
|
77
|
+
return new Promise((resolve, reject) => {
|
|
78
|
+
/**
|
|
79
|
+
*
|
|
80
|
+
*/
|
|
81
|
+
const onReadable = () => {
|
|
82
|
+
cleanup()
|
|
83
|
+
const next = stream.read()
|
|
84
|
+
resolve(next ? Buffer.from(next) : Buffer.alloc(0))
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
*
|
|
88
|
+
*/
|
|
89
|
+
const onEnd = () => {
|
|
90
|
+
cleanup()
|
|
91
|
+
resolve(null)
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* @param {Error} err 错误
|
|
95
|
+
*/
|
|
96
|
+
const onError = err => {
|
|
97
|
+
cleanup()
|
|
98
|
+
reject(err)
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
*
|
|
102
|
+
*/
|
|
103
|
+
const cleanup = () => {
|
|
104
|
+
stream.off('readable', onReadable)
|
|
105
|
+
stream.off('end', onEnd)
|
|
106
|
+
stream.off('error', onError)
|
|
107
|
+
}
|
|
108
|
+
stream.once('readable', onReadable)
|
|
109
|
+
stream.once('end', onEnd)
|
|
110
|
+
stream.once('error', onError)
|
|
111
|
+
})
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* 将 manifest 各密文块按 part.size 拼齐后解密,串联为明文可读流。
|
|
75
116
|
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
76
117
|
* @param {import('node:stream').Readable[]} partStreams 按序密文流
|
|
77
118
|
* @param {Buffer | null} contentKey 随机密钥
|
|
78
119
|
* @returns {Readable} 明文流
|
|
79
120
|
*/
|
|
80
121
|
export function createManifestPlaintextStream(manifest, partStreams, contentKey) {
|
|
122
|
+
if (partStreams.length !== manifest.parts.length)
|
|
123
|
+
throw new Error('part stream count mismatch')
|
|
124
|
+
|
|
81
125
|
let partIndex = 0
|
|
82
|
-
|
|
126
|
+
/** @type {Buffer} */
|
|
127
|
+
let pending = Buffer.alloc(0)
|
|
83
128
|
const digest = createHash('sha256')
|
|
129
|
+
let finished = false
|
|
84
130
|
|
|
85
131
|
return new Readable({
|
|
86
132
|
/**
|
|
87
133
|
*
|
|
88
134
|
*/
|
|
89
135
|
async read() {
|
|
136
|
+
if (finished) return
|
|
90
137
|
try {
|
|
91
|
-
while (
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
138
|
+
while (partIndex < manifest.parts.length) {
|
|
139
|
+
const need = Number(manifest.parts[partIndex].size) || 0
|
|
140
|
+
const stream = partStreams[partIndex]
|
|
141
|
+
while (pending.length < need) {
|
|
142
|
+
const more = await readStreamChunk(stream)
|
|
143
|
+
if (more === null) break
|
|
144
|
+
if (more.length) pending = Buffer.concat([pending, more])
|
|
145
|
+
}
|
|
146
|
+
if (pending.length < need) {
|
|
147
|
+
this.destroy(new Error('short ciphertext part'))
|
|
148
|
+
return
|
|
98
149
|
}
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
currentStream.resume()
|
|
105
|
-
})
|
|
106
|
-
if (currentStream === null) {
|
|
107
|
-
currentStream = null
|
|
108
|
-
continue
|
|
109
|
-
}
|
|
150
|
+
const enc = pending.subarray(0, need)
|
|
151
|
+
pending = pending.subarray(need)
|
|
152
|
+
if (pending.length) {
|
|
153
|
+
this.destroy(new Error('trailing ciphertext in part stream'))
|
|
154
|
+
return
|
|
110
155
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
156
|
+
const plain = decryptPart(enc, manifest, contentKey, partIndex)
|
|
157
|
+
if (!plain) {
|
|
158
|
+
this.destroy(new Error('decrypt failed'))
|
|
159
|
+
return
|
|
160
|
+
}
|
|
161
|
+
digest.update(plain)
|
|
162
|
+
partIndex++
|
|
163
|
+
this.push(plain)
|
|
164
|
+
return
|
|
165
|
+
}
|
|
166
|
+
if (manifest.contentHash) {
|
|
167
|
+
const got = digest.digest('hex')
|
|
168
|
+
if (got !== String(manifest.contentHash).toLowerCase()) {
|
|
169
|
+
this.destroy(new Error('contentHash mismatch'))
|
|
120
170
|
return
|
|
121
171
|
}
|
|
122
172
|
}
|
|
173
|
+
finished = true
|
|
174
|
+
this.push(null)
|
|
123
175
|
}
|
|
124
176
|
catch (error) {
|
|
125
177
|
this.destroy(error instanceof Error ? error : new Error(String(error)))
|
|
@@ -32,12 +32,6 @@ export function unregisterChunkProviders(ownerId) {
|
|
|
32
32
|
nodeHashProvidersByOwner.delete(key)
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
/** @returns {void} */
|
|
36
|
-
export function clearChunkProviderRegistry() {
|
|
37
|
-
federationFetchersByOwner.clear()
|
|
38
|
-
nodeHashProvidersByOwner.clear()
|
|
39
|
-
}
|
|
40
|
-
|
|
41
35
|
/**
|
|
42
36
|
* @param {string} username 用户
|
|
43
37
|
* @param {string} groupId 群 ID
|
|
@@ -71,7 +71,7 @@ export function attachNodeScopeFedChunkResponder(username, wire) {
|
|
|
71
71
|
* Trystero room:注册带 requestId 的 fed_chunk_* + fed_manifest_*(TrustGraph 全局 miss)。
|
|
72
72
|
* @param {string} username 用户
|
|
73
73
|
* @param {object} room Trystero room
|
|
74
|
-
* @param {{ enqueue: (prio: number,
|
|
74
|
+
* @param {{ enqueue: (prio: number, cleanup: () => void) => void }} [fedOut] 出站队列
|
|
75
75
|
* @param {(roomKey: string, action: string, rtcLimits: object) => boolean} [guardGet] RTC 负载守卫
|
|
76
76
|
* @param {object} [rtcLimits] RTC 限额
|
|
77
77
|
* @param {string} [roomKey] 房间键
|
package/files/chunk_store.mjs
CHANGED
|
@@ -2,7 +2,6 @@ import { Buffer } from 'node:buffer'
|
|
|
2
2
|
import fs from 'node:fs'
|
|
3
3
|
import fsp from 'node:fs/promises'
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
|
-
import { Readable } from 'node:stream'
|
|
6
5
|
import { pipeline } from 'node:stream/promises'
|
|
7
6
|
|
|
8
7
|
import { isHex64 } from '../core/hexIds.mjs'
|
|
@@ -74,14 +73,5 @@ export async function putChunk(hash, data) {
|
|
|
74
73
|
export async function putChunkFromStream(hash, readable) {
|
|
75
74
|
const filePath = chunkStorePath(hash)
|
|
76
75
|
await fsp.mkdir(dirname(filePath), { recursive: true })
|
|
77
|
-
|
|
78
|
-
await pipeline(readable, writable)
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* @param {string} hash 64 位十六进制
|
|
83
|
-
* @returns {Readable} chunk 可读流
|
|
84
|
-
*/
|
|
85
|
-
export function chunkToReadable(hash) {
|
|
86
|
-
return createChunkReadStream(hash)
|
|
76
|
+
await pipeline(readable, fs.createWriteStream(filePath))
|
|
87
77
|
}
|
package/files/evfs.mjs
CHANGED
|
@@ -4,13 +4,51 @@ import { Readable } from 'node:stream'
|
|
|
4
4
|
import { FEDERATION_CHUNK_MAX_BYTES } from '../core/constants.mjs'
|
|
5
5
|
import { getEntityStore } from '../node/instance.mjs'
|
|
6
6
|
|
|
7
|
-
import { buildFileManifestFromEnc, encryptPlaintextToParts,
|
|
8
|
-
import { encryptReadableToParts } from './assemble_stream.mjs'
|
|
7
|
+
import { buildFileManifestFromEnc, encryptPlaintextToMultiPartsAsync, encryptPlaintextToParts, manifestPartsForPersist } from './assemble.mjs'
|
|
8
|
+
import { createManifestPlaintextStream, encryptReadableToParts } from './assemble_stream.mjs'
|
|
9
9
|
import { fetchChunk } from './chunk_fetch.mjs'
|
|
10
|
-
import { getChunk, hasChunk, putChunk } from './chunk_store.mjs'
|
|
10
|
+
import { createChunkReadStream, getChunk, hasChunk, putChunk } from './chunk_store.mjs'
|
|
11
11
|
import { normalizeFileManifest, publicTransferKeyDescriptor } from './manifest.mjs'
|
|
12
|
-
import { assembleManifestPlaintext } from './transfer_key.mjs'
|
|
13
|
-
import { readDagManifestPlaintext,
|
|
12
|
+
import { assembleManifestPlaintext, resolveContentKey } from './transfer_key.mjs'
|
|
13
|
+
import { readDagManifestPlaintext, resolveTransferKeyDependencies } from './transfer_key_registry.mjs'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} replicaUsername 副本用户名
|
|
17
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
18
|
+
* @returns {{ getGroupFileMasterKey?: Function, getVaultMasterKey?: Function }} 密钥依赖
|
|
19
|
+
*/
|
|
20
|
+
function transferKeyDependenciesForReplica(replicaUsername, manifest) {
|
|
21
|
+
const rawDependencies = resolveTransferKeyDependencies(undefined, manifest)
|
|
22
|
+
return {
|
|
23
|
+
getGroupFileMasterKey: rawDependencies.getGroupFileMasterKey
|
|
24
|
+
? (groupId, keyGeneration) => rawDependencies.getGroupFileMasterKey(replicaUsername, groupId, keyGeneration)
|
|
25
|
+
: undefined,
|
|
26
|
+
getVaultMasterKey: rawDependencies.getVaultMasterKey
|
|
27
|
+
? entityHash => rawDependencies.getVaultMasterKey(replicaUsername, entityHash)
|
|
28
|
+
: undefined,
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* @param {string} username 拉取身份
|
|
34
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
35
|
+
* @param {{ fetchChunk?: Function }} [options] miss 拉取
|
|
36
|
+
* @returns {Promise<boolean>} 全部 part 是否已就位
|
|
37
|
+
*/
|
|
38
|
+
async function ensureManifestPartsLocal(username, manifest, options = {}) {
|
|
39
|
+
for (const part of manifest.parts) {
|
|
40
|
+
if (await hasChunk(part.hash)) continue
|
|
41
|
+
const fetchedChunk = await (options.fetchChunk || fetchChunk)({
|
|
42
|
+
username,
|
|
43
|
+
ciphertextHash: part.hash,
|
|
44
|
+
ownerEntityHash: manifest.ownerEntityHash,
|
|
45
|
+
groupId: manifest.transferKeyDescriptor.groupId,
|
|
46
|
+
})
|
|
47
|
+
if (!fetchedChunk) return false
|
|
48
|
+
await putChunk(part.hash, fetchedChunk)
|
|
49
|
+
}
|
|
50
|
+
return true
|
|
51
|
+
}
|
|
14
52
|
|
|
15
53
|
/**
|
|
16
54
|
* @param {string} ownerEntityHash 所有者
|
|
@@ -43,61 +81,50 @@ export async function storeManifestParts(manifest, partBytes) {
|
|
|
43
81
|
/**
|
|
44
82
|
* @param {string} replicaUsername 副本用户名
|
|
45
83
|
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
46
|
-
* @param {{ username?: string, fetchChunk?: Function }} [
|
|
84
|
+
* @param {{ username?: string, fetchChunk?: Function }} [options] miss 拉取
|
|
47
85
|
* @returns {Promise<Buffer | null>} 明文内容
|
|
48
86
|
*/
|
|
49
|
-
export async function readManifestPlaintext(replicaUsername, manifest,
|
|
87
|
+
export async function readManifestPlaintext(replicaUsername, manifest, options = {}) {
|
|
50
88
|
const dagGroupId = manifest.meta?.groupId
|
|
51
89
|
if (Array.isArray(manifest.meta?.dagParts) && dagGroupId) {
|
|
52
90
|
const dagPlain = await readDagManifestPlaintext(replicaUsername, manifest)
|
|
53
91
|
if (dagPlain) return dagPlain
|
|
54
92
|
}
|
|
55
93
|
|
|
56
|
-
const username =
|
|
94
|
+
const username = options.username || replicaUsername
|
|
95
|
+
if (!await ensureManifestPartsLocal(username, manifest, options)) return null
|
|
96
|
+
|
|
57
97
|
/** @type {Buffer[]} */
|
|
58
98
|
const partBytes = []
|
|
59
|
-
for (const part of manifest.parts)
|
|
60
|
-
|
|
61
|
-
if (await hasChunk(part.hash))
|
|
62
|
-
ciphertextBytes = await getChunk(part.hash)
|
|
63
|
-
else {
|
|
64
|
-
const fetchedChunk = await (opts.fetchChunk || fetchChunk)({
|
|
65
|
-
username,
|
|
66
|
-
ciphertextHash: part.hash,
|
|
67
|
-
ownerEntityHash: manifest.ownerEntityHash,
|
|
68
|
-
groupId: manifest.transferKeyDescriptor.groupId,
|
|
69
|
-
})
|
|
70
|
-
if (fetchedChunk) {
|
|
71
|
-
await putChunk(part.hash, fetchedChunk)
|
|
72
|
-
ciphertextBytes = Buffer.from(fetchedChunk)
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
if (!ciphertextBytes) return null
|
|
76
|
-
partBytes.push(ciphertextBytes)
|
|
77
|
-
}
|
|
99
|
+
for (const part of manifest.parts)
|
|
100
|
+
partBytes.push(await getChunk(part.hash))
|
|
78
101
|
|
|
79
|
-
|
|
80
|
-
const deps = {
|
|
81
|
-
getGroupFileMasterKey: rawDeps.getGroupFileMasterKey
|
|
82
|
-
? (groupId, keyGeneration) => rawDeps.getGroupFileMasterKey(replicaUsername, groupId, keyGeneration)
|
|
83
|
-
: undefined,
|
|
84
|
-
getVaultMasterKey: rawDeps.getVaultMasterKey
|
|
85
|
-
? entityHash => rawDeps.getVaultMasterKey(replicaUsername, entityHash)
|
|
86
|
-
: undefined,
|
|
87
|
-
}
|
|
88
|
-
return assembleManifestPlaintext(manifest, partBytes, deps)
|
|
102
|
+
return assembleManifestPlaintext(manifest, partBytes, transferKeyDependenciesForReplica(replicaUsername, manifest))
|
|
89
103
|
}
|
|
90
104
|
|
|
91
105
|
/**
|
|
92
106
|
* @param {string} replicaUsername 副本用户名
|
|
93
107
|
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
94
|
-
* @param {{ username?: string, fetchChunk?: Function }} [
|
|
108
|
+
* @param {{ username?: string, fetchChunk?: Function }} [options] miss 拉取
|
|
95
109
|
* @returns {Promise<import('node:stream').Readable | null>} 明文流
|
|
96
110
|
*/
|
|
97
|
-
export async function readManifestPlaintextStream(replicaUsername, manifest,
|
|
98
|
-
const
|
|
99
|
-
if (
|
|
100
|
-
|
|
111
|
+
export async function readManifestPlaintextStream(replicaUsername, manifest, options = {}) {
|
|
112
|
+
const dagGroupId = manifest.meta?.groupId
|
|
113
|
+
if (Array.isArray(manifest.meta?.dagParts) && dagGroupId) {
|
|
114
|
+
const plain = await readManifestPlaintext(replicaUsername, manifest, options)
|
|
115
|
+
if (!plain) return null
|
|
116
|
+
return Readable.from([plain])
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const username = options.username || replicaUsername
|
|
120
|
+
if (!await ensureManifestPartsLocal(username, manifest, options)) return null
|
|
121
|
+
|
|
122
|
+
const dependencies = transferKeyDependenciesForReplica(replicaUsername, manifest)
|
|
123
|
+
const contentKey = await resolveContentKey(manifest.transferKeyDescriptor, manifest, dependencies)
|
|
124
|
+
if (manifest.ceMode === 'random' && !contentKey) return null
|
|
125
|
+
|
|
126
|
+
const partStreams = manifest.parts.map(part => createChunkReadStream(part.hash))
|
|
127
|
+
return createManifestPlaintextStream(manifest, partStreams, contentKey)
|
|
101
128
|
}
|
|
102
129
|
|
|
103
130
|
/**
|
|
@@ -178,7 +205,7 @@ export async function putFileManifestFromStream(params) {
|
|
|
178
205
|
size: plainSize,
|
|
179
206
|
contentHash: enc.contentHash,
|
|
180
207
|
ceMode,
|
|
181
|
-
parts: enc.parts,
|
|
208
|
+
parts: manifestPartsForPersist(enc.parts),
|
|
182
209
|
transferKeyDescriptor: transferKeyDescriptor || publicTransferKeyDescriptor(),
|
|
183
210
|
meta,
|
|
184
211
|
})
|
|
@@ -192,16 +219,16 @@ export async function putFileManifestFromStream(params) {
|
|
|
192
219
|
* @param {string} replicaUsername 副本用户名
|
|
193
220
|
* @param {string} entityHash owner entityHash
|
|
194
221
|
* @param {string} logicalPath EVFS 逻辑路径
|
|
195
|
-
* @param {{ username?: string, fetchChunk?: Function }} [
|
|
222
|
+
* @param {{ username?: string, fetchChunk?: Function }} [options] miss 拉取
|
|
196
223
|
* @returns {Promise<Buffer | null>} 明文或 null
|
|
197
224
|
*/
|
|
198
|
-
export async function readPublicFile(replicaUsername, entityHash, logicalPath,
|
|
225
|
+
export async function readPublicFile(replicaUsername, entityHash, logicalPath, options = {}) {
|
|
199
226
|
const { fetchPublicManifest } = await import('./manifest_fetch.mjs')
|
|
200
227
|
const manifest = await fetchPublicManifest({
|
|
201
|
-
username:
|
|
228
|
+
username: options.username || replicaUsername,
|
|
202
229
|
ownerEntityHash: entityHash,
|
|
203
230
|
logicalPath,
|
|
204
231
|
})
|
|
205
232
|
if (!manifest) return null
|
|
206
|
-
return readManifestPlaintext(replicaUsername, manifest,
|
|
233
|
+
return readManifestPlaintext(replicaUsername, manifest, options)
|
|
207
234
|
}
|
package/files/manifest.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { isHex64 } from '../core/hexIds.mjs'
|
|
|
3
3
|
|
|
4
4
|
/** @typedef {'plain' | 'convergent' | 'random'} CeMode */
|
|
5
5
|
|
|
6
|
-
/** @typedef {{ hash: string, size: number }} ManifestPart */
|
|
6
|
+
/** @typedef {{ hash: string, size: number, contentHash?: string }} ManifestPart */
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* @typedef {{
|
|
@@ -48,7 +48,16 @@ export function normalizeFileManifest(input) {
|
|
|
48
48
|
if (!CE_MODES.has(ceMode)) return null
|
|
49
49
|
const parts = Array.isArray(input.parts)
|
|
50
50
|
? input.parts
|
|
51
|
-
.map(part =>
|
|
51
|
+
.map(part => {
|
|
52
|
+
/** @type {ManifestPart} */
|
|
53
|
+
const out = {
|
|
54
|
+
hash: String(part?.hash || '').trim().toLowerCase(),
|
|
55
|
+
size: Number(part?.size) || 0,
|
|
56
|
+
}
|
|
57
|
+
const partContentHash = String(part?.contentHash || '').trim().toLowerCase()
|
|
58
|
+
if (isHex64(partContentHash)) out.contentHash = partContentHash
|
|
59
|
+
return out
|
|
60
|
+
})
|
|
52
61
|
.filter(part => isHex64(part.hash))
|
|
53
62
|
: []
|
|
54
63
|
if (!parts.length) return null
|
|
@@ -61,12 +61,6 @@ export function unregisterManifestAclMatcher(ownerId) {
|
|
|
61
61
|
matchersByOwner.delete(String(ownerId))
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
/** @returns {void} */
|
|
65
|
-
export function clearManifestAclRegistry() {
|
|
66
|
-
handlersByType.clear()
|
|
67
|
-
matchersByOwner.clear()
|
|
68
|
-
}
|
|
69
|
-
|
|
70
64
|
/**
|
|
71
65
|
* @param {import('./manifest.mjs').FileManifest | null | undefined} manifest manifest
|
|
72
66
|
* @param {string} ownerEntityHash 所有者
|
package/files/transfer_key.mjs
CHANGED
|
@@ -15,26 +15,26 @@ import {
|
|
|
15
15
|
/**
|
|
16
16
|
* @param {TransferKeyDescriptor} descriptor 传递密钥描述符
|
|
17
17
|
* @param {FileManifest} manifest manifest
|
|
18
|
-
* @param {{ getGroupFileMasterKey?: (groupId: string, keyGeneration?: number) => Promise<Buffer | string | null>, getVaultMasterKey?: (entityHash: string) => Promise<Buffer | string | null> }}
|
|
18
|
+
* @param {{ getGroupFileMasterKey?: (groupId: string, keyGeneration?: number) => Promise<Buffer | string | null>, getVaultMasterKey?: (entityHash: string) => Promise<Buffer | string | null> }} dependencies 密钥源
|
|
19
19
|
* @returns {Promise<Buffer | null>} contentKey;plain/convergent 返回 null(按 contentHash 派生)
|
|
20
20
|
*/
|
|
21
|
-
export async function resolveContentKey(descriptor, manifest,
|
|
21
|
+
export async function resolveContentKey(descriptor, manifest, dependencies = {}) {
|
|
22
22
|
const type = descriptor?.type || 'public'
|
|
23
23
|
if (type === 'public' || manifest.ceMode === 'plain' || manifest.ceMode === 'convergent')
|
|
24
24
|
return null
|
|
25
25
|
|
|
26
26
|
if (type === 'file-master-key-wrap') {
|
|
27
27
|
const { groupId, fileId } = descriptor
|
|
28
|
-
if (!groupId || !fileId || !descriptor.wrappedKey || !
|
|
29
|
-
const groupKey = await
|
|
28
|
+
if (!groupId || !fileId || !descriptor.wrappedKey || !dependencies.getGroupFileMasterKey) return null
|
|
29
|
+
const groupKey = await dependencies.getGroupFileMasterKey(String(groupId), descriptor.keyGeneration)
|
|
30
30
|
if (!groupKey) return null
|
|
31
31
|
return unwrapContentKey(descriptor.wrappedKey, groupKey, fileId)
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
if (type === 'vault-wrap') {
|
|
35
35
|
const { entityHash, fileId } = descriptor
|
|
36
|
-
if (!entityHash || !fileId || !descriptor.wrappedKey || !
|
|
37
|
-
const vaultKey = await
|
|
36
|
+
if (!entityHash || !fileId || !descriptor.wrappedKey || !dependencies.getVaultMasterKey) return null
|
|
37
|
+
const vaultKey = await dependencies.getVaultMasterKey(String(entityHash))
|
|
38
38
|
if (!vaultKey) return null
|
|
39
39
|
return unwrapContentKey(descriptor.wrappedKey, vaultKey, fileId)
|
|
40
40
|
}
|
|
@@ -46,17 +46,23 @@ export async function resolveContentKey(descriptor, manifest, deps = {}) {
|
|
|
46
46
|
* @param {Buffer | Uint8Array} encryptedPartBytes 密文块
|
|
47
47
|
* @param {FileManifest} manifest manifest
|
|
48
48
|
* @param {Buffer | null} contentKey random 模式密钥
|
|
49
|
+
* @param {number} [partIndex] 分块下标(多块 convergent 用 part.contentHash)
|
|
49
50
|
* @returns {Buffer | null} 明文
|
|
50
51
|
*/
|
|
51
|
-
export function decryptPart(encryptedPartBytes, manifest, contentKey) {
|
|
52
|
+
export function decryptPart(encryptedPartBytes, manifest, contentKey, partIndex = 0) {
|
|
52
53
|
if (manifest.ceMode === 'plain')
|
|
53
54
|
return Buffer.from(encryptedPartBytes)
|
|
54
55
|
|
|
55
|
-
if (manifest.ceMode === 'convergent')
|
|
56
|
-
|
|
56
|
+
if (manifest.ceMode === 'convergent') {
|
|
57
|
+
const partPlainHash = manifest.parts[partIndex]?.contentHash || manifest.contentHash
|
|
58
|
+
return decryptConvergentCiphertext(encryptedPartBytes, partPlainHash)
|
|
59
|
+
}
|
|
57
60
|
|
|
58
|
-
if (manifest.ceMode === 'random' && contentKey)
|
|
59
|
-
|
|
61
|
+
if (manifest.ceMode === 'random' && contentKey) {
|
|
62
|
+
// 多块时各块明文 hash ≠ 整文件 contentHash;完整性在 assemble 末尾校验
|
|
63
|
+
const verifyHash = manifest.parts.length === 1 ? manifest.contentHash : ''
|
|
64
|
+
return decryptRandomCiphertext(encryptedPartBytes, contentKey, verifyHash)
|
|
65
|
+
}
|
|
60
66
|
|
|
61
67
|
return null
|
|
62
68
|
}
|
|
@@ -64,16 +70,16 @@ export function decryptPart(encryptedPartBytes, manifest, contentKey) {
|
|
|
64
70
|
/**
|
|
65
71
|
* @param {FileManifest} manifest manifest
|
|
66
72
|
* @param {Array<Buffer | Uint8Array>} partBytes 按序密文块
|
|
67
|
-
* @param {{ getGroupFileMasterKey?: Function, getVaultMasterKey?: Function }}
|
|
73
|
+
* @param {{ getGroupFileMasterKey?: Function, getVaultMasterKey?: Function }} dependencies 密钥源
|
|
68
74
|
* @returns {Promise<Buffer | null>} 完整明文
|
|
69
75
|
*/
|
|
70
|
-
export async function assembleManifestPlaintext(manifest, partBytes,
|
|
76
|
+
export async function assembleManifestPlaintext(manifest, partBytes, dependencies = {}) {
|
|
71
77
|
if (partBytes.length !== manifest.parts.length) return null
|
|
72
|
-
const contentKey = await resolveContentKey(manifest.transferKeyDescriptor, manifest,
|
|
78
|
+
const contentKey = await resolveContentKey(manifest.transferKeyDescriptor, manifest, dependencies)
|
|
73
79
|
/** @type {Buffer[]} */
|
|
74
80
|
const plains = []
|
|
75
81
|
for (let index = 0; index < manifest.parts.length; index++) {
|
|
76
|
-
const plain = decryptPart(partBytes[index], manifest, contentKey)
|
|
82
|
+
const plain = decryptPart(partBytes[index], manifest, contentKey, index)
|
|
77
83
|
if (!plain) return null
|
|
78
84
|
plains.push(plain)
|
|
79
85
|
}
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
/** @type {Map<string, { getGroupFileMasterKey?: (replicaUsername: string, groupId: string, keyGeneration?: number) => Promise<Buffer | string | null>, getVaultMasterKey?: (replicaUsername: string, entityHash: string) => Promise<Buffer | string | null> }>} */
|
|
2
|
-
const
|
|
2
|
+
const transferDependenciesByOwner = new Map()
|
|
3
3
|
|
|
4
4
|
/** @type {Map<string, (replicaUsername: string, manifest: import('./manifest.mjs').FileManifest) => Promise<Buffer | null>>} */
|
|
5
5
|
const dagPlaintextReadersByOwner = new Map()
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* @param {string} ownerId 注册方
|
|
9
|
-
* @param {{ getGroupFileMasterKey?: (replicaUsername: string, groupId: string, keyGeneration?: number) => Promise<Buffer | string | null>, getVaultMasterKey?: (replicaUsername: string, entityHash: string) => Promise<Buffer | string | null> }}
|
|
9
|
+
* @param {{ getGroupFileMasterKey?: (replicaUsername: string, groupId: string, keyGeneration?: number) => Promise<Buffer | string | null>, getVaultMasterKey?: (replicaUsername: string, entityHash: string) => Promise<Buffer | string | null> }} dependencies 密钥源
|
|
10
10
|
* @returns {void}
|
|
11
11
|
*/
|
|
12
|
-
export function
|
|
12
|
+
export function registerTransferKeyDependencies(ownerId, dependencies) {
|
|
13
13
|
const key = String(ownerId)
|
|
14
|
-
const prev =
|
|
15
|
-
|
|
14
|
+
const prev = transferDependenciesByOwner.get(key) || {}
|
|
15
|
+
transferDependenciesByOwner.set(key, { ...prev, ...dependencies })
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
/**
|
|
@@ -50,20 +50,13 @@ export function unregisterManifestOwnerMatchers(ownerId) {
|
|
|
50
50
|
* @param {string} ownerId 注册方
|
|
51
51
|
* @returns {void}
|
|
52
52
|
*/
|
|
53
|
-
export function
|
|
53
|
+
export function unregisterTransferKeyDependencies(ownerId) {
|
|
54
54
|
const key = String(ownerId)
|
|
55
|
-
|
|
55
|
+
transferDependenciesByOwner.delete(key)
|
|
56
56
|
dagPlaintextReadersByOwner.delete(key)
|
|
57
57
|
unregisterManifestOwnerMatchers(key)
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
/** @returns {void} */
|
|
61
|
-
export function clearTransferKeyRegistry() {
|
|
62
|
-
transferDepsByOwner.clear()
|
|
63
|
-
dagPlaintextReadersByOwner.clear()
|
|
64
|
-
manifestOwnerMatchers.length = 0
|
|
65
|
-
}
|
|
66
|
-
|
|
67
60
|
/**
|
|
68
61
|
* 从 manifest 推断 transfer key 注册方 id(按 registerManifestOwnerMatcher 注册顺序匹配)。
|
|
69
62
|
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
@@ -80,9 +73,9 @@ export function resolveManifestTransferOwnerId(manifest) {
|
|
|
80
73
|
* @param {import('./manifest.mjs').FileManifest} [manifest] 用于推断 ownerId
|
|
81
74
|
* @returns {{ getGroupFileMasterKey?: (replicaUsername: string, groupId: string, keyGeneration?: number) => Promise<Buffer | string | null>, getVaultMasterKey?: (replicaUsername: string, entityHash: string) => Promise<Buffer | string | null> }} 依赖
|
|
82
75
|
*/
|
|
83
|
-
export function
|
|
76
|
+
export function resolveTransferKeyDependencies(ownerId, manifest) {
|
|
84
77
|
const resolved = ownerId || (manifest ? resolveManifestTransferOwnerId(manifest) : null)
|
|
85
|
-
if (resolved) return
|
|
78
|
+
if (resolved) return transferDependenciesByOwner.get(String(resolved)) || {}
|
|
86
79
|
return {}
|
|
87
80
|
}
|
|
88
81
|
|
package/governance/branch.mjs
CHANGED
|
@@ -93,17 +93,6 @@ export function descendantClosureFromTip(rootId, byId) {
|
|
|
93
93
|
return out
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
/**
|
|
97
|
-
* 事件是否在 root 的后代闭包中(含 root)。
|
|
98
|
-
* @param {string} eventId 事件 id
|
|
99
|
-
* @param {string} rootId 根 id
|
|
100
|
-
* @param {Map<string, object>} byId id→事件
|
|
101
|
-
* @returns {boolean} 是否为 root 的后代(含 root 自身)
|
|
102
|
-
*/
|
|
103
|
-
export function isDescendantOfTip(eventId, rootId, byId) {
|
|
104
|
-
return descendantClosureFromTip(rootId, byId).has(eventId)
|
|
105
|
-
}
|
|
106
|
-
|
|
107
96
|
/**
|
|
108
97
|
* @param {string} tipId 叶 id
|
|
109
98
|
* @param {Map<string, object>} byId id→事件
|