@steve02081504/fount-p2p 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/core/bytes_codec.mjs +0 -27
- package/core/constants.mjs +0 -31
- package/core/hexIds.mjs +0 -13
- package/{entity → core}/logical_entity.mjs +2 -1
- package/crypto/key.mjs +14 -15
- package/dag/index.mjs +0 -73
- package/dag/storage.mjs +0 -22
- package/federation/entity_key_chain.mjs +0 -9
- package/federation/topo_order_memo.mjs +0 -15
- package/{entity/files → files}/acl.mjs +3 -3
- package/files/assemble.mjs +55 -39
- package/files/assemble_stream.mjs +96 -44
- package/files/chunk_provider_registry.mjs +0 -6
- package/files/chunk_store.mjs +1 -11
- package/{entity/files → files}/evfs.mjs +81 -55
- package/{entity/files → files}/evfs_ref.mjs +3 -5
- package/files/manifest.mjs +11 -2
- package/{entity/files → files}/manifest_acl_registry.mjs +4 -10
- package/files/manifest_fetch.mjs +2 -2
- package/files/public_manifest.mjs +1 -1
- package/files/transfer_key.mjs +12 -6
- package/files/transfer_key_registry.mjs +0 -7
- package/governance/branch.mjs +0 -11
- package/link/channel_mux.mjs +2 -0
- package/mailbox/importance.mjs +0 -33
- package/mailbox/settings.mjs +0 -9
- package/node/denylist.mjs +0 -24
- package/node/identity.mjs +33 -2
- package/node/network.mjs +0 -51
- package/node/personal_block.mjs +1 -36
- package/node/reputation_store.mjs +0 -9
- package/node/retention_policy.mjs +0 -15
- package/package.json +1 -3
- package/registries/event_type.mjs +0 -12
- package/registries/inbound.mjs +0 -22
- package/reputation/engine.mjs +0 -11
- package/schemas/part_query.mjs +156 -0
- package/transport/ice_servers.mjs +0 -8
- package/transport/peer_identity_maps.mjs +0 -58
- package/transport/remote_user_room.mjs +0 -12
- package/transport/rtc_connection_budget.mjs +2 -0
- package/transport/user_room.mjs +2 -11
- package/trust_graph/cache.mjs +0 -13
- package/trust_graph/engine.mjs +0 -24
- package/utils/async_mutex.mjs +0 -9
- package/wire/part_fanout.mjs +0 -19
- package/wire/part_query.mjs +487 -0
- package/wire/part_query.tunables.json +14 -0
- package/wire/part_query_cache.mjs +94 -0
- package/entity/files/replica_host_cache.mjs +0 -46
- package/entity/hosting_registry.mjs +0 -47
- package/entity/logical_entity_id_registry.mjs +0 -40
- package/entity/node_hash.mjs +0 -15
- package/entity/replica.mjs +0 -20
- package/entity/session_snapshot_registry.mjs +0 -38
- package/registries/p2p_viewer.mjs +0 -35
|
@@ -1,18 +1,15 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer'
|
|
2
|
-
import { createHash } from 'node:crypto'
|
|
2
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
3
3
|
import { Readable } from 'node:stream'
|
|
4
4
|
|
|
5
5
|
import { FEDERATION_CHUNK_MAX_BYTES } from '../core/constants.mjs'
|
|
6
|
-
import {
|
|
7
|
-
encryptConvergentPlaintext,
|
|
8
|
-
encryptRandomPlaintext,
|
|
9
|
-
} from '../crypto/key.mjs'
|
|
10
6
|
|
|
7
|
+
import { encryptSliceToPart } from './assemble.mjs'
|
|
11
8
|
import { decryptPart } from './transfer_key.mjs'
|
|
12
9
|
|
|
13
10
|
/**
|
|
14
11
|
* @typedef {import('./manifest.mjs').CeMode} CeMode
|
|
15
|
-
* @typedef {{ hash: string, size: number, raw: Buffer }} EncryptedPart
|
|
12
|
+
* @typedef {{ hash: string, size: number, raw: Buffer, contentHash?: string }} EncryptedPart
|
|
16
13
|
*/
|
|
17
14
|
|
|
18
15
|
/**
|
|
@@ -21,15 +18,15 @@ import { decryptPart } from './transfer_key.mjs'
|
|
|
21
18
|
* @param {CeMode} [ceMode] 加密模式
|
|
22
19
|
* @param {(part: EncryptedPart) => Promise<void>} onPart 每块回调
|
|
23
20
|
* @param {number} [maxBytes] 最大字节
|
|
24
|
-
* @returns {Promise<{ contentHash: string, parts: Array<{ hash: string, size: number }>, contentKey?: Buffer }>} 分块结果
|
|
21
|
+
* @returns {Promise<{ contentHash: string, parts: Array<{ hash: string, size: number, contentHash?: string }>, contentKey?: Buffer }>} 分块结果
|
|
25
22
|
*/
|
|
26
23
|
export async function encryptReadableToParts(readable, ceMode = 'convergent', onPart, maxBytes = Infinity) {
|
|
27
24
|
const digest = createHash('sha256')
|
|
28
25
|
/** @type {Buffer[]} */
|
|
29
26
|
let pending = Buffer.alloc(0)
|
|
30
|
-
/** @type {Array<{ hash: string, size: number }>} */
|
|
27
|
+
/** @type {Array<{ hash: string, size: number, contentHash?: string }>} */
|
|
31
28
|
const parts = []
|
|
32
|
-
|
|
29
|
+
const contentKey = ceMode === 'random' ? randomBytes(32) : null
|
|
33
30
|
let total = 0
|
|
34
31
|
|
|
35
32
|
/**
|
|
@@ -38,12 +35,11 @@ export async function encryptReadableToParts(readable, ceMode = 'convergent', on
|
|
|
38
35
|
*/
|
|
39
36
|
const flushSlice = async (slice) => {
|
|
40
37
|
digest.update(slice)
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
if (
|
|
45
|
-
|
|
46
|
-
parts.push({ hash: part.hash, size: part.size })
|
|
38
|
+
const part = encryptSliceToPart(slice, ceMode, contentKey)
|
|
39
|
+
/** @type {{ hash: string, size: number, contentHash?: string }} */
|
|
40
|
+
const meta = { hash: part.hash, size: part.size }
|
|
41
|
+
if (part.contentHash) meta.contentHash = part.contentHash
|
|
42
|
+
parts.push(meta)
|
|
47
43
|
await onPart(part)
|
|
48
44
|
}
|
|
49
45
|
|
|
@@ -71,55 +67,111 @@ export async function encryptReadableToParts(readable, ceMode = 'convergent', on
|
|
|
71
67
|
}
|
|
72
68
|
|
|
73
69
|
/**
|
|
74
|
-
*
|
|
70
|
+
* @param {import('node:stream').Readable} stream 密文流
|
|
71
|
+
* @returns {Promise<Buffer | null>} 下一可读块;流结束为 null
|
|
72
|
+
*/
|
|
73
|
+
function readStreamChunk(stream) {
|
|
74
|
+
const chunk = stream.read()
|
|
75
|
+
if (chunk) return Promise.resolve(Buffer.from(chunk))
|
|
76
|
+
if (stream.readableEnded) return Promise.resolve(null)
|
|
77
|
+
return new Promise((resolve, reject) => {
|
|
78
|
+
/**
|
|
79
|
+
*
|
|
80
|
+
*/
|
|
81
|
+
const onReadable = () => {
|
|
82
|
+
cleanup()
|
|
83
|
+
const next = stream.read()
|
|
84
|
+
resolve(next ? Buffer.from(next) : Buffer.alloc(0))
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
*
|
|
88
|
+
*/
|
|
89
|
+
const onEnd = () => {
|
|
90
|
+
cleanup()
|
|
91
|
+
resolve(null)
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* @param {Error} err 错误
|
|
95
|
+
*/
|
|
96
|
+
const onError = err => {
|
|
97
|
+
cleanup()
|
|
98
|
+
reject(err)
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
*
|
|
102
|
+
*/
|
|
103
|
+
const cleanup = () => {
|
|
104
|
+
stream.off('readable', onReadable)
|
|
105
|
+
stream.off('end', onEnd)
|
|
106
|
+
stream.off('error', onError)
|
|
107
|
+
}
|
|
108
|
+
stream.once('readable', onReadable)
|
|
109
|
+
stream.once('end', onEnd)
|
|
110
|
+
stream.once('error', onError)
|
|
111
|
+
})
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* 将 manifest 各密文块按 part.size 拼齐后解密,串联为明文可读流。
|
|
75
116
|
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
76
117
|
* @param {import('node:stream').Readable[]} partStreams 按序密文流
|
|
77
118
|
* @param {Buffer | null} contentKey 随机密钥
|
|
78
119
|
* @returns {Readable} 明文流
|
|
79
120
|
*/
|
|
80
121
|
export function createManifestPlaintextStream(manifest, partStreams, contentKey) {
|
|
122
|
+
if (partStreams.length !== manifest.parts.length)
|
|
123
|
+
throw new Error('part stream count mismatch')
|
|
124
|
+
|
|
81
125
|
let partIndex = 0
|
|
82
|
-
|
|
126
|
+
/** @type {Buffer} */
|
|
127
|
+
let pending = Buffer.alloc(0)
|
|
83
128
|
const digest = createHash('sha256')
|
|
129
|
+
let finished = false
|
|
84
130
|
|
|
85
131
|
return new Readable({
|
|
86
132
|
/**
|
|
87
133
|
*
|
|
88
134
|
*/
|
|
89
135
|
async read() {
|
|
136
|
+
if (finished) return
|
|
90
137
|
try {
|
|
91
|
-
while (
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
138
|
+
while (partIndex < manifest.parts.length) {
|
|
139
|
+
const need = Number(manifest.parts[partIndex].size) || 0
|
|
140
|
+
const stream = partStreams[partIndex]
|
|
141
|
+
while (pending.length < need) {
|
|
142
|
+
const more = await readStreamChunk(stream)
|
|
143
|
+
if (more === null) break
|
|
144
|
+
if (more.length) pending = Buffer.concat([pending, more])
|
|
145
|
+
}
|
|
146
|
+
if (pending.length < need) {
|
|
147
|
+
this.destroy(new Error('short ciphertext part'))
|
|
148
|
+
return
|
|
98
149
|
}
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
currentStream.resume()
|
|
105
|
-
})
|
|
106
|
-
if (currentStream === null) {
|
|
107
|
-
currentStream = null
|
|
108
|
-
continue
|
|
109
|
-
}
|
|
150
|
+
const enc = pending.subarray(0, need)
|
|
151
|
+
pending = pending.subarray(need)
|
|
152
|
+
if (pending.length) {
|
|
153
|
+
this.destroy(new Error('trailing ciphertext in part stream'))
|
|
154
|
+
return
|
|
110
155
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
156
|
+
const plain = decryptPart(enc, manifest, contentKey, partIndex)
|
|
157
|
+
if (!plain) {
|
|
158
|
+
this.destroy(new Error('decrypt failed'))
|
|
159
|
+
return
|
|
160
|
+
}
|
|
161
|
+
digest.update(plain)
|
|
162
|
+
partIndex++
|
|
163
|
+
this.push(plain)
|
|
164
|
+
return
|
|
165
|
+
}
|
|
166
|
+
if (manifest.contentHash) {
|
|
167
|
+
const got = digest.digest('hex')
|
|
168
|
+
if (got !== String(manifest.contentHash).toLowerCase()) {
|
|
169
|
+
this.destroy(new Error('contentHash mismatch'))
|
|
120
170
|
return
|
|
121
171
|
}
|
|
122
172
|
}
|
|
173
|
+
finished = true
|
|
174
|
+
this.push(null)
|
|
123
175
|
}
|
|
124
176
|
catch (error) {
|
|
125
177
|
this.destroy(error instanceof Error ? error : new Error(String(error)))
|
|
@@ -32,12 +32,6 @@ export function unregisterChunkProviders(ownerId) {
|
|
|
32
32
|
nodeHashProvidersByOwner.delete(key)
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
/** @returns {void} */
|
|
36
|
-
export function clearChunkProviderRegistry() {
|
|
37
|
-
federationFetchersByOwner.clear()
|
|
38
|
-
nodeHashProvidersByOwner.clear()
|
|
39
|
-
}
|
|
40
|
-
|
|
41
35
|
/**
|
|
42
36
|
* @param {string} username 用户
|
|
43
37
|
* @param {string} groupId 群 ID
|
package/files/chunk_store.mjs
CHANGED
|
@@ -2,7 +2,6 @@ import { Buffer } from 'node:buffer'
|
|
|
2
2
|
import fs from 'node:fs'
|
|
3
3
|
import fsp from 'node:fs/promises'
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
|
-
import { Readable } from 'node:stream'
|
|
6
5
|
import { pipeline } from 'node:stream/promises'
|
|
7
6
|
|
|
8
7
|
import { isHex64 } from '../core/hexIds.mjs'
|
|
@@ -74,14 +73,5 @@ export async function putChunk(hash, data) {
|
|
|
74
73
|
export async function putChunkFromStream(hash, readable) {
|
|
75
74
|
const filePath = chunkStorePath(hash)
|
|
76
75
|
await fsp.mkdir(dirname(filePath), { recursive: true })
|
|
77
|
-
|
|
78
|
-
await pipeline(readable, writable)
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* @param {string} hash 64 位十六进制
|
|
83
|
-
* @returns {Readable} chunk 可读流
|
|
84
|
-
*/
|
|
85
|
-
export function chunkToReadable(hash) {
|
|
86
|
-
return createChunkReadStream(hash)
|
|
76
|
+
await pipeline(readable, fs.createWriteStream(filePath))
|
|
87
77
|
}
|
|
@@ -1,21 +1,59 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer'
|
|
2
2
|
import { Readable } from 'node:stream'
|
|
3
3
|
|
|
4
|
-
import { FEDERATION_CHUNK_MAX_BYTES } from '
|
|
5
|
-
import {
|
|
6
|
-
import { encryptReadableToParts } from '../../files/assemble_stream.mjs'
|
|
7
|
-
import { fetchChunk } from '../../files/chunk_fetch.mjs'
|
|
8
|
-
import { getChunk, hasChunk, putChunk } from '../../files/chunk_store.mjs'
|
|
9
|
-
import { normalizeFileManifest, publicTransferKeyDescriptor } from '../../files/manifest.mjs'
|
|
10
|
-
import { assembleManifestPlaintext } from '../../files/transfer_key.mjs'
|
|
11
|
-
import { readDagManifestPlaintext, resolveTransferKeyDeps } from '../../files/transfer_key_registry.mjs'
|
|
12
|
-
import { getEntityStore } from '../../node/instance.mjs'
|
|
4
|
+
import { FEDERATION_CHUNK_MAX_BYTES } from '../core/constants.mjs'
|
|
5
|
+
import { getEntityStore } from '../node/instance.mjs'
|
|
13
6
|
|
|
7
|
+
import { buildFileManifestFromEnc, encryptPlaintextToMultiPartsAsync, encryptPlaintextToParts, manifestPartsForPersist } from './assemble.mjs'
|
|
8
|
+
import { createManifestPlaintextStream, encryptReadableToParts } from './assemble_stream.mjs'
|
|
9
|
+
import { fetchChunk } from './chunk_fetch.mjs'
|
|
10
|
+
import { createChunkReadStream, getChunk, hasChunk, putChunk } from './chunk_store.mjs'
|
|
11
|
+
import { normalizeFileManifest, publicTransferKeyDescriptor } from './manifest.mjs'
|
|
12
|
+
import { assembleManifestPlaintext, resolveContentKey } from './transfer_key.mjs'
|
|
13
|
+
import { readDagManifestPlaintext, resolveTransferKeyDeps } from './transfer_key_registry.mjs'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} replicaUsername 副本用户名
|
|
17
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
18
|
+
* @returns {{ getGroupFileMasterKey?: Function, getVaultMasterKey?: Function }} 密钥依赖
|
|
19
|
+
*/
|
|
20
|
+
function transferKeyDepsForReplica(replicaUsername, manifest) {
|
|
21
|
+
const rawDeps = resolveTransferKeyDeps(undefined, manifest)
|
|
22
|
+
return {
|
|
23
|
+
getGroupFileMasterKey: rawDeps.getGroupFileMasterKey
|
|
24
|
+
? (groupId, keyGeneration) => rawDeps.getGroupFileMasterKey(replicaUsername, groupId, keyGeneration)
|
|
25
|
+
: undefined,
|
|
26
|
+
getVaultMasterKey: rawDeps.getVaultMasterKey
|
|
27
|
+
? entityHash => rawDeps.getVaultMasterKey(replicaUsername, entityHash)
|
|
28
|
+
: undefined,
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* @param {string} username 拉取身份
|
|
34
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
35
|
+
* @param {{ fetchChunk?: Function }} [opts] miss 拉取
|
|
36
|
+
* @returns {Promise<boolean>} 全部 part 是否已就位
|
|
37
|
+
*/
|
|
38
|
+
async function ensureManifestPartsLocal(username, manifest, opts = {}) {
|
|
39
|
+
for (const part of manifest.parts) {
|
|
40
|
+
if (await hasChunk(part.hash)) continue
|
|
41
|
+
const fetchedChunk = await (opts.fetchChunk || fetchChunk)({
|
|
42
|
+
username,
|
|
43
|
+
ciphertextHash: part.hash,
|
|
44
|
+
ownerEntityHash: manifest.ownerEntityHash,
|
|
45
|
+
groupId: manifest.transferKeyDescriptor.groupId,
|
|
46
|
+
})
|
|
47
|
+
if (!fetchedChunk) return false
|
|
48
|
+
await putChunk(part.hash, fetchedChunk)
|
|
49
|
+
}
|
|
50
|
+
return true
|
|
51
|
+
}
|
|
14
52
|
|
|
15
53
|
/**
|
|
16
54
|
* @param {string} ownerEntityHash 所有者
|
|
17
55
|
* @param {string} logicalPath 路径
|
|
18
|
-
* @returns {Promise<import('
|
|
56
|
+
* @returns {Promise<import('./manifest.mjs').FileManifest | null>} 归一化 manifest
|
|
19
57
|
*/
|
|
20
58
|
export async function loadFileManifest(ownerEntityHash, logicalPath) {
|
|
21
59
|
const manifest = await getEntityStore().readManifest(ownerEntityHash, logicalPath)
|
|
@@ -23,7 +61,7 @@ export async function loadFileManifest(ownerEntityHash, logicalPath) {
|
|
|
23
61
|
}
|
|
24
62
|
|
|
25
63
|
/**
|
|
26
|
-
* @param {import('
|
|
64
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
27
65
|
* @returns {Promise<void>}
|
|
28
66
|
*/
|
|
29
67
|
export async function saveFileManifest(manifest) {
|
|
@@ -31,7 +69,7 @@ export async function saveFileManifest(manifest) {
|
|
|
31
69
|
}
|
|
32
70
|
|
|
33
71
|
/**
|
|
34
|
-
* @param {import('
|
|
72
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
35
73
|
* @param {Array<Buffer | Uint8Array>} partBytes 密文块
|
|
36
74
|
* @returns {Promise<void>}
|
|
37
75
|
*/
|
|
@@ -42,7 +80,7 @@ export async function storeManifestParts(manifest, partBytes) {
|
|
|
42
80
|
|
|
43
81
|
/**
|
|
44
82
|
* @param {string} replicaUsername 副本用户名
|
|
45
|
-
* @param {import('
|
|
83
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
46
84
|
* @param {{ username?: string, fetchChunk?: Function }} [opts] miss 拉取
|
|
47
85
|
* @returns {Promise<Buffer | null>} 明文内容
|
|
48
86
|
*/
|
|
@@ -54,50 +92,39 @@ export async function readManifestPlaintext(replicaUsername, manifest, opts = {}
|
|
|
54
92
|
}
|
|
55
93
|
|
|
56
94
|
const username = opts.username || replicaUsername
|
|
95
|
+
if (!await ensureManifestPartsLocal(username, manifest, opts)) return null
|
|
96
|
+
|
|
57
97
|
/** @type {Buffer[]} */
|
|
58
98
|
const partBytes = []
|
|
59
|
-
for (const part of manifest.parts)
|
|
60
|
-
|
|
61
|
-
if (await hasChunk(part.hash))
|
|
62
|
-
ciphertextBytes = await getChunk(part.hash)
|
|
63
|
-
else {
|
|
64
|
-
const fetchedChunk = await (opts.fetchChunk || fetchChunk)({
|
|
65
|
-
username,
|
|
66
|
-
ciphertextHash: part.hash,
|
|
67
|
-
ownerEntityHash: manifest.ownerEntityHash,
|
|
68
|
-
groupId: manifest.transferKeyDescriptor.groupId,
|
|
69
|
-
})
|
|
70
|
-
if (fetchedChunk) {
|
|
71
|
-
await putChunk(part.hash, fetchedChunk)
|
|
72
|
-
ciphertextBytes = Buffer.from(fetchedChunk)
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
if (!ciphertextBytes) return null
|
|
76
|
-
partBytes.push(ciphertextBytes)
|
|
77
|
-
}
|
|
99
|
+
for (const part of manifest.parts)
|
|
100
|
+
partBytes.push(await getChunk(part.hash))
|
|
78
101
|
|
|
79
|
-
|
|
80
|
-
const deps = {
|
|
81
|
-
getGroupFileMasterKey: rawDeps.getGroupFileMasterKey
|
|
82
|
-
? (groupId, keyGeneration) => rawDeps.getGroupFileMasterKey(replicaUsername, groupId, keyGeneration)
|
|
83
|
-
: undefined,
|
|
84
|
-
getVaultMasterKey: rawDeps.getVaultMasterKey
|
|
85
|
-
? entityHash => rawDeps.getVaultMasterKey(replicaUsername, entityHash)
|
|
86
|
-
: undefined,
|
|
87
|
-
}
|
|
88
|
-
return assembleManifestPlaintext(manifest, partBytes, deps)
|
|
102
|
+
return assembleManifestPlaintext(manifest, partBytes, transferKeyDepsForReplica(replicaUsername, manifest))
|
|
89
103
|
}
|
|
90
104
|
|
|
91
105
|
/**
|
|
92
106
|
* @param {string} replicaUsername 副本用户名
|
|
93
|
-
* @param {import('
|
|
107
|
+
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
|
94
108
|
* @param {{ username?: string, fetchChunk?: Function }} [opts] miss 拉取
|
|
95
109
|
* @returns {Promise<import('node:stream').Readable | null>} 明文流
|
|
96
110
|
*/
|
|
97
111
|
export async function readManifestPlaintextStream(replicaUsername, manifest, opts = {}) {
|
|
98
|
-
const
|
|
99
|
-
if (
|
|
100
|
-
|
|
112
|
+
const dagGroupId = manifest.meta?.groupId
|
|
113
|
+
if (Array.isArray(manifest.meta?.dagParts) && dagGroupId) {
|
|
114
|
+
const plain = await readManifestPlaintext(replicaUsername, manifest, opts)
|
|
115
|
+
if (!plain) return null
|
|
116
|
+
return Readable.from([plain])
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const username = opts.username || replicaUsername
|
|
120
|
+
if (!await ensureManifestPartsLocal(username, manifest, opts)) return null
|
|
121
|
+
|
|
122
|
+
const deps = transferKeyDepsForReplica(replicaUsername, manifest)
|
|
123
|
+
const contentKey = await resolveContentKey(manifest.transferKeyDescriptor, manifest, deps)
|
|
124
|
+
if (manifest.ceMode === 'random' && !contentKey) return null
|
|
125
|
+
|
|
126
|
+
const partStreams = manifest.parts.map(part => createChunkReadStream(part.hash))
|
|
127
|
+
return createManifestPlaintextStream(manifest, partStreams, contentKey)
|
|
101
128
|
}
|
|
102
129
|
|
|
103
130
|
/**
|
|
@@ -107,13 +134,12 @@ export async function readManifestPlaintextStream(replicaUsername, manifest, opt
|
|
|
107
134
|
* @param {Buffer | Uint8Array} params.plaintext 明文
|
|
108
135
|
* @param {string} [params.name] 文件名
|
|
109
136
|
* @param {string} [params.mimeType] MIME
|
|
110
|
-
* @param {import('
|
|
111
|
-
* @param {import('
|
|
137
|
+
* @param {import('./manifest.mjs').CeMode} [params.ceMode] 模式
|
|
138
|
+
* @param {import('./manifest.mjs').TransferKeyDescriptor} [params.transferKeyDescriptor] 传递密钥
|
|
112
139
|
* @param {object} [params.meta] meta
|
|
113
|
-
* @returns {Promise<import('
|
|
140
|
+
* @returns {Promise<import('./manifest.mjs').FileManifest>} 写入后的 manifest
|
|
114
141
|
*/
|
|
115
142
|
export async function putFileManifest(params) {
|
|
116
|
-
const { encryptPlaintextToParts, encryptPlaintextToMultiPartsAsync } = await import('../../files/assemble.mjs')
|
|
117
143
|
const {
|
|
118
144
|
ownerEntityHash,
|
|
119
145
|
logicalPath,
|
|
@@ -152,10 +178,10 @@ export async function putFileManifest(params) {
|
|
|
152
178
|
* @param {number} params.plainSize 明文字节数
|
|
153
179
|
* @param {string} [params.name] 文件名
|
|
154
180
|
* @param {string} [params.mimeType] MIME
|
|
155
|
-
* @param {import('
|
|
156
|
-
* @param {import('
|
|
181
|
+
* @param {import('./manifest.mjs').CeMode} [params.ceMode] 模式
|
|
182
|
+
* @param {import('./manifest.mjs').TransferKeyDescriptor} [params.transferKeyDescriptor] 传递密钥
|
|
157
183
|
* @param {object} [params.meta] meta
|
|
158
|
-
* @returns {Promise<import('
|
|
184
|
+
* @returns {Promise<import('./manifest.mjs').FileManifest>} 写入后的 manifest
|
|
159
185
|
*/
|
|
160
186
|
export async function putFileManifestFromStream(params) {
|
|
161
187
|
const {
|
|
@@ -179,7 +205,7 @@ export async function putFileManifestFromStream(params) {
|
|
|
179
205
|
size: plainSize,
|
|
180
206
|
contentHash: enc.contentHash,
|
|
181
207
|
ceMode,
|
|
182
|
-
parts: enc.parts,
|
|
208
|
+
parts: manifestPartsForPersist(enc.parts),
|
|
183
209
|
transferKeyDescriptor: transferKeyDescriptor || publicTransferKeyDescriptor(),
|
|
184
210
|
meta,
|
|
185
211
|
})
|
|
@@ -197,7 +223,7 @@ export async function putFileManifestFromStream(params) {
|
|
|
197
223
|
* @returns {Promise<Buffer | null>} 明文或 null
|
|
198
224
|
*/
|
|
199
225
|
export async function readPublicFile(replicaUsername, entityHash, logicalPath, opts = {}) {
|
|
200
|
-
const { fetchPublicManifest } = await import('
|
|
226
|
+
const { fetchPublicManifest } = await import('./manifest_fetch.mjs')
|
|
201
227
|
const manifest = await fetchPublicManifest({
|
|
202
228
|
username: opts.username || replicaUsername,
|
|
203
229
|
ownerEntityHash: entityHash,
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
-
import { isEntityHash128 } from '
|
|
2
|
-
import { assertSafeEvfsLogicalPath } from '
|
|
1
|
+
import { isEntityHash128 } from '../core/entity_id.mjs'
|
|
2
|
+
import { assertSafeEvfsLogicalPath } from '../core/evfs_logical_path.mjs'
|
|
3
3
|
|
|
4
|
-
/**
|
|
5
|
-
*
|
|
6
|
-
*/
|
|
4
|
+
/** @type {string} evfs URI scheme */
|
|
7
5
|
export const EVFS_SCHEME = 'evfs:'
|
|
8
6
|
|
|
9
7
|
/**
|
package/files/manifest.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { isHex64 } from '../core/hexIds.mjs'
|
|
|
3
3
|
|
|
4
4
|
/** @typedef {'plain' | 'convergent' | 'random'} CeMode */
|
|
5
5
|
|
|
6
|
-
/** @typedef {{ hash: string, size: number }} ManifestPart */
|
|
6
|
+
/** @typedef {{ hash: string, size: number, contentHash?: string }} ManifestPart */
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* @typedef {{
|
|
@@ -48,7 +48,16 @@ export function normalizeFileManifest(input) {
|
|
|
48
48
|
if (!CE_MODES.has(ceMode)) return null
|
|
49
49
|
const parts = Array.isArray(input.parts)
|
|
50
50
|
? input.parts
|
|
51
|
-
.map(part =>
|
|
51
|
+
.map(part => {
|
|
52
|
+
/** @type {ManifestPart} */
|
|
53
|
+
const out = {
|
|
54
|
+
hash: String(part?.hash || '').trim().toLowerCase(),
|
|
55
|
+
size: Number(part?.size) || 0,
|
|
56
|
+
}
|
|
57
|
+
const partContentHash = String(part?.contentHash || '').trim().toLowerCase()
|
|
58
|
+
if (isHex64(partContentHash)) out.contentHash = partContentHash
|
|
59
|
+
return out
|
|
60
|
+
})
|
|
52
61
|
.filter(part => isHex64(part.hash))
|
|
53
62
|
: []
|
|
54
63
|
if (!parts.length) return null
|
|
@@ -5,14 +5,14 @@
|
|
|
5
5
|
/** @type {Map<string, { ownerId: string, handler: (context: ManifestAclContext, logicalPath?: string) => Promise<boolean> }>} */
|
|
6
6
|
const handlersByType = new Map()
|
|
7
7
|
|
|
8
|
-
/** @type {Map<string, { ownerId: string, match: (manifest: import('
|
|
8
|
+
/** @type {Map<string, { ownerId: string, match: (manifest: import('./manifest.mjs').FileManifest | null | undefined, ownerEntityHash: string) => string | null }>} */
|
|
9
9
|
const matchersByOwner = new Map()
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* @typedef {{
|
|
13
13
|
* replicaUsername: string,
|
|
14
14
|
* ownerEntityHash: string,
|
|
15
|
-
* manifest: import('
|
|
15
|
+
* manifest: import('./manifest.mjs').FileManifest,
|
|
16
16
|
* }} ManifestAclContext
|
|
17
17
|
*/
|
|
18
18
|
|
|
@@ -46,7 +46,7 @@ export function unregisterManifestAcl(type, ownerId) {
|
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
48
|
* @param {string} ownerId 注册方
|
|
49
|
-
* @param {(manifest: import('
|
|
49
|
+
* @param {(manifest: import('./manifest.mjs').FileManifest | null | undefined, ownerEntityHash: string) => string | null} match ACL 类型匹配
|
|
50
50
|
* @returns {void}
|
|
51
51
|
*/
|
|
52
52
|
export function registerManifestAclMatcher(ownerId, match) {
|
|
@@ -61,14 +61,8 @@ export function unregisterManifestAclMatcher(ownerId) {
|
|
|
61
61
|
matchersByOwner.delete(String(ownerId))
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
/** @returns {void} */
|
|
65
|
-
export function clearManifestAclRegistry() {
|
|
66
|
-
handlersByType.clear()
|
|
67
|
-
matchersByOwner.clear()
|
|
68
|
-
}
|
|
69
|
-
|
|
70
64
|
/**
|
|
71
|
-
* @param {import('
|
|
65
|
+
* @param {import('./manifest.mjs').FileManifest | null | undefined} manifest manifest
|
|
72
66
|
* @param {string} ownerEntityHash 所有者
|
|
73
67
|
* @returns {string | null} ACL 类型
|
|
74
68
|
*/
|
package/files/manifest_fetch.mjs
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto'
|
|
2
2
|
|
|
3
|
-
import { loadFileManifest, saveFileManifest } from '../entity/files/evfs.mjs'
|
|
4
|
-
import { isWritableLocalEntity } from '../entity/replica.mjs'
|
|
5
3
|
import {
|
|
6
4
|
manifestFetchExpectedKey,
|
|
7
5
|
MAX_PENDING_MANIFEST_FETCHES,
|
|
8
6
|
pendingManifestFetches,
|
|
9
7
|
registerManifestFetchWait,
|
|
10
8
|
} from '../federation/manifest_fetch_pending.mjs'
|
|
9
|
+
import { isWritableLocalEntity } from '../node/identity.mjs'
|
|
11
10
|
import { getEntityStore } from '../node/instance.mjs'
|
|
12
11
|
|
|
13
12
|
import { resolveNodeHash } from './chunk_provider_registry.mjs'
|
|
13
|
+
import { loadFileManifest, saveFileManifest } from './evfs.mjs'
|
|
14
14
|
import { fanoutFedFetch } from './fetch_fanout.mjs'
|
|
15
15
|
import { normalizeFileManifest } from './manifest.mjs'
|
|
16
16
|
import { shouldPreferIncomingPublicManifest } from './public_manifest.mjs'
|
|
@@ -6,6 +6,7 @@ import { assertSafeEvfsLogicalPath } from '../core/evfs_logical_path.mjs'
|
|
|
6
6
|
import { isHex64, normalizeHex64 } from '../core/hexIds.mjs'
|
|
7
7
|
import { sign, verify } from '../crypto/crypto.mjs'
|
|
8
8
|
|
|
9
|
+
import { putFileManifest, saveFileManifest } from './evfs.mjs'
|
|
9
10
|
import { normalizeFileManifest, publicTransferKeyDescriptor } from './manifest.mjs'
|
|
10
11
|
|
|
11
12
|
/** 实体公开 manifest 签名域 */
|
|
@@ -137,7 +138,6 @@ export async function publishPublicFile(params) {
|
|
|
137
138
|
entityPubKeyHex,
|
|
138
139
|
publishedAt = Date.now(),
|
|
139
140
|
} = params
|
|
140
|
-
const { putFileManifest, saveFileManifest } = await import('../entity/files/evfs.mjs')
|
|
141
141
|
const base = await putFileManifest({
|
|
142
142
|
ownerEntityHash,
|
|
143
143
|
logicalPath,
|
package/files/transfer_key.mjs
CHANGED
|
@@ -46,17 +46,23 @@ export async function resolveContentKey(descriptor, manifest, deps = {}) {
|
|
|
46
46
|
* @param {Buffer | Uint8Array} encryptedPartBytes 密文块
|
|
47
47
|
* @param {FileManifest} manifest manifest
|
|
48
48
|
* @param {Buffer | null} contentKey random 模式密钥
|
|
49
|
+
* @param {number} [partIndex] 分块下标(多块 convergent 用 part.contentHash)
|
|
49
50
|
* @returns {Buffer | null} 明文
|
|
50
51
|
*/
|
|
51
|
-
export function decryptPart(encryptedPartBytes, manifest, contentKey) {
|
|
52
|
+
export function decryptPart(encryptedPartBytes, manifest, contentKey, partIndex = 0) {
|
|
52
53
|
if (manifest.ceMode === 'plain')
|
|
53
54
|
return Buffer.from(encryptedPartBytes)
|
|
54
55
|
|
|
55
|
-
if (manifest.ceMode === 'convergent')
|
|
56
|
-
|
|
56
|
+
if (manifest.ceMode === 'convergent') {
|
|
57
|
+
const partPlainHash = manifest.parts[partIndex]?.contentHash || manifest.contentHash
|
|
58
|
+
return decryptConvergentCiphertext(encryptedPartBytes, partPlainHash)
|
|
59
|
+
}
|
|
57
60
|
|
|
58
|
-
if (manifest.ceMode === 'random' && contentKey)
|
|
59
|
-
|
|
61
|
+
if (manifest.ceMode === 'random' && contentKey) {
|
|
62
|
+
// 多块时各块明文 hash ≠ 整文件 contentHash;完整性在 assemble 末尾校验
|
|
63
|
+
const verifyHash = manifest.parts.length === 1 ? manifest.contentHash : ''
|
|
64
|
+
return decryptRandomCiphertext(encryptedPartBytes, contentKey, verifyHash)
|
|
65
|
+
}
|
|
60
66
|
|
|
61
67
|
return null
|
|
62
68
|
}
|
|
@@ -73,7 +79,7 @@ export async function assembleManifestPlaintext(manifest, partBytes, deps = {})
|
|
|
73
79
|
/** @type {Buffer[]} */
|
|
74
80
|
const plains = []
|
|
75
81
|
for (let index = 0; index < manifest.parts.length; index++) {
|
|
76
|
-
const plain = decryptPart(partBytes[index], manifest, contentKey)
|
|
82
|
+
const plain = decryptPart(partBytes[index], manifest, contentKey, index)
|
|
77
83
|
if (!plain) return null
|
|
78
84
|
plains.push(plain)
|
|
79
85
|
}
|
|
@@ -57,13 +57,6 @@ export function unregisterTransferKeyDeps(ownerId) {
|
|
|
57
57
|
unregisterManifestOwnerMatchers(key)
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
/** @returns {void} */
|
|
61
|
-
export function clearTransferKeyRegistry() {
|
|
62
|
-
transferDepsByOwner.clear()
|
|
63
|
-
dagPlaintextReadersByOwner.clear()
|
|
64
|
-
manifestOwnerMatchers.length = 0
|
|
65
|
-
}
|
|
66
|
-
|
|
67
60
|
/**
|
|
68
61
|
* 从 manifest 推断 transfer key 注册方 id(按 registerManifestOwnerMatcher 注册顺序匹配)。
|
|
69
62
|
* @param {import('./manifest.mjs').FileManifest} manifest 清单
|
package/governance/branch.mjs
CHANGED
|
@@ -93,17 +93,6 @@ export function descendantClosureFromTip(rootId, byId) {
|
|
|
93
93
|
return out
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
/**
|
|
97
|
-
* 事件是否在 root 的后代闭包中(含 root)。
|
|
98
|
-
* @param {string} eventId 事件 id
|
|
99
|
-
* @param {string} rootId 根 id
|
|
100
|
-
* @param {Map<string, object>} byId id→事件
|
|
101
|
-
* @returns {boolean} 是否为 root 的后代(含 root 自身)
|
|
102
|
-
*/
|
|
103
|
-
export function isDescendantOfTip(eventId, rootId, byId) {
|
|
104
|
-
return descendantClosureFromTip(rootId, byId).has(eventId)
|
|
105
|
-
}
|
|
106
|
-
|
|
107
96
|
/**
|
|
108
97
|
* @param {string} tipId 叶 id
|
|
109
98
|
* @param {Map<string, object>} byId id→事件
|
package/link/channel_mux.mjs
CHANGED