libp2r2p 0.10.28 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/nip27/index.js +1 -0
- package/nip94/README.md +13 -0
- package/nip94/index.js +56 -2
- package/package.json +4 -1
- package/private-channel/helpers/chunk-size.js +1 -1
- package/private-channel/helpers/chunks.js +1 -1
- package/private-channel/helpers/event.js +14 -3
- package/private-channel/index.js +25 -7
- package/private-channel/services/received-chunks.js +6 -3
- package/private-message/index.js +21 -7
- package/private-messenger/event-store/README.md +110 -0
- package/private-messenger/event-store/helpers/codec.js +136 -0
- package/private-messenger/event-store/index.js +176 -0
- package/private-messenger/file/README.md +152 -0
- package/private-messenger/file/helpers/cache.js +159 -0
- package/private-messenger/file/helpers/catalog.js +87 -0
- package/private-messenger/file/helpers/stream.js +31 -0
- package/private-messenger/file/index.js +433 -0
- package/private-messenger/index.js +45 -10
- package/private-messenger/recovery/index.js +1 -1
- package/private-messenger/services/storage-maintenance.js +3 -0
- package/private-messenger/session/README.md +39 -0
- package/private-messenger/session/helpers/publication.js +40 -0
- package/private-messenger/session/helpers/signer.js +13 -0
- package/private-messenger/session/helpers/work-storage.js +32 -0
- package/private-messenger/session/index.js +307 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { ValidationError } from '../../../error/index.js'
|
|
2
|
+
import { createQueue } from '../../../idb-queue/index.js'
|
|
3
|
+
|
|
4
|
+
const localLocks = new WeakMap()
|
|
5
|
+
const keyFor = row => `${row.controlChannelPubkey}:${row.fileChannelPubkey}:${row.receiverPubkey}`
|
|
6
|
+
|
|
7
|
+
// A grant contains no payload, key material or retention reference in the file
|
|
8
|
+
// store. Serialize read/merge/write with other coordinators for this identity.
|
|
9
|
+
export function createFileCatalog ({ messenger, now, storage }) {
|
|
10
|
+
if (storage) {
|
|
11
|
+
const valid = row => {
|
|
12
|
+
const parent = messenger.channels.get(row.controlChannelPubkey)
|
|
13
|
+
return parent?.mode === 'seeder' && row.expiresAt > now() && row.sharedAt + messenger.offlineRecoverySecondsFor(parent) > now()
|
|
14
|
+
}
|
|
15
|
+
return {
|
|
16
|
+
ready: Promise.resolve(),
|
|
17
|
+
async authorize (file, receivers, sharedAt) {
|
|
18
|
+
const { controlChannelPubkey, fileChannelPubkey, peerPubkey, root, size } = file
|
|
19
|
+
for (const receiverPubkey of receivers) {
|
|
20
|
+
const row = { controlChannelPubkey, fileChannelPubkey, peerPubkey, root, size, receiverPubkey, sharedAt, expiresAt: sharedAt + messenger.offlineRecoverySecondsFor(controlChannelPubkey) }
|
|
21
|
+
if (valid(row)) await storage.put(row)
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
async find (controlChannelPubkey, fileChannelPubkey, receiverPubkey) {
|
|
25
|
+
const row = await storage.find({ controlChannelPubkey, fileChannelPubkey, receiverPubkey })
|
|
26
|
+
return row && valid(row) ? row : null
|
|
27
|
+
},
|
|
28
|
+
prune: () => storage.prune({ now: now() }),
|
|
29
|
+
async close () {}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const prefix = `${messenger.prefix}:file-authorizations`
|
|
33
|
+
let database
|
|
34
|
+
const db = () => (database ??= createQueue({ prefix, indexedDB: messenger._indexedDB, evictionPolicy: 'reject', indexes: { key: { keyPath: 'key', unique: true }, channel: 'fileChannelPubkey' } }))
|
|
35
|
+
function lock (work) {
|
|
36
|
+
if (globalThis.navigator?.locks?.request) return navigator.locks.request(prefix, work)
|
|
37
|
+
let locks = localLocks.get(messenger._indexedDB)
|
|
38
|
+
if (!locks) localLocks.set(messenger._indexedDB, (locks = new Map()))
|
|
39
|
+
const task = (locks.get(prefix) || Promise.resolve()).catch(() => {}).then(work)
|
|
40
|
+
locks.set(prefix, task.catch(() => {}))
|
|
41
|
+
return task
|
|
42
|
+
}
|
|
43
|
+
function isValid (row) {
|
|
44
|
+
const parent = messenger.channels.get(row.controlChannelPubkey)
|
|
45
|
+
return parent?.mode === 'seeder' && row.expiresAt > now() && row.sharedAt + messenger.offlineRecoverySecondsFor(parent) > now()
|
|
46
|
+
}
|
|
47
|
+
async function put (file, receiverPubkey, sharedAt, expiresAt) {
|
|
48
|
+
const { controlChannelPubkey, fileChannelPubkey, peerPubkey, root, size } = file
|
|
49
|
+
const row = { controlChannelPubkey, fileChannelPubkey, peerPubkey, root, size, receiverPubkey, sharedAt, expiresAt }
|
|
50
|
+
row.key = keyFor(row)
|
|
51
|
+
if (!isValid(row)) return
|
|
52
|
+
const queue = await db()
|
|
53
|
+
const prior = await queue.getBy('key', row.key)
|
|
54
|
+
if (prior && (prior.root !== root || prior.peerPubkey !== peerPubkey)) throw new ValidationError('FILE_AUTHORIZATION_CONFLICT')
|
|
55
|
+
// Replaying an older share cannot shorten or renew a newer grant.
|
|
56
|
+
if (prior && prior.sharedAt >= sharedAt && isValid(prior)) return
|
|
57
|
+
await queue.putBy('key', row)
|
|
58
|
+
}
|
|
59
|
+
async function pruneUnlocked () {
|
|
60
|
+
await (await db()).removeWhere(row => !isValid(row))
|
|
61
|
+
}
|
|
62
|
+
const ready = lock(pruneUnlocked)
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
ready,
|
|
66
|
+
async authorize (file, receivers, sharedAt) {
|
|
67
|
+
await ready
|
|
68
|
+
return lock(async () => {
|
|
69
|
+
await pruneUnlocked()
|
|
70
|
+
const expiresAt = sharedAt + messenger.offlineRecoverySecondsFor(file.controlChannelPubkey)
|
|
71
|
+
for (const receiver of receivers) await put(file, receiver, sharedAt, expiresAt)
|
|
72
|
+
})
|
|
73
|
+
},
|
|
74
|
+
async find (controlChannelPubkey, fileChannelPubkey, receiverPubkey) {
|
|
75
|
+
await ready
|
|
76
|
+
return lock(async () => {
|
|
77
|
+
const queue = await db()
|
|
78
|
+
const key = keyFor({ controlChannelPubkey, fileChannelPubkey, receiverPubkey })
|
|
79
|
+
const row = await queue.getBy('key', key)
|
|
80
|
+
if (row && !isValid(row)) { await queue.removeBy('key', key); return null }
|
|
81
|
+
return row
|
|
82
|
+
})
|
|
83
|
+
},
|
|
84
|
+
async prune () { await ready; return lock(pruneUnlocked) },
|
|
85
|
+
async close () { await ready.catch(() => {}); await lock(async () => { if (database) await (await database.catch(() => null))?.close() }) }
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { decodeIrfsChunk } from '../../../irfs/index.js'
|
|
2
|
+
import { ValidationError } from '../../../error/index.js'
|
|
3
|
+
|
|
4
|
+
export function chunkStream ({ read, root, size, signal, release = async () => {} }) {
|
|
5
|
+
let index = 0, total, bytes = 0, settled = false, controller
|
|
6
|
+
const finish = async () => {
|
|
7
|
+
if (settled) return
|
|
8
|
+
settled = true; signal?.removeEventListener('abort', abort)
|
|
9
|
+
await release()
|
|
10
|
+
}
|
|
11
|
+
const abort = () => { if (settled) return; controller.error(signal.reason); finish().catch(() => {}) }
|
|
12
|
+
return new ReadableStream({
|
|
13
|
+
start (value) { controller = value; signal?.addEventListener('abort', abort, { once: true }); if (signal?.aborted) abort() },
|
|
14
|
+
async pull () {
|
|
15
|
+
try {
|
|
16
|
+
signal?.throwIfAborted()
|
|
17
|
+
const event = await read(root, index)
|
|
18
|
+
signal?.throwIfAborted()
|
|
19
|
+
if (settled) return
|
|
20
|
+
if (!event) throw new Error('FILE_UNAVAILABLE')
|
|
21
|
+
const chunk = decodeIrfsChunk(event)
|
|
22
|
+
if (chunk.root !== root || chunk.index !== index || (total !== undefined && total !== chunk.total)) throw new ValidationError('FILE_CHUNK_DESCRIPTOR_MISMATCH')
|
|
23
|
+
total = chunk.total; index++; bytes += chunk.contentBytes.length
|
|
24
|
+
if (index === total && size !== undefined && bytes !== size) throw new ValidationError('FILE_SIZE_MISMATCH')
|
|
25
|
+
controller.enqueue(chunk.contentBytes)
|
|
26
|
+
if (index === total) { await finish(); controller.close() }
|
|
27
|
+
} catch (error) { if (!settled) { await finish(); controller.error(error) } }
|
|
28
|
+
},
|
|
29
|
+
cancel: finish
|
|
30
|
+
}, { highWaterMark: 1 })
|
|
31
|
+
}
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
import NMMR from 'nmmr'
|
|
2
|
+
import { chunkStream } from './helpers/stream.js'
|
|
3
|
+
import { createFileCache } from './helpers/cache.js'
|
|
4
|
+
export { createFileCache, DEFAULT_FILE_CACHE_BYTES } from './helpers/cache.js'
|
|
5
|
+
import { createFileCatalog } from './helpers/catalog.js'
|
|
6
|
+
import { ValidationError } from '../../error/index.js'
|
|
7
|
+
import { IRFS_CHUNK_BYTES, decodeIrfsChunk } from '../../irfs/index.js'
|
|
8
|
+
import { createQueue } from '../../idb-queue/index.js'
|
|
9
|
+
import * as channelTransport from '../../private-channel/index.js'
|
|
10
|
+
import * as messages from '../../private-message/index.js'
|
|
11
|
+
import { readFileChunkIndex } from '../../private-channel/helpers/event.js'
|
|
12
|
+
import { compactSeedRouterRows, compactRecordsFromSeed, createEventReplyPacker, ROUTER_SEED_RECORD_TYPE } from '../recovery/index.js'
|
|
13
|
+
|
|
14
|
+
export const FILE_CHUNKS_REQUEST_CODE = 'fileChunksRequest_p5cc'
|
|
15
|
+
export const FILE_CHUNKS_REPLY_CODE = 'fileChunksReply_p5cc'
|
|
16
|
+
export const IRFS_CHUNK_RECORD_TYPE = 'irfsChunk_v1'
|
|
17
|
+
export const AUTO_DOWNLOAD_BYTES = 1024 * 1024
|
|
18
|
+
export const FILE_REQUEST_CHUNKS = 16
|
|
19
|
+
const HEX = /^[0-9a-f]{64}$/
|
|
20
|
+
const now = () => Math.floor(Date.now() / 1000)
|
|
21
|
+
|
|
22
|
+
export function fileChannelInfo (root) {
|
|
23
|
+
if (typeof root !== 'string' || !HEX.test(root)) throw new ValidationError('INVALID_FILE_ROOT')
|
|
24
|
+
return `dm:media:${root}`
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function decodeMissingRanges (ranges) {
|
|
28
|
+
if (!Array.isArray(ranges) || !ranges.length) throw new ValidationError('INVALID_FILE_CHUNK_RANGES')
|
|
29
|
+
const indices = []
|
|
30
|
+
let previous = -1
|
|
31
|
+
for (const range of ranges) {
|
|
32
|
+
if (!Array.isArray(range) || range.length !== 2) throw new ValidationError('INVALID_FILE_CHUNK_RANGES')
|
|
33
|
+
const [first, last] = range
|
|
34
|
+
if (![first, last].every(Number.isSafeInteger) || first < 0 || first <= previous || last < first || last - first + 1 > FILE_REQUEST_CHUNKS - indices.length) throw new ValidationError('INVALID_FILE_CHUNK_RANGES')
|
|
35
|
+
for (let index = first; index <= last; index++) indices.push(index)
|
|
36
|
+
previous = last
|
|
37
|
+
}
|
|
38
|
+
return indices
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function rangesFor (indices) {
|
|
42
|
+
const ranges = []
|
|
43
|
+
for (const index of indices) {
|
|
44
|
+
const last = ranges.at(-1)
|
|
45
|
+
if (last && last[1] + 1 === index) last[1] = index
|
|
46
|
+
else ranges.push([index, index])
|
|
47
|
+
}
|
|
48
|
+
return ranges
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function assertPublished (result) {
|
|
52
|
+
const reports = result?.delivery?.reports
|
|
53
|
+
if (!Array.isArray(reports) || !reports.length || !reports.every(report => report?.success === true)) throw Object.assign(new Error('FILE_NOT_PUBLISHED'), { code: 'MESSAGE_NOT_PUBLISHED' })
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function waitForTransfer (promise, signal) {
|
|
57
|
+
signal?.throwIfAborted()
|
|
58
|
+
if (!signal) return promise
|
|
59
|
+
return new Promise((resolve, reject) => {
|
|
60
|
+
const aborted = () => reject(signal.reason)
|
|
61
|
+
signal.addEventListener('abort', aborted, { once: true })
|
|
62
|
+
promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', aborted))
|
|
63
|
+
})
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// One coordinator per account. The caller resolves signers and may supply
|
|
67
|
+
// storage; recovery, validation and scheduling belong here.
|
|
68
|
+
export function createPrivateFileTransfer ({ messenger, resolveChannel, storage, authorizationStorage, seedStorage, cacheMaxBytes, onError = () => {}, _transport = channelTransport, _messages = messages, _hedgeMs = 2000, _idleMs = 30000, _now = now }) {
|
|
69
|
+
if (!messenger?.extensions || typeof resolveChannel !== 'function' || (storage && (!storage.read || !storage.save))) throw new ValidationError('INVALID_FILE_TRANSFER_OPTIONS')
|
|
70
|
+
const ownsStorage = !storage
|
|
71
|
+
storage ||= createFileCache({ prefix: messenger.prefix, indexedDB: messenger._indexedDB, maxBytes: cacheMaxBytes })
|
|
72
|
+
const uploadLeases = new Map()
|
|
73
|
+
const registrations = new Map()
|
|
74
|
+
const jobs = new Map()
|
|
75
|
+
const listeners = new Set()
|
|
76
|
+
const successes = new Map()
|
|
77
|
+
const work = new Set()
|
|
78
|
+
const requests = []
|
|
79
|
+
let replying = 0
|
|
80
|
+
let active = 0
|
|
81
|
+
let closed = false
|
|
82
|
+
let queuePromise
|
|
83
|
+
const queue = () => (queuePromise ??= createQueue({ prefix: `${messenger.prefix}:file-seeds`, indexedDB: messenger._indexedDB, maxBytes: 64 * 1024 * 1024, evictionPolicy: 'fifo', indexes: { key: { keyPath: 'key', unique: true }, channel: 'fileChannelPubkey' } }))
|
|
84
|
+
const catalog = createFileCatalog({ messenger, now: _now, storage: authorizationStorage })
|
|
85
|
+
catalog.ready.catch(onError)
|
|
86
|
+
const keyFor = (control, root) => `${control}:${root}`
|
|
87
|
+
const emit = job => { for (const listener of listeners) listener({ root: job.file.root, controlChannelPubkey: job.file.controlChannelPubkey, status: job.status, completed: job.completed, total: job.file.size, error: job.error }) }
|
|
88
|
+
const usable = file => {
|
|
89
|
+
if (closed) throw new Error('FILE_TRANSFER_CLOSED')
|
|
90
|
+
messenger.requireWritableChannel(file.controlChannelPubkey)
|
|
91
|
+
if (!messenger.desiredChannels.has(file.controlChannelPubkey)) throw new Error('FILE_TRANSFER_CHANNEL_UNWATCHED')
|
|
92
|
+
}
|
|
93
|
+
async function register ({ controlChannelPubkey, peerPubkey, root, size, sharedAt }) {
|
|
94
|
+
fileChannelInfo(root)
|
|
95
|
+
if (!HEX.test(controlChannelPubkey) || !HEX.test(peerPubkey) || (size !== undefined && (!Number.isSafeInteger(size) || size < 1))) throw new ValidationError('INVALID_FILE_TRANSFER_DESCRIPTOR')
|
|
96
|
+
if (sharedAt !== undefined && (!Number.isSafeInteger(sharedAt) || sharedAt < 0)) throw new ValidationError('INVALID_FILE_SHARED_AT')
|
|
97
|
+
const key = keyFor(controlChannelPubkey, root)
|
|
98
|
+
const previous = registrations.get(key)
|
|
99
|
+
if (previous && (previous.peerPubkey !== peerPubkey || (size !== undefined && previous.size !== undefined && previous.size !== size))) throw new ValidationError('CONFLICTING_FILE_TRANSFER_DESCRIPTOR')
|
|
100
|
+
const file = previous || { controlChannelPubkey, peerPubkey, root }
|
|
101
|
+
if (size !== undefined) file.size = size
|
|
102
|
+
if (sharedAt !== undefined) file.sharedAt = Math.max(file.sharedAt ?? 0, sharedAt)
|
|
103
|
+
registrations.set(key, file)
|
|
104
|
+
file.ready ||= (async () => {
|
|
105
|
+
file.signer = await resolveChannel({ controlChannelPubkey, peerPubkey, root, info: fileChannelInfo(root) })
|
|
106
|
+
file.fileChannelPubkey = await file.signer.getPublicKey()
|
|
107
|
+
if (!HEX.test(file.fileChannelPubkey)) throw new ValidationError('INVALID_FILE_CHANNEL_PUBKEY')
|
|
108
|
+
})().catch(error => { if (registrations.get(key) === file) registrations.delete(key); throw error })
|
|
109
|
+
await file.ready
|
|
110
|
+
return file
|
|
111
|
+
}
|
|
112
|
+
function seedEnabled (file) {
|
|
113
|
+
const parent = messenger.channels.get(file.controlChannelPubkey)
|
|
114
|
+
return ['seeder', 'watchtower'].includes(parent?.mode) && messenger.offlineRecoverySecondsFor(parent) > 0
|
|
115
|
+
}
|
|
116
|
+
async function authorizeSeeding (descriptor, { receiverPubkeys, sharedAt } = {}) {
|
|
117
|
+
if (!Array.isArray(receiverPubkeys) || !receiverPubkeys.length || !receiverPubkeys.every(key => typeof key === 'string' && HEX.test(key))) throw new ValidationError('INVALID_FILE_AUTHORIZATION_RECEIVERS')
|
|
118
|
+
if (!Number.isSafeInteger(sharedAt) || sharedAt < 0) throw new ValidationError('INVALID_FILE_SHARED_AT')
|
|
119
|
+
const file = await register(descriptor)
|
|
120
|
+
usable(file)
|
|
121
|
+
await catalog.authorize(file, [...new Set(receiverPubkeys)], sharedAt)
|
|
122
|
+
}
|
|
123
|
+
async function prune () {
|
|
124
|
+
await catalog.prune()
|
|
125
|
+
if (seedStorage) { await seedStorage.prune({ now: _now() }); return }
|
|
126
|
+
if (!queuePromise) return
|
|
127
|
+
await (await queue()).removeWhere(row => {
|
|
128
|
+
const parent = messenger.channels.get(row.controlChannelPubkey)
|
|
129
|
+
return parent?.mode !== 'watchtower' || !seedEnabled(row) || row.expiresAt <= _now() || row.receivedAt + messenger.offlineRecoverySecondsFor(parent) <= _now()
|
|
130
|
+
})
|
|
131
|
+
}
|
|
132
|
+
async function saveSeed (file, seed) {
|
|
133
|
+
if (messenger.channels.get(file.controlChannelPubkey)?.mode !== 'watchtower' || !seedEnabled(file)) return
|
|
134
|
+
const index = readFileChunkIndex(seed.router)
|
|
135
|
+
if (index === undefined) return
|
|
136
|
+
const db = seedStorage ? null : await queue()
|
|
137
|
+
for (const row of compactSeedRouterRows(seed)) {
|
|
138
|
+
if (![file.peerPubkey, messenger.userPubkey].includes(row.receiverPubkey)) continue
|
|
139
|
+
const key = `${file.controlChannelPubkey}:${file.fileChannelPubkey}:${row.receiverPubkey}:${index}`
|
|
140
|
+
const record = { ...row, key, root: file.root, controlChannelPubkey: file.controlChannelPubkey, fileChannelPubkey: file.fileChannelPubkey, peerPubkey: file.peerPubkey, chunkIndex: index, receivedAt: _now(), expiresAt: _now() + messenger.offlineRecoverySecondsFor(file.controlChannelPubkey) }
|
|
141
|
+
if (seedStorage) await seedStorage.put(record)
|
|
142
|
+
else await db.putBy('key', record)
|
|
143
|
+
}
|
|
144
|
+
await prune()
|
|
145
|
+
}
|
|
146
|
+
function checked (file, event, index) {
|
|
147
|
+
const decoded = decodeIrfsChunk(event)
|
|
148
|
+
if (decoded.root !== file.root || (index !== undefined && decoded.index !== index) || (file.size !== undefined && (decoded.total !== Math.ceil(file.size / IRFS_CHUNK_BYTES) || (decoded.index === decoded.total - 1 && decoded.contentBytes.length !== file.size - decoded.index * IRFS_CHUNK_BYTES)))) throw new ValidationError('FILE_CHUNK_DESCRIPTOR_MISMATCH')
|
|
149
|
+
return decoded
|
|
150
|
+
}
|
|
151
|
+
async function routing (file, receiverPubkey) {
|
|
152
|
+
const parent = messenger.requireWritableChannel(file.controlChannelPubkey)
|
|
153
|
+
return messenger.resolveSendRouting({ channel: parent, receiverPubkeys: [receiverPubkey] })
|
|
154
|
+
}
|
|
155
|
+
function sendOptions (file, routes) {
|
|
156
|
+
return { senderSigner: messenger.userSigner, imkcSigner: messenger.contentKeySigner, privateChannelSigner: file.signer, ...routes, temporaryStorageArea: messenger.temporaryStorageArea, expirationSeconds: messenger.eventExpirationSecondsFor(file.controlChannelPubkey), _getIykcProofs: messenger.contentKeyLookup() }
|
|
157
|
+
}
|
|
158
|
+
async function publishChunk (descriptor, event) {
|
|
159
|
+
const file = await register(descriptor)
|
|
160
|
+
usable(file)
|
|
161
|
+
const decoded = checked(file, event)
|
|
162
|
+
await catalog.ready
|
|
163
|
+
if (ownsStorage) {
|
|
164
|
+
if (!uploadLeases.has(file.root)) uploadLeases.set(file.root, await storage.reserve(file.root, file.size))
|
|
165
|
+
try { await storage.save(event, file) } catch (error) { await uploadLeases.get(file.root)?.(); uploadLeases.delete(file.root); throw error }
|
|
166
|
+
}
|
|
167
|
+
let result
|
|
168
|
+
try {
|
|
169
|
+
const options = { ...sendOptions(file, await routing(file, file.peerPubkey)), receiverPubkeys: [file.peerPubkey], fileChunkIndex: decoded.index, ...(messenger.channels.get(file.controlChannelPubkey)?.mode === 'watchtower' ? { onPreparedSeed: seed => saveSeed(file, seed) } : {}) }
|
|
170
|
+
result = event.sig ? await _messages.broadcastEvent({ ...options, event }) : await _messages.broadcastRumor({ ...options, rumor: event })
|
|
171
|
+
assertPublished(result)
|
|
172
|
+
} catch (error) { await uploadLeases.get(file.root)?.(); uploadLeases.delete(file.root); throw error }
|
|
173
|
+
if (decoded.index === decoded.total - 1) { await uploadLeases.get(file.root)?.(); uploadLeases.delete(file.root) }
|
|
174
|
+
// A scheduler turn between chunks lets queued text/control publications run.
|
|
175
|
+
await new Promise(resolve => setTimeout(resolve, 0))
|
|
176
|
+
return result
|
|
177
|
+
}
|
|
178
|
+
async function replyToRequest (controlChannelPubkey, message) {
|
|
179
|
+
if (message.payload?.code !== FILE_CHUNKS_REQUEST_CODE) return false
|
|
180
|
+
const question = message.question || message.event
|
|
181
|
+
if (message.provenance === 'hearsay' || message.senderPubkey !== question?.pubkey || question.tags?.find(tag => tag[0] === 'r')?.[1] !== messenger.userPubkey) return true
|
|
182
|
+
const payload = message.payload.payload
|
|
183
|
+
if (!HEX.test(payload?.fileChannelPubkey)) throw new ValidationError('INVALID_FILE_TRANSFER_DESCRIPTOR')
|
|
184
|
+
const indices = new Set(decodeMissingRanges(payload.missingRanges))
|
|
185
|
+
await prune()
|
|
186
|
+
const grant = await catalog.find(controlChannelPubkey, payload.fileChannelPubkey, question.pubkey)
|
|
187
|
+
if (grant) {
|
|
188
|
+
const file = await register(grant)
|
|
189
|
+
usable(file)
|
|
190
|
+
if (file.fileChannelPubkey !== payload.fileChannelPubkey) throw new ValidationError('FILE_CHANNEL_MISMATCH')
|
|
191
|
+
const routes = await routing(file, question.pubkey)
|
|
192
|
+
const packer = createEventReplyPacker({
|
|
193
|
+
messenger: {
|
|
194
|
+
reply: async options => {
|
|
195
|
+
usable(file)
|
|
196
|
+
if (!await catalog.find(controlChannelPubkey, file.fileChannelPubkey, question.pubkey)) return
|
|
197
|
+
return _messages.reply({ ...sendOptions(file, routes), ...options })
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
channelPubkey: file.fileChannelPubkey, question, receiverPubkey: question.pubkey,
|
|
201
|
+
code: FILE_CHUNKS_REPLY_CODE, eventsPerChunk: 1, recordsFromInput: record => [record]
|
|
202
|
+
})
|
|
203
|
+
for (const index of indices) {
|
|
204
|
+
usable(file)
|
|
205
|
+
if (!await catalog.find(controlChannelPubkey, file.fileChannelPubkey, question.pubkey)) break
|
|
206
|
+
const event = await storage.read(file.root, index, file)
|
|
207
|
+
if (!event) continue
|
|
208
|
+
let decoded
|
|
209
|
+
try { decoded = checked(file, event, index) } catch (error) { if (error instanceof ValidationError) { onError(error); continue } throw error }
|
|
210
|
+
const proof = event.tags.find(tag => tag[0] === 'mmr')[3]
|
|
211
|
+
await packer.update({ recordType: IRFS_CHUNK_RECORD_TYPE, index, total: decoded.total, proof, content: event.content })
|
|
212
|
+
await new Promise(resolve => setTimeout(resolve, 0))
|
|
213
|
+
}
|
|
214
|
+
await packer.finalize()
|
|
215
|
+
return true
|
|
216
|
+
}
|
|
217
|
+
if (messenger.channels.get(controlChannelPubkey)?.mode !== 'watchtower') return true
|
|
218
|
+
const db = seedStorage ? null : await queue()
|
|
219
|
+
let packer
|
|
220
|
+
const candidates = seedStorage ? seedStorage.iterate({ controlChannelPubkey, fileChannelPubkey: payload.fileChannelPubkey, receiverPubkey: question.pubkey, indices: [...indices] }) : db.storedItemsBy('channel', payload.fileChannelPubkey)
|
|
221
|
+
for await (const seed of candidates) {
|
|
222
|
+
if (seed.expiresAt <= _now() || seed.receivedAt + messenger.offlineRecoverySecondsFor(controlChannelPubkey) <= _now() || (seedStorage && !await seedStorage.has(seed))) continue
|
|
223
|
+
if (seed.controlChannelPubkey !== controlChannelPubkey || seed.receiverPubkey !== question.pubkey || !indices.has(seed.chunkIndex) || !seedEnabled(seed)) continue
|
|
224
|
+
const file = await register(seed)
|
|
225
|
+
usable(file)
|
|
226
|
+
if (file.fileChannelPubkey !== payload.fileChannelPubkey) throw new ValidationError('FILE_CHANNEL_MISMATCH')
|
|
227
|
+
if (!packer) {
|
|
228
|
+
const routes = await routing(file, question.pubkey)
|
|
229
|
+
packer = createEventReplyPacker({ messenger: { reply: async options => { usable(file); return _messages.reply({ ...sendOptions(file, routes), ...options }) } }, channelPubkey: file.fileChannelPubkey, question, receiverPubkey: question.pubkey, code: FILE_CHUNKS_REPLY_CODE, eventsPerChunk: 1, recordsFromInput: row => compactRecordsFromSeed(row, { receiverPubkey: question.pubkey }) })
|
|
230
|
+
}
|
|
231
|
+
await packer.update(seed)
|
|
232
|
+
await new Promise(resolve => setTimeout(resolve, 0))
|
|
233
|
+
}
|
|
234
|
+
await packer?.finalize()
|
|
235
|
+
return true
|
|
236
|
+
}
|
|
237
|
+
async function run (job) {
|
|
238
|
+
const file = job.file
|
|
239
|
+
const signal = job.controller.signal
|
|
240
|
+
let sub, release
|
|
241
|
+
const historyController = new AbortController()
|
|
242
|
+
let ingest = Promise.resolve()
|
|
243
|
+
let total = file.size === undefined ? undefined : Math.ceil(file.size / IRFS_CHUNK_BYTES)
|
|
244
|
+
const persisted = new Set()
|
|
245
|
+
let lastProgress = Date.now()
|
|
246
|
+
let requestRoundAt = Date.now()
|
|
247
|
+
const requestedSeeders = new Set()
|
|
248
|
+
const validSeeders = new Set([file.peerPubkey, ...messenger.recoverySeeders(file.controlChannelPubkey)])
|
|
249
|
+
let terminalError
|
|
250
|
+
const fail = error => { terminalError ||= error }
|
|
251
|
+
async function accept (event, index, { save = true } = {}) {
|
|
252
|
+
signal.throwIfAborted()
|
|
253
|
+
usable(file)
|
|
254
|
+
const chunk = checked(file, event, index)
|
|
255
|
+
if (total !== undefined && chunk.total !== total) throw new ValidationError('FILE_CHUNK_TOTAL_MISMATCH')
|
|
256
|
+
total = chunk.total
|
|
257
|
+
if (persisted.has(chunk.index)) return
|
|
258
|
+
if (save) await storage.save(event, file)
|
|
259
|
+
signal.throwIfAborted()
|
|
260
|
+
persisted.add(chunk.index)
|
|
261
|
+
job.completed += chunk.contentBytes.length
|
|
262
|
+
lastProgress = Date.now()
|
|
263
|
+
if (chunk.index === total - 1 && file.size === undefined) file.size = (total - 1) * IRFS_CHUNK_BYTES + chunk.contentBytes.length
|
|
264
|
+
emit(job)
|
|
265
|
+
}
|
|
266
|
+
const consume = (event, outer, meta) => {
|
|
267
|
+
const task = ingest.then(async () => {
|
|
268
|
+
if (signal.aborted) return
|
|
269
|
+
if (event.kind === messages.REPLY_KIND) {
|
|
270
|
+
const reply = messages.parseRumorContent(event)
|
|
271
|
+
if (reply.code !== FILE_CHUNKS_REPLY_CODE || meta.provenance === 'hearsay' || meta.senderPubkey !== event.pubkey || !validSeeders.has(event.pubkey) || event.tags?.find(tag => tag[0] === 'r')?.[1] !== messenger.userPubkey) return
|
|
272
|
+
const jsonl = reply.payload?.jsonl
|
|
273
|
+
if (typeof jsonl !== 'string') throw new ValidationError('INVALID_FILE_CHUNK_REPLY')
|
|
274
|
+
const rows = jsonl.split('\n').filter(Boolean)
|
|
275
|
+
if (rows.length > FILE_REQUEST_CHUNKS) throw new ValidationError('INVALID_FILE_CHUNK_REPLY')
|
|
276
|
+
for (const line of rows) {
|
|
277
|
+
let record
|
|
278
|
+
try { record = JSON.parse(line) } catch { throw new ValidationError('INVALID_FILE_CHUNK_REPLY') }
|
|
279
|
+
if (record.recordType === IRFS_CHUNK_RECORD_TYPE) {
|
|
280
|
+
if (!Number.isSafeInteger(record.index) || record.index < 0 || !Number.isSafeInteger(record.total) || record.total <= record.index || typeof record.content !== 'string' || record.content.length > 100000 || typeof record.proof !== 'string' || record.proof.length > 8192) throw new ValidationError('INVALID_FILE_CHUNK_REPLY')
|
|
281
|
+
const chunk = { kind: 34601, created_at: event.created_at, tags: [['d', NMMR.deriveChunkId(file.root, record.index)], ['mmr', String(record.index), String(record.total), record.proof]], content: record.content }
|
|
282
|
+
await accept(chunk, record.index)
|
|
283
|
+
successes.set(event.pubkey, Date.now())
|
|
284
|
+
continue
|
|
285
|
+
}
|
|
286
|
+
if (record.recordType !== ROUTER_SEED_RECORD_TYPE) continue
|
|
287
|
+
const index = readFileChunkIndex(record.router)
|
|
288
|
+
if (index === undefined) throw new ValidationError('INVALID_FILE_CHUNK_INDEX')
|
|
289
|
+
const original = await _transport.unwrapRouterEvent({ router: record.router, receiverSigner: messenger.userSigner, receiverPubkey: messenger.userPubkey, channelPubkey: file.fileChannelPubkey })
|
|
290
|
+
if (original) { await accept(original, index); successes.set(event.pubkey, Date.now()) }
|
|
291
|
+
}
|
|
292
|
+
} else {
|
|
293
|
+
if (meta.senderPubkey !== file.peerPubkey) return
|
|
294
|
+
const index = readFileChunkIndex(meta.router)
|
|
295
|
+
if (index === undefined) throw new ValidationError('INVALID_FILE_CHUNK_INDEX')
|
|
296
|
+
await accept(event, index)
|
|
297
|
+
// Only direct, validated delivery proves this identity was a recipient.
|
|
298
|
+
// Local cache hits and recovery replies never create/renew a grant.
|
|
299
|
+
if (file.sharedAt !== undefined) await authorizeSeeding(file, { receiverPubkeys: [messenger.userPubkey], sharedAt: file.sharedAt })
|
|
300
|
+
}
|
|
301
|
+
})
|
|
302
|
+
// Malformed remote records do not poison the ingestion chain. Operational
|
|
303
|
+
// persistence failures stop the job so completion cannot outrun storage.
|
|
304
|
+
ingest = task.catch(error => { if (error instanceof ValidationError) onError(error); else if (!signal.aborted) fail(error) })
|
|
305
|
+
return ingest
|
|
306
|
+
}
|
|
307
|
+
try {
|
|
308
|
+
release = await storage.reserve?.(file.root, file.size, { signal })
|
|
309
|
+
job.status = 'downloading'; emit(job)
|
|
310
|
+
// Resume from independently verified local chunks, never from an ACK.
|
|
311
|
+
for (let index = 0; index < (total ?? 1); index++) {
|
|
312
|
+
signal.throwIfAborted()
|
|
313
|
+
const event = await storage.read(file.root, index, file)
|
|
314
|
+
if (event) await accept(event, index, { save: false })
|
|
315
|
+
}
|
|
316
|
+
if (total !== undefined && persisted.size === total) return
|
|
317
|
+
const parent = messenger.requireWritableChannel(file.controlChannelPubkey)
|
|
318
|
+
const relays = await messenger.resolveWatchRelays(parent)
|
|
319
|
+
const options = { receiverSigner: messenger.userSigner, privateChannelSigner: file.signer, privateChannelPubkey: file.fileChannelPubkey, receiverPubkey: messenger.userPubkey, relays, receivedChunkIndexedDB: messenger._indexedDB, receivedChunkScope: `${messenger.userPubkey}:file:${file.fileChannelPubkey}`, onEvent: consume, onSeedEvent: seed => saveSeed(file, seed), mode: messenger.channels.get(file.controlChannelPubkey)?.mode === 'watchtower' ? 'watchtower' : 'leecher', onError }
|
|
320
|
+
sub = _transport.subscribe({ ...options, liveOnly: true })
|
|
321
|
+
const stop = () => { sub?.close() }
|
|
322
|
+
signal.addEventListener('abort', stop, { once: true })
|
|
323
|
+
try {
|
|
324
|
+
await sub.ready
|
|
325
|
+
signal.throwIfAborted()
|
|
326
|
+
// Partial/failed history still permits recovery from peers.
|
|
327
|
+
const history = _transport.fetch({ ...options, signal: AbortSignal.any([signal, historyController.signal]), since: 0 }).catch(error => { if (!signal.aborted) onError(error) })
|
|
328
|
+
work.add(history); history.finally(() => work.delete(history))
|
|
329
|
+
const seeders = [...validSeeders].sort((a, b) => (successes.get(b) || 0) - (successes.get(a) || 0) || Math.random() - 0.5)
|
|
330
|
+
while (true) {
|
|
331
|
+
signal.throwIfAborted(); usable(file)
|
|
332
|
+
await ingest
|
|
333
|
+
if (terminalError) throw terminalError
|
|
334
|
+
if (total !== undefined && persisted.size === total) break
|
|
335
|
+
const missing = []
|
|
336
|
+
for (let index = 0; index < (total ?? 1) && missing.length < FILE_REQUEST_CHUNKS; index++) if (!persisted.has(index)) missing.push(index)
|
|
337
|
+
const remaining = seeders.filter(peer => !requestedSeeders.has(peer))
|
|
338
|
+
if (remaining.length && Date.now() >= requestRoundAt) {
|
|
339
|
+
const peer = remaining[0]
|
|
340
|
+
requestedSeeders.add(peer)
|
|
341
|
+
requestRoundAt = Date.now() + _hedgeMs
|
|
342
|
+
// The request is on dm; its compact reply is on the file channel.
|
|
343
|
+
await messenger.ask({ channelPubkey: file.controlChannelPubkey, receiverPubkey: peer, code: FILE_CHUNKS_REQUEST_CODE, payload: { fileChannelPubkey: file.fileChannelPubkey, missingRanges: rangesFor(missing) } }).catch(onError)
|
|
344
|
+
}
|
|
345
|
+
if (!remaining.length && Date.now() - Math.max(lastProgress, requestRoundAt - _hedgeMs) >= _idleMs) throw new Error('FILE_DOWNLOAD_STALLED')
|
|
346
|
+
// Start a fresh batch when all positions of the previous batch arrived.
|
|
347
|
+
if (job.batch?.every(index => persisted.has(index))) { requestedSeeders.clear(); requestRoundAt = Date.now(); job.batch = null }
|
|
348
|
+
job.batch ||= missing
|
|
349
|
+
await new Promise(resolve => { const timer = setTimeout(done, Math.min(100, _hedgeMs)); function done () { clearTimeout(timer); signal.removeEventListener('abort', done); resolve() } signal.addEventListener('abort', done, { once: true }) })
|
|
350
|
+
}
|
|
351
|
+
} finally { signal.removeEventListener('abort', stop) }
|
|
352
|
+
} finally { historyController.abort(); await sub?.close(); await ingest; await release?.() }
|
|
353
|
+
if (job.completed !== file.size) throw new ValidationError('FILE_SIZE_MISMATCH')
|
|
354
|
+
}
|
|
355
|
+
function pump () {
|
|
356
|
+
if (closed) return
|
|
357
|
+
for (const job of [...jobs.values()].filter(job => job.status === 'queued').sort((a, b) => b.priority - a.priority)) {
|
|
358
|
+
if (active >= 2) break
|
|
359
|
+
active++; job.status = 'starting'
|
|
360
|
+
const task = run(job).then(() => { job.status = 'complete'; emit(job); job.resolve(job.file) }, error => { job.status = job.pauseRequested ? 'paused' : job.controller.signal.aborted ? 'cancelled' : 'error'; job.error = error; emit(job); job.reject(error) }).finally(() => { active--; work.delete(task); pump() })
|
|
361
|
+
work.add(task)
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
async function download (descriptor, { manual = false, thumbnail = false, signal } = {}) {
|
|
365
|
+
const file = await register(descriptor)
|
|
366
|
+
usable(file)
|
|
367
|
+
const existing = jobs.get(keyFor(file.controlChannelPubkey, file.root))
|
|
368
|
+
if (existing && !['error', 'cancelled', 'paused'].includes(existing.status)) return waitForTransfer(existing.promise, signal)
|
|
369
|
+
if (!manual && (file.size === undefined || file.size > (thumbnail ? IRFS_CHUNK_BYTES : AUTO_DOWNLOAD_BYTES))) {
|
|
370
|
+
// Locally complete files need no network consent, including the sender's
|
|
371
|
+
// own originals and files already downloaded before a restart.
|
|
372
|
+
let total, completed = 0
|
|
373
|
+
for (let index = 0; index < (total ?? 1); index++) {
|
|
374
|
+
signal?.throwIfAborted(); usable(file)
|
|
375
|
+
const event = await storage.read(file.root, index, file)
|
|
376
|
+
if (!event) throw new Error('FILE_DOWNLOAD_REQUIRES_ACTION')
|
|
377
|
+
const chunk = checked(file, event, index)
|
|
378
|
+
if (total !== undefined && total !== chunk.total) throw new ValidationError('FILE_CHUNK_TOTAL_MISMATCH')
|
|
379
|
+
total = chunk.total; completed += chunk.contentBytes.length
|
|
380
|
+
}
|
|
381
|
+
file.size ??= completed
|
|
382
|
+
if (completed !== file.size) throw new ValidationError('FILE_SIZE_MISMATCH')
|
|
383
|
+
const job = { file, completed, status: 'complete', promise: Promise.resolve(file) }
|
|
384
|
+
jobs.set(keyFor(file.controlChannelPubkey, file.root), job); emit(job)
|
|
385
|
+
return file
|
|
386
|
+
}
|
|
387
|
+
const key = keyFor(file.controlChannelPubkey, file.root)
|
|
388
|
+
let job = jobs.get(key)
|
|
389
|
+
if (!job || ['error', 'cancelled', 'paused', ...(ownsStorage ? ['complete'] : [])].includes(job.status)) {
|
|
390
|
+
const controller = new AbortController()
|
|
391
|
+
const { promise, resolve, reject } = Promise.withResolvers()
|
|
392
|
+
job = { file, controller, resolve, reject, promise, completed: 0, manual, thumbnail, priority: thumbnail ? 2 : manual ? 1 : 0, status: 'queued' }
|
|
393
|
+
jobs.set(key, job); emit(job); pump()
|
|
394
|
+
}
|
|
395
|
+
// A view leaving does not cancel another consumer's shared download.
|
|
396
|
+
return waitForTransfer(job.promise, signal)
|
|
397
|
+
}
|
|
398
|
+
const cancelJob = job => {
|
|
399
|
+
if (['complete', 'error', 'cancelled'].includes(job.status)) return
|
|
400
|
+
const error = new DOMException('Download cancelled', 'AbortError')
|
|
401
|
+
job.controller.abort(error)
|
|
402
|
+
if (job.status === 'queued') { job.status = job.pauseRequested ? 'paused' : 'cancelled'; job.reject(error); emit(job) }
|
|
403
|
+
}
|
|
404
|
+
function drainRequests () {
|
|
405
|
+
if (closed) return
|
|
406
|
+
while (replying < 2 && requests.length) {
|
|
407
|
+
const [channel, message] = requests.shift()
|
|
408
|
+
replying++
|
|
409
|
+
const task = replyToRequest(channel, message).catch(error => { if (!(closed && error.message === 'FILE_TRANSFER_CLOSED')) onError(error) }).finally(() => { replying--; work.delete(task); drainRequests() })
|
|
410
|
+
work.add(task)
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
const extension = {
|
|
414
|
+
handleAsk (channel, message) {
|
|
415
|
+
if (message.payload?.code !== FILE_CHUNKS_REQUEST_CODE) return false
|
|
416
|
+
// Never hold the DM dispatcher while publishing a batch of file bytes.
|
|
417
|
+
// Bound both workers and queued requests; discarded excess is retryable.
|
|
418
|
+
if (!closed && requests.length < 16) { requests.push([channel, message]); drainRequests() }
|
|
419
|
+
return true
|
|
420
|
+
},
|
|
421
|
+
pause () { requests.length = 0; for (const job of jobs.values()) { if (!['complete', 'error', 'cancelled'].includes(job.status)) job.pauseRequested = true; cancelJob(job) } },
|
|
422
|
+
async resume () {
|
|
423
|
+
await Promise.allSettled([...work])
|
|
424
|
+
for (const job of jobs.values()) if (job.pauseRequested && job.status !== 'complete') download(job.file, { manual: job.manual, thumbnail: job.thumbnail }).catch(onError)
|
|
425
|
+
},
|
|
426
|
+
unwatch (channels) { for (const job of jobs.values()) if (channels.includes(job.file.controlChannelPubkey)) cancelJob(job) },
|
|
427
|
+
async close () { if (closed) return; closed = true; extension.pause(); clearInterval(timer); await Promise.allSettled([...work]); await catalog.close(); for (const release of uploadLeases.values()) await release(); uploadLeases.clear(); if (ownsStorage) await storage.close(); if (queuePromise) await (await queuePromise).close(); messenger.extensions.delete(extension); listeners.clear() }
|
|
428
|
+
}
|
|
429
|
+
const timer = setInterval(() => { prune().catch(onError) }, 60000)
|
|
430
|
+
timer.unref?.()
|
|
431
|
+
messenger.extensions.add(extension)
|
|
432
|
+
return { register, authorizeSeeding, publishChunk, download, readChunk: (root, index) => storage.read(root, index), stream: (descriptor, options = {}) => storage.stream ? storage.stream(descriptor, options) : chunkStream({ read: storage.read.bind(storage), ...descriptor, ...options }), close: extension.close, cancel (control, root) { const job = jobs.get(keyFor(control, root)); if (job) { job.pauseRequested = false; cancelJob(job) } }, observe (listener) { listeners.add(listener); for (const job of jobs.values()) emit(job); return () => listeners.delete(listener) } }
|
|
433
|
+
}
|