libp2r2p 0.11.21 → 0.11.23

package/private-messenger/services/decryption-scheduler.js

@@ -0,0 +1,135 @@
+// Admission only: never wait while holding an inbox operation, IDB transaction
+// or the cross-instance retry lock. A busy channel cannot stall another one.
+export function createDecryptionScheduler ({ ready = true, now = Date.now, setTimer = setTimeout, clearTimer = clearTimeout } = {}) {
+  const channels = new Map()
+  const readyWaiters = new Set()
+  let timer
+  let running = false
+  let closed
+  let foreground = false
+  let foregroundAt = now()
+  let priority = null
+  let previous
+  let batch = 0
+  const stateFor = pubkey => {
+    if (!channels.has(pubkey)) channels.set(pubkey, { active: 0, at: now(), deadline: null, jobs: [] })
+    return channels.get(pubkey)
+  }
+  const quietAt = state => state.active || foreground ? Infinity : Math.max(state.at, foregroundAt) + 250
+  const schedule = () => {
+    if (timer !== undefined) clearTimer(timer)
+    timer = undefined
+    if (closed || !ready || running) return
+    const pending = [...channels.values()].filter(state => state.jobs.length)
+    if (!pending.length) return
+    const at = Math.min(...pending.map(state => Math.min(quietAt(state), state.deadline)))
+    timer = setTimer(pump, Math.max(0, at - now()))
+  }
+  async function pump () {
+    timer = undefined
+    if (closed || !ready || running) return
+    const time = now()
+    const eligible = [...channels].filter(([, state]) => state.jobs.length && Math.min(quietAt(state), state.deadline) <= time)
+    eligible.sort(([a, x], [b, y]) => {
+      const overdue = Number(y.deadline <= time) - Number(x.deadline <= time)
+      if (overdue) return overdue
+      if (x.deadline <= time && x.deadline !== y.deadline) return x.deadline - y.deadline
+      if (a === previous && batch >= 4) return 1
+      if (b === previous && batch >= 4) return -1
+      return Number(b === priority) - Number(a === priority)
+    })
+    if (!eligible.length) { schedule(); return }
+    const [pubkey, state] = eligible[0]
+    const job = state.jobs.shift()
+    running = true
+    if (previous === pubkey) batch++
+    else { previous = pubkey; batch = 1 }
+    try {
+      job.signal?.throwIfAborted()
+      job.resolve(await job.work())
+    } catch (error) { job.reject(error) } finally {
+      job.signal?.removeEventListener('abort', job.abort)
+      state.deadline = now() + 5000
+      running = false
+      // Always use a new task, including between records in the same batch.
+      schedule()
+    }
+  }
+  const rejectJobs = (state, error) => {
+    for (const job of state.jobs.splice(0)) {
+      job.signal?.removeEventListener('abort', job.abort)
+      job.reject(error)
+    }
+  }
+  return {
+    activity (pubkey) {
+      if (closed) return () => {}
+      const state = stateFor(pubkey)
+      state.active++; state.at = now(); schedule()
+      let released = false
+      return () => {
+        if (released) return
+        released = true
+        state.active--; state.at = now(); schedule()
+      }
+    },
+    foreground (active) { foreground = active; foregroundAt = now(); schedule() },
+    priority (pubkey) { priority = pubkey; schedule() },
+    setReady (value) {
+      if (ready === value || closed) return
+      ready = value
+      if (ready) {
+        for (const state of channels.values()) { state.at = now(); state.deadline = state.jobs.length ? now() + 5000 : null }
+        for (const wake of [...readyWaiters]) wake()
+      }
+      schedule()
+    },
+    async waitReady (signal) {
+      while (true) {
+        signal?.throwIfAborted()
+        if (closed) throw closed
+        if (ready) return
+        await new Promise((resolve, reject) => {
+          const finish = () => {
+            readyWaiters.delete(finish); signal?.removeEventListener('abort', finish)
+            if (closed || signal?.aborted) reject(closed || signal.reason)
+            else resolve()
+          }
+          readyWaiters.add(finish); signal?.addEventListener('abort', finish, { once: true })
+        })
+      }
+    },
+    run (pubkey, work, signal) {
+      signal?.throwIfAborted()
+      if (closed) return Promise.reject(closed)
+      const state = stateFor(pubkey)
+      state.deadline ??= now() + 5000
+      return new Promise((resolve, reject) => {
+        const job = { work, resolve, reject, signal }
+        job.abort = () => {
+          const index = state.jobs.indexOf(job)
+          if (index < 0) return
+          state.jobs.splice(index, 1); reject(signal.reason); schedule()
+        }
+        signal?.addEventListener('abort', job.abort, { once: true })
+        state.jobs.push(job); schedule()
+      })
+    },
+    remove (pubkey) {
+      const state = channels.get(pubkey)
+      if (state) rejectJobs(state, new Error('PRIVATE_CHANNEL_CHANGED'))
+      channels.delete(pubkey); schedule()
+    },
+    finish (pubkey) {
+      const state = channels.get(pubkey)
+      if (state && !state.jobs.length) state.deadline = null
+    },
+    close () {
+      closed ||= new Error('PRIVATE_MESSENGER_CLOSED')
+      if (timer !== undefined) clearTimer(timer)
+      for (const state of channels.values()) rejectJobs(state, closed)
+      channels.clear()
+      for (const wake of [...readyWaiters]) wake()
+    }
+  }
+}

package/private-messenger/services/migrate-message-queue.js

@@ -0,0 +1,95 @@
+import { deliveryInfo } from '../../private-channel/helpers/rumor.js'
+
+const encoder = new TextEncoder()
+const indexName = 'byChannelTypeEventId'
+const oldPath = ['item.channelPubkey', 'item.type', 'item.event.id']
+const newPath = [...oldPath, 'item.provenance']
+const equal = (a, b) => JSON.stringify(a) === JSON.stringify(b)
+
+function open (indexedDB, name, version, upgrade) {
+  return new Promise((resolve, reject) => {
+    const request = indexedDB.open(name, version)
+    let rejected = false
+    request.onblocked = () => { rejected = true; reject(new Error('IDB_DATABASE_BLOCKED')) }
+    request.onerror = () => reject(request.error)
+    request.onupgradeneeded = () => {
+      if (rejected) { request.transaction.abort(); return }
+      try { upgrade?.(request.result, request.transaction) } catch (error) {
+        request.transaction.abort(); reject(error)
+      }
+    }
+    request.onsuccess = () => {
+      const db = request.result
+      db.onversionchange = () => db.close()
+      if (rejected) db.close()
+      else resolve(db)
+    }
+  })
+}
+
+function matchesLegacy (items) {
+  if (!items.indexNames.contains(indexName)) return false
+  const index = items.index(indexName)
+  return equal(items.keyPath, 'position') && equal(index.keyPath, oldPath) && index.unique && !index.multiEntry &&
+    items.indexNames.contains('byChannel') && items.index('byChannel').keyPath === 'item.channelPubkey' &&
+    !items.index('byChannel').unique && !items.index('byChannel').multiEntry
+}
+
+// The generic queue remains strict. Only this historical messenger schema is
+// migrated, atomically with its records and capacity accounting.
+export async function migrateMessageQueue (indexedDB, prefix) {
+  if (!indexedDB?.open) throw new Error('IDB_UNAVAILABLE')
+  const name = `${prefix}:idb-queue`
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const db = await open(indexedDB, name)
+    if (!db.objectStoreNames.contains('items') || !db.objectStoreNames.contains('state')) { db.close(); return }
+    const tx = db.transaction(['items', 'state'], 'readonly')
+    const legacy = matchesLegacy(tx.objectStore('items')) && tx.objectStore('state').keyPath === 'key'
+    const version = db.version + 1
+    await new Promise((resolve, reject) => { tx.oncomplete = resolve; tx.onabort = () => reject(tx.error) })
+    db.close()
+    if (!legacy) return
+    try {
+      const upgraded = await open(indexedDB, name, version, (_db, tx) => {
+        const items = tx.objectStore('items')
+        if (!matchesLegacy(items)) return
+        items.deleteIndex(indexName)
+        items.createIndex(indexName, newPath, { unique: true })
+        let delta = 0
+        const cursor = items.openCursor()
+        cursor.onsuccess = () => {
+          try {
+            const row = cursor.result
+            if (!row) {
+              const state = tx.objectStore('state')
+              const request = state.get('queue')
+              request.onsuccess = () => {
+                if (request.result) state.put({ ...request.result, usedBytes: request.result.usedBytes + delta })
+              }
+              return
+            }
+            const record = row.value
+            const item = { ...record.item }
+            const sender = item.senderPubkey || item.meta?.senderPubkey || item.meta?.router?.tags?.find(tag => tag[0] === 'f')?.[1] || ''
+            const info = deliveryInfo(item.event, sender)
+            item.senderPubkey = sender
+            item.provenance = item.provenance === 'hearsay' ? 'hearsay' : info.provenance
+            let byteSize = 0
+            while (true) {
+              const size = encoder.encode(JSON.stringify({ byteSize, item })).length
+              if (size === byteSize) break
+              byteSize = size
+            }
+            delta += byteSize - record.byteSize
+            row.update({ ...record, item, byteSize })
+            row.continue()
+          } catch { tx.abort() }
+        }
+      })
+      upgraded.close()
+      return
+    } catch (error) {
+      if (error?.name !== 'VersionError' || attempt === 2) throw error
+    }
+  }
+}

package/private-messenger/services/pending-decryption.js

@@ -0,0 +1,126 @@
+import { sha256 } from '@noble/hashes/sha2.js'
+import { bytesToHex } from '../../base16/index.js'
+import { base64ToBytes } from '../../base64/index.js'
+import { createQueue } from '../../idb-queue/index.js'
+import { isDeferredDecryptionError } from '../../private-channel/helpers/decryption-error.js'
+
+const encoder = new TextEncoder()
+const decoder = new TextDecoder()
+export const PENDING_DECRYPTION_MAX_BYTES = 16 * 1024 * 1024
+
+function storedBytes (item) {
+  let byteSize = 0
+  while (true) {
+    const size = encoder.encode(JSON.stringify({ byteSize, item })).length
+    if (size === byteSize) return size
+    byteSize = size
+  }
+}
+
+function packetSnapshot (packet) {
+  const event = (value, router = false) => {
+    const copy = Object.fromEntries(['id', 'kind', 'pubkey', 'created_at', 'content', 'sig'].filter(key => value?.[key] !== undefined).map(key => [key, value[key]]))
+    if (value?.tags) {
+      copy.tags = router
+        ? value.tags.filter(tag => ['f', 'p', 'imkc', 'c', 'i'].includes(tag[0])).map(tag => tag.slice(0, tag[0] === 'imkc' || tag[0] === 'c' ? 3 : 2))
+        : structuredClone(value.tags)
+    }
+    return copy
+  }
+  return { format: packet.format, channelPubkey: packet.channelPubkey, outer: event(packet.outer), ...(packet.router ? { router: event(packet.router, true) } : {}) }
+}
+
+// Stable across live, history and synthesized recovery wrappers.
+function packetId (record) {
+  const rows = record.format === 'routerRow_v1'
+    ? decoder.decode(base64ToBytes(record.router.content)).split('\n').filter(Boolean).map(line => {
+      const row = JSON.parse(line)
+      return row.length === 1 ? row : [row[0], row[1], row[2] || '', row[3] || '']
+    })
+    : null
+  const value = rows ? [record.channelPubkey, record.router.pubkey, record.router.tags.filter(tag => tag[0] !== 'c'), rows] : [record.channelPubkey, record.outer.id]
+  return bytesToHex(sha256(encoder.encode(JSON.stringify(value))))
+}
+
+export async function createPendingDecryption ({ prefix, indexedDB, maxBytes = PENDING_DECRYPTION_MAX_BYTES, report = () => {} }) {
+  const queue = await createQueue({
+    prefix: `${prefix}:pending-decryption`, indexedDB, maxBytes, evictionPolicy: 'fifo',
+    indexes: { byId: { keyPath: 'id', unique: true }, byChannel: 'channelPubkey' }
+  })
+  let tail = Promise.resolve()
+  let timer
+  let counts = {}
+  const flush = () => {
+    clearTimeout(timer); timer = null
+    if (Object.keys(counts).length) {
+      const summary = counts; counts = {}
+      report({ action: 'summary', counts: summary })
+      console.debug('private-messenger pending decryption', JSON.stringify(summary))
+    }
+  }
+  const serial = work => { const result = tail.then(work); tail = result.catch(() => {}); return result }
+  const notify = (action, detail) => {
+    report({ action, ...detail })
+    const key = `${action}:${detail.code || detail.reason || ''}`
+    counts[key] = { count: (counts[key]?.count || 0) + 1, example: detail }
+    if (!timer) { timer = setTimeout(flush, 1000); timer.unref?.() }
+  }
+  return {
+    queue,
+    has: record => serial(() => queue.someBy('byId', packetId(packetSnapshot(record)))),
+    save ({ error, record }) {
+      record = packetSnapshot(record)
+      return serial(async () => {
+        const id = packetId(record)
+        if (await queue.someBy('byId', id)) return true
+        const item = { id, channelPubkey: record.channelPubkey, packet: structuredClone(record), code: error.code, macAttempts: error.code === 'INVALID_MAC' ? 1 : 0 }
+        const requiredBytes = storedBytes(item)
+        if (requiredBytes > maxBytes) {
+          notify('discarded', { reason: 'oversized', channelPubkey: item.channelPubkey, id }); return true
+        }
+        const before = await queue.getCapacity()
+        try { await queue.enqueue(item) } catch (error) {
+          if (error.name === 'ConstraintError') return true
+          throw error
+        }
+        const after = await queue.getCapacity()
+        const evictedBytes = Math.max(0, before.usedBytes + requiredBytes - after.usedBytes)
+        if (evictedBytes) notify('discarded', { reason: 'capacity', bytes: evictedBytes })
+        const coordinates = Object.fromEntries(['eventId', 'eventKind', 'senderPubkey', 'receiverPubkey', 'ownContentPubkey', 'peerContentPubkey', 'hasOwnContentKey']
+          .filter(key => ['string', 'boolean', 'number'].includes(typeof error[key])).map(key => [key, error[key]]))
+        notify('deferred', { ...coordinates, code: error.code, stage: error.stage, id, channelPubkey: item.channelPubkey, capacityBytesBefore: before.usedBytes, capacityBytesAfter: after.usedBytes })
+        return true
+      })
+    },
+    async records ({ force = false } = {}) {
+      await tail
+      const records = []
+      for await (const item of queue.storedItems()) if (force || item.macAttempts < 3) records.push(item)
+      return records
+    },
+    current (item, { force = false } = {}) {
+      return serial(async () => {
+        const stored = await queue.getBy('byId', item.id)
+        return stored && (force || stored.macAttempts < 3) ? stored : null
+      })
+    },
+    recovered (item) {
+      return serial(async () => {
+        await queue.removeBy('byId', item.id)
+        notify('recovered', { id: item.id, channelPubkey: item.channelPubkey })
+      })
+    },
+    failed (item, error) {
+      return serial(async () => {
+        if (!isDeferredDecryptionError(error)) throw error
+        const current = await queue.getBy('byId', item.id)
+        if (!current) return
+        const updated = { ...current, code: error.code, macAttempts: current.macAttempts + (error.code === 'INVALID_MAC' ? 1 : 0) }
+        await queue.putBy('byId', updated, { existingOnly: true })
+        if (updated.macAttempts >= 3 && item.macAttempts < 3) notify('suspended', { id: item.id, code: error.code, channelPubkey: item.channelPubkey })
+      })
+    },
+    clearChannel: channel => serial(() => queue.removeBy('byChannel', channel)),
+    async close () { await tail; flush(); await queue.close() }
+  }
+}

package/private-messenger/services/storage-maintenance.js

@@ -185,6 +185,7 @@ function storageDatabaseNames (userPubkey) {
   const prefix = `libp2r2p:private-messenger:${userPubkey}`
   return [
     `${prefix}:idb-queue`,
+    `${prefix}:pending-decryption:idb-queue`,
     `${prefix}:seeds:idb-queue`,
     `${prefix}:file-seeds:idb-queue`,
     `${prefix}:file-authorizations:idb-queue`,

package/private-messenger/session/README.md

@@ -4,6 +4,25 @@
 outbox and IRFS attachments without presentation state. It builds on
 PrivateMessenger and the file coordinator, not another recovery protocol.
 
+`retryPendingDecryptions({ force = false } = {})` forwards a background pass to
+the account messenger while available. Call after committed content-key changes;
+ordinary availability recovery already requests a pass. Ciphertext stays in the
+separate bounded FIFO until admitted to the normal inbox, evicted or explicitly
+cleared. This does not retry application storage/permission failures as crypto.
+
+Construction accepts `pendingDecryptionReady` (default `true`). Pass `false` to
+hold pending queue reads/replay until `setPendingDecryptionReady(true)` releases
+them after initial presentation. This never pauses live reception, sends or
+channel preparation. Recovery waits for 250 ms of channel inactivity; a five-second
+maximum admits one packet during sustained activity. It yields between records,
+with at most four consecutive preferred records before another eligible channel.
+`setPendingDecryptionPriority(peer | null)` retains a peer preference through
+preparation; self-chat/`null` or removal clear it. Deadlines prevent preference
+from starving other channels. Account unavailability/close still cancel recovery;
+`force` bypasses only the MAC attempt limit. Awaiting the retry API awaits its
+scheduled pass, so do not await it on the interface's critical startup path.
+
+
 Provide `owner`, a byte-based NIP-44 v3 `signer` with `withSharedKey`, and either
 `eventStore` or a `messageStorage.save(event, { peerPubkey, hearsay })` adapter.
 Optional `chunkStorage` and `recoveryStorage` (`seeds`, `authorizations`) select

package/private-messenger/session/helpers/foreground-work.js

@@ -3,15 +3,19 @@
 export function createForegroundWork () {
   let active = 0
   const waiters = new Set()
+  const observers = new Set()
+  const notify = () => { for (const listener of observers) listener(active > 0) }
   const lifetime = new AbortController()
   return {
     enter () {
       active++
+      notify()
       let released = false
       return () => {
         if (released) return
         released = true
         if (--active === 0) for (const wake of [...waiters]) wake()
+        notify()
       }
     },
     wait (signal) {
@@ -24,6 +28,7 @@ export function createForegroundWork () {
         combined.addEventListener('abort', finish, { once: true })
       })
     },
-    close () { lifetime.abort(); waiters.clear() }
+    observe (listener) { observers.add(listener); listener(active > 0); return () => observers.delete(listener) },
+    close () { lifetime.abort(); waiters.clear(); observers.clear() }
   }
 }

package/private-messenger/session/index.js

@@ -44,9 +44,11 @@ function messengerStatusSnapshot (value) {
   return { closed: value.closed, paused: value.paused, pauseReasons: [...value.pauseReasons] }
 }
 
-export function createPrivateMessageSession ({ owner, signer, eventStore, messageStorage, chunkStorage, recoveryStorage, fallbackRelays = [], fallbackDelayMs = null, mode = 'seeder', seedersForPeer = peer => [peer], allowedKinds = [5, 9, 1063, 34601], useContentKeys = true, onMedia = () => {}, onOutbox = () => {}, onError = () => {}, onSendError = () => {}, Messenger = createPrivateMessenger, openOutbox = createChatOutbox, FileTransfer = createPrivateFileTransfer, _onOnline = onOnline, _isOnline = isOnline, _getIykcProofs = getIykcProofs, _setTimeout = setTimeout, _clearTimeout = clearTimeout, _random = Math.random, openDownloads = options => createChatOutbox({ ...options, namespace: 'downloads' }) }) {
+export function createPrivateMessageSession ({ owner, signer, eventStore, messageStorage, chunkStorage, recoveryStorage, fallbackRelays = [], fallbackDelayMs = null, mode = 'seeder', seedersForPeer = peer => [peer], allowedKinds = [5, 9, 1063, 34601], useContentKeys = true, pendingDecryptionReady = true, onMedia = () => {}, onOutbox = () => {}, onError = () => {}, onSendError = () => {}, Messenger = createPrivateMessenger, openOutbox = createChatOutbox, FileTransfer = createPrivateFileTransfer, _onOnline = onOnline, _isOnline = isOnline, _getIykcProofs = getIykcProofs, _setTimeout = setTimeout, _clearTimeout = clearTimeout, _random = Math.random, openDownloads = options => createChatOutbox({ ...options, namespace: 'downloads' }) }) {
   fallbackRelays = normalizeFallbackRelays(fallbackRelays)
   fallbackDelayMs = normalizeFallbackDelay(fallbackDelayMs)
+  if (typeof pendingDecryptionReady !== 'boolean') throw new ValidationError('INVALID_PENDING_DECRYPTION_READY')
+  let pendingDecryptionPeer = null
   const reportedErrors = new WeakSet()
   const reportError = error => {
     if (error && (typeof error === 'object' || typeof error === 'function')) {
@@ -139,7 +141,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     let candidate
     try {
       candidate = await Messenger({
-        fallbackRelays, fallbackDelayMs, seedStorage: recoveryStorage?.seeds, userSigner, channels: [], useContentKeys, _waitForForeground: foreground.wait,
+        fallbackRelays, fallbackDelayMs, pendingDecryptionReady, seedStorage: recoveryStorage?.seeds, userSigner, channels: [], useContentKeys, _waitForForeground: foreground.wait, _observeForeground: foreground.observe,
         onMessageQueued: () => { if (messengerBinding === binding) return drain() },
         onStateChanged: state => {
           if (closed || messengerBinding !== binding) return
@@ -152,6 +154,8 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
       if (typeof candidate?.readStatus !== 'function' || !notified) throw new ValidationError('MESSENGER_STATE_CONTRACT_REQUIRED')
       const status = messengerStatusSnapshot(candidate.readStatus())
       messenger = candidate
+      messenger.setPendingDecryptionReady?.(pendingDecryptionReady)
+      messenger.setPendingDecryptionPriority?.(channels.get(pendingDecryptionPeer)?.pubkey ?? null)
       stateChanged(status)
     } catch (error) {
       messengerBinding = null
@@ -168,6 +172,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
       signal.throwIfAborted()
       const channel = { signer: scoped, pubkey, readerPubkey: pubkey, mode, seeders: seedersForPeer(peer) }
       channels.set(peer, channel)
+      if (peer === pendingDecryptionPeer) messenger?.setPendingDecryptionPriority?.(channel.pubkey)
       await schedule()
       signal.throwIfAborted()
       return channel
@@ -558,6 +563,21 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     await writeDownloadIntent(() => downloads?.remove(id))
   }
   return {
+    setPendingDecryptionReady (ready) {
+      if (typeof ready !== 'boolean') throw new ValidationError('INVALID_PENDING_DECRYPTION_READY')
+      pendingDecryptionReady = ready
+      if (!closed) messenger?.setPendingDecryptionReady?.(ready)
+    },
+    setPendingDecryptionPriority (peer) {
+      if (peer !== null && !/^[0-9a-f]{64}$/.test(peer || '')) throw new ValidationError('INVALID_PENDING_DECRYPTION_PRIORITY')
+      pendingDecryptionPeer = peer === owner ? null : peer
+      if (!closed) messenger?.setPendingDecryptionPriority?.(channels.get(pendingDecryptionPeer)?.pubkey ?? null)
+    },
+    async retryPendingDecryptions (options) {
+      if (closed || !available) return
+      await ensureMessenger()
+      if (!closed && available) return messenger?.retryPendingDecryptions(options)
+    },
     download, cancelDownload, preparePeer,
     // Peers-based so callers can prefetch as soon as a chat route opens, before
     // the shared-key channel exists. When group chats get real membership, add
@@ -600,6 +620,10 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
       if (next.size === peers.size && [...next].every(peer => peers.has(peer))) return configuring
       peers.clear(); for (const peer of next) peers.add(peer)
       preparation.reconcile(peers)
+      if (pendingDecryptionPeer && !peers.has(pendingDecryptionPeer)) {
+        pendingDecryptionPeer = null
+        messenger?.setPendingDecryptionPriority?.(null)
+      }
       for (const peer of channels.keys()) {
         if (!peers.has(peer)) {
           channels.delete(peer); pendingPriorityRanges.delete(peer)