libp2r2p 0.11.24 → 0.11.26

package/private-messenger/index.js

@@ -233,6 +233,7 @@ export class PrivateMessenger {
     messageQueueMaxBytes = DEFAULT_MESSAGE_QUEUE_MAX_BYTES,
     seedQueueMaxBytes = DEFAULT_SEED_QUEUE_MAX_BYTES,
     seedStorage,
+    _pendingSeedRecords = () => [],
     temporaryStorageArea = globalThis.sessionStorage,
     autoDeletionCapability = true,
     _indexedDB = globalThis.indexedDB,
@@ -290,6 +291,7 @@ export class PrivateMessenger {
     this.messageQueueMaxBytes = messageQueueMaxBytes
     this.seedQueueMaxBytes = seedQueueMaxBytes
     this.seedStorage = seedStorage
+    this._pendingSeedRecords = _pendingSeedRecords
     this.temporaryStorageArea = temporaryStorageArea
     this.autoDeletionCapability = normalizeAutoDeletionCapability(autoDeletionCapability)
     this._indexedDB = _indexedDB
@@ -887,7 +889,8 @@ export class PrivateMessenger {
     return uniq([...primary, ...this.fallbackRelays].map(normalizeRelayUrl))
   }
 
-  async resolveSendRouting ({ channel, receiverPubkeys, relays, relayToReceivers, signal }) {
+  async resolveSendRouting ({ channel, receiverPubkeys, relays, relayToReceivers, signal, attemptStartedAt, onDebug }) {
+    if (attemptStartedAt !== undefined && (!Number.isFinite(attemptStartedAt) || attemptStartedAt < 0)) throw new ValidationError('INVALID_ATTEMPT_STARTED_AT')
     const recoveryRelays = await this.recoveryMirrorRelays(channel.pubkey)
     if (relayToReceivers && !this.fallbackRelays.length) return { relayToReceivers, recoveryRelays }
     const fixed = relayToReceivers ? null : relays?.length ? uniq(relays) : channel.sendRelays.length ? channel.sendRelays : channel.relays.length ? channel.relays : null
@@ -903,12 +906,17 @@ export class PrivateMessenger {
       peers: fixed && !recipients.length ? [this.userPubkey] : recipients, relaysByPubkey, recoveryRelays, primaryRelays: fixed?.map(normalizeRelayUrl), primaryRelayToReceivers: relayToReceivers, fallbackRelays: this.fallbackRelays,
       signal: signal ? AbortSignal.any([this.sendRoutingLifetime.signal, signal]) : this.sendRoutingLifetime.signal,
       exclusions: this.sendRelayExclusions.get(key).exclusions,
-      fallbackDelayMs: this.fallbackDelayMs, pauseSignal: this.sendRoutingPause.signal,
+      fallbackDelayMs: this.fallbackDelayMs, attemptStartedAt, pauseSignal: this.sendRoutingPause.signal,
       setTimer: this._setTimeout, clearTimer: this._clearTimeout,
       pickRelays: this._pickRelaysForPubkeys,
       publish: this._privateChannel.publish || privateChannel.publish,
       publishNymEvent: this._privateChannel.publishNymEvent || privateChannel.publishNymEvent,
-      sendEvent: (...args) => relayPool.sendEvent(...args),
+      sendEvent: async (...args) => {
+        const startedAt = onDebug ? performance.now() : 0
+        const result = await relayPool.sendEvent(...args)
+        if (result.success && onDebug) { try { onDebug({ stage: 'first-ack', publicationId: args[0].id, elapsedMs: performance.now() - startedAt }) } catch {} }
+        return result
+      },
       isOnline: this._isOnline,
       isCurrent: () => !this.closePromise && !this.pauseReasons.size && this.channels.has(channel.pubkey)
     })
@@ -2401,11 +2409,11 @@ export class PrivateMessenger {
     } finally { releaseActivity() }
   }
 
-  async broadcastRumor ({ channelPubkey = this.defaultChannelPubkey(), receiverPubkeys, relays, relayToReceivers, rumor, deletionPubkey, signal }) {
+  async broadcastRumor ({ channelPubkey = this.defaultChannelPubkey(), receiverPubkeys, relays, relayToReceivers, rumor, deletionPubkey, signal, attemptStartedAt, onPreparedSeed, preparedPublication, onDebug }) {
     const channel = this.requireWritableChannel(channelPubkey)
     const releaseActivity = this.decryptionScheduler.activity(channelPubkey)
     try {
-      const routing = await this.resolveSendRouting({ channel, receiverPubkeys, relays, relayToReceivers, signal })
+      const routing = await this.resolveSendRouting({ channel, receiverPubkeys, relays, relayToReceivers, signal, attemptStartedAt, onDebug })
       this.debugSend('broadcastRumor', channelPubkey, { receiverPubkeys })
       return await this._privateMessage.broadcastRumor({
         senderSigner: this.userSigner,
@@ -2419,17 +2427,18 @@ export class PrivateMessenger {
         deletionPubkey,
         autoDeletionCapability: this.autoDeletionCapabilityFor(channel),
         rumor,
-        onPreparedSeed: this.outgoingSeedHandler(channelPubkey),
+        onPreparedSeed: onPreparedSeed || this.outgoingSeedHandler(channelPubkey),
+        preparedPublication,
         _getIykcProofs: this.contentKeyLookup()
       })
     } finally { releaseActivity() }
   }
 
-  async broadcastEvent ({ channelPubkey = this.defaultChannelPubkey(), receiverPubkeys, relays, relayToReceivers, event, deletionPubkey, signal }) {
+  async broadcastEvent ({ channelPubkey = this.defaultChannelPubkey(), receiverPubkeys, relays, relayToReceivers, event, deletionPubkey, signal, attemptStartedAt, onPreparedSeed, preparedPublication, onDebug }) {
     const channel = this.requireWritableChannel(channelPubkey)
     const releaseActivity = this.decryptionScheduler.activity(channelPubkey)
     try {
-      const routing = await this.resolveSendRouting({ channel, receiverPubkeys, relays, relayToReceivers, signal })
+      const routing = await this.resolveSendRouting({ channel, receiverPubkeys, relays, relayToReceivers, signal, attemptStartedAt, onDebug })
       this.debugSend('broadcastEvent', channelPubkey, { receiverPubkeys })
       return await this._privateMessage.broadcastEvent({
         senderSigner: this.userSigner,
@@ -2443,7 +2452,8 @@ export class PrivateMessenger {
         deletionPubkey,
         autoDeletionCapability: this.autoDeletionCapabilityFor(channel),
         event,
-        onPreparedSeed: this.outgoingSeedHandler(channelPubkey),
+        onPreparedSeed: onPreparedSeed || this.outgoingSeedHandler(channelPubkey),
+        preparedPublication,
         _getIykcProofs: this.contentKeyLookup()
       })
     } finally { releaseActivity() }
@@ -2782,8 +2792,17 @@ export class PrivateMessenger {
     })
 
     if (this.offlineRecoverySecondsFor(channelPubkey)) {
+      const seen = new Set()
+      const key = seed => seed.router ? routerSeedRowKey(seed) : nymCarrierSeedKey(seed)
+      const pending = this._pendingSeedRecords({ channelPubkey, receiverPubkey: message.event?.pubkey, since, until })
+      for (const seed of pending) {
+        if (seed.channelPubkey !== channelPubkey || (seed.receiverPubkey && seed.receiverPubkey !== message.event?.pubkey) || seed.expiresAt <= nowSeconds()) continue
+        if (since !== undefined && (seed.lastSeenAt || seed.router.created_at) < since) continue
+        if (until !== undefined && (seed.firstSeenAt || seed.router.created_at) > until) continue
+        seen.add(key(seed)); await packer.update(seed)
+      }
       for await (const seed of this.seedStorage ? this.seedStorage.iterate({ channelPubkey, receiverPubkey: message.event?.pubkey, since, until }) : this.seedQueue.storedItemsBy('byChannel', channelPubkey)) {
-        if (seedRecordTime(seed) < nowSeconds() - this.offlineRecoverySecondsFor(channelPubkey)) continue
+        if (seen.has(key(seed)) || seedRecordTime(seed) < nowSeconds() - this.offlineRecoverySecondsFor(channelPubkey)) continue
         if (this.seedStorage && !await this.seedStorage.has(seed)) continue
         await packer.update(seed)
       }

package/private-messenger/session/README.md

@@ -170,8 +170,11 @@ Optional `fallbackDelayMs` defaults to `null` (sequential primary exhaustion).
 A nonnegative integer up to 2147483647 opts automatic NIP-65 routing into early
 fallback; explicit relay lists/maps retain the sequential policy. With `3000`,
 start at most two eligible fallback relays after three seconds without complete
-acceptance, or immediately after primary exhaustion. The deadline belongs to
-one signed outer event; later batches do not renew it. Primary operations retain
+acceptance, or immediately after primary exhaustion. Optional `enqueue({ ..., attemptStartedAt })`
+uses `performance.now()` from the caller's realm to include preparation in the
+budget. Without it, the deadline starts at publication. Later batches do not
+renew it; retries/reloads start a new attempt and uploads budget each chunk.
+Primary operations retain
 their deadlines and can still win. Failed fallback operations leave primary
 alternatives available; no relay is tried twice. Multiple recipients require
 accepted coverage of every encrypted recipient subset. No wrapper, ID, recipient
@@ -249,3 +252,34 @@ event/peer identifiers and elapsed milliseconds for admission, local processing,
 channel preparation, publication acceptance and cleanup. Callback failures never
 change delivery. Connect only in development; never log events or crypto buffers.
 Custom outbox factories also receive this optional callback for storage timings.
+
+## Durable prepared publications
+
+Each remote entry may contain optional `prepared` progress in its existing
+encrypted JSON. It holds the signed envelope fragments, compact recovery seeds,
+recipient/channel and inner-event identities, event/chunk position, retention
+and accepted/archived flags. Old entries without it retain `localSaved`, index
+and file/chunk progress and enter the same preparation path; there is no database
+migration or alternate publisher. Replays validate signatures and identity and
+reuse the exact envelopes without repeating cryptography or extending expiry.
+
+The existing `onPreparedSeed` callback now also supplies `publication`. The
+session commits that minimal journal before transmitting; direct messengers
+retain their local seed-store durability. Only after that commit can relay
+publication proceed. One independent serial worker converts recovery seeds into
+their configured archive. Pending durable rows participate in recovery reads
+immediately, including during a blocked archive write. No new seeds are enabled
+for modes which did not previously retain them.
+
+First acceptance of every required envelope emits saved independently of archive
+or cleanup. ACK progress checkpoints serialize with other entry writes. A crash
+between ACK and its checkpoint may replay the same envelope; acknowledged durable
+progress only archives. Archive failure never becomes `onSendError` or changes
+saved status. Removal waits for archival and outstanding entry writes.
+
+At most one unit awaits archival per entry; a stalled upload yields its remote
+slot to other texts rather than accumulating the entire file. Cancel stops wire
+work but retains potentially transmitted seeds until archival finishes. Close
+drains active local operations and leaves durable backoff work for restart.
+Diagnostics add publication/seed IDs, checkpoint time, archive wait/execution and
+first-ACK timing, without message or ciphertext contents.

package/private-messenger/session/helpers/publication-journal.js

@@ -0,0 +1,137 @@
+import { compactSeedRouterRows } from '../../index.js'
+import { encodeRecord } from '../../event-store/helpers/codec.js'
+import { isRecoverableStorageFailure } from '../../helpers/pause-recovery.js'
+
+export function createPublicationJournal ({ entries, store, archive, changed, onError, debug, setTimer = setTimeout, clearTimer = clearTimeout, random = Math.random }) {
+  const writes = new Map()
+  const failed = new Set()
+  const durable = new WeakSet()
+  const queued = new WeakMap()
+  let working; let stopped = false; let enabled = false; let timer; let delay = 1000
+  function write (entry) {
+    const previous = writes.get(entry.id) || Promise.resolve()
+    const work = previous.catch(() => {}).then(() => store().put(structuredClone(entry), { existing: true }))
+    writes.set(entry.id, work)
+    work.finally(() => { if (writes.get(entry.id) === work) writes.delete(entry.id) }).catch(() => {})
+    return work
+  }
+  function rows (entry) {
+    const packet = entry.prepared
+    if (!packet?.recover) return []
+    return packet.publication.seeds.flatMap(seed => compactSeedRouterRows(seed).map(row => {
+      const descriptor = packet.target.file
+      const value = {
+        ...row, channelPubkey: seed.channelPubkey, receivedAt: packet.receivedAt,
+        expiresAt: row.router.created_at + packet.retentionSeconds,
+        ...(descriptor
+          ? {
+              root: descriptor.root, controlChannelPubkey: descriptor.controlChannelPubkey,
+              fileChannelPubkey: packet.publication.channelPubkey, peerPubkey: descriptor.peerPubkey,
+              chunkIndex: packet.target.chunkIndex,
+              key: `${descriptor.controlChannelPubkey}:${packet.publication.channelPubkey}:${row.receiverPubkey}:${packet.target.chunkIndex}`
+            }
+          : {})
+      }
+      value.recordId = encodeRecord('seed', value).tags[0][1]
+      return value
+    }))
+  }
+  function retryLater () {
+    if (stopped || !enabled || timer || !failed.size) return
+    timer = setTimer(() => { timer = null; failed.clear(); wake() }, Math.min(30000, delay * (0.8 + random() * 0.4)))
+    timer?.unref?.()
+    delay = Math.min(30000, delay * 2)
+  }
+  function wake () {
+    if (stopped || !enabled || working) return working
+    const work = Promise.resolve().then(async () => {
+      for (const entry of entries.values()) {
+        if (stopped || !enabled) break
+        const packet = entry.prepared
+        if (!packet || failed.has(entry.id)) continue
+        if (packet.archived && !packet.accepted && !entry.cancelled) continue
+        const started = Date.now()
+        try {
+          if (!durable.has(packet)) { await write(entry); durable.add(packet) }
+          if (!packet.archived) {
+            debug(entry, 'recovery-archive-start', queued.get(packet), { publicationId: packet.publication.events[0].id })
+            if (packet.recover) await archive(entry, rows(entry))
+            packet.archived = true
+            debug(entry, 'recovery-archived', started)
+          }
+          if (packet.accepted || entry.cancelled) delete entry.prepared
+          await write(entry)
+          changed(entry)
+        } catch (error) {
+          failed.add(entry.id)
+          onError(error)
+          if (isRecoverableStorageFailure(error)) retryLater()
+        }
+      }
+    }).finally(() => {
+      if (working === work) working = null
+      if (!failed.size) delay = 1000
+      // An ACK can arrive while another entry is being archived.
+      if (!stopped && [...entries.values()].some(entry => entry.prepared && !failed.has(entry.id) && (!entry.prepared.archived || entry.prepared.accepted || entry.cancelled))) wake()
+    })
+    working = work
+    return work
+  }
+  return {
+    write,
+    async flush (entry) { while (writes.has(entry.id)) await writes.get(entry.id) },
+    adopt (entry) { if (entry.prepared) { durable.add(entry.prepared); queued.set(entry.prepared, Date.now()) } },
+    async stage (entry, seed, target, retentionSeconds) {
+      const started = Date.now()
+      if (entry.prepared && entry.prepared.publication.events[0].id !== seed.publication.events[0].id) throw new Error('PUBLICATION_ARCHIVE_PENDING')
+      entry.prepared ||= { publication: structuredClone(seed.publication), target, receivedAt: Math.floor(Date.now() / 1000), retentionSeconds, recover: retentionSeconds > 0, accepted: false, archived: retentionSeconds <= 0 }
+      if (durable.has(entry.prepared)) return
+      queued.set(entry.prepared, Date.now())
+      await write(entry)
+      durable.add(entry.prepared)
+      debug(entry, 'publication-checkpoint', started, { publicationId: seed.publication.events[0].id })
+      for (const row of rows(entry)) debug(entry, 'recovery-pending', undefined, { seedRecordId: row.recordId, publicationId: seed.publication.events[0].id })
+      wake()
+    },
+    wake,
+    accept (entry) {
+      entry.prepared.accepted = true
+      // An unrelated archive may be blocked. Its worker must not hold this ACK.
+      write(entry).catch(error => { failed.add(entry.id); onError(error); retryLater() }).finally(wake)
+      wake()
+    },
+    setAvailable (value) {
+      enabled = value
+      if (!enabled) { if (timer) clearTimer(timer); timer = null } else { failed.clear(); wake() }
+    },
+    retry () { failed.clear(); if (timer) clearTimer(timer); timer = null; return wake() },
+    pendingRows () { return [...entries.values()].flatMap(rows).filter(row => row.expiresAt > Math.floor(Date.now() / 1000)) },
+    async close () {
+      stopped = true
+      if (timer) clearTimer(timer)
+      await working
+      await Promise.allSettled([...writes.values()])
+    }
+  }
+}
+
+export function pendingSeedMatches (row, query) {
+  for (const key of ['channelPubkey', 'controlChannelPubkey', 'fileChannelPubkey', 'receiverPubkey', 'recordId']) if (query[key] && row[key] !== query[key]) return false
+  if (query.indices && !query.indices.includes(row.chunkIndex)) return false
+  if (query.since !== undefined && (row.lastSeenAt ?? row.router.created_at) < query.since) return false
+  if (query.until !== undefined && (row.firstSeenAt ?? row.router.created_at) > query.until) return false
+  return true
+}
+
+export function withPendingSeeds (storage, pending) {
+  return {
+    ...storage,
+    async * iterate (query = {}) {
+      const seen = new Set()
+      // Pending durable rows are immediately usable, even while store.put waits.
+      for (const row of pending()) if (pendingSeedMatches(row, query)) { seen.add(row.recordId); yield row }
+      for await (const row of storage.iterate(query)) if (!seen.has(row.recordId)) yield row
+    },
+    has: row => pending().some(value => value.recordId === row.recordId) || storage.has(row)
+  }
+}