libp2r2p 0.11.5 → 0.11.6

package/README.md

@@ -534,6 +534,42 @@ returned events; received messages may already have been queued. This is a
 history completeness failure, not evidence that an outgoing message failed.
 Existing rewatch/resume and relay-list refresh paths retry pending intervals;
 there is no dedicated periodic retry timer for a failed history fetch.
+`private-channel.fetchHistory(options)` (0.11.6) is the complete-interval API
+used by messenger recovery. It accepts the signers, processor/fragment options,
+callbacks and abort signal of `fetch`, plus inclusive `since`/`until` bounds
+(defaults: 0 and the start time of the call). It returns only
+`{ oldestCreatedAt, receivedEventCount, relays }`; an empty scan has a null oldest
+time. No complete-history event array is retained. `fetch` keeps its existing
+single-query array contract and optional limit.
+
+History queries each relay independently with limit 16. A response with 16 events
+or status `satisfied` is saturated, even if it already reports EOSE or contains
+fewer valid events. Saturated intervals split into nonoverlapping inclusive
+halves; completed pages process chronologically, oldest half first. Single-second
+intervals grow through 16, 32, 64, 128 and 256. Each response is also bounded to
+4 MiB of serialized UTF-8 event data. At the dense-second or volume ceiling,
+`PRIVATE_CHANNEL_FETCH_INCOMPLETE` retains the operational cause
+`PRIVATE_CHANNEL_HISTORY_PAGE_LIMIT`; the interval stays pending. There is no
+nonstandard cursor, disk spool, dropped excess or successful incomplete scan.
+
+A fragment processor and the existing scoped fragment storage span the entire
+call, including pages and relays. Callbacks are awaited before requesting the
+next page. Processing/persistence failures propagate; completed pages may already
+be ingested and retries remain subject to the existing idempotence contract.
+The summary count includes processed outer events per relay, including replicas,
+not unique decrypted messages or discarded saturated probes. It does not include
+all events observed in those probes.
+
+Recovery admits at most two pages per PrivateMessenger and one per normalized
+relay across the same library instance, including page processing time. Continuous
+live watches are outside those recovery gates. Waiters are abortable and FIFO,
+with at most 256 queued per gate (`PRIVATE_CHANNEL_HISTORY_QUEUE_FULL` on excess).
+The five-second network timeout begins after admission; reported read time also
+includes admission but excludes processing. Seeders still supplement the oldest
+relay result. Checkpoints advance only after the entire interval and existing
+seeder stages succeed; interruption preserves its original pending range. No
+persisted state format changes or migrations are needed.
+
 Subscription relay errors forwarded to `onError` include `relay` when known
 and `operation: 'private-channel.subscribe'`. A per-notification wrapper retains
 the native message, name, code, category and WebSocket close fields; `cause`

package/helpers/abortable-semaphore.js

@@ -0,0 +1,41 @@
+// A bounded FIFO for asynchronous work. Acquired leases release explicitly;
+// queued cancellations never consume capacity or keep idle keyed gates alive.
+export function createAbortableSemaphore (capacity, onIdle = () => {}) {
+  const queue = []
+  let active = 0
+  const drain = () => {
+    while (active < capacity && queue.length) {
+      const job = queue.shift()
+      job.signal?.removeEventListener('abort', job.abort)
+      active++
+      let released = false
+      job.resolve(() => {
+        if (released) return
+        released = true
+        active--
+        drain()
+      })
+    }
+    if (!active && !queue.length) onIdle()
+  }
+  return {
+    acquire (signal) {
+      signal?.throwIfAborted()
+      if (queue.length >= 256) return Promise.reject(Object.assign(new Error('PRIVATE_CHANNEL_HISTORY_QUEUE_FULL'), { code: 'PRIVATE_CHANNEL_HISTORY_QUEUE_FULL' }))
+      return new Promise((resolve, reject) => {
+        const job = {
+          signal, resolve, abort: () => {
+            const index = queue.indexOf(job)
+            if (index < 0) return
+            queue.splice(index, 1)
+            reject(signal.reason)
+            drain()
+          }
+        }
+        signal?.addEventListener('abort', job.abort, { once: true })
+        queue.push(job)
+        drain()
+      })
+    }
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libp2r2p",
-  "version": "0.11.5",
+  "version": "0.11.6",
   "description": "Peer-to-relay-to-peer",
   "keywords": [
     "p2r2p",
@@ -22,6 +22,7 @@
     "ecdh",
     "error",
     "event",
+    "helpers",
     "idb",
     "idb-queue",
     "index.js",

package/private-channel/helpers/history.js

@@ -0,0 +1,121 @@
+import { ValidationError } from '../../error/index.js'
+import { normalizeRelayUrl } from '../../url/index.js'
+import { createAbortableSemaphore } from '../../helpers/abortable-semaphore.js'
+import { incompleteFetchError } from './fetch-error.js'
+
+const PAGE_SIZE = 16
+const MAX_PAGE_SIZE = 256
+const MAX_PAGE_BYTES = 4 * 1024 * 1024
+const TIMEOUT_MS = 5000
+const relayGates = new Map()
+const encoder = new TextEncoder()
+const pageLimitError = () => Object.assign(new Error('PRIVATE_CHANNEL_HISTORY_PAGE_LIMIT'), { code: 'PRIVATE_CHANNEL_HISTORY_PAGE_LIMIT' })
+
+function acquireRelay (relay, signal) {
+  signal?.throwIfAborted()
+  let gate = relayGates.get(relay)
+  if (!gate) {
+    gate = createAbortableSemaphore(1, () => { if (relayGates.get(relay) === gate) relayGates.delete(relay) })
+    relayGates.set(relay, gate)
+  }
+  return gate.acquire(signal)
+}
+
+// Queries bounded temporal leaves, oldest first. A saturated one-second leaf
+// cannot be split using standard NIP-01; expand it only up to the safe ceiling.
+export async function readHistory ({ filter, relays, receiverPubkey, signal, getEvents, processEvent, acquirePage }) {
+  const since = filter.since ?? 0
+  const until = filter.until ?? Math.floor(Date.now() / 1000)
+  if (![since, until].every(value => Number.isSafeInteger(value) && value >= 0) || since > until) throw new ValidationError('INVALID_PRIVATE_CHANNEL_HISTORY_RANGE')
+  if (!Array.isArray(relays) || !relays.length) throw new ValidationError('NO_RELAYS')
+  const urls = [...new Set(relays.map(normalizeRelayUrl))]
+  const report = []
+  const errors = []
+  let oldestCreatedAt = null
+  let receivedEventCount = 0
+  let elapsedMs = 0
+  const process = async result => {
+    result.sort((a, b) => a.event.created_at - b.event.created_at || String(a.event.id).localeCompare(String(b.event.id)))
+    for (const { event } of result) {
+      signal?.throwIfAborted()
+      await processEvent(event)
+      receivedEventCount++
+      oldestCreatedAt = Math.min(oldestCreatedAt ?? event.created_at, event.created_at)
+    }
+  }
+  for (const relay of urls) {
+    const intervals = [{ since, until, limit: PAGE_SIZE }]
+    let outcome = { relay, status: 'eose' }
+    while (intervals.length) {
+      signal?.throwIfAborted()
+      const interval = intervals.pop()
+      const started = performance.now()
+      let releaseRelay
+      let releasePage
+      let page
+      try {
+        let bytes = 0
+        try {
+          releaseRelay = await acquireRelay(relay, signal)
+          releasePage = await acquirePage?.(signal)
+          signal?.throwIfAborted()
+          page = await getEvents({ ...filter, ...interval }, [relay], {
+            signal, timeout: TIMEOUT_MS, timeoutAfterFirstEose: null,
+            callback: item => {
+              if (item.type !== 'event') return
+              bytes += encoder.encode(JSON.stringify(item.event)).byteLength
+              if (bytes > MAX_PAGE_BYTES) throw pageLimitError()
+            }
+          })
+          // Also enforce the contract for injected readers that omit callbacks.
+          let resultBytes = 0
+          for (const { event } of page.result) {
+            resultBytes += encoder.encode(JSON.stringify(event)).byteLength
+            if (resultBytes > MAX_PAGE_BYTES) throw pageLimitError()
+          }
+        } catch (error) {
+          signal?.throwIfAborted()
+          errors.push({ relay, reason: error })
+          outcome = { relay, status: 'error', error }
+          break
+        } finally { elapsedMs += performance.now() - started }
+        signal?.throwIfAborted()
+        const status = page.relays?.find(item => item.relay === relay)
+        if (page.errors?.length || !status || !['eose', 'satisfied'].includes(status.status)) {
+          await process(page.result)
+          errors.push(...(page.errors ?? []))
+          outcome = status ?? { relay, status: 'error' }
+          break
+        }
+        if (status.status === 'satisfied' || page.result.length >= interval.limit) {
+          if (interval.since < interval.until) {
+            const middle = Math.floor(interval.since + (interval.until - interval.since) / 2)
+            intervals.push({ since: middle + 1, until: interval.until, limit: PAGE_SIZE }, { since: interval.since, until: middle, limit: PAGE_SIZE })
+          } else if (interval.limit < MAX_PAGE_SIZE) {
+            intervals.push({ ...interval, limit: interval.limit * 2 })
+          } else {
+            await process(page.result)
+            const error = pageLimitError()
+            errors.push({ relay, reason: error })
+            outcome = { relay, status: 'error', error }
+            break
+          }
+        } else {
+          await process(page.result)
+        }
+        signal?.throwIfAborted()
+      } finally {
+        releasePage?.()
+        releaseRelay?.()
+      }
+    }
+    report.push(outcome)
+  }
+  signal?.throwIfAborted()
+  if (errors.length || report.some(item => item.status !== 'eose')) {
+    const error = incompleteFetchError({ errors, report, receivedEventCount, elapsedMs: Math.round(elapsedMs), request: { relays: urls, channelPubkeys: filter.authors ?? [], receiverPubkey, since, until, limit: PAGE_SIZE, timeoutMs: TIMEOUT_MS } })
+    error.operation = 'private-channel.fetchHistory'
+    throw error
+  }
+  return { oldestCreatedAt, receivedEventCount, relays: report }
+}

package/private-channel/index.js

@@ -7,6 +7,7 @@ import { getIykcProofs } from '../content-key/index.js'
 import { ValidationError } from '../error/index.js'
 import { normalizeRumor, deliveryInfo } from './helpers/rumor.js'
 import { incompleteFetchError } from './helpers/fetch-error.js'
+import { readHistory } from './helpers/history.js'
 import { subscriptionError } from './helpers/subscription-error.js'
 import * as nip44v3 from '../nip44-v3/index.js'
 import { relayPool } from '../relay/index.js'
@@ -955,6 +956,17 @@ export async function fetch ({ signal, receiverSigner, iykcSigner, privateChanne
   }
 }
 
+// Complete interval recovery with bounded pages and one fragment processor.
+// Existing fetch() retains its one-query/event-array contract.
+export async function fetchHistory ({ receiverSigner, privateChannelSigner = receiverSigner, _getEvents = getEvents, _acquirePage, ...options }) {
+  const authors = privateChannelPubkeyList(options)
+  const filter = { kinds: [PRIVATE_BROADCAST_KIND], ...(authors.length ? { authors } : {}), since: options.since, until: options.until }
+  const processOuterEvent = createProcessor({ ...options, receiverSigner, privateChannelSigner, onError: error => { options.onError?.(error); throw error } })
+  try {
+    return await readHistory({ filter, relays: options.relays, receiverPubkey: options.receiverPubkey, signal: options.signal, getEvents: _getEvents, processEvent: processOuterEvent, acquirePage: _acquirePage })
+  } finally { processOuterEvent.close() }
+}
+
 export function subscribe ({ receiverSigner, iykcSigner, privateChannelSigner = receiverSigner, privateChannelSignersByPubkey, privateChannelReaderSigner = privateChannelSigner, privateChannelReaderSignersByPubkey, privateChannelReaderPubkey, privateChannelReaderPubkeysByPubkey, privateChannelPubkey, privateChannelPubkeys, receiverPubkey, relays, onChunk, onEvent, onNymEvent, onSeedEvent, onContentKeyUsage, onError, since = nowSeconds() - 5, limit, liveOnly = false, mode = 'leecher', modeByPubkey, receivedChunkScope = '', receivedChunkTtlMs = DEFAULT_RECEIVED_CHUNK_TTL_MS, receivedChunkTtlMsByPubkey, receivedChunkMaxBytes = DEFAULT_RECEIVED_CHUNK_MAX_BYTES, receivedChunkIndexedDB = globalThis.indexedDB, ignoredGroupTtlMs = DEFAULT_IGNORED_GROUP_TTL_MS, ignoredGroupMaxEntries = DEFAULT_IGNORED_GROUP_MAX_ENTRIES, _liveEventsGenerator = getLiveEventsGenerator, _eventsFeedGenerator = getEventsFeedGenerator }) {
   if (!relays?.length) throw new ValidationError('NO_RELAYS')
   if (receiverSigner && !receiverSigner?.nip44DecryptDoubleDH && !receiverSigner?.nip44v3Decrypt) throw new ValidationError('RECEIVER_SIGNER_NIP44V3_DECRYPT_UNSUPPORTED')

package/private-messenger/index.js

@@ -42,6 +42,7 @@
 // - Seeder replies stream compact routers with createMissingMessageReplyPacker({ messenger, question }).update(seed), then finalize(optionalLastSeed).
 // - For other event-list replies, use createEventReplyPacker({ messenger, question, code }).update(event).
 
+import { createAbortableSemaphore } from '../helpers/abortable-semaphore.js'
 import * as privateMessage from '../private-message/index.js'
 import { isOnline } from '../network/index.js'
 import { createSendRelayRouting, normalizeFallbackRelays } from './helpers/send-routing.js'
@@ -285,6 +286,7 @@ export class PrivateMessenger {
     this.deliveryReads = new Set()
     this.deliveryWaiters = new Set()
     this.recoveries = new Map()
+    this.recoveryAdmission = createAbortableSemaphore(2)
     this.recoveryControllers = new Set()
     this.resumeWork = null
     this.capacityTimer = null
@@ -1950,12 +1952,11 @@ export class PrivateMessenger {
   }
 
   async askSeedersForRelayLeftEdge (channelPubkey, range, fetchedEvents) {
-    const { asks } = await this.#askSeedersForRelayLeftEdgeAttempt(channelPubkey, range, fetchedEvents)
+    const { asks } = await this.#askSeedersForRelayLeftEdgeAttempt(channelPubkey, range, oldestCreatedAt(fetchedEvents))
     return asks
   }
 
-  async #askSeedersForRelayLeftEdgeAttempt (channelPubkey, range, fetchedEvents) {
-    const oldest = oldestCreatedAt(fetchedEvents)
+  async #askSeedersForRelayLeftEdgeAttempt (channelPubkey, range, oldest) {
     const until = oldest == null ? range.end : Math.min(range.end, oldest)
     if (until < range.start) return { asks: [], failures: [] }
     return this.#askSeedersForMissingRangeAttempt(channelPubkey, range.start, until)
@@ -2098,7 +2099,8 @@ export class PrivateMessenger {
           if (this.pauseReasons.size || this.closePromise || !this.desiredChannels.has(pubkey)) throw new Error('PRIVATE_MESSENGER_PAUSED')
           const fetchRelays = await this.resolveWatchRelays(channel)
           controller.signal.throwIfAborted()
-          const fetchedEvents = await this._privateChannel.fetch({
+          const history = await this._privateChannel.fetchHistory({
+            _acquirePage: signal => this.recoveryAdmission.acquire(signal),
             signal: controller.signal,
             receivedChunkScope: this.storageLeaseId,
             receiverSigner: this.userSigner,
@@ -2120,9 +2122,9 @@ export class PrivateMessenger {
             onSeedEvent: seed => this.receive(pubkey, () => this.enqueueSeed(pubkey, seed), seed),
             onContentKeyUsage: usage => this.handleContentKeyUsage(pubkey, usage),
             onError: err => { throw err }
-          }) || []
+          })
           controller.signal.throwIfAborted()
-          const attempt = await this.#askSeedersForRelayLeftEdgeAttempt(pubkey, range, fetchedEvents)
+          const attempt = await this.#askSeedersForRelayLeftEdgeAttempt(pubkey, range, history.oldestCreatedAt)
           const lifecycleChanged = this.closePromise || !this.channels.has(pubkey) || !this.stopByChannel.has(pubkey) ||
             (this.watchRevisionByChannel.get(pubkey) || 0) !== watchRevision
           if (lifecycleChanged || attempt.failures.length) remaining.push(range)