libp2r2p 0.11.27 → 0.11.29

package/README.md

@@ -1257,3 +1257,16 @@ The 0.11.27 session publishes independently of local event-store persistence aft
 its initial durable intent and exact-envelope checkpoint. `deliveryConfirmed`
 and the copied `localPersistence` summary distinguish relay acceptance from
 local storage. See `private-messenger/session/README.md` for retry and cleanup.
+
+The 0.11.28 private messenger preserves classified publication failures for
+seeder recovery asks. Diagnostics include the logical ask ID, channel, seeder
+and range, plus report and relay classifications; arbitrary relay text is not
+logged. Native report causes remain available privately to recovery retry
+policy. Required publication coverage, first-ACK semantics and range recovery
+behavior are unchanged.
+
+The 0.11.29 event-store recovery adapter batches immutable seed existence checks
+and writes in groups of at most 100. Batched checks combine one recovery-type
+AND filter with coordinate OR filters, preserve input order and continue to
+decode and validate every returned snapshot. Custom adapters without batched
+methods retain the sequential fallback.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libp2r2p",
-  "version": "0.11.27",
+  "version": "0.11.29",
   "description": "Peer-to-relay-to-peer",
   "keywords": [
     "p2r2p",

package/private-messenger/event-store/README.md

@@ -18,10 +18,16 @@ without decrypting the archive; the store may also purge expired wrappers itself
 
 ## Contracts and ownership
 
-- Seeds: `put(record)`, `iterate(query)`, `has(record)`, `removeLocal(query)`,
-  `prune({ now })`, and `revoke(records)`.
+- Seeds: `put(record)`, `putMany(records)`, `iterate(query)`, `has(record)`,
+  `hasMany(records)`, `removeLocal(query)`, `prune({ now })`, and
+  `revoke(records)`.
 - Authorizations: the same operations plus `find(query)`, choosing the newest
   valid share, then the shorter expiry and lexicographically smaller record ID.
+- Batched methods preserve input order and process at most 100 coordinates per
+  query. Identical inputs share one immutable result; expired inputs return
+  `undefined` without a query or write. Batched checks use the recovery type
+  as an AND filter and the record coordinates as an OR filter, then still decode
+  and validate every returned snapshot.
 - `iterate` is an async generator. Queries can name `recordId`, `channelPubkey`,
   `receiverPubkey`, `since`/`until`, or the file tuple `controlChannelPubkey`,
   `fileChannelPubkey`, `receiverPubkey`, and `indices`. `signal` cancels reads.

package/private-messenger/event-store/index.js

@@ -6,6 +6,9 @@ import { encodeRecord, decodeRecord, selector, days, PREFIX } from './helpers/co
 export { encodeRecord as encodeRecoveryRecord, decodeRecord as decodeRecoveryRecord } from './helpers/codec.js'
 
 const nowSeconds = () => Math.floor(Date.now() / 1000)
+const RECOVERY_BATCH_SIZE = 100
+const chunks = values => Array.from({ length: Math.ceil(values.length / RECOVERY_BATCH_SIZE) }, (_, index) =>
+  values.slice(index * RECOVERY_BATCH_SIZE, (index + 1) * RECOVERY_BATCH_SIZE))
 function assertOptions (eventStore, signer) {
   if (!eventStore?.query || !eventStore?.addPersonalCopy || !signer?.getPublicKey || !signer?.obfuscate || !signer?.nip44v3?.decrypt) throw new ValidationError('INVALID_EVENT_STORE_OPTIONS')
 }
@@ -49,6 +52,11 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
     const [owner, context] = await ready()
     return { kinds: [1006], authors: [owner], '#k': ['30078'], '#c': [context], '#v': ['0', '1'], '&o': [await mirror('t', `${PREFIX}${type}`)] }
   }
+  async function exactFilterFor (type) {
+    check()
+    const [owner] = await ready()
+    return { kinds: [1006], authors: [owner], '&o': [await mirror('t', `${PREFIX}${type}`)] }
+  }
   async function read (wrapper) {
     check()
     // Query the store before every use; only decryption of an immutable wrapper
@@ -65,9 +73,13 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
     return { ...decoded, wrapper }
   }
   async function * records (type, query = {}) {
-    const filter = await filterFor(type)
-    if (query.recordId) filter['&o'].push(await mirror('d', query.recordId))
-    else if (query.fileChannelPubkey && query.receiverPubkey) {
+    const exact = typeof query.recordId === 'string' || Array.isArray(query.recordIds)
+    const filter = exact ? await exactFilterFor(type) : await filterFor(type)
+    if (query.recordId) filter['#o'] = [await mirror('d', query.recordId)]
+    else if (Array.isArray(query.recordIds)) {
+      if (!query.recordIds.length) return
+      filter['#o'] = await Promise.all([...new Set(query.recordIds)].map(recordId => mirror('d', recordId)))
+    } else if (query.fileChannelPubkey && query.receiverPubkey) {
       const indices = type === 'seed' ? query.indices : [undefined]
       if (indices?.length) filter['#o'] = await Promise.all(indices.map(chunkIndex => mirror('s', selector(type, { ...query, chunkIndex }))))
     } else if (query.channelPubkey) {
@@ -92,24 +104,44 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
     }
   }
   function store (type) {
-    return {
-      async put (row) {
-        check()
+    async function putMany (rows) {
+      check()
+      if (!Array.isArray(rows)) throw new ValidationError('INVALID_RECOVERY_RECORDS')
+      const encoded = rows.map(row => {
+        if (row.expiresAt <= now()) return null
         const event = encodeRecord(type, row)
-        if (row.expiresAt <= now()) return
-        const recordId = event.tags[0][1]
-        const existing = records(type, { recordId, includeExpired: true })
-        try { const first = await existing.next(); if (!first.done) return first.value.row } finally { await existing.return() }
-        const saved = committed(await eventStore.addPersonalCopy(event, { context: '' }))
+        return { event, recordId: event.tags[0][1], row }
+      })
+      const unique = new Map(encoded.filter(Boolean).map(record => [record.recordId, record]))
+      const stored = new Map()
+      for (const recordIds of chunks([...unique.keys()])) {
+        for await (const { row } of records(type, { recordIds, includeExpired: true })) stored.set(row.recordId, row)
+      }
+      for (const [recordId, record] of unique) {
+        if (stored.has(recordId)) continue
+        const saved = committed(await eventStore.addPersonalCopy(record.event, { context: '' }))
         // A pending kind-5 tombstone can reject the insertion; do not cache it.
-        if (saved.result.stored === false && saved.result.code === 'blocked') return
-        return { ...row, recordId }
-      },
+        if (saved.result.stored === false && saved.result.code === 'blocked') stored.set(recordId, undefined)
+        else stored.set(recordId, { ...record.row, recordId })
+      }
+      return encoded.map(record => record ? stored.get(record.recordId) : undefined)
+    }
+    async function hasMany (rows) {
+      check()
+      if (!Array.isArray(rows)) throw new ValidationError('INVALID_RECOVERY_RECORDS')
+      const recordIds = rows.map(row => row.recordId || encodeRecord(type, row).tags[0][1])
+      const present = new Set()
+      for (const batch of chunks([...new Set(recordIds)])) {
+        for await (const { row } of records(type, { recordIds: batch, includeExpired: true })) present.add(row.recordId)
+      }
+      return recordIds.map(recordId => present.has(recordId))
+    }
+    return {
+      async put (row) { return (await putMany([row]))[0] },
+      putMany,
       async * iterate (query = {}) { for await (const { row } of records(type, query)) yield row },
-      async has (row) {
-        const iterator = records(type, { recordId: row.recordId || encodeRecord(type, row).tags[0][1] })
-        try { return !(await iterator.next()).done } finally { await iterator.return() }
-      },
+      async has (row) { return (await hasMany([row]))[0] },
+      hasMany,
       async removeLocal (query = {}) {
         for await (const { wrapper } of records(type, { ...query, includeExpired: true })) await eventStore.remove([['e', wrapper.id]])
       },

package/private-messenger/helpers/publication.js

@@ -1,5 +1,96 @@
 import { getEventHash } from '../../event/index.js'
 
+const safeTokenPattern = /^[A-Za-z0-9][A-Za-z0-9:_-]{0,159}$/
+const safeToken = value => typeof value === 'string' && safeTokenPattern.test(value) ? value : undefined
+const safeRelay = value => {
+  if (typeof value !== 'string') return undefined
+  try {
+    const url = new URL(value)
+    if (url.protocol !== 'ws:' && url.protocol !== 'wss:') return undefined
+    return `${url.protocol}//${url.host}`
+  } catch {
+    return undefined
+  }
+}
+
+function reasonDiagnostic (reason) {
+  if (typeof reason === 'string') {
+    const prefix = /^([a-z][a-z0-9-]{0,31}):/i.exec(reason)?.[1]
+    return { code: safeToken(reason) ?? safeToken(prefix ? `${prefix}:` : undefined) ?? 'UNKNOWN' }
+  }
+  const prefix = typeof reason?.message === 'string' && /^([a-z][a-z0-9-]{0,31}):/i.exec(reason.message)?.[1]
+  const category = safeToken(reason?.category)
+  return {
+    code: safeToken(reason?.code) ?? safeToken(prefix ? `${prefix}:` : undefined) ?? category ?? 'UNKNOWN',
+    ...(category ? { category } : {}),
+    ...(safeToken(reason?.name) ? { name: safeToken(reason.name) } : {}),
+    ...(Number.isSafeInteger(reason?.retryAt) ? { retryAt: reason.retryAt } : {}),
+    ...(Number.isSafeInteger(reason?.retryAfterMs) ? { retryAfterMs: reason.retryAfterMs } : {})
+  }
+}
+
+function sanitizedReport (report, index) {
+  return {
+    index,
+    ...(Number.isSafeInteger(report?.total) ? { total: report.total } : {}),
+    ...(Number.isSafeInteger(report?.fulfilled) ? { fulfilled: report.fulfilled } : {}),
+    ...(report?.retryWhenAvailable === true ? { retryWhenAvailable: true } : {}),
+    ...(report?.retryWhenOnline === true ? { retryWhenOnline: true } : {}),
+    errors: (Array.isArray(report?.errors) ? report.errors : []).map(entry => {
+      const relay = safeRelay(entry?.relay)
+      const detail = reasonDiagnostic(entry?.reason ?? entry)
+      return {
+        ...(relay ? { relay } : {}),
+        ...(safeToken(entry?.code) ? { ...detail, code: entry.code } : detail)
+      }
+    })
+  }
+}
+
+const contextFields = ['operation', 'channelPubkey', 'seederPubkey', 'receiverPubkey', 'eventId']
+const contextNumbers = ['eventKind', 'since', 'until']
+
+export function sanitizePublicationError (error, context = {}) {
+  const source = error instanceof Error ? error : new Error('PRIVATE_MESSAGE_NOT_PUBLISHED')
+  const reports = Array.isArray(source.reports) ? source.reports : []
+  const retryReasons = reports.flatMap(report => (Array.isArray(report?.errors) ? report.errors : [])
+    .map(entry => entry?.reason).filter(Boolean))
+  if (source.code !== 'MESSAGE_NOT_PUBLISHED' && !reports.length) {
+    for (const key of contextFields) if (safeToken(context[key])) source[key] = context[key]
+    for (const key of contextNumbers) if (Number.isSafeInteger(context[key])) source[key] = context[key]
+    return source
+  }
+
+  const sanitized = new Error('PRIVATE_MESSAGE_NOT_PUBLISHED')
+  if (source.stack) sanitized.stack = source.stack
+  Object.assign(sanitized, {
+    code: safeToken(source.code) ?? 'MESSAGE_NOT_PUBLISHED',
+    ...(safeToken(source.reason) ? { reason: source.reason } : {}),
+    ...(safeToken(source.eventId) ? { eventId: source.eventId } : {}),
+    ...(Number.isSafeInteger(source.eventKind) ? { eventKind: source.eventKind } : {}),
+    ...(source.retryWhenAvailable === true ? { retryWhenAvailable: true } : {}),
+    ...(source.retryWhenOnline === true ? { retryWhenOnline: true } : {}),
+    reports: reports.map(sanitizedReport)
+  })
+  for (const key of contextFields) if (safeToken(context[key])) sanitized[key] = context[key]
+  for (const key of contextNumbers) if (Number.isSafeInteger(context[key])) sanitized[key] = context[key]
+  Object.defineProperty(sanitized, 'retryReasons', { value: retryReasons })
+  return sanitized
+}
+
+export function publicationFailureDiagnostic (error) {
+  const result = {}
+  for (const key of [...contextFields, ...contextNumbers]) {
+    const value = error?.[key]
+    if (safeToken(value) && contextFields.includes(key)) result[key] = value
+    else if (Number.isSafeInteger(value) && contextNumbers.includes(key)) result[key] = value
+  }
+  for (const key of ['code', 'reason', 'eventId']) if (safeToken(error?.[key])) result[key] = error[key]
+  for (const key of ['retryWhenAvailable', 'retryWhenOnline']) if (error?.[key] === true) result[key] = true
+  result.reports = (Array.isArray(error?.reports) ? error.reports : []).map(sanitizedReport)
+  return result
+}
+
 // A report describes one outer event, not one relay. Each outer event needs
 // accepted publication covering its routed recipients. Keep the successful fast
 // path immediate; a partially covered multi-recipient report remains a failure.

package/private-messenger/helpers/recovery-retry.js

@@ -19,6 +19,11 @@ export function recoveryRetryDelay (errors, delay) {
       if (['timeout', 'cutoff', 'closed'].includes(entry.status)) retry = true
       visit(entry.error)
     }
+    for (const report of error.reports || []) {
+      if (report.retryWhenAvailable || report.retryWhenOnline) retry = true
+      for (const entry of report.errors || []) visit(entry.reason ?? entry.error)
+    }
+    for (const reason of error.retryReasons || []) visit(reason)
     visit(error.cause)
   }
   errors.forEach(visit)
@@ -36,6 +41,10 @@ export function isPermanentRecoveryError (error) {
         /^(auth-required:|restricted:|blocked:|invalid:|pow:)/.test(value.message || '')) return true
     for (const child of value.errors || []) if (visit(child)) return true
     for (const entry of value.relays || []) if (visit(entry.error)) return true
+    for (const report of value.reports || []) {
+      for (const entry of report.errors || []) if (visit(entry.reason ?? entry.error)) return true
+    }
+    for (const reason of value.retryReasons || []) if (visit(reason)) return true
     return visit(value.cause)
   }
   return visit(error)

package/private-messenger/index.js

@@ -52,6 +52,7 @@ import { backgroundSigner } from './helpers/background-signer.js'
 import { createSendRelayRouting, normalizeFallbackRelays, normalizeFallbackDelay } from './helpers/send-routing.js'
 import { incompleteFetchError } from '../private-channel/helpers/fetch-error.js'
 import { recoveryRetryDelay, isPermanentRecoveryError } from './helpers/recovery-retry.js'
+import { assertMessagePublished, publicationFailureDiagnostic, sanitizePublicationError } from './helpers/publication.js'
 import { normalizeRelayUrl } from '../url/index.js'
 import { deliveryInfo } from '../private-channel/helpers/rumor.js'
 import { migrateMessageQueue } from './services/migrate-message-queue.js'
@@ -105,6 +106,7 @@ export {
 
 const DEFAULT_OFFLINE_RECOVERY_SECONDS = 7 * 24 * 60 * 60
 const MAX_OFFLINE_RECOVERY_SECONDS = Math.floor(Number.MAX_SAFE_INTEGER / 1000)
+const RECOVERY_STORAGE_BATCH_SIZE = 100
 const STORAGE_POLICY_BROADCAST_CHANNEL = 'libp2r2p:private-messenger:storage-policy'
 const DEFAULT_OFFLINE_SKEW_SECONDS = 30
 const DEFAULT_RELOAD_GAP_DELAY_MS = 500
@@ -2147,9 +2149,13 @@ export class PrivateMessenger {
       const rows = seed.recordType === NYM_CARRIER_SEED_RECORD_TYPE || seed.carriers?.length
         ? [{ recordType: NYM_CARRIER_SEED_RECORD_TYPE, carriers: compactSeedNymCarriers(seed.carriers) }]
         : compactSeedRouterRows(seed)
-      for (const row of rows) {
+      const records = rows.map(row => {
         const time = seedRecordTime(row) || receivedAt
-        await this.seedStorage.put({ ...row, channelPubkey, receivedAt, expiresAt: time + this.offlineRecoverySecondsFor(channelPubkey) })
+        return { ...row, channelPubkey, receivedAt, expiresAt: time + this.offlineRecoverySecondsFor(channelPubkey) }
+      })
+      if (typeof this.seedStorage.putMany === 'function') await this.seedStorage.putMany(records)
+      else {
+        for (const row of records) await this.seedStorage.put(row)
       }
       return
     }
@@ -2755,13 +2761,17 @@ export class PrivateMessenger {
           payload: { since, until }
         })
         asks.push(ask)
-        const reports = ask?.delivery?.reports
-        if (!Array.isArray(reports) || !reports.length || reports.some(report => report?.success !== true)) {
-          throw new Error('PRIVATE_MESSAGE_NOT_PUBLISHED')
-        }
+        await assertMessagePublished(ask, ask.question)
       } catch (err) {
-        failures.push({ seeder, error: err })
-        console.warn('private-messenger seeder recovery ask failed', seeder, err?.message ?? err)
+        const error = sanitizePublicationError(err, {
+          operation: 'seeder-recovery-ask',
+          channelPubkey,
+          seederPubkey: seeder,
+          since,
+          until
+        })
+        failures.push({ seeder, error })
+        console.warn('private-messenger seeder recovery ask failed', JSON.stringify(publicationFailureDiagnostic(error)))
       }
     }
     return { asks, failures }
@@ -2794,6 +2804,16 @@ export class PrivateMessenger {
     if (this.offlineRecoverySecondsFor(channelPubkey)) {
       const seen = new Set()
       const key = seed => seed.router ? routerSeedRowKey(seed) : nymCarrierSeedKey(seed)
+      const sendStored = async seeds => {
+        let present
+        if (typeof this.seedStorage?.hasMany === 'function') present = await this.seedStorage.hasMany(seeds)
+        else {
+          present = []
+          for (const seed of seeds) present.push(await this.seedStorage.has(seed))
+        }
+        if (!Array.isArray(present) || present.length !== seeds.length) throw new ValidationError('INVALID_RECOVERY_STORAGE_RESULT')
+        for (let index = 0; index < seeds.length; index++) if (present[index]) await packer.update(seeds[index])
+      }
       const pending = this._pendingSeedRecords({ channelPubkey, receiverPubkey: message.event?.pubkey, since, until })
       for (const seed of pending) {
         if (seed.channelPubkey !== channelPubkey || (seed.receiverPubkey && seed.receiverPubkey !== message.event?.pubkey) || seed.expiresAt <= nowSeconds()) continue
@@ -2801,11 +2821,17 @@ export class PrivateMessenger {
         if (until !== undefined && (seed.firstSeenAt || seed.router.created_at) > until) continue
         seen.add(key(seed)); await packer.update(seed)
       }
+      const stored = []
       for await (const seed of this.seedStorage ? this.seedStorage.iterate({ channelPubkey, receiverPubkey: message.event?.pubkey, since, until }) : this.seedQueue.storedItemsBy('byChannel', channelPubkey)) {
         if (seen.has(key(seed)) || seedRecordTime(seed) < nowSeconds() - this.offlineRecoverySecondsFor(channelPubkey)) continue
-        if (this.seedStorage && !await this.seedStorage.has(seed)) continue
-        await packer.update(seed)
+        if (!this.seedStorage) {
+          await packer.update(seed)
+          continue
+        }
+        stored.push(seed)
+        if (stored.length === RECOVERY_STORAGE_BATCH_SIZE) await sendStored(stored.splice(0))
       }
+      if (stored.length) await sendStored(stored)
     }
     await packer.finalize()
   }