libp2r2p 0.11.31 → 0.11.33

package/README.md

@@ -1284,3 +1284,14 @@ The 0.11.31 pending-decryption retry groups records by
 remaining records only when the local signer confirms `CONTENT_KEY_NOT_FOUND`
 with `hasOwnContentKey: false`. MAC, proof, parsing and operational failures are
 never generalized to the group, and every deferred record remains persisted.
+
+The 0.11.32 local recovery-seed queue serializes incoming and outgoing writes
+within one PrivateMessenger. Duplicate `bySeedKey` races retry up to five times:
+NYM rows remain idempotent and router rows re-read the winning timestamps before
+replacement. External event-store seed adapters retain their independent batched
+writes.
+
+The 0.11.33 recovery adapter checks immutable wrapper mirrors without decrypting
+snapshots and serializes recovery mutations per event store. Session archives
+wait for foreground quiet for up to five seconds, execute one batch, then
+re-evaluate so pending recovery cannot monopolize local persistence.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libp2r2p",
-  "version": "0.11.31",
+  "version": "0.11.33",
   "description": "Peer-to-relay-to-peer",
   "keywords": [
     "p2r2p",

package/private-messenger/event-store/README.md

@@ -27,8 +27,13 @@ without decrypting the archive; the store may also purge expired wrappers itself
   query. Identical inputs share one immutable result. `putMany` returns
   `undefined` for an expired input without querying or writing; `hasMany` returns
   `false` for expired snapshots. Batched checks use the recovery type as an AND
-  filter and the record coordinates as an OR filter, then still decode and
-  validate every returned snapshot.
+  filter and the record coordinates as an OR filter. Existence checks compare the
+  immutable wrapper `o` mirrors to the requested record IDs and do not decrypt or
+  decode snapshots. Duplicate `putMany` inputs return their supplied immutable row.
+- Mutating recovery stores share one operation tail per event store, so seed and
+  grant writes cannot overlap for the same store. Writes pass the internal
+  `recovery: true` scheduling hint to `addPersonalCopy`; row-returning reads keep
+  normal decrypt-and-validate behavior.
 - `iterate` is an async generator. Queries can name `recordId`, `channelPubkey`,
   `receiverPubkey`, `since`/`until`, or the file tuple `controlChannelPubkey`,
   `fileChannelPubkey`, `receiverPubkey`, and `indices`. `signal` cancels reads.

package/private-messenger/event-store/index.js

@@ -7,8 +7,19 @@ export { encodeRecord as encodeRecoveryRecord, decodeRecord as decodeRecoveryRec
 
 const nowSeconds = () => Math.floor(Date.now() / 1000)
 const RECOVERY_BATCH_SIZE = 100
+const mutationTails = new WeakMap()
 const chunks = values => Array.from({ length: Math.ceil(values.length / RECOVERY_BATCH_SIZE) }, (_, index) =>
   values.slice(index * RECOVERY_BATCH_SIZE, (index + 1) * RECOVERY_BATCH_SIZE))
+function serializeMutation (eventStore, operation) {
+  const previous = mutationTails.get(eventStore) ?? Promise.resolve()
+  const run = previous.catch(() => {}).then(operation)
+  const tail = run.catch(() => {})
+  mutationTails.set(eventStore, tail)
+  tail.then(() => {
+    if (mutationTails.get(eventStore) === tail) mutationTails.delete(eventStore)
+  })
+  return run
+}
 function assertOptions (eventStore, signer) {
   if (!eventStore?.query || !eventStore?.addPersonalCopy || !signer?.getPublicKey || !signer?.obfuscate || !signer?.nip44v3?.decrypt) throw new ValidationError('INVALID_EVENT_STORE_OPTIONS')
 }
@@ -103,8 +114,27 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
       yield { row, wrapper }
     }
   }
+  async function * existingRecordIds (type, values, { includeExpired = false } = {}) {
+    const ids = [...new Set(values)]
+    if (!ids.length) return
+    const filter = await exactFilterFor(type)
+    const mirrorsById = new Map(await Promise.all(ids.map(async recordId => [await mirror('d', recordId), recordId])))
+    filter['#o'] = [...mirrorsById.keys()]
+    for await (const wrapper of pages(eventStore, filter)) {
+      const expiry = Number(wrapper.tags?.find(tag => Array.isArray(tag) && tag[0] === 'expiration')?.[1])
+      if (!includeExpired && (!Number.isSafeInteger(expiry) || expiry <= now())) continue
+      for (const tag of wrapper.tags || []) {
+        if (!Array.isArray(tag) || tag[0] !== 'o') continue
+        const recordId = mirrorsById.get(tag[1])
+        if (recordId) yield recordId
+      }
+    }
+  }
   function store (type) {
     async function putMany (rows) {
+      return serializeMutation(eventStore, () => putManyNow(rows))
+    }
+    async function putManyNow (rows) {
       check()
       if (!Array.isArray(rows)) throw new ValidationError('INVALID_RECOVERY_RECORDS')
       const encoded = rows.map(row => {
@@ -115,11 +145,14 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
       const unique = new Map(encoded.filter(Boolean).map(record => [record.recordId, record]))
       const stored = new Map()
       for (const recordIds of chunks([...unique.keys()])) {
-        for await (const { row } of records(type, { recordIds, includeExpired: true })) stored.set(row.recordId, row)
+        for await (const recordId of existingRecordIds(type, recordIds, { includeExpired: true })) {
+          const record = unique.get(recordId)
+          if (record) stored.set(recordId, { ...record.row, recordId })
+        }
       }
       for (const [recordId, record] of unique) {
         if (stored.has(recordId)) continue
-        const saved = committed(await eventStore.addPersonalCopy(record.event, { context: '' }))
+        const saved = committed(await eventStore.addPersonalCopy(record.event, { context: '', recovery: true }))
         // A pending kind-5 tombstone can reject the insertion; do not cache it.
         if (saved.result.stored === false && saved.result.code === 'blocked') stored.set(recordId, undefined)
         else stored.set(recordId, { ...record.row, recordId })
@@ -132,7 +165,7 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
       const recordIds = rows.map(row => row.recordId || encodeRecord(type, row).tags[0][1])
       const present = new Set()
       for (const batch of chunks([...new Set(recordIds)])) {
-        for await (const { row } of records(type, { recordIds: batch })) present.add(row.recordId)
+        for await (const recordId of existingRecordIds(type, batch)) present.add(recordId)
       }
       return recordIds.map(recordId => present.has(recordId))
     }
@@ -154,20 +187,22 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
         }
       },
       async revoke (rows) {
-        check()
-        const [owner] = await ready()
-        for (let i = 0; i < rows.length; i += 100) {
-          const group = rows.slice(i, i + 100).filter(row => row.expiresAt > now())
-          if (!group.length) continue
-          const tags = []; let createdAt = now(), expiresAt = 0
-          for (const row of group) {
-            const recordId = row.recordId || encodeRecord(type, row).tags[0][1]
-            for await (const { wrapper } of records(type, { recordId })) createdAt = Math.max(createdAt, wrapper.created_at)
-            tags.push(['a', `30078:${owner}:${recordId}`]); expiresAt = Math.max(expiresAt, row.expiresAt)
+        return serializeMutation(eventStore, async () => {
+          check()
+          const [owner] = await ready()
+          for (let i = 0; i < rows.length; i += 100) {
+            const group = rows.slice(i, i + 100).filter(row => row.expiresAt > now())
+            if (!group.length) continue
+            const tags = []; let createdAt = now(), expiresAt = 0
+            for (const row of group) {
+              const recordId = row.recordId || encodeRecord(type, row).tags[0][1]
+              for await (const { wrapper } of records(type, { recordId })) createdAt = Math.max(createdAt, wrapper.created_at)
+              tags.push(['a', `30078:${owner}:${recordId}`]); expiresAt = Math.max(expiresAt, row.expiresAt)
+            }
+            if (expiresAt <= createdAt) continue
+            committed(await eventStore.addPersonalCopy({ kind: 5, created_at: createdAt, tags: [...tags, ['k', '30078'], ['expiration', String(expiresAt)]], content: '' }, { context: '', recovery: true }))
           }
-          if (expiresAt <= createdAt) continue
-          committed(await eventStore.addPersonalCopy({ kind: 5, created_at: createdAt, tags: [...tags, ['k', '30078'], ['expiration', String(expiresAt)]], content: '' }, { context: '' }))
-        }
+        })
       }
     }
   }

package/private-messenger/index.js

@@ -108,6 +108,7 @@ export {
 const DEFAULT_OFFLINE_RECOVERY_SECONDS = 7 * 24 * 60 * 60
 const MAX_OFFLINE_RECOVERY_SECONDS = Math.floor(Number.MAX_SAFE_INTEGER / 1000)
 const RECOVERY_STORAGE_BATCH_SIZE = 100
+const SEED_QUEUE_UPSERT_ATTEMPTS = 5
 const STORAGE_POLICY_BROADCAST_CHANNEL = 'libp2r2p:private-messenger:storage-policy'
 const DEFAULT_OFFLINE_SKEW_SECONDS = 30
 const DEFAULT_RELOAD_GAP_DELAY_MS = 500
@@ -374,6 +375,7 @@ export class PrivateMessenger {
     this.watchRevisionByChannel = new Map()
     this.presenceTimers = new Map()
     this.presenceJobs = new Map()
+    this.seedQueueOperationTail = Promise.resolve()
     this.stopRelayListWatcher = null
     this.relayListWatcherPubkey = ''
     this.relayListRefreshPromise = null
@@ -686,6 +688,13 @@ export class PrivateMessenger {
     return this.runQueueOperation(operation)
   }
 
+  runSeedQueueOperation (operation) {
+    if (this.closePromise) return Promise.reject(new Error('PRIVATE_MESSENGER_CLOSED'))
+    const run = this.seedQueueOperationTail.then(operation)
+    this.seedQueueOperationTail = run.catch(() => {})
+    return run
+  }
+
   debug (action, detail = {}) {
     try {
       this.onDebug?.({ source: 'private-messenger', action, ...detail })
@@ -2140,8 +2149,8 @@ export class PrivateMessenger {
 
   async enqueueSeed (channelPubkey, seed) {
     if (!this.offlineRecoverySecondsFor(channelPubkey)) return
-    const receivedAt = nowSeconds()
     if (this.seedStorage) {
+      const receivedAt = nowSeconds()
       const rows = seed.recordType === NYM_CARRIER_SEED_RECORD_TYPE || seed.carriers?.length
         ? [{ recordType: NYM_CARRIER_SEED_RECORD_TYPE, carriers: compactSeedNymCarriers(seed.carriers) }]
         : compactSeedRouterRows(seed)
@@ -2155,22 +2164,37 @@ export class PrivateMessenger {
       }
       return
     }
+    return this.runSeedQueueOperation(() => this.enqueueQueueSeed(channelPubkey, seed))
+  }
+
+  async enqueueQueueSeed (channelPubkey, seed) {
+    this.assertOpen()
+    if (!this.channels.has(channelPubkey) || !this.offlineRecoverySecondsFor(channelPubkey)) return
+    const receivedAt = nowSeconds()
     if (seed.recordType === NYM_CARRIER_SEED_RECORD_TYPE || seed.carriers?.length) {
       const carriers = compactSeedNymCarriers(seed.carriers)
       const recordTime = nymCarrierRecordTime({ carriers }) || seed.outer?.created_at || receivedAt
       const key = nymCarrierSeedKey({ channelPubkey, carriers })
       const seedKey = key ? `nym:${key}` : ''
-      if (seedKey && await this.seedQueue.someBy('bySeedKey', seedKey)) return
-      await this.seedQueue.enqueue({
-        type: 'seed',
-        recordType: NYM_CARRIER_SEED_RECORD_TYPE,
-        channelPubkey,
-        receivedAt,
-        carriers,
-        meta: { channelPubkey: seed.channelPubkey },
-        [SEED_KEY]: seedKey || undefined,
-        [SEED_TIME]: recordTime
-      })
+      for (let attempt = 0; attempt < SEED_QUEUE_UPSERT_ATTEMPTS; attempt++) {
+        if (seedKey && await this.seedQueue.someBy('bySeedKey', seedKey)) break
+        try {
+          await this.seedQueue.enqueue({
+            type: 'seed',
+            recordType: NYM_CARRIER_SEED_RECORD_TYPE,
+            channelPubkey,
+            receivedAt,
+            carriers,
+            meta: { channelPubkey: seed.channelPubkey },
+            [SEED_KEY]: seedKey || undefined,
+            [SEED_TIME]: recordTime
+          })
+          break
+        } catch (error) {
+          if (error?.name !== 'ConstraintError') throw error
+          if (attempt === SEED_QUEUE_UPSERT_ATTEMPTS - 1) throw error
+        }
+      }
       await this.pruneStoredSeeds(channelPubkey)
       return
     }
@@ -2180,21 +2204,29 @@ export class PrivateMessenger {
       const rowTime = row.lastSeenAt || row.router?.created_at || receivedAt
       const rowKey = routerSeedRowKey({ ...row, channelPubkey })
       const seedKey = `router:${rowKey}`
-      const [previous] = await this.seedQueue.removeBy('bySeedKey', seedKey)
-      const firstSeenAt = Math.min(previous?.firstSeenAt ?? rowTime, row.firstSeenAt ?? rowTime)
-      const lastSeenAt = Math.max(previous?.lastSeenAt ?? rowTime, row.lastSeenAt ?? rowTime)
-      await this.seedQueue.enqueue({
-        ...row,
-        type: 'seed',
-        recordType: ROUTER_SEED_RECORD_TYPE,
-        channelPubkey,
-        receivedAt,
-        firstSeenAt,
-        lastSeenAt,
-        meta: { channelPubkey: seed.channelPubkey },
-        [SEED_KEY]: seedKey,
-        [SEED_TIME]: lastSeenAt || rowTime
-      })
+      for (let attempt = 0; attempt < SEED_QUEUE_UPSERT_ATTEMPTS; attempt++) {
+        const [previous] = await this.seedQueue.removeBy('bySeedKey', seedKey)
+        const firstSeenAt = Math.min(previous?.firstSeenAt ?? rowTime, row.firstSeenAt ?? rowTime)
+        const lastSeenAt = Math.max(previous?.lastSeenAt ?? rowTime, row.lastSeenAt ?? rowTime)
+        try {
+          await this.seedQueue.enqueue({
+            ...row,
+            type: 'seed',
+            recordType: ROUTER_SEED_RECORD_TYPE,
+            channelPubkey,
+            receivedAt,
+            firstSeenAt,
+            lastSeenAt,
+            meta: { channelPubkey: seed.channelPubkey },
+            [SEED_KEY]: seedKey,
+            [SEED_TIME]: lastSeenAt || rowTime
+          })
+          break
+        } catch (error) {
+          if (error?.name !== 'ConstraintError') throw error
+          if (attempt === SEED_QUEUE_UPSERT_ATTEMPTS - 1) throw error
+        }
+      }
     }
     await this.pruneStoredSeeds(channelPubkey)
   }

package/private-messenger/session/README.md

@@ -130,6 +130,8 @@ reevaluate a local failure without clearing another owner's storage pause.
 
 `enqueue({ peer, event, context, requiredFiles, deletion })` preserves prepared
 identities. Local message commits and remote stages have independent checkpoints.
+Recovery archival waits up to five seconds for foreground work, then runs one
+durable batch and re-evaluates; postponement never drops packet rows.
 It sends original/thumbnail chunks before announcements, yields for small messages,
 and never interprets relay rejection text as signer permission denial. Inbox ACK
 follows durable storage; failed storage nacks and pauses ingestion. Hearsay cannot

package/private-messenger/session/helpers/foreground-work.js

@@ -1,6 +1,6 @@
 // Local outbox persistence and user publications take precedence over the next
 // optional presence signer call. Already-started signer calls are not recalled.
-export function createForegroundWork () {
+export function createForegroundWork ({ setTimer = setTimeout, clearTimer = clearTimeout } = {}) {
   let active = 0
   const waiters = new Set()
   const observers = new Set()
@@ -28,6 +28,35 @@ export function createForegroundWork () {
         combined.addEventListener('abort', finish, { once: true })
       })
     },
+    waitUntilIdle ({ maxWaitMs = 5000, signal } = {}) {
+      if (!Number.isFinite(maxWaitMs) || maxWaitMs < 0) throw new TypeError('INVALID_FOREGROUND_MAX_WAIT')
+      const combined = signal ? AbortSignal.any([signal, lifetime.signal]) : lifetime.signal
+      combined.throwIfAborted()
+      if (!active) return Promise.resolve(true)
+      return new Promise((resolve, reject) => {
+        const timeout = { timer: undefined }
+        const cleanup = () => {
+          waiters.delete(finishIdle)
+          if (timeout.timer !== undefined) clearTimer(timeout.timer)
+          combined.removeEventListener('abort', finishAbort)
+        }
+        const finishIdle = () => {
+          cleanup()
+          resolve(true)
+        }
+        const finishAbort = () => {
+          cleanup()
+          reject(combined.reason)
+        }
+        waiters.add(finishIdle)
+        combined.addEventListener('abort', finishAbort, { once: true })
+        timeout.timer = setTimer(() => {
+          cleanup()
+          resolve(false)
+        }, maxWaitMs)
+        timeout.timer?.unref?.()
+      })
+    },
     observe (listener) { observers.add(listener); listener(active > 0); return () => observers.delete(listener) },
     close () { lifetime.abort(); waiters.clear(); observers.clear() }
   }

package/private-messenger/session/index.js

@@ -28,6 +28,7 @@ const unavailableSend = /PRIVATE_MESSENGER_PAUSED|CHAT_UNAVAILABLE|MESSENGER_UNA
 const isUnavailableSend = error => unavailableSend.test(`${error?.code || ''} ${error?.message || ''}`)
 const SEND_RETRY_MIN_MS = 1000
 const SEND_RETRY_MAX_MS = 30000
+const ARCHIVE_FOREGROUND_WAIT_MS = 5000
 // A paused or unavailable messenger must keep the outbox entry retryable
 // instead of surfacing a send failure the user has to retry by hand.
 const unavailableError = (error, signal) => Object.assign(new Error(error?.message || 'CHAT_UNAVAILABLE'), {
@@ -59,7 +60,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     onError(error)
   }
   const userSigner = messengerSigner(signer)
-  const foreground = createForegroundWork()
+  const foreground = createForegroundWork({ setTimer: _setTimeout, clearTimer: _clearTimeout })
   messageStorage ||= createEventStoreMessageStorage({ eventStore })
   chunkStorage ||= eventStore ? createEventStoreChunkStorage({ eventStore }) : undefined
   const peers = new Set()
@@ -165,6 +166,16 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
   const journal = createPublicationJournal({
     entries, store: () => storage, onError: reportError, debug, setTimer: _setTimeout, clearTimer: _clearTimeout, random: _random,
     archive: async (entry, rows) => {
+      const waitedAt = Date.now()
+      let idle
+      try {
+        idle = await foreground.waitUntilIdle({ maxWaitMs: ARCHIVE_FOREGROUND_WAIT_MS })
+      } catch (error) {
+        if (closed) return
+        throw error
+      }
+      if (closed) return
+      debug(entry, 'recovery-archive-window', waitedAt, { foregroundIdle: idle, rowCount: rows.length })
       if (recoveryStorage?.seeds) {
         if (typeof recoveryStorage.seeds.putMany === 'function') await recoveryStorage.seeds.putMany(rows)
         else for (const row of rows) await recoveryStorage.seeds.put(row)