libp2r2p 0.11.32 → 0.11.34

package/README.md

@@ -1290,3 +1290,14 @@ within one PrivateMessenger. Duplicate `bySeedKey` races retry up to five times:
 NYM rows remain idempotent and router rows re-read the winning timestamps before
 replacement. External event-store seed adapters retain their independent batched
 writes.
+
+The 0.11.33 recovery adapter checks immutable wrapper mirrors without decrypting
+snapshots and serializes recovery mutations per event store. Session archives
+wait for foreground quiet for up to five seconds, execute one batch, then
+re-evaluate so pending recovery cannot monopolize local persistence.
+
+The 0.11.34 recovery adapter marks paged event-store reads with the internal
+`recovery: true` hint so a compatible launcher can prioritize foreground writes
+over new background read transactions. `putMany` also accepts an optional
+development diagnostic callback with aggregate existing/inserted/skipped counts;
+it never exposes ciphertext or record payloads.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libp2r2p",
-  "version": "0.11.32",
+  "version": "0.11.34",
   "description": "Peer-to-relay-to-peer",
   "keywords": [
     "p2r2p",

package/private-messenger/event-store/README.md

@@ -27,8 +27,21 @@ without decrypting the archive; the store may also purge expired wrappers itself
   query. Identical inputs share one immutable result. `putMany` returns
   `undefined` for an expired input without querying or writing; `hasMany` returns
   `false` for expired snapshots. Batched checks use the recovery type as an AND
-  filter and the record coordinates as an OR filter, then still decode and
-  validate every returned snapshot.
+  filter and the record coordinates as an OR filter. Existence checks compare the
+  immutable wrapper `o` mirrors to the requested record IDs and do not decrypt or
+  decode snapshots. Duplicate `putMany` inputs return their supplied immutable row.
+- Recovery pages pass the optional internal `{ recovery: true }` query hint.
+  Launchers may use it to admit the readonly transaction as background without
+  granting an app elevated priority. Stores that do not recognize the hint keep
+  the previous query behavior.
+- `putMany(rows, { onDiagnostic })` accepts an optional development callback. One
+  sanitized report identifies row, unique, existing, inserted, blocked, expired,
+  stored and skipped counts after each batch. It never includes ciphertext,
+  decoded rows, record payloads or signer material.
+- Mutating recovery stores share one operation tail per event store, so seed and
+  grant writes cannot overlap for the same store. Writes and paged reads pass the
+  internal `recovery: true` scheduling hint to the corresponding event-store
+  methods; row-returning reads keep normal decrypt-and-validate behavior.
 - `iterate` is an async generator. Queries can name `recordId`, `channelPubkey`,
   `receiverPubkey`, `since`/`until`, or the file tuple `controlChannelPubkey`,
   `fileChannelPubkey`, `receiverPubkey`, and `indices`. `signal` cancels reads.

package/private-messenger/event-store/index.js

@@ -7,8 +7,20 @@ export { encodeRecord as encodeRecoveryRecord, decodeRecord as decodeRecoveryRec
 
 const nowSeconds = () => Math.floor(Date.now() / 1000)
 const RECOVERY_BATCH_SIZE = 100
+const RECOVERY_QUERY_OPTIONS = Object.freeze({ recovery: true })
+const mutationTails = new WeakMap()
 const chunks = values => Array.from({ length: Math.ceil(values.length / RECOVERY_BATCH_SIZE) }, (_, index) =>
   values.slice(index * RECOVERY_BATCH_SIZE, (index + 1) * RECOVERY_BATCH_SIZE))
+function serializeMutation (eventStore, operation) {
+  const previous = mutationTails.get(eventStore) ?? Promise.resolve()
+  const run = previous.catch(() => {}).then(operation)
+  const tail = run.catch(() => {})
+  mutationTails.set(eventStore, tail)
+  tail.then(() => {
+    if (mutationTails.get(eventStore) === tail) mutationTails.delete(eventStore)
+  })
+  return run
+}
 function assertOptions (eventStore, signer) {
   if (!eventStore?.query || !eventStore?.addPersonalCopy || !signer?.getPublicKey || !signer?.obfuscate || !signer?.nip44v3?.decrypt) throw new ValidationError('INVALID_EVENT_STORE_OPTIONS')
 }
@@ -21,7 +33,10 @@ async function * pages (eventStore, filter, signal) {
   let until = filter.until, excluded = []
   while (true) {
     signal?.throwIfAborted()
-    const { results } = await eventStore.query({ ...filter, ...(until === undefined ? {} : { until }), ...(excluded.length ? { '!ids': excluded } : {}), limit: 128, search: 'sort:desc' })
+    const { results } = await eventStore.query(
+      { ...filter, ...(until === undefined ? {} : { until }), ...(excluded.length ? { '!ids': excluded } : {}), limit: 128, search: 'sort:desc' },
+      RECOVERY_QUERY_OPTIONS
+    )
     if (!results.length) return
     for (const event of results) { signal?.throwIfAborted(); yield event }
     const boundary = results.at(-1).created_at
@@ -103,8 +118,27 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
       yield { row, wrapper }
     }
   }
+  async function * existingRecordIds (type, values, { includeExpired = false } = {}) {
+    const ids = [...new Set(values)]
+    if (!ids.length) return
+    const filter = await exactFilterFor(type)
+    const mirrorsById = new Map(await Promise.all(ids.map(async recordId => [await mirror('d', recordId), recordId])))
+    filter['#o'] = [...mirrorsById.keys()]
+    for await (const wrapper of pages(eventStore, filter)) {
+      const expiry = Number(wrapper.tags?.find(tag => Array.isArray(tag) && tag[0] === 'expiration')?.[1])
+      if (!includeExpired && (!Number.isSafeInteger(expiry) || expiry <= now())) continue
+      for (const tag of wrapper.tags || []) {
+        if (!Array.isArray(tag) || tag[0] !== 'o') continue
+        const recordId = mirrorsById.get(tag[1])
+        if (recordId) yield recordId
+      }
+    }
+  }
   function store (type) {
-    async function putMany (rows) {
+    async function putMany (rows, options = {}) {
+      return serializeMutation(eventStore, () => putManyNow(rows, options))
+    }
+    async function putManyNow (rows, { onDiagnostic } = {}) {
       check()
       if (!Array.isArray(rows)) throw new ValidationError('INVALID_RECOVERY_RECORDS')
       const encoded = rows.map(row => {
@@ -115,15 +149,40 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
       const unique = new Map(encoded.filter(Boolean).map(record => [record.recordId, record]))
       const stored = new Map()
       for (const recordIds of chunks([...unique.keys()])) {
-        for await (const { row } of records(type, { recordIds, includeExpired: true })) stored.set(row.recordId, row)
+        for await (const recordId of existingRecordIds(type, recordIds, { includeExpired: true })) {
+          const record = unique.get(recordId)
+          if (record) stored.set(recordId, { ...record.row, recordId })
+        }
       }
+      const existingCount = stored.size
+      let insertedCount = 0
+      let blockedCount = 0
       for (const [recordId, record] of unique) {
         if (stored.has(recordId)) continue
-        const saved = committed(await eventStore.addPersonalCopy(record.event, { context: '' }))
+        const saved = committed(await eventStore.addPersonalCopy(record.event, { context: '', recovery: true }))
         // A pending kind-5 tombstone can reject the insertion; do not cache it.
-        if (saved.result.stored === false && saved.result.code === 'blocked') stored.set(recordId, undefined)
-        else stored.set(recordId, { ...record.row, recordId })
+        if (saved.result.stored === false && saved.result.code === 'blocked') {
+          stored.set(recordId, undefined)
+          blockedCount++
+        } else {
+          stored.set(recordId, { ...record.row, recordId })
+          insertedCount++
+        }
       }
+      const expiredCount = encoded.filter(record => record === null).length
+      try {
+        onDiagnostic?.({
+          type,
+          rowCount: rows.length,
+          uniqueCount: unique.size,
+          existingCount,
+          insertedCount,
+          blockedCount,
+          expiredCount,
+          storedCount: existingCount + insertedCount,
+          skippedCount: existingCount + blockedCount + expiredCount
+        })
+      } catch {}
       return encoded.map(record => record ? stored.get(record.recordId) : undefined)
     }
     async function hasMany (rows) {
@@ -132,7 +191,7 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
       const recordIds = rows.map(row => row.recordId || encodeRecord(type, row).tags[0][1])
       const present = new Set()
       for (const batch of chunks([...new Set(recordIds)])) {
-        for await (const { row } of records(type, { recordIds: batch })) present.add(row.recordId)
+        for await (const recordId of existingRecordIds(type, batch)) present.add(recordId)
       }
       return recordIds.map(recordId => present.has(recordId))
     }
@@ -154,20 +213,22 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
         }
       },
       async revoke (rows) {
-        check()
-        const [owner] = await ready()
-        for (let i = 0; i < rows.length; i += 100) {
-          const group = rows.slice(i, i + 100).filter(row => row.expiresAt > now())
-          if (!group.length) continue
-          const tags = []; let createdAt = now(), expiresAt = 0
-          for (const row of group) {
-            const recordId = row.recordId || encodeRecord(type, row).tags[0][1]
-            for await (const { wrapper } of records(type, { recordId })) createdAt = Math.max(createdAt, wrapper.created_at)
-            tags.push(['a', `30078:${owner}:${recordId}`]); expiresAt = Math.max(expiresAt, row.expiresAt)
+        return serializeMutation(eventStore, async () => {
+          check()
+          const [owner] = await ready()
+          for (let i = 0; i < rows.length; i += 100) {
+            const group = rows.slice(i, i + 100).filter(row => row.expiresAt > now())
+            if (!group.length) continue
+            const tags = []; let createdAt = now(), expiresAt = 0
+            for (const row of group) {
+              const recordId = row.recordId || encodeRecord(type, row).tags[0][1]
+              for await (const { wrapper } of records(type, { recordId })) createdAt = Math.max(createdAt, wrapper.created_at)
+              tags.push(['a', `30078:${owner}:${recordId}`]); expiresAt = Math.max(expiresAt, row.expiresAt)
+            }
+            if (expiresAt <= createdAt) continue
+            committed(await eventStore.addPersonalCopy({ kind: 5, created_at: createdAt, tags: [...tags, ['k', '30078'], ['expiration', String(expiresAt)]], content: '' }, { context: '', recovery: true }))
           }
-          if (expiresAt <= createdAt) continue
-          committed(await eventStore.addPersonalCopy({ kind: 5, created_at: createdAt, tags: [...tags, ['k', '30078'], ['expiration', String(expiresAt)]], content: '' }, { context: '' }))
-        }
+        })
       }
     }
   }

package/private-messenger/session/README.md

@@ -130,6 +130,8 @@ reevaluate a local failure without clearing another owner's storage pause.
 
 `enqueue({ peer, event, context, requiredFiles, deletion })` preserves prepared
 identities. Local message commits and remote stages have independent checkpoints.
+Recovery archival waits up to five seconds for foreground work, then runs one
+durable batch and re-evaluates; postponement never drops packet rows.
 It sends original/thumbnail chunks before announcements, yields for small messages,
 and never interprets relay rejection text as signer permission denial. Inbox ACK
 follows durable storage; failed storage nacks and pauses ingestion. Hearsay cannot
@@ -268,6 +270,10 @@ event/peer identifiers and elapsed milliseconds for admission, local processing,
 channel preparation, publication acceptance and cleanup. Callback failures never
 change delivery. Connect only in development; never log events or crypto buffers.
 Custom outbox factories also receive this optional callback for storage timings.
+Recovery archive completion also reports `publicationId`, one aggregate
+`recovery-archive-result` per `putMany`, and sanitized row/unique/existing/
+inserted/blocked/expired/stored/skipped counts. A single-row batch may include
+its opaque `seedRecordId`.
 
 ## Durable prepared publications
 

package/private-messenger/session/helpers/foreground-work.js

@@ -1,6 +1,6 @@
 // Local outbox persistence and user publications take precedence over the next
 // optional presence signer call. Already-started signer calls are not recalled.
-export function createForegroundWork () {
+export function createForegroundWork ({ setTimer = setTimeout, clearTimer = clearTimeout } = {}) {
   let active = 0
   const waiters = new Set()
   const observers = new Set()
@@ -28,6 +28,35 @@ export function createForegroundWork () {
         combined.addEventListener('abort', finish, { once: true })
       })
     },
+    waitUntilIdle ({ maxWaitMs = 5000, signal } = {}) {
+      if (!Number.isFinite(maxWaitMs) || maxWaitMs < 0) throw new TypeError('INVALID_FOREGROUND_MAX_WAIT')
+      const combined = signal ? AbortSignal.any([signal, lifetime.signal]) : lifetime.signal
+      combined.throwIfAborted()
+      if (!active) return Promise.resolve(true)
+      return new Promise((resolve, reject) => {
+        const timeout = { timer: undefined }
+        const cleanup = () => {
+          waiters.delete(finishIdle)
+          if (timeout.timer !== undefined) clearTimer(timeout.timer)
+          combined.removeEventListener('abort', finishAbort)
+        }
+        const finishIdle = () => {
+          cleanup()
+          resolve(true)
+        }
+        const finishAbort = () => {
+          cleanup()
+          reject(combined.reason)
+        }
+        waiters.add(finishIdle)
+        combined.addEventListener('abort', finishAbort, { once: true })
+        timeout.timer = setTimer(() => {
+          cleanup()
+          resolve(false)
+        }, maxWaitMs)
+        timeout.timer?.unref?.()
+      })
+    },
     observe (listener) { observers.add(listener); listener(active > 0); return () => observers.delete(listener) },
     close () { lifetime.abort(); waiters.clear(); observers.clear() }
   }

package/private-messenger/session/index.js

@@ -28,6 +28,7 @@ const unavailableSend = /PRIVATE_MESSENGER_PAUSED|CHAT_UNAVAILABLE|MESSENGER_UNA
 const isUnavailableSend = error => unavailableSend.test(`${error?.code || ''} ${error?.message || ''}`)
 const SEND_RETRY_MIN_MS = 1000
 const SEND_RETRY_MAX_MS = 30000
+const ARCHIVE_FOREGROUND_WAIT_MS = 5000
 // A paused or unavailable messenger must keep the outbox entry retryable
 // instead of surfacing a send failure the user has to retry by hand.
 const unavailableError = (error, signal) => Object.assign(new Error(error?.message || 'CHAT_UNAVAILABLE'), {
@@ -59,7 +60,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     onError(error)
   }
   const userSigner = messengerSigner(signer)
-  const foreground = createForegroundWork()
+  const foreground = createForegroundWork({ setTimer: _setTimeout, clearTimer: _clearTimeout })
   messageStorage ||= createEventStoreMessageStorage({ eventStore })
   chunkStorage ||= eventStore ? createEventStoreChunkStorage({ eventStore }) : undefined
   const peers = new Set()
@@ -165,14 +166,49 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
   const journal = createPublicationJournal({
     entries, store: () => storage, onError: reportError, debug, setTimer: _setTimeout, clearTimer: _clearTimeout, random: _random,
     archive: async (entry, rows) => {
+      const waitedAt = Date.now()
+      let idle
+      try {
+        idle = await foreground.waitUntilIdle({ maxWaitMs: ARCHIVE_FOREGROUND_WAIT_MS })
+      } catch (error) {
+        if (closed) return
+        throw error
+      }
+      if (closed) return
+      const archiveStartedAt = Date.now()
+      debug(entry, 'recovery-archive-window', waitedAt, { foregroundIdle: idle, rowCount: rows.length })
+      let archived
+      let diagnosticReported = false
+      const onDiagnostic = details => {
+        diagnosticReported = true
+        debug(entry, 'recovery-archive-result', archiveStartedAt, {
+          publicationId: entry.prepared.publication.events[0].id,
+          ...details
+        })
+      }
       if (recoveryStorage?.seeds) {
-        if (typeof recoveryStorage.seeds.putMany === 'function') await recoveryStorage.seeds.putMany(rows)
-        else for (const row of rows) await recoveryStorage.seeds.put(row)
+        if (typeof recoveryStorage.seeds.putMany === 'function') archived = await recoveryStorage.seeds.putMany(rows, { onDiagnostic })
+        else {
+          archived = []
+          for (const row of rows) archived.push(await recoveryStorage.seeds.put(row))
+        }
       } else if (entry.prepared.target.file) {
         for (const seed of entry.prepared.publication.seeds) await fileTransfers.archiveSeed(entry.prepared.target.file, seed)
       } else {
         for (const seed of entry.prepared.publication.seeds) await messenger.enqueueSeed(seed.channelPubkey, seed)
       }
+      if (!diagnosticReported) {
+        const results = Array.isArray(archived) ? archived : []
+        const storedCount = results.filter(Boolean).length
+        debug(entry, 'recovery-archive-result', archiveStartedAt, {
+          publicationId: entry.prepared.publication.events[0].id,
+          rowCount: rows.length,
+          uniqueCount: new Set(rows.map(row => row.recordId).filter(Boolean)).size,
+          storedCount,
+          skippedCount: results.length ? results.length - storedCount : undefined,
+          ...(rows.length === 1 && rows[0].recordId ? { seedRecordId: rows[0].recordId } : {})
+        })
+      }
     },
     changed: entry => {
       watchOfflineSends()