libp2r2p 0.11.33 → 0.11.35

package/README.md

@@ -1295,3 +1295,15 @@ The 0.11.33 recovery adapter checks immutable wrapper mirrors without decrypting
 snapshots and serializes recovery mutations per event store. Session archives
 wait for foreground quiet for up to five seconds, execute one batch, then
 re-evaluate so pending recovery cannot monopolize local persistence.
+
+The 0.11.34 recovery adapter marks paged event-store reads with the internal
+`recovery: true` hint so a compatible launcher can prioritize foreground writes
+over new background read transactions. `putMany` also accepts an optional
+development diagnostic callback with aggregate existing/inserted/skipped counts;
+it never exposes ciphertext or record payloads.
+
+The 0.11.35 session lets self-chat local persistence and its initial encrypted
+outbox checkpoint run concurrently. The entry remains hidden and ineligible for
+saved/publication/cleanup until admission; remote entries keep checkpoint-first
+ordering. A failed local commit after admission remains retryable through the
+existing outbox path.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libp2r2p",
-  "version": "0.11.33",
+  "version": "0.11.35",
   "description": "Peer-to-relay-to-peer",
   "keywords": [
     "p2r2p",

package/private-messenger/event-store/README.md

@@ -30,10 +30,18 @@ without decrypting the archive; the store may also purge expired wrappers itself
   filter and the record coordinates as an OR filter. Existence checks compare the
   immutable wrapper `o` mirrors to the requested record IDs and do not decrypt or
   decode snapshots. Duplicate `putMany` inputs return their supplied immutable row.
+- Recovery pages pass the optional internal `{ recovery: true }` query hint.
+  Launchers may use it to admit the readonly transaction as background without
+  granting an app elevated priority. Stores that do not recognize the hint keep
+  the previous query behavior.
+- `putMany(rows, { onDiagnostic })` accepts an optional development callback. One
+  sanitized report identifies row, unique, existing, inserted, blocked, expired,
+  stored and skipped counts after each batch. It never includes ciphertext,
+  decoded rows, record payloads or signer material.
 - Mutating recovery stores share one operation tail per event store, so seed and
-  grant writes cannot overlap for the same store. Writes pass the internal
-  `recovery: true` scheduling hint to `addPersonalCopy`; row-returning reads keep
-  normal decrypt-and-validate behavior.
+  grant writes cannot overlap for the same store. Writes and paged reads pass the
+  internal `recovery: true` scheduling hint to the corresponding event-store
+  methods; row-returning reads keep normal decrypt-and-validate behavior.
 - `iterate` is an async generator. Queries can name `recordId`, `channelPubkey`,
   `receiverPubkey`, `since`/`until`, or the file tuple `controlChannelPubkey`,
   `fileChannelPubkey`, `receiverPubkey`, and `indices`. `signal` cancels reads.

package/private-messenger/event-store/index.js

@@ -7,6 +7,7 @@ export { encodeRecord as encodeRecoveryRecord, decodeRecord as decodeRecoveryRec
 
 const nowSeconds = () => Math.floor(Date.now() / 1000)
 const RECOVERY_BATCH_SIZE = 100
+const RECOVERY_QUERY_OPTIONS = Object.freeze({ recovery: true })
 const mutationTails = new WeakMap()
 const chunks = values => Array.from({ length: Math.ceil(values.length / RECOVERY_BATCH_SIZE) }, (_, index) =>
   values.slice(index * RECOVERY_BATCH_SIZE, (index + 1) * RECOVERY_BATCH_SIZE))
@@ -32,7 +33,10 @@ async function * pages (eventStore, filter, signal) {
   let until = filter.until, excluded = []
   while (true) {
     signal?.throwIfAborted()
-    const { results } = await eventStore.query({ ...filter, ...(until === undefined ? {} : { until }), ...(excluded.length ? { '!ids': excluded } : {}), limit: 128, search: 'sort:desc' })
+    const { results } = await eventStore.query(
+      { ...filter, ...(until === undefined ? {} : { until }), ...(excluded.length ? { '!ids': excluded } : {}), limit: 128, search: 'sort:desc' },
+      RECOVERY_QUERY_OPTIONS
+    )
     if (!results.length) return
     for (const event of results) { signal?.throwIfAborted(); yield event }
     const boundary = results.at(-1).created_at
@@ -131,10 +135,10 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
     }
   }
   function store (type) {
-    async function putMany (rows) {
-      return serializeMutation(eventStore, () => putManyNow(rows))
+    async function putMany (rows, options = {}) {
+      return serializeMutation(eventStore, () => putManyNow(rows, options))
     }
-    async function putManyNow (rows) {
+    async function putManyNow (rows, { onDiagnostic } = {}) {
       check()
       if (!Array.isArray(rows)) throw new ValidationError('INVALID_RECOVERY_RECORDS')
       const encoded = rows.map(row => {
@@ -150,13 +154,35 @@ export function createPersonalCopyRecoveryStorage ({ eventStore, signer, now = n
           if (record) stored.set(recordId, { ...record.row, recordId })
         }
       }
+      const existingCount = stored.size
+      let insertedCount = 0
+      let blockedCount = 0
       for (const [recordId, record] of unique) {
         if (stored.has(recordId)) continue
         const saved = committed(await eventStore.addPersonalCopy(record.event, { context: '', recovery: true }))
         // A pending kind-5 tombstone can reject the insertion; do not cache it.
-        if (saved.result.stored === false && saved.result.code === 'blocked') stored.set(recordId, undefined)
-        else stored.set(recordId, { ...record.row, recordId })
+        if (saved.result.stored === false && saved.result.code === 'blocked') {
+          stored.set(recordId, undefined)
+          blockedCount++
+        } else {
+          stored.set(recordId, { ...record.row, recordId })
+          insertedCount++
+        }
       }
+      const expiredCount = encoded.filter(record => record === null).length
+      try {
+        onDiagnostic?.({
+          type,
+          rowCount: rows.length,
+          uniqueCount: unique.size,
+          existingCount,
+          insertedCount,
+          blockedCount,
+          expiredCount,
+          storedCount: existingCount + insertedCount,
+          skippedCount: existingCount + blockedCount + expiredCount
+        })
+      } catch {}
       return encoded.map(record => record ? stored.get(record.recordId) : undefined)
     }
     async function hasMany (rows) {

package/private-messenger/session/README.md

@@ -251,13 +251,19 @@ creates a fresh publication lifetime, without treating the pause as a user abort
 ## Independent local persistence and publication
 
 One encrypted outbox feeds two independent serial processors. Local persistence
-never waits for peer preparation or an in-flight publication. After the initial encrypted intent commits, remote preparation and publication
-never wait for event-store persistence. Only the exact prepared-envelope
-checkpoint gates transmission. Text continues passing uploads between chunks;
-its local stage need not have completed. Both stages use the admitted inner ID.
-
-Self entries emit `status: 'saved'` as soon as local persistence succeeds. They
-do not write a redundant localSaved checkpoint before removal. Remote entries
+never waits for peer preparation or an in-flight publication. Remote entries
+join those processors only after their initial encrypted intent commits, while
+self entries may start local persistence alongside that checkpoint. An
+in-flight self entry remains internal until admission: it is absent from
+`onOutbox`, cannot publish or clean up, and cannot report saved. Remote
+preparation and publication never wait for event-store persistence. Only the
+exact prepared-envelope checkpoint gates transmission. Text continues passing
+uploads between chunks; its local stage need not have completed. Both stages use
+the admitted inner ID.
+
+Self entries emit `status: 'saved'` once local persistence succeeds and the
+initial encrypted checkpoint admits the entry. They do not write a redundant
+localSaved checkpoint before removal. Remote entries
 emit that status after the first valid relay ACK for every required publication,
 without waiting for redundant relays. Outbox cleanup runs independently; failures
 retain saved status and retry only removal with the session backoff, never
@@ -270,6 +276,10 @@ event/peer identifiers and elapsed milliseconds for admission, local processing,
 channel preparation, publication acceptance and cleanup. Callback failures never
 change delivery. Connect only in development; never log events or crypto buffers.
 Custom outbox factories also receive this optional callback for storage timings.
+Recovery archive completion also reports `publicationId`, one aggregate
+`recovery-archive-result` per `putMany`, and sanitized row/unique/existing/
+inserted/blocked/expired/stored/skipped counts. A single-row batch may include
+its opaque `seedRecordId`.
 
 ## Durable prepared publications
 

package/private-messenger/session/index.js

@@ -67,6 +67,12 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
   const channels = new Map()
   const entries = new Map()
   const cancelled = new Set()
+  // Self entries can persist locally while their encrypted checkpoint is in
+  // flight, but they are not visible to consumers until that checkpoint wins.
+  const unadmitted = new WeakSet()
+  const admissionFailed = new WeakSet()
+  const localWorks = new WeakMap()
+  const localErrors = new WeakMap()
   const attemptStarts = new Map()
   const networkUrgency = new Map()
   const sendControllers = new Map()
@@ -106,14 +112,14 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
       for (const entry of entries.values()) {
         if (entry.peer !== owner && entry.failed && entry.status === 'pending') pausedSends.add(entry.id)
       }
-      if (![...entries.values()].some(entry => (entry.localFailed && entry.localRetryable) || (pendingRetryable(entry) && !pausedSends.has(entry.id)))) clearSendRetry()
+      if (![...entries.values()].some(entry => !unadmitted.has(entry) && ((entry.localFailed && entry.localRetryable) || (pendingRetryable(entry) && !pausedSends.has(entry.id))))) clearSendRetry()
       watchOfflineSends()
       return
     }
     clearSendRetry()
     for (const id of pausedSends) { const entry = entries.get(id); if (entry) { entry.failed = false; attemptStarts.set(entry.id, performance.now()) } }
     pausedSends.clear()
-    if ([...entries.values()].some(entry => entry.localFailed && entry.localRetryable)) scheduleSendRetry()
+    if ([...entries.values()].some(entry => !unadmitted.has(entry) && entry.localFailed && entry.localRetryable)) scheduleSendRetry()
     watchOfflineSends()
     // The callback can run inside construction/resume, before configure settles.
     queueMicrotask(() => pump())
@@ -158,8 +164,8 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
   const idleWaiters = new Set()
   const settled = () => { if (!sending && !localSaving && !enqueuing && !draining && !cleanups.size) { for (const resolve of idleWaiters) resolve(); idleWaiters.clear() } }
   const localComplete = entry => entry.localSaved.every(Boolean)
-  const completed = entry => localComplete(entry) && (entry.peer === owner || entry.deliveryConfirmed)
-  const emit = () => onOutbox([...entries.values()].filter(entry => !entry.deletion && !entry.cancelled).map(entry => ({
+  const completed = entry => !unadmitted.has(entry) && localComplete(entry) && (entry.peer === owner || entry.deliveryConfirmed)
+  const emit = () => onOutbox([...entries.values()].filter(entry => !unadmitted.has(entry) && !entry.deletion && !entry.cancelled).map(entry => ({
     ...entry,
     localPersistence: { state: localComplete(entry) ? 'saved' : entry.localFailed ? 'failed' : 'pending', retryable: Boolean(entry.localFailed && entry.localRetryable && !localRetries.has(entry.id)) }
   })))
@@ -175,15 +181,40 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
         throw error
       }
       if (closed) return
+      const archiveStartedAt = Date.now()
       debug(entry, 'recovery-archive-window', waitedAt, { foregroundIdle: idle, rowCount: rows.length })
+      let archived
+      let diagnosticReported = false
+      const onDiagnostic = details => {
+        diagnosticReported = true
+        debug(entry, 'recovery-archive-result', archiveStartedAt, {
+          publicationId: entry.prepared.publication.events[0].id,
+          ...details
+        })
+      }
       if (recoveryStorage?.seeds) {
-        if (typeof recoveryStorage.seeds.putMany === 'function') await recoveryStorage.seeds.putMany(rows)
-        else for (const row of rows) await recoveryStorage.seeds.put(row)
+        if (typeof recoveryStorage.seeds.putMany === 'function') archived = await recoveryStorage.seeds.putMany(rows, { onDiagnostic })
+        else {
+          archived = []
+          for (const row of rows) archived.push(await recoveryStorage.seeds.put(row))
+        }
       } else if (entry.prepared.target.file) {
         for (const seed of entry.prepared.publication.seeds) await fileTransfers.archiveSeed(entry.prepared.target.file, seed)
       } else {
         for (const seed of entry.prepared.publication.seeds) await messenger.enqueueSeed(seed.channelPubkey, seed)
       }
+      if (!diagnosticReported) {
+        const results = Array.isArray(archived) ? archived : []
+        const storedCount = results.filter(Boolean).length
+        debug(entry, 'recovery-archive-result', archiveStartedAt, {
+          publicationId: entry.prepared.publication.events[0].id,
+          rowCount: rows.length,
+          uniqueCount: new Set(rows.map(row => row.recordId).filter(Boolean)).size,
+          storedCount,
+          skippedCount: results.length ? results.length - storedCount : undefined,
+          ...(rows.length === 1 && rows[0].recordId ? { seedRecordId: rows[0].recordId } : {})
+        })
+      }
     },
     changed: entry => {
       watchOfflineSends()
@@ -459,6 +490,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     await assertMessagePublished(report, event)
   }
   async function cleanupEntry (entry) {
+    if (unadmitted.has(entry)) return
     if (entry.prepared) { journal.wake(); return }
     if (!entry.cancelled && !completed(entry)) return
     if (entry.failed && entry.status === 'saved') return
@@ -478,10 +510,11 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     return work
   }
   async function saveEntry (entry) {
-    if (entry.peer === owner) { entry.status = 'pending'; emit() }
+    if (entry.peer === owner && !unadmitted.has(entry)) { entry.status = 'pending'; emit() }
     const releaseForeground = foreground.enter()
     const active = async () => {
-      if (storage.has && !await storage.has(entry.id)) { cancelled.add(entry.id); sendControllers.get(entry.id)?.abort() }
+      if (admissionFailed.has(entry)) throw new Error('CHAT_UNAVAILABLE')
+      if (!unadmitted.has(entry) && storage.has && !await storage.has(entry.id)) { cancelled.add(entry.id); sendControllers.get(entry.id)?.abort() }
       if (closed || !available || cancelled.has(entry.id) || (entry.peer !== owner && !peers.has(entry.peer))) throw new Error('CHAT_UNAVAILABLE')
     }
     const startedAt = Date.now()
@@ -499,12 +532,18 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
       }
       entry.localFailed = false
       debug(entry, 'local-stored', startedAt)
+      if (unadmitted.has(entry)) return
       if (entry.peer === owner) entry.status = 'saved'
       emit()
       if (completed(entry)) cleanupEntry(entry)
       else await journal.write(entry)
     } catch (error) {
+      if (admissionFailed.has(entry)) return
       if (cancelled.has(entry.id)) { sendControllers.get(entry.id)?.abort(); entry.cancelled = true; cleanupEntry(entry) } else if (!closed && available) {
+        if (unadmitted.has(entry)) {
+          entry.localFailed = true; entry.localRetryable = retryable(error); localErrors.set(entry, error)
+          return
+        }
         entry.localFailed = true; entry.localRetryable = retryable(error)
         if (entry.peer === owner) { entry.status = 'error'; entry.retryable = entry.localRetryable }
         await journal.write(entry).catch(reportError)
@@ -628,7 +667,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     clearSendRetry()
     sendRetryDelay = SEND_RETRY_MIN_MS
   }
-  const pendingRetryable = entry => Boolean(!cancelled.has(entry.id) && ((entry.localFailed && entry.localRetryable) || (entry.failed && (entry.retryable || entry.status === 'pending'))))
+  const pendingRetryable = entry => Boolean(!unadmitted.has(entry) && !cancelled.has(entry.id) && ((entry.localFailed && entry.localRetryable) || (entry.failed && (entry.retryable || entry.status === 'pending'))))
   function scheduleSendRetry () {
     if (closed || !available || sendRetryTimer) return
     let nextRetryAt = 0
@@ -716,14 +755,21 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
         relocal = false
         for (const entry of entries.values()) {
           if (closed || !available) break
+          if (admissionFailed.has(entry)) continue
           if ((entry.localFailed && !localRetries.has(entry.id)) || cancelled.has(entry.id) || (entry.peer !== owner && !peers.has(entry.peer))) continue
-          if (localComplete(entry) && !entry.localFailed) { if (entry.status === 'saved') cleanupEntry(entry); continue }
-          await saveEntry(entry)
+          if (localComplete(entry) && !entry.localFailed) {
+            if (!unadmitted.has(entry) && entry.peer === owner && entry.status !== 'saved') await saveEntry(entry)
+            else if (entry.status === 'saved') cleanupEntry(entry)
+            continue
+          }
+          const work = saveEntry(entry)
+          localWorks.set(entry, work)
+          try { await work } finally { localWorks.delete(entry) }
           pumpRemote()
         }
       } while (relocal)
     } finally {
-      if ([...entries.values()].some(entry => (entry.localFailed && entry.localRetryable) || (entry.status === 'saved' && entry.failed && entry.retryable))) scheduleSendRetry()
+      if ([...entries.values()].some(entry => !unadmitted.has(entry) && ((entry.localFailed && entry.localRetryable) || (entry.status === 'saved' && entry.failed && entry.retryable)))) scheduleSendRetry()
       localSaving = false; for (const resolve of localIdleWaiters) resolve(); localIdleWaiters.clear(); settled()
     }
   }
@@ -736,7 +782,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
         repump = false
         for (const entry of entries.values()) {
           if (closed || !available) break
-          if (entry.prepared?.accepted) { journal.wake(); continue }
+          if (unadmitted.has(entry) || entry.prepared?.accepted) { if (entry.prepared?.accepted) journal.wake(); continue }
           if (entry.peer === owner || entry.status === 'saved' || !peers.has(entry.peer) || entry.failed || cancelled.has(entry.id)) continue
           if (!preparation.get(entry.peer) && !preparation.error(entry.peer)) {
             preparation.request(entry.peer).catch(() => {})
@@ -754,6 +800,12 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
     }
   }
   async function pump () { await Promise.all([pumpLocal(), pumpRemote()]) }
+  async function discardUnadmitted (entry) {
+    admissionFailed.add(entry)
+    unadmitted.delete(entry)
+    if (entries.get(entry.id) === entry) entries.delete(entry.id)
+    await localWorks.get(entry)
+  }
   const intentId = file => getEventHash({ kind: 0, pubkey: owner, created_at: 0, tags: [], content: `${file.peer}:${file.root}` })
   async function download (file, options = {}) {
     if (file.peer === owner) return file
@@ -879,7 +931,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
       journal.setAvailable(true)
       pumpLocal()
       for (const entry of entries.values()) {
-        if (entry.localRetryable) localRetries.add(entry.id)
+        if (!unadmitted.has(entry) && entry.localRetryable) localRetries.add(entry.id)
         if (entry.failed && (entry.retryable || entry.status === 'pending')) { entry.failed = false; attemptStarts.set(entry.id, performance.now()) }
       }
       if (storageBlocked) inboxRecovery.start({ immediate: true })
@@ -901,17 +953,45 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
         const files = events.filter(value => value.kind === 1063).map(value => ({ ...decodeFileMetadata(value), optional: !requiredFiles.includes(getEventHash(value)) })).filter(file => file.service === 'irfs').flatMap(file => [file, ...(file.thumbnail?.root ? [{ ...file.thumbnail, optional: file.optional }] : [])])
         const entry = { id, peer, event: { ...main, id }, events, files, index: 0, fileIndex: 0, chunkIndex: 0, localSaved: context.map(() => true).concat(false), status: 'pending', deliveryConfirmed: false, deletion }
         const startedAt = Date.now()
+        const earlyLocal = peer === owner
+        if (earlyLocal) {
+          queuedAt.set(entry, Date.now())
+          unadmitted.add(entry)
+          entries.set(id, entry)
+          pumpLocal()
+        }
         debug(entry, 'outbox-start')
-        await storage.put(entry)
+        try {
+          await storage.put(entry)
+        } catch (error) {
+          if (earlyLocal) await discardUnadmitted(entry)
+          throw error
+        }
         debug(entry, 'outbox-stored', startedAt)
         queuedAt.set(entry, Date.now())
         if (attemptStartedAt !== undefined) attemptStarts.set(id, attemptStartedAt)
-        if (cancelled.has(id)) { await storage.remove(id); throw new Error('CHAT_UNAVAILABLE') }
+        if (cancelled.has(id)) {
+          if (earlyLocal) await discardUnadmitted(entry)
+          await storage.remove(id)
+          throw new Error('CHAT_UNAVAILABLE')
+        }
         // An accepted durable intent survives shutdown/locking for replay. Only
         // explicit cancellation removes it; no late callback or work is admitted.
-        if (closed) return id
-        entries.set(id, entry)
+        if (closed) {
+          if (earlyLocal) await discardUnadmitted(entry)
+          return id
+        }
+        if (earlyLocal) unadmitted.delete(entry)
+        else entries.set(id, entry)
         if (peer !== owner && available && lifecycle === urgencyLifecycle) networkUrgency.set(id, urgencyUntil)
+        if (earlyLocal && entry.localFailed) {
+          entry.status = 'error'; entry.retryable = entry.localRetryable
+          const error = localErrors.get(entry)
+          if (error) {
+            reportError(error)
+            if (!entry.deletion) onSendError(error, { id: entry.id, peer: entry.peer })
+          }
+        }
         watchOfflineSends(); emit(); pump()
         if (available && peer !== owner) preparePeer(peer).catch(() => {})
         return id
@@ -921,7 +1001,7 @@ export function createPrivateMessageSession ({ owner, signer, eventStore, messag
       if (storageBlocked) inboxRecovery.start({ immediate: true })
       await ready()
       const entry = entries.get(id)
-      if (!entry) return
+      if (!entry || unadmitted.has(entry)) return
       journal.retry()
       if (entry.deliveryConfirmed) {
         // This action belongs to local recovery, never a fresh network attempt.