libp2r2p 0.6.0 → 0.8.0

package/private-channel/services/received-chunks.js

@@ -5,7 +5,7 @@ export const DEFAULT_RECEIVED_CHUNK_TTL_MS = 60 * 60 * 1000 // 1 hour
 export const DEFAULT_RECEIVED_CHUNK_MAX_BYTES = 16 * 1024 * 1024 // 16 MiB
 
 const DEFAULT_PREFIX = 'libp2r2p:private-channel:received'
-const DATABASE_VERSION = 1
+const DATABASE_VERSION = 2
 const GROUPS_STORE = 'groups'
 const CHUNKS_STORE = 'chunks'
 const STATE_STORE = 'state'
@@ -16,7 +16,7 @@ const decoder = new TextDecoder()
 /*
 IndexedDB schema, scoped per received-chunk prefix:
 
-database `${prefix}:idb`, version 1
+database `${prefix}:idb`, version 2
 
 groups, keyPath "groupKey"
   groupKey                  `${channelPubkey}:${routerPubkey}`
@@ -28,10 +28,12 @@ groups, keyPath "groupKey"
   receiverPubkeys           deduplicated intended receivers
   byteSize                  logical bytes charged to this group
   createdAt/updatedAt        lifecycle timestamps in milliseconds
+  ttlMs/expiresAt            group-specific retention and absolute expiry
   drainToken/drainUntil      optional drain lease owner and expiry
 
 groups indexes
   byUpdatedAt  updatedAt
+  byExpiresAt  expiresAt
 
 chunks, keyPath ["groupKey", "index"]
   groupKey  owning group coordinate
@@ -80,6 +82,7 @@ function openDatabase (indexedDB, name) {
         groups = tx.objectStore(GROUPS_STORE)
       }
       if (!groups.indexNames.contains('byUpdatedAt')) groups.createIndex('byUpdatedAt', 'updatedAt')
+      if (!groups.indexNames.contains('byExpiresAt')) groups.createIndex('byExpiresAt', 'expiresAt')
 
       let chunks
       if (!db.objectStoreNames.contains(CHUNKS_STORE)) {
@@ -119,12 +122,14 @@ function normalizeReceived (received) {
   )
 }
 
-function normalizeMeta (meta) {
+function normalizeMeta (meta, fallbackTtlMs = DEFAULT_RECEIVED_CHUNK_TTL_MS) {
   if (!meta || typeof meta !== 'object') return null
   const total = Number(meta.total)
   const nextIndex = Number(meta.nextIndex)
   const rowIndex = Number(meta.rowIndex)
   if (!meta.groupKey || !Number.isSafeInteger(total) || total < 1) return null
+  const updatedAt = Number(meta.updatedAt) || Date.now()
+  const ttlMs = Number.isFinite(meta.ttlMs) && meta.ttlMs >= 0 ? meta.ttlMs : fallbackTtlMs
   return {
     groupKey: String(meta.groupKey),
     channelPubkey: String(meta.channelPubkey || ''),
@@ -139,7 +144,9 @@ function normalizeMeta (meta) {
     receiverPubkeys: uniq(meta.receiverPubkeys),
     byteSize: Math.max(0, Number(meta.byteSize) || 0),
     createdAt: Number(meta.createdAt) || Date.now(),
-    updatedAt: Number(meta.updatedAt) || Date.now(),
+    updatedAt,
+    ttlMs,
+    expiresAt: Number(meta.expiresAt) || updatedAt + ttlMs,
     drainToken: typeof meta.drainToken === 'string' ? meta.drainToken : '',
     drainUntil: Math.max(0, Number(meta.drainUntil) || 0)
   }
@@ -180,10 +187,18 @@ export function createReceivedChunkStore ({
   const configuredMaxBytes = Number.isFinite(maxBytes) && maxBytes > 0 ? maxBytes : Infinity
   let dbPromise
   let readyPromise
+  let closed = false
 
   function database () {
+    if (closed) return Promise.reject(new Error('RECEIVED_CHUNK_STORE_CLOSED'))
     dbPromise ||= openDatabase(indexedDB, `${prefix}:idb`)
-    return dbPromise
+    return dbPromise.then(db => {
+      if (closed) {
+        db.close()
+        throw new Error('RECEIVED_CHUNK_STORE_CLOSED')
+      }
+      return db
+    })
   }
 
   async function transaction (storeNames, mode, work) {
@@ -215,7 +230,7 @@ export function createReceivedChunkStore ({
   }
 
   async function deleteGroupInTransaction (tx, groupKey, usage, knownMeta) {
-    const meta = knownMeta || normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result)
+    const meta = knownMeta || normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
     const keys = (await run('getAllKeys', [globalThis.IDBKeyRange?.only?.(groupKey) ?? groupKey], CHUNKS_STORE, 'byGroup', { tx })).result
     for (const key of keys) await run('delete', [key], CHUNKS_STORE, null, { tx })
     await run('delete', [groupKey], GROUPS_STORE, null, { tx })
@@ -226,19 +241,18 @@ export function createReceivedChunkStore ({
   async function oldestGroupsInTransaction (tx, except = '') {
     const values = (await run('getAll', [], GROUPS_STORE, null, { tx })).result
     return values
-      .map(normalizeMeta)
+      .map(value => normalizeMeta(value, configuredTtlMs))
       .filter(meta => meta && meta.groupKey !== except)
       .sort((left, right) => left.updatedAt - right.updatedAt || left.groupKey.localeCompare(right.groupKey))
   }
 
   async function cleanupStaleRaw (nowMs = Date.now()) {
-    const cutoff = nowMs - configuredTtlMs
     return transaction([GROUPS_STORE, CHUNKS_STORE, STATE_STORE], 'readwrite', async tx => {
       const usage = await readUsage(tx)
       const groups = await oldestGroupsInTransaction(tx)
       let removed = 0
       for (const meta of groups) {
-        if (meta.updatedAt > cutoff && (!Number.isFinite(configuredMaxBytes) || usage.usedBytes <= configuredMaxBytes)) continue
+        if (meta.expiresAt > nowMs && (!Number.isFinite(configuredMaxBytes) || usage.usedBytes <= configuredMaxBytes)) continue
         await deleteGroupInTransaction(tx, meta.groupKey, usage, meta)
         removed++
       }
@@ -247,26 +261,26 @@ export function createReceivedChunkStore ({
     })
   }
 
-  function ready () {
-    readyPromise ||= cleanupStaleRaw()
+  function ready (nowMs = Date.now()) {
+    readyPromise ||= cleanupStaleRaw(nowMs)
     return readyPromise
   }
 
   async function cleanupStale (nowMs = Date.now()) {
-    await ready()
+    await ready(nowMs)
     return cleanupStaleRaw(nowMs)
   }
 
-  async function putOnce ({ channelPubkey, routerPubkey, index, total, contentBytes }) {
+  async function putOnce ({ channelPubkey, routerPubkey, index, total, contentBytes, ttlMs }) {
     const groupKey = groupKeyFor(channelPubkey, routerPubkey)
     const bytes = normalizeBytes(contentBytes)
     const now = Date.now()
+    const groupTtlMs = Number.isFinite(ttlMs) && ttlMs >= 0 ? ttlMs : configuredTtlMs
 
     return transaction([GROUPS_STORE, CHUNKS_STORE, STATE_STORE], 'readwrite', async tx => {
       const usage = await readUsage(tx)
-      let meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result)
-      const staleCutoff = now - configuredTtlMs
-      if (meta && meta.updatedAt <= staleCutoff) {
+      let meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
+      if (meta && meta.expiresAt <= now) {
         await deleteGroupInTransaction(tx, groupKey, usage, meta)
         meta = null
       }
@@ -289,8 +303,10 @@ export function createReceivedChunkStore ({
           receiverPubkeys: [],
           byteSize: 0,
           createdAt: now,
-          updatedAt: now
-        })
+          updatedAt: now,
+          ttlMs: groupTtlMs,
+          expiresAt: now + groupTtlMs
+        }, configuredTtlMs)
       }
 
       const existing = (await run('get', [[groupKey, index]], CHUNKS_STORE, null, { tx })).result
@@ -304,11 +320,11 @@ export function createReceivedChunkStore ({
         const requiredBytes = bytes.byteLength
         const candidates = await oldestGroupsInTransaction(tx, groupKey)
         for (const candidate of candidates) {
-          if (candidate.updatedAt > staleCutoff) continue
+          if (candidate.expiresAt > now) continue
           await deleteGroupInTransaction(tx, candidate.groupKey, usage, candidate)
         }
         for (const candidate of candidates) {
-          if (candidate.updatedAt <= staleCutoff) continue
+          if (candidate.expiresAt <= now) continue
           if (!Number.isFinite(configuredMaxBytes) || usage.usedBytes + requiredBytes <= configuredMaxBytes) break
           await deleteGroupInTransaction(tx, candidate.groupKey, usage, candidate)
         }
@@ -327,6 +343,7 @@ export function createReceivedChunkStore ({
 
       meta.total = total
       meta.updatedAt = now
+      meta.expiresAt = now + meta.ttlMs
       await run('put', [meta], GROUPS_STORE, null, { tx })
       await writeUsage(tx, usage)
       return { tooLarge: false, meta }
@@ -344,7 +361,7 @@ export function createReceivedChunkStore ({
     })
   }
 
-  async function put ({ channelPubkey, routerPubkey, index, total, contentBytes }) {
+  async function put ({ channelPubkey, routerPubkey, index, total, contentBytes, ttlMs }) {
     if (!channelPubkey || !routerPubkey) throw new Error('RECEIVED_CHUNK_GROUP_REQUIRED')
     if (!Number.isSafeInteger(index) || !Number.isSafeInteger(total) || index < 0 || total < 1 || index >= total) {
       throw new Error('INVALID_RECEIVED_CHUNK_INDEX')
@@ -353,7 +370,7 @@ export function createReceivedChunkStore ({
     await ready()
     while (true) {
       try {
-        const result = await putOnce({ channelPubkey, routerPubkey, index, total, contentBytes: bytes })
+        const result = await putOnce({ channelPubkey, routerPubkey, index, total, contentBytes: bytes, ttlMs })
         if (result.tooLarge) throw new Error('RECEIVED_CHUNK_GROUP_TOO_LARGE')
         return result.meta
       } catch (err) {
@@ -365,12 +382,12 @@ export function createReceivedChunkStore ({
   async function readMeta (groupKey) {
     await ready()
     return transaction([GROUPS_STORE], 'readonly', async tx => {
-      return normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result)
+      return normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
     })
   }
 
   async function status (metaOrGroupKey) {
-    const meta = typeof metaOrGroupKey === 'string' ? await readMeta(metaOrGroupKey) : normalizeMeta(metaOrGroupKey)
+    const meta = typeof metaOrGroupKey === 'string' ? await readMeta(metaOrGroupKey) : normalizeMeta(metaOrGroupKey, configuredTtlMs)
     if (!meta) return { received: 0, missing: [] }
     const missing = []
     let received = 0
@@ -393,13 +410,15 @@ export function createReceivedChunkStore ({
     while (true) {
       const now = Date.now()
       const result = await transaction([GROUPS_STORE], 'readwrite', async tx => {
-        const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result)
+        const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
         if (!meta) return { meta: null, waitMs: 0 }
         if (meta.drainToken && meta.drainToken !== token && meta.drainUntil > now) {
           return { meta: null, waitMs: Math.min(DRAIN_LEASE_MS, meta.drainUntil - now) }
         }
         meta.drainToken = token
         meta.drainUntil = now + DRAIN_LEASE_MS
+        meta.updatedAt = now
+        meta.expiresAt = now + meta.ttlMs
         await run('put', [meta], GROUPS_STORE, null, { tx })
         return { meta, waitMs: 0 }
       })
@@ -410,7 +429,7 @@ export function createReceivedChunkStore ({
 
   async function readDrainChunk (groupKey, token) {
     return transaction([GROUPS_STORE, CHUNKS_STORE], 'readonly', async tx => {
-      const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result)
+      const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
       if (!meta || meta.drainToken !== token) return { meta: null, bytes: null }
       if (meta.nextIndex >= meta.total) return { meta, bytes: null }
       const chunk = (await run('get', [[groupKey, meta.nextIndex]], CHUNKS_STORE, null, { tx })).result
@@ -420,7 +439,7 @@ export function createReceivedChunkStore ({
 
   async function commitDrainMeta (nextMeta, token) {
     return transaction([GROUPS_STORE], 'readwrite', async tx => {
-      const current = normalizeMeta((await run('get', [nextMeta.groupKey], GROUPS_STORE, null, { tx })).result)
+      const current = normalizeMeta((await run('get', [nextMeta.groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
       if (!current || current.drainToken !== token) return null
       current.nextIndex = nextMeta.nextIndex
       current.rowIndex = nextMeta.rowIndex
@@ -428,6 +447,7 @@ export function createReceivedChunkStore ({
       current.payloadCiphertext = nextMeta.payloadCiphertext
       current.receiverPubkeys = uniq(nextMeta.receiverPubkeys)
       current.updatedAt = nextMeta.updatedAt
+      current.expiresAt = current.updatedAt + current.ttlMs
       current.drainUntil = Date.now() + DRAIN_LEASE_MS
       await run('put', [current], GROUPS_STORE, null, { tx })
       return current
@@ -436,7 +456,7 @@ export function createReceivedChunkStore ({
 
   async function releaseDrain (groupKey, token) {
     return transaction([GROUPS_STORE], 'readwrite', async tx => {
-      const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result)
+      const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
       if (!meta || meta.drainToken !== token) return false
       meta.drainToken = ''
       meta.drainUntil = 0
@@ -504,7 +524,7 @@ export function createReceivedChunkStore ({
   async function readChunkBytes (groupKey) {
     await ready()
     return transaction([GROUPS_STORE, CHUNKS_STORE], 'readonly', async tx => {
-      const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result)
+      const meta = normalizeMeta((await run('get', [groupKey], GROUPS_STORE, null, { tx })).result, configuredTtlMs)
       if (!meta) return { meta: null, chunks: [] }
       const chunks = []
       for (let index = 0; index < meta.total; index++) {
@@ -551,6 +571,12 @@ export function createReceivedChunkStore ({
     })
   }
 
+  function close () {
+    if (closed) return
+    closed = true
+    dbPromise?.then(db => db.close()).catch(() => {})
+  }
+
   return {
     cleanupStale,
     drainAvailable,
@@ -559,7 +585,22 @@ export function createReceivedChunkStore ({
     readChunkContents,
     readEnvelopeBundleContent,
     readEnvelopeBundleText,
+    ready,
     removeGroup,
-    status
+    status,
+    close
+  }
+}
+
+export async function cleanupReceivedChunkStorage ({
+  indexedDB = globalThis.indexedDB,
+  prefix = DEFAULT_PREFIX,
+  now = Date.now()
+} = {}) {
+  const store = createReceivedChunkStore({ prefix, indexedDB })
+  try {
+    return await store.cleanupStale(now)
+  } finally {
+    store.close()
   }
 }

package/private-message/index.js

@@ -12,6 +12,7 @@ const HEX_PUBKEY = /^[0-9a-f]{64}$/i
 
 const watchesByChannel = new Map()
 const subsByRelay = new Map()
+let nextWatchRevision = 1
 
 function nowSeconds () {
   return Math.floor(Date.now() / 1000)
@@ -254,6 +255,27 @@ function maxWatchNumber (channels, field) {
   return values.length ? Math.max(...values) : undefined
 }
 
+function watchNumbersForChannels (channels, field) {
+  const out = {}
+  for (const channel of channels) {
+    const value = watchesByChannel.get(channel)?.[field]
+    if (Number.isFinite(value)) out[channel] = value
+  }
+  return out
+}
+
+function watchRevisionsForChannels (channels) {
+  return Object.fromEntries(channels.map(channel => [channel, watchesByChannel.get(channel)?.revision || 0]))
+}
+
+function subscriptionMatches (current, channels) {
+  if (!current || !setEquals(current.channels, channels)) return false
+  for (const channel of channels) {
+    if (current.revisions?.[channel] !== watchesByChannel.get(channel)?.revision) return false
+  }
+  return true
+}
+
 function firstWatchValue (channels, field) {
   for (const channel of channels) {
     const value = watchesByChannel.get(channel)?.[field]
@@ -263,25 +285,34 @@ function firstWatchValue (channels, field) {
 }
 
 function closeSubscription (sub, gracefulClose) {
-  if (gracefulClose) setTimeout(() => sub.close(), RESUBSCRIBE_GRACE_MS)
-  else sub.close()
+  if (gracefulClose) {
+    setTimeout(() => Promise.resolve().then(() => sub.close()).catch(() => {}), RESUBSCRIBE_GRACE_MS)
+    return null
+  }
+  try {
+    return Promise.resolve(sub.close())
+  } catch (err) {
+    return Promise.reject(err)
+  }
 }
 
 function rebuildSubscriptions ({ _subscribe = privateChannel.subscribe, gracefulClose = true } = {}) {
   const desired = desiredRelayState()
+  const closing = []
 
   for (const [relay, current] of subsByRelay) {
     const nextChannels = desired.get(relay)
-    if (nextChannels && setEquals(current.channels, nextChannels)) continue
+    if (nextChannels && subscriptionMatches(current, nextChannels)) continue
     if (!nextChannels) {
-      closeSubscription(current.sub, gracefulClose)
+      const close = closeSubscription(current.sub, gracefulClose)
+      if (close) closing.push(close)
       subsByRelay.delete(relay)
     }
   }
 
   for (const [relay, channels] of desired) {
     const current = subsByRelay.get(relay)
-    if (current && setEquals(current.channels, channels)) continue
+    if (subscriptionMatches(current, channels)) continue
 
     const channelList = [...channels]
     const firstWatch = watchesByChannel.get(channelList[0])
@@ -300,6 +331,7 @@ function rebuildSubscriptions ({ _subscribe = privateChannel.subscribe, graceful
       mode: firstWatch.mode,
       modeByPubkey: modesForChannels(channelList),
       receivedChunkTtlMs: maxWatchNumber(channelList, 'receivedChunkTtlMs'),
+      receivedChunkTtlMsByPubkey: watchNumbersForChannels(channelList, 'receivedChunkTtlMs'),
       receivedChunkMaxBytes: maxWatchNumber(channelList, 'receivedChunkMaxBytes'),
       receivedChunkIndexedDB: firstWatchValue(channelList, 'receivedChunkIndexedDB'),
       ignoredGroupTtlMs: maxWatchNumber(channelList, 'ignoredGroupTtlMs'),
@@ -321,9 +353,17 @@ function rebuildSubscriptions ({ _subscribe = privateChannel.subscribe, graceful
       onError: err => firstWatch.callbacks.onError?.(err)
     })
 
-    subsByRelay.set(relay, { channels: new Set(channels), sub })
-    if (current) closeSubscription(current.sub, gracefulClose)
+    subsByRelay.set(relay, {
+      channels: new Set(channels),
+      revisions: watchRevisionsForChannels(channelList),
+      sub
+    })
+    if (current) {
+      const close = closeSubscription(current.sub, gracefulClose)
+      if (close) closing.push(close)
+    }
   }
+  return Promise.allSettled(closing)
 }
 
 export async function watch ({
@@ -379,19 +419,23 @@ export async function watch ({
       since
     }
     const current = watchesByChannel.get(channel)
-    if (
+    const sameSettings = Boolean(
       current &&
-      setEquals(new Set(current.relays), new Set(next.relays)) &&
+      current.receiverSigner === next.receiverSigner &&
+      current.iykcSigner === next.iykcSigner &&
       current.privateChannelSigner === next.privateChannelSigner &&
       current.privateChannelReaderSigner === next.privateChannelReaderSigner &&
       current.privateChannelReaderPubkey === next.privateChannelReaderPubkey &&
+      current.receiverPubkey === next.receiverPubkey &&
       current.mode === next.mode &&
       current.receivedChunkTtlMs === next.receivedChunkTtlMs &&
       current.receivedChunkMaxBytes === next.receivedChunkMaxBytes &&
       current.receivedChunkIndexedDB === next.receivedChunkIndexedDB &&
       current.ignoredGroupTtlMs === next.ignoredGroupTtlMs &&
       current.ignoredGroupMaxEntries === next.ignoredGroupMaxEntries
-    ) {
+    )
+    next.revision = sameSettings ? current.revision : nextWatchRevision++
+    if (sameSettings && setEquals(new Set(current.relays), new Set(next.relays))) {
       current.callbacks = callbacks
       continue
     }
@@ -399,18 +443,19 @@ export async function watch ({
     changed = true
   }
 
-  if (changed) rebuildSubscriptions({ _subscribe })
+  if (changed) await rebuildSubscriptions({ _subscribe })
   return () => unwatch(channelList)
 }
 
 export function unwatch (channels) {
   const channelList = channels ? uniq(Array.isArray(channels) ? channels : [channels]) : [...watchesByChannel.keys()]
   for (const channel of channelList) watchesByChannel.delete(channel)
-  rebuildSubscriptions({ gracefulClose: false })
+  return rebuildSubscriptions({ gracefulClose: false })
 }
 
 export function clearChannelState (channelPubkey) {
-  if (watchesByChannel.has(channelPubkey)) unwatch(channelPubkey)
+  if (watchesByChannel.has(channelPubkey)) return unwatch(channelPubkey)
+  return Promise.resolve([])
 }
 
 async function sendPrivateMessage ({