nappup 1.5.2 → 1.5.4

package/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/44billion/nappup.git"
   },
   "license": "MIT",
-  "version": "1.5.2",
+  "version": "1.5.4",
   "description": "Nostr App Uploader",
   "type": "module",
   "scripts": {

package/src/helpers/event.js

@@ -0,0 +1,26 @@
+export function stringifyEvent (event) {
+  event = { ...event }
+
+  if (typeof event.content === 'string' && event.content.length > 70) {
+    event.content = `${event.content.slice(0, 70)}...(${event.content.length})`
+  }
+
+  if (typeof event.sig === 'string' && event.sig.length > 3) {
+    event.sig = `${event.sig.slice(0, 3)}...(${event.sig.length})`
+  }
+
+  if (Array.isArray(event.tags)) {
+    const totalTagsCount = event.tags.length
+    event.tags = event.tags.slice(0, 5).map(tag =>
+      Array.isArray(tag)
+        ? tag.map(val => typeof val === 'string' && val.length > 64 ? `${val.slice(0, 64)}...(${val.length})` : val)
+        : tag
+    )
+
+    if (totalTagsCount > 5) {
+      event.tags.push(`... and ${totalTagsCount - 5} more tags`)
+    }
+  }
+
+  return JSON.stringify(event, null, 2)
+}

package/src/index.js

@@ -6,6 +6,7 @@ import NostrSigner from '#services/nostr-signer.js'
 import { streamToChunks, streamToText } from '#helpers/stream.js'
 import { isNostrAppDTagSafe, deriveNostrAppDTag } from '#helpers/app.js'
 import { extractHtmlMetadata, findFavicon, findIndexFile } from '#helpers/app-metadata.js'
+import { stringifyEvent } from '#helpers/event.js'
 import { NAPP_CATEGORIES } from '#config/napp-categories.js'
 
 export default async function (...args) {
@@ -165,13 +166,19 @@ export async function toApp (fileList, nostrSigner, { log = () => {}, dTag, dTag
 
 async function uploadBinaryDataChunks ({ nmmr, signer, filename, chunkLength, log, pause = 0, mimeType, shouldReupload = false }) {
   const writeRelays = (await signer.getRelays()).write
+  const relays = [...new Set([...writeRelays, ...nappRelays])]
   let chunkIndex = 0
   for await (const chunk of nmmr.getChunks()) {
     const dTag = chunk.x
     const currentCtag = `${chunk.rootX}:${chunk.index}`
-    const { otherCtags, hasCurrentCtag } = await getPreviousCtags(dTag, currentCtag, writeRelays, signer)
+    const { otherCtags, hasCurrentCtag, foundEvent, missingRelays } = await getPreviousCtags(dTag, currentCtag, relays, signer)
     if (!shouldReupload && hasCurrentCtag) {
-      log(`${filename}: Skipping chunk ${++chunkIndex} of ${chunkLength} (already uploaded)`)
+      if (missingRelays.length === 0) {
+        log(`${filename}: Skipping chunk ${++chunkIndex} of ${chunkLength} (already uploaded)`)
+        continue
+      }
+      log(`${filename}: Re-uploading chunk ${++chunkIndex} of ${chunkLength} to ${missingRelays.length} missing relays (out of ${relays.length})`)
+      ;({ pause } = (await throttledSendEvent(foundEvent, missingRelays, { pause, log, trailingPause: true, minSuccessfulRelays: 0 })))
       continue
     }
     const binaryDataChunk = {
@@ -188,7 +195,6 @@ async function uploadBinaryDataChunks ({ nmmr, signer, filename, chunkLength, lo
     }
 
     const event = await signer.signEvent(binaryDataChunk)
-    const relays = [...new Set([...writeRelays, ...nappRelays])]
     const fallbackRelayCount = relays.length - writeRelays.length
     log(`${filename}: Uploading file part ${++chunkIndex} of ${chunkLength} to ${writeRelays.length} relays${fallbackRelayCount > 0 ? ` (+${fallbackRelayCount} fallback)` : ''}`)
     ;({ pause } = (await throttledSendEvent(event, relays, { pause, log, trailingPause: true })))
@@ -217,7 +223,10 @@ async function throttledSendEvent (event, relays, {
       else r[1].push(v)
       return r
     }, [[], []])
-  log(`${unretryableErrors.length} Unretryable errors\n: ${unretryableErrors.map(v => `${v.relay}: ${v.reason.message}`).join('; ')}`)
+  if (unretryableErrors.length > 0) {
+    log(`${unretryableErrors.length} unretryable errors:\n${unretryableErrors.map(v => `${v.relay}: ${v.reason.message}`).join('; ')}`)
+    console.log('Erroed event:', stringifyEvent(event))
+  }
   const unretryableErrorsLength = errors.length - rateLimitErrors.length
   const maybeSuccessfulRelays = relays.length - unretryableErrorsLength
   const hasReachedMaxRetries = retries > maxRetries
@@ -241,20 +250,24 @@ async function throttledSendEvent (event, relays, {
   })
 }
 
-async function getPreviousCtags (dTagValue, currentCtagValue, writeRelays, signer) {
+async function getPreviousCtags (dTagValue, currentCtagValue, relays, signer) {
+  const targetRelays = [...new Set([...relays, ...nappRelays])]
   const storedEvents = (await nostrRelays.getEvents({
     kinds: [34600],
     authors: [await signer.getPublicKey()],
     '#d': [dTagValue],
     limit: 1
-  }, [...new Set([...writeRelays, ...nappRelays])])).result
+  }, targetRelays)).result
 
   let hasCurrentCtag = false
   const hasEvent = storedEvents.length > 0
   if (!hasEvent) return { otherCtags: [], hasEvent, hasCurrentCtag }
 
   const cTagValues = { [currentCtagValue]: true }
-  const prevTags = storedEvents.sort((a, b) => b.created_at - a.created_at)[0].tags
+  storedEvents.sort((a, b) => b.created_at - a.created_at)
+  const bestEvent = storedEvents[0]
+  const prevTags = bestEvent.tags
+
   if (!Array.isArray(prevTags)) return { otherCtags: [], hasEvent, hasCurrentCtag }
 
   hasCurrentCtag = prevTags.some(tag =>
@@ -277,7 +290,11 @@ async function getPreviousCtags (dTagValue, currentCtagValue, writeRelays, signe
       return isCTag && isntDuplicate
     })
 
-  return { otherCtags, hasEvent, hasCurrentCtag }
+  const matchingEvents = storedEvents.filter(e => e.id === bestEvent.id)
+  const coveredRelays = new Set(matchingEvents.map(e => e.meta?.relay).filter(Boolean))
+  const missingRelays = targetRelays.filter(r => !coveredRelays.has(r))
+
+  return { otherCtags, hasEvent, hasCurrentCtag, foundEvent: bestEvent, missingRelays }
 }
 
 async function uploadBundle ({ dTag, channel, fileMetadata, signer, pause = 0, shouldReupload = false, log = () => {} }) {
@@ -299,7 +316,8 @@ async function uploadBundle ({ dTag, channel, fileMetadata, signer, pause = 0, s
     const events = (await nostrRelays.getEvents({
       kinds: [kind],
       authors: [await signer.getPublicKey()],
-      '#d': [dTag]
+      '#d': [dTag],
+      limit: 1
     }, writeRelays)).result
 
     if (events.length > 0) {
@@ -311,21 +329,32 @@ async function uploadBundle ({ dTag, channel, fileMetadata, signer, pause = 0, s
       })
 
       const mostRecentEvent = events[0]
-      const recentFileTags = mostRecentEvent.tags.filter(t => t[0] === 'file')
+      const recentFileTags = mostRecentEvent.tags
+        .filter(t => t[0] === 'file' && t[2] !== '.well-known/napp.json')
+        .sort((a, b) => (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0))
 
-      const isSame = fileTags.length === recentFileTags.length && fileTags.every((t, i) => {
+      const currentFileTags = fileTags
+        .filter(t => t[2] !== '.well-known/napp.json')
+        .sort((a, b) => (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0))
+
+      const isSame = currentFileTags.length === recentFileTags.length && currentFileTags.every((t, i) => {
         const rt = recentFileTags[i]
         return rt.length >= 4 && rt[1] === t[1] && rt[2] === t[2] && rt[3] === t[3]
       })
 
       if (isSame) {
-        log(`Bundle based on ${fileTags.length} files is up to date (id: ${mostRecentEvent.id} - created_at: ${new Date(mostRecentEvent.created_at * 1000).toISOString()})`)
-        if (events.length === writeRelays.length && events.every(e => e.id === mostRecentEvent.id)) return mostRecentEvent
+        log(`Bundle based on ${fileTags.length} files is up-to-date (id: ${mostRecentEvent.id} - created_at: ${new Date(mostRecentEvent.created_at * 1000).toISOString()})`)
+
+        const matchingEvents = events.filter(e => e.id === mostRecentEvent.id)
+        const coveredRelays = new Set(matchingEvents.map(e => e.meta?.relay).filter(Boolean))
+        const missingRelays = writeRelays.filter(r => !coveredRelays.has(r))
+
+        if (missingRelays.length === 0) return mostRecentEvent
 
         // nostrRelays.getEvents currently doesn't tell us which event came from which relay,
         // so we re-upload to all relays to ensure consistency
-        log(`Re-uploading existing bundle event to all ${writeRelays.length} relays`)
-        await throttledSendEvent(mostRecentEvent, writeRelays, { pause, trailingPause: true, log })
+        log(`Re-uploading existing bundle event to ${missingRelays.length} missing relays (out of ${writeRelays.length})`)
+        await throttledSendEvent(mostRecentEvent, missingRelays, { pause, trailingPause: true, log, minSuccessfulRelays: 0 })
         return mostRecentEvent
       }
     }
@@ -369,12 +398,14 @@ async function maybeUploadStall ({
   const hasMetadata = Boolean(trimmedName) || Boolean(trimmedSummary) || Boolean(iconRootHash) ||
     Boolean(self) || (countries && countries.length > 0) || (categories && categories.length > 0) || (hashtags && hashtags.length > 0)
 
-  const previous = await getPreviousStall(dTag, writeRelays, signer, channel)
+  const relays = [...new Set([...writeRelays, ...nappRelays])]
+
+  const previousResult = await getPreviousStall(dTag, relays, signer, channel)
+  const previous = previousResult?.previous
   if (!previous && !hasMetadata) return { pause }
 
   const publishStall = async (event) => {
     const signedEvent = await signer.signEvent(event)
-    const relays = [...new Set([...writeRelays, ...nappRelays])]
     return await throttledSendEvent(signedEvent, relays, { pause, log, trailingPause: true })
   }
 
@@ -600,7 +631,18 @@ async function maybeUploadStall ({
     }
   }
 
-  if (!changed) return { pause }
+  if (!changed) {
+    const { storedEvents } = previousResult
+
+    const matchingEvents = storedEvents.filter(e => e.id === previous.id)
+    const coveredRelays = new Set(matchingEvents.map(e => e.meta?.relay).filter(Boolean))
+    const missingRelays = relays.filter(r => !coveredRelays.has(r))
+
+    if (missingRelays.length === 0) return { pause }
+
+    log(`Re-uploading existing stall event to ${missingRelays.length} missing relays (out of ${relays.length})`)
+    return await throttledSendEvent(previous, missingRelays, { pause, log, trailingPause: true, minSuccessfulRelays: 0 })
+  }
 
   return await publishStall({
     kind,
@@ -610,7 +652,7 @@ async function maybeUploadStall ({
   })
 }
 
-async function getPreviousStall (dTagValue, writeRelays, signer, channel) {
+async function getPreviousStall (dTagValue, relays, signer, channel) {
   const kind = {
     main: 37348,
     next: 37349,
@@ -622,8 +664,15 @@ async function getPreviousStall (dTagValue, writeRelays, signer, channel) {
     authors: [await signer.getPublicKey()],
     '#d': [dTagValue],
     limit: 1
-  }, [...new Set([...writeRelays, ...nappRelays])])).result
+  }, relays)).result
 
   if (storedEvents.length === 0) return null
-  return storedEvents.sort((a, b) => b.created_at - a.created_at)[0]
+
+  storedEvents.sort((a, b) => b.created_at - a.created_at)
+
+  return {
+    previous: storedEvents[0],
+    storedEvents,
+    targetRelayCount: relays.length
+  }
 }

package/src/services/nostr-relays.js

@@ -80,6 +80,7 @@ export class NostrRelays {
         const relay = await this.#getRelay(url)
         sub = relay.subscribe([filter], {
           onevent: (event) => {
+            event.meta = { relay: url }
             events.push(event)
           },
           onclose: err => {
@@ -115,6 +116,9 @@ export class NostrRelays {
 
   // Send an event to a list of relays
   async sendEvent (event, relays, timeout = 3000) {
+    const eventToSend = event.meta ? { ...event } : event
+    if (eventToSend.meta) delete eventToSend.meta
+
     const promises = relays.map(async (url) => {
       let timer
       const p = Promise.withResolvers()
@@ -124,7 +128,7 @@ export class NostrRelays {
         }, timeout))
 
         const relay = await this.#getRelay(url)
-        await relay.publish(event)
+        await relay.publish(eventToSend)
         p.resolve()
       } catch (err) {
         if (err.message?.startsWith('duplicate:')) return p.resolve()