nappup 1.0.9 → 1.0.12

package/src/index.js

@@ -1,10 +1,11 @@
 import NMMR from 'nmmr'
 import { appEncode } from '#helpers/nip19.js'
-import Base122Encoder from '#services/base122-encoder.js'
+import Base93Encoder from '#services/base93-encoder.js'
 import nostrRelays from '#services/nostr-relays.js'
 import NostrSigner from '#services/nostr-signer.js'
-import { streamToChunks } from '#helpers/stream.js'
+import { streamToChunks, streamToText } from '#helpers/stream.js'
 import { isNostrAppDTagSafe, deriveNostrAppDTag } from '#helpers/app.js'
+import { extractHtmlMetadata, findFavicon, findIndexFile } from '#helpers/app-metadata.js'
 
 export default async function (...args) {
   try {
@@ -14,47 +15,86 @@ export default async function (...args) {
   }
 }
 
-export async function toApp (fileList, nostrSigner, { log = () => {}, dTag, channel = 'main' } = {}) {
+export async function toApp (fileList, nostrSigner, { log = () => {}, dTag, channel = 'main', shouldReupload = false } = {}) {
   if (!nostrSigner && typeof window !== 'undefined') nostrSigner = window.nostr
   if (!nostrSigner) throw new Error('No Nostr signer found')
   if (typeof window !== 'undefined' && nostrSigner === window.nostr) {
     nostrSigner.getRelays = NostrSigner.prototype.getRelays
   }
+  const writeRelays = (await nostrSigner.getRelays()).write
+  log(`Found ${writeRelays.length} outbox relays for pubkey ${nostrSigner.getPublicKey()}:\n${writeRelays.join(', ')}`)
+  if (writeRelays.length === 0) throw new Error('No outbox relays found')
 
   if (typeof dTag === 'string') {
     if (!isNostrAppDTagSafe(dTag)) throw new Error('dTag should be [A-Za-z0-9] with length ranging from 1 to 19')
   } else {
     dTag = fileList[0].webkitRelativePath.split('/')[0].trim()
-    if (!isNostrAppDTagSafe(dTag)) dTag = deriveNostrAppDTag(dTag || Math.random().toString(36))
+    if (!isNostrAppDTagSafe(dTag)) dTag = await deriveNostrAppDTag(dTag || Math.random().toString(36))
   }
   let nmmr
   const fileMetadata = []
 
+  const indexFile = findIndexFile(fileList)
+  let stallName, stallSummary
+  if (indexFile) {
+    try {
+      const htmlContent = await streamToText(indexFile.stream())
+      const { name, description } = extractHtmlMetadata(htmlContent)
+      stallName = name
+      stallSummary = description
+    } catch (err) {
+      log('Error extracting HTML metadata:', err)
+    }
+  }
+  const faviconFile = findFavicon(fileList)
+  let iconMetadata
+
   log(`Processing ${fileList.length} files`)
+  let pause = 1000
   for (const file of fileList) {
     nmmr = new NMMR()
     const stream = file.stream()
 
     let chunkLength = 0
-    for await (const chunk of streamToChunks(stream, 54600)) {
+    for await (const chunk of streamToChunks(stream, 51000)) {
       chunkLength++
-      nmmr.append(chunk)
+      await nmmr.append(chunk)
     }
     if (chunkLength) {
       // remove root dir
       const filename = file.webkitRelativePath.split('/').slice(1).join('/')
       log(`Uploading ${chunkLength} file parts of ${filename}`)
-      await uploadBinaryDataChunks(nmmr, nostrSigner, { mimeType: file.type || 'application/octet-stream' })
+      ;({ pause } = (await uploadBinaryDataChunks({ nmmr, signer: nostrSigner, filename, chunkLength, log, pause, mimeType: file.type || 'application/octet-stream', shouldReupload })))
       fileMetadata.push({
         rootHash: nmmr.getRoot(),
         filename,
         mimeType: file.type || 'application/octet-stream'
       })
+
+      if (faviconFile && file === faviconFile) {
+        iconMetadata = {
+          rootHash: nmmr.getRoot(),
+          mimeType: file.type || 'application/octet-stream'
+        }
+      }
     }
   }
 
+  log(`Uploading stall event for #${dTag}`)
+  ;({ pause } = (await maybeUploadStall({
+    dTag,
+    channel,
+    name: stallName,
+    summary: stallSummary,
+    icon: iconMetadata,
+    signer: nostrSigner,
+    writeRelays,
+    log,
+    pause
+  })))
+
   log(`Uploading bundle #${dTag}`)
-  const bundle = await uploadBundle(dTag, channel, fileMetadata, nostrSigner)
+  const bundle = await uploadBundle({ dTag, channel, fileMetadata, signer: nostrSigner, pause })
 
   const appEntity = appEncode({
     dTag: bundle.tags.find(v => v[0] === 'd')[1],
@@ -65,43 +105,105 @@ export async function toApp (fileList, nostrSigner, { log = () => {}, dTag, chan
   log(`Visit at https://44billion.net/${appEntity}`)
 }
 
-async function uploadBinaryDataChunks (nmmr, signer, { mimeType } = {}) {
+async function uploadBinaryDataChunks ({ nmmr, signer, filename, chunkLength, log, pause = 0, mimeType, shouldReupload = false }) {
   const writeRelays = (await signer.getRelays()).write
+  let chunkIndex = 0
   for await (const chunk of nmmr.getChunks()) {
     const dTag = chunk.x
     const currentCtag = `${chunk.rootX}:${chunk.index}`
-    const prevCTags = await getPreviousCtags(dTag, currentCtag, writeRelays, signer)
+    const { otherCtags, hasCurrentCtag } = await getPreviousCtags(dTag, currentCtag, writeRelays, signer)
+    if (!shouldReupload && hasCurrentCtag) {
+      log(`${filename}: Skipping chunk ${++chunkIndex} of ${chunkLength} (already uploaded)`)
+      continue
+    }
     const binaryDataChunk = {
       kind: 34600,
       tags: [
         ['d', dTag],
-        ...prevCTags,
+        ...otherCtags,
         ['c', currentCtag, chunk.length, ...chunk.proof],
         ...(mimeType ? [['m', mimeType]] : [])
       ],
-      // These chunks already have the expected size of 54600 bytes
-      content: new Base122Encoder().update(chunk.contentBytes).getEncoded(),
+      // These chunks already have the expected size of 51000 bytes
+      content: new Base93Encoder().update(chunk.contentBytes).getEncoded(),
       created_at: Math.floor(Date.now() / 1000)
     }
 
     const event = await signer.signEvent(binaryDataChunk)
-    await nostrRelays.sendEvent(event, writeRelays)
+    log(`${filename}: Uploading file part ${++chunkIndex} of ${chunkLength} to ${writeRelays.length} relays`)
+    ;({ pause } = (await throttledSendEvent(event, writeRelays, { pause, log, trailingPause: true })))
+  }
+  return { pause }
+}
+
+async function throttledSendEvent (event, relays, {
+  pause, log,
+  retries = 0, maxRetries = 10,
+  minSuccessfulRelays = 1,
+  leadingPause = false, trailingPause = false
+}) {
+  if (pause && leadingPause) await new Promise(resolve => setTimeout(resolve, pause))
+  if (retries > 0) log(`Retrying upload to ${relays.length} relays: ${relays.join(', ')}`)
+
+  const { errors } = (await nostrRelays.sendEvent(event, relays, 15000))
+  if (errors.length === 0) {
+    if (pause && trailingPause) await new Promise(resolve => setTimeout(resolve, pause))
+    return { pause }
   }
+
+  const [rateLimitErrors, unretryableErrors] =
+    errors.reduce((r, v) => {
+      if ((v.reason?.message ?? '').startsWith('rate-limited:')) r[0].push(v)
+      else r[1].push(v)
+      return r
+    }, [[], []])
+  log(`${unretryableErrors.length} Unretryable errors\n: ${unretryableErrors.map(v => `${v.relay}: ${v.reason.message}`).join('; ')}`)
+  const unretryableErrorsLength = errors.length - rateLimitErrors.length
+  const maybeSuccessfulRelays = relays.length - unretryableErrorsLength
+  const hasReachedMaxRetries = retries > maxRetries
+  if (
+    hasReachedMaxRetries ||
+    maybeSuccessfulRelays < minSuccessfulRelays
+  ) throw new Error(errors.map(v => `\n${v.relay}: ${v.reason}`).join('\n'))
+
+  if (rateLimitErrors.length === 0) {
+    if (pause && trailingPause) await new Promise(resolve => setTimeout(resolve, pause))
+    return { pause }
+  }
+
+  const erroedRelays = rateLimitErrors.map(v => v.relay)
+  log(`Rate limited by ${erroedRelays.length} relays, pausing for ${pause + 2000} ms`)
+  await new Promise(resolve => setTimeout(resolve, (pause += 2000)))
+
+  minSuccessfulRelays = Math.max(0, minSuccessfulRelays - (relays.length - erroedRelays.length))
+  return await throttledSendEvent(event, erroedRelays, {
+    pause, log, retries: ++retries, maxRetries, minSuccessfulRelays, leadingPause: false, trailingPause
+  })
 }
 
 async function getPreviousCtags (dTagValue, currentCtagValue, writeRelays, signer) {
-  const storedEvents = await nostrRelays.getEvents({
+  const storedEvents = (await nostrRelays.getEvents({
     kinds: [34600],
     authors: [await signer.getPublicKey()],
     '#d': [dTagValue],
     limit: 1
-  }, writeRelays)
-  if (storedEvents.length === 0) return []
+  }, writeRelays)).result
+
+  let hasCurrentCtag = false
+  const hasEvent = storedEvents.length > 0
+  if (!hasEvent) return { otherCtags: [], hasEvent, hasCurrentCtag }
 
   const cTagValues = { [currentCtagValue]: true }
   const prevTags = storedEvents.sort((a, b) => b.created_at - a.created_at)[0].tags
-  if (!Array.isArray(prevTags)) return []
-  return prevTags
+  if (!Array.isArray(prevTags)) return { otherCtags: [], hasEvent, hasCurrentCtag }
+
+  hasCurrentCtag = prevTags.some(tag =>
+    Array.isArray(tag) &&
+    tag[0] === 'c' &&
+    tag[1] === currentCtagValue
+  )
+
+  const otherCtags = prevTags
     .filter(v => {
       const isCTag =
         Array.isArray(v) &&
@@ -114,9 +216,11 @@ async function getPreviousCtags (dTagValue, currentCtagValue, writeRelays, signe
       cTagValues[v[1]] = true
       return isCTag && isntDuplicate
     })
+
+  return { otherCtags, hasEvent, hasCurrentCtag }
 }
 
-async function uploadBundle (dTag, channel, fileMetadata, signer) {
+async function uploadBundle ({ dTag, channel, fileMetadata, signer, pause = 0 }) {
   const kind = {
     main: 37448, // stable
     next: 37449, // insider
@@ -132,6 +236,159 @@ async function uploadBundle (dTag, channel, fileMetadata, signer) {
     created_at: Math.floor(Date.now() / 1000)
   }
   const event = await signer.signEvent(appBundle)
-  await nostrRelays.sendEvent(event, (await signer.getRelays()).write)
+  await throttledSendEvent(event, (await signer.getRelays()).write, { pause, trailingPause: true })
   return event
 }
+
+async function maybeUploadStall ({
+  dTag,
+  channel,
+  name,
+  summary,
+  icon,
+  signer,
+  writeRelays,
+  log,
+  pause
+}) {
+  const trimmedName = typeof name === 'string' ? name.trim() : ''
+  const trimmedSummary = typeof summary === 'string' ? summary.trim() : ''
+  const iconRootHash = icon?.rootHash
+  const iconMimeType = icon?.mimeType
+  const hasMetadata = Boolean(trimmedName) || Boolean(trimmedSummary) || Boolean(iconRootHash)
+
+  const previous = await getPreviousStall(dTag, writeRelays, signer, channel)
+  if (!previous && !hasMetadata) return { pause }
+
+  const publishStall = async (event) => {
+    const signedEvent = await signer.signEvent(event)
+    return await throttledSendEvent(signedEvent, writeRelays, { pause, log, trailingPause: true })
+  }
+
+  const createdAt = Math.floor(Date.now() / 1000)
+  const kind = {
+    main: 37348,
+    next: 37349,
+    draft: 37350
+  }[channel] ?? 37348
+
+  if (!previous) {
+    const tags = [
+      ['d', dTag],
+      ['c', '*']
+    ]
+
+    let hasIcon = false
+    let hasName = false
+    if (iconRootHash && iconMimeType) {
+      hasIcon = true
+      tags.push(['icon', iconRootHash, iconMimeType])
+      tags.push(['auto', 'icon'])
+    }
+
+    if (trimmedName) {
+      hasName = true
+      tags.push(['name', trimmedName])
+      tags.push(['auto', 'name'])
+    }
+
+    if (trimmedSummary) {
+      tags.push(['summary', trimmedSummary])
+      tags.push(['auto', 'summary'])
+    }
+
+    if (!hasIcon || !hasName) return { pause }
+
+    return await publishStall({
+      kind,
+      tags,
+      content: '',
+      created_at: createdAt
+    })
+  }
+
+  const tags = Array.isArray(previous.tags)
+    ? previous.tags.map(tag => (Array.isArray(tag) ? [...tag] : tag))
+    : []
+  let changed = false
+
+  const ensureTagValue = (key, updater) => {
+    const index = tags.findIndex(tag => Array.isArray(tag) && tag[0] === key)
+    if (index === -1) {
+      const next = updater(null)
+      if (!next) return
+      tags.push(next)
+      changed = true
+      return
+    }
+
+    const next = updater(tags[index])
+    if (!next) return
+    if (!tags[index] || tags[index].some((value, idx) => value !== next[idx])) {
+      tags[index] = next
+      changed = true
+    }
+  }
+
+  ensureTagValue('d', (existing) => {
+    if (existing && existing[1] === dTag) return existing
+    return ['d', dTag]
+  })
+
+  ensureTagValue('c', (existing) => {
+    if (!existing) return ['c', '*']
+    const currentValue = typeof existing[1] === 'string' ? existing[1].trim() : ''
+    if (currentValue === '') return ['c', '*']
+    return existing
+  })
+
+  const hasAuto = (field) => tags.some(tag => Array.isArray(tag) && tag[0] === 'auto' && tag[1] === field)
+
+  if (trimmedName && hasAuto('name')) {
+    ensureTagValue('name', (existing) => {
+      if (existing && existing[1] === trimmedName) return existing
+      return ['name', trimmedName]
+    })
+  }
+
+  if (trimmedSummary && hasAuto('summary')) {
+    ensureTagValue('summary', (existing) => {
+      if (existing && existing[1] === trimmedSummary) return existing
+      return ['summary', trimmedSummary]
+    })
+  }
+
+  if (iconRootHash && iconMimeType && hasAuto('icon')) {
+    ensureTagValue('icon', (existing) => {
+      if (existing && existing[1] === iconRootHash && existing[2] === iconMimeType) return existing
+      return ['icon', iconRootHash, iconMimeType]
+    })
+  }
+
+  if (!changed) return { pause }
+
+  return await publishStall({
+    kind,
+    tags,
+    content: typeof previous.content === 'string' ? previous.content : '',
+    created_at: createdAt
+  })
+}
+
+async function getPreviousStall (dTagValue, writeRelays, signer, channel) {
+  const kind = {
+    main: 37348,
+    next: 37349,
+    draft: 37350
+  }[channel] ?? 37348
+
+  const storedEvents = (await nostrRelays.getEvents({
+    kinds: [kind],
+    authors: [await signer.getPublicKey()],
+    '#d': [dTagValue],
+    limit: 1
+  }, writeRelays)).result
+
+  if (storedEvents.length === 0) return null
+  return storedEvents.sort((a, b) => b.created_at - a.created_at)[0]
+}

package/src/services/base93-decoder.js

@@ -0,0 +1,107 @@
+// https://github.com/ticlo/arrow-code/blob/master/src/base93.ts
+// https://github.com/ticlo/arrow-code/blob/master/LICENSE - Apache 2.0
+
+// JSON-safe (space included; " and \ excluded)
+const BASE93_ALPHABET =
+  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&'()*+,-./:;<=>?@[]^_`{|}~ "
+
+const DECODING_TABLE = (() => {
+  const out = new Int16Array(128)
+  out.fill(93) // sentinel = invalid
+  for (let i = 0; i < 93; i++) out[BASE93_ALPHABET.charCodeAt(i)] = i
+  return out
+})()
+
+/**
+ * Decode Base93 string to Uint8Array
+ * @param {string} str - The Base93 encoded string to decode
+ * @param {number} [offset=0] - The starting position in the string
+ * @param {number} [length=-1] - The number of characters to decode, or -1 for all remaining
+ * @returns {Uint8Array} The decoded bytes
+ */
+export function decode (str, offset = 0, length = -1) {
+  let end = offset + length
+  if (length < 0 || end > str.length) end = str.length
+
+  // Over-allocate; we’ll trim at the end
+  const out = new Uint8Array(Math.ceil((end - offset) * 7 / 8))
+
+  let dbq = 0
+  let dn = 0
+  let dv = -1
+  let pos = 0
+
+  for (let i = offset; i < end; i++) {
+    const code = str.charCodeAt(i)
+    if (code > 126) continue // ignore non-ASCII
+    const v = DECODING_TABLE[code]
+    if (v === 93) continue // ignore invalids
+    if (dv === -1) {
+      dv = v
+    } else {
+      const t = dv + v * 93
+      dv = -1
+      dbq |= t << dn
+      dn += ((t & 0x1fff) > 456 ? 13 : 14)
+      while (dn > 7) {
+        out[pos++] = dbq & 0xff
+        dbq >>>= 8
+        dn -= 8
+      }
+    }
+  }
+
+  if (dv !== -1) {
+    out[pos++] = (dbq | (dv << dn)) & 0xff
+  }
+  return out.subarray(0, pos)
+}
+
+export default class Base93Decoder {
+  constructor (source, { mimeType = '', preferTextStreamDecoding = false } = {}) {
+    this.sourceIterator = source?.[Symbol.iterator]?.() || source?.[Symbol.asyncIterator]?.() || source()
+    this.asTextStream = preferTextStreamDecoding && mimeType.startsWith('text/')
+    if (this.asTextStream) this.textDecoder = new TextDecoder()
+  }
+
+  // decoder generator
+  * [Symbol.iterator] (base93String) {
+    if (this.asTextStream) {
+      while (base93String) {
+        // stream=true avoids cutting a multi-byte character
+        base93String = yield this.textDecoder.decode(decode(base93String), { stream: true })
+      }
+    } else {
+      while (base93String) {
+        base93String = yield decode(base93String)
+      }
+    }
+  }
+
+  // Gets the decoded data.
+  getDecoded () { return iteratorToStream(this, this.sourceIterator) }
+}
+
+function iteratorToStream (decoder, sourceIterator) {
+  return new ReadableStream({
+    decoderIterator: null,
+    async start (controller) {
+      const { value: chunk, done } = await sourceIterator.next()
+      if (done) return controller.close()
+
+      // Pass first chunk when instantiating the decoder generator
+      this.decoderIterator = decoder[Symbol.iterator](chunk)
+      const { value } = this.decoderIterator.next()
+      if (value) controller.enqueue(value)
+    },
+    async pull (controller) {
+      if (!this.decoderIterator) return
+
+      const { value: chunk, done: sourceDone } = await sourceIterator.next()
+      const { value, done } = this.decoderIterator.next(chunk)
+
+      if (value) controller.enqueue(value)
+      if (done || sourceDone) controller.close()
+    }
+  })
+}

package/src/services/base93-encoder.js

@@ -0,0 +1,96 @@
+// https://github.com/ticlo/arrow-code/blob/master/src/base93.ts
+// https://github.com/ticlo/arrow-code/blob/master/LICENSE - Apache 2.0
+
+// JSON-safe (space included; " and \ excluded)
+const BASE93_ALPHABET =
+  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&'()*+,-./:;<=>?@[]^_`{|}~ "
+const ENCODING_TABLE = (() => {
+  const out = new Uint16Array(93)
+  for (let i = 0; i < 93; i++) out[i] = BASE93_ALPHABET.charCodeAt(i)
+  return out
+})()
+
+function codesToString (codes, len) {
+  const CHUNK = 16384 // 16k chars per slice
+  let s = ''
+  for (let i = 0; i < len; i += CHUNK) {
+    const end = i + CHUNK < len ? i + CHUNK : len
+    s += String.fromCharCode.apply(
+      null,
+      Array.prototype.slice.call(codes, i, end)
+    )
+  }
+  return s
+}
+
+export default class Base93Encoder {
+  constructor (prefix = '') {
+    // bit reservoir
+    this._ebq = 0 // queued bits
+    this._en = 0  // number of bits in ebq
+
+    // output parts
+    this._parts = []
+    this._finished = false
+
+    if (prefix) this._parts.push(prefix)
+  }
+
+  // Stream bytes; keeps reservoir across calls.
+  update (bytes) {
+    if (this._finished) throw new Error('Encoder already finalized.')
+    const src = bytes instanceof Uint8Array ? bytes : Uint8Array.from(bytes)
+
+    // Over-allocate for this update; we’ll trim to 'pos'
+    const outCodes = new Uint16Array(Math.ceil(src.length * 8 / 6.5) + 4)
+    let pos = 0
+
+    let ebq = this._ebq
+    let en = this._en
+    let ev = 0
+
+    for (let i = 0; i < src.length; i++) {
+      ebq |= (src[i] & 0xff) << en
+      en += 8
+      if (en > 13) {
+        ev = ebq & 0x1fff
+        if (ev > 456) {
+          ebq >>>= 13
+          en -= 13
+        } else {
+          ev = ebq & 0x3fff
+          ebq >>>= 14
+          en -= 14
+        }
+        outCodes[pos++] = ENCODING_TABLE[ev % 93]
+        outCodes[pos++] = ENCODING_TABLE[(ev / 93) | 0]
+      }
+    }
+
+    // persist reservoir
+    this._ebq = ebq
+    this._en = en
+
+    if (pos) this._parts.push(codesToString(outCodes, pos))
+    return this
+  }
+
+  // Finalize on first call: flush trailing partial block, join, lock.
+  getEncoded () {
+    if (!this._finished) {
+      if (this._en > 0) {
+        const outCodes = new Uint16Array(2)
+        let pos = 0
+        outCodes[pos++] = ENCODING_TABLE[this._ebq % 93]
+        if (this._en > 7 || this._ebq > 92) {
+          outCodes[pos++] = ENCODING_TABLE[(this._ebq / 93) | 0]
+        }
+        this._parts.push(codesToString(outCodes, pos))
+      }
+      this._finished = true
+      this._ebq = 0
+      this._en = 0
+    }
+    return this._parts.join('')
+  }
+}