@helia/ipns 0.0.0

package/src/index.ts

@@ -0,0 +1,296 @@
+/**
+ * @packageDocumentation
+ *
+ * IPNS operations using a Helia node
+ *
+ * @example
+ *
+ * ```typescript
+ * import { gossipsub } from '@chainsafe/libp2p'
+ * import { kadDHT } from '@libp2p/kad-dht'
+ * import { createLibp2p } from 'libp2p'
+ * import { createHelia } from 'helia'
+ * import { ipns, ipnsValidator, ipnsSelector } from '@helia/ipns'
+ * import { dht, pubsub } from '@helia/ipns/routing'
+ * import { unixfs } from '@helia/unixfs
+ *
+ * const libp2p = await createLibp2p({
+ *   dht: kadDHT({
+ *    validators: {
+ *      ipns: ipnsValidator
+ *    },
+ *    selectors: {
+ *      ipns: ipnsSelector
+ *    }
+ *   }),
+ *   pubsub: gossipsub()
+ * })
+ *
+ * const helia = await createHelia({
+ *   libp2p,
+ *   //.. other options
+ * })
+ * const name = ipns(helia, [
+ *   dht(helia)
+ *   pubsub(helia)
+ * ])
+ *
+ * // create a public key to publish as an IPNS name
+ * const keyInfo = await helia.libp2p.keychain.createKey('my-key')
+ * const peerId = await helia.libp2p.keychain.exportPeerId(keyInfo.name)
+ *
+ * // store some data to publish
+ * const fs = unixfs(helia)
+ * const cid = await fs.add(Uint8Array.from([0, 1, 2, 3, 4]))
+ *
+ * // publish the name
+ * await name.publish(peerId, cid)
+ *
+ * // resolve the name
+ * const cid = name.resolve(peerId)
+ * ```
+ *
+ * @example
+ *
+ * ```typescript
+ * // resolve a CID from a TXT record in a DNS zone file, eg:
+ * // > dig ipfs.io TXT
+ * // ;; ANSWER SECTION:
+ * // ipfs.io.           435     IN      TXT     "dnslink=/ipfs/Qmfoo"
+ *
+ * const cid = name.resolveDns('ipfs.io')
+ * ```
+ */
+
+import type { AbortOptions } from '@libp2p/interfaces'
+import { isPeerId, PeerId } from '@libp2p/interface-peer-id'
+import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns'
+import type { IPNSEntry } from 'ipns'
+import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js'
+import { ipnsValidator } from 'ipns/validator'
+import { CID } from 'multiformats/cid'
+import { resolveDnslink } from './utils/resolve-dns-link.js'
+import { logger } from '@libp2p/logger'
+import { peerIdFromString } from '@libp2p/peer-id'
+import type { ProgressEvent, ProgressOptions } from 'progress-events'
+import { CustomProgressEvent } from 'progress-events'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import type { Datastore } from 'interface-datastore'
+import { localStore, LocalStore } from './routing/local-store.js'
+
+const log = logger('helia:ipns')
+
+const MINUTE = 60 * 1000
+const HOUR = 60 * MINUTE
+
+const DEFAULT_LIFETIME_MS = 24 * HOUR
+const DEFAULT_REPUBLISH_INTERVAL_MS = 23 * HOUR
+
+export type PublishProgressEvents =
+  ProgressEvent<'ipns:publish:start'> |
+  ProgressEvent<'ipns:publish:success', IPNSEntry> |
+  ProgressEvent<'ipns:publish:error', Error>
+
+export type ResolveProgressEvents =
+  ProgressEvent<'ipns:resolve:start', unknown> |
+  ProgressEvent<'ipns:resolve:success', IPNSEntry> |
+  ProgressEvent<'ipns:resolve:error', Error>
+
+export type RepublishProgressEvents =
+  ProgressEvent<'ipns:republish:start', unknown> |
+  ProgressEvent<'ipns:republish:success', IPNSEntry> |
+  ProgressEvent<'ipns:republish:error', { record: IPNSEntry, err: Error }>
+
+export interface PublishOptions extends AbortOptions, ProgressOptions<PublishProgressEvents | IPNSRoutingEvents> {
+  /**
+   * Time duration of the record in ms
+   */
+  lifetime?: number
+}
+
+export interface ResolveOptions extends AbortOptions, ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents> {
+  /**
+   * do not use cached entries
+   */
+  nocache?: boolean
+}
+
+export interface RepublishOptions extends AbortOptions, ProgressOptions<RepublishProgressEvents | IPNSRoutingEvents> {
+  /**
+   * The republish interval in ms (default: 24hrs)
+   */
+  interval?: number
+}
+
+export interface IPNS {
+  /**
+   * Creates an IPNS record signed by the passed PeerId that will resolve to the passed value
+   *
+   * If the valid is a PeerId, a recursive IPNS record will be created.
+   */
+  publish: (key: PeerId, value: CID | PeerId, options?: PublishOptions) => Promise<IPNSEntry>
+
+  /**
+   * Accepts a public key formatted as a libp2p PeerID and resolves the IPNS record
+   * corresponding to that public key until a value is found
+   */
+  resolve: (key: PeerId, options?: ResolveOptions) => Promise<CID>
+
+  /**
+   * Resolve a CID from a dns-link style IPNS record
+   */
+  resolveDns: (domain: string, options?: ResolveOptions) => Promise<CID>
+
+  /**
+   * Periodically republish all IPNS records found in the datastore
+   */
+  republish: (options?: RepublishOptions) => void
+}
+
+export type { IPNSRouting } from './routing/index.js'
+
+export interface IPNSComponents {
+  datastore: Datastore
+}
+
+class DefaultIPNS implements IPNS {
+  private readonly routers: IPNSRouting[]
+  private readonly localStore: LocalStore
+  private timeout?: ReturnType<typeof setTimeout>
+
+  constructor (components: IPNSComponents, routers: IPNSRouting[] = []) {
+    this.routers = routers
+    this.localStore = localStore(components.datastore)
+  }
+
+  async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise<IPNSEntry> {
+    try {
+      let sequenceNumber = 1n
+      const routingKey = peerIdToRoutingKey(key)
+
+      if (await this.localStore.has(routingKey, options)) {
+        // if we have published under this key before, increment the sequence number
+        const buf = await this.localStore.get(routingKey, options)
+        const existingRecord = unmarshal(buf)
+        sequenceNumber = existingRecord.sequence + 1n
+      }
+
+      let str
+
+      if (isPeerId(value)) {
+        str = `/ipns/${value.toString()}`
+      } else {
+        str = `/ipfs/${value.toString()}`
+      }
+
+      const bytes = uint8ArrayFromString(str)
+
+      // create record
+      const record = await create(key, bytes, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS)
+      const marshaledRecord = marshal(record)
+
+      await this.localStore.put(routingKey, marshaledRecord, options)
+
+      // publish record to routing
+      await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) }))
+
+      return record
+    } catch (err: any) {
+      options.onProgress?.(new CustomProgressEvent<Error>('ipns:publish:error', err))
+      throw err
+    }
+  }
+
+  async resolve (key: PeerId, options: ResolveOptions = {}): Promise<CID> {
+    const routingKey = peerIdToRoutingKey(key)
+    const record = await this.#findIpnsRecord(routingKey, options)
+    const str = uint8ArrayToString(record.value)
+
+    return await this.#resolve(str)
+  }
+
+  async resolveDns (domain: string, options: ResolveOptions = {}): Promise<CID> {
+    const dnslink = await resolveDnslink(domain, options)
+
+    return await this.#resolve(dnslink)
+  }
+
+  republish (options: RepublishOptions = {}): void {
+    if (this.timeout != null) {
+      throw new Error('Republish is already running')
+    }
+
+    options.signal?.addEventListener('abort', () => {
+      clearTimeout(this.timeout)
+    })
+
+    async function republish (): Promise<void> {
+      const startTime = Date.now()
+
+      options.onProgress?.(new CustomProgressEvent('ipns:republish:start'))
+
+      const finishType = Date.now()
+      const timeTaken = finishType - startTime
+      let nextInterval = DEFAULT_REPUBLISH_INTERVAL_MS - timeTaken
+
+      if (nextInterval < 0) {
+        nextInterval = options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS
+      }
+
+      setTimeout(() => {
+        republish().catch(err => {
+          log.error('error republishing', err)
+        })
+      }, nextInterval)
+    }
+
+    this.timeout = setTimeout(() => {
+      republish().catch(err => {
+        log.error('error republishing', err)
+      })
+    }, options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS)
+  }
+
+  async #resolve (ipfsPath: string): Promise<CID> {
+    const parts = ipfsPath.split('/')
+
+    if (parts.length === 3) {
+      const scheme = parts[1]
+
+      if (scheme === 'ipns') {
+        return await this.resolve(peerIdFromString(parts[2]))
+      } else if (scheme === 'ipfs') {
+        return CID.parse(parts[2])
+      }
+    }
+
+    log.error('invalid ipfs path %s', ipfsPath)
+    throw new Error('Invalid value')
+  }
+
+  async #findIpnsRecord (routingKey: Uint8Array, options: AbortOptions): Promise<IPNSEntry> {
+    const routers = [
+      this.localStore,
+      ...this.routers
+    ]
+
+    const unmarshaledRecord = await Promise.any(
+      routers.map(async (router) => {
+        const unmarshaledRecord = await router.get(routingKey, options)
+        await ipnsValidator(routingKey, unmarshaledRecord)
+
+        return unmarshaledRecord
+      })
+    )
+
+    return unmarshal(unmarshaledRecord)
+  }
+}
+
+export function ipns (components: IPNSComponents, routers: IPNSRouting[] = []): IPNS {
+  return new DefaultIPNS(components, routers)
+}
+
+export { ipnsValidator }
+export { ipnsSelector } from 'ipns/selector'

package/src/routing/dht.ts

@@ -0,0 +1,85 @@
+import { logger } from '@libp2p/logger'
+import type { IPNSRouting } from '../index.js'
+import type { DHT, QueryEvent } from '@libp2p/interface-dht'
+import type { GetOptions, PutOptions } from './index.js'
+import { CustomProgressEvent, ProgressEvent } from 'progress-events'
+
+const log = logger('helia:ipns:routing:dht')
+
+export interface DHTRoutingComponents {
+  libp2p: {
+    dht: DHT
+  }
+}
+
+export type DHTProgressEvents =
+  ProgressEvent<'ipns:routing:dht:query', QueryEvent> |
+  ProgressEvent<'ipns:routing:dht:error', Error>
+
+export class DHTRouting implements IPNSRouting {
+  private readonly dht: DHT
+
+  constructor (components: DHTRoutingComponents) {
+    this.dht = components.libp2p.dht
+  }
+
+  async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}): Promise<void> {
+    let putValue = false
+
+    try {
+      for await (const event of this.dht.put(routingKey, marshaledRecord, options)) {
+        logEvent('DHT put event', event)
+
+        options.onProgress?.(new CustomProgressEvent<QueryEvent>('ipns:routing:dht:query', event))
+
+        if (event.name === 'PEER_RESPONSE' && event.messageName === 'PUT_VALUE') {
+          putValue = true
+        }
+      }
+    } catch (err: any) {
+      options.onProgress?.(new CustomProgressEvent<Error>('ipns:routing:dht:error', err))
+    }
+
+    if (!putValue) {
+      throw new Error('Could not put value to DHT')
+    }
+  }
+
+  async get (routingKey: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {
+    try {
+      for await (const event of this.dht.get(routingKey, options)) {
+        logEvent('DHT get event', event)
+
+        options.onProgress?.(new CustomProgressEvent<QueryEvent>('ipns:routing:dht:query', event))
+
+        if (event.name === 'VALUE') {
+          return event.value
+        }
+      }
+    } catch (err: any) {
+      options.onProgress?.(new CustomProgressEvent<Error>('ipns:routing:dht:error', err))
+    }
+
+    throw new Error('Not found')
+  }
+}
+
+function logEvent (prefix: string, event: QueryEvent): void {
+  if (event.name === 'SENDING_QUERY') {
+    log(prefix, event.name, event.messageName, '->', event.to.toString())
+  } else if (event.name === 'PEER_RESPONSE') {
+    log(prefix, event.name, event.messageName, '<-', event.from.toString())
+  } else if (event.name === 'FINAL_PEER') {
+    log(prefix, event.name, event.peer.id.toString())
+  } else if (event.name === 'QUERY_ERROR') {
+    log(prefix, event.name, event.error.message)
+  } else if (event.name === 'PROVIDER') {
+    log(prefix, event.name, event.providers.map(p => p.id.toString()).join(', '))
+  } else {
+    log(prefix, event.name)
+  }
+}
+
+export function dht (components: DHTRoutingComponents): IPNSRouting {
+  return new DHTRouting(components)
+}

package/src/routing/index.ts

@@ -0,0 +1,26 @@
+import type { ProgressOptions } from 'progress-events'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { DHTProgressEvents } from './dht.js'
+import type { DatastoreProgressEvents } from './local-store.js'
+import type { PubSubProgressEvents } from './pubsub.js'
+
+export interface PutOptions extends AbortOptions, ProgressOptions {
+
+}
+
+export interface GetOptions extends AbortOptions, ProgressOptions {
+
+}
+
+export interface IPNSRouting {
+  put: (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions) => Promise<void>
+  get: (routingKey: Uint8Array, options?: GetOptions) => Promise<Uint8Array>
+}
+
+export type IPNSRoutingEvents =
+  DatastoreProgressEvents |
+  DHTProgressEvents |
+  PubSubProgressEvents
+
+export { dht } from './dht.js'
+export { pubsub } from './pubsub.js'

package/src/routing/local-store.ts

@@ -0,0 +1,63 @@
+import { CustomProgressEvent, ProgressEvent } from 'progress-events'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { Libp2pRecord } from '@libp2p/record'
+import { Datastore, Key } from 'interface-datastore'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import type { GetOptions, IPNSRouting, PutOptions } from '../routing'
+
+function dhtRoutingKey (key: Uint8Array): Key {
+  return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false)
+}
+
+export type DatastoreProgressEvents =
+  ProgressEvent<'ipns:routing:datastore:put'> |
+  ProgressEvent<'ipns:routing:datastore:get'> |
+  ProgressEvent<'ipns:routing:datastore:error', Error>
+
+export interface LocalStore extends IPNSRouting {
+  has: (routingKey: Uint8Array, options?: AbortOptions) => Promise<boolean>
+}
+
+/**
+ * Returns an IPNSRouting implementation that reads/writes IPNS records to the
+ * datastore as DHT records. This lets us publish IPNS records offline then
+ * serve them to the network later in response to DHT queries.
+ */
+export function localStore (datastore: Datastore): LocalStore {
+  return {
+    async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}) {
+      try {
+        const key = dhtRoutingKey(routingKey)
+
+        // Marshal to libp2p record as the DHT does
+        const record = new Libp2pRecord(routingKey, marshaledRecord, new Date())
+
+        options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:put'))
+        await datastore.put(key, record.serialize(), options)
+      } catch (err: any) {
+        options.onProgress?.(new CustomProgressEvent<Error>('ipns:routing:datastore:error', err))
+        throw err
+      }
+    },
+    async get (routingKey: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {
+      try {
+        const key = dhtRoutingKey(routingKey)
+
+        options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:get'))
+        const buf = await datastore.get(key, options)
+
+        // Unmarshal libp2p record as the DHT does
+        const record = Libp2pRecord.deserialize(buf)
+
+        return record.value
+      } catch (err: any) {
+        options.onProgress?.(new CustomProgressEvent<Error>('ipns:routing:datastore:error', err))
+        throw err
+      }
+    },
+    async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise<boolean> {
+      const key = dhtRoutingKey(routingKey)
+      return await datastore.has(key, options)
+    }
+  }
+}

package/src/routing/pubsub.ts

@@ -0,0 +1,195 @@
+import { peerIdToRoutingKey } from 'ipns'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { logger } from '@libp2p/logger'
+import type { PeerId } from '@libp2p/interface-peer-id'
+import type { Message, PublishResult, PubSub } from '@libp2p/interface-pubsub'
+import type { Datastore } from 'interface-datastore'
+import type { GetOptions, IPNSRouting, PutOptions } from './index.js'
+import { CodeError } from '@libp2p/interfaces/errors'
+import { localStore, LocalStore } from './local-store.js'
+import { ipnsValidator } from 'ipns/validator'
+import { ipnsSelector } from 'ipns/selector'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import { CustomProgressEvent, ProgressEvent } from 'progress-events'
+
+const log = logger('helia:ipns:routing:pubsub')
+
+export interface PubsubRoutingComponents {
+  datastore: Datastore
+  libp2p: {
+    peerId: PeerId
+    pubsub: PubSub
+  }
+}
+
+export type PubSubProgressEvents =
+  ProgressEvent<'ipns:pubsub:publish', { topic: string, result: PublishResult }> |
+  ProgressEvent<'ipns:pubsub:subscribe', { topic: string }> |
+  ProgressEvent<'ipns:pubsub:error', Error>
+
+/**
+ * This IPNS routing receives IPNS record updates via dedicated
+ * pubsub topic.
+ *
+ * Note we must first be subscribed to the topic in order to receive
+ * updated records, so the first call to `.get` should be expected
+ * to fail!
+ */
+class PubSubRouting implements IPNSRouting {
+  private subscriptions: string[]
+  private readonly localStore: LocalStore
+  private readonly peerId: PeerId
+  private readonly pubsub: PubSub
+
+  constructor (components: PubsubRoutingComponents) {
+    this.subscriptions = []
+    this.localStore = localStore(components.datastore)
+    this.peerId = components.libp2p.peerId
+    this.pubsub = components.libp2p.pubsub
+
+    this.pubsub.addEventListener('message', (evt) => {
+      const message = evt.detail
+
+      if (!this.subscriptions.includes(message.topic)) {
+        return
+      }
+
+      this.#processPubSubMessage(message).catch(err => {
+        log.error('Error processing message', err)
+      })
+    })
+  }
+
+  async #processPubSubMessage (message: Message): Promise<void> {
+    log('message received for topic', message.topic)
+
+    if (message.type !== 'signed') {
+      log.error('unsigned message received, this module can only work with signed messages')
+      return
+    }
+
+    if (message.from.equals(this.peerId)) {
+      log('not storing record from self')
+      return
+    }
+
+    const routingKey = topicToKey(message.topic)
+
+    await ipnsValidator(routingKey, message.data)
+
+    if (await this.localStore.has(routingKey)) {
+      const currentRecord = await this.localStore.get(routingKey)
+
+      if (uint8ArrayEquals(currentRecord, message.data)) {
+        log('not storing record as we already have it')
+        return
+      }
+
+      const records = [currentRecord, message.data]
+      const index = ipnsSelector(routingKey, records)
+
+      if (index === 0) {
+        log('not storing record as the one we have is better')
+        return
+      }
+    }
+
+    await this.localStore.put(routingKey, message.data)
+  }
+
+  /**
+   * Put a value to the pubsub datastore indexed by the received key properly encoded
+   */
+  async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}): Promise<void> {
+    try {
+      const topic = keyToTopic(routingKey)
+
+      log('publish value for topic %s', topic)
+      const result = await this.pubsub.publish(topic, marshaledRecord)
+
+      log('published record on topic %s to %d recipients', topic, result.recipients)
+      options.onProgress?.(new CustomProgressEvent('ipns:pubsub:publish', { topic, result }))
+    } catch (err: any) {
+      options.onProgress?.(new CustomProgressEvent<Error>('ipns:pubsub:error', err))
+      throw err
+    }
+  }
+
+  /**
+   * Get a value from the pubsub datastore indexed by the received key properly encoded.
+   * Also, the identifier topic is subscribed to and the pubsub datastore records will be
+   * updated once new publishes occur
+   */
+  async get (routingKey: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {
+    try {
+      const topic = keyToTopic(routingKey)
+
+      // ensure we are subscribed to topic
+      if (!this.pubsub.getTopics().includes(topic)) {
+        log('add subscription for topic', topic)
+        this.pubsub.subscribe(topic)
+        this.subscriptions.push(topic)
+
+        options.onProgress?.(new CustomProgressEvent('ipns:pubsub:subscribe', { topic }))
+      }
+
+      // chain through to local store
+      return await this.localStore.get(routingKey, options)
+    } catch (err: any) {
+      options.onProgress?.(new CustomProgressEvent<Error>('ipns:pubsub:error', err))
+      throw err
+    }
+  }
+
+  /**
+   * Get pubsub subscriptions related to ipns
+   */
+  getSubscriptions (): string[] {
+    return this.subscriptions
+  }
+
+  /**
+   * Cancel pubsub subscriptions related to ipns
+   */
+  cancel (key: PeerId): void {
+    const routingKey = peerIdToRoutingKey(key)
+    const topic = keyToTopic(routingKey)
+
+    // Not found topic
+    if (!this.subscriptions.includes(topic)) {
+      return
+    }
+
+    this.pubsub.unsubscribe(topic)
+    this.subscriptions = this.subscriptions.filter(t => t !== topic)
+  }
+}
+
+const PUBSUB_NAMESPACE = '/record/'
+
+/**
+ * converts a binary record key to a pubsub topic key
+ */
+function keyToTopic (key: Uint8Array): string {
+  const b64url = uint8ArrayToString(key, 'base64url')
+
+  return `${PUBSUB_NAMESPACE}${b64url}`
+}
+
+/**
+ * converts a pubsub topic key to a binary record key
+ */
+function topicToKey (topic: string): Uint8Array {
+  if (topic.substring(0, PUBSUB_NAMESPACE.length) !== PUBSUB_NAMESPACE) {
+    throw new CodeError('topic received is not from a record', 'ERR_TOPIC_IS_NOT_FROM_RECORD_NAMESPACE')
+  }
+
+  const key = topic.substring(PUBSUB_NAMESPACE.length)
+
+  return uint8ArrayFromString(key, 'base64url')
+}
+
+export function pubsub (components: PubsubRoutingComponents): IPNSRouting {
+  return new PubSubRouting(components)
+}