@helia/ipns 10.1.1 → 10.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/ipns",
-  "version": "10.1.1",
+  "version": "10.2.0",
   "description": "An implementation of IPNS for Helia",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/ipns#readme",
@@ -66,7 +66,7 @@
     "interface-datastore": "^10.0.1",
     "multiformats": "^14.0.0",
     "progress-events": "^1.1.0",
-    "protons-runtime": "^7.1.0",
+    "protons-runtime": "^7.1.1",
     "race-signal": "^2.0.0",
     "timestamp-nano": "^1.0.1",
     "uint8arraylist": "^3.0.2",
@@ -78,7 +78,7 @@
     "datastore-core": "^12.0.1",
     "it-drain": "^3.0.12",
     "it-last": "^3.0.11",
-    "protons": "^9.1.0",
+    "protons": "^9.1.1",
     "sinon": "^22.0.0",
     "sinon-ts": "^2.0.0"
   },

package/src/errors.ts

@@ -1,3 +1,5 @@
+import type { IPNSEntry } from './pb/ipns.ts'
+
 export class RecordsFailedValidationError extends Error {
   static name = 'RecordsFailedValidationError'
   name = 'RecordsFailedValidationError'
@@ -18,6 +20,22 @@ export class RecordNotFoundError extends Error {
   name = 'RecordNotFoundError'
 }
 
+export class RecordObsoleteError extends Error {
+  static name = 'RecordObsoleteError'
+  name = 'RecordObsoleteError'
+  /**
+   * The more suitable record that made the one being imported or republished
+   * obsolete.
+   */
+  readonly record: IPNSEntry
+
+  constructor (message: string, record: IPNSEntry) {
+    super(message)
+
+    this.record = record
+  }
+}
+
 export class SignatureCreationError extends Error {
   static name = 'SignatureCreationError'
   name = 'SignatureCreationError'

package/src/index.ts

@@ -135,6 +135,46 @@
  *   console.info(result.record.value)
  * }
  * ```
+ *
+ * @example Republishing an existing IPNS record
+ *
+ * It is sometimes useful to be able to republish an existing IPNS record
+ * without needing the private key. This allows you to extend the availability
+ * of a record that was created elsewhere.
+ *
+ * There should be only one republisher per IPNS key. Multiple machines
+ * republishing the same key flood the routers with redundant writes.
+ *
+ * ```TypeScript
+ * import { createHelia } from 'helia'
+ * import { ipns } from '@helia/ipns'
+ * import { delegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client'
+ * import { defaultLogger } from 'birnam'
+ * import { CID } from 'multiformats/cid'
+ *
+ * const helia = await createHelia()
+ * const name = ipns(helia)
+ *
+ * const ipnsName = 'k51qzi5uqu5dktsyfv7xz8h631pri4ct7osmb43nibxiojpttxzoft6hdyyzg4'
+ * const parsedCid: CID<unknown, 114, 0 | 18, 1> = CID.parse(ipnsName)
+ * const delegatedClient = delegatedRoutingV1HttpApiClient({
+ *   url: 'https://delegated-ipfs.dev'
+ * })({
+ *   logger: defaultLogger()
+ * })
+ * const record = await delegatedClient.getIPNS(parsedCid)
+ *
+ * // import the record into the local store (validates and stores it locally,
+ * // but does not publish it to the routers)
+ * await name.import(parsedCid, record)
+ *
+ * // start republishing it; throws RecordObsoleteError if the routers
+ * // already have a newer record for this key
+ * const { record: latestRecord } = await name.republish(parsedCid)
+ *
+ * // stop republishing a key
+ * await name.unpublish(parsedCid)
+ * ```
  */
 
 import { CID } from 'multiformats/cid'
@@ -155,6 +195,7 @@ import type { ProgressEvent, ProgressOptions } from 'progress-events'
 
 export * from './routing/index.ts'
 export * from './pb/ipns.ts'
+export * from './errors.ts'
 
 export {
   multihashFromIPNSRoutingKey,
@@ -184,6 +225,14 @@ export type DatastoreProgressEvents =
   ProgressEvent<'ipns:routing:datastore:list'> |
   ProgressEvent<'ipns:routing:datastore:error', Error>
 
+/**
+ * The automated upkeep policy for a stored IPNS record. `republish()` accepts
+ * every policy except `reissue`, which needs the private key to re-sign.
+ *
+ * These strings must match the `Upkeep` enum values in `pb/metadata.proto`.
+ */
+export type UpkeepPolicy = 'reissue' | 'rebroadcast' | 'none'
+
 export interface PublishOptions extends AbortOptions, ProgressOptions<PublishProgressEvents | IPNSRoutingProgressEvents> {
   /**
    * Time duration of the signature validity in ms - after this many ms have
@@ -219,6 +268,17 @@ export interface PublishOptions extends AbortOptions, ProgressOptions<PublishPro
    */
   ttl?: number
 
+  /**
+   * Automated record upkeep policy.
+   *
+   * - `reissue`: re-sign the record with a new validity
+   * - `rebroadcast`: re-publish the existing record until it expires
+   * - `none`: disable automated publishing
+   *
+   * @default 'reissue'
+   */
+  upkeep?: UpkeepPolicy
+
   /**
    * Extensible data that will be added to the IPNS record data and signed to
    * verify it's integrity.
@@ -273,6 +333,43 @@ export interface IPNSResolveOptions extends AbortOptions, ProgressOptions<Resolv
   validate?: boolean
 }
 
+export interface RepublishOptions extends AbortOptions, ProgressOptions<IPNSRoutingProgressEvents> {
+  /**
+   * Skip resolution of latest record before republishing.
+   *
+   * It's important to resolve the latest record before republishing to routers
+   *
+   * Resolution should only be skipped when confident the latest record is
+   * already known.
+   *
+   * @default false
+   */
+  skipResolution?: boolean
+
+  /**
+   * Automated record upkeep policy.
+   *
+   * Defaults to `rebroadcast` since `republish()` cannot sign new records
+   * without the private key.
+   *
+   * - `rebroadcast`: re-publish the existing record until it expires
+   * - `none`: disable automated publishing
+   *
+   * @default 'rebroadcast'
+   */
+  upkeep?: Exclude<UpkeepPolicy, 'reissue'>
+}
+
+export interface UnpublishOptions extends AbortOptions {
+  /**
+   * Also delete the record from the local store, so the node stops serving it
+   * immediately instead of keeping it until it expires.
+   *
+   * @default false
+   */
+  removeRecord?: boolean
+}
+
 export interface IPNSResolveResult {
   /**
    * The resolved record
@@ -302,6 +399,18 @@ export interface IPNSPublishResult {
   publicKey: PublicKey
 }
 
+export interface RepublishResult {
+  /**
+   * The published record
+   */
+  record: IPNSEntry
+
+  /**
+   * The public key that the record is published under
+   */
+  publicKey: PublicKey
+}
+
 export interface IPNSResolver {
   /**
    * Accepts a CID with the libp2p-key codec and either the identity hash (for
@@ -345,7 +454,7 @@ export interface IPNS {
    *   signal: AbortSignal.timeout(5_000)
    * })
    *
-   * console.info(result) // { answer: ... }
+   * console.info(result) // { record, name, publicKey }
    * ```
    */
   publish(keyName: string, value: CID | PublicKey | MultihashDigest | string, options?: PublishOptions): Promise<IPNSPublishResult>
@@ -357,17 +466,46 @@ export interface IPNS {
    * (e.g. the value can be parsed as a string that does not start with
    * `/ipns/`).
    */
-  resolve(name: CID | PublicKey | MultihashDigest | string, options?: IPNSResolveOptions): AsyncGenerator<IPNSResolveResult>
+  resolve(name: CID<unknown, 0x72> | PublicKey | MultihashDigest | string, options?: IPNSResolveOptions): AsyncGenerator<IPNSResolveResult>
+
+  /**
+   * Stop automatically republishing an IPNS record
+   *
+   * By default this removes only the republishing metadata, keeping the record
+   * in the datastore so it is still served until it expires. Pass
+   * `removeRecord: true` to also delete the record so the node stops serving it.
+   * If a key name is passed, the key remains in the keychain.
+   */
+  unpublish(key: CID<unknown, 0x72> | PublicKey | MultihashDigest | string, options?: UnpublishOptions): Promise<void>
+
+  /**
+   * Republish the record already in the local store, setting its upkeep policy.
+   *
+   * Sets the record's upkeep policy to `options.upkeep` (default `rebroadcast`)
+   * so the background republisher keeps it available on the routers.
+   *
+   * @throws {NotFoundError} when there is no local record to republish
+   * @throws {RecordObsoleteError} when the routers already have a newer record
+   */
+  republish(key: CID<unknown, 0x72> | PublicKey | MultihashDigest | string, options?: RepublishOptions): Promise<RepublishResult>
 
   /**
-   * Stop republishing of an IPNS record.
+   * Import an existing IPNS record into the local store without publishing it.
+   *
+   * The record is validated and stored so this node serves it on a GET, but it
+   * is not broadcast to the routers. A newly imported record (for a key with no
+   * existing upkeep policy) is not automatically republished; call `republish`
+   * with an upkeep policy to start keeping it alive.
    *
-   * This will delete the last signed IPNS record from the datastore.
+   * Overwrites an older record already stored under this key, or rejects with
+   * `RecordObsoleteError` if the stored record is newer. Importing never changes
+   * the key's upkeep metadata, so a key that already has an upkeep policy keeps
+   * it and continues to be republished automatically.
    *
-   * Note that the record may still be resolved by other peers until it expires
-   * or is otherwise no longer valid.
+   * @throws {RecordExpiredError} (or another validation error) when the record fails validation
+   * @throws {RecordObsoleteError} when the stored record is more suitable than the one being imported
    */
-  unpublish(keyName: string, options?: AbortOptions): Promise<void>
+  import(key: CID<unknown, 0x72> | PublicKey | MultihashDigest | string, record: IPNSEntry | Uint8Array, options?: AbortOptions): Promise<void>
 }
 
 export interface IPNSComponents {

package/src/ipns/publisher.ts

@@ -2,9 +2,10 @@ import { base36 } from 'multiformats/bases/base36'
 import { CustomProgressEvent } from 'progress-events'
 import { DEFAULT_LIFETIME_MS } from '../constants.ts'
 import { IPNSEntry } from '../pb/ipns.ts'
+import { Upkeep } from '../pb/metadata.ts'
 import { createIPNSRecord } from '../records.ts'
 import { decodeExtensibleData, multihashToIPNSRoutingKey } from '../utils.ts'
-import type { IPNSPublishResult, PublishOptions } from '../index.ts'
+import type { IPNSPublishResult, PublishOptions, UnpublishOptions } from '../index.ts'
 import type { LocalStore } from '../local-store.ts'
 import type { IPNSRouting } from '../routing/index.ts'
 import type { Keychain, PrivateKey, PublicKey } from '@helia/interface'
@@ -58,25 +59,14 @@ export class IPNSPublisher {
       })
       const marshaledRecord = IPNSEntry.encode(record)
 
+      const metadata = { keyName, lifetime, upkeep: Upkeep[options.upkeep ?? 'reissue'] }
       if (options.offline === true) {
         // only store record locally
-        await this.localStore.put(routingKey, marshaledRecord, {
-          ...options,
-          metadata: {
-            keyName,
-            lifetime
-          }
-        })
+        await this.localStore.put(routingKey, marshaledRecord, { ...options, metadata })
       } else {
         // publish record to routing (including the local store)
         await Promise.all(this.routers.map(async r => {
-          await r.put(routingKey, marshaledRecord, {
-            ...options,
-            metadata: {
-              keyName,
-              lifetime
-            }
-          })
+          await r.put(routingKey, marshaledRecord, { ...options, metadata })
         }))
       }
 
@@ -108,10 +98,22 @@ export class IPNSPublisher {
     }
   }
 
-  async unpublish (keyName: string, options?: AbortOptions): Promise<void> {
-    const key = await this.keychain.exportKey(keyName, options)
-    const digest = key.publicKey.toMultihash()
-    const routingKey = multihashToIPNSRoutingKey(digest)
-    await this.localStore.delete(routingKey, options)
+  async unpublish (keyName: string | MultihashDigest, options?: UnpublishOptions): Promise<void> {
+    if (typeof keyName === 'string') {
+      const key = await this.keychain.exportKey(keyName, options)
+      keyName = key.publicKey.toMultihash()
+    }
+
+    const routingKey = multihashToIPNSRoutingKey(keyName)
+
+    if (options?.removeRecord === true) {
+      // delete the record and its metadata so this node stops serving it
+      await this.localStore.delete(routingKey, options)
+      return
+    }
+
+    // remove only the metadata so the record is still served but no longer
+    // automatically republished; it expires naturally at its validity
+    await this.localStore.deleteMetadata(routingKey, options)
   }
 }

package/src/ipns/republisher.ts

@@ -1,15 +1,24 @@
+import { NotFoundError } from '@libp2p/interface'
 import { Queue, repeatingTask } from '@libp2p/utils'
+import NanoDate from 'timestamp-nano'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { DEFAULT_REPUBLISH_CONCURRENCY, DEFAULT_REPUBLISH_INTERVAL_MS, DEFAULT_TTL_NS } from '../constants.ts'
+import { RecordObsoleteError } from '../errors.ts'
 import { IPNSEntry } from '../pb/ipns.ts'
+import { Upkeep } from '../pb/metadata.ts'
 import { createIPNSRecord } from '../records.ts'
-import { decodeExtensibleData, shouldRepublish } from '../utils.ts'
-import type { IPNSRecordData } from '../index.ts'
+import { isLocalStoreRouting, routerName } from '../routing/index.ts'
+import { ipnsSelector } from '../selector.ts'
+import { decodeExtensibleData, multihashToIPNSRoutingKey, shouldRepublish } from '../utils.ts'
+import { extractPublicKey, ipnsValidator } from '../validator.ts'
+import { findRoutingRecords } from './resolver.ts'
+import type { IPNSRecordData, RepublishOptions, RepublishResult } from '../index.ts'
 import type { LocalStore } from '../local-store.ts'
-import type { IPNSRouting } from '../routing/index.ts'
+import type { IPNSRouting, IPNSRoutingPutOptions } from '../routing/index.ts'
 import type { Keychain, PrivateKey } from '@helia/interface'
 import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface'
 import type { RepeatingTask } from '@libp2p/utils'
+import type { MultihashDigest } from 'multiformats'
 
 export interface IPNSRepublisherComponents {
   logger: ComponentLogger
@@ -79,7 +88,8 @@ export class IPNSRepublisher {
     })
 
     try {
-      const recordsToRepublish: Array<{ routingKey: Uint8Array, record: IPNSEntry }> = []
+      const recordsToReissue: Array<{ routingKey: Uint8Array, record: IPNSEntry }> = []
+      const recordsToRebroadcast: Array<{ routingKey: Uint8Array, record: IPNSEntry }> = []
       let listed = 0
 
       // Find all records using the localStore.list method
@@ -87,23 +97,31 @@ export class IPNSRepublisher {
         listed++
 
         if (metadata == null) {
-          // Skip if no metadata is found from before we started
-          // storing metadata or for records republished without a key
+          // no metadata: an imported record not yet given an upkeep policy, or a
+          // record stored before we tracked metadata
           this.log('no metadata found for record %b, skipping', routingKey)
           continue
         }
 
+        if (metadata.upkeep === Upkeep.none) {
+          // Skip republishing, disabled for this record
+          this.log('republishing is disabled for record %b, skipping', routingKey)
+          continue
+        }
+
         let ipnsRecord: IPNSEntry
 
         try {
           ipnsRecord = IPNSEntry.decode(record)
         } catch (err) {
-          this.log.trace('skipping invalid record %b because could not decode', routingKey)
+          // the record was valid when stored, so a decode failure now means
+          // on-disk corruption or a format change worth surfacing
+          this.log.error('skipping stored record %b, could not decode it - %e', routingKey, err)
           continue
         }
 
         if (ipnsRecord.data == null) {
-          this.log.trace('skipping record %b because data was missing', routingKey)
+          this.log.error('skipping stored record %b, its data is missing', routingKey)
           continue
         }
 
@@ -111,17 +129,40 @@ export class IPNSRepublisher {
 
         try {
           data = decodeExtensibleData(ipnsRecord.data)
-        } catch {
-          this.log.trace('skipping record %b because could not decode data', routingKey)
+        } catch (err) {
+          this.log.error('skipping stored record %b, could not decode its data - %e', routingKey, err)
+          continue
+        }
+
+        let recordExpiry: Date
+
+        try {
+          recordExpiry = NanoDate.fromString(uint8ArrayToString(data.Validity)).toDate()
+        } catch (err) {
+          this.log.error('skipping stored record %b, could not parse its validity - %e', routingKey, err)
           continue
         }
 
         // Only republish records that are within the DHT or record expiry threshold
-        if (!shouldRepublish(created, new Date(uint8ArrayToString(data.Validity)))) {
+        if (!shouldRepublish(created, recordExpiry)) {
           this.log.trace('skipping record %b within republish threshold', routingKey)
           continue
         }
 
+        if (metadata.upkeep === Upkeep.rebroadcast) {
+          // only re-broadcast records that are still valid: an expired record
+          // cannot be revived without re-signing, which the rebroadcast policy
+          // has no key for
+          if (recordExpiry.getTime() <= Date.now()) {
+            this.log.trace('skipping expired record %b for rebroadcast', routingKey)
+            continue
+          }
+
+          recordsToRebroadcast.push({ routingKey, record: ipnsRecord })
+          continue
+        }
+
+        // Upkeep.reissue: re-sign the record with a new validity
         const sequenceNumber = data.Sequence + 1n
         const ttlNs = data.TTL ?? DEFAULT_TTL_NS
         let privKey: PrivateKey
@@ -138,36 +179,151 @@ export class IPNSRepublisher {
             ...options,
             ttlNs
           })
-          recordsToRepublish.push({
+
+          recordsToReissue.push({
             routingKey,
             record: updatedRecord
           })
         } catch (err: any) {
-          this.log.error('error creating updated IPNS record for %s - %e', routingKey, err)
+          this.log.error('error creating updated IPNS record for %b - %e', routingKey, err)
           continue
         }
       }
 
-      this.log(`found ${recordsToRepublish.length}/${listed} records to republish`)
+      this.log(`reissuing ${recordsToReissue.length} and rebroadcasting ${recordsToRebroadcast.length} of ${listed} records`)
+
+      // reissue: re-publish the re-signed record as-is. the jobs run
+      // fire-and-forget, so catch any failure (including a local-store write
+      // that #publishToRouters surfaces) rather than leaving an unhandled rejection
+      for (const { routingKey, record } of recordsToReissue) {
+        queue.add(async () => this.#publishToRouters(routingKey, IPNSEntry.encode(record), options), options)
+          .catch((err: any) => { this.log.error('failed to reissue record %b - %e', routingKey, err) })
+      }
 
-      // Republish each record
-      for (const { routingKey, record } of recordsToRepublish) {
-        // Add job to queue to republish the record to all routers
+      // rebroadcast: resolve the latest record from the network and re-broadcast
+      // whichever of the local and network records is most suitable, so we adopt
+      // a newer record rather than pushing a stale local copy
+      for (const { routingKey, record } of recordsToRebroadcast) {
         queue.add(async () => {
-          try {
-            const marshaledRecord = IPNSEntry.encode(record)
-            await Promise.all(
-              this.routers.map(r => r.put(routingKey, marshaledRecord, options))
-            )
-          } catch (err: any) {
-            this.log.error('error republishing record - %e', err)
-          }
+          const { records } = await findRoutingRecords(this.routers, routingKey, this.keychain, this.log, options)
+          const candidates = [record, ...records]
+          const best = candidates[ipnsSelector(routingKey, candidates)]
+
+          await this.#publishToRouters(routingKey, IPNSEntry.encode(best), { ...options, overwrite: true })
         }, options)
+          .catch((err: any) => { this.log.error('failed to rebroadcast record %b - %e', routingKey, err) })
       }
     } catch (err: any) {
       this.log.error('error during republish - %e', err)
     }
+  }
+
+  /**
+   * Persist the record to the local store first, throwing if that fails: it
+   * holds the record and its upkeep policy, so that failure must surface rather
+   * than be swallowed. Then broadcast to the network routers best-effort,
+   * logging any that fail.
+   */
+  async #publishToRouters (routingKey: Uint8Array, marshaledRecord: Uint8Array, putOptions: IPNSRoutingPutOptions): Promise<void> {
+    const localStoreRouter = this.routers.find(isLocalStoreRouting)
+
+    if (localStoreRouter != null) {
+      await localStoreRouter.put(routingKey, marshaledRecord, putOptions)
+      this.log('published record %b to %s', routingKey, routerName(localStoreRouter))
+    }
+
+    await Promise.all(
+      this.routers
+        .filter(router => router !== localStoreRouter)
+        .map(async (router) => {
+          try {
+            await router.put(routingKey, marshaledRecord, putOptions)
+            this.log('published record %b to %s', routingKey, routerName(router))
+          } catch (err) {
+            this.log.error('failed to publish record %b to %s - %e', routingKey, routerName(router), err)
+          }
+        })
+    )
+  }
+
+  async republish (key: MultihashDigest, options: RepublishOptions = {}): Promise<RepublishResult> {
+    const routingKey = multihashToIPNSRoutingKey(key)
+
+    // read the record we hold for this key directly, it was validated when it
+    // was imported or published, and reading it through resolve() would recurse
+    // and rewrite the local store
+    let record: Uint8Array<ArrayBuffer>
+
+    try {
+      ({ record } = await this.localStore.get(routingKey, options))
+    } catch (err: any) {
+      if (err.name !== 'NotFoundError') {
+        throw err
+      }
+
+      throw new NotFoundError('No local record found to republish - import it first')
+    }
+
+    // re-validate: a record that was valid when stored can have expired since
+    const localRecord = await ipnsValidator(routingKey, record, this.keychain, options)
+
+    // unless we are told not to look, check whether the routing already has a
+    // more suitable record; if so, back off
+    if (options.skipResolution !== true) {
+      const { records } = await findRoutingRecords(this.routers, routingKey, this.keychain, this.log, options)
+      const candidates = [localRecord, ...records]
+      const best = ipnsSelector(routingKey, candidates)
+
+      if (best !== 0) {
+        throw new RecordObsoleteError('A newer record is already published for this key', candidates[best])
+      }
+    }
+
+    const metadata = {
+      upkeep: Upkeep[options.upkeep ?? 'rebroadcast']
+    }
+
+    // a publish that fails on every router does not throw; use onProgress to see
+    // which routers worked and watch the logs
+    await this.#publishToRouters(routingKey, record, {
+      ...options,
+      overwrite: true,
+      metadata
+    })
+
+    return {
+      record: localRecord,
+      publicKey: await extractPublicKey(routingKey, localRecord, this.keychain, options)
+    }
+  }
+
+  async import (key: MultihashDigest, record: IPNSEntry | Uint8Array, options?: AbortOptions): Promise<void> {
+    const routingKey = multihashToIPNSRoutingKey(key)
+    const entry = record instanceof Uint8Array ? IPNSEntry.decode(record) : record
+    const marshaledRecord = IPNSEntry.encode(entry)
+
+    // validate the record, throws if it is invalid (bad signature, expired, etc)
+    await ipnsValidator(routingKey, marshaledRecord, this.keychain, options)
+
+    // do not downgrade a record we already hold
+    try {
+      const { record: existingBytes } = await this.localStore.get(routingKey, options)
+      const existing = IPNSEntry.decode(existingBytes)
+
+      // if the record we already have wins the selector, the incoming one is obsolete
+      if (ipnsSelector(routingKey, [entry, existing]) !== 0) {
+        throw new RecordObsoleteError('A newer record is already stored for this key', existing)
+      }
+    } catch (err: any) {
+      // holding no record for this key is the normal case, everything else
+      // (the RecordObsoleteError above included) is a real failure
+      if (err.name !== 'NotFoundError') {
+        throw err
+      }
+    }
 
-    await queue.onIdle(options) // Wait for all jobs to complete
+    // store the record locally only, without metadata, so it is served but not
+    // automatically republished until republish() gives it an upkeep policy
+    await this.localStore.put(routingKey, marshaledRecord, options)
   }
 }