helia 0.0.0-270bb98

package/src/pins.ts

@@ -0,0 +1,238 @@
+import type { AddOptions, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
+import { Datastore, Key } from 'interface-datastore'
+import { CID, Version } from 'multiformats/cid'
+import * as cborg from 'cborg'
+import { base36 } from 'multiformats/bases/base36'
+import type { Blockstore } from 'interface-blockstore'
+import PQueue from 'p-queue'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import defer from 'p-defer'
+import type { DAGWalker } from './index.js'
+import { cborWalker, dagPbWalker, jsonWalker, rawWalker } from './utils/dag-walkers.js'
+
+const DEFAULT_DAG_WALKERS = [
+  rawWalker,
+  dagPbWalker,
+  cborWalker,
+  jsonWalker
+]
+
+interface DatastorePin {
+  /**
+   * 0 for a direct pin or an arbitrary (+ve, whole) number or Infinity
+   */
+  depth: number
+
+  /**
+   * User-specific metadata for the pin
+   */
+  metadata: Record<string, string | number | boolean>
+}
+
+interface DatastorePinnedBlock {
+  pinCount: number
+  pinnedBy: Uint8Array[]
+}
+
+const DATASTORE_PIN_PREFIX = '/pin/'
+const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
+const DATASTORE_ENCODING = base36
+// const DAG_WALK_MAX_QUEUE_LENGTH = 10
+const DAG_WALK_QUEUE_CONCURRENCY = 1
+
+interface WalkDagOptions extends AbortOptions {
+  depth: number
+}
+
+function toDSKey (cid: CID): Key {
+  if (cid.version === 0) {
+    cid = cid.toV1()
+  }
+
+  return new Key(`${DATASTORE_PIN_PREFIX}${cid.toString(DATASTORE_ENCODING)}`)
+}
+
+export class PinsImpl implements Pins {
+  private readonly datastore: Datastore
+  private readonly blockstore: Blockstore
+  private dagWalkers: Record<number, DAGWalker>
+
+  constructor (datastore: Datastore, blockstore: Blockstore, dagWalkers: DAGWalker[]) {
+    this.datastore = datastore
+    this.blockstore = blockstore
+    this.dagWalkers = {}
+
+    ;[...DEFAULT_DAG_WALKERS, ...dagWalkers].forEach(dagWalker => {
+      this.dagWalkers[dagWalker.codec] = dagWalker
+    })
+  }
+
+  async add (cid: CID<unknown, number, number, Version>, options: AddOptions = {}): Promise<Pin> {
+    const pinKey = toDSKey(cid)
+
+    if (await this.datastore.has(pinKey)) {
+      throw new Error('Already pinned')
+    }
+
+    const depth = Math.round(options.depth ?? Infinity)
+
+    if (depth < 0) {
+      throw new Error('Depth must be greater than or equal to 0')
+    }
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
+    const queue = new PQueue({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+    void queue.add(async () => {
+      await this.#walkDag(cid, queue, (pinnedBlock) => {
+        // do not update pinned block if this block is already pinned by this CID
+        if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
+          return
+        }
+
+        pinnedBlock.pinCount++
+        pinnedBlock.pinnedBy.push(cid.bytes)
+      }, {
+        ...options,
+        depth
+      })
+    })
+
+    // if a job in the queue errors, throw that error
+    const deferred = defer()
+
+    queue.on('error', (err) => {
+      queue.clear()
+      deferred.reject(err)
+    })
+
+    // wait for the queue to complete or error
+    await Promise.race([
+      queue.onIdle(),
+      deferred.promise
+    ])
+
+    const pin: DatastorePin = {
+      depth,
+      metadata: options.metadata ?? {}
+    }
+
+    await this.datastore.put(pinKey, cborg.encode(pin), options)
+
+    return {
+      cid,
+      ...pin
+    }
+  }
+
+  /**
+   * Walk the DAG behind the passed CID, ensure all blocks are present in the blockstore
+   * and update the pin count for them
+   */
+  async #walkDag (cid: CID, queue: PQueue, withPinnedBlock: (pinnedBlock: DatastorePinnedBlock) => void, options: WalkDagOptions): Promise<void> {
+    if (options.depth === -1) {
+      return
+    }
+
+    const dagWalker = this.dagWalkers[cid.code]
+
+    if (dagWalker == null) {
+      throw new Error(`No dag walker found for cid codec ${cid.code}`)
+    }
+
+    const block = await this.blockstore.get(cid)
+
+    await this.#updatePinnedBlock(cid, withPinnedBlock, options)
+
+    // walk dag, ensure all blocks are present
+    for await (const cid of dagWalker.walk(block)) {
+      void queue.add(async () => {
+        await this.#walkDag(cid, queue, withPinnedBlock, {
+          ...options,
+          depth: options.depth - 1
+        })
+      })
+    }
+  }
+
+  /**
+   * Update the pin count for the CID
+   */
+  async #updatePinnedBlock (cid: CID, withPinnedBlock: (pinnedBlock: DatastorePinnedBlock) => void, options: AbortOptions): Promise<void> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    let pinnedBlock: DatastorePinnedBlock = {
+      pinCount: 0,
+      pinnedBy: []
+    }
+
+    try {
+      pinnedBlock = cborg.decode(await this.datastore.get(blockKey, options))
+    } catch (err: any) {
+      if (err.code !== 'ERR_NOT_FOUND') {
+        throw err
+      }
+    }
+
+    withPinnedBlock(pinnedBlock)
+
+    if (pinnedBlock.pinCount === 0) {
+      if (await this.datastore.has(blockKey)) {
+        await this.datastore.delete(blockKey)
+        return
+      }
+    }
+
+    await this.datastore.put(blockKey, cborg.encode(pinnedBlock), options)
+  }
+
+  async rm (cid: CID<unknown, number, number, Version>, options: RmOptions = {}): Promise<Pin> {
+    const pinKey = toDSKey(cid)
+    const buf = await this.datastore.get(pinKey, options)
+    const pin = cborg.decode(buf)
+
+    await this.datastore.delete(pinKey, options)
+
+    // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
+    const queue = new PQueue({
+      concurrency: DAG_WALK_QUEUE_CONCURRENCY
+    })
+    void queue.add(async () => {
+      await this.#walkDag(cid, queue, (pinnedBlock) => {
+        pinnedBlock.pinCount--
+        pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))
+      }, {
+        ...options,
+        depth: pin.depth
+      })
+    })
+    await queue.onIdle()
+
+    return {
+      cid,
+      ...pin
+    }
+  }
+
+  async * ls (options: LsOptions = {}): AsyncGenerator<Pin, void, undefined> {
+    for await (const { key, value } of this.datastore.query({
+      prefix: DATASTORE_PIN_PREFIX + (options.cid != null ? `${options.cid.toString(base36)}` : '')
+    }, options)) {
+      const cid = CID.parse(key.toString().substring(5), base36)
+      const pin = cborg.decode(value)
+
+      yield {
+        cid,
+        ...pin
+      }
+    }
+  }
+
+  async isPinned (cid: CID, options: IsPinnedOptions = {}): Promise<boolean> {
+    const blockKey = new Key(`${DATASTORE_BLOCK_PREFIX}${DATASTORE_ENCODING.encode(cid.multihash.bytes)}`)
+
+    return await this.datastore.has(blockKey, options)
+  }
+}

package/src/storage.ts

@@ -0,0 +1,209 @@
+import { BaseBlockstore } from 'blockstore-core'
+import merge from 'it-merge'
+import { pushable } from 'it-pushable'
+import filter from 'it-filter'
+import type { Blockstore, KeyQuery, Query } from 'interface-blockstore'
+import type { Bitswap } from 'ipfs-bitswap'
+import type { CID } from 'multiformats/cid'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { AwaitIterable } from 'interface-store'
+import type { Mortice } from 'mortice'
+import createMortice from 'mortice'
+import type { Pins } from '@helia/interface/pins'
+
+export interface BlockStorageOptions extends AbortOptions {
+  progress?: (evt: Event) => void
+}
+
+/**
+ * BlockStorage is a hybrid blockstore that puts/gets blocks from a configured
+ * blockstore (that may be on disk, s3, or something else). If the blocks are
+ * not present Bitswap will be used to fetch them from network peers.
+ */
+export class BlockStorage extends BaseBlockstore implements Blockstore {
+  public lock: Mortice
+  private readonly child: Blockstore
+  private readonly bitswap: Bitswap
+  private readonly pins: Pins
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (blockstore: Blockstore, bitswap: Bitswap, pins: Pins) {
+    super()
+
+    this.child = blockstore
+    this.bitswap = bitswap
+    this.pins = pins
+    this.lock = createMortice()
+  }
+
+  async open (): Promise<void> {
+    await this.child.open()
+  }
+
+  async close (): Promise<void> {
+    await this.child.close()
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  /**
+   * Put a block to the underlying datastore
+   */
+  async put (cid: CID, block: Uint8Array, options: AbortOptions = {}): Promise<void> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      if (this.bitswap.isStarted()) {
+        await this.bitswap.put(cid, block, options)
+      } else {
+        await this.child.put(cid, block, options)
+      }
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Put a multiple blocks to the underlying datastore
+   */
+  async * putMany (blocks: AwaitIterable<{ key: CID, value: Uint8Array }>, options: AbortOptions = {}): AsyncGenerator<{ key: CID, value: Uint8Array }, void, undefined> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      const missingBlocks = filter(blocks, async ({ key }) => { return !(await this.child.has(key)) })
+
+      if (this.bitswap.isStarted()) {
+        yield * this.bitswap.putMany(missingBlocks, options)
+      } else {
+        yield * this.child.putMany(missingBlocks, options)
+      }
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get a block by cid
+   */
+  async get (cid: CID, options: BlockStorageOptions = {}): Promise<Uint8Array> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      if (!(await this.has(cid)) && this.bitswap.isStarted()) {
+        return await this.bitswap.get(cid, options)
+      } else {
+        return await this.child.get(cid, options)
+      }
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Get multiple blocks back from an array of cids
+   */
+  async * getMany (cids: AwaitIterable<CID>, options: BlockStorageOptions = {}): AsyncGenerator<Uint8Array, void, undefined> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      const getFromBitswap = pushable<CID>({ objectMode: true })
+      const getFromChild = pushable<CID>({ objectMode: true })
+
+      void Promise.resolve().then(async () => {
+        for await (const cid of cids) {
+          if (!(await this.has(cid)) && this.bitswap.isStarted()) {
+            getFromBitswap.push(cid)
+          } else {
+            getFromChild.push(cid)
+          }
+        }
+
+        getFromBitswap.end()
+        getFromChild.end()
+      }).catch(err => {
+        getFromBitswap.throw(err)
+      })
+
+      yield * merge(
+        this.bitswap.getMany(getFromBitswap, options),
+        this.child.getMany(getFromChild, options)
+      )
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete a block from the blockstore
+   */
+  async delete (cid: CID, options: AbortOptions = {}): Promise<void> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      if (await this.pins.isPinned(cid)) {
+        throw new Error('CID was pinned')
+      }
+
+      await this.child.delete(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  /**
+   * Delete multiple blocks from the blockstore
+   */
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const releaseLock = await this.lock.writeLock()
+
+    try {
+      const storage = this
+
+      yield * this.child.deleteMany((async function * () {
+        for await (const cid of cids) {
+          if (await storage.pins.isPinned(cid)) {
+            throw new Error('CID was pinned')
+          }
+
+          yield cid
+        }
+      }()), options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      return await this.child.has(cid, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async * query (q: Query, options: AbortOptions = {}): AsyncGenerator<{ key: CID, value: Uint8Array }, void, undefined> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.query(q, options)
+    } finally {
+      releaseLock()
+    }
+  }
+
+  async * queryKeys (q: KeyQuery, options: AbortOptions = {}): AsyncGenerator<CID, void, undefined> {
+    const releaseLock = await this.lock.readLock()
+
+    try {
+      yield * this.child.queryKeys(q, options)
+    } finally {
+      releaseLock()
+    }
+  }
+}

package/src/utils/dag-walkers.ts

@@ -0,0 +1,169 @@
+/* eslint max-depth: ["error", 7] */
+
+import * as dagPb from '@ipld/dag-pb'
+import * as cborg from 'cborg'
+import { Type, Token } from 'cborg'
+import * as cborgJson from 'cborg/json'
+import type { DAGWalker } from '../index.js'
+import * as raw from 'multiformats/codecs/raw'
+import { CID } from 'multiformats'
+import { base64 } from 'multiformats/bases/base64'
+
+/**
+ * Dag walker for dag-pb CIDs
+ */
+export const dagPbWalker: DAGWalker = {
+  codec: dagPb.code,
+  async * walk (block) {
+    const node = dagPb.decode(block)
+
+    yield * node.Links.map(l => l.Hash)
+  }
+}
+
+/**
+ * Dag walker for raw CIDs
+ */
+export const rawWalker: DAGWalker = {
+  codec: raw.code,
+  async * walk () {
+    // no embedded CIDs in a raw block
+  }
+}
+
+// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692
+const CID_TAG = 42
+
+/**
+ * Dag walker for dag-cbor CIDs. Does not actually use dag-cbor since
+ * all we are interested in is extracting the the CIDs from the block
+ * so we can just use cborg for that.
+ */
+export const cborWalker: DAGWalker = {
+  codec: 0x71,
+  async * walk (block) {
+    const cids: CID[] = []
+    const tags: cborg.TagDecoder[] = []
+    tags[CID_TAG] = (bytes) => {
+      if (bytes[0] !== 0) {
+        throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00')
+      }
+
+      const cid = CID.decode(bytes.subarray(1)) // ignore leading 0x00
+
+      cids.push(cid)
+
+      return cid
+    }
+
+    cborg.decode(block, {
+      tags
+    })
+
+    yield * cids
+  }
+}
+
+/**
+ * Borrowed from @ipld/dag-json
+ */
+class DagJsonTokenizer extends cborgJson.Tokenizer {
+  private readonly tokenBuffer: cborg.Token[]
+
+  constructor (data: Uint8Array, options?: cborg.DecodeOptions) {
+    super(data, options)
+
+    this.tokenBuffer = []
+  }
+
+  done (): boolean {
+    return this.tokenBuffer.length === 0 && super.done()
+  }
+
+  _next (): cborg.Token {
+    if (this.tokenBuffer.length > 0) {
+      // @ts-expect-error https://github.com/Microsoft/TypeScript/issues/30406
+      return this.tokenBuffer.pop()
+    }
+    return super.next()
+  }
+
+  /**
+   * Implements rules outlined in https://github.com/ipld/specs/pull/356
+   */
+  next (): cborg.Token {
+    const token = this._next()
+
+    if (token.type === Type.map) {
+      const keyToken = this._next()
+      if (keyToken.type === Type.string && keyToken.value === '/') {
+        const valueToken = this._next()
+        if (valueToken.type === Type.string) { // *must* be a CID
+          const breakToken = this._next() // swallow the end-of-map token
+          if (breakToken.type !== Type.break) {
+            throw new Error('Invalid encoded CID form')
+          }
+          this.tokenBuffer.push(valueToken) // CID.parse will pick this up after our tag token
+          return new Token(Type.tag, 42, 0)
+        }
+        if (valueToken.type === Type.map) {
+          const innerKeyToken = this._next()
+          if (innerKeyToken.type === Type.string && innerKeyToken.value === 'bytes') {
+            const innerValueToken = this._next()
+            if (innerValueToken.type === Type.string) { // *must* be Bytes
+              for (let i = 0; i < 2; i++) {
+                const breakToken = this._next() // swallow two end-of-map tokens
+                if (breakToken.type !== Type.break) {
+                  throw new Error('Invalid encoded Bytes form')
+                }
+              }
+              const bytes = base64.decode(`m${innerValueToken.value}`)
+              return new Token(Type.bytes, bytes, innerValueToken.value.length)
+            }
+            this.tokenBuffer.push(innerValueToken) // bail
+          }
+          this.tokenBuffer.push(innerKeyToken) // bail
+        }
+        this.tokenBuffer.push(valueToken) // bail
+      }
+      this.tokenBuffer.push(keyToken) // bail
+    }
+    return token
+  }
+}
+
+/**
+ * Dag walker for dag-json CIDs. Does not actually use dag-json since
+ * all we are interested in is extracting the the CIDs from the block
+ * so we can just use cborg/json for that.
+ */
+export const jsonWalker: DAGWalker = {
+  codec: 0x0129,
+  async * walk (block) {
+    const cids: CID[] = []
+    const tags: cborg.TagDecoder[] = []
+    tags[CID_TAG] = (string) => {
+      const cid = CID.parse(string)
+
+      cids.push(cid)
+
+      return cid
+    }
+
+    cborgJson.decode(block, {
+      tags,
+      tokenizer: new DagJsonTokenizer(block, {
+        tags,
+        allowIndefinite: true,
+        allowUndefined: true,
+        allowNaN: true,
+        allowInfinity: true,
+        allowBigInt: true,
+        strict: false,
+        rejectDuplicateMapKeys: false
+      })
+    })
+
+    yield * cids
+  }
+}

package/src/utils/datastore-version.ts

@@ -0,0 +1,23 @@
+import { Datastore, Key } from 'interface-datastore'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+
+const DS_VERSION_KEY = new Key('/version')
+const CURRENT_VERSION = 1
+
+export async function assertDatastoreVersionIsCurrent (datastore: Datastore): Promise<void> {
+  if (!(await datastore.has(DS_VERSION_KEY))) {
+    await datastore.put(DS_VERSION_KEY, uint8ArrayFromString(`${CURRENT_VERSION}`))
+
+    return
+  }
+
+  const buf = await datastore.get(DS_VERSION_KEY)
+  const str = uint8ArrayToString(buf)
+  const version = parseInt(str, 10)
+
+  if (version !== CURRENT_VERSION) {
+    // TODO: write migrations when we break compatibility - for an example, see https://github.com/ipfs/js-ipfs-repo/tree/master/packages/ipfs-repo-migrations
+    throw new Error('Unknown datastore version, a datastore migration may be required')
+  }
+}