helia 3.0.1 → 4.0.0-e554493

package/src/storage.ts

@@ -1,172 +0,0 @@
-import { start, stop } from '@libp2p/interface'
-import createMortice from 'mortice'
-import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions } from '@helia/interface/blocks'
-import type { Pins } from '@helia/interface/pins'
-import type { AbortOptions, Startable } from '@libp2p/interface'
-import type { Blockstore } from 'interface-blockstore'
-import type { AwaitIterable } from 'interface-store'
-import type { Mortice } from 'mortice'
-import type { CID } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
-
-export interface BlockStorageInit {
-  holdGcLock?: boolean
-}
-
-export interface GetOptions extends AbortOptions {
-  progress?(evt: Event): void
-}
-
-/**
- * BlockStorage is a hybrid blockstore that puts/gets blocks from a configured
- * blockstore (that may be on disk, s3, or something else). If the blocks are
- * not present Bitswap will be used to fetch them from network peers.
- */
-export class BlockStorage implements Blocks, Startable {
-  public lock: Mortice
-  private readonly child: Blockstore
-  private readonly pins: Pins
-  private started: boolean
-
-  /**
-   * Create a new BlockStorage
-   */
-  constructor (blockstore: Blockstore, pins: Pins, options: BlockStorageInit = {}) {
-    this.child = blockstore
-    this.pins = pins
-    this.lock = createMortice({
-      singleProcess: options.holdGcLock
-    })
-    this.started = false
-  }
-
-  isStarted (): boolean {
-    return this.started
-  }
-
-  async start (): Promise<void> {
-    await start(this.child)
-    this.started = true
-  }
-
-  async stop (): Promise<void> {
-    await stop(this.child)
-    this.started = false
-  }
-
-  unwrap (): Blockstore {
-    return this.child
-  }
-
-  /**
-   * Put a block to the underlying datastore
-   */
-  async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      return await this.child.put(cid, block, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Put a multiple blocks to the underlying datastore
-   */
-  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      yield * this.child.putMany(blocks, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Get a block by cid
-   */
-  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      return await this.child.get(cid, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Get multiple blocks back from an (async) iterable of cids
-   */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      yield * this.child.getMany(cids, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Delete a block from the blockstore
-   */
-  async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
-    const releaseLock = await this.lock.writeLock()
-
-    try {
-      if (await this.pins.isPinned(cid)) {
-        throw new Error('CID was pinned')
-      }
-
-      await this.child.delete(cid, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  /**
-   * Delete multiple blocks from the blockstore
-   */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
-    const releaseLock = await this.lock.writeLock()
-
-    try {
-      const storage = this
-
-      yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
-        for await (const cid of cids) {
-          if (await storage.pins.isPinned(cid)) {
-            throw new Error('CID was pinned')
-          }
-
-          yield cid
-        }
-      }()), options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      return await this.child.has(cid, options)
-    } finally {
-      releaseLock()
-    }
-  }
-
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncIterable<Pair> {
-    const releaseLock = await this.lock.readLock()
-
-    try {
-      yield * this.child.getAll(options)
-    } finally {
-      releaseLock()
-    }
-  }
-}

package/src/utils/dag-walkers.ts

@@ -1,181 +0,0 @@
-/* eslint max-depth: ["error", 7] */
-
-import * as dagCbor from '@ipld/dag-cbor'
-import * as dagJson from '@ipld/dag-json'
-import * as dagPb from '@ipld/dag-pb'
-import * as cborg from 'cborg'
-import { Type, Token } from 'cborg'
-import * as cborgJson from 'cborg/json'
-import { CID } from 'multiformats'
-import { base64 } from 'multiformats/bases/base64'
-import * as json from 'multiformats/codecs/json'
-import * as raw from 'multiformats/codecs/raw'
-import type { DAGWalker } from '../index.js'
-
-/**
- * Dag walker for dag-pb CIDs
- */
-export const dagPbWalker: DAGWalker = {
-  codec: dagPb.code,
-  * walk (block) {
-    const node = dagPb.decode(block)
-
-    yield * node.Links.map(l => l.Hash)
-  }
-}
-
-/**
- * Dag walker for raw CIDs
- */
-export const rawWalker: DAGWalker = {
-  codec: raw.code,
-  * walk () {
-    // no embedded CIDs in a raw block
-  }
-}
-
-// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692
-const CID_TAG = 42
-
-/**
- * Dag walker for dag-cbor CIDs. Does not actually use dag-cbor since
- * all we are interested in is extracting the the CIDs from the block
- * so we can just use cborg for that.
- */
-export const dagCborWalker: DAGWalker = {
-  codec: dagCbor.code,
-  * walk (block) {
-    const cids: CID[] = []
-    const tags: cborg.TagDecoder[] = []
-    tags[CID_TAG] = (bytes) => {
-      if (bytes[0] !== 0) {
-        throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00')
-      }
-
-      const cid = CID.decode(bytes.subarray(1)) // ignore leading 0x00
-
-      cids.push(cid)
-
-      return cid
-    }
-
-    cborg.decode(block, {
-      tags
-    })
-
-    yield * cids
-  }
-}
-
-/**
- * Borrowed from @ipld/dag-json
- */
-class DagJsonTokenizer extends cborgJson.Tokenizer {
-  private readonly tokenBuffer: cborg.Token[]
-
-  constructor (data: Uint8Array, options?: cborg.DecodeOptions) {
-    super(data, options)
-
-    this.tokenBuffer = []
-  }
-
-  done (): boolean {
-    return this.tokenBuffer.length === 0 && super.done()
-  }
-
-  _next (): cborg.Token {
-    if (this.tokenBuffer.length > 0) {
-      // @ts-expect-error https://github.com/Microsoft/TypeScript/issues/30406
-      return this.tokenBuffer.pop()
-    }
-    return super.next()
-  }
-
-  /**
-   * Implements rules outlined in https://github.com/ipld/specs/pull/356
-   */
-  next (): cborg.Token {
-    const token = this._next()
-
-    if (token.type === Type.map) {
-      const keyToken = this._next()
-      if (keyToken.type === Type.string && keyToken.value === '/') {
-        const valueToken = this._next()
-        if (valueToken.type === Type.string) { // *must* be a CID
-          const breakToken = this._next() // swallow the end-of-map token
-          if (breakToken.type !== Type.break) {
-            throw new Error('Invalid encoded CID form')
-          }
-          this.tokenBuffer.push(valueToken) // CID.parse will pick this up after our tag token
-          return new Token(Type.tag, 42, 0)
-        }
-        if (valueToken.type === Type.map) {
-          const innerKeyToken = this._next()
-          if (innerKeyToken.type === Type.string && innerKeyToken.value === 'bytes') {
-            const innerValueToken = this._next()
-            if (innerValueToken.type === Type.string) { // *must* be Bytes
-              for (let i = 0; i < 2; i++) {
-                const breakToken = this._next() // swallow two end-of-map tokens
-                if (breakToken.type !== Type.break) {
-                  throw new Error('Invalid encoded Bytes form')
-                }
-              }
-              const bytes = base64.decode(`m${innerValueToken.value}`)
-              return new Token(Type.bytes, bytes, innerValueToken.value.length)
-            }
-            this.tokenBuffer.push(innerValueToken) // bail
-          }
-          this.tokenBuffer.push(innerKeyToken) // bail
-        }
-        this.tokenBuffer.push(valueToken) // bail
-      }
-      this.tokenBuffer.push(keyToken) // bail
-    }
-    return token
-  }
-}
-
-/**
- * Dag walker for dag-json CIDs. Does not actually use dag-json since
- * all we are interested in is extracting the the CIDs from the block
- * so we can just use cborg/json for that.
- */
-export const dagJsonWalker: DAGWalker = {
-  codec: dagJson.code,
-  * walk (block) {
-    const cids: CID[] = []
-    const tags: cborg.TagDecoder[] = []
-    tags[CID_TAG] = (string) => {
-      const cid = CID.parse(string)
-
-      cids.push(cid)
-
-      return cid
-    }
-
-    cborgJson.decode(block, {
-      tags,
-      tokenizer: new DagJsonTokenizer(block, {
-        tags,
-        allowIndefinite: true,
-        allowUndefined: true,
-        allowNaN: true,
-        allowInfinity: true,
-        allowBigInt: true,
-        strict: false,
-        rejectDuplicateMapKeys: false
-      })
-    })
-
-    yield * cids
-  }
-}
-
-/**
- * Dag walker for json CIDs. JSON has no facility for linking to
- * external blocks so the walker is a no-op.
- */
-export const jsonWalker: DAGWalker = {
-  codec: json.code,
-  * walk () {}
-}

package/src/utils/datastore-version.ts

@@ -1,23 +0,0 @@
-import { type Datastore, Key } from 'interface-datastore'
-import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
-import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-
-const DS_VERSION_KEY = new Key('/version')
-const CURRENT_VERSION = 1
-
-export async function assertDatastoreVersionIsCurrent (datastore: Datastore): Promise<void> {
-  if (!(await datastore.has(DS_VERSION_KEY))) {
-    await datastore.put(DS_VERSION_KEY, uint8ArrayFromString(`${CURRENT_VERSION}`))
-
-    return
-  }
-
-  const buf = await datastore.get(DS_VERSION_KEY)
-  const str = uint8ArrayToString(buf)
-  const version = parseInt(str, 10)
-
-  if (version !== CURRENT_VERSION) {
-    // TODO: write migrations when we break compatibility - for an example, see https://github.com/ipfs/js-ipfs-repo/tree/master/packages/ipfs-repo-migrations
-    throw new Error('Unknown datastore version, a datastore migration may be required')
-  }
-}

package/src/utils/default-hashers.ts

@@ -1,12 +0,0 @@
-import { identity } from 'multiformats/hashes/identity'
-import { sha256, sha512 } from 'multiformats/hashes/sha2'
-import type { MultihashHasher } from 'multiformats/hashes/interface'
-
-export function defaultHashers (hashers: MultihashHasher[] = []): MultihashHasher[] {
-  return [
-    sha256,
-    sha512,
-    identity,
-    ...hashers
-  ]
-}