@audius/sdk 0.0.0

package/src/utils/fileHasher.ts

@@ -0,0 +1,278 @@
+import {
+  ImportCandidate,
+  importer,
+  UserImporterOptions
+} from 'ipfs-unixfs-importer'
+import fs from 'fs'
+import { hrtime } from 'process'
+import { promisify } from 'util'
+import { Stream } from 'stream'
+import type { Blockstore, Options } from 'interface-blockstore'
+import type {
+  AwaitIterable,
+  Pair,
+  Batch,
+  Query,
+  KeyQuery
+} from 'interface-store'
+import type { CID } from 'multiformats/cid'
+
+// Base functionality for only hash logic taken from https://github.com/alanshaw/ipfs-only-hash/blob/master/index.js
+
+export type Content = ReadableStream | Buffer | string
+export interface ImageHasher {
+  options: UserImporterOptions
+  content: ImportCandidate
+}
+export interface NonImageHasher {
+  options: UserImporterOptions
+  content: Uint8Array
+}
+export interface HashedImage {
+  path: string | undefined
+  cid: string
+  size: number
+}
+
+const block: Blockstore = {
+  get: async (key: CID, _options?: Options) => {
+    throw new Error(`unexpected block API get for ${key}`)
+  },
+  put: async (_key: CID, _val: Uint8Array, _options?: Options) => {
+    throw new Error('unexpected block API put')
+  },
+  open: async function (): Promise<void> {
+    throw new Error('Function not implemented.')
+  },
+  close: async function (): Promise<void> {
+    throw new Error('Function not implemented.')
+  },
+  has: async function (_key: CID, _options?: Options): Promise<boolean> {
+    throw new Error('Function not implemented.')
+  },
+  delete: async function (_key: CID, _options?: Options): Promise<void> {
+    throw new Error('Function not implemented.')
+  },
+  putMany: function (
+    _source: AwaitIterable<Pair<CID, Uint8Array>>,
+    _options?: Options
+  ): AsyncIterable<Pair<CID, Uint8Array>> {
+    throw new Error('Function not implemented.')
+  },
+  getMany: function (
+    _source: AwaitIterable<CID>,
+    _options?: Options
+  ): AsyncIterable<Uint8Array> {
+    throw new Error('Function not implemented.')
+  },
+  deleteMany: function (
+    _source: AwaitIterable<CID>,
+    _options?: Options
+  ): AsyncIterable<CID> {
+    throw new Error('Function not implemented.')
+  },
+  batch: function (): Batch<CID, Uint8Array> {
+    throw new Error('Function not implemented.')
+  },
+  query: function (
+    _query: Query<CID, Uint8Array>,
+    _options?: Options
+  ): AsyncIterable<Pair<CID, Uint8Array>> {
+    throw new Error('Function not implemented.')
+  },
+  queryKeys: function (
+    _query: KeyQuery<CID>,
+    _options?: Options
+  ): AsyncIterable<CID> {
+    throw new Error('Function not implemented.')
+  }
+}
+
+export const fileHasher = {
+  convertNanosToMillis(nanoSeconds: bigint) {
+    return nanoSeconds / BigInt(1000000)
+  },
+
+  /**
+   * Used to initalize the only hash fns. See Alan Shaw's reference code for more context.
+   */
+  initImageHasher(
+    content: ImportCandidate,
+    options: UserImporterOptions
+  ): ImageHasher {
+    options = options || {}
+    options.onlyHash = true
+    options.cidVersion = 0
+
+    return { options, content }
+  },
+
+  /**
+   * Used to iniitalize the only hash fns. See Alan Shaw's reference code for more context.
+   */
+  initNonImageHasher(
+    content: Uint8Array,
+    options: UserImporterOptions
+  ): NonImageHasher {
+    options = options || {}
+    options.onlyHash = true
+    options.cidVersion = 0
+
+    return { options, content }
+  },
+
+  /**
+   * Convert content to a buffer; used in `generateNonImageCid()`.
+   * @param {ReadStream|Buffer|string} content if string, should be file path
+   * @param {Object} logger
+   * @returns buffer version of content
+   */
+  async convertToBuffer(content: Content, logger: any): Promise<Buffer> {
+    if (Buffer.isBuffer(content)) return content
+
+    let buffer: any
+    try {
+      if (content instanceof Stream.Readable) {
+        await new Promise((resolve, reject) => {
+          content.on('data', (chunk: any) => buffer.push(chunk))
+          content.on('end', () => resolve(Buffer.concat(buffer)))
+          content.on('error', (err: any) => reject(err))
+        })
+      } else {
+        const fsReadFile = promisify(fs.readFile)
+        buffer = await fsReadFile(content as string)
+      }
+    } catch (e: any) {
+      const errMsg = `[fileHasher - convertToBuffer()] Could not convert content into buffer: ${e.toString()}`
+      logger.error(errMsg)
+      throw new Error(errMsg)
+    }
+
+    return buffer
+  },
+
+  /**
+   * Custom fn to generate the content-hashing logic
+   * @param content a buffer of the content
+   * @param options options for importer
+   * @returns the CID from content addressing logic
+   */
+  async hashNonImages(
+    content: Uint8Array,
+    options: UserImporterOptions = {}
+  ): Promise<string> {
+    ;({ options, content } = fileHasher.initNonImageHasher(content, options))
+
+    let lastCid: string = ''
+    for await (const { cid } of importer([{ content }], block, options)) {
+      lastCid = `${cid}`
+    }
+
+    return lastCid
+  },
+
+  /**
+   * Custom fn to generate the content-hashing logic
+   * @param content an Object[] with the structure [{ path: string, content: buffer }, ...]
+   * @param options options for importer
+   * @returns an Object[] with the structure [{path: <string>, cid: <string>, size: <number>}]
+   *
+   * Example with adding a profile picture:
+   * [
+      {
+        "cid": "QmSRyKvnXwoxPZ9UxqxXPR8NXjcPYBEf1qbNrXyo5USqLL",
+        "path": "blob/150x150.jpg",
+        "size": 3091
+      },
+      {
+        "cid": "QmQQMV9TXxRmDKafZiRvMVkqUNtUu9WGAfukUBS1yCk2ht",
+        "path": "blob/480x480.jpg",
+        "size": 20743
+      },
+      {
+        "cid": "Qmd8cDdDGcWVaLEoJPVFtkKhYMqvHXZTvXcisYjubFxv1F",
+        "path": "blob/1000x1000.jpg",
+        "size": 72621
+      },
+      {
+        "cid": "QmaYCPUH8G14yxetsMgW5J5tpTqPaTp3HMd3EAyffZKSvm",
+        "path": "blob/original.jpg",
+        "size": 185844
+      },
+      {
+        "cid": "QmW8FUFhvaxv1MZmVcUcmR7Tg9WZhGf8xDNBesT9XepwrK",
+        "path": "blob",
+        "size": 282525
+      }
+    ]
+  */
+  async hashImages(
+    content: ImportCandidate,
+    options: UserImporterOptions = {}
+  ): Promise<HashedImage[]> {
+    ;({ options, content } = fileHasher.initImageHasher(content, options))
+
+    const result: HashedImage[] = []
+    for await (const file of importer(content, block, options)) {
+      result.push({
+        path: file.path,
+        cid: `${file.cid}`,
+        size: file.size
+      })
+    }
+
+    // Note: According to https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs-importer#example,
+    // the importer will return the root as the last file resp. This means that the dir should always be the last index.
+    // (As we need it to be in resizeImage.js)
+    return result
+  },
+
+  /**
+   * Generates CID for a non-image file (track segment, track transcode, metadata)
+   * @param {Buffer|ReadStream|string} content a single Buffer, a ReadStream, or path to an existing file
+   * @param {Object?} logger
+   * @returns {string} only hash response cid
+   */
+  async generateNonImageCid(
+    content: Content,
+    logger: any = console
+  ): Promise<string> {
+    const buffer = await fileHasher.convertToBuffer(content, logger)
+
+    const startHashing: bigint = hrtime.bigint()
+    const cid = await fileHasher.hashNonImages(buffer)
+
+    const hashDurationMs = fileHasher.convertNanosToMillis(
+      hrtime.bigint() - startHashing
+    )
+
+    logger.info(
+      `[fileHasher - generateNonImageCid()] CID=${cid} hashDurationMs=${hashDurationMs}ms`
+    )
+
+    return cid
+  },
+
+  /**
+   * Wrapper that generates multihashes for image files
+   * @param {Object[]} content an Object[] with the structure [{ path: string, content: buffer }, ...]
+   * @param {Object?} logger
+   * @returns {HashedImage[]} only hash responses with the structure [{path: <string>, cid: <string>, size: <number>}]
+   */
+  async generateImageCids(
+    content: ImportCandidate,
+    logger: any = console
+  ): Promise<HashedImage[]> {
+    const startHashing: bigint = hrtime.bigint()
+    const hashedImages: HashedImage[] = await fileHasher.hashImages(content)
+    const hashDurationMs = fileHasher.convertNanosToMillis(
+      hrtime.bigint() - startHashing
+    )
+
+    const hashedImagesStr = JSON.stringify(hashedImages)
+    logger.info(
+      `[fileHasher - generateImageCids()] hashedImages=${hashedImagesStr} hashImagesDurationMs=${hashDurationMs}ms`
+    )
+    return hashedImages
+  }
+}

package/src/utils/importContractABI.d.ts

@@ -0,0 +1,9 @@
+import type { AbiItem } from 'web3-utils'
+
+export type ContractABI = {
+  abi: AbiItem[]
+  contractName: string
+}
+
+export const importDataContractABIs: (pathStr: string) => ContractABI
+export const importEthContractABIs: (pathStr: string) => ContractABI

package/src/utils/importContractABI.js

@@ -0,0 +1,19 @@
+const importDataContractABIs = (pathStr) => {
+  // need to specify part of path here because of https://github.com/webpack/webpack/issues/4921#issuecomment-357147299
+  const importFile = require(`../../data-contracts/ABIs/${pathStr}`)
+
+  if (importFile) return importFile
+  else throw new Error(`Data contract ABI not found ${pathStr}`)
+}
+
+const importEthContractABIs = (pathStr) => {
+  // need to specify part of path here because of https://github.com/webpack/webpack/issues/4921#issuecomment-357147299
+  const importFile = require(`../../eth-contracts/ABIs/${pathStr}`)
+
+  if (importFile) return importFile
+  else throw new Error(`Eth contract ABI not found ${pathStr}`)
+}
+
+module.exports = {
+  importDataContractABIs, importEthContractABIs
+}

package/src/utils/index.ts

@@ -0,0 +1,11 @@
+export * from './utils'
+export * from './apiSigning'
+export * from './captcha'
+export * from './estimateGas'
+export * from './multiProvider'
+export * from './promiseFight'
+export * from './signatures'
+export * from './uuid'
+export * from './network'
+export * from './importContractABI'
+export * from './types'

package/src/utils/multiProvider.ts

@@ -0,0 +1,72 @@
+import { callbackify, promisify } from 'util'
+import Web3 from '../web3'
+import { shuffle } from 'lodash'
+import type { HttpProvider, AbstractProvider } from 'web3-core'
+import type { JsonRpcPayload } from 'web3-core-helpers'
+
+const getSendMethod = (provider: HttpProvider | AbstractProvider) => {
+  if ('sendAsync' in provider) {
+    return provider.sendAsync
+  }
+  return provider.send
+}
+
+type Providers = [HttpProvider, ...Array<HttpProvider | AbstractProvider>]
+
+/**
+ * web3 consumes a provider object on initialization
+ * ref: https://github.com/ChainSafe/web3.js/blob/1.x/packages/web3/types/index.d.ts#L31
+ * which references: https://github.com/ChainSafe/web3.js/blob/1.x/packages/web3-core/types/index.d.ts#L436
+ * MultiProvider implements HttpProvider which can be consumed by web3
+ * ref for HttpProvider: https://github.com/ChainSafe/web3.js/blob/1.x/packages/web3-providers-http/types/index.d.ts#L46-L66
+ */
+export class MultiProvider extends Web3.providers.HttpProvider {
+  providers: Providers
+  /**
+   * Creates a MultiProvider
+   * @param {Array<string | Provider> | string} - The providers to use.
+   */
+  constructor(providers: string[] | string) {
+    let web3Providers: string[]
+    if (typeof providers === 'string') {
+      web3Providers = providers.split(',')
+    } else if (!Array.isArray(providers)) {
+      web3Providers = [providers]
+    } else {
+      web3Providers = providers
+    }
+
+    // The below line ensures that we support different types of providers i.e. comma separated strings, an array of strings or an array of providers.
+    const web3ProviderInstances = web3Providers.map(
+      (provider) => new Web3(provider).eth.currentProvider
+    ) as Providers
+    super(web3ProviderInstances[0]?.host)
+
+    if (!web3ProviderInstances.every(getSendMethod)) {
+      throw new Error('Some providers do not have a send method to use.')
+    }
+
+    this.providers = web3ProviderInstances
+
+    // We replace HttpProvider.send with a custom function that supports fallback providers.
+    this.send = callbackify(this._send.bind(this)) // web3 only supports callback functions and not async
+  }
+
+  /**
+   * @method _send
+   * @param {Object} payload
+   */
+  async _send(payload: JsonRpcPayload) {
+    for (const provider of shuffle(this.providers)) {
+      try {
+        const send = promisify(getSendMethod(provider).bind(provider))
+        const result = await send(payload)
+        return result
+      } catch (e) {
+        console.log(e)
+      }
+    }
+
+    throw new Error('All requests failed')
+  }
+}

package/src/utils/network.test.ts

@@ -0,0 +1,127 @@
+import assert from 'assert'
+import nock from 'nock'
+
+import { timeRequests } from './network'
+
+const setupRequest = (
+  url: string,
+  delay: number,
+  version: string,
+  status = 200
+) => {
+  const req = { url }
+  nock(req.url).get('/').delay(delay).reply(status, {
+    data: {
+      version
+    }
+  })
+  return req
+}
+
+describe('timeRequests()', () => {
+  it('sortByVersion = true', async () => {
+    const requests = [
+      setupRequest('https://fastest.audius.co', 50, '1.2.3'),
+      setupRequest('https://fastAndAhead.audius.co', 100, '1.2.4'),
+      setupRequest('https://behind.audius.co', 100, '1.2.2'),
+      setupRequest('https://slow.audius.co', 500, '1.2.3'),
+      setupRequest('https://error.audius.co', 500, '1.2.3', 404)
+    ]
+
+    const res = await timeRequests({
+      requests,
+      sortByVersion: true
+    })
+
+    assert.strictEqual(res[0]?.request.url, 'https://fastAndAhead.audius.co')
+    assert.strictEqual(res[1]?.request.url, 'https://fastest.audius.co')
+    assert.strictEqual(res[2]?.request.url, 'https://slow.audius.co')
+    assert.strictEqual(res[3]?.request.url, 'https://behind.audius.co')
+    assert.strictEqual(res[4]?.request.url, 'https://error.audius.co')
+  })
+
+  it('sortByVersion = false', async () => {
+    const requests = [
+      setupRequest('https://fastest.audius.co', 50, '1.2.3'),
+      setupRequest('https://fastAndAhead.audius.co', 100, '1.2.4'),
+      setupRequest('https://fastAndBehind.audius.co', 100, '1.2.2'),
+      setupRequest('https://slow.audius.co', 500, '1.2.3'),
+      setupRequest('https://slowAndError.audius.co', 500, '1.2.3', 404)
+    ]
+
+    const res = await timeRequests({
+      requests,
+      sortByVersion: false,
+      currentVersion: '1.2.3'
+    })
+
+    // All healthy nodes with valid version should be sorted by request duration, remaining by version then duration
+    assert.strictEqual(res[0]?.request.url, 'https://fastest.audius.co')
+    assert.strictEqual(res[1]?.request.url, 'https://fastAndAhead.audius.co')
+    assert.strictEqual(res[2]?.request.url, 'https://slow.audius.co')
+    assert.strictEqual(res[3]?.request.url, 'https://fastAndBehind.audius.co')
+    assert.strictEqual(res[4]?.request.url, 'https://slowAndError.audius.co')
+  })
+
+  it('respects an equivalency delta', async () => {
+    const allResults: string[] = []
+    for (let i = 0; i < 20; ++i) {
+      const requests = [
+        setupRequest('https://cohort1a.audius.co', 1, '1.2.3'),
+        setupRequest('https://cohort1b.audius.co', 1, '1.2.3'),
+        setupRequest('https://cohort1c.audius.co', 1, '1.2.3'),
+
+        setupRequest('https://cohort2a.audius.co', 100, '1.2.3'),
+        setupRequest('https://cohort2b.audius.co', 101, '1.2.3'),
+        setupRequest('https://cohort2c.audius.co', 102, '1.2.3'),
+
+        setupRequest('https://cohort3a.audius.co', 200, '1.2.3'),
+        setupRequest('https://cohort3b.audius.co', 220, '1.2.3'),
+        setupRequest('https://cohort3c.audius.co', 205, '1.2.3')
+      ]
+      const res = await timeRequests({
+        requests,
+        sortByVersion: true,
+        equivalencyDelta: 50
+      })
+      allResults.push(res.map((r) => r.request.url).join(''))
+
+      // Ensure that each round of testing separates by cohors
+      assert(res[0]?.request.url.startsWith('https://cohort1'))
+      assert(res[1]?.request.url.startsWith('https://cohort1'))
+      assert(res[2]?.request.url.startsWith('https://cohort1'))
+
+      assert(res[3]?.request.url.startsWith('https://cohort2'))
+      assert(res[4]?.request.url.startsWith('https://cohort2'))
+      assert(res[5]?.request.url.startsWith('https://cohort2'))
+
+      assert(res[6]?.request.url.startsWith('https://cohort3'))
+      assert(res[7]?.request.url.startsWith('https://cohort3'))
+      assert(res[8]?.request.url.startsWith('https://cohort3'))
+    }
+
+    // Make sure there is some variance
+    assert(!allResults.every((val) => val === allResults[0]))
+  }).timeout(10000)
+
+  it('filterNonResponsive = true', async () => {
+    const requests = [
+      setupRequest('https://fastest.audius.co', 50, '1.2.3'),
+      setupRequest('https://fast.audius.co', 100, '1.2.3'),
+      setupRequest('https://fastAndBehind.audius.co', 100, '1.2.2'),
+      setupRequest('https://slow.audius.co', 500, '1.2.3'),
+      setupRequest('https://slowAndError.audius.co', 500, '1.2.3', 404)
+    ]
+
+    const res = await timeRequests({
+      requests,
+      sortByVersion: true,
+      filterNonResponsive: true,
+      timeout: 150
+    })
+
+    assert.strictEqual(res[0]?.request.url, 'https://fastest.audius.co')
+    assert.strictEqual(res[1]?.request.url, 'https://fast.audius.co')
+    assert.strictEqual(res[2]?.request.url, 'https://fastAndBehind.audius.co')
+  })
+})