@graphprotocol/graph-cli 0.24.0 → 0.25.2

package/src/commands/test.js

@@ -2,61 +2,125 @@ const { Binary } = require('binary-install-raw')
 const os = require('os')
 const chalk = require('chalk')
 const fetch = require('node-fetch')
+const { filesystem, print } = require('gluegun')
+const { fixParameters } = require('../command-helpers/gluegun')
 const semver = require('semver')
+const { spawn, exec } = require('child_process')
+const yaml = require('js-yaml')
 
 const HELP = `
 ${chalk.bold('graph test')} ${chalk.dim('[options]')} ${chalk.bold('<datasource>')}
 
 ${chalk.dim('Options:')}
-
-  -f  --force                   Overwrite folder + file when downloading
+  -c, --coverage                Run the tests in coverage mode. Works with v0.2.1 and above
+  -d, --docker                  Run the tests in a docker container(Note: Please execute from the root folder of the subgraph)
+  -f  --force                   Binary - overwrites folder + file when downloading. Docker - rebuilds the docker image
   -h, --help                    Show usage information
   -l, --logs                    Logs to the console information about the OS, CPU model and download url (debugging purposes)
+  -r, --recompile               Force-recompile tests (Not available in 0.2.2 and earlier versions)
   -v, --version <tag>           Choose the version of the rust binary that you want to be downloaded/used
   `
 
 module.exports = {
   description: 'Runs rust binary for subgraph testing',
   run: async toolbox => {
-    // Obtain tools
-    let { print } = toolbox
-
     // Read CLI parameters
-    let { f, force, h, help, l, logs, v, version } = toolbox.parameters.options
-    let datasource = toolbox.parameters.first
+    let {
+      c,
+      coverage,
+      d,
+      docker,
+      f,
+      force,
+      h,
+      help,
+      l,
+      logs,
+      r,
+      recompile,
+      v,
+      version,
+    } = toolbox.parameters.options
 
+    let opts = new Map()
     // Support both long and short option variants
-    force = force || f
-    help = help || h
-    logs = logs || l
-    version = version || v
+    opts.set("coverage", coverage || c)
+    opts.set("docker", docker || d)
+    opts.set("force", force || f)
+    opts.set("help", help || h)
+    opts.set("logs", logs || l)
+    opts.set("recompile", recompile || r)
+    opts.set("version", version || v)
+
+    // Fix if a boolean flag (e.g -c, --coverage) has an argument
+    try {
+      fixParameters(toolbox.parameters, {
+        h,
+        help,
+        c,
+        coverage,
+        d,
+        docker,
+        f,
+        force,
+        l,
+        logs,
+        r,
+        recompile
+      })
+    } catch (e) {
+      print.error(e.message)
+      process.exitCode = 1
+      return
+    }
+
+    let datasource = toolbox.parameters.first || toolbox.parameters.array[0]
 
     // Show help text if requested
-    if (help) {
+    if (opts.get("help")) {
       print.info(HELP)
       return
     }
 
-    const platform = getPlatform(logs)
-    if (!version) {
-      let result = await fetch('https://api.github.com/repos/LimeChain/matchstick/releases/latest')
-      let json = await result.json()
-      version = json.tag_name
+    let result = await fetch('https://api.github.com/repos/LimeChain/matchstick/releases/latest')
+    let json = await result.json()
+    opts.set("latestVersion", json.tag_name)
+
+    if(opts.get("docker")) {
+      runDocker(datasource, opts)
+    } else {
+      runBinary(datasource, opts)
     }
+  }
+}
 
-    const url = `https://github.com/LimeChain/matchstick/releases/download/${version}/${platform}`
+async function runBinary(datasource, opts) {
+  let coverageOpt = opts.get("coverage")
+  let forceOpt = opts.get("force")
+  let logsOpt = opts.get("logs")
+  let versionOpt = opts.get("version")
+  let latestVersion = opts.get("latestVersion")
+  let recompileOpt = opts.get("recompile")
 
-    if (logs) {
-      console.log(`Download link: ${url}`)
-    }
+  const platform = getPlatform(logsOpt)
 
-    let binary = new Binary(platform, url, version)
-    await binary.install(force)
-    datasource ? binary.run(datasource) : binary.run()
+  const url = `https://github.com/LimeChain/matchstick/releases/download/${versionOpt || latestVersion}/${platform}`
+
+  if (logsOpt) {
+    print.info(`Download link: ${url}`)
   }
+
+  let binary = new Binary(platform, url, versionOpt || latestVersion)
+  forceOpt ? await binary.install(true) : await binary.install(false)
+  let args = new Array()
+
+  if (coverageOpt) args.push('-c')
+  if (recompileOpt) args.push('-r')
+  if (datasource) args.push(datasource)
+  args.length > 0 ? binary.run(...args) : binary.run()
 }
 
-function getPlatform(logs) {
+function getPlatform(logsOpt) {
   const type = os.type()
   const arch = os.arch()
   const release = os.release()
@@ -64,8 +128,8 @@ function getPlatform(logs) {
   const majorVersion = semver.major(release)
   const isM1 = cpuCore.model.includes("Apple M1")
 
-  if (logs) {
-    console.log(`OS type: ${type}\nOS arch: ${arch}\nOS release: ${release}\nOS major version: ${majorVersion}\nCPU model: ${cpuCore.model}`)
+  if (logsOpt) {
+    print.info(`OS type: ${type}\nOS arch: ${arch}\nOS release: ${release}\nOS major version: ${majorVersion}\nCPU model: ${cpuCore.model}`)
   }
 
   if (arch === 'x64' || (arch === 'arm64' && isM1)) {
@@ -90,3 +154,122 @@ function getPlatform(logs) {
 
   throw new Error(`Unsupported platform: ${type} ${arch} ${majorVersion}`)
 }
+
+async function runDocker(datasource, opts) {
+  let coverageOpt = opts.get("coverage")
+  let forceOpt = opts.get("force")
+  let versionOpt = opts.get("version")
+  let latestVersion = opts.get("latestVersion")
+  let recompileOpt = opts.get("recompile")
+
+  // Remove binary-install-raw binaries, because docker has permission issues
+  // when building the docker images
+  await filesystem.remove("./node_modules/binary-install-raw/bin")
+
+  // Get current working directory
+  let current_folder = await filesystem.cwd()
+
+  // Build the Dockerfile location. Defaults to ./tests/.docker if
+  // a custom testsFolder is not declared in the subgraph.yaml
+  let dockerDir = ""
+
+  try {
+    let doc = await yaml.load(filesystem.read('subgraph.yaml', 'utf8'))
+    testsFolder = doc.testsFolder || './tests'
+    dockerDir = testsFolder.endsWith('/') ? testsFolder + '.docker' : testsFolder + '/.docker'
+  } catch (error) {
+    print.error(error.message)
+    return
+  }
+
+  // Create the Dockerfile
+  try {
+    await filesystem.write(`${dockerDir}/Dockerfile`, dockerfile(versionOpt, latestVersion))
+    print.info('Successfully generated Dockerfile.')
+  } catch (error) {
+    print.info('A problem occurred while generating the Dockerfile. Please attend to the errors below:')
+    print.error(error.message)
+    return
+  }
+
+  // Run a command to check if matchstick image already exists
+  exec('docker images -q matchstick', (error, stdout, stderr) => {
+    // Collect all(if any) flags and options that have to be passed to the matchstick binary
+    let testArgs = ''
+    if (coverageOpt) testArgs = testArgs + ' -c'
+    if (recompileOpt) testArgs = testArgs + ' -r'
+    if (datasource) testArgs = testArgs + ' ' + datasource
+
+    // Build the `docker run` command options and flags
+    let dockerRunOpts = ['run', '-it', '--rm', '--mount', `type=bind,source=${current_folder},target=/matchstick`]
+
+    if(testArgs !== '') {
+      dockerRunOpts.push('-e')
+      dockerRunOpts.push(`ARGS=${testArgs.trim()}`)
+    }
+
+    dockerRunOpts.push('matchstick')
+
+    // If a matchstick image does not exists, the command returns an empty string,
+    // else it'll return the image ID. Skip `docker build` if an image already exists
+    // If `-v/--version` is specified, delete current image(if any) and rebuild.
+    // Use spawn() and {stdio: 'inherit'} so we can see the logs in real time.
+    if(stdout === '' || versionOpt || forceOpt) {
+      if ((stdout !== '' && versionOpt) || forceOpt) {
+        exec('docker image rm matchstick', (error, stdout, stderr) => {
+          print.info(chalk.bold(`Removing matchstick image\n${stdout}`))
+        })
+      }
+      // Build a docker image. If the process has executed successfully
+      // run a container from that image.
+      spawn(
+        'docker',
+        ['build', '--no-cache', '-f', `${dockerDir}/Dockerfile`, '-t', 'matchstick', '.'],
+        { stdio: 'inherit' }
+      ).on('close', code => {
+        if (code === 0) {
+           spawn('docker', dockerRunOpts, { stdio: 'inherit' })
+        }
+      })
+    } else {
+      print.info("Docker image already exists. Skipping `docker build` command.")
+      // Run the container from the existing matchstick docker image
+      spawn('docker', dockerRunOpts, { stdio: 'inherit' })
+    }
+  })
+}
+
+// TODO: Move these in separate file (in a function maybe)
+function dockerfile(versionOpt, latestVersion) {
+  return `
+  FROM ubuntu:20.04
+  ENV ARGS=""
+
+  # Install necessary packages
+  RUN apt update
+  RUN apt install -y nodejs
+  RUN apt install -y npm
+  RUN apt install -y git
+  RUN apt install -y postgresql
+  RUN apt install -y curl
+  RUN apt install -y cmake
+  RUN npm install -g @graphprotocol/graph-cli
+
+  # Download the latest linux binary
+  RUN curl -OL https://github.com/LimeChain/matchstick/releases/download/${versionOpt || latestVersion}/binary-linux-20
+
+  # Make it executable
+  RUN chmod a+x binary-linux-20
+
+  # Create a matchstick dir where the host will be copied
+  RUN mkdir matchstick
+  WORKDIR matchstick
+
+  # Copy host to /matchstick
+  COPY ../ .
+
+  RUN graph codegen
+  RUN graph build
+
+  CMD ../binary-linux-20 \${ARGS}`
+}

package/src/protocols/ethereum/contract.js

@@ -0,0 +1,22 @@
+module.exports = class EthereumContract {
+  static identifierName() {
+    return 'address'
+  }
+
+  constructor(address) {
+    this.address = address
+  }
+
+  validate() {
+    const pattern = /^(0x)?[0-9a-fA-F]{40}$/
+
+    const errorMessage = "Must be 40 hexadecimal characters, with an optional '0x' prefix."
+
+    const valid = pattern.test(this.address)
+
+    return {
+      valid,
+      error: valid ? null : errorMessage,
+    }
+  }
+}

package/src/protocols/ethereum/scaffold/manifest.js

@@ -0,0 +1,32 @@
+const { abiEvents } = require('../../../scaffold/schema')
+const ABI = require('../abi')
+
+const source = ({ contract, contractName }) => `
+      address: '${contract}'
+      abi: ${contractName}`
+
+const mapping = ({ abi, contractName }) => `
+      kind: ethereum/events
+      apiVersion: 0.0.5
+      language: wasm/assemblyscript
+      entities:
+        ${abiEvents(abi)
+          .map(event => `- ${event.get('_alias')}`)
+          .join('\n        ')}
+      abis:
+        - name: ${contractName}
+          file: ./abis/${contractName}.json
+      eventHandlers:
+        ${abiEvents(abi)
+          .map(
+            event => `
+        - event: ${ABI.eventSignature(event)}
+          handler: handle${event.get('_alias')}`,
+          )
+          .join('')}
+      file: ./src/mapping.ts`
+
+module.exports = {
+  source,
+  mapping,
+}

package/src/protocols/ethereum/scaffold/mapping.js

@@ -0,0 +1,76 @@
+const { generateEventFieldAssignments } = require('../../../scaffold/mapping')
+
+const generatePlaceholderHandlers = ({ abi, events, contractName }) =>
+  `
+  import { BigInt } from '@graphprotocol/graph-ts'
+  import { ${contractName}, ${events.map(event => event._alias)} }
+    from '../generated/${contractName}/${contractName}'
+  import { ExampleEntity } from '../generated/schema'
+
+  ${events
+    .map((event, index) =>
+      index === 0
+        ? `
+    export function handle${event._alias}(event: ${event._alias}): void {
+      // Entities can be loaded from the store using a string ID; this ID
+      // needs to be unique across all entities of the same type
+      let entity = ExampleEntity.load(event.transaction.from.toHex())
+
+      // Entities only exist after they have been saved to the store;
+      // \`null\` checks allow to create entities on demand
+      if (!entity) {
+        entity = new ExampleEntity(event.transaction.from.toHex())
+
+        // Entity fields can be set using simple assignments
+        entity.count = BigInt.fromI32(0)
+      }
+
+      // BigInt and BigDecimal math are supported
+      entity.count = entity.count + BigInt.fromI32(1)
+
+      // Entity fields can be set based on event parameters
+      ${generateEventFieldAssignments(event)
+        .slice(0, 2)
+        .join('\n')}
+
+      // Entities can be written to the store with \`.save()\`
+      entity.save()
+
+      // Note: If a handler doesn't require existing field values, it is faster
+      // _not_ to load the entity from the store. Instead, create it fresh with
+      // \`new Entity(...)\`, set the fields that should be updated and save the
+      // entity back to the store. Fields that were not set or unset remain
+      // unchanged, allowing for partial updates to be applied.
+
+      // It is also possible to access smart contracts from mappings. For
+      // example, the contract that has emitted the event can be connected to
+      // with:
+      //
+      // let contract = Contract.bind(event.address)
+      //
+      // The following functions can then be called on this contract to access
+      // state variables and other data:
+      //
+      // ${
+        abi
+          .codeGenerator()
+          .callableFunctions()
+          .isEmpty()
+          ? 'None'
+          : abi
+              .codeGenerator()
+              .callableFunctions()
+              .map(fn => `- contract.${fn.get('name')}(...)`)
+              .join('\n// ')
+      }
+    }
+    `
+        : `
+export function handle${event._alias}(event: ${event._alias}): void {}
+`,
+    )
+    .join('\n')}`
+
+module.exports = {
+  generatePlaceholderHandlers,
+}

package/src/protocols/ethereum/subgraph.js

@@ -1,7 +1,6 @@
 const immutable = require('immutable')
 const ABI = require('./abi')
 const DataSourcesExtractor = require('../../command-helpers/data-sources')
-const { validateContractValues } = require('../../validation')
 
 module.exports = class EthereumSubgraph {
   constructor(options = {}) {
@@ -12,7 +11,6 @@ module.exports = class EthereumSubgraph {
 
   validateManifest() {
     return this.validateAbis()
-      .concat(this.validateContractAddresses())
       .concat(this.validateEvents())
       .concat(this.validateCallFunctions())
   }
@@ -72,18 +70,6 @@ ${abiNames
     return nameErrors.concat(fileErrors)
   }
 
-  validateContractAddresses() {
-    const ethereumAddressPattern = /^(0x)?[0-9a-fA-F]{40}$/
-
-    return validateContractValues(
-      this.manifest,
-      this.protocol,
-      'address',
-      address => ethereumAddressPattern.test(address),
-      "Must be 40 hexadecimal characters, with an optional '0x' prefix.",
-    )
-  }
-
   validateEvents() {
     const dataSourcesAndTemplates = DataSourcesExtractor.fromManifest(this.manifest, this.protocol)
 

package/src/protocols/index.js

@@ -4,6 +4,12 @@ const EthereumTemplateCodeGen = require('./ethereum/codegen/template')
 const EthereumABI = require('./ethereum/abi')
 const EthereumSubgraph = require('./ethereum/subgraph')
 const NearSubgraph = require('./near/subgraph')
+const EthereumContract = require('./ethereum/contract')
+const NearContract = require('./near/contract')
+const EthereumManifestScaffold = require('./ethereum/scaffold/manifest')
+const NearManifestScaffold = require('./near/scaffold/manifest')
+const EthereumMappingScaffold = require('./ethereum/scaffold/mapping')
+const NearMappingScaffold = require('./near/scaffold/mapping')
 
 module.exports = class Protocol {
   static fromDataSources(dataSourcesAndTemplates) {
@@ -15,7 +21,7 @@ module.exports = class Protocol {
     this.name = this.normalizeName(name)
   }
 
-  availableProtocols() {
+  static availableProtocols() {
     return immutable.fromJS({
       // `ethereum/contract` is kept for backwards compatibility.
       // New networks (or protocol perhaps) shouldn't have the `/contract` anymore (unless a new case makes use of it).
@@ -24,15 +30,62 @@ module.exports = class Protocol {
     })
   }
 
+  static availableNetworks() {
+    return immutable.fromJS({
+      ethereum: [
+        'mainnet',
+        'kovan',
+        'rinkeby',
+        'ropsten',
+        'goerli',
+        'poa-core',
+        'poa-sokol',
+        'xdai',
+        'matic',
+        'mumbai',
+        'fantom',
+        'bsc',
+        'chapel',
+        'clover',
+        'avalanche',
+        'fuji',
+        'celo',
+        'celo-alfajores',
+        'fuse',
+        'mbase',
+        'arbitrum-one',
+        'arbitrum-rinkeby',
+        'optimism',
+        'optimism-kovan',
+        'aurora',
+        'aurora-testnet',
+      ],
+      near: [
+        'near-mainnet',
+        'near-testnet'
+      ],
+    })
+  }
+
   normalizeName(name) {
-    return this.availableProtocols()
-      .findKey(possibleNames => possibleNames.includes(name))
+    return Protocol.availableProtocols().findKey(possibleNames =>
+      possibleNames.includes(name),
+    )
+  }
+
+  displayName() {
+    switch (this.name) {
+      case 'ethereum':
+        return 'Ethereum'
+      case 'near':
+        return 'NEAR'
+    }
   }
 
   // Receives a data source kind, and checks if it's valid
   // for the given protocol instance (this).
   isValidKindName(kind) {
-    return this.availableProtocols()
+    return Protocol.availableProtocols()
       .get(this.name, immutable.List())
       .includes(kind)
   }
@@ -40,7 +93,15 @@ module.exports = class Protocol {
   hasABIs() {
     switch (this.name) {
       case 'ethereum':
-      case 'ethereum/contract':
+        return true
+      case 'near':
+        return false
+    }
+  }
+
+  hasEvents() {
+    switch (this.name) {
+      case 'ethereum':
         return true
       case 'near':
         return false
@@ -50,7 +111,6 @@ module.exports = class Protocol {
   getTypeGenerator(options) {
     switch (this.name) {
       case 'ethereum':
-      case 'ethereum/contract':
         return new EthereumTypeGenerator(options)
       case 'near':
         return null
@@ -60,7 +120,6 @@ module.exports = class Protocol {
   getTemplateCodeGen(template) {
     switch (this.name) {
       case 'ethereum':
-      case 'ethereum/contract':
         return new EthereumTemplateCodeGen(template)
       default:
         throw new Error(
@@ -72,7 +131,6 @@ module.exports = class Protocol {
   getABI() {
     switch (this.name) {
       case 'ethereum':
-      case 'ethereum/contract':
         return EthereumABI
       case 'near':
         return null
@@ -84,14 +142,38 @@ module.exports = class Protocol {
 
     switch (this.name) {
       case 'ethereum':
-      case 'ethereum/contract':
         return new EthereumSubgraph(optionsWithProtocol)
       case 'near':
         return new NearSubgraph(optionsWithProtocol)
       default:
-        throw new Error(
-          `Data sources with kind '${this.name}' are not supported yet`,
-        )
+        throw new Error(`Data sources with kind '${this.name}' are not supported yet`)
+    }
+  }
+
+  getContract() {
+    switch (this.name) {
+      case 'ethereum':
+        return EthereumContract
+      case 'near':
+        return NearContract
+    }
+  }
+
+  getManifestScaffold() {
+    switch (this.name) {
+      case 'ethereum':
+        return EthereumManifestScaffold
+      case 'near':
+        return NearManifestScaffold
+    }
+  }
+
+  getMappingScaffold() {
+    switch (this.name) {
+      case 'ethereum':
+        return EthereumMappingScaffold
+      case 'near':
+        return NearMappingScaffold
     }
   }
 }

package/src/protocols/near/contract.js

@@ -0,0 +1,46 @@
+const MINIMUM_ACCOUNT_ID_LENGTH = 2
+const MAXIMUM_ACCOUNT_ID_LENGTH = 64
+
+const RULES_URL = 'https://docs.near.org/docs/concepts/account#account-id-rules'
+
+module.exports = class NearContract {
+  static identifierName() {
+    return 'account'
+  }
+
+  constructor(account) {
+    this.account = account
+  }
+
+  _validateLength() {
+    return this.account.length >= MINIMUM_ACCOUNT_ID_LENGTH &&
+      this.account.length <= MAXIMUM_ACCOUNT_ID_LENGTH
+  }
+
+  _validateFormat() {
+    const pattern = /^(([a-z\d]+[\-_])*[a-z\d]+\.)*([a-z\d]+[\-_])*[a-z\d]+$/
+
+    return pattern.test(this.account)
+  }
+
+  validate() {
+    if (!this._validateLength(this.account)) {
+      return {
+        valid: false,
+        error: `Account must be between '${MINIMUM_ACCOUNT_ID_LENGTH}' and '${MAXIMUM_ACCOUNT_ID_LENGTH}' characters, see ${RULES_URL}`,
+      }
+    }
+
+    if (!this._validateFormat()) {
+      return {
+        valid: false,
+        error: `Account must conform to the rules on ${RULES_URL}`,
+      }
+    }
+
+    return {
+      valid: true,
+      error: null,
+    }
+  }
+}

package/src/protocols/near/scaffold/manifest.js

@@ -0,0 +1,16 @@
+const source = ({ contract }) => `
+      account: '${contract}'`
+
+const mapping = () => `
+      apiVersion: 0.0.5
+      language: wasm/assemblyscript
+      entities:
+        - ExampleEntity
+      receiptHandlers:
+        - handler: handleReceipt
+      file: ./src/mapping.ts`
+
+module.exports = {
+  source,
+  mapping,
+}