@graphprotocol/graph-cli 0.29.2 → 0.30.0

package/src/command-helpers/scaffold.js

@@ -1,8 +1,28 @@
 const fs = require('fs-extra')
 const path = require('path')
+const prettier = require('prettier')
+const yaml = require('yaml')
 
 const { step } = require('./spinner')
 const Scaffold = require('../scaffold')
+const { generateEventIndexingHandlers } = require('../scaffold/mapping')
+const { generateEventType, abiEvents } = require('../scaffold/schema')
+const { strings } = require('gluegun')
+const { Map } = require('immutable')
+
+const generateDataSource = async (protocol, contractName, network, contractAddress, abi) => {
+  const protocolManifest = protocol.getManifestScaffold()
+
+  return Map.of(
+    'kind', protocol.name,
+    'name', contractName,
+    'network', network,
+    'source', yaml.parse(prettier.format(protocolManifest.source({contract: contractAddress, contractName}),
+      {parser: 'yaml'})),
+    'mapping', yaml.parse(prettier.format(protocolManifest.mapping({abi, contractName}),
+      {parser: 'yaml'}))
+  ).asMutable()
+}
 
 const generateScaffold = async (
   {
@@ -13,6 +33,7 @@ const generateScaffold = async (
     subgraphName,
     indexEvents,
     contractName = 'Contract',
+    dataSourceName,
     node,
   },
   spinner,
@@ -27,6 +48,7 @@ const generateScaffold = async (
     network,
     contractName,
     subgraphName,
+    dataSourceName,
     node,
   })
 
@@ -59,8 +81,53 @@ const writeScaffold = async (scaffold, directory, spinner) => {
   await writeScaffoldDirectory(scaffold, directory, spinner)
 }
 
+const writeABI = async (abi, contractName, abiPath = `./abis/${contractName}.json`) => {
+  let data = prettier.format(JSON.stringify(abi.data), {
+    parser: 'json',
+  })
+
+  await fs.writeFile(abiPath, data, { encoding: 'utf-8' })
+}
+
+const writeSchema = async (abi, protocol, schemaPath, entities) => {
+  const events = protocol.hasEvents()
+    ? abiEvents(abi).filter(event => entities.indexOf(event.get('name')) === -1).toJS()
+    : []
+
+  let data = prettier.format(
+    events.map(
+        event => generateEventType(event, protocol.name)
+      ).join('\n\n'),
+    {
+      parser: 'graphql',
+    },
+  )
+
+  await fs.appendFile(schemaPath, data, { encoding: 'utf-8' })
+}
+
+const writeMapping = async (abi, protocol, contractName, entities) => {
+  const events = protocol.hasEvents()
+    ? abiEvents(abi).filter(event => entities.indexOf(event.get('name')) === -1).toJS()
+    : []
+
+  let mapping = prettier.format(
+    generateEventIndexingHandlers(
+        events,
+        contractName,
+      ),
+    { parser: 'typescript', semi: false },
+  )
+
+  await fs.writeFile(`./src/${strings.kebabCase(contractName)}.ts`, mapping, { encoding: 'utf-8' })
+}
+
 module.exports = {
   ...module.exports,
   generateScaffold,
   writeScaffold,
+  generateDataSource,
+  writeABI,
+  writeSchema,
+  writeMapping,
 }

package/src/commands/add.js

@@ -0,0 +1,198 @@
+const chalk = require('chalk')
+const toolbox = require('gluegun/toolbox')
+const immutable = require('immutable')
+const { withSpinner } = require('../command-helpers/spinner')
+const Subgraph = require('../subgraph')
+const Protocol = require('../protocols')
+const DataSourcesExtractor = require('../command-helpers/data-sources')
+const { generateDataSource, writeABI, writeSchema, writeMapping } = require('../command-helpers/scaffold')
+const { loadAbiFromEtherscan, loadAbiFromBlockScout } = require('../command-helpers/abi')
+const EthereumABI = require('../protocols/ethereum/abi')
+const { fixParameters } = require('../command-helpers/gluegun')
+
+const HELP = `
+${chalk.bold('graph add')} <address> [<subgraph-manifest default: "./subgraph.yaml">]
+
+${chalk.dim('Options:')}
+
+      --abi <path>              Path to the contract ABI (default: download from Etherscan)
+      --contract-name           Name of the contract (default: Contract)
+      --merge-entities          Whether to merge entities with the same name (default: false)
+  -h, --help                    Show usage information
+`
+
+module.exports = {
+  description: 'Creates a new subgraph with basic scaffolding',
+  run: async toolbox => {
+    // Obtain tools
+    let { print, system } = toolbox
+
+    // Read CLI parameters
+    let {
+      abi,
+      contractName,
+      h,
+      help,
+      mergeEntities
+    } = toolbox.parameters.options
+
+    let address = toolbox.parameters.first
+    let manifestPath = toolbox.parameters.second || './subgraph.yaml'
+    contractName = contractName || 'Contract'
+
+    // Validate the address
+    if (!address) {
+      print.error('No contract address provided')
+      process.exitCode = 1
+      return
+    }
+
+    try {
+      fixParameters(toolbox.parameters, {
+        h,
+        help,
+        mergeEntities,
+      })
+    } catch (e) {
+      print.error(e.message)
+      process.exitCode = 1
+      return
+    }
+
+    // Show help text if requested
+    if (help || h) {
+      print.info(HELP)
+      return
+    }
+
+    const dataSourcesAndTemplates = await DataSourcesExtractor.fromFilePath(manifestPath)
+    let protocol = Protocol.fromDataSources(dataSourcesAndTemplates)
+    let manifest = await Subgraph.load(manifestPath, { protocol })
+    let network = manifest.result.get('dataSources').get(0).get('network')
+    let result = manifest.result.asMutable()
+
+    let entities = getEntities(manifest)
+    let contractNames = getContractNames(manifest)
+    if (contractNames.indexOf(contractName) !== -1) {
+      print.error(
+        `Datasource or template with name ${contractName} already exists, please choose a different name`,
+      )
+      process.exitCode = 1
+      return
+    }
+
+    let ethabi = null
+    if (abi) {
+      ethabi = EthereumABI.load(contractName, abi)
+    } else {
+      if (network === 'poa-core') {
+        ethabi = await loadAbiFromBlockScout(EthereumABI, network, address)
+      } else {
+        ethabi = await loadAbiFromEtherscan(EthereumABI, network, address)
+      }
+    }
+
+    let { collisionEntities, onlyCollisions, abiData } = updateEventNamesOnCollision(ethabi, entities, contractName, mergeEntities)
+    ethabi.data = abiData
+
+    await writeABI(ethabi, contractName, abi)
+    await writeSchema(ethabi, protocol, result.getIn(['schema', 'file']), collisionEntities)
+    await writeMapping(ethabi, protocol, contractName, collisionEntities)
+
+    let dataSources = result.get('dataSources')
+    let dataSource = await generateDataSource(protocol, 
+      contractName, network, address, ethabi)
+
+    // Handle the collisions edge case by copying another data source yaml data
+    if (mergeEntities && onlyCollisions) {
+      let firstDataSource = dataSources.get(0)
+      let source = dataSource.get('source')
+      let mapping = firstDataSource.get('mapping').asMutable()
+
+      // Save the address of the new data source
+      source.abi = firstDataSource.get('source').get('abi')
+
+      dataSource.set('mapping', mapping)
+      dataSource.set('source', source)
+    }
+
+    result.set('dataSources', dataSources.push(dataSource))
+
+    await Subgraph.write(result, manifestPath)
+
+    // Detect Yarn and/or NPM
+    let yarn = await system.which('yarn')
+    let npm = await system.which('npm')
+    if (!yarn && !npm) {
+      print.error(
+        `Neither Yarn nor NPM were found on your system. Please install one of them.`,
+      )
+      process.exitCode = 1
+      return
+    }
+
+    await withSpinner(
+      'Running codegen',
+      'Failed to run codegen',
+      'Warning during codegen',
+      async spinner => {
+        await system.run(yarn ? 'yarn codegen' : 'npm run codegen')
+      }
+    )
+  }
+}
+
+const getEntities = (manifest) => {
+  let dataSources = manifest.result.get('dataSources', immutable.List())
+  let templates = manifest.result.get('templates', immutable.List())
+
+  return dataSources
+    .concat(templates)
+    .map(dataSource => dataSource.getIn(['mapping', 'entities']))
+    .flatten()
+}
+
+const getContractNames = (manifest) => {
+  let dataSources = manifest.result.get('dataSources', immutable.List())
+  let templates = manifest.result.get('templates', immutable.List())
+
+  return dataSources
+    .concat(templates)
+    .map(dataSource => dataSource.get('name'))
+}
+
+const updateEventNamesOnCollision = (ethabi, entities, contractName, mergeEntities) => {
+  let abiData = ethabi.data
+  let { print } = toolbox
+  let collisionEntities = []
+  let onlyCollisions = true
+
+  for (let i = 0; i < abiData.size; i++) {
+    let dataRow = abiData.get(i).asMutable()
+
+    if (dataRow.get('type') === 'event'){
+      if (entities.indexOf(dataRow.get('name')) !== -1) {
+        if (entities.indexOf(`${contractName}${dataRow.get('name')}`) !== -1) {
+          print.error(`Contract name ('${contractName}') 
+            + event name ('${dataRow.get('name')}') entity already exists.`)
+          process.exitCode = 1
+          return
+        }
+        
+        if (mergeEntities) {
+          collisionEntities.push(dataRow.get('name'))
+          abiData = abiData.asImmutable().delete(i) // needs to be immutable when deleting, yes you read that right - https://github.com/immutable-js/immutable-js/issues/1901
+          i-- // deletion also shifts values to the left
+          continue
+        } else {
+          dataRow.set('name', `${contractName}${dataRow.get('name')}`)
+        }
+      } else {
+        onlyCollisions = false
+      }
+    }
+    abiData = abiData.asMutable().set(i, dataRow)
+  }
+
+  return { abiData, collisionEntities, onlyCollisions }
+}

package/src/commands/deploy.js

@@ -294,7 +294,6 @@ $ graph create --node ${node} ${subgraphName}`)
             const base = requestUrl.protocol + '//' + requestUrl.hostname
             let playground = res.playground
             let queries = res.queries
-            let subscriptions = res.subscriptions
 
             // Add a base URL if graph-node did not return the full URL
             if (playground.charAt(0) === ':') {
@@ -303,9 +302,6 @@ $ graph create --node ${node} ${subgraphName}`)
             if (queries.charAt(0) === ':') {
               queries = base + queries
             }
-            if (subscriptions.charAt(0) === ':') {
-              subscriptions = base + subscriptions
-            }
 
             if (isHostedService) {
               print.success(
@@ -318,7 +314,6 @@ $ graph create --node ${node} ${subgraphName}`)
             }
             print.info('\nSubgraph endpoints:')
             print.info(`Queries (HTTP):     ${queries}`)
-            print.info(`Subscriptions (WS): ${subscriptions}`)
             print.info(``)
           }
         },

package/src/commands/init.js

@@ -16,6 +16,7 @@ const { initNetworksConfig } = require('../command-helpers/network')
 const { withSpinner, step } = require('../command-helpers/spinner')
 const { fixParameters } = require('../command-helpers/gluegun')
 const { chooseNodeUrl } = require('../command-helpers/node')
+const { loadAbiFromEtherscan, loadAbiFromBlockScout } = require('../command-helpers/abi')
 const { generateScaffold, writeScaffold } = require('../command-helpers/scaffold')
 const { abiEvents } = require('../scaffold/schema')
 const { validateContract } = require('../validation')
@@ -57,6 +58,11 @@ ${chalk.dim.underline('NEAR:')}
 
       --network <${availableNetworks.get('near').join('|')}>
                                  Selects the network the contract is deployed to
+
+${chalk.dim.underline('Cosmos:')}
+
+      --network <${availableNetworks.get('cosmos').join('|')}>
+                                 Selects the network the contract is deployed to
 `
 
 const processInitForm = async (
@@ -169,6 +175,11 @@ const processInitForm = async (
         return value
       },
     },
+    // TODO:
+    //
+    // protocols that don't support contract
+    // - arweave
+    // - tendermint
     {
       type: 'input',
       name: 'contract',
@@ -176,10 +187,10 @@ const processInitForm = async (
         ProtocolContract = protocolInstance.getContract()
         return `Contract ${ProtocolContract.identifierName()}`
       },
-      skip: fromExample !== undefined,
+      skip: () => fromExample !== undefined || !protocolInstance.hasContract(),
       initial: contract,
       validate: async value => {
-        if (fromExample !== undefined) {
+        if (fromExample !== undefined || !protocolInstance.hasContract()) {
           return true
         }
 
@@ -237,11 +248,11 @@ const processInitForm = async (
       name: 'contractName',
       message: 'Contract Name',
       initial: contractName || 'Contract',
-      skip: () => fromExample !== undefined,
+      skip: () => fromExample !== undefined || !protocolInstance.hasContract(),
       validate: value => value && value.length > 0,
       result: value => {
-        contractName = value;
-        return value;
+        contractName = value
+        return value
       }
     },
   ]
@@ -254,67 +265,6 @@ const processInitForm = async (
   }
 }
 
-const loadAbiFromBlockScout = async (ABI, network, address) =>
-  await withSpinner(
-    `Fetching ABI from BlockScout`,
-    `Failed to fetch ABI from BlockScout`,
-    `Warnings while fetching ABI from BlockScout`,
-    async spinner => {
-      let result = await fetch(
-        `https://blockscout.com/${
-          network.replace('-', '/')
-        }/api?module=contract&action=getabi&address=${address}`,
-      )
-      let json = await result.json()
-
-      // BlockScout returns a JSON object that has a `status`, a `message` and
-      // a `result` field. The `status` is '0' in case of errors and '1' in
-      // case of success
-      if (json.status === '1') {
-        return new ABI('Contract', undefined, immutable.fromJS(JSON.parse(json.result)))
-      } else {
-        throw new Error('ABI not found, try loading it from a local file')
-      }
-    },
-  )
-
-const getEtherscanLikeAPIUrl = (network) => {
-  switch(network){
-    case "mainnet": return `https://api.etherscan.io/api`;
-    case "arbitrum-one": return `https://api.arbiscan.io/api`;
-    case "bsc": return `https://api.bscscan.com/api`;
-    case "matic": return `https://api.polygonscan.com/api`;
-    case "mumbai": return `https://api-testnet.polygonscan.com/api`;
-    case "aurora": return `https://api.aurorascan.dev/api`;
-    case "aurora-testnet": return `https://api-testnet.aurorascan.dev/api`;
-    case "optimism-kovan": return `https://api-kovan-optimistic.etherscan.io/api`;
-    default: return `https://api-${network}.etherscan.io/api`;
-  }
-}
-
-const loadAbiFromEtherscan = async (ABI, network, address) =>
-  await withSpinner(
-    `Fetching ABI from Etherscan`,
-    `Failed to fetch ABI from Etherscan`,
-    `Warnings while fetching ABI from Etherscan`,
-    async spinner => {
-      const scanApiUrl = getEtherscanLikeAPIUrl(network);
-      let result = await fetch(
-        `${scanApiUrl}?module=contract&action=getabi&address=${address}`,
-      )
-      let json = await result.json()
-
-      // Etherscan returns a JSON object that has a `status`, a `message` and
-      // a `result` field. The `status` is '0' in case of errors and '1' in
-      // case of success
-      if (json.status === '1') {
-        return new ABI('Contract', undefined, immutable.fromJS(JSON.parse(json.result)))
-      } else {
-        throw new Error('ABI not found, try loading it from a local file')
-      }
-    },
-  )
-
 const loadAbiFromFile = async (ABI, filename) => {
   let exists = await toolbox.filesystem.exists(filename)
 
@@ -768,6 +718,7 @@ const initSubgraphFromContract = async (
     contract,
     indexEvents,
     contractName,
+    dataSourceName,
     node,
     studio,
     product,
@@ -822,6 +773,7 @@ const initSubgraphFromContract = async (
           contract,
           indexEvents,
           contractName,
+          dataSourceName,
           node,
         },
         spinner,
@@ -835,11 +787,13 @@ const initSubgraphFromContract = async (
     return
   }
 
-  let identifierName = protocolInstance.getContract().identifierName()
-  let networkConf = await initNetworksConfig(toolbox, directory, identifierName)
-  if (networkConf !== true) {
-    process.exitCode = 1
-    return
+  if (protocolInstance.hasContract()) {
+    let identifierName = protocolInstance.getContract().identifierName()
+    let networkConf = await initNetworksConfig(toolbox, directory, identifierName)
+    if (networkConf !== true) {
+      process.exitCode = 1
+      return
+    }
   }
 
   // Initialize a fresh Git repository

package/src/migrations/mapping_api_version_0_0_5.js

@@ -21,7 +21,7 @@ module.exports = {
 
     let manifest = loadManifest(manifestFile)
     return (
-      // Only migrate if the graph-ts version is >= 0.23.0...
+      // Only migrate if the graph-ts version is >= 0.24.0...
       // Coerce needed because we may be dealing with an alpha version
       // and in the `semver` library this would not return true on equality.
       semver.gte(semver.coerce(graphTsVersion), '0.24.0') &&

package/src/migrations/mapping_api_version_0_0_6.js

@@ -0,0 +1,74 @@
+const fs = require('fs-extra')
+const semver = require('semver')
+const toolbox = require('gluegun/toolbox')
+const yaml = require('js-yaml')
+const { loadManifest } = require('./util/load-manifest')
+const { getGraphTsVersion } = require('./util/versions')
+
+// If any of the manifest apiVersions are 0.0.6, replace them with 0.0.7
+module.exports = {
+  name: 'Bump mapping apiVersion from 0.0.6 to 0.0.7',
+  predicate: async ({ sourceDir, manifestFile }) => {
+    // Obtain the graph-ts version, if possible
+    let graphTsVersion
+    try {
+      graphTsVersion = await getGraphTsVersion(sourceDir)
+    } catch (_) {
+      // If we cannot obtain the version, return a hint that the graph-ts
+      // hasn't been installed yet
+      return 'graph-ts dependency not installed yet'
+    }
+
+    let manifest = loadManifest(manifestFile)
+    return (
+      // Only migrate if the graph-ts version is >= 0.27.0...
+      // Coerce needed because we may be dealing with an alpha version
+      // and in the `semver` library this would not return true on equality.
+      semver.gte(semver.coerce(graphTsVersion), '0.27.0') &&
+      // ...and we have a manifest with mapping > apiVersion = 0.0.6
+      manifest &&
+      typeof manifest === 'object' &&
+      Array.isArray(manifest.dataSources) &&
+      (manifest.dataSources.reduce(
+        (hasOldMappings, dataSource) =>
+          hasOldMappings ||
+          (typeof dataSource === 'object' &&
+            dataSource.mapping &&
+            typeof dataSource.mapping === 'object' &&
+            dataSource.mapping.apiVersion === '0.0.6'),
+        false,
+      ) ||
+        (Array.isArray(manifest.templates) &&
+          manifest.templates.reduce(
+            (hasOldMappings, template) =>
+              hasOldMappings ||
+              (typeof template === 'object' &&
+                template.mapping &&
+                typeof template.mapping === 'object' &&
+                template.mapping.apiVersion === '0.0.6'),
+            false,
+          )))
+    )
+  },
+  apply: async ({ manifestFile }) => {
+    // Make sure we catch all variants; we could load the manifest
+    // and replace the values in the data structures here; unfortunately
+    // writing that back to the file messes with the formatting more than
+    // we'd like; that's why for now, we use a simple patching approach
+    await toolbox.patching.replace(
+      manifestFile,
+      new RegExp('apiVersion: 0.0.6', 'g'),
+      'apiVersion: 0.0.7',
+    )
+    await toolbox.patching.replace(
+      manifestFile,
+      new RegExp("apiVersion: '0.0.6'", 'g'),
+      "apiVersion: '0.0.7'",
+    )
+    await toolbox.patching.replace(
+      manifestFile,
+      new RegExp('apiVersion: "0.0.6"', 'g'),
+      'apiVersion: "0.0.7"',
+    )
+  },
+}

package/src/migrations/spec_version_0_0_2.js

@@ -6,7 +6,7 @@ const { loadManifest } = require('./util/load-manifest')
 // Spec version to 0.0.2 uses top level templates. graph-cli no longer supports
 // 0.0.1 which used nested templates.
 module.exports = {
-  name: 'Bump mapping specVersion from 0.0.1 to 0.0.2',
+  name: 'Bump manifest specVersion from 0.0.1 to 0.0.2',
   predicate: async ({ sourceDir, manifestFile }) => {
     let manifest = await loadManifest(manifestFile)
     return (

package/src/migrations/spec_version_0_0_3.js

@@ -0,0 +1,29 @@
+const fs = require('fs-extra')
+const toolbox = require('gluegun/toolbox')
+const yaml = require('js-yaml')
+const { loadManifest } = require('./util/load-manifest')
+
+// Spec version 0.0.4 uses feature management, but features are
+// detected and validated by the graph-node instance during subgraph
+// deployment.
+//
+// Also, we skip spec version 0.0.3, which is considered invalid and
+// non-canonical.
+module.exports = {
+  name: 'Bump manifest specVersion from 0.0.2 to 0.0.4',
+  predicate: async ({ sourceDir, manifestFile }) => {
+    let manifest = await loadManifest(manifestFile)
+    return (
+      manifest &&
+      typeof manifest === 'object' &&
+      manifest.specVersion &&
+      (manifest.specVersion === '0.0.2' || manifest.specVersion === '0.0.3')
+    )
+  },
+  apply: async ({ manifestFile }) => {
+    await toolbox.patching.patch(manifestFile, {
+      insert: 'specVersion: 0.0.4',
+      replace: new RegExp(`specVersion: ['"]?0.0.[23]['"]?`),
+    })
+  },
+}

package/src/migrations/spec_version_0_0_4.js

@@ -0,0 +1,25 @@
+const fs = require('fs-extra')
+const toolbox = require('gluegun/toolbox')
+const yaml = require('js-yaml')
+const { loadManifest } = require('./util/load-manifest')
+
+// Spec version 0.0.5 let event handlers require transaction receipt
+// data to be accessed in the runtime.
+module.exports = {
+  name: 'Bump manifest specVersion from 0.0.4 to 0.0.5',
+  predicate: async ({ sourceDir, manifestFile }) => {
+    let manifest = await loadManifest(manifestFile)
+    return (
+      manifest &&
+      typeof manifest === 'object' &&
+      manifest.specVersion &&
+      manifest.specVersion === '0.0.4'
+    )
+  },
+  apply: async ({ manifestFile }) => {
+    await toolbox.patching.patch(manifestFile, {
+      insert: 'specVersion: 0.0.5',
+      replace: new RegExp(`specVersion: ['"]?0.0.4['"]?`),
+    })
+  },
+}

package/src/migrations.js

@@ -5,7 +5,11 @@ const MIGRATIONS = [
   require('./migrations/mapping_api_version_0_0_2'),
   require('./migrations/mapping_api_version_0_0_3'),
   require('./migrations/mapping_api_version_0_0_4'),
+  require('./migrations/mapping_api_version_0_0_5'),
+  require('./migrations/mapping_api_version_0_0_6'),
   require('./migrations/spec_version_0_0_2'),
+  require('./migrations/spec_version_0_0_3'),
+  require('./migrations/spec_version_0_0_4'),
 ]
 
 const applyMigrations = async options =>