@sentio/cli 1.0.0-development

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@sentio/cli",
+  "license": "Apache-2.0",
+  "version": "1.0.0-development",
+  "scripts": {
+    "compile": "tsc -p . && cp src/webpack.config.js lib/",
+    "build": "yarn compile",
+    "cli": "ts-node src/cli.ts",
+    "test": "jest"
+  },
+  "dependencies": {
+    "command-line-args": "^5.2.1",
+    "command-line-usage": "^6.1.3",
+    "express": "^4.18.2",
+    "form-data": "^4.0.0",
+    "fs-extra": "^11.0.0",
+    "js-yaml": "^4.1.0",
+    "node-fetch": "2",
+    "open": "^8.4.0",
+    "ts-loader": "^9.3.0",
+    "webpack": "^5.72.1",
+    "webpack-cli": "^5.0.0"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.3.1",
+    "@types/command-line-args": "^5.2.0",
+    "@types/command-line-usage": "^5.0.2",
+    "@types/expect": "^24.3.0",
+    "@types/express": "^4.17.14",
+    "@types/fs-extra": "^9.0.13",
+    "@types/google-protobuf": "^3.15.6",
+    "@types/js-yaml": "^4.0.5",
+    "@types/mkdirp": "^1.0.2",
+    "@types/node": "^18.0.4",
+    "@types/node-fetch": "^2.6.2",
+    "chai": "^4.3.6",
+    "clean-publish": "^4.0.1",
+    "conventional-changelog-conventionalcommits": "^5.0.0",
+    "eslint": "^8.27.0",
+    "eslint-import-resolver-typescript": "^3.5.2",
+    "eslint-plugin-import": "^2.26.0",
+    "jest": "^29.0.3",
+    "semantic-release": "^19.0.5",
+    "ts-jest": "^29.0.3",
+    "ts-node": "^10.9.1",
+    "tsconfig-paths": "^4.0.0",
+    "typescript": "^4.8.0"
+  },
+  "bin": {
+    "sentio": "./lib/cli.js"
+  },
+  "main": "./lib/index.js",
+  "types": "./lib/index.d.ts",
+  "module": "./lib/index.js",
+  "files": [
+    "{lib,src,templates}",
+    "!{lib,src}/tests",
+    "!**/*.test.{js,ts}"
+  ],
+  "engines": {
+    "node": ">=16"
+  }
+}

package/src/build.ts

@@ -0,0 +1,96 @@
+import chalk from 'chalk'
+import path from 'path'
+import fs from 'fs'
+import { exec } from 'child_process'
+
+export async function buildProcessor(onlyGen: boolean) {
+  if (!onlyGen) {
+    await installDeps()
+  }
+
+  // targets.forEach(async (target) => await buildProcessorForTarget(onlyGen, target))
+  // for (const target) {
+  await buildProcessorForTarget(onlyGen)
+  // }
+
+  if (!onlyGen) {
+    const WEBPACK_CONFIG = path.join(__dirname, 'webpack.config.js')
+    await execStep('yarn tsc -p .', 'Compile')
+    await execStep('yarn webpack --config=' + WEBPACK_CONFIG, 'Packaging')
+  }
+}
+
+async function buildProcessorForTarget(onlyGen: boolean) {
+  await codeGenEthersProcessor(path.join('abis', 'evm'))
+
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const solanaModule = require('@sentio/sdk/lib/solana-codegen/codegen')
+  solanaModule.codeGenSolanaProcessor(path.join('abis', 'solana'))
+
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const aptosModuole = require('@sentio/sdk/lib/aptos-codegen/codegen')
+  aptosModuole.codeGenAptosProcessor(path.join('abis', 'aptos'))
+
+  if (onlyGen) {
+    return
+  }
+}
+
+async function installDeps() {
+  await execStep('yarn install --ignore-scripts', 'Yarn Install')
+}
+
+export async function codeGenEthersProcessor(
+  abisDir: string,
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  ETHERS_TARGET = path.dirname(require.resolve('@sentio/sdk/lib/target-ethers-sentio')),
+  outDir = 'src/types/internal'
+) {
+  if (!fs.existsSync(abisDir)) {
+    return
+  }
+
+  let haveJson = false
+  const files = fs.readdirSync(abisDir)
+  for (const file of files) {
+    if (file.toLowerCase().endsWith('.json')) {
+      haveJson = true
+      break
+    }
+  }
+  if (!haveJson) {
+    return
+  }
+
+  console.log(chalk.green('Generated Types for EVM'))
+
+  // TODO this will fail during postinstall, need to locate real typechain path
+  await execStep(
+    'yarn typechain --target ' + ETHERS_TARGET + ` --out-dir ${outDir} ${path.join(abisDir, '*.json')}`,
+    'Type definitions gen'
+  )
+}
+
+async function execStep(cmd: string, stepName: string) {
+  const child = exec(cmd)
+  console.log(chalk.blue(stepName + ' begin'))
+
+  if (!child.stdout || !child.stderr) {
+    console.error(chalk.red(stepName + ' failed'))
+    process.exit(1)
+  }
+
+  child.stdout.pipe(process.stdout)
+  child.stderr.pipe(process.stderr)
+
+  await new Promise((resolve) => {
+    child.on('close', resolve)
+  })
+
+  if (child.exitCode) {
+    console.error(chalk.red(stepName + ' failed'))
+    process.exit(child.exitCode)
+  }
+  console.log(chalk.blue(stepName + ' success'))
+  console.log()
+}

package/src/cli.ts

@@ -0,0 +1,184 @@
+#!/usr/bin/env node
+
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import fs from 'fs'
+import path from 'path'
+
+import yaml from 'js-yaml'
+import { finalizeHost, FinalizeProjectName, SentioProjectConfig } from './config'
+import { uploadFile } from './upload'
+import chalk from 'chalk'
+import { buildProcessor } from './build'
+import { runCreate } from './commands/run-create'
+import { runVersion } from './commands/run-version'
+import { runLogin } from './commands/run-login'
+
+const mainDefinitions = [{ name: 'command', defaultOption: true }]
+const mainOptions = commandLineArgs(mainDefinitions, {
+  stopAtFirstUnknown: true,
+})
+const argv = mainOptions._unknown || []
+
+if (!mainOptions.command) {
+  usage()
+}
+
+if (mainOptions.command === 'login') {
+  runLogin(argv)
+} else if (mainOptions.command === 'create') {
+  runCreate(argv)
+} else if (mainOptions.command === 'version') {
+  runVersion(argv)
+} else {
+  // For all the commands that need read project configs
+  // TODO move them to their own modules
+
+  // Process configs
+  let processorConfig: SentioProjectConfig = { host: '', project: '', build: true, debug: false }
+  // Fist step, read from project yaml file
+  try {
+    console.log(chalk.blue('Loading Process config'))
+    // TODO correctly located sentio.yaml
+    const pwd = process.cwd()
+    const packageJson = path.join(pwd, 'package.json')
+    if (!fs.existsSync(packageJson)) {
+      console.error('package.json not found, please run this command in the root of your project')
+      process.exit(1)
+    }
+
+    const yamlPath = path.join(pwd, 'sentio.yaml')
+    if (!fs.existsSync(yamlPath)) {
+      console.error('sentio.yaml not found, please create one according to: TODO docs')
+      process.exit(1)
+    }
+
+    processorConfig = yaml.load(fs.readFileSync('sentio.yaml', 'utf8')) as SentioProjectConfig
+    if (!processorConfig.project === undefined) {
+      console.error('Config yaml must have contain a valid project identifier')
+      process.exit(1)
+    }
+    if (processorConfig.build === undefined) {
+      processorConfig.build = true
+    }
+    if (!processorConfig.host) {
+      processorConfig.host = 'prod'
+    }
+    if (processorConfig.debug === undefined) {
+      processorConfig.debug = false
+    }
+
+    // if (!processorConfig.source) {
+    //   processorConfig.source = 'src/processor.ts'
+    // }
+    // if (!processorConfig.targets) {
+    //   console.warn('targets is not defined, use EVM as the default target')
+    //   processorConfig.targets = []
+    // }
+    // if (processorConfig.targets.length === 0) {
+    //   // By default evm
+    //   processorConfig.targets.push({ chain: EVM })
+    // }
+  } catch (e) {
+    console.error(e)
+    process.exit(1)
+  }
+
+  if (mainOptions.command === 'upload') {
+    const optionDefinitions = [
+      {
+        name: 'help',
+        alias: 'h',
+        type: Boolean,
+        description: 'Display this usage guide.',
+      },
+      {
+        name: 'api-key',
+        type: String,
+        description: '(Optional) Manually provide API key rather than use saved credential',
+      },
+      {
+        name: 'host',
+        description: '(Optional) Override Sentio Host name',
+        type: String,
+      },
+      {
+        name: 'owner',
+        description: '(Optional) Override Project owner',
+        type: String,
+      },
+      {
+        name: 'nobuild',
+        description: '(Optional) Skip build & pack file before uploading, default false',
+        type: Boolean,
+      },
+      {
+        name: 'debug',
+        description: '(Optional) Set driver logging level to debug',
+        type: Boolean,
+      },
+    ]
+    const options = commandLineArgs(optionDefinitions, { argv })
+    if (options.help) {
+      const usage = commandLineUsage([
+        {
+          header: 'Sentio upload',
+          content: 'sentio upload',
+        },
+        {
+          header: 'Options',
+          optionList: optionDefinitions,
+        },
+      ])
+      console.log(usage)
+    } else {
+      if (options.host) {
+        processorConfig.host = options.host
+      }
+      if (options.nobuild) {
+        processorConfig.build = false
+      }
+      if (options.debug) {
+        processorConfig.debug = true
+      }
+      finalizeHost(processorConfig)
+      FinalizeProjectName(processorConfig, options.owner)
+      console.log(processorConfig)
+
+      let apiOverride = undefined
+      if (options['api-key']) {
+        apiOverride = options['api-key']
+      }
+      uploadFile(processorConfig, apiOverride)
+    }
+  } else if (mainOptions.command === 'build') {
+    buildProcessor(false)
+  } else if (mainOptions.command === 'gen') {
+    buildProcessor(true)
+  } else {
+    usage()
+  }
+}
+
+function usage() {
+  const usage = commandLineUsage([
+    {
+      header: 'Sentio',
+      content: 'Login & Manage your project files to Sentio.',
+    },
+    {
+      header: 'Usage',
+      content: [
+        'sentio <command> --help\t\tshow detail usage of specific command',
+        'sentio login\t\t\t\tlogin to sentio',
+        'sentio create\t\t\t\tcreate a template project',
+        'sentio upload\t\t\t\tbuild and upload processor to sentio',
+        'sentio gen\t\t\t\tgenerate abi',
+        'sentio build\t\t\t\tgenerate abi and build',
+        'sentio version\t\t\tcurrent cli version',
+      ],
+    },
+  ])
+  console.log(usage)
+  process.exit(1)
+}

package/src/commands/login-server.ts

@@ -0,0 +1,119 @@
+import express from 'express'
+import { getAuthConfig, getFinalizedHost } from '../config'
+import url, { URL } from 'url'
+import fetch from 'node-fetch'
+import { getCliVersion } from '../utils'
+import { WriteKey } from '../key'
+import chalk from 'chalk'
+import http from 'http'
+import os from 'os'
+import * as crypto from 'crypto'
+
+interface AuthParams {
+  serverPort: number
+  sentioHost: string
+  codeVerifier: string
+}
+
+const app = express()
+
+let server: http.Server
+let authParams: AuthParams
+
+export function startServer(params: AuthParams) {
+  authParams = params
+  server = app.listen(params.serverPort)
+}
+
+app.get('/callback', async (req, res) => {
+  res.end('login success, please go back to CLI to continue')
+  const host = getFinalizedHost(authParams.sentioHost)
+  const code = req.query.code
+  if (!code || (code as string).length == 0) {
+    return
+  }
+
+  // exchange token
+  const tokenResRaw = await getToken(host, code as string)
+  if (!tokenResRaw.ok) {
+    console.error(chalk.red('request token error, code:', tokenResRaw.status, tokenResRaw.statusText))
+    return
+  }
+  const tokenRes = await tokenResRaw.json()
+  const accessToken = tokenRes['access_token']
+
+  // check if the account is ready
+  const userResRaw = await getUser(host, accessToken)
+  if (!userResRaw.ok) {
+    if (userResRaw.status == 401) {
+      console.error(chalk.red('please sign up on sentio first'))
+    } else {
+      console.error(chalk.red('get user error, code:', userResRaw.status, userResRaw.statusText))
+    }
+    return
+  }
+  const userRes = await userResRaw.json()
+  if (!userRes.emailVerified) {
+    console.error(chalk.red('please verify your email first'))
+    return
+  }
+
+  // create API key
+  const apiKeyName = `${os.hostname()}-${crypto.randomBytes(4).toString('hex')}`
+  const createApiKeyResRaw = await createApiKey(host, apiKeyName, 'sdk_generated', accessToken)
+  if (!createApiKeyResRaw.ok) {
+    console.error(chalk.red('create api key error, code:', createApiKeyResRaw.status, createApiKeyResRaw.statusText))
+    return
+  }
+  const createApiKeyRes = await createApiKeyResRaw.json()
+  const apiKey = createApiKeyRes['key']
+  WriteKey(host, apiKey)
+  console.log(chalk.green('login success, new API key: ' + apiKey))
+
+  server.close()
+})
+
+async function getToken(host: string, code: string) {
+  const authConf = getAuthConfig(host)
+  const params = new url.URLSearchParams({
+    grant_type: 'authorization_code',
+    client_id: authConf.clientId,
+    code_verifier: authParams.codeVerifier,
+    code: code,
+    redirect_uri: `http://localhost:${authParams.serverPort}/callback`,
+  })
+  return fetch(`${authConf.domain}/oauth/token`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded',
+    },
+    body: params.toString(),
+  })
+}
+
+async function createApiKey(host: string, name: string, source: string, accessToken: string) {
+  const createApiKeyUrl = new URL('/api/v1/keys', host)
+  return fetch(createApiKeyUrl, {
+    method: 'POST',
+    headers: {
+      Authorization: 'Bearer ' + accessToken,
+      version: getCliVersion(),
+    },
+    body: JSON.stringify({
+      name: name,
+      scopes: ['write:project'],
+      source: source,
+    }),
+  })
+}
+
+async function getUser(host: string, accessToken: string) {
+  const getUserUrl = new URL('/api/v1/users', host)
+  return fetch(getUserUrl, {
+    method: 'GET',
+    headers: {
+      Authorization: 'Bearer ' + accessToken,
+      version: getCliVersion(),
+    },
+  })
+}

package/src/commands/run-create.ts

@@ -0,0 +1,116 @@
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import path from 'path'
+import fs from 'fs-extra'
+import chalk from 'chalk'
+import { getCliVersion } from '../utils'
+
+export function runCreate(argv: string[]) {
+  const optionDefinitions = [
+    {
+      name: 'help',
+      alias: 'h',
+      type: Boolean,
+      description: 'Display this usage guide.',
+    },
+    {
+      name: 'name',
+      alias: 'n',
+      defaultOption: true,
+      type: String,
+      description: 'Project name',
+    },
+    {
+      name: 'directory',
+      alias: 'd',
+      description: '(Optional) The root direct new project will be created, default current working dir',
+      type: String,
+    },
+    {
+      name: 'chain-type',
+      alias: 'c',
+      description:
+        'The type of project you want to create, can be evm, aptos, raw (if you want to start from scratch and support multiple types of chains)',
+      type: String,
+      defaultValue: 'evm',
+    },
+  ]
+
+  const options = commandLineArgs(optionDefinitions, { argv })
+  const usage = commandLineUsage([
+    {
+      header: 'Create a template project',
+      content: 'sentio create <name>',
+    },
+    {
+      header: 'Options',
+      optionList: optionDefinitions,
+    },
+  ])
+
+  if (options.help || !options.name) {
+    console.log(usage)
+  } else {
+    const chainType: string = options['chain-type'].toLowerCase()
+    switch (chainType) {
+      case 'evm':
+        break
+      case 'aptos':
+        break
+      case 'raw':
+        break
+      default:
+        console.error(chalk.red('non supported chain-type for template creation, use --help for more information.'))
+        console.log(usage)
+        process.exit(1)
+    }
+    const templateFolder = path.resolve(__dirname, '../../templates', chainType)
+    const projectName = options.name || 'default'
+
+    const rootDir = options.directory || process.cwd()
+    const dstFolder = path.resolve(rootDir, projectName)
+    if (fs.existsSync(dstFolder)) {
+      console.error(chalk.red("can't create project '" + projectName + "', directory already existed"))
+      process.exit(1)
+    }
+
+    fs.copySync(templateFolder, dstFolder, {
+      filter: (src, _) => {
+        // TODO read from .gitignore to be more reliable
+        if (
+          src.endsWith('types') ||
+          src.endsWith('dist') ||
+          src.endsWith('node_modules') ||
+          src.endsWith('.lock') ||
+          src.endsWith('dist')
+        ) {
+          return false
+        }
+        return true
+      },
+      overwrite: false,
+    })
+    if (options.name) {
+      const sentioYamlPath = path.resolve(dstFolder, 'sentio.yaml')
+      fs.writeFileSync(sentioYamlPath, 'project: ' + projectName + '\n', { flag: 'w+' })
+
+      const packageJsonPath = path.resolve(dstFolder, 'package.json')
+      const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
+
+      let cliVersion = getCliVersion()
+      if (!cliVersion.startsWith('^')) {
+        cliVersion = '^' + cliVersion
+      }
+
+      packageJson.dependencies['@sentio/sdk'] = cliVersion
+      packageJson.dependencies['@sentio/cli'] = cliVersion
+      packageJson.name = projectName
+
+      // Don't add directly to avoid deps issue
+      packageJson.scripts.postinstall = 'sentio gen'
+
+      fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2))
+    }
+    console.log(chalk.green("successfully create project '" + projectName + "'"))
+  }
+}

package/src/commands/run-login.ts

@@ -0,0 +1,111 @@
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import { getAuthConfig, getFinalizedHost } from '../config'
+import { startServer } from './login-server'
+import url, { URL } from 'url'
+import * as crypto from 'crypto'
+import chalk from 'chalk'
+import { WriteKey } from '../key'
+import fetch from 'node-fetch'
+import open from 'open'
+
+const port = 20000
+
+export function runLogin(argv: string[]) {
+  const optionDefinitions = [
+    {
+      name: 'help',
+      alias: 'h',
+      type: Boolean,
+      description: 'Display this usage guide.',
+    },
+    {
+      name: 'host',
+      description: '(Optional) Override Sentio Host name',
+      type: String,
+    },
+    {
+      name: 'api-key',
+      type: String,
+      description: '(Optional) Your API key',
+    },
+  ]
+  const options = commandLineArgs(optionDefinitions, { argv })
+
+  const host = getFinalizedHost(options.host)
+  if (options.help) {
+    const usage = commandLineUsage([
+      {
+        header: 'Login to Sentio',
+        content: 'sentio login',
+      },
+      {
+        header: 'Options',
+        optionList: optionDefinitions,
+      },
+    ])
+    console.log(usage)
+  } else if (options['api-key']) {
+    console.log(chalk.blue('login to ' + host))
+    const apiKey = options['api-key']
+    checkKey(host, apiKey).then((res) => {
+      if (res.status == 200) {
+        WriteKey(host, apiKey)
+        console.log(chalk.green('login success'))
+      } else {
+        console.error(chalk.red('login failed, code:', res.status, res.statusText))
+      }
+    })
+  } else {
+    // https://auth0.com/docs/get-started/authentication-and-authorization-flow/call-your-api-using-the-authorization-code-flow-with-pkce
+    const verifier = base64URLEncode(crypto.randomBytes(32))
+    const challenge = base64URLEncode(sha256(verifier))
+
+    const conf = getAuthConfig(host)
+    if (conf.domain === '') {
+      console.error(chalk.red('invalid host, try login with an API key if it is a dev env'))
+      return
+    }
+    const authURL = new URL(conf.domain + `/authorize?`)
+    const params = new url.URLSearchParams({
+      response_type: 'code',
+      code_challenge: challenge,
+      code_challenge_method: 'S256',
+      client_id: conf.clientId,
+      redirect_uri: `http://localhost:${port}/callback`,
+      audience: conf.audience,
+      prompt: 'login',
+    })
+    authURL.search = params.toString()
+
+    console.log('Continue your authorization in the browser')
+    open(authURL.toString()).catch((reason) => {
+      console.error(chalk.red('Unable to open browser: ' + reason))
+      console.error(chalk.red('Open this url in your browser: ' + authURL.toString()))
+    })
+
+    startServer({
+      serverPort: port,
+      sentioHost: options.host,
+      codeVerifier: verifier,
+    })
+  }
+}
+
+function base64URLEncode(str: Buffer) {
+  return str.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '')
+}
+
+function sha256(str: string) {
+  return crypto.createHash('sha256').update(str).digest()
+}
+
+async function checkKey(host: string, apiKey: string) {
+  const checkApiKeyUrl = new URL('/api/v1/processors/check_key', host)
+  return fetch(checkApiKeyUrl, {
+    method: 'GET',
+    headers: {
+      'api-key': apiKey,
+    },
+  })
+}