@certik/skynet 0.10.7 → 0.10.8

package/deploy.js

@@ -1,676 +1,676 @@
-const fs = require("fs/promises");
-const fso = require("fs");
-const execa = require("execa");
-const meow = require("meow");
-const chalk = require("chalk");
-const which = require("which");
-const { getSelectorFlags, getSelectorDesc } = require("./selector");
-const { getEnvOrThrow } = require("./env");
-const { getBinaryName, detectSkynetDirectory } = require("./cli");
-
-const INTERVAL_ALIASES = {
-  secondly: "*/1 * * * * * *",
-  "@secondly": "*/1 * * * * * *",
-  minutely: "0 * * * * * *",
-  "@minutely": "0 * * * * * *",
-  hourly: "0 0 * * * * *",
-  "@hourly": "0 0 * * * * *",
-  daily: "0 0 0 * * * *",
-  "@daily": "0 0 0 * * * *",
-  weekly: "0 0 0 * * 0 *",
-  "@weekly": "0 0 0 * * 0 *",
-};
-
-function buildEnvTemplate(additionalEnv, isProduction) {
-  return Object.keys(additionalEnv)
-    .map((key) => {
-      return `${key}="${additionalEnv[key] || getEnvironmentVariableValue(key, isProduction)}"`;
-    })
-    .join("\n");
-}
-
-function getEnvironmentVariableValue(name, isProduction) {
-  if (isProduction) {
-    return `{{key "secrets/${name}"}}`;
-  } else {
-    if (!process.env[name]) {
-      return "";
-    }
-
-    return process.env[name];
-  }
-}
-
-const genConfig = ({
-  jobName,
-  workingDirectory,
-  cmd,
-  cron,
-  restart,
-  cpu,
-  mem,
-  service,
-  additionalEnv = [],
-  region = "us-east-1",
-  isProduction,
-}) => `job "${jobName}" {
-  datacenters = ["${region}"]
-
-  type = "batch"
-
-  ${
-    cron
-      ? `# Triggers periodically
-  periodic {
-    cron             = "${cron}"
-    prohibit_overlap = true
-  }`
-      : ""
-  }
-
-  constraint {
-    attribute = "\${meta.has_nodejs}"
-    value     = "true"
-  }
-
-  constraint {
-    attribute = "\${meta.has_skynet}"
-    value     = "true"
-  }
-
-  group "default" {
-    reschedule {
-      attempts  = 0
-      unlimited = false
-    }
-
-    ${
-      service
-        ? `# Setup Service Network
-    network {
-      port "http" {
-        static = ${service.port}
-      }
-    }`
-        : ""
-    }
-
-    task "log-shipper" {
-      driver = "raw_exec"
-
-      config {
-        command = "sh"
-        args = [
-          "-c",
-          "cd \${meta.skynet_code_path}/infra-nomad/log-shipper && yarn install --silent && exec bin/shipper --path ${jobName}"
-        ]
-      }
-
-      env {
-        SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
-        AWS_REGION="${region}"
-      }
-
-      kill_timeout = "120s"
-
-      resources {
-        cpu    = 100 # MHz
-        memory = 100 # MB
-      }
-
-      restart {
-        attempts = 0
-        mode     = "fail"
-      }
-    }
-
-    task "main" {
-      driver = "raw_exec"
-
-      config {
-        command = "sh"
-        args = [
-          "-c",
-          "cd \${meta.skynet_code_path}/${workingDirectory} && yarn install --silent && exec ${cmd}"
-        ]
-      }
-
-      ${
-        service
-          ? `# Setup API Routes
-      service {
-        name = "${jobName}"
-        port = "http"
-
-        tags = [
-          "urlprefix-${service.prefix} strip=${service.prefix}",
-        ]
-
-        check {
-          type     = "http"
-          path     = "/"
-          interval = "10s"
-          timeout  = "2s"
-        }
-      }
-`
-          : ""
-      }
-
-      template {
-        change_mode = "restart"
-        destination = "secrets/context.env"
-        env         = true
-
-        data = <<EOH
-${buildEnvTemplate(additionalEnv, isProduction)}
-EOH
-      }
-
-      # It is possible to set environment variables which will be
-      # available to the task when it runs.
-      env {
-        SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
-        AWS_REGION="${region}"
-      }
-
-      # Specify the maximum resources required to run the task,
-      # include CPU and memory.
-      resources {
-        cpu    = ${cpu} # MHz
-        memory = ${mem} # MB
-      }
-
-      # Setting the server task as the leader of the task group allows Nomad to
-      # signal the log shipper task to gracefully shutdown when the server exits.
-      leader = true
-
-      ${
-        restart
-          ? `
-      # Restart the job if it fails
-      restart {
-        attempts = ${restart.attempts ?? 2}
-        mode     = "${restart.mode ?? "fail"}"
-        interval = "${restart.interval ?? "30m"}"
-        delay    = "${restart.delay ?? "15s"}"
-      }
-      `
-          : `
-      # do not retry from the periodical job will reschedule anyway
-      restart {
-        attempts = 0
-        mode     = "fail"
-      }`
-      }
-    }
-  }
-}`;
-
-function normalizeSelectorValue(v) {
-  return v.replace(/[^A-Za-z0-9]+/g, "-");
-}
-
-function getJobName(name, selectorFlags, mode = null) {
-  const selectorNamePart = Object.keys(selectorFlags)
-    .sort()
-    .map((name) => selectorFlags[name])
-    .join("-");
-
-  let jobName = name;
-
-  if (mode) {
-    jobName += `-${mode}`;
-  }
-
-  if (selectorNamePart.length > 0) {
-    // handle special case
-    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
-  }
-
-  return jobName;
-}
-
-async function prepareNomad(isProduction) {
-  if (isProduction) {
-    console.log("Deploy to Production");
-  } else {
-    const skynetDir = detectSkynetDirectory();
-
-    if (!fso.existsSync("/tmp/skynet")) {
-      await execa("ln", ["-s", skynetDir, "/tmp/skynet"]);
-    }
-
-    console.log("Deploy locally, please start nomad server in a separate terminal");
-    console.log(`You can start nomad server by running ${chalk.inverse(`${skynetDir}/infra-nomad/dev/start.sh`)}`);
-    console.log(
-      `Then you can visit ${chalk.underline("http://localhost:4646/ui/jobs")} to check submitted dev jobs.\n`
-    );
-  }
-}
-
-async function getNomadAddr(isProduction) {
-  let nomadAddr;
-
-  if (isProduction) {
-    nomadAddr = getEnvOrThrow("SKYNET_NOMAD_PRODUCTION_ADDR");
-  } else {
-    nomadAddr = "http://127.0.0.1:4646";
-  }
-
-  return nomadAddr;
-}
-
-async function getNomadPath() {
-  try {
-    return await which("nomad");
-  } catch (missingNomad) {
-    console.log(
-      `Deploy requires ${chalk.bold("nomad")} binary, please follow ${chalk.underline(
-        "https://learn.hashicorp.com/tutorials/nomad/get-started-install"
-      )} for installation`
-    );
-
-    throw new Error("missing nomad binary");
-  }
-}
-
-async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, isStop, isDryRun) {
-  try {
-    if (isStop) {
-      const nomad = execa(nomadPath, ["job", "stop", jobName], {
-        env: {
-          NOMAD_ADDR: nomadAddr,
-        },
-      });
-      nomad.stdout.pipe(process.stdout);
-      await nomad;
-
-      console.log(chalk.green(`Stopped nomad job ${jobName} in ${nomadAddr}`));
-    } else if (isDryRun) {
-      console.log("Definition for", jobName);
-      console.log("========================================");
-      console.log(nomadJobDefinition);
-    } else {
-      const jobFileName = `/tmp/job-${jobName}`;
-
-      await fs.writeFile(jobFileName, nomadJobDefinition);
-
-      const nomad = execa(nomadPath, ["job", "run", jobFileName], {
-        env: {
-          NOMAD_ADDR: nomadAddr,
-        },
-      });
-      nomad.stdout.pipe(process.stdout);
-      await nomad;
-
-      console.log(chalk.green(`Deployed nomad job ${jobName} to ${nomadAddr}`));
-    }
-  } catch (nomadExecErr) {
-    console.log("Nomad Execution Error:");
-    console.log(nomadExecErr.message);
-    console.log("");
-
-    console.log(
-      `Failed to run ${chalk.bold("nomad")} commands, please ensure nomad server is accessible at ${chalk.bold(
-        nomadAddr
-      )}`
-    );
-
-    throw new Error("nomad execution error");
-  }
-}
-
-function createModeDeploy({
-  binaryName,
-  name,
-  workingDirectory,
-  bin = "bin/indexer",
-  selector = {},
-  env = {},
-  region = "us-east-1",
-  check,
-  deltaSchedule,
-  validateSchedule,
-  deltaCpu,
-  deltaMem,
-  rebuildCpu,
-  rebuildMem,
-  validateCpu,
-  validateMem,
-}) {
-  async function deployMode({
-    mode,
-    from,
-    to,
-    stop,
-    production,
-    dryRun,
-    verbose,
-    schedule: cmdSchedule,
-    ...selectorFlags
-  }) {
-    if (mode === "delta") {
-      // delta mode will ignore from/to flags
-      from = 0;
-      to = 0;
-    }
-
-    const isPeriodic = from === 0 && to === 0 && ["delta", "validate"].includes(mode);
-
-    const jobName = getJobName(name, selectorFlags, mode);
-
-    const selectorCmdPart = Object.keys(selectorFlags)
-      .sort()
-      .map((name) => `--${name} ${selectorFlags[name]}`)
-      .join(" ");
-    let args = `--mode ${mode} ${selectorCmdPart}`;
-
-    if (verbose) {
-      args += ` --verbose`;
-    }
-
-    let rangeArgs = "";
-
-    if (from > 0) {
-      rangeArgs += ` --from ${from}`;
-    }
-
-    if (to > 0) {
-      rangeArgs += ` --to ${to}`;
-    }
-
-    const modeResouces = {
-      rebuild: { cpu: rebuildCpu, mem: rebuildMem },
-      "resume-rebuild": { cpu: rebuildCpu, mem: rebuildMem },
-      validate: { cpu: validateCpu || rebuildCpu, mem: validateMem || rebuildMem },
-      delta: { cpu: deltaCpu, mem: deltaMem },
-    };
-
-    // by default use delta cpu/mem settings
-    const { cpu, mem } = modeResouces[mode] || modeResouces.delta;
-
-    let deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
-
-    if (deltaSchedule && cmdSchedule) {
-      deltaCron = cmdSchedule;
-    }
-
-    let validateCron = typeof validateSchedule === "function" ? validateSchedule(jobName) : validateSchedule;
-
-    if (validateSchedule && cmdSchedule) {
-      validateCron = cmdSchedule;
-    }
-
-    const modeIntervals = {
-      delta: INTERVAL_ALIASES[deltaCron] || deltaCron,
-      validate: INTERVAL_ALIASES[validateCron] || validateCron,
-    };
-
-    const mainJobDefinition = genConfig({
-      jobName,
-      cron: isPeriodic && modeIntervals[mode],
-      workingDirectory,
-      additionalEnv: env,
-      region,
-      cmd: `${bin} ${args} ${rangeArgs}`,
-      cpu,
-      mem,
-      isProduction: production,
-    });
-
-    const nomadPath = await getNomadPath();
-    await prepareNomad(production);
-    const nomadAddr = await getNomadAddr(production);
-
-    await runNomadJob(nomadPath, nomadAddr, jobName, mainJobDefinition, stop, dryRun);
-
-    if (check && check.bin) {
-      console.log("");
-
-      const monitorJobName = `${jobName}-monitor`;
-      const monitorJobDefinition = genConfig({
-        jobName: monitorJobName,
-        cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
-        workingDirectory,
-        additionalEnv: {
-          ...env,
-          SKYNET_NOMAD_PRODUCTION_ADDR: null,
-          SKYNET_SLACK_TOKEN: null,
-          OPSGENIE_API_KEY: null,
-          OPSGENIE_END_POINT: null,
-        },
-        region,
-        cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
-        cpu: check.cpu || 100,
-        mem: check.mem || 100,
-        isProduction: production,
-      });
-
-      await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop, dryRun);
-    }
-  }
-
-  function deploy() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const cli = meow(
-      `
-        Usage
-
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset
-          --from          min id to build
-          --to            max id to build
-          --stop          stop job instead of running the job
-          --production    deploy to production, default is development
-          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
-          --verbose       Output debug messages
-          --dry-run       print nomad job file but do not really execute it
-
-        Examples
-          ${binaryName} --mode delta
-          ${binaryName} --mode rebuild
-          ${binaryName} --mode validate
-      `,
-      {
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          mode: {
-            type: "string",
-            default: "delta",
-          },
-          from: {
-            alias: "since",
-            type: "number",
-            default: 0,
-          },
-          to: {
-            alias: "until",
-            type: "number",
-            default: 0,
-          },
-          schedule: {
-            type: "string",
-          },
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-          production: {
-            alias: "prd",
-            type: "boolean",
-            default: false,
-          },
-          dryRun: {
-            type: "boolean",
-            default: false,
-          },
-          stop: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    deployMode(cli.flags).catch((err) => {
-      console.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { deploy };
-}
-
-function createDeploy({
-  binaryName,
-  name,
-  workingDirectory,
-  bin = "bin/indexer",
-  selector = {},
-  region = "us-east-1",
-  env = {},
-  check,
-  schedule,
-  restart,
-  cpu,
-  mem,
-  service,
-}) {
-  async function deployModeless({ production, stop, dryRun, verbose, schedule: cmdSchedule, ...selectorFlags }) {
-    const jobName = getJobName(name, selectorFlags, null);
-
-    const selectorCmdPart = Object.keys(selectorFlags)
-      .sort()
-      .map((name) => `--${name} ${selectorFlags[name]}`)
-      .join(" ");
-    let args = `${selectorCmdPart}`;
-
-    if (verbose) {
-      args += ` --verbose`;
-    }
-
-    let cron = typeof schedule === "function" ? schedule(jobName) : schedule;
-
-    if (schedule && cmdSchedule) {
-      // cmd schedule has higher priority
-      cron = cmdSchedule;
-    }
-
-    const nomadJobDefinition = genConfig({
-      jobName,
-      cron: INTERVAL_ALIASES[cron] || cron,
-      restart,
-      workingDirectory,
-      additionalEnv: env,
-      region,
-      cmd: `${bin} ${args}`,
-      cpu,
-      mem,
-      service,
-      isProduction: production,
-    });
-
-    const nomadPath = await getNomadPath();
-    await prepareNomad(production);
-    const nomadAddr = await getNomadAddr(production);
-
-    await runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, stop, dryRun);
-
-    if (check && check.bin) {
-      console.log("");
-
-      const monitorJobName = `${jobName}-monitor`;
-      const monitorJobDefinition = genConfig({
-        jobName: monitorJobName,
-        cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
-        workingDirectory,
-        additionalEnv: {
-          ...env,
-          SKYNET_NOMAD_PRODUCTION_ADDR: null,
-          SKYNET_SLACK_TOKEN: null,
-          OPSGENIE_API_KEY: null,
-          OPSGENIE_END_POINT: null,
-        },
-        region,
-        cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
-        cpu: check.cpu || 100,
-        mem: check.mem || 100,
-        isProduction: production,
-      });
-
-      await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop, dryRun);
-    }
-  }
-
-  function deploy() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const cli = meow(
-      `
-        Usage
-
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --stop          stop job instead of running the job
-          --production    deploy to production, default is development
-          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
-          --verbose       Output debug messages
-          --dry-run       print nomad job file but do not really execute it
-      `,
-      {
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          schedule: {
-            type: "string",
-          },
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-          production: {
-            alias: "prd",
-            type: "boolean",
-            default: false,
-          },
-          dryRun: {
-            type: "boolean",
-            default: false,
-          },
-          stop: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    deployModeless(cli.flags).catch((err) => {
-      console.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { deploy };
-}
-
-module.exports = {
-  getJobName,
-  getNomadAddr,
-  createModeDeploy,
-  createDeploy,
-};
+const fs = require("fs/promises");
+const fso = require("fs");
+const execa = require("execa");
+const meow = require("meow");
+const chalk = require("chalk");
+const which = require("which");
+const { getSelectorFlags, getSelectorDesc } = require("./selector");
+const { getEnvOrThrow } = require("./env");
+const { getBinaryName, detectSkynetDirectory } = require("./cli");
+
+const INTERVAL_ALIASES = {
+  secondly: "*/1 * * * * * *",
+  "@secondly": "*/1 * * * * * *",
+  minutely: "0 * * * * * *",
+  "@minutely": "0 * * * * * *",
+  hourly: "0 0 * * * * *",
+  "@hourly": "0 0 * * * * *",
+  daily: "0 0 0 * * * *",
+  "@daily": "0 0 0 * * * *",
+  weekly: "0 0 0 * * 0 *",
+  "@weekly": "0 0 0 * * 0 *",
+};
+
+function buildEnvTemplate(additionalEnv, isProduction) {
+  return Object.keys(additionalEnv)
+    .map((key) => {
+      return `${key}="${additionalEnv[key] || getEnvironmentVariableValue(key, isProduction)}"`;
+    })
+    .join("\n");
+}
+
+function getEnvironmentVariableValue(name, isProduction) {
+  if (isProduction) {
+    return `{{key "secrets/${name}"}}`;
+  } else {
+    if (!process.env[name]) {
+      return "";
+    }
+
+    return process.env[name];
+  }
+}
+
+const genConfig = ({
+  jobName,
+  workingDirectory,
+  cmd,
+  cron,
+  restart,
+  cpu,
+  mem,
+  service,
+  additionalEnv = [],
+  region = "us-east-1",
+  isProduction,
+}) => `job "${jobName}" {
+  datacenters = ["${region}"]
+
+  type = "batch"
+
+  ${
+    cron
+      ? `# Triggers periodically
+  periodic {
+    cron             = "${cron}"
+    prohibit_overlap = true
+  }`
+      : ""
+  }
+
+  constraint {
+    attribute = "\${meta.has_nodejs}"
+    value     = "true"
+  }
+
+  constraint {
+    attribute = "\${meta.has_skynet}"
+    value     = "true"
+  }
+
+  group "default" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    ${
+      service
+        ? `# Setup Service Network
+    network {
+      port "http" {
+        static = ${service.port}
+      }
+    }`
+        : ""
+    }
+
+    task "log-shipper" {
+      driver = "raw_exec"
+
+      config {
+        command = "sh"
+        args = [
+          "-c",
+          "cd \${meta.skynet_code_path}/infra-nomad/log-shipper && yarn install --silent && exec bin/shipper --path ${jobName} --opensearch"
+        ]
+      }
+
+      env {
+        SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
+        AWS_REGION="${region}"
+      }
+
+      kill_timeout = "120s"
+
+      resources {
+        cpu    = 100 # MHz
+        memory = 100 # MB
+      }
+
+      restart {
+        attempts = 0
+        mode     = "fail"
+      }
+    }
+
+    task "main" {
+      driver = "raw_exec"
+
+      config {
+        command = "sh"
+        args = [
+          "-c",
+          "cd \${meta.skynet_code_path}/${workingDirectory} && yarn install --silent && exec ${cmd}"
+        ]
+      }
+
+      ${
+        service
+          ? `# Setup API Routes
+      service {
+        name = "${jobName}"
+        port = "http"
+
+        tags = [
+          "urlprefix-${service.prefix} strip=${service.prefix}",
+        ]
+
+        check {
+          type     = "http"
+          path     = "/"
+          interval = "10s"
+          timeout  = "2s"
+        }
+      }
+`
+          : ""
+      }
+
+      template {
+        change_mode = "restart"
+        destination = "secrets/context.env"
+        env         = true
+
+        data = <<EOH
+${buildEnvTemplate(additionalEnv, isProduction)}
+EOH
+      }
+
+      # It is possible to set environment variables which will be
+      # available to the task when it runs.
+      env {
+        SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
+        AWS_REGION="${region}"
+      }
+
+      # Specify the maximum resources required to run the task,
+      # include CPU and memory.
+      resources {
+        cpu    = ${cpu} # MHz
+        memory = ${mem} # MB
+      }
+
+      # Setting the server task as the leader of the task group allows Nomad to
+      # signal the log shipper task to gracefully shutdown when the server exits.
+      leader = true
+
+      ${
+        restart
+          ? `
+      # Restart the job if it fails
+      restart {
+        attempts = ${restart.attempts ?? 2}
+        mode     = "${restart.mode ?? "fail"}"
+        interval = "${restart.interval ?? "30m"}"
+        delay    = "${restart.delay ?? "15s"}"
+      }
+      `
+          : `
+      # do not retry from the periodical job will reschedule anyway
+      restart {
+        attempts = 0
+        mode     = "fail"
+      }`
+      }
+    }
+  }
+}`;
+
+function normalizeSelectorValue(v) {
+  return v.replace(/[^A-Za-z0-9]+/g, "-");
+}
+
+function getJobName(name, selectorFlags, mode = null) {
+  const selectorNamePart = Object.keys(selectorFlags)
+    .sort()
+    .map((name) => selectorFlags[name])
+    .join("-");
+
+  let jobName = name;
+
+  if (mode) {
+    jobName += `-${mode}`;
+  }
+
+  if (selectorNamePart.length > 0) {
+    // handle special case
+    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
+  }
+
+  return jobName;
+}
+
+async function prepareNomad(isProduction) {
+  if (isProduction) {
+    console.log("Deploy to Production");
+  } else {
+    const skynetDir = detectSkynetDirectory();
+
+    if (!fso.existsSync("/tmp/skynet")) {
+      await execa("ln", ["-s", skynetDir, "/tmp/skynet"]);
+    }
+
+    console.log("Deploy locally, please start nomad server in a separate terminal");
+    console.log(`You can start nomad server by running ${chalk.inverse(`${skynetDir}/infra-nomad/dev/start.sh`)}`);
+    console.log(
+      `Then you can visit ${chalk.underline("http://localhost:4646/ui/jobs")} to check submitted dev jobs.\n`
+    );
+  }
+}
+
+async function getNomadAddr(isProduction) {
+  let nomadAddr;
+
+  if (isProduction) {
+    nomadAddr = getEnvOrThrow("SKYNET_NOMAD_PRODUCTION_ADDR");
+  } else {
+    nomadAddr = "http://127.0.0.1:4646";
+  }
+
+  return nomadAddr;
+}
+
+async function getNomadPath() {
+  try {
+    return await which("nomad");
+  } catch (missingNomad) {
+    console.log(
+      `Deploy requires ${chalk.bold("nomad")} binary, please follow ${chalk.underline(
+        "https://learn.hashicorp.com/tutorials/nomad/get-started-install"
+      )} for installation`
+    );
+
+    throw new Error("missing nomad binary");
+  }
+}
+
+async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, isStop, isDryRun) {
+  try {
+    if (isStop) {
+      const nomad = execa(nomadPath, ["job", "stop", jobName], {
+        env: {
+          NOMAD_ADDR: nomadAddr,
+        },
+      });
+      nomad.stdout.pipe(process.stdout);
+      await nomad;
+
+      console.log(chalk.green(`Stopped nomad job ${jobName} in ${nomadAddr}`));
+    } else if (isDryRun) {
+      console.log("Definition for", jobName);
+      console.log("========================================");
+      console.log(nomadJobDefinition);
+    } else {
+      const jobFileName = `/tmp/job-${jobName}`;
+
+      await fs.writeFile(jobFileName, nomadJobDefinition);
+
+      const nomad = execa(nomadPath, ["job", "run", jobFileName], {
+        env: {
+          NOMAD_ADDR: nomadAddr,
+        },
+      });
+      nomad.stdout.pipe(process.stdout);
+      await nomad;
+
+      console.log(chalk.green(`Deployed nomad job ${jobName} to ${nomadAddr}`));
+    }
+  } catch (nomadExecErr) {
+    console.log("Nomad Execution Error:");
+    console.log(nomadExecErr.message);
+    console.log("");
+
+    console.log(
+      `Failed to run ${chalk.bold("nomad")} commands, please ensure nomad server is accessible at ${chalk.bold(
+        nomadAddr
+      )}`
+    );
+
+    throw new Error("nomad execution error");
+  }
+}
+
+function createModeDeploy({
+  binaryName,
+  name,
+  workingDirectory,
+  bin = "bin/indexer",
+  selector = {},
+  env = {},
+  region = "us-east-1",
+  check,
+  deltaSchedule,
+  validateSchedule,
+  deltaCpu,
+  deltaMem,
+  rebuildCpu,
+  rebuildMem,
+  validateCpu,
+  validateMem,
+}) {
+  async function deployMode({
+    mode,
+    from,
+    to,
+    stop,
+    production,
+    dryRun,
+    verbose,
+    schedule: cmdSchedule,
+    ...selectorFlags
+  }) {
+    if (mode === "delta") {
+      // delta mode will ignore from/to flags
+      from = 0;
+      to = 0;
+    }
+
+    const isPeriodic = from === 0 && to === 0 && ["delta", "validate"].includes(mode);
+
+    const jobName = getJobName(name, selectorFlags, mode);
+
+    const selectorCmdPart = Object.keys(selectorFlags)
+      .sort()
+      .map((name) => `--${name} ${selectorFlags[name]}`)
+      .join(" ");
+    let args = `--mode ${mode} ${selectorCmdPart}`;
+
+    if (verbose) {
+      args += ` --verbose`;
+    }
+
+    let rangeArgs = "";
+
+    if (from > 0) {
+      rangeArgs += ` --from ${from}`;
+    }
+
+    if (to > 0) {
+      rangeArgs += ` --to ${to}`;
+    }
+
+    const modeResouces = {
+      rebuild: { cpu: rebuildCpu, mem: rebuildMem },
+      "resume-rebuild": { cpu: rebuildCpu, mem: rebuildMem },
+      validate: { cpu: validateCpu || rebuildCpu, mem: validateMem || rebuildMem },
+      delta: { cpu: deltaCpu, mem: deltaMem },
+    };
+
+    // by default use delta cpu/mem settings
+    const { cpu, mem } = modeResouces[mode] || modeResouces.delta;
+
+    let deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
+
+    if (deltaSchedule && cmdSchedule) {
+      deltaCron = cmdSchedule;
+    }
+
+    let validateCron = typeof validateSchedule === "function" ? validateSchedule(jobName) : validateSchedule;
+
+    if (validateSchedule && cmdSchedule) {
+      validateCron = cmdSchedule;
+    }
+
+    const modeIntervals = {
+      delta: INTERVAL_ALIASES[deltaCron] || deltaCron,
+      validate: INTERVAL_ALIASES[validateCron] || validateCron,
+    };
+
+    const mainJobDefinition = genConfig({
+      jobName,
+      cron: isPeriodic && modeIntervals[mode],
+      workingDirectory,
+      additionalEnv: env,
+      region,
+      cmd: `${bin} ${args} ${rangeArgs}`,
+      cpu,
+      mem,
+      isProduction: production,
+    });
+
+    const nomadPath = await getNomadPath();
+    await prepareNomad(production);
+    const nomadAddr = await getNomadAddr(production);
+
+    await runNomadJob(nomadPath, nomadAddr, jobName, mainJobDefinition, stop, dryRun);
+
+    if (check && check.bin) {
+      console.log("");
+
+      const monitorJobName = `${jobName}-monitor`;
+      const monitorJobDefinition = genConfig({
+        jobName: monitorJobName,
+        cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
+        workingDirectory,
+        additionalEnv: {
+          ...env,
+          SKYNET_NOMAD_PRODUCTION_ADDR: null,
+          SKYNET_SLACK_TOKEN: null,
+          OPSGENIE_API_KEY: null,
+          OPSGENIE_END_POINT: null,
+        },
+        region,
+        cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
+        cpu: check.cpu || 100,
+        mem: check.mem || 100,
+        isProduction: production,
+      });
+
+      await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop, dryRun);
+    }
+  }
+
+  function deploy() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const cli = meow(
+      `
+        Usage
+
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset
+          --from          min id to build
+          --to            max id to build
+          --stop          stop job instead of running the job
+          --production    deploy to production, default is development
+          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
+          --verbose       Output debug messages
+          --dry-run       print nomad job file but do not really execute it
+
+        Examples
+          ${binaryName} --mode delta
+          ${binaryName} --mode rebuild
+          ${binaryName} --mode validate
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          mode: {
+            type: "string",
+            default: "delta",
+          },
+          from: {
+            alias: "since",
+            type: "number",
+            default: 0,
+          },
+          to: {
+            alias: "until",
+            type: "number",
+            default: 0,
+          },
+          schedule: {
+            type: "string",
+          },
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+          production: {
+            alias: "prd",
+            type: "boolean",
+            default: false,
+          },
+          dryRun: {
+            type: "boolean",
+            default: false,
+          },
+          stop: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    deployMode(cli.flags).catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { deploy };
+}
+
+function createDeploy({
+  binaryName,
+  name,
+  workingDirectory,
+  bin = "bin/indexer",
+  selector = {},
+  region = "us-east-1",
+  env = {},
+  check,
+  schedule,
+  restart,
+  cpu,
+  mem,
+  service,
+}) {
+  async function deployModeless({ production, stop, dryRun, verbose, schedule: cmdSchedule, ...selectorFlags }) {
+    const jobName = getJobName(name, selectorFlags, null);
+
+    const selectorCmdPart = Object.keys(selectorFlags)
+      .sort()
+      .map((name) => `--${name} ${selectorFlags[name]}`)
+      .join(" ");
+    let args = `${selectorCmdPart}`;
+
+    if (verbose) {
+      args += ` --verbose`;
+    }
+
+    let cron = typeof schedule === "function" ? schedule(jobName) : schedule;
+
+    if (schedule && cmdSchedule) {
+      // cmd schedule has higher priority
+      cron = cmdSchedule;
+    }
+
+    const nomadJobDefinition = genConfig({
+      jobName,
+      cron: INTERVAL_ALIASES[cron] || cron,
+      restart,
+      workingDirectory,
+      additionalEnv: env,
+      region,
+      cmd: `${bin} ${args}`,
+      cpu,
+      mem,
+      service,
+      isProduction: production,
+    });
+
+    const nomadPath = await getNomadPath();
+    await prepareNomad(production);
+    const nomadAddr = await getNomadAddr(production);
+
+    await runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, stop, dryRun);
+
+    if (check && check.bin) {
+      console.log("");
+
+      const monitorJobName = `${jobName}-monitor`;
+      const monitorJobDefinition = genConfig({
+        jobName: monitorJobName,
+        cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
+        workingDirectory,
+        additionalEnv: {
+          ...env,
+          SKYNET_NOMAD_PRODUCTION_ADDR: null,
+          SKYNET_SLACK_TOKEN: null,
+          OPSGENIE_API_KEY: null,
+          OPSGENIE_END_POINT: null,
+        },
+        region,
+        cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
+        cpu: check.cpu || 100,
+        mem: check.mem || 100,
+        isProduction: production,
+      });
+
+      await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop, dryRun);
+    }
+  }
+
+  function deploy() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const cli = meow(
+      `
+        Usage
+
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --stop          stop job instead of running the job
+          --production    deploy to production, default is development
+          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
+          --verbose       Output debug messages
+          --dry-run       print nomad job file but do not really execute it
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          schedule: {
+            type: "string",
+          },
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+          production: {
+            alias: "prd",
+            type: "boolean",
+            default: false,
+          },
+          dryRun: {
+            type: "boolean",
+            default: false,
+          },
+          stop: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    deployModeless(cli.flags).catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { deploy };
+}
+
+module.exports = {
+  getJobName,
+  getNomadAddr,
+  createModeDeploy,
+  createDeploy,
+};