@certik/skynet 0.7.20 → 0.8.0

package/deploy.js

@@ -1,27 +1,37 @@
 const fs = require("fs/promises");
+const fso = require("fs");
 const execa = require("execa");
-const path = require("path");
 const meow = require("meow");
 const chalk = require("chalk");
 const which = require("which");
 const { getSelectorFlags, getSelectorDesc } = require("./selector");
 const { getEnvOrThrow } = require("./env");
+const { getBinaryName, detectSkynetDirectory } = require("./cli");
+
+const INTERVAL_ALIASES = {
+  "@secondly": "*/1 * * * * * *",
+  "@minutely": "0 * * * * * *",
+  "@hourly": "0 0 * * * * *",
+  "@daily": "0 0 0 * * * *",
+  "@weekly": "0 0 0 * * 0 *",
+};
 
-function buildEnvSection(additionalEnv, isProduction) {
-  const defaultEnv = {
-    SKYNET_AWS_ACCESS_KEY_ID: getEnvOrThrow("SKYNET_AWS_ACCESS_KEY_ID"),
-    SKYNET_AWS_SECRET_ACCESS_KEY: getEnvOrThrow("SKYNET_AWS_SECRET_ACCESS_KEY"),
-    SKYNET_AWS_REGION: process.env.SKYNET_AWS_REGION || "us-east-1",
-    SKYNET_ENVIRONMENT: isProduction ? "prd" : "dev"
-  };
-
-  const obj = { ...defaultEnv, ...additionalEnv };
-
-  return Object.keys(obj)
-    .map(key => `${key} = "${obj[key]}"`)
+function buildEnvTemplate(additionalEnv, isProduction) {
+  return Object.keys(additionalEnv)
+    .map((key) => {
+      return `${key} = "${additionalEnv[key] || getEnvironmentVariableValue(key, isProduction)}"`;
+    })
     .join("\n        ");
 }
 
+function getEnvironmentVariableValue(name, isProduction) {
+  if (isProduction) {
+    return `{{key "${name}"}}`;
+  } else {
+    return process.env[name];
+  }
+}
+
 const genConfig = ({
   jobName,
   workingDirectory,
@@ -31,13 +41,12 @@ const genConfig = ({
   cpu,
   mem,
   additionalEnv = [],
-  dc = "us-east-1",
-  isProduction
+  region = "us-east-1",
+  isProduction,
 }) => `
 job "${jobName}" {
-  datacenters = ["${dc}"]
+  datacenters = ["${region}"]
 
-  # Run this job as a "batch" type
   type = "batch"
 
   ${
@@ -66,9 +75,7 @@ job "${jobName}" {
       unlimited = false
     }
 
-    ${
-      isProduction
-        ? `task "log-shipper" {
+    task "log-shipper" {
       driver = "raw_exec"
 
       config {
@@ -79,16 +86,13 @@ job "${jobName}" {
         ]
       }
 
-      # It is possible to set environment variables which will be
-      # available to the task when it runs.
       env {
-        ${buildEnvSection({}, isProduction)}
+        SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
+        AWS_REGION="${region}"
       }
 
       kill_timeout = "120s"
 
-      # Specify the maximum resources required to run the task,
-      # include CPU and memory.
       resources {
         cpu    = 100 # MHz
         memory = 100 # MB
@@ -98,11 +102,9 @@ job "${jobName}" {
         attempts = 3
         mode     = "fail"
       }
-    }`
-        : ""
     }
 
-    task "indexer" {
+    task "main" {
       driver = "raw_exec"
 
       config {
@@ -113,10 +115,21 @@ job "${jobName}" {
         ]
       }
 
+      template {
+        change_mode = "restart"
+        destination = "secrets/context.env"
+        env         = true
+
+        data = <<EOH
+        ${buildEnvTemplate(additionalEnv, isProduction)}
+        EOH
+      }
+
       # It is possible to set environment variables which will be
       # available to the task when it runs.
       env {
-        ${buildEnvSection(additionalEnv, isProduction)}
+        SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
+        AWS_REGION="${region}"
       }
 
       # Specify the maximum resources required to run the task,
@@ -152,10 +165,10 @@ job "${jobName}" {
   }
 }`;
 
-function getJobName(name, selectorFlags, mode = null, periodic = false) {
+function getJobName(name, selectorFlags, mode = null) {
   const selectorNamePart = Object.keys(selectorFlags)
     .sort()
-    .map(name => `${name}:${selectorFlags[name]}`)
+    .map((name) => selectorFlags[name])
     .join("-");
 
   let jobName = name;
@@ -168,14 +181,10 @@ function getJobName(name, selectorFlags, mode = null, periodic = false) {
     jobName += `-${selectorNamePart}`;
   }
 
-  if (periodic) {
-    jobName += `-periodic`;
-  }
-
   return jobName;
 }
 
-function getNomadAddr(isProduction) {
+async function getNomadAddr(isProduction) {
   let nomadAddr;
 
   if (isProduction) {
@@ -183,18 +192,16 @@ function getNomadAddr(isProduction) {
 
     nomadAddr = getEnvOrThrow("SKYNET_NOMAD_PRODUCTION_ADDR");
   } else {
+    const skynetDir = detectSkynetDirectory();
+
+    if (!fso.existsSync("/tmp/skynet")) {
+      await execa("ln", ["-s", skynetDir, "/tmp/skynet"]);
+    }
+
+    console.log("Deploy locally, please start nomad server in a separate terminal");
+    console.log(`You can start nomad server by running ${chalk.inverse(`${skynetDir}/infra-nomad/dev/start.sh`)}`);
     console.log(
-      "Deploy locally, please start nomad server in a separate terminal"
-    );
-    console.log(
-      `You can start nomad server by running ${chalk.inverse(
-        "cd <skynet-repo-root> && ln -s . /opt/certik/skynet && infra-nomad/dev/start.sh"
-      )}`
-    );
-    console.log(
-      `Then you can visit ${chalk.underline(
-        "http://localhost:4646/ui/jobs"
-      )} to check submitted dev jobs.\n`
+      `Then you can visit ${chalk.underline("http://localhost:4646/ui/jobs")} to check submitted dev jobs.\n`
     );
 
     nomadAddr = "http://127.0.0.1:4646";
@@ -203,31 +210,27 @@ function getNomadAddr(isProduction) {
   return nomadAddr;
 }
 
-async function runNomadJob(jobName, nomadJobDefinition, isStop, isProduction) {
-  let nomadPath;
-
+async function getNomadPath() {
   try {
-    nomadPath = await which("nomad");
+    return await which("nomad");
   } catch (missingNomad) {
     console.log(
-      `Deploy requires ${chalk.bold(
-        "nomad"
-      )} binary, please follow ${chalk.underline(
+      `Deploy requires ${chalk.bold("nomad")} binary, please follow ${chalk.underline(
         "https://learn.hashicorp.com/tutorials/nomad/get-started-install"
       )} for installation`
     );
 
     throw new Error("missing nomad binary");
   }
+}
 
-  const nomadAddr = getNomadAddr(isProduction);
-
+async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, isStop) {
   try {
     if (isStop) {
       const nomad = execa(nomadPath, ["job", "stop", jobName], {
         env: {
-          NOMAD_ADDR: nomadAddr
-        }
+          NOMAD_ADDR: nomadAddr,
+        },
       });
       nomad.stdout.pipe(process.stdout);
       await nomad;
@@ -240,8 +243,8 @@ async function runNomadJob(jobName, nomadJobDefinition, isStop, isProduction) {
 
       const nomad = execa(nomadPath, ["job", "run", jobFileName], {
         env: {
-          NOMAD_ADDR: nomadAddr
-        }
+          NOMAD_ADDR: nomadAddr,
+        },
       });
       nomad.stdout.pipe(process.stdout);
       await nomad;
@@ -249,10 +252,12 @@ async function runNomadJob(jobName, nomadJobDefinition, isStop, isProduction) {
       console.log(chalk.green(`deployed ${jobName}`));
     }
   } catch (nomadExecErr) {
+    console.log("Nomad Execution Error:");
+    console.log(nomadExecErr.message);
+    console.log("");
+
     console.log(
-      `Failed to run ${chalk.bold(
-        "nomad"
-      )} commands, please ensure nomad server is accessible at ${chalk.bold(
+      `Failed to run ${chalk.bold("nomad")} commands, please ensure nomad server is accessible at ${chalk.bold(
         nomadAddr
       )}`
     );
@@ -262,106 +267,118 @@ async function runNomadJob(jobName, nomadJobDefinition, isStop, isProduction) {
 }
 
 function createModeDeploy({
+  binaryName,
   name,
   workingDirectory,
   bin = "bin/indexer",
   selector = {},
   env = {},
+  region = "us-east-1",
+  check,
   deltaSchedule,
   validateSchedule,
   deltaCpu,
   deltaMem,
   rebuildCpu,
-  rebuildMem
+  rebuildMem,
+  validateCpu,
+  validateMem,
 }) {
-  async function deployMode({
-    mode,
-    from,
-    to,
-    stop,
-    production,
-    verbose,
-    ...selectorFlags
-  }) {
+  async function deployMode({ mode, from, to, stop, production, verbose, ...selectorFlags }) {
     if (mode === "delta") {
       // delta mode will ignore from/to flags
       from = 0;
       to = 0;
     }
 
-    const isPeriodic =
-      from === 0 && to === 0 && ["delta", "validate"].includes(mode);
+    const isPeriodic = from === 0 && to === 0 && ["delta", "validate"].includes(mode);
 
-    const jobName = getJobName(name, selectorFlags, mode, isPeriodic);
+    const jobName = getJobName(name, selectorFlags, mode);
 
     const selectorCmdPart = Object.keys(selectorFlags)
       .sort()
-      .map(name => `--${name} ${selectorFlags[name]}`)
+      .map((name) => `--${name} ${selectorFlags[name]}`)
       .join(" ");
-    let cmd = `${bin} --mode ${mode} ${selectorCmdPart}`;
+    let args = `--mode ${mode} ${selectorCmdPart}`;
 
-    if (from > 0) {
-      cmd += ` --from ${from}`;
+    if (verbose) {
+      args += ` --verbose`;
     }
 
-    if (to > 0) {
-      cmd += ` --to ${to}`;
+    let rangeArgs = "";
+
+    if (from > 0) {
+      rangeArgs += ` --from ${from}`;
     }
 
-    if (verbose) {
-      cmd += ` --verbose`;
+    if (to > 0) {
+      rangeArgs += ` --to ${to}`;
     }
 
     const modeResouces = {
       rebuild: { cpu: rebuildCpu, mem: rebuildMem },
       "resume-rebuild": { cpu: rebuildCpu, mem: rebuildMem },
-      validate: { cpu: rebuildCpu, mem: rebuildMem },
-      delta: { cpu: deltaCpu, mem: deltaMem }
+      validate: { cpu: validateCpu || rebuildCpu, mem: validateMem || rebuildMem },
+      delta: { cpu: deltaCpu, mem: deltaMem },
     };
 
     // by default use delta cpu/mem settings
     const { cpu, mem } = modeResouces[mode] || modeResouces.delta;
 
-    const intervalAliases = {
-      "@secondly": "*/1 * * * * * *",
-      "@minutely": "0 * * * * * *",
-      "@hourly": "0 0 * * * * *",
-      "@daily": "0 0 0 * * * *",
-      "@weekly": "0 0 0 * * 0 *"
-    };
-
-    const deltaCron =
-      typeof deltaSchedule === "function"
-        ? deltaSchedule(jobName)
-        : deltaSchedule;
+    const deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
 
-    const validateCron =
-      typeof validateSchedule === "function"
-        ? validateSchedule(jobName)
-        : validateSchedule;
+    const validateCron = typeof validateSchedule === "function" ? validateSchedule(jobName) : validateSchedule;
 
     const modeIntervals = {
-      delta: intervalAliases[deltaCron] || deltaCron,
-      validate: intervalAliases[validateCron] || validateCron
+      delta: INTERVAL_ALIASES[deltaCron] || deltaCron,
+      validate: INTERVAL_ALIASES[validateCron] || validateCron,
     };
 
-    const nomadJobDefinition = genConfig({
+    const mainJobDefinition = genConfig({
       jobName,
       cron: isPeriodic && modeIntervals[mode],
       workingDirectory,
       additionalEnv: env,
-      cmd,
+      region,
+      cmd: `${bin} ${args} ${rangeArgs}`,
       cpu,
       mem,
-      isProduction: production
+      isProduction: production,
     });
 
-    await runNomadJob(jobName, nomadJobDefinition, stop, production);
+    const nomadPath = await getNomadPath();
+    const nomadAddr = await getNomadAddr(production);
+
+    await runNomadJob(nomadPath, nomadAddr, jobName, mainJobDefinition, stop);
+
+    if (check && check.bin) {
+      console.log("");
+
+      const monitorJobName = `${jobName}-monitor`;
+      const monitorJobDefinition = genConfig({
+        jobName: monitorJobName,
+        cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
+        workingDirectory,
+        additionalEnv: {
+          ...env,
+          SKYNET_SLACK_TOKEN: null,
+          SKYNET_NOMAD_URL: null,
+        },
+        region,
+        cmd: `${check.bin} ${args}`,
+        cpu: check.cpu || 100,
+        mem: check.mem || 100,
+        isProduction: production,
+      });
+
+      await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop);
+    }
   }
 
   function deploy() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `
@@ -389,36 +406,36 @@ ${getSelectorDesc(selector)}
           ...getSelectorFlags(selector),
           mode: {
             type: "string",
-            default: "delta"
+            default: "delta",
           },
           from: {
             alias: "since",
             type: "number",
-            default: 0
+            default: 0,
           },
           to: {
             alias: "until",
             type: "number",
-            default: 0
+            default: 0,
           },
           verbose: {
             type: "boolean",
-            default: false
+            default: false,
           },
           production: {
             alias: "prd",
             type: "boolean",
-            default: false
+            default: false,
           },
           stop: {
             type: "boolean",
-            default: false
-          }
-        }
+            default: false,
+          },
+        },
       }
     );
 
-    deployMode(cli.flags).catch(err => {
+    deployMode(cli.flags).catch((err) => {
       console.error(err);
       process.exit(1);
     });
@@ -428,54 +445,81 @@ ${getSelectorDesc(selector)}
 }
 
 function createDeploy({
+  binaryName,
   name,
   workingDirectory,
   bin = "bin/indexer",
   selector = {},
+  region = "us-east-1",
   env = {},
+  check,
   schedule,
   restart,
   cpu,
-  mem
+  mem,
 }) {
-  async function deployModeless({
-    production,
-    stop,
-    verbose,
-    ...selectorFlags
-  }) {
-    const jobName = getJobName(name, selectorFlags, null, !!schedule);
+  async function deployModeless({ production, stop, verbose, ...selectorFlags }) {
+    const jobName = getJobName(name, selectorFlags, null);
 
     const selectorCmdPart = Object.keys(selectorFlags)
       .sort()
-      .map(name => `--${name} ${selectorFlags[name]}`)
+      .map((name) => `--${name} ${selectorFlags[name]}`)
       .join(" ");
-    let cmd = `${bin} ${selectorCmdPart}`;
+    let args = `${selectorCmdPart}`;
 
     if (verbose) {
-      cmd += ` --verbose`;
+      args += ` --verbose`;
     }
 
     const cron = typeof schedule === "function" ? schedule(jobName) : schedule;
 
     const nomadJobDefinition = genConfig({
       jobName,
-      cron,
+      cron: INTERVAL_ALIASES[cron] || cron,
       restart,
       workingDirectory,
       additionalEnv: env,
-      cmd,
+      region,
+      cmd: `${bin} ${args}`,
       cpu,
       mem,
-      isProduction: production
+      check,
+      isProduction: production,
     });
 
-    await runNomadJob(jobName, nomadJobDefinition, stop, production);
+    const nomadPath = await getNomadPath();
+    const nomadAddr = await getNomadAddr(production);
+
+    await runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, stop);
+
+    if (check && check.bin) {
+      console.log("");
+
+      const monitorJobName = `${jobName}-monitor`;
+      const monitorJobDefinition = genConfig({
+        jobName: monitorJobName,
+        cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
+        workingDirectory,
+        additionalEnv: {
+          ...env,
+          SKYNET_SLACK_TOKEN: null,
+          SKYNET_NOMAD_URL: null,
+        },
+        region,
+        cmd: `${check.bin} ${args}`,
+        cpu: check.cpu || 100,
+        mem: check.mem || 100,
+        isProduction: production,
+      });
+
+      await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop);
+    }
   }
 
   function deploy() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `
@@ -495,22 +539,22 @@ ${getSelectorDesc(selector)}
           ...getSelectorFlags(selector),
           verbose: {
             type: "boolean",
-            default: false
+            default: false,
           },
           production: {
             alias: "prd",
             type: "boolean",
-            default: false
+            default: false,
           },
           stop: {
             type: "boolean",
-            default: false
-          }
-        }
+            default: false,
+          },
+        },
       }
     );
 
-    deployModeless(cli.flags).catch(err => {
+    deployModeless(cli.flags).catch((err) => {
       console.error(err);
       process.exit(1);
     });
@@ -520,6 +564,7 @@ ${getSelectorDesc(selector)}
 }
 
 module.exports = {
+  getJobName,
   createModeDeploy,
-  createDeploy
+  createDeploy,
 };

package/examples/consumer

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+// this file is executable and is for kafka producer testing
+// a few test commands to try on
+//   $ examples/consumer run --protocol bsc --verbose
+//   $ examples/consumer run --protocol eth
+//   $ examples/consumer check --protocol eth
+//   $ examples/consumer deploy --protocol eth
+//   $ examples/consumer --help
+
+const { consumer, SENSITIVE_VALUE, ERROR_LEVEL } = require("../app");
+
+async function consume({ protocol, messages, verbose }) {
+  console.log("consume called with", protocol, messages, verbose);
+}
+
+async function check({ protocol, state, verbose }) {
+  console.log("check called with", protocol, state, verbose);
+
+  const errors = [];
+
+  errors.push({ type: ERROR_LEVEL.CRITICAL, message: "processed height lagged behind" });
+
+  return errors;
+}
+
+const app = consumer({
+  name: "LibSkynetExampleConsumer",
+  selector: { protocol: { type: "string" } },
+
+  env: {
+    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
+    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
+    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
+  },
+
+  consume: {
+    func: consume,
+    topic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dev`,
+    maxRetry: 1,
+
+    cpu: 600,
+    mem: 200,
+  },
+
+  check: {
+    func: check,
+    schedule: "@minutely",
+    slackChannel: "skynet-notifications-local-dev",
+  },
+});
+
+app();