@certik/skynet 0.7.20 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.8.0
+
+- Added `app` library to support one binary developer experience
+
 ## 0.7.20
 
 - Added `listKeys` method to `s3.js`

package/app.js

@@ -0,0 +1,476 @@
+const { EOL } = require("os");
+const { createIndexerApp, createModeIndexerApp } = require("./indexer");
+const { createDeploy, createModeDeploy } = require("./deploy");
+const { createProducerApp, createConsumerApp } = require("./kafka");
+const { createMonitor, ERROR_LEVEL } = require("./monitor");
+const { getBinaryName, detectBin, detectWorkingDirectory } = require("./cli");
+
+function printAppHelp() {
+  console.log(`
+    Usage
+      $ ${getBinaryName()} run <options>
+      $ ${getBinaryName()} check <options>
+      $ ${getBinaryName()} deploy <options>
+      $ ${getBinaryName()} delete <options>
+  `);
+}
+
+function isDeleteCommand(command) {
+  return ["delete", "stop", "remove"].includes(command);
+}
+
+function createApp({ parameterErrors, onRun, onDeploy, onCheck }) {
+  if (parameterErrors.length > 0) {
+    console.log(`Parameter Validation Failed:${EOL}- ${parameterErrors.join(EOL + "- ")}`);
+
+    process.exit(1);
+  }
+
+  return () => {
+    const subCommand = process.argv[2];
+
+    // emulate command line call without subcmd
+    process.argv = [process.argv[0], process.argv[1], ...process.argv.slice(3)];
+
+    if (subCommand === "run") {
+      onRun();
+    } else if (subCommand === "deploy" || isDeleteCommand(subCommand)) {
+      if (isDeleteCommand(subCommand)) {
+        process.argv.push("--stop");
+      }
+
+      onDeploy();
+    } else if (subCommand === "check") {
+      onCheck();
+    } else {
+      printAppHelp();
+    }
+  };
+}
+
+function checkEnvParameter(env, required = []) {
+  const errors = [];
+
+  const envKeys = Object.keys(env);
+
+  required.forEach((k) => {
+    if (!envKeys.includes(k)) {
+      errors.push(`must define env.${k}`);
+    }
+  });
+
+  envKeys.forEach((k) => {
+    if (!env[k] && env[k] !== SENSITIVE_VALUE) {
+      errors.push(`must have valid non-empty value for env.${k}`);
+    }
+  });
+
+  return errors;
+}
+
+function checkIndexerBuildParameter(build) {
+  const errors = [];
+
+  if (!build?.func) {
+    errors.push("must define build.func");
+  }
+
+  if (!build?.cpu) {
+    errors.push("must define build.cpu");
+  }
+
+  if (!build?.mem) {
+    errors.push("must define build.mem");
+  }
+
+  return errors;
+}
+
+function checkCheckParameter(check, env) {
+  const errors = [];
+
+  if (!check?.func) {
+    errors.push("must define check.func");
+  }
+
+  if (!check?.schedule) {
+    errors.push("must define check.schedule");
+  }
+
+  if (check.slackChannel && !Object.keys(env).includes("SKYNET_SLACK_TOKEN")) {
+    errors.push("must define env.SKYNET_SLACK_TOKEN when check.slackChannel is present");
+  }
+
+  return errors;
+}
+
+function checkStateParameter(state) {
+  const errors = [];
+
+  if (!state?.type) {
+    errors.push("must define state.type");
+  }
+
+  if (!state?.getMaxId) {
+    errors.push("must define state.getMaxId");
+  }
+
+  return errors;
+}
+
+function indexer({ name, selector, build, check, env = {}, region = "us-east-1" }) {
+  return createApp({
+    parameterErrors: [
+      ...checkIndexerBuildParameter(build),
+      ...checkCheckParameter(check, env),
+      ...checkEnvParameter(env),
+    ],
+    onRun: () => {
+      const { run } = createIndexerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        build: build.func,
+        maxRetry: build.maxRetry,
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+        schedule: build.schedule,
+        restart: build.restart,
+        cpu: build.cpu,
+        mem: build.mem,
+        check: {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+          hasSlack: check.slackChannel,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        type: "stateless",
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
+        slackChannel: check.slackChannel,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkModeIndexerBuildParameter(build) {
+  const errors = [];
+
+  if (!build?.func) {
+    errors.push("must define build.func");
+  }
+
+  if (!build?.cpu) {
+    errors.push("must define build.cpu");
+  }
+
+  if (!build?.mem) {
+    errors.push("must define build.mem");
+  }
+
+  return errors;
+}
+
+function checkModeIndexerValidateParameter(validate) {
+  const errors = [];
+
+  if (validate) {
+    if (!validate.func) {
+      errors.push("must define validate.func");
+    }
+
+    if (!validate.cpu) {
+      errors.push("must define validate.cpu");
+    }
+
+    if (!validate.mem) {
+      errors.push("must define validate.mem");
+    }
+  }
+
+  return errors;
+}
+
+function modeIndexer({ name, selector, state, build, validate, check, env = {}, region = "us-east-1" }) {
+  return createApp({
+    parameterErrors: [
+      ...checkModeIndexerBuildParameter(build),
+      ...checkModeIndexerValidateParameter(validate),
+      ...checkStateParameter(state),
+      ...checkCheckParameter(check, env),
+      ...checkEnvParameter(env),
+    ],
+    onRun: () => {
+      const { run } = createModeIndexerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        build: build.func,
+        maxRetry: build.maxRetry,
+        buildBatchSize: build.batchSize,
+        buildConcurrency: build.concurrency,
+        validate: validate && validate.func,
+        validateBatchSize: validate && validate.batchSize,
+        validateConcurrency: validate && validate.concurrency,
+        state,
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createModeDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+
+        deltaSchedule: build.schedule,
+        deltaCpu: build.cpu,
+        deltaMem: build.mem,
+        rebuildCpu: build.cpu,
+        rebuildMem: build.mem,
+
+        validateSchedule: validate && validate.schedule,
+        validateCpu: validate && validate.cpu,
+        validateMem: validate && validate.mem,
+
+        check: {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+          hasSlack: check.slackChannel,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        type: "stateful",
+        selector,
+        mode: true,
+        check: check.func,
+        maxRetry: check.maxRetry,
+        slackChannel: check.slackChannel,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkProducerProduceParameter(produce) {
+  const errors = [];
+
+  if (!produce?.func) {
+    errors.push("must define produce.func");
+  }
+
+  if (!produce?.topic) {
+    errors.push("must define produce.topic");
+  }
+
+  if (!produce?.deadLetterTopic) {
+    errors.push("must define produce.deadLetterTopic");
+  }
+
+  if (!produce?.cpu) {
+    errors.push("must define produce.cpu");
+  }
+
+  if (!produce?.mem) {
+    errors.push("must define produce.mem");
+  }
+
+  return errors;
+}
+
+function producer({ name, selector, produce, check, state, env = {}, region = "us-east-1" }) {
+  return createApp({
+    parameterErrors: [
+      ...checkProducerProduceParameter(produce),
+      ...checkStateParameter(state),
+      ...checkCheckParameter(check, env),
+      ...checkEnvParameter(env, ["SKYNET_KAFKA_SERVER", "SKYNET_KAFKA_USERNAME", "SKYNET_KAFKA_PASSWORD"]),
+    ],
+    onRun: () => {
+      const { run } = createProducerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+
+        producer: {
+          topic: produce.topic,
+          deadLetterTopic: produce.deadLetterTopic,
+          produce: produce.func,
+          batchSize: produce.batchSize,
+          maxRetry: produce.maxRetry,
+        },
+
+        state,
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+        schedule: "@minutely",
+        cpu: produce.cpu,
+        mem: produce.mem,
+        check: {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+          hasSlack: check.slackChannel,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        type: "stateful",
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
+        slackChannel: check.slackChannel,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkConsumerConsumeParameter(consume) {
+  const errors = [];
+
+  if (!consume?.func) {
+    errors.push("must define consume.func");
+  }
+
+  if (!consume?.topic) {
+    errors.push("must define consume.topic");
+  }
+
+  if (!consume?.cpu) {
+    errors.push("must define consume.cpu");
+  }
+
+  if (!consume?.mem) {
+    errors.push("must define consume.mem");
+  }
+
+  return errors;
+}
+
+function consumer({ name, selector, consume, check, env = {}, region = "us-east-1" }) {
+  return createApp({
+    parameterErrors: [
+      ...checkConsumerConsumeParameter(consume),
+      ...checkCheckParameter(check, env),
+      ...checkEnvParameter(env, ["SKYNET_KAFKA_SERVER", "SKYNET_KAFKA_USERNAME", "SKYNET_KAFKA_PASSWORD"]),
+    ],
+    onRun: () => {
+      const { run } = createConsumerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+
+        consumer: {
+          topic: consume.topic,
+          consume: consume.func,
+          maxRetry: consume.maxRetry,
+        },
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+        schedule: "@minutely",
+        cpu: consume.cpu,
+        mem: consume.mem,
+        check: {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+          hasSlack: check.slackChannel,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        type: "stateless",
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
+        slackChannel: check.slackChannel,
+      });
+
+      monitor();
+    },
+  });
+}
+
+const SENSITIVE_VALUE = null;
+
+module.exports = { indexer, modeIndexer, producer, consumer, SENSITIVE_VALUE, ERROR_LEVEL };

package/cli.js

@@ -0,0 +1,51 @@
+const path = require("path");
+const fs = require("fs");
+
+function getBinaryName() {
+  const binaryNameParts = process.argv[1].split(path.sep);
+  const binaryName = binaryNameParts[binaryNameParts.length - 1];
+
+  return binaryName;
+}
+
+function detectSkynetDirectory() {
+  return detectDirectory(process.argv[1], "SkynetAPIDefinitions.yml");
+}
+
+function detectWorkingDirectory() {
+  const wd = detectDirectory(process.argv[1], "package.json");
+  const skynetd = detectDirectory(process.argv[1], "SkynetAPIDefinitions.yml");
+
+  return wd.slice(skynetd.length + path.sep.length);
+}
+
+function detectDirectory(fullBinPath, sentinel = "package.json") {
+  const binaryNameParts = fullBinPath.split(path.sep);
+
+  const parentFolders = [];
+
+  for (let i = binaryNameParts.length - 1; i > 1; i--) {
+    parentFolders.push(path.join(path.sep, ...binaryNameParts.slice(0, i)));
+  }
+
+  for (let folder of parentFolders) {
+    if (fs.existsSync(path.join(folder, sentinel))) {
+      return folder;
+    }
+  }
+
+  throw new Error("Cannot detect current working directory");
+}
+
+function detectBin() {
+  const wd = detectDirectory(process.argv[1], "package.json");
+
+  return process.argv[1].slice(wd.length + path.sep.length);
+}
+
+module.exports = {
+  getBinaryName,
+  detectSkynetDirectory,
+  detectWorkingDirectory,
+  detectBin,
+};