@certik/skynet 0.10.7 → 0.10.8

package/app.js

@@ -1,709 +1,709 @@
-const { EOL } = require("os");
-const fetch = require("node-fetch");
-const { SKYNET_API_PREFIX } = require("./const");
-const {
-  createIndexerApp,
-  createModeIndexerApp,
-  getIndexerLatestId,
-  getIndexerValidatedId,
-  getIndexerState,
-} = require("./indexer");
-const { createDeploy, createModeDeploy } = require("./deploy");
-const { createProducerApp, createConsumerApp, getProducerLatestId } = require("./kafka");
-const { startApiApp } = require("./api");
-const { createMonitor, ERROR_LEVEL } = require("./monitor");
-const { getBinaryName, detectBin, detectWorkingDirectory } = require("./cli");
-
-function printAppHelp(hasCheck) {
-  console.log(`
-    Usage
-
-      $ ${getBinaryName()} run <options>${hasCheck ? `\n     $ ${getBinaryName()} check <options>` : ""}
-      $ ${getBinaryName()} deploy <options>
-      $ ${getBinaryName()} delete <options>
-  `);
-}
-
-function isDeleteCommand(command) {
-  return ["delete", "stop", "remove"].includes(command);
-}
-
-async function checkDeployEnv(env) {
-  // check sensitive env
-  const missingEnvs = [];
-
-  for (let key of Object.keys(env)) {
-    if (env[key] === SENSITIVE_VALUE) {
-      const res = await fetch(`${SKYNET_API_PREFIX}/secrets/verify?name=${key}`);
-
-      if (res.ok) {
-        const hasEnv = await res.json();
-
-        if (!hasEnv) {
-          missingEnvs.push(key);
-        }
-      }
-    }
-  }
-
-  if (missingEnvs.length > 0) {
-    console.log(`Missing the following sensitive environment on nomad server:${EOL}- ${missingEnvs.join(EOL + "- ")}`);
-
-    process.exit(1);
-  }
-}
-
-function checkAndSetEnv(env) {
-  const missingEnvs = [];
-
-  for (let key of Object.keys(env)) {
-    if (env[key]) {
-      process.env[key] = env[key];
-    } else if (!process.env[key]) {
-      missingEnvs.push(key);
-    }
-  }
-
-  if (missingEnvs.length > 0) {
-    console.log(`The following environment value shouldn't be empty:${EOL}- ${missingEnvs.join(EOL + "- ")}`);
-
-    process.exit(1);
-  }
-}
-
-function createApp({ parameterErrors, env, check, onRun, onDeploy, onCheck }) {
-  if (parameterErrors.length > 0) {
-    console.log(`Parameter Validation Failed:${EOL}- ${parameterErrors.join(EOL + "- ")}`);
-
-    process.exit(1);
-  }
-
-  return () => {
-    const subCommand = process.argv[2];
-
-    // emulate command line call without subcmd
-    process.argv = [process.argv[0], process.argv[1], ...process.argv.slice(3)];
-
-    if (subCommand === "run") {
-      checkAndSetEnv(env);
-
-      onRun();
-    } else if (subCommand === "deploy" || isDeleteCommand(subCommand)) {
-      if (isDeleteCommand(subCommand)) {
-        process.argv.push("--stop");
-      }
-
-      if (process.argv.includes("--production")) {
-        checkDeployEnv(env)
-          .then(() => onDeploy())
-          .catch(console.error);
-      } else {
-        onDeploy();
-      }
-    } else if (!!check && subCommand === "check") {
-      checkAndSetEnv(env);
-
-      onCheck();
-    } else {
-      printAppHelp(!!check);
-    }
-  };
-}
-
-function checkEnvParameter(env) {
-  const errors = [];
-
-  const envKeys = Object.keys(env);
-
-  envKeys.forEach((k) => {
-    if (!env[k] && env[k] !== SENSITIVE_VALUE) {
-      errors.push(`must have valid non-empty value for env.${k}`);
-    }
-  });
-
-  return errors;
-}
-
-function checkIndexerBuildParameter(build) {
-  const errors = [];
-
-  if (!build?.func) {
-    errors.push("must define build.func");
-  }
-
-  if (!build?.cpu) {
-    errors.push("must define build.cpu");
-  }
-
-  if (!build?.mem) {
-    errors.push("must define build.mem");
-  }
-
-  return errors;
-}
-
-function checkCheckParameter(check) {
-  if (!check) {
-    return [];
-  }
-
-  const errors = [];
-
-  if (!check?.func) {
-    errors.push("must define check.func");
-  }
-
-  if (!check?.schedule) {
-    errors.push("must define check.schedule");
-  }
-
-  return errors;
-}
-
-function checkStateParameter(state) {
-  const errors = [];
-
-  if (!state?.type) {
-    errors.push("must define state.type");
-  }
-
-  if (!state?.getMaxId) {
-    errors.push("must define state.getMaxId");
-  }
-
-  return errors;
-}
-
-function indexer({ name, selector, build, check, env = {}, region = "us-east-1" }) {
-  return createApp({
-    parameterErrors: [...checkIndexerBuildParameter(build), ...checkCheckParameter(check), ...checkEnvParameter(env)],
-    env,
-    check,
-    onRun: () => {
-      const { run } = createIndexerApp({
-        binaryName: `${getBinaryName()} run`,
-        name,
-        selector,
-        build: build.func,
-        maxRetry: build.maxRetry,
-      });
-
-      run();
-    },
-    onDeploy: () => {
-      const bin = detectBin();
-
-      const { deploy } = createDeploy({
-        binaryName: `${getBinaryName()} deploy`,
-        name,
-        workingDirectory: detectWorkingDirectory(),
-        bin: `${bin} run`,
-        selector,
-        region,
-        env,
-        schedule: build.schedule,
-        restart: build.restart,
-        cpu: build.cpu,
-        mem: build.mem,
-        check: check && {
-          bin: `${bin} check`,
-          schedule: check.schedule,
-          cpu: check.cpu || 100,
-          mem: check.mem || 100,
-        },
-      });
-
-      deploy();
-    },
-    onCheck: () => {
-      const { monitor } = createMonitor({
-        binaryName: `${getBinaryName()} check`,
-        name,
-        type: "stateless",
-        selector,
-        check: check.func,
-        maxRetry: check.maxRetry,
-      });
-
-      monitor();
-    },
-  });
-}
-
-function checkModeIndexerBuildParameter(build) {
-  const errors = [];
-
-  if (!build?.func) {
-    errors.push("must define build.func");
-  }
-
-  if (!build?.cpu) {
-    errors.push("must define build.cpu");
-  }
-
-  if (!build?.mem) {
-    errors.push("must define build.mem");
-  }
-
-  return errors;
-}
-
-function checkModeIndexerValidateParameter(validate) {
-  const errors = [];
-
-  if (validate) {
-    if (!validate.func) {
-      errors.push("must define validate.func");
-    }
-
-    if (!validate.cpu) {
-      errors.push("must define validate.cpu");
-    }
-
-    if (!validate.mem) {
-      errors.push("must define validate.mem");
-    }
-  }
-
-  return errors;
-}
-
-function modeIndexer({ name, selector, state, build, validate, check, env = {}, region = "us-east-1" }) {
-  return createApp({
-    parameterErrors: [
-      ...checkModeIndexerBuildParameter(build),
-      ...checkModeIndexerValidateParameter(validate),
-      ...checkStateParameter(state),
-      ...checkCheckParameter(check),
-      ...checkEnvParameter(env),
-    ],
-    env,
-    check,
-    onRun: () => {
-      const { run } = createModeIndexerApp({
-        binaryName: `${getBinaryName()} run`,
-        name,
-        selector,
-        build: build.func,
-        maxRetry: build.maxRetry,
-        buildBatchSize: build.batchSize,
-        buildConcurrency: build.concurrency,
-        validate: validate && validate.func,
-        validateBatchSize: validate && validate.batchSize,
-        validateConcurrency: validate && validate.concurrency,
-        state,
-      });
-
-      run();
-    },
-    onDeploy: () => {
-      const bin = detectBin();
-
-      const { deploy } = createModeDeploy({
-        binaryName: `${getBinaryName()} deploy`,
-        name,
-        workingDirectory: detectWorkingDirectory(),
-        bin: `${bin} run`,
-        selector,
-        region,
-        env,
-
-        deltaSchedule: build.schedule,
-        deltaCpu: build.cpu,
-        deltaMem: build.mem,
-        rebuildCpu: build.cpu,
-        rebuildMem: build.mem,
-
-        validateSchedule: validate && validate.schedule,
-        validateCpu: validate && validate.cpu,
-        validateMem: validate && validate.mem,
-
-        check: check && {
-          bin: `${bin} check`,
-          schedule: check.schedule,
-          cpu: check.cpu || 100,
-          mem: check.mem || 100,
-        },
-      });
-
-      deploy();
-    },
-    onCheck: () => {
-      const { monitor } = createMonitor({
-        binaryName: `${getBinaryName()} check`,
-        name,
-        getState: async (nam, selectorFlags) => {
-          const latestId = await getIndexerLatestId(nam, selectorFlags);
-          const validatedId = await getIndexerValidatedId(nam, selectorFlags);
-          const buildState = await getIndexerState(nam, selectorFlags);
-
-          return { latestId, validatedId, buildState };
-        },
-        selector,
-        mode: true,
-        check: check.func,
-        maxRetry: check.maxRetry,
-      });
-
-      monitor();
-    },
-  });
-}
-
-function checkProducerProduceParameter(produce) {
-  const errors = [];
-
-  if (!produce?.func) {
-    errors.push("must define produce.func");
-  }
-
-  if (!produce?.topic) {
-    errors.push("must define produce.topic");
-  }
-
-  if (!produce?.deadLetterTopic) {
-    errors.push("must define produce.deadLetterTopic");
-  }
-
-  if (!produce?.cpu) {
-    errors.push("must define produce.cpu");
-  }
-
-  if (!produce?.mem) {
-    errors.push("must define produce.mem");
-  }
-
-  return errors;
-}
-
-function producer({ name, selector, produce, check, state, env = {}, region = "us-east-1" }) {
-  const envWithDefaultValues = {
-    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
-    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
-    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
-    ...env,
-  };
-
-  return createApp({
-    parameterErrors: [
-      ...checkProducerProduceParameter(produce),
-      ...checkStateParameter(state),
-      ...checkCheckParameter(check),
-      ...checkEnvParameter(env),
-    ],
-    env: envWithDefaultValues,
-    check,
-    onRun: () => {
-      const { run } = createProducerApp({
-        binaryName: `${getBinaryName()} run`,
-        name,
-        selector,
-
-        producer: {
-          topic: produce.topic,
-          deadLetterTopic: produce.deadLetterTopic,
-          produce: produce.func,
-          batchSize: produce.batchSize,
-          maxRetry: produce.maxRetry,
-        },
-
-        state,
-      });
-
-      run();
-    },
-    onDeploy: () => {
-      const bin = detectBin();
-
-      const { deploy } = createDeploy({
-        binaryName: `${getBinaryName()} deploy`,
-        name,
-        workingDirectory: detectWorkingDirectory(),
-        bin: `${bin} run`,
-        selector,
-        region,
-        env: envWithDefaultValues,
-        schedule: "@minutely",
-        cpu: produce.cpu,
-        mem: produce.mem,
-        check: check && {
-          bin: `${bin} check`,
-          schedule: check.schedule,
-          cpu: check.cpu || 100,
-          mem: check.mem || 100,
-        },
-      });
-
-      deploy();
-    },
-    onCheck: () => {
-      const { monitor } = createMonitor({
-        binaryName: `${getBinaryName()} check`,
-        name,
-        getState: async (nam, selectorFlags) => {
-          const latestId = await getProducerLatestId(nam, selectorFlags);
-
-          return { latestId };
-        },
-        selector,
-        check: check.func,
-        maxRetry: check.maxRetry,
-      });
-
-      monitor();
-    },
-  });
-}
-
-function checkConsumerConsumeParameter(consume) {
-  const errors = [];
-
-  if (!consume?.func) {
-    errors.push("must define consume.func");
-  }
-
-  if (!consume?.topic) {
-    errors.push("must define consume.topic");
-  }
-
-  if (!consume?.cpu) {
-    errors.push("must define consume.cpu");
-  }
-
-  if (!consume?.mem) {
-    errors.push("must define consume.mem");
-  }
-
-  return errors;
-}
-
-function consumer({ name, selector, consume, check, env = {}, region = "us-east-1" }) {
-  const envWithDefaultValues = {
-    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
-    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
-    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
-    ...env,
-  };
-
-  return createApp({
-    parameterErrors: [
-      ...checkConsumerConsumeParameter(consume),
-      ...checkCheckParameter(check),
-      ...checkEnvParameter(env),
-    ],
-    env: envWithDefaultValues,
-    check,
-    onRun: () => {
-      const { run } = createConsumerApp({
-        binaryName: `${getBinaryName()} run`,
-        name,
-        selector,
-
-        consumer: {
-          topic: consume.topic,
-          consume: consume.func,
-          maxRetry: consume.maxRetry,
-        },
-      });
-
-      run();
-    },
-    onDeploy: () => {
-      const bin = detectBin();
-
-      const { deploy } = createDeploy({
-        binaryName: `${getBinaryName()} deploy`,
-        name,
-        workingDirectory: detectWorkingDirectory(),
-        bin: `${bin} run`,
-        selector,
-        region,
-        env: envWithDefaultValues,
-        schedule: "@minutely",
-        cpu: consume.cpu,
-        mem: consume.mem,
-        check: check && {
-          bin: `${bin} check`,
-          schedule: check.schedule,
-          cpu: check.cpu || 100,
-          mem: check.mem || 100,
-        },
-      });
-
-      deploy();
-    },
-    onCheck: () => {
-      const { monitor } = createMonitor({
-        binaryName: `${getBinaryName()} check`,
-        name,
-        selector,
-        check: check.func,
-        maxRetry: check.maxRetry,
-      });
-
-      monitor();
-    },
-  });
-}
-
-function checkApiServeParameter(serve, routes) {
-  const errors = [];
-
-  if (!serve?.prefix) {
-    errors.push("must define serve.prefix");
-  } else if (!serve.prefix.startsWith("/")) {
-    errors.push("server.prefix must start with /, e.g. /my-api");
-  }
-
-  if (!serve?.port) {
-    errors.push("must define serve.port");
-  }
-
-  if (!serve?.cpu) {
-    errors.push("must define serve.cpu");
-  }
-
-  if (!serve?.mem) {
-    errors.push("must define serve.mem");
-  }
-
-  if (routes.some((r) => r.protected) && !serve.apiKey) {
-    errors.push("must define serve.apiKey since some routes are protected");
-  }
-
-  return errors;
-}
-
-function checkApiRoutesParameter(routes) {
-  const errors = [];
-
-  if (!Array.isArray(routes)) {
-    errors.push("routes must be an array");
-  } else {
-    for (let i = 0; i < routes.length; i++) {
-      const route = routes[i];
-
-      if (!route.path) {
-        errors.push(`routes[${i}] must define path`);
-      }
-
-      if (!route.handler) {
-        errors.push(`routes[${i}] must define handler`);
-      }
-
-      if (route.middlewares && !Array.isArray(route.middlewares)) {
-        errors.push(`routes[${i}].middlewares must be an array`);
-      }
-    }
-  }
-
-  return errors;
-}
-
-function api({ name, routes, serve, env = {}, region = "us-east-1" }) {
-  // do not support selector for now
-  const selector = {};
-
-  // hard code a simple health check for now
-  const check = {
-    func: async () => {
-      const errors = [];
-
-      const url = `http://localhost:9999${serve.prefix}/`;
-      const res = await fetch(url);
-
-      if (!res.ok) {
-        errors.push({
-          type: ERROR_LEVEL.ERROR,
-          message: `service ${name} is not healthy`,
-        });
-      }
-
-      return errors;
-    },
-    schedule: every(5).minutes,
-    maxRetry: 0,
-  };
-
-  return createApp({
-    parameterErrors: [
-      ...checkApiRoutesParameter(routes),
-      ...checkApiServeParameter(serve, routes),
-      ...checkCheckParameter(check),
-      ...checkEnvParameter(env),
-    ],
-    env,
-    check,
-    onRun: () => {
-      startApiApp({
-        binaryName: `${getBinaryName()} run`,
-        name,
-        selector,
-        routes,
-        serve,
-      });
-    },
-    onDeploy: () => {
-      const bin = detectBin();
-
-      const { deploy } = createDeploy({
-        binaryName: `${getBinaryName()} deploy`,
-        name,
-        workingDirectory: detectWorkingDirectory(),
-        bin: `${bin} run`,
-        selector,
-        region,
-        env,
-        schedule: "@minutely",
-        cpu: serve.cpu,
-        mem: serve.mem,
-        service: {
-          prefix: serve.prefix,
-          port: serve.port,
-        },
-        check: check && {
-          bin: `${bin} check`,
-          schedule: check.schedule,
-          cpu: check.cpu || 100,
-          mem: check.mem || 100,
-        },
-      });
-
-      deploy();
-    },
-    onCheck: () => {
-      const { monitor } = createMonitor({
-        binaryName: `${getBinaryName()} check`,
-        name,
-        selector,
-        check: check.func,
-        maxRetry: check.maxRetry,
-      });
-
-      monitor();
-    },
-  });
-}
-
-const SENSITIVE_VALUE = null;
-
-const every = (n = 1) => {
-  if (n === 1) {
-    return {
-      second: "*/1 * * * * * *",
-      minute: "0 * * * * * *",
-      hour: "0 0 * * * * *",
-      day: "0 0 0 * * * *",
-      week: "0 0 0 * * 0 *",
-    };
-  }
-
-  return {
-    seconds: `*/${n} * * * * * *`,
-    minutes: `0 */${n} * * * * *`,
-    hours: `0 0 */${n} * * * *`,
-    days: `0 0 0 */${n} * * *`,
-  };
-};
-
-module.exports = { indexer, modeIndexer, producer, consumer, api, every, SENSITIVE_VALUE, ERROR_LEVEL };
+const { EOL } = require("os");
+const fetch = require("node-fetch");
+const { SKYNET_API_PREFIX } = require("./const");
+const {
+  createIndexerApp,
+  createModeIndexerApp,
+  getIndexerLatestId,
+  getIndexerValidatedId,
+  getIndexerState,
+} = require("./indexer");
+const { createDeploy, createModeDeploy } = require("./deploy");
+const { createProducerApp, createConsumerApp, getProducerLatestId } = require("./kafka");
+const { startApiApp } = require("./api");
+const { createMonitor, ERROR_LEVEL } = require("./monitor");
+const { getBinaryName, detectBin, detectWorkingDirectory } = require("./cli");
+
+function printAppHelp(hasCheck) {
+  console.log(`
+    Usage
+
+      $ ${getBinaryName()} run <options>${hasCheck ? `\n     $ ${getBinaryName()} check <options>` : ""}
+      $ ${getBinaryName()} deploy <options>
+      $ ${getBinaryName()} delete <options>
+  `);
+}
+
+function isDeleteCommand(command) {
+  return ["delete", "stop", "remove"].includes(command);
+}
+
+async function checkDeployEnv(env) {
+  // check sensitive env
+  const missingEnvs = [];
+
+  for (let key of Object.keys(env)) {
+    if (env[key] === SENSITIVE_VALUE) {
+      const res = await fetch(`${SKYNET_API_PREFIX}/secrets/verify?name=${key}`);
+
+      if (res.ok) {
+        const hasEnv = await res.json();
+
+        if (!hasEnv) {
+          missingEnvs.push(key);
+        }
+      }
+    }
+  }
+
+  if (missingEnvs.length > 0) {
+    console.log(`Missing the following sensitive environment on nomad server:${EOL}- ${missingEnvs.join(EOL + "- ")}`);
+
+    process.exit(1);
+  }
+}
+
+function checkAndSetEnv(env) {
+  const missingEnvs = [];
+
+  for (let key of Object.keys(env)) {
+    if (env[key]) {
+      process.env[key] = env[key];
+    } else if (!process.env[key]) {
+      missingEnvs.push(key);
+    }
+  }
+
+  if (missingEnvs.length > 0) {
+    console.log(`The following environment value shouldn't be empty:${EOL}- ${missingEnvs.join(EOL + "- ")}`);
+
+    process.exit(1);
+  }
+}
+
+function createApp({ parameterErrors, env, check, onRun, onDeploy, onCheck }) {
+  if (parameterErrors.length > 0) {
+    console.log(`Parameter Validation Failed:${EOL}- ${parameterErrors.join(EOL + "- ")}`);
+
+    process.exit(1);
+  }
+
+  return () => {
+    const subCommand = process.argv[2];
+
+    // emulate command line call without subcmd
+    process.argv = [process.argv[0], process.argv[1], ...process.argv.slice(3)];
+
+    if (subCommand === "run") {
+      checkAndSetEnv(env);
+
+      onRun();
+    } else if (subCommand === "deploy" || isDeleteCommand(subCommand)) {
+      if (isDeleteCommand(subCommand)) {
+        process.argv.push("--stop");
+      }
+
+      if (process.argv.includes("--production")) {
+        checkDeployEnv(env)
+          .then(() => onDeploy())
+          .catch(console.error);
+      } else {
+        onDeploy();
+      }
+    } else if (!!check && subCommand === "check") {
+      checkAndSetEnv(env);
+
+      onCheck();
+    } else {
+      printAppHelp(!!check);
+    }
+  };
+}
+
+function checkEnvParameter(env) {
+  const errors = [];
+
+  const envKeys = Object.keys(env);
+
+  envKeys.forEach((k) => {
+    if (!env[k] && env[k] !== SENSITIVE_VALUE) {
+      errors.push(`must have valid non-empty value for env.${k}`);
+    }
+  });
+
+  return errors;
+}
+
+function checkIndexerBuildParameter(build) {
+  const errors = [];
+
+  if (!build?.func) {
+    errors.push("must define build.func");
+  }
+
+  if (!build?.cpu) {
+    errors.push("must define build.cpu");
+  }
+
+  if (!build?.mem) {
+    errors.push("must define build.mem");
+  }
+
+  return errors;
+}
+
+function checkCheckParameter(check) {
+  if (!check) {
+    return [];
+  }
+
+  const errors = [];
+
+  if (!check?.func) {
+    errors.push("must define check.func");
+  }
+
+  if (!check?.schedule) {
+    errors.push("must define check.schedule");
+  }
+
+  return errors;
+}
+
+function checkStateParameter(state) {
+  const errors = [];
+
+  if (!state?.type) {
+    errors.push("must define state.type");
+  }
+
+  if (!state?.getMaxId) {
+    errors.push("must define state.getMaxId");
+  }
+
+  return errors;
+}
+
+function indexer({ name, selector, build, check, env = {}, region = "us-east-1" }) {
+  return createApp({
+    parameterErrors: [...checkIndexerBuildParameter(build), ...checkCheckParameter(check), ...checkEnvParameter(env)],
+    env,
+    check,
+    onRun: () => {
+      const { run } = createIndexerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        build: build.func,
+        maxRetry: build.maxRetry,
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+        schedule: build.schedule,
+        restart: build.restart,
+        cpu: build.cpu,
+        mem: build.mem,
+        check: check && {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        type: "stateless",
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkModeIndexerBuildParameter(build) {
+  const errors = [];
+
+  if (!build?.func) {
+    errors.push("must define build.func");
+  }
+
+  if (!build?.cpu) {
+    errors.push("must define build.cpu");
+  }
+
+  if (!build?.mem) {
+    errors.push("must define build.mem");
+  }
+
+  return errors;
+}
+
+function checkModeIndexerValidateParameter(validate) {
+  const errors = [];
+
+  if (validate) {
+    if (!validate.func) {
+      errors.push("must define validate.func");
+    }
+
+    if (!validate.cpu) {
+      errors.push("must define validate.cpu");
+    }
+
+    if (!validate.mem) {
+      errors.push("must define validate.mem");
+    }
+  }
+
+  return errors;
+}
+
+function modeIndexer({ name, selector, state, build, validate, check, env = {}, region = "us-east-1" }) {
+  return createApp({
+    parameterErrors: [
+      ...checkModeIndexerBuildParameter(build),
+      ...checkModeIndexerValidateParameter(validate),
+      ...checkStateParameter(state),
+      ...checkCheckParameter(check),
+      ...checkEnvParameter(env),
+    ],
+    env,
+    check,
+    onRun: () => {
+      const { run } = createModeIndexerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        build: build.func,
+        maxRetry: build.maxRetry,
+        buildBatchSize: build.batchSize,
+        buildConcurrency: build.concurrency,
+        validate: validate && validate.func,
+        validateBatchSize: validate && validate.batchSize,
+        validateConcurrency: validate && validate.concurrency,
+        state,
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createModeDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+
+        deltaSchedule: build.schedule,
+        deltaCpu: build.cpu,
+        deltaMem: build.mem,
+        rebuildCpu: build.cpu,
+        rebuildMem: build.mem,
+
+        validateSchedule: validate && validate.schedule,
+        validateCpu: validate && validate.cpu,
+        validateMem: validate && validate.mem,
+
+        check: check && {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        getState: async (nam, selectorFlags) => {
+          const latestId = await getIndexerLatestId(nam, selectorFlags);
+          const validatedId = await getIndexerValidatedId(nam, selectorFlags);
+          const buildState = await getIndexerState(nam, selectorFlags);
+
+          return { latestId, validatedId, buildState };
+        },
+        selector,
+        mode: true,
+        check: check.func,
+        maxRetry: check.maxRetry,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkProducerProduceParameter(produce) {
+  const errors = [];
+
+  if (!produce?.func) {
+    errors.push("must define produce.func");
+  }
+
+  if (!produce?.topic) {
+    errors.push("must define produce.topic");
+  }
+
+  if (!produce?.deadLetterTopic) {
+    errors.push("must define produce.deadLetterTopic");
+  }
+
+  if (!produce?.cpu) {
+    errors.push("must define produce.cpu");
+  }
+
+  if (!produce?.mem) {
+    errors.push("must define produce.mem");
+  }
+
+  return errors;
+}
+
+function producer({ name, selector, produce, check, state, env = {}, region = "us-east-1" }) {
+  const envWithDefaultValues = {
+    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
+    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
+    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
+    ...env,
+  };
+
+  return createApp({
+    parameterErrors: [
+      ...checkProducerProduceParameter(produce),
+      ...checkStateParameter(state),
+      ...checkCheckParameter(check),
+      ...checkEnvParameter(env),
+    ],
+    env: envWithDefaultValues,
+    check,
+    onRun: () => {
+      const { run } = createProducerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+
+        producer: {
+          topic: produce.topic,
+          deadLetterTopic: produce.deadLetterTopic,
+          produce: produce.func,
+          batchSize: produce.batchSize,
+          maxRetry: produce.maxRetry,
+        },
+
+        state,
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env: envWithDefaultValues,
+        schedule: "@minutely",
+        cpu: produce.cpu,
+        mem: produce.mem,
+        check: check && {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        getState: async (nam, selectorFlags) => {
+          const latestId = await getProducerLatestId(nam, selectorFlags);
+
+          return { latestId };
+        },
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkConsumerConsumeParameter(consume) {
+  const errors = [];
+
+  if (!consume?.func) {
+    errors.push("must define consume.func");
+  }
+
+  if (!consume?.topic) {
+    errors.push("must define consume.topic");
+  }
+
+  if (!consume?.cpu) {
+    errors.push("must define consume.cpu");
+  }
+
+  if (!consume?.mem) {
+    errors.push("must define consume.mem");
+  }
+
+  return errors;
+}
+
+function consumer({ name, selector, consume, check, env = {}, region = "us-east-1" }) {
+  const envWithDefaultValues = {
+    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
+    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
+    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
+    ...env,
+  };
+
+  return createApp({
+    parameterErrors: [
+      ...checkConsumerConsumeParameter(consume),
+      ...checkCheckParameter(check),
+      ...checkEnvParameter(env),
+    ],
+    env: envWithDefaultValues,
+    check,
+    onRun: () => {
+      const { run } = createConsumerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+
+        consumer: {
+          topic: consume.topic,
+          consume: consume.func,
+          maxRetry: consume.maxRetry,
+        },
+      });
+
+      run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env: envWithDefaultValues,
+        schedule: "@minutely",
+        cpu: consume.cpu,
+        mem: consume.mem,
+        check: check && {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkApiServeParameter(serve, routes) {
+  const errors = [];
+
+  if (!serve?.prefix) {
+    errors.push("must define serve.prefix");
+  } else if (!serve.prefix.startsWith("/")) {
+    errors.push("server.prefix must start with /, e.g. /my-api");
+  }
+
+  if (!serve?.port) {
+    errors.push("must define serve.port");
+  }
+
+  if (!serve?.cpu) {
+    errors.push("must define serve.cpu");
+  }
+
+  if (!serve?.mem) {
+    errors.push("must define serve.mem");
+  }
+
+  if (routes.some((r) => r.protected) && !serve.apiKey) {
+    errors.push("must define serve.apiKey since some routes are protected");
+  }
+
+  return errors;
+}
+
+function checkApiRoutesParameter(routes) {
+  const errors = [];
+
+  if (!Array.isArray(routes)) {
+    errors.push("routes must be an array");
+  } else {
+    for (let i = 0; i < routes.length; i++) {
+      const route = routes[i];
+
+      if (!route.path) {
+        errors.push(`routes[${i}] must define path`);
+      }
+
+      if (!route.handler) {
+        errors.push(`routes[${i}] must define handler`);
+      }
+
+      if (route.middlewares && !Array.isArray(route.middlewares)) {
+        errors.push(`routes[${i}].middlewares must be an array`);
+      }
+    }
+  }
+
+  return errors;
+}
+
+function api({ name, routes, serve, env = {}, region = "us-east-1" }) {
+  // do not support selector for now
+  const selector = {};
+
+  // hard code a simple health check for now
+  const check = {
+    func: async () => {
+      const errors = [];
+
+      const url = `http://localhost:9999${serve.prefix}/`;
+      const res = await fetch(url);
+
+      if (!res.ok) {
+        errors.push({
+          type: ERROR_LEVEL.ERROR,
+          message: `service ${name} is not healthy`,
+        });
+      }
+
+      return errors;
+    },
+    schedule: every(5).minutes,
+    maxRetry: 0,
+  };
+
+  return createApp({
+    parameterErrors: [
+      ...checkApiRoutesParameter(routes),
+      ...checkApiServeParameter(serve, routes),
+      ...checkCheckParameter(check),
+      ...checkEnvParameter(env),
+    ],
+    env,
+    check,
+    onRun: () => {
+      startApiApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        routes,
+        serve,
+      });
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+        schedule: "@minutely",
+        cpu: serve.cpu,
+        mem: serve.mem,
+        service: {
+          prefix: serve.prefix,
+          port: serve.port,
+        },
+        check: check && {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
+      });
+
+      monitor();
+    },
+  });
+}
+
+const SENSITIVE_VALUE = null;
+
+const every = (n = 1) => {
+  if (n === 1) {
+    return {
+      second: "*/1 * * * * * *",
+      minute: "0 * * * * * *",
+      hour: "0 0 * * * * *",
+      day: "0 0 0 * * * *",
+      week: "0 0 0 * * 0 *",
+    };
+  }
+
+  return {
+    seconds: `*/${n} * * * * * *`,
+    minutes: `0 */${n} * * * * *`,
+    hours: `0 0 */${n} * * * *`,
+    days: `0 0 0 */${n} * * *`,
+  };
+};
+
+module.exports = { indexer, modeIndexer, producer, consumer, api, every, SENSITIVE_VALUE, ERROR_LEVEL };