@certik/skynet 0.9.5 → 0.10.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.10.0
+
+- Added a new application type `api` to the `app` library
+
+## 0.9.7
+
+- Added Polygon multicall support
+
+## 0.9.6
+
+- Updated `inline` method in `log` library to print a timestamp in front of each log line
+
 ## 0.9.5
 
 - Fixed deploy issue when selector value contains special characters

package/api.js

@@ -0,0 +1,105 @@
+const express = require("express");
+const meow = require("meow");
+const { getSelectorFlags, getSelectorDesc } = require("./selector");
+const { isProduction } = require("./env");
+
+const apiKeyMiddleware = (key) => {
+  async function requireAPIKey(req, res, next) {
+    try {
+      const apiKey = req.get("x-api-key");
+
+      if (!apiKey) {
+        console.log("request without api key");
+
+        res.status(400).send("require x-api-key header");
+
+        return;
+      }
+
+      if (apiKey !== key) {
+        console.log("request has an invalid api key");
+
+        res.status(400).send("invalid api key");
+
+        return;
+      }
+
+      next();
+    } catch (err) {
+      console.log("check api key error", err);
+
+      res.status(500).send("internal error");
+    }
+  }
+
+  return requireAPIKey;
+};
+
+function startApiApp({ binaryName, name, selector = {}, routes, serve }) {
+  const app = express();
+  app.use(express.json());
+
+  const cli = meow(
+    `
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --verbose       Output debug messages
+      `,
+    {
+      description: false,
+      version: false,
+      flags: {
+        ...getSelectorFlags(selector),
+        verbose: {
+          type: "boolean",
+          default: false,
+        },
+      },
+    }
+  );
+
+  const { verbose, ...selectorFlags } = cli.flags;
+
+  // for health check
+  app.get("/", (req, res) => {
+    res.send("ok");
+  });
+
+  for (const route of routes) {
+    const method = route.method ? route.method.toLowerCase() : "get";
+    const middlewares = route.middlewares || [];
+
+    if (route.protected) {
+      middlewares.push(apiKeyMiddleware(serve.apiKey));
+    }
+
+    if (app[method]) {
+      if (verbose) {
+        console.log(`registering ${method} ${route.path}`);
+      }
+
+      app[method](route.path, ...middlewares, async (req, res) => {
+        try {
+          await route.handler({ req, res, ...selectorFlags });
+        } catch (routeErr) {
+          console.log("caught route err", routeErr);
+
+          res.status(500).send(`internal server error: ${routeErr.message}`);
+        }
+      });
+    }
+  }
+
+  app.listen(serve.port, () => {
+    if (isProduction()) {
+      console.log(`${name} listening at https://api.certik-skynet.com/${serve.prefix}`);
+    } else {
+      console.log(`${name} listening at http://localhost:${serve.port}`);
+    }
+  });
+}
+
+module.exports = { startApiApp };

package/app.js

@@ -4,6 +4,7 @@ const { SKYNET_API_PREFIX } = require("./const");
 const { createIndexerApp, createModeIndexerApp } = require("./indexer");
 const { createDeploy, createModeDeploy } = require("./deploy");
 const { createProducerApp, createConsumerApp } = require("./kafka");
+const { startApiApp } = require("./api");
 const { createMonitor, ERROR_LEVEL } = require("./monitor");
 const { getBinaryName, detectBin, detectWorkingDirectory } = require("./cli");
 
@@ -202,7 +203,7 @@ function indexer({ name, selector, build, check, env = {}, region = "us-east-1"
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100
+          mem: check.mem || 100,
         },
       });
 
@@ -215,7 +216,7 @@ function indexer({ name, selector, build, check, env = {}, region = "us-east-1"
         type: "stateless",
         selector,
         check: check.func,
-        maxRetry: check.maxRetry
+        maxRetry: check.maxRetry,
       });
 
       monitor();
@@ -315,7 +316,7 @@ function modeIndexer({ name, selector, state, build, validate, check, env = {},
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100
+          mem: check.mem || 100,
         },
       });
 
@@ -329,7 +330,7 @@ function modeIndexer({ name, selector, state, build, validate, check, env = {},
         selector,
         mode: true,
         check: check.func,
-        maxRetry: check.maxRetry
+        maxRetry: check.maxRetry,
       });
 
       monitor();
@@ -417,7 +418,7 @@ function producer({ name, selector, produce, check, state, env = {}, region = "u
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100
+          mem: check.mem || 100,
         },
       });
 
@@ -430,7 +431,7 @@ function producer({ name, selector, produce, check, state, env = {}, region = "u
         type: "stateful",
         selector,
         check: check.func,
-        maxRetry: check.maxRetry
+        maxRetry: check.maxRetry,
       });
 
       monitor();
@@ -509,7 +510,7 @@ function consumer({ name, selector, consume, check, env = {}, region = "us-east-
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100
+          mem: check.mem || 100,
         },
       });
 
@@ -522,7 +523,147 @@ function consumer({ name, selector, consume, check, env = {}, region = "us-east-
         type: "stateless",
         selector,
         check: check.func,
-        maxRetry: check.maxRetry
+        maxRetry: check.maxRetry,
+      });
+
+      monitor();
+    },
+  });
+}
+
+function checkApiServeParameter(serve, routes) {
+  const errors = [];
+
+  if (!serve?.prefix) {
+    errors.push("must define serve.prefix");
+  } else if (!serve.prefix.startsWith("/")) {
+    errors.push("server.prefix must start with /, e.g. /my-api");
+  }
+
+  if (!serve?.port) {
+    errors.push("must define serve.port");
+  }
+
+  if (!serve?.cpu) {
+    errors.push("must define serve.cpu");
+  }
+
+  if (!serve?.mem) {
+    errors.push("must define serve.mem");
+  }
+
+  if (routes.some((r) => r.protected) && !serve.apiKey) {
+    errors.push("must define serve.apiKey since some routes are protected");
+  }
+
+  return errors;
+}
+
+function checkApiRoutesParameter(routes) {
+  const errors = [];
+
+  if (!Array.isArray(routes)) {
+    errors.push("routes must be an array");
+  } else {
+    for (let i = 0; i < routes.length; i++) {
+      const route = routes[i];
+
+      if (!route.path) {
+        errors.push(`routes[${i}] must define path`);
+      }
+
+      if (!route.handler) {
+        errors.push(`routes[${i}] must define handler`);
+      }
+
+      if (route.middlewares && !Array.isArray(route.middlewares)) {
+        errors.push(`routes[${i}].middlewares must be an array`);
+      }
+    }
+  }
+
+  return errors;
+}
+
+function api({ name, routes, serve, env = {}, region = "us-east-1" }) {
+  // do not support selector for now
+  const selector = {};
+
+  // hard code a simple health check for now
+  const check = {
+    func: async () => {
+      const errors = [];
+
+      const url = `${serve.prefix}/`;
+      const res = await fetch(url);
+
+      if (!res.ok) {
+        errors.push({
+          type: ERROR_LEVEL.ERROR,
+          message: `service ${name} is not healthy`,
+        });
+      }
+
+      return errors;
+    },
+    schedule: every(5).minutes,
+    maxRetry: 0,
+  };
+
+  return createApp({
+    parameterErrors: [
+      ...checkApiRoutesParameter(routes),
+      ...checkApiServeParameter(serve, routes),
+      ...checkCheckParameter(check),
+      ...checkEnvParameter(env),
+    ],
+    env,
+    check,
+    onRun: () => {
+      startApiApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        routes,
+        serve,
+      });
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: `${bin} run`,
+        selector,
+        region,
+        env,
+        schedule: "@minutely",
+        cpu: serve.cpu,
+        mem: serve.mem,
+        service: {
+          prefix: serve.prefix,
+          port: serve.port,
+        },
+        check: check && {
+          bin: `${bin} check`,
+          schedule: check.schedule,
+          cpu: check.cpu || 100,
+          mem: check.mem || 100,
+        },
+      });
+
+      deploy();
+    },
+    onCheck: () => {
+      const { monitor } = createMonitor({
+        binaryName: `${getBinaryName()} check`,
+        name,
+        type: "stateless",
+        selector,
+        check: check.func,
+        maxRetry: check.maxRetry,
       });
 
       monitor();
@@ -551,4 +692,4 @@ const every = (n = 1) => {
   };
 };
 
-module.exports = { indexer, modeIndexer, producer, consumer, every, SENSITIVE_VALUE, ERROR_LEVEL };
+module.exports = { indexer, modeIndexer, producer, consumer, api, every, SENSITIVE_VALUE, ERROR_LEVEL };

package/const.js

@@ -59,7 +59,7 @@ const PROTOCOLS = {
       endpoint: "https://api.polygonscan.com/api",
       key: getPolygonScanApiKey(),
     },
-    multiCallProvider: "", // TODO
+    multiCallProvider: "0x8eC86392e0aDB57d00fDffbA39b8870e107c0757",
     scanUrl: "https://polygonscan.com/",
   },
 };

package/deploy.js

@@ -49,6 +49,7 @@ const genConfig = ({
   restart,
   cpu,
   mem,
+  service,
   additionalEnv = [],
   region = "us-east-1",
   isProduction,
@@ -123,6 +124,34 @@ const genConfig = ({
         ]
       }
 
+      ${
+        service
+          ? `# Setup API Routes
+      network {
+        port "http" {
+          static = ${service.port}
+        }
+      }
+
+      service {
+        name = "${jobName}"
+        port = "http"
+
+        tags = [
+          "urlprefix-${service.prefix} strip=${service.prefix}",
+        ]
+
+        check {
+          type     = "http"
+          path     = "/"
+          interval = "10s"
+          timeout  = "2s"
+        }
+      }
+`
+          : ""
+      }
+
       template {
         change_mode = "restart"
         destination = "secrets/context.env"
@@ -507,6 +536,7 @@ function createDeploy({
   restart,
   cpu,
   mem,
+  service,
 }) {
   async function deployModeless({ production, stop, dryRun, verbose, schedule: cmdSchedule, ...selectorFlags }) {
     const jobName = getJobName(name, selectorFlags, null);
@@ -538,6 +568,7 @@ function createDeploy({
       cmd: `${bin} ${args}`,
       cpu,
       mem,
+      service,
       isProduction: production,
     });
 

package/examples/api

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+
+// this file is executable and is for kafka producer testing
+// a few test commands to try on
+//   $ examples/api run --verbose
+//   $ examples/api run
+//   $ examples/api check
+//   $ examples/api deploy
+//   $ examples/api --help
+
+const { api, every, SENSITIVE_VALUE, ERROR_LEVEL } = require("../app");
+
+// an example middleware
+async function logRequest(req, res, next) {
+  const start = new Date();
+  next();
+  const end = new Date();
+  const logInfo = { start, end, elapsed: `${end - start}ms` };
+  console.log(logInfo);
+}
+
+async function getProjectsHandler({ req, res, verbose }) {
+  console.log("receieved call", req.query, verbose);
+
+  const type = req.query.type;
+
+  if (!type) {
+    res.status(400).end("must provide type parameter");
+    return;
+  }
+
+  res.json([{ id: "proj-1", name: "Test Project 1" }]);
+}
+
+async function createProjectHandler({ req, res, verbose }) {
+  console.log("receieved call", req.body, verbose);
+
+  res.json({ success: true });
+}
+
+const app = api({
+  name: "example-api",
+
+  env: {
+    EXAMPLE_API_KEY: SENSITIVE_VALUE,
+  },
+
+  routes: [
+    {
+      method: "GET",
+      path: "/projects",
+      handler: getProjectsHandler,
+      middlewares: [logRequest],
+    },
+    {
+      method: "POST",
+      path: "/projects",
+      handler: createProjectHandler,
+      protected: true,
+      middlewares: [logRequest],
+    },
+  ],
+
+  serve: {
+    prefix: "/example-api", // deployed to a sub path
+    port: 12345,
+    apiKey: process.env.EXAMPLE_API_KEY,
+    cpu: 600,
+    mem: 200,
+  },
+});
+
+app();

package/examples/consumer

@@ -25,7 +25,7 @@ async function check({ protocol, state, verbose }) {
 }
 
 const app = consumer({
-  name: "LibSkynetExampleConsumer",
+  name: "example-consumer",
   selector: { protocol: { type: "string", description: "from which chain to consume data" } },
 
   consume: {

package/examples/indexer

@@ -35,7 +35,7 @@ async function check({ protocol, state, verbose }) {
 }
 
 const app = indexer({
-  name: "LibSkynetExampleIndexer",
+  name: "example-indexer",
 
   selector: {
     // for more flags check meow documentation at https://github.com/sindresorhus/meow

package/examples/mode-indexer

@@ -32,7 +32,7 @@ async function check({ mode, protocol, state, verbose }) {
 }
 
 const app = modeIndexer({
-  name: "LibSkynetExampleModeIndexer",
+  name: "example-mode-indexer",
 
   selector: {
     // for more flags check meow documentation at https://github.com/sindresorhus/meow

package/examples/producer

@@ -32,7 +32,7 @@ async function check({ protocol, state, verbose }) {
 }
 
 const app = producer({
-  name: "LibSkynetExampleProducer",
+  name: "example-producer",
   selector: { protocol: { type: "string", description: "for which chain to produce data" } },
 
   env: {

package/log.js

@@ -1,5 +1,5 @@
 function getLine(params) {
-  let line = `${new Date().toISOString()}\t`;
+  let line = "";
 
   // Convert to string and filter out newline to tabs (AWS Athena)
   for (let i = 0, l = params.length; i < l; i++) {
@@ -11,12 +11,16 @@ function getLine(params) {
   return line.trim();
 }
 
+function timestamp() {
+  return new Date().toISOString();
+}
+
 const inline = {
   log: function (...args) {
-    console.log(getLine(args));
+    console.log(`${timestamp()} ${getLine(args)}`);
   },
   error: function (...args) {
-    console.error(getLine(args));
+    console.error(`${timestamp()} ${getLine(args)}`);
   },
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.9.5",
+  "version": "0.10.0",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",
@@ -16,6 +16,7 @@
     "aws-sdk": "^2.932.0",
     "chalk": "^4.1.1",
     "execa": "^5.0.0",
+    "express": "^4.18.1",
     "kafkajs": "^1.15.0",
     "meow": "^7.0.1",
     "node-fetch": "^2.6.1",

package/examples/legacy-deploy-consumer

@@ -1,24 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for deployer testing
-// try this test command
-//   $ examples/deploy-consumer --protocol bsc
-
-const { createDeploy } = require("../deploy");
-
-const { deploy } = createDeploy({
-  name: "lib-skynet-test-consumer",
-  workingDirectory: "lib-skynet",
-  bin: "examples/kafka-consumer",
-  selector: { protocol: { type: "string" } },
-  env: {
-    SKYNET_KAFKA_SERVER: process.env.SKYNET_KAFKA_SERVER,
-    SKYNET_KAFKA_USERNAME: process.env.SKYNET_KAFKA_USERNAME,
-    SKYNET_KAFKA_PASSWORD: process.env.SKYNET_KAFKA_PASSWORD
-  },
-  restart: { attempts: 3 },
-  cpu: 600,
-  mem: 200
-});
-
-deploy();

package/examples/legacy-deploy-indexer

@@ -1,19 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for indexer testing
-// try this test command
-//   $ examples/deploy-indexer --protocol bsc
-
-const { createDeploy } = require("../deploy");
-
-const { deploy } = createDeploy({
-  name: "lib-skynet-test-indexer",
-  workingDirectory: "lib-skynet",
-  bin: "examples/indexer",
-  selector: { protocol: { type: "string" } },
-  schedule: "@minutely",
-  cpu: 600,
-  mem: 200,
-});
-
-deploy();

package/examples/legacy-deploy-mode-indexer

@@ -1,22 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for indexer with mode testing
-// try this test command
-//   $ examples/deploy-mode-indexer --protocol bsc
-
-const { createModeDeploy } = require("../deploy");
-
-const { deploy } = createModeDeploy({
-  name: "lib-skynet-test-mode-indexer",
-  workingDirectory: "lib-skynet",
-  bin: "examples/mode-indexer",
-  selector: { protocol: { type: "string" } },
-  deltaSchedule: "@minutely",
-  validateSchedule: "@hourly",
-  deltaCpu: 600,
-  deltaMem: 200,
-  rebuildCpu: 800,
-  rebuildMem: 400
-});
-
-deploy();

package/examples/legacy-deploy-producer

@@ -1,25 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for deployer testing
-// try this test command
-//   $ examples/deploy-producer --protocol bsc --from 1
-
-const path = require("path");
-
-const { createDeploy } = require("../deploy");
-
-const { deploy } = createDeploy({
-  name: "lib-skynet-test-producer",
-  workingDirectory: "lib-skynet", // where package.json was located
-  bin: "examples/kafka-producer",
-  selector: { protocol: { type: "string" } },
-  env: {
-    SKYNET_KAFKA_SERVER: process.env.SKYNET_KAFKA_SERVER,
-    SKYNET_KAFKA_USERNAME: process.env.SKYNET_KAFKA_USERNAME,
-    SKYNET_KAFKA_PASSWORD: process.env.SKYNET_KAFKA_PASSWORD
-  },
-  cpu: 600,
-  mem: 200
-});
-
-deploy();

package/examples/legacy-indexer

@@ -1,25 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for kafka producer testing
-// a few test commands to try on
-//   $ examples/indexer --protocol bsc --verbose
-//   $ examples/indexer --protocol eth
-//   $ examples/indexer --help
-
-const { createIndexerApp } = require("../indexer");
-
-async function build({ protocol, verbose }) {
-  console.log("build called with", protocol, verbose);
-
-  if (protocol === "eth") {
-    throw new Error("when problem is not recoverable and cannot be resolved by retry, throw error here");
-  }
-}
-
-const { run } = createIndexerApp({
-  name: "lib-skynet-test-indexer",
-  selector: { protocol: { type: "string" } },
-  build
-});
-
-run();

package/examples/legacy-kafka-consumer

@@ -1,33 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for kafka consumer testing
-// a few test commands to try on
-//   $ examples/kafka-consumer --protocol bsc --verbose
-//   $ examples/kafka-consumer --help
-
-const { createConsumerApp } = require("../kafka");
-
-async function consume({ protocol, messages, verbose }) {
-  console.log("consume called with", protocol, messages, verbose);
-
-  messages.forEach(message => {
-    // if (message.id === 4) {
-    //   throw new Error(
-    //     "when critical error that cannot be fixed happened, we can throw and stop processing here"
-    //   );
-    // }
-  });
-}
-
-const { run } = createConsumerApp({
-  name: "ShareJSTestConsumer",
-  selector: { protocol: { type: "string" } },
-
-  consumer: {
-    topic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dev`,
-    consume,
-    maxRetry: 1 // default to 2
-  }
-});
-
-run();

package/examples/legacy-kafka-producer

@@ -1,64 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for kafka producer testing
-// a few test commands to try on
-//   $ examples/kafka-producer --protocol bsc --verbose
-//   $ examples/kafka-producer --protocol bsc --from 4 --verbose
-//   $ examples/kafka-producer --protocol bsc --from 4 --to 5 --verbose
-//   $ examples/kafka-producer --protocol bsc --status
-//   $ examples/kafka-producer --help
-
-const { createProducerApp } = require("../kafka");
-
-async function produce({ protocol, from, to, verbose, send }) {
-  console.log("produce called with", protocol, from, to, verbose);
-
-  if (from <= 50) {
-    // send records to topic
-    const items = [];
-    for (let i = from; i <= to; i++) {
-      items.push({ id: i, name: `User ${i}` });
-    }
-
-    await send(items);
-
-    return []; // no errors
-  } else if (from > 80) {
-    throw new Error("critical error, cannot continue, better to throw");
-  } else {
-    const failed = [];
-
-    for (let i = from; i <= to; i++) {
-      failed.push(i);
-    }
-
-    // return failed item ids, which will be send to dead letter topic
-    return failed;
-  }
-}
-
-const { run } = createProducerApp({
-  name: "ShareJSTestProducer",
-  selector: { protocol: { type: "string" } },
-
-  producer: {
-    topic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dev`,
-    deadLetterTopic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dead-letter-dev`, // problematic ids will be sent to dead letter topic for later retry
-    produce,
-    batchSize: 10, // default to 50
-    maxRetry: 1 // default to 2
-  },
-
-  state: {
-    type: "block", // can be omitted, default is block
-    updateInterval: ({ protocol }) => (protocol === "bsc" ? 3000 : 20000), // how often max id is increasing, will determine poll interval, default to 5000ms
-    getMinId: async () => 4, // default returns 1
-    getMaxId: async ({ protocol }) => {
-      console.log("getMaxId called", protocol);
-
-      return 100;
-    }
-  }
-});
-
-run();

package/examples/legacy-mode-indexer

@@ -1,64 +0,0 @@
-#!/usr/bin/env node
-
-// this file is executable and is for kafka producer testing
-// a few test commands to try on
-//   $ examples/indexer --protocol bsc --verbose
-//   $ examples/indexer --help
-
-const { createModeIndexerApp } = require("../indexer");
-const { readFile, writeFile } = require("fs/promises");
-
-async function build({ protocol, from, to, verbose }) {
-  console.log("build called with", protocol, from, to, verbose);
-
-  for (let i = from; i <= to; i++) {
-    if (i !== 5) {
-      // simulate a gap, id = 5
-      // we can use validate mode to fix
-      const filename = `/tmp/ShareJSTestModeIndexer-${protocol}-${i}`;
-
-      await writeFile(filename, "helloworld");
-    }
-  }
-
-  console.log("build done", protocol, from, to);
-}
-
-async function validate({ protocol, from, to, verbose }) {
-  console.log("validate called with", protocol, from, to, verbose);
-
-  for (let i = from; i <= to; i++) {
-    const filename = `/tmp/ShareJSTestModeIndexer-${protocol}-${i}`;
-
-    try {
-      await readFile(filename);
-    } catch (notFound) {
-      // fix missing data
-      await writeFile(filename, "helloworld");
-    }
-  }
-
-  console.log("validate done", protocol, from, to);
-}
-
-const { run } = createModeIndexerApp({
-  name: "ShareJSTestModeIndexer",
-  selector: { protocol: { type: "string" } },
-
-  build,
-  buildBatchSize: 5,
-  buildConcurrency: 2, // default is 1, no concurrency
-  validate,
-  validateBatchSize: 2,
-  validateConcurrency: 2, // default is 1, no concurrency
-
-  state: {
-    type: "block", // can be omitted, default is block
-    getMinId: async () => 2, // default returns 1
-    getMaxId: async ({ protocol }) => {
-      return 10;
-    }
-  }
-});
-
-run();