@certik/skynet 0.10.6 → 0.10.9

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.10.9
+
+- Improved `inline.log` to print one level of object
+
+## 0.10.8
+
+- Added logMiddleware to `api` apps by default
+- Enabled opensearch logging for log-shipper
+
+## 0.10.7
+
+- Added `ably` library
+
 ## 0.10.6
 
 - Fixed producer check function

package/ably.js

@@ -0,0 +1,29 @@
+const Ably = require("ably");
+const Bottleneck = require("bottleneck/es5");
+const { ensureAndGet } = require("./env");
+
+const ably = new Ably.Realtime(ensureAndGet("ABLY_ROOT_API_KEY"));
+ably.connection.on("connected", () => {
+  console.log("ably connection successful");
+});
+
+async function publishMessage(channel, messageName, messageData) {
+  const channelObj = ably.channels.get(channel);
+
+  const messageDataJson = JSON.stringify(messageData);
+
+  try {
+    await channelObj.publish(messageName, messageDataJson);
+  } catch (error) {
+    console.error("error publishing to ably: ", error);
+  }
+}
+
+const rateLimitedPublishMessage = new Bottleneck({
+  minTime: 1000 / 50,
+}).wrap(publishMessage);
+
+module.exports = {
+  publishMessage,
+  rateLimitedPublishMessage,
+};

package/api.js

@@ -2,6 +2,25 @@ const express = require("express");
 const meow = require("meow");
 const { getSelectorFlags, getSelectorDesc } = require("./selector");
 const { isProduction } = require("./env");
+const { inline } = require("./log");
+
+async function logMiddleware(req, res, next) {
+  const start = new Date();
+  next();
+  const end = new Date();
+  const logInfo = {
+    start,
+    end,
+    elapsed: `${end - start}ms`,
+    endpoint: req.path,
+    host: req.hostname,
+    status: res.statusCode,
+  };
+  if (res.statusMessage) {
+    logInfo.errorMessage = res.statusMessage;
+  }
+  inline.log(JSON.stringify(logInfo));
+}
 
 const apiKeyMiddleware = (key) => {
   async function requireAPIKey(req, res, next) {
@@ -9,7 +28,7 @@ const apiKeyMiddleware = (key) => {
       const apiKey = req.get("x-api-key");
 
       if (!apiKey) {
-        console.log("request without api key");
+        inline.log("request without api key");
 
         res.status(400).send("require x-api-key header");
 
@@ -17,7 +36,7 @@ const apiKeyMiddleware = (key) => {
       }
 
       if (apiKey !== key) {
-        console.log("request has an invalid api key");
+        inline.log("request has an invalid api key");
 
         res.status(400).send("invalid api key");
 
@@ -26,7 +45,7 @@ const apiKeyMiddleware = (key) => {
 
       next();
     } catch (err) {
-      console.log("check api key error", err);
+      inline.log("check api key error", err);
 
       res.status(500).send("internal error");
     }
@@ -72,20 +91,24 @@ ${getSelectorDesc(selector)}
     const method = route.method ? route.method.toLowerCase() : "get";
     const middlewares = route.middlewares || [];
 
+    if (route.log !== false) {
+      middlewares.unshift(logMiddleware);
+    }
+
     if (route.protected) {
-      middlewares.push(apiKeyMiddleware(serve.apiKey));
+      middlewares.unshift(apiKeyMiddleware(serve.apiKey));
     }
 
     if (app[method]) {
       if (verbose) {
-        console.log(`registering ${method} ${route.path}`);
+        inline.log(`registering ${method} ${route.path}`);
       }
 
       app[method](route.path, ...middlewares, async (req, res) => {
         try {
           await route.handler({ req, res, ...selectorFlags });
         } catch (routeErr) {
-          console.log("caught route err", routeErr);
+          inline.log("caught route err", routeErr);
 
           res.status(500).send(`internal server error: ${routeErr.message}`);
         }
@@ -95,9 +118,9 @@ ${getSelectorDesc(selector)}
 
   app.listen(serve.port, () => {
     if (isProduction()) {
-      console.log(`${name} listening at https://api.certik-skynet.com/${serve.prefix}`);
+      inline.log(`${name} listening at https://api.certik-skynet.com/${serve.prefix}`);
     } else {
-      console.log(`${name} listening at http://localhost:${serve.port}`);
+      inline.log(`${name} listening at http://localhost:${serve.port}`);
     }
   });
 }

package/deploy.js

@@ -102,7 +102,7 @@ const genConfig = ({
         command = "sh"
         args = [
           "-c",
-          "cd \${meta.skynet_code_path}/infra-nomad/log-shipper && yarn install --silent && exec bin/shipper --path ${jobName}"
+          "cd \${meta.skynet_code_path}/infra-nomad/log-shipper && yarn install --silent && exec bin/shipper --path ${jobName} --opensearch"
         ]
       }
 

package/examples/api

@@ -2,21 +2,18 @@
 
 // this file is executable and is for kafka producer testing
 // a few test commands to try on
-//   $ examples/api run --verbose
-//   $ examples/api run
-//   $ examples/api check
+//   $ EXAMPLE_API_KEY=abc examples/api run --verbose
+//   $ EXAMPLE_API_KEY=abc examples/api run
 //   $ examples/api deploy
 //   $ examples/api --help
 
 const { api, every, SENSITIVE_VALUE, ERROR_LEVEL } = require("../app");
 
 // an example middleware
-async function logRequest(req, res, next) {
-  const start = new Date();
+async function exampleMiddleware(req, res, next) {
+  console.log("before request!");
   next();
-  const end = new Date();
-  const logInfo = { start, end, elapsed: `${end - start}ms` };
-  console.log(logInfo);
+  console.log("after request!");
 }
 
 async function getProjectsHandler({ req, res, verbose }) {
@@ -50,14 +47,14 @@ const app = api({
       method: "GET",
       path: "/projects",
       handler: getProjectsHandler,
-      middlewares: [logRequest],
+      middlewares: [exampleMiddleware],
     },
     {
       method: "POST",
       path: "/projects",
       handler: createProjectHandler,
       protected: true,
-      middlewares: [logRequest],
+      middlewares: [exampleMiddleware],
     },
   ],
 

package/indexer.js

@@ -5,6 +5,7 @@ const { exponentialRetry } = require("./availability");
 const { range, fillRange } = require("./util");
 const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { getBinaryName } = require("./cli");
+const { inline } = require("./log");
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -90,7 +91,7 @@ function createModeIndexerApp({
         name: `${name}Validate(${toSelectorString(selectorFlags)})`,
       });
 
-      console.log(`RebuildState=${stateItem?.value} Since=${fromItem?.value} Validated=${validateItem?.value}`);
+      inline.log(`RebuildState=${stateItem?.value} Since=${fromItem?.value} Validated=${validateItem?.value}`);
 
       process.exit(0);
     }
@@ -99,7 +100,7 @@ function createModeIndexerApp({
       from = await finalState.getMinId(selectorFlags);
     }
 
-    console.log(
+    inline.log(
       `[MODE INDEXER] mode=${mode}, from=${from}, to=${
         to > 0 ? to : "undefined"
       }, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`
@@ -135,7 +136,7 @@ function createModeIndexerApp({
       );
     } else if (mode === "one") {
       if (to > 0) {
-        console.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
+        inline.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
       }
       await runRange(selectorFlags, from, from, verbose);
     } else if (mode === "range") {
@@ -148,7 +149,7 @@ function createModeIndexerApp({
   async function runRange(selectorFlags, from, to, verbose) {
     const startTime = Date.now();
 
-    console.log(
+    inline.log(
       `[MODE INDEXER] building range, from=${from}, to=${to}, ${toSelectorString(
         selectorFlags,
         ", "
@@ -158,23 +159,23 @@ function createModeIndexerApp({
     const failedIds = await execBuild(selectorFlags, from, to, verbose, false);
 
     if (failedIds.length > 0) {
-      console.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
+      inline.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
       process.exit(1);
     } else {
-      console.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+      inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
       process.exit(0);
     }
   }
 
   async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
     if (!validate) {
-      console.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
+      inline.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
       process.exit(1);
     }
 
     const startTime = Date.now();
 
-    console.log(
+    inline.log(
       `[MODE INDEXER] validating, from=${from}, to=${to}, ${toSelectorString(
         selectorFlags,
         ", "
@@ -183,12 +184,12 @@ function createModeIndexerApp({
 
     const windows = range(from, to, validateBatchSize * validateConcurrency);
 
-    console.log(
+    inline.log(
       `[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`
     );
 
     for (let [windowStart, windowEnd] of windows) {
-      console.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
+      inline.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
 
       const batches = range(windowStart, windowEnd, validateBatchSize);
 
@@ -207,7 +208,7 @@ function createModeIndexerApp({
 
                 return true;
               } catch (err) {
-                console.error(`got error in validation`, err);
+                inline.error(`got error in validation`, err);
 
                 return false;
               }
@@ -232,12 +233,12 @@ function createModeIndexerApp({
         });
 
         if (verbose) {
-          console.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
         }
       }
     }
 
-    console.log(
+    inline.log(
       `[MODE INDEXER] validated ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
     );
   }
@@ -248,7 +249,7 @@ function createModeIndexerApp({
     const windows = range(from, to, buildBatchSize * buildConcurrency);
 
     for (let [windowStart, windowEnd] of windows) {
-      console.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
+      inline.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
 
       const batches = range(windowStart, windowEnd, buildBatchSize);
 
@@ -271,7 +272,7 @@ function createModeIndexerApp({
                   return false;
                 }
               } catch (err) {
-                console.error(`[MODE INDEXER] got error in build`, err);
+                inline.error(`[MODE INDEXER] got error in build`, err);
 
                 return fillRange(batchStart, batchEnd);
               }
@@ -292,7 +293,7 @@ function createModeIndexerApp({
         });
 
         if (verbose) {
-          console.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
         }
       }
 
@@ -311,7 +312,7 @@ function createModeIndexerApp({
   async function runRebuild(selectorFlags, from, to, verbose) {
     const startTime = Date.now();
 
-    console.log(
+    inline.log(
       `[MODE INDEXER] rebuilding, from=${from}, to=${to}, ${toSelectorString(
         selectorFlags,
         ", "
@@ -339,13 +340,13 @@ function createModeIndexerApp({
     });
 
     if (failedIds.length > 0) {
-      console.log(
+      inline.log(
         `[MODE INDEXER] built ${to - from + 1} ${finalState.type} with some failed ${finalState.type}`,
         failedIds
       );
       process.exit(1);
     } else {
-      console.log(
+      inline.log(
         `[MODE INDEXER] built ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
       );
       process.exit(0);
@@ -365,7 +366,7 @@ function createModeIndexerApp({
       const to = await state.getMaxId(selectorFlags);
 
       if (to <= from) {
-        console.log(
+        inline.log(
           `[MODE INDEXER] skip delta because there're no items need to be processed, ${toSelectorString(
             selectorFlags,
             ", "
@@ -375,7 +376,7 @@ function createModeIndexerApp({
         return;
       }
 
-      console.log(
+      inline.log(
         `[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(
           selectorFlags,
           ", "
@@ -386,7 +387,7 @@ function createModeIndexerApp({
         const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
 
         if (failedIds.length > 0) {
-          console.log("[MODE INDEXER] built with some failed txs", failedIds);
+          inline.log("[MODE INDEXER] built with some failed txs", failedIds);
 
           await createRecord(STATE_TABLE_NAME, {
             name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
@@ -410,25 +411,25 @@ function createModeIndexerApp({
             value: to,
           });
 
-          console.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+          inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
           process.exit(0);
         }
       } catch (err) {
-        console.error("[MODE INDEXER] delta build failed", from, to, err);
+        inline.error("[MODE INDEXER] delta build failed", from, to, err);
 
         process.exit(1);
       }
     } else {
-      console.log("[MODE INDEXER] skip because rebuild hasn't done yet");
+      inline.log("[MODE INDEXER] skip because rebuild hasn't done yet");
     }
   }
 
   async function runReset(selectorFlags) {
     const startTime = Date.now();
 
-    console.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
+    inline.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
 
-    console.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
+    inline.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
     await createRecord(STATE_TABLE_NAME, {
       name: `${name}Since(${toSelectorString(selectorFlags)})`,
       value: 0,
@@ -444,7 +445,7 @@ function createModeIndexerApp({
       value: "init",
     });
 
-    console.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
+    inline.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
   }
 
   function run() {
@@ -498,7 +499,7 @@ ${getSelectorDesc(selector)}
     );
 
     runMode(cli.flags).catch((err) => {
-      console.error(err);
+      inline.error(err);
       process.exit(1);
     });
   }
@@ -541,9 +542,9 @@ ${getSelectorDesc(selector)}
       const startTime = Date.now();
 
       if (Object.keys(selectorFlags).length > 0) {
-        console.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
+        inline.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
       } else {
-        console.log(`[INDEXER] starting build`);
+        inline.log(`[INDEXER] starting build`);
       }
 
       const result = await exponentialRetry(
@@ -553,7 +554,7 @@ ${getSelectorDesc(selector)}
 
             return true;
           } catch (err) {
-            console.log(`[INDEXER] got error in build`, err);
+            inline.log(`[INDEXER] got error in build`, err);
 
             return false;
           }
@@ -574,11 +575,11 @@ ${getSelectorDesc(selector)}
         value: new Date().toISOString(),
       });
 
-      console.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
+      inline.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
     }
 
     runBuild(cli.flags).catch((err) => {
-      console.error(err);
+      inline.error(err);
       process.exit(1);
     });
   }

package/kafka.js

@@ -6,6 +6,7 @@ const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selec
 const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
 const { exponentialRetry } = require("./availability");
 const { getBinaryName } = require("./cli");
+const { inline } = require("./log");
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -35,7 +36,7 @@ async function deleteProducerLatestId(name, selectorFlags, verbose) {
 function sendToTopic(producer, topic, verbose) {
   async function send(records) {
     if (verbose) {
-      console.log(`sending ${records.length} records to topic`, records);
+      inline.log(`sending ${records.length} records to topic`, records);
     }
 
     await producer.send({
@@ -204,7 +205,7 @@ ${getSelectorDesc(selector)}
           await finalState.onReset(selectorFlags, verbose);
         }
 
-        console.log("producer reset");
+        inline.log("producer reset");
 
         process.exit(0);
       }
@@ -212,7 +213,7 @@ ${getSelectorDesc(selector)}
       if (status) {
         const latestId = await getProducerLatestId(name, selectorFlags);
 
-        console.log(`LatestID=${latestId}`);
+        inline.log(`LatestID=${latestId}`);
 
         process.exit(0);
       }
@@ -239,7 +240,7 @@ ${getSelectorDesc(selector)}
 
       const producerInstance = await initProducer(name, kafkaCredential);
 
-      console.log("producing to topic", finalTopic);
+      inline.log("producing to topic", finalTopic);
 
       // if to flag was not provided, run this program indefinitely
       const isLongRunning = to === 0;
@@ -275,7 +276,7 @@ ${getSelectorDesc(selector)}
           const batchStart = i;
           const batchEnd = Math.min(batchStart + batchSize - 1, lastId);
 
-          console.log(`building batch ${batchStart}~${batchEnd}`);
+          inline.log(`building batch ${batchStart}~${batchEnd}`);
 
           // add a retry for errors
           const result = await exponentialRetry(
@@ -295,7 +296,7 @@ ${getSelectorDesc(selector)}
 
                 return true;
               } catch (err) {
-                console.error(`Critical error processing batch ${batchStart}~${batchEnd}`, err);
+                inline.error(`Critical error processing batch ${batchStart}~${batchEnd}`, err);
 
                 return false;
               }
@@ -330,7 +331,7 @@ ${getSelectorDesc(selector)}
     }
 
     return build(cli.flags).catch((err) => {
-      console.error(err);
+      inline.error(err);
       process.exit(1);
     });
   }
@@ -381,12 +382,12 @@ ${getSelectorDesc(selector)}
 
         await consumerInstance.subscribe({ topic: finalTopic });
 
-        console.log("subscribed to topic", finalTopic);
+        inline.log("subscribed to topic", finalTopic);
 
         await consumerInstance.run({
           eachBatch: async ({ batch }) => {
             if (verbose) {
-              console.log("received batch, number of items =", batch.messages.length);
+              inline.log("received batch, number of items =", batch.messages.length);
             }
 
             const messages = batch.messages.map((m) => JSON.parse(m.value));
@@ -399,7 +400,7 @@ ${getSelectorDesc(selector)}
 
                   return true;
                 } catch (err) {
-                  console.error(`got critical error`, err);
+                  inline.error(`got critical error`, err);
 
                   return false;
                 }
@@ -412,21 +413,21 @@ ${getSelectorDesc(selector)}
             );
 
             if (!result) {
-              console.error("Terminate consumer due to consume errors, likely bug");
+              inline.error("Terminate consumer due to consume errors, likely bug");
 
               process.exit(1);
             }
           },
         });
       } catch (err) {
-        console.error("Terminate consumer due to critical kafka connection error", err);
+        inline.error("Terminate consumer due to critical kafka connection error", err);
 
         process.exit(1);
       }
     }
 
     return build(cli.flags).catch((err) => {
-      console.error(err);
+      inline.error(err);
       process.exit(1);
     });
   }

package/log.js

@@ -1,3 +1,19 @@
+function isObject(a) {
+  return !!a && a.constructor === Object;
+}
+
+function print(o) {
+  if (Array.isArray(o)) {
+    return `[${o.map(print).join(", ")}]`
+  }
+
+  if (isObject(o)) {
+    return `{${Object.keys(o).map(k => `${k}: ${o[k]}`).join(", ")}}`;
+  }
+
+  return `${o}`;
+}
+
 function getLine(params) {
   let line = "";
 
@@ -5,7 +21,7 @@ function getLine(params) {
   for (let i = 0, l = params.length; i < l; i++) {
     // Certain objects don't get converted
     // Note using JSON.stringfy may be too slow for large objects
-    line += `${params[i]} `.replace(/\n/gm, "\t");
+    line += `${print(params[i])} `.replace(/\n/gm, "\t");
   }
 
   return line.trim();
@@ -25,5 +41,6 @@ const inline = {
 };
 
 module.exports = {
+  print,
   inline,
 };

package/monitor.js

@@ -106,7 +106,12 @@ ${
 
     async function runCheck({ verbose, production, mode, ...selectorFlags }) {
       const startTime = Date.now();
-      console.log(`[MONITOR] starting check, ${toSelectorString(selectorFlags, ", ")}`);
+
+      if (Object.keys(selectorFlags).length > 0) {
+        console.log(`[MONITOR] starting check, ${toSelectorString(selectorFlags, ", ")}`);
+      } else {
+        console.log(`[MONITOR] starting check`);
+      }
 
       let checkState = {};
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.10.6",
+  "version": "0.10.9",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",
@@ -13,7 +13,9 @@
   },
   "dependencies": {
     "@slack/web-api": "^6.4.0",
+    "ably": "^1.2.22",
     "aws-sdk": "^2.932.0",
+    "bottleneck": "^2.19.5",
     "chalk": "^4.1.1",
     "execa": "^5.0.0",
     "express": "^4.18.1",