@certik/skynet 0.10.8 → 0.10.11

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.10.11
+
+- Added distributed lock for `producer` and `api` type apps
+
+## 0.10.10
+
+- Updated `every` helper function in `app` library to support usage like `every(1).days` and `every(10).day`
+
+## 0.10.9
+
+- Improved `inline.log` to print one level of object
+
 ## 0.10.8
 
 - Added logMiddleware to `api` apps by default

package/api.js

@@ -2,6 +2,7 @@ const express = require("express");
 const meow = require("meow");
 const { getSelectorFlags, getSelectorDesc } = require("./selector");
 const { isProduction } = require("./env");
+const { useLock } = require("./distributed-lock");
 const { inline } = require("./log");
 
 async function logMiddleware(req, res, next) {
@@ -54,7 +55,7 @@ const apiKeyMiddleware = (key) => {
   return requireAPIKey;
 };
 
-function startApiApp({ binaryName, name, selector = {}, routes, serve }) {
+async function startApiApp({ binaryName, name, selector = {}, routes, serve }) {
   const app = express();
   app.use(express.json());
 
@@ -82,6 +83,8 @@ ${getSelectorDesc(selector)}
 
   const { verbose, ...selectorFlags } = cli.flags;
 
+  await useLock({ name, ttl: 120, verbose });
+
   // for health check
   app.get("/", (req, res) => {
     res.send("ok");

package/app.js

@@ -635,7 +635,7 @@ function api({ name, routes, serve, env = {}, region = "us-east-1" }) {
     env,
     check,
     onRun: () => {
-      startApiApp({
+      return startApiApp({
         binaryName: `${getBinaryName()} run`,
         name,
         selector,
@@ -691,17 +691,26 @@ const every = (n = 1) => {
   if (n === 1) {
     return {
       second: "*/1 * * * * * *",
+      seconds: "*/1 * * * * * *",
       minute: "0 * * * * * *",
+      minutes: "0 * * * * * *",
       hour: "0 0 * * * * *",
+      hours: "0 0 * * * * *",
       day: "0 0 0 * * * *",
+      days: "0 0 0 * * * *",
       week: "0 0 0 * * 0 *",
+      weeks: "0 0 0 * * 0 *",
     };
   }
 
   return {
+    second: `*/${n} * * * * * *`,
     seconds: `*/${n} * * * * * *`,
+    minute: `0 */${n} * * * * *`,
     minutes: `0 */${n} * * * * *`,
+    hour: `0 0 */${n} * * * *`,
     hours: `0 0 */${n} * * * *`,
+    day: `0 0 0 */${n} * * *`,
     days: `0 0 0 */${n} * * *`,
   };
 };

package/distributed-lock.js

@@ -0,0 +1,111 @@
+const fetch = require("node-fetch");
+
+async function acquireLock(name, ttl) {
+  try {
+    const res = await fetch("http://localhost:8500/v1/session/create", {
+      method: "PUT",
+      body: JSON.stringify({
+        Name: name,
+        TTL: `${ttl}s`,
+        Behavior: "delete",
+      }),
+    });
+
+    if (res.ok) {
+      const { ID: id } = await res.json();
+
+      return id;
+    } else {
+      console.log(res, await res.text());
+
+      return null;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return null;
+  }
+}
+
+async function renewLock(uid) {
+  try {
+    const res = await fetch(`http://localhost:8500/v1/session/renew/${uid}`, {
+      method: "PUT",
+    });
+
+    if (res.ok) {
+      return uid;
+    } else {
+      console.log(res, await res.text());
+
+      return null;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return null;
+  }
+}
+
+async function hasLock(name, uid = null) {
+  try {
+    const res = await fetch(`http://localhost:8500/v1/session/list`);
+
+    if (res.ok) {
+      const sessions = await res.json();
+
+      const sessionsWithTheSameName = sessions.filter((s) => s.Name === name);
+
+      if (!uid) {
+        return sessionsWithTheSameName.length === 0;
+      }
+
+      if (sessionsWithTheSameName.length !== 1) {
+        return false;
+      }
+
+      // only one should have the same uid
+      return sessionsWithTheSameName[0].ID === uid;
+    } else {
+      console.log(await res.text());
+
+      return true;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return true;
+  }
+}
+
+async function useLock({ name, ttl, verbose }) {
+  const lockAvailable = await hasLock(name);
+
+  if (!lockAvailable) {
+    console.log("only one process with the same name should be running, quit");
+    process.exit(0);
+  }
+
+  let uid = await acquireLock(name, ttl);
+
+  setInterval(async () => {
+    if (!(await hasLock(name, uid))) {
+      console.log("only one process with the same name should be running, quit");
+      process.exit(0);
+    }
+
+    if (uid) {
+      if (verbose) {
+        console.log("renewing", name, uid);
+      }
+
+      uid = await renewLock(uid);
+    } else {
+      uid = await acquireLock(name, ttl);
+    }
+  }, (ttl * 1000) / 2);
+
+  return uid;
+}
+
+module.exports = { useLock };

package/kafka.js

@@ -5,6 +5,7 @@ const { Kafka, logLevel } = require("kafkajs");
 const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
 const { exponentialRetry } = require("./availability");
+const { useLock } = require("./distributed-lock");
 const { getBinaryName } = require("./cli");
 const { inline } = require("./log");
 
@@ -218,6 +219,8 @@ ${getSelectorDesc(selector)}
         process.exit(0);
       }
 
+      await useLock({ name, ttl: 120, verbose });
+
       if (!from) {
         const prevId = await getProducerLatestId(name, selectorFlags);
 

package/log.js

@@ -1,3 +1,19 @@
+function isObject(a) {
+  return !!a && a.constructor === Object;
+}
+
+function print(o) {
+  if (Array.isArray(o)) {
+    return `[${o.map(print).join(", ")}]`
+  }
+
+  if (isObject(o)) {
+    return `{${Object.keys(o).map(k => `${k}: ${o[k]}`).join(", ")}}`;
+  }
+
+  return `${o}`;
+}
+
 function getLine(params) {
   let line = "";
 
@@ -5,7 +21,7 @@ function getLine(params) {
   for (let i = 0, l = params.length; i < l; i++) {
     // Certain objects don't get converted
     // Note using JSON.stringfy may be too slow for large objects
-    line += `${params[i]} `.replace(/\n/gm, "\t");
+    line += `${print(params[i])} `.replace(/\n/gm, "\t");
   }
 
   return line.trim();
@@ -25,5 +41,6 @@ const inline = {
 };
 
 module.exports = {
+  print,
   inline,
 };

package/monitor.js

@@ -106,7 +106,12 @@ ${
 
     async function runCheck({ verbose, production, mode, ...selectorFlags }) {
       const startTime = Date.now();
-      console.log(`[MONITOR] starting check, ${toSelectorString(selectorFlags, ", ")}`);
+
+      if (Object.keys(selectorFlags).length > 0) {
+        console.log(`[MONITOR] starting check, ${toSelectorString(selectorFlags, ", ")}`);
+      } else {
+        console.log(`[MONITOR] starting check`);
+      }
 
       let checkState = {};
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.10.8",
+  "version": "0.10.11",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",