@certik/skynet 0.10.9 → 0.10.12

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.10.12
+
+- Fixed distributed lock name for `producer` and `api` type apps
+
+## 0.10.11
+
+- Added distributed lock for `producer` and `api` type apps
+
+## 0.10.10
+
+- Updated `every` helper function in `app` library to support usage like `every(1).days` and `every(10).day`
+
 ## 0.10.9
 
 - Improved `inline.log` to print one level of object

package/api.js

@@ -1,7 +1,8 @@
 const express = require("express");
 const meow = require("meow");
-const { getSelectorFlags, getSelectorDesc } = require("./selector");
+const { getJobName, getSelectorFlags, getSelectorDesc } = require("./selector");
 const { isProduction } = require("./env");
+const { useLock } = require("./distributed-lock");
 const { inline } = require("./log");
 
 async function logMiddleware(req, res, next) {
@@ -54,7 +55,7 @@ const apiKeyMiddleware = (key) => {
   return requireAPIKey;
 };
 
-function startApiApp({ binaryName, name, selector = {}, routes, serve }) {
+async function startApiApp({ binaryName, name, selector = {}, routes, serve }) {
   const app = express();
   app.use(express.json());
 
@@ -82,6 +83,8 @@ ${getSelectorDesc(selector)}
 
   const { verbose, ...selectorFlags } = cli.flags;
 
+  await useLock({ name: getJobName(name, selectorFlags), ttl: 120, verbose });
+
   // for health check
   app.get("/", (req, res) => {
     res.send("ok");

package/app.js

@@ -635,7 +635,7 @@ function api({ name, routes, serve, env = {}, region = "us-east-1" }) {
     env,
     check,
     onRun: () => {
-      startApiApp({
+      return startApiApp({
         binaryName: `${getBinaryName()} run`,
         name,
         selector,
@@ -691,17 +691,26 @@ const every = (n = 1) => {
   if (n === 1) {
     return {
       second: "*/1 * * * * * *",
+      seconds: "*/1 * * * * * *",
       minute: "0 * * * * * *",
+      minutes: "0 * * * * * *",
       hour: "0 0 * * * * *",
+      hours: "0 0 * * * * *",
       day: "0 0 0 * * * *",
+      days: "0 0 0 * * * *",
       week: "0 0 0 * * 0 *",
+      weeks: "0 0 0 * * 0 *",
     };
   }
 
   return {
+    second: `*/${n} * * * * * *`,
     seconds: `*/${n} * * * * * *`,
+    minute: `0 */${n} * * * * *`,
     minutes: `0 */${n} * * * * *`,
+    hour: `0 0 */${n} * * * *`,
     hours: `0 0 */${n} * * * *`,
+    day: `0 0 0 */${n} * * *`,
     days: `0 0 0 */${n} * * *`,
   };
 };

package/deploy.js

@@ -4,7 +4,7 @@ const execa = require("execa");
 const meow = require("meow");
 const chalk = require("chalk");
 const which = require("which");
-const { getSelectorFlags, getSelectorDesc } = require("./selector");
+const { getJobName, getSelectorFlags, getSelectorDesc } = require("./selector");
 const { getEnvOrThrow } = require("./env");
 const { getBinaryName, detectSkynetDirectory } = require("./cli");
 
@@ -207,30 +207,6 @@ EOH
   }
 }`;
 
-function normalizeSelectorValue(v) {
-  return v.replace(/[^A-Za-z0-9]+/g, "-");
-}
-
-function getJobName(name, selectorFlags, mode = null) {
-  const selectorNamePart = Object.keys(selectorFlags)
-    .sort()
-    .map((name) => selectorFlags[name])
-    .join("-");
-
-  let jobName = name;
-
-  if (mode) {
-    jobName += `-${mode}`;
-  }
-
-  if (selectorNamePart.length > 0) {
-    // handle special case
-    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
-  }
-
-  return jobName;
-}
-
 async function prepareNomad(isProduction) {
   if (isProduction) {
     console.log("Deploy to Production");

package/distributed-lock.js

@@ -0,0 +1,113 @@
+const fetch = require("node-fetch");
+
+async function acquireLock(name, ttl) {
+  try {
+    const res = await fetch("http://localhost:8500/v1/session/create", {
+      method: "PUT",
+      body: JSON.stringify({
+        Name: name,
+        TTL: `${ttl}s`,
+        Behavior: "delete",
+      }),
+    });
+
+    if (res.ok) {
+      const { ID: id } = await res.json();
+
+      return id;
+    } else {
+      console.log(res, await res.text());
+
+      return null;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return null;
+  }
+}
+
+async function renewLock(uid) {
+  try {
+    const res = await fetch(`http://localhost:8500/v1/session/renew/${uid}`, {
+      method: "PUT",
+    });
+
+    if (res.ok) {
+      return uid;
+    } else {
+      console.log(res, await res.text());
+
+      return null;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return null;
+  }
+}
+
+async function hasLock(name, uid = null) {
+  try {
+    const res = await fetch(`http://localhost:8500/v1/session/list`);
+
+    if (res.ok) {
+      const sessions = await res.json();
+
+      const sessionsWithTheSameName = sessions.filter((s) => s.Name === name);
+
+      if (!uid) {
+        return sessionsWithTheSameName.length === 0;
+      }
+
+      if (sessionsWithTheSameName.length !== 1) {
+        return false;
+      }
+
+      // only one should have the same uid
+      return sessionsWithTheSameName[0].ID === uid;
+    } else {
+      console.log(await res.text());
+
+      return true;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return true;
+  }
+}
+
+async function useLock({ name, ttl, verbose }) {
+  const lockWarningMessage = `only one process with the same lock name ${name} should be running, terminate current process`;
+
+  const lockAvailable = await hasLock(name);
+
+  if (!lockAvailable) {
+    console.log(lockWarningMessage);
+    process.exit(0);
+  }
+
+  let uid = await acquireLock(name, ttl);
+
+  setInterval(async () => {
+    if (!(await hasLock(name, uid))) {
+      console.log(lockWarningMessage);
+      process.exit(0);
+    }
+
+    if (uid) {
+      if (verbose) {
+        console.log("renewing", name, uid);
+      }
+
+      uid = await renewLock(uid);
+    } else {
+      uid = await acquireLock(name, ttl);
+    }
+  }, (ttl * 1000) / 2);
+
+  return uid;
+}
+
+module.exports = { useLock };

package/kafka.js

@@ -2,9 +2,10 @@ const meow = require("meow");
 const { getEnvironment, getEnvOrThrow } = require("./env");
 const { wait } = require("./availability");
 const { Kafka, logLevel } = require("kafkajs");
-const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { getJobName, getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
 const { exponentialRetry } = require("./availability");
+const { useLock } = require("./distributed-lock");
 const { getBinaryName } = require("./cli");
 const { inline } = require("./log");
 
@@ -218,6 +219,8 @@ ${getSelectorDesc(selector)}
         process.exit(0);
       }
 
+      await useLock({ name: getJobName(name, selectorFlags), ttl: 120, verbose });
+
       if (!from) {
         const prevId = await getProducerLatestId(name, selectorFlags);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.10.9",
+  "version": "0.10.12",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",

package/selector.js

@@ -46,7 +46,32 @@ function toSelectorString(selectorFlags, delim = ",") {
     .join(delim);
 }
 
+function normalizeSelectorValue(v) {
+  return v.replace(/[^A-Za-z0-9]+/g, "-");
+}
+
+function getJobName(name, selectorFlags, mode = null) {
+  const selectorNamePart = Object.keys(selectorFlags)
+    .sort()
+    .map((name) => selectorFlags[name])
+    .join("-");
+
+  let jobName = name;
+
+  if (mode) {
+    jobName += `-${mode}`;
+  }
+
+  if (selectorNamePart.length > 0) {
+    // handle special case
+    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
+  }
+
+  return jobName;
+}
+
 module.exports = {
+  getJobName,
   getSelectorDesc,
   getSelectorFlags,
   toSelectorString,