@certik/skynet 0.10.10 → 0.10.13

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.10.13
+
+- Initialize Ably after user request publishing messages
+
+- Increase wait time for Ably message sending bottleneck
+
+- Update packages
+
+## 0.10.12
+
+- Fixed distributed lock name for `producer` and `api` type apps
+
+## 0.10.11
+
+- Added distributed lock for `producer` and `api` type apps
+
 ## 0.10.10
 
 - Updated `every` helper function in `app` library to support usage like `every(1).days` and `every(10).day`

package/ably.js

@@ -2,12 +2,23 @@ const Ably = require("ably");
 const Bottleneck = require("bottleneck/es5");
 const { ensureAndGet } = require("./env");
 
-const ably = new Ably.Realtime(ensureAndGet("ABLY_ROOT_API_KEY"));
-ably.connection.on("connected", () => {
-  console.log("ably connection successful");
-});
+let ABLY;
+
+async function getAbly() {
+  if (!ABLY) {
+    ABLY = await new Promise((resolve) => {
+      const ably = new Ably.Realtime(ensureAndGet("ABLY_ROOT_API_KEY"));
+      ably.connection.on("connected", () => {
+        console.log("ably connection successful");
+        resolve(ably);
+      });
+    });
+  }
+  return ABLY;
+}
 
 async function publishMessage(channel, messageName, messageData) {
+  const ably = await getAbly();
   const channelObj = ably.channels.get(channel);
 
   const messageDataJson = JSON.stringify(messageData);
@@ -20,7 +31,7 @@ async function publishMessage(channel, messageName, messageData) {
 }
 
 const rateLimitedPublishMessage = new Bottleneck({
-  minTime: 1000 / 50,
+  minTime: 1000 / 60,
 }).wrap(publishMessage);
 
 module.exports = {

package/api.js

@@ -1,7 +1,8 @@
 const express = require("express");
 const meow = require("meow");
-const { getSelectorFlags, getSelectorDesc } = require("./selector");
+const { getJobName, getSelectorFlags, getSelectorDesc } = require("./selector");
 const { isProduction } = require("./env");
+const { useLock } = require("./distributed-lock");
 const { inline } = require("./log");
 
 async function logMiddleware(req, res, next) {
@@ -54,7 +55,7 @@ const apiKeyMiddleware = (key) => {
   return requireAPIKey;
 };
 
-function startApiApp({ binaryName, name, selector = {}, routes, serve }) {
+async function startApiApp({ binaryName, name, selector = {}, routes, serve }) {
   const app = express();
   app.use(express.json());
 
@@ -82,6 +83,8 @@ ${getSelectorDesc(selector)}
 
   const { verbose, ...selectorFlags } = cli.flags;
 
+  await useLock({ name: getJobName(name, selectorFlags), ttl: 120, verbose });
+
   // for health check
   app.get("/", (req, res) => {
     res.send("ok");

package/app.js

@@ -635,7 +635,7 @@ function api({ name, routes, serve, env = {}, region = "us-east-1" }) {
     env,
     check,
     onRun: () => {
-      startApiApp({
+      return startApiApp({
         binaryName: `${getBinaryName()} run`,
         name,
         selector,

package/deploy.js

@@ -4,7 +4,7 @@ const execa = require("execa");
 const meow = require("meow");
 const chalk = require("chalk");
 const which = require("which");
-const { getSelectorFlags, getSelectorDesc } = require("./selector");
+const { getJobName, getSelectorFlags, getSelectorDesc } = require("./selector");
 const { getEnvOrThrow } = require("./env");
 const { getBinaryName, detectSkynetDirectory } = require("./cli");
 
@@ -207,30 +207,6 @@ EOH
   }
 }`;
 
-function normalizeSelectorValue(v) {
-  return v.replace(/[^A-Za-z0-9]+/g, "-");
-}
-
-function getJobName(name, selectorFlags, mode = null) {
-  const selectorNamePart = Object.keys(selectorFlags)
-    .sort()
-    .map((name) => selectorFlags[name])
-    .join("-");
-
-  let jobName = name;
-
-  if (mode) {
-    jobName += `-${mode}`;
-  }
-
-  if (selectorNamePart.length > 0) {
-    // handle special case
-    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
-  }
-
-  return jobName;
-}
-
 async function prepareNomad(isProduction) {
   if (isProduction) {
     console.log("Deploy to Production");

package/distributed-lock.js

@@ -0,0 +1,113 @@
+const fetch = require("node-fetch");
+
+async function acquireLock(name, ttl) {
+  try {
+    const res = await fetch("http://localhost:8500/v1/session/create", {
+      method: "PUT",
+      body: JSON.stringify({
+        Name: name,
+        TTL: `${ttl}s`,
+        Behavior: "delete",
+      }),
+    });
+
+    if (res.ok) {
+      const { ID: id } = await res.json();
+
+      return id;
+    } else {
+      console.log(res, await res.text());
+
+      return null;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return null;
+  }
+}
+
+async function renewLock(uid) {
+  try {
+    const res = await fetch(`http://localhost:8500/v1/session/renew/${uid}`, {
+      method: "PUT",
+    });
+
+    if (res.ok) {
+      return uid;
+    } else {
+      console.log(res, await res.text());
+
+      return null;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return null;
+  }
+}
+
+async function hasLock(name, uid = null) {
+  try {
+    const res = await fetch(`http://localhost:8500/v1/session/list`);
+
+    if (res.ok) {
+      const sessions = await res.json();
+
+      const sessionsWithTheSameName = sessions.filter((s) => s.Name === name);
+
+      if (!uid) {
+        return sessionsWithTheSameName.length === 0;
+      }
+
+      if (sessionsWithTheSameName.length !== 1) {
+        return false;
+      }
+
+      // only one should have the same uid
+      return sessionsWithTheSameName[0].ID === uid;
+    } else {
+      console.log(await res.text());
+
+      return true;
+    }
+  } catch (fetchErr) {
+    console.log("error fetching", fetchErr);
+
+    return true;
+  }
+}
+
+async function useLock({ name, ttl, verbose }) {
+  const lockWarningMessage = `only one process with the same lock name ${name} should be running, terminate current process`;
+
+  const lockAvailable = await hasLock(name);
+
+  if (!lockAvailable) {
+    console.log(lockWarningMessage);
+    process.exit(0);
+  }
+
+  let uid = await acquireLock(name, ttl);
+
+  setInterval(async () => {
+    if (!(await hasLock(name, uid))) {
+      console.log(lockWarningMessage);
+      process.exit(0);
+    }
+
+    if (uid) {
+      if (verbose) {
+        console.log("renewing", name, uid);
+      }
+
+      uid = await renewLock(uid);
+    } else {
+      uid = await acquireLock(name, ttl);
+    }
+  }, (ttl * 1000) / 2);
+
+  return uid;
+}
+
+module.exports = { useLock };

package/kafka.js

@@ -2,9 +2,10 @@ const meow = require("meow");
 const { getEnvironment, getEnvOrThrow } = require("./env");
 const { wait } = require("./availability");
 const { Kafka, logLevel } = require("kafkajs");
-const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { getJobName, getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
 const { exponentialRetry } = require("./availability");
+const { useLock } = require("./distributed-lock");
 const { getBinaryName } = require("./cli");
 const { inline } = require("./log");
 
@@ -218,6 +219,8 @@ ${getSelectorDesc(selector)}
         process.exit(0);
       }
 
+      await useLock({ name: getJobName(name, selectorFlags), ttl: 120, verbose });
+
       if (!from) {
         const prevId = await getProducerLatestId(name, selectorFlags);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.10.10",
+  "version": "0.10.13",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",

package/selector.js

@@ -46,7 +46,32 @@ function toSelectorString(selectorFlags, delim = ",") {
     .join(delim);
 }
 
+function normalizeSelectorValue(v) {
+  return v.replace(/[^A-Za-z0-9]+/g, "-");
+}
+
+function getJobName(name, selectorFlags, mode = null) {
+  const selectorNamePart = Object.keys(selectorFlags)
+    .sort()
+    .map((name) => selectorFlags[name])
+    .join("-");
+
+  let jobName = name;
+
+  if (mode) {
+    jobName += `-${mode}`;
+  }
+
+  if (selectorNamePart.length > 0) {
+    // handle special case
+    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
+  }
+
+  return jobName;
+}
+
 module.exports = {
+  getJobName,
   getSelectorDesc,
   getSelectorFlags,
   toSelectorString,