@certik/skynet 0.7.20 → 0.8.0

package/monitor.js

@@ -0,0 +1,246 @@
+const meow = require("meow");
+const fetch = require("node-fetch");
+const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { getBinaryName } = require("./cli");
+const { exponentialRetry } = require("./availability");
+const { getIndexerLatestId, getIndexerValidatedId, getIndexerState } = require("./indexer");
+const { postMessage } = require("./slack");
+const { getJobName } = require("./deploy");
+
+const ERROR_LEVEL = {
+  INFO: "Info",
+  CRITICAL: "Critical",
+  WARNING: "Warning",
+};
+
+const LEVEL_EMOJI = {
+  Critical: ":exclamation:",
+  Warning: ":warning:",
+  Info: "",
+};
+
+const LEVEL_PRIORITY = {
+  Critical: 3,
+  Warning: 2,
+  Info: 1,
+};
+
+function sortErrors(errors) {
+  return errors.sort((e1, e2) => (LEVEL_PRIORITY[e2.type] || 0) - (LEVEL_PRIORITY[e1.type] || 0));
+}
+
+function getNomadUrl() {
+  return process.env.SKYNET_NOMAD_URL || "http://localhost:4646";
+}
+
+async function checkMostRecentAllocationStatus(name) {
+  const jobsRes = await fetch(`http://localhost:4646/v1/jobs?prefix=${name}`);
+
+  if (!jobsRes.ok) {
+    console.log(`request local nomad API failed`);
+
+    return [];
+  }
+
+  const jobs = await jobsRes.json();
+
+  if (jobs.length === 0) {
+    return [];
+  }
+
+  const mostRecentJob = jobs[jobs.length - 1];
+
+  const allocationsRes = await fetch(`http://localhost:4646/v1/job/${mostRecentJob.ID}/allocations`);
+
+  if (!allocationsRes.ok) {
+    console.log(`request local nomad API failed`);
+
+    return [];
+  }
+
+  const allocations = await allocationsRes.json();
+
+  if (allocations.length === 0) {
+    return [];
+  }
+
+  const mostRecentAllocation = allocations[allocations.length - 1];
+
+  const tasks = Object.keys(mostRecentAllocation.TaskStates);
+
+  for (let task of tasks) {
+    if (mostRecentAllocation.TaskStates[task].Failed) {
+      // TODO if we could include a link to the failed allocation
+      // which could be very useful
+      return [
+        {
+          type: ERROR_LEVEL.CRITICAL,
+          message: `Job ${name}'s most recent allocation failed, please investigate`,
+        },
+      ];
+    }
+  }
+
+  return [];
+}
+
+function createMonitor({
+  binaryName,
+  name,
+  type = "stateless",
+  mode = false,
+  slackChannel,
+  selector = {},
+  check,
+  maxRetry = 2,
+}) {
+  function monitor() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const cli = meow(
+      `
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${
+  mode ? "          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset\n" : ""
+}${getSelectorDesc(selector)}
+          --verbose       Output debug messages
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          ...(mode && {
+            mode: {
+              type: "string",
+              default: "delta",
+            },
+          }),
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    async function runCheck({ verbose, mode, ...selectorFlags }) {
+      const startTime = Date.now();
+      console.log(`[MONITOR] starting check, ${toSelectorString(selectorFlags, ", ")}`);
+
+      const state = {};
+
+      if (type === "stateful") {
+        state.latestId = await getIndexerLatestId(name, selectorFlags);
+        state.validatedId = await getIndexerValidatedId(name, selectorFlags);
+        state.buildState = await getIndexerState(name, selectorFlags);
+      }
+
+      let result = await exponentialRetry(
+        async () => {
+          try {
+            const errors = await check({ verbose, state, mode, ...selectorFlags });
+
+            if (!Array.isArray(errors)) {
+              throw new Error(`check function must return array of error messages`);
+            }
+
+            return errors;
+          } catch (err) {
+            console.log(`[MONITOR] got error in check`, err);
+
+            return [`${err.message}`];
+          }
+        },
+        {
+          maxRetry,
+          initialDuration: 10000,
+          growFactor: 3,
+          test: (r) => r.length === 0,
+          verbose,
+        }
+      );
+
+      const allocationErrors = await checkMostRecentAllocationStatus(name);
+
+      result = result.concat(allocationErrors);
+
+      if (result.length > 0) {
+        console.log("Found Errors", result);
+
+        if (slackChannel) {
+          await postMessage(slackChannel, {
+            text: `[Monitor] ${name} Job Errors: ${result.join("\n")}`,
+            blocks: [
+              {
+                type: "header",
+                text: {
+                  type: "plain_text",
+                  text: `${name} Monitor Errors`,
+                  emoji: true,
+                },
+              },
+              {
+                type: "divider",
+              },
+              ...sortErrors(result)
+                .map((m) => {
+                  return [
+                    {
+                      type: "context",
+                      elements: [
+                        {
+                          type: "plain_text",
+                          text: `${LEVEL_EMOJI[m.type || ERROR_LEVEL.CRITICAL]} ${m.type || ERROR_LEVEL.CRITICAL}`,
+                        },
+                      ],
+                    },
+                    {
+                      type: "section",
+                      text: {
+                        type: "mrkdwn",
+                        text: m.message || m,
+                      },
+                    },
+                  ];
+                })
+                .flat(),
+              {
+                type: "actions",
+                elements: [
+                  {
+                    type: "button",
+                    text: {
+                      type: "plain_text",
+                      text: "View Details",
+                    },
+                    value: "view_details",
+                    url: `${getNomadUrl()}/ui/jobs/${getJobName(name, selectorFlags, mode)}`,
+                  },
+                ],
+              },
+            ],
+          });
+        }
+
+        throw new Error(`[MONITOR] failed due to critical errors`);
+      }
+
+      console.log(`[MONITOR] check successfully in ${Date.now() - startTime}ms`);
+    }
+
+    runCheck(cli.flags).catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { monitor };
+}
+
+module.exports = { createMonitor, ERROR_LEVEL };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.7.20",
+  "version": "0.8.0",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",

package/selector.js

@@ -9,9 +9,9 @@ function getSpaces(num) {
 
 function getSelectorDesc(selector) {
   return Object.keys(selector)
-    .map(name => {
+    .map((name) => {
       return `          --${name}${getSpaces(14 - name.length)}${
-        selector[name].desc
+        selector[name].desc || selector[name].description || ""
       }`;
     })
     .join("\n");
@@ -21,7 +21,7 @@ function getSelectorFlags(selector) {
   return Object.keys(selector).reduce((acc, name) => {
     const flag = {
       type: selector[name].type || "string",
-      isRequired: true
+      isRequired: true,
     };
 
     if (selector[name].default) {
@@ -40,7 +40,7 @@ function getSelectorFlags(selector) {
 function toSelectorString(selectorFlags, delim = ",") {
   return Object.keys(selectorFlags)
     .sort() // deterministic
-    .map(flag => {
+    .map((flag) => {
       return `${flag}=${selectorFlags[flag]}`;
     })
     .join(delim);
@@ -49,5 +49,5 @@ function toSelectorString(selectorFlags, delim = ",") {
 module.exports = {
   getSelectorDesc,
   getSelectorFlags,
-  toSelectorString
+  toSelectorString,
 };

package/slack.js

@@ -1,46 +1,62 @@
-const { WebClient, LogLevel } = require("@slack/web-api");
+const { WebClient } = require("@slack/web-api");
 
-const token = process.env.SLACK_TOKEN;
+function getToken() {
+  return process.env.SKYNET_SLACK_TOKEN;
+}
 
-// WebClient insantiates a client that can call API methods
-const client = new WebClient(token, {
-  logLevel: LogLevel.DEBUG,
-});
+function getClient() {
+  const token = getToken();
 
-async function findConversation(name) {
-  try {
-    // Call the conversations.list method using the built-in WebClient
-    const result = await client.conversations.list({
-      token: token,
-      limit: 1000,
-    });
+  if (!token) {
+    throw new Error("Cannot communicate with slack due to missing slack token: SKYNET_SLACK_TOKEN");
+  }
+
+  const client = new WebClient(token);
+
+  return client;
+}
+
+async function findConversation(client, name) {
+  const { conversations } = client;
+
+  // Call the conversations.list method using the built-in WebClient
+  let result = await conversations.list({
+    limit: 1000,
+  });
 
-    for (const channel of result.channels) {
-      if (channel.name === name) {
-        const conversationId = channel.id;
+  for (const channel of result.channels) {
+    if (channel.name === name) {
+      const conversationId = channel.id;
 
-        console.log("Found conversation ID: " + conversationId);
-        return conversationId;
-      }
+      return conversationId;
     }
-  } catch (error) {
-    console.error(error);
   }
+
+  return null;
 }
 
-async function publishMessage(id, text) {
+async function postMessage(channel, message) {
   try {
-    const result = await client.chat.postMessage({
-      token: token,
-      channel: id,
-      text: text,
-      // You could also use a blocks[] array to send richer content
-    });
+    const client = getClient();
+
+    const conversationId = await findConversation(client, channel);
 
-    console.log(result);
+    let post = {};
+
+    if (typeof message === "string") {
+      post.text = message;
+    } else {
+      post = message;
+    }
+
+    await client.chat.postMessage({
+      channel: conversationId,
+      ...post,
+    });
   } catch (error) {
-    console.error(error);
+    // no blocking
+    console.error("failed to post slack message", error);
   }
 }
 
-module.exports = { publishMessage, findConversation };
+module.exports = { postMessage };