@certik/skynet 0.8.13 → 0.8.16

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.8.16
+
+- Changed monitor exit behavior
+- Improved OpsGenie message for service checks
+
+## 0.8.15
+
+- Fixed OpsGenie message bug 
+
+## 0.8.14
+
+- Switched monitor to send to opsgenie only
+
 ## 0.8.13
 
 - Added `inline.error` function to `log` library

package/app.js

@@ -202,8 +202,7 @@ function indexer({ name, selector, build, check, env = {}, region = "us-east-1"
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100,
-          hasSlack: check.slackChannel,
+          mem: check.mem || 100
         },
       });
 
@@ -216,8 +215,7 @@ function indexer({ name, selector, build, check, env = {}, region = "us-east-1"
         type: "stateless",
         selector,
         check: check.func,
-        maxRetry: check.maxRetry,
-        slackChannel: check.slackChannel,
+        maxRetry: check.maxRetry
       });
 
       monitor();
@@ -317,8 +315,7 @@ function modeIndexer({ name, selector, state, build, validate, check, env = {},
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100,
-          hasSlack: check.slackChannel,
+          mem: check.mem || 100
         },
       });
 
@@ -332,8 +329,7 @@ function modeIndexer({ name, selector, state, build, validate, check, env = {},
         selector,
         mode: true,
         check: check.func,
-        maxRetry: check.maxRetry,
-        slackChannel: check.slackChannel,
+        maxRetry: check.maxRetry
       });
 
       monitor();
@@ -421,8 +417,7 @@ function producer({ name, selector, produce, check, state, env = {}, region = "u
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100,
-          hasSlack: check.slackChannel,
+          mem: check.mem || 100
         },
       });
 
@@ -435,8 +430,7 @@ function producer({ name, selector, produce, check, state, env = {}, region = "u
         type: "stateful",
         selector,
         check: check.func,
-        maxRetry: check.maxRetry,
-        slackChannel: check.slackChannel,
+        maxRetry: check.maxRetry
       });
 
       monitor();
@@ -515,8 +509,7 @@ function consumer({ name, selector, consume, check, env = {}, region = "us-east-
           bin: `${bin} check`,
           schedule: check.schedule,
           cpu: check.cpu || 100,
-          mem: check.mem || 100,
-          hasSlack: check.slackChannel,
+          mem: check.mem || 100
         },
       });
 
@@ -529,8 +522,7 @@ function consumer({ name, selector, consume, check, env = {}, region = "us-east-
         type: "stateless",
         selector,
         check: check.func,
-        maxRetry: check.maxRetry,
-        slackChannel: check.slackChannel,
+        maxRetry: check.maxRetry
       });
 
       monitor();

package/monitor.js

@@ -4,9 +4,8 @@ const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selec
 const { getBinaryName } = require("./cli");
 const { exponentialRetry } = require("./availability");
 const { getIndexerLatestId, getIndexerValidatedId, getIndexerState } = require("./indexer");
-const { postMessage } = require("./slack");
 const { postGenieMessage } = require("./opsgenie");
-const { getJobName, getNomadAddr } = require("./deploy");
+const { getJobName } = require("./deploy");
 
 const ERROR_LEVEL = {
   INFO: "Info",
@@ -31,70 +30,44 @@ function sortErrors(errors) {
 }
 
 async function getMostRecentJobLaunch(name) {
-  const jobsRes = await fetch(`http://localhost:4646/v1/jobs?prefix=${name}`);
+  try {
+    const jobsRes = await fetch(`http://localhost:4646/v1/jobs?prefix=${name}`);
 
-  if (!jobsRes.ok) {
-    console.log(`[MONITOR] request local nomad API failed`);
+    if (!jobsRes.ok) {
+      console.log(`[MONITOR] request local nomad API failed`);
 
-    return null;
-  }
-
-  const jobs = await jobsRes.json();
-
-  if (jobs.length === 0) {
-    console.log(`[MONITOR] did not see any jobs prefixed with ${name}`);
-
-    return null;
-  }
+      return null;
+    }
 
-  const recentFinishedJob = jobs.reverse().find((job) => {
-    // filter out monitor job
-    return job.ID.indexOf("-monitor") === -1 && job.Status === "dead";
-  });
+    const jobs = await jobsRes.json();
 
-  if (!recentFinishedJob) {
-    console.log(`[MONITOR] did not see any dead jobs`);
+    if (jobs.length === 0) {
+      console.log(`[MONITOR] did not see any jobs prefixed with ${name}`);
 
-    return null;
-  }
+      return null;
+    }
 
-  console.log("[MONITOR]", "most recent job info", recentFinishedJob.ID, recentFinishedJob.JobSummary.Summary);
+    const recentFinishedJob = jobs.reverse().find((job) => {
+      // filter out monitor job
+      return job.ID.indexOf("-monitor") === -1 && job.Status === "dead";
+    });
 
-  return recentFinishedJob;
-}
+    if (!recentFinishedJob) {
+      console.log(`[MONITOR] did not see any dead jobs`);
 
-function checkMostRecentJobFailures(recentFinishedJob) {
-  if (!recentFinishedJob) {
-    return [];
-  }
+      return null;
+    }
 
-  const summary = recentFinishedJob.JobSummary.Summary;
-  const groups = Object.keys(summary);
+    console.log("[MONITOR]", "most recent job info", recentFinishedJob.ID, recentFinishedJob.JobSummary.Summary);
 
-  for (let group of groups) {
-    if (summary[group].Failed > 0) {
-      return [
-        {
-          type: ERROR_LEVEL.CRITICAL,
-          message: `The most recent launch returned non-zero exit value, please investigate`,
-        },
-      ];
-    }
+    return recentFinishedJob;
+  } catch (getJobError) {
+    console.log("[MONITOR]", "cannot get most recent job", getJobError.message);
+    return null;
   }
-
-  return [];
 }
 
-function createMonitor({
-  binaryName,
-  name,
-  type = "stateless",
-  mode = false,
-  slackChannel,
-  selector = {},
-  check,
-  maxRetry = 2,
-}) {
+function createMonitor({ binaryName, name, type = "stateless", mode = false, selector = {}, check, maxRetry = 2 }) {
   function monitor() {
     if (!binaryName) {
       binaryName = getBinaryName();
@@ -171,80 +144,26 @@ ${
 
       const jobName = getJobName(name, selectorFlags, mode);
       const mostRecentJob = await getMostRecentJobLaunch(jobName);
-      const recentJobFailures = checkMostRecentJobFailures(mostRecentJob);
-
-      result = result.concat(recentJobFailures);
 
       if (result.length > 0) {
         console.log("Found Errors", result);
 
-        if (slackChannel && production) {
-          // TODO: deduplicate
-          await postMessage(
-            slackChannel,
-            {
-              text: `${jobName} Monitor Errors: ${result.map((m) => m.message || m).join("\n")}`,
-              blocks: [
-                {
-                  type: "header",
-                  text: {
-                    type: "plain_text",
-                    text: `${jobName} Monitor Errors`,
-                    emoji: true,
-                  },
-                },
-                {
-                  type: "divider",
-                },
-                ...sortErrors(result)
-                  .map((m) => {
-                    return [
-                      {
-                        type: "context",
-                        elements: [
-                          {
-                            type: "plain_text",
-                            text: `${LEVEL_EMOJI[m.type || ERROR_LEVEL.CRITICAL]} ${m.type || ERROR_LEVEL.CRITICAL}`,
-                          },
-                        ],
-                      },
-                      {
-                        type: "section",
-                        text: {
-                          type: "mrkdwn",
-                          text: m.message || m,
-                        },
-                      },
-                    ];
-                  })
-                  .flat(),
-                {
-                  type: "actions",
-                  elements: [
-                    {
-                      type: "button",
-                      text: {
-                        type: "plain_text",
-                        text: "View Details",
-                      },
-                      value: "view_details",
-                      url: `${await getNomadAddr(production)}/ui/jobs/${jobName}`,
-                    },
-                  ],
-                },
-              ],
-            },
-            verbose
-          );
+        if (production) {
+          const nomadAddr = process.env.SKYNET_NOMAD_PRODUCTION_ADDR;
 
-          // Also alert on opsgenie (func prevents duplicate alerts)
+          // alert on opsgenie
           await postGenieMessage(
-            `${jobName} Monitor Errors: ${result.map((m) => m.message || m).join("\n")}`,
-            m.message
+            `Failed Service Check: ${jobName}`,
+            `<p><b>Service:</b><a href="${nomadAddr}/ui/jobs/${jobName}" target="_blank">${jobName}</a></p><p><b>Issues</b></p><ul>${sortErrors(
+              result
+            )
+              .map((m) => `<li><b>${m.type}:</b> ${m.message}</li>`)
+              .join("")}</ul>`,
+            verbose
           );
+        } else {
+          console.log("skip sending messages to opsgenie in dev env");
         }
-
-        throw new Error(`failed due to critical errors`);
       }
 
       console.log(`[MONITOR] check successfully in ${Date.now() - startTime}ms`);
@@ -270,4 +189,4 @@ ${
   return { monitor };
 }
 
-module.exports = { createMonitor, ERROR_LEVEL };
+module.exports = { createMonitor, ERROR_LEVEL, LEVEL_EMOJI };

package/opsgenie.js

@@ -26,7 +26,7 @@ async function postGenieMessage(msg, desc, verbose) {
     };
 
     // Prevents duplicate alerts (See Opsgenie doc about alias)
-    bodyHash = hash(body);
+    const bodyHash = hash(body);
     body.alias = bodyHash;
 
     if (verbose) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.8.13",
+  "version": "0.8.16",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",