@certik/skynet 0.7.17 → 0.8.0

package/indexer.js

@@ -1,10 +1,10 @@
-const path = require("path");
 const meow = require("meow");
 const { createRecord, getRecordByKey } = require("./dynamodb");
 const { getEnvironment } = require("./env");
 const { exponentialRetry } = require("./availability");
 const { range, fillRange } = require("./util");
 const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { getBinaryName } = require("./cli");
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -49,12 +49,21 @@ async function getIndexerValidatedId(name, selectorFlags) {
 // managing state would be helpful to reduce the build time
 // and avoid unnecessary computation & storage
 function createModeIndexerApp({
+  binaryName,
   name,
   selector = {},
   build,
+  // number of items run in a batch, determines the { from, to } to the build function
   buildBatchSize = 1,
+  // number of build functions calling at a time
+  buildConcurrency = 1,
+  // commit updates every rolling window = buildBatchSize * buildConcurrency
   validate,
+  // number of items run in a batch, determines the { from, to } to the validate function
   validateBatchSize = 1,
+  // number of validate functions calling at a time
+  validateConcurrency = 1,
+  // commit updates every rolling window = validateBatchSize * validateConcurrency
   maxRetry = 2,
   state,
 }) {
@@ -91,10 +100,9 @@ function createModeIndexerApp({
     }
 
     console.log(
-      `mode=${mode}, from=${from}, to=${to > 0 ? to : "undefined"}, env=${getEnvironment()}, ${toSelectorString(
-        selectorFlags,
-        ", "
-      )}`
+      `[MODE INDEXER] mode=${mode}, from=${from}, to=${
+        to > 0 ? to : "undefined"
+      }, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`
     );
 
     if (mode === "reset") {
@@ -127,7 +135,7 @@ function createModeIndexerApp({
       );
     } else if (mode === "one") {
       if (to > 0) {
-        console.log("one mode ignores --to option. you may want to use range mode instead");
+        console.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
       }
       await runRange(selectorFlags, from, from, verbose);
     } else if (mode === "range") {
@@ -140,48 +148,60 @@ function createModeIndexerApp({
   async function runRange(selectorFlags, from, to, verbose) {
     const startTime = Date.now();
 
-    console.log(`building range, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
+    console.log(
+      `[MODE INDEXER] building range, from=${from}, to=${to}, ${toSelectorString(
+        selectorFlags,
+        ", "
+      )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+    );
 
     const failedIds = await execBuild(selectorFlags, from, to, verbose, false);
 
     if (failedIds.length > 0) {
-      console.log(`built with some failed ${finalState.type}`, failedIds);
+      console.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
       process.exit(1);
     } else {
-      console.log(`built successfully in ${Date.now() - startTime}ms`);
+      console.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
       process.exit(0);
     }
   }
 
   async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
     if (!validate) {
-      console.log(`the indexer doesn't support validate mode, validate function not implemented`);
+      console.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
       process.exit(1);
     }
 
     const startTime = Date.now();
 
-    console.log(`validating, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
+    console.log(
+      `[MODE INDEXER] validating, from=${from}, to=${to}, ${toSelectorString(
+        selectorFlags,
+        ", "
+      )}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`
+    );
 
-    const batches = range(from, to, validateBatchSize);
+    const windows = range(from, to, validateBatchSize * validateConcurrency);
 
-    console.log(`from=${from}, to=${to}, num batches=${batches.length}`);
+    console.log(
+      `[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`
+    );
 
-    for (let [batchStart, batchEnd] of batches) {
-      console.log(`validating batch ${batchStart}~${batchEnd}`);
+    for (let [windowStart, windowEnd] of windows) {
+      console.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
 
-      const batchIds = fillRange(batchStart, batchEnd);
+      const batches = range(windowStart, windowEnd, validateBatchSize);
 
       // add a retry for errors
       await Promise.all(
-        batchIds.map(async (id) => {
+        batches.map(async ([batchStart, batchEnd]) => {
           const result = await exponentialRetry(
             async () => {
               try {
                 await validate({
                   ...selectorFlags,
-                  from: id,
-                  to: id,
+                  from: batchStart,
+                  to: batchEnd,
                   verbose,
                 });
 
@@ -204,50 +224,56 @@ function createModeIndexerApp({
           }
         })
       );
-    }
 
-    console.log(`validated ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`);
+      if (shouldSaveState) {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+          value: to,
+        });
 
-    if (shouldSaveState) {
-      await createRecord(STATE_TABLE_NAME, {
-        name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-        value: to,
-      });
+        if (verbose) {
+          console.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+        }
+      }
     }
+
+    console.log(
+      `[MODE INDEXER] validated ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
+    );
   }
 
   async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
     let failedIds = [];
 
-    const batches = range(from, to, buildBatchSize);
+    const windows = range(from, to, buildBatchSize * buildConcurrency);
 
-    for (let [batchStart, batchEnd] of batches) {
-      console.log(`building batch ${batchStart}~${batchEnd}`);
+    for (let [windowStart, windowEnd] of windows) {
+      console.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
 
-      const batchIds = fillRange(batchStart, batchEnd);
+      const batches = range(windowStart, windowEnd, buildBatchSize);
 
       // add a retry for errors
       const batchResults = await Promise.all(
-        batchIds.map(async (id) => {
+        batches.map(async ([batchStart, batchEnd]) => {
           await exponentialRetry(
             async () => {
               try {
                 const ids = await build({
                   ...selectorFlags,
-                  from: id,
-                  to: id,
+                  from: batchStart,
+                  to: batchEnd,
                   verbose,
                 });
 
                 if (ids && ids.length > 0) {
-                  return id;
+                  return ids;
                 } else {
                   return false;
                 }
               } catch (err) {
-                console.error(`got error in build`, err);
+                console.error(`[MODE INDEXER] got error in build`, err);
 
-                return id;
+                return fillRange(batchStart, batchEnd);
               }
             },
             {
@@ -262,11 +288,19 @@ function createModeIndexerApp({
       if (shouldSaveState) {
         await createRecord(STATE_TABLE_NAME, {
           name: `${name}Since(${toSelectorString(selectorFlags)})`,
-          value: batchEnd,
+          value: windowEnd,
         });
+
+        if (verbose) {
+          console.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+        }
       }
 
-      failedIds = failedIds.concat(batchResults.filter((r) => r));
+      batchResults.forEach((ids) => {
+        if (ids) {
+          failedIds = failedIds.concat(ids);
+        }
+      });
     }
 
     failedIds.sort();
@@ -277,7 +311,12 @@ function createModeIndexerApp({
   async function runRebuild(selectorFlags, from, to, verbose) {
     const startTime = Date.now();
 
-    console.log(`rebuilding, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
+    console.log(
+      `[MODE INDEXER] rebuilding, from=${from}, to=${to}, ${toSelectorString(
+        selectorFlags,
+        ", "
+      )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+    );
 
     // add a flag to stop delta from running
     await createRecord(STATE_TABLE_NAME, {
@@ -300,10 +339,15 @@ function createModeIndexerApp({
     });
 
     if (failedIds.length > 0) {
-      console.log(`built ${to - from + 1} ${finalState.type} with some failed ${finalState.type}`, failedIds);
+      console.log(
+        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} with some failed ${finalState.type}`,
+        failedIds
+      );
       process.exit(1);
     } else {
-      console.log(`built ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`);
+      console.log(
+        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
+      );
       process.exit(0);
     }
   }
@@ -322,19 +366,27 @@ function createModeIndexerApp({
 
       if (to <= from) {
         console.log(
-          `skip delta because there're no items need to be processed, ${toSelectorString(selectorFlags, ", ")}`
+          `[MODE INDEXER] skip delta because there're no items need to be processed, ${toSelectorString(
+            selectorFlags,
+            ", "
+          )}`
         );
 
         return;
       }
 
-      console.log(`starting delta, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
+      console.log(
+        `[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(
+          selectorFlags,
+          ", "
+        )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+      );
 
       try {
         const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
 
         if (failedIds.length > 0) {
-          console.log("built with some failed txs", failedIds);
+          console.log("[MODE INDEXER] built with some failed txs", failedIds);
 
           await createRecord(STATE_TABLE_NAME, {
             name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
@@ -358,25 +410,25 @@ function createModeIndexerApp({
             value: to,
           });
 
-          console.log(`built successfully in ${Date.now() - startTime}ms`);
+          console.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
           process.exit(0);
         }
       } catch (err) {
-        console.error("delta build failed", from, to, err);
+        console.error("[MODE INDEXER] delta build failed", from, to, err);
 
         process.exit(1);
       }
     } else {
-      console.log("skip because rebuild hasn't done yet");
+      console.log("[MODE INDEXER] skip because rebuild hasn't done yet");
     }
   }
 
   async function runReset(selectorFlags) {
     const startTime = Date.now();
 
-    console.log(`starting reset, ${toSelectorString(selectorFlags, ", ")}`);
+    console.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
 
-    console.log("reset state", STATE_TABLE_NAME);
+    console.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
     await createRecord(STATE_TABLE_NAME, {
       name: `${name}Since(${toSelectorString(selectorFlags)})`,
       value: 0,
@@ -392,12 +444,13 @@ function createModeIndexerApp({
       value: "init",
     });
 
-    console.log(`reset successfully in ${Date.now() - startTime}ms`);
+    console.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
   }
 
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `
@@ -455,10 +508,11 @@ ${getSelectorDesc(selector)}
 // for those indexers that does not rely on a curosr
 // e.g. should always rebuild everything from scratch
 // or that the state can be easily inferred from existing data
-function createIndexerApp({ name, selector = {}, build, maxRetry = 2 }) {
+function createIndexerApp({ binaryName, name, selector = {}, build, maxRetry = 2 }) {
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `
@@ -483,6 +537,9 @@ ${getSelectorDesc(selector)}
     );
 
     async function runBuild({ verbose, ...selectorFlags }) {
+      const startTime = Date.now();
+      console.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
+
       const result = await exponentialRetry(
         async () => {
           try {
@@ -490,7 +547,7 @@ ${getSelectorDesc(selector)}
 
             return true;
           } catch (err) {
-            console.log(`got error in build`, err);
+            console.log(`[INDEXER] got error in build`, err);
 
             return false;
           }
@@ -503,13 +560,15 @@ ${getSelectorDesc(selector)}
       );
 
       if (!result) {
-        throw new Error(`Build failed due to critical errors`);
+        throw new Error(`[INDEXER] Build failed due to critical errors`);
       }
 
       await createRecord(STATE_TABLE_NAME, {
         name: `${name}UpdatedAt(${toSelectorString(selectorFlags)})`,
         value: new Date().toISOString(),
       });
+
+      console.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
     }
 
     runBuild(cli.flags).catch((err) => {

package/kafka.js

@@ -1,4 +1,3 @@
-const path = require("path");
 const meow = require("meow");
 const { getEnvironment, getEnvOrThrow } = require("./env");
 const { wait } = require("./availability");
@@ -6,6 +5,7 @@ const { Kafka, logLevel } = require("kafkajs");
 const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
 const { exponentialRetry } = require("./availability");
+const { getBinaryName } = require("./cli");
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -137,10 +137,11 @@ async function consumeMessages(consumerId, topic, callback) {
   return stopConsumeMessages;
 }
 
-function createProducerApp({ name, selector = {}, producer, state }) {
+function createProducerApp({ binaryName, name, selector = {}, producer, state }) {
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const finalState = {
       type: "block",
@@ -337,10 +338,11 @@ ${getSelectorDesc(selector)}
   return { run };
 }
 
-function createConsumerApp({ name, selector = {}, consumer }) {
+function createConsumerApp({ binaryName, name, selector = {}, consumer }) {
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `

package/monitor.js

@@ -0,0 +1,246 @@
+const meow = require("meow");
+const fetch = require("node-fetch");
+const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { getBinaryName } = require("./cli");
+const { exponentialRetry } = require("./availability");
+const { getIndexerLatestId, getIndexerValidatedId, getIndexerState } = require("./indexer");
+const { postMessage } = require("./slack");
+const { getJobName } = require("./deploy");
+
+const ERROR_LEVEL = {
+  INFO: "Info",
+  CRITICAL: "Critical",
+  WARNING: "Warning",
+};
+
+const LEVEL_EMOJI = {
+  Critical: ":exclamation:",
+  Warning: ":warning:",
+  Info: "",
+};
+
+const LEVEL_PRIORITY = {
+  Critical: 3,
+  Warning: 2,
+  Info: 1,
+};
+
+function sortErrors(errors) {
+  return errors.sort((e1, e2) => (LEVEL_PRIORITY[e2.type] || 0) - (LEVEL_PRIORITY[e1.type] || 0));
+}
+
+function getNomadUrl() {
+  return process.env.SKYNET_NOMAD_URL || "http://localhost:4646";
+}
+
+async function checkMostRecentAllocationStatus(name) {
+  const jobsRes = await fetch(`http://localhost:4646/v1/jobs?prefix=${name}`);
+
+  if (!jobsRes.ok) {
+    console.log(`request local nomad API failed`);
+
+    return [];
+  }
+
+  const jobs = await jobsRes.json();
+
+  if (jobs.length === 0) {
+    return [];
+  }
+
+  const mostRecentJob = jobs[jobs.length - 1];
+
+  const allocationsRes = await fetch(`http://localhost:4646/v1/job/${mostRecentJob.ID}/allocations`);
+
+  if (!allocationsRes.ok) {
+    console.log(`request local nomad API failed`);
+
+    return [];
+  }
+
+  const allocations = await allocationsRes.json();
+
+  if (allocations.length === 0) {
+    return [];
+  }
+
+  const mostRecentAllocation = allocations[allocations.length - 1];
+
+  const tasks = Object.keys(mostRecentAllocation.TaskStates);
+
+  for (let task of tasks) {
+    if (mostRecentAllocation.TaskStates[task].Failed) {
+      // TODO if we could include a link to the failed allocation
+      // which could be very useful
+      return [
+        {
+          type: ERROR_LEVEL.CRITICAL,
+          message: `Job ${name}'s most recent allocation failed, please investigate`,
+        },
+      ];
+    }
+  }
+
+  return [];
+}
+
+function createMonitor({
+  binaryName,
+  name,
+  type = "stateless",
+  mode = false,
+  slackChannel,
+  selector = {},
+  check,
+  maxRetry = 2,
+}) {
+  function monitor() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const cli = meow(
+      `
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${
+  mode ? "          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset\n" : ""
+}${getSelectorDesc(selector)}
+          --verbose       Output debug messages
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          ...(mode && {
+            mode: {
+              type: "string",
+              default: "delta",
+            },
+          }),
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    async function runCheck({ verbose, mode, ...selectorFlags }) {
+      const startTime = Date.now();
+      console.log(`[MONITOR] starting check, ${toSelectorString(selectorFlags, ", ")}`);
+
+      const state = {};
+
+      if (type === "stateful") {
+        state.latestId = await getIndexerLatestId(name, selectorFlags);
+        state.validatedId = await getIndexerValidatedId(name, selectorFlags);
+        state.buildState = await getIndexerState(name, selectorFlags);
+      }
+
+      let result = await exponentialRetry(
+        async () => {
+          try {
+            const errors = await check({ verbose, state, mode, ...selectorFlags });
+
+            if (!Array.isArray(errors)) {
+              throw new Error(`check function must return array of error messages`);
+            }
+
+            return errors;
+          } catch (err) {
+            console.log(`[MONITOR] got error in check`, err);
+
+            return [`${err.message}`];
+          }
+        },
+        {
+          maxRetry,
+          initialDuration: 10000,
+          growFactor: 3,
+          test: (r) => r.length === 0,
+          verbose,
+        }
+      );
+
+      const allocationErrors = await checkMostRecentAllocationStatus(name);
+
+      result = result.concat(allocationErrors);
+
+      if (result.length > 0) {
+        console.log("Found Errors", result);
+
+        if (slackChannel) {
+          await postMessage(slackChannel, {
+            text: `[Monitor] ${name} Job Errors: ${result.join("\n")}`,
+            blocks: [
+              {
+                type: "header",
+                text: {
+                  type: "plain_text",
+                  text: `${name} Monitor Errors`,
+                  emoji: true,
+                },
+              },
+              {
+                type: "divider",
+              },
+              ...sortErrors(result)
+                .map((m) => {
+                  return [
+                    {
+                      type: "context",
+                      elements: [
+                        {
+                          type: "plain_text",
+                          text: `${LEVEL_EMOJI[m.type || ERROR_LEVEL.CRITICAL]} ${m.type || ERROR_LEVEL.CRITICAL}`,
+                        },
+                      ],
+                    },
+                    {
+                      type: "section",
+                      text: {
+                        type: "mrkdwn",
+                        text: m.message || m,
+                      },
+                    },
+                  ];
+                })
+                .flat(),
+              {
+                type: "actions",
+                elements: [
+                  {
+                    type: "button",
+                    text: {
+                      type: "plain_text",
+                      text: "View Details",
+                    },
+                    value: "view_details",
+                    url: `${getNomadUrl()}/ui/jobs/${getJobName(name, selectorFlags, mode)}`,
+                  },
+                ],
+              },
+            ],
+          });
+        }
+
+        throw new Error(`[MONITOR] failed due to critical errors`);
+      }
+
+      console.log(`[MONITOR] check successfully in ${Date.now() - startTime}ms`);
+    }
+
+    runCheck(cli.flags).catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { monitor };
+}
+
+module.exports = { createMonitor, ERROR_LEVEL };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.7.17",
+  "version": "0.8.0",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",