@certik/skynet 0.8.15 → 0.9.1

package/CHANGELOG.md

@@ -1,8 +1,21 @@
 # Changelog
 
+## 0.9.1
+
+- Support using `--schedule` argument to override default schedule when deploying
+
+## 0.9.0
+
+- Added `web3` library with call support
+
+## 0.8.16
+
+- Changed monitor exit behavior
+- Improved OpsGenie message for service checks
+
 ## 0.8.15
 
-- OpsGenie bug fix
+- Fixed OpsGenie message bug 
 
 ## 0.8.14
 

package/abi.js

@@ -6,35 +6,35 @@ const ERC20 = [
     outputs: [
       {
         name: "",
-        type: "string"
-      }
+        type: "string",
+      },
     ],
     payable: false,
     stateMutability: "view",
-    type: "function"
+    type: "function",
   },
   {
     constant: false,
     inputs: [
       {
         name: "_spender",
-        type: "address"
+        type: "address",
       },
       {
         name: "_value",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     name: "approve",
     outputs: [
       {
         name: "",
-        type: "bool"
-      }
+        type: "bool",
+      },
     ],
     payable: false,
     stateMutability: "nonpayable",
-    type: "function"
+    type: "function",
   },
   {
     constant: true,
@@ -43,39 +43,39 @@ const ERC20 = [
     outputs: [
       {
         name: "",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     payable: false,
     stateMutability: "view",
-    type: "function"
+    type: "function",
   },
   {
     constant: false,
     inputs: [
       {
         name: "_from",
-        type: "address"
+        type: "address",
       },
       {
         name: "_to",
-        type: "address"
+        type: "address",
       },
       {
         name: "_value",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     name: "transferFrom",
     outputs: [
       {
         name: "",
-        type: "bool"
-      }
+        type: "bool",
+      },
     ],
     payable: false,
     stateMutability: "nonpayable",
-    type: "function"
+    type: "function",
   },
   {
     constant: true,
@@ -84,31 +84,31 @@ const ERC20 = [
     outputs: [
       {
         name: "",
-        type: "uint8"
-      }
+        type: "uint8",
+      },
     ],
     payable: false,
     stateMutability: "view",
-    type: "function"
+    type: "function",
   },
   {
     constant: true,
     inputs: [
       {
         name: "_owner",
-        type: "address"
-      }
+        type: "address",
+      },
     ],
     name: "balanceOf",
     outputs: [
       {
         name: "balance",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     payable: false,
     stateMutability: "view",
-    type: "function"
+    type: "function",
   },
   {
     constant: true,
@@ -117,63 +117,63 @@ const ERC20 = [
     outputs: [
       {
         name: "",
-        type: "string"
-      }
+        type: "string",
+      },
     ],
     payable: false,
     stateMutability: "view",
-    type: "function"
+    type: "function",
   },
   {
     constant: false,
     inputs: [
       {
         name: "_to",
-        type: "address"
+        type: "address",
       },
       {
         name: "_value",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     name: "transfer",
     outputs: [
       {
         name: "",
-        type: "bool"
-      }
+        type: "bool",
+      },
     ],
     payable: false,
     stateMutability: "nonpayable",
-    type: "function"
+    type: "function",
   },
   {
     constant: true,
     inputs: [
       {
         name: "_owner",
-        type: "address"
+        type: "address",
       },
       {
         name: "_spender",
-        type: "address"
-      }
+        type: "address",
+      },
     ],
     name: "allowance",
     outputs: [
       {
         name: "",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     payable: false,
     stateMutability: "view",
-    type: "function"
+    type: "function",
   },
   {
     payable: true,
     stateMutability: "payable",
-    type: "fallback"
+    type: "fallback",
   },
   {
     anonymous: false,
@@ -181,21 +181,21 @@ const ERC20 = [
       {
         indexed: true,
         name: "owner",
-        type: "address"
+        type: "address",
       },
       {
         indexed: true,
         name: "spender",
-        type: "address"
+        type: "address",
       },
       {
         indexed: false,
         name: "value",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     name: "Approval",
-    type: "event"
+    type: "event",
   },
   {
     anonymous: false,
@@ -203,25 +203,151 @@ const ERC20 = [
       {
         indexed: true,
         name: "from",
-        type: "address"
+        type: "address",
       },
       {
         indexed: true,
         name: "to",
-        type: "address"
+        type: "address",
       },
       {
         indexed: false,
         name: "value",
-        type: "uint256"
-      }
+        type: "uint256",
+      },
     ],
     name: "Transfer",
-    type: "event"
-  }
+    type: "event",
+  },
+];
+
+const MULTICALL = [
+  {
+    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
+    name: "callBalanceOf",
+    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callBlockNumber",
+    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [{ internalType: "uint256", name: "_i", type: "uint256" }],
+    name: "callBlockhash",
+    outputs: [{ internalType: "bytes32", name: "", type: "bytes32" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callChainId",
+    outputs: [{ internalType: "uint256", name: "id", type: "uint256" }],
+    stateMutability: "pure",
+    type: "function",
+  },
+  {
+    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
+    name: "callCode",
+    outputs: [{ internalType: "bytes", name: "code", type: "bytes" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
+    name: "callCodeHash",
+    outputs: [{ internalType: "bytes32", name: "codeHash", type: "bytes32" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
+    name: "callCodeSize",
+    outputs: [{ internalType: "uint256", name: "size", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callCoinbase",
+    outputs: [{ internalType: "address", name: "", type: "address" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callDifficulty",
+    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callGasLeft",
+    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callGasLimit",
+    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callGasPrice",
+    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callOrigin",
+    outputs: [{ internalType: "address", name: "", type: "address" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [],
+    name: "callTimestamp",
+    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
+    stateMutability: "view",
+    type: "function",
+  },
+  {
+    inputs: [
+      {
+        components: [
+          { internalType: "bool", name: "delegateCall", type: "bool" },
+          { internalType: "bool", name: "revertOnError", type: "bool" },
+          { internalType: "uint256", name: "gasLimit", type: "uint256" },
+          { internalType: "address", name: "target", type: "address" },
+          { internalType: "uint256", name: "value", type: "uint256" },
+          { internalType: "bytes", name: "data", type: "bytes" },
+        ],
+        internalType: "struct IModuleCalls.Transaction[]",
+        name: "_txs",
+        type: "tuple[]",
+      },
+    ],
+    name: "multiCall",
+    outputs: [
+      { internalType: "bool[]", name: "_successes", type: "bool[]" },
+      { internalType: "bytes[]", name: "_results", type: "bytes[]" },
+    ],
+    stateMutability: "payable",
+    type: "function",
+  },
 ];
 
 module.exports = {
+  MULTICALL,
   ERC20,
-  BEP20: ERC20
+  BEP20: ERC20,
 };

package/const.js

@@ -23,6 +23,7 @@ const PROTOCOLS = {
       endpoint: "https://api.etherscan.io/api",
       key: getEtherScanApiKey(),
     },
+    multiCallProvider: "0xCa731e0f33Afbcfa9363d6F7449d1f5447d10C80",
     scanUrl: "https://etherscan.io/",
   },
   bsc: {
@@ -39,6 +40,7 @@ const PROTOCOLS = {
       endpoint: "https://api.bscscan.com/api",
       key: getBscScanApiKey(),
     },
+    multiCallProvider: "0xe7144e57d832c9005D252f415d205b4b8D78228e",
     scanUrl: "https://bscscan.com/",
   },
   polygon: {
@@ -56,8 +58,9 @@ const PROTOCOLS = {
       endpoint: "https://api.polygonscan.com/api",
       key: getPolygonScanApiKey(),
     },
+    multiCallProvider: "", // TODO
     scanUrl: "https://polygonscan.com/",
-  },
+  }
 };
 
 const TIME = {

package/deploy.js

@@ -9,10 +9,15 @@ const { getEnvOrThrow } = require("./env");
 const { getBinaryName, detectSkynetDirectory } = require("./cli");
 
 const INTERVAL_ALIASES = {
+  secondly: "*/1 * * * * * *",
   "@secondly": "*/1 * * * * * *",
+  minutely: "0 * * * * * *",
   "@minutely": "0 * * * * * *",
+  hourly: "0 0 * * * * *",
   "@hourly": "0 0 * * * * *",
+  daily: "0 0 0 * * * *",
   "@daily": "0 0 0 * * * *",
+  weekly: "0 0 0 * * 0 *",
   "@weekly": "0 0 0 * * 0 *",
 };
 
@@ -295,7 +300,17 @@ function createModeDeploy({
   validateCpu,
   validateMem,
 }) {
-  async function deployMode({ mode, from, to, stop, production, dryRun, verbose, ...selectorFlags }) {
+  async function deployMode({
+    mode,
+    from,
+    to,
+    stop,
+    production,
+    dryRun,
+    verbose,
+    schedule: cmdSchedule,
+    ...selectorFlags
+  }) {
     if (mode === "delta") {
       // delta mode will ignore from/to flags
       from = 0;
@@ -336,9 +351,17 @@ function createModeDeploy({
     // by default use delta cpu/mem settings
     const { cpu, mem } = modeResouces[mode] || modeResouces.delta;
 
-    const deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
+    let deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
 
-    const validateCron = typeof validateSchedule === "function" ? validateSchedule(jobName) : validateSchedule;
+    if (deltaSchedule && cmdSchedule) {
+      deltaCron = cmdSchedule;
+    }
+
+    let validateCron = typeof validateSchedule === "function" ? validateSchedule(jobName) : validateSchedule;
+
+    if (validateSchedule && cmdSchedule) {
+      validateCron = cmdSchedule;
+    }
 
     const modeIntervals = {
       delta: INTERVAL_ALIASES[deltaCron] || deltaCron,
@@ -404,7 +427,8 @@ ${getSelectorDesc(selector)}
           --from          min id to build
           --to            max id to build
           --stop          stop job instead of running the job
-          --production    deploy to production environment
+          --production    deploy to production, default is development
+          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
           --verbose       Output debug messages
           --dry-run       print nomad job file but do not really execute it
 
@@ -432,6 +456,9 @@ ${getSelectorDesc(selector)}
             type: "number",
             default: 0,
           },
+          schedule: {
+            type: "string",
+          },
           verbose: {
             type: "boolean",
             default: false,
@@ -476,7 +503,7 @@ function createDeploy({
   cpu,
   mem,
 }) {
-  async function deployModeless({ production, stop, dryRun, verbose, ...selectorFlags }) {
+  async function deployModeless({ production, stop, dryRun, verbose, schedule: cmdSchedule, ...selectorFlags }) {
     const jobName = getJobName(name, selectorFlags, null);
 
     const selectorCmdPart = Object.keys(selectorFlags)
@@ -489,7 +516,12 @@ function createDeploy({
       args += ` --verbose`;
     }
 
-    const cron = typeof schedule === "function" ? schedule(jobName) : schedule;
+    let cron = typeof schedule === "function" ? schedule(jobName) : schedule;
+
+    if (schedule && cmdSchedule) {
+      // cmd schedule has higher priority
+      cron = cmdSchedule;
+    }
 
     const nomadJobDefinition = genConfig({
       jobName,
@@ -549,6 +581,7 @@ function createDeploy({
 ${getSelectorDesc(selector)}
           --stop          stop job instead of running the job
           --production    deploy to production, default is development
+          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
           --verbose       Output debug messages
           --dry-run       print nomad job file but do not really execute it
       `,
@@ -557,6 +590,9 @@ ${getSelectorDesc(selector)}
         version: false,
         flags: {
           ...getSelectorFlags(selector),
+          schedule: {
+            type: "string",
+          },
           verbose: {
             type: "boolean",
             default: false,

package/monitor.js

@@ -30,69 +30,44 @@ function sortErrors(errors) {
 }
 
 async function getMostRecentJobLaunch(name) {
-  const jobsRes = await fetch(`http://localhost:4646/v1/jobs?prefix=${name}`);
+  try {
+    const jobsRes = await fetch(`http://localhost:4646/v1/jobs?prefix=${name}`);
 
-  if (!jobsRes.ok) {
-    console.log(`[MONITOR] request local nomad API failed`);
+    if (!jobsRes.ok) {
+      console.log(`[MONITOR] request local nomad API failed`);
 
-    return null;
-  }
-
-  const jobs = await jobsRes.json();
-
-  if (jobs.length === 0) {
-    console.log(`[MONITOR] did not see any jobs prefixed with ${name}`);
-
-    return null;
-  }
+      return null;
+    }
 
-  const recentFinishedJob = jobs.reverse().find((job) => {
-    // filter out monitor job
-    return job.ID.indexOf("-monitor") === -1 && job.Status === "dead";
-  });
+    const jobs = await jobsRes.json();
 
-  if (!recentFinishedJob) {
-    console.log(`[MONITOR] did not see any dead jobs`);
+    if (jobs.length === 0) {
+      console.log(`[MONITOR] did not see any jobs prefixed with ${name}`);
 
-    return null;
-  }
+      return null;
+    }
 
-  console.log("[MONITOR]", "most recent job info", recentFinishedJob.ID, recentFinishedJob.JobSummary.Summary);
+    const recentFinishedJob = jobs.reverse().find((job) => {
+      // filter out monitor job
+      return job.ID.indexOf("-monitor") === -1 && job.Status === "dead";
+    });
 
-  return recentFinishedJob;
-}
+    if (!recentFinishedJob) {
+      console.log(`[MONITOR] did not see any dead jobs`);
 
-function checkMostRecentJobFailures(recentFinishedJob) {
-  if (!recentFinishedJob) {
-    return [];
-  }
+      return null;
+    }
 
-  const summary = recentFinishedJob.JobSummary.Summary;
-  const groups = Object.keys(summary);
+    console.log("[MONITOR]", "most recent job info", recentFinishedJob.ID, recentFinishedJob.JobSummary.Summary);
 
-  for (let group of groups) {
-    if (summary[group].Failed > 0) {
-      return [
-        {
-          type: ERROR_LEVEL.CRITICAL,
-          message: `The most recent launch returned non-zero exit value, please investigate`,
-        },
-      ];
-    }
+    return recentFinishedJob;
+  } catch (getJobError) {
+    console.log("[MONITOR]", "cannot get most recent job", getJobError.message);
+    return null;
   }
-
-  return [];
 }
 
-function createMonitor({
-  binaryName,
-  name,
-  type = "stateless",
-  mode = false,
-  selector = {},
-  check,
-  maxRetry = 2,
-}) {
+function createMonitor({ binaryName, name, type = "stateless", mode = false, selector = {}, check, maxRetry = 2 }) {
   function monitor() {
     if (!binaryName) {
       binaryName = getBinaryName();
@@ -169,25 +144,26 @@ ${
 
       const jobName = getJobName(name, selectorFlags, mode);
       const mostRecentJob = await getMostRecentJobLaunch(jobName);
-      const recentJobFailures = checkMostRecentJobFailures(mostRecentJob);
-
-      result = result.concat(recentJobFailures);
 
       if (result.length > 0) {
         console.log("Found Errors", result);
 
         if (production) {
-          // Also alert on opsgenie (func prevents duplicate alerts)
+          const nomadAddr = process.env.SKYNET_NOMAD_PRODUCTION_ADDR;
+
+          // alert on opsgenie
           await postGenieMessage(
-            `${jobName} Monitor Errors`,
-            `${sortErrors(result)
-              .map((m) => m.message || m)
-              .join("\n")}`,
+            `Failed Service Check: ${jobName}`,
+            `<p><b>Service:</b><a href="${nomadAddr}/ui/jobs/${jobName}" target="_blank">${jobName}</a></p><p><b>Issues</b></p><ul>${sortErrors(
+              result
+            )
+              .map((m) => `<li><b>${m.type}:</b> ${m.message}</li>`)
+              .join("")}</ul>`,
             verbose
           );
+        } else {
+          console.log("skip sending messages to opsgenie in dev env");
         }
-
-        throw new Error(`failed due to critical errors`);
       }
 
       console.log(`[MONITOR] check successfully in ${Date.now() - startTime}ms`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.8.15",
+  "version": "0.9.1",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",

package/util.js

@@ -10,10 +10,7 @@ function partition(startAt, endAt, numGroups) {
   const step = Math.max(Math.floor(span / numGroups), 1);
   const adjustedStep = span % step === 0 ? step : step + 1;
 
-  return [
-    [startAt, startAt + adjustedStep - 1],
-    ...partition(startAt + adjustedStep, endAt, numGroups - 1)
-  ];
+  return [[startAt, startAt + adjustedStep - 1], ...partition(startAt + adjustedStep, endAt, numGroups - 1)];
 }
 
 // Inclusive range
@@ -50,9 +47,21 @@ function fillRange(start, end) {
   return result;
 }
 
+function chunk(array, count) {
+  if (count == null || count < 1) return [];
+  var result = [];
+  var i = 0,
+    length = array.length;
+  while (i < length) {
+    result.push(array.slice(i, (i += count)));
+  }
+  return result;
+}
+
 module.exports = {
   arrayGroup,
   partition,
   range,
-  fillRange
+  fillRange,
+  chunk
 };

package/web3.js

@@ -0,0 +1,117 @@
+const Web3 = require("web3");
+const { PROTOCOLS } = require("./const");
+const { MULTICALL } = require("./abi");
+const { chunk } = require("./util");
+const MULTICALL_CHUNK_SIZE = 400;
+
+function newWeb3ByProtocol(protocol) {
+  if (!Object.keys(PROTOCOLS).includes(protocol)) {
+    return null;
+  }
+
+  return new Web3(PROTOCOLS[protocol].endpoint);
+}
+
+function normalizeCallParams(params) {
+  if (params === undefined) {
+    return [];
+  }
+
+  if (Array.isArray(params)) {
+    return params;
+  }
+
+  return [params];
+}
+
+async function singleCall(protocol, abi, target, params) {
+  const web3 = newWeb3ByProtocol(protocol);
+
+  if (!web3) {
+    throw new Error(`unsupported protocol`);
+  }
+
+  const contract = new web3.eth.Contract([abi], target);
+  const functionSignature = web3.eth.abi.encodeFunctionSignature(abi);
+  const method = contract.methods[functionSignature];
+
+  const result = await method(...normalizeCallParams(params)).call();
+
+  return result;
+}
+
+// limiter if provided, should be an instance of Bottleneck, which is part of bottleneck package
+async function multiCall({ protocol, limiter = null, target, abi, calls }) {
+  const web3 = newWeb3ByProtocol(protocol);
+
+  if (!web3) {
+    throw new Error(`unsupported protocol`);
+  }
+
+  const multiCallProviderAddress = PROTOCOLS[protocol].multiCallProvider;
+
+  if (!multiCallProviderAddress) {
+    throw new Error("protocol doesn't support multicall yet");
+  }
+
+  if (calls.length === 0) return { callCount: 0, output: [] };
+
+  const multiCallContract = new web3.eth.Contract(MULTICALL, multiCallProviderAddress);
+
+  let maybeRateLimitedCallChunk = async (callChunk) => {
+    const txs = callChunk.map((call) => ({
+      delegateCall: false,
+      revertOnError: false,
+      gasLimit: 0,
+      target: call.target || target,
+      value: 0,
+      data: web3.eth.abi.encodeFunctionCall(abi, normalizeCallParams(call.params)),
+    }));
+
+    const response = await multiCallContract.methods.multiCall(txs).call();
+
+    return response._results.map((res, idx) => {
+      const input = {
+        target: callChunk[idx].target || target,
+        params: normalizeCallParams(callChunk[idx].params),
+      };
+
+      try {
+        const callResult = web3.eth.abi.decodeParameters(abi.outputs, res);
+
+        return {
+          input,
+          success: response._successes[idx],
+          // according to SDK doc, if there's only one, return result directly
+          output: abi.outputs.length === 1 ? callResult[0] : callResult,
+        };
+      } catch (decodeErr) {
+        console.log("decode err", abi.name, callChunk[idx].target || target, callChunk[idx].params, decodeErr.message);
+
+        return {
+          input,
+          success: false,
+          output: null,
+        };
+      }
+    });
+  };
+
+  if (limiter) {
+    maybeRateLimitedCallChunk = limiter.wrap(maybeRateLimitedCallChunk);
+  }
+
+  const callChunks = chunk(calls, MULTICALL_CHUNK_SIZE);
+
+  const responses = (
+    await Promise.all(callChunks.map(async (callChunk) => await maybeRateLimitedCallChunk(callChunk)))
+  ).flat();
+
+  return { actualCallCount: callChunks.length, output: responses };
+}
+
+module.exports = {
+  newWeb3ByProtocol,
+  singleCall,
+  multiCall,
+};