@certik/skynet 0.10.44 → 0.10.47

package/CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
-## 0.10.44
+## 0.10.47
+
+- Added killTimeout config to `app`
+
+## 0.10.46
+
+- Added chain id to `const.js`, [data source](https://chainlist.org/).
+
+## 0.10.44 && 0.10.45
 
 - Fixed modeIndexer delta bug
 

package/app.d.ts

@@ -16,6 +16,7 @@ export type Serve = {
     prefix: string;
     port: number;
     apiKey: string;
+    killTimeout: string;
     cpu: number;
     mem: number;
     instances: number;
@@ -29,6 +30,7 @@ export type Consume = {
     }) => Promise<void>;
     topic: ({ protocol }: { protocol: string }) => string;
     maxRetry: number;
+    killTimeout: string;
     cpu: number;
     mem: number;
 }
@@ -52,6 +54,7 @@ export type Produce = {
     deadLetterTopic: ({ protocol }) => string;
     batchSize: number;
     maxRetry: number;
+    killTimeout: string;
     cpu: number;
     mem: number;
 }
@@ -61,6 +64,9 @@ export type Check = {
         => ({ type: string, message: string });
     schedule: string;
     slackChannel: string;
+    killTimeout: string;
+    cpu: number;
+    mem: number;
 }
 
 export type Build = {
@@ -68,6 +74,7 @@ export type Build = {
         => void;
     batchSize: number;
     schedule: string;
+    killTimeout: string;
     cpu: number;
     mem: number;
 }
@@ -84,6 +91,7 @@ export type Validate = {
         => void;
     batchSize: number;
     schedule: string;
+    killTimeout: string;
     cpu: number;
     mem: number;
 }

package/app.js

@@ -205,11 +205,13 @@ function indexer({ name, selector, build, check, env = {}, region = "us-east-1"
         env,
         schedule: build.schedule,
         restart: build.restart,
+        killTimeout: build.killTimeout,
         cpu: build.cpu,
         mem: build.mem,
         check: check && {
           bin: `${bin} check`,
           schedule: check.schedule,
+          killTimeout: check.killTimeout,
           cpu: check.cpu || 100,
           mem: check.mem || 100,
         },
@@ -313,18 +315,23 @@ function modeIndexer({ name, selector, state, build, validate, check, env = {},
         env,
 
         deltaSchedule: build.schedule,
+        deltaKillTimeout: build.killTimeout,
         deltaCpu: build.cpu,
         deltaMem: build.mem,
+
+        rebuildKillTimeout: build.killTimeout,
         rebuildCpu: build.cpu,
         rebuildMem: build.mem,
 
         validateSchedule: validate && validate.schedule,
+        validateKillTimeout: validate && validate.killTimeout,
         validateCpu: validate && validate.cpu,
         validateMem: validate && validate.mem,
 
         check: check && {
           bin: `${bin} check`,
           schedule: check.schedule,
+          killTimeout: check.killTimeout,
           cpu: check.cpu || 100,
           mem: check.mem || 100,
         },
@@ -430,11 +437,13 @@ function producer({ name, selector, produce, check, state, env = {}, region = "u
         region,
         env: envWithDefaultValues,
         schedule: "@minutely",
+        killTimeout: produce.killTimeout,
         cpu: produce.cpu,
         mem: produce.mem,
         check: check && {
           bin: `${bin} check`,
           schedule: check.schedule,
+          killTimeout: check.killTimeout,
           cpu: check.cpu || 100,
           mem: check.mem || 100,
         },
@@ -528,11 +537,13 @@ function consumer({ name, selector, consume, check, env = {}, region = "us-east-
         region,
         env: envWithDefaultValues,
         schedule: "@minutely",
+        killTimeout: consume.killTimeout,
         cpu: consume.cpu,
         mem: consume.mem,
         check: check && {
           bin: `${bin} check`,
           schedule: check.schedule,
+          killTimeout: check.killTimeout,
           cpu: check.cpu || 100,
           mem: check.mem || 100,
         },
@@ -682,6 +693,7 @@ function api({ name, routes, serve, beforeListen, env = {}, region = "us-east-1"
           interval: "30m",
         },
         count: serve.instances,
+        killTimeout: serve.killTimeout,
         cpu: serve.cpu,
         mem: serve.mem,
         service: {
@@ -691,6 +703,7 @@ function api({ name, routes, serve, beforeListen, env = {}, region = "us-east-1"
         check: check && {
           bin: `${bin} check`,
           schedule: check.schedule,
+          killTimeout: check.killTimeout,
           cpu: check.cpu || 100,
           mem: check.mem || 100,
         },

package/const.js

@@ -1,9 +1,4 @@
-const {
-  getGetBlockApiKey,
-  getAlchemyApiKey,
-  getNodeRealApiKey,
-  ensureAndGet,
-} = require("./env");
+const { getGetBlockApiKey, getAlchemyApiKey, getNodeRealApiKey, ensureAndGet } = require("./env");
 
 const SKYNET_API_PREFIX = "https://api.certik-skynet.com";
 
@@ -25,6 +20,7 @@ const PROTOCOLS = {
     },
     multiCallProvider: "0xCa731e0f33Afbcfa9363d6F7449d1f5447d10C80",
     scanUrl: "https://etherscan.io/",
+    chainId: 1,
   },
   bsc: {
     nativeTokenName: "Binance Coin",
@@ -43,6 +39,7 @@ const PROTOCOLS = {
     },
     multiCallProvider: "0xe7144e57d832c9005D252f415d205b4b8D78228e",
     scanUrl: "https://bscscan.com/",
+    chainId: 56,
   },
   polygon: {
     nativeTokenName: "Polygon",
@@ -62,6 +59,7 @@ const PROTOCOLS = {
     },
     multiCallProvider: "0x8eC86392e0aDB57d00fDffbA39b8870e107c0757",
     scanUrl: "https://polygonscan.com/",
+    chainId: 137,
   },
   heco: {
     nativeTokenName: "Heco",
@@ -71,7 +69,8 @@ const PROTOCOLS = {
     endpoint: `https://http-mainnet.hecochain.com`, // use heco public RPC node
     tokenStandard: "HRC20",
     multiCallProvider: "0xe7144e57d832c9005d252f415d205b4b8d78228e",
-    scanUrl: "https://hecoinfo.com/"
+    scanUrl: "https://hecoinfo.com/",
+    chainId: 128,
   },
   avax: {
     nativeTokenName: "Avalanche",
@@ -87,7 +86,8 @@ const PROTOCOLS = {
       endpoint: "https://api.snowtrace.io/api",
       key: ensureAndGet("SKYNET_AVASCAN_API_KEY"),
     },
-    scanUrl: "https://snowtrace.io/"
+    scanUrl: "https://snowtrace.io/",
+    chainId: 43114,
   },
   ftm: {
     nativeTokenName: "Fantom",
@@ -103,7 +103,8 @@ const PROTOCOLS = {
       endpoint: "https://api.ftmscan.com/api",
       key: ensureAndGet("SKYNET_FTMSCAN_API_KEY"),
     },
-    scanUrl: "https://ftmscan.com/"
+    scanUrl: "https://ftmscan.com/",
+    chainId: 250,
   },
   algo: {
     nativeTokenName: "Algorand",
@@ -115,7 +116,7 @@ const PROTOCOLS = {
     nativeTokenCmcId: 4030,
     endpoint: "https://algo-node.certik-skynet.com/",
     tokenStandard: "ASA",
-  }
+  },
 };
 
 const TIME = {

package/deploy.js

@@ -48,6 +48,7 @@ const genConfig = ({
   cron,
   count,
   restart,
+  killTimeout,
   cpu,
   mem,
   service,
@@ -185,6 +186,8 @@ EOH
         SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
       }
 
+      kill_timeout = "${killTimeout || "60s"}"
+
       # Specify the maximum resources required to run the task,
       # include CPU and memory.
       resources {
@@ -319,10 +322,13 @@ function createModeDeploy({
   check,
   deltaSchedule,
   validateSchedule,
+  deltaKillTimeout,
   deltaCpu,
   deltaMem,
+  rebuildKillTimeout,
   rebuildCpu,
   rebuildMem,
+  validateKillTimeout,
   validateCpu,
   validateMem,
 }) {
@@ -368,10 +374,14 @@ function createModeDeploy({
     }
 
     const modeResouces = {
-      rebuild: { cpu: rebuildCpu, mem: rebuildMem },
-      "resume-rebuild": { cpu: rebuildCpu, mem: rebuildMem },
-      validate: { cpu: validateCpu || rebuildCpu, mem: validateMem || rebuildMem },
-      delta: { cpu: deltaCpu, mem: deltaMem },
+      rebuild: { cpu: rebuildCpu, mem: rebuildMem, killTimeout: rebuildKillTimeout },
+      "resume-rebuild": { cpu: rebuildCpu, mem: rebuildMem, killTimeout: rebuildKillTimeout },
+      validate: {
+        cpu: validateCpu || rebuildCpu,
+        mem: validateMem || rebuildMem,
+        killTimeout: validateKillTimeout || rebuildKillTimeout,
+      },
+      delta: { cpu: deltaCpu, mem: deltaMem, killTimeout: deltaKillTimeout },
     };
 
     // added an always changing env var
@@ -382,7 +392,7 @@ function createModeDeploy({
     };
 
     // by default use delta cpu/mem settings
-    const { cpu, mem } = modeResouces[mode] || modeResouces.delta;
+    const { cpu, mem, killTimeout } = modeResouces[mode] || modeResouces.delta;
 
     let deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
 
@@ -408,6 +418,7 @@ function createModeDeploy({
       additionalEnv: env,
       region,
       cmd: `${bin} ${args} ${rangeArgs}`,
+      killTimeout,
       cpu,
       mem,
       isProduction: production,
@@ -436,6 +447,7 @@ function createModeDeploy({
         },
         region,
         cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
+        killTimeout: check.killTimeout || "10s",
         cpu: check.cpu || 100,
         mem: check.mem || 100,
         isProduction: production,
@@ -537,6 +549,7 @@ function createDeploy({
   check,
   schedule,
   restart,
+  killTimeout,
   cpu,
   mem,
   service,
@@ -578,6 +591,7 @@ function createDeploy({
       region,
       type,
       cmd: `${bin} ${args}`,
+      killTimeout,
       cpu,
       mem,
       service,
@@ -607,6 +621,7 @@ function createDeploy({
         },
         region,
         cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
+        killTimeout: check.killTimeout || "10s",
         cpu: check.cpu || 100,
         mem: check.mem || 100,
         isProduction: production,

package/examples/indexer

@@ -50,6 +50,7 @@ const app = indexer({
   build: {
     func: build,
     schedule: every(1).minute,
+    killTimeout: "90s",
     cpu: 600,
     mem: 200,
   },

package/indexer.js

@@ -126,53 +126,71 @@ function createModeIndexerApp({
       process.exit(0);
     }
 
-    if (!from) {
-      from = await finalState.getMinId(selectorFlags);
-    }
-
-    inline.log(
-      `[MODE INDEXER] mode=${mode}, from=${from}, to=${
-        to ? to : "undefined"
-      }, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`
-    );
+    inline.log(`[MODE INDEXER] mode=${mode}, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`);
 
     if (mode === "reset") {
       await runReset(selectorFlags);
     } else if (mode === "rebuild") {
+      const rebuildFrom = from || (await finalState.getMinId(selectorFlags));
+      const rebuildTo = to || (await finalState.getMaxId(selectorFlags));
+
       await runReset(selectorFlags);
-      await runRebuild(selectorFlags, from, to ? to : await finalState.getMaxId(selectorFlags), verbose);
+      await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
     } else if (mode === "resume-rebuild") {
       const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
 
-      await runRebuild(
-        selectorFlags,
-        previousRebuildEnds ? increaseId(finalState.type, previousRebuildEnds, 1) : from,
-        to ? to : await finalState.getMaxId(selectorFlags),
-        verbose
-      );
+      const rebuildFrom =
+        from || increaseId(finalState.type, previousRebuildEnds, 1) || (await finalState.getMinId(selectorFlags));
+      const rebuildTo = to || (await finalState.getMaxId(selectorFlags));
+
+      await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
     } else if (mode === "validate" || mode === "validation") {
       const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
 
+      if (!previousRebuildEnds) {
+        inline.log(`[MODE INDEXER] cannot validate without a successful rebuild`);
+        process.exit(0);
+      }
+
       const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
 
-      const shouldSaveState = !to;
+      const validateFrom = from || previousValidatedTo || (await finalState.getMinId(selectorFlags));
+      const validateTo = to || previousRebuildEnds;
+      const shouldSaveState = !to; // should not save state for manual validations, those are for testing
 
-      await runValidate(
-        selectorFlags,
-        from ? from : previousValidatedTo,
-        to ? to : previousRebuildEnds,
-        shouldSaveState,
-        verbose
-      );
+      await runValidate(selectorFlags, validateFrom, validateTo, shouldSaveState, verbose);
     } else if (mode === "one") {
       if (to) {
         inline.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
       }
+      if (!from) {
+        inline.log(`[MODE INDEXER] must provide --from option for one mode`);
+        process.exit(1);
+      }
+
       await runRange(selectorFlags, from, from, verbose);
     } else if (mode === "range") {
+      if (!from || !to) {
+        inline.log(`[MODE INDEXER] must provide --from and --to option for range mode`);
+        process.exit(1);
+      }
+
       await runRange(selectorFlags, from, to, verbose);
     } else {
-      await runDelta(selectorFlags, verbose);
+      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      });
+
+      // only build when rebuild succeed
+      if (!stateItem || stateItem.value !== "succeed") {
+        inline.log("[MODE INDEXER] skip because rebuild hasn't done yet");
+        process.exit(0);
+      }
+
+      const deltaFrom = increaseId(finalState.type, await getIndexerLatestId(name, selectorFlags), 1);
+      const deltaTo = await state.getMaxId(selectorFlags);
+
+      await runDelta(selectorFlags, deltaFrom, deltaTo, verbose);
     }
   }
 
@@ -385,74 +403,62 @@ function createModeIndexerApp({
     }
   }
 
-  async function runDelta(selectorFlags, verbose) {
-    const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
-      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-    });
-
-    // only build when rebuild succeed
-    if (stateItem && stateItem.value === "succeed") {
-      const from = increaseId(finalState.type, await getIndexerLatestId(name, selectorFlags), 1);
-
-      const startTime = Date.now();
-      const to = await state.getMaxId(selectorFlags);
-
-      if (to <= from) {
-        inline.log(
-          `[MODE INDEXER] skip delta because there're no items need to be processed, ${toSelectorString(
-            selectorFlags,
-            ", "
-          )}`
-        );
-
-        return;
-      }
+  async function runDelta(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
 
+    if (to < from) {
       inline.log(
-        `[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(
+        `[MODE INDEXER] skip delta, there're no more items need to be processed, from=${from}, to=${to}, ${toSelectorString(
           selectorFlags,
           ", "
-        )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+        )}`
       );
 
-      try {
-        const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
-
-        if (failedIds.length > 0) {
-          inline.log("[MODE INDEXER] built with some failed txs", failedIds);
+      return;
+    }
 
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
-            value: "failed",
-          });
+    inline.log(
+      `[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(
+        selectorFlags,
+        ", "
+      )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+    );
 
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}Since(${toSelectorString(selectorFlags)})`,
-            value: Math.min(to, failedIds[0]),
-          });
+    try {
+      const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
 
-          process.exit(1);
-        } else {
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
-            value: "succeed",
-          });
+      if (failedIds.length > 0) {
+        inline.log("[MODE INDEXER] built with some failed txs", failedIds);
 
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}Since(${toSelectorString(selectorFlags)})`,
-            value: to,
-          });
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+          value: "failed",
+        });
 
-          inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
-          process.exit(0);
-        }
-      } catch (err) {
-        inline.error("[MODE INDEXER] delta build failed", from, to, err);
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: Math.min(to, failedIds[0]),
+        });
 
         process.exit(1);
+      } else {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+          value: "succeed",
+        });
+
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: to,
+        });
+
+        inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+        process.exit(0);
       }
-    } else {
-      inline.log("[MODE INDEXER] skip because rebuild hasn't done yet");
+    } catch (err) {
+      inline.error("[MODE INDEXER] delta build failed", from, to, err);
+
+      process.exit(1);
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.10.44",
+  "version": "0.10.47",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",