@certik/skynet 0.7.20 → 0.8.0

package/examples/indexer

@@ -2,24 +2,55 @@
 
 // this file is executable and is for kafka producer testing
 // a few test commands to try on
-//   $ examples/indexer --protocol bsc --verbose
-//   $ examples/indexer --protocol eth
+//   $ examples/indexer run --protocol bsc --verbose
+//   $ examples/indexer run --protocol eth
+//   $ examples/indexer check --protocol eth
+//   $ examples/indexer deploy --protocol eth
 //   $ examples/indexer --help
 
-const { createIndexerApp } = require("../indexer");
+const { indexer, ERROR_LEVEL } = require("../app");
 
 async function build({ protocol, verbose }) {
   console.log("build called with", protocol, verbose);
 
   if (protocol === "eth") {
-    throw new Error("when problem is not recoverable and cannot be resolved by retry, throw error here");
+    throw new Error("use eth to test error handling");
   }
 }
 
-const { run } = createIndexerApp({
-  name: "ShareJSTestIndexer",
+async function check({ protocol, state, verbose }) {
+  console.log("check called with", protocol, state, verbose);
+
+  // latestId
+  // latestIdOnChain
+
+  const errors = [];
+
+  errors.push({
+    type: ERROR_LEVEL.WARNING,
+    message: "processed *height* lagged behind", // can be markdown
+  });
+
+  return errors;
+}
+
+const app = indexer({
+  name: "LibSkynetExampleIndexer",
   selector: { protocol: { type: "string" } },
-  build
+
+  build: {
+    func: build,
+    schedule: "@minutely",
+    cpu: 600,
+    mem: 200,
+  },
+
+  check: {
+    func: check,
+    schedule: "@minutely",
+    maxRetry: 0,
+    slackChannel: "skynet-notifications-local-dev",
+  },
 });
 
-run();
+app();

package/examples/{deploy-consumer → legacy-deploy-consumer}

@@ -7,8 +7,8 @@
 const { createDeploy } = require("../deploy");
 
 const { deploy } = createDeploy({
-  name: "shared-js-test-consumer",
-  workingDirectory: "shared-js",
+  name: "lib-skynet-test-consumer",
+  workingDirectory: "lib-skynet",
   bin: "examples/kafka-consumer",
   selector: { protocol: { type: "string" } },
   env: {

package/examples/{deploy-indexer → legacy-deploy-indexer}

@@ -7,12 +7,13 @@
 const { createDeploy } = require("../deploy");
 
 const { deploy } = createDeploy({
-  name: "ShareJSTestIndexer",
-  workingDirectory: "shared-js",
+  name: "lib-skynet-test-indexer",
+  workingDirectory: "lib-skynet",
   bin: "examples/indexer",
   selector: { protocol: { type: "string" } },
+  schedule: "@minutely",
   cpu: 600,
-  mem: 200
+  mem: 200,
 });
 
 deploy();

package/examples/{deploy-mode-indexer → legacy-deploy-mode-indexer}

@@ -7,8 +7,8 @@
 const { createModeDeploy } = require("../deploy");
 
 const { deploy } = createModeDeploy({
-  name: "ShareJSTestModeIndexer",
-  workingDirectory: "shared-js",
+  name: "lib-skynet-test-mode-indexer",
+  workingDirectory: "lib-skynet",
   bin: "examples/mode-indexer",
   selector: { protocol: { type: "string" } },
   deltaSchedule: "@minutely",

package/examples/{deploy-producer → legacy-deploy-producer}

@@ -9,8 +9,8 @@ const path = require("path");
 const { createDeploy } = require("../deploy");
 
 const { deploy } = createDeploy({
-  name: "shared-js-test-producer",
-  workingDirectory: "shared-js", // where package.json was located
+  name: "lib-skynet-test-producer",
+  workingDirectory: "lib-skynet", // where package.json was located
   bin: "examples/kafka-producer",
   selector: { protocol: { type: "string" } },
   env: {

package/examples/legacy-indexer

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+
+// this file is executable and is for kafka producer testing
+// a few test commands to try on
+//   $ examples/indexer --protocol bsc --verbose
+//   $ examples/indexer --protocol eth
+//   $ examples/indexer --help
+
+const { createIndexerApp } = require("../indexer");
+
+async function build({ protocol, verbose }) {
+  console.log("build called with", protocol, verbose);
+
+  if (protocol === "eth") {
+    throw new Error("when problem is not recoverable and cannot be resolved by retry, throw error here");
+  }
+}
+
+const { run } = createIndexerApp({
+  name: "lib-skynet-test-indexer",
+  selector: { protocol: { type: "string" } },
+  build
+});
+
+run();

package/examples/{kafka-consumer → legacy-kafka-consumer}

@@ -24,7 +24,7 @@ const { run } = createConsumerApp({
   selector: { protocol: { type: "string" } },
 
   consumer: {
-    topic: ({ protocol }) => `shared-js-test-kafka-${protocol}-dev`,
+    topic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dev`,
     consume,
     maxRetry: 1 // default to 2
   }

package/examples/{kafka-producer → legacy-kafka-producer}

@@ -42,8 +42,8 @@ const { run } = createProducerApp({
   selector: { protocol: { type: "string" } },
 
   producer: {
-    topic: ({ protocol }) => `shared-js-test-kafka-${protocol}-dev`,
-    deadLetterTopic: ({ protocol }) => `shared-js-test-kafka-${protocol}-dead-letter-dev`, // problematic ids will be sent to dead letter topic for later retry
+    topic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dev`,
+    deadLetterTopic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dead-letter-dev`, // problematic ids will be sent to dead letter topic for later retry
     produce,
     batchSize: 10, // default to 50
     maxRetry: 1 // default to 2

package/examples/legacy-mode-indexer

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+
+// this file is executable and is for kafka producer testing
+// a few test commands to try on
+//   $ examples/indexer --protocol bsc --verbose
+//   $ examples/indexer --help
+
+const { createModeIndexerApp } = require("../indexer");
+const { readFile, writeFile } = require("fs/promises");
+
+async function build({ protocol, from, to, verbose }) {
+  console.log("build called with", protocol, from, to, verbose);
+
+  for (let i = from; i <= to; i++) {
+    if (i !== 5) {
+      // simulate a gap, id = 5
+      // we can use validate mode to fix
+      const filename = `/tmp/ShareJSTestModeIndexer-${protocol}-${i}`;
+
+      await writeFile(filename, "helloworld");
+    }
+  }
+
+  console.log("build done", protocol, from, to);
+}
+
+async function validate({ protocol, from, to, verbose }) {
+  console.log("validate called with", protocol, from, to, verbose);
+
+  for (let i = from; i <= to; i++) {
+    const filename = `/tmp/ShareJSTestModeIndexer-${protocol}-${i}`;
+
+    try {
+      await readFile(filename);
+    } catch (notFound) {
+      // fix missing data
+      await writeFile(filename, "helloworld");
+    }
+  }
+
+  console.log("validate done", protocol, from, to);
+}
+
+const { run } = createModeIndexerApp({
+  name: "ShareJSTestModeIndexer",
+  selector: { protocol: { type: "string" } },
+
+  build,
+  buildBatchSize: 5,
+  buildConcurrency: 2, // default is 1, no concurrency
+  validate,
+  validateBatchSize: 2,
+  validateConcurrency: 2, // default is 1, no concurrency
+
+  state: {
+    type: "block", // can be omitted, default is block
+    getMinId: async () => 2, // default returns 1
+    getMaxId: async ({ protocol }) => {
+      return 10;
+    }
+  }
+});
+
+run();

package/examples/mode-indexer

@@ -2,63 +2,76 @@
 
 // this file is executable and is for kafka producer testing
 // a few test commands to try on
-//   $ examples/indexer --protocol bsc --verbose
-//   $ examples/indexer --help
+//   $ examples/mode-indexer run --protocol bsc --verbose
+//   $ examples/mode-indexer run --mode rebuild --protocol bsc --verbose
+//   $ examples/mode-indexer run --protocol eth
+//   $ examples/mode-indexer check --protocol eth
+//   $ examples/mode-indexer deploy --protocol eth
+//   $ examples/mode-indexer --help
 
-const { createModeIndexerApp } = require("../indexer");
-const { readFile, writeFile } = require("fs/promises");
+const { modeIndexer } = require("../app");
 
 async function build({ protocol, from, to, verbose }) {
   console.log("build called with", protocol, from, to, verbose);
-
-  for (let i = from; i <= to; i++) {
-    if (i !== 5) {
-      // simulate a gap, id = 5
-      // we can use validate mode to fix
-      const filename = `/tmp/ShareJSTestModeIndexer-${protocol}-${i}`;
-
-      await writeFile(filename, "helloworld");
-    }
-  }
-
-  console.log("build done", protocol, from, to);
 }
 
 async function validate({ protocol, from, to, verbose }) {
   console.log("validate called with", protocol, from, to, verbose);
+}
+
+async function check({ mode, protocol, state, verbose }) {
+  console.log("check called with", mode, protocol, state, verbose);
 
-  for (let i = from; i <= to; i++) {
-    const filename = `/tmp/ShareJSTestModeIndexer-${protocol}-${i}`;
+  const errors = [];
 
-    try {
-      await readFile(filename);
-    } catch (notFound) {
-      // fix missing data
-      await writeFile(filename, "helloworld");
-    }
-  }
+  // if (state.latestId < getBlockLatestHeight(protocol)) {
+  //   errors.push({ type: ERROR_LEVEL.INFO, message: "processed height lagged behind" });
+  // }
 
-  console.log("validate done", protocol, from, to);
+  return errors;
 }
 
-const { run } = createModeIndexerApp({
-  name: "ShareJSTestModeIndexer",
+const app = modeIndexer({
+  name: "LibSkynetExampleModeIndexer",
   selector: { protocol: { type: "string" } },
 
-  build,
-  buildBatchSize: 5,
-  buildConcurrency: 2, // default is 1, no concurrency
-  validate,
-  validateBatchSize: 2,
-  validateConcurrency: 2, // default is 1, no concurrency
-
   state: {
     type: "block", // can be omitted, default is block
     getMinId: async () => 2, // default returns 1
     getMaxId: async ({ protocol }) => {
       return 10;
-    }
-  }
+    },
+  },
+
+  env: {
+    SKYNET_SLACK_TOKEN: SENSITIVE_VALUE,
+  },
+
+  build: {
+    func: build,
+    batchSize: 5,
+    concurrency: 2, // default is 1, no concurrency
+
+    schedule: "@minutely",
+    cpu: 800,
+    mem: 400,
+  },
+
+  validate: {
+    func: validate,
+    batchSize: 2,
+    concurrency: 2, // default is 1, no concurrency
+
+    schedule: "@hourly",
+    cpu: 600,
+    mem: 200,
+  },
+
+  check: {
+    func: check,
+    schedule: "@minutely",
+    slackChannel: "skynet-notifications-local-dev",
+  },
 });
 
-run();
+app();

package/examples/producer

@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+
+// this file is executable and is for kafka producer testing
+// a few test commands to try on
+//   $ examples/producer run --protocol bsc --verbose
+//   $ examples/producer run --protocol eth
+//   $ examples/producer check --protocol eth
+//   $ examples/producer deploy --protocol eth
+//   $ examples/producer --help
+
+const { producer, SENSITIVE_VALUE, ERROR_LEVEL } = require("../app");
+
+async function produce({ protocol, from, to, verbose, send }) {
+  console.log("produce called with", protocol, from, to, verbose);
+}
+
+async function check({ protocol, state, verbose }) {
+  console.log("check called with", protocol, state, verbose);
+
+  const errors = [];
+
+  errors.push({ type: ERROR_LEVEL.CRITICAL, message: "producer error" });
+
+  return errors;
+}
+
+const app = producer({
+  name: "LibSkynetExampleProducer",
+  selector: { protocol: { type: "string" } },
+
+  state: {
+    type: "block", // can be omitted, default is block
+    updateInterval: ({ protocol }) => (protocol === "bsc" ? 3000 : 20000), // how often max id is increasing, will determine poll interval, default to 5000ms
+    getMinId: async () => 4, // default returns 1
+    getMaxId: async ({ protocol }) => {
+      console.log("getMaxId called", protocol);
+
+      return 100;
+    },
+  },
+
+  env: {
+    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
+    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
+    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
+  },
+
+  produce: {
+    func: produce,
+
+    topic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dev`,
+    deadLetterTopic: ({ protocol }) => `lib-skynet-test-kafka-${protocol}-dead-letter-dev`, // problematic ids will be sent to dead letter topic for later retry
+
+    batchSize: 10, // default 50
+    maxRetry: 1, // default 2
+
+    cpu: 600,
+    mem: 200,
+  },
+
+  check: {
+    func: check,
+    schedule: "@minutely",
+    slackChannel: "skynet-notifications-local-dev",
+  },
+});
+
+app();

package/indexer.js

@@ -1,10 +1,10 @@
-const path = require("path");
 const meow = require("meow");
 const { createRecord, getRecordByKey } = require("./dynamodb");
 const { getEnvironment } = require("./env");
 const { exponentialRetry } = require("./availability");
 const { range, fillRange } = require("./util");
 const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { getBinaryName } = require("./cli");
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -49,6 +49,7 @@ async function getIndexerValidatedId(name, selectorFlags) {
 // managing state would be helpful to reduce the build time
 // and avoid unnecessary computation & storage
 function createModeIndexerApp({
+  binaryName,
   name,
   selector = {},
   build,
@@ -447,8 +448,9 @@ function createModeIndexerApp({
   }
 
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `
@@ -506,10 +508,11 @@ ${getSelectorDesc(selector)}
 // for those indexers that does not rely on a curosr
 // e.g. should always rebuild everything from scratch
 // or that the state can be easily inferred from existing data
-function createIndexerApp({ name, selector = {}, build, maxRetry = 2 }) {
+function createIndexerApp({ binaryName, name, selector = {}, build, maxRetry = 2 }) {
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `
@@ -565,7 +568,7 @@ ${getSelectorDesc(selector)}
         value: new Date().toISOString(),
       });
 
-      console.log(`[INDEXER] reset successfully in ${Date.now() - startTime}ms`);
+      console.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
     }
 
     runBuild(cli.flags).catch((err) => {

package/kafka.js

@@ -1,4 +1,3 @@
-const path = require("path");
 const meow = require("meow");
 const { getEnvironment, getEnvOrThrow } = require("./env");
 const { wait } = require("./availability");
@@ -6,6 +5,7 @@ const { Kafka, logLevel } = require("kafkajs");
 const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
 const { exponentialRetry } = require("./availability");
+const { getBinaryName } = require("./cli");
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -137,10 +137,11 @@ async function consumeMessages(consumerId, topic, callback) {
   return stopConsumeMessages;
 }
 
-function createProducerApp({ name, selector = {}, producer, state }) {
+function createProducerApp({ binaryName, name, selector = {}, producer, state }) {
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const finalState = {
       type: "block",
@@ -337,10 +338,11 @@ ${getSelectorDesc(selector)}
   return { run };
 }
 
-function createConsumerApp({ name, selector = {}, consumer }) {
+function createConsumerApp({ binaryName, name, selector = {}, consumer }) {
   function run() {
-    const binaryNameParts = process.argv[1].split(path.sep);
-    const binaryName = binaryNameParts[binaryNameParts.length - 1];
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
 
     const cli = meow(
       `