@certik/skynet 0.10.7 → 0.10.10

package/indexer.js

@@ -1,595 +1,596 @@
-const meow = require("meow");
-const { createRecord, getRecordByKey } = require("./dynamodb");
-const { getEnvironment } = require("./env");
-const { exponentialRetry } = require("./availability");
-const { range, fillRange } = require("./util");
-const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
-const { getBinaryName } = require("./cli");
-
-const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
-
-async function getIndexerState(name, selectorFlags) {
-  const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
-    name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-  });
-
-  if (!stateItem) {
-    return "N/A";
-  }
-
-  return stateItem.value;
-}
-
-async function getIndexerLatestId(name, selectorFlags) {
-  const record = await getRecordByKey(STATE_TABLE_NAME, {
-    name: `${name}Since(${toSelectorString(selectorFlags)})`,
-  });
-
-  if (record) {
-    return record.value;
-  }
-
-  return 0;
-}
-
-async function getIndexerValidatedId(name, selectorFlags) {
-  const record = await getRecordByKey(STATE_TABLE_NAME, {
-    name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-  });
-
-  if (record) {
-    return record.value;
-  }
-
-  return 0;
-}
-
-// for those indexers that can have progress tracked by a numeric state.type
-// such as block height, or timestamp
-// managing state would be helpful to reduce the build time
-// and avoid unnecessary computation & storage
-function createModeIndexerApp({
-  binaryName,
-  name,
-  selector = {},
-  build,
-  // number of items run in a batch, determines the { from, to } to the build function
-  buildBatchSize = 1,
-  // number of build functions calling at a time
-  buildConcurrency = 1,
-  // commit updates every rolling window = buildBatchSize * buildConcurrency
-  validate,
-  // number of items run in a batch, determines the { from, to } to the validate function
-  validateBatchSize = 1,
-  // number of validate functions calling at a time
-  validateConcurrency = 1,
-  // commit updates every rolling window = validateBatchSize * validateConcurrency
-  maxRetry = 2,
-  state,
-}) {
-  const finalState = {
-    type: "block",
-    getMinId: async () => 1,
-    getMaxId: async () => {
-      throw new Error("must implement getMaxId");
-    },
-    ...state,
-  };
-
-  async function runMode({ mode, from, to, status, verbose, ...selectorFlags }) {
-    if (status) {
-      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
-        name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-      });
-
-      const fromItem = await getRecordByKey(STATE_TABLE_NAME, {
-        name: `${name}Since(${toSelectorString(selectorFlags)})`,
-      });
-
-      const validateItem = await getRecordByKey(STATE_TABLE_NAME, {
-        name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-      });
-
-      console.log(`RebuildState=${stateItem?.value} Since=${fromItem?.value} Validated=${validateItem?.value}`);
-
-      process.exit(0);
-    }
-
-    if (from === 0) {
-      from = await finalState.getMinId(selectorFlags);
-    }
-
-    console.log(
-      `[MODE INDEXER] mode=${mode}, from=${from}, to=${
-        to > 0 ? to : "undefined"
-      }, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`
-    );
-
-    if (mode === "reset") {
-      await runReset(selectorFlags, from);
-    } else if (mode === "rebuild") {
-      await runReset(selectorFlags);
-      await runRebuild(selectorFlags, from, to > 0 ? to : await finalState.getMaxId(selectorFlags), verbose);
-    } else if (mode === "resume-rebuild") {
-      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
-
-      await runRebuild(
-        selectorFlags,
-        previousRebuildEnds ? previousRebuildEnds + 1 : from,
-        to > 0 ? to : await finalState.getMaxId(selectorFlags),
-        verbose
-      );
-    } else if (mode === "validate" || mode === "validation") {
-      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
-
-      const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
-
-      const shouldSaveState = to === 0;
-
-      await runValidate(
-        selectorFlags,
-        from > 0 ? from : previousValidatedTo,
-        to > 0 ? to : previousRebuildEnds,
-        shouldSaveState,
-        verbose
-      );
-    } else if (mode === "one") {
-      if (to > 0) {
-        console.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
-      }
-      await runRange(selectorFlags, from, from, verbose);
-    } else if (mode === "range") {
-      await runRange(selectorFlags, from, to, verbose);
-    } else {
-      await runDelta(selectorFlags, verbose);
-    }
-  }
-
-  async function runRange(selectorFlags, from, to, verbose) {
-    const startTime = Date.now();
-
-    console.log(
-      `[MODE INDEXER] building range, from=${from}, to=${to}, ${toSelectorString(
-        selectorFlags,
-        ", "
-      )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
-    );
-
-    const failedIds = await execBuild(selectorFlags, from, to, verbose, false);
-
-    if (failedIds.length > 0) {
-      console.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
-      process.exit(1);
-    } else {
-      console.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
-      process.exit(0);
-    }
-  }
-
-  async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
-    if (!validate) {
-      console.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
-      process.exit(1);
-    }
-
-    const startTime = Date.now();
-
-    console.log(
-      `[MODE INDEXER] validating, from=${from}, to=${to}, ${toSelectorString(
-        selectorFlags,
-        ", "
-      )}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`
-    );
-
-    const windows = range(from, to, validateBatchSize * validateConcurrency);
-
-    console.log(
-      `[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`
-    );
-
-    for (let [windowStart, windowEnd] of windows) {
-      console.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
-
-      const batches = range(windowStart, windowEnd, validateBatchSize);
-
-      // add a retry for errors
-      await Promise.all(
-        batches.map(async ([batchStart, batchEnd]) => {
-          const result = await exponentialRetry(
-            async () => {
-              try {
-                await validate({
-                  ...selectorFlags,
-                  from: batchStart,
-                  to: batchEnd,
-                  verbose,
-                });
-
-                return true;
-              } catch (err) {
-                console.error(`got error in validation`, err);
-
-                return false;
-              }
-            },
-            {
-              maxRetry,
-              test: (r) => r,
-              verbose,
-            }
-          );
-
-          if (!result) {
-            throw new Error(`Terminate validation due to critical errors, from=${batchStart}, to=${batchEnd}`);
-          }
-        })
-      );
-
-      if (shouldSaveState) {
-        await createRecord(STATE_TABLE_NAME, {
-          name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-          value: to,
-        });
-
-        if (verbose) {
-          console.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
-        }
-      }
-    }
-
-    console.log(
-      `[MODE INDEXER] validated ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
-    );
-  }
-
-  async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
-    let failedIds = [];
-
-    const windows = range(from, to, buildBatchSize * buildConcurrency);
-
-    for (let [windowStart, windowEnd] of windows) {
-      console.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
-
-      const batches = range(windowStart, windowEnd, buildBatchSize);
-
-      // add a retry for errors
-      const batchResults = await Promise.all(
-        batches.map(async ([batchStart, batchEnd]) => {
-          await exponentialRetry(
-            async () => {
-              try {
-                const ids = await build({
-                  ...selectorFlags,
-                  from: batchStart,
-                  to: batchEnd,
-                  verbose,
-                });
-
-                if (ids && ids.length > 0) {
-                  return ids;
-                } else {
-                  return false;
-                }
-              } catch (err) {
-                console.error(`[MODE INDEXER] got error in build`, err);
-
-                return fillRange(batchStart, batchEnd);
-              }
-            },
-            {
-              maxRetry,
-              test: (r) => !r,
-              verbose,
-            }
-          );
-        })
-      );
-
-      if (shouldSaveState) {
-        await createRecord(STATE_TABLE_NAME, {
-          name: `${name}Since(${toSelectorString(selectorFlags)})`,
-          value: windowEnd,
-        });
-
-        if (verbose) {
-          console.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
-        }
-      }
-
-      batchResults.forEach((ids) => {
-        if (ids) {
-          failedIds = failedIds.concat(ids);
-        }
-      });
-    }
-
-    failedIds.sort();
-
-    return failedIds;
-  }
-
-  async function runRebuild(selectorFlags, from, to, verbose) {
-    const startTime = Date.now();
-
-    console.log(
-      `[MODE INDEXER] rebuilding, from=${from}, to=${to}, ${toSelectorString(
-        selectorFlags,
-        ", "
-      )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
-    );
-
-    // add a flag to stop delta from running
-    await createRecord(STATE_TABLE_NAME, {
-      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-      value: "running",
-    });
-
-    const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
-
-    // even if some transactions are failed we should continue to allow delta job running
-    // because rebuild is usally heavy
-    await createRecord(STATE_TABLE_NAME, {
-      name: `${name}Since(${toSelectorString(selectorFlags)})`,
-      value: to,
-    });
-
-    await createRecord(STATE_TABLE_NAME, {
-      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-      value: "succeed",
-    });
-
-    if (failedIds.length > 0) {
-      console.log(
-        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} with some failed ${finalState.type}`,
-        failedIds
-      );
-      process.exit(1);
-    } else {
-      console.log(
-        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
-      );
-      process.exit(0);
-    }
-  }
-
-  async function runDelta(selectorFlags, verbose) {
-    const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
-      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-    });
-
-    // only build when rebuild succeed
-    if (stateItem && stateItem.value === "succeed") {
-      const from = (await getIndexerLatestId(name, selectorFlags)) + 1;
-
-      const startTime = Date.now();
-      const to = await state.getMaxId(selectorFlags);
-
-      if (to <= from) {
-        console.log(
-          `[MODE INDEXER] skip delta because there're no items need to be processed, ${toSelectorString(
-            selectorFlags,
-            ", "
-          )}`
-        );
-
-        return;
-      }
-
-      console.log(
-        `[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(
-          selectorFlags,
-          ", "
-        )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
-      );
-
-      try {
-        const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
-
-        if (failedIds.length > 0) {
-          console.log("[MODE INDEXER] built with some failed txs", failedIds);
-
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
-            value: "failed",
-          });
-
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}Since(${toSelectorString(selectorFlags)})`,
-            value: Math.min(to, failedIds[0]),
-          });
-
-          process.exit(1);
-        } else {
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
-            value: "succeed",
-          });
-
-          await createRecord(STATE_TABLE_NAME, {
-            name: `${name}Since(${toSelectorString(selectorFlags)})`,
-            value: to,
-          });
-
-          console.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
-          process.exit(0);
-        }
-      } catch (err) {
-        console.error("[MODE INDEXER] delta build failed", from, to, err);
-
-        process.exit(1);
-      }
-    } else {
-      console.log("[MODE INDEXER] skip because rebuild hasn't done yet");
-    }
-  }
-
-  async function runReset(selectorFlags) {
-    const startTime = Date.now();
-
-    console.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
-
-    console.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
-    await createRecord(STATE_TABLE_NAME, {
-      name: `${name}Since(${toSelectorString(selectorFlags)})`,
-      value: 0,
-    });
-
-    await createRecord(STATE_TABLE_NAME, {
-      name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-      value: 0,
-    });
-
-    await createRecord(STATE_TABLE_NAME, {
-      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-      value: "init",
-    });
-
-    console.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
-  }
-
-  function run() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const cli = meow(
-      `
-        Usage
-
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset
-          --from          min ${finalState.type} to build
-          --to            max ${finalState.type} to build
-          --status        print status of indexer and exit
-          --verbose       Output debug messages
-      `,
-      {
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          mode: {
-            type: "string",
-            default: "delta",
-          },
-          from: {
-            alias: "since",
-            type: "number",
-            default: 0,
-          },
-          to: {
-            alias: "until",
-            type: "number",
-            default: 0,
-          },
-          status: {
-            type: "boolean",
-            default: false,
-          },
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    runMode(cli.flags).catch((err) => {
-      console.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { run };
-}
-
-// for those indexers that does not rely on a curosr
-// e.g. should always rebuild everything from scratch
-// or that the state can be easily inferred from existing data
-function createIndexerApp({ binaryName, name, selector = {}, build, maxRetry = 2 }) {
-  function run() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const cli = meow(
-      `
-        Usage
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --verbose       Output debug messages
-      `,
-      {
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    async function runBuild({ verbose, ...selectorFlags }) {
-      const startTime = Date.now();
-
-      if (Object.keys(selectorFlags).length > 0) {
-        console.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
-      } else {
-        console.log(`[INDEXER] starting build`);
-      }
-
-      const result = await exponentialRetry(
-        async () => {
-          try {
-            await build({ verbose, ...selectorFlags });
-
-            return true;
-          } catch (err) {
-            console.log(`[INDEXER] got error in build`, err);
-
-            return false;
-          }
-        },
-        {
-          maxRetry,
-          test: (r) => r,
-          verbose,
-        }
-      );
-
-      if (!result) {
-        throw new Error(`[INDEXER] Build failed due to critical errors`);
-      }
-
-      await createRecord(STATE_TABLE_NAME, {
-        name: `${name}UpdatedAt(${toSelectorString(selectorFlags)})`,
-        value: new Date().toISOString(),
-      });
-
-      console.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
-    }
-
-    runBuild(cli.flags).catch((err) => {
-      console.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { run };
-}
-
-module.exports = {
-  createModeIndexerApp,
-  createIndexerApp,
-  getIndexerState,
-  getIndexerLatestId,
-  getIndexerValidatedId,
-};
+const meow = require("meow");
+const { createRecord, getRecordByKey } = require("./dynamodb");
+const { getEnvironment } = require("./env");
+const { exponentialRetry } = require("./availability");
+const { range, fillRange } = require("./util");
+const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { getBinaryName } = require("./cli");
+const { inline } = require("./log");
+
+const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
+
+async function getIndexerState(name, selectorFlags) {
+  const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+  });
+
+  if (!stateItem) {
+    return "N/A";
+  }
+
+  return stateItem.value;
+}
+
+async function getIndexerLatestId(name, selectorFlags) {
+  const record = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}Since(${toSelectorString(selectorFlags)})`,
+  });
+
+  if (record) {
+    return record.value;
+  }
+
+  return 0;
+}
+
+async function getIndexerValidatedId(name, selectorFlags) {
+  const record = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+  });
+
+  if (record) {
+    return record.value;
+  }
+
+  return 0;
+}
+
+// for those indexers that can have progress tracked by a numeric state.type
+// such as block height, or timestamp
+// managing state would be helpful to reduce the build time
+// and avoid unnecessary computation & storage
+function createModeIndexerApp({
+  binaryName,
+  name,
+  selector = {},
+  build,
+  // number of items run in a batch, determines the { from, to } to the build function
+  buildBatchSize = 1,
+  // number of build functions calling at a time
+  buildConcurrency = 1,
+  // commit updates every rolling window = buildBatchSize * buildConcurrency
+  validate,
+  // number of items run in a batch, determines the { from, to } to the validate function
+  validateBatchSize = 1,
+  // number of validate functions calling at a time
+  validateConcurrency = 1,
+  // commit updates every rolling window = validateBatchSize * validateConcurrency
+  maxRetry = 2,
+  state,
+}) {
+  const finalState = {
+    type: "block",
+    getMinId: async () => 1,
+    getMaxId: async () => {
+      throw new Error("must implement getMaxId");
+    },
+    ...state,
+  };
+
+  async function runMode({ mode, from, to, status, verbose, ...selectorFlags }) {
+    if (status) {
+      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      });
+
+      const fromItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}Since(${toSelectorString(selectorFlags)})`,
+      });
+
+      const validateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+      });
+
+      inline.log(`RebuildState=${stateItem?.value} Since=${fromItem?.value} Validated=${validateItem?.value}`);
+
+      process.exit(0);
+    }
+
+    if (from === 0) {
+      from = await finalState.getMinId(selectorFlags);
+    }
+
+    inline.log(
+      `[MODE INDEXER] mode=${mode}, from=${from}, to=${
+        to > 0 ? to : "undefined"
+      }, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`
+    );
+
+    if (mode === "reset") {
+      await runReset(selectorFlags, from);
+    } else if (mode === "rebuild") {
+      await runReset(selectorFlags);
+      await runRebuild(selectorFlags, from, to > 0 ? to : await finalState.getMaxId(selectorFlags), verbose);
+    } else if (mode === "resume-rebuild") {
+      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+
+      await runRebuild(
+        selectorFlags,
+        previousRebuildEnds ? previousRebuildEnds + 1 : from,
+        to > 0 ? to : await finalState.getMaxId(selectorFlags),
+        verbose
+      );
+    } else if (mode === "validate" || mode === "validation") {
+      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+
+      const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
+
+      const shouldSaveState = to === 0;
+
+      await runValidate(
+        selectorFlags,
+        from > 0 ? from : previousValidatedTo,
+        to > 0 ? to : previousRebuildEnds,
+        shouldSaveState,
+        verbose
+      );
+    } else if (mode === "one") {
+      if (to > 0) {
+        inline.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
+      }
+      await runRange(selectorFlags, from, from, verbose);
+    } else if (mode === "range") {
+      await runRange(selectorFlags, from, to, verbose);
+    } else {
+      await runDelta(selectorFlags, verbose);
+    }
+  }
+
+  async function runRange(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+
+    inline.log(
+      `[MODE INDEXER] building range, from=${from}, to=${to}, ${toSelectorString(
+        selectorFlags,
+        ", "
+      )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+    );
+
+    const failedIds = await execBuild(selectorFlags, from, to, verbose, false);
+
+    if (failedIds.length > 0) {
+      inline.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
+      process.exit(1);
+    } else {
+      inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+      process.exit(0);
+    }
+  }
+
+  async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
+    if (!validate) {
+      inline.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
+      process.exit(1);
+    }
+
+    const startTime = Date.now();
+
+    inline.log(
+      `[MODE INDEXER] validating, from=${from}, to=${to}, ${toSelectorString(
+        selectorFlags,
+        ", "
+      )}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`
+    );
+
+    const windows = range(from, to, validateBatchSize * validateConcurrency);
+
+    inline.log(
+      `[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`
+    );
+
+    for (let [windowStart, windowEnd] of windows) {
+      inline.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
+
+      const batches = range(windowStart, windowEnd, validateBatchSize);
+
+      // add a retry for errors
+      await Promise.all(
+        batches.map(async ([batchStart, batchEnd]) => {
+          const result = await exponentialRetry(
+            async () => {
+              try {
+                await validate({
+                  ...selectorFlags,
+                  from: batchStart,
+                  to: batchEnd,
+                  verbose,
+                });
+
+                return true;
+              } catch (err) {
+                inline.error(`got error in validation`, err);
+
+                return false;
+              }
+            },
+            {
+              maxRetry,
+              test: (r) => r,
+              verbose,
+            }
+          );
+
+          if (!result) {
+            throw new Error(`Terminate validation due to critical errors, from=${batchStart}, to=${batchEnd}`);
+          }
+        })
+      );
+
+      if (shouldSaveState) {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+          value: to,
+        });
+
+        if (verbose) {
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+        }
+      }
+    }
+
+    inline.log(
+      `[MODE INDEXER] validated ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
+    );
+  }
+
+  async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
+    let failedIds = [];
+
+    const windows = range(from, to, buildBatchSize * buildConcurrency);
+
+    for (let [windowStart, windowEnd] of windows) {
+      inline.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
+
+      const batches = range(windowStart, windowEnd, buildBatchSize);
+
+      // add a retry for errors
+      const batchResults = await Promise.all(
+        batches.map(async ([batchStart, batchEnd]) => {
+          await exponentialRetry(
+            async () => {
+              try {
+                const ids = await build({
+                  ...selectorFlags,
+                  from: batchStart,
+                  to: batchEnd,
+                  verbose,
+                });
+
+                if (ids && ids.length > 0) {
+                  return ids;
+                } else {
+                  return false;
+                }
+              } catch (err) {
+                inline.error(`[MODE INDEXER] got error in build`, err);
+
+                return fillRange(batchStart, batchEnd);
+              }
+            },
+            {
+              maxRetry,
+              test: (r) => !r,
+              verbose,
+            }
+          );
+        })
+      );
+
+      if (shouldSaveState) {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: windowEnd,
+        });
+
+        if (verbose) {
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+        }
+      }
+
+      batchResults.forEach((ids) => {
+        if (ids) {
+          failedIds = failedIds.concat(ids);
+        }
+      });
+    }
+
+    failedIds.sort();
+
+    return failedIds;
+  }
+
+  async function runRebuild(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+
+    inline.log(
+      `[MODE INDEXER] rebuilding, from=${from}, to=${to}, ${toSelectorString(
+        selectorFlags,
+        ", "
+      )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+    );
+
+    // add a flag to stop delta from running
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "running",
+    });
+
+    const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
+
+    // even if some transactions are failed we should continue to allow delta job running
+    // because rebuild is usally heavy
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Since(${toSelectorString(selectorFlags)})`,
+      value: to,
+    });
+
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "succeed",
+    });
+
+    if (failedIds.length > 0) {
+      inline.log(
+        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} with some failed ${finalState.type}`,
+        failedIds
+      );
+      process.exit(1);
+    } else {
+      inline.log(
+        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
+      );
+      process.exit(0);
+    }
+  }
+
+  async function runDelta(selectorFlags, verbose) {
+    const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+    });
+
+    // only build when rebuild succeed
+    if (stateItem && stateItem.value === "succeed") {
+      const from = (await getIndexerLatestId(name, selectorFlags)) + 1;
+
+      const startTime = Date.now();
+      const to = await state.getMaxId(selectorFlags);
+
+      if (to <= from) {
+        inline.log(
+          `[MODE INDEXER] skip delta because there're no items need to be processed, ${toSelectorString(
+            selectorFlags,
+            ", "
+          )}`
+        );
+
+        return;
+      }
+
+      inline.log(
+        `[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(
+          selectorFlags,
+          ", "
+        )}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`
+      );
+
+      try {
+        const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
+
+        if (failedIds.length > 0) {
+          inline.log("[MODE INDEXER] built with some failed txs", failedIds);
+
+          await createRecord(STATE_TABLE_NAME, {
+            name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+            value: "failed",
+          });
+
+          await createRecord(STATE_TABLE_NAME, {
+            name: `${name}Since(${toSelectorString(selectorFlags)})`,
+            value: Math.min(to, failedIds[0]),
+          });
+
+          process.exit(1);
+        } else {
+          await createRecord(STATE_TABLE_NAME, {
+            name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+            value: "succeed",
+          });
+
+          await createRecord(STATE_TABLE_NAME, {
+            name: `${name}Since(${toSelectorString(selectorFlags)})`,
+            value: to,
+          });
+
+          inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+          process.exit(0);
+        }
+      } catch (err) {
+        inline.error("[MODE INDEXER] delta build failed", from, to, err);
+
+        process.exit(1);
+      }
+    } else {
+      inline.log("[MODE INDEXER] skip because rebuild hasn't done yet");
+    }
+  }
+
+  async function runReset(selectorFlags) {
+    const startTime = Date.now();
+
+    inline.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
+
+    inline.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Since(${toSelectorString(selectorFlags)})`,
+      value: 0,
+    });
+
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+      value: 0,
+    });
+
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "init",
+    });
+
+    inline.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
+  }
+
+  function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const cli = meow(
+      `
+        Usage
+
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset
+          --from          min ${finalState.type} to build
+          --to            max ${finalState.type} to build
+          --status        print status of indexer and exit
+          --verbose       Output debug messages
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          mode: {
+            type: "string",
+            default: "delta",
+          },
+          from: {
+            alias: "since",
+            type: "number",
+            default: 0,
+          },
+          to: {
+            alias: "until",
+            type: "number",
+            default: 0,
+          },
+          status: {
+            type: "boolean",
+            default: false,
+          },
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    runMode(cli.flags).catch((err) => {
+      inline.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { run };
+}
+
+// for those indexers that does not rely on a curosr
+// e.g. should always rebuild everything from scratch
+// or that the state can be easily inferred from existing data
+function createIndexerApp({ binaryName, name, selector = {}, build, maxRetry = 2 }) {
+  function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const cli = meow(
+      `
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --verbose       Output debug messages
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    async function runBuild({ verbose, ...selectorFlags }) {
+      const startTime = Date.now();
+
+      if (Object.keys(selectorFlags).length > 0) {
+        inline.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
+      } else {
+        inline.log(`[INDEXER] starting build`);
+      }
+
+      const result = await exponentialRetry(
+        async () => {
+          try {
+            await build({ verbose, ...selectorFlags });
+
+            return true;
+          } catch (err) {
+            inline.log(`[INDEXER] got error in build`, err);
+
+            return false;
+          }
+        },
+        {
+          maxRetry,
+          test: (r) => r,
+          verbose,
+        }
+      );
+
+      if (!result) {
+        throw new Error(`[INDEXER] Build failed due to critical errors`);
+      }
+
+      await createRecord(STATE_TABLE_NAME, {
+        name: `${name}UpdatedAt(${toSelectorString(selectorFlags)})`,
+        value: new Date().toISOString(),
+      });
+
+      inline.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
+    }
+
+    runBuild(cli.flags).catch((err) => {
+      inline.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { run };
+}
+
+module.exports = {
+  createModeIndexerApp,
+  createIndexerApp,
+  getIndexerState,
+  getIndexerLatestId,
+  getIndexerValidatedId,
+};