@certik/skynet 0.22.1 → 0.22.3

package/dist/indexer.js

@@ -1,381 +1,987 @@
-import meow from "meow";
-import { createRecord, getRecordByKey } from "./dynamodb";
-import { getEnvironment } from "./env";
-import { exponentialRetry } from "./availability";
-import { range as numberRange, fillRange as fillNumberRange } from "./util";
-import { getSelectorDesc, getSelectorFlags, toSelectorString } from "./selector";
-import { getBinaryName } from "./cli";
-import { inline } from "./log";
-import { findDateAfter, dateRange, daysInRange as fillDateRange } from "./date";
-const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
-async function getIndexerLatestId(name, selectorFlags) {
-    const record = await getRecordByKey(STATE_TABLE_NAME, {
-        name: `${name}Since(${toSelectorString(selectorFlags)})`,
+// src/env.ts
+function ensureAndGet(envName, defaultValue) {
+  return process.env[envName] || defaultValue;
+}
+function getEnvironment() {
+  return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
+}
+function getEnvOrThrow(envName) {
+  if (!process.env[envName]) {
+    throw new Error(`Must set environment variable ${envName}`);
+  }
+  return process.env[envName];
+}
+function isProduction() {
+  return getEnvironment() === "prd";
+}
+function isDev() {
+  return getEnvironment() === "dev";
+}
+// src/util.ts
+function range(startAt, endAt, step) {
+  const arr = [];
+  for (let i = startAt;i <= endAt; i += step) {
+    arr.push([i, Math.min(endAt, i + step - 1)]);
+  }
+  return arr;
+}
+function arrayGroup(array, groupSize) {
+  const groups = [];
+  for (let i = 0;i < array.length; i += groupSize) {
+    groups.push(array.slice(i, i + groupSize));
+  }
+  return groups;
+}
+function fillRange(start, end) {
+  const result = [];
+  for (let i = start;i <= end; i++) {
+    result.push(i);
+  }
+  return result;
+}
+// src/date.ts
+var MS_IN_A_DAY = 3600 * 24 * 1000;
+function getDateOnly(date) {
+  return new Date(date).toISOString().split("T")[0];
+}
+function findDateAfter(date, n) {
+  const d = new Date(date);
+  const after = new Date(d.getTime() + MS_IN_A_DAY * n);
+  return getDateOnly(after);
+}
+function daysInRange(from, to) {
+  const fromTime = new Date(from).getTime();
+  const toTime = new Date(to).getTime();
+  if (fromTime > toTime) {
+    throw new Error(`range to date couldn't be earlier than range from date`);
+  }
+  const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
+  const dates = [getDateOnly(new Date(fromTime))];
+  for (let i = 1;i <= daysBetween; i += 1) {
+    dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
+  }
+  return dates;
+}
+function dateRange(from, to, step) {
+  const days = daysInRange(from, to);
+  const windows = arrayGroup(days, step);
+  return windows.map((w) => [w[0], w[w.length - 1]]);
+}
+// src/object-hash.ts
+import xh from "@node-rs/xxhash";
+function getHash(obj) {
+  const xxh3 = xh.xxh3.Xxh3.withSeed();
+  hash(obj, xxh3);
+  return xxh3.digest().toString(16);
+}
+function hash(obj, xxh3) {
+  if (obj === null) {
+    xxh3.update("null");
+  } else if (obj === undefined) {
+    xxh3.update("undefined");
+  } else if (typeof obj === "string") {
+    xxh3.update(obj);
+  } else if (typeof obj === "number") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "boolean") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "bigint") {
+    xxh3.update(obj.toString());
+  } else if (obj instanceof Date) {
+    xxh3.update(obj.toISOString());
+  } else if (Array.isArray(obj)) {
+    arrayHash(obj, xxh3);
+  } else if (obj instanceof Set) {
+    setHash(obj, xxh3);
+  } else if (obj instanceof Map) {
+    mapHash(obj, xxh3);
+  } else if (typeof obj === "object") {
+    objectHash(obj, xxh3);
+  } else {
+    throw new Error(`Unsupported type: ${obj}`);
+  }
+}
+function arrayHash(array, xxh3) {
+  xxh3.update("[");
+  for (const obj of array) {
+    hash(obj, xxh3);
+  }
+  xxh3.update("]");
+}
+function setHash(_set, _xxh3) {
+  throw new Error("Set hashing not implemented");
+}
+function mapHash(map, xxh3) {
+  const array = Array.from(map.entries()).sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}
+function objectHash(obj, xxh3) {
+  const array = Object.entries(obj).sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}
+
+// src/availability.ts
+import pThrottle from "p-throttle";
+import pMemoize from "p-memoize";
+import QuickLRU from "quick-lru";
+async function wait(time) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, time);
+  });
+}
+async function exponentialRetry(func, {
+  maxRetry,
+  initialDuration,
+  growFactor,
+  test,
+  verbose
+}) {
+  let retries = maxRetry;
+  let duration = initialDuration || 5000;
+  const growFactorFinal = growFactor || 2;
+  let result = await func();
+  while (!test(result) && retries > 0) {
+    if (verbose) {
+      console.log("failed attempt result", result);
+      console.log(`sleep for ${duration}ms after failed attempt, remaining ${retries} attempts`);
+    }
+    retries = retries - 1;
+    await wait(duration);
+    result = await func();
+    duration = duration * growFactorFinal;
+  }
+  if (verbose) {
+    console.log(`function to retry ends with status ${test(result)}, number of retries done: ${maxRetry - retries}}`);
+  }
+  return result;
+}
+function withRetry(func, options) {
+  let retries = options?.maxRetry || 3;
+  let duration = options?.initialDuration || 500;
+  const growFactorFinal = options?.growFactor || 2;
+  return async (...args) => {
+    do {
+      try {
+        return await func(...args);
+      } catch (error) {
+        retries = retries - 1;
+        if (retries <= 0) {
+          throw error;
+        }
+        await wait(duration);
+        duration = duration * growFactorFinal;
+      }
+    } while (retries > 0);
+    throw new Error("unreachable");
+  };
+}
+function memoize(func, options) {
+  if (!options) {
+    options = {};
+  }
+  if (!options.cache) {
+    options.cache = new QuickLRU({ maxSize: options.lruMaxSize || 1e4 });
+  }
+  if (!options.cacheKey) {
+    options.cacheKey = (args) => getHash(args);
+  }
+  return pMemoize(func, options);
+}
+// src/log.ts
+function isObject(a) {
+  return !!a && a.constructor === Object;
+}
+function print(o) {
+  if (Array.isArray(o)) {
+    return `[${o.map(print).join(", ")}]`;
+  }
+  if (isObject(o)) {
+    return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
+  }
+  return `${o}`;
+}
+function getLine(params) {
+  let line = "";
+  for (let i = 0, l = params.length;i < l; i++) {
+    line += `${print(params[i])} `.replace(/\n/gm, "\t");
+  }
+  return line.trim();
+}
+function timestamp() {
+  return new Date().toISOString();
+}
+var inline = {
+  debug: function(...args) {
+    if (true) {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
+  },
+  log: function(...args) {
+    if (true) {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
+  },
+  error: function(...args) {
+    if (true) {
+      console.error(`${timestamp()} ${getLine(args)}`);
+    }
+  }
+};
+var logger = {
+  debug: function(...args) {
+    if (true) {
+      console.log(`[${timestamp()}]`, ...args);
+    }
+  },
+  log: function(...args) {
+    if (true) {
+      console.log(`[${timestamp()}]`, ...args);
+    }
+  },
+  error: function(...args) {
+    if (true) {
+      console.error(`[${timestamp()}]`, ...args);
+    }
+  }
+};
+// src/dynamodb.ts
+import {
+  DynamoDBDocumentClient,
+  ScanCommand,
+  BatchWriteCommand,
+  GetCommand,
+  PutCommand,
+  QueryCommand,
+  UpdateCommand
+} from "@aws-sdk/lib-dynamodb";
+import { DynamoDBClient, DescribeTableCommand } from "@aws-sdk/client-dynamodb";
+var _dynamoDB;
+var _docClient;
+function getDynamoDB(forceNew = false) {
+  if (!_dynamoDB || forceNew) {
+    _dynamoDB = new DynamoDBClient;
+  }
+  return _dynamoDB;
+}
+function getDocClient(forceNew = false) {
+  const marshallOptions = {
+    convertEmptyValues: true,
+    removeUndefinedValues: true,
+    convertClassInstanceToMap: true
+  };
+  const unmarshallOptions = {
+    wrapNumbers: false
+  };
+  if (!_docClient || forceNew) {
+    _docClient = DynamoDBDocumentClient.from(getDynamoDB(), {
+      marshallOptions,
+      unmarshallOptions
     });
-    return record?.value;
+  }
+  return _docClient;
 }
-async function getIndexerValidatedId(name, selectorFlags) {
-    const record = await getRecordByKey(STATE_TABLE_NAME, {
-        name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+async function scanWholeTable(options) {
+  const dynamodb = getDocClient();
+  let items = [];
+  let count = 0;
+  let scannedCount = 0;
+  let data = await dynamodb.send(new ScanCommand(options));
+  while (data.LastEvaluatedKey) {
+    if (data.Items) {
+      items = items.concat(data.Items);
+    }
+    count += data.Count || 0;
+    scannedCount += data.ScannedCount || 0;
+    data = await dynamodb.send(new ScanCommand({ ...options, ExclusiveStartKey: data.LastEvaluatedKey }));
+  }
+  if (data.Items) {
+    items = items.concat(data.Items);
+  }
+  count += data.Count || 0;
+  scannedCount += data.ScannedCount || 0;
+  return {
+    Items: items,
+    Count: count,
+    ScannedCount: scannedCount
+  };
+}
+async function batchCreateRecords(tableName, records, maxWritingCapacity, verbose = false) {
+  if (verbose) {
+    console.log(`creating ${records.length} items in ${tableName}`);
+  }
+  const docClient = getDocClient();
+  let remainingItems = records;
+  let prevRemainingCount = remainingItems.length + 1;
+  let factor = 1;
+  let rejection = undefined;
+  while (remainingItems.length > 0 && factor <= 128 && !rejection) {
+    if (prevRemainingCount === remainingItems.length) {
+      await wait(5000 * factor);
+      factor = factor * 2;
+    }
+    if (factor >= 32) {
+      console.log(`WARNING: no progress for a long time for batchCreateRecords, please check`);
+    }
+    const slices = arrayGroup(remainingItems.slice(0, maxWritingCapacity), 25);
+    const results = await Promise.allSettled(slices.map((rs) => docClient.send(new BatchWriteCommand({
+      RequestItems: {
+        [tableName]: rs.map((record) => ({ PutRequest: { Item: record } }))
+      }
+    }))));
+    const isFulfilled = (p) => p.status === "fulfilled";
+    const isRejected = (p) => p.status === "rejected";
+    prevRemainingCount = remainingItems.length;
+    remainingItems = remainingItems.slice(maxWritingCapacity);
+    results.forEach((rs, idx) => {
+      if (isRejected(rs)) {
+        remainingItems = remainingItems.concat(slices[idx]);
+        rejection = rs;
+      } else if (isFulfilled(rs) && rs.value.UnprocessedItems && Object.keys(rs.value.UnprocessedItems).length > 0) {
+        const unprocessedItems = rs.value.UnprocessedItems[tableName].map((it) => it.PutRequest?.Item ?? []).flat();
+        remainingItems = remainingItems.concat(unprocessedItems);
+      }
     });
-    if (record) {
-        return record.value;
+    if (verbose) {
+      console.log(`processed=${prevRemainingCount - remainingItems.length}, remaining=${remainingItems.length}`);
     }
-    return undefined;
+  }
+  if (rejection) {
+    console.log("batchCreateRecords rejected", rejection);
+    throw new Error(`batchCreateRecords rejected, failed items=${remainingItems.length}`);
+  }
+  if (remainingItems.length > 0) {
+    console.log(`failed batchCreateRecords, failed items=${remainingItems.length}`);
+    throw new Error(`batchCreateRecords retry failed, failed items=${remainingItems.length}`);
+  }
 }
-function increaseId(type, currentId, n) {
-    if (type === "date") {
-        if (typeof currentId !== "string") {
-            throw new Error("invalid type for date id");
-        }
-        return findDateAfter(currentId, n);
-    }
-    if (typeof currentId !== "number") {
-        throw new Error("Invalid type for numeric id");
-    }
-    return (currentId + n);
-}
-// for those indexers that can have progress tracked by a numeric state.type
-// such as block height, or timestamp
-// managing state would be helpful to reduce the build time
-// and avoid unnecessary computation & storage
-function createModeIndexerApp({ binaryName, name, selector = {}, build, 
-// number of items run in a batch, determines the { from, to } to the build function
-buildBatchSize = 1, 
-// number of build functions calling at a time
-buildConcurrency = 1, 
-// commit updates every rolling window = buildBatchSize * buildConcurrency
-validate, 
-// number of items run in a batch, determines the { from, to } to the validate function
-validateBatchSize = 1, 
-// number of validate functions calling at a time
-validateConcurrency = 1, 
-// commit updates every rolling window = validateBatchSize * validateConcurrency
-maxRetry = 2, state, }) {
-    const defaultState = {
-        type: "block",
-        getMinId: async () => 1,
-        getMaxId: async () => {
-            throw new Error("must implement getMaxId");
-        },
+async function createRecord(tableName, fields, verbose = false) {
+  if (verbose) {
+    console.log("creating", tableName, fields);
+  }
+  const docClient = getDocClient();
+  const params = {
+    TableName: tableName,
+    Item: fields
+  };
+  return docClient.send(new PutCommand(params));
+}
+async function readRecord(tableName, key, verbose = false) {
+  if (verbose) {
+    console.log("reading", tableName, key);
+  }
+  const docClient = getDocClient();
+  const record = await docClient.send(new GetCommand({
+    TableName: tableName,
+    Key: key
+  }));
+  return record.Item;
+}
+async function getRecordsByKey(tableName, keys, indexName) {
+  const docClient = getDocClient();
+  const keyNames = Object.keys(keys);
+  const conditionExpression = keyNames.map((key) => `#${key} = :${key}`).join(" and ");
+  const params = {
+    TableName: tableName,
+    KeyConditionExpression: conditionExpression,
+    ExpressionAttributeNames: generateExpressionNames(keyNames),
+    ExpressionAttributeValues: generateExpressionValues(keyNames, keys)
+  };
+  if (indexName) {
+    params.IndexName = indexName;
+  }
+  try {
+    let data = await docClient.send(new QueryCommand(params));
+    let items = data.Items ?? [];
+    while (data.LastEvaluatedKey) {
+      data = await docClient.send(new QueryCommand({
+        ...params,
+        ExclusiveStartKey: data.LastEvaluatedKey
+      }));
+      if (data.Items) {
+        items = items.concat(data.Items);
+      }
+    }
+    return items;
+  } catch (err) {
+    console.log(err);
+    if (err instanceof Error && "statusCode" in err && err.statusCode === 400) {
+      return null;
+    }
+    throw err;
+  }
+}
+async function getRecordByKey(tableName, keys, indexName) {
+  if (indexName) {
+    const records = await getRecordsByKey(tableName, keys, indexName);
+    if (records) {
+      return records[0];
+    } else {
+      return null;
+    }
+  } else {
+    return readRecord(tableName, keys);
+  }
+}
+function generateExpressionNames(keys) {
+  return keys.reduce((acc, key) => ({ ...acc, [`#${key}`]: key }), {});
+}
+function generateExpressionValues(keys, fields) {
+  return keys.reduce((acc, key) => ({ ...acc, [`:${key}`]: fields[key] }), {});
+}
+async function updateRecordByKey(tableName, idKey, fields, conditionExpressions = null, verbose = false) {
+  if (verbose) {
+    console.log("update", tableName, idKey, fields);
+  }
+  const docClient = getDocClient();
+  const idKeyNames = Object.keys(idKey);
+  const fieldsToDelete = Object.keys(fields).filter((f) => fields[f] === undefined);
+  const fieldsToUpdate = Object.keys(fields).filter((k) => !idKeyNames.includes(k) && !fieldsToDelete.includes(k));
+  let data;
+  if (fieldsToDelete.length > 0) {
+    if (verbose) {
+      console.log("delete fields", tableName, fieldsToDelete);
+    }
+    const deleteParams = {
+      TableName: tableName,
+      Key: idKey,
+      ExpressionAttributeNames: generateExpressionNames(fieldsToDelete),
+      UpdateExpression: `REMOVE ${fieldsToDelete.map((f) => `#${f}`).join(", ")}`,
+      ReturnValues: "ALL_NEW"
     };
-    const finalState = {
-        ...defaultState,
-        ...state,
+    if (conditionExpressions) {
+      deleteParams.ConditionExpression = conditionExpressions;
+    }
+    data = await docClient.send(new UpdateCommand(deleteParams));
+  }
+  if (fieldsToUpdate.length > 0) {
+    if (verbose) {
+      console.log("update fields", tableName, fieldsToUpdate);
+    }
+    const updateExpressions = fieldsToUpdate.map((key) => `#${key} = :${key}`);
+    const params = {
+      TableName: tableName,
+      Key: idKey,
+      ExpressionAttributeNames: generateExpressionNames(fieldsToUpdate),
+      ExpressionAttributeValues: generateExpressionValues(fieldsToUpdate, fields),
+      UpdateExpression: `SET ${updateExpressions.join(", ")}`,
+      ReturnValues: "ALL_NEW"
     };
-    // type based range functions
-    function range(from, to, step) {
-        if (typeof from === "string" && typeof to === "string") {
-            if (finalState.type === "date") {
-                return dateRange(from, to, step);
-            }
-            throw new Error("Invalid type for numeric range");
-        }
-        if (typeof from === "number" && typeof to === "number") {
-            return numberRange(from, to, step);
-        }
-        throw new Error("Invalid type for range");
-    }
-    function fillRange(from, to) {
-        if (typeof from === "string" && typeof to === "string") {
-            if (finalState.type === "date") {
-                return fillDateRange(from, to);
-            }
-            throw new Error("Invalid type for numeric range");
-        }
-        if (typeof from === "number" && typeof to === "number") {
-            return fillNumberRange(from, to);
-        }
-        throw new Error("Invalid type for range");
-    }
-    function offsetRange(from, to) {
-        return fillRange(from, to).length;
-    }
-    async function runMode(flags) {
-        const { mode, from: fromUntyped, to: toUntyped, status, verbose: verboseUntyped, ...untypeSelectorFlags } = flags;
-        const from = fromUntyped;
-        const to = toUntyped;
-        const verbose = verboseUntyped;
-        const selectorFlags = untypeSelectorFlags;
-        if (status) {
-            const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
-                name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-            });
-            const fromItem = await getRecordByKey(STATE_TABLE_NAME, {
-                name: `${name}Since(${toSelectorString(selectorFlags)})`,
-            });
-            const validateItem = await getRecordByKey(STATE_TABLE_NAME, {
-                name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-            });
-            inline.log(`RebuildState=${stateItem?.value} Since=${fromItem?.value} Validated=${validateItem?.value}`);
-            process.exit(0);
-        }
-        inline.log(`[MODE INDEXER] mode=${mode}, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`);
-        if (mode === "reset") {
-            await runReset(selectorFlags);
-        }
-        else if (mode === "rebuild") {
-            const rebuildFrom = from || (await finalState.getMinId(selectorFlags));
-            const rebuildTo = to || (await finalState.getMaxId(selectorFlags));
-            await runReset(selectorFlags);
-            await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
-        }
-        else if (mode === "resume-rebuild") {
-            const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
-            const rebuildFrom = from ||
-                (previousRebuildEnds !== undefined && increaseId(finalState.type, previousRebuildEnds, 1)) ||
-                (await finalState.getMinId(selectorFlags));
-            const rebuildTo = to || (await finalState.getMaxId(selectorFlags));
-            await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
-        }
-        else if (mode === "validate" || mode === "validation") {
-            const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
-            if (!previousRebuildEnds) {
-                inline.log(`[MODE INDEXER] cannot validate without a successful rebuild`);
-                process.exit(0);
-            }
-            const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
-            const validateFrom = from || previousValidatedTo || (await finalState.getMinId(selectorFlags));
-            const validateTo = to || previousRebuildEnds;
-            const shouldSaveState = !to; // should not save state for manual validations, those are for testing
-            await runValidate(selectorFlags, validateFrom, validateTo, shouldSaveState, verbose);
-        }
-        else if (mode === "one") {
-            if (to) {
-                inline.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
-            }
-            if (!from) {
-                inline.log(`[MODE INDEXER] must provide --from option for one mode`);
-                process.exit(1);
-            }
-            await runRange(selectorFlags, from, from, verbose);
-        }
-        else if (mode === "range") {
-            if (!from || !to) {
-                inline.log(`[MODE INDEXER] must provide --from and --to option for range mode`);
-                process.exit(1);
-            }
-            await runRange(selectorFlags, from, to, verbose);
-        }
-        else {
-            const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
-                name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-            });
-            // only build when rebuild succeed
-            if (!stateItem || stateItem.value !== "succeed") {
-                inline.log("[MODE INDEXER] skip because rebuild hasn't done yet");
-                process.exit(0);
-            }
-            const latestId = await getIndexerLatestId(name, selectorFlags);
-            if (!latestId) {
-                throw new Error(`[MODE INDEXER] cannot find the latest ${finalState.type}`);
-            }
-            const deltaFrom = increaseId(finalState.type, latestId, 1);
-            const deltaTo = await state.getMaxId(selectorFlags);
-            await runDelta(selectorFlags, deltaFrom, deltaTo, verbose);
-        }
+    if (conditionExpressions) {
+      params.ConditionExpression = conditionExpressions;
     }
-    async function runRange(selectorFlags, from, to, verbose) {
-        const startTime = Date.now();
-        inline.log(`[MODE INDEXER] building range, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
-        const failedIds = await execBuild(selectorFlags, from, to, verbose, false);
-        if (failedIds.length > 0) {
-            inline.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
-            process.exit(1);
-        }
-        else {
-            inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
-            process.exit(0);
-        }
+    data = await docClient.send(new UpdateCommand(params));
+  }
+  return data?.Attributes;
+}
+async function batchDeleteRecords(tableName, keys) {
+  const docClient = getDocClient();
+  for (let start = 0;start < keys.length; start += 25) {
+    const slice = keys.slice(start, start + 25);
+    await docClient.send(new BatchWriteCommand({
+      RequestItems: {
+        [tableName]: slice.map((key) => {
+          return { DeleteRequest: { Key: key } };
+        })
+      }
+    }));
+  }
+}
+function getKeyName(keySchema, type) {
+  const key = keySchema.find((k) => k.KeyType === type);
+  return key?.AttributeName;
+}
+function getIndexKeyName(globalSecondaryIndexes, indexName, type) {
+  const idx = globalSecondaryIndexes.find((i) => i.IndexName === indexName);
+  return idx?.KeySchema && getKeyName(idx.KeySchema, type);
+}
+async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbose = false) {
+  const docClient = getDocClient();
+  const meta = await getDynamoDB().send(new DescribeTableCommand({ TableName: tableName }));
+  if (!meta.Table) {
+    throw new Error(`cannot find table ${tableName}`);
+  }
+  if (indexName && !meta.Table.GlobalSecondaryIndexes) {
+    throw new Error(`cannot find global secondary indexes for table ${tableName}`);
+  }
+  if (!meta.Table.KeySchema) {
+    throw new Error(`cannot find key schema for table ${tableName}`);
+  }
+  const hashKeyName = indexName ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes, indexName, "HASH") : getKeyName(meta.Table.KeySchema, "HASH");
+  if (!hashKeyName) {
+    throw new Error(`cannot find hash key name for table ${tableName}`);
+  }
+  const mainHashKeyName = getKeyName(meta.Table.KeySchema, "HASH");
+  if (!mainHashKeyName) {
+    throw new Error(`cannot find main hash key name for table ${tableName}`);
+  }
+  const mainRangeKeyName = getKeyName(meta.Table.KeySchema, "RANGE");
+  if (!mainRangeKeyName) {
+    throw new Error(`cannot find main range key name for table ${tableName}`);
+  }
+  let totalDeleted = 0;
+  const params = {
+    TableName: tableName,
+    KeyConditionExpression: "#hashKeyName = :hashKeyValue",
+    ExpressionAttributeNames: { "#hashKeyName": hashKeyName },
+    ExpressionAttributeValues: { ":hashKeyValue": hashKeyValue }
+  };
+  if (indexName) {
+    params.IndexName = indexName;
+  }
+  let data = await docClient.send(new QueryCommand(params));
+  if (data.Items) {
+    await batchDeleteRecords(tableName, data.Items.map((item) => mainRangeKeyName ? {
+      [mainHashKeyName]: item[mainHashKeyName],
+      [mainRangeKeyName]: item[mainRangeKeyName]
+    } : {
+      [mainHashKeyName]: item[mainHashKeyName]
+    }));
+    totalDeleted += data.Items.length;
+  }
+  while (data.LastEvaluatedKey) {
+    data = await docClient.send(new QueryCommand({
+      ...params,
+      ExclusiveStartKey: data.LastEvaluatedKey
+    }));
+    if (data.Items) {
+      await batchDeleteRecords(tableName, data.Items.map((item) => mainRangeKeyName ? {
+        [mainHashKeyName]: item[mainHashKeyName],
+        [mainRangeKeyName]: item[mainRangeKeyName]
+      } : {
+        [mainHashKeyName]: item[mainHashKeyName]
+      }));
+      totalDeleted += data.Items.length;
     }
-    async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
-        if (!validate) {
-            inline.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
-            process.exit(1);
-        }
-        const startTime = Date.now();
-        inline.log(`[MODE INDEXER] validating, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`);
-        const windows = range(from, to, validateBatchSize * validateConcurrency);
-        inline.log(`[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`);
-        for (const [windowStart, windowEnd] of windows) {
-            inline.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
-            const batches = range(windowStart, windowEnd, validateBatchSize);
-            // add a retry for errors
-            await Promise.all(batches.map(async ([batchStart, batchEnd]) => {
-                const result = await exponentialRetry(async () => {
-                    try {
-                        await validate({
-                            ...selectorFlags,
-                            from: batchStart,
-                            to: batchEnd,
-                            verbose,
-                        });
-                        return true;
-                    }
-                    catch (err) {
-                        inline.error(`got error in validation`, err);
-                        return false;
-                    }
-                }, {
-                    maxRetry,
-                    test: (r) => r,
-                    verbose,
-                });
-                if (!result) {
-                    throw new Error(`Terminate validation due to critical errors, from=${batchStart}, to=${batchEnd}`);
-                }
-            }));
-            if (shouldSaveState) {
-                await createRecord(STATE_TABLE_NAME, {
-                    name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-                    value: to,
-                });
-                if (verbose) {
-                    inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
-                }
-            }
-        }
-        inline.log(`[MODE INDEXER] validated ${offsetRange(from, to)} ${finalState.type} successfully in ${Date.now() - startTime}ms`);
-    }
-    async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
-        let failedIds = [];
-        const windows = range(from, to, buildBatchSize * buildConcurrency);
-        for (const [windowStart, windowEnd] of windows) {
-            inline.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
-            const batches = range(windowStart, windowEnd, buildBatchSize);
-            // add a retry for errors
-            const batchResults = await Promise.all(batches.map(async ([batchStart, batchEnd]) => await exponentialRetry(async () => {
-                try {
-                    const ids = await build({
-                        ...selectorFlags,
-                        from: batchStart,
-                        to: batchEnd,
-                        verbose,
-                    });
-                    if (ids && ids.length > 0) {
-                        return ids;
-                    }
-                    else {
-                        return false;
-                    }
-                }
-                catch (err) {
-                    inline.error(`[MODE INDEXER] got error in build`, err);
-                    return fillRange(batchStart, batchEnd);
-                }
-            }, {
-                maxRetry,
-                test: (r) => !r,
-                verbose,
-            })));
-            if (shouldSaveState) {
-                await createRecord(STATE_TABLE_NAME, {
-                    name: `${name}Since(${toSelectorString(selectorFlags)})`,
-                    value: windowEnd,
-                });
-                if (verbose) {
-                    inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
-                }
-            }
-            batchResults.forEach((ids) => {
-                if (ids) {
-                    failedIds = failedIds.concat(ids);
-                }
-            });
-        }
-        failedIds.sort();
-        return failedIds;
+  }
+  if (verbose) {
+    console.log(`successfully delete ${totalDeleted} items`);
+  }
+  return totalDeleted;
+}
+// src/selector.ts
+function getSelectorDesc(selector) {
+  return Object.keys(selector).map((name) => {
+    return `          --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
+  }).join(`
+`);
+}
+function getSelectorFlags(selector) {
+  return Object.keys(selector).reduce((acc, name) => {
+    const flag = {
+      type: selector[name].type || "string",
+      ...selector[name]
+    };
+    if (!selector[name].optional && selector[name].isRequired !== false) {
+      flag.isRequired = true;
     }
-    async function runRebuild(selectorFlags, from, to, verbose) {
-        const startTime = Date.now();
-        inline.log(`[MODE INDEXER] rebuilding, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
-        // add a flag to stop delta from running
-        await createRecord(STATE_TABLE_NAME, {
-            name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-            value: "running",
-        });
-        const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
-        // even if some transactions are failed we should continue to allow delta job running
-        // because rebuild is usally heavy
-        await createRecord(STATE_TABLE_NAME, {
-            name: `${name}Since(${toSelectorString(selectorFlags)})`,
-            value: to,
+    return { ...acc, [name]: flag };
+  }, {});
+}
+function toSelectorString(selectorFlags, delim = ",") {
+  return Object.keys(selectorFlags).sort().map((flag) => {
+    return `${flag}=${selectorFlags[flag]}`;
+  }).join(delim);
+}
+function normalizeSelectorValue(v) {
+  return v.replace(/[^A-Za-z0-9]+/g, "-");
+}
+function getJobName(name, selectorFlags, mode) {
+  const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
+  let jobName = name;
+  if (mode) {
+    jobName += `-${mode}`;
+  }
+  if (selectorNamePart.length > 0) {
+    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
+  }
+  return jobName;
+}
+// src/cli.ts
+import path from "path";
+import fs from "fs";
+function getBinaryName() {
+  const binaryNameParts = process.argv[1].split(path.sep);
+  const binaryName = binaryNameParts[binaryNameParts.length - 1];
+  return binaryName;
+}
+function detectSkynetDirectory() {
+  return detectDirectory(process.argv[1], "SkynetAPIDefinitions.yml");
+}
+function detectWorkingDirectory() {
+  const wd = detectDirectory(process.argv[1], "package.json");
+  const skynetd = detectDirectory(process.argv[1], "SkynetAPIDefinitions.yml");
+  return wd.slice(skynetd.length + path.sep.length).replace(path.sep, "/");
+}
+function detectDirectory(fullBinPath, sentinel = "package.json") {
+  let parentFolder = path.dirname(fullBinPath);
+  while (parentFolder) {
+    const sentinelPath = path.join(parentFolder, sentinel);
+    if (fs.existsSync(sentinelPath)) {
+      return parentFolder;
+    }
+    const newParentFolder = path.dirname(parentFolder);
+    if (newParentFolder === parentFolder) {
+      break;
+    }
+    parentFolder = newParentFolder;
+  }
+  throw new Error("Cannot detect current working directory");
+}
+function detectBin() {
+  const wd = detectDirectory(process.argv[1], "package.json");
+  return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
+}
+// src/indexer.ts
+import meow from "meow";
+var STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
+async function getIndexerLatestId(name, selectorFlags) {
+  const record = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}Since(${toSelectorString(selectorFlags)})`
+  });
+  return record?.value;
+}
+async function getIndexerValidatedId(name, selectorFlags) {
+  const record = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}Validate(${toSelectorString(selectorFlags)})`
+  });
+  if (record) {
+    return record.value;
+  }
+  return;
+}
+function increaseId(type, currentId, n) {
+  if (type === "date") {
+    if (typeof currentId !== "string") {
+      throw new Error("invalid type for date id");
+    }
+    return findDateAfter(currentId, n);
+  }
+  if (typeof currentId !== "number") {
+    throw new Error("Invalid type for numeric id");
+  }
+  return currentId + n;
+}
+function createModeIndexerApp({
+  binaryName,
+  name,
+  selector = {},
+  build,
+  buildBatchSize = 1,
+  buildConcurrency = 1,
+  validate,
+  validateBatchSize = 1,
+  validateConcurrency = 1,
+  maxRetry = 2,
+  state
+}) {
+  const defaultState = {
+    type: "block",
+    getMinId: async () => 1,
+    getMaxId: async () => {
+      throw new Error("must implement getMaxId");
+    }
+  };
+  const finalState = {
+    ...defaultState,
+    ...state
+  };
+  function range2(from, to, step) {
+    if (typeof from === "string" && typeof to === "string") {
+      if (finalState.type === "date") {
+        return dateRange(from, to, step);
+      }
+      throw new Error("Invalid type for numeric range");
+    }
+    if (typeof from === "number" && typeof to === "number") {
+      return range(from, to, step);
+    }
+    throw new Error("Invalid type for range");
+  }
+  function fillRange2(from, to) {
+    if (typeof from === "string" && typeof to === "string") {
+      if (finalState.type === "date") {
+        return daysInRange(from, to);
+      }
+      throw new Error("Invalid type for numeric range");
+    }
+    if (typeof from === "number" && typeof to === "number") {
+      return fillRange(from, to);
+    }
+    throw new Error("Invalid type for range");
+  }
+  function offsetRange(from, to) {
+    return fillRange2(from, to).length;
+  }
+  async function runMode(flags) {
+    const { mode, from: fromUntyped, to: toUntyped, status, verbose: verboseUntyped, ...untypeSelectorFlags } = flags;
+    const from = fromUntyped;
+    const to = toUntyped;
+    const verbose = verboseUntyped;
+    const selectorFlags = untypeSelectorFlags;
+    if (status) {
+      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}RebuildState(${toSelectorString(selectorFlags)})`
+      });
+      const fromItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}Since(${toSelectorString(selectorFlags)})`
+      });
+      const validateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}Validate(${toSelectorString(selectorFlags)})`
+      });
+      inline.log(`RebuildState=${stateItem?.value} Since=${fromItem?.value} Validated=${validateItem?.value}`);
+      process.exit(0);
+    }
+    inline.log(`[MODE INDEXER] mode=${mode}, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`);
+    if (mode === "reset") {
+      await runReset(selectorFlags);
+    } else if (mode === "rebuild") {
+      const rebuildFrom = from || await finalState.getMinId(selectorFlags);
+      const rebuildTo = to || await finalState.getMaxId(selectorFlags);
+      await runReset(selectorFlags);
+      await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
+    } else if (mode === "resume-rebuild") {
+      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+      const rebuildFrom = from || previousRebuildEnds !== undefined && increaseId(finalState.type, previousRebuildEnds, 1) || await finalState.getMinId(selectorFlags);
+      const rebuildTo = to || await finalState.getMaxId(selectorFlags);
+      await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
+    } else if (mode === "validate" || mode === "validation") {
+      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+      if (!previousRebuildEnds) {
+        inline.log(`[MODE INDEXER] cannot validate without a successful rebuild`);
+        process.exit(0);
+      }
+      const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
+      const validateFrom = from || previousValidatedTo || await finalState.getMinId(selectorFlags);
+      const validateTo = to || previousRebuildEnds;
+      const shouldSaveState = !to;
+      await runValidate(selectorFlags, validateFrom, validateTo, shouldSaveState, verbose);
+    } else if (mode === "one") {
+      if (to) {
+        inline.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
+      }
+      if (!from) {
+        inline.log(`[MODE INDEXER] must provide --from option for one mode`);
+        process.exit(1);
+      }
+      await runRange(selectorFlags, from, from, verbose);
+    } else if (mode === "range") {
+      if (!from || !to) {
+        inline.log(`[MODE INDEXER] must provide --from and --to option for range mode`);
+        process.exit(1);
+      }
+      await runRange(selectorFlags, from, to, verbose);
+    } else {
+      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}RebuildState(${toSelectorString(selectorFlags)})`
+      });
+      if (!stateItem || stateItem.value !== "succeed") {
+        inline.log("[MODE INDEXER] skip because rebuild hasn't done yet");
+        process.exit(0);
+      }
+      const latestId = await getIndexerLatestId(name, selectorFlags);
+      if (!latestId) {
+        throw new Error(`[MODE INDEXER] cannot find the latest ${finalState.type}`);
+      }
+      const deltaFrom = increaseId(finalState.type, latestId, 1);
+      const deltaTo = await state.getMaxId(selectorFlags);
+      await runDelta(selectorFlags, deltaFrom, deltaTo, verbose);
+    }
+  }
+  async function runRange(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] building range, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
+    const failedIds = await execBuild(selectorFlags, from, to, verbose, false);
+    if (failedIds.length > 0) {
+      inline.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
+      process.exit(1);
+    } else {
+      inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+      process.exit(0);
+    }
+  }
+  async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
+    if (!validate) {
+      inline.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
+      process.exit(1);
+    }
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] validating, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`);
+    const windows = range2(from, to, validateBatchSize * validateConcurrency);
+    inline.log(`[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`);
+    for (const [windowStart, windowEnd] of windows) {
+      inline.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
+      const batches = range2(windowStart, windowEnd, validateBatchSize);
+      await Promise.all(batches.map(async ([batchStart, batchEnd]) => {
+        const result = await exponentialRetry(async () => {
+          try {
+            await validate({
+              ...selectorFlags,
+              from: batchStart,
+              to: batchEnd,
+              verbose
+            });
+            return true;
+          } catch (err) {
+            inline.error(`got error in validation`, err);
+            return false;
+          }
+        }, {
+          maxRetry,
+          test: (r) => r,
+          verbose
         });
+        if (!result) {
+          throw new Error(`Terminate validation due to critical errors, from=${batchStart}, to=${batchEnd}`);
+        }
+      }));
+      if (shouldSaveState) {
         await createRecord(STATE_TABLE_NAME, {
-            name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-            value: "succeed",
+          name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+          value: to
         });
-        if (failedIds.length > 0) {
-            inline.log(`[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) with some failed ${finalState.type}`, failedIds);
-            process.exit(1);
-        }
-        else {
-            inline.log(`[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) successfully in ${Date.now() - startTime}ms`);
-            process.exit(0);
+        if (verbose) {
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
         }
+      }
     }
-    async function runDelta(selectorFlags, from, to, verbose) {
-        const startTime = Date.now();
-        if (to < from) {
-            inline.log(`[MODE INDEXER] skip delta, there're no more items need to be processed, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
-            return;
-        }
-        inline.log(`[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
+    inline.log(`[MODE INDEXER] validated ${offsetRange(from, to)} ${finalState.type} successfully in ${Date.now() - startTime}ms`);
+  }
+  async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
+    let failedIds = [];
+    const windows = range2(from, to, buildBatchSize * buildConcurrency);
+    for (const [windowStart, windowEnd] of windows) {
+      inline.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
+      const batches = range2(windowStart, windowEnd, buildBatchSize);
+      const batchResults = await Promise.all(batches.map(async ([batchStart, batchEnd]) => await exponentialRetry(async () => {
         try {
-            const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
-            if (failedIds.length > 0) {
-                inline.log("[MODE INDEXER] built with some failed txs", failedIds);
-                await createRecord(STATE_TABLE_NAME, {
-                    name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
-                    value: "failed",
-                });
-                await createRecord(STATE_TABLE_NAME, {
-                    name: `${name}Since(${toSelectorString(selectorFlags)})`,
-                    value: to < failedIds[0] ? to : failedIds[0],
-                });
-                process.exit(1);
-            }
-            else {
-                await createRecord(STATE_TABLE_NAME, {
-                    name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
-                    value: "succeed",
-                });
-                await createRecord(STATE_TABLE_NAME, {
-                    name: `${name}Since(${toSelectorString(selectorFlags)})`,
-                    value: to,
-                });
-                inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
-                process.exit(0);
-            }
+          const ids = await build({
+            ...selectorFlags,
+            from: batchStart,
+            to: batchEnd,
+            verbose
+          });
+          if (ids && ids.length > 0) {
+            return ids;
+          } else {
+            return false;
+          }
+        } catch (err) {
+          inline.error(`[MODE INDEXER] got error in build`, err);
+          return fillRange2(batchStart, batchEnd);
         }
-        catch (err) {
-            inline.error("[MODE INDEXER] delta build failed", from, to, err);
-            process.exit(1);
+      }, {
+        maxRetry,
+        test: (r) => !r,
+        verbose
+      })));
+      if (shouldSaveState) {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: windowEnd
+        });
+        if (verbose) {
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
         }
+      }
+      batchResults.forEach((ids) => {
+        if (ids) {
+          failedIds = failedIds.concat(ids);
+        }
+      });
     }
-    async function runReset(selectorFlags) {
-        const startTime = Date.now();
-        inline.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
-        inline.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
+    failedIds.sort();
+    return failedIds;
+  }
+  async function runRebuild(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] rebuilding, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "running"
+    });
+    const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Since(${toSelectorString(selectorFlags)})`,
+      value: to
+    });
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "succeed"
+    });
+    if (failedIds.length > 0) {
+      inline.log(`[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) with some failed ${finalState.type}`, failedIds);
+      process.exit(1);
+    } else {
+      inline.log(`[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) successfully in ${Date.now() - startTime}ms`);
+      process.exit(0);
+    }
+  }
+  async function runDelta(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+    if (to < from) {
+      inline.log(`[MODE INDEXER] skip delta, there're no more items need to be processed, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
+      return;
+    }
+    inline.log(`[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
+    try {
+      const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
+      if (failedIds.length > 0) {
+        inline.log("[MODE INDEXER] built with some failed txs", failedIds);
         await createRecord(STATE_TABLE_NAME, {
-            name: `${name}Since(${toSelectorString(selectorFlags)})`,
-            value: 0,
+          name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+          value: "failed"
         });
         await createRecord(STATE_TABLE_NAME, {
-            name: `${name}Validate(${toSelectorString(selectorFlags)})`,
-            value: 0,
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: to < failedIds[0] ? to : failedIds[0]
         });
+        process.exit(1);
+      } else {
         await createRecord(STATE_TABLE_NAME, {
-            name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-            value: "init",
+          name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+          value: "succeed"
         });
-        inline.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: to
+        });
+        inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+        process.exit(0);
+      }
+    } catch (err) {
+      inline.error("[MODE INDEXER] delta build failed", from, to, err);
+      process.exit(1);
     }
-    async function run() {
-        if (!binaryName) {
-            binaryName = getBinaryName();
-        }
-        const cli = meow(`
+  }
+  async function runReset(selectorFlags) {
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
+    inline.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Since(${toSelectorString(selectorFlags)})`,
+      value: 0
+    });
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+      value: 0
+    });
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "init"
+    });
+    inline.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
+  }
+  async function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+    const cli = meow(`
         Usage
 
           $ ${binaryName} <options>
@@ -388,52 +994,53 @@ ${selector ? getSelectorDesc(selector) : ""}
           --status        print status of indexer and exit
           --verbose       Output debug messages
       `, {
-            importMeta: import.meta,
-            description: false,
-            version: false,
-            flags: {
-                ...getSelectorFlags(selector),
-                mode: {
-                    type: "string",
-                    default: "delta",
-                },
-                from: {
-                    aliases: ["since"],
-                    type: "string",
-                },
-                to: {
-                    aliases: ["until"],
-                    type: "string",
-                },
-                status: {
-                    type: "boolean",
-                    default: false,
-                },
-                verbose: {
-                    type: "boolean",
-                    default: false,
-                },
-            },
-        });
-        try {
-            return runMode(cli.flags);
-        }
-        catch (err) {
-            inline.error(err);
-            process.exit(1);
+      importMeta: import.meta,
+      description: false,
+      version: false,
+      flags: {
+        ...getSelectorFlags(selector),
+        mode: {
+          type: "string",
+          default: "delta"
+        },
+        from: {
+          aliases: ["since"],
+          type: "string"
+        },
+        to: {
+          aliases: ["until"],
+          type: "string"
+        },
+        status: {
+          type: "boolean",
+          default: false
+        },
+        verbose: {
+          type: "boolean",
+          default: false
         }
+      }
+    });
+    try {
+      return runMode(cli.flags);
+    } catch (err) {
+      inline.error(err);
+      process.exit(1);
     }
-    return { run };
+  }
+  return { run };
 }
-// for indexers that don't rely on a cursor
-// e.g. should always rebuild everything from scratch
-// or that the state can be easily inferred from existing data
-function createIndexerApp({ binaryName, selector = {}, build, maxRetry = 2, }) {
-    async function run() {
-        if (!binaryName) {
-            binaryName = getBinaryName();
-        }
-        const cli = meow(`
+function createIndexerApp({
+  binaryName,
+  selector = {},
+  build,
+  maxRetry = 2
+}) {
+  async function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+    const cli = meow(`
         Usage
           $ ${binaryName} <options>
 
@@ -441,52 +1048,54 @@ function createIndexerApp({ binaryName, selector = {}, build, maxRetry = 2, }) {
 ${selector ? getSelectorDesc(selector) : ""}
           --verbose       Output debug messages
       `, {
-            importMeta: import.meta,
-            description: false,
-            version: false,
-            flags: {
-                ...getSelectorFlags(selector),
-                verbose: {
-                    type: "boolean",
-                    default: false,
-                },
-            },
-        });
-        async function runBuild(flags) {
-            const { verbose: untypedVerbose, ...untypedSelectorFlags } = flags;
-            const verbose = untypedVerbose;
-            const selectorFlags = untypedSelectorFlags;
-            const startTime = Date.now();
-            if (Object.keys(selectorFlags).length > 0) {
-                inline.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
-            }
-            else {
-                inline.log(`[INDEXER] starting build`);
-            }
-            const result = await exponentialRetry(async () => {
-                try {
-                    await build(flags);
-                    return true;
-                }
-                catch (err) {
-                    inline.log(`[INDEXER] got error in build`, err);
-                    return false;
-                }
-            }, {
-                maxRetry,
-                test: (r) => r,
-                verbose: verbose,
-            });
-            if (!result) {
-                throw new Error(`[INDEXER] Build failed due to critical errors`);
-            }
-            inline.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
+      importMeta: import.meta,
+      description: false,
+      version: false,
+      flags: {
+        ...getSelectorFlags(selector),
+        verbose: {
+          type: "boolean",
+          default: false
         }
-        return runBuild(cli.flags).catch((err) => {
-            inline.error(err);
-            process.exit(1);
-        });
+      }
+    });
+    async function runBuild(flags) {
+      const { verbose: untypedVerbose, ...untypedSelectorFlags } = flags;
+      const verbose = untypedVerbose;
+      const selectorFlags = untypedSelectorFlags;
+      const startTime = Date.now();
+      if (Object.keys(selectorFlags).length > 0) {
+        inline.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
+      } else {
+        inline.log(`[INDEXER] starting build`);
+      }
+      const result = await exponentialRetry(async () => {
+        try {
+          await build(flags);
+          return true;
+        } catch (err) {
+          inline.log(`[INDEXER] got error in build`, err);
+          return false;
+        }
+      }, {
+        maxRetry,
+        test: (r) => r,
+        verbose
+      });
+      if (!result) {
+        throw new Error(`[INDEXER] Build failed due to critical errors`);
+      }
+      inline.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
     }
-    return { run };
+    return runBuild(cli.flags).catch((err) => {
+      inline.error(err);
+      process.exit(1);
+    });
+  }
+  return { run };
 }
-export { increaseId, createModeIndexerApp, createIndexerApp };
+export {
+  increaseId,
+  createModeIndexerApp,
+  createIndexerApp
+};