@certik/skynet 0.25.0 → 0.25.1

package/dist/app.js

@@ -0,0 +1,2077 @@
+// src/env.ts
+function ensureAndGet(envName, defaultValue) {
+  return process.env[envName] || defaultValue;
+}
+function getEnvironment() {
+  return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
+}
+function getEnvOrThrow(envName) {
+  if (!process.env[envName]) {
+    throw new Error(`Must set environment variable ${envName}`);
+  }
+  return process.env[envName];
+}
+function isProduction() {
+  return getEnvironment() === "prd";
+}
+function isDev() {
+  return getEnvironment() === "dev";
+}
+// src/util.ts
+function range(startAt, endAt, step) {
+  const arr = [];
+  for (let i = startAt;i <= endAt; i += step) {
+    arr.push([i, Math.min(endAt, i + step - 1)]);
+  }
+  return arr;
+}
+function arrayGroup(array, groupSize) {
+  const groups = [];
+  for (let i = 0;i < array.length; i += groupSize) {
+    groups.push(array.slice(i, i + groupSize));
+  }
+  return groups;
+}
+function fillRange(start, end) {
+  const result = [];
+  for (let i = start;i <= end; i++) {
+    result.push(i);
+  }
+  return result;
+}
+// src/date.ts
+var MS_IN_A_DAY = 3600 * 24 * 1000;
+function getDateOnly(date) {
+  return new Date(date).toISOString().split("T")[0];
+}
+function findDateAfter(date, n) {
+  const d = new Date(date);
+  const after = new Date(d.getTime() + MS_IN_A_DAY * n);
+  return getDateOnly(after);
+}
+function daysInRange(from, to) {
+  const fromTime = new Date(from).getTime();
+  const toTime = new Date(to).getTime();
+  if (fromTime > toTime) {
+    throw new Error(`range to date couldn't be earlier than range from date`);
+  }
+  const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
+  const dates = [getDateOnly(new Date(fromTime))];
+  for (let i = 1;i <= daysBetween; i += 1) {
+    dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
+  }
+  return dates;
+}
+function dateRange(from, to, step) {
+  const days = daysInRange(from, to);
+  const windows = arrayGroup(days, step);
+  return windows.map((w) => [w[0], w[w.length - 1]]);
+}
+// src/object-hash.ts
+import xh from "@node-rs/xxhash";
+function getHash(obj) {
+  const xxh3 = xh.xxh3.Xxh3.withSeed();
+  hash(obj, xxh3);
+  return xxh3.digest().toString(16);
+}
+function hash(obj, xxh3) {
+  if (obj === null) {
+    xxh3.update("null");
+  } else if (obj === undefined) {
+    xxh3.update("undefined");
+  } else if (typeof obj === "string") {
+    xxh3.update(obj);
+  } else if (typeof obj === "number") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "boolean") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "bigint") {
+    xxh3.update(obj.toString());
+  } else if (obj instanceof Date) {
+    xxh3.update(obj.toISOString());
+  } else if (Array.isArray(obj)) {
+    arrayHash(obj, xxh3);
+  } else if (obj instanceof Set) {
+    setHash(obj, xxh3);
+  } else if (obj instanceof Map) {
+    mapHash(obj, xxh3);
+  } else if (typeof obj === "object") {
+    objectHash(obj, xxh3);
+  } else {
+    throw new Error(`Unsupported type: ${obj}`);
+  }
+}
+function arrayHash(array, xxh3) {
+  xxh3.update("[");
+  for (const obj of array) {
+    hash(obj, xxh3);
+  }
+  xxh3.update("]");
+}
+function setHash(_set, _xxh3) {
+  throw new Error("Set hashing not implemented");
+}
+function mapHash(map, xxh3) {
+  const array = Array.from(map.entries()).sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}
+function objectHash(obj, xxh3) {
+  const array = Object.entries(obj).sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}
+
+// src/availability.ts
+import pThrottle from "p-throttle";
+import pMemoize from "p-memoize";
+import QuickLRU from "quick-lru";
+async function wait(time) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, time);
+  });
+}
+async function exponentialRetry(func, {
+  maxRetry,
+  initialDuration,
+  growFactor,
+  test,
+  verbose
+}) {
+  let retries = maxRetry;
+  let duration = initialDuration || 5000;
+  const growFactorFinal = growFactor || 2;
+  let result = await func();
+  while (!test(result) && retries > 0) {
+    if (verbose) {
+      console.log("failed attempt result", result);
+      console.log(`sleep for ${duration}ms after failed attempt, remaining ${retries} attempts`);
+    }
+    retries = retries - 1;
+    await wait(duration);
+    result = await func();
+    duration = duration * growFactorFinal;
+  }
+  if (verbose) {
+    console.log(`function to retry ends with status ${test(result)}, number of retries done: ${maxRetry - retries}}`);
+  }
+  return result;
+}
+function withRetry(func, options) {
+  let retries = options?.maxRetry || 3;
+  let duration = options?.initialDuration || 500;
+  const growFactorFinal = options?.growFactor || 2;
+  return async (...args) => {
+    do {
+      try {
+        return await func(...args);
+      } catch (error) {
+        retries = retries - 1;
+        if (retries <= 0) {
+          throw error;
+        }
+        await wait(duration);
+        duration = duration * growFactorFinal;
+      }
+    } while (retries > 0);
+    throw new Error("unreachable");
+  };
+}
+function memoize(func, options) {
+  if (!options) {
+    options = {};
+  }
+  if (!options.cache) {
+    options.cache = new QuickLRU({ maxSize: options.lruMaxSize || 1e4 });
+  }
+  if (!options.cacheKey) {
+    options.cacheKey = (args) => getHash(args);
+  }
+  return pMemoize(func, options);
+}
+// src/log.ts
+function isObject(a) {
+  return !!a && a.constructor === Object;
+}
+function print(o) {
+  if (Array.isArray(o)) {
+    return `[${o.map(print).join(", ")}]`;
+  }
+  if (isObject(o)) {
+    return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
+  }
+  return `${o}`;
+}
+function getLine(params) {
+  let line = "";
+  for (let i = 0, l = params.length;i < l; i++) {
+    line += `${print(params[i])} `.replace(/\n/gm, "\t");
+  }
+  return line.trim();
+}
+function timestamp() {
+  return new Date().toISOString();
+}
+var inline = {
+  debug: function(...args) {
+    if (true) {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
+  },
+  log: function(...args) {
+    if (true) {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
+  },
+  error: function(...args) {
+    if (true) {
+      console.error(`${timestamp()} ${getLine(args)}`);
+    }
+  }
+};
+var logger = {
+  debug: function(...args) {
+    if (true) {
+      console.log(`[${timestamp()}]`, ...args);
+    }
+  },
+  log: function(...args) {
+    if (true) {
+      console.log(`[${timestamp()}]`, ...args);
+    }
+  },
+  error: function(...args) {
+    if (true) {
+      console.error(`[${timestamp()}]`, ...args);
+    }
+  }
+};
+// src/dynamodb.ts
+import {
+  DynamoDBDocumentClient,
+  ScanCommand,
+  BatchWriteCommand,
+  GetCommand,
+  PutCommand,
+  QueryCommand,
+  UpdateCommand
+} from "@aws-sdk/lib-dynamodb";
+import { DynamoDBClient, DescribeTableCommand } from "@aws-sdk/client-dynamodb";
+var _dynamoDB;
+var _docClient;
+function getDynamoDB(forceNew = false) {
+  if (!_dynamoDB || forceNew) {
+    _dynamoDB = new DynamoDBClient;
+  }
+  return _dynamoDB;
+}
+function getDocClient(forceNew = false) {
+  const marshallOptions = {
+    convertEmptyValues: true,
+    removeUndefinedValues: true,
+    convertClassInstanceToMap: true
+  };
+  const unmarshallOptions = {
+    wrapNumbers: false
+  };
+  if (!_docClient || forceNew) {
+    _docClient = DynamoDBDocumentClient.from(getDynamoDB(), {
+      marshallOptions,
+      unmarshallOptions
+    });
+  }
+  return _docClient;
+}
+async function scanWholeTable(options) {
+  const dynamodb = getDocClient();
+  let items = [];
+  let count = 0;
+  let scannedCount = 0;
+  let data = await dynamodb.send(new ScanCommand(options));
+  while (data.LastEvaluatedKey) {
+    if (data.Items) {
+      items = items.concat(data.Items);
+    }
+    count += data.Count || 0;
+    scannedCount += data.ScannedCount || 0;
+    data = await dynamodb.send(new ScanCommand({ ...options, ExclusiveStartKey: data.LastEvaluatedKey }));
+  }
+  if (data.Items) {
+    items = items.concat(data.Items);
+  }
+  count += data.Count || 0;
+  scannedCount += data.ScannedCount || 0;
+  return {
+    Items: items,
+    Count: count,
+    ScannedCount: scannedCount
+  };
+}
+async function batchCreateRecords(tableName, records, maxWritingCapacity, verbose = false) {
+  if (verbose) {
+    console.log(`creating ${records.length} items in ${tableName}`);
+  }
+  const docClient = getDocClient();
+  let remainingItems = records;
+  let prevRemainingCount = remainingItems.length + 1;
+  let factor = 1;
+  let rejection = undefined;
+  while (remainingItems.length > 0 && factor <= 128 && !rejection) {
+    if (prevRemainingCount === remainingItems.length) {
+      await wait(5000 * factor);
+      factor = factor * 2;
+    }
+    if (factor >= 32) {
+      console.log(`WARNING: no progress for a long time for batchCreateRecords, please check`);
+    }
+    const slices = arrayGroup(remainingItems.slice(0, maxWritingCapacity), 25);
+    const results = await Promise.allSettled(slices.map((rs) => docClient.send(new BatchWriteCommand({
+      RequestItems: {
+        [tableName]: rs.map((record) => ({ PutRequest: { Item: record } }))
+      }
+    }))));
+    const isFulfilled = (p) => p.status === "fulfilled";
+    const isRejected = (p) => p.status === "rejected";
+    prevRemainingCount = remainingItems.length;
+    remainingItems = remainingItems.slice(maxWritingCapacity);
+    results.forEach((rs, idx) => {
+      if (isRejected(rs)) {
+        remainingItems = remainingItems.concat(slices[idx]);
+        rejection = rs;
+      } else if (isFulfilled(rs) && rs.value.UnprocessedItems && Object.keys(rs.value.UnprocessedItems).length > 0) {
+        const unprocessedItems = rs.value.UnprocessedItems[tableName].map((it) => it.PutRequest?.Item ?? []).flat();
+        remainingItems = remainingItems.concat(unprocessedItems);
+      }
+    });
+    if (verbose) {
+      console.log(`processed=${prevRemainingCount - remainingItems.length}, remaining=${remainingItems.length}`);
+    }
+  }
+  if (rejection) {
+    console.log("batchCreateRecords rejected", rejection);
+    throw new Error(`batchCreateRecords rejected, failed items=${remainingItems.length}`);
+  }
+  if (remainingItems.length > 0) {
+    console.log(`failed batchCreateRecords, failed items=${remainingItems.length}`);
+    throw new Error(`batchCreateRecords retry failed, failed items=${remainingItems.length}`);
+  }
+}
+async function createRecord(tableName, fields, verbose = false) {
+  if (verbose) {
+    console.log("creating", tableName, fields);
+  }
+  const docClient = getDocClient();
+  const params = {
+    TableName: tableName,
+    Item: fields
+  };
+  return docClient.send(new PutCommand(params));
+}
+async function readRecord(tableName, key, verbose = false) {
+  if (verbose) {
+    console.log("reading", tableName, key);
+  }
+  const docClient = getDocClient();
+  const record = await docClient.send(new GetCommand({
+    TableName: tableName,
+    Key: key
+  }));
+  return record.Item;
+}
+async function getRecordsByKey(tableName, keys, indexName) {
+  const docClient = getDocClient();
+  const keyNames = Object.keys(keys);
+  const conditionExpression = keyNames.map((key) => `#${key} = :${key}`).join(" and ");
+  const params = {
+    TableName: tableName,
+    KeyConditionExpression: conditionExpression,
+    ExpressionAttributeNames: generateExpressionNames(keyNames),
+    ExpressionAttributeValues: generateExpressionValues(keyNames, keys)
+  };
+  if (indexName) {
+    params.IndexName = indexName;
+  }
+  try {
+    let data = await docClient.send(new QueryCommand(params));
+    let items = data.Items ?? [];
+    while (data.LastEvaluatedKey) {
+      data = await docClient.send(new QueryCommand({
+        ...params,
+        ExclusiveStartKey: data.LastEvaluatedKey
+      }));
+      if (data.Items) {
+        items = items.concat(data.Items);
+      }
+    }
+    return items;
+  } catch (err) {
+    console.log(err);
+    if (err instanceof Error && "statusCode" in err && err.statusCode === 400) {
+      return null;
+    }
+    throw err;
+  }
+}
+async function getRecordByKey(tableName, keys, indexName) {
+  if (indexName) {
+    const records = await getRecordsByKey(tableName, keys, indexName);
+    if (records) {
+      return records[0];
+    } else {
+      return null;
+    }
+  } else {
+    return readRecord(tableName, keys);
+  }
+}
+function generateExpressionNames(keys) {
+  return keys.reduce((acc, key) => ({ ...acc, [`#${key}`]: key }), {});
+}
+function generateExpressionValues(keys, fields) {
+  return keys.reduce((acc, key) => ({ ...acc, [`:${key}`]: fields[key] }), {});
+}
+async function updateRecordByKey(tableName, idKey, fields, conditionExpressions = null, verbose = false) {
+  if (verbose) {
+    console.log("update", tableName, idKey, fields);
+  }
+  const docClient = getDocClient();
+  const idKeyNames = Object.keys(idKey);
+  const fieldsToDelete = Object.keys(fields).filter((f) => fields[f] === undefined);
+  const fieldsToUpdate = Object.keys(fields).filter((k) => !idKeyNames.includes(k) && !fieldsToDelete.includes(k));
+  let data;
+  if (fieldsToDelete.length > 0) {
+    if (verbose) {
+      console.log("delete fields", tableName, fieldsToDelete);
+    }
+    const deleteParams = {
+      TableName: tableName,
+      Key: idKey,
+      ExpressionAttributeNames: generateExpressionNames(fieldsToDelete),
+      UpdateExpression: `REMOVE ${fieldsToDelete.map((f) => `#${f}`).join(", ")}`,
+      ReturnValues: "ALL_NEW"
+    };
+    if (conditionExpressions) {
+      deleteParams.ConditionExpression = conditionExpressions;
+    }
+    data = await docClient.send(new UpdateCommand(deleteParams));
+  }
+  if (fieldsToUpdate.length > 0) {
+    if (verbose) {
+      console.log("update fields", tableName, fieldsToUpdate);
+    }
+    const updateExpressions = fieldsToUpdate.map((key) => `#${key} = :${key}`);
+    const params = {
+      TableName: tableName,
+      Key: idKey,
+      ExpressionAttributeNames: generateExpressionNames(fieldsToUpdate),
+      ExpressionAttributeValues: generateExpressionValues(fieldsToUpdate, fields),
+      UpdateExpression: `SET ${updateExpressions.join(", ")}`,
+      ReturnValues: "ALL_NEW"
+    };
+    if (conditionExpressions) {
+      params.ConditionExpression = conditionExpressions;
+    }
+    data = await docClient.send(new UpdateCommand(params));
+  }
+  return data?.Attributes;
+}
+async function batchDeleteRecords(tableName, keys) {
+  const docClient = getDocClient();
+  for (let start = 0;start < keys.length; start += 25) {
+    const slice = keys.slice(start, start + 25);
+    await docClient.send(new BatchWriteCommand({
+      RequestItems: {
+        [tableName]: slice.map((key) => {
+          return { DeleteRequest: { Key: key } };
+        })
+      }
+    }));
+  }
+}
+function getKeyName(keySchema, type) {
+  const key = keySchema.find((k) => k.KeyType === type);
+  return key?.AttributeName;
+}
+function getIndexKeyName(globalSecondaryIndexes, indexName, type) {
+  const idx = globalSecondaryIndexes.find((i) => i.IndexName === indexName);
+  return idx?.KeySchema && getKeyName(idx.KeySchema, type);
+}
+async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbose = false) {
+  const docClient = getDocClient();
+  const meta = await getDynamoDB().send(new DescribeTableCommand({ TableName: tableName }));
+  if (!meta.Table) {
+    throw new Error(`cannot find table ${tableName}`);
+  }
+  if (indexName && !meta.Table.GlobalSecondaryIndexes) {
+    throw new Error(`cannot find global secondary indexes for table ${tableName}`);
+  }
+  if (!meta.Table.KeySchema) {
+    throw new Error(`cannot find key schema for table ${tableName}`);
+  }
+  const hashKeyName = indexName ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes, indexName, "HASH") : getKeyName(meta.Table.KeySchema, "HASH");
+  if (!hashKeyName) {
+    throw new Error(`cannot find hash key name for table ${tableName}`);
+  }
+  const mainHashKeyName = getKeyName(meta.Table.KeySchema, "HASH");
+  if (!mainHashKeyName) {
+    throw new Error(`cannot find main hash key name for table ${tableName}`);
+  }
+  const mainRangeKeyName = getKeyName(meta.Table.KeySchema, "RANGE");
+  if (!mainRangeKeyName) {
+    throw new Error(`cannot find main range key name for table ${tableName}`);
+  }
+  let totalDeleted = 0;
+  const params = {
+    TableName: tableName,
+    KeyConditionExpression: "#hashKeyName = :hashKeyValue",
+    ExpressionAttributeNames: { "#hashKeyName": hashKeyName },
+    ExpressionAttributeValues: { ":hashKeyValue": hashKeyValue }
+  };
+  if (indexName) {
+    params.IndexName = indexName;
+  }
+  let data = await docClient.send(new QueryCommand(params));
+  if (data.Items) {
+    await batchDeleteRecords(tableName, data.Items.map((item) => mainRangeKeyName ? {
+      [mainHashKeyName]: item[mainHashKeyName],
+      [mainRangeKeyName]: item[mainRangeKeyName]
+    } : {
+      [mainHashKeyName]: item[mainHashKeyName]
+    }));
+    totalDeleted += data.Items.length;
+  }
+  while (data.LastEvaluatedKey) {
+    data = await docClient.send(new QueryCommand({
+      ...params,
+      ExclusiveStartKey: data.LastEvaluatedKey
+    }));
+    if (data.Items) {
+      await batchDeleteRecords(tableName, data.Items.map((item) => mainRangeKeyName ? {
+        [mainHashKeyName]: item[mainHashKeyName],
+        [mainRangeKeyName]: item[mainRangeKeyName]
+      } : {
+        [mainHashKeyName]: item[mainHashKeyName]
+      }));
+      totalDeleted += data.Items.length;
+    }
+  }
+  if (verbose) {
+    console.log(`successfully delete ${totalDeleted} items`);
+  }
+  return totalDeleted;
+}
+// src/selector.ts
+function getSelectorDesc(selector) {
+  return Object.keys(selector).map((name) => {
+    return `          --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
+  }).join(`
+`);
+}
+function getSelectorFlags(selector) {
+  return Object.keys(selector).reduce((acc, name) => {
+    const flag = {
+      type: selector[name].type || "string",
+      ...selector[name]
+    };
+    if (!selector[name].optional && selector[name].isRequired !== false) {
+      flag.isRequired = true;
+    }
+    return { ...acc, [name]: flag };
+  }, {});
+}
+function toSelectorString(selectorFlags, delim = ",") {
+  return Object.keys(selectorFlags).sort().map((flag) => {
+    return `${flag}=${selectorFlags[flag]}`;
+  }).join(delim);
+}
+function normalizeSelectorValue(v) {
+  return v.replace(/[^A-Za-z0-9]+/g, "-");
+}
+function getJobName(name, selectorFlags, mode) {
+  const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
+  let jobName = name;
+  if (mode) {
+    jobName += `-${mode}`;
+  }
+  if (selectorNamePart.length > 0) {
+    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
+  }
+  return jobName;
+}
+// src/cli.ts
+import path from "path";
+import fs from "fs";
+function getBinaryName() {
+  const binaryNameParts = process.argv[1].split(path.sep);
+  const binaryName = binaryNameParts[binaryNameParts.length - 1];
+  return binaryName;
+}
+function detectSkynetDirectory() {
+  return detectDirectory(process.argv[1], "SkynetAPIDefinitions.yml");
+}
+function detectWorkingDirectory() {
+  const wd = detectDirectory(process.argv[1], "package.json");
+  const skynetd = detectDirectory(process.argv[1], "SkynetAPIDefinitions.yml");
+  return wd.slice(skynetd.length + path.sep.length).replace(path.sep, "/");
+}
+function detectDirectory(fullBinPath, sentinel = "package.json") {
+  let parentFolder = path.dirname(fullBinPath);
+  while (parentFolder) {
+    const sentinelPath = path.join(parentFolder, sentinel);
+    if (fs.existsSync(sentinelPath)) {
+      return parentFolder;
+    }
+    const newParentFolder = path.dirname(parentFolder);
+    if (newParentFolder === parentFolder) {
+      break;
+    }
+    parentFolder = newParentFolder;
+  }
+  throw new Error("Cannot detect current working directory");
+}
+function detectBin() {
+  const wd = detectDirectory(process.argv[1], "package.json");
+  return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
+}
+// src/indexer.ts
+import meow from "meow";
+var STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
+async function getIndexerLatestId(name, selectorFlags) {
+  const record = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}Since(${toSelectorString(selectorFlags)})`
+  });
+  return record?.value;
+}
+async function getIndexerValidatedId(name, selectorFlags) {
+  const record = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}Validate(${toSelectorString(selectorFlags)})`
+  });
+  if (record) {
+    return record.value;
+  }
+  return;
+}
+function increaseId(type, currentId, n) {
+  if (type === "date") {
+    if (typeof currentId !== "string") {
+      throw new Error("invalid type for date id");
+    }
+    return findDateAfter(currentId, n);
+  }
+  if (typeof currentId !== "number") {
+    throw new Error("Invalid type for numeric id");
+  }
+  return currentId + n;
+}
+function createModeIndexerApp({
+  binaryName,
+  name,
+  selector = {},
+  build,
+  buildBatchSize = 1,
+  buildConcurrency = 1,
+  validate,
+  validateBatchSize = 1,
+  validateConcurrency = 1,
+  maxRetry = 2,
+  state
+}) {
+  const defaultState = {
+    type: "block",
+    getMinId: async () => 1,
+    getMaxId: async () => {
+      throw new Error("must implement getMaxId");
+    }
+  };
+  const finalState = {
+    ...defaultState,
+    ...state
+  };
+  function range2(from, to, step) {
+    if (typeof from === "string" && typeof to === "string") {
+      if (finalState.type === "date") {
+        return dateRange(from, to, step);
+      }
+      throw new Error("Invalid type for numeric range");
+    }
+    if (typeof from === "number" && typeof to === "number") {
+      return range(from, to, step);
+    }
+    throw new Error("Invalid type for range");
+  }
+  function fillRange2(from, to) {
+    if (typeof from === "string" && typeof to === "string") {
+      if (finalState.type === "date") {
+        return daysInRange(from, to);
+      }
+      throw new Error("Invalid type for numeric range");
+    }
+    if (typeof from === "number" && typeof to === "number") {
+      return fillRange(from, to);
+    }
+    throw new Error("Invalid type for range");
+  }
+  function offsetRange(from, to) {
+    return fillRange2(from, to).length;
+  }
+  async function runMode(flags) {
+    const { mode, from: fromUntyped, to: toUntyped, status, verbose: verboseUntyped, ...untypeSelectorFlags } = flags;
+    const from = fromUntyped;
+    const to = toUntyped;
+    const verbose = verboseUntyped;
+    const selectorFlags = untypeSelectorFlags;
+    if (status) {
+      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}RebuildState(${toSelectorString(selectorFlags)})`
+      });
+      const fromItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}Since(${toSelectorString(selectorFlags)})`
+      });
+      const validateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}Validate(${toSelectorString(selectorFlags)})`
+      });
+      inline.log(`RebuildState=${stateItem?.value} Since=${fromItem?.value} Validated=${validateItem?.value}`);
+      process.exit(0);
+    }
+    inline.log(`[MODE INDEXER] mode=${mode}, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`);
+    if (mode === "reset") {
+      await runReset(selectorFlags);
+    } else if (mode === "rebuild") {
+      const rebuildFrom = from || await finalState.getMinId(selectorFlags);
+      const rebuildTo = to || await finalState.getMaxId(selectorFlags);
+      await runReset(selectorFlags);
+      await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
+    } else if (mode === "resume-rebuild") {
+      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+      const rebuildFrom = from || previousRebuildEnds !== undefined && increaseId(finalState.type, previousRebuildEnds, 1) || await finalState.getMinId(selectorFlags);
+      const rebuildTo = to || await finalState.getMaxId(selectorFlags);
+      await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
+    } else if (mode === "validate" || mode === "validation") {
+      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+      if (!previousRebuildEnds) {
+        inline.log(`[MODE INDEXER] cannot validate without a successful rebuild`);
+        process.exit(0);
+      }
+      const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
+      const validateFrom = from || previousValidatedTo || await finalState.getMinId(selectorFlags);
+      const validateTo = to || previousRebuildEnds;
+      const shouldSaveState = !to;
+      await runValidate(selectorFlags, validateFrom, validateTo, shouldSaveState, verbose);
+    } else if (mode === "one") {
+      if (to) {
+        inline.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
+      }
+      if (!from) {
+        inline.log(`[MODE INDEXER] must provide --from option for one mode`);
+        process.exit(1);
+      }
+      await runRange(selectorFlags, from, from, verbose);
+    } else if (mode === "range") {
+      if (!from || !to) {
+        inline.log(`[MODE INDEXER] must provide --from and --to option for range mode`);
+        process.exit(1);
+      }
+      await runRange(selectorFlags, from, to, verbose);
+    } else {
+      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+        name: `${name}RebuildState(${toSelectorString(selectorFlags)})`
+      });
+      if (!stateItem || stateItem.value !== "succeed") {
+        inline.log("[MODE INDEXER] skip because rebuild hasn't done yet");
+        process.exit(0);
+      }
+      const latestId = await getIndexerLatestId(name, selectorFlags);
+      if (!latestId) {
+        throw new Error(`[MODE INDEXER] cannot find the latest ${finalState.type}`);
+      }
+      const deltaFrom = increaseId(finalState.type, latestId, 1);
+      const deltaTo = await state.getMaxId(selectorFlags);
+      await runDelta(selectorFlags, deltaFrom, deltaTo, verbose);
+    }
+  }
+  async function runRange(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] building range, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
+    const failedIds = await execBuild(selectorFlags, from, to, verbose, false);
+    if (failedIds.length > 0) {
+      inline.log(`[MODE INDEXER] built with some failed ${finalState.type}`, failedIds);
+      process.exit(1);
+    } else {
+      inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+      process.exit(0);
+    }
+  }
+  async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
+    if (!validate) {
+      inline.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
+      process.exit(1);
+    }
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] validating, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`);
+    const windows = range2(from, to, validateBatchSize * validateConcurrency);
+    inline.log(`[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`);
+    for (const [windowStart, windowEnd] of windows) {
+      inline.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
+      const batches = range2(windowStart, windowEnd, validateBatchSize);
+      await Promise.all(batches.map(async ([batchStart, batchEnd]) => {
+        const result = await exponentialRetry(async () => {
+          try {
+            await validate({
+              ...selectorFlags,
+              from: batchStart,
+              to: batchEnd,
+              verbose
+            });
+            return true;
+          } catch (err) {
+            inline.error(`got error in validation`, err);
+            return false;
+          }
+        }, {
+          maxRetry,
+          test: (r) => r,
+          verbose
+        });
+        if (!result) {
+          throw new Error(`Terminate validation due to critical errors, from=${batchStart}, to=${batchEnd}`);
+        }
+      }));
+      if (shouldSaveState) {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+          value: to
+        });
+        if (verbose) {
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+        }
+      }
+    }
+    inline.log(`[MODE INDEXER] validated ${offsetRange(from, to)} ${finalState.type} successfully in ${Date.now() - startTime}ms`);
+  }
+  async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
+    let failedIds = [];
+    const windows = range2(from, to, buildBatchSize * buildConcurrency);
+    for (const [windowStart, windowEnd] of windows) {
+      inline.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
+      const batches = range2(windowStart, windowEnd, buildBatchSize);
+      const batchResults = await Promise.all(batches.map(async ([batchStart, batchEnd]) => await exponentialRetry(async () => {
+        try {
+          const ids = await build({
+            ...selectorFlags,
+            from: batchStart,
+            to: batchEnd,
+            verbose
+          });
+          if (ids && ids.length > 0) {
+            return ids;
+          } else {
+            return false;
+          }
+        } catch (err) {
+          inline.error(`[MODE INDEXER] got error in build`, err);
+          return fillRange2(batchStart, batchEnd);
+        }
+      }, {
+        maxRetry,
+        test: (r) => !r,
+        verbose
+      })));
+      if (shouldSaveState) {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: windowEnd
+        });
+        if (verbose) {
+          inline.log(`[MODE INDEXER] updated processed ${finalState.type} to ${windowEnd}`);
+        }
+      }
+      batchResults.forEach((ids) => {
+        if (ids) {
+          failedIds = failedIds.concat(ids);
+        }
+      });
+    }
+    failedIds.sort();
+    return failedIds;
+  }
+  async function runRebuild(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] rebuilding, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "running"
+    });
+    const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Since(${toSelectorString(selectorFlags)})`,
+      value: to
+    });
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "succeed"
+    });
+    if (failedIds.length > 0) {
+      inline.log(`[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) with some failed ${finalState.type}`, failedIds);
+      process.exit(1);
+    } else {
+      inline.log(`[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) successfully in ${Date.now() - startTime}ms`);
+      process.exit(0);
+    }
+  }
+  async function runDelta(selectorFlags, from, to, verbose) {
+    const startTime = Date.now();
+    if (to < from) {
+      inline.log(`[MODE INDEXER] skip delta, there're no more items need to be processed, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
+      return;
+    }
+    inline.log(`[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
+    try {
+      const failedIds = await execBuild(selectorFlags, from, to, verbose, true);
+      if (failedIds.length > 0) {
+        inline.log("[MODE INDEXER] built with some failed txs", failedIds);
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+          value: "failed"
+        });
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: to < failedIds[0] ? to : failedIds[0]
+        });
+        process.exit(1);
+      } else {
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}DeltaState(${toSelectorString(selectorFlags)})`,
+          value: "succeed"
+        });
+        await createRecord(STATE_TABLE_NAME, {
+          name: `${name}Since(${toSelectorString(selectorFlags)})`,
+          value: to
+        });
+        inline.log(`[MODE INDEXER] built successfully in ${Date.now() - startTime}ms`);
+        process.exit(0);
+      }
+    } catch (err) {
+      inline.error("[MODE INDEXER] delta build failed", from, to, err);
+      process.exit(1);
+    }
+  }
+  async function runReset(selectorFlags) {
+    const startTime = Date.now();
+    inline.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
+    inline.log("[MODE INDEXER] reset state", STATE_TABLE_NAME);
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Since(${toSelectorString(selectorFlags)})`,
+      value: 0
+    });
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}Validate(${toSelectorString(selectorFlags)})`,
+      value: 0
+    });
+    await createRecord(STATE_TABLE_NAME, {
+      name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
+      value: "init"
+    });
+    inline.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
+  }
+  async function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+    const cli = meow(`
+        Usage
+
+          $ ${binaryName} <options>
+
+        Options
+${selector ? getSelectorDesc(selector) : ""}
+          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset
+          --from          min ${finalState.type} to build
+          --to            max ${finalState.type} to build
+          --status        print status of indexer and exit
+          --verbose       Output debug messages
+      `, {
+      importMeta: import.meta,
+      description: false,
+      flags: {
+        ...getSelectorFlags(selector),
+        mode: {
+          type: "string",
+          default: "delta"
+        },
+        from: {
+          aliases: ["since"],
+          type: "string"
+        },
+        to: {
+          aliases: ["until"],
+          type: "string"
+        },
+        status: {
+          type: "boolean",
+          default: false
+        },
+        verbose: {
+          type: "boolean",
+          default: false
+        }
+      }
+    });
+    try {
+      return runMode(cli.flags);
+    } catch (err) {
+      inline.error(err);
+      process.exit(1);
+    }
+  }
+  return { run };
+}
+function createIndexerApp({
+  binaryName,
+  selector = {},
+  build,
+  maxRetry = 2
+}) {
+  async function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+    const cli = meow(`
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${selector ? getSelectorDesc(selector) : ""}
+          --verbose       Output debug messages
+      `, {
+      importMeta: import.meta,
+      description: false,
+      flags: {
+        ...getSelectorFlags(selector),
+        verbose: {
+          type: "boolean",
+          default: false
+        }
+      }
+    });
+    async function runBuild(flags) {
+      const { verbose: untypedVerbose, ...untypedSelectorFlags } = flags;
+      const verbose = untypedVerbose;
+      const selectorFlags = untypedSelectorFlags;
+      const startTime = Date.now();
+      if (Object.keys(selectorFlags).length > 0) {
+        inline.log(`[INDEXER] starting build, ${toSelectorString(selectorFlags, ", ")}`);
+      } else {
+        inline.log(`[INDEXER] starting build`);
+      }
+      const result = await exponentialRetry(async () => {
+        try {
+          await build(flags);
+          return true;
+        } catch (err) {
+          inline.log(`[INDEXER] got error in build`, err);
+          return false;
+        }
+      }, {
+        maxRetry,
+        test: (r) => r,
+        verbose
+      });
+      if (!result) {
+        throw new Error(`[INDEXER] Build failed due to critical errors`);
+      }
+      inline.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
+    }
+    return runBuild(cli.flags).catch((err) => {
+      inline.error(err);
+      process.exit(1);
+    });
+  }
+  return { run };
+}
+// src/deploy.ts
+import fs2 from "fs/promises";
+import fso from "fs";
+import { execa } from "execa";
+import meow2 from "meow";
+import chalk from "chalk";
+import which from "which";
+var INTERVAL_ALIASES = {
+  secondly: "*/1 * * * * * *",
+  "@secondly": "*/1 * * * * * *",
+  minutely: "0 * * * * * *",
+  "@minutely": "0 * * * * * *",
+  hourly: "0 0 * * * * *",
+  "@hourly": "0 0 * * * * *",
+  daily: "0 0 0 * * * *",
+  "@daily": "0 0 0 * * * *",
+  weekly: "0 0 0 * * 0 *",
+  "@weekly": "0 0 0 * * 0 *"
+};
+var genConfig = ({
+  jobName,
+  workingDirectory,
+  cmd,
+  cron,
+  count,
+  restart,
+  killTimeout,
+  cpu,
+  mem,
+  service,
+  additionalEnv = {},
+  type = "batch",
+  region = "skynet-dc1",
+  isProduction: isProduction2
+}) => `job "${jobName}" {
+  datacenters = ["${region}"]
+
+  type = "${type}"
+
+  ${cron ? `# Triggers periodically
+  periodic {
+    crons            = ["${cron}"]
+    prohibit_overlap = true
+  }` : ""}
+
+  constraint {
+    attribute = "\${meta.has_nodejs}"
+    value     = "true"
+  }
+
+  constraint {
+    attribute = "\${meta.has_skynet}"
+    value     = "true"
+  }
+
+  group "default" {
+    ${count && count > 1 ? `count = ${count}` : ""}
+    ${count && count > 1 ? `# Rolling Update
+    update {
+      max_parallel = 1
+      min_healthy_time = "10s"
+    }` : ""}
+
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    ${service ? `# Setup Service Network
+    network {
+      port "http" {
+        static = ${service.port}
+      }
+    }` : ""}
+
+    task "main" {
+      driver = "raw_exec"
+
+      config {
+        command = "sh"
+        args = [
+          "-c",
+          "cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi && exec ${cmd}"
+        ]
+      }
+
+      ${service ? `# Setup API Routes
+      service {
+        name = "${jobName}"
+        port = "http"
+
+        tags = [
+          "urlprefix-${service.prefix} strip=${service.prefix}",
+        ]
+
+        check {
+          type     = "http"
+          path     = "/"
+          interval = "10s"
+          timeout  = "2s"
+        }
+      }
+` : ""}
+
+      # doppler integration support
+      # it is always there but a project can decide to not use it
+      template {
+        change_mode = "restart"
+        destination = "secrets/context.env"
+        env         = true
+        data        = "DOPPLER_TOKEN={{key \\"infra-nomad/doppler-token\\"}}"
+      }
+
+      # always update SKYNET_DEPLOYED_AT so that new deployment always triggers
+      env {
+        SKYNET_DEPLOYED_AT="${new Date().toISOString()}"
+        HOME="/root"
+        DOPPLER_PROJECT="${workingDirectory}"
+        DOPPLER_CONFIG="${isProduction2 ? "prd" : "dev"}"
+        SKYNET_ENVIRONMENT="${isProduction2 ? "prd" : "dev"}"
+        ${Object.entries(additionalEnv).filter((kv) => !!kv[1]).map(([key, value]) => `${key}="${value}"`).join(`          
+`)}
+      }
+
+      kill_timeout = "${killTimeout || "60s"}"
+
+      # Specify the maximum resources required to run the task,
+      # include CPU and memory.
+      resources {
+        cpu    = ${cpu} # MHz
+        memory = ${mem} # MB
+      }
+
+      # Setting the server task as the leader of the task group allows Nomad to
+      # signal the log shipper task to gracefully shutdown when the server exits.
+      leader = true
+
+      ${restart ? `
+      # Restart the job if it fails
+      restart {
+        attempts = ${restart.attempts ?? 2}
+        mode     = "${restart.mode ?? "fail"}"
+        interval = "${restart.interval ?? "30m"}"
+        delay    = "${restart.delay ?? "15s"}"
+      }
+      ` : `
+      # do not retry from the periodical job will reschedule anyway
+      restart {
+        attempts = 0
+        mode     = "fail"
+      }`}
+    }
+  }
+}`;
+async function prepareNomad(isProduction2) {
+  if (isProduction2) {
+    console.log("Deploy to Production");
+  } else {
+    const skynetDir = detectSkynetDirectory();
+    if (!fso.existsSync("/tmp/skynet")) {
+      await execa("ln", ["-s", skynetDir, "/tmp/skynet"]);
+    }
+    console.log("Deploy locally, please start nomad server in a separate terminal");
+    console.log(`You can start nomad server by running ${chalk.inverse(`${skynetDir}/infra-nomad/dev/start.sh`)}`);
+    console.log(`Then you can visit ${chalk.underline("http://localhost:4646/ui/jobs")} to check submitted dev jobs.
+`);
+  }
+}
+function getNomadAddr(isProduction2) {
+  return isProduction2 ? getEnvOrThrow("NOMAD_ADDR") : "http://127.0.0.1:4646";
+}
+async function getNomadPath() {
+  try {
+    return await which("nomad");
+  } catch (missingNomad) {
+    console.log(`Deploy requires ${chalk.bold("nomad")} binary, please follow ${chalk.underline("https://learn.hashicorp.com/tutorials/nomad/get-started-install")} for installation`, missingNomad);
+    throw new Error("missing nomad binary");
+  }
+}
+async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, isStop, isDryRun) {
+  try {
+    if (isStop) {
+      const nomad = execa(nomadPath, ["job", "stop", jobName], {
+        env: {
+          NOMAD_ADDR: nomadAddr
+        }
+      });
+      nomad.stdout.pipe(process.stdout);
+      await nomad;
+      console.log(chalk.green(`Stopped nomad job ${jobName} in ${nomadAddr}`));
+    } else if (isDryRun) {
+      console.log("Definition for", jobName);
+      console.log("========================================");
+      console.log(nomadJobDefinition);
+    } else {
+      const jobFileName = `/tmp/job-${jobName}`;
+      await fs2.writeFile(jobFileName, nomadJobDefinition);
+      const nomad = execa(nomadPath, ["job", "run", jobFileName], {
+        env: {
+          NOMAD_ADDR: nomadAddr
+        }
+      });
+      nomad.stdout.pipe(process.stdout);
+      await nomad;
+      console.log(chalk.green(`Deployed nomad job ${jobName} to ${nomadAddr}`));
+    }
+  } catch (nomadExecErr) {
+    if (nomadExecErr instanceof Error) {
+      console.log("Nomad Execution Error:");
+      console.log(nomadExecErr.message);
+      console.log("");
+    }
+    console.log(`Failed to run ${chalk.bold("nomad")} commands, please ensure nomad server is accessible at ${chalk.bold(nomadAddr)}`);
+    throw new Error("nomad execution error");
+  }
+}
+function createModeDeploy({
+  binaryName,
+  name,
+  workingDirectory,
+  bin = "bin/indexer",
+  selector = {},
+  env = {},
+  region = "skynet-dc1",
+  deltaSchedule,
+  validateSchedule,
+  deltaKillTimeout,
+  deltaCpu,
+  deltaMem,
+  rebuildKillTimeout,
+  rebuildCpu,
+  rebuildMem,
+  validateKillTimeout,
+  validateCpu,
+  validateMem
+}) {
+  async function deployMode({
+    mode,
+    from,
+    to,
+    stop,
+    production,
+    dryRun,
+    verbose,
+    schedule: cmdSchedule,
+    ...selectorFlags
+  }) {
+    if (mode === "delta") {
+      from = 0;
+      to = 0;
+    }
+    const isPeriodic = from === 0 && to === 0 && ["delta", "validate"].includes(mode);
+    const jobName = getJobName(name, selectorFlags, mode);
+    const selectorCmdPart = Object.entries(selectorFlags).sort().map(([name2, value]) => `--${name2} ${value}`).join(" ");
+    let args = `--mode ${mode} ${selectorCmdPart}`;
+    if (verbose) {
+      args += ` --verbose`;
+    }
+    let rangeArgs = "";
+    if (from > 0) {
+      rangeArgs += ` --from ${from}`;
+    }
+    if (to > 0) {
+      rangeArgs += ` --to ${to}`;
+    }
+    const modeResouces = {
+      rebuild: { cpu: rebuildCpu, mem: rebuildMem, killTimeout: rebuildKillTimeout },
+      "resume-rebuild": { cpu: rebuildCpu, mem: rebuildMem, killTimeout: rebuildKillTimeout },
+      validate: {
+        cpu: validateCpu || rebuildCpu,
+        mem: validateMem || rebuildMem,
+        killTimeout: validateKillTimeout || rebuildKillTimeout
+      },
+      delta: { cpu: deltaCpu, mem: deltaMem, killTimeout: deltaKillTimeout }
+    };
+    const cpu = modeResouces[mode]?.cpu || deltaCpu;
+    const mem = modeResouces[mode]?.mem || deltaMem;
+    const killTimeout = modeResouces[mode]?.killTimeout || deltaKillTimeout;
+    let deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
+    if (deltaSchedule && cmdSchedule) {
+      deltaCron = cmdSchedule;
+    }
+    let validateCron = typeof validateSchedule === "function" ? validateSchedule(jobName) : validateSchedule;
+    if (validateSchedule && cmdSchedule) {
+      validateCron = cmdSchedule;
+    }
+    const modeIntervals = {
+      delta: deltaCron ? INTERVAL_ALIASES[deltaCron] || deltaCron : undefined,
+      validate: validateCron ? INTERVAL_ALIASES[validateCron] || validateCron : undefined
+    };
+    const mainJobDefinition = genConfig({
+      jobName,
+      cron: isPeriodic ? modeIntervals[mode] : undefined,
+      workingDirectory,
+      additionalEnv: env,
+      region,
+      cmd: `${bin} ${args} ${rangeArgs}`,
+      killTimeout,
+      cpu,
+      mem,
+      isProduction: production
+    });
+    const nomadPath = await getNomadPath();
+    await prepareNomad(production);
+    const nomadAddr = getNomadAddr(production);
+    await runNomadJob(nomadPath, nomadAddr, jobName, mainJobDefinition, stop, dryRun);
+  }
+  async function deploy() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+    const cli = meow2(`
+        Usage
+
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset
+          --from          min id to build
+          --to            max id to build
+          --stop          stop job instead of running the job
+          --production    deploy to production, default is development
+          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
+          --verbose       Output debug messages
+          --dry-run       print nomad job file but do not really execute it
+
+        Examples
+          ${binaryName} --mode delta
+          ${binaryName} --mode rebuild
+          ${binaryName} --mode validate
+      `, {
+      importMeta: import.meta,
+      description: false,
+      flags: {
+        ...getSelectorFlags(selector),
+        mode: {
+          type: "string",
+          default: "delta"
+        },
+        from: {
+          aliases: ["since"],
+          type: "number",
+          default: 0
+        },
+        to: {
+          aliases: ["until"],
+          type: "number",
+          default: 0
+        },
+        schedule: {
+          type: "string"
+        },
+        verbose: {
+          type: "boolean",
+          default: false
+        },
+        production: {
+          aliases: ["prd"],
+          type: "boolean",
+          default: false
+        },
+        dryRun: {
+          type: "boolean",
+          default: false
+        },
+        stop: {
+          type: "boolean",
+          default: false
+        }
+      }
+    });
+    try {
+      return deployMode(cli.flags);
+    } catch (err) {
+      console.error(err);
+      process.exit(1);
+    }
+  }
+  return { deploy };
+}
+function createDeploy({
+  binaryName,
+  name,
+  workingDirectory,
+  bin = "bin/indexer",
+  selector = {},
+  region = "skynet-dc1",
+  type = "batch",
+  env = {},
+  count,
+  schedule,
+  restart,
+  killTimeout,
+  cpu,
+  mem,
+  service
+}) {
+  async function deployModeless({
+    production,
+    stop,
+    dryRun,
+    verbose,
+    schedule: cmdSchedule,
+    ...selectorFlags
+  }) {
+    const jobName = getJobName(name, selectorFlags);
+    const selectorCmdPart = Object.entries(selectorFlags).sort().map(([name2, value]) => `--${name2} ${value}`).join(" ");
+    let args = `${selectorCmdPart}`;
+    if (verbose) {
+      args += ` --verbose`;
+    }
+    let cron = typeof schedule === "function" ? schedule(jobName) : schedule;
+    if (schedule && cmdSchedule) {
+      cron = cmdSchedule;
+    }
+    const nomadJobDefinition = genConfig({
+      jobName,
+      cron: cron ? INTERVAL_ALIASES[cron] || cron : undefined,
+      count,
+      restart,
+      workingDirectory,
+      additionalEnv: env,
+      region,
+      type,
+      cmd: `${bin} ${args}`,
+      killTimeout,
+      cpu,
+      mem,
+      service,
+      isProduction: production
+    });
+    const nomadPath = await getNomadPath();
+    await prepareNomad(production);
+    const nomadAddr = getNomadAddr(production);
+    await runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, stop, dryRun);
+  }
+  async function deploy() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+    const cli = meow2(`
+        Usage
+
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --stop          stop job instead of running the job
+          --production    deploy to production, default is development
+          --schedule      override default schedule, support aliases: secondly, minutely, hourly, daily, weekly
+          --verbose       Output debug messages
+          --dry-run       print nomad job file but do not really execute it
+      `, {
+      importMeta: import.meta,
+      description: false,
+      flags: {
+        ...getSelectorFlags(selector),
+        schedule: {
+          type: "string"
+        },
+        verbose: {
+          type: "boolean",
+          default: false
+        },
+        production: {
+          aliases: ["prd"],
+          type: "boolean",
+          default: false
+        },
+        dryRun: {
+          type: "boolean",
+          default: false
+        },
+        stop: {
+          type: "boolean",
+          default: false
+        }
+      }
+    });
+    try {
+      return deployModeless(cli.flags);
+    } catch (err) {
+      console.error(err);
+      process.exit(1);
+    }
+  }
+  return { deploy };
+}
+// src/api.ts
+import osModule from "os";
+import express from "express";
+import meow3 from "meow";
+async function logStartMiddleware(_, res, next) {
+  const start = new Date;
+  res.set("x-requested-at", start.toISOString());
+  next();
+}
+async function contextMiddleware(_, res, next) {
+  res.set("x-instance-id", osModule.hostname());
+  next();
+}
+async function logEndMiddleware(req, res, next) {
+  const requestedAt = res.get("x-requested-at");
+  if (!requestedAt) {
+    inline.log("missing x-requested-at header");
+    next();
+    return;
+  }
+  const start = new Date(requestedAt).getTime();
+  const end = new Date().getTime();
+  const logInfo = {
+    start,
+    end,
+    elapsed: `${end - start}ms`,
+    endpoint: req.path,
+    host: req.hostname,
+    status: res.statusCode
+  };
+  if (res.statusMessage) {
+    logInfo.errorMessage = res.statusMessage;
+  }
+  inline.log(JSON.stringify(logInfo));
+  next();
+}
+var apiKeyMiddleware = (key) => {
+  async function requireAPIKey(req, res, next) {
+    try {
+      const apiKey = req.get("x-api-key") || req.query["api-key"];
+      if (!apiKey) {
+        inline.log("request without api key");
+        res.status(400).send("require x-api-key header");
+        return;
+      }
+      if (typeof key === "string") {
+        if (apiKey !== key) {
+          inline.log("request has an invalid api key");
+          res.status(400).send("invalid api key");
+          return;
+        }
+        inline.log(`requested by valid key ${key.slice(0, 6)}`);
+      } else {
+        const name = key[apiKey];
+        if (!name) {
+          inline.log("request has an invalid api key");
+          res.status(400).send("invalid api key");
+          return;
+        }
+        inline.log(`requested by authorized user ${name}`);
+      }
+      next();
+    } catch (err) {
+      inline.log("check api key error", err);
+      res.status(500).send("internal error");
+    }
+  }
+  return requireAPIKey;
+};
+async function startApiApp({
+  binaryName,
+  name,
+  selector = {},
+  routes,
+  serve,
+  beforeListen
+}) {
+  const app = express();
+  app.use(express.json({ limit: "20mb" }));
+  const cli = meow3(`
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --verbose       Output debug messages
+      `, {
+    importMeta: import.meta,
+    description: false,
+    flags: {
+      ...getSelectorFlags(selector),
+      verbose: {
+        type: "boolean",
+        default: false
+      }
+    }
+  });
+  const { verbose, ...selectorFlags } = cli.flags;
+  for (const route of routes) {
+    const method = route.method ? route.method.toLowerCase() : "get";
+    const middlewares = route.middlewares || [];
+    if (route.protected) {
+      if (!serve.apiKey) {
+        throw new Error("serve.apiKey is required for protected route");
+      }
+      middlewares.unshift(apiKeyMiddleware(serve.apiKey));
+    }
+    if (app[method]) {
+      if (verbose) {
+        inline.log(`registering ${method} ${route.path}`);
+      }
+      app[method](route.path, contextMiddleware, logStartMiddleware, ...middlewares, async (req, res, next) => {
+        try {
+          await route.handler({ req, res, ...selectorFlags });
+        } catch (routeErr) {
+          if (routeErr instanceof Error) {
+            inline.log("caught route err", routeErr, routeErr.stack);
+            res.status(500).send(`internal server error: ${routeErr.message}`);
+          } else {
+            inline.log("caught route err", routeErr);
+            res.status(500).send("internal server error");
+          }
+        }
+        next();
+      }, logEndMiddleware);
+    }
+  }
+  if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
+    app.get("/", (_, res) => {
+      res.send("ok");
+    });
+  }
+  if (beforeListen) {
+    await beforeListen({ app });
+  }
+  app.listen(serve.port, () => {
+    if (isProduction()) {
+      inline.log(`${name} listening at https://api.wf.corp.certik.com${serve.prefix}`);
+    } else {
+      inline.log(`${name} listening at http://localhost:${serve.port}`);
+    }
+  });
+}
+
+// src/app.ts
+import { EOL } from "os";
+function printAppHelp() {
+  console.log(`
+    Usage
+
+      $ ${getBinaryName()} run <options>
+      $ ${getBinaryName()} deploy <options>
+      $ ${getBinaryName()} delete <options>
+  `);
+}
+function isDeleteCommand(command) {
+  return ["delete", "stop", "remove"].includes(command);
+}
+function checkAndSetEnv(env) {
+  const missingEnvs = [];
+  for (const key of Object.keys(env)) {
+    if (env[key]) {
+      process.env[key] = env[key];
+    } else if (!process.env[key]) {
+      missingEnvs.push(key);
+    }
+  }
+  if (missingEnvs.length > 0) {
+    console.log(`The following environment value shouldn't be empty:${EOL}- ${missingEnvs.join(EOL + "- ")}`);
+    process.exit(1);
+  }
+}
+function createApp({
+  parameterErrors,
+  env,
+  onRun,
+  onDeploy
+}) {
+  if (parameterErrors.length > 0) {
+    console.log(`Parameter Validation Failed:${EOL}- ${parameterErrors.join(EOL + "- ")}`);
+    process.exit(1);
+  }
+  return async () => {
+    const subCommand = process.argv[2];
+    process.argv = [process.argv[0], process.argv[1], ...process.argv.slice(3)];
+    if (subCommand === "run") {
+      checkAndSetEnv(env);
+      await onRun();
+    } else if (subCommand === "deploy" || isDeleteCommand(subCommand)) {
+      if (isDeleteCommand(subCommand)) {
+        process.argv.push("--stop");
+      }
+      await onDeploy();
+    } else {
+      printAppHelp();
+    }
+  };
+}
+function checkEnvParameter(env) {
+  const errors = [];
+  const envKeys = Object.keys(env);
+  envKeys.forEach((k) => {
+    if (!env[k] && env[k] !== SENSITIVE_VALUE) {
+      errors.push(`must have valid non-empty value for env.${k}`);
+    }
+  });
+  return errors;
+}
+function checkIndexerBuildParameter(build) {
+  const errors = [];
+  if (!build?.func) {
+    errors.push("must define build.func");
+  }
+  if (!build?.cpu) {
+    errors.push("must define build.cpu");
+  }
+  if (!build?.mem) {
+    errors.push("must define build.mem");
+  }
+  return errors;
+}
+function checkStateParameter(state) {
+  const errors = [];
+  if (!state?.type) {
+    errors.push("must define state.type");
+  }
+  if (!state?.getMaxId) {
+    errors.push("must define state.getMaxId");
+  }
+  return errors;
+}
+function indexer({
+  name,
+  selector,
+  build,
+  env = {},
+  region = "skynet-dc1"
+}) {
+  return createApp({
+    parameterErrors: [...checkIndexerBuildParameter(build), ...checkEnvParameter(env)],
+    env,
+    onRun: () => {
+      const { run } = createIndexerApp({
+        binaryName: `${getBinaryName()} run`,
+        selector,
+        build: build.func,
+        maxRetry: build.maxRetry
+      });
+      process.title = name;
+      return run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+      const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
+        selector,
+        region,
+        env,
+        schedule: build.schedule,
+        restart: build.restart,
+        killTimeout: build.killTimeout,
+        cpu: build.cpu,
+        mem: build.mem
+      });
+      return deploy();
+    }
+  });
+}
+function checkModeIndexerBuildParameter(build) {
+  const errors = [];
+  if (!build?.func) {
+    errors.push("must define build.func");
+  }
+  if (!build?.cpu) {
+    errors.push("must define build.cpu");
+  }
+  if (!build?.mem) {
+    errors.push("must define build.mem");
+  }
+  return errors;
+}
+function checkModeIndexerValidateParameter(validate) {
+  const errors = [];
+  if (validate) {
+    if (!validate.func) {
+      errors.push("must define validate.func");
+    }
+    if (!validate.cpu) {
+      errors.push("must define validate.cpu");
+    }
+    if (!validate.mem) {
+      errors.push("must define validate.mem");
+    }
+  }
+  return errors;
+}
+function modeIndexer({
+  name,
+  selector,
+  state,
+  build,
+  validate,
+  env = {},
+  region = "skynet-dc1"
+}) {
+  return createApp({
+    parameterErrors: [
+      ...checkModeIndexerBuildParameter(build),
+      ...checkModeIndexerValidateParameter(validate),
+      ...checkStateParameter(state),
+      ...checkEnvParameter(env)
+    ],
+    env,
+    onRun: () => {
+      const { run } = createModeIndexerApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        build: build.func,
+        maxRetry: build.maxRetry,
+        buildBatchSize: build.batchSize,
+        buildConcurrency: build.concurrency,
+        validate: validate && validate.func,
+        validateBatchSize: validate && validate.batchSize,
+        validateConcurrency: validate && validate.concurrency,
+        state
+      });
+      process.title = name;
+      return run();
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+      const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
+      const { deploy } = createModeDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
+        selector,
+        region,
+        env,
+        deltaSchedule: build.schedule,
+        deltaKillTimeout: build.killTimeout,
+        deltaCpu: build.cpu,
+        deltaMem: build.mem,
+        rebuildKillTimeout: build.killTimeout,
+        rebuildCpu: build.cpu,
+        rebuildMem: build.mem,
+        validateSchedule: validate && validate.schedule,
+        validateKillTimeout: validate && validate.killTimeout,
+        validateCpu: validate && validate.cpu,
+        validateMem: validate && validate.mem
+      });
+      return deploy();
+    }
+  });
+}
+function checkApiServeParameter(serve, routes) {
+  const errors = [];
+  if (!serve?.prefix) {
+    errors.push("must define serve.prefix");
+  } else if (!serve.prefix.startsWith("/")) {
+    errors.push("server.prefix must start with /, e.g. /my-api");
+  }
+  if (!serve?.port) {
+    errors.push("must define serve.port");
+  }
+  if (!serve?.cpu) {
+    errors.push("must define serve.cpu");
+  }
+  if (!serve?.mem) {
+    errors.push("must define serve.mem");
+  }
+  if (routes.some((r) => r.protected) && !serve.apiKey) {
+    errors.push("must define serve.apiKey since some routes are protected");
+  }
+  return errors;
+}
+function checkApiRoutesParameter(routes) {
+  const errors = [];
+  if (!Array.isArray(routes)) {
+    errors.push("routes must be an array");
+  } else {
+    for (let i = 0;i < routes.length; i++) {
+      const route = routes[i];
+      if (!route.path) {
+        errors.push(`routes[${i}] must define path`);
+      }
+      if (!route.handler) {
+        errors.push(`routes[${i}] must define handler`);
+      }
+      if (route.middlewares && !Array.isArray(route.middlewares)) {
+        errors.push(`routes[${i}].middlewares must be an array`);
+      }
+    }
+  }
+  return errors;
+}
+function api({
+  name,
+  routes,
+  serve,
+  beforeListen,
+  env = {},
+  region = "skynet-dc1"
+}) {
+  const selector = {};
+  return createApp({
+    parameterErrors: [
+      ...checkApiRoutesParameter(routes),
+      ...checkApiServeParameter(serve, routes),
+      ...checkEnvParameter(env)
+    ],
+    env,
+    onRun: () => {
+      process.title = name;
+      return startApiApp({
+        binaryName: `${getBinaryName()} run`,
+        name,
+        selector,
+        routes,
+        serve,
+        beforeListen
+      });
+    },
+    onDeploy: () => {
+      const bin = detectBin();
+      const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
+      const { deploy } = createDeploy({
+        binaryName: `${getBinaryName()} deploy`,
+        name,
+        workingDirectory: detectWorkingDirectory(),
+        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
+        selector,
+        region,
+        env,
+        type: "service",
+        restart: {
+          attempts: 3,
+          delay: "15s",
+          mode: "delay",
+          interval: "2m"
+        },
+        count: serve.instances,
+        killTimeout: serve.killTimeout,
+        cpu: serve.cpu,
+        mem: serve.mem,
+        service: {
+          prefix: serve.prefix,
+          port: serve.port
+        }
+      });
+      return deploy();
+    }
+  });
+}
+var SENSITIVE_VALUE = null;
+var every = (n = 1) => {
+  if (n === 1) {
+    return {
+      second: "*/1 * * * * * *",
+      seconds: "*/1 * * * * * *",
+      minute: "0 * * * * * *",
+      minutes: "0 * * * * * *",
+      hour: "0 0 * * * * *",
+      hours: "0 0 * * * * *",
+      day: "0 0 0 * * * *",
+      days: "0 0 0 * * * *",
+      week: "0 0 0 * * 0 *",
+      weeks: "0 0 0 * * 0 *"
+    };
+  }
+  return {
+    second: `*/${n} * * * * * *`,
+    seconds: `*/${n} * * * * * *`,
+    minute: `0 */${n} * * * * *`,
+    minutes: `0 */${n} * * * * *`,
+    hour: `0 0 */${n} * * * *`,
+    hours: `0 0 */${n} * * * *`,
+    day: `0 0 0 */${n} * * *`,
+    days: `0 0 0 */${n} * * *`
+  };
+};
+export {
+  modeIndexer,
+  indexer,
+  every,
+  api,
+  SENSITIVE_VALUE
+};