@certik/skynet 0.18.9 → 0.19.0

package/examples/{mode-indexer.js → mode-indexer.ts}

@@ -8,13 +8,19 @@
 //   $ examples/mode-indexer deploy --protocol eth
 //   $ examples/mode-indexer --help
 
-import { modeIndexer, every, SENSITIVE_VALUE } from "../app.js";
+import { modeIndexer, every } from "../app.ts";
 
-async function build({ protocol, from, to, verbose }) {
-  console.log("build called with", protocol, from, to, verbose);
-}
-
-async function validate({ protocol, from, to, verbose }) {
+async function validate({
+  protocol,
+  from,
+  to,
+  verbose,
+}: {
+  protocol?: string;
+  from?: string;
+  to?: string;
+  verbose?: boolean;
+}) {
   console.log("validate called with", protocol, from, to, verbose);
 }
 
@@ -33,14 +39,16 @@ const app = modeIndexer({
 
   state: {
     type: "date", // can be omitted, default is block
-    getMinId: async () => "2022-09-26", // default returns 1
-    getMaxId: async ({ protocol }) => {
-      return "2022-10-18";
+    getMinId: async () => "2022-09-26" as string,
+    getMaxId: async () => {
+      return "2022-10-18" as string;
     },
   },
 
   build: {
-    func: build,
+    func: ({ protocol, from, to, verbose }) => {
+      console.log("build called with", protocol, from, to, verbose);
+    },
     batchSize: 5,
     concurrency: 2, // default is 1, no concurrency
 

package/graphql.ts

@@ -0,0 +1,43 @@
+type GraphqlVariables = { [key: string]: unknown };
+
+type GraphQLError = {
+  message: string;
+  locations?: { line: number; column: number }[];
+  path?: string[];
+  extensions?: Record<string, unknown>;
+};
+
+type GraphQLResponse<TData> = {
+  data: TData;
+  errors?: GraphQLError[];
+};
+
+export async function gql<T>(query: string, variables: GraphqlVariables = {}) {
+  const endpoint = process.env["SKYNET_GRAPHQL_ENDPOINT"];
+  const apiKey = process.env["SKYNET_GRAPHQL_API_KEY"];
+
+  if (!endpoint || !apiKey) {
+    throw new Error("SKYNET_GRAPHQL_ENDPOINT or SKYNET_GRAPHQL_API_KEY is not set");
+  }
+
+  const res = await fetch(endpoint, {
+    method: "POST",
+    headers: {
+      "x-api-key": apiKey,
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({ query: query.trim(), variables }),
+  });
+
+  if (res.ok) {
+    const { data, errors }: GraphQLResponse<T> = await res.json();
+
+    if (errors && errors.length > 0) {
+      throw new Error(JSON.stringify(errors, null, 2));
+    }
+
+    return data;
+  } else {
+    throw new Error(await res.text());
+  }
+}

package/{indexer.js → indexer.ts}

@@ -1,41 +1,78 @@
 import meow from "meow";
-import { createRecord, getRecordByKey } from "./dynamodb.js";
-import { getEnvironment } from "./env.js";
-import { exponentialRetry } from "./availability.js";
-import { range as numberRange, fillRange as fillNumberRange } from "./util.js";
-import { getSelectorFlags, getSelectorDesc, toSelectorString } from "./selector.js";
-import { getBinaryName } from "./cli.js";
-import { inline } from "./log.js";
-import { findDateAfter, dateRange, daysInRange as fillDateRange } from "./date.js";
+import { createRecord, getRecordByKey } from "./dynamodb.ts";
+import { getEnvironment } from "./env.ts";
+import { exponentialRetry } from "./availability.ts";
+import { range as numberRange, fillRange as fillNumberRange } from "./util.ts";
+import { getSelectorDesc, getSelectorFlags, toSelectorString } from "./selector.ts";
+import type { SelectorFlags, Selector } from "./selector.ts";
+import { getBinaryName } from "./cli.ts";
+import { inline } from "./log.ts";
+import { findDateAfter, dateRange, daysInRange as fillDateRange } from "./date.ts";
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
-async function getIndexerState(name, selectorFlags) {
-  const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
-    name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
-  });
-
-  if (!stateItem) {
-    return "N/A";
+type IndexerStateValue = string | number;
+type RebuildStateValue = "init" | "running" | "succeed" | "failed";
+
+type StatelessIndexerFlags<TSelector extends Selector> = SelectorFlags<
+  TSelector & { verbose: { type: "boolean"; default: false } }
+>;
+type StatelessBuildFunction<TSelector extends Selector> = ({
+  verbose,
+  ...selectorFlags
+}: StatelessIndexerFlags<TSelector>) => Promise<void> | void;
+
+type ModeIndexerFlags<TSelector extends Selector> = SelectorFlags<
+  TSelector & {
+    verbose: { type: "boolean"; default: false };
+    mode: { type: "string"; default: "delta" };
+    from: { aliases: ["since"]; type: "string" };
+    to: { aliases: ["until"]; type: "string" };
+    status: { type: "boolean"; default: false };
   }
+>;
+type ModeBuildFunction<T extends IndexerStateValue, TSelector extends Selector> = ({
+  from,
+  to,
+  verbose,
+  ...selectorFlags
+}: { from: T; to: T; verbose: boolean } & ModeIndexerFlags<TSelector>) => Promise<void | T[]> | void | T[];
+
+type ValidateFunction<TSelector extends Selector> = ({
+  from,
+  to,
+  verbose,
+  ...selectorFlags
+}: ModeIndexerFlags<TSelector>) => Promise<void> | void;
+
+type State<T extends IndexerStateValue, TSelector extends Selector> = {
+  type: string;
+  updateInterval?: () => number;
+  getMinId: (selectorFlags: ModeIndexerFlags<TSelector>) => Promise<T>;
+  getMaxId: (selectorFlags: ModeIndexerFlags<TSelector>) => Promise<T>;
+};
 
-  return stateItem.value;
-}
+type IndexerState<T> = {
+  name: string;
+  value: T;
+};
 
-async function getIndexerLatestId(name, selectorFlags) {
-  const record = await getRecordByKey(STATE_TABLE_NAME, {
+async function getIndexerLatestId<T extends IndexerStateValue, TSelector extends Selector>(
+  name: string,
+  selectorFlags: ModeIndexerFlags<TSelector>,
+) {
+  const record = await getRecordByKey<IndexerState<T>>(STATE_TABLE_NAME, {
     name: `${name}Since(${toSelectorString(selectorFlags)})`,
   });
 
-  if (record) {
-    return record.value;
-  }
-
-  return 0;
+  return record?.value;
 }
 
-async function getIndexerValidatedId(name, selectorFlags) {
-  const record = await getRecordByKey(STATE_TABLE_NAME, {
+async function getIndexerValidatedId<T extends IndexerStateValue, TSelector extends Selector>(
+  name: string,
+  selectorFlags: ModeIndexerFlags<TSelector>,
+) {
+  const record = await getRecordByKey<IndexerState<T>>(STATE_TABLE_NAME, {
     name: `${name}Validate(${toSelectorString(selectorFlags)})`,
   });
 
@@ -43,25 +80,33 @@ async function getIndexerValidatedId(name, selectorFlags) {
     return record.value;
   }
 
-  return 0;
+  return undefined;
 }
 
-function increaseId(type, currentId, n) {
+function increaseId<T extends IndexerStateValue = IndexerStateValue>(type: string, currentId: T, n: number): T {
   if (type === "date") {
-    return findDateAfter(currentId, n);
+    if (typeof currentId !== "string") {
+      throw new Error("invalid type for date id");
+    }
+
+    return findDateAfter(currentId, n) as T;
+  }
+
+  if (typeof currentId !== "number") {
+    throw new Error("Invalid type for numeric id");
   }
 
-  return currentId + n;
+  return (currentId + n) as T;
 }
 
 // for those indexers that can have progress tracked by a numeric state.type
 // such as block height, or timestamp
 // managing state would be helpful to reduce the build time
 // and avoid unnecessary computation & storage
-function createModeIndexerApp({
+function createModeIndexerApp<T extends IndexerStateValue, TSelector extends Selector>({
   binaryName,
   name,
-  selector = {},
+  selector = {} as TSelector,
   build,
   // number of items run in a batch, determines the { from, to } to the build function
   buildBatchSize = 1,
@@ -76,48 +121,86 @@ function createModeIndexerApp({
   // commit updates every rolling window = validateBatchSize * validateConcurrency
   maxRetry = 2,
   state,
+}: {
+  binaryName: string;
+  name: string;
+  selector: TSelector;
+  build: ModeBuildFunction<T, TSelector>;
+  maxRetry?: number;
+  buildBatchSize?: number;
+  buildConcurrency?: number;
+  validate?: ValidateFunction<TSelector>;
+  validateBatchSize?: number;
+  validateConcurrency?: number;
+  state: State<T, TSelector>;
 }) {
-  const finalState = {
+  const defaultState = {
     type: "block",
     getMinId: async () => 1,
     getMaxId: async () => {
       throw new Error("must implement getMaxId");
     },
+  };
+  const finalState = {
+    ...defaultState,
     ...state,
   };
 
   // type based range functions
-  function range(from, to, step) {
-    if (finalState.type === "date") {
-      return dateRange(from, to, step);
+  function range<T extends IndexerStateValue>(from: T, to: T, step: number) {
+    if (typeof from === "string" && typeof to === "string") {
+      if (finalState.type === "date") {
+        return dateRange(from, to, step) as [T, T][];
+      }
+
+      throw new Error("Invalid type for numeric range");
+    }
+
+    if (typeof from === "number" && typeof to === "number") {
+      return numberRange(from, to, step) as [T, T][];
     }
 
-    return numberRange(from, to, step);
+    throw new Error("Invalid type for range");
   }
 
-  function fillRange(from, to) {
-    if (finalState.type === "date") {
-      return fillDateRange(from, to);
+  function fillRange(from: T, to: T) {
+    if (typeof from === "string" && typeof to === "string") {
+      if (finalState.type === "date") {
+        return fillDateRange(from, to) as T[];
+      }
+
+      throw new Error("Invalid type for numeric range");
+    }
+
+    if (typeof from === "number" && typeof to === "number") {
+      return fillNumberRange(from, to) as T[];
     }
 
-    return fillNumberRange(from, to);
+    throw new Error("Invalid type for range");
   }
 
-  function offsetRange(from, to) {
+  function offsetRange(from: T, to: T) {
     return fillRange(from, to).length;
   }
 
-  async function runMode({ mode, from, to, status, verbose, ...selectorFlags }) {
+  async function runMode(flags: ModeIndexerFlags<TSelector>) {
+    const { mode, from: fromUntyped, to: toUntyped, status, verbose: verboseUntyped, ...untypeSelectorFlags } = flags;
+
+    const from = fromUntyped as T;
+    const to = toUntyped as T;
+    const verbose = verboseUntyped as boolean;
+    const selectorFlags = untypeSelectorFlags as ModeIndexerFlags<TSelector>;
+
     if (status) {
-      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+      const stateItem = await getRecordByKey<IndexerState<RebuildStateValue>>(STATE_TABLE_NAME, {
         name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
       });
 
-      const fromItem = await getRecordByKey(STATE_TABLE_NAME, {
+      const fromItem = await getRecordByKey<IndexerState<T>>(STATE_TABLE_NAME, {
         name: `${name}Since(${toSelectorString(selectorFlags)})`,
       });
 
-      const validateItem = await getRecordByKey(STATE_TABLE_NAME, {
+      const validateItem = await getRecordByKey<IndexerState<T>>(STATE_TABLE_NAME, {
         name: `${name}Validate(${toSelectorString(selectorFlags)})`,
       });
 
@@ -137,22 +220,27 @@ function createModeIndexerApp({
       await runReset(selectorFlags);
       await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
     } else if (mode === "resume-rebuild") {
-      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+      const previousRebuildEnds = await getIndexerLatestId<T, TSelector>(
+        name,
+        selectorFlags as ModeIndexerFlags<TSelector>,
+      );
 
       const rebuildFrom =
-        from || increaseId(finalState.type, previousRebuildEnds, 1) || (await finalState.getMinId(selectorFlags));
+        from ||
+        (previousRebuildEnds !== undefined && increaseId(finalState.type, previousRebuildEnds, 1)) ||
+        (await finalState.getMinId(selectorFlags));
       const rebuildTo = to || (await finalState.getMaxId(selectorFlags));
 
       await runRebuild(selectorFlags, rebuildFrom, rebuildTo, verbose);
     } else if (mode === "validate" || mode === "validation") {
-      const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
+      const previousRebuildEnds = await getIndexerLatestId<T, TSelector>(name, selectorFlags);
 
       if (!previousRebuildEnds) {
         inline.log(`[MODE INDEXER] cannot validate without a successful rebuild`);
         process.exit(0);
       }
 
-      const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
+      const previousValidatedTo = await getIndexerValidatedId<T, TSelector>(name, selectorFlags);
 
       const validateFrom = from || previousValidatedTo || (await finalState.getMinId(selectorFlags));
       const validateTo = to || previousRebuildEnds;
@@ -177,7 +265,7 @@ function createModeIndexerApp({
 
       await runRange(selectorFlags, from, to, verbose);
     } else {
-      const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
+      const stateItem = await getRecordByKey<IndexerState<string>>(STATE_TABLE_NAME, {
         name: `${name}RebuildState(${toSelectorString(selectorFlags)})`,
       });
 
@@ -187,14 +275,20 @@ function createModeIndexerApp({
         process.exit(0);
       }
 
-      const deltaFrom = increaseId(finalState.type, await getIndexerLatestId(name, selectorFlags), 1);
+      const latestId = await getIndexerLatestId<T, TSelector>(name, selectorFlags);
+
+      if (!latestId) {
+        throw new Error(`[MODE INDEXER] cannot find the latest ${finalState.type}`);
+      }
+
+      const deltaFrom = increaseId(finalState.type, latestId, 1);
       const deltaTo = await state.getMaxId(selectorFlags);
 
       await runDelta(selectorFlags, deltaFrom, deltaTo, verbose);
     }
   }
 
-  async function runRange(selectorFlags, from, to, verbose) {
+  async function runRange(selectorFlags: ModeIndexerFlags<TSelector>, from: T, to: T, verbose: boolean) {
     const startTime = Date.now();
 
     inline.log(
@@ -215,7 +309,13 @@ function createModeIndexerApp({
     }
   }
 
-  async function runValidate(selectorFlags, from, to, shouldSaveState, verbose) {
+  async function runValidate(
+    selectorFlags: ModeIndexerFlags<TSelector>,
+    from: T,
+    to: T,
+    shouldSaveState?: boolean,
+    verbose?: boolean,
+  ) {
     if (!validate) {
       inline.log(`[MODE INDEXER] the indexer doesn't support validate mode, validate function not implemented`);
       process.exit(1);
@@ -236,7 +336,7 @@ function createModeIndexerApp({
       `[MODE INDEXER] from=${from}, to=${to}, batchSize=${validateBatchSize}, concurrency=${validateConcurrency}`,
     );
 
-    for (let [windowStart, windowEnd] of windows) {
+    for (const [windowStart, windowEnd] of windows) {
       inline.log(`[MODE INDEXER] validating window ${windowStart}~${windowEnd}, concurrency=${validateConcurrency}`);
 
       const batches = range(windowStart, windowEnd, validateBatchSize);
@@ -291,47 +391,54 @@ function createModeIndexerApp({
     );
   }
 
-  async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
-    let failedIds = [];
+  async function execBuild(
+    selectorFlags: ModeIndexerFlags<TSelector>,
+    from: T,
+    to: T,
+    verbose: boolean,
+    shouldSaveState = false,
+  ) {
+    let failedIds: T[] = [];
 
     const windows = range(from, to, buildBatchSize * buildConcurrency);
 
-    for (let [windowStart, windowEnd] of windows) {
+    for (const [windowStart, windowEnd] of windows) {
       inline.log(`[MODE INDEXER] building window ${windowStart}~${windowEnd}, concurrency = ${buildConcurrency}`);
 
       const batches = range(windowStart, windowEnd, buildBatchSize);
 
       // add a retry for errors
       const batchResults = await Promise.all(
-        batches.map(async ([batchStart, batchEnd]) => {
-          await exponentialRetry(
-            async () => {
-              try {
-                const ids = await build({
-                  ...selectorFlags,
-                  from: batchStart,
-                  to: batchEnd,
-                  verbose,
-                });
-
-                if (ids && ids.length > 0) {
-                  return ids;
-                } else {
-                  return false;
+        batches.map(
+          async ([batchStart, batchEnd]) =>
+            await exponentialRetry(
+              async () => {
+                try {
+                  const ids = await build({
+                    ...selectorFlags,
+                    from: batchStart,
+                    to: batchEnd,
+                    verbose,
+                  });
+
+                  if (ids && ids.length > 0) {
+                    return ids;
+                  } else {
+                    return false;
+                  }
+                } catch (err) {
+                  inline.error(`[MODE INDEXER] got error in build`, err);
+
+                  return fillRange(batchStart, batchEnd);
                 }
-              } catch (err) {
-                inline.error(`[MODE INDEXER] got error in build`, err);
-
-                return fillRange(batchStart, batchEnd);
-              }
-            },
-            {
-              maxRetry,
-              test: (r) => !r,
-              verbose,
-            },
-          );
-        }),
+              },
+              {
+                maxRetry,
+                test: (r) => !r,
+                verbose,
+              },
+            ),
+        ),
       );
 
       if (shouldSaveState) {
@@ -357,7 +464,7 @@ function createModeIndexerApp({
     return failedIds;
   }
 
-  async function runRebuild(selectorFlags, from, to, verbose) {
+  async function runRebuild(selectorFlags: ModeIndexerFlags<TSelector>, from: T, to: T, verbose: boolean) {
     const startTime = Date.now();
 
     inline.log(
@@ -403,7 +510,7 @@ function createModeIndexerApp({
     }
   }
 
-  async function runDelta(selectorFlags, from, to, verbose) {
+  async function runDelta(selectorFlags: ModeIndexerFlags<TSelector>, from: T, to: T, verbose: boolean) {
     const startTime = Date.now();
 
     if (to < from) {
@@ -437,7 +544,7 @@ function createModeIndexerApp({
 
         await createRecord(STATE_TABLE_NAME, {
           name: `${name}Since(${toSelectorString(selectorFlags)})`,
-          value: Math.min(to, failedIds[0]),
+          value: to < failedIds[0] ? to : failedIds[0],
         });
 
         process.exit(1);
@@ -462,7 +569,7 @@ function createModeIndexerApp({
     }
   }
 
-  async function runReset(selectorFlags) {
+  async function runReset(selectorFlags: ModeIndexerFlags<TSelector>) {
     const startTime = Date.now();
 
     inline.log(`[MODE INDEXER] starting reset, ${toSelectorString(selectorFlags, ", ")}`);
@@ -486,7 +593,7 @@ function createModeIndexerApp({
     inline.log(`[MODE INDEXER] reset successfully in ${Date.now() - startTime}ms`);
   }
 
-  function run() {
+  async function run() {
     if (!binaryName) {
       binaryName = getBinaryName();
     }
@@ -498,7 +605,7 @@ function createModeIndexerApp({
           $ ${binaryName} <options>
 
         Options
-${getSelectorDesc(selector)}
+${selector ? getSelectorDesc(selector) : ""}
           --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset
           --from          min ${finalState.type} to build
           --to            max ${finalState.type} to build
@@ -535,20 +642,32 @@ ${getSelectorDesc(selector)}
       },
     );
 
-    return runMode(cli.flags).catch((err) => {
+    try {
+      return runMode(cli.flags);
+    } catch (err) {
       inline.error(err);
       process.exit(1);
-    });
+    }
   }
 
   return { run };
 }
 
-// for those indexers that does not rely on a curosr
+// for indexers that don't rely on a cursor
 // e.g. should always rebuild everything from scratch
 // or that the state can be easily inferred from existing data
-function createIndexerApp({ binaryName, selector = {}, build, maxRetry = 2 }) {
-  function run() {
+function createIndexerApp<TSelector extends Selector>({
+  binaryName,
+  selector = {} as TSelector,
+  build,
+  maxRetry = 2,
+}: {
+  binaryName: string;
+  selector?: TSelector;
+  build: StatelessBuildFunction<TSelector>;
+  maxRetry?: number;
+}) {
+  async function run() {
     if (!binaryName) {
       binaryName = getBinaryName();
     }
@@ -559,7 +678,7 @@ function createIndexerApp({ binaryName, selector = {}, build, maxRetry = 2 }) {
           $ ${binaryName} <options>
 
         Options
-${getSelectorDesc(selector)}
+${selector ? getSelectorDesc(selector) : ""}
           --verbose       Output debug messages
       `,
       {
@@ -576,7 +695,12 @@ ${getSelectorDesc(selector)}
       },
     );
 
-    async function runBuild({ verbose, ...selectorFlags }) {
+    async function runBuild(flags: StatelessIndexerFlags<TSelector>) {
+      const { verbose: untypedVerbose, ...untypedSelectorFlags } = flags;
+
+      const verbose = untypedVerbose as boolean;
+      const selectorFlags = untypedSelectorFlags as StatelessIndexerFlags<TSelector>;
+
       const startTime = Date.now();
 
       if (Object.keys(selectorFlags).length > 0) {
@@ -588,7 +712,7 @@ ${getSelectorDesc(selector)}
       const result = await exponentialRetry(
         async () => {
           try {
-            await build({ verbose, ...selectorFlags });
+            await build(flags);
 
             return true;
           } catch (err) {
@@ -600,7 +724,7 @@ ${getSelectorDesc(selector)}
         {
           maxRetry,
           test: (r) => r,
-          verbose,
+          verbose: verbose as boolean,
         },
       );
 
@@ -620,11 +744,6 @@ ${getSelectorDesc(selector)}
   return { run };
 }
 
-export {
-  increaseId,
-  createModeIndexerApp,
-  createIndexerApp,
-  getIndexerState,
-  getIndexerLatestId,
-  getIndexerValidatedId,
-};
+export { increaseId, createModeIndexerApp, createIndexerApp };
+
+export type { IndexerStateValue, State, StatelessBuildFunction, ModeBuildFunction, ValidateFunction };

package/{log.js → log.ts}

@@ -1,8 +1,8 @@
-function isObject(a) {
+function isObject(a: unknown): a is Record<string, unknown> {
   return !!a && a.constructor === Object;
 }
 
-function print(o) {
+function print(o: unknown): string {
   if (Array.isArray(o)) {
     return `[${o.map(print).join(", ")}]`;
   }
@@ -16,7 +16,7 @@ function print(o) {
   return `${o}`;
 }
 
-function getLine(params) {
+function getLine(params: unknown[]) {
   let line = "";
 
   // Convert to string and filter out newline to tabs (AWS Athena)
@@ -34,12 +34,12 @@ function timestamp() {
 }
 
 const inline = {
-  log: function (...args) {
+  log: function (...args: unknown[]) {
     console.log(`${timestamp()} ${getLine(args)}`);
   },
-  error: function (...args) {
+  error: function (...args: unknown[]) {
     console.error(`${timestamp()} ${getLine(args)}`);
   },
 };
 
-export { getLine, print, inline };
+export { print, getLine, inline };