@certik/skynet 0.18.9 → 0.19.0

package/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "conventionalCommits.scopes": [
+        "lib-skynet"
+    ]
+}

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.19.0
+
+- BREAKING: remove multicall from web3 module
+- BREAKING: remove retry from util module
+- BREAKING: remove createTableIfNotExist from dynamodb module
+- BREAKING: remove transactWrite from dynamodb module
+- BREAKING: remove sqs module
+- Updated: Typescript rewrite
+
 ## 0.18.9
 
 - Added options for `executeSql` function in `databricks` library

package/README.md

@@ -13,11 +13,11 @@ const { scanWholeTable } = require("@certik/skynet/dynamodb");
 ## Test
 
 ```bash
-bun run test
+bun test
 ```
 
 ## Publish
 
 ```bash
-bun run pub
+bun publish --access public
 ```

package/{abi.js → abi.ts}

@@ -1,4 +1,6 @@
-export const ERC20 = [
+import type { AbiFunctionFragment } from "web3";
+
+export const ERC20: AbiFunctionFragment[] = [
   {
     constant: true,
     inputs: [],
@@ -171,12 +173,12 @@ export const ERC20 = [
     type: "function",
   },
   {
+    name: "Approval",
     payable: true,
     stateMutability: "payable",
     type: "fallback",
   },
   {
-    anonymous: false,
     inputs: [
       {
         indexed: true,
@@ -198,7 +200,6 @@ export const ERC20 = [
     type: "event",
   },
   {
-    anonymous: false,
     inputs: [
       {
         indexed: true,
@@ -221,9 +222,8 @@ export const ERC20 = [
   },
 ];
 
-export const ERC721 = [
+export const ERC721: AbiFunctionFragment[] = [
   {
-    anonymous: false,
     inputs: [
       {
         indexed: true,
@@ -248,7 +248,6 @@ export const ERC721 = [
     type: "event",
   },
   {
-    anonymous: false,
     inputs: [
       {
         indexed: true,
@@ -273,7 +272,6 @@ export const ERC721 = [
     type: "event",
   },
   {
-    anonymous: false,
     inputs: [
       {
         indexed: true,
@@ -568,129 +566,4 @@ export const ERC721 = [
   },
 ];
 
-export const MULTICALL = [
-  {
-    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
-    name: "callBalanceOf",
-    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callBlockNumber",
-    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [{ internalType: "uint256", name: "_i", type: "uint256" }],
-    name: "callBlockhash",
-    outputs: [{ internalType: "bytes32", name: "", type: "bytes32" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callChainId",
-    outputs: [{ internalType: "uint256", name: "id", type: "uint256" }],
-    stateMutability: "pure",
-    type: "function",
-  },
-  {
-    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
-    name: "callCode",
-    outputs: [{ internalType: "bytes", name: "code", type: "bytes" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
-    name: "callCodeHash",
-    outputs: [{ internalType: "bytes32", name: "codeHash", type: "bytes32" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [{ internalType: "address", name: "_addr", type: "address" }],
-    name: "callCodeSize",
-    outputs: [{ internalType: "uint256", name: "size", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callCoinbase",
-    outputs: [{ internalType: "address", name: "", type: "address" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callDifficulty",
-    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callGasLeft",
-    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callGasLimit",
-    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callGasPrice",
-    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callOrigin",
-    outputs: [{ internalType: "address", name: "", type: "address" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [],
-    name: "callTimestamp",
-    outputs: [{ internalType: "uint256", name: "", type: "uint256" }],
-    stateMutability: "view",
-    type: "function",
-  },
-  {
-    inputs: [
-      {
-        components: [
-          { internalType: "bool", name: "delegateCall", type: "bool" },
-          { internalType: "bool", name: "revertOnError", type: "bool" },
-          { internalType: "uint256", name: "gasLimit", type: "uint256" },
-          { internalType: "address", name: "target", type: "address" },
-          { internalType: "uint256", name: "value", type: "uint256" },
-          { internalType: "bytes", name: "data", type: "bytes" },
-        ],
-        internalType: "struct IModuleCalls.Transaction[]",
-        name: "_txs",
-        type: "tuple[]",
-      },
-    ],
-    name: "multiCall",
-    outputs: [
-      { internalType: "bool[]", name: "_successes", type: "bool[]" },
-      { internalType: "bytes[]", name: "_results", type: "bytes[]" },
-    ],
-    stateMutability: "payable",
-    type: "function",
-  },
-];
-
 export const BEP20 = ERC20;

package/{address.js → address.ts}

@@ -1,9 +1,9 @@
-export function parseAddress(address) {
+export function parseAddress(address: string) {
   // in case of address has multiple colons, we only split the first one
   const separatorIndex = address.indexOf(":");
 
-  let protocolPart;
-  let addressPart;
+  let protocolPart: string;
+  let addressPart: string;
 
   if (separatorIndex === -1) {
     // if there is no colon, we assume it is an ethereum address
@@ -21,6 +21,6 @@ export function parseAddress(address) {
   return [protocolPart, addressPart];
 }
 
-export function composeAddress(protocol, addr) {
+export function composeAddress(protocol: string, addr: string) {
   return `${protocol}:${addr}`;
 }

package/{api.js → api.ts}

@@ -1,11 +1,29 @@
 import osModule from "os";
 import express from "express";
+import type { Request, Response, NextFunction } from "express";
 import meow from "meow";
-import { getSelectorFlags, getSelectorDesc } from "./selector.js";
-import { isProduction } from "./env.js";
-import { inline } from "./log.js";
+import { getSelectorDesc, getSelectorFlags } from "./selector.ts";
+import { isProduction } from "./env.ts";
+import { inline } from "./log.ts";
+import type { Selector } from "./selector.ts";
+
+export type Middleware = (req: Request, res: Response, next: NextFunction) => Promise<void>;
+
+type Route = {
+  method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+  path: string;
+  middlewares?: Middleware[];
+  handler: (args: { req: Request; res: Response; verbose?: boolean }) => Promise<void>;
+  protected?: boolean;
+};
+
+type Serve = {
+  prefix: string;
+  port: number;
+  apiKey?: string;
+};
 
-async function logStartMiddleware(req, res, next) {
+async function logStartMiddleware(req: Request, res: Response, next: NextFunction) {
   const start = new Date();
 
   res.set("x-requested-at", start.toISOString());
@@ -13,17 +31,36 @@ async function logStartMiddleware(req, res, next) {
   next();
 }
 
-async function contextMiddleware(req, res, next) {
+async function contextMiddleware(req: Request, res: Response, next: NextFunction) {
   res.set("x-instance-id", osModule.hostname());
 
   next();
 }
 
-async function logEndMiddleware(req, res, next) {
-  const start = new Date(res.get("x-requested-at"));
-  const end = new Date();
+type LogInfo = {
+  start: number;
+  end: number;
+  elapsed: string;
+  endpoint: string;
+  host: string;
+  status: number;
+  errorMessage?: string;
+};
 
-  const logInfo = {
+async function logEndMiddleware(req: Request, res: Response, next: NextFunction) {
+  const requestedAt = res.get("x-requested-at");
+
+  if (!requestedAt) {
+    inline.log("missing x-requested-at header");
+
+    next();
+    return;
+  }
+
+  const start = new Date(requestedAt).getTime();
+  const end = new Date().getTime();
+
+  const logInfo: LogInfo = {
     start,
     end,
     elapsed: `${end - start}ms`,
@@ -31,6 +68,7 @@ async function logEndMiddleware(req, res, next) {
     host: req.hostname,
     status: res.statusCode,
   };
+
   if (res.statusMessage) {
     logInfo.errorMessage = res.statusMessage;
   }
@@ -39,10 +77,10 @@ async function logEndMiddleware(req, res, next) {
   next();
 }
 
-const apiKeyMiddleware = (key) => {
-  async function requireAPIKey(req, res, next) {
+const apiKeyMiddleware = (key: string | Record<string, string>) => {
+  async function requireAPIKey(req: Request, res: Response, next: NextFunction) {
     try {
-      const apiKey = req.get("x-api-key") || req.query["api-key"]; // api key present either in header or query string
+      const apiKey = req.get("x-api-key") || (req.query["api-key"] as string); // api key present either in header or query string
 
       if (!apiKey) {
         inline.log("request without api key");
@@ -88,7 +126,21 @@ const apiKeyMiddleware = (key) => {
   return requireAPIKey;
 };
 
-export async function startApiApp({ binaryName, name, selector = {}, routes, serve, beforeListen }) {
+export async function startApiApp({
+  binaryName,
+  name,
+  selector = {},
+  routes,
+  serve,
+  beforeListen,
+}: {
+  binaryName: string;
+  name: string;
+  selector?: Selector;
+  routes: Route[];
+  serve: Serve;
+  beforeListen?: (args: { app: ReturnType<typeof express> }) => Promise<void>;
+}) {
   const app = express();
   app.use(express.json({ limit: "20mb" }));
 
@@ -118,10 +170,14 @@ ${getSelectorDesc(selector)}
   const { verbose, ...selectorFlags } = cli.flags;
 
   for (const route of routes) {
-    const method = route.method ? route.method.toLowerCase() : "get";
+    const method = route.method ? (route.method.toLowerCase() as "get" | "post" | "put" | "patch" | "delete") : "get";
     const middlewares = route.middlewares || [];
 
     if (route.protected) {
+      if (!serve.apiKey) {
+        throw new Error("serve.apiKey is required for protected route");
+      }
+
       middlewares.unshift(apiKeyMiddleware(serve.apiKey));
     }
 
@@ -135,13 +191,17 @@ ${getSelectorDesc(selector)}
         contextMiddleware,
         logStartMiddleware,
         ...middlewares,
-        async (req, res, next) => {
+        async (req: Request, res: Response, next: NextFunction) => {
           try {
             await route.handler({ req, res, ...selectorFlags });
           } catch (routeErr) {
-            inline.log("caught route err", routeErr, routeErr.stack);
-
-            res.status(500).send(`internal server error: ${routeErr.message}`);
+            if (routeErr instanceof Error) {
+              inline.log("caught route err", routeErr, routeErr.stack);
+              res.status(500).send(`internal server error: ${routeErr.message}`);
+            } else {
+              inline.log("caught route err", routeErr);
+              res.status(500).send("internal server error");
+            }
           }
 
           next();
@@ -151,7 +211,7 @@ ${getSelectorDesc(selector)}
     }
   }
 
-  if (!routes.some((r) => r.path === "/" && r.method.toUpperCase() === "GET")) {
+  if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
     // for health check
     app.get("/", (req, res) => {
       res.send("ok");

package/{app.js → app.ts}

@@ -1,8 +1,64 @@
 import { EOL } from "os";
-import { createIndexerApp, createModeIndexerApp } from "./indexer.js";
-import { createDeploy, createModeDeploy } from "./deploy.js";
+import { createIndexerApp, createModeIndexerApp } from "./indexer.ts";
+import type {
+  IndexerStateValue,
+  State,
+  ModeBuildFunction,
+  StatelessBuildFunction,
+  ValidateFunction,
+} from "./indexer.ts";
+import { createDeploy, createModeDeploy } from "./deploy.ts";
+import type { Env } from "./deploy.ts";
 import { startApiApp } from "./api.js";
+import type { Middleware } from "./api.ts";
 import { getBinaryName, detectBin, detectWorkingDirectory } from "./cli.js";
+import type { Selector } from "./selector.ts";
+import type { Request, Response, Application } from "express";
+
+type RouteDefinition = {
+  method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+  path: string;
+  middlewares?: Middleware[];
+  handler: (args: { req: Request; res: Response; verbose?: boolean }) => Promise<void>;
+  protected?: boolean;
+};
+
+type ServeDefinition = {
+  prefix: string;
+  port: number;
+  apiKey?: string;
+  killTimeout?: string;
+  cpu: number;
+  mem: number;
+  instances?: number;
+};
+
+type BuildDefinition<TFunc> = {
+  func: TFunc;
+  killTimeout?: string;
+  maxRetry?: number;
+  cpu: number;
+  mem: number;
+  schedule: string;
+  restart?: {
+    attempts: number;
+    mode: string;
+    interval: string;
+    delay: string;
+  };
+  batchSize?: number;
+  concurrency?: number;
+};
+
+type ValidateDefinition<TFunc> = {
+  func: TFunc;
+  batchSize: number;
+  killTimeout?: string;
+  cpu: number;
+  mem: number;
+  concurrency?: number;
+  schedule: string;
+};
 
 function printAppHelp() {
   console.log(`
@@ -14,14 +70,14 @@ function printAppHelp() {
   `);
 }
 
-function isDeleteCommand(command) {
+function isDeleteCommand(command: string) {
   return ["delete", "stop", "remove"].includes(command);
 }
 
-function checkAndSetEnv(env) {
-  const missingEnvs = [];
+function checkAndSetEnv(env: Env) {
+  const missingEnvs: string[] = [];
 
-  for (let key of Object.keys(env)) {
+  for (const key of Object.keys(env)) {
     if (env[key]) {
       process.env[key] = env[key];
     } else if (!process.env[key]) {
@@ -36,7 +92,17 @@ function checkAndSetEnv(env) {
   }
 }
 
-function createApp({ parameterErrors, env, onRun, onDeploy }) {
+function createApp({
+  parameterErrors,
+  env,
+  onRun,
+  onDeploy,
+}: {
+  parameterErrors: string[];
+  env: Env;
+  onRun: () => Promise<void>;
+  onDeploy: () => Promise<void>;
+}) {
   if (parameterErrors.length > 0) {
     console.log(`Parameter Validation Failed:${EOL}- ${parameterErrors.join(EOL + "- ")}`);
 
@@ -65,8 +131,8 @@ function createApp({ parameterErrors, env, onRun, onDeploy }) {
   };
 }
 
-function checkEnvParameter(env) {
-  const errors = [];
+function checkEnvParameter(env: Env) {
+  const errors: string[] = [];
 
   const envKeys = Object.keys(env);
 
@@ -79,8 +145,10 @@ function checkEnvParameter(env) {
   return errors;
 }
 
-function checkIndexerBuildParameter(build) {
-  const errors = [];
+function checkIndexerBuildParameter<TSelector extends Selector>(
+  build: BuildDefinition<StatelessBuildFunction<TSelector>>,
+) {
+  const errors: string[] = [];
 
   if (!build?.func) {
     errors.push("must define build.func");
@@ -97,8 +165,8 @@ function checkIndexerBuildParameter(build) {
   return errors;
 }
 
-function checkStateParameter(state) {
-  const errors = [];
+function checkStateParameter<T extends IndexerStateValue, TSelector extends Selector>(state: State<T, TSelector>) {
+  const errors: string[] = [];
 
   if (!state?.type) {
     errors.push("must define state.type");
@@ -111,14 +179,25 @@ function checkStateParameter(state) {
   return errors;
 }
 
-function indexer({ name, selector, build, env = {}, region = "skynet-dc1" }) {
+function indexer<TSelector extends Selector>({
+  name,
+  selector,
+  build,
+  env = {},
+  region = "skynet-dc1",
+}: {
+  name: string;
+  selector: TSelector;
+  build: BuildDefinition<StatelessBuildFunction<TSelector>>;
+  env?: Env;
+  region?: string;
+}) {
   return createApp({
     parameterErrors: [...checkIndexerBuildParameter(build), ...checkEnvParameter(env)],
     env,
     onRun: () => {
       const { run } = createIndexerApp({
         binaryName: `${getBinaryName()} run`,
-        name,
         selector,
         build: build.func,
         maxRetry: build.maxRetry,
@@ -152,8 +231,10 @@ function indexer({ name, selector, build, env = {}, region = "skynet-dc1" }) {
   });
 }
 
-function checkModeIndexerBuildParameter(build) {
-  const errors = [];
+function checkModeIndexerBuildParameter<T extends IndexerStateValue, TSelector extends Selector>(
+  build: BuildDefinition<ModeBuildFunction<T, TSelector>>,
+) {
+  const errors: string[] = [];
 
   if (!build?.func) {
     errors.push("must define build.func");
@@ -170,8 +251,10 @@ function checkModeIndexerBuildParameter(build) {
   return errors;
 }
 
-function checkModeIndexerValidateParameter(validate) {
-  const errors = [];
+function checkModeIndexerValidateParameter<TSelector extends Selector>(
+  validate?: ValidateDefinition<ValidateFunction<TSelector>>,
+) {
+  const errors: string[] = [];
 
   if (validate) {
     if (!validate.func) {
@@ -190,7 +273,23 @@ function checkModeIndexerValidateParameter(validate) {
   return errors;
 }
 
-function modeIndexer({ name, selector, state, build, validate, env = {}, region = "skynet-dc1" }) {
+function modeIndexer<T extends IndexerStateValue, TSelector extends Selector>({
+  name,
+  selector,
+  state,
+  build,
+  validate,
+  env = {},
+  region = "skynet-dc1",
+}: {
+  name: string;
+  selector: TSelector;
+  state: State<T, TSelector>;
+  build: BuildDefinition<ModeBuildFunction<T, TSelector>>;
+  validate?: ValidateDefinition<ValidateFunction<TSelector>>;
+  env?: Env;
+  region?: string;
+}) {
   return createApp({
     parameterErrors: [
       ...checkModeIndexerBuildParameter(build),
@@ -251,8 +350,8 @@ function modeIndexer({ name, selector, state, build, validate, env = {}, region
   });
 }
 
-function checkApiServeParameter(serve, routes) {
-  const errors = [];
+function checkApiServeParameter(serve: ServeDefinition, routes: RouteDefinition[]) {
+  const errors: string[] = [];
 
   if (!serve?.prefix) {
     errors.push("must define serve.prefix");
@@ -279,8 +378,8 @@ function checkApiServeParameter(serve, routes) {
   return errors;
 }
 
-function checkApiRoutesParameter(routes) {
-  const errors = [];
+function checkApiRoutesParameter(routes: RouteDefinition[]) {
+  const errors: string[] = [];
 
   if (!Array.isArray(routes)) {
     errors.push("routes must be an array");
@@ -305,7 +404,21 @@ function checkApiRoutesParameter(routes) {
   return errors;
 }
 
-function api({ name, routes, serve, beforeListen, env = {}, region = "skynet-dc1" }) {
+function api({
+  name,
+  routes,
+  serve,
+  beforeListen,
+  env = {},
+  region = "skynet-dc1",
+}: {
+  name: string;
+  routes: RouteDefinition[];
+  serve: ServeDefinition;
+  beforeListen?: ({ app }: { app: Application }) => Promise<void>;
+  env?: Env;
+  region?: string;
+}) {
   // do not support selector for now
   const selector = {};