@certik/skynet 0.18.9 → 0.19.0

package/{availability.js → availability.ts}

@@ -1,31 +1,28 @@
-export async function wait(time) {
+export async function wait(time: number) {
   return new Promise((resolve) => {
     setTimeout(resolve, time);
   });
 }
 
-export async function retry(times, verbose, func) {
-  try {
-    await func();
-  } catch (err) {
-    if (times <= 1) {
-      console.log("failed to retry", err);
-      throw err;
-    } else {
-      if (verbose) {
-        console.log("retry catch error", err);
-        console.log("retrying remaining times=", times - 1);
-      }
-      await wait(5000);
-      await retry(times - 1, verbose, func);
-    }
-  }
-}
-
-export async function exponentialRetry(func, { maxRetry, initialDuration, growFactor, test, verbose }) {
+export async function exponentialRetry<T>(
+  func: () => Promise<T>,
+  {
+    maxRetry,
+    initialDuration,
+    growFactor,
+    test,
+    verbose,
+  }: {
+    maxRetry: number;
+    initialDuration?: number;
+    growFactor?: number;
+    test: (result: T) => boolean;
+    verbose?: boolean;
+  },
+) {
   let retries = maxRetry;
   let duration = initialDuration || 5000;
-  let growFactorFinal = growFactor || 2;
+  const growFactorFinal = growFactor || 2;
 
   let result = await func();
 

package/{cli.js → cli.ts}

@@ -19,7 +19,7 @@ function detectWorkingDirectory() {
   return wd.slice(skynetd.length + path.sep.length).replace(path.sep, "/");
 }
 
-function detectDirectory(fullBinPath, sentinel = "package.json") {
+function detectDirectory(fullBinPath: string, sentinel = "package.json") {
   let parentFolder = path.dirname(fullBinPath);
 
   while (parentFolder) {

package/{const.js → const.ts}

@@ -1,6 +1,41 @@
-import { getNodeRealApiKey, ensureAndGet } from "./env.js";
+import { ensureAndGet } from "./env.ts";
 
-const PROTOCOLS = {
+type Protocol = {
+  nativeTokenName: string;
+  nativeTokenSymbol: string;
+  nativeTokenDecimals: number;
+  nativeTokenAddress: string;
+  nativeTokenLogo?: string;
+  nativeTokenCoinGeckoId?: string;
+  nativeTokenCmcId?: number;
+  endpoint: string;
+  backupEndpoint?: string;
+  archiveEndpoint?: string;
+  tokenStandard: string;
+  scanApi?: {
+    endpoint: string;
+    key?: string;
+  };
+  multiCallProvider?: string;
+  scanUrl?: string;
+  chainId: number;
+};
+
+type TimeIntervals = {
+  SECOND: number;
+  MINUTE: number;
+  HOUR: number;
+  DAY: number;
+  WEEK: number;
+  YEAR: number;
+};
+
+function getNodeRealApiKey(identifier: string) {
+  // NodeReal API keys are different for each NodeReal app
+  return ensureAndGet(`SKYNET_NODEREAL_API_${identifier.toUpperCase()}_KEY`) ?? ensureAndGet("SKYNET_NODEREAL_API_KEY");
+}
+
+const PROTOCOLS: Record<string, Protocol> = {
   eth: {
     nativeTokenName: "Ethereum",
     nativeTokenSymbol: "ETH",
@@ -114,10 +149,14 @@ const PROTOCOLS = {
     nativeTokenCmcId: 4030,
     endpoint: "https://algo-node.certik-skynet.com/",
     tokenStandard: "ASA",
+    chainId: 1300,
   },
 };
 
-const TIME = {
+const TIME: {
+  BY_MS: TimeIntervals;
+  BY_S: TimeIntervals;
+} = {
   BY_MS: {
     SECOND: 1000,
     MINUTE: 1000 * 60,

package/databricks.ts

@@ -0,0 +1,82 @@
+import { DBSQLClient, DBSQLParameter } from "@databricks/sql";
+import { getEnvOrThrow } from "./env.js";
+import { exponentialRetry } from "./availability.js";
+import type IDBSQLSession from "@databricks/sql/dist/contracts/IDBSQLSession.js";
+import type { DBSQLParameterValue } from "@databricks/sql/dist/DBSQLParameter.js";
+
+let client: DBSQLClient | null = null;
+
+let session: IDBSQLSession | null = null;
+
+async function initSession() {
+  if (!client) {
+    client = new DBSQLClient();
+    await client.connect({
+      authType: "access-token",
+      token: getEnvOrThrow("SKYNET_DATABRICKS_TOKEN"),
+      host: getEnvOrThrow("SKYNET_DATABRICKS_SERVER_HOSTNAME"),
+      path: getEnvOrThrow("SKYNET_DATABRICKS_HTTP_PATH"),
+    });
+  }
+  if (!session) {
+    session = await client.openSession();
+  }
+}
+
+async function executeSql<T>(sql: string, bindings: Record<string, DBSQLParameterValue>) {
+  // retry sql query to avoid socket hang up error
+  const results = await exponentialRetry(
+    async () => {
+      let queryOperation;
+      try {
+        await initSession();
+        if (!session) throw new Error("Session is not initialized");
+        queryOperation = await session.executeStatement(sql, {
+          runAsync: true,
+          namedParameters: objToParameter(bindings),
+        });
+
+        const result = await queryOperation.fetchAll();
+        return result;
+      } catch (err) {
+        // If the error is not a Databricks error, throw it
+        if (!checkIsDatabricksError(err)) {
+          throw err;
+        }
+        return err;
+      } finally {
+        await queryOperation?.close();
+      }
+    },
+    {
+      maxRetry: 3,
+      initialDuration: 500,
+      test: (result) => {
+        return !checkIsDatabricksError(result);
+      },
+    },
+  );
+
+  if (checkIsDatabricksError(results)) {
+    // still error after retry, throw it
+    throw results;
+  }
+  return results as T[];
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function checkIsDatabricksError(err: any) {
+  return err?.errno === "ECONNRESET" || err?.message?.includes("java.lang.NullPointerException");
+}
+
+function objToParameter(obj: Record<string, DBSQLParameterValue>) {
+  return Object.entries(obj).reduce(
+    (acc, [key, value]) => {
+      acc[key] = new DBSQLParameter({ value });
+      return acc;
+    },
+    {} as Record<string, DBSQLParameter>,
+  );
+}
+
+export { executeSql };

package/{date.js → date.ts}

@@ -1,19 +1,19 @@
-import { arrayGroup } from "./util.js";
+import { arrayGroup } from "./util.ts";
 
 const MS_IN_A_DAY = 3600 * 24 * 1000;
 
-function getDateOnly(date) {
+function getDateOnly(date: number | string | Date) {
   return new Date(date).toISOString().split("T")[0];
 }
 
-function findDateAfter(date, n) {
+function findDateAfter(date: number | string | Date, n: number) {
   const d = new Date(date);
   const after = new Date(d.getTime() + MS_IN_A_DAY * n);
 
   return getDateOnly(after);
 }
 
-function daysInRange(from, to) {
+function daysInRange(from: number | string | Date, to: number | string | Date) {
   const fromTime = new Date(from).getTime();
   const toTime = new Date(to).getTime();
 
@@ -32,7 +32,7 @@ function daysInRange(from, to) {
   return dates;
 }
 
-function dateRange(from, to, step) {
+function dateRange(from: number | string | Date, to: number | string | Date, step: number): [string, string][] {
   const days = daysInRange(from, to);
 
   const windows = arrayGroup(days, step);

package/{deploy.js → deploy.ts}

@@ -4,11 +4,43 @@ import { execa } from "execa";
 import meow from "meow";
 import chalk from "chalk";
 import which from "which";
-import { getJobName, getSelectorFlags, getSelectorDesc } from "./selector.js";
-import { getEnvOrThrow } from "./env.js";
-import { getBinaryName, detectSkynetDirectory } from "./cli.js";
+import { getJobName, getSelectorDesc, getSelectorFlags } from "./selector.ts";
+import type { Selector } from "./selector.ts";
+import { getEnvOrThrow } from "./env.ts";
+import { getBinaryName, detectSkynetDirectory } from "./cli.ts";
+import type { Writable } from "stream";
+
+export type Env = {
+  [key: string]: string | null;
+};
+
+type Service = {
+  prefix: string;
+  port: number;
+};
+
+type Schedule = string | ((jobName: string) => string);
+
+type Check = {
+  func: ({ protocol, state, verbose }: { protocol: string; state: string; verbose: boolean }) => {
+    type: string;
+    message: string;
+  };
+  schedule: Schedule;
+  slackChannel: string;
+  killTimeout?: string;
+  cpu: number;
+  mem: number;
+};
+
+type Restart = {
+  attempts: number;
+  mode: string;
+  interval: string;
+  delay: string;
+};
 
-const INTERVAL_ALIASES = {
+const INTERVAL_ALIASES: Record<string, string> = {
   secondly: "*/1 * * * * * *",
   "@secondly": "*/1 * * * * * *",
   minutely: "0 * * * * * *",
@@ -32,10 +64,25 @@ const genConfig = ({
   cpu,
   mem,
   service,
-  additionalEnv = [],
+  additionalEnv = {},
   type = "batch",
   region = "skynet-dc1",
   isProduction,
+}: {
+  jobName: string;
+  workingDirectory: string;
+  cmd: string;
+  cron?: string;
+  count?: number;
+  restart?: Restart;
+  killTimeout?: string;
+  cpu: number;
+  mem: number;
+  service?: Service;
+  additionalEnv?: Env;
+  type?: string;
+  region?: string;
+  isProduction?: boolean;
 }) => `job "${jobName}" {
   datacenters = ["${region}"]
 
@@ -179,7 +226,7 @@ const genConfig = ({
   }
 }`;
 
-async function prepareNomad(isProduction) {
+async function prepareNomad(isProduction: boolean) {
   if (isProduction) {
     console.log("Deploy to Production");
   } else {
@@ -197,16 +244,8 @@ async function prepareNomad(isProduction) {
   }
 }
 
-async function getNomadAddr(isProduction) {
-  let nomadAddr;
-
-  if (isProduction) {
-    nomadAddr = getEnvOrThrow("NOMAD_ADDR");
-  } else {
-    nomadAddr = "http://127.0.0.1:4646";
-  }
-
-  return nomadAddr;
+function getNomadAddr(isProduction: boolean) {
+  return isProduction ? getEnvOrThrow("NOMAD_ADDR") : "http://127.0.0.1:4646";
 }
 
 async function getNomadPath() {
@@ -217,13 +256,21 @@ async function getNomadPath() {
       `Deploy requires ${chalk.bold("nomad")} binary, please follow ${chalk.underline(
         "https://learn.hashicorp.com/tutorials/nomad/get-started-install",
       )} for installation`,
+      missingNomad,
     );
 
     throw new Error("missing nomad binary");
   }
 }
 
-async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, isStop, isDryRun) {
+async function runNomadJob(
+  nomadPath: string,
+  nomadAddr: string,
+  jobName: string,
+  nomadJobDefinition: string,
+  isStop?: boolean,
+  isDryRun?: boolean,
+) {
   try {
     if (isStop) {
       const nomad = execa(nomadPath, ["job", "stop", jobName], {
@@ -231,7 +278,7 @@ async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, is
           NOMAD_ADDR: nomadAddr,
         },
       });
-      nomad.stdout.pipe(process.stdout);
+      nomad.stdout.pipe(process.stdout as Writable);
       await nomad;
 
       console.log(chalk.green(`Stopped nomad job ${jobName} in ${nomadAddr}`));
@@ -249,15 +296,17 @@ async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, is
           NOMAD_ADDR: nomadAddr,
         },
       });
-      nomad.stdout.pipe(process.stdout);
+      nomad.stdout.pipe(process.stdout as Writable);
       await nomad;
 
       console.log(chalk.green(`Deployed nomad job ${jobName} to ${nomadAddr}`));
     }
   } catch (nomadExecErr) {
-    console.log("Nomad Execution Error:");
-    console.log(nomadExecErr.message);
-    console.log("");
+    if (nomadExecErr instanceof Error) {
+      console.log("Nomad Execution Error:");
+      console.log(nomadExecErr.message);
+      console.log("");
+    }
 
     console.log(
       `Failed to run ${chalk.bold("nomad")} commands, please ensure nomad server is accessible at ${chalk.bold(
@@ -274,7 +323,7 @@ function createModeDeploy({
   name,
   workingDirectory,
   bin = "bin/indexer",
-  selector = {},
+  selector = {} as Selector,
   env = {},
   region = "skynet-dc1",
   deltaSchedule,
@@ -288,6 +337,26 @@ function createModeDeploy({
   validateKillTimeout,
   validateCpu,
   validateMem,
+}: {
+  binaryName: string;
+  name: string;
+  workingDirectory: string;
+  bin?: string;
+  selector?: Selector;
+  env?: Env;
+  region?: string;
+  check?: Check;
+  deltaSchedule?: Schedule;
+  validateSchedule?: Schedule;
+  deltaKillTimeout?: string;
+  deltaCpu: number;
+  deltaMem: number;
+  rebuildKillTimeout?: string;
+  rebuildCpu?: number;
+  rebuildMem?: number;
+  validateKillTimeout?: string;
+  validateCpu?: number;
+  validateMem?: number;
 }) {
   async function deployMode({
     mode,
@@ -299,6 +368,15 @@ function createModeDeploy({
     verbose,
     schedule: cmdSchedule,
     ...selectorFlags
+  }: {
+    mode: string;
+    from: number;
+    to: number;
+    stop: boolean;
+    production: boolean;
+    dryRun: boolean;
+    verbose: boolean;
+    schedule?: string;
   }) {
     if (mode === "delta") {
       // delta mode will ignore from/to flags
@@ -310,9 +388,9 @@ function createModeDeploy({
 
     const jobName = getJobName(name, selectorFlags, mode);
 
-    const selectorCmdPart = Object.keys(selectorFlags)
+    const selectorCmdPart = Object.entries(selectorFlags)
       .sort()
-      .map((name) => `--${name} ${selectorFlags[name]}`)
+      .map(([name, value]) => `--${name} ${value}`)
       .join(" ");
     let args = `--mode ${mode} ${selectorCmdPart}`;
 
@@ -330,7 +408,7 @@ function createModeDeploy({
       rangeArgs += ` --to ${to}`;
     }
 
-    const modeResouces = {
+    const modeResouces: Record<string, { cpu?: number; mem?: number; killTimeout?: string }> = {
       rebuild: { cpu: rebuildCpu, mem: rebuildMem, killTimeout: rebuildKillTimeout },
       "resume-rebuild": { cpu: rebuildCpu, mem: rebuildMem, killTimeout: rebuildKillTimeout },
       validate: {
@@ -342,7 +420,9 @@ function createModeDeploy({
     };
 
     // by default use delta cpu/mem settings
-    const { cpu, mem, killTimeout } = modeResouces[mode] || modeResouces.delta;
+    const cpu = modeResouces[mode]?.cpu || deltaCpu;
+    const mem = modeResouces[mode]?.mem || deltaMem;
+    const killTimeout = modeResouces[mode]?.killTimeout || deltaKillTimeout;
 
     let deltaCron = typeof deltaSchedule === "function" ? deltaSchedule(jobName) : deltaSchedule;
 
@@ -356,14 +436,14 @@ function createModeDeploy({
       validateCron = cmdSchedule;
     }
 
-    const modeIntervals = {
-      delta: INTERVAL_ALIASES[deltaCron] || deltaCron,
-      validate: INTERVAL_ALIASES[validateCron] || validateCron,
+    const modeIntervals: Record<string, string | undefined> = {
+      delta: deltaCron ? INTERVAL_ALIASES[deltaCron] || deltaCron : undefined,
+      validate: validateCron ? INTERVAL_ALIASES[validateCron] || validateCron : undefined,
     };
 
     const mainJobDefinition = genConfig({
       jobName,
-      cron: isPeriodic && modeIntervals[mode],
+      cron: isPeriodic ? modeIntervals[mode] : undefined,
       workingDirectory,
       additionalEnv: env,
       region,
@@ -376,12 +456,12 @@ function createModeDeploy({
 
     const nomadPath = await getNomadPath();
     await prepareNomad(production);
-    const nomadAddr = await getNomadAddr(production);
+    const nomadAddr = getNomadAddr(production);
 
     await runNomadJob(nomadPath, nomadAddr, jobName, mainJobDefinition, stop, dryRun);
   }
 
-  function deploy() {
+  async function deploy() {
     if (!binaryName) {
       binaryName = getBinaryName();
     }
@@ -452,10 +532,12 @@ ${getSelectorDesc(selector)}
       },
     );
 
-    return deployMode(cli.flags).catch((err) => {
+    try {
+      return deployMode(cli.flags);
+    } catch (err) {
       console.error(err);
       process.exit(1);
-    });
+    }
   }
 
   return { deploy };
@@ -477,13 +559,42 @@ function createDeploy({
   cpu,
   mem,
   service,
+}: {
+  binaryName: string;
+  name: string;
+  workingDirectory: string;
+  bin?: string;
+  selector?: Selector;
+  env?: Env;
+  region?: string;
+  type?: string;
+  count?: number;
+  schedule?: Schedule;
+  restart?: Restart;
+  killTimeout?: string;
+  cpu: number;
+  mem: number;
+  service?: Service;
 }) {
-  async function deployModeless({ production, stop, dryRun, verbose, schedule: cmdSchedule, ...selectorFlags }) {
-    const jobName = getJobName(name, selectorFlags, null);
+  async function deployModeless({
+    production,
+    stop,
+    dryRun,
+    verbose,
+    schedule: cmdSchedule,
+    ...selectorFlags
+  }: {
+    production: boolean;
+    stop: boolean;
+    dryRun: boolean;
+    verbose: boolean;
+    schedule?: string;
+  }) {
+    const jobName = getJobName(name, selectorFlags);
 
-    const selectorCmdPart = Object.keys(selectorFlags)
+    const selectorCmdPart = Object.entries(selectorFlags)
       .sort()
-      .map((name) => `--${name} ${selectorFlags[name]}`)
+      .map(([name, value]) => `--${name} ${value}`)
       .join(" ");
     let args = `${selectorCmdPart}`;
 
@@ -500,7 +611,7 @@ function createDeploy({
 
     const nomadJobDefinition = genConfig({
       jobName,
-      cron: INTERVAL_ALIASES[cron] || cron,
+      cron: cron ? INTERVAL_ALIASES[cron] || cron : undefined,
       count,
       restart,
       workingDirectory,
@@ -517,12 +628,12 @@ function createDeploy({
 
     const nomadPath = await getNomadPath();
     await prepareNomad(production);
-    const nomadAddr = await getNomadAddr(production);
+    const nomadAddr = getNomadAddr(production);
 
     await runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, stop, dryRun);
   }
 
-  function deploy() {
+  async function deploy() {
     if (!binaryName) {
       binaryName = getBinaryName();
     }
@@ -571,10 +682,12 @@ ${getSelectorDesc(selector)}
       },
     );
 
-    return deployModeless(cli.flags).catch((err) => {
+    try {
+      return deployModeless(cli.flags);
+    } catch (err) {
       console.error(err);
       process.exit(1);
-    });
+    }
   }
 
   return { deploy };