@certik/skynet 0.10.65 → 0.11.0

package/ably.js

@@ -1,6 +1,6 @@
-const Ably = require("ably");
-const Bottleneck = require("bottleneck/es5");
-const { ensureAndGet } = require("./env");
+import Ably from "ably";
+import Bottleneck from "bottleneck/es5";
+import { ensureAndGet } from "./env.js";
 
 let ABLY;
 
@@ -17,7 +17,7 @@ async function getAbly() {
   return ABLY;
 }
 
-async function publishMessage(channel, messageName, messageData) {
+export async function publishMessage(channel, messageName, messageData) {
   const ably = await getAbly();
   const channelObj = ably.channels.get(channel);
 
@@ -30,11 +30,6 @@ async function publishMessage(channel, messageName, messageData) {
   }
 }
 
-const rateLimitedPublishMessage = new Bottleneck({
+export const rateLimitedPublishMessage = new Bottleneck({
   minTime: 1000 / 40,
 }).wrap(publishMessage);
-
-module.exports = {
-  publishMessage,
-  rateLimitedPublishMessage,
-};

package/address.d.ts

@@ -1,2 +1,2 @@
 export function parseAddress(address: string): [string, string];
-export function composeAddress(protocol: string, addr: string): string; 
+export function composeAddress(protocol: string, addr: string): string;

package/address.js

@@ -1,4 +1,4 @@
-function parseAddress(address) {
+export function parseAddress(address) {
   // in case of address has multiple colons, we only split the first one
   const separatorIndex = address.indexOf(":");
 
@@ -21,11 +21,6 @@ function parseAddress(address) {
   return [protocolPart, addressPart];
 }
 
-function composeAddress(protocol, addr) {
+export function composeAddress(protocol, addr) {
   return `${protocol}:${addr}`;
 }
-
-module.exports = {
-  parseAddress,
-  composeAddress,
-};

package/api.d.ts

@@ -1,6 +1,6 @@
 import { Selector } from "./selector";
 
-export function startApiApp(params: { 
+export function startApiApp(params: {
     binaryName: string;
     name: string;
     selector: Selector;

package/api.js

@@ -1,9 +1,9 @@
-const osModule = require("os");
-const express = require("express");
-const meow = require("meow");
-const { getSelectorFlags, getSelectorDesc } = require("./selector");
-const { isProduction } = require("./env");
-const { inline } = require("./log");
+import osModule from "os";
+import express from "express";
+import meow from "meow";
+import { getSelectorFlags, getSelectorDesc } from "./selector.js";
+import { isProduction } from "./env.js";
+import { inline } from "./log.js";
 
 async function logStartMiddleware(req, res, next) {
   const start = new Date();
@@ -88,7 +88,7 @@ const apiKeyMiddleware = (key) => {
   return requireAPIKey;
 };
 
-async function startApiApp({ binaryName, name, selector = {}, routes, serve, beforeListen }) {
+export async function startApiApp({ binaryName, name, selector = {}, routes, serve, beforeListen }) {
   const app = express();
   app.use(express.json({ limit: "20mb" }));
 
@@ -167,5 +167,3 @@ ${getSelectorDesc(selector)}
     }
   });
 }
-
-module.exports = { startApiApp };

package/app.js

@@ -1,18 +1,19 @@
-const { EOL } = require("os");
-const fetch = require("node-fetch");
-const { SKYNET_API_PREFIX } = require("./const");
-const {
+import { EOL } from "os";
+import { SKYNET_API_PREFIX } from "./const.js";
+
+import {
   createIndexerApp,
   createModeIndexerApp,
   getIndexerLatestId,
   getIndexerValidatedId,
   getIndexerState,
-} = require("./indexer");
-const { createDeploy, createModeDeploy } = require("./deploy");
-const { createProducerApp, createConsumerApp, getProducerLatestId } = require("./kafka");
-const { startApiApp } = require("./api");
-const { createMonitor, ERROR_LEVEL } = require("./monitor");
-const { getBinaryName, detectBin, detectWorkingDirectory } = require("./cli");
+} from "./indexer.js";
+
+import { createDeploy, createModeDeploy } from "./deploy.js";
+import { createProducerApp, createConsumerApp, getProducerLatestId } from "./kafka.js";
+import { startApiApp } from "./api.js";
+import { createMonitor, ERROR_LEVEL } from "./monitor.js";
+import { getBinaryName, detectBin, detectWorkingDirectory } from "./cli.js";
 
 function printAppHelp(hasCheck) {
   console.log(`
@@ -759,4 +760,4 @@ const every = (n = 1) => {
   };
 };
 
-module.exports = { indexer, modeIndexer, producer, consumer, api, every, SENSITIVE_VALUE, ERROR_LEVEL };
+export { indexer, modeIndexer, producer, consumer, api, every, SENSITIVE_VALUE, ERROR_LEVEL };

package/availability.js

@@ -1,10 +1,10 @@
-async function wait(time) {
+export async function wait(time) {
   return new Promise((resolve) => {
     setTimeout(resolve, time);
   });
 }
 
-async function retry(times, verbose, func) {
+export async function retry(times, verbose, func) {
   try {
     await func();
   } catch (err) {
@@ -22,7 +22,7 @@ async function retry(times, verbose, func) {
   }
 }
 
-async function exponentialRetry(func, { maxRetry, initialDuration, growFactor, test, verbose }) {
+export async function exponentialRetry(func, { maxRetry, initialDuration, growFactor, test, verbose }) {
   let retries = maxRetry;
   let duration = initialDuration || 5000;
   let growFactorFinal = growFactor || 2;
@@ -49,9 +49,3 @@ async function exponentialRetry(func, { maxRetry, initialDuration, growFactor, t
 
   return result;
 }
-
-module.exports = {
-  wait,
-  retry,
-  exponentialRetry,
-};

package/block.js

@@ -1,9 +1,8 @@
-const { getEnvironment } = require("./env");
-const { PROTOCOLS } = require("./const");
-const { readFile } = require("./s3");
-const { getRecordByKey } = require("./dynamodb");
-const fetch = require("node-fetch");
-const Web3 = require("web3");
+import { Web3 } from "web3";
+import { getEnvironment } from "./env.js";
+import { PROTOCOLS } from "./const.js";
+import { readFile } from "./s3.js";
+import { getRecordByKey } from "./dynamodb.js";
 
 const S3_BUCKET_NAME =
   getEnvironment() === "prd" ? "certik-skynet" : "certik-skynet-dev";
@@ -74,7 +73,7 @@ async function getBlockFromS3(protocol, height) {
   return JSON.parse(body);
 }
 
-module.exports = {
+export {
   getLatestBlockHeight,
   getBlock,
   getBlockS3Path,

package/cli.js

@@ -1,5 +1,5 @@
-const path = require("path");
-const fs = require("fs");
+import path from "path";
+import fs from "fs";
 
 function getBinaryName() {
   const binaryNameParts = process.argv[1].split(path.sep);
@@ -45,7 +45,7 @@ function detectBin() {
   return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
 }
 
-module.exports = {
+export {
   getBinaryName,
   detectSkynetDirectory,
   detectWorkingDirectory,

package/const.js

@@ -1,4 +1,4 @@
-const { getNodeRealApiKey, ensureAndGet } = require("./env");
+import { getNodeRealApiKey, ensureAndGet } from "./env.js";
 
 const SKYNET_API_PREFIX = "https://api.certik-skynet.com";
 
@@ -137,7 +137,7 @@ const TIME = {
   },
 };
 
-module.exports = {
+export {
   SKYNET_API_PREFIX,
   PROTOCOLS,
   TIME,

package/date.js

@@ -1,4 +1,4 @@
-const { arrayGroup } = require("./util");
+import { arrayGroup } from "./util.js";
 
 const MS_IN_A_DAY = 3600 * 24 * 1000;
 
@@ -40,7 +40,7 @@ function dateRange(from, to, step) {
   return windows.map((w) => [w[0], w[w.length - 1]]);
 }
 
-module.exports = {
+export {
   getDateOnly,
   findDateAfter,
   daysInRange,

package/deploy.js

@@ -1,12 +1,12 @@
-const fs = require("fs/promises");
-const fso = require("fs");
-const execa = require("execa");
-const meow = require("meow");
-const chalk = require("chalk");
-const which = require("which");
-const { getJobName, getSelectorFlags, getSelectorDesc } = require("./selector");
-const { getEnvOrThrow } = require("./env");
-const { getBinaryName, detectSkynetDirectory } = require("./cli");
+import fs from "fs/promises";
+import fso from "fs";
+import { execa } from "execa";
+import meow from "meow";
+import chalk from "chalk";
+import which from "which";
+import { getJobName, getSelectorFlags, getSelectorDesc } from "./selector.js";
+import { getEnvOrThrow } from "./env.js";
+import { getBinaryName, detectSkynetDirectory } from "./cli.js";
 
 const INTERVAL_ALIASES = {
   secondly: "*/1 * * * * * *",
@@ -118,7 +118,7 @@ const genConfig = ({
         command = "sh"
         args = [
           "-c",
-          "cd \${meta.skynet_code_path}/infra-nomad/log-shipper && yarn install --silent && exec bin/shipper --path ${jobName} --opensearch"
+          "cd \${meta.skynet_code_path}/infra-nomad/log-shipper && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi && exec bin/shipper --path ${jobName} --opensearch"
         ]
       }
 
@@ -146,7 +146,7 @@ const genConfig = ({
         command = "sh"
         args = [
           "-c",
-          "cd \${meta.skynet_code_path}/${workingDirectory} && yarn install --silent && exec ${cmd}"
+          "cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi&& exec ${cmd}"
         ]
       }
 
@@ -236,7 +236,7 @@ async function prepareNomad(isProduction) {
     console.log("Deploy locally, please start nomad server in a separate terminal");
     console.log(`You can start nomad server by running ${chalk.inverse(`${skynetDir}/infra-nomad/dev/start.sh`)}`);
     console.log(
-      `Then you can visit ${chalk.underline("http://localhost:4646/ui/jobs")} to check submitted dev jobs.\n`
+      `Then you can visit ${chalk.underline("http://localhost:4646/ui/jobs")} to check submitted dev jobs.\n`,
     );
   }
 }
@@ -259,8 +259,8 @@ async function getNomadPath() {
   } catch (missingNomad) {
     console.log(
       `Deploy requires ${chalk.bold("nomad")} binary, please follow ${chalk.underline(
-        "https://learn.hashicorp.com/tutorials/nomad/get-started-install"
-      )} for installation`
+        "https://learn.hashicorp.com/tutorials/nomad/get-started-install",
+      )} for installation`,
     );
 
     throw new Error("missing nomad binary");
@@ -305,8 +305,8 @@ async function runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, is
 
     console.log(
       `Failed to run ${chalk.bold("nomad")} commands, please ensure nomad server is accessible at ${chalk.bold(
-        nomadAddr
-      )}`
+        nomadAddr,
+      )}`,
     );
 
     throw new Error("nomad execution error");
@@ -526,7 +526,7 @@ ${getSelectorDesc(selector)}
             default: false,
           },
         },
-      }
+      },
     );
 
     return deployMode(cli.flags).catch((err) => {
@@ -678,7 +678,7 @@ ${getSelectorDesc(selector)}
             default: false,
           },
         },
-      }
+      },
     );
 
     return deployModeless(cli.flags).catch((err) => {
@@ -690,9 +690,4 @@ ${getSelectorDesc(selector)}
   return { deploy };
 }
 
-module.exports = {
-  getJobName,
-  getNomadAddr,
-  createModeDeploy,
-  createDeploy,
-};
+export { getJobName, getNomadAddr, createModeDeploy, createDeploy };

package/distributed-lock.js

@@ -1,5 +1,3 @@
-const fetch = require("node-fetch");
-
 async function acquireLock(name, ttl) {
   try {
     const res = await fetch("http://localhost:8500/v1/session/create", {
@@ -78,7 +76,7 @@ async function hasLock(name, uid = null) {
   }
 }
 
-async function useLock({ name, ttl, verbose }) {
+export async function useLock({ name, ttl, verbose }) {
   const lockWarningMessage = `only one process with the same lock name ${name} should be running, terminate current process`;
 
   const lockAvailable = await hasLock(name);
@@ -109,5 +107,3 @@ async function useLock({ name, ttl, verbose }) {
 
   return uid;
 }
-
-module.exports = { useLock };

package/dns.js

@@ -1,7 +1,7 @@
 // https://nodejs.org/api/dns.html#dns
-const dns = require("dns").promises;
+import { promises as dns } from "dns";
 
-async function hasTxtRecord(domainName, txtRecord) {
+export async function hasTxtRecord(domainName, txtRecord) {
   let nsAddresses;
   // get all nameservers for the domain
   const nameservers = await dns.resolveNs(domainName);
@@ -25,5 +25,3 @@ async function hasTxtRecord(domainName, txtRecord) {
   }
   return false;
 }
-
-module.exports = { hasTxtRecord };

package/dynamodb.js

@@ -1,21 +1,35 @@
-const { DynamoDB } = require("aws-sdk");
-const { getAWSAccessKeyId, getAWSSecretAccessKey, getAWSRegion } = require("./env");
-const { wait } = require("./availability");
+import {
+  DynamoDBDocumentClient,
+  ScanCommand,
+  BatchWriteCommand,
+  PutCommand,
+  QueryCommand,
+  UpdateCommand,
+} from "@aws-sdk/lib-dynamodb";
+import { DynamoDBClient, DescribeTableCommand } from "@aws-sdk/client-dynamodb";
+import { getAWSAccessKeyId, getAWSSecretAccessKey, getAWSRegion } from "./env.js";
+import { wait } from "./availability.js";
 
 function getDynamoDB() {
-  return new DynamoDB({
-    accessKeyId: getAWSAccessKeyId(),
-    secretAccessKey: getAWSSecretAccessKey(),
+  return new DynamoDBClient({
+    credentials: {
+      accessKeyId: getAWSAccessKeyId(),
+      secretAccessKey: getAWSSecretAccessKey(),
+    },
     region: getAWSRegion(),
   });
 }
 
 function getDocClient() {
-  return new DynamoDB.DocumentClient({
-    accessKeyId: getAWSAccessKeyId(),
-    secretAccessKey: getAWSSecretAccessKey(),
-    region: getAWSRegion(),
-  });
+  return DynamoDBDocumentClient.from(
+    new DynamoDBClient({
+      credentials: {
+        accessKeyId: getAWSAccessKeyId(),
+        secretAccessKey: getAWSSecretAccessKey(),
+      },
+      region: getAWSRegion(),
+    }),
+  );
 }
 
 function mergeQueries(q1, q2) {
@@ -35,17 +49,12 @@ async function scanWholeTable(options) {
     ScannedCount: 0,
   };
 
-  let data = await dynamodb.scan(options).promise();
+  let data = await dynamodb.send(new ScanCommand(options));
 
   while (data.LastEvaluatedKey) {
     items = mergeQueries(items, data);
 
-    data = await dynamodb
-      .scan({
-        ExclusiveStartKey: data.LastEvaluatedKey,
-        ...options,
-      })
-      .promise();
+    data = await dynamodb.send(new ScanCommand({ ...options, ExclusiveStartKey: data.LastEvaluatedKey }));
   }
 
   items = mergeQueries(items, data);
@@ -96,8 +105,8 @@ async function batchCreateRecords(tableName, records, maxWritingCapacity, verbos
           },
         };
 
-        return docClient.batchWrite(params).promise();
-      })
+        return docClient.send(new BatchWriteCommand(params));
+      }),
     );
 
     prevRemainingCount = remainingItems.length;
@@ -144,7 +153,7 @@ async function createRecord(tableName, fields, verbose = false) {
     Item: fields,
   };
 
-  return docClient.put(params).promise();
+  return docClient.send(new PutCommand(params));
 }
 
 async function getRecordsByKey(tableName, keys, indexName) {
@@ -167,17 +176,17 @@ async function getRecordsByKey(tableName, keys, indexName) {
   let data;
 
   try {
-    data = await docClient.query(params).promise();
+    data = await docClient.send(new QueryCommand(params));
 
     let items = data.Items;
 
     while (data.LastEvaluatedKey) {
-      data = await docClient
-        .query({
-          ExclusiveStartKey: data.LastEvaluatedKey,
+      data = await docClient.send(
+        new QueryCommand({
           ...params,
-        })
-        .promise();
+          ExclusiveStartKey: data.LastEvaluatedKey,
+        }),
+      );
 
       items = items.concat(data.Items);
     }
@@ -247,7 +256,7 @@ async function updateRecordByKey(tableName, idKey, fields, conditionExpressions
       deleteParams.ConditionExpression = conditionExpressions;
     }
 
-    data = await docClient.update(deleteParams).promise();
+    data = await docClient.send(new UpdateCommand(deleteParams));
   }
 
   if (fieldsToUpdate.length > 0) {
@@ -270,7 +279,7 @@ async function updateRecordByKey(tableName, idKey, fields, conditionExpressions
       params.ConditionExpression = conditionExpressions;
     }
 
-    data = await docClient.update(params).promise();
+    data = await docClient.send(new UpdateCommand(params));
   }
 
   return data && data.Attributes;
@@ -291,12 +300,12 @@ async function createTableIfNotExist(
     },
   ],
   otherOptions = {},
-  verbose = false
+  verbose = false,
 ) {
   const dynamodb = getDynamoDB();
 
   try {
-    const table = await dynamodb.describeTable({ TableName: tableName }).promise();
+    const table = await dynamodb.send(new DescribeTableCommand({ TableName: tableName }));
 
     if (verbose) {
       console.log(`table ${tableName} already exist`, table);
@@ -325,7 +334,7 @@ async function createTableIfNotExist(
     while (!isCreated) {
       await wait(5000);
 
-      const t = await dynamodb.describeTable({ TableName: tableName }).promise();
+      const t = await dynamodb.send(new DescribeTableCommand({ TableName: tableName }));
 
       console.log("waiting for table to become active");
 
@@ -342,15 +351,15 @@ async function batchDeleteRecords(tableName, keys) {
   for (let start = 0; start < keys.length; start += 25) {
     const slice = keys.slice(start, start + 25);
 
-    await docClient
-      .batchWrite({
+    await docClient.send(
+      new BatchWriteCommand({
         RequestItems: {
           [tableName]: slice.map((key) => {
             return { DeleteRequest: { Key: key } };
           }),
         },
-      })
-      .promise();
+      }),
+    );
   }
 }
 
@@ -369,7 +378,7 @@ function getIndexKeyName(globalSecondaryIndexes, indexName, type) {
 async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbose = false) {
   const docClient = getDocClient();
 
-  const meta = await getDynamoDB().describeTable({ TableName: tableName }).promise();
+  const meta = await getDynamoDB().send(new DescribeTableCommand({ TableName: tableName }));
 
   const hashKeyName = indexName
     ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes, indexName, "HASH")
@@ -395,7 +404,7 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
     params.IndexName = indexName;
   }
 
-  let data = await docClient.query(params).promise();
+  let data = await docClient.send(new QueryCommand(params));
 
   await batchDeleteRecords(
     tableName,
@@ -407,19 +416,19 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
           }
         : {
             [mainHashKeyName]: item[mainHashKeyName],
-          }
-    )
+          },
+    ),
   );
 
   totalDeleted += data.Items.length;
 
   while (data.LastEvaluatedKey) {
-    data = await docClient
-      .query({
+    data = await docClient.send(
+      new QueryCommand({
         ...params,
         ExclusiveStartKey: data.LastEvaluatedKey,
-      })
-      .promise();
+      }),
+    );
 
     await batchDeleteRecords(
       tableName,
@@ -431,8 +440,8 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
             }
           : {
               [mainHashKeyName]: item[mainHashKeyName],
-            }
-      )
+            },
+      ),
     );
 
     totalDeleted += data.Items.length;
@@ -445,7 +454,7 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
   return totalDeleted;
 }
 
-module.exports = {
+export {
   getDocClient,
   scanWholeTable,
   batchCreateRecords,

package/env.js

@@ -51,7 +51,7 @@ function isDev() {
   return getEnvironment() === "dev";
 }
 
-module.exports = {
+export {
   ensureAndGet,
   getEnvOrThrow,
   getAWSAccessKeyId,

package/examples/api

@@ -7,7 +7,7 @@
 //   $ examples/api deploy
 //   $ examples/api --help
 
-const { api, every, SENSITIVE_VALUE, ERROR_LEVEL } = require("../app");
+import { api, every, SENSITIVE_VALUE, ERROR_LEVEL } from "../app";
 
 // an example middleware
 async function exampleMiddleware(req, res, next) {

package/examples/consumer

@@ -8,7 +8,7 @@
 //   $ examples/consumer deploy --protocol eth
 //   $ examples/consumer --help
 
-const { consumer, every, ERROR_LEVEL } = require("../app");
+import { consumer, every, ERROR_LEVEL } from "../app";
 
 async function consume({ protocol, messages, verbose }) {
   console.log("consume called with", protocol, messages, verbose);

package/examples/indexer

@@ -8,7 +8,7 @@
 //   $ examples/indexer deploy --protocol eth
 //   $ examples/indexer --help
 
-const { indexer, every, ERROR_LEVEL } = require("../app");
+import { indexer, every, ERROR_LEVEL } from "../app";
 
 async function build({ protocol, verbose }) {
   console.log("build called with", protocol, verbose);
@@ -41,8 +41,8 @@ const app = indexer({
     // for more flags check meow documentation at https://github.com/sindresorhus/meow
     protocol: {
       type: "string",
-      description: "which chain to index"
-    }
+      description: "which chain to index",
+    },
   },
 
   env: {},

package/examples/mode-indexer

@@ -9,7 +9,7 @@
 //   $ examples/mode-indexer deploy --protocol eth
 //   $ examples/mode-indexer --help
 
-const { modeIndexer, every, SENSITIVE_VALUE } = require("../app");
+import { modeIndexer, every, SENSITIVE_VALUE } from "../app";
 
 async function build({ protocol, from, to, verbose }) {
   console.log("build called with", protocol, from, to, verbose);