@certik/skynet 0.11.3 → 0.12.0

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.12.0
+
+- BREAKING: removed token module (@certik/skynet/token)
+- BREAKING: removed block module (@certik/skynet/block)
+- BREAKING: removed transaction module (@certik/skynet/transaction)
+- BREAKING: removed primitive module (@certik/skynet/primitive)
+- BREAKING: removed price module (@certik/skynet/price)
+- BREAKING: removed distributed-lock module (@certik/skynet/distributed-lock)
+
+## 0.11.4
+
+- support api key in query string
+
 ## 0.11.3
 
 - Fixed: get aws credential for production

package/api.js

@@ -42,7 +42,7 @@ async function logEndMiddleware(req, res, next) {
 const apiKeyMiddleware = (key) => {
   async function requireAPIKey(req, res, next) {
     try {
-      const apiKey = req.get("x-api-key");
+      const apiKey = req.get("x-api-key") || req.query["api-key"]; // api key present either in header or query string
 
       if (!apiKey) {
         inline.log("request without api key");

package/kafka.js

@@ -1,10 +1,9 @@
 import meow from "meow";
 import { Kafka, logLevel } from "kafkajs";
 import { getEnvironment, getEnvOrThrow } from "./env.js";
-import { getJobName, getSelectorFlags, getSelectorDesc, toSelectorString } from "./selector.js";
+import { getSelectorFlags, getSelectorDesc, toSelectorString } from "./selector.js";
 import { createRecord, getRecordByKey, deleteRecordsByHashKey } from "./dynamodb.js";
 import { wait, exponentialRetry } from "./availability.js";
-import { useLock } from "./distributed-lock.js";
 import { getBinaryName } from "./cli.js";
 import { inline } from "./log.js";
 
@@ -219,8 +218,6 @@ ${getSelectorDesc(selector)}
         process.exit(0);
       }
 
-      await useLock({ name: getJobName(name, selectorFlags), ttl: 50, verbose });
-
       if (!from) {
         const prevId = await getProducerLatestId(name, selectorFlags);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.11.3",
+  "version": "0.12.0",
   "description": "Skynet Shared JS library",
   "type": "module",
   "main": "index.js",
@@ -15,13 +15,13 @@
     "node": ">= 18"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "^3.409.0",
-    "@aws-sdk/client-s3": "^3.409.0",
-    "@aws-sdk/client-sqs": "^3.409.0",
-    "@aws-sdk/lib-dynamodb": "^3.409.0",
-    "@opensearch-project/opensearch": "^2.3.1",
-    "@slack/web-api": "^6.9.0",
-    "ably": "^1.2.44",
+    "@aws-sdk/client-dynamodb": "^3.449.0",
+    "@aws-sdk/client-s3": "^3.449.0",
+    "@aws-sdk/client-sqs": "^3.449.0",
+    "@aws-sdk/lib-dynamodb": "^3.449.0",
+    "@opensearch-project/opensearch": "^2.4.0",
+    "@slack/web-api": "^6.9.1",
+    "ably": "^1.2.47",
     "bottleneck": "^2.19.5",
     "chalk": "^5.3.0",
     "execa": "^8.0.1",
@@ -29,8 +29,8 @@
     "kafkajs": "^2.2.4",
     "md5": "^2.3.0",
     "meow": "^12.1.1",
-    "snowflake-sdk": "^1.8.0",
-    "web3": "^4.1.1",
+    "snowflake-sdk": "^1.9.0",
+    "web3": "^4.2.2",
     "which": "^4.0.0"
   },
   "devDependencies": {

package/block.d.ts

@@ -1,49 +0,0 @@
-// See https://www.quicknode.com/docs/ethereum/eth_getBlockByNumber for a complete reference.
-
-type Transaction = {
-    blockHash: string;
-    blockNumber: string;
-    hash: string;
-    to: string;
-    from: string;
-    transactionIndex: string;
-    value: string | undefined;
-    nonce: string;
-    gas: string;
-    gasPrice: string;
-    input: string;
-    type: string;
-    chainId: string | undefined;
-    v: string;
-    r: string;
-    s: string;
-}
-
-type BlockInfo = {
-    number: string;
-    timestamp: string;
-    difficulty: string;
-    extraData: string;
-    gasLimit: string;
-    gasUsed: string;
-    hash: string;
-    logsBloom: string;
-    miner: string;
-    mixHash: string;
-    nonce: string;
-    parentHash: string;
-    receiptsRoot: string;
-    sha3Uncles: string;
-    size: string;
-    stateRoot: string;
-    totalDifficulty: string;
-    transactions: Transaction[];
-    transactionsRoot: string;
-    uncles: string[];
-}
-
-export function getLatestBlockHeight(protocol: string): Promise<number>;
-export function getBlock(protocol: string, height: number): Promise<BlockInfo>;
-export function getBlockS3Path(protocol: string, height: number): string;
-export function getLatestS3BlockHeight(protocol: string): Promise<number>;
-export function getBlockFromS3(protocol: string, height: number): Promise<BlockInfo>;

package/block.js

@@ -1,82 +0,0 @@
-import { Web3 } from "web3";
-import { getEnvironment } from "./env.js";
-import { PROTOCOLS } from "./const.js";
-import { readFile } from "./s3.js";
-import { getRecordByKey } from "./dynamodb.js";
-
-const S3_BUCKET_NAME =
-  getEnvironment() === "prd" ? "certik-skynet" : "certik-skynet-dev";
-const INDEXER_STATE_TABLE = `skynet-${getEnvironment()}-indexer-state`;
-
-function leftPad3(num) {
-  if (num === 0) {
-    return "000";
-  } else if (num < 10) {
-    return "00" + String(num);
-  } else if (num < 100) {
-    return "0" + String(num);
-  } else {
-    return String(num);
-  }
-}
-
-function getBlockS3Path(protocol, height) {
-  return `chaindata/${protocol}/${Math.floor(height / 1_000_000)}/${leftPad3(
-    Math.floor(height / 1_000) % 1000
-  )}/${leftPad3(height % 1000)}.json`;
-}
-
-// We use this function instead of web3 because the gas limits on bsc exceed
-// the size supported by javascript numbers, and web3 does not support this
-async function getBlock(protocol, height) {
-  const body = {
-    jsonrpc: "2.0",
-    method: "eth_getBlockByNumber",
-    params: [`0x${height.toString(16)}`, true],
-    id: 1
-  };
-  const response = await fetch(PROTOCOLS[protocol].endpoint, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json"
-    },
-    body: JSON.stringify(body)
-  });
-
-  const { result: block } = await response.json();
-
-  return block;
-}
-
-async function getLatestBlockHeight(protocol) {
-  const web3 = new Web3(PROTOCOLS[protocol].endpoint);
-  const isSyncing = await web3.eth.isSyncing();
-
-  if (isSyncing) {
-    return isSyncing.currentBlock;
-  } else {
-    const latestBlock = await web3.eth.getBlock("latest");
-
-    return latestBlock.number;
-  }
-}
-
-async function getLatestS3BlockHeight(protocol) {
-  const name = `AMLBlockIndexerSince(protocol=${protocol})`;
-  const { value: height } = await getRecordByKey(INDEXER_STATE_TABLE, { name });
-  return height;
-}
-
-async function getBlockFromS3(protocol, height) {
-  const body = await readFile(S3_BUCKET_NAME, getBlockS3Path(protocol, height));
-
-  return JSON.parse(body);
-}
-
-export {
-  getLatestBlockHeight,
-  getBlock,
-  getBlockS3Path,
-  getLatestS3BlockHeight,
-  getBlockFromS3
-};

package/distributed-lock.d.ts

@@ -1 +0,0 @@
-export function useLock(useLockParams: { name: string, ttl: string, verbose: boolean }): Promise<string | null>

package/distributed-lock.js

@@ -1,109 +0,0 @@
-async function acquireLock(name, ttl) {
-  try {
-    const res = await fetch("http://localhost:8500/v1/session/create", {
-      method: "PUT",
-      body: JSON.stringify({
-        Name: name,
-        TTL: `${ttl}s`,
-        Behavior: "delete",
-      }),
-    });
-
-    if (res.ok) {
-      const { ID: id } = await res.json();
-
-      return id;
-    } else {
-      console.log(res, await res.text());
-
-      return null;
-    }
-  } catch (fetchErr) {
-    // console.log("error fetching", fetchErr);
-
-    return null;
-  }
-}
-
-async function renewLock(uid) {
-  try {
-    const res = await fetch(`http://localhost:8500/v1/session/renew/${uid}`, {
-      method: "PUT",
-    });
-
-    if (res.ok) {
-      return uid;
-    } else {
-      console.log(res, await res.text());
-
-      return null;
-    }
-  } catch (fetchErr) {
-    // console.log("error fetching", fetchErr);
-
-    return null;
-  }
-}
-
-async function hasLock(name, uid = null) {
-  try {
-    const res = await fetch(`http://localhost:8500/v1/session/list`);
-
-    if (res.ok) {
-      const sessions = await res.json();
-
-      const sessionsWithTheSameName = sessions.filter((s) => s.Name === name);
-
-      if (!uid) {
-        return sessionsWithTheSameName.length === 0;
-      }
-
-      if (sessionsWithTheSameName.length !== 1) {
-        return false;
-      }
-
-      // only one should have the same uid
-      return sessionsWithTheSameName[0].ID === uid;
-    } else {
-      console.log(await res.text());
-
-      return true;
-    }
-  } catch (fetchErr) {
-    // console.log("error fetching", fetchErr);
-
-    return true;
-  }
-}
-
-export async function useLock({ name, ttl, verbose }) {
-  const lockWarningMessage = `only one process with the same lock name ${name} should be running, terminate current process`;
-
-  const lockAvailable = await hasLock(name);
-
-  if (!lockAvailable) {
-    console.log(lockWarningMessage);
-    process.exit(0);
-  }
-
-  let uid = await acquireLock(name, ttl);
-
-  setInterval(async () => {
-    if (!(await hasLock(name, uid))) {
-      console.log(lockWarningMessage);
-      process.exit(0);
-    }
-
-    if (uid) {
-      if (verbose) {
-        console.log("renewing", name, uid);
-      }
-
-      uid = await renewLock(uid);
-    } else {
-      uid = await acquireLock(name, ttl);
-    }
-  }, (ttl * 1000) / 2);
-
-  return uid;
-}

package/price.d.ts

@@ -1,8 +0,0 @@
-type PricePreviousRecord = {
-    address: string;
-    price: number;
-    timestamp: number;
-}
-
-export function getPriceClosestTo(tableName: string, address: string, timestamp: number): Promise<number>;
-export function getPricePreviousRecord(tableName: string, address: string, timestamp: number): Promise<PricePreviousRecord | null>;

package/price.js

@@ -1,46 +0,0 @@
-import { QueryCommand } from "@aws-sdk/lib-dynamodb";
-import { getDocClient } from "./dynamodb.js";
-
-async function queryPrice(tableName, address, timestamp, op = "<") {
-  const docClient = getDocClient();
-
-  return await docClient.send(
-    new QueryCommand({
-      TableName: tableName,
-      KeyConditionExpression: `#address = :address and #timestamp ${op} :timestamp`,
-      ExpressionAttributeNames: {
-        "#timestamp": "timestamp",
-        "#address": "address",
-      },
-      ExpressionAttributeValues: {
-        ":timestamp": timestamp,
-        ":address": address,
-      },
-      Limit: 1,
-      ScanIndexForward: false,
-    }),
-  );
-}
-
-async function getPricePreviousRecord(tableName, address, timestamp) {
-  const query = await queryPrice(tableName, address, timestamp, "<");
-
-  if (query.Count === 0) {
-    return null;
-  } else {
-    return query.Items[0];
-  }
-}
-
-async function getPriceClosestTo(tableName, address, timestamp) {
-  const query = await queryPrice(tableName, address, timestamp, "<=");
-
-  if (query.Count === 0) {
-    return 0;
-  } else {
-    const { price } = query.Items[0];
-    return price;
-  }
-}
-
-export { getPriceClosestTo, getPricePreviousRecord };

package/primitive.d.ts

@@ -1,29 +0,0 @@
-import { DynamoDB } from "aws-sdk";
-
-type Issue = {
-    code: string;
-    link: string;
-    reduction: number;
-    title: string;
-}
-
-export function getOverrides(
-    docClient: DynamoDB.DocumentClient, 
-    projectId: string, 
-    address: string, 
-    log: (...data: any[]) => void, 
-    primitive: string
-): Promise<string[] | null>;
-
-export function overrideScoreAndIssues(
-    docClient: DynamoDB.DocumentClient,
-    projectId: string,
-    address: string,
-    score: number,
-    issues: Issue[],
-    log: (...data: any[]) => void,
-    primitive: string
-): Promise<{
-    score: number;
-    issues: Issue[];
-}>

package/primitive.js

@@ -1,78 +0,0 @@
-async function getOverrides(docClient, projectId, address, log, primitive) {
-  try {
-    console.log("ProjectId:" + projectId + " address:" + address);
-
-    const whitelist = [];
-    const queryProjectOverrides = await docClient
-      .get({
-        TableName: "skynet-prd-primitive-manual-overrides",
-        Key: {
-          primitive: primitive,
-          overrideId: "project/" + projectId,
-        },
-      })
-      .promise();
-    if (queryProjectOverrides.Item && queryProjectOverrides.Item.whitelist) {
-      whitelist.push(...queryProjectOverrides.Item.whitelist);
-    }
-
-    const queryAddressOverrides = await docClient
-      .get({
-        TableName: "skynet-prd-primitive-manual-overrides",
-        Key: {
-          primitive: primitive,
-          overrideId: "address/" + address,
-        },
-      })
-      .promise();
-
-    if (queryAddressOverrides.Item && queryAddressOverrides.Item.whitelist) {
-      console.log(queryAddressOverrides.Item.whitelist);
-      whitelist.push(...queryAddressOverrides.Item.whitelist);
-    }
-    if (whitelist.length === 0) {
-      log(
-        "ProjectId " + projectId + " ,Address " + address + " has no override"
-      );
-    }
-    console.log("Whitelist:" + whitelist);
-    return whitelist;
-  } catch (scanFeedErr) {
-    log("scan feed error, check dynamodb table", scanFeedErr);
-    return null;
-  }
-}
-
-async function overrideScoreAndIssues(
-  docClient,
-  projectId,
-  address,
-  score,
-  issues,
-  log,
-  primitive
-) {
-  const overrides = await getOverrides(
-    docClient,
-    projectId,
-    address,
-    log,
-    primitive
-  );
-
-  issues = issues.filter((issue) => {
-    if (overrides.includes(issue.code)) {
-      score += issue.reduction;
-      return false;
-    } else return true;
-  });
-  return {
-    score,
-    issues,
-  };
-}
-
-export {
-  getOverrides,
-  overrideScoreAndIssues,
-};

package/token.d.ts

@@ -1,5 +0,0 @@
-import { BigNumber } from "bignumber.js";
-
-export function getTokenPriceAt(tokenAddress: string, timestamp: number, useCache: boolean | undefined): Promise<number>;
-export function toNativeDecimal(bigNumber: BigNumber, decimals: number): Promise<BigNumber>;
-export function toHumanDecimal(bigNumber: BigNumber, decimals: number): Promise<BigNumber>;

package/token.js

@@ -1,46 +0,0 @@
-import { getEnvironment } from "./env.js";
-import { getPriceClosestTo } from "./price.js";
-import BigNumber from "bignumber.js";
-
-const TOKEN_PRICE_TABLE_NAME = `skynet-${getEnvironment()}-token-prices`;
-const TOKEN_PRICE_CACHE = {};
-
-async function getTokenPriceAt(tokenAddress, timestamp, useCache = true) {
-  // to avoid huge amount of dynamodb query
-  // "ceil" timestamp to the closest proximate timestamp
-  // 100_000 seconds ~= a bit more than 1 day 86_400 seconds
-  // and use cache
-  const proximateTimestamp = timestamp + 100_000 - (timestamp % 100_000);
-
-  if (useCache && TOKEN_PRICE_CACHE[tokenAddress] && TOKEN_PRICE_CACHE[tokenAddress][proximateTimestamp]) {
-    return TOKEN_PRICE_CACHE[tokenAddress][proximateTimestamp];
-  }
-
-  const price = await getPriceClosestTo(TOKEN_PRICE_TABLE_NAME, tokenAddress, proximateTimestamp);
-
-  if (useCache) {
-    if (!TOKEN_PRICE_CACHE[tokenAddress]) {
-      TOKEN_PRICE_CACHE[tokenAddress] = {};
-    }
-
-    TOKEN_PRICE_CACHE[tokenAddress][proximateTimestamp] = price;
-  }
-
-  return price;
-}
-
-function toNativeDecimal(bigNumber, decimals) {
-  const dividend = new BigNumber(10).exponentiatedBy(new BigNumber(decimals));
-  return new BigNumber(bigNumber).multipliedBy(dividend);
-}
-
-function toHumanDecimal(bigNumber, decimals) {
-  const dividend = new BigNumber(10).exponentiatedBy(new BigNumber(decimals));
-  return new BigNumber(bigNumber).dividedBy(dividend);
-}
-
-export {
-  getTokenPriceAt,
-  toNativeDecimal,
-  toHumanDecimal,
-};

package/transaction.d.ts

@@ -1,30 +0,0 @@
-type Log = {
-    address: string,
-    topics: string[],
-    data: string,
-    blockNumber: string,
-    transactionHash: string,
-    transactionIndex: string,
-    blockHash: string,
-    logIndex: string,
-    removed: boolean
-}
-
-type TransactionReceipt = {
-    blockHash: string,
-    blockNumber: string,
-    contractAddress: string | null,
-    cumulativeGasUsed: string,
-    effectiveGasPrice: string,
-    from: string,
-    gasUsed: string,
-    logs: Log[],
-    logsBloom: string,
-    status: string,
-    to: string,
-    transactionHash: string,
-    transactionIndex: string,
-    type: string
-}
-
-export function getTxReceipt(protocol: string, txHash: string, verbose: boolean): Promise<TransactionReceipt | null>;

package/transaction.js

@@ -1,42 +0,0 @@
-import { PROTOCOLS } from "./const.js";
-import { exponentialRetry } from "./availability.js";
-
-async function getTxReceipt(protocol, txHash, verbose = false) {
-  const { endpoint } = PROTOCOLS[protocol];
-
-  console.log("got endpoint", protocol, endpoint);
-
-  const body = {
-    jsonrpc: "2.0",
-    method: "eth_getTransactionReceipt",
-    params: [txHash],
-    id: 1
-  };
-
-  const response = await exponentialRetry(
-    () => {
-      return fetch(endpoint, {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify(body)
-      });
-    },
-    {
-      maxRetry: 6,
-      test: r => r.ok,
-      verbose
-    }
-  );
-
-  if (!response.ok) {
-    return null;
-  }
-
-  const { result } = await response.json();
-
-  return result;
-}
-
-export {
-  getTxReceipt
-};