@certik/skynet 0.7.10 → 0.7.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.7.10",
+  "version": "0.7.14",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",
@@ -28,4 +28,4 @@
     "sinon": "^11.1.2"
   },
   "license": "MIT"
-}
+}

package/price.js

@@ -1,48 +1,48 @@
-const { getDocClient } = require("./dynamodb");
-
-async function queryPrice(tableName, address, timestamp, op = "<") {
-  const docClient = getDocClient();
-
-  return await docClient
-    .query({
-      TableName: tableName,
-      KeyConditionExpression: `#address = :address and #timestamp ${op} :timestamp`,
-      ExpressionAttributeNames: {
-        "#timestamp": "timestamp",
-        "#address": "address"
-      },
-      ExpressionAttributeValues: {
-        ":timestamp": timestamp,
-        ":address": address
-      },
-      Limit: 1,
-      ScanIndexForward: false
-    })
-    .promise();
-}
-
-async function getPricePreviousRecord(tableName, address, timestamp) {
-  const query = await queryPrice(tableName, address, timestamp, "<");
-
-  if (query.Count === 0) {
-    return null;
-  } else {
-    return query.Items[0];
-  }
-}
-
-async function getPriceClosestTo(tableName, address, timestamp) {
-  const query = await queryPrice(tableName, address, timestamp, "<=");
-
-  if (query.Count === 0) {
-    return 0;
-  } else {
-    const { price } = query.Items[0];
-    return price;
-  }
-}
-
-module.exports = {
-  getPriceClosestTo,
-  getPricePreviousRecord
-};
+const { getDocClient } = require("./dynamodb");
+
+async function queryPrice(tableName, address, timestamp, op = "<") {
+  const docClient = getDocClient();
+
+  return await docClient
+    .query({
+      TableName: tableName,
+      KeyConditionExpression: `#address = :address and #timestamp ${op} :timestamp`,
+      ExpressionAttributeNames: {
+        "#timestamp": "timestamp",
+        "#address": "address"
+      },
+      ExpressionAttributeValues: {
+        ":timestamp": timestamp,
+        ":address": address
+      },
+      Limit: 1,
+      ScanIndexForward: false
+    })
+    .promise();
+}
+
+async function getPricePreviousRecord(tableName, address, timestamp) {
+  const query = await queryPrice(tableName, address, timestamp, "<");
+
+  if (query.Count === 0) {
+    return null;
+  } else {
+    return query.Items[0];
+  }
+}
+
+async function getPriceClosestTo(tableName, address, timestamp) {
+  const query = await queryPrice(tableName, address, timestamp, "<=");
+
+  if (query.Count === 0) {
+    return 0;
+  } else {
+    const { price } = query.Items[0];
+    return price;
+  }
+}
+
+module.exports = {
+  getPriceClosestTo,
+  getPricePreviousRecord
+};

package/primitive.js

@@ -1,77 +1,77 @@
-async function getOverrides(docClient, projectId, address, log, primitive) {
-  try {
-    console.log("ProjectId:" + projectId + " address:" + address);
-
-    const whitelist = [];
-    const queryProjectOverrides = await docClient
-      .get({
-        TableName: "skynet-prd-primitive-manual-overrides",
-        Key: {
-          primitive: primitive,
-          overrideId: "project/" + projectId,
-        },
-      })
-      .promise();
-    if (queryProjectOverrides.Item && queryProjectOverrides.Item.whitelist) {
-      whitelist.push(...queryProjectOverrides.Item.whitelist);
-    }
-
-    const queryAddressOverrides = await docClient
-      .get({
-        TableName: "skynet-prd-primitive-manual-overrides",
-        Key: {
-          primitive: primitive,
-          overrideId: "address/" + address,
-        },
-      })
-      .promise();
-
-    if (queryAddressOverrides.Item && queryAddressOverrides.Item.whitelist) {
-      console.log(queryAddressOverrides.Item.whitelist);
-      whitelist.push(...queryAddressOverrides.Item.whitelist);
-    }
-    if (whitelist.length === 0) {
-      log(
-        "ProjectId " + projectId + " ,Address " + address + " has no override"
-      );
-    }
-    console.log("Whitelist:" + whitelist);
-    return whitelist;
-  } catch (scanFeedErr) {
-    log("scan feed error, check dynamodb table", scanFeedErr);
-    return null;
-  }
-}
-
-async function overrideScoreAndIssues(
-  docClient,
-  projectId,
-  address,
-  score,
-  issues,
-  log,
-  primitive
-) {
-  const overrides = await getOverrides(
-    docClient,
-    projectId,
-    address,
-    log,
-    primitive
-  );
-
-  issues = issues.filter((issue) => {
-    if (overrides.includes(issue.code)) {
-      score += issue.reduction;
-      return false;
-    } else return true;
-  });
-  return {
-    score,
-    issues,
-  };
-}
-module.exports = {
-  getOverrides,
-  overrideScoreAndIssues,
-};
+async function getOverrides(docClient, projectId, address, log, primitive) {
+  try {
+    console.log("ProjectId:" + projectId + " address:" + address);
+
+    const whitelist = [];
+    const queryProjectOverrides = await docClient
+      .get({
+        TableName: "skynet-prd-primitive-manual-overrides",
+        Key: {
+          primitive: primitive,
+          overrideId: "project/" + projectId,
+        },
+      })
+      .promise();
+    if (queryProjectOverrides.Item && queryProjectOverrides.Item.whitelist) {
+      whitelist.push(...queryProjectOverrides.Item.whitelist);
+    }
+
+    const queryAddressOverrides = await docClient
+      .get({
+        TableName: "skynet-prd-primitive-manual-overrides",
+        Key: {
+          primitive: primitive,
+          overrideId: "address/" + address,
+        },
+      })
+      .promise();
+
+    if (queryAddressOverrides.Item && queryAddressOverrides.Item.whitelist) {
+      console.log(queryAddressOverrides.Item.whitelist);
+      whitelist.push(...queryAddressOverrides.Item.whitelist);
+    }
+    if (whitelist.length === 0) {
+      log(
+        "ProjectId " + projectId + " ,Address " + address + " has no override"
+      );
+    }
+    console.log("Whitelist:" + whitelist);
+    return whitelist;
+  } catch (scanFeedErr) {
+    log("scan feed error, check dynamodb table", scanFeedErr);
+    return null;
+  }
+}
+
+async function overrideScoreAndIssues(
+  docClient,
+  projectId,
+  address,
+  score,
+  issues,
+  log,
+  primitive
+) {
+  const overrides = await getOverrides(
+    docClient,
+    projectId,
+    address,
+    log,
+    primitive
+  );
+
+  issues = issues.filter((issue) => {
+    if (overrides.includes(issue.code)) {
+      score += issue.reduction;
+      return false;
+    } else return true;
+  });
+  return {
+    score,
+    issues,
+  };
+}
+module.exports = {
+  getOverrides,
+  overrideScoreAndIssues,
+};

package/rateLimit.js

@@ -1,21 +1,21 @@
-const fetch = require("node-fetch");
-
-/**
- * Record the api access and return the rate limit result.
- * @async
- * @param {string} name - The name of the key.
- * @param {number} unitTime - The sliding window for rate limiting, in seconds.
- * @param {number} maxAccess - The total count of allowed accesses in a sliding window.
- * @returns {Promise<boolean>} This request should be allowed or not.
- */
-exports.isRateLimited = async (name, unitTime, maxAccess) => {
-  try {
-    const url = `https://rate-limit.certik-skynet.com?name=${name}&unitTime=${unitTime}&maxAccess=${maxAccess}`;
-    const response = await fetch(url);
-    const result = await response.json();
-    if (result.remaining > 0) return false;
-    return true;
-  } catch {
-    return false;
-  }
-};
+const fetch = require("node-fetch");
+
+/**
+ * Record the api access and return the rate limit result.
+ * @async
+ * @param {string} name - The name of the key.
+ * @param {number} unitTime - The sliding window for rate limiting, in seconds.
+ * @param {number} maxAccess - The total count of allowed accesses in a sliding window.
+ * @returns {Promise<boolean>} This request should be allowed or not.
+ */
+exports.isRateLimited = async (name, unitTime, maxAccess) => {
+  try {
+    const url = `https://rate-limit.certik-skynet.com?name=${name}&unitTime=${unitTime}&maxAccess=${maxAccess}`;
+    const response = await fetch(url);
+    const result = await response.json();
+    if (result.remaining > 0) return false;
+    return true;
+  } catch {
+    return false;
+  }
+};

package/s3.js

@@ -1,93 +1,93 @@
-const {
-  getAWSAccessKeyId,
-  getAWSSecretAccessKey,
-  getAWSRegion
-} = require("./env");
-const { S3 } = require("aws-sdk");
-
-function getS3() {
-  return new S3({
-    accessKeyId: getAWSAccessKeyId(),
-    secretAccessKey: getAWSSecretAccessKey(),
-    region: getAWSRegion()
-  });
-}
-
-async function readFile(bucketName, key, verbose = false) {
-  const s3 = getS3();
-
-  const params = { Bucket: bucketName, Key: key };
-
-  try {
-    const result = await s3.getObject(params).promise();
-    return result.Body.toString("utf-8");
-  } catch (noSuchKeyErr) {
-    if (verbose) {
-      console.log("no such bucket or key", bucketName, key);
-    }
-    // do nothing
-    return null;
-  }
-}
-
-async function hasFile(bucketName, key) {
-  const s3 = getS3();
-
-  try {
-    await s3.headObject({ Bucket: bucketName, Key: key }).promise();
-
-    return true;
-  } catch (headErr) {
-    if (headErr.statusCode === 404) {
-      return false;
-    }
-
-    throw headErr;
-  }
-}
-
-async function writeFile(bucketName, key, body, options = {}) {
-  const s3 = getS3();
-
-  if (options.skipIfExists) {
-    if (await hasFile(bucketName, key)) {
-      return;
-    }
-  }
-
-  const verbose = options.verbose || false;
-
-  const params = { Bucket: bucketName, Key: key, Body: body };
-
-  if (options.acl) {
-    params.ACL = options.acl;
-  }
-
-  if (verbose) {
-    console.log("uploading", key);
-  }
-
-  await s3.upload(params).promise();
-}
-
-async function deleteFile(bucketName, key, verbose = false) {
-  const s3 = getS3();
-
-  const params = { Bucket: bucketName, Key: key };
-
-  try {
-    await s3.deleteObject(params).promise();
-  } catch (noSuchKeyErr) {
-    if (verbose) {
-      console.log("no such bucket or key", bucketName, key);
-    }
-  }
-}
-
-module.exports = {
-  getS3,
-  hasFile,
-  readFile,
-  writeFile,
-  deleteFile
-};
+const {
+  getAWSAccessKeyId,
+  getAWSSecretAccessKey,
+  getAWSRegion
+} = require("./env");
+const { S3 } = require("aws-sdk");
+
+function getS3() {
+  return new S3({
+    accessKeyId: getAWSAccessKeyId(),
+    secretAccessKey: getAWSSecretAccessKey(),
+    region: getAWSRegion()
+  });
+}
+
+async function readFile(bucketName, key, verbose = false) {
+  const s3 = getS3();
+
+  const params = { Bucket: bucketName, Key: key };
+
+  try {
+    const result = await s3.getObject(params).promise();
+    return result.Body.toString("utf-8");
+  } catch (noSuchKeyErr) {
+    if (verbose) {
+      console.log("no such bucket or key", bucketName, key);
+    }
+    // do nothing
+    return null;
+  }
+}
+
+async function hasFile(bucketName, key) {
+  const s3 = getS3();
+
+  try {
+    await s3.headObject({ Bucket: bucketName, Key: key }).promise();
+
+    return true;
+  } catch (headErr) {
+    if (headErr.statusCode === 404) {
+      return false;
+    }
+
+    throw headErr;
+  }
+}
+
+async function writeFile(bucketName, key, body, options = {}) {
+  const s3 = getS3();
+
+  if (options.skipIfExists) {
+    if (await hasFile(bucketName, key)) {
+      return;
+    }
+  }
+
+  const verbose = options.verbose || false;
+
+  const params = { Bucket: bucketName, Key: key, Body: body };
+
+  if (options.acl) {
+    params.ACL = options.acl;
+  }
+
+  if (verbose) {
+    console.log("uploading", key);
+  }
+
+  await s3.upload(params).promise();
+}
+
+async function deleteFile(bucketName, key, verbose = false) {
+  const s3 = getS3();
+
+  const params = { Bucket: bucketName, Key: key };
+
+  try {
+    await s3.deleteObject(params).promise();
+  } catch (noSuchKeyErr) {
+    if (verbose) {
+      console.log("no such bucket or key", bucketName, key);
+    }
+  }
+}
+
+module.exports = {
+  getS3,
+  hasFile,
+  readFile,
+  writeFile,
+  deleteFile
+};

package/scan.js

@@ -1,68 +1,68 @@
-const { exponentialRetry, wait } = require("@certik/skynet/availability");
-const { PROTOCOLS } = require("@certik/skynet/const");
-const fetch = require("node-fetch");
-
-const BATCH_SIZE = 10_000; // Max number of transactions fetched from scan api at once
-
-function getScanApiEndpoint(protocol, addr, startBlock = 0, endBlock = null) {
-  const { endpoint, key } = PROTOCOLS[protocol].scanApi;
-  let url = `${endpoint}?module=account&action=txlist&address=${addr}&apikey=${key}&sort=asc&startblock=${startBlock}&page=1&offset=${BATCH_SIZE}`;
-
-  if (endBlock) {
-    url += `&endblock=${endBlock}`;
-  }
-
-  return url;
-}
-
-async function fetchTxs(protocol, addr, startBlock, to, verbose) {
-  const url = getScanApiEndpoint(protocol, addr, startBlock, to);
-
-  const txs = await exponentialRetry(
-    async () => {
-      const res = await fetch(url);
-      const scanJSON = await res.json();
-      return scanJSON.result;
-    },
-    { maxRetry: 6, test: Array.isArray, verbose }
-  )
-
-  console.log(`start block=${startBlock}, total items=${txs.length}`);
-  return txs;
-}
-
-async function streamTxs(address, since, to, callback, verbose) {
-  let startBlock = since;
-  let hasMorePage = true;
-  let lastTxHash = null;
-
-  const [protocol, addr] = address.split(":");
-
-  while (hasMorePage) {
-    const txs = await fetchTxs(protocol, addr, startBlock, to, verbose);
-
-    // This will only ever happen if the total number of txs is a multiple of our batch size
-    if (txs.length === 0) {
-      break;
-    }
-
-    // If our start block overlaps with the last request's end block, then we may have some overlapping txs
-    // To avoid adding those txs to the db twice, we filter them out
-    const newTxsStartIdx = txs.indexOf(({ hash }) => hash === lastTxHash);
-
-    const newTxs = txs.slice(newTxsStartIdx !== -1 ? newTxsStartIdx : 0);
-
-    for (const tx of newTxs) {
-      callback(tx);
-    }
-
-    hasMorePage = txs.length === BATCH_SIZE;
-
-    startBlock = parseInt(txs[txs.length - 1].blockNumber, 10) + 1;
-    lastTxHash = txs[txs.length - 1].hash;
-
-    await wait(2000);
-  }
-}
-
+const { exponentialRetry, wait } = require("@certik/skynet/availability");
+const { PROTOCOLS } = require("@certik/skynet/const");
+const fetch = require("node-fetch");
+
+const BATCH_SIZE = 10_000; // Max number of transactions fetched from scan api at once
+
+function getScanApiEndpoint(protocol, addr, startBlock = 0, endBlock = null) {
+  const { endpoint, key } = PROTOCOLS[protocol].scanApi;
+  let url = `${endpoint}?module=account&action=txlist&address=${addr}&apikey=${key}&sort=asc&startblock=${startBlock}&page=1&offset=${BATCH_SIZE}`;
+
+  if (endBlock) {
+    url += `&endblock=${endBlock}`;
+  }
+
+  return url;
+}
+
+async function fetchTxs(protocol, addr, startBlock, to, verbose) {
+  const url = getScanApiEndpoint(protocol, addr, startBlock, to);
+
+  const txs = await exponentialRetry(
+    async () => {
+      const res = await fetch(url);
+      const scanJSON = await res.json();
+      return scanJSON.result;
+    },
+    { maxRetry: 6, test: Array.isArray, verbose }
+  )
+
+  console.log(`start block=${startBlock}, total items=${txs.length}`);
+  return txs;
+}
+
+async function streamTxs(address, since, to, callback, verbose) {
+  let startBlock = since;
+  let hasMorePage = true;
+  let lastTxHash = null;
+
+  const [protocol, addr] = address.split(":");
+
+  while (hasMorePage) {
+    const txs = await fetchTxs(protocol, addr, startBlock, to, verbose);
+
+    // This will only ever happen if the total number of txs is a multiple of our batch size
+    if (txs.length === 0) {
+      break;
+    }
+
+    // If our start block overlaps with the last request's end block, then we may have some overlapping txs
+    // To avoid adding those txs to the db twice, we filter them out
+    const newTxsStartIdx = txs.indexOf(({ hash }) => hash === lastTxHash);
+
+    const newTxs = txs.slice(newTxsStartIdx !== -1 ? newTxsStartIdx : 0);
+
+    for (const tx of newTxs) {
+      callback(tx);
+    }
+
+    hasMorePage = txs.length === BATCH_SIZE;
+
+    startBlock = parseInt(txs[txs.length - 1].blockNumber, 10) + 1;
+    lastTxHash = txs[txs.length - 1].hash;
+
+    await wait(2000);
+  }
+}
+
 module.exports = { streamTxs };

package/sqs.js

@@ -1,12 +1,12 @@
-const { SQS } = require("aws-sdk");
-const { getAWSAccessKeyId, getAWSSecretAccessKey, getAWSRegion } = require("./env");
-
-function getSQS() {
-  return new SQS({
-    accessKeyId: getAWSAccessKeyId(),
-    secretAccessKey: getAWSSecretAccessKey(),
-    region: getAWSRegion(),
-  });
-}
-
-module.exports = { getSQS };
+const { SQS } = require("aws-sdk");
+const { getAWSAccessKeyId, getAWSSecretAccessKey, getAWSRegion } = require("./env");
+
+function getSQS() {
+  return new SQS({
+    accessKeyId: getAWSAccessKeyId(),
+    secretAccessKey: getAWSSecretAccessKey(),
+    region: getAWSRegion(),
+  });
+}
+
+module.exports = { getSQS };