@certik/skynet 0.10.39 → 0.10.41

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.10.41
+
+- Added support for "date" state type for modeIndexer
+- Retired no longer used `inquiry`, `labeling`, `rateLimit` libraries
+
+## 0.10.40
+
+- Added graphql (copied from frontend)
+
 ## 0.10.38 & 0.10.39
 
 - Added x-instance-id header for api service

package/date.js

@@ -0,0 +1,48 @@
+const { arrayGroup } = require("./util");
+
+const MS_IN_A_DAY = 3600 * 24 * 1000;
+
+function getDateOnly(date) {
+  return date.toISOString().split("T")[0];
+}
+
+function findDateAfter(date, n) {
+  const d = new Date(date);
+  const after = new Date(d.getTime() + MS_IN_A_DAY * n);
+
+  return getDateOnly(after);
+}
+
+function daysInRange(from, to) {
+  const fromTime = new Date(from).getTime();
+  const toTime = new Date(to).getTime();
+
+  if (fromTime > toTime) {
+    throw new Error(`range to date couldn't be earlier than range from date`);
+  }
+
+  const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
+
+  const dates = [getDateOnly(new Date(fromTime))];
+
+  for (let i = 1; i <= daysBetween; i += 1) {
+    dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
+  }
+
+  return dates;
+}
+
+function dateRange(from, to, step) {
+  const days = daysInRange(from, to);
+
+  const windows = arrayGroup(days, step);
+
+  return windows.map((w) => [w[0], w[w.length - 1]]);
+}
+
+module.exports = {
+  getDateOnly,
+  findDateAfter,
+  daysInRange,
+  dateRange,
+};

package/examples/mode-indexer

@@ -45,10 +45,10 @@ const app = modeIndexer({
   env: {},
 
   state: {
-    type: "block", // can be omitted, default is block
-    getMinId: async () => 2, // default returns 1
+    type: "date", // can be omitted, default is block
+    getMinId: async () => "2022-09-26", // default returns 1
     getMaxId: async ({ protocol }) => {
-      return 10;
+      return "2022-10-18";
     },
   },
 

package/graphql.d.ts

@@ -0,0 +1,5 @@
+import { Response } from "node-fetch";
+
+type GraphqlVariables = { [key: string]: any };
+
+export function gql(query: string, variables?: GraphqlVariables): Promise<Response>;

package/graphql.js

@@ -0,0 +1,30 @@
+const fetch = require("node-fetch");
+
+const GRAPHQL_API_KEY = process.env.SKYNET_GRAPHQL_API_KEY || process.env.GRAPHQL_API_KEY;
+
+async function gql(url, query, variables = {}) {
+  const res = await fetch(url, {
+    method: "POST",
+    headers: {
+      "x-api-key": GRAPHQL_API_KEY,
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({ query: query.trim(), variables }),
+  });
+
+  if (res.ok) {
+    const { data, errors } = await res.json();
+
+    if (errors && errors.length > 0) {
+      throw new Error(JSON.stringify(errors, null, 2));
+    }
+
+    return data;
+  } else {
+    throw new Error(await res.text());
+  }
+}
+
+module.exports = {
+  gql,
+};

package/indexer.js

@@ -2,10 +2,11 @@ const meow = require("meow");
 const { createRecord, getRecordByKey } = require("./dynamodb");
 const { getEnvironment } = require("./env");
 const { exponentialRetry } = require("./availability");
-const { range, fillRange } = require("./util");
+const { range: numberRange, fillRange: fillNumberRange } = require("./util");
 const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { getBinaryName } = require("./cli");
 const { inline } = require("./log");
+const { findDateAfter, dateRange, daysInRange: fillDateRange } = require("./date");
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -45,6 +46,14 @@ async function getIndexerValidatedId(name, selectorFlags) {
   return 0;
 }
 
+function increaseId(type, currentId, n) {
+  if (type === "date") {
+    return findDateAfter(currentId, n);
+  }
+
+  return currentId + n;
+}
+
 // for those indexers that can have progress tracked by a numeric state.type
 // such as block height, or timestamp
 // managing state would be helpful to reduce the build time
@@ -77,6 +86,23 @@ function createModeIndexerApp({
     ...state,
   };
 
+  // type based range functions
+  function range(from, to, step) {
+    if (finalState.type === "date") {
+      return dateRange(from, to, step);
+    }
+
+    return numberRange(from, to, step);
+  }
+
+  function fillRange(from, to) {
+    if (finalState.type === "date") {
+      return fillDateRange(from, to);
+    }
+
+    return fillNumberRange(from, to);
+  }
+
   async function runMode({ mode, from, to, status, verbose, ...selectorFlags }) {
     if (status) {
       const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
@@ -107,7 +133,7 @@ function createModeIndexerApp({
     );
 
     if (mode === "reset") {
-      await runReset(selectorFlags, from);
+      await runReset(selectorFlags);
     } else if (mode === "rebuild") {
       await runReset(selectorFlags);
       await runRebuild(selectorFlags, from, to > 0 ? to : await finalState.getMaxId(selectorFlags), verbose);
@@ -116,7 +142,7 @@ function createModeIndexerApp({
 
       await runRebuild(
         selectorFlags,
-        previousRebuildEnds ? previousRebuildEnds + 1 : from,
+        previousRebuildEnds ? increaseId(finalState.type, previousRebuildEnds, 1) : from,
         to > 0 ? to : await finalState.getMaxId(selectorFlags),
         verbose
       );
@@ -588,6 +614,7 @@ ${getSelectorDesc(selector)}
 }
 
 module.exports = {
+  increaseId,
   createModeIndexerApp,
   createIndexerApp,
   getIndexerState,

package/log.js

@@ -41,6 +41,7 @@ const inline = {
 };
 
 module.exports = {
+  getLine,
   print,
   inline,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.10.39",
+  "version": "0.10.41",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",
@@ -29,7 +29,7 @@
     "which": "^2.0.2"
   },
   "devDependencies": {
-    "ava": "^4.3.0",
+    "ava": "^5.0.1",
     "eslint": "^8.18.0",
     "eslint-plugin-import": "^2.26.0",
     "sinon": "^14.0.0"

package/transaction.js

@@ -3,7 +3,9 @@ const { PROTOCOLS } = require("./const");
 const { exponentialRetry } = require("./availability");
 
 async function getTxReceipt(protocol, txHash, verbose = false) {
-  let { endpoint } = PROTOCOLS[protocol];
+  const { endpoint } = PROTOCOLS[protocol];
+
+  console.log("got endpoint", protocol, endpoint);
 
   const body = {
     jsonrpc: "2.0",

package/inquiry.d.ts

@@ -1,3 +0,0 @@
-import { Response } from "node-fetch";
-
-export function createAddressRecord(address: string): Promise<Response>;

package/inquiry.js

@@ -1,15 +0,0 @@
-const fetch = require("node-fetch");
-const { getEnvironment } = require("./env");
-
-const ADDRESS_INQUIRY_URL = getEnvironment() === "prd" ? (
-  "https://aml-inquiry.certik-skynet.com/"
-) : (
-  "https://2kbfm0t10j.execute-api.us-east-1.amazonaws.com/dev/"
-);
-
-async function createAddressRecord(address) {
-  // Use skynet-aml-address-inquiry to add addresses as EOA or smart contract
-  return await fetch(`${ADDRESS_INQUIRY_URL}?address=${address}`);
-}
-
-module.exports = { createAddressRecord };

package/labelling.d.ts

@@ -1,15 +0,0 @@
-type Tag = {
-    type: string;
-    label: string;
-}
-
-export function addTagsToAddress(
-    address: string,
-    tags: Tag[],
-    verbose: boolean
-): Promise<{
-    tableName: string;
-    updateItemTags: Tag[];
-} | undefined>;
-
-export function queryAddressTags(address: string): Promise<Tag[]>;

package/labelling.js

@@ -1,90 +0,0 @@
-const fetch = require("node-fetch");
-const { getRecordByKey, updateRecordByKey } = require("./dynamodb");
-const { getEnvironment } = require("./env");
-
-const ADDRESSES_TABLE_NAME = `skynet-${getEnvironment()}-aml-addresses`;
-const SMART_CONTRACTS_TABLE_NAME = `skynet-${getEnvironment()}-aml-smart-contracts`;
-
-async function addTagsToAddress(address, tags, verbose) {
-  let tableName = null;
-  let updateItemTags;
-
-  // Validate tags
-  tags.forEach((tag) => {
-    if (!tag.type || !tag.label) {
-      throw new Error("error: tag must be in format { type, label }");
-    }
-  });
-
-  // check if address is a smart contract
-  let queryResult = await getRecordByKey(SMART_CONTRACTS_TABLE_NAME, { address });
-
-  if (queryResult) {
-    tableName = SMART_CONTRACTS_TABLE_NAME;
-    updateItemTags = queryResult.tags ?? [];
-  } else {
-    // check if address is an EOA
-    queryResult = await getRecordByKey(ADDRESSES_TABLE_NAME, { address });
-
-    if (queryResult) {
-      tableName = ADDRESSES_TABLE_NAME;
-      updateItemTags = queryResult.tags ?? [];
-    } else {
-      // Unknown address type, inquiry endpoint to create a record
-      const url = `https://aml-inquiry.certik-skynet.com/address?address=${address}`;
-      const response = await fetch(url);
-      const json = await response.json();
-      if (!json.type) {
-        throw new Error(`Unknown address type in inquiry response: ${address}`);
-      }
-      const { type } = json;
-      tableName = type === "EOA" ? ADDRESSES_TABLE_NAME : SMART_CONTRACTS_TABLE_NAME;
-      updateItemTags = [];
-    }
-  }
-
-  if (tableName) {
-    // record available, add tag
-    const newTags = tags.filter((tag) => !updateItemTags.includes(tag));
-
-    if (newTags.length + updateItemTags.length === 0) {
-      throw new Error("error: no tags to add and none already in database");
-    }
-
-    if (newTags.length === 0) {
-      return;
-    }
-
-    updateItemTags = [...updateItemTags, ...newTags];
-
-    // keep distinct tags
-    updateItemTags = updateItemTags.filter(
-      (value, index, self) => self.findIndex((t) => t.type === value.type && t.label === value.label) === index
-    );
-
-    await updateRecordByKey(tableName, { address: address }, { tags: updateItemTags }, null, verbose);
-  }
-
-  return { tableName, updateItemTags };
-}
-
-async function queryAddressTags(address) {
-  let updateItemTags = [];
-
-  let queryResult = await getRecordByKey(SMART_CONTRACTS_TABLE_NAME, { address });
-
-  if (queryResult) {
-    updateItemTags = queryResult.tags ?? [];
-  } else {
-    // check if address is an EOA
-    queryResult = await getRecordByKey(ADDRESSES_TABLE_NAME, { address });
-
-    if (queryResult) {
-      updateItemTags = queryResult.tags ?? [];
-    }
-  }
-
-  return updateItemTags;
-}
-
-module.exports = { addTagsToAddress, queryAddressTags };

package/rateLimit.d.ts

@@ -1 +0,0 @@
-export function isRateLimited(name: string, unitTime: number, maxAccess: number): Promise<boolean>;

package/rateLimit.js

@@ -1,21 +0,0 @@
-const fetch = require("node-fetch");
-
-/**
- * Record the api access and return the rate limit result.
- * @async
- * @param {string} name - The name of the key.
- * @param {number} unitTime - The sliding window for rate limiting, in seconds.
- * @param {number} maxAccess - The total count of allowed accesses in a sliding window.
- * @returns {Promise<boolean>} This request should be allowed or not.
- */
-exports.isRateLimited = async (name, unitTime, maxAccess) => {
-  try {
-    const url = `https://rate-limit.certik-skynet.com?name=${name}&unitTime=${unitTime}&maxAccess=${maxAccess}`;
-    const response = await fetch(url);
-    const result = await response.json();
-    if (result.remaining > 0) return false;
-    return true;
-  } catch {
-    return false;
-  }
-};