@certik/skynet 0.20.2 → 0.22.0

package/dist/util.d.ts

@@ -0,0 +1,5 @@
+declare function range(startAt: number, endAt: number, step: number): [number, number][];
+declare function arrayGroup<T>(array: T[], groupSize: number): T[][];
+declare function fillRange(start: number, end: number): number[];
+export { arrayGroup, range, fillRange };
+//# sourceMappingURL=util.d.ts.map

package/dist/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../util.ts"],"names":[],"mappings":"AACA,iBAAS,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,sBAQ1D;AAED,iBAAS,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,SAQnD;AAKD,iBAAS,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,YAQ5C;AAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC"}

package/dist/util.js

@@ -0,0 +1,72 @@
+// @bun
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __toESM = (mod, isNodeMode, target) => {
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: () => mod[key],
+        enumerable: true
+      });
+  return to;
+};
+var __moduleCache = /* @__PURE__ */ new WeakMap;
+var __toCommonJS = (from) => {
+  var entry = __moduleCache.get(from), desc;
+  if (entry)
+    return entry;
+  entry = __defProp({}, "__esModule", { value: true });
+  if (from && typeof from === "object" || typeof from === "function")
+    __getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
+      get: () => from[key],
+      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+    }));
+  __moduleCache.set(from, entry);
+  return entry;
+};
+var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: (newValue) => all[name] = () => newValue
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+var __require = import.meta.require;
+
+// util.ts
+function range(startAt, endAt, step) {
+  const arr = [];
+  for (let i = startAt;i <= endAt; i += step) {
+    arr.push([i, Math.min(endAt, i + step - 1)]);
+  }
+  return arr;
+}
+function arrayGroup(array, groupSize) {
+  const groups = [];
+  for (let i = 0;i < array.length; i += groupSize) {
+    groups.push(array.slice(i, i + groupSize));
+  }
+  return groups;
+}
+function fillRange(start, end) {
+  const result = [];
+  for (let i = start;i <= end; i++) {
+    result.push(i);
+  }
+  return result;
+}
+export {
+  range,
+  fillRange,
+  arrayGroup
+};

package/examples/api.ts

@@ -8,7 +8,7 @@
 //   $ examples/api --help
 
 import type { Request, Response, NextFunction } from "express";
-import { api, SENSITIVE_VALUE } from "../app.ts";
+import { api, SENSITIVE_VALUE } from "../app";
 
 // an example middleware
 async function exampleMiddleware(req: Request, res: Response, next: NextFunction) {

package/examples/indexer.ts

@@ -7,7 +7,7 @@
 //   $ examples/indexer deploy --protocol eth
 //   $ examples/indexer --help
 
-import { indexer, every } from "../app.ts";
+import { indexer, every } from "../app";
 
 const app = indexer({
   name: "example-indexer",

package/examples/mode-indexer.ts

@@ -8,7 +8,7 @@
 //   $ examples/mode-indexer deploy --protocol eth
 //   $ examples/mode-indexer --help
 
-import { modeIndexer, every } from "../app.ts";
+import { modeIndexer, every } from "../app";
 
 async function validate({
   protocol,

package/indexer.ts

@@ -1,13 +1,13 @@
 import meow from "meow";
-import { createRecord, getRecordByKey } from "./dynamodb.ts";
-import { getEnvironment } from "./env.ts";
-import { exponentialRetry } from "./availability.ts";
-import { range as numberRange, fillRange as fillNumberRange } from "./util.ts";
-import { getSelectorDesc, getSelectorFlags, toSelectorString } from "./selector.ts";
-import type { SelectorFlags, Selector } from "./selector.ts";
-import { getBinaryName } from "./cli.ts";
-import { inline } from "./log.ts";
-import { findDateAfter, dateRange, daysInRange as fillDateRange } from "./date.ts";
+import { createRecord, getRecordByKey } from "./dynamodb";
+import { getEnvironment } from "./env";
+import { exponentialRetry } from "./availability";
+import { range as numberRange, fillRange as fillNumberRange } from "./util";
+import { getSelectorDesc, getSelectorFlags, toSelectorString } from "./selector";
+import type { SelectorFlags, Selector } from "./selector";
+import { getBinaryName } from "./cli";
+import { inline } from "./log";
+import { findDateAfter, dateRange, daysInRange as fillDateRange } from "./date";
 
 const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
@@ -643,7 +643,7 @@ ${selector ? getSelectorDesc(selector) : ""}
     );
 
     try {
-      return runMode(cli.flags);
+      return runMode(cli.flags as ModeIndexerFlags<TSelector>);
     } catch (err) {
       inline.error(err);
       process.exit(1);
@@ -735,7 +735,7 @@ ${selector ? getSelectorDesc(selector) : ""}
       inline.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
     }
 
-    return runBuild(cli.flags).catch((err) => {
+    return runBuild(cli.flags as ModeIndexerFlags<TSelector>).catch((err) => {
       inline.error(err);
       process.exit(1);
     });

package/log.ts

@@ -34,20 +34,38 @@ function timestamp() {
 }
 
 const inline = {
+  debug: function (...args: unknown[]) {
+    if (process.env.NODE_ENV === "development") {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
+  },
   log: function (...args: unknown[]) {
-    console.log(`${timestamp()} ${getLine(args)}`);
+    if (process.env.NODE_ENV !== "test") {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
   },
   error: function (...args: unknown[]) {
-    console.error(`${timestamp()} ${getLine(args)}`);
+    if (process.env.NODE_ENV !== "test") {
+      console.error(`${timestamp()} ${getLine(args)}`);
+    }
   },
 };
 
 const logger = {
+  debug: function (...args: unknown[]) {
+    if (process.env.NODE_ENV === "development") {
+      console.log(`[${timestamp()}]`, ...args);
+    }
+  },
   log: function (...args: unknown[]) {
-    console.log(`[${timestamp()}]`, ...args);
+    if (process.env.NODE_ENV !== "test") {
+      console.log(`[${timestamp()}]`, ...args);
+    }
   },
   error: function (...args: unknown[]) {
-    console.error(`[${timestamp()}]`, ...args);
+    if (process.env.NODE_ENV !== "test") {
+      console.error(`[${timestamp()}]`, ...args);
+    }
   },
 };
 

package/object-hash.ts

@@ -0,0 +1,66 @@
+import xh from "@node-rs/xxhash";
+
+export function getHash(obj: unknown) {
+  const xxh3 = xh.xxh3.Xxh3.withSeed();
+  hash(obj, xxh3);
+  return xxh3.digest().toString(16);
+}
+
+function hash(obj: unknown, xxh3: xh.xxh3.Xxh3) {
+  if (obj === null) {
+    xxh3.update("null");
+  } else if (obj === undefined) {
+    xxh3.update("undefined");
+  } else if (typeof obj === "string") {
+    xxh3.update(obj);
+  } else if (typeof obj === "number") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "boolean") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "bigint") {
+    xxh3.update(obj.toString());
+  } else if (obj instanceof Date) {
+    xxh3.update(obj.toISOString());
+  } else if (Array.isArray(obj)) {
+    arrayHash(obj, xxh3);
+  } else if (obj instanceof Set) {
+    setHash(obj, xxh3);
+  } else if (obj instanceof Map) {
+    mapHash(obj, xxh3);
+  } else if (typeof obj === "object") {
+    objectHash(obj, xxh3);
+  } else {
+    throw new Error(`Unsupported type: ${obj}`);
+  }
+}
+
+function arrayHash(array: unknown[], xxh3: xh.xxh3.Xxh3) {
+  xxh3.update("[");
+  for (const obj of array) {
+    hash(obj, xxh3);
+  }
+  xxh3.update("]");
+}
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+function setHash(_set: Set<unknown>, _xxh3: xh.xxh3.Xxh3) {
+  throw new Error("Set hashing not implemented");
+}
+
+function mapHash(map: Map<unknown, unknown>, xxh3: xh.xxh3.Xxh3) {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const array = Array.from(map.entries()).sort(([aKey], [bKey]) => (aKey as any).localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function objectHash(obj: any, xxh3: xh.xxh3.Xxh3) {
+  const array = Object.entries(obj).sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}

package/package.json

@@ -1,11 +1,106 @@
 {
   "name": "@certik/skynet",
-  "version": "0.20.2",
+  "version": "0.22.0",
   "description": "Skynet Shared JS library",
   "type": "module",
-  "main": "index.ts",
+  "exports": {
+    "./abi": {
+      "import": "./dist/abi.js",
+      "types": "./dist/abi.d.ts"
+    },
+    "./address": {
+      "import": "./dist/address.js",
+      "types": "./dist/address.d.ts"
+    },
+    "./api": {
+      "import": "./dist/api.js",
+      "types": "./dist/api.d.ts"
+    },
+    "./app": {
+      "import": "./dist/app.js",
+      "types": "./dist/app.d.ts"
+    },
+    "./availability": {
+      "import": "./dist/availability.js",
+      "types": "./dist/availability.d.ts"
+    },
+    "./cli": {
+      "import": "./dist/cli.js",
+      "types": "./dist/cli.d.ts"
+    },
+    "./const": {
+      "import": "./dist/const.js",
+      "types": "./dist/const.d.ts"
+    },
+    "./databricks": {
+      "import": "./dist/databricks.js",
+      "types": "./dist/databricks.d.ts"
+    },
+    "./date": {
+      "import": "./dist/date.js",
+      "types": "./dist/date.d.ts"
+    },
+    "./deploy": {
+      "import": "./dist/deploy.js",
+      "types": "./dist/deploy.d.ts"
+    },
+    "./dynamodb": {
+      "import": "./dist/dynamodb.js",
+      "types": "./dist/dynamodb.d.ts"
+    },
+    "./env": {
+      "import": "./dist/env.js",
+      "types": "./dist/env.d.ts"
+    },
+    "./graphql": {
+      "import": "./dist/graphql.js",
+      "types": "./dist/graphql.d.ts"
+    },
+    "./indexer": {
+      "import": "./dist/indexer.js",
+      "types": "./dist/indexer.d.ts"
+    },
+    "./log": {
+      "import": "./dist/log.js",
+      "types": "./dist/log.d.ts"
+    },
+    "./object-hash": {
+      "import": "./dist/object-hash.js",
+      "types": "./dist/object-hash.d.ts"
+    },
+    "./opsgenie": {
+      "import": "./dist/opsgenie.js",
+      "types": "./dist/opsgenie.d.ts"
+    },
+    "./por": {
+      "import": "./dist/por.js",
+      "types": "./dist/por.d.ts"
+    },
+    "./s3": {
+      "import": "./dist/s3.js",
+      "types": "./dist/s3.d.ts"
+    },
+    "./search": {
+      "import": "./dist/search.js",
+      "types": "./dist/search.d.ts"
+    },
+    "./selector": {
+      "import": "./dist/selector.js",
+      "types": "./dist/selector.d.ts"
+    },
+    "./slack": {
+      "import": "./dist/slack.js",
+      "types": "./dist/slack.d.ts"
+    },
+    "./util": {
+      "import": "./dist/util.js",
+      "types": "./dist/util.d.ts"
+    }
+  },
   "author": "CertiK Engineering",
   "scripts": {
+    "build": "bun build --outdir dist --target bun --format esm *.ts && tsc --project tsconfig.build.json --emitDeclarationOnly",
+    "clean": "rimraf dist",
     "lint": "eslint '*.ts' test/*.ts",
     "test": "doppler run -- bun test",
     "format": "prettier --write '*.ts' '**/*.ts'",
@@ -21,14 +116,17 @@
     "@aws-sdk/lib-dynamodb": "^3.758.0",
     "@databricks/sql": "^1.9.0",
     "@elastic/elasticsearch": "^8.17.1",
+    "@node-rs/xxhash": "^1.7.6",
     "@slack/web-api": "^6.13.0",
     "chalk": "^5.4.1",
     "execa": "^9.5.2",
     "express": "^4.21.2",
     "md5": "^2.3.0",
     "meow": "^13.2.0",
+    "p-memoize": "^7.1.1",
+    "p-throttle": "^7.0.0",
+    "quick-lru": "^7.0.0",
     "type-fest": "^4.35.0",
-    "web3": "^4.16.0",
     "which": "^5.0.0"
   },
   "devDependencies": {
@@ -43,7 +141,7 @@
     "eslint-plugin-prettier": "^5.2.3",
     "prettier": "^3.5.2",
     "typescript": "^5.8.2",
-    "typescript-eslint": "^8.25.0"
+    "rimraf": "^6.0.1"
   },
   "license": "MIT",
   "publishConfig": {

package/por.ts

@@ -0,0 +1,181 @@
+import { CryptoHasher }  from "bun";
+
+export interface UserLiability {
+  userHash: string;
+  balance: number;
+  token?: string;
+}
+
+export interface MerkleProof {
+  userHash: string;
+  balance: number;
+  token?: string;
+  proof: string[];
+  positions: ('left' | 'right')[];
+  rootHash: string;
+  leafIndex: number;
+  hashFunction?: (data: string) => string;
+}
+
+export interface MerkleNode {
+  hash: string;
+  left?: MerkleNode;
+  right?: MerkleNode;
+  data?: UserLiability;
+}
+
+export class MerkleTreePoR {
+  private leaves: string[] = [];
+  private tree: MerkleNode[][] = [];
+  private userMap: Map<string, number> = new Map();
+  private userData: UserLiability[] = [];
+  private hashFunction: (data: string) => string;
+
+  constructor(users: UserLiability[], hashFunction?: (data: string) => string) {
+    this.userData = [...users];
+    this.hashFunction = hashFunction || this.defaultSHA256Hash;
+    this.buildTree();
+  }
+
+  // Default SHA-256 hash function
+  private static defaultSHA256Hash(data: string): string {
+    const hasher = new CryptoHasher("sha256");
+    hasher.update(data);
+    return hasher.digest("hex");
+  }
+
+  private defaultSHA256Hash(data: string): string {
+    return MerkleTreePoR.defaultSHA256Hash(data);
+  }
+
+  // Configurable hash function
+  private hash(data: string): string {
+    return this.hashFunction(data);
+  }
+
+
+  // hash user data
+  private createLeafHash(user: UserLiability): string {
+    const leafData = `${user.userHash}:${user.balance}:${user.token || ''}`;
+    return this.hash(leafData);
+  }
+
+  private buildTree(): void {
+    if (this.userData.length === 0) {
+      throw new Error('Cannot build tree with empty user data');
+    }
+
+    // Create leaf hashes and store user mapping
+    this.leaves = this.userData.map((user, index) => {
+      const leafHash = this.createLeafHash(user);
+      this.userMap.set(`${user.userHash}:${user.token || ''}`, index);
+      return leafHash;
+    });
+
+    // Start with leaf level
+    let currentLevel = this.leaves;
+    this.tree.push(currentLevel.map(hash => ({ hash })));
+
+    // Build tree bottom-up
+    while (currentLevel.length > 1) {
+      const nextLevel: string[] = [];
+      const nextLevelNodes: MerkleNode[] = [];
+
+      for (let i = 0; i < currentLevel.length; i += 2) {
+        const left = currentLevel[i];
+        const right = i + 1 < currentLevel.length 
+          ? currentLevel[i + 1] 
+          : left; // Duplicate if odd number
+
+        const parentHash = this.hash(left + right);
+        nextLevel.push(parentHash);
+
+        const leftNode = this.tree[this.tree.length - 1][i];
+        const rightNode = i + 1 < currentLevel.length 
+          ? this.tree[this.tree.length - 1][i + 1] 
+          : leftNode;
+
+        nextLevelNodes.push({
+          hash: parentHash,
+          left: leftNode,
+          right: rightNode
+        });
+      }
+
+      currentLevel = nextLevel;
+      this.tree.push(nextLevelNodes);
+    }
+  }
+
+  // get root hash of created tree
+  getRootHash(): string {
+    if (this.tree.length === 0) {
+      throw new Error('Merkle tree not built');
+    }
+    return this.tree[this.tree.length - 1][0].hash;
+  }
+
+  // generate merkle proof
+  generateProof(userHash: string, token?: string): MerkleProof {
+    const key = `${userHash}:${token || ''}`;
+    const leafIndex = this.userMap.get(key);
+    if (leafIndex === undefined) {
+      throw new Error(`${userHash} not found in merkle tree for token ${token}`);
+    }
+
+    const user = this.userData[leafIndex];
+    const proof: string[] = [];
+    const positions: ('left' | 'right')[] = [];
+
+    let currentIndex = leafIndex;
+
+    // Walk up the tree collecting sibling hashes
+    for (let level = 0; level < this.tree.length - 1; level++) {
+      const currentLevelSize = this.tree[level].length;
+      const isRightNode = currentIndex % 2 === 1;
+      const siblingIndex = isRightNode ? currentIndex - 1 : currentIndex + 1;
+
+      if (siblingIndex < currentLevelSize) {
+        proof.push(this.tree[level][siblingIndex].hash);
+        positions.push(isRightNode ? 'left' : 'right');
+      } else {
+        // When odd number, the last node is duplicated
+        proof.push(this.tree[level][currentIndex].hash);
+        positions.push(isRightNode ? 'left' : 'right');
+      }
+
+      currentIndex = Math.floor(currentIndex / 2);
+    }
+
+    return {
+      userHash: user.userHash,
+      balance: user.balance,
+      token: user.token,
+      proof,
+      positions,
+      rootHash: this.getRootHash(),
+      leafIndex,
+      hashFunction: this.hashFunction !== this.defaultSHA256Hash ? this.hashFunction : undefined
+    };
+  }
+
+  // verify inclusion using proof
+  static verifyProof(merkleProof: MerkleProof): boolean {
+    const leafData = `${merkleProof.userHash}:${merkleProof.balance}:${merkleProof.token || ''}`;
+    const hashFunction = merkleProof.hashFunction || MerkleTreePoR.defaultSHA256Hash;
+    let currentHash = hashFunction(leafData);
+    
+    for (let i = 0; i < merkleProof.proof.length; i++) {
+      const siblingHash = merkleProof.proof[i];
+      const position = merkleProof.positions[i];
+
+      if (position === 'left') {
+        currentHash = hashFunction(siblingHash + currentHash);
+      } else {
+        currentHash = hashFunction(currentHash + siblingHash);
+      }
+    }
+
+    return currentHash === merkleProof.rootHash;
+  }
+}

package/search.ts

@@ -1,5 +1,5 @@
-import { isProduction } from "./env.ts";
-import { inline } from "./log.ts";
+import { isProduction } from "./env";
+import { inline } from "./log";
 import { Client, HttpConnection } from "@elastic/elasticsearch";
 import osModule from "os";
 

package/tsconfig.build.json

@@ -0,0 +1,24 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "noEmit": false,
+        "emitDeclarationOnly": true,
+        "outDir": "dist",
+        "declaration": true,
+        "declarationMap": true,
+        "target": "ES2020",
+        "module": "ES2020",
+        "moduleResolution": "bundler",
+        "allowImportingTsExtensions": false,
+        "verbatimModuleSyntax": false
+    },
+    "include": [
+        "*.ts"
+    ],
+    "exclude": [
+        "test/**/*",
+        "examples/**/*",
+        "dist",
+        "node_modules"
+    ]
+}

package/tsconfig.json

@@ -1,23 +1,23 @@
 {
   "compilerOptions": {
     // Enable latest features
-    "lib": ["ESNext", "DOM"],
+    "lib": [
+      "ESNext",
+      "DOM"
+    ],
     "target": "ESNext",
     "module": "ESNext",
     "moduleDetection": "force",
     "allowJs": true,
-
     // Bundler mode
     "moduleResolution": "bundler",
     "allowImportingTsExtensions": true,
     "verbatimModuleSyntax": true,
     "noEmit": true,
-
     // Best practices
-    "strict": true,
+    "strict": false,
     "skipLibCheck": true,
     "noFallthroughCasesInSwitch": true,
-
     // Some stricter flags
     "noUnusedLocals": true,
     "noUnusedParameters": true,