@certik/skynet 0.18.9 → 0.19.0

package/proxy.js

@@ -1,154 +0,0 @@
-const PROXY_INSTANCES = [
-  "https://uer87capga.execute-api.us-east-1.amazonaws.com/", // 0
-  "https://t7utq0a5hb.execute-api.us-east-1.amazonaws.com/",
-  "https://3n8prspdj7.execute-api.us-east-1.amazonaws.com/",
-  "https://u2yvkoxcij.execute-api.us-east-1.amazonaws.com/",
-  "https://am6yzsqgsj.execute-api.us-east-1.amazonaws.com/",
-  "https://7bgxb3dza0.execute-api.us-east-1.amazonaws.com/",
-  "https://f46a3arqm0.execute-api.us-east-1.amazonaws.com/",
-  "https://07hvmxo9u3.execute-api.us-east-1.amazonaws.com/",
-  "https://rwj17dzple.execute-api.us-east-1.amazonaws.com/",
-  "https://siqmv1wrj1.execute-api.us-east-1.amazonaws.com/",
-  "https://hz9vj47r66.execute-api.us-east-1.amazonaws.com/", // 10
-  "https://edqblj52qh.execute-api.us-east-1.amazonaws.com/",
-  "https://bt990cdkkc.execute-api.us-east-1.amazonaws.com/",
-  "https://9pqoo01160.execute-api.us-east-1.amazonaws.com/",
-  "https://kg2swyefu3.execute-api.us-east-1.amazonaws.com/",
-  "https://hjtezlyp5h.execute-api.us-east-1.amazonaws.com/",
-  "https://uv9tqbu127.execute-api.us-east-1.amazonaws.com/",
-  "https://hypj6yxuif.execute-api.us-east-1.amazonaws.com/",
-  "https://q8rwozuheh.execute-api.us-east-1.amazonaws.com/",
-  "https://owk290z9a8.execute-api.us-east-1.amazonaws.com/",
-  "https://rqjqjvz7d6.execute-api.us-east-1.amazonaws.com/", // 20
-  "https://g8jxt5xr74.execute-api.us-east-1.amazonaws.com/",
-  "https://k2psewlm7h.execute-api.us-east-1.amazonaws.com/",
-  "https://kitlcntj2m.execute-api.us-east-1.amazonaws.com/",
-  "https://dxcdo7fmy2.execute-api.us-east-1.amazonaws.com/",
-  "https://idzyuxsy0h.execute-api.us-east-1.amazonaws.com/",
-  "https://osl2t5witk.execute-api.us-east-1.amazonaws.com/",
-  "https://sjnkscdgul.execute-api.us-east-1.amazonaws.com/",
-  "https://rqo11cixlc.execute-api.us-east-1.amazonaws.com/",
-  "https://06zifludsc.execute-api.us-east-1.amazonaws.com/",
-  "https://u9k7jhl2g9.execute-api.us-east-1.amazonaws.com/", // 30
-  "https://uu2jwf0lnf.execute-api.us-east-1.amazonaws.com/",
-  "https://xrqcf8ijfk.execute-api.us-east-1.amazonaws.com/",
-  "https://w5pqx1nnkb.execute-api.us-east-1.amazonaws.com/",
-  "https://qt16b1ikhk.execute-api.us-east-1.amazonaws.com/",
-  "https://y0x4ahq1ak.execute-api.us-east-1.amazonaws.com/",
-  "https://fujoj0uyo9.execute-api.us-east-1.amazonaws.com/",
-  "https://8u8g10va1m.execute-api.us-east-1.amazonaws.com/",
-  "https://ez9lw2f68l.execute-api.us-east-1.amazonaws.com/",
-  "https://20floesmp3.execute-api.us-east-1.amazonaws.com/",
-  "https://15m55yshpi.execute-api.us-east-1.amazonaws.com/", // 40
-  "https://juvdtsrnz2.execute-api.us-east-1.amazonaws.com/",
-  "https://qzyqh8tc9d.execute-api.us-east-1.amazonaws.com/",
-  "https://tcd3dev5bl.execute-api.us-east-1.amazonaws.com/",
-  "https://0heozuo48k.execute-api.us-east-1.amazonaws.com/",
-  "https://6rgbg2eyt4.execute-api.us-east-1.amazonaws.com/",
-  "https://8xcixylxsk.execute-api.us-east-1.amazonaws.com/",
-  "https://gu56mjdx4g.execute-api.us-east-1.amazonaws.com/",
-  "https://2kou7rlmel.execute-api.us-east-1.amazonaws.com/",
-  "https://783td1vsme.execute-api.us-east-1.amazonaws.com/",
-  "https://cb71cnbcwb.execute-api.us-east-1.amazonaws.com/", // 50
-  "https://mjmcciml16.execute-api.us-east-1.amazonaws.com/",
-  "https://rzdrzof2gh.execute-api.us-east-1.amazonaws.com/",
-  "https://r9r1kjdtgb.execute-api.us-east-1.amazonaws.com/",
-  "https://9je54malq1.execute-api.us-east-1.amazonaws.com/",
-  "https://djigu0l9dl.execute-api.us-east-1.amazonaws.com/",
-  "https://bsg5re0koc.execute-api.us-east-1.amazonaws.com/",
-  "https://2uobbakcck.execute-api.us-east-1.amazonaws.com/",
-  "https://qdk9s4u3n5.execute-api.us-east-1.amazonaws.com/",
-  "https://s0qzagys0f.execute-api.us-east-1.amazonaws.com/",
-  "https://t20jva77ek.execute-api.us-east-1.amazonaws.com/", // 60
-  "https://rnz8llxvqf.execute-api.us-east-1.amazonaws.com/",
-  "https://t9cy9qtxp6.execute-api.us-east-1.amazonaws.com/",
-  "https://uph5hi51e9.execute-api.us-east-1.amazonaws.com/",
-  "https://uhas1ctjl2.execute-api.us-east-1.amazonaws.com/",
-  "https://h602ynhw5m.execute-api.us-east-1.amazonaws.com/",
-  "https://qt87x78zw9.execute-api.us-east-1.amazonaws.com/",
-  "https://8gwu72zzkh.execute-api.us-east-1.amazonaws.com/",
-  "https://cvlp54ehig.execute-api.us-east-1.amazonaws.com/",
-  "https://9c99a5ek7c.execute-api.us-east-1.amazonaws.com/",
-  "https://hrz3qhqs0j.execute-api.us-east-1.amazonaws.com/", // 70
-  "https://cm0gfte4kh.execute-api.us-east-1.amazonaws.com/",
-  "https://0xorkhltdi.execute-api.us-east-1.amazonaws.com/",
-  "https://amqsj875b4.execute-api.us-east-1.amazonaws.com/",
-  "https://4ve0o19yr5.execute-api.us-east-1.amazonaws.com/",
-  "https://ektqo7i1t1.execute-api.us-east-1.amazonaws.com/",
-  "https://1hms6u81fe.execute-api.us-east-1.amazonaws.com/",
-  "https://40tydiwluj.execute-api.us-east-1.amazonaws.com/",
-  "https://cehlt4lgsl.execute-api.us-east-1.amazonaws.com/",
-  "https://80fbnad9k8.execute-api.us-east-1.amazonaws.com/",
-  "https://g0bwbsyo8g.execute-api.us-east-1.amazonaws.com/", // 80
-  "https://5b4cqe6sjl.execute-api.us-east-1.amazonaws.com/",
-  "https://f9trwl8thg.execute-api.us-east-1.amazonaws.com/",
-  "https://gqn9p2pka0.execute-api.us-east-1.amazonaws.com/",
-  "https://ibei256b64.execute-api.us-east-1.amazonaws.com/",
-  "https://bzb989tye7.execute-api.us-east-1.amazonaws.com/",
-  "https://hsylj0lmhb.execute-api.us-east-1.amazonaws.com/",
-  "https://2cmgyet6c9.execute-api.us-east-1.amazonaws.com/",
-  "https://fbug2oau1a.execute-api.us-east-1.amazonaws.com/",
-  "https://4dkwwtxnk1.execute-api.us-east-1.amazonaws.com/",
-  "https://hkbqu2rd88.execute-api.us-east-1.amazonaws.com/", // 90
-  "https://i3of390pqe.execute-api.us-east-1.amazonaws.com/",
-  "https://i17mmutqil.execute-api.us-east-1.amazonaws.com/",
-  "https://284yk97cnc.execute-api.us-east-1.amazonaws.com/",
-  "https://jmf3m87fcd.execute-api.us-east-1.amazonaws.com/",
-  "https://k4rohifagh.execute-api.us-east-1.amazonaws.com/",
-  "https://8ttlnvdn5e.execute-api.us-east-1.amazonaws.com/",
-  "https://7vek7u8571.execute-api.us-east-1.amazonaws.com/",
-  "https://hwmema8c6b.execute-api.us-east-1.amazonaws.com/",
-  "https://80xmk8qjdb.execute-api.us-east-1.amazonaws.com/", // 99
-];
-
-async function proxyFetch(url, options = {}) {
-  const rn = Math.floor(Math.random() * PROXY_INSTANCES.length);
-
-  const proxy = PROXY_INSTANCES[rn];
-
-  if (options.verbose) {
-    console.log(`fetching ${url} using ${proxy}`);
-  }
-
-  const controller = new AbortController();
-  const timeout = setTimeout(() => {
-    controller.abort();
-  }, options.timeout || 30_000);
-
-  try {
-    const res = await fetch(proxy, {
-      signal: controller.signal,
-      ...options,
-      headers: {
-        "fetch-url": url,
-        ...options.headers,
-      },
-    });
-
-    if (options.verbose) {
-      console.log(`DONE fetching ${url}`);
-    }
-
-    clearTimeout(timeout);
-
-    return res;
-  } catch (fetchErr) {
-    clearTimeout(timeout);
-
-    if (fetchErr.name === "AbortError") {
-      return {
-        ok: false,
-        status: 408,
-        text: () => Promise.resolve("Timeout"),
-      };
-    }
-
-    return {
-      ok: false,
-      status: 500,
-      text: () => Promise.resolve(fetchErr.message),
-    };
-  }
-}
-
-export default proxyFetch;

package/s3.d.ts

@@ -1,36 +0,0 @@
-import {
-  S3Client,
-  GetObjectCommand,
-  HeadObjectCommand,
-  PutObjectCommand,
-  DeleteObjectCommand,
-  ListObjectsV2Command,
-  NotFound,
-} from "@aws-sdk/client-s3";
-
-type WriteOptions = {
-  verbose: boolean;
-  acl: string;
-  skipIfExists: boolean;
-  contentType: string;
-};
-
-export function getS3(): S3Client;
-export function readFile(bucketName: string, key: string, verbose?: boolean): Promise<string | null>;
-export function hasFile(bucketName: string, key: string): Promise<boolean>;
-export function getFileSize(bucketName: string, key: string): Promise<number>;
-export function writeFile(
-  bucketName: string,
-  key: string,
-  body: string,
-  options?: Partial<WriteOptions>,
-): Promise<void>;
-export function deleteFile(bucketName: string, key: string, verbose?: boolean): Promise<void>;
-export function listKeys(
-  bucketname: string,
-  prefix: string,
-  continuationToken?: string,
-): Promise<{
-  keys: string[];
-  continuationToken: string | null;
-} | null>;

package/search.d.ts

@@ -1,5 +0,0 @@
-export function sendToSearch<T>(
-  indexPrefix: string,
-  record: string,
-  throws?: boolean,
-): Promise<{ status: string; result: T }>;

package/search.js

@@ -1,29 +0,0 @@
-import { isProduction } from "./env.js";
-import { inline } from "./log.js";
-import { Client } from "@elastic/elasticsearch";
-import osModule from "os";
-
-export async function sendToSearch(indexPrefix, record, throws = false) {
-  const client = new Client({
-    cloud: { id: process.env.SKYNET_ELASTICSEARCH_CLOUD_ID },
-    auth: { apiKey: process.env.SKYNET_ELASTICSEARCH_API_KEY },
-  });
-
-  const now = new Date();
-  const indexName = [indexPrefix, isProduction() ? "prod" : "dev", now.toISOString().slice(0, 7)].join("-");
-
-  // best as possible delivery
-  // by default not throw
-  try {
-    await client.index({
-      index: indexName,
-      document: { instance: osModule.hostname(), timestamp: now, record },
-    });
-  } catch (err) {
-    inline.error(err);
-
-    if (throws) {
-      throw err;
-    }
-  }
-}

package/selector.d.ts

@@ -1,19 +0,0 @@
-export type SelectorFlags = {
-  [flag: string]: string;
-};
-
-export type Selector = {
-  [key: string]: {
-    type?: "string" | "number" | "boolean" | string;
-    description?: string;
-    desc?: string;
-    aliases?: string[];
-    isRequired?: boolean;
-    default?: any;
-  };
-};
-
-export function getJobName(name: string, selectorFlags: SelectorFlags, mode?: string): string;
-export function getSelectorDesc(selector: Selector): string;
-export function getSelectorFlags(selector: Selector): SelectorFlags;
-export function toSelectorString(selectorFlags: SelectorFlags, delim?: string): string;

package/selector.js

@@ -1,71 +0,0 @@
-function getSpaces(num) {
-  let str = "";
-  for (let i = 0; i < num; i++) {
-    str += " ";
-  }
-
-  return str;
-}
-
-function getSelectorDesc(selector) {
-  return Object.keys(selector)
-    .map((name) => {
-      return `          --${name}${getSpaces(14 - name.length)}${
-        selector[name].desc || selector[name].description || ""
-      }`;
-    })
-    .join("\n");
-}
-
-// check https://github.com/sindresorhus/meow for selector config options
-function getSelectorFlags(selector) {
-  return Object.keys(selector).reduce((acc, name) => {
-    const flag = {
-      type: selector[name].type || "string",
-      ...selector[name],
-    };
-
-    // by default to be required
-    if (!selector[name].optional && selector[name].isRequired !== false) {
-      flag.isRequired = true;
-    }
-
-    acc[name] = flag;
-    return acc;
-  }, {});
-}
-
-function toSelectorString(selectorFlags, delim = ",") {
-  return Object.keys(selectorFlags)
-    .sort() // deterministic
-    .map((flag) => {
-      return `${flag}=${selectorFlags[flag]}`;
-    })
-    .join(delim);
-}
-
-function normalizeSelectorValue(v) {
-  return v.replace(/[^A-Za-z0-9]+/g, "-");
-}
-
-function getJobName(name, selectorFlags, mode = null) {
-  const selectorNamePart = Object.keys(selectorFlags)
-    .sort()
-    .map((name) => selectorFlags[name])
-    .join("-");
-
-  let jobName = name;
-
-  if (mode) {
-    jobName += `-${mode}`;
-  }
-
-  if (selectorNamePart.length > 0) {
-    // handle special case
-    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
-  }
-
-  return jobName;
-}
-
-export { getJobName, getSelectorDesc, getSelectorFlags, toSelectorString };

package/slack.d.ts

@@ -1,10 +0,0 @@
-import { ChatPostMessageArguments } from "@slack/web-api";
-
-interface PostMessageParams {
-  conversationId: string;
-  message: string | Partial<ChatPostMessageArguments>;
-  token: string;
-  verbose?: boolean;
-}
-
-async function postMessageToConversation(params: PostMessageParams): Promise<void>;

package/snowflake.d.ts

@@ -1,4 +0,0 @@
-import { ConnectionOptions, Connection, Binds } from "snowflake-sdk";
-
-export function getConnection(options: ConnectionOptions): Promise<Connection>;
-export function executeSql<T>(options: ConnectionOptions, sql: string, binds?: Binds): Promise<T[]>;

package/snowflake.js

@@ -1,33 +0,0 @@
-import snowflake from "snowflake-sdk";
-import { promisify } from "util";
-import { ensureAndGet } from "./env.js";
-
-async function getConnection(options) {
-  const connection = snowflake.createConnection({
-    account: ensureAndGet("SKYNET_SNOWFLAKE_ACCOUNT"),
-    username: ensureAndGet("SKYNET_SNOWFLAKE_USERNAME"),
-    password: ensureAndGet("SKYNET_SNOWFLAKE_PASSWORD"),
-    ...options,
-  });
-  return await promisify(connection.connect).bind(connection)();
-}
-
-async function executeSql(options, sql, binds) {
-  const connection = await getConnection(options);
-  return await new Promise((resolve, reject) => {
-    connection.execute({
-      sqlText: sql,
-      binds: binds,
-      complete: (err, statement, rows) => {
-        // console.log(statement.getSqlText());
-        if (err) {
-          reject(err);
-        } else {
-          resolve(rows);
-        }
-      },
-    });
-  });
-}
-
-export { getConnection, executeSql };

package/sqs.d.ts

@@ -1,3 +0,0 @@
-import { SQSClient } from "@aws-sdk/client-sqs";
-
-export function getSQS(): SQSClient;

package/sqs.js

@@ -1,5 +0,0 @@
-import { SQSClient } from "@aws-sdk/client-sqs";
-
-export function getSQS() {
-  return new SQSClient();
-}

package/util.d.ts

@@ -1,5 +0,0 @@
-export function arrayGroup<T>(array: T[], groupSize: number): T[][];
-export function partition(startAt: number, endAt: number, numGroups: number): [number, number][];
-export function range(startAt: number, endAt: number, step: number): [number, number][];
-export function fillRange(startAt: number, endAt: number): number[];
-export function chunk<T>(array: T[], count: number): T[][];

package/util.js

@@ -1,61 +0,0 @@
-// Inclusive endAt
-function partition(startAt, endAt, numGroups) {
-  if (numGroups === 0 || startAt > endAt) {
-    return [];
-  } else if (startAt === endAt) {
-    return [[startAt, endAt]];
-  }
-
-  const span = endAt - startAt + 1;
-  const step = Math.max(Math.floor(span / numGroups), 1);
-  const adjustedStep = span % step === 0 ? step : step + 1;
-
-  return [[startAt, startAt + adjustedStep - 1], ...partition(startAt + adjustedStep, endAt, numGroups - 1)];
-}
-
-// Inclusive range
-function range(startAt, endAt, step) {
-  let arr = [];
-
-  for (let i = startAt; i <= endAt; i += step) {
-    arr.push([i, Math.min(endAt, i + step - 1)]);
-  }
-
-  return arr;
-}
-
-function arrayGroup(array, groupSize) {
-  const groups = [];
-
-  for (let i = 0; i < array.length; i += groupSize) {
-    groups.push(array.slice(i, i + groupSize));
-  }
-
-  return groups;
-}
-
-// return an array with numbers of given inclusive range
-// given 1, 5
-// return [1, 2, 3, 4, 5]
-function fillRange(start, end) {
-  let result = [];
-
-  for (let i = start; i <= end; i++) {
-    result.push(i);
-  }
-
-  return result;
-}
-
-function chunk(array, count) {
-  if (count == null || count < 1) return [];
-  var result = [];
-  var i = 0,
-    length = array.length;
-  while (i < length) {
-    result.push(array.slice(i, (i += count)));
-  }
-  return result;
-}
-
-export { arrayGroup, partition, range, fillRange, chunk };

package/web3.d.ts

@@ -1,25 +0,0 @@
-import Bottleneck from "bottleneck";
-import Web3 from "web3";
-import { ABI } from "./abi";
-
-type Call =
-  | {
-      params: any | any[];
-      functionName?: string;
-      target?: string;
-    }
-  | any[];
-
-export function newWeb3ByProtocol(protocol: string): Web3;
-export function singleCall(protocol: string, abi: ABI, target: string, params: any[]): Promise<any>;
-export function multiCall(multiCallParams: {
-  protocol: string;
-  limiter?: Bottleneck;
-  target: string;
-  abi: ABI;
-  calls: Call[];
-}): Promise<{
-  callCount: number;
-  actualCallCount?: number;
-  output: any[];
-}>;

package/web3.js

@@ -1,113 +0,0 @@
-import { Web3 } from "web3";
-import { PROTOCOLS } from "./const.js";
-import { MULTICALL } from "./abi.js";
-import { chunk } from "./util.js";
-const MULTICALL_CHUNK_SIZE = 400;
-
-function newWeb3ByProtocol(protocol) {
-  if (!Object.keys(PROTOCOLS).includes(protocol)) {
-    return null;
-  }
-
-  return new Web3(PROTOCOLS[protocol].endpoint);
-}
-
-function normalizeCallParams(params) {
-  if (params === undefined) {
-    return [];
-  }
-
-  if (Array.isArray(params)) {
-    return params;
-  }
-
-  return [params];
-}
-
-async function singleCall(protocol, abi, target, params) {
-  const web3 = newWeb3ByProtocol(protocol);
-
-  if (!web3) {
-    throw new Error(`unsupported protocol`);
-  }
-
-  const contract = new web3.eth.Contract([abi], target);
-  const functionSignature = web3.eth.abi.encodeFunctionSignature(abi);
-  const method = contract.methods[functionSignature];
-
-  const result = await method(...normalizeCallParams(params)).call();
-
-  return result;
-}
-
-// limiter if provided, should be an instance of Bottleneck, which is part of bottleneck package
-async function multiCall({ protocol, limiter = null, target, abi, calls }) {
-  const web3 = newWeb3ByProtocol(protocol);
-
-  if (!web3) {
-    throw new Error(`unsupported protocol`);
-  }
-
-  const multiCallProviderAddress = PROTOCOLS[protocol].multiCallProvider;
-
-  if (!multiCallProviderAddress) {
-    throw new Error("protocol doesn't support multicall yet");
-  }
-
-  if (calls.length === 0) return { callCount: 0, output: [] };
-
-  const multiCallContract = new web3.eth.Contract(MULTICALL, multiCallProviderAddress);
-
-  let maybeRateLimitedCallChunk = async (callChunk) => {
-    const txs = callChunk.map((call) => ({
-      delegateCall: false,
-      revertOnError: false,
-      gasLimit: 0,
-      target: call.target || target,
-      value: 0,
-      data: web3.eth.abi.encodeFunctionCall(abi, normalizeCallParams(call.params)),
-    }));
-
-    const response = await multiCallContract.methods.multiCall(txs).call();
-
-    return response._results.map((res, idx) => {
-      const input = {
-        target: callChunk[idx].target || target,
-        params: normalizeCallParams(callChunk[idx].params),
-      };
-
-      try {
-        const callResult = web3.eth.abi.decodeParameters(abi.outputs, res);
-
-        return {
-          input,
-          success: response._successes[idx],
-          // according to SDK doc, if there's only one, return result directly
-          output: abi.outputs.length === 1 ? callResult[0] : callResult,
-        };
-      } catch (decodeErr) {
-        console.log("decode err", abi.name, callChunk[idx].target || target, callChunk[idx].params, decodeErr.message);
-
-        return {
-          input,
-          success: false,
-          output: null,
-        };
-      }
-    });
-  };
-
-  if (limiter) {
-    maybeRateLimitedCallChunk = limiter.wrap(maybeRateLimitedCallChunk);
-  }
-
-  const callChunks = chunk(calls, MULTICALL_CHUNK_SIZE);
-
-  const responses = (
-    await Promise.all(callChunks.map(async (callChunk) => await maybeRateLimitedCallChunk(callChunk)))
-  ).flat();
-
-  return { actualCallCount: callChunks.length, output: responses };
-}
-
-export { newWeb3ByProtocol, singleCall, multiCall };