@certik/skynet 0.12.0 → 0.13.1

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.13.1
+
+- Added: s3 getFileSize (@certik/skynet/s3)
+
+## 0.13.0
+
+- BREAKING: removed ably library (@certik/skynet/ably)
+- BREAKING: removed metric library (@certik/skynet/metric)
+- BREAKING: removed scan library (@certik/skynet/scan)
+- BREAKING: redesigned slack library (@certik/skynet/slack)
+- Upgraded: @aws-sdk to support new S3 directory bucket capability
+- Deprecated: kafka / web3
+
 ## 0.12.0
 
 - BREAKING: removed token module (@certik/skynet/token)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.12.0",
+  "version": "0.13.1",
   "description": "Skynet Shared JS library",
   "type": "module",
   "main": "index.js",
@@ -15,13 +15,12 @@
     "node": ">= 18"
   },
   "dependencies": {
-    "@aws-sdk/client-dynamodb": "^3.449.0",
-    "@aws-sdk/client-s3": "^3.449.0",
-    "@aws-sdk/client-sqs": "^3.449.0",
-    "@aws-sdk/lib-dynamodb": "^3.449.0",
+    "@aws-sdk/client-dynamodb": "^3.478.0",
+    "@aws-sdk/client-s3": "^3.478.0",
+    "@aws-sdk/client-sqs": "^3.478.0",
+    "@aws-sdk/lib-dynamodb": "^3.478.0",
     "@opensearch-project/opensearch": "^2.4.0",
-    "@slack/web-api": "^6.9.1",
-    "ably": "^1.2.47",
+    "@slack/web-api": "^6.11.0",
     "bottleneck": "^2.19.5",
     "chalk": "^5.3.0",
     "execa": "^8.0.1",
@@ -29,16 +28,16 @@
     "kafkajs": "^2.2.4",
     "md5": "^2.3.0",
     "meow": "^12.1.1",
-    "snowflake-sdk": "^1.9.0",
-    "web3": "^4.2.2",
+    "snowflake-sdk": "^1.9.2",
+    "web3": "^4.3.0",
     "which": "^4.0.0"
   },
   "devDependencies": {
-    "ava": "^5.3.1",
-    "eslint": "^8.49.0",
-    "eslint-plugin-import": "^2.28.1",
-    "prettier": "^3.0.3",
-    "sinon": "^15.2.0"
+    "ava": "^6.0.1",
+    "eslint": "^8.56.0",
+    "eslint-plugin-import": "^2.29.1",
+    "prettier": "^3.1.1",
+    "sinon": "^17.0.1"
   },
   "license": "MIT",
   "publishConfig": {

package/s3.js

@@ -46,6 +46,18 @@ async function hasFile(bucketName, key) {
   }
 }
 
+async function getFileSize(bucketName, key) {
+  const s3 = getS3();
+
+  const result = await s3.send(
+    new HeadObjectCommand({
+      Bucket: bucketName,
+      Key: key,
+    }),
+  );
+  return result.ContentLength;
+}
+
 async function writeFile(bucketName, key, body, options = {}) {
   const s3 = getS3();
 
@@ -116,4 +128,4 @@ async function listKeys(bucketname, prefix, continuationToken) {
   return res;
 }
 
-export { getS3, hasFile, readFile, writeFile, deleteFile, listKeys };
+export { getS3, hasFile, readFile, writeFile, deleteFile, getFileSize, listKeys };

package/slack.js

@@ -1,71 +1,16 @@
 import { WebClient } from "@slack/web-api";
 
-function getToken() {
-  return process.env.SKYNET_SLACK_TOKEN;
-}
-
-function getClient(t) {
-  const token = t || getToken();
-
-  if (!token) {
-    throw new Error("Cannot communicate with slack due to missing slack token: process.env.SKYNET_SLACK_TOKEN");
-  }
-
+function getClient(token) {
   const client = new WebClient(token);
 
   return client;
 }
 
-async function findConversation(client, name) {
-  const { conversations } = client;
-
-  let channels = [];
-
-  let result = await conversations.list({
-    types: "public_channel,private_channel",
-    limit: 1000,
-  });
-
-  channels = channels.concat(result.channels);
-
-  while (result.response_metadata.next_cursor) {
-    result = await conversations.list({
-      types: "public_channel,private_channel",
-      cursor: result.response_metadata.next_cursor,
-      limit: 1000,
-    });
-
-    channels = channels.concat(result.channels);
-  }
-
-  for (const channel of channels) {
-    if (channel.name === name) {
-      const conversationId = channel.id;
-
-      return conversationId;
-    }
-  }
-
-  return null;
-}
-
-function isString(s) {
-  return typeof s === "string" || s instanceof String;
-}
-
-async function postMessage(channel, message, verbose) {
+// you can identify conversation ID at the bottom of the channel settings page
+async function postMessageToConversation({ conversationId, message, token, verbose }) {
   try {
-    const token = isString(channel) ? null : channel.token;
     const client = getClient(token);
 
-    let conversationId = isString(channel) ? await findConversation(client, channel) : channel.id;
-
-    if (!conversationId) {
-      throw new Error(
-        `cannot find slack public/private channel: ${channel}, you may have to invite the @CertiK Skynet bot to the channel`
-      );
-    }
-
     let post = {};
 
     if (typeof message === "string") {
@@ -75,7 +20,7 @@ async function postMessage(channel, message, verbose) {
     }
 
     if (verbose) {
-      console.log(`posting to slack channel ${channel} (${conversationId}):`, JSON.stringify(post, null, 2));
+      console.log(`posting to slack conversation ${conversationId}:`, JSON.stringify(post, null, 2));
     }
 
     await client.chat.postMessage({
@@ -89,4 +34,4 @@ async function postMessage(channel, message, verbose) {
   }
 }
 
-export { getClient, findConversation, postMessage };
+export { postMessageToConversation };

package/ably.d.ts

@@ -1,4 +0,0 @@
-import Bottleneck from "bottleneck";
-
-export function publishMessage(channel: string, messageName: string, messageData: any): Promise<void>;
-export const rateLimitedPublishMessage: Bottleneck;

package/ably.js

@@ -1,35 +0,0 @@
-import Ably from "ably";
-import Bottleneck from "bottleneck/es5";
-import { ensureAndGet } from "./env.js";
-
-let ABLY;
-
-async function getAbly() {
-  if (!ABLY) {
-    ABLY = await new Promise((resolve) => {
-      const ably = new Ably.Realtime(ensureAndGet("ABLY_ROOT_API_KEY"));
-      ably.connection.on("connected", () => {
-        console.log("ably connection successful");
-        resolve(ably);
-      });
-    });
-  }
-  return ABLY;
-}
-
-export async function publishMessage(channel, messageName, messageData) {
-  const ably = await getAbly();
-  const channelObj = ably.channels.get(channel);
-
-  const messageDataJson = JSON.stringify(messageData);
-
-  try {
-    await channelObj.publish(messageName, messageDataJson);
-  } catch (error) {
-    console.error("error publishing to ably: ", error);
-  }
-}
-
-export const rateLimitedPublishMessage = new Bottleneck({
-  minTime: 1000 / 40,
-}).wrap(publishMessage);

package/metric.d.ts

@@ -1,10 +0,0 @@
-export function getMetricAt(
-    tableName: string,
-    name: string,
-    timestamp: number
-): Promise<any>;
-export function getMetricPreviousValue(
-    tableName: string,
-    name: string,
-    timestamp: number
-): Promise<any>;

package/metric.js

@@ -1,60 +0,0 @@
-import { getDocClient } from "./dynamodb.js";
-
-/* Assume table has name/timestamp/value fields */
-
-export async function getMetricAt(tableName, name, timestamp) {
-  const docClient = getDocClient();
-
-  const query = await docClient
-    .query({
-      TableName: tableName,
-      KeyConditionExpression: "#name = :name and #timestamp <= :timestamp",
-      ExpressionAttributeNames: {
-        "#name": "name",
-        "#timestamp": "timestamp"
-      },
-      ExpressionAttributeValues: {
-        ":name": name,
-        ":timestamp": timestamp
-      },
-      Limit: 1,
-      ScanIndexForward: false
-    })
-    .promise();
-
-  if (query.Count > 0) {
-    const { value } = query.Items[0];
-
-    return value;
-  } else {
-    return null;
-  }
-}
-
-export async function getMetricPreviousValue(tableName, name, timestamp) {
-  const docClient = getDocClient();
-
-  const query = await docClient
-    .query({
-      TableName: tableName,
-      KeyConditionExpression: "#name = :name and #timestamp < :timestamp",
-      ExpressionAttributeNames: {
-        "#timestamp": "timestamp",
-        "#name": "name"
-      },
-      ExpressionAttributeValues: {
-        ":timestamp": timestamp,
-        ":name": name
-      },
-      Limit: 1,
-      ScanIndexForward: false
-    })
-    .promise();
-
-  if (query.Count === 0) {
-    return 0;
-  } else {
-    const { value } = query.Items[0];
-    return value;
-  }
-}

package/scan.d.ts

@@ -1,47 +0,0 @@
-type Transaction = {
-    blockNumber: string,
-    timeStamp: string,
-    hash: string,
-    nonce: string,
-    blockHash: string,
-    transactionIndex: string,
-    from: string,
-    to: string,
-    value: string | undefined,
-    gas: string,
-    gasPrice: string,
-    isError: string,
-    txreceipt_status: string,
-    input: string,
-    contractAddress: string,
-    cumulativeGasUsed: string,
-    gasUsed: string,
-    confirmations: string,
-    methodId: string,
-    functionName: string
-}
-
-export function streamTxs(
-    address: string, 
-    since: number | undefined, 
-    to: number | undefined,
-    callback: (tx: Transaction) => void, 
-    verbose: boolean
-): Promise<void>;
-
-export function fetchTxs(
-    protocol: string, 
-    addr: string, 
-    since: number | undefined, 
-    to: number | undefined, 
-    offset: number | undefined,
-): Promise<Transaction[] | null>;
-
-export function fetchTxsWithRetry(
-    protocol: string, 
-    addr: string, 
-    since: number | undefined, 
-    to: number | undefined, 
-    offset: number | undefined,
-    verbose: boolean
-): Promise<Transaction[] | null>

package/scan.js

@@ -1,74 +0,0 @@
-import { exponentialRetry, wait } from "./availability.js";
-import { PROTOCOLS } from "./const.js";
-
-const BATCH_SIZE = 10_000; // Max number of transactions fetched from scan api at once
-
-function getScanApiEndpoint(protocol, addr, startBlock = 0, endBlock = null, offset=BATCH_SIZE) {
-  const { endpoint, key } = PROTOCOLS[protocol].scanApi;
-  let url = `${endpoint}?module=account&action=txlist&address=${addr}&apikey=${key}&sort=asc&startblock=${startBlock}&page=1&offset=${offset}`;
-
-  if (endBlock) {
-    url += `&endblock=${endBlock}`;
-  }
-
-  return url;
-}
-
-async function fetchTxs(protocol, addr, since, to, offset) {
-  const url = getScanApiEndpoint(protocol, addr, since, to, offset);
-
-  try {
-    const res = await fetch(url);
-    const scanJSON = await res.json();
-    return scanJSON.result;
-  } catch (err) {
-    console.log(`error fetching txs: ${err}`)
-    return null;
-  }
-}
-
-async function fetchTxsWithRetry(protocol, addr, since, to, offset, verbose) {
-  const txs = await exponentialRetry(
-    async () => fetchTxs(protocol, addr, since, to, offset),
-    { maxRetry: 6, test: Array.isArray, verbose }
-  )
-
-  return txs;
-}
-
-async function streamTxs(address, since, to, callback, verbose) {
-  let startBlock = since;
-  let hasMorePage = true;
-  let lastTxHash = null;
-
-  const [protocol, addr] = address.split(":");
-
-  while (hasMorePage) {
-    const txs = await fetchTxs(protocol, addr, startBlock, to, BATCH_SIZE, verbose);
-    console.log(`start block=${since}, total items=${txs.length}`);
-
-    // This will only ever happen if the total number of txs is a multiple of our batch size
-    if (txs.length === 0) {
-      break;
-    }
-
-    // If our start block overlaps with the last request's end block, then we may have some overlapping txs
-    // To avoid adding those txs to the db twice, we filter them out
-    const newTxsStartIdx = txs.indexOf(({ hash }) => hash === lastTxHash);
-
-    const newTxs = txs.slice(newTxsStartIdx !== -1 ? newTxsStartIdx : 0);
-
-    for (const tx of newTxs) {
-      callback(tx);
-    }
-
-    hasMorePage = txs.length === BATCH_SIZE;
-
-    startBlock = parseInt(txs[txs.length - 1].blockNumber, 10) + 1;
-    lastTxHash = txs[txs.length - 1].hash;
-
-    await wait(2000);
-  }
-}
-
-export { streamTxs, fetchTxs, fetchTxsWithRetry };