@certik/skynet 0.7.10 → 0.7.14

package/examples/kafka-producer

@@ -17,7 +17,7 @@ async function produce({ protocol, from, to, verbose, send }) {
     // send records to topic
     const items = [];
     for (let i = from; i <= to; i++) {
-      items.push({ id: i });
+      items.push({ id: i, name: `User ${i}` });
     }
 
     await send(items);

package/inquiry.js

@@ -1,15 +1,15 @@
-const fetch = require("node-fetch");
-const { getEnvironment } = require("./env");
-
-const ADDRESS_INQUIRY_URL = getEnvironment() === "prd" ? (
-  "https://aml-inquiry.certik-skynet.com/"
-) : (
-  "https://2kbfm0t10j.execute-api.us-east-1.amazonaws.com/dev/"
-);
-
-async function createAddressRecord(address) {
-  // Use skynet-aml-address-inquiry to add addresses as EOA or smart contract
-  return await fetch(`${ADDRESS_INQUIRY_URL}?address=${address}`);
-}
-
+const fetch = require("node-fetch");
+const { getEnvironment } = require("./env");
+
+const ADDRESS_INQUIRY_URL = getEnvironment() === "prd" ? (
+  "https://aml-inquiry.certik-skynet.com/"
+) : (
+  "https://2kbfm0t10j.execute-api.us-east-1.amazonaws.com/dev/"
+);
+
+async function createAddressRecord(address) {
+  // Use skynet-aml-address-inquiry to add addresses as EOA or smart contract
+  return await fetch(`${ADDRESS_INQUIRY_URL}?address=${address}`);
+}
+
 module.exports = { createAddressRecord };

package/kafka.js

@@ -3,11 +3,7 @@ const meow = require("meow");
 const { getEnvironment, getEnvOrThrow } = require("./env");
 const { wait } = require("./availability");
 const { Kafka, logLevel } = require("kafkajs");
-const {
-  getSelectorFlags,
-  getSelectorDesc,
-  toSelectorString
-} = require("./selector");
+const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
 const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
 const { exponentialRetry } = require("./availability");
 
@@ -15,7 +11,7 @@ const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 
 async function getProducerLatestId(name, selectorFlags) {
   const record = await getRecordByKey(STATE_TABLE_NAME, {
-    name: `${name}At(${toSelectorString(selectorFlags)})`
+    name: `${name}At(${toSelectorString(selectorFlags)})`,
   });
 
   if (record) {
@@ -28,17 +24,12 @@ async function getProducerLatestId(name, selectorFlags) {
 async function setProducerLatestId(name, selectorFlags, value) {
   await createRecord(STATE_TABLE_NAME, {
     name: `${name}At(${toSelectorString(selectorFlags)})`,
-    value
+    value,
   });
 }
 
 async function deleteProducerLatestId(name, selectorFlags, verbose) {
-  await deleteRecordsByHashKey(
-    STATE_TABLE_NAME,
-    null,
-    `${name}At(${toSelectorString(selectorFlags)})`,
-    verbose
-  );
+  await deleteRecordsByHashKey(STATE_TABLE_NAME, null, `${name}At(${toSelectorString(selectorFlags)})`, verbose);
 }
 
 function sendToTopic(producer, topic, verbose) {
@@ -49,9 +40,9 @@ function sendToTopic(producer, topic, verbose) {
 
     await producer.send({
       topic,
-      messages: records.map(r => ({
-        value: Buffer.from(typeof r === "string" ? r : JSON.stringify(r))
-      }))
+      messages: records.map((r) => ({
+        value: Buffer.from(typeof r === "string" ? r : JSON.stringify(r)),
+      })),
     });
   }
 
@@ -62,8 +53,88 @@ function getDefaultKafkaCredential() {
   return {
     server: getEnvOrThrow("SKYNET_KAFKA_SERVER"),
     username: getEnvOrThrow("SKYNET_KAFKA_USERNAME"),
-    password: getEnvOrThrow("SKYNET_KAFKA_PASSWORD")
+    password: getEnvOrThrow("SKYNET_KAFKA_PASSWORD"),
+  };
+}
+
+async function initProducer(clientId, customKafkaCredential = undefined) {
+  const kafkaCredential = customKafkaCredential ?? getDefaultKafkaCredential();
+  const kafka = new Kafka({
+    clientId,
+    brokers: [kafkaCredential.server],
+    logLevel: logLevel.ERROR,
+    ssl: true,
+    sasl: {
+      mechanism: "plain",
+      username: kafkaCredential.username,
+      password: kafkaCredential.password,
+    },
+  });
+  const producerInstance = kafka.producer();
+  await producerInstance.connect();
+
+  return producerInstance;
+}
+
+async function closeProducer(producerInstance) {
+  await producerInstance.disconnect();
+}
+
+async function initConsumer(clientId, customKafkaCredential = undefined) {
+  const kafkaCredential = customKafkaCredential ? customKafkaCredential : getDefaultKafkaCredential();
+  const kafka = new Kafka({
+    clientId,
+    brokers: [kafkaCredential.server],
+    logLevel: logLevel.ERROR,
+    ssl: true,
+    sasl: {
+      mechanism: "plain",
+      username: kafkaCredential.username,
+      password: kafkaCredential.password,
+    },
+  });
+  const consumerInstance = kafka.consumer({ groupId: clientId });
+
+  await consumerInstance.connect();
+
+  return consumerInstance;
+}
+
+async function closeConsumer(consumerInstance) {
+  await consumerInstance.disconnect();
+}
+
+async function produceMessages(producerId, callback) {
+  const producerInstance = await initProducer(producerId);
+  const send = async (topic, messages) => {
+    let records;
+    if (Array.isArray(messages)) {
+      records = messages;
+    } else {
+      records = [messages];
+    }
+    return sendToTopic(producerInstance, topic, false)(records);
+  };
+  await callback(send);
+  await closeProducer(producerInstance);
+}
+
+async function consumeMessages(consumerId, topic, callback) {
+  const consumerInstance = await initConsumer(consumerId);
+
+  await consumerInstance.subscribe({ topic });
+
+  const stopConsumeMessages = async () => {
+    await closeConsumer(consumerInstance);
   };
+  await consumerInstance.run({
+    eachMessage: async ({ topic: receivedTopic, message }) => {
+      if (topic === receivedTopic) {
+        await callback(message, stopConsumeMessages);
+      }
+    },
+  });
+  return stopConsumeMessages;
 }
 
 function createProducerApp({ name, selector = {}, producer, state }) {
@@ -78,7 +149,7 @@ function createProducerApp({ name, selector = {}, producer, state }) {
       getMaxId: async () => {
         throw new Error("must implement state.getMaxId");
       },
-      ...state
+      ...state,
     };
 
     const cli = meow(
@@ -102,29 +173,28 @@ ${getSelectorDesc(selector)}
           from: {
             alias: "since",
             type: "number",
-            default: 0
+            default: 0,
           },
           to: {
             alias: "until",
             type: "number",
-            default: 0
+            default: 0,
           },
           verbose: {
             type: "boolean",
-            default: false
-          }
-        }
+            default: false,
+          },
+        },
       }
     );
 
     const { topic, deadLetterTopic, produce, batchSize, maxRetry } = {
       batchSize: 50,
       maxRetry: 2,
-      ...producer
+      ...producer,
     };
 
-    const kafkaCredential =
-      (producer && producer.kafkaCredential) || getDefaultKafkaCredential();
+    const kafkaCredential = (producer && producer.kafkaCredential) || getDefaultKafkaCredential();
 
     async function build({ from, to, status, reset, verbose, ...selectorFlags }) {
       if (reset) {
@@ -157,33 +227,16 @@ ${getSelectorDesc(selector)}
         }
       }
 
-      const finalTopic =
-        typeof topic === "function" ? topic(selectorFlags) : topic;
+      const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
       const finalDeadLetterTopic =
-        typeof deadLetterTopic === "function"
-          ? deadLetterTopic(selectorFlags)
-          : deadLetterTopic;
+        typeof deadLetterTopic === "function" ? deadLetterTopic(selectorFlags) : deadLetterTopic;
 
       const updateInterval =
         typeof finalState.updateInterval === "function"
           ? finalState.updateInterval(selectorFlags)
           : finalState.updateInterval;
 
-      const kafka = new Kafka({
-        clientId: name,
-        brokers: [kafkaCredential.server],
-        logLevel: logLevel.ERROR,
-        ssl: true,
-        sasl: {
-          mechanism: "plain",
-          username: kafkaCredential.username,
-          password: kafkaCredential.password
-        }
-      });
-
-      const producerInstance = kafka.producer();
-
-      await producerInstance.connect();
+      const producerInstance = await initProducer(name, kafkaCredential);
 
       console.log("producing to topic", finalTopic);
 
@@ -197,7 +250,7 @@ ${getSelectorDesc(selector)}
       async function cleanUpForLongRunningMode() {
         clearInterval(pollHandle);
 
-        await producerInstance.disconnect();
+        await closeProducer(producerInstance);
 
         process.exit(0);
       }
@@ -214,11 +267,7 @@ ${getSelectorDesc(selector)}
       // the main loop
       for (let i = from; isLongRunning || i <= lastId; ) {
         if (i > lastId) {
-          console.log(
-            "no more items to process, sleep for",
-            updateInterval,
-            "ms"
-          );
+          console.log("no more items to process, sleep for", updateInterval, "ms");
 
           await wait(updateInterval);
         } else {
@@ -236,31 +285,24 @@ ${getSelectorDesc(selector)}
                   from: batchStart,
                   to: batchEnd,
                   verbose,
-                  send: sendToTopic(producerInstance, finalTopic, verbose)
+                  send: sendToTopic(producerInstance, finalTopic, verbose),
                 });
 
                 if (Array.isArray(failed) && failed.length > 0) {
-                  await sendToTopic(
-                    producerInstance,
-                    finalDeadLetterTopic,
-                    verbose
-                  )(failed);
+                  await sendToTopic(producerInstance, finalDeadLetterTopic, verbose)(failed);
                 }
 
                 return true;
               } catch (err) {
-                console.error(
-                  `Critical error processing batch ${batchStart}~${batchEnd}`,
-                  err
-                );
+                console.error(`Critical error processing batch ${batchStart}~${batchEnd}`, err);
 
                 return false;
               }
             },
             {
               maxRetry,
-              test: r => r,
-              verbose
+              test: (r) => r,
+              verbose,
             }
           );
 
@@ -269,13 +311,11 @@ ${getSelectorDesc(selector)}
               cleanUpForLongRunningMode();
             }
 
-            throw new Error(
-              `Terminate producer due to critical errors, batch ${batchStart}~${batchEnd}`
-            );
+            throw new Error(`Terminate producer due to critical errors, batch ${batchStart}~${batchEnd}`);
           }
 
           await setProducerLatestId(name, selectorFlags, batchEnd);
-          
+
           if (i + batchSize <= lastId) i += batchSize;
           else i = batchEnd + 1;
         }
@@ -285,12 +325,10 @@ ${getSelectorDesc(selector)}
         }
       }
 
-      producerInstance.flush(500, () => {
-        producerInstance.disconnect();
-      });
+      await closeProducer(producerInstance);
     }
 
-    return build(cli.flags).catch(err => {
+    return build(cli.flags).catch((err) => {
       console.error(err);
       process.exit(1);
     });
@@ -320,55 +358,36 @@ ${getSelectorDesc(selector)}
           ...getSelectorFlags(selector),
           verbose: {
             type: "boolean",
-            default: false
-          }
-        }
+            default: false,
+          },
+        },
       }
     );
 
     const { topic, consume, maxRetry } = {
       maxRetry: 2,
-      ...consumer
+      ...consumer,
     };
 
-    const kafkaCredential =
-      (consume && consumer.kafkaCredential) || getDefaultKafkaCredential();
+    const kafkaCredential = (consume && consumer.kafkaCredential) || getDefaultKafkaCredential();
 
     async function build({ verbose, ...selectorFlags }) {
-      const kafka = new Kafka({
-        clientId: name,
-        brokers: [kafkaCredential.server],
-        logLevel: logLevel.ERROR,
-        ssl: true,
-        sasl: {
-          mechanism: "plain",
-          username: kafkaCredential.username,
-          password: kafkaCredential.password
-        }
-      });
-
-      const consumerInstance = kafka.consumer({ groupId: name });
+      const consumerInstance = await initConsumer(name, kafkaCredential);
 
       try {
-        await consumerInstance.connect();
-
-        const finalTopic =
-          typeof topic === "function" ? topic(selectorFlags) : topic;
+        const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
 
-        consumerInstance.subscribe({ topic: finalTopic });
+        await consumerInstance.subscribe({ topic: finalTopic });
 
         console.log("subscribed to topic", finalTopic);
 
         await consumerInstance.run({
           eachBatch: async ({ batch }) => {
             if (verbose) {
-              console.log(
-                "received batch, number of items =",
-                batch.messages.length
-              );
+              console.log("received batch, number of items =", batch.messages.length);
             }
 
-            const messages = batch.messages.map(m => JSON.parse(m.value));
+            const messages = batch.messages.map((m) => JSON.parse(m.value));
 
             // add a retry for errors
             const result = await exponentialRetry(
@@ -385,31 +404,26 @@ ${getSelectorDesc(selector)}
               },
               {
                 maxRetry,
-                test: r => r,
-                verbose
+                test: (r) => r,
+                verbose,
               }
             );
 
             if (!result) {
-              console.error(
-                "Terminate consumer due to consume errors, likely bug"
-              );
+              console.error("Terminate consumer due to consume errors, likely bug");
 
               process.exit(1);
             }
-          }
+          },
         });
       } catch (err) {
-        console.error(
-          "Terminate consumer due to critical kafka connection error",
-          err
-        );
+        console.error("Terminate consumer due to critical kafka connection error", err);
 
         process.exit(1);
       }
     }
 
-    return build(cli.flags).catch(err => {
+    return build(cli.flags).catch((err) => {
       console.error(err);
       process.exit(1);
     });
@@ -420,5 +434,7 @@ ${getSelectorDesc(selector)}
 
 module.exports = {
   createProducerApp,
-  createConsumerApp
+  createConsumerApp,
+  produceMessages,
+  consumeMessages,
 };

package/labelling.js

@@ -1,3 +1,4 @@
+const fetch = require("node-fetch");
 const { getRecordByKey, updateRecordByKey } = require("./dynamodb");
 const { getEnvironment } = require("./env");
 
@@ -9,11 +10,11 @@ async function addTagsToAddress(address, tags, verbose) {
   let updateItemTags;
 
   // Validate tags
-  tags.forEach(tag => {
+  tags.forEach((tag) => {
     if (!tag.type || !tag.label) {
       throw new Error("error: tag must be in format { type, label }");
     }
-  })
+  });
 
   // check if address is a smart contract
   let queryResult = await getRecordByKey(SMART_CONTRACTS_TABLE_NAME, { address });
@@ -28,12 +29,23 @@ async function addTagsToAddress(address, tags, verbose) {
     if (queryResult) {
       tableName = ADDRESSES_TABLE_NAME;
       updateItemTags = queryResult.tags ?? [];
+    } else {
+      // Unknown address type, inquiry endpoint to create a record
+      const url = `https://aml-inquiry.certik-skynet.com/address?address=${address}`;
+      const response = await fetch(url);
+      const json = await response.json();
+      if (!json.type) {
+        throw new Error(`Unknown address type in inquiry response: ${address}`);
+      }
+      const { type } = json;
+      tableName = type === "EOA" ? ADDRESSES_TABLE_NAME : SMART_CONTRACTS_TABLE_NAME;
+      updateItemTags = [];
     }
   }
 
   if (tableName) {
     // record available, add tag
-    const newTags = tags.filter(tag => !updateItemTags.includes(tag));
+    const newTags = tags.filter((tag) => !updateItemTags.includes(tag));
 
     if (newTags.length + updateItemTags.length === 0) {
       throw new Error("error: no tags to add and none already in database");
@@ -45,10 +57,34 @@ async function addTagsToAddress(address, tags, verbose) {
 
     updateItemTags = [...updateItemTags, ...newTags];
 
+    // keep distinct tags
+    updateItemTags = updateItemTags.filter(
+      (value, index, self) => self.findIndex((t) => t.type === value.type && t.label === value.label) === index
+    );
+
     await updateRecordByKey(tableName, { address: address }, { tags: updateItemTags }, null, verbose);
   }
 
   return { tableName, updateItemTags };
 }
 
-module.exports = { addTagsToAddress };
+async function queryAddressTags(address) {
+  let updateItemTags = [];
+
+  let queryResult = await getRecordByKey(SMART_CONTRACTS_TABLE_NAME, { address });
+
+  if (queryResult) {
+    updateItemTags = queryResult.tags ?? [];
+  } else {
+    // check if address is an EOA
+    queryResult = await getRecordByKey(ADDRESSES_TABLE_NAME, { address });
+
+    if (queryResult) {
+      updateItemTags = queryResult.tags ?? [];
+    }
+  }
+
+  return updateItemTags;
+}
+
+module.exports = { addTagsToAddress, queryAddressTags };

package/metric.js

@@ -1,65 +1,65 @@
-const { getDocClient } = require("./dynamodb");
-
-/* Assume table has name/timestamp/value fields */
-
-async function getMetricAt(tableName, name, timestamp) {
-  const docClient = getDocClient();
-
-  const query = await docClient
-    .query({
-      TableName: tableName,
-      KeyConditionExpression: "#name = :name and #timestamp <= :timestamp",
-      ExpressionAttributeNames: {
-        "#name": "name",
-        "#timestamp": "timestamp"
-      },
-      ExpressionAttributeValues: {
-        ":name": name,
-        ":timestamp": timestamp
-      },
-      Limit: 1,
-      ScanIndexForward: false
-    })
-    .promise();
-
-  if (query.Count > 0) {
-    const { value } = query.Items[0];
-
-    return value;
-  } else {
-    return null;
-  }
-}
-
-async function getMetricPreviousValue(tableName, name, timestamp) {
-  const docClient = getDocClient();
-
-  const query = await docClient
-    .query({
-      TableName: tableName,
-      KeyConditionExpression: "#name = :name and #timestamp < :timestamp",
-      ExpressionAttributeNames: {
-        "#timestamp": "timestamp",
-        "#name": "name"
-      },
-      ExpressionAttributeValues: {
-        ":timestamp": timestamp,
-        ":name": name
-      },
-      Limit: 1,
-      ScanIndexForward: false
-    })
-    .promise();
-
-  if (query.Count === 0) {
-    return 0;
-  } else {
-    const { value } = query.Items[0];
-    return value;
-  }
-}
-
-module.exports = {
-  getMetricAt,
-  getMetricPreviousValue
-}
+const { getDocClient } = require("./dynamodb");
+
+/* Assume table has name/timestamp/value fields */
+
+async function getMetricAt(tableName, name, timestamp) {
+  const docClient = getDocClient();
+
+  const query = await docClient
+    .query({
+      TableName: tableName,
+      KeyConditionExpression: "#name = :name and #timestamp <= :timestamp",
+      ExpressionAttributeNames: {
+        "#name": "name",
+        "#timestamp": "timestamp"
+      },
+      ExpressionAttributeValues: {
+        ":name": name,
+        ":timestamp": timestamp
+      },
+      Limit: 1,
+      ScanIndexForward: false
+    })
+    .promise();
+
+  if (query.Count > 0) {
+    const { value } = query.Items[0];
+
+    return value;
+  } else {
+    return null;
+  }
+}
+
+async function getMetricPreviousValue(tableName, name, timestamp) {
+  const docClient = getDocClient();
+
+  const query = await docClient
+    .query({
+      TableName: tableName,
+      KeyConditionExpression: "#name = :name and #timestamp < :timestamp",
+      ExpressionAttributeNames: {
+        "#timestamp": "timestamp",
+        "#name": "name"
+      },
+      ExpressionAttributeValues: {
+        ":timestamp": timestamp,
+        ":name": name
+      },
+      Limit: 1,
+      ScanIndexForward: false
+    })
+    .promise();
+
+  if (query.Count === 0) {
+    return 0;
+  } else {
+    const { value } = query.Items[0];
+    return value;
+  }
+}
+
+module.exports = {
+  getMetricAt,
+  getMetricPreviousValue
+}