@certik/skynet 0.10.4 → 0.10.7

package/kafka.js

@@ -1,442 +1,443 @@
-const meow = require("meow");
-const { getEnvironment, getEnvOrThrow } = require("./env");
-const { wait } = require("./availability");
-const { Kafka, logLevel } = require("kafkajs");
-const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
-const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
-const { exponentialRetry } = require("./availability");
-const { getBinaryName } = require("./cli");
-
-const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
-
-async function getProducerLatestId(name, selectorFlags) {
-  const record = await getRecordByKey(STATE_TABLE_NAME, {
-    name: `${name}At(${toSelectorString(selectorFlags)})`,
-  });
-
-  if (record) {
-    return record.value;
-  }
-
-  return 0;
-}
-
-async function setProducerLatestId(name, selectorFlags, value) {
-  await createRecord(STATE_TABLE_NAME, {
-    name: `${name}At(${toSelectorString(selectorFlags)})`,
-    value,
-  });
-}
-
-async function deleteProducerLatestId(name, selectorFlags, verbose) {
-  await deleteRecordsByHashKey(STATE_TABLE_NAME, null, `${name}At(${toSelectorString(selectorFlags)})`, verbose);
-}
-
-function sendToTopic(producer, topic, verbose) {
-  async function send(records) {
-    if (verbose) {
-      console.log(`sending ${records.length} records to topic`, records);
-    }
-
-    await producer.send({
-      topic,
-      messages: records.map((r) => ({
-        value: Buffer.from(typeof r === "string" ? r : JSON.stringify(r)),
-      })),
-    });
-  }
-
-  return send;
-}
-
-function getDefaultKafkaCredential() {
-  return {
-    server: getEnvOrThrow("SKYNET_KAFKA_SERVER"),
-    username: getEnvOrThrow("SKYNET_KAFKA_USERNAME"),
-    password: getEnvOrThrow("SKYNET_KAFKA_PASSWORD"),
-  };
-}
-
-async function initProducer(clientId, customKafkaCredential = undefined) {
-  const kafkaCredential = customKafkaCredential ?? getDefaultKafkaCredential();
-  const kafka = new Kafka({
-    clientId,
-    brokers: [kafkaCredential.server],
-    logLevel: logLevel.ERROR,
-    ssl: true,
-    sasl: {
-      mechanism: "plain",
-      username: kafkaCredential.username,
-      password: kafkaCredential.password,
-    },
-  });
-  const producerInstance = kafka.producer();
-  await producerInstance.connect();
-
-  return producerInstance;
-}
-
-async function closeProducer(producerInstance) {
-  await producerInstance.disconnect();
-}
-
-async function initConsumer(clientId, customKafkaCredential = undefined) {
-  const kafkaCredential = customKafkaCredential ? customKafkaCredential : getDefaultKafkaCredential();
-  const kafka = new Kafka({
-    clientId,
-    brokers: [kafkaCredential.server],
-    logLevel: logLevel.ERROR,
-    ssl: true,
-    sasl: {
-      mechanism: "plain",
-      username: kafkaCredential.username,
-      password: kafkaCredential.password,
-    },
-  });
-  const consumerInstance = kafka.consumer({ groupId: clientId });
-
-  await consumerInstance.connect();
-
-  return consumerInstance;
-}
-
-async function closeConsumer(consumerInstance) {
-  await consumerInstance.disconnect();
-}
-
-async function produceMessages(producerId, callback) {
-  const producerInstance = await initProducer(producerId);
-  const send = async (topic, messages) => {
-    let records;
-    if (Array.isArray(messages)) {
-      records = messages;
-    } else {
-      records = [messages];
-    }
-    return sendToTopic(producerInstance, topic, false)(records);
-  };
-  await callback(send);
-  await closeProducer(producerInstance);
-}
-
-async function consumeMessages(consumerId, topic, callback) {
-  const consumerInstance = await initConsumer(consumerId);
-
-  await consumerInstance.subscribe({ topic });
-
-  const stopConsumeMessages = async () => {
-    await closeConsumer(consumerInstance);
-  };
-  await consumerInstance.run({
-    eachMessage: async ({ topic: receivedTopic, message }) => {
-      if (topic === receivedTopic) {
-        await callback(message, stopConsumeMessages);
-      }
-    },
-  });
-  return stopConsumeMessages;
-}
-
-function createProducerApp({ binaryName, name, selector = {}, producer, state }) {
-  function run() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const finalState = {
-      type: "block",
-      updateInterval: 5000,
-      getMinId: async () => 1,
-      getMaxId: async () => {
-        throw new Error("must implement state.getMaxId");
-      },
-      ...state,
-    };
-
-    const cli = meow(
-      `
-        Usage
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --from          min ${finalState.type} to produce
-          --to            max ${finalState.type} to produce
-          --status        print current status and exit
-          --verbose       Output debug messages
-          --reset         restart producer
-      `,
-      {
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          from: {
-            alias: "since",
-            type: "number",
-            default: 0,
-          },
-          to: {
-            alias: "until",
-            type: "number",
-            default: 0,
-          },
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    const { topic, deadLetterTopic, produce, batchSize, maxRetry } = {
-      batchSize: 50,
-      maxRetry: 2,
-      ...producer,
-    };
-
-    const kafkaCredential = (producer && producer.kafkaCredential) || getDefaultKafkaCredential();
-
-    async function build({ from, to, status, reset, verbose, ...selectorFlags }) {
-      if (reset) {
-        await deleteProducerLatestId(name, selectorFlags);
-        if (finalState.onReset) {
-          await finalState.onReset(selectorFlags, verbose);
-        }
-
-        console.log("producer reset");
-
-        process.exit(0);
-      }
-
-      if (status) {
-        const latestId = await getProducerLatestId(name, selectorFlags);
-
-        console.log(`LatestID=${latestId}`);
-
-        process.exit(0);
-      }
-
-      if (!from) {
-        const prevId = await getProducerLatestId(name, selectorFlags);
-
-        if (prevId) {
-          from = prevId + 1;
-        } else {
-          // initial build
-          from = await finalState.getMinId(selectorFlags);
-        }
-      }
-
-      const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
-      const finalDeadLetterTopic =
-        typeof deadLetterTopic === "function" ? deadLetterTopic(selectorFlags) : deadLetterTopic;
-
-      const updateInterval =
-        typeof finalState.updateInterval === "function"
-          ? finalState.updateInterval(selectorFlags)
-          : finalState.updateInterval;
-
-      const producerInstance = await initProducer(name, kafkaCredential);
-
-      console.log("producing to topic", finalTopic);
-
-      // if to flag was not provided, run this program indefinitely
-      const isLongRunning = to === 0;
-
-      let lastId = to > 0 ? to : await finalState.getMaxId(selectorFlags);
-
-      let pollHandle;
-
-      async function cleanUpForLongRunningMode() {
-        clearInterval(pollHandle);
-
-        await closeProducer(producerInstance);
-
-        process.exit(0);
-      }
-
-      if (isLongRunning) {
-        pollHandle = setInterval(async () => {
-          lastId = await finalState.getMaxId(selectorFlags);
-        }, updateInterval);
-
-        // clean up before exit
-        process.on("SIGINT", cleanUpForLongRunningMode);
-      }
-
-      // the main loop
-      for (let i = from; isLongRunning || i <= lastId; ) {
-        if (i > lastId) {
-          console.log("no more items to process, sleep for", updateInterval, "ms");
-
-          await wait(updateInterval);
-        } else {
-          const batchStart = i;
-          const batchEnd = Math.min(batchStart + batchSize - 1, lastId);
-
-          console.log(`building batch ${batchStart}~${batchEnd}`);
-
-          // add a retry for errors
-          const result = await exponentialRetry(
-            async () => {
-              try {
-                const failed = await produce({
-                  ...selectorFlags,
-                  from: batchStart,
-                  to: batchEnd,
-                  verbose,
-                  send: sendToTopic(producerInstance, finalTopic, verbose),
-                });
-
-                if (Array.isArray(failed) && failed.length > 0) {
-                  await sendToTopic(producerInstance, finalDeadLetterTopic, verbose)(failed);
-                }
-
-                return true;
-              } catch (err) {
-                console.error(`Critical error processing batch ${batchStart}~${batchEnd}`, err);
-
-                return false;
-              }
-            },
-            {
-              maxRetry,
-              test: (r) => r,
-              verbose,
-            }
-          );
-
-          if (!result) {
-            if (isLongRunning) {
-              cleanUpForLongRunningMode();
-            }
-
-            throw new Error(`Terminate producer due to critical errors, batch ${batchStart}~${batchEnd}`);
-          }
-
-          await setProducerLatestId(name, selectorFlags, batchEnd);
-
-          if (i + batchSize <= lastId) i += batchSize;
-          else i = batchEnd + 1;
-        }
-
-        if (isLongRunning) {
-          lastId = await finalState.getMaxId(selectorFlags);
-        }
-      }
-
-      await closeProducer(producerInstance);
-    }
-
-    return build(cli.flags).catch((err) => {
-      console.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { run };
-}
-
-function createConsumerApp({ binaryName, name, selector = {}, consumer }) {
-  function run() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const cli = meow(
-      `
-        Usage
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --verbose       Output debug messages
-      `,
-      {
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    const { topic, consume, maxRetry } = {
-      maxRetry: 2,
-      ...consumer,
-    };
-
-    const kafkaCredential = (consume && consumer.kafkaCredential) || getDefaultKafkaCredential();
-
-    async function build({ verbose, ...selectorFlags }) {
-      const consumerInstance = await initConsumer(name, kafkaCredential);
-
-      try {
-        const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
-
-        await consumerInstance.subscribe({ topic: finalTopic });
-
-        console.log("subscribed to topic", finalTopic);
-
-        await consumerInstance.run({
-          eachBatch: async ({ batch }) => {
-            if (verbose) {
-              console.log("received batch, number of items =", batch.messages.length);
-            }
-
-            const messages = batch.messages.map((m) => JSON.parse(m.value));
-
-            // add a retry for errors
-            const result = await exponentialRetry(
-              async () => {
-                try {
-                  await consume({ ...selectorFlags, messages, verbose });
-
-                  return true;
-                } catch (err) {
-                  console.error(`got critical error`, err);
-
-                  return false;
-                }
-              },
-              {
-                maxRetry,
-                test: (r) => r,
-                verbose,
-              }
-            );
-
-            if (!result) {
-              console.error("Terminate consumer due to consume errors, likely bug");
-
-              process.exit(1);
-            }
-          },
-        });
-      } catch (err) {
-        console.error("Terminate consumer due to critical kafka connection error", err);
-
-        process.exit(1);
-      }
-    }
-
-    return build(cli.flags).catch((err) => {
-      console.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { run };
-}
-
-module.exports = {
-  createProducerApp,
-  createConsumerApp,
-  produceMessages,
-  consumeMessages,
-};
+const meow = require("meow");
+const { getEnvironment, getEnvOrThrow } = require("./env");
+const { wait } = require("./availability");
+const { Kafka, logLevel } = require("kafkajs");
+const { getSelectorFlags, getSelectorDesc, toSelectorString } = require("./selector");
+const { createRecord, getRecordByKey, deleteRecordsByHashKey } = require("./dynamodb");
+const { exponentialRetry } = require("./availability");
+const { getBinaryName } = require("./cli");
+
+const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
+
+async function getProducerLatestId(name, selectorFlags) {
+  const record = await getRecordByKey(STATE_TABLE_NAME, {
+    name: `${name}At(${toSelectorString(selectorFlags)})`,
+  });
+
+  if (record) {
+    return record.value;
+  }
+
+  return 0;
+}
+
+async function setProducerLatestId(name, selectorFlags, value) {
+  await createRecord(STATE_TABLE_NAME, {
+    name: `${name}At(${toSelectorString(selectorFlags)})`,
+    value,
+  });
+}
+
+async function deleteProducerLatestId(name, selectorFlags, verbose) {
+  await deleteRecordsByHashKey(STATE_TABLE_NAME, null, `${name}At(${toSelectorString(selectorFlags)})`, verbose);
+}
+
+function sendToTopic(producer, topic, verbose) {
+  async function send(records) {
+    if (verbose) {
+      console.log(`sending ${records.length} records to topic`, records);
+    }
+
+    await producer.send({
+      topic,
+      messages: records.map((r) => ({
+        value: Buffer.from(typeof r === "string" ? r : JSON.stringify(r)),
+      })),
+    });
+  }
+
+  return send;
+}
+
+function getDefaultKafkaCredential() {
+  return {
+    server: getEnvOrThrow("SKYNET_KAFKA_SERVER"),
+    username: getEnvOrThrow("SKYNET_KAFKA_USERNAME"),
+    password: getEnvOrThrow("SKYNET_KAFKA_PASSWORD"),
+  };
+}
+
+async function initProducer(clientId, customKafkaCredential = undefined) {
+  const kafkaCredential = customKafkaCredential ?? getDefaultKafkaCredential();
+  const kafka = new Kafka({
+    clientId,
+    brokers: [kafkaCredential.server],
+    logLevel: logLevel.ERROR,
+    ssl: true,
+    sasl: {
+      mechanism: "plain",
+      username: kafkaCredential.username,
+      password: kafkaCredential.password,
+    },
+  });
+  const producerInstance = kafka.producer();
+  await producerInstance.connect();
+
+  return producerInstance;
+}
+
+async function closeProducer(producerInstance) {
+  await producerInstance.disconnect();
+}
+
+async function initConsumer(clientId, customKafkaCredential = undefined) {
+  const kafkaCredential = customKafkaCredential ? customKafkaCredential : getDefaultKafkaCredential();
+  const kafka = new Kafka({
+    clientId,
+    brokers: [kafkaCredential.server],
+    logLevel: logLevel.ERROR,
+    ssl: true,
+    sasl: {
+      mechanism: "plain",
+      username: kafkaCredential.username,
+      password: kafkaCredential.password,
+    },
+  });
+  const consumerInstance = kafka.consumer({ groupId: clientId });
+
+  await consumerInstance.connect();
+
+  return consumerInstance;
+}
+
+async function closeConsumer(consumerInstance) {
+  await consumerInstance.disconnect();
+}
+
+async function produceMessages(producerId, callback) {
+  const producerInstance = await initProducer(producerId);
+  const send = async (topic, messages) => {
+    let records;
+    if (Array.isArray(messages)) {
+      records = messages;
+    } else {
+      records = [messages];
+    }
+    return sendToTopic(producerInstance, topic, false)(records);
+  };
+  await callback(send);
+  await closeProducer(producerInstance);
+}
+
+async function consumeMessages(consumerId, topic, callback) {
+  const consumerInstance = await initConsumer(consumerId);
+
+  await consumerInstance.subscribe({ topic });
+
+  const stopConsumeMessages = async () => {
+    await closeConsumer(consumerInstance);
+  };
+  await consumerInstance.run({
+    eachMessage: async ({ topic: receivedTopic, message }) => {
+      if (topic === receivedTopic) {
+        await callback(message, stopConsumeMessages);
+      }
+    },
+  });
+  return stopConsumeMessages;
+}
+
+function createProducerApp({ binaryName, name, selector = {}, producer, state }) {
+  function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const finalState = {
+      type: "block",
+      updateInterval: 5000,
+      getMinId: async () => 1,
+      getMaxId: async () => {
+        throw new Error("must implement state.getMaxId");
+      },
+      ...state,
+    };
+
+    const cli = meow(
+      `
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --from          min ${finalState.type} to produce
+          --to            max ${finalState.type} to produce
+          --status        print current status and exit
+          --verbose       Output debug messages
+          --reset         restart producer
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          from: {
+            alias: "since",
+            type: "number",
+            default: 0,
+          },
+          to: {
+            alias: "until",
+            type: "number",
+            default: 0,
+          },
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    const { topic, deadLetterTopic, produce, batchSize, maxRetry } = {
+      batchSize: 50,
+      maxRetry: 2,
+      ...producer,
+    };
+
+    const kafkaCredential = (producer && producer.kafkaCredential) || getDefaultKafkaCredential();
+
+    async function build({ from, to, status, reset, verbose, ...selectorFlags }) {
+      if (reset) {
+        await deleteProducerLatestId(name, selectorFlags);
+        if (finalState.onReset) {
+          await finalState.onReset(selectorFlags, verbose);
+        }
+
+        console.log("producer reset");
+
+        process.exit(0);
+      }
+
+      if (status) {
+        const latestId = await getProducerLatestId(name, selectorFlags);
+
+        console.log(`LatestID=${latestId}`);
+
+        process.exit(0);
+      }
+
+      if (!from) {
+        const prevId = await getProducerLatestId(name, selectorFlags);
+
+        if (prevId) {
+          from = prevId + 1;
+        } else {
+          // initial build
+          from = await finalState.getMinId(selectorFlags);
+        }
+      }
+
+      const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
+      const finalDeadLetterTopic =
+        typeof deadLetterTopic === "function" ? deadLetterTopic(selectorFlags) : deadLetterTopic;
+
+      const updateInterval =
+        typeof finalState.updateInterval === "function"
+          ? finalState.updateInterval(selectorFlags)
+          : finalState.updateInterval;
+
+      const producerInstance = await initProducer(name, kafkaCredential);
+
+      console.log("producing to topic", finalTopic);
+
+      // if to flag was not provided, run this program indefinitely
+      const isLongRunning = to === 0;
+
+      let lastId = to > 0 ? to : await finalState.getMaxId(selectorFlags);
+
+      let pollHandle;
+
+      async function cleanUpForLongRunningMode() {
+        clearInterval(pollHandle);
+
+        await closeProducer(producerInstance);
+
+        process.exit(0);
+      }
+
+      if (isLongRunning) {
+        pollHandle = setInterval(async () => {
+          lastId = await finalState.getMaxId(selectorFlags);
+        }, updateInterval);
+
+        // clean up before exit
+        process.on("SIGINT", cleanUpForLongRunningMode);
+      }
+
+      // the main loop
+      for (let i = from; isLongRunning || i <= lastId; ) {
+        if (i > lastId) {
+          console.log("no more items to process, sleep for", updateInterval, "ms");
+
+          await wait(updateInterval);
+        } else {
+          const batchStart = i;
+          const batchEnd = Math.min(batchStart + batchSize - 1, lastId);
+
+          console.log(`building batch ${batchStart}~${batchEnd}`);
+
+          // add a retry for errors
+          const result = await exponentialRetry(
+            async () => {
+              try {
+                const failed = await produce({
+                  ...selectorFlags,
+                  from: batchStart,
+                  to: batchEnd,
+                  verbose,
+                  send: sendToTopic(producerInstance, finalTopic, verbose),
+                });
+
+                if (Array.isArray(failed) && failed.length > 0) {
+                  await sendToTopic(producerInstance, finalDeadLetterTopic, verbose)(failed);
+                }
+
+                return true;
+              } catch (err) {
+                console.error(`Critical error processing batch ${batchStart}~${batchEnd}`, err);
+
+                return false;
+              }
+            },
+            {
+              maxRetry,
+              test: (r) => r,
+              verbose,
+            }
+          );
+
+          if (!result) {
+            if (isLongRunning) {
+              cleanUpForLongRunningMode();
+            }
+
+            throw new Error(`Terminate producer due to critical errors, batch ${batchStart}~${batchEnd}`);
+          }
+
+          await setProducerLatestId(name, selectorFlags, batchEnd);
+
+          if (i + batchSize <= lastId) i += batchSize;
+          else i = batchEnd + 1;
+        }
+
+        if (isLongRunning) {
+          lastId = await finalState.getMaxId(selectorFlags);
+        }
+      }
+
+      await closeProducer(producerInstance);
+    }
+
+    return build(cli.flags).catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { run };
+}
+
+function createConsumerApp({ binaryName, name, selector = {}, consumer }) {
+  function run() {
+    if (!binaryName) {
+      binaryName = getBinaryName();
+    }
+
+    const cli = meow(
+      `
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --verbose       Output debug messages
+      `,
+      {
+        description: false,
+        version: false,
+        flags: {
+          ...getSelectorFlags(selector),
+          verbose: {
+            type: "boolean",
+            default: false,
+          },
+        },
+      }
+    );
+
+    const { topic, consume, maxRetry } = {
+      maxRetry: 2,
+      ...consumer,
+    };
+
+    const kafkaCredential = (consume && consumer.kafkaCredential) || getDefaultKafkaCredential();
+
+    async function build({ verbose, ...selectorFlags }) {
+      const consumerInstance = await initConsumer(name, kafkaCredential);
+
+      try {
+        const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
+
+        await consumerInstance.subscribe({ topic: finalTopic });
+
+        console.log("subscribed to topic", finalTopic);
+
+        await consumerInstance.run({
+          eachBatch: async ({ batch }) => {
+            if (verbose) {
+              console.log("received batch, number of items =", batch.messages.length);
+            }
+
+            const messages = batch.messages.map((m) => JSON.parse(m.value));
+
+            // add a retry for errors
+            const result = await exponentialRetry(
+              async () => {
+                try {
+                  await consume({ ...selectorFlags, messages, verbose });
+
+                  return true;
+                } catch (err) {
+                  console.error(`got critical error`, err);
+
+                  return false;
+                }
+              },
+              {
+                maxRetry,
+                test: (r) => r,
+                verbose,
+              }
+            );
+
+            if (!result) {
+              console.error("Terminate consumer due to consume errors, likely bug");
+
+              process.exit(1);
+            }
+          },
+        });
+      } catch (err) {
+        console.error("Terminate consumer due to critical kafka connection error", err);
+
+        process.exit(1);
+      }
+    }
+
+    return build(cli.flags).catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+  }
+
+  return { run };
+}
+
+module.exports = {
+  createProducerApp,
+  createConsumerApp,
+  produceMessages,
+  consumeMessages,
+  getProducerLatestId,
+};