@certik/skynet 0.17.0 → 0.18.1

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.18.1
+
+- BREAKING: opensearch feature has been replaced by elastic search
+- Fixed: s3 library listKeys function when search result is empty
+
+## 0.18.0
+
+- BREAKING: remove dependencies on `SKYNET_AWS_ACCESS_KEY_ID` and `SKYNET_AWS_SECRET_ACCESS_KEY` and `SKYNET_AWS_REGION`
+- BREAKING: rename OPSGENIE_API_KEY to SKYNET_OPSGENIE_API_KEY
+- BREAKING: removed producer and consumer support
+
 ## 0.17.0
 
 - BREAKING: changed the way an indexer integrate with added doppler integration support in deploy

package/app.d.ts

@@ -1,4 +1,3 @@
-import { ERROR_LEVEL } from "./monitor";
 import { Selector } from "./selector";
 import express from "express";
 
@@ -110,8 +109,6 @@ type ValidateWithConcurrency = Validate & { concurrency: number | undefined };
 
 export const SENSITIVE_VALUE: null;
 
-export { ERROR_LEVEL };
-
 export function every(n: number | undefined): {
   second: string;
   seconds: string;
@@ -134,30 +131,10 @@ export function api(apiParams: {
   beforeListen: (args: { app: ReturnType<typeof express> }) => void;
 }): () => void;
 
-export function consumer(consumerParams: {
-  name: string;
-  selector?: Selector;
-  consume: Consume;
-  check?: Check;
-  env: Env | undefined;
-  region: string | undefined;
-}): () => void;
-
-export function producer(producerParams: {
-  name: string;
-  selector?: Selector;
-  produce: Produce;
-  check?: Check;
-  state: State;
-  env?: Env;
-  region?: string;
-}): () => void;
-
 export function indexer(indexerParams: {
   name: string;
   selector?: Selector;
   build: BuildWithRestart;
-  check?: Check;
   env?: Env;
   region?: string;
 }): () => void;
@@ -168,7 +145,6 @@ export function modeIndexer(modeIndexerParams: {
   state: State;
   build: BuildWithConcurrency;
   validate?: ValidateWithConcurrency;
-  check?: Check;
   env?: Env;
   region?: string;
 }): () => void;

package/app.js

@@ -1,17 +1,7 @@
 import { EOL } from "os";
-
-import {
-  createIndexerApp,
-  createModeIndexerApp,
-  getIndexerLatestId,
-  getIndexerValidatedId,
-  getIndexerState,
-} from "./indexer.js";
-
+import { createIndexerApp, createModeIndexerApp } from "./indexer.js";
 import { createDeploy, createModeDeploy } from "./deploy.js";
-import { createProducerApp, createConsumerApp, getProducerLatestId } from "./kafka.js";
 import { startApiApp } from "./api.js";
-import { createMonitor, ERROR_LEVEL } from "./monitor.js";
 import { getBinaryName, detectBin, detectWorkingDirectory } from "./cli.js";
 
 function printAppHelp() {
@@ -261,164 +251,6 @@ function modeIndexer({ name, selector, state, build, validate, env = {}, region
   });
 }
 
-function checkProducerProduceParameter(produce) {
-  const errors = [];
-
-  if (!produce?.func) {
-    errors.push("must define produce.func");
-  }
-
-  if (!produce?.topic) {
-    errors.push("must define produce.topic");
-  }
-
-  if (!produce?.deadLetterTopic) {
-    errors.push("must define produce.deadLetterTopic");
-  }
-
-  if (!produce?.cpu) {
-    errors.push("must define produce.cpu");
-  }
-
-  if (!produce?.mem) {
-    errors.push("must define produce.mem");
-  }
-
-  return errors;
-}
-
-function producer({ name, selector, produce, state, env = {}, region = "skynet-dc1" }) {
-  const envWithDefaultValues = {
-    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
-    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
-    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
-    ...env,
-  };
-
-  return createApp({
-    parameterErrors: [
-      ...checkProducerProduceParameter(produce),
-      ...checkStateParameter(state),
-      ...checkEnvParameter(env),
-    ],
-    env: envWithDefaultValues,
-    onRun: () => {
-      const { run } = createProducerApp({
-        binaryName: `${getBinaryName()} run`,
-        name,
-        selector,
-
-        producer: {
-          topic: produce.topic,
-          deadLetterTopic: produce.deadLetterTopic,
-          produce: produce.func,
-          batchSize: produce.batchSize,
-          maxRetry: produce.maxRetry,
-        },
-
-        state,
-      });
-
-      process.title = name;
-
-      return run();
-    },
-    onDeploy: () => {
-      const bin = detectBin();
-      const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
-
-      const { deploy } = createDeploy({
-        binaryName: `${getBinaryName()} deploy`,
-        name,
-        workingDirectory: detectWorkingDirectory(),
-        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
-        selector,
-        region,
-        env: envWithDefaultValues,
-        schedule: "@minutely",
-        killTimeout: produce.killTimeout,
-        cpu: produce.cpu,
-        mem: produce.mem,
-      });
-
-      return deploy();
-    },
-  });
-}
-
-function checkConsumerConsumeParameter(consume) {
-  const errors = [];
-
-  if (!consume?.func) {
-    errors.push("must define consume.func");
-  }
-
-  if (!consume?.topic) {
-    errors.push("must define consume.topic");
-  }
-
-  if (!consume?.cpu) {
-    errors.push("must define consume.cpu");
-  }
-
-  if (!consume?.mem) {
-    errors.push("must define consume.mem");
-  }
-
-  return errors;
-}
-
-function consumer({ name, selector, consume, env = {}, region = "skynet-dc1" }) {
-  const envWithDefaultValues = {
-    SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
-    SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
-    SKYNET_KAFKA_PASSWORD: SENSITIVE_VALUE,
-    ...env,
-  };
-
-  return createApp({
-    parameterErrors: [...checkConsumerConsumeParameter(consume), ...checkEnvParameter(env)],
-    env: envWithDefaultValues,
-    onRun: () => {
-      const { run } = createConsumerApp({
-        binaryName: `${getBinaryName()} run`,
-        name,
-        selector,
-
-        consumer: {
-          topic: consume.topic,
-          consume: consume.func,
-          maxRetry: consume.maxRetry,
-        },
-      });
-
-      process.title = name;
-
-      return run();
-    },
-    onDeploy: () => {
-      const bin = detectBin();
-      const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
-
-      const { deploy } = createDeploy({
-        binaryName: `${getBinaryName()} deploy`,
-        name,
-        workingDirectory: detectWorkingDirectory(),
-        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
-        selector,
-        region,
-        env: envWithDefaultValues,
-        schedule: "@minutely",
-        killTimeout: consume.killTimeout,
-        cpu: consume.cpu,
-        mem: consume.mem,
-      });
-
-      return deploy();
-    },
-  });
-}
-
 function checkApiServeParameter(serve, routes) {
   const errors = [];
 
@@ -560,4 +392,4 @@ const every = (n = 1) => {
   };
 };
 
-export { indexer, modeIndexer, producer, consumer, api, every, SENSITIVE_VALUE, ERROR_LEVEL };
+export { indexer, modeIndexer, api, every, SENSITIVE_VALUE };

package/deploy.js

@@ -63,7 +63,9 @@ const genConfig = ({
 
   group "default" {
     ${count && count > 1 ? `count = ${count}` : ""}
-    ${count && count > 1 ? `# Rolling Update
+    ${
+      count && count > 1
+        ? `# Rolling Update
     update {
       max_parallel = 1
       min_healthy_time = "10s"
@@ -137,7 +139,7 @@ const genConfig = ({
         DOPPLER_CONFIG="${isProduction ? "prd" : "dev"}"
         SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
         ${Object.entries(additionalEnv)
-          .filter(([k, v]) => !!v)
+          .filter((kv) => !!kv[1])
           .map(([key, value]) => `${key}="${value}"`)
           .join("          \n")}
       }

package/dynamodb.js

@@ -9,7 +9,6 @@ import {
   TransactWriteCommand,
 } from "@aws-sdk/lib-dynamodb";
 import { DynamoDBClient, DescribeTableCommand } from "@aws-sdk/client-dynamodb";
-import { getAWSSDKConfig } from "./env.js";
 import { wait } from "./availability.js";
 
 let _dynamoDB;
@@ -18,7 +17,7 @@ let _docClient;
 
 function getDynamoDB(forceNew = false) {
   if (!_dynamoDB || forceNew) {
-    _dynamoDB = new DynamoDBClient(getAWSSDKConfig());
+    _dynamoDB = new DynamoDBClient();
   }
   return _dynamoDB;
 }

package/env.d.ts

@@ -1,15 +1,5 @@
 export function ensureAndGet<T>(envName: string, defaultValue: T): string | T;
 export function getEnvOrThrow(envName: string): string;
-export function getAWSAccessKeyId(): string | undefined;
-export function getAWSSecretAccessKey(): string | undefined;
-export function getAWSSDKConfig(): {
-  region: string;
-  credentials?: {
-    accessKeyId: string;
-    secretAccessKey: string;
-  };
-};
-export function getAWSRegion(): string;
 export function getEnvironment(): string;
 export function getNodeRealApiKey(identifier: string): string | undefined;
 export function isProduction(): boolean;

package/env.js

@@ -1,29 +1,3 @@
-function getAWSAccessKeyId() {
-  return getEnvOrThrow("SKYNET_AWS_ACCESS_KEY_ID");
-}
-
-function getAWSSecretAccessKey() {
-  return getEnvOrThrow("SKYNET_AWS_SECRET_ACCESS_KEY");
-}
-
-function getAWSSDKConfig() {
-  const region = getAWSRegion();
-  const accessKeyId = getAWSAccessKeyId();
-  const secretAccessKey = getAWSSecretAccessKey();
-
-  return {
-    region,
-    credentials: {
-      accessKeyId,
-      secretAccessKey,
-    },
-  };
-}
-
-function getAWSRegion() {
-  return ensureAndGet(["SKYNET_AWS_REGION"], "us-east-1");
-}
-
 function getEnvironment() {
   return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
 }
@@ -68,10 +42,6 @@ function isDev() {
 export {
   ensureAndGet,
   getEnvOrThrow,
-  getAWSAccessKeyId,
-  getAWSSecretAccessKey,
-  getAWSSDKConfig,
-  getAWSRegion,
   getEnvironment,
   isProduction,
   isDev,

package/indexer.js

@@ -547,7 +547,7 @@ ${getSelectorDesc(selector)}
 // for those indexers that does not rely on a curosr
 // e.g. should always rebuild everything from scratch
 // or that the state can be easily inferred from existing data
-function createIndexerApp({ binaryName, name, selector = {}, build, maxRetry = 2 }) {
+function createIndexerApp({ binaryName, selector = {}, build, maxRetry = 2 }) {
   function run() {
     if (!binaryName) {
       binaryName = getBinaryName();

package/opsgenie.d.ts

@@ -3,5 +3,6 @@ export function postGenieMessage<T>(
     alias?: string;
     message: string;
   },
+  apiKey: string,
   verbose?: boolean,
 ): Promise<{ result: T }>;

package/opsgenie.js

@@ -1,11 +1,11 @@
 import md5 from "md5";
 
 function getGenieKey(key) {
-  return key || process.env.OPSGENIE_API_KEY;
+  return key || process.env.SKYNET_OPSGENIE_API_KEY;
 }
 
 function getGenieEndPoint() {
-  return process.env.OPSGENIE_END_POINT || "https://api.opsgenie.com/v2/alerts";
+  return process.env.SKYNET_OPSGENIE_END_POINT || "https://api.opsgenie.com/v2/alerts";
 }
 
 export async function postGenieMessage(body, apiKey, verbose) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.17.0",
+  "version": "0.18.1",
   "description": "Skynet Shared JS library",
   "type": "module",
   "main": "index.js",
@@ -19,25 +19,24 @@
     "@aws-sdk/client-s3": "^3.478.0",
     "@aws-sdk/client-sqs": "^3.478.0",
     "@aws-sdk/lib-dynamodb": "^3.478.0",
-    "@opensearch-project/opensearch": "^2.4.0",
+    "@elastic/elasticsearch": "^8.14.0",
     "@slack/web-api": "^6.11.0",
     "bottleneck": "^2.19.5",
     "chalk": "^5.3.0",
-    "execa": "^8.0.1",
-    "express": "^4.18.2",
-    "kafkajs": "^2.2.4",
+    "execa": "^9.3.0",
+    "express": "^4.19.2",
     "md5": "^2.3.0",
-    "meow": "^12.1.1",
-    "snowflake-sdk": "^1.9.2",
-    "web3": "^4.3.0",
+    "meow": "^13.2.0",
+    "snowflake-sdk": "^1.11.0",
+    "web3": "^4.11.0",
     "which": "^4.0.0"
   },
   "devDependencies": {
-    "ava": "^6.0.1",
-    "eslint": "^8.56.0",
+    "ava": "^6.1.3",
+    "eslint": "8",
     "eslint-plugin-import": "^2.29.1",
-    "prettier": "^3.1.1",
-    "sinon": "^17.0.1"
+    "prettier": "^3.3.3",
+    "sinon": "^18.0.0"
   },
   "license": "MIT",
   "publishConfig": {

package/s3.js

@@ -8,13 +8,12 @@ import {
   NotFound,
   NoSuchKey,
 } from "@aws-sdk/client-s3";
-import { getAWSSDKConfig } from "./env.js";
 
 let _s3Client;
 
 function getS3(forceNew = false) {
   if (!_s3Client || forceNew) {
-    _s3Client = new S3Client(getAWSSDKConfig());
+    _s3Client = new S3Client();
   }
   return _s3Client;
 }
@@ -136,6 +135,10 @@ async function listKeys(bucketname, prefix, continuationToken) {
     throw `unable to list keys with prefix ${prefix}: ${err}`;
   }
 
+  if (!data.Contents) {
+    return { keys: [] };
+  }
+
   let res = { keys: data.Contents.map(({ Key }) => Key) };
   if (data.IsTruncated) {
     res.continuationToken = data.NextContinuationToken;

package/search.d.ts

@@ -0,0 +1,6 @@
+export function sendToSearch<T>(
+  appName: string,
+  indexPrefix: string,
+  record: string,
+  throws?: boolean,
+): Promise<{ status: string; result: T }>;

package/search.js

@@ -0,0 +1,29 @@
+import { isProduction } from "./env.js";
+import { inline } from "./log.js";
+import { Client } from "@elastic/elasticsearch";
+import osModule from "os";
+
+export async function sendToSearch(appName, indexPrefix, record, throws = false) {
+  const client = new Client({
+    cloud: { id: process.env.SKYNET_ELASTICSEARCH_CLOUD_ID },
+    auth: { apiKey: process.env.SKYNET_ELASTICSEARCH_API_KEY },
+  });
+
+  const now = new Date();
+  const indexName = [indexPrefix, isProduction() ? "prod" : "dev", now.toISOString().slice(0, 7)].join("-");
+
+  // best as possible delivery
+  // by default not throw
+  try {
+    await client.index({
+      index: indexName,
+      document: { app: appName, instance: osModule.hostname(), timestamp: now, record },
+    });
+  } catch (err) {
+    inline.error(err);
+
+    if (throws) {
+      throw err;
+    }
+  }
+}

package/sqs.js

@@ -1,6 +1,5 @@
 import { SQSClient } from "@aws-sdk/client-sqs";
-import { getAWSSDKConfig } from "./env.js";
 
 export function getSQS() {
-  return new SQSClient(getAWSSDKConfig());
+  return new SQSClient();
 }

package/kafka.d.ts

@@ -1,31 +0,0 @@
-import { KafkaMessage } from "kafkajs";
-import { Produce, Consume, State } from "./app";
-import { SelectorFlags, Selector } from "./selector";
-
-export function createProducerApp(createProducerAppParams: {
-  binaryName: string;
-  name: string;
-  selector?: Selector;
-  producer: Produce;
-  state: State;
-}): { run: () => Promise<void> };
-
-export function createConsumerApp(createConsumerAppParams: {
-  binaryName: string;
-  name: string;
-  selector?: Selector;
-  consumer: Consume;
-}): { run: () => Promise<void> };
-
-export function produceMessages(
-  producerId: string,
-  callback: (send: <T>(topic: string, messages: T) => Promise<void>) => Promise<void>,
-): Promise<void>;
-
-export function consumeMessages(
-  consumerId: string,
-  topic: string | RegExp,
-  callback: (message: KafkaMessage, stopConsumeMessages: () => Promise<void>) => Promise<void>,
-): Promise<() => Promise<void>>;
-
-export function getProducerLatestId(name: string, selectorFlags: SelectorFlags): Promise<number>;

package/kafka.js

@@ -1,445 +0,0 @@
-import meow from "meow";
-import { Kafka, logLevel } from "kafkajs";
-import { getEnvironment, getEnvOrThrow } from "./env.js";
-import { getSelectorFlags, getSelectorDesc, toSelectorString } from "./selector.js";
-import { createRecord, getRecordByKey, deleteRecordsByHashKey } from "./dynamodb.js";
-import { wait, exponentialRetry } from "./availability.js";
-import { getBinaryName } from "./cli.js";
-import { inline } from "./log.js";
-
-const STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
-
-async function getProducerLatestId(name, selectorFlags) {
-  const record = await getRecordByKey(STATE_TABLE_NAME, {
-    name: `${name}At(${toSelectorString(selectorFlags)})`,
-  });
-
-  if (record) {
-    return record.value;
-  }
-
-  return 0;
-}
-
-async function setProducerLatestId(name, selectorFlags, value) {
-  await createRecord(STATE_TABLE_NAME, {
-    name: `${name}At(${toSelectorString(selectorFlags)})`,
-    value,
-  });
-}
-
-async function deleteProducerLatestId(name, selectorFlags, verbose) {
-  await deleteRecordsByHashKey(STATE_TABLE_NAME, null, `${name}At(${toSelectorString(selectorFlags)})`, verbose);
-}
-
-function sendToTopic(producer, topic, verbose) {
-  async function send(records) {
-    if (verbose) {
-      inline.log(`sending ${records.length} records to topic`, records);
-    }
-
-    await producer.send({
-      topic,
-      messages: records.map((r) => ({
-        value: Buffer.from(typeof r === "string" ? r : JSON.stringify(r)),
-      })),
-    });
-  }
-
-  return send;
-}
-
-function getDefaultKafkaCredential() {
-  return {
-    server: getEnvOrThrow("SKYNET_KAFKA_SERVER"),
-    username: getEnvOrThrow("SKYNET_KAFKA_USERNAME"),
-    password: getEnvOrThrow("SKYNET_KAFKA_PASSWORD"),
-  };
-}
-
-async function initProducer(clientId, customKafkaCredential = undefined) {
-  const kafkaCredential = customKafkaCredential ?? getDefaultKafkaCredential();
-  const kafka = new Kafka({
-    clientId,
-    brokers: [kafkaCredential.server],
-    logLevel: logLevel.ERROR,
-    ssl: true,
-    sasl: {
-      mechanism: "plain",
-      username: kafkaCredential.username,
-      password: kafkaCredential.password,
-    },
-  });
-  const producerInstance = kafka.producer();
-  await producerInstance.connect();
-
-  return producerInstance;
-}
-
-async function closeProducer(producerInstance) {
-  await producerInstance.disconnect();
-}
-
-async function initConsumer(clientId, customKafkaCredential = undefined) {
-  const kafkaCredential = customKafkaCredential ? customKafkaCredential : getDefaultKafkaCredential();
-  const kafka = new Kafka({
-    clientId,
-    brokers: [kafkaCredential.server],
-    logLevel: logLevel.ERROR,
-    ssl: true,
-    sasl: {
-      mechanism: "plain",
-      username: kafkaCredential.username,
-      password: kafkaCredential.password,
-    },
-  });
-  const consumerInstance = kafka.consumer({ groupId: clientId });
-
-  await consumerInstance.connect();
-
-  return consumerInstance;
-}
-
-async function closeConsumer(consumerInstance) {
-  await consumerInstance.disconnect();
-}
-
-async function produceMessages(producerId, callback) {
-  const producerInstance = await initProducer(producerId);
-  const send = async (topic, messages) => {
-    let records;
-    if (Array.isArray(messages)) {
-      records = messages;
-    } else {
-      records = [messages];
-    }
-    return sendToTopic(producerInstance, topic, false)(records);
-  };
-  await callback(send);
-  await closeProducer(producerInstance);
-}
-
-async function consumeMessages(consumerId, topic, callback) {
-  const consumerInstance = await initConsumer(consumerId);
-
-  await consumerInstance.subscribe({ topic });
-
-  const stopConsumeMessages = async () => {
-    await closeConsumer(consumerInstance);
-  };
-  await consumerInstance.run({
-    eachMessage: async ({ topic: receivedTopic, message }) => {
-      if (topic === receivedTopic) {
-        await callback(message, stopConsumeMessages);
-      }
-    },
-  });
-  return stopConsumeMessages;
-}
-
-function createProducerApp({ binaryName, name, selector = {}, producer, state }) {
-  function run() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const finalState = {
-      type: "block",
-      updateInterval: 5000,
-      getMinId: async () => 1,
-      getMaxId: async () => {
-        throw new Error("must implement state.getMaxId");
-      },
-      ...state,
-    };
-
-    const cli = meow(
-      `
-        Usage
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --from          min ${finalState.type} to produce
-          --to            max ${finalState.type} to produce
-          --status        print current status and exit
-          --verbose       Output debug messages
-          --reset         restart producer
-      `,
-      {
-        importMeta: import.meta,
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          from: {
-            aliases: ["since"],
-            type: "number",
-            default: 0,
-          },
-          to: {
-            aliases: ["until"],
-            type: "number",
-            default: 0,
-          },
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    const { topic, deadLetterTopic, produce, batchSize, maxRetry } = {
-      batchSize: 50,
-      maxRetry: 2,
-      ...producer,
-    };
-
-    const kafkaCredential = (producer && producer.kafkaCredential) || getDefaultKafkaCredential();
-
-    async function build({ from, to, status, reset, verbose, ...selectorFlags }) {
-      if (reset) {
-        await deleteProducerLatestId(name, selectorFlags);
-        if (finalState.onReset) {
-          await finalState.onReset(selectorFlags, verbose);
-        }
-
-        inline.log("producer reset");
-
-        process.exit(0);
-      }
-
-      if (status) {
-        const latestId = await getProducerLatestId(name, selectorFlags);
-
-        inline.log(`LatestID=${latestId}`);
-
-        process.exit(0);
-      }
-
-      if (!from) {
-        const prevId = await getProducerLatestId(name, selectorFlags);
-
-        if (prevId) {
-          from = prevId + 1;
-        } else {
-          // initial build
-          from = await finalState.getMinId(selectorFlags);
-        }
-      }
-
-      const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
-      const finalDeadLetterTopic =
-        typeof deadLetterTopic === "function" ? deadLetterTopic(selectorFlags) : deadLetterTopic;
-
-      const updateInterval =
-        typeof finalState.updateInterval === "function"
-          ? finalState.updateInterval(selectorFlags)
-          : finalState.updateInterval;
-
-      const producerInstance = await initProducer(name, kafkaCredential);
-
-      inline.log("producing to topic", finalTopic);
-
-      // if to flag was not provided, run this program indefinitely
-      const isLongRunning = to === 0;
-
-      let lastId = to > 0 ? to : await finalState.getMaxId(selectorFlags);
-
-      let pollHandle;
-
-      async function cleanUpForLongRunningMode() {
-        clearInterval(pollHandle);
-
-        await closeProducer(producerInstance);
-
-        process.exit(0);
-      }
-
-      if (isLongRunning) {
-        pollHandle = setInterval(async () => {
-          lastId = await finalState.getMaxId(selectorFlags);
-        }, updateInterval);
-
-        // clean up before exit
-        process.on("SIGINT", cleanUpForLongRunningMode);
-      }
-
-      // the main loop
-      for (let i = from; isLongRunning || i <= lastId; ) {
-        if (i > lastId) {
-          console.log("no more items to process, sleep for", updateInterval, "ms");
-
-          await wait(updateInterval);
-        } else {
-          const batchStart = i;
-          const batchEnd = Math.min(batchStart + batchSize - 1, lastId);
-
-          inline.log(`building batch ${batchStart}~${batchEnd}`);
-
-          // add a retry for errors
-          const result = await exponentialRetry(
-            async () => {
-              try {
-                const failed = await produce({
-                  ...selectorFlags,
-                  from: batchStart,
-                  to: batchEnd,
-                  verbose,
-                  send: sendToTopic(producerInstance, finalTopic, verbose),
-                });
-
-                if (Array.isArray(failed) && failed.length > 0) {
-                  await sendToTopic(producerInstance, finalDeadLetterTopic, verbose)(failed);
-                }
-
-                return true;
-              } catch (err) {
-                inline.error(`Critical error processing batch ${batchStart}~${batchEnd}`, err);
-
-                return false;
-              }
-            },
-            {
-              maxRetry,
-              test: (r) => r,
-              verbose,
-            }
-          );
-
-          if (!result) {
-            if (isLongRunning) {
-              cleanUpForLongRunningMode();
-            }
-
-            throw new Error(`Terminate producer due to critical errors, batch ${batchStart}~${batchEnd}`);
-          }
-
-          await setProducerLatestId(name, selectorFlags, batchEnd);
-
-          if (i + batchSize <= lastId) i += batchSize;
-          else i = batchEnd + 1;
-        }
-
-        if (isLongRunning) {
-          lastId = await finalState.getMaxId(selectorFlags);
-        }
-      }
-
-      await closeProducer(producerInstance);
-    }
-
-    return build(cli.flags).catch((err) => {
-      inline.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { run };
-}
-
-function createConsumerApp({ binaryName, name, selector = {}, consumer }) {
-  function run() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const cli = meow(
-      `
-        Usage
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --verbose       Output debug messages
-      `,
-      {
-        importMeta: import.meta,
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    const { topic, consume, maxRetry } = {
-      maxRetry: 2,
-      ...consumer,
-    };
-
-    const kafkaCredential = (consume && consumer.kafkaCredential) || getDefaultKafkaCredential();
-
-    async function build({ verbose, ...selectorFlags }) {
-      const consumerInstance = await initConsumer(name, kafkaCredential);
-
-      try {
-        const finalTopic = typeof topic === "function" ? topic(selectorFlags) : topic;
-
-        await consumerInstance.subscribe({ topic: finalTopic });
-
-        inline.log("subscribed to topic", finalTopic);
-
-        await consumerInstance.run({
-          eachBatch: async ({ batch }) => {
-            if (verbose) {
-              inline.log("received batch, number of items =", batch.messages.length);
-            }
-
-            const messages = batch.messages.map((m) => JSON.parse(m.value));
-
-            // add a retry for errors
-            const result = await exponentialRetry(
-              async () => {
-                try {
-                  await consume({ ...selectorFlags, messages, verbose });
-
-                  return true;
-                } catch (err) {
-                  inline.error(`got critical error`, err);
-
-                  return false;
-                }
-              },
-              {
-                maxRetry,
-                test: (r) => r,
-                verbose,
-              }
-            );
-
-            if (!result) {
-              inline.error("Terminate consumer due to consume errors, likely bug");
-
-              process.exit(1);
-            }
-          },
-        });
-      } catch (err) {
-        inline.error("Terminate consumer due to critical kafka connection error", err);
-
-        process.exit(1);
-      }
-    }
-
-    return build(cli.flags).catch((err) => {
-      inline.error(err);
-      process.exit(1);
-    });
-  }
-
-  return { run };
-}
-
-export {
-  createProducerApp,
-  createConsumerApp,
-  produceMessages,
-  consumeMessages,
-  getProducerLatestId,
-};

package/monitor.d.ts

@@ -1,24 +0,0 @@
-import { Check } from "./app";
-import { SelectorFlags, Selector } from "./selector";
-
-export const ERROR_LEVEL: {
-  INFO: string;
-  CRITICAL: string;
-  WARNING: string;
-};
-
-export const LEVEL_EMOJI: {
-  Info: string;
-  Critical: string;
-  Warning: string;
-};
-
-export function createMonitor(createMonitorParams: {
-  binaryName: string;
-  name: string;
-  getState?: (name: string, selectorFlags: SelectorFlags) => Promise<void>;
-  mode?: boolean;
-  selector?: Selector;
-  check?: Check;
-  maxRetry?: number;
-}): { monitor: () => void };

package/monitor.js

@@ -1,197 +0,0 @@
-import meow from "meow";
-import { getSelectorFlags, getSelectorDesc, toSelectorString } from "./selector.js";
-import { getBinaryName } from "./cli.js";
-import { exponentialRetry } from "./availability.js";
-import { postGenieMessage } from "./opsgenie.js";
-import { getJobName } from "./deploy.js";
-
-const ERROR_LEVEL = {
-  INFO: "Info",
-  CRITICAL: "Critical",
-  WARNING: "Warning",
-};
-
-const LEVEL_EMOJI = {
-  Critical: ":exclamation:",
-  Warning: ":warning:",
-  Info: "",
-};
-
-const LEVEL_PRIORITY = {
-  Critical: 3,
-  Warning: 2,
-  Info: 1,
-};
-
-function sortErrors(errors) {
-  return errors.sort((e1, e2) => (LEVEL_PRIORITY[e2.type] || 0) - (LEVEL_PRIORITY[e1.type] || 0));
-}
-
-async function getMostRecentJobLaunch(name) {
-  try {
-    const jobsRes = await fetch(`http://localhost:4646/v1/jobs?prefix=${name}`);
-
-    if (!jobsRes.ok) {
-      console.log(`[MONITOR] request local nomad API failed`);
-
-      return null;
-    }
-
-    const jobs = await jobsRes.json();
-
-    if (jobs.length === 0) {
-      console.log(`[MONITOR] did not see any jobs prefixed with ${name}`);
-
-      return null;
-    }
-
-    const recentFinishedJob = jobs.reverse().find((job) => {
-      // filter out monitor job
-      return job.ID.indexOf("-monitor") === -1 && job.Status === "dead";
-
-    });
-
-    if (!recentFinishedJob) {
-      console.log(`[MONITOR] did not see any dead jobs`);
-
-      return null;
-    }
-
-    console.log("[MONITOR]", "most recent job info", recentFinishedJob.ID, recentFinishedJob.JobSummary.Summary);
-
-    return recentFinishedJob;
-  } catch (getJobError) {
-    console.log("[MONITOR]", "cannot get most recent job", getJobError.message);
-    return null;
-  }
-}
-
-function createMonitor({ binaryName, name, getState = null, mode = false, selector = {}, check, maxRetry = 2 }) {
-  function monitor() {
-    if (!binaryName) {
-      binaryName = getBinaryName();
-    }
-
-    const cli = meow(
-      `
-        Usage
-
-          $ ${binaryName} <options>
-
-        Options
-${
-  mode ? "          --mode          could be delta/rebuild/resume-rebuild/validate/one/range/reset\n" : ""
-}${getSelectorDesc(selector)}
-          --production    run in production mode
-          --verbose       Output debug messages
-      `,
-      {
-        importMeta: import.meta,
-        description: false,
-        version: false,
-        flags: {
-          ...getSelectorFlags(selector),
-          ...(mode && {
-            mode: {
-              type: "string",
-              default: "delta",
-            },
-          }),
-          verbose: {
-            type: "boolean",
-            default: false,
-          },
-        },
-      }
-    );
-
-    async function runCheck({ verbose, production, mode, ...selectorFlags }) {
-      const startTime = Date.now();
-
-      if (Object.keys(selectorFlags).length > 0) {
-        console.log(`[MONITOR] starting check, ${toSelectorString(selectorFlags, ", ")}`);
-      } else {
-        console.log(`[MONITOR] starting check`);
-      }
-
-      let checkState = {};
-
-      if (getState) {
-        checkState = await getState(name, selectorFlags);
-      }
-
-      let result = await exponentialRetry(
-        async () => {
-          try {
-            const errors = await check({ verbose, state: checkState, mode, ...selectorFlags });
-
-            if (!Array.isArray(errors)) {
-              throw new Error(`check function must return array of error messages`);
-            }
-
-            return errors;
-          } catch (err) {
-            console.log(`[MONITOR] got error in check`, err);
-
-            return [{ type: ERROR_LEVEL.ERROR, message: `${err.message}` }];
-          }
-        },
-        {
-          maxRetry,
-          initialDuration: 10000,
-          growFactor: 3,
-          test: (r) => r.length === 0,
-          verbose,
-        }
-      );
-
-      const jobName = getJobName(name, selectorFlags, mode);
-      const mostRecentJob = await getMostRecentJobLaunch(jobName);
-
-      if (result.length > 0) {
-        console.log("Found Errors", result);
-
-        if (production) {
-          const nomadAddr = process.env.SKYNET_NOMAD_PRODUCTION_ADDR;
-
-          // alert on opsgenie
-          await postGenieMessage(
-            {
-              message: `Failed Service Check: ${jobName}`,
-              description: `<p><b>Service:</b><a href="${nomadAddr}/ui/jobs/${jobName}" target="_blank">${jobName}</a></p><p><b>Issues</b></p><ul>${sortErrors(
-                result
-              )
-                .map((m) => `<li><b>${m.type}:</b> ${m.message}</li>`)
-                .join("")}</ul>`,
-            },
-            verbose
-          );
-        } else {
-          console.log("skip sending messages to opsgenie in dev env");
-        }
-      }
-
-      console.log(`[MONITOR] check successfully in ${Date.now() - startTime}ms`);
-
-      if (mostRecentJob && mostRecentJob.ParentID === "" && mostRecentJob.Periodic === false) {
-        // stop monitor job if the main job was a one time job and completed
-        const stopRes = await fetch(`http://localhost:4646/v1/job/${jobName}-monitor`, { method: "DELETE" });
-
-        if (stopRes.ok) {
-          console.log(`[MONITOR] monitor job stopped since the completed main job was a one time job`);
-        } else {
-          console.log(`[MONITOR] error stopping monitor job`);
-        }
-      }
-    }
-
-    return runCheck(cli.flags).catch((err) => {
-      console.log("[MONITOR]", err);
-      process.exit(1);
-    });
-  }
-
-  return { monitor };
-}
-
-export { createMonitor, ERROR_LEVEL, LEVEL_EMOJI };

package/opensearch.d.ts

@@ -1,16 +0,0 @@
-import { Client, ClientOptions } from "@opensearch-project/opensearch";
-
-export declare function newOpensearchClient(
-  endpoint: string,
-  key: string,
-  secret: string,
-  options?: ClientOptions,
-): Client;
-
-export declare async function sendToOpensearch<T>(
-  client: Client,
-  appName: string,
-  index: string,
-  record: string,
-  indexSuffixes?: string[],
-): Promise<{ status: string; result: T }>;

package/opensearch.js

@@ -1,34 +0,0 @@
-import { Client } from "@opensearch-project/opensearch";
-
-// newOpensearchClient creates a new opensearch client with the given endpoint, key, secret and extra opensearch client options.
-export function newOpensearchClient(endpoint, key, secret, options = {}) {
-  return new Client({
-    node: endpoint,
-    auth: {
-      username: key,
-      password: secret,
-    },
-    ...options,
-  });
-}
-
-// sendToOpensearch sends a record object to the given index using the given opensearch client. An appName is also attached to the record
-// as an identifier for who sends the record. The indexSuffixes is an array of strings that will be appended to the index name using `-`.
-// A typical usage of indexSuffixes could be the date e.g. ["2023", "08"] so that new indexes are created on the monthly basis to prevent
-// the index from growing too large.
-// Returns an object with status and result properties. The status is either "ok" or "error". The result is the response from opensearch
-// if the status is "ok" or the error message if the status is "error".
-export async function sendToOpensearch(client, appName, index, record, indexSuffixes = []) {
-  try {
-    const result = await client.index({
-      index: indexSuffixes ? [index, ...indexSuffixes].join("-") : index,
-      body: { app: appName, timestamp: new Date(), record },
-    });
-    if (result.statusCode === 201) {
-      return { status: "ok", result };
-    }
-    return { status: "error", result };
-  } catch (err) {
-    return { status: "error", result: err.stack ? err.stack : err };
-  }
-}