@certik/skynet 0.8.10 → 0.8.11

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.8.11
+
+- Improved `updateRecordByKeys` in `dynamodb` library to support fields deletion
+- Added `opsgenie` library
+
 ## 0.8.10
 
 - Improved `postMessage` in `slack` library to support private channels

package/dynamodb.js

@@ -1,16 +1,12 @@
 const { DynamoDB } = require("aws-sdk");
-const {
-  getAWSAccessKeyId,
-  getAWSSecretAccessKey,
-  getAWSRegion
-} = require("./env");
+const { getAWSAccessKeyId, getAWSSecretAccessKey, getAWSRegion } = require("./env");
 const { wait } = require("./availability");
 
 function getDynamoDB() {
   return new DynamoDB({
     accessKeyId: getAWSAccessKeyId(),
     secretAccessKey: getAWSSecretAccessKey(),
-    region: getAWSRegion()
+    region: getAWSRegion(),
   });
 }
 
@@ -18,7 +14,7 @@ function getDocClient() {
   return new DynamoDB.DocumentClient({
     accessKeyId: getAWSAccessKeyId(),
     secretAccessKey: getAWSSecretAccessKey(),
-    region: getAWSRegion()
+    region: getAWSRegion(),
   });
 }
 
@@ -26,7 +22,7 @@ function mergeQueries(q1, q2) {
   return {
     Items: q1.Items.concat(q2.Items),
     Count: q1.Count + q2.Count,
-    ScannedCount: q1.ScannedCount + q2.ScannedCount
+    ScannedCount: q1.ScannedCount + q2.ScannedCount,
   };
 }
 
@@ -36,7 +32,7 @@ async function scanWholeTable(options) {
   let items = {
     Items: [],
     Count: 0,
-    ScannedCount: 0
+    ScannedCount: 0,
   };
 
   let data = await dynamodb.scan(options).promise();
@@ -47,7 +43,7 @@ async function scanWholeTable(options) {
     data = await dynamodb
       .scan({
         ExclusiveStartKey: data.LastEvaluatedKey,
-        ...options
+        ...options,
       })
       .promise();
   }
@@ -67,12 +63,7 @@ function toSlices(items) {
   return slices;
 }
 
-async function batchCreateRecords(
-  tableName,
-  records,
-  maxWritingCapacity,
-  verbose = false
-) {
+async function batchCreateRecords(tableName, records, maxWritingCapacity, verbose = false) {
   if (verbose) {
     console.log(`creating ${records.length} items in ${tableName}`);
   }
@@ -91,20 +82,18 @@ async function batchCreateRecords(
     }
 
     if (factor >= 32) {
-      console.log(
-        `WARNING: no progress for a long time for batchCreateRecords, please check`
-      );
+      console.log(`WARNING: no progress for a long time for batchCreateRecords, please check`);
     }
 
     const slices = toSlices(remainingItems.slice(0, maxWritingCapacity));
     const results = await Promise.allSettled(
-      slices.map(rs => {
+      slices.map((rs) => {
         const params = {
           RequestItems: {
-            [tableName]: rs.map(record => {
+            [tableName]: rs.map((record) => {
               return { PutRequest: { Item: record } };
-            })
-          }
+            }),
+          },
         };
 
         return docClient.batchWrite(params).promise();
@@ -120,39 +109,27 @@ async function batchCreateRecords(
         rejection = rs;
       } else if (Object.keys(rs.value.UnprocessedItems).length > 0) {
         // partially fails, redo unprocessedItems
-        acc = acc.concat(
-          rs.value.UnprocessedItems[tableName].map(it => it.PutRequest.Item)
-        );
+        acc = acc.concat(rs.value.UnprocessedItems[tableName].map((it) => it.PutRequest.Item));
       }
 
       return acc;
     }, remainingItems.slice(maxWritingCapacity));
 
     if (verbose) {
-      console.log(
-        `processed=${prevRemainingCount - remainingItems.length}, remaining=${
-          remainingItems.length
-        }`
-      );
+      console.log(`processed=${prevRemainingCount - remainingItems.length}, remaining=${remainingItems.length}`);
     }
   }
 
   if (rejection) {
     console.log("batchCreateRecords rejected", rejection);
 
-    throw new Error(
-      `batchCreateRecords rejected, failed items=${remainingItems.length}`
-    );
+    throw new Error(`batchCreateRecords rejected, failed items=${remainingItems.length}`);
   }
 
   if (remainingItems.length > 0) {
-    console.log(
-      `failed batchCreateRecords, failed items=${remainingItems.length}`
-    );
+    console.log(`failed batchCreateRecords, failed items=${remainingItems.length}`);
 
-    throw new Error(
-      `batchCreateRecords retry failed, failed items=${remainingItems.length}`
-    );
+    throw new Error(`batchCreateRecords retry failed, failed items=${remainingItems.length}`);
   }
 }
 
@@ -164,7 +141,7 @@ async function createRecord(tableName, fields, verbose = false) {
   const docClient = getDocClient();
   const params = {
     TableName: tableName,
-    Item: fields
+    Item: fields,
   };
 
   return docClient.put(params).promise();
@@ -174,23 +151,13 @@ async function getRecordsByKey(tableName, keys, indexName) {
   const docClient = getDocClient();
 
   const keyNames = Object.keys(keys);
-  const conditionExpression = keyNames
-    .map(key => `#${key} = :${key}`)
-    .join(" and ");
-  const expressionNames = keyNames.reduce((acc, key) => {
-    acc[`#${key}`] = key;
-    return acc;
-  }, {});
-  const expressionValues = keyNames.reduce((acc, key) => {
-    acc[`:${key}`] = keys[key];
-    return acc;
-  }, {});
+  const conditionExpression = keyNames.map((key) => `#${key} = :${key}`).join(" and ");
 
   const params = {
     TableName: tableName,
     KeyConditionExpression: conditionExpression,
-    ExpressionAttributeNames: expressionNames,
-    ExpressionAttributeValues: expressionValues
+    ExpressionAttributeNames: generateExpressionNames(keyNames),
+    ExpressionAttributeValues: generateExpressionValues(keyNames, keys),
   };
 
   if (indexName) {
@@ -221,45 +188,77 @@ async function getRecordByKey(tableName, keys, indexName) {
   }
 }
 
-async function updateRecordByKey(
-  tableName,
-  idKey,
-  fields,
-  conditionExpressions = null,
-  verbose = false
-) {
-  if (verbose) {
-    console.log("update", tableName, idKey, fields);
-  }
-
-  const idKeyNames = Object.keys(idKey);
-  const keys = Object.keys(fields).filter(k => !idKeyNames.includes(k));
-  const updateExpressions = keys.map(key => `#${key} = :${key}`);
-  const expressionNames = keys.reduce((acc, key) => {
+function generateExpressionNames(keys) {
+  return keys.reduce((acc, key) => {
     acc[`#${key}`] = key;
     return acc;
   }, {});
-  const expressionValues = keys.reduce((acc, key) => {
+}
+
+function generateExpressionValues(keys, fields) {
+  return keys.reduce((acc, key) => {
     acc[`:${key}`] = fields[key];
     return acc;
   }, {});
+}
+
+async function updateRecordByKey(tableName, idKey, fields, conditionExpressions = null, verbose = false) {
+  if (verbose) {
+    console.log("update", tableName, idKey, fields);
+  }
+
   const docClient = getDocClient();
-  const params = {
-    TableName: tableName,
-    Key: idKey,
-    ExpressionAttributeNames: expressionNames,
-    ExpressionAttributeValues: expressionValues,
-    UpdateExpression: `SET ${updateExpressions.join(", ")}`,
-    ReturnValues: "ALL_NEW"
-  };
 
-  if (conditionExpressions) {
-    params.ConditionExpression = conditionExpressions;
+  const idKeyNames = Object.keys(idKey);
+  const fieldsToDelete = Object.keys(fields).filter((f) => fields[f] === undefined);
+  const fieldsToUpdate = Object.keys(fields).filter((k) => !idKeyNames.includes(k) && !fieldsToDelete.includes(k));
+
+  let data;
+
+  if (fieldsToDelete.length > 0) {
+    if (verbose) {
+      console.log("delete fields", tableName, fieldsToDelete);
+    }
+
+    const deleteParams = {
+      TableName: tableName,
+      Key: idKey,
+      ExpressionAttributeNames: generateExpressionNames(fieldsToDelete),
+      UpdateExpression: `REMOVE ${fieldsToDelete.map((f) => `#${f}`).join(", ")}`,
+      ReturnValues: "ALL_NEW",
+    };
+
+    if (conditionExpressions) {
+      deleteParams.ConditionExpression = conditionExpressions;
+    }
+
+    data = await docClient.update(deleteParams).promise();
   }
 
-  const data = await docClient.update(params).promise();
+  if (fieldsToUpdate.length > 0) {
+    if (verbose) {
+      console.log("update fields", tableName, fieldsToUpdate);
+    }
+
+    const updateExpressions = fieldsToUpdate.map((key) => `#${key} = :${key}`);
+
+    const params = {
+      TableName: tableName,
+      Key: idKey,
+      ExpressionAttributeNames: generateExpressionNames(fieldsToUpdate),
+      ExpressionAttributeValues: generateExpressionValues(fieldsToUpdate, fields),
+      UpdateExpression: `SET ${updateExpressions.join(", ")}`,
+      ReturnValues: "ALL_NEW",
+    };
 
-  return data.Attributes;
+    if (conditionExpressions) {
+      params.ConditionExpression = conditionExpressions;
+    }
+
+    data = await docClient.update(params).promise();
+  }
+
+  return data && data.Attributes;
 }
 
 async function createTableIfNotExist(
@@ -267,14 +266,14 @@ async function createTableIfNotExist(
   attributeDefinitions = [
     {
       AttributeName: "id",
-      AttributeType: "S"
-    }
+      AttributeType: "S",
+    },
   ],
   keySchema = [
     {
       AttributeName: "id",
-      KeyType: "HASH"
-    }
+      KeyType: "HASH",
+    },
   ],
   otherOptions = {},
   verbose = false
@@ -282,9 +281,7 @@ async function createTableIfNotExist(
   const dynamodb = getDynamoDB();
 
   try {
-    const table = await dynamodb
-      .describeTable({ TableName: tableName })
-      .promise();
+    const table = await dynamodb.describeTable({ TableName: tableName }).promise();
 
     if (verbose) {
       console.log(`table ${tableName} already exist`, table);
@@ -298,9 +295,9 @@ async function createTableIfNotExist(
         KeySchema: keySchema,
         ProvisionedThroughput: {
           ReadCapacityUnits: 5,
-          WriteCapacityUnits: 5
+          WriteCapacityUnits: 5,
         },
-        ...otherOptions
+        ...otherOptions,
       })
       .promise();
 
@@ -313,9 +310,7 @@ async function createTableIfNotExist(
     while (!isCreated) {
       await wait(5000);
 
-      const t = await dynamodb
-        .describeTable({ TableName: tableName })
-        .promise();
+      const t = await dynamodb.describeTable({ TableName: tableName }).promise();
 
       console.log("waiting for table to become active");
 
@@ -335,38 +330,31 @@ async function batchDeleteRecords(tableName, keys) {
     await docClient
       .batchWrite({
         RequestItems: {
-          [tableName]: slice.map(key => {
+          [tableName]: slice.map((key) => {
             return { DeleteRequest: { Key: key } };
-          })
-        }
+          }),
+        },
       })
       .promise();
   }
 }
 
 function getKeyName(keySchema, type) {
-  const key = keySchema.find(k => k.KeyType === type);
+  const key = keySchema.find((k) => k.KeyType === type);
 
   return key && key.AttributeName;
 }
 
 function getIndexKeyName(globalSecondaryIndexes, indexName, type) {
-  const idx = globalSecondaryIndexes.find(i => i.IndexName === indexName);
+  const idx = globalSecondaryIndexes.find((i) => i.IndexName === indexName);
 
   return idx && getKeyName(idx.KeySchema, type);
 }
 
-async function deleteRecordsByHashKey(
-  tableName,
-  indexName,
-  hashKeyValue,
-  verbose = false
-) {
+async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbose = false) {
   const docClient = getDocClient();
 
-  const meta = await getDynamoDB()
-    .describeTable({ TableName: tableName })
-    .promise();
+  const meta = await getDynamoDB().describeTable({ TableName: tableName }).promise();
 
   const hashKeyName = indexName
     ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes, indexName, "HASH")
@@ -385,7 +373,7 @@ async function deleteRecordsByHashKey(
     TableName: tableName,
     KeyConditionExpression: "#hashKeyName = :hashKeyValue",
     ExpressionAttributeNames: { "#hashKeyName": hashKeyName },
-    ExpressionAttributeValues: { ":hashKeyValue": hashKeyValue }
+    ExpressionAttributeValues: { ":hashKeyValue": hashKeyValue },
   };
 
   if (indexName) {
@@ -396,14 +384,14 @@ async function deleteRecordsByHashKey(
 
   await batchDeleteRecords(
     tableName,
-    data.Items.map(item =>
+    data.Items.map((item) =>
       mainRangeKeyName
         ? {
             [mainHashKeyName]: item[mainHashKeyName],
-            [mainRangeKeyName]: item[mainRangeKeyName]
+            [mainRangeKeyName]: item[mainRangeKeyName],
           }
         : {
-            [mainHashKeyName]: item[mainHashKeyName]
+            [mainHashKeyName]: item[mainHashKeyName],
           }
     )
   );
@@ -414,20 +402,20 @@ async function deleteRecordsByHashKey(
     data = await docClient
       .query({
         ...params,
-        ExclusiveStartKey: data.LastEvaluatedKey
+        ExclusiveStartKey: data.LastEvaluatedKey,
       })
       .promise();
 
     await batchDeleteRecords(
       tableName,
-      data.Items.map(item =>
+      data.Items.map((item) =>
         mainRangeKeyName
           ? {
               [mainHashKeyName]: item[mainHashKeyName],
-              [mainRangeKeyName]: item[mainRangeKeyName]
+              [mainRangeKeyName]: item[mainRangeKeyName],
             }
           : {
-              [mainHashKeyName]: item[mainHashKeyName]
+              [mainHashKeyName]: item[mainHashKeyName],
             }
       )
     );
@@ -452,5 +440,5 @@ module.exports = {
   updateRecordByKey,
   batchDeleteRecords,
   deleteRecordsByHashKey,
-  createTableIfNotExist
+  createTableIfNotExist,
 };

package/monitor.js

@@ -5,6 +5,7 @@ const { getBinaryName } = require("./cli");
 const { exponentialRetry } = require("./availability");
 const { getIndexerLatestId, getIndexerValidatedId, getIndexerState } = require("./indexer");
 const { postMessage } = require("./slack");
+const { postGenieMessage } = require("./opsgenie");
 const { getJobName, getNomadAddr } = require("./deploy");
 
 const ERROR_LEVEL = {
@@ -235,6 +236,12 @@ ${
             },
             verbose
           );
+
+          // Also alert on opsgenie (func prevents duplicate alerts)
+          await postGenieMessage(
+            `${jobName} Monitor Errors: ${result.map((m) => m.message || m).join("\n")}`,
+            m.message
+          );
         }
 
         throw new Error(`failed due to critical errors`);

package/opsgenie.js

@@ -0,0 +1,60 @@
+const fetch = require("node-fetch");
+const hash = require("object-hash");
+
+function getGenieKey() {
+  return process.env.OPSGENIE_API_KEY;
+}
+
+function getGenieEndPoint() {
+  const key = getGenieKey();
+
+  if (!key) {
+    throw new Error("Cannot communicate with opsgenie due to missing API key: process.env.OPSGENIE_API_KEY");
+  }
+
+  return process.env.OPSGENIE_END_POINT;
+}
+
+async function postGenieMessage(msg, desc, verbose) {
+  try {
+    const genieKey = getGenieKey();
+    const genieEndPoint = getGenieEndPoint();
+
+    const body = {
+      message: msg,
+      description: desc
+    };
+
+    // Prevents duplicate alerts (See Opsgenie doc about alias)
+    bodyHash = hash(body);
+    body.alias = bodyHash;
+
+    if (verbose) {
+      console.log(`Making API call to Opsgenie ${msg} (${desc}):`, JSON.stringify(body, null, 2));
+    }
+
+    // Makes the call using fetch and ENV variables
+    const response = await fetch(genieEndPoint, {
+      method: "POST",
+      headers: {
+          "Content-Type": "application/json",
+          "Authorization": "GenieKey " + genieKey
+      },
+      body: JSON.stringify(body)
+    });
+
+    const result = await response.json();
+    if (verbose) {
+      console.log(`Result of API call to Opsgenie... ${result}`);
+    }
+
+    return result
+
+  } catch (error) {
+    console.error("Failed to make opsgenie API call", error);
+
+    throw error;
+  }
+}
+
+module.exports = { postGenieMessage };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.8.10",
+  "version": "0.8.11",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",
@@ -19,6 +19,7 @@
     "kafkajs": "^1.15.0",
     "meow": "^7.0.1",
     "node-fetch": "^2.6.1",
+    "object-hash": "^2.2.0",
     "snowflake-sdk": "^1.6.3",
     "web3": "^1.3.5",
     "which": "^2.0.2"