@certik/skynet 0.7.10 → 0.7.14

package/dynamodb.js

@@ -1,456 +1,456 @@
-const { DynamoDB } = require("aws-sdk");
-const {
-  getAWSAccessKeyId,
-  getAWSSecretAccessKey,
-  getAWSRegion
-} = require("./env");
-const { wait } = require("./availability");
-
-function getDynamoDB() {
-  return new DynamoDB({
-    accessKeyId: getAWSAccessKeyId(),
-    secretAccessKey: getAWSSecretAccessKey(),
-    region: getAWSRegion()
-  });
-}
-
-function getDocClient() {
-  return new DynamoDB.DocumentClient({
-    accessKeyId: getAWSAccessKeyId(),
-    secretAccessKey: getAWSSecretAccessKey(),
-    region: getAWSRegion()
-  });
-}
-
-function mergeQueries(q1, q2) {
-  return {
-    Items: q1.Items.concat(q2.Items),
-    Count: q1.Count + q2.Count,
-    ScannedCount: q1.ScannedCount + q2.ScannedCount
-  };
-}
-
-async function scanWholeTable(options) {
-  const dynamodb = getDocClient();
-
-  let items = {
-    Items: [],
-    Count: 0,
-    ScannedCount: 0
-  };
-
-  let data = await dynamodb.scan(options).promise();
-
-  while (data.LastEvaluatedKey) {
-    items = mergeQueries(items, data);
-
-    data = await dynamodb
-      .scan({
-        ExclusiveStartKey: data.LastEvaluatedKey,
-        ...options
-      })
-      .promise();
-  }
-
-  items = mergeQueries(items, data);
-
-  return items;
-}
-
-function toSlices(items) {
-  let slices = [];
-
-  for (let start = 0; start < items.length; start += 25) {
-    slices.push(items.slice(start, start + 25));
-  }
-
-  return slices;
-}
-
-async function batchCreateRecords(
-  tableName,
-  records,
-  maxWritingCapacity,
-  verbose = false
-) {
-  if (verbose) {
-    console.log(`creating ${records.length} items in ${tableName}`);
-  }
-  const docClient = getDocClient();
-
-  let remainingItems = records;
-  // an arbitrary number to ensure while first loop is entered
-  let prevRemainingCount = remainingItems.length + 1;
-  let factor = 1;
-  let rejection = undefined;
-
-  while (remainingItems.length > 0 && factor <= 128 && !rejection) {
-    if (prevRemainingCount === remainingItems.length) {
-      await wait(5000 * factor);
-      factor = factor * 2;
-    }
-
-    if (factor >= 32) {
-      console.log(
-        `WARNING: no progress for a long time for batchCreateRecords, please check`
-      );
-    }
-
-    const slices = toSlices(remainingItems.slice(0, maxWritingCapacity));
-    const results = await Promise.allSettled(
-      slices.map(rs => {
-        const params = {
-          RequestItems: {
-            [tableName]: rs.map(record => {
-              return { PutRequest: { Item: record } };
-            })
-          }
-        };
-
-        return docClient.batchWrite(params).promise();
-      })
-    );
-
-    prevRemainingCount = remainingItems.length;
-
-    remainingItems = results.reduce((acc, rs, idx) => {
-      if (rs.status === "rejected") {
-        // whole slice fails, redo whole slice
-        acc = acc.concat(slices[idx]);
-        rejection = rs;
-      } else if (Object.keys(rs.value.UnprocessedItems).length > 0) {
-        // partially fails, redo unprocessedItems
-        acc = acc.concat(
-          rs.value.UnprocessedItems[tableName].map(it => it.PutRequest.Item)
-        );
-      }
-
-      return acc;
-    }, remainingItems.slice(maxWritingCapacity));
-
-    if (verbose) {
-      console.log(
-        `processed=${prevRemainingCount - remainingItems.length}, remaining=${
-          remainingItems.length
-        }`
-      );
-    }
-  }
-
-  if (rejection) {
-    console.log("batchCreateRecords rejected", rejection);
-
-    throw new Error(
-      `batchCreateRecords rejected, failed items=${remainingItems.length}`
-    );
-  }
-
-  if (remainingItems.length > 0) {
-    console.log(
-      `failed batchCreateRecords, failed items=${remainingItems.length}`
-    );
-
-    throw new Error(
-      `batchCreateRecords retry failed, failed items=${remainingItems.length}`
-    );
-  }
-}
-
-async function createRecord(tableName, fields, verbose = false) {
-  if (verbose) {
-    console.log("creating", tableName, fields);
-  }
-
-  const docClient = getDocClient();
-  const params = {
-    TableName: tableName,
-    Item: fields
-  };
-
-  return docClient.put(params).promise();
-}
-
-async function getRecordsByKey(tableName, keys, indexName) {
-  const docClient = getDocClient();
-
-  const keyNames = Object.keys(keys);
-  const conditionExpression = keyNames
-    .map(key => `#${key} = :${key}`)
-    .join(" and ");
-  const expressionNames = keyNames.reduce((acc, key) => {
-    acc[`#${key}`] = key;
-    return acc;
-  }, {});
-  const expressionValues = keyNames.reduce((acc, key) => {
-    acc[`:${key}`] = keys[key];
-    return acc;
-  }, {});
-
-  const params = {
-    TableName: tableName,
-    KeyConditionExpression: conditionExpression,
-    ExpressionAttributeNames: expressionNames,
-    ExpressionAttributeValues: expressionValues
-  };
-
-  if (indexName) {
-    params.IndexName = indexName;
-  }
-
-  let data;
-
-  try {
-    data = await docClient.query(params).promise();
-    return data.Items;
-  } catch (err) {
-    if (err.statusCode && err.statusCode === 400) {
-      return null;
-    }
-
-    throw err;
-  }
-}
-
-async function getRecordByKey(tableName, keys, indexName) {
-  const records = await getRecordsByKey(tableName, keys, indexName);
-
-  if (records) {
-    return records[0];
-  } else {
-    return null;
-  }
-}
-
-async function updateRecordByKey(
-  tableName,
-  idKey,
-  fields,
-  conditionExpressions = null,
-  verbose = false
-) {
-  if (verbose) {
-    console.log("update", tableName, idKey, fields);
-  }
-
-  const idKeyNames = Object.keys(idKey);
-  const keys = Object.keys(fields).filter(k => !idKeyNames.includes(k));
-  const updateExpressions = keys.map(key => `#${key} = :${key}`);
-  const expressionNames = keys.reduce((acc, key) => {
-    acc[`#${key}`] = key;
-    return acc;
-  }, {});
-  const expressionValues = keys.reduce((acc, key) => {
-    acc[`:${key}`] = fields[key];
-    return acc;
-  }, {});
-  const docClient = getDocClient();
-  const params = {
-    TableName: tableName,
-    Key: idKey,
-    ExpressionAttributeNames: expressionNames,
-    ExpressionAttributeValues: expressionValues,
-    UpdateExpression: `SET ${updateExpressions.join(", ")}`,
-    ReturnValues: "ALL_NEW"
-  };
-
-  if (conditionExpressions) {
-    params.ConditionExpression = conditionExpressions;
-  }
-
-  const data = await docClient.update(params).promise();
-
-  return data.Attributes;
-}
-
-async function createTableIfNotExist(
-  tableName,
-  attributeDefinitions = [
-    {
-      AttributeName: "id",
-      AttributeType: "S"
-    }
-  ],
-  keySchema = [
-    {
-      AttributeName: "id",
-      KeyType: "HASH"
-    }
-  ],
-  otherOptions = {},
-  verbose = false
-) {
-  const dynamodb = getDynamoDB();
-
-  try {
-    const table = await dynamodb
-      .describeTable({ TableName: tableName })
-      .promise();
-
-    if (verbose) {
-      console.log(`table ${tableName} already exist`, table);
-    }
-    return table;
-  } catch (_tableDoesNotExistErr) {
-    const createRequest = await dynamodb
-      .createTable({
-        TableName: tableName,
-        AttributeDefinitions: attributeDefinitions,
-        KeySchema: keySchema,
-        ProvisionedThroughput: {
-          ReadCapacityUnits: 5,
-          WriteCapacityUnits: 5
-        },
-        ...otherOptions
-      })
-      .promise();
-
-    if (verbose) {
-      console.log(`table ${tableName} creating`);
-    }
-
-    let isCreated = false;
-
-    while (!isCreated) {
-      await wait(5000);
-
-      const t = await dynamodb
-        .describeTable({ TableName: tableName })
-        .promise();
-
-      console.log("waiting for table to become active");
-
-      isCreated = t.Table.TableStatus === "ACTIVE";
-    }
-
-    return createRequest;
-  }
-}
-
-async function batchDeleteRecords(tableName, keys) {
-  const docClient = getDocClient();
-
-  for (let start = 0; start < keys.length; start += 25) {
-    const slice = keys.slice(start, start + 25);
-
-    await docClient
-      .batchWrite({
-        RequestItems: {
-          [tableName]: slice.map(key => {
-            return { DeleteRequest: { Key: key } };
-          })
-        }
-      })
-      .promise();
-  }
-}
-
-function getKeyName(keySchema, type) {
-  const key = keySchema.find(k => k.KeyType === type);
-
-  return key && key.AttributeName;
-}
-
-function getIndexKeyName(globalSecondaryIndexes, indexName, type) {
-  const idx = globalSecondaryIndexes.find(i => i.IndexName === indexName);
-
-  return idx && getKeyName(idx.KeySchema, type);
-}
-
-async function deleteRecordsByHashKey(
-  tableName,
-  indexName,
-  hashKeyValue,
-  verbose = false
-) {
-  const docClient = getDocClient();
-
-  const meta = await getDynamoDB()
-    .describeTable({ TableName: tableName })
-    .promise();
-
-  const hashKeyName = indexName
-    ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes, indexName, "HASH")
-    : getKeyName(meta.Table.KeySchema, "HASH");
-
-  if (!hashKeyName) {
-    throw new Error(`cannot find corresponding index`, indexName);
-  }
-
-  const mainHashKeyName = getKeyName(meta.Table.KeySchema, "HASH");
-  const mainRangeKeyName = getKeyName(meta.Table.KeySchema, "RANGE");
-
-  let totalDeleted = 0;
-
-  let params = {
-    TableName: tableName,
-    KeyConditionExpression: "#hashKeyName = :hashKeyValue",
-    ExpressionAttributeNames: { "#hashKeyName": hashKeyName },
-    ExpressionAttributeValues: { ":hashKeyValue": hashKeyValue }
-  };
-
-  if (indexName) {
-    params.IndexName = indexName;
-  }
-
-  let data = await docClient.query(params).promise();
-
-  await batchDeleteRecords(
-    tableName,
-    data.Items.map(item =>
-      mainRangeKeyName
-        ? {
-            [mainHashKeyName]: item[mainHashKeyName],
-            [mainRangeKeyName]: item[mainRangeKeyName]
-          }
-        : {
-            [mainHashKeyName]: item[mainHashKeyName]
-          }
-    )
-  );
-
-  totalDeleted += data.Items.length;
-
-  while (data.LastEvaluatedKey) {
-    data = await docClient
-      .query({
-        ...params,
-        ExclusiveStartKey: data.LastEvaluatedKey
-      })
-      .promise();
-
-    await batchDeleteRecords(
-      tableName,
-      data.Items.map(item =>
-        mainRangeKeyName
-          ? {
-              [mainHashKeyName]: item[mainHashKeyName],
-              [mainRangeKeyName]: item[mainRangeKeyName]
-            }
-          : {
-              [mainHashKeyName]: item[mainHashKeyName]
-            }
-      )
-    );
-
-    totalDeleted += data.Items.length;
-  }
-
-  if (verbose) {
-    console.log(`successfully delete ${totalDeleted} items`);
-  }
-
-  return totalDeleted;
-}
-
-module.exports = {
-  getDocClient,
-  scanWholeTable,
-  batchCreateRecords,
-  createRecord,
-  getRecordsByKey,
-  getRecordByKey,
-  updateRecordByKey,
-  batchDeleteRecords,
-  deleteRecordsByHashKey,
-  createTableIfNotExist
-};
+const { DynamoDB } = require("aws-sdk");
+const {
+  getAWSAccessKeyId,
+  getAWSSecretAccessKey,
+  getAWSRegion
+} = require("./env");
+const { wait } = require("./availability");
+
+function getDynamoDB() {
+  return new DynamoDB({
+    accessKeyId: getAWSAccessKeyId(),
+    secretAccessKey: getAWSSecretAccessKey(),
+    region: getAWSRegion()
+  });
+}
+
+function getDocClient() {
+  return new DynamoDB.DocumentClient({
+    accessKeyId: getAWSAccessKeyId(),
+    secretAccessKey: getAWSSecretAccessKey(),
+    region: getAWSRegion()
+  });
+}
+
+function mergeQueries(q1, q2) {
+  return {
+    Items: q1.Items.concat(q2.Items),
+    Count: q1.Count + q2.Count,
+    ScannedCount: q1.ScannedCount + q2.ScannedCount
+  };
+}
+
+async function scanWholeTable(options) {
+  const dynamodb = getDocClient();
+
+  let items = {
+    Items: [],
+    Count: 0,
+    ScannedCount: 0
+  };
+
+  let data = await dynamodb.scan(options).promise();
+
+  while (data.LastEvaluatedKey) {
+    items = mergeQueries(items, data);
+
+    data = await dynamodb
+      .scan({
+        ExclusiveStartKey: data.LastEvaluatedKey,
+        ...options
+      })
+      .promise();
+  }
+
+  items = mergeQueries(items, data);
+
+  return items;
+}
+
+function toSlices(items) {
+  let slices = [];
+
+  for (let start = 0; start < items.length; start += 25) {
+    slices.push(items.slice(start, start + 25));
+  }
+
+  return slices;
+}
+
+async function batchCreateRecords(
+  tableName,
+  records,
+  maxWritingCapacity,
+  verbose = false
+) {
+  if (verbose) {
+    console.log(`creating ${records.length} items in ${tableName}`);
+  }
+  const docClient = getDocClient();
+
+  let remainingItems = records;
+  // an arbitrary number to ensure while first loop is entered
+  let prevRemainingCount = remainingItems.length + 1;
+  let factor = 1;
+  let rejection = undefined;
+
+  while (remainingItems.length > 0 && factor <= 128 && !rejection) {
+    if (prevRemainingCount === remainingItems.length) {
+      await wait(5000 * factor);
+      factor = factor * 2;
+    }
+
+    if (factor >= 32) {
+      console.log(
+        `WARNING: no progress for a long time for batchCreateRecords, please check`
+      );
+    }
+
+    const slices = toSlices(remainingItems.slice(0, maxWritingCapacity));
+    const results = await Promise.allSettled(
+      slices.map(rs => {
+        const params = {
+          RequestItems: {
+            [tableName]: rs.map(record => {
+              return { PutRequest: { Item: record } };
+            })
+          }
+        };
+
+        return docClient.batchWrite(params).promise();
+      })
+    );
+
+    prevRemainingCount = remainingItems.length;
+
+    remainingItems = results.reduce((acc, rs, idx) => {
+      if (rs.status === "rejected") {
+        // whole slice fails, redo whole slice
+        acc = acc.concat(slices[idx]);
+        rejection = rs;
+      } else if (Object.keys(rs.value.UnprocessedItems).length > 0) {
+        // partially fails, redo unprocessedItems
+        acc = acc.concat(
+          rs.value.UnprocessedItems[tableName].map(it => it.PutRequest.Item)
+        );
+      }
+
+      return acc;
+    }, remainingItems.slice(maxWritingCapacity));
+
+    if (verbose) {
+      console.log(
+        `processed=${prevRemainingCount - remainingItems.length}, remaining=${
+          remainingItems.length
+        }`
+      );
+    }
+  }
+
+  if (rejection) {
+    console.log("batchCreateRecords rejected", rejection);
+
+    throw new Error(
+      `batchCreateRecords rejected, failed items=${remainingItems.length}`
+    );
+  }
+
+  if (remainingItems.length > 0) {
+    console.log(
+      `failed batchCreateRecords, failed items=${remainingItems.length}`
+    );
+
+    throw new Error(
+      `batchCreateRecords retry failed, failed items=${remainingItems.length}`
+    );
+  }
+}
+
+async function createRecord(tableName, fields, verbose = false) {
+  if (verbose) {
+    console.log("creating", tableName, fields);
+  }
+
+  const docClient = getDocClient();
+  const params = {
+    TableName: tableName,
+    Item: fields
+  };
+
+  return docClient.put(params).promise();
+}
+
+async function getRecordsByKey(tableName, keys, indexName) {
+  const docClient = getDocClient();
+
+  const keyNames = Object.keys(keys);
+  const conditionExpression = keyNames
+    .map(key => `#${key} = :${key}`)
+    .join(" and ");
+  const expressionNames = keyNames.reduce((acc, key) => {
+    acc[`#${key}`] = key;
+    return acc;
+  }, {});
+  const expressionValues = keyNames.reduce((acc, key) => {
+    acc[`:${key}`] = keys[key];
+    return acc;
+  }, {});
+
+  const params = {
+    TableName: tableName,
+    KeyConditionExpression: conditionExpression,
+    ExpressionAttributeNames: expressionNames,
+    ExpressionAttributeValues: expressionValues
+  };
+
+  if (indexName) {
+    params.IndexName = indexName;
+  }
+
+  let data;
+
+  try {
+    data = await docClient.query(params).promise();
+    return data.Items;
+  } catch (err) {
+    if (err.statusCode && err.statusCode === 400) {
+      return null;
+    }
+
+    throw err;
+  }
+}
+
+async function getRecordByKey(tableName, keys, indexName) {
+  const records = await getRecordsByKey(tableName, keys, indexName);
+
+  if (records) {
+    return records[0];
+  } else {
+    return null;
+  }
+}
+
+async function updateRecordByKey(
+  tableName,
+  idKey,
+  fields,
+  conditionExpressions = null,
+  verbose = false
+) {
+  if (verbose) {
+    console.log("update", tableName, idKey, fields);
+  }
+
+  const idKeyNames = Object.keys(idKey);
+  const keys = Object.keys(fields).filter(k => !idKeyNames.includes(k));
+  const updateExpressions = keys.map(key => `#${key} = :${key}`);
+  const expressionNames = keys.reduce((acc, key) => {
+    acc[`#${key}`] = key;
+    return acc;
+  }, {});
+  const expressionValues = keys.reduce((acc, key) => {
+    acc[`:${key}`] = fields[key];
+    return acc;
+  }, {});
+  const docClient = getDocClient();
+  const params = {
+    TableName: tableName,
+    Key: idKey,
+    ExpressionAttributeNames: expressionNames,
+    ExpressionAttributeValues: expressionValues,
+    UpdateExpression: `SET ${updateExpressions.join(", ")}`,
+    ReturnValues: "ALL_NEW"
+  };
+
+  if (conditionExpressions) {
+    params.ConditionExpression = conditionExpressions;
+  }
+
+  const data = await docClient.update(params).promise();
+
+  return data.Attributes;
+}
+
+async function createTableIfNotExist(
+  tableName,
+  attributeDefinitions = [
+    {
+      AttributeName: "id",
+      AttributeType: "S"
+    }
+  ],
+  keySchema = [
+    {
+      AttributeName: "id",
+      KeyType: "HASH"
+    }
+  ],
+  otherOptions = {},
+  verbose = false
+) {
+  const dynamodb = getDynamoDB();
+
+  try {
+    const table = await dynamodb
+      .describeTable({ TableName: tableName })
+      .promise();
+
+    if (verbose) {
+      console.log(`table ${tableName} already exist`, table);
+    }
+    return table;
+  } catch (_tableDoesNotExistErr) {
+    const createRequest = await dynamodb
+      .createTable({
+        TableName: tableName,
+        AttributeDefinitions: attributeDefinitions,
+        KeySchema: keySchema,
+        ProvisionedThroughput: {
+          ReadCapacityUnits: 5,
+          WriteCapacityUnits: 5
+        },
+        ...otherOptions
+      })
+      .promise();
+
+    if (verbose) {
+      console.log(`table ${tableName} creating`);
+    }
+
+    let isCreated = false;
+
+    while (!isCreated) {
+      await wait(5000);
+
+      const t = await dynamodb
+        .describeTable({ TableName: tableName })
+        .promise();
+
+      console.log("waiting for table to become active");
+
+      isCreated = t.Table.TableStatus === "ACTIVE";
+    }
+
+    return createRequest;
+  }
+}
+
+async function batchDeleteRecords(tableName, keys) {
+  const docClient = getDocClient();
+
+  for (let start = 0; start < keys.length; start += 25) {
+    const slice = keys.slice(start, start + 25);
+
+    await docClient
+      .batchWrite({
+        RequestItems: {
+          [tableName]: slice.map(key => {
+            return { DeleteRequest: { Key: key } };
+          })
+        }
+      })
+      .promise();
+  }
+}
+
+function getKeyName(keySchema, type) {
+  const key = keySchema.find(k => k.KeyType === type);
+
+  return key && key.AttributeName;
+}
+
+function getIndexKeyName(globalSecondaryIndexes, indexName, type) {
+  const idx = globalSecondaryIndexes.find(i => i.IndexName === indexName);
+
+  return idx && getKeyName(idx.KeySchema, type);
+}
+
+async function deleteRecordsByHashKey(
+  tableName,
+  indexName,
+  hashKeyValue,
+  verbose = false
+) {
+  const docClient = getDocClient();
+
+  const meta = await getDynamoDB()
+    .describeTable({ TableName: tableName })
+    .promise();
+
+  const hashKeyName = indexName
+    ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes, indexName, "HASH")
+    : getKeyName(meta.Table.KeySchema, "HASH");
+
+  if (!hashKeyName) {
+    throw new Error(`cannot find corresponding index`, indexName);
+  }
+
+  const mainHashKeyName = getKeyName(meta.Table.KeySchema, "HASH");
+  const mainRangeKeyName = getKeyName(meta.Table.KeySchema, "RANGE");
+
+  let totalDeleted = 0;
+
+  let params = {
+    TableName: tableName,
+    KeyConditionExpression: "#hashKeyName = :hashKeyValue",
+    ExpressionAttributeNames: { "#hashKeyName": hashKeyName },
+    ExpressionAttributeValues: { ":hashKeyValue": hashKeyValue }
+  };
+
+  if (indexName) {
+    params.IndexName = indexName;
+  }
+
+  let data = await docClient.query(params).promise();
+
+  await batchDeleteRecords(
+    tableName,
+    data.Items.map(item =>
+      mainRangeKeyName
+        ? {
+            [mainHashKeyName]: item[mainHashKeyName],
+            [mainRangeKeyName]: item[mainRangeKeyName]
+          }
+        : {
+            [mainHashKeyName]: item[mainHashKeyName]
+          }
+    )
+  );
+
+  totalDeleted += data.Items.length;
+
+  while (data.LastEvaluatedKey) {
+    data = await docClient
+      .query({
+        ...params,
+        ExclusiveStartKey: data.LastEvaluatedKey
+      })
+      .promise();
+
+    await batchDeleteRecords(
+      tableName,
+      data.Items.map(item =>
+        mainRangeKeyName
+          ? {
+              [mainHashKeyName]: item[mainHashKeyName],
+              [mainRangeKeyName]: item[mainRangeKeyName]
+            }
+          : {
+              [mainHashKeyName]: item[mainHashKeyName]
+            }
+      )
+    );
+
+    totalDeleted += data.Items.length;
+  }
+
+  if (verbose) {
+    console.log(`successfully delete ${totalDeleted} items`);
+  }
+
+  return totalDeleted;
+}
+
+module.exports = {
+  getDocClient,
+  scanWholeTable,
+  batchCreateRecords,
+  createRecord,
+  getRecordsByKey,
+  getRecordByKey,
+  updateRecordByKey,
+  batchDeleteRecords,
+  deleteRecordsByHashKey,
+  createTableIfNotExist
+};