@certik/skynet 0.18.9 → 0.19.1

package/{dynamodb.js → dynamodb.ts}

@@ -6,14 +6,24 @@ import {
   PutCommand,
   QueryCommand,
   UpdateCommand,
-  TransactWriteCommand,
+} from "@aws-sdk/lib-dynamodb";
+import type {
+  ScanCommandInput,
+  PutCommandInput,
+  GetCommandInput,
+  QueryCommandInput,
+  UpdateCommandOutput,
+  UpdateCommandInput,
+  ScanCommandOutput,
 } from "@aws-sdk/lib-dynamodb";
 import { DynamoDBClient, DescribeTableCommand } from "@aws-sdk/client-dynamodb";
+import type { KeySchemaElement, GlobalSecondaryIndexDescription } from "@aws-sdk/client-dynamodb";
 import { wait } from "./availability.js";
+import { arrayGroup } from "./util.js";
 
-let _dynamoDB;
+let _dynamoDB: DynamoDBClient;
 
-let _docClient;
+let _docClient: DynamoDBDocumentClient;
 
 function getDynamoDB(forceNew = false) {
   if (!_dynamoDB || forceNew) {
@@ -46,47 +56,44 @@ function getDocClient(forceNew = false) {
   return _docClient;
 }
 
-function mergeQueries(q1, q2) {
-  return {
-    Items: q1.Items.concat(q2.Items),
-    Count: q1.Count + q2.Count,
-    ScannedCount: q1.ScannedCount + q2.ScannedCount,
-  };
-}
-
-async function scanWholeTable(options) {
+async function scanWholeTable(options: ScanCommandInput) {
   const dynamodb = getDocClient();
 
-  let items = {
-    Items: [],
-    Count: 0,
-    ScannedCount: 0,
-  };
+  let items: ScanCommandOutput["Items"] = [];
+  let count = 0;
+  let scannedCount = 0;
 
   let data = await dynamodb.send(new ScanCommand(options));
 
   while (data.LastEvaluatedKey) {
-    items = mergeQueries(items, data);
+    if (data.Items) {
+      items = items.concat(data.Items);
+    }
+    count += data.Count || 0;
+    scannedCount += data.ScannedCount || 0;
 
     data = await dynamodb.send(new ScanCommand({ ...options, ExclusiveStartKey: data.LastEvaluatedKey }));
   }
 
-  items = mergeQueries(items, data);
-
-  return items;
-}
-
-function toSlices(items) {
-  let slices = [];
-
-  for (let start = 0; start < items.length; start += 25) {
-    slices.push(items.slice(start, start + 25));
+  if (data.Items) {
+    items = items.concat(data.Items);
   }
+  count += data.Count || 0;
+  scannedCount += data.ScannedCount || 0;
 
-  return slices;
+  return {
+    Items: items,
+    Count: count,
+    ScannedCount: scannedCount,
+  };
 }
 
-async function batchCreateRecords(tableName, records, maxWritingCapacity, verbose = false) {
+async function batchCreateRecords(
+  tableName: string,
+  records: Record<string, unknown>[],
+  maxWritingCapacity?: number,
+  verbose = false,
+) {
   if (verbose) {
     console.log(`creating ${records.length} items in ${tableName}`);
   }
@@ -96,7 +103,7 @@ async function batchCreateRecords(tableName, records, maxWritingCapacity, verbos
   // an arbitrary number to ensure while first loop is entered
   let prevRemainingCount = remainingItems.length + 1;
   let factor = 1;
-  let rejection = undefined;
+  let rejection: PromiseRejectedResult | undefined = undefined;
 
   while (remainingItems.length > 0 && factor <= 128 && !rejection) {
     if (prevRemainingCount === remainingItems.length) {
@@ -108,35 +115,36 @@ async function batchCreateRecords(tableName, records, maxWritingCapacity, verbos
       console.log(`WARNING: no progress for a long time for batchCreateRecords, please check`);
     }
 
-    const slices = toSlices(remainingItems.slice(0, maxWritingCapacity));
+    const slices = arrayGroup(remainingItems.slice(0, maxWritingCapacity), 25);
     const results = await Promise.allSettled(
-      slices.map((rs) => {
-        const params = {
-          RequestItems: {
-            [tableName]: rs.map((record) => {
-              return { PutRequest: { Item: record } };
-            }),
-          },
-        };
-
-        return docClient.send(new BatchWriteCommand(params));
-      }),
+      slices.map((rs) =>
+        docClient.send(
+          new BatchWriteCommand({
+            RequestItems: {
+              [tableName]: rs.map((record) => ({ PutRequest: { Item: record } })),
+            },
+          }),
+        ),
+      ),
     );
 
+    const isFulfilled = <T>(p: PromiseSettledResult<T>): p is PromiseFulfilledResult<T> => p.status === "fulfilled";
+    const isRejected = <T>(p: PromiseSettledResult<T>): p is PromiseRejectedResult => p.status === "rejected";
+
     prevRemainingCount = remainingItems.length;
+    remainingItems = remainingItems.slice(maxWritingCapacity);
 
-    remainingItems = results.reduce((acc, rs, idx) => {
-      if (rs.status === "rejected") {
+    results.forEach((rs, idx) => {
+      if (isRejected(rs)) {
         // whole slice fails, redo whole slice
-        acc = acc.concat(slices[idx]);
+        remainingItems = remainingItems.concat(slices[idx]);
         rejection = rs;
-      } else if (Object.keys(rs.value.UnprocessedItems).length > 0) {
+      } else if (isFulfilled(rs) && rs.value.UnprocessedItems && Object.keys(rs.value.UnprocessedItems).length > 0) {
         // partially fails, redo unprocessedItems
-        acc = acc.concat(rs.value.UnprocessedItems[tableName].map((it) => it.PutRequest.Item));
+        const unprocessedItems = rs.value.UnprocessedItems[tableName].map((it) => it.PutRequest?.Item ?? []).flat();
+        remainingItems = remainingItems.concat(unprocessedItems);
       }
-
-      return acc;
-    }, remainingItems.slice(maxWritingCapacity));
+    });
 
     if (verbose) {
       console.log(`processed=${prevRemainingCount - remainingItems.length}, remaining=${remainingItems.length}`);
@@ -156,7 +164,7 @@ async function batchCreateRecords(tableName, records, maxWritingCapacity, verbos
   }
 }
 
-async function createRecord(tableName, fields, verbose = false) {
+async function createRecord(tableName: string, fields: PutCommandInput["Item"], verbose = false) {
   if (verbose) {
     console.log("creating", tableName, fields);
   }
@@ -170,7 +178,7 @@ async function createRecord(tableName, fields, verbose = false) {
   return docClient.send(new PutCommand(params));
 }
 
-async function readRecord(tableName, key, verbose = false) {
+async function readRecord<T>(tableName: string, key: GetCommandInput["Key"], verbose = false) {
   if (verbose) {
     console.log("reading", tableName, key);
   }
@@ -184,16 +192,20 @@ async function readRecord(tableName, key, verbose = false) {
     }),
   );
 
-  return record.Item;
+  return record.Item as T;
 }
 
-async function getRecordsByKey(tableName, keys, indexName) {
+async function getRecordsByKey<TReturn, TKey extends Record<string, unknown> = Record<string, unknown>>(
+  tableName: string,
+  keys: TKey,
+  indexName?: string,
+) {
   const docClient = getDocClient();
 
   const keyNames = Object.keys(keys);
   const conditionExpression = keyNames.map((key) => `#${key} = :${key}`).join(" and ");
 
-  const params = {
+  const params: QueryCommandInput = {
     TableName: tableName,
     KeyConditionExpression: conditionExpression,
     ExpressionAttributeNames: generateExpressionNames(keyNames),
@@ -204,12 +216,10 @@ async function getRecordsByKey(tableName, keys, indexName) {
     params.IndexName = indexName;
   }
 
-  let data;
-
   try {
-    data = await docClient.send(new QueryCommand(params));
+    let data = await docClient.send(new QueryCommand(params));
 
-    let items = data.Items;
+    let items = data.Items ?? [];
 
     while (data.LastEvaluatedKey) {
       data = await docClient.send(
@@ -219,13 +229,15 @@ async function getRecordsByKey(tableName, keys, indexName) {
         }),
       );
 
-      items = items.concat(data.Items);
+      if (data.Items) {
+        items = items.concat(data.Items);
+      }
     }
 
-    return items;
+    return items as TReturn[];
   } catch (err) {
     console.log(err);
-    if (err.statusCode && err.statusCode === 400) {
+    if (err instanceof Error && "statusCode" in err && err.statusCode === 400) {
       return null;
     }
 
@@ -234,9 +246,9 @@ async function getRecordsByKey(tableName, keys, indexName) {
 }
 
 // Dual purpose for compatibility. If indexName is provided, it will use query command to get the record; if not, use get command which is most efficient.
-async function getRecordByKey(tableName, keys, indexName) {
+async function getRecordByKey<T>(tableName: string, keys: Record<string, unknown>, indexName?: string) {
   if (indexName) {
-    const records = await getRecordsByKey(tableName, keys, indexName);
+    const records = await getRecordsByKey<T>(tableName, keys, indexName);
 
     if (records) {
       return records[0];
@@ -244,25 +256,28 @@ async function getRecordByKey(tableName, keys, indexName) {
       return null;
     }
   } else {
-    return readRecord(tableName, keys);
+    return readRecord<T>(tableName, keys);
   }
 }
 
-function generateExpressionNames(keys) {
-  return keys.reduce((acc, key) => {
-    acc[`#${key}`] = key;
-    return acc;
-  }, {});
+function generateExpressionNames(keys: string[]): QueryCommandInput["ExpressionAttributeNames"] {
+  return keys.reduce((acc, key) => ({ ...acc, [`#${key}`]: key }), {});
 }
 
-function generateExpressionValues(keys, fields) {
-  return keys.reduce((acc, key) => {
-    acc[`:${key}`] = fields[key];
-    return acc;
-  }, {});
+function generateExpressionValues(
+  keys: string[],
+  fields: Record<string, unknown>,
+): QueryCommandInput["ExpressionAttributeValues"] {
+  return keys.reduce((acc, key) => ({ ...acc, [`:${key}`]: fields[key] }), {});
 }
 
-async function updateRecordByKey(tableName, idKey, fields, conditionExpressions = null, verbose = false) {
+async function updateRecordByKey(
+  tableName: string,
+  idKey: Record<string, unknown>,
+  fields: Record<string, unknown>,
+  conditionExpressions = null,
+  verbose = false,
+) {
   if (verbose) {
     console.log("update", tableName, idKey, fields);
   }
@@ -273,14 +288,14 @@ async function updateRecordByKey(tableName, idKey, fields, conditionExpressions
   const fieldsToDelete = Object.keys(fields).filter((f) => fields[f] === undefined);
   const fieldsToUpdate = Object.keys(fields).filter((k) => !idKeyNames.includes(k) && !fieldsToDelete.includes(k));
 
-  let data;
+  let data: UpdateCommandOutput | undefined;
 
   if (fieldsToDelete.length > 0) {
     if (verbose) {
       console.log("delete fields", tableName, fieldsToDelete);
     }
 
-    const deleteParams = {
+    const deleteParams: UpdateCommandInput = {
       TableName: tableName,
       Key: idKey,
       ExpressionAttributeNames: generateExpressionNames(fieldsToDelete),
@@ -302,7 +317,7 @@ async function updateRecordByKey(tableName, idKey, fields, conditionExpressions
 
     const updateExpressions = fieldsToUpdate.map((key) => `#${key} = :${key}`);
 
-    const params = {
+    const params: UpdateCommandInput = {
       TableName: tableName,
       Key: idKey,
       ExpressionAttributeNames: generateExpressionNames(fieldsToUpdate),
@@ -318,70 +333,10 @@ async function updateRecordByKey(tableName, idKey, fields, conditionExpressions
     data = await docClient.send(new UpdateCommand(params));
   }
 
-  return data && data.Attributes;
-}
-
-async function createTableIfNotExist(
-  tableName,
-  attributeDefinitions = [
-    {
-      AttributeName: "id",
-      AttributeType: "S",
-    },
-  ],
-  keySchema = [
-    {
-      AttributeName: "id",
-      KeyType: "HASH",
-    },
-  ],
-  otherOptions = {},
-  verbose = false,
-) {
-  const dynamodb = getDynamoDB();
-
-  try {
-    const table = await dynamodb.send(new DescribeTableCommand({ TableName: tableName }));
-
-    if (verbose) {
-      console.log(`table ${tableName} already exist`, table);
-    }
-    return table;
-  } catch (_tableDoesNotExistErr) {
-    const createRequest = await dynamodb
-      .createTable({
-        TableName: tableName,
-        AttributeDefinitions: attributeDefinitions,
-        KeySchema: keySchema,
-        ProvisionedThroughput: {
-          ReadCapacityUnits: 5,
-          WriteCapacityUnits: 5,
-        },
-        ...otherOptions,
-      })
-      .promise();
-
-    if (verbose) {
-      console.log(`table ${tableName} creating`);
-    }
-
-    let isCreated = false;
-
-    while (!isCreated) {
-      await wait(5000);
-
-      const t = await dynamodb.send(new DescribeTableCommand({ TableName: tableName }));
-
-      console.log("waiting for table to become active");
-
-      isCreated = t.Table.TableStatus === "ACTIVE";
-    }
-
-    return createRequest;
-  }
+  return data?.Attributes;
 }
 
-async function batchDeleteRecords(tableName, keys) {
+async function batchDeleteRecords(tableName: string, keys: Record<string, unknown>[]) {
   const docClient = getDocClient();
 
   for (let start = 0; start < keys.length; start += 25) {
@@ -399,37 +354,63 @@ async function batchDeleteRecords(tableName, keys) {
   }
 }
 
-function getKeyName(keySchema, type) {
+function getKeyName(keySchema: KeySchemaElement[], type: string) {
   const key = keySchema.find((k) => k.KeyType === type);
 
-  return key && key.AttributeName;
+  return key?.AttributeName;
 }
 
-function getIndexKeyName(globalSecondaryIndexes, indexName, type) {
+function getIndexKeyName(globalSecondaryIndexes: GlobalSecondaryIndexDescription[], indexName: string, type: string) {
   const idx = globalSecondaryIndexes.find((i) => i.IndexName === indexName);
 
-  return idx && getKeyName(idx.KeySchema, type);
+  return idx?.KeySchema && getKeyName(idx.KeySchema, type);
 }
 
-async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbose = false) {
+async function deleteRecordsByHashKey(
+  tableName: string,
+  indexName: string | undefined,
+  hashKeyValue: string,
+  verbose = false,
+) {
   const docClient = getDocClient();
 
   const meta = await getDynamoDB().send(new DescribeTableCommand({ TableName: tableName }));
 
+  if (!meta.Table) {
+    throw new Error(`cannot find table ${tableName}`);
+  }
+
+  if (indexName && !meta.Table.GlobalSecondaryIndexes) {
+    throw new Error(`cannot find global secondary indexes for table ${tableName}`);
+  }
+
+  if (!meta.Table.KeySchema) {
+    throw new Error(`cannot find key schema for table ${tableName}`);
+  }
+
   const hashKeyName = indexName
-    ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes, indexName, "HASH")
+    ? getIndexKeyName(meta.Table.GlobalSecondaryIndexes!, indexName, "HASH")
     : getKeyName(meta.Table.KeySchema, "HASH");
 
   if (!hashKeyName) {
-    throw new Error(`cannot find corresponding index`, indexName);
+    throw new Error(`cannot find hash key name for table ${tableName}`);
   }
 
   const mainHashKeyName = getKeyName(meta.Table.KeySchema, "HASH");
+
+  if (!mainHashKeyName) {
+    throw new Error(`cannot find main hash key name for table ${tableName}`);
+  }
+
   const mainRangeKeyName = getKeyName(meta.Table.KeySchema, "RANGE");
 
+  if (!mainRangeKeyName) {
+    throw new Error(`cannot find main range key name for table ${tableName}`);
+  }
+
   let totalDeleted = 0;
 
-  let params = {
+  const params: QueryCommandInput = {
     TableName: tableName,
     KeyConditionExpression: "#hashKeyName = :hashKeyValue",
     ExpressionAttributeNames: { "#hashKeyName": hashKeyName },
@@ -442,30 +423,7 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
 
   let data = await docClient.send(new QueryCommand(params));
 
-  await batchDeleteRecords(
-    tableName,
-    data.Items.map((item) =>
-      mainRangeKeyName
-        ? {
-            [mainHashKeyName]: item[mainHashKeyName],
-            [mainRangeKeyName]: item[mainRangeKeyName],
-          }
-        : {
-            [mainHashKeyName]: item[mainHashKeyName],
-          },
-    ),
-  );
-
-  totalDeleted += data.Items.length;
-
-  while (data.LastEvaluatedKey) {
-    data = await docClient.send(
-      new QueryCommand({
-        ...params,
-        ExclusiveStartKey: data.LastEvaluatedKey,
-      }),
-    );
-
+  if (data.Items) {
     await batchDeleteRecords(
       tableName,
       data.Items.map((item) =>
@@ -483,6 +441,33 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
     totalDeleted += data.Items.length;
   }
 
+  while (data.LastEvaluatedKey) {
+    data = await docClient.send(
+      new QueryCommand({
+        ...params,
+        ExclusiveStartKey: data.LastEvaluatedKey,
+      }),
+    );
+
+    if (data.Items) {
+      await batchDeleteRecords(
+        tableName,
+        data.Items.map((item) =>
+          mainRangeKeyName
+            ? {
+                [mainHashKeyName]: item[mainHashKeyName],
+                [mainRangeKeyName]: item[mainRangeKeyName],
+              }
+            : {
+                [mainHashKeyName]: item[mainHashKeyName],
+              },
+        ),
+      );
+
+      totalDeleted += data.Items.length;
+    }
+  }
+
   if (verbose) {
     console.log(`successfully delete ${totalDeleted} items`);
   }
@@ -490,16 +475,6 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
   return totalDeleted;
 }
 
-async function transactWrite(items) {
-  const docClient = getDocClient();
-
-  return docClient.send(
-    new TransactWriteCommand({
-      TransactItems: items,
-    }),
-  );
-}
-
 export {
   getDocClient,
   // export common dynamodb commands to make it easier to use
@@ -517,6 +492,4 @@ export {
   updateRecordByKey,
   batchDeleteRecords,
   deleteRecordsByHashKey,
-  createTableIfNotExist,
-  transactWrite,
 };

package/env.ts

@@ -0,0 +1,25 @@
+function ensureAndGet(envName: string, defaultValue?: string) {
+  return process.env[envName] || defaultValue;
+}
+
+function getEnvironment() {
+  return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
+}
+
+function getEnvOrThrow(envName: string) {
+  if (!process.env[envName]) {
+    throw new Error(`Must set environment variable ${envName}`);
+  }
+
+  return process.env[envName];
+}
+
+function isProduction() {
+  return getEnvironment() === "prd";
+}
+
+function isDev() {
+  return getEnvironment() === "dev";
+}
+
+export { ensureAndGet, getEnvOrThrow, getEnvironment, isProduction, isDev };

package/eslint.config.mjs

@@ -0,0 +1,7 @@
+import eslint from '@eslint/js';
+import tseslint from 'typescript-eslint';
+
+export default tseslint.config(
+  eslint.configs.recommended,
+  tseslint.configs.recommended,
+);

package/examples/{api.js → api.ts}

@@ -7,19 +7,20 @@
 //   $ examples/api deploy
 //   $ examples/api --help
 
-import { api, every, SENSITIVE_VALUE, ERROR_LEVEL } from "../app.js";
+import type { Request, Response, NextFunction } from "express";
+import { api, SENSITIVE_VALUE } from "../app.ts";
 
 // an example middleware
-async function exampleMiddleware(req, res, next) {
+async function exampleMiddleware(req: Request, res: Response, next: NextFunction) {
   console.log("before request!");
   next();
   console.log("after request!");
 }
 
-async function getProjectsHandler({ req, res, verbose }) {
+async function getProjectsHandler({ req, res, verbose }: { req: Request; res: Response; verbose?: boolean }) {
   console.log("receieved call", req.query, verbose);
 
-  const type = req.query.type;
+  const type = req.query["type"];
 
   if (!type) {
     res.status(400).end("must provide type parameter");
@@ -29,7 +30,7 @@ async function getProjectsHandler({ req, res, verbose }) {
   res.json([{ id: "proj-1", name: "Test Project 1" }]);
 }
 
-async function createProjectHandler({ req, res, verbose }) {
+async function createProjectHandler({ req, res, verbose }: { req: Request; res: Response; verbose?: boolean }) {
   console.log("receieved call", req.body, verbose);
 
   res.json({ success: true });
@@ -61,7 +62,7 @@ const app = api({
   serve: {
     prefix: "/example-api", // deployed to a sub path
     port: 12345,
-    apiKey: process.env.EXAMPLE_API_KEY,
+    apiKey: process.env["EXAMPLE_API_KEY"],
     cpu: 600,
     mem: 600,
     instances: 3, // run 3 instances

package/examples/{indexer.js → indexer.ts}

@@ -7,15 +7,7 @@
 //   $ examples/indexer deploy --protocol eth
 //   $ examples/indexer --help
 
-import { indexer, every, ERROR_LEVEL } from "../app.js";
-
-async function build({ protocol, verbose }) {
-  console.log("build called with", protocol, verbose);
-
-  if (protocol === "eth") {
-    throw new Error("use eth to test error handling");
-  }
-}
+import { indexer, every } from "../app.ts";
 
 const app = indexer({
   name: "example-indexer",
@@ -23,6 +15,7 @@ const app = indexer({
   selector: {
     // for more flags check meow documentation at https://github.com/sindresorhus/meow
     protocol: {
+      isRequired: true,
       type: "string",
       description: "which chain to index",
     },
@@ -33,7 +26,13 @@ const app = indexer({
   },
 
   build: {
-    func: build,
+    func: ({ protocol, verbose }) => {
+      console.log("build called with", protocol, verbose);
+
+      if (protocol === "eth") {
+        throw new Error("use eth to test error handling");
+      }
+    },
     schedule: every(1).minute,
     killTimeout: "90s",
     cpu: 600,