@certik/skynet 0.10.42 → 0.10.44

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.10.44
+
+- Fixed modeIndexer delta bug
+
+## 0.10.43
+
+- Fixed modeIndexer one and range mode
+- Fixed `dynamodb`.`getRecordsByKey` pagination handling bug
+
 ## 0.10.42
 
 - Added "contentType" option for `s3`.`writeFile`

package/dynamodb.js

@@ -168,8 +168,23 @@ async function getRecordsByKey(tableName, keys, indexName) {
 
   try {
     data = await docClient.query(params).promise();
-    return data.Items;
+
+    let items = data.Items;
+
+    while (data.LastEvaluatedKey) {
+      data = await docClient
+        .query({
+          ExclusiveStartKey: data.LastEvaluatedKey,
+          ...params,
+        })
+        .promise();
+
+      items = items.concat(data.Items);
+    }
+
+    return items;
   } catch (err) {
+    console.log(err);
     if (err.statusCode && err.statusCode === 400) {
       return null;
     }

package/indexer.js

@@ -103,6 +103,10 @@ function createModeIndexerApp({
     return fillNumberRange(from, to);
   }
 
+  function offsetRange(from, to) {
+    return fillRange(from, to).length;
+  }
+
   async function runMode({ mode, from, to, status, verbose, ...selectorFlags }) {
     if (status) {
       const stateItem = await getRecordByKey(STATE_TABLE_NAME, {
@@ -122,13 +126,13 @@ function createModeIndexerApp({
       process.exit(0);
     }
 
-    if (from === 0) {
+    if (!from) {
       from = await finalState.getMinId(selectorFlags);
     }
 
     inline.log(
       `[MODE INDEXER] mode=${mode}, from=${from}, to=${
-        to > 0 ? to : "undefined"
+        to ? to : "undefined"
       }, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`
     );
 
@@ -136,14 +140,14 @@ function createModeIndexerApp({
       await runReset(selectorFlags);
     } else if (mode === "rebuild") {
       await runReset(selectorFlags);
-      await runRebuild(selectorFlags, from, to > 0 ? to : await finalState.getMaxId(selectorFlags), verbose);
+      await runRebuild(selectorFlags, from, to ? to : await finalState.getMaxId(selectorFlags), verbose);
     } else if (mode === "resume-rebuild") {
       const previousRebuildEnds = await getIndexerLatestId(name, selectorFlags);
 
       await runRebuild(
         selectorFlags,
         previousRebuildEnds ? increaseId(finalState.type, previousRebuildEnds, 1) : from,
-        to > 0 ? to : await finalState.getMaxId(selectorFlags),
+        to ? to : await finalState.getMaxId(selectorFlags),
         verbose
       );
     } else if (mode === "validate" || mode === "validation") {
@@ -151,17 +155,17 @@ function createModeIndexerApp({
 
       const previousValidatedTo = await getIndexerValidatedId(name, selectorFlags);
 
-      const shouldSaveState = to === 0;
+      const shouldSaveState = !to;
 
       await runValidate(
         selectorFlags,
-        from > 0 ? from : previousValidatedTo,
-        to > 0 ? to : previousRebuildEnds,
+        from ? from : previousValidatedTo,
+        to ? to : previousRebuildEnds,
         shouldSaveState,
         verbose
       );
     } else if (mode === "one") {
-      if (to > 0) {
+      if (to) {
         inline.log("[MODE INDEXER] one mode ignores --to option. you may want to use range mode instead");
       }
       await runRange(selectorFlags, from, from, verbose);
@@ -265,7 +269,7 @@ function createModeIndexerApp({
     }
 
     inline.log(
-      `[MODE INDEXER] validated ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
+      `[MODE INDEXER] validated ${offsetRange(from, to)} ${finalState.type} successfully in ${Date.now() - startTime}ms`
     );
   }
 
@@ -367,13 +371,15 @@ function createModeIndexerApp({
 
     if (failedIds.length > 0) {
       inline.log(
-        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} with some failed ${finalState.type}`,
+        `[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) with some failed ${finalState.type}`,
         failedIds
       );
       process.exit(1);
     } else {
       inline.log(
-        `[MODE INDEXER] built ${to - from + 1} ${finalState.type} successfully in ${Date.now() - startTime}ms`
+        `[MODE INDEXER] built ${offsetRange(from, to)} ${finalState.type}(s) successfully in ${
+          Date.now() - startTime
+        }ms`
       );
       process.exit(0);
     }
@@ -386,7 +392,7 @@ function createModeIndexerApp({
 
     // only build when rebuild succeed
     if (stateItem && stateItem.value === "succeed") {
-      const from = (await getIndexerLatestId(name, selectorFlags)) + 1;
+      const from = increaseId(finalState.type, await getIndexerLatestId(name, selectorFlags), 1);
 
       const startTime = Date.now();
       const to = await state.getMaxId(selectorFlags);
@@ -504,13 +510,11 @@ ${getSelectorDesc(selector)}
           },
           from: {
             alias: "since",
-            type: "number",
-            default: 0,
+            type: "string",
           },
           to: {
             alias: "until",
-            type: "number",
-            default: 0,
+            type: "string",
           },
           status: {
             type: "boolean",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@certik/skynet",
-  "version": "0.10.42",
+  "version": "0.10.44",
   "description": "Skynet Shared JS library",
   "main": "index.js",
   "author": "CertiK Engineering",