@certik/skynet 0.25.5 → 0.25.6

package/dist/app.js

@@ -1,41 +1,3 @@
-// src/selector.ts
-function getSelectorDesc(selector) {
-  return Object.keys(selector).map((name) => {
-    return `          --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
-  }).join(`
-`);
-}
-function getSelectorFlags(selector) {
-  return Object.keys(selector).reduce((acc, name) => {
-    const flag = {
-      type: selector[name].type || "string",
-      ...selector[name]
-    };
-    if (!selector[name].optional && selector[name].isRequired !== false) {
-      flag.isRequired = true;
-    }
-    return { ...acc, [name]: flag };
-  }, {});
-}
-function toSelectorString(selectorFlags, delim = ",") {
-  return Object.keys(selectorFlags).sort().map((flag) => {
-    return `${flag}=${selectorFlags[flag]}`;
-  }).join(delim);
-}
-function normalizeSelectorValue(v) {
-  return v.replace(/[^A-Za-z0-9]+/g, "-");
-}
-function getJobName(name, selectorFlags, mode) {
-  const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
-  let jobName = name;
-  if (mode) {
-    jobName += `-${mode}`;
-  }
-  if (selectorNamePart.length > 0) {
-    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
-  }
-  return jobName;
-}
 // src/env.ts
 function ensureAndGet(envName, defaultValue) {
   return process.env[envName] || defaultValue;
@@ -55,207 +17,56 @@ function isProduction() {
 function isDev() {
   return getEnvironment() === "dev";
 }
-// src/log.ts
-function isObject(a) {
-  return !!a && a.constructor === Object;
-}
-function print(o) {
-  if (Array.isArray(o)) {
-    return `[${o.map(print).join(", ")}]`;
-  }
-  if (isObject(o)) {
-    return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
+// src/util.ts
+function range(startAt, endAt, step) {
+  const arr = [];
+  for (let i = startAt;i <= endAt; i += step) {
+    arr.push([i, Math.min(endAt, i + step - 1)]);
   }
-  return `${o}`;
+  return arr;
 }
-function getLine(params) {
-  let line = "";
-  for (let i = 0, l = params.length;i < l; i++) {
-    line += `${print(params[i])} `.replace(/\n/gm, "\t");
+function arrayGroup(array, groupSize) {
+  const groups = [];
+  for (let i = 0;i < array.length; i += groupSize) {
+    groups.push(array.slice(i, i + groupSize));
   }
-  return line.trim();
-}
-function timestamp() {
-  return new Date().toISOString();
+  return groups;
 }
-var inline = {
-  debug: function(...args) {
-    if (true) {
-      console.log(`${timestamp()} ${getLine(args)}`);
-    }
-  },
-  log: function(...args) {
-    if (true) {
-      console.log(`${timestamp()} ${getLine(args)}`);
-    }
-  },
-  error: function(...args) {
-    if (true) {
-      console.error(`${timestamp()} ${getLine(args)}`);
-    }
-  }
-};
-var logger = {
-  debug: function(...args) {
-    if (true) {
-      console.log(`[${timestamp()}]`, ...args);
-    }
-  },
-  log: function(...args) {
-    if (true) {
-      console.log(`[${timestamp()}]`, ...args);
-    }
-  },
-  error: function(...args) {
-    if (true) {
-      console.error(`[${timestamp()}]`, ...args);
-    }
+function fillRange(start, end) {
+  const result = [];
+  for (let i = start;i <= end; i++) {
+    result.push(i);
   }
-};
-// src/api.ts
-import osModule from "os";
-import express from "express";
-import meow from "meow";
-async function logStartMiddleware(_, res, next) {
-  const start = new Date;
-  res.set("x-requested-at", start.toISOString());
-  next();
+  return result;
 }
-async function contextMiddleware(_, res, next) {
-  res.set("x-instance-id", osModule.hostname());
-  next();
+// src/date.ts
+var MS_IN_A_DAY = 3600 * 24 * 1000;
+function getDateOnly(date) {
+  return new Date(date).toISOString().split("T")[0];
 }
-async function logEndMiddleware(req, res, next) {
-  const requestedAt = res.get("x-requested-at");
-  if (!requestedAt) {
-    inline.log("missing x-requested-at header");
-    next();
-    return;
-  }
-  const start = new Date(requestedAt).getTime();
-  const end = new Date().getTime();
-  const logInfo = {
-    start,
-    end,
-    elapsed: `${end - start}ms`,
-    endpoint: req.path,
-    host: req.hostname,
-    status: res.statusCode
-  };
-  if (res.statusMessage) {
-    logInfo.errorMessage = res.statusMessage;
-  }
-  inline.log(JSON.stringify(logInfo));
-  next();
+function findDateAfter(date, n) {
+  const d = new Date(date);
+  const after = new Date(d.getTime() + MS_IN_A_DAY * n);
+  return getDateOnly(after);
 }
-var apiKeyMiddleware = (key) => {
-  async function requireAPIKey(req, res, next) {
-    try {
-      const apiKey = req.get("x-api-key") || req.query["api-key"];
-      if (!apiKey) {
-        inline.log("request without api key");
-        res.status(400).send("require x-api-key header");
-        return;
-      }
-      if (typeof key === "string") {
-        if (apiKey !== key) {
-          inline.log("request has an invalid api key");
-          res.status(400).send("invalid api key");
-          return;
-        }
-        inline.log(`requested by valid key ${key.slice(0, 6)}`);
-      } else {
-        const name = key[apiKey];
-        if (!name) {
-          inline.log("request has an invalid api key");
-          res.status(400).send("invalid api key");
-          return;
-        }
-        inline.log(`requested by authorized user ${name}`);
-      }
-      next();
-    } catch (err) {
-      inline.log("check api key error", err);
-      res.status(500).send("internal error");
-    }
-  }
-  return requireAPIKey;
-};
-async function startApiApp({
-  binaryName,
-  name,
-  selector = {},
-  routes,
-  serve,
-  beforeListen
-}) {
-  const app = express();
-  app.use(express.json({ limit: "20mb" }));
-  const cli = meow(`
-        Usage
-          $ ${binaryName} <options>
-
-        Options
-${getSelectorDesc(selector)}
-          --verbose       Output debug messages
-      `, {
-    importMeta: import.meta,
-    description: false,
-    flags: {
-      ...getSelectorFlags(selector),
-      verbose: {
-        type: "boolean",
-        default: false
-      }
-    }
-  });
-  const { verbose, ...selectorFlags } = cli.flags;
-  for (const route of routes) {
-    const method = route.method ? route.method.toLowerCase() : "get";
-    const middlewares = route.middlewares || [];
-    if (route.protected) {
-      if (!serve.apiKey) {
-        throw new Error("serve.apiKey is required for protected route");
-      }
-      middlewares.unshift(apiKeyMiddleware(serve.apiKey));
-    }
-    if (app[method]) {
-      if (verbose) {
-        inline.log(`registering ${method} ${route.path}`);
-      }
-      app[method](route.path, contextMiddleware, logStartMiddleware, ...middlewares, async (req, res, next) => {
-        try {
-          await route.handler({ req, res, ...selectorFlags });
-        } catch (routeErr) {
-          if (routeErr instanceof Error) {
-            inline.log("caught route err", routeErr, routeErr.stack);
-            res.status(500).send(`internal server error: ${routeErr.message}`);
-          } else {
-            inline.log("caught route err", routeErr);
-            res.status(500).send("internal server error");
-          }
-        }
-        next();
-      }, logEndMiddleware);
-    }
-  }
-  if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
-    app.get("/", (_, res) => {
-      res.send("ok");
-    });
+function daysInRange(from, to) {
+  const fromTime = new Date(from).getTime();
+  const toTime = new Date(to).getTime();
+  if (fromTime > toTime) {
+    throw new Error(`range to date couldn't be earlier than range from date`);
   }
-  if (beforeListen) {
-    await beforeListen({ app });
+  const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
+  const dates = [getDateOnly(new Date(fromTime))];
+  for (let i = 1;i <= daysBetween; i += 1) {
+    dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
   }
-  app.listen(serve.port, () => {
-    if (isProduction()) {
-      inline.log(`${name} listening at https://api.wf.corp.certik.com${serve.prefix}`);
-    } else {
-      inline.log(`${name} listening at http://localhost:${serve.port}`);
-    }
-  });
+  return dates;
+}
+function dateRange(from, to, step) {
+  const days = daysInRange(from, to);
+  const windows = arrayGroup(days, step);
+  return windows.map((w) => [w[0], w[w.length - 1]]);
 }
-
 // src/object-hash.ts
 import xh from "@node-rs/xxhash";
 function getHash(obj) {
@@ -382,28 +193,63 @@ function memoize(func, options) {
   }
   return pMemoize(func, options);
 }
-// src/util.ts
-function range(startAt, endAt, step) {
-  const arr = [];
-  for (let i = startAt;i <= endAt; i += step) {
-    arr.push([i, Math.min(endAt, i + step - 1)]);
+// src/log.ts
+function isObject(a) {
+  return !!a && a.constructor === Object;
+}
+function print(o) {
+  if (Array.isArray(o)) {
+    return `[${o.map(print).join(", ")}]`;
   }
-  return arr;
+  if (isObject(o)) {
+    return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
+  }
+  return `${o}`;
 }
-function arrayGroup(array, groupSize) {
-  const groups = [];
-  for (let i = 0;i < array.length; i += groupSize) {
-    groups.push(array.slice(i, i + groupSize));
+function getLine(params) {
+  let line = "";
+  for (let i = 0, l = params.length;i < l; i++) {
+    line += `${print(params[i])} `.replace(/\n/gm, "\t");
+  }
+  return line.trim();
+}
+function timestamp() {
+  return new Date().toISOString();
+}
+var inline = {
+  debug: function(...args) {
+    if (true) {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
+  },
+  log: function(...args) {
+    if (true) {
+      console.log(`${timestamp()} ${getLine(args)}`);
+    }
+  },
+  error: function(...args) {
+    if (true) {
+      console.error(`${timestamp()} ${getLine(args)}`);
+    }
   }
-  return groups;
-}
-function fillRange(start, end) {
-  const result = [];
-  for (let i = start;i <= end; i++) {
-    result.push(i);
+};
+var logger = {
+  debug: function(...args) {
+    if (true) {
+      console.log(`[${timestamp()}]`, ...args);
+    }
+  },
+  log: function(...args) {
+    if (true) {
+      console.log(`[${timestamp()}]`, ...args);
+    }
+  },
+  error: function(...args) {
+    if (true) {
+      console.error(`[${timestamp()}]`, ...args);
+    }
   }
-  return result;
-}
+};
 // src/dynamodb.ts
 import {
   DynamoDBDocumentClient,
@@ -718,6 +564,58 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
   }
   return totalDeleted;
 }
+function destroyDynamoDB() {
+  if (_dynamoDB) {
+    _dynamoDB.destroy();
+  }
+}
+// src/selector.ts
+function getSelectorDesc(selector) {
+  return Object.keys(selector).map((name) => {
+    return `          --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
+  }).join(`
+`);
+}
+function getSelectorFlags(selector) {
+  return Object.keys(selector).reduce((acc, name) => {
+    const flag = {
+      type: selector[name].type || "string",
+      ...selector[name]
+    };
+    if (!selector[name].optional && selector[name].isRequired !== false) {
+      flag.isRequired = true;
+    }
+    return { ...acc, [name]: flag };
+  }, {});
+}
+function toSelectorString(selectorFlags, delim = ",") {
+  return Object.keys(selectorFlags).sort().map((flag) => {
+    return `${flag}=${selectorFlags[flag]}`;
+  }).join(delim);
+}
+function normalizeSelectorValue(v) {
+  return v.replace(/[^A-Za-z0-9]+/g, "-");
+}
+function includeSelectorValueInJobName(value) {
+  if (value === false)
+    return false;
+  if (value === undefined || value === null)
+    return false;
+  if (value === "")
+    return false;
+  return true;
+}
+function getJobName(name, selectorFlags, mode) {
+  const selectorNamePart = Object.keys(selectorFlags).sort().map((key) => selectorFlags[key]).filter(includeSelectorValueInJobName).map((value) => String(value)).join("-");
+  let jobName = name;
+  if (mode) {
+    jobName += `-${mode}`;
+  }
+  if (selectorNamePart.length > 0) {
+    jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
+  }
+  return jobName;
+}
 // src/cli.ts
 import path from "path";
 import fs from "fs";
@@ -753,36 +651,11 @@ function detectBin() {
   const wd = detectDirectory(process.argv[1], "package.json");
   return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
 }
-// src/date.ts
-var MS_IN_A_DAY = 3600 * 24 * 1000;
-function getDateOnly(date) {
-  return new Date(date).toISOString().split("T")[0];
-}
-function findDateAfter(date, n) {
-  const d = new Date(date);
-  const after = new Date(d.getTime() + MS_IN_A_DAY * n);
-  return getDateOnly(after);
-}
-function daysInRange(from, to) {
-  const fromTime = new Date(from).getTime();
-  const toTime = new Date(to).getTime();
-  if (fromTime > toTime) {
-    throw new Error(`range to date couldn't be earlier than range from date`);
-  }
-  const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
-  const dates = [getDateOnly(new Date(fromTime))];
-  for (let i = 1;i <= daysBetween; i += 1) {
-    dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
-  }
-  return dates;
-}
-function dateRange(from, to, step) {
-  const days = daysInRange(from, to);
-  const windows = arrayGroup(days, step);
-  return windows.map((w) => [w[0], w[w.length - 1]]);
+function getDeployBin(bin) {
+  return bin.endsWith(".ts") ? `bun ${bin}` : bin;
 }
 // src/indexer.ts
-import meow2 from "meow";
+import meow from "meow";
 var STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
 async function getIndexerLatestId(name, selectorFlags) {
   const record = await getRecordByKey(STATE_TABLE_NAME, {
@@ -884,6 +757,7 @@ function createModeIndexerApp({
     inline.log(`[MODE INDEXER] mode=${mode}, env=${getEnvironment()}, ${toSelectorString(selectorFlags, ", ")}`);
     if (mode === "reset") {
       await runReset(selectorFlags);
+      process.exit(0);
     } else if (mode === "rebuild") {
       const rebuildFrom = from || await finalState.getMinId(selectorFlags);
       const rebuildTo = to || await finalState.getMaxId(selectorFlags);
@@ -995,6 +869,7 @@ function createModeIndexerApp({
       }
     }
     inline.log(`[MODE INDEXER] validated ${offsetRange(from, to)} ${finalState.type} successfully in ${Date.now() - startTime}ms`);
+    process.exit(0);
   }
   async function execBuild(selectorFlags, from, to, verbose, shouldSaveState = false) {
     let failedIds = [];
@@ -1070,7 +945,7 @@ function createModeIndexerApp({
     const startTime = Date.now();
     if (to < from) {
       inline.log(`[MODE INDEXER] skip delta, there're no more items need to be processed, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}`);
-      return;
+      process.exit(0);
     }
     inline.log(`[MODE INDEXER] starting delta, from=${from}, to=${to}, ${toSelectorString(selectorFlags, ", ")}, batchSize=${buildBatchSize}, concurrency=${buildConcurrency}`);
     try {
@@ -1125,7 +1000,7 @@ function createModeIndexerApp({
     if (!binaryName) {
       binaryName = getBinaryName();
     }
-    const cli = meow2(`
+    const cli = meow(`
         Usage
 
           $ ${binaryName} <options>
@@ -1169,6 +1044,8 @@ ${selector ? getSelectorDesc(selector) : ""}
     } catch (err) {
       inline.error(err);
       process.exit(1);
+    } finally {
+      destroyDynamoDB();
     }
   }
   return { run };
@@ -1183,7 +1060,7 @@ function createIndexerApp({
     if (!binaryName) {
       binaryName = getBinaryName();
     }
-    const cli = meow2(`
+    const cli = meow(`
         Usage
           $ ${binaryName} <options>
 
@@ -1228,6 +1105,7 @@ ${selector ? getSelectorDesc(selector) : ""}
         throw new Error(`[INDEXER] Build failed due to critical errors`);
       }
       inline.log(`[INDEXER] build successfully in ${Date.now() - startTime}ms`);
+      process.exit(0);
     }
     return runBuild(cli.flags).catch((err) => {
       inline.error(err);
@@ -1240,7 +1118,7 @@ ${selector ? getSelectorDesc(selector) : ""}
 import fs2 from "fs/promises";
 import fso from "fs";
 import { execa } from "execa";
-import meow3 from "meow";
+import meow2 from "meow";
 import chalk from "chalk";
 import which from "which";
 var INTERVAL_ALIASES = {
@@ -1547,7 +1425,7 @@ function createModeDeploy({
     if (!binaryName) {
       binaryName = getBinaryName();
     }
-    const cli = meow3(`
+    const cli = meow2(`
         Usage
 
           $ ${binaryName} <options>
@@ -1677,7 +1555,7 @@ function createDeploy({
     if (!binaryName) {
       binaryName = getBinaryName();
     }
-    const cli = meow3(`
+    const cli = meow2(`
         Usage
 
           $ ${binaryName} <options>
@@ -1725,6 +1603,150 @@ ${getSelectorDesc(selector)}
   }
   return { deploy };
 }
+// src/api.ts
+import osModule from "os";
+import express from "express";
+import meow3 from "meow";
+async function logStartMiddleware(_, res, next) {
+  const start = new Date;
+  res.set("x-requested-at", start.toISOString());
+  next();
+}
+async function contextMiddleware(_, res, next) {
+  res.set("x-instance-id", osModule.hostname());
+  next();
+}
+async function logEndMiddleware(req, res, next) {
+  const requestedAt = res.get("x-requested-at");
+  if (!requestedAt) {
+    inline.log("missing x-requested-at header");
+    next();
+    return;
+  }
+  const start = new Date(requestedAt).getTime();
+  const end = new Date().getTime();
+  const logInfo = {
+    start,
+    end,
+    elapsed: `${end - start}ms`,
+    endpoint: req.path,
+    host: req.hostname,
+    status: res.statusCode
+  };
+  if (res.statusMessage) {
+    logInfo.errorMessage = res.statusMessage;
+  }
+  inline.log(JSON.stringify(logInfo));
+  next();
+}
+var apiKeyMiddleware = (key) => {
+  async function requireAPIKey(req, res, next) {
+    try {
+      const apiKey = req.get("x-api-key") || req.query["api-key"];
+      if (!apiKey) {
+        inline.log("request without api key");
+        res.status(400).send("require x-api-key header");
+        return;
+      }
+      if (typeof key === "string") {
+        if (apiKey !== key) {
+          inline.log("request has an invalid api key");
+          res.status(400).send("invalid api key");
+          return;
+        }
+        inline.log(`requested by valid key ${key.slice(0, 6)}`);
+      } else {
+        const name = key[apiKey];
+        if (!name) {
+          inline.log("request has an invalid api key");
+          res.status(400).send("invalid api key");
+          return;
+        }
+        inline.log(`requested by authorized user ${name}`);
+      }
+      next();
+    } catch (err) {
+      inline.log("check api key error", err);
+      res.status(500).send("internal error");
+    }
+  }
+  return requireAPIKey;
+};
+async function startApiApp({
+  binaryName,
+  name,
+  selector = {},
+  routes,
+  serve,
+  beforeListen
+}) {
+  const app = express();
+  app.use(express.json({ limit: "20mb" }));
+  const cli = meow3(`
+        Usage
+          $ ${binaryName} <options>
+
+        Options
+${getSelectorDesc(selector)}
+          --verbose       Output debug messages
+      `, {
+    importMeta: import.meta,
+    description: false,
+    flags: {
+      ...getSelectorFlags(selector),
+      verbose: {
+        type: "boolean",
+        default: false
+      }
+    }
+  });
+  const { verbose, ...selectorFlags } = cli.flags;
+  for (const route of routes) {
+    const method = route.method ? route.method.toLowerCase() : "get";
+    const middlewares = route.middlewares || [];
+    if (route.protected) {
+      if (!serve.apiKey) {
+        throw new Error("serve.apiKey is required for protected route");
+      }
+      middlewares.unshift(apiKeyMiddleware(serve.apiKey));
+    }
+    if (app[method]) {
+      if (verbose) {
+        inline.log(`registering ${method} ${route.path}`);
+      }
+      app[method](route.path, contextMiddleware, logStartMiddleware, ...middlewares, async (req, res, next) => {
+        try {
+          await route.handler({ req, res, ...selectorFlags });
+        } catch (routeErr) {
+          if (routeErr instanceof Error) {
+            inline.log("caught route err", routeErr, routeErr.stack);
+            res.status(500).send(`internal server error: ${routeErr.message}`);
+          } else {
+            inline.log("caught route err", routeErr);
+            res.status(500).send("internal server error");
+          }
+        }
+        next();
+      }, logEndMiddleware);
+    }
+  }
+  if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
+    app.get("/", (_, res) => {
+      res.send("ok");
+    });
+  }
+  if (beforeListen) {
+    await beforeListen({ app });
+  }
+  app.listen(serve.port, () => {
+    if (isProduction()) {
+      inline.log(`${name} listening at https://api.wf.corp.certik.com${serve.prefix}`);
+    } else {
+      inline.log(`${name} listening at http://localhost:${serve.port}`);
+    }
+  });
+}
+
 // src/app.ts
 import { EOL } from "os";
 function printAppHelp() {
@@ -1834,12 +1856,13 @@ function indexer({
     },
     onDeploy: () => {
       const bin = detectBin();
+      const runBin = getDeployBin(bin);
       const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
       const { deploy } = createDeploy({
         binaryName: `${getBinaryName()} deploy`,
         name,
         workingDirectory: detectWorkingDirectory(),
-        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
+        bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
         selector,
         region,
         env,
@@ -1917,12 +1940,13 @@ function modeIndexer({
     },
     onDeploy: () => {
       const bin = detectBin();
+      const runBin = getDeployBin(bin);
       const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
       const { deploy } = createModeDeploy({
         binaryName: `${getBinaryName()} deploy`,
         name,
         workingDirectory: detectWorkingDirectory(),
-        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
+        bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
         selector,
         region,
         env,
@@ -2012,12 +2036,13 @@ function api({
     },
     onDeploy: () => {
       const bin = detectBin();
+      const runBin = getDeployBin(bin);
       const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
       const { deploy } = createDeploy({
         binaryName: `${getBinaryName()} deploy`,
         name,
         workingDirectory: detectWorkingDirectory(),
-        bin: needDoppler ? `doppler run -- ${bin} run` : `${bin} run`,
+        bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
         selector,
         region,
         env,