@certik/skynet 0.22.1 → 0.22.2

package/dist/databricks.js

@@ -1,68 +1,208 @@
-import { DBSQLClient, DBSQLParameter } from "@databricks/sql";
-import { getEnvOrThrow } from "./env.js";
-import { exponentialRetry } from "./availability.js";
-let client = null;
-let session = null;
-async function initSession() {
-    if (!client) {
-        client = new DBSQLClient();
-        await client.connect({
-            authType: "access-token",
-            token: getEnvOrThrow("SKYNET_DATABRICKS_TOKEN"),
-            host: getEnvOrThrow("SKYNET_DATABRICKS_SERVER_HOSTNAME"),
-            path: getEnvOrThrow("SKYNET_DATABRICKS_HTTP_PATH"),
-        });
-    }
-    if (!session) {
-        session = await client.openSession();
+// src/env.ts
+function ensureAndGet(envName, defaultValue) {
+  return process.env[envName] || defaultValue;
+}
+function getEnvironment() {
+  return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
+}
+function getEnvOrThrow(envName) {
+  if (!process.env[envName]) {
+    throw new Error(`Must set environment variable ${envName}`);
+  }
+  return process.env[envName];
+}
+function isProduction() {
+  return getEnvironment() === "prd";
+}
+function isDev() {
+  return getEnvironment() === "dev";
+}
+// src/object-hash.ts
+import xh from "@node-rs/xxhash";
+function getHash(obj) {
+  const xxh3 = xh.xxh3.Xxh3.withSeed();
+  hash(obj, xxh3);
+  return xxh3.digest().toString(16);
+}
+function hash(obj, xxh3) {
+  if (obj === null) {
+    xxh3.update("null");
+  } else if (obj === undefined) {
+    xxh3.update("undefined");
+  } else if (typeof obj === "string") {
+    xxh3.update(obj);
+  } else if (typeof obj === "number") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "boolean") {
+    xxh3.update(obj.toString());
+  } else if (typeof obj === "bigint") {
+    xxh3.update(obj.toString());
+  } else if (obj instanceof Date) {
+    xxh3.update(obj.toISOString());
+  } else if (Array.isArray(obj)) {
+    arrayHash(obj, xxh3);
+  } else if (obj instanceof Set) {
+    setHash(obj, xxh3);
+  } else if (obj instanceof Map) {
+    mapHash(obj, xxh3);
+  } else if (typeof obj === "object") {
+    objectHash(obj, xxh3);
+  } else {
+    throw new Error(`Unsupported type: ${obj}`);
+  }
+}
+function arrayHash(array, xxh3) {
+  xxh3.update("[");
+  for (const obj of array) {
+    hash(obj, xxh3);
+  }
+  xxh3.update("]");
+}
+function setHash(_set, _xxh3) {
+  throw new Error("Set hashing not implemented");
+}
+function mapHash(map, xxh3) {
+  const array = Array.from(map.entries()).sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}
+function objectHash(obj, xxh3) {
+  const array = Object.entries(obj).sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
+  for (const [key, value] of array) {
+    hash(key, xxh3);
+    hash(value, xxh3);
+  }
+}
+
+// src/availability.ts
+import pThrottle from "p-throttle";
+import pMemoize from "p-memoize";
+import QuickLRU from "quick-lru";
+async function wait(time) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, time);
+  });
+}
+async function exponentialRetry(func, {
+  maxRetry,
+  initialDuration,
+  growFactor,
+  test,
+  verbose
+}) {
+  let retries = maxRetry;
+  let duration = initialDuration || 5000;
+  const growFactorFinal = growFactor || 2;
+  let result = await func();
+  while (!test(result) && retries > 0) {
+    if (verbose) {
+      console.log("failed attempt result", result);
+      console.log(`sleep for ${duration}ms after failed attempt, remaining ${retries} attempts`);
     }
+    retries = retries - 1;
+    await wait(duration);
+    result = await func();
+    duration = duration * growFactorFinal;
+  }
+  if (verbose) {
+    console.log(`function to retry ends with status ${test(result)}, number of retries done: ${maxRetry - retries}}`);
+  }
+  return result;
 }
-async function executeSql(sql, bindings) {
-    // retry sql query to avoid socket hang up error
-    const results = await exponentialRetry(async () => {
-        let queryOperation;
-        try {
-            await initSession();
-            if (!session)
-                throw new Error("Session is not initialized");
-            queryOperation = await session.executeStatement(sql, {
-                runAsync: true,
-                namedParameters: objToParameter(bindings),
-            });
-            const result = await queryOperation.fetchAll();
-            return result;
+function withRetry(func, options) {
+  let retries = options?.maxRetry || 3;
+  let duration = options?.initialDuration || 500;
+  const growFactorFinal = options?.growFactor || 2;
+  return async (...args) => {
+    do {
+      try {
+        return await func(...args);
+      } catch (error) {
+        retries = retries - 1;
+        if (retries <= 0) {
+          throw error;
         }
-        catch (err) {
-            // If the error is not a Databricks error, throw it
-            if (!checkIsDatabricksError(err)) {
-                throw err;
-            }
-            return err;
-        }
-        finally {
-            await queryOperation?.close();
-        }
-    }, {
-        maxRetry: 3,
-        initialDuration: 500,
-        test: (result) => {
-            return !checkIsDatabricksError(result);
-        },
+        await wait(duration);
+        duration = duration * growFactorFinal;
+      }
+    } while (retries > 0);
+    throw new Error("unreachable");
+  };
+}
+function memoize(func, options) {
+  if (!options) {
+    options = {};
+  }
+  if (!options.cache) {
+    options.cache = new QuickLRU({ maxSize: options.lruMaxSize || 1e4 });
+  }
+  if (!options.cacheKey) {
+    options.cacheKey = (args) => getHash(args);
+  }
+  return pMemoize(func, options);
+}
+// src/databricks.ts
+import { DBSQLClient, DBSQLParameter } from "@databricks/sql";
+var client = null;
+var session = null;
+async function initSession() {
+  if (!client) {
+    client = new DBSQLClient;
+    await client.connect({
+      authType: "access-token",
+      token: getEnvOrThrow("SKYNET_DATABRICKS_TOKEN"),
+      host: getEnvOrThrow("SKYNET_DATABRICKS_SERVER_HOSTNAME"),
+      path: getEnvOrThrow("SKYNET_DATABRICKS_HTTP_PATH")
     });
-    if (checkIsDatabricksError(results)) {
-        // still error after retry, throw it
-        throw results;
+  }
+  if (!session) {
+    session = await client.openSession();
+  }
+}
+async function executeSql(sql, bindings) {
+  const results = await exponentialRetry(async () => {
+    let queryOperation;
+    try {
+      await initSession();
+      if (!session)
+        throw new Error("Session is not initialized");
+      queryOperation = await session.executeStatement(sql, {
+        runAsync: true,
+        namedParameters: objToParameter(bindings)
+      });
+      const result = await queryOperation.fetchAll();
+      return result;
+    } catch (err) {
+      if (!checkIsDatabricksError(err)) {
+        throw err;
+      }
+      return err;
+    } finally {
+      await queryOperation?.close();
+    }
+  }, {
+    maxRetry: 3,
+    initialDuration: 500,
+    test: (result) => {
+      return !checkIsDatabricksError(result);
     }
-    return results;
+  });
+  if (checkIsDatabricksError(results)) {
+    throw results;
+  }
+  return results;
 }
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
 function checkIsDatabricksError(err) {
-    return err?.errno === "ECONNRESET" || err?.message?.includes("java.lang.NullPointerException");
+  return err?.errno === "ECONNRESET" || err?.message?.includes("java.lang.NullPointerException");
 }
 function objToParameter(obj) {
-    return Object.entries(obj).reduce((acc, [key, value]) => {
-        acc[key] = new DBSQLParameter({ value });
-        return acc;
-    }, {});
+  return Object.entries(obj).reduce((acc, [key, value]) => {
+    acc[key] = new DBSQLParameter({ value });
+    return acc;
+  }, {});
 }
-export { executeSql };
+export {
+  executeSql
+};

package/dist/date.d.ts

@@ -3,4 +3,3 @@ declare function findDateAfter(date: number | string | Date, n: number): string;
 declare function daysInRange(from: number | string | Date, to: number | string | Date): string[];
 declare function dateRange(from: number | string | Date, to: number | string | Date, step: number): [string, string][];
 export { getDateOnly, findDateAfter, daysInRange, dateRange };
-//# sourceMappingURL=date.d.ts.map

package/dist/date.js

@@ -1,29 +1,56 @@
-import { arrayGroup } from "./util";
-const MS_IN_A_DAY = 3600 * 24 * 1000;
+// src/util.ts
+function range(startAt, endAt, step) {
+  const arr = [];
+  for (let i = startAt;i <= endAt; i += step) {
+    arr.push([i, Math.min(endAt, i + step - 1)]);
+  }
+  return arr;
+}
+function arrayGroup(array, groupSize) {
+  const groups = [];
+  for (let i = 0;i < array.length; i += groupSize) {
+    groups.push(array.slice(i, i + groupSize));
+  }
+  return groups;
+}
+function fillRange(start, end) {
+  const result = [];
+  for (let i = start;i <= end; i++) {
+    result.push(i);
+  }
+  return result;
+}
+// src/date.ts
+var MS_IN_A_DAY = 3600 * 24 * 1000;
 function getDateOnly(date) {
-    return new Date(date).toISOString().split("T")[0];
+  return new Date(date).toISOString().split("T")[0];
 }
 function findDateAfter(date, n) {
-    const d = new Date(date);
-    const after = new Date(d.getTime() + MS_IN_A_DAY * n);
-    return getDateOnly(after);
+  const d = new Date(date);
+  const after = new Date(d.getTime() + MS_IN_A_DAY * n);
+  return getDateOnly(after);
 }
 function daysInRange(from, to) {
-    const fromTime = new Date(from).getTime();
-    const toTime = new Date(to).getTime();
-    if (fromTime > toTime) {
-        throw new Error(`range to date couldn't be earlier than range from date`);
-    }
-    const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
-    const dates = [getDateOnly(new Date(fromTime))];
-    for (let i = 1; i <= daysBetween; i += 1) {
-        dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
-    }
-    return dates;
+  const fromTime = new Date(from).getTime();
+  const toTime = new Date(to).getTime();
+  if (fromTime > toTime) {
+    throw new Error(`range to date couldn't be earlier than range from date`);
+  }
+  const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
+  const dates = [getDateOnly(new Date(fromTime))];
+  for (let i = 1;i <= daysBetween; i += 1) {
+    dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
+  }
+  return dates;
 }
 function dateRange(from, to, step) {
-    const days = daysInRange(from, to);
-    const windows = arrayGroup(days, step);
-    return windows.map((w) => [w[0], w[w.length - 1]]);
+  const days = daysInRange(from, to);
+  const windows = arrayGroup(days, step);
+  return windows.map((w) => [w[0], w[w.length - 1]]);
 }
-export { getDateOnly, findDateAfter, daysInRange, dateRange };
+export {
+  getDateOnly,
+  findDateAfter,
+  daysInRange,
+  dateRange
+};

package/dist/deploy.d.ts

@@ -73,4 +73,3 @@ declare function createDeploy({ binaryName, name, workingDirectory, bin, selecto
     deploy: () => Promise<void>;
 };
 export { getJobName, getNomadAddr, createModeDeploy, createDeploy };
-//# sourceMappingURL=deploy.d.ts.map