@tim-code/my-util 0.3.0 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tim-code/my-util",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "",
   "type": "module",
   "author": "Tim Sprowl",

package/src/fs.js

@@ -1,47 +1,40 @@
 import { readFile, stat, writeFile } from "node:fs/promises"
 import { tmpdir } from "node:os"
 import { promisify } from "node:util"
-import { gunzip as _gunzip } from "node:zlib"
+import { gunzip as _gunzip, gzip as _gzip } from "node:zlib"
 
 const gunzip = promisify(_gunzip)
+const gzip = promisify(_gzip)
 
 // the integration tests for this file are the s3-fs integration tests in lambda-integrations
 // changes to this file should be tested against lambda-integrations' integration tests
 
 /**
- * Get JSON from a path.
+ * Get JSON from a path. If path ends with .gz, will automatically decompress data.
  * @param {string} path
  * @returns {Object|Array}
  */
-export async function getJSON(path) {
-  const buffer = await readFile(path)
+export async function readJSON(path) {
+  let buffer = await readFile(path)
+  if (path.endsWith(".gz")) {
+    buffer = await gunzip(buffer)
+  }
   return JSON.parse(buffer.toString())
 }
 
 /**
- * Write JSON to a path.
+ * Write JSON to a path. If path ends with .gz, will automatically compress data.
  * @param {string} path
  * @param {Object|Array} object
  * @param {Object} $1
  * @param {number=} $1.indent Indent used to format JSON object. Default 2. If 0, does not indent object.
  */
 export async function writeJSON(path, object, { indent = 2 } = {}) {
-  const string = JSON.stringify(object, undefined, indent)
-  await writeFile(path, string)
-}
-
-/**
- * Get gzipped JSON from a path.
- * @param {string} path
- * @returns {Object|Array}
- */
-export async function getCompressedJSON(path) {
-  if (!path.endsWith(".gz")) {
-    throw new Error("path does not suggest a compressed file")
+  let data = JSON.stringify(object, undefined, indent)
+  if (path.endsWith(".gz")) {
+    data = await gzip(data)
   }
-  const buffer = await readFile(path)
-  const uncompressed = await gunzip(buffer)
-  return JSON.parse(uncompressed.toString())
+  await writeFile(path, data)
 }
 
 /**

package/src/fs.test.js

@@ -4,6 +4,7 @@ const readFileMock = jest.fn()
 const statMock = jest.fn()
 const tmpdirMock = jest.fn()
 const gunzipMock = jest.fn()
+const gzipMock = jest.fn()
 const writeFileMock = jest.fn()
 jest.unstable_mockModule("node:fs/promises", () => ({
   readFile: readFileMock,
@@ -15,6 +16,7 @@ jest.unstable_mockModule("node:os", () => ({
 }))
 jest.unstable_mockModule("node:zlib", () => ({
   gunzip: gunzipMock,
+  gzip: gzipMock,
 }))
 jest.unstable_mockModule("node:util", () => ({
   promisify: (mock) => mock,
@@ -22,16 +24,24 @@ jest.unstable_mockModule("node:util", () => ({
 
 // Now import the module under test
 const mod = await import("./fs.js")
-const { getJSON, writeJSON, getCompressedJSON, pathExists, makeTempDirectory } = mod
+const { readJSON, writeJSON, pathExists, makeTempDirectory } = mod
 
-describe("getJSON", () => {
+describe("readJSON", () => {
   beforeEach(() => jest.clearAllMocks())
   it("parses JSON from file", async () => {
     readFileMock.mockResolvedValue(Buffer.from('{"a":1}'))
-    const result = await getJSON("foo.json")
+    const result = await readJSON("foo.json")
     expect(result).toEqual({ a: 1 })
     expect(readFileMock).toHaveBeenCalledWith("foo.json")
   })
+  it("decompresses and parses JSON from .gz file", async () => {
+    readFileMock.mockResolvedValue(Buffer.from("gzipped"))
+    gunzipMock.mockResolvedValue(Buffer.from('{"b":2}'))
+    const result = await readJSON("bar.gz")
+    expect(result).toEqual({ b: 2 })
+    expect(readFileMock).toHaveBeenCalledWith("bar.gz")
+    expect(gunzipMock).toHaveBeenCalledWith(Buffer.from("gzipped"))
+  })
 })
 
 describe("writeJSON", () => {
@@ -55,20 +65,18 @@ describe("writeJSON", () => {
     await writeJSON("foo.json", { x: 2 }, { indent: 0 })
     expect(writeFileMock).toHaveBeenCalledWith("foo.json", '{"x":2}')
   })
-})
-
-describe("getCompressedJSON", () => {
-  beforeEach(() => jest.clearAllMocks())
-  it("reads, decompresses, and parses JSON", async () => {
-    readFileMock.mockResolvedValue(Buffer.from("gzipped"))
-    gunzipMock.mockResolvedValue(Buffer.from('{"b":2}'))
-    const result = await getCompressedJSON("bar.gz")
-    expect(result).toEqual({ b: 2 })
-    expect(readFileMock).toHaveBeenCalledWith("bar.gz")
-    expect(gunzipMock).toHaveBeenCalled()
-  })
-  it("throws for bad filename", async () => {
-    await expect(getCompressedJSON("bar")).rejects.toThrow(/a compressed file/u)
+  it("compresses and writes JSON to .gz file", async () => {
+    gzipMock.mockResolvedValue(Buffer.from("compressed"))
+    await writeJSON("foo.gz", { y: 3 })
+    // Should gzip the JSON string and write the compressed buffer
+    expect(gzipMock).toHaveBeenCalledWith(JSON.stringify({ y: 3 }, undefined, 2))
+    expect(writeFileMock).toHaveBeenCalledWith("foo.gz", Buffer.from("compressed"))
+  })
+  it("compresses and writes JSON with custom indent to .gz file", async () => {
+    gzipMock.mockResolvedValue(Buffer.from("compressed"))
+    await writeJSON("foo.gz", { y: 3 }, { indent: 0 })
+    expect(gzipMock).toHaveBeenCalledWith(JSON.stringify({ y: 3 }, undefined, 0))
+    expect(writeFileMock).toHaveBeenCalledWith("foo.gz", Buffer.from("compressed"))
   })
 })
 

package/src/promise.js

@@ -44,11 +44,14 @@ export function poll({ ms, wait = false, attempts = undefined }, callback) {
 
 /**
  * Sleep for X milliseconds.
- * @param {number} milliseconds
+ * @param {number} ms Milliseconds; returns immediately if negative
  */
-export async function sleep(milliseconds) {
+export async function sleep(ms) {
+  if (ms < 0) {
+    return
+  }
   await new Promise((resolve) => {
-    setTimeout(resolve, milliseconds)
+    setTimeout(resolve, ms)
   })
 }
 
@@ -63,12 +66,14 @@ export async function sleep(milliseconds) {
  *  - "errors": The "reason" property for each "result" object that did not have a status of "fulfilled".
  * @param {Object} $1
  * @param {Array} $1.array
- * @param {number=} $1.limit If not provided, each call is done in parallel.
+ * @param {number=} $1.limit The number of calls to do in parallel. If not provided, each call is done in parallel.
+ * @param {Function=} $1.limiter A function awaited after a group of parallel calls is processed.
+ *  It is called with the number of parallel calls processed. Could be as simple as `() => sleep(10000)` if you wanted to wait 10 seconds between.
  * @param {boolean=} $1.flatten Flattens values before returning; useful if promises return arrays
  * @param {Function} callback
  * @returns {Object} {results, values, returned, errors}
  */
-export async function allSettled({ array, limit, flatten = false }, callback) {
+export async function allSettled({ array, limit, limiter, flatten = false }, callback) {
   const results = []
   let returned = []
   let values = []
@@ -87,6 +92,7 @@ export async function allSettled({ array, limit, flatten = false }, callback) {
         errors.push(reason)
       }
     }
+    await limiter?.(elements.length)
   }
   if (flatten) {
     values = values.flat()
@@ -95,6 +101,28 @@ export async function allSettled({ array, limit, flatten = false }, callback) {
   return { values, returned, errors, results }
 }
 
+/**
+ * Creates a function that can be used with allSettled to limit the number of elements processed in a time interval.
+ * Once the limit is reached, waits until the start of a new interval before returning.
+ * @param {Object} $1
+ * @param {number} $1.limit The maximum number of elements to be processed in the interval
+ * @param {Function=} $1.interval The length of the interval in milliseconds. Default is one minute.
+ * @returns {Function} Returned function expects to be called with the number of elements added since last call.
+ */
+export function intervalLimiter({ limit, interval = 1000 * 60 }) {
+  let count = 0
+  let startTimestamp = Date.now()
+  return async (added) => {
+    count += added
+    if (count >= limit) {
+      const currentTimestamp = Date.now()
+      await sleep(interval - (currentTimestamp - startTimestamp))
+      startTimestamp = Date.now()
+      count = 0
+    }
+  }
+}
+
 /**
  * A convenience method to throw the result of allSettled().
  * Useful in testing contexts when simply propagating the error is enough.
@@ -111,6 +139,7 @@ export function alert(result) {
   return result
 }
 
+// unused but included for reference
 /**
  * Parallelize executions of a function using `Promise.all()`.
  * This is useful because usually you want to set a limit to the number of parallel requests possible at once.

package/src/promise.test.js

@@ -2,7 +2,7 @@
 /* eslint-disable prefer-promise-reject-errors */
 import { jest } from "@jest/globals"
 
-import { alert, allSettled, poll, PollError, sleep, throwFirstReject } from "./promise.js"
+import { alert, allSettled, poll, PollError, sleep, throwFirstReject, intervalLimiter } from "./promise.js"
 
 describe("poll", () => {
   it("resolves immediately if callback returns a non-undefined/null/false value", async () => {
@@ -104,6 +104,14 @@ describe("sleep", () => {
     // Allow for some jitter
     expect(after - before).toBeGreaterThanOrEqual(5)
   })
+
+  it("resolves immediately if ms is negative", async () => {
+    const before = Date.now()
+    const promise = sleep(-10)
+    await expect(promise).resolves.toBeUndefined()
+    const after = Date.now()
+    expect(after - before).toBeLessThan(5)
+  })
 })
 
 describe("allSettled", () => {
@@ -154,6 +162,60 @@ describe("allSettled", () => {
     expect(result.errors).toEqual([])
     expect(result.results).toEqual([])
   })
+
+  it("calls limiter after each chunk if provided", async () => {
+    const arr = [1, 2, 3, 4, 5]
+    const calls = []
+    const limiterCalls = []
+    const cb = (x) => {
+      calls.push(x)
+      return x
+    }
+    const limiter = jest.fn(async (n) => {
+      limiterCalls.push(n)
+      // simulate async delay
+      await sleep(1)
+    })
+    const result = await allSettled({ array: arr, limit: 2, limiter }, cb)
+    expect(result.values).toEqual([1, 2, 3, 4, 5])
+    expect(calls).toEqual([1, 2, 3, 4, 5])
+    expect(limiter).toHaveBeenCalledTimes(3)
+    expect(limiterCalls).toEqual([2, 2, 1])
+  })
+})
+
+describe("intervalLimiter", () => {
+  it("does not delay until limit is reached", async () => {
+    const limiter = intervalLimiter({ limit: 3, interval: 10 })
+    const before = Date.now()
+    await limiter(1)
+    await limiter(1)
+    await limiter(1) // should reach limit here, triggers wait
+    const after = Date.now()
+    expect(after - before).toBeGreaterThanOrEqual(10)
+  })
+
+  it("resets count and interval after waiting", async () => {
+    const limiter = intervalLimiter({ limit: 2, interval: 5 })
+    const before = Date.now()
+    await limiter(1)
+    await limiter(1) // triggers wait
+    const afterFirst = Date.now()
+    await limiter(1)
+    await limiter(1) // triggers wait again
+    const afterSecond = Date.now()
+    expect(afterFirst - before).toBeGreaterThanOrEqual(5)
+    expect(afterSecond - afterFirst).toBeGreaterThanOrEqual(5)
+  })
+
+  it("handles added < limit with no delay", async () => {
+    const limiter = intervalLimiter({ limit: 10, interval: 5 })
+    const before = Date.now()
+    await limiter(3)
+    await limiter(3)
+    const after = Date.now()
+    expect(after - before).toBeLessThan(5)
+  })
 })
 
 describe("alert", () => {