@autonomys/asynchronous 1.4.0

package/README.md

@@ -0,0 +1,5 @@
+# @autonomys/asynchronous
+
+## Description
+
+A toolbox library for managing asyncronous problems such as concurrency or async generator functions.

package/__test__/weightedRequestsConcurrencyController.spec.ts

@@ -0,0 +1,94 @@
+import { weightedRequestConcurrencyController } from '../src/concurrency/weightedRequestsConcurrencyController'
+
+const mockRequest = async (id: number) => {
+  return new Promise((resolve) => {
+    setTimeout(() => {
+      resolve(`Response from request ${id}`)
+    }, 100)
+  })
+}
+
+const mockTimestampedRequest = async () => {
+  const timestamp = Date.now()
+  return new Promise<number>((resolve) => {
+    setTimeout(() => {
+      resolve(timestamp)
+    }, 100)
+  })
+}
+
+describe('MultirequestAsyncController', () => {
+  it('should handle multiple requests resolving in order', async () => {
+    const maxConcurrency = 2
+    const controller = weightedRequestConcurrencyController(maxConcurrency)
+    const requests = Array.from({ length: 5 }, (_, i) => () => mockRequest(i + 1))
+
+    const results = await Promise.all(requests.map((request) => controller(request, 1)))
+
+    expect(results).toEqual([
+      'Response from request 1',
+      'Response from request 2',
+      'Response from request 3',
+      'Response from request 4',
+      'Response from request 5',
+    ])
+  })
+
+  it('should handle multiple requests applying without exceeding max concurrency', async () => {
+    let counter = 0
+    const maxConcurrency = 3
+    const requests = Array.from({ length: 5 }, () => async () => {
+      if (counter > maxConcurrency) throw new Error('Too many requests')
+      counter++
+      await mockRequest(counter)
+      counter--
+    })
+
+    const controller = weightedRequestConcurrencyController(maxConcurrency)
+    await expect(
+      Promise.all(requests.map((request) => controller(request, 1))),
+    ).resolves.not.toThrow()
+  })
+
+  it('should handle multiple requests applying without exceeding max concurrency (different weights)', async () => {
+    const maxConcurrency = 10
+    let counter = 0
+    const requests = Array.from({ length: 10 }, (_, i) => async () => {
+      if (counter > maxConcurrency) throw new Error('Too many requests')
+      counter += i
+      await mockRequest(counter)
+      counter -= i
+    })
+
+    const controller = weightedRequestConcurrencyController(maxConcurrency)
+    await expect(
+      Promise.all(requests.map((request, i) => controller(request, i))),
+    ).resolves.not.toThrow()
+  })
+
+  it('should handle light weight requests', async () => {
+    const maxConcurrency = 4
+    const weights = [3, 3, 3, 1]
+    const requests = weights.map(() => mockTimestampedRequest)
+
+    const controller = weightedRequestConcurrencyController(maxConcurrency)
+    const results = await Promise.all(requests.map((request, i) => controller(request, weights[i])))
+
+    expect(results[0]).toBeLessThanOrEqual(results[3])
+    expect(results[3]).toBeLessThanOrEqual(results[1])
+    expect(results[1]).toBeLessThanOrEqual(results[2])
+  })
+
+  it('should respect order of execution (using ensureOrder flag)', async () => {
+    const maxConcurrency = 4
+    const weights = [3, 3, 3, 1]
+    const requests = weights.map(() => mockTimestampedRequest)
+
+    const controller = weightedRequestConcurrencyController(maxConcurrency, true)
+    const results = await Promise.all(requests.map((request, i) => controller(request, weights[i])))
+
+    for (let i = 0; i < results.length - 1; i++) {
+      expect(results[i]).toBeLessThanOrEqual(results[i + 1])
+    }
+  })
+})

package/jest.config.js

@@ -0,0 +1,5 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@autonomys/asynchronous",
+  "packageManager": "yarn@4.7.0",
+  "version": "1.4.0",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/autonomys/auto-sdk"
+  },
+  "author": {
+    "name": "Autonomys",
+    "url": "https://www.autonomys.xyz"
+  },
+  "bugs": {
+    "url": "https://github.com/autonomys/auto-sdk/issues"
+  },
+  "scripts": {
+    "build": "yarn tsc",
+    "test": "jest"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "eslint": "^9.21.0",
+    "interface-store": "^6.0.2",
+    "jest": "^29.7.0",
+    "prettier": "^3.2.5",
+    "ts-jest": "^29.1.4",
+    "typescript": "^5.4.5"
+  },
+  "gitHead": "53cf74430ba08ed3b8f38ccc2fcee7b373a7934c"
+}

package/src/asyncGenerators/index.ts

@@ -0,0 +1,116 @@
+import { PassThrough, Readable } from 'stream'
+
+export type AwaitIterable<T> = AsyncIterable<T> | Iterable<T>
+
+export const asyncIterableMap = async <T, R>(
+  iterable: AwaitIterable<T>,
+  fn: (value: T) => Promise<R>,
+): Promise<R[]> => {
+  const result = []
+  for await (const value of iterable) {
+    result.push(await fn(value))
+  }
+  return result
+}
+
+export const asyncIterableForEach = async <T>(
+  iterable: AwaitIterable<T>,
+  fn: (value: T[]) => Promise<void>,
+  concurrency: number,
+): Promise<void> => {
+  let batch: T[] = []
+  for await (const value of iterable) {
+    batch.push(value)
+    if (batch.length === concurrency) {
+      await fn(batch)
+      batch = []
+    }
+  }
+
+  if (batch.length > 0) {
+    await fn(batch)
+  }
+}
+
+export const asyncIterableToPromiseOfArray = async <T>(
+  iterable: AwaitIterable<T>,
+): Promise<T[]> => {
+  const result = []
+  for await (const value of iterable) {
+    result.push(value)
+  }
+  return result
+}
+
+export const bufferToAsyncIterable = (buffer: Buffer): AsyncIterable<Buffer> => {
+  return (async function* () {
+    yield buffer
+  })()
+}
+
+export const asyncIterableToBuffer = async (iterable: AwaitIterable<Buffer>): Promise<Buffer> => {
+  let buffer = Buffer.alloc(0)
+  for await (const chunk of iterable) {
+    buffer = Buffer.concat([buffer, chunk])
+  }
+  return buffer
+}
+
+export const asyncByChunk = async function* (
+  iterable: AwaitIterable<Buffer>,
+  chunkSize: number,
+  ignoreLastChunk: boolean = false,
+): AsyncIterable<Buffer> {
+  let accumulated = Buffer.alloc(0)
+  for await (const chunk of iterable) {
+    accumulated = Buffer.concat([accumulated, chunk])
+    while (accumulated.length >= chunkSize) {
+      yield accumulated.subarray(0, chunkSize)
+      accumulated = accumulated.subarray(chunkSize)
+    }
+  }
+
+  if (accumulated.length > 0 && !ignoreLastChunk) {
+    yield accumulated
+  }
+}
+
+export async function forkAsyncIterable(
+  asyncIterable: AwaitIterable<Uint8Array>,
+): Promise<[Readable, Readable]> {
+  const passThrough1 = new PassThrough()
+  const passThrough2 = new PassThrough()
+
+  for await (const chunk of asyncIterable) {
+    passThrough1.write(chunk)
+    passThrough2.write(chunk)
+  }
+  passThrough1.end()
+  passThrough2.end()
+
+  return [passThrough1, passThrough2]
+}
+
+export const asyncFromStream = async function* (
+  stream: ReadableStream<Uint8Array>,
+): AsyncIterable<Buffer> {
+  const reader = stream.getReader()
+  let result = await reader.read()
+  while (!result.done) {
+    yield Buffer.from(result.value)
+    result = await reader.read()
+  }
+}
+
+export const bufferToIterable = async function* (buffer: Buffer): AsyncIterable<Buffer> {
+  yield buffer
+}
+
+export const fileToIterable = async function* (
+  file: File | Blob,
+  chunkSize: number = 1024 * 1024,
+): AsyncIterable<Buffer> {
+  for (let i = 0; i < file.size; i += chunkSize) {
+    yield Buffer.from(await file.slice(i, i + chunkSize).arrayBuffer())
+  }
+}

package/src/concurrency/index.ts

@@ -0,0 +1 @@
+export * from './weightedRequestsConcurrencyController.js'

package/src/concurrency/weightedRequestsConcurrencyController.ts

@@ -0,0 +1,66 @@
+export type Job<T> = () => Promise<T>
+
+type ConcurrencyController = <T>(job: Job<T>, concurrency: number) => Promise<T>
+
+interface ScheduleTask<T> {
+  job: Job<T>
+  concurrency: number
+}
+
+export const weightedRequestConcurrencyController = (
+  maxConcurrency: number,
+  ensureOrder: boolean = false,
+): ConcurrencyController => {
+  if (maxConcurrency <= 0) {
+    throw new Error('Max concurrency must be greater than 0')
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const queue: ScheduleTask<any>[] = []
+  let active = 0
+
+  const enqueue = <T>(task: Job<T>, concurrency: number): Promise<T> => {
+    return new Promise<T>((resolve, reject) => {
+      queue.push({
+        job: () => task().then(resolve).catch(reject),
+        concurrency,
+      })
+    })
+  }
+
+  const findExecutableTask = () => {
+    const index = queue.findIndex((e) => active + e.concurrency <= maxConcurrency)
+    const notMatched = index === -1
+    const notFirst = ensureOrder && index !== 0
+    if (notMatched || notFirst) {
+      return null
+    }
+
+    return queue.splice(index, 1).at(0)
+  }
+
+  const manageJobFinalization = async (concurrency: number) => {
+    active -= concurrency
+    const executableTask = findExecutableTask()
+    if (executableTask) {
+      setImmediate(() => internalRunJob(executableTask.job, executableTask.concurrency))
+    }
+  }
+
+  const internalRunJob = async <T>(job: Job<T>, concurrency: number): Promise<T> => {
+    active += concurrency
+    return job().finally(() => manageJobFinalization(concurrency))
+  }
+
+  const runJob = async <T>(job: Job<T>, concurrency: number): Promise<T> => {
+    const exceededMaxConcurrency = active + concurrency > maxConcurrency
+    const shouldRespectOrder = ensureOrder && queue.length > 0
+    if (exceededMaxConcurrency || shouldRespectOrder) {
+      return enqueue(job, concurrency)
+    }
+
+    return internalRunJob(job, concurrency)
+  }
+
+  return runJob
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from './asyncGenerators'
+export * from './concurrency'

package/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "resolveJsonModule": true
+  },
+  "include": ["src"]
+}