@xylabs/threads 3.0.4

package/test/lib/serialization.ts

@@ -0,0 +1,38 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable import/no-internal-modules */
+import { JsonSerializable, SerializerImplementation } from '../../src/index'
+
+export class Foo<T> {
+  private readonly value: T
+
+  constructor(value: T) {
+    this.value = value
+  }
+
+  getValue() {
+    return this.value
+  }
+}
+
+interface SerializedFoo<T extends JsonSerializable> {
+  __type: '$$foo'
+  val: T
+}
+
+const isSerializedFoo = (thing: any): thing is SerializedFoo<JsonSerializable> =>
+  thing && typeof thing === 'object' && '__type' in thing && thing.__type === '$$foo'
+
+export const fooSerializer: SerializerImplementation = {
+  deserialize(serialized, fallback) {
+    return isSerializedFoo(serialized) ? new Foo(serialized.val) : fallback(serialized)
+  },
+
+  serialize(data, fallback) {
+    return data instanceof Foo ?
+        {
+          __type: '$$foo',
+          val: data.getValue(),
+        }
+      : fallback(data)
+  },
+}

package/test/observable-promise.test.ts

@@ -0,0 +1,189 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable import/no-internal-modules */
+import test from 'ava'
+import { Observable } from 'observable-fns'
+
+import { ObservablePromise } from '../src/observable-promise'
+
+const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
+
+test('can create an observable promise', async (t) => {
+  t.plan(1)
+
+  await new ObservablePromise((observer) => {
+    t.true(true)
+    observer.complete()
+  })
+})
+
+test('init function is only called once', async (t) => {
+  let initCallCount = 0
+
+  const async = new ObservablePromise((observer) => {
+    initCallCount++
+    setTimeout(() => observer.complete(), 10)
+  })
+
+  await Promise.all([async.then(() => t.true(true)), async.then(() => t.true(true)), async.then(() => t.true(true))])
+
+  t.is(initCallCount, 1)
+})
+
+test('can proxy a promise fulfillment', async (t) => {
+  t.plan(2)
+
+  const async = new ObservablePromise((observer) => {
+    setTimeout(() => {
+      observer.next(123)
+
+      // Ignore all values after the first one
+      observer.next(456)
+      observer.complete()
+    }, 1)
+  })
+
+  const promise1 = async.then((value) => t.is(value, 123), t.fail)
+  await delay(10)
+  const promise2 = async.then((value) => t.is(value, 123), t.fail)
+
+  await Promise.all([promise1, promise2])
+})
+
+test('can proxy a promise rejection', async (t) => {
+  let handlerCallCount = 0
+
+  const async = new ObservablePromise((observer) => {
+    setTimeout(() => observer.error(Error('I am supposed to be rejected.')), 1)
+  })
+
+  const promise1 = async.then(
+    () => t.fail('Promise should not become fulfilled'),
+    () => handlerCallCount++,
+  )
+  await delay(10)
+  const promise2 = async.then(
+    () => t.fail('Promise should not become fulfilled'),
+    () => handlerCallCount++,
+  )
+
+  await Promise.all([promise1.catch(() => true), promise2.catch(() => true)])
+  t.is(handlerCallCount, 2)
+})
+
+test('can proxy a promise rejection caused by a sync throw', async (t) => {
+  let handlerCallCount = 0
+
+  const async = new ObservablePromise(() => {
+    throw new Error('I am supposed to be rejected.')
+  })
+
+  const promise1 = async.then(
+    () => t.fail('Promise should not become fulfilled'),
+    () => handlerCallCount++,
+  )
+  await delay(10)
+  const promise2 = async.then(
+    () => t.fail('Promise should not become fulfilled'),
+    () => handlerCallCount++,
+  )
+
+  await Promise.all([promise1, promise2])
+  t.is(handlerCallCount, 2)
+})
+
+test('can subscribe to values and completion', async (t) => {
+  const capturedValues: any[] = []
+  let capturedCompletions = 0
+
+  const async = new ObservablePromise((observer) => {
+    setTimeout(() => observer.next(1), 10)
+    setTimeout(() => observer.next(2), 20)
+    setTimeout(() => observer.complete(), 30)
+  })
+
+  for (let index = 0; index < 2; index++) {
+    async.subscribe(
+      (value) => capturedValues.push(value),
+      () => {},
+      () => capturedCompletions++,
+    )
+  }
+
+  await async.finally()
+  await delay(1)
+
+  t.deepEqual(capturedValues, [1, 1, 2, 2])
+  t.is(capturedCompletions, 2)
+})
+
+test('can subscribe to errors', async (t) => {
+  const capturedErrorMessages: string[] = []
+  const capturedValues: any[] = []
+  let capturedCompletions = 0
+
+  const async = new ObservablePromise((observer) => {
+    setTimeout(() => observer.next(1), 10)
+    setTimeout(() => observer.error(Error('Fails as expected.')), 20)
+    setTimeout(() => observer.next(2), 30)
+    setTimeout(() => observer.complete(), 40)
+  })
+
+  for (let index = 0; index < 2; index++) {
+    async.subscribe(
+      (value) => capturedValues.push(value),
+      (error) => capturedErrorMessages.push(error.message),
+      () => capturedCompletions++,
+    )
+  }
+
+  await async.finally()
+  await delay(35)
+
+  t.deepEqual(capturedValues, [1, 1])
+  t.deepEqual(capturedErrorMessages, ['Fails as expected.', 'Fails as expected.'])
+  t.is(capturedCompletions, 0)
+})
+
+test('from(Observable) works', async (t) => {
+  const capturedErrorMessages: string[] = []
+  const capturedValues: any[] = []
+  let capturedCompletions = 0
+
+  const async = ObservablePromise.from(
+    new Observable((observer) => {
+      setTimeout(() => observer.next(1), 10)
+      setTimeout(() => observer.error(Error('Fails as expected.')), 20)
+      setTimeout(() => observer.next(2), 30)
+      setTimeout(() => observer.complete(), 40)
+    }),
+  )
+
+  for (let index = 0; index < 2; index++) {
+    async.subscribe(
+      (value) => capturedValues.push(value),
+      (error) => capturedErrorMessages.push(error.message),
+      () => capturedCompletions++,
+    )
+  }
+
+  await async.finally()
+  await delay(35)
+
+  t.deepEqual(capturedValues, [1, 1])
+  t.deepEqual(capturedErrorMessages, ['Fails as expected.', 'Fails as expected.'])
+  t.is(capturedCompletions, 0)
+})
+
+test('from(Promise) works', async (t) => {
+  const resolved = ObservablePromise.from(
+    new Promise((resolve) => {
+      setTimeout(() => resolve('Works'), 10)
+    }),
+  )
+  t.is(await resolved, 'Works')
+
+  const rejected = ObservablePromise.from(Promise.reject(Error('Fails')))
+  const error = await t.throwsAsync(rejected)
+
+  t.is(error.message, 'Fails')
+})

package/test/observable.test.ts

@@ -0,0 +1,86 @@
+/* eslint-disable require-await */
+/* eslint-disable import/no-internal-modules */
+import test from 'ava'
+
+import { Observable, Subject } from '../src/observable'
+
+test('Observable subject emits values and completion event', async (t) => {
+  let completed1 = false
+  const values1: number[] = []
+  let completed2 = false
+  const values2: number[] = []
+  let completed3 = false
+  const values3: number[] = []
+
+  const subject = new Subject<number>()
+  const observable = Observable.from(subject)
+
+  const subscription1 = subject.subscribe(
+    (value) => values1.push(value),
+    undefined,
+    () => (completed1 = true),
+  )
+  subject.subscribe(
+    (value) => values2.push(value),
+    undefined,
+    () => (completed2 = true),
+  )
+  observable.subscribe(
+    (value) => values3.push(value),
+    undefined,
+    () => (completed3 = true),
+  )
+
+  subject.next(1)
+  subscription1.unsubscribe()
+
+  subject.next(2)
+  subject.complete()
+
+  t.deepEqual(values1, [1])
+  t.deepEqual(values2, [1, 2])
+  t.deepEqual(values3, [1, 2])
+  t.is(completed1, false)
+  t.is(completed2, true)
+  t.is(completed3, true)
+})
+
+test('Observable subject propagates errors', async (t) => {
+  let completed1 = false
+  let error1: Error | undefined
+  let completed2 = false
+  let error2: Error | undefined
+  let completed3 = false
+  let error3: Error | undefined
+
+  const subject = new Subject<number>()
+  const observable = Observable.from(subject)
+
+  const subscription1 = subject.subscribe(
+    () => {},
+    (error) => (error1 = error),
+    () => (completed1 = true),
+  )
+  subject.subscribe(
+    () => {},
+    (error) => (error2 = error),
+    () => (completed2 = true),
+  )
+  observable.subscribe(
+    () => {},
+    (error) => (error3 = error),
+    () => (completed3 = true),
+  )
+
+  const testingError = Error('Test, test!')
+
+  subscription1.unsubscribe()
+  subject.error(testingError)
+
+  t.is(completed1, false)
+  t.is(error1, undefined)
+  t.is(completed2, false)
+  t.is(error2, testingError)
+  t.is(completed3, false)
+  t.is(error3, testingError)
+})

package/test/pool.test.ts

@@ -0,0 +1,173 @@
+/* eslint-disable require-await */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-floating-promises */
+/* eslint-disable import/no-internal-modules */
+import test from 'ava'
+
+import { Pool, spawn, Worker } from '../src/index'
+import { PoolEventType, QueuedTask } from '../src/master/pool'
+
+test.serial('thread pool basics work and events are emitted', async (t) => {
+  const events: Pool.Event[] = []
+  let spawnCalled = 0
+  let taskFnCalled = 0
+
+  const spawnHelloWorld = () => {
+    spawnCalled++
+    return spawn<() => string>(new Worker('./workers/hello-world'))
+  }
+  const pool = Pool(spawnHelloWorld, 3)
+  pool.events().subscribe((event) => events.push(event))
+
+  // Just to make sure all worker threads are initialized before starting to queue
+  // This is only necessary for testing to make sure that this is the first event recorded
+  await new Promise((resolve, reject) => {
+    pool
+      .events()
+      .filter((event) => event.type === PoolEventType.initialized)
+      .subscribe(resolve, reject)
+  })
+
+  await pool.queue(async (helloWorld) => {
+    taskFnCalled++
+    const result = await helloWorld()
+    t.is(result, 'Hello World')
+    return result
+  })
+
+  await pool.terminate()
+  t.is(spawnCalled, 3)
+  t.is(taskFnCalled, 1)
+
+  t.deepEqual(events, [
+    {
+      size: 3,
+      type: Pool.EventType.initialized,
+    },
+    {
+      taskID: 1,
+      type: Pool.EventType.taskQueued,
+    },
+    {
+      taskID: 1,
+      type: Pool.EventType.taskStart,
+      workerID: 1,
+    },
+    {
+      returnValue: 'Hello World',
+      taskID: 1,
+      type: Pool.EventType.taskCompleted,
+      workerID: 1,
+    },
+    {
+      type: Pool.EventType.taskQueueDrained,
+    },
+    {
+      remainingQueue: [],
+      type: Pool.EventType.terminated,
+    },
+  ])
+})
+
+test.serial('pool.completed() works', async (t) => {
+  const returned: any[] = []
+
+  const spawnHelloWorld = () => spawn(new Worker('./workers/hello-world'))
+  const pool = Pool(spawnHelloWorld, 2)
+
+  for (let i = 0; i < 3; i++) {
+    pool.queue(async (helloWorld) => {
+      returned.push(await helloWorld())
+    })
+  }
+
+  await pool.completed()
+
+  t.deepEqual(returned, ['Hello World', 'Hello World', 'Hello World'])
+})
+
+test.serial('pool.completed() proxies errors', async (t) => {
+  const spawnHelloWorld = () => spawn(new Worker('./workers/hello-world'))
+  const pool = Pool(spawnHelloWorld, 2)
+
+  pool.queue(async () => {
+    throw new Error('Ooopsie')
+  })
+
+  const error = await t.throwsAsync(() => pool.completed())
+  t.is(error.message, 'Ooopsie')
+})
+
+test.serial('pool.completed(true) works', async (t) => {
+  const spawnHelloWorld = () => spawn(new Worker('./workers/hello-world'))
+  const pool = Pool(spawnHelloWorld, 2)
+
+  await pool.completed(true)
+  t.pass()
+})
+
+test.serial('pool.settled() does not reject on task failure', async (t) => {
+  const returned: any[] = []
+
+  const spawnHelloWorld = () => spawn(new Worker('./workers/hello-world'))
+  const pool = Pool(spawnHelloWorld, 2)
+
+  pool.queue(async (helloWorld) => {
+    returned.push(await helloWorld())
+  })
+  pool.queue(async () => {
+    throw new Error('Test error one')
+  })
+  pool.queue(async () => {
+    throw new Error('Test error two')
+  })
+
+  const errors = await pool.settled()
+  t.is(errors.length, 2)
+  t.deepEqual(errors.map((error) => error.message).sort(), ['Test error one', 'Test error two'])
+})
+
+test.serial('pool.settled(true) works', async (t) => {
+  const spawnHelloWorld = () => spawn(new Worker('./workers/hello-world'))
+  const pool = Pool(spawnHelloWorld, 2)
+
+  await pool.settled(true)
+  t.pass()
+})
+
+test.serial('task.cancel() works', async (t) => {
+  const events: Pool.Event[] = []
+  const spawnHelloWorld = () => spawn(new Worker('./workers/hello-world'))
+  const pool = Pool(spawnHelloWorld, 1)
+
+  pool.events().subscribe((event) => events.push(event))
+
+  let executionCount = 0
+  const tasks: QueuedTask<any, any>[] = []
+
+  for (let i = 0; i < 4; i++) {
+    const task = pool.queue((helloWorld) => {
+      executionCount++
+      return helloWorld()
+    })
+    tasks.push(task)
+  }
+
+  tasks[2].cancel()
+  tasks[3].cancel()
+
+  await pool.completed()
+  t.is(executionCount, 2)
+
+  const cancellationEvents = events.filter((event) => event.type === 'taskCanceled')
+  t.deepEqual(cancellationEvents, [
+    {
+      taskID: 3,
+      type: PoolEventType.taskCanceled,
+    },
+    {
+      taskID: 4,
+      type: PoolEventType.taskCanceled,
+    },
+  ])
+})

package/test/serialization.test.ts

@@ -0,0 +1,21 @@
+/* eslint-disable import/no-internal-modules */
+import test from 'ava'
+
+import { registerSerializer, spawn, Thread, Worker } from '../src/index'
+import { Foo, fooSerializer } from './lib/serialization'
+
+registerSerializer(fooSerializer)
+
+test('can use a custom serializer', async (t) => {
+  const run = await spawn(new Worker('./workers/serialization.ts'))
+
+  try {
+    const input = new Foo('Test')
+    const result: Foo<string> = await run(input)
+
+    t.true(result instanceof Foo)
+    t.is(result.getValue(), 'TestTest')
+  } finally {
+    await Thread.terminate(run)
+  }
+})

package/test/spawn.chromium.mocha.ts

@@ -0,0 +1,49 @@
+/* eslint-disable no-restricted-imports */
+/* eslint-disable import/no-internal-modules */
+/*
+ * This code here will be run in a headless Chromium browser using `puppet-run`.
+ * Check the package.json scripts `test:puppeteer:*`.
+ */
+
+// We need this as a work-around to make our threads Worker global, since
+// the bundler would otherwise not recognize `new Worker()` as a web worker
+import '../src/master/register'
+
+import { expect } from 'chai'
+
+import { BlobWorker, spawn, Thread } from '..'
+
+describe('threads in browser', function () {
+  it('can spawn and terminate a thread', async function () {
+    const helloWorld = await spawn<() => string>(new Worker('./workers/hello-world.js'))
+    expect(await helloWorld()).to.equal('Hello World')
+    await Thread.terminate(helloWorld)
+  })
+
+  it('can call a function thread more than once', async function () {
+    const increment = await spawn<() => number>(new Worker('./workers/increment.js'))
+    expect(await increment()).to.equal(1)
+    expect(await increment()).to.equal(2)
+    expect(await increment()).to.equal(3)
+    await Thread.terminate(increment)
+  })
+
+  it('can spawn and use a blob worker', async function () {
+    const baseUrl = new URL(window.location.href).origin
+    const workerSource = `
+      // Makes expose() available on global scope
+      importScripts(${JSON.stringify(baseUrl + '/worker.js')})
+
+      let counter = 0
+
+      expose(function() {
+        return ++counter
+      })
+    `
+    const increment = await spawn<() => number>(BlobWorker.fromText(workerSource))
+    expect(await increment()).to.equal(1)
+    expect(await increment()).to.equal(2)
+    expect(await increment()).to.equal(3)
+    await Thread.terminate(increment)
+  })
+})

package/test/spawn.test.ts

@@ -0,0 +1,71 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable import/no-internal-modules */
+import test from 'ava'
+import { Observable } from 'observable-fns'
+
+import { spawn, Thread, Worker } from '../src/index'
+import { Counter } from './workers/counter'
+
+test('can spawn and terminate a thread', async (t) => {
+  // We also test here that running spawn() without type parameters works
+  const helloWorld = await spawn(new Worker('./workers/hello-world'))
+  t.is(await helloWorld(), 'Hello World')
+  await Thread.terminate(helloWorld)
+  t.pass()
+})
+
+test('can call a function thread more than once', async (t) => {
+  const increment = await spawn<() => number>(new Worker('./workers/increment'))
+  t.is(await increment(), 1)
+  t.is(await increment(), 2)
+  t.is(await increment(), 3)
+  await Thread.terminate(increment)
+})
+
+test('can subscribe to an observable returned by a thread call', async (t) => {
+  const countToFive = await spawn<() => Observable<number>>(new Worker('./workers/count-to-five'))
+  const encounteredValues: any[] = []
+
+  const observable = countToFive()
+  observable.subscribe((value) => encounteredValues.push(value))
+  await observable
+
+  t.deepEqual(encounteredValues, [1, 2, 3, 4, 5])
+  await Thread.terminate(countToFive)
+})
+
+test('can spawn a module thread', async (t) => {
+  const counter = await spawn<Counter>(new Worker('./workers/counter'))
+  t.is(await counter.getCount(), 0)
+  await Promise.all([counter.increment(), counter.increment()])
+  t.is(await counter.getCount(), 2)
+  await counter.decrement()
+  t.is(await counter.getCount(), 1)
+  await Thread.terminate(counter)
+})
+
+test('thread job errors are handled', async (t) => {
+  const fail = await spawn<() => Promise<never>>(new Worker('./workers/faulty-function'))
+  await t.throwsAsync(fail(), null, 'I am supposed to fail.')
+  await Thread.terminate(fail)
+})
+
+test('thread transfer errors are handled', async (t) => {
+  const builtin = require('node:module').builtinModules
+  if (builtin.includes('worker_threads')) {
+    // test is actual for native worker_threads only
+    const helloWorld = await spawn(new Worker('./workers/hello-world'))
+    const badTransferObj = { fn: () => {} }
+    await t.throwsAsync(helloWorld(badTransferObj), { name: 'DataCloneError' })
+    await Thread.terminate(helloWorld)
+  } else {
+    t.pass()
+  }
+})
+
+test('catches top-level thread errors', async (t) => {
+  await t.throwsAsync(spawn(new Worker('./workers/top-level-throw')), null, 'Top-level worker error')
+})
+
+test.todo('can subscribe to thread events')

package/test/streaming.test.ts

@@ -0,0 +1,27 @@
+/* eslint-disable import/no-internal-modules */
+import test from 'ava'
+
+import { spawn, Thread, Worker } from '../src/index'
+
+test('can use worker returning an observable subject', async (t) => {
+  const captured: Array<{ max: number; min: number }> = []
+
+  const minmax = await spawn(new Worker('./workers/minmax'))
+  minmax.values().subscribe((values) => captured.push(values))
+
+  await minmax.push(2)
+  await minmax.push(3)
+  await minmax.push(4)
+  await minmax.push(1)
+  await minmax.push(5)
+  await minmax.finish()
+
+  await Thread.terminate(minmax)
+  t.deepEqual(captured, [
+    { max: 2, min: 2 },
+    { max: 3, min: 2 },
+    { max: 4, min: 2 },
+    { max: 4, min: 1 },
+    { max: 5, min: 1 },
+  ])
+})