@kingsy/objectloader2 2.25.7

package/src/operations/databases/memoryDatabase.spec.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryDatabase } from './memoryDatabase.js'
+import { Base, Item } from '../../types/types.js'
+
+const makeItem = (id: string, foo = 'bar'): Item => ({
+  baseId: id,
+  base: { foo } as unknown as Base
+})
+
+describe('MemoryDatabase', () => {
+  let db: MemoryDatabase
+
+  beforeEach(() => {
+    db = new MemoryDatabase()
+  })
+
+  it('should return undefined for missing keys', async () => {
+    const result = await db.getAll(['missing'])
+    expect(result).toEqual([undefined])
+  })
+
+  it('should add and retrieve a single item', async () => {
+    const item = makeItem('id1')
+    await db.cacheSaveBatch({ batch: [item] })
+    const result = await db.getAll(['id1'])
+    expect(result).toEqual([item])
+  })
+
+  it('should add and retrieve multiple items', async () => {
+    const items = [makeItem('id1'), makeItem('id2', 'baz')]
+    await db.cacheSaveBatch({ batch: items })
+    const result = await db.getAll(['id1', 'id2'])
+    expect(result).toEqual(items)
+  })
+
+  it('should overwrite items with the same key', async () => {
+    const item1 = makeItem('id1', 'foo')
+    const item2 = makeItem('id1', 'bar')
+    await db.cacheSaveBatch({ batch: [item1] })
+    await db.cacheSaveBatch({ batch: [item2] })
+    const result = await db.getAll(['id1'])
+    expect(result).toEqual([item2])
+  })
+
+  it('disposeAsync should resolve', async () => {
+    await expect(db.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/operations/databases/memoryDatabase.ts

@@ -0,0 +1,35 @@
+import { Base, Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { MemoryDatabaseOptions } from '../options.js'
+
+export class MemoryDatabase implements Database {
+  private items: Map<string, Base>
+
+  constructor(options?: MemoryDatabaseOptions) {
+    this.items = options?.items || new Map<string, Base>()
+  }
+
+  getAll(keys: string[]): Promise<(Item | undefined)[]> {
+    const found: (Item | undefined)[] = []
+    for (const key of keys) {
+      const item = this.items.get(key)
+      if (item) {
+        found.push({ baseId: key, base: item })
+      } else {
+        found.push(undefined)
+      }
+    }
+    return Promise.resolve(found)
+  }
+
+  cacheSaveBatch({ batch }: { batch: Item[] }): Promise<void> {
+    for (const item of batch) {
+      this.items.set(item.baseId, item.base)
+    }
+    return Promise.resolve()
+  }
+
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
+}

package/src/operations/downloaders/__snapshots__/memoryDownloader.spec.ts.snap

@@ -0,0 +1,12 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`MemoryDownloader > should add found item to results queue 1`] = `
+Map {
+  "id1" => {
+    "foo": "foo",
+  },
+  "id2" => {
+    "foo": "bar",
+  },
+}
+`;

package/src/operations/downloaders/__snapshots__/serverDownloader.spec.ts.snap

@@ -0,0 +1,92 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`downloader > add extra header 1`] = `
+{
+  "base": {
+    "__closure": {
+      "childIds": 1,
+    },
+    "id": "id",
+    "kingsy_type": "type",
+  },
+  "baseId": "id",
+  "size": 0,
+}
+`;
+
+exports[`downloader > download batch of one 1`] = `
+[
+  {
+    "base": {
+      "id": "id",
+      "kingsy_type": "type",
+    },
+    "baseId": "id",
+    "size": 33,
+  },
+]
+`;
+
+exports[`downloader > download batch of three 1`] = `
+[
+  {
+    "base": {
+      "id": "id1",
+      "kingsy_type": "type",
+    },
+    "baseId": "id1",
+    "size": 34,
+  },
+  {
+    "base": {
+      "id": "id2",
+      "kingsy_type": "type",
+    },
+    "baseId": "id2",
+    "size": 34,
+  },
+  {
+    "base": {
+      "id": "id3",
+      "kingsy_type": "type",
+    },
+    "baseId": "id3",
+    "size": 34,
+  },
+]
+`;
+
+exports[`downloader > download batch of two 1`] = `
+[
+  {
+    "base": {
+      "id": "id1",
+      "kingsy_type": "type",
+    },
+    "baseId": "id1",
+    "size": 34,
+  },
+  {
+    "base": {
+      "id": "id2",
+      "kingsy_type": "type",
+    },
+    "baseId": "id2",
+    "size": 34,
+  },
+]
+`;
+
+exports[`downloader > download single exists 1`] = `
+{
+  "base": {
+    "__closure": {
+      "childIds": 1,
+    },
+    "id": "id",
+    "kingsy_type": "type",
+  },
+  "baseId": "id",
+  "size": 0,
+}
+`;

package/src/operations/downloaders/memoryDownloader.spec.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryDownloader } from './memoryDownloader.js'
+import { Base, Item } from '../../types/types.js'
+import Queue from '../../helpers/queue.js'
+import BufferQueue from '../../helpers/bufferQueue.js'
+
+const makeBase = (foo: string): Base => ({ foo } as unknown as Base)
+
+describe('MemoryDownloader', () => {
+  let items: Map<string, Base>
+  let downloader: MemoryDownloader
+  let results: Queue<Item>
+
+  beforeEach(() => {
+    items = new Map([
+      ['id1', makeBase('foo')],
+      ['id2', makeBase('bar')]
+    ])
+    downloader = new MemoryDownloader('id1', items)
+    results = new BufferQueue<Item>()
+  })
+
+  it('should download the root item', async () => {
+    const item = await downloader.downloadSingle()
+    expect(item).toEqual({ baseId: 'id1', base: { foo: 'foo' } })
+  })
+
+  it('should throw if root item is missing', async () => {
+    const missingDownloader = new MemoryDownloader('missing', items)
+    await expect(missingDownloader.downloadSingle()).rejects.toThrow(
+      'Method not implemented.'
+    )
+  })
+
+  it('should add found item to results queue', () => {
+    downloader.initializePool({ results, total: 2 })
+    downloader.add('id2')
+    expect(items).toMatchSnapshot()
+  })
+
+  it('should throw if added item is missing', () => {
+    downloader.initializePool({ results, total: 2 })
+    expect(() => downloader.add('missing')).toThrow()
+  })
+
+  it('disposeAsync should resolve', async () => {
+    await expect(downloader.disposeAsync()).resolves.not.toThrow()
+  })
+})

package/src/operations/downloaders/memoryDownloader.ts

@@ -0,0 +1,39 @@
+import Queue from '../../helpers/queue.js'
+import { Base, Item } from '../../types/types.js'
+import { Downloader } from '../interfaces.js'
+
+export class MemoryDownloader implements Downloader {
+  #items: Map<string, Base>
+  #rootId: string
+  #results?: Queue<Item>
+
+  constructor(rootId: string, items: Map<string, Base>) {
+    this.#rootId = rootId
+    this.#items = items
+  }
+  initializePool(params: {
+    results: Queue<Item>
+    total: number
+    maxDownloadBatchWait?: number
+  }): void {
+    this.#results = params.results
+  }
+  downloadSingle(): Promise<Item> {
+    const root = this.#items.get(this.#rootId)
+    if (root) {
+      return Promise.resolve({ baseId: this.#rootId, base: root })
+    }
+    return Promise.reject(new Error('Method not implemented.'))
+  }
+  disposeAsync(): Promise<void> {
+    return Promise.resolve()
+  }
+  add(id: string): void {
+    const base = this.#items.get(id)
+    if (base) {
+      this.#results?.add({ baseId: id, base })
+      return
+    }
+    throw new Error('Method not implemented.')
+  }
+}

package/src/operations/downloaders/serverDownloader.spec.ts

@@ -0,0 +1,161 @@
+import { describe, expect, test } from 'vitest'
+import createFetchMock from 'vitest-fetch-mock'
+import { vi } from 'vitest'
+import { Item } from '../../types/types.js'
+import ServerDownloader from './serverDownloader.js'
+import { MemoryPump } from '../../helpers/memoryPump.js'
+
+describe('downloader', () => {
+  test('download batch of one', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = { baseId: 'id', base: { id: 'id', kingsy_type: 'type' } }
+    fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
+    const pump = new MemoryPump()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://kingsy.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+      fetch: fetchMocker
+    })
+    downloader.initializePool({ results: pump, total: 1, maxDownloadBatchWait: 200 })
+    downloader.add('id')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of pump.gather([i.baseId])) {
+      r.push(x)
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('download batch of two', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', kingsy_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id2', kingsy_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
+    )
+
+    const pump = new MemoryPump()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://kingsy.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 })
+    downloader.add('id1')
+    downloader.add('id2')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of pump.gather([i1.baseId, i2.baseId])) {
+      r.push(x)
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('download batch of three', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', kingsy_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id2', kingsy_type: 'type' } }
+    const i3: Item = { baseId: 'id3', base: { id: 'id3', kingsy_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' +
+        JSON.stringify(i1.base) +
+        '\nid2\t' +
+        JSON.stringify(i2.base) +
+        '\nid3\t' +
+        JSON.stringify(i3.base) +
+        '\n'
+    )
+
+    const pump = new MemoryPump()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://kingsy.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    downloader.initializePool({ results: pump, total: 3, maxDownloadBatchWait: 200 })
+    downloader.add('id1')
+    downloader.add('id2')
+    downloader.add('id3')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) {
+      r.push(x)
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('download single exists', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = {
+      baseId: 'id',
+      base: { id: 'id', kingsy_type: 'type', __closure: { childIds: 1 } }
+    }
+    fetchMocker.mockResponseOnce(JSON.stringify(i.base))
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://kingsy.test',
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    const x = await downloader.downloadSingle()
+    expect(x).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('add extra header', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = {
+      baseId: 'id',
+      base: { id: 'id', kingsy_type: 'type', __closure: { childIds: 1 } }
+    }
+    fetchMocker.mockResponseIf(
+      (req) => req.headers.get('x-test') === 'asdf',
+      JSON.stringify(i.base)
+    )
+    const headers = new Headers()
+    headers.set('x-test', 'asdf')
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://kingsy.test',
+      headers,
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    const x = await downloader.downloadSingle()
+    expect(x).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('can dispose used', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const headers = new Headers()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://kingsy.test',
+      headers,
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    await downloader.disposeAsync()
+  })
+})

package/src/operations/downloaders/serverDownloader.ts

@@ -0,0 +1,217 @@
+import BatchedPool from '../../helpers/batchedPool.js'
+import Queue from '../../helpers/queue.js'
+import { ObjectLoaderRuntimeError } from '../../types/errors.js'
+import { Fetcher, isBase, Item, take } from '../../types/types.js'
+import { Downloader } from '../interfaces.js'
+
+export interface ServerDownloaderOptions {
+  serverUrl: string
+  streamId: string
+  objectId: string
+  token?: string
+  headers?: Headers
+  fetch?: Fetcher
+}
+
+export default class ServerDownloader implements Downloader {
+  #requestUrlRootObj: string
+  #requestUrlChildren: string
+  #headers: HeadersInit
+  #options: ServerDownloaderOptions
+  #fetch: Fetcher
+  #results?: Queue<Item>
+
+  #downloadQueue?: BatchedPool<string>
+  #decoder = new TextDecoder()
+
+  constructor(options: ServerDownloaderOptions) {
+    this.#options = options
+    this.#fetch =
+      options.fetch ?? ((...args): Promise<Response> => globalThis.fetch(...args))
+
+    this.#headers = {}
+    if (options.headers) {
+      for (const header of options.headers.entries()) {
+        this.#headers[header[0]] = header[1]
+      }
+    }
+    this.#headers['Accept'] = `text/plain`
+
+    if (this.#options.token) {
+      this.#headers['Authorization'] = `Bearer ${this.#options.token}`
+    }
+    this.#requestUrlChildren = `${this.#options.serverUrl}/api/getobjects/${
+      this.#options.streamId
+    }`
+    this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${
+      this.#options.streamId
+    }/${this.#options.objectId}/single`
+  }
+
+  #getDownloadCountAndSizes(total: number): number[] {
+    if (total <= 50) {
+      return [total]
+    }
+
+    return [10000, 25000, 10000, 1000]
+  }
+
+  initializePool(params: {
+    results: Queue<Item>
+    total: number
+    maxDownloadBatchWait?: number
+  }): void {
+    const { results, total } = params
+    this.#results = results
+    this.#downloadQueue = new BatchedPool<string>({
+      concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
+      maxWaitTime: params.maxDownloadBatchWait,
+      processFunction: (batch: string[]): Promise<void> =>
+        this.downloadBatch({
+          batch,
+          url: this.#requestUrlChildren,
+          headers: this.#headers
+        })
+    })
+  }
+
+  #getPool(): BatchedPool<string> {
+    if (this.#downloadQueue) {
+      return this.#downloadQueue
+    }
+    throw new Error('Download pool is not initialized')
+  }
+
+  add(id: string): void {
+    this.#getPool().add(id)
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#downloadQueue?.disposeAsync()
+  }
+
+  #processJson(baseId: string, unparsedBase: string): Item {
+    let base: unknown
+    try {
+      base = JSON.parse(unparsedBase)
+    } catch (e: unknown) {
+      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
+    }
+    if (isBase(base)) {
+      return { baseId, base }
+    } else {
+      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+    }
+  }
+
+  async downloadBatch(params: {
+    batch: string[]
+    url: string
+    headers: HeadersInit
+  }): Promise<void> {
+    const { batch, url, headers } = params
+    const keys = new Set<string>(batch)
+    const response = await this.#fetch(url, {
+      method: 'POST',
+      headers: { ...headers, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ objects: JSON.stringify(batch) })
+    })
+
+    this.#validateResponse(response)
+    if (!response.body) {
+      throw new Error('ReadableStream not supported or response has no body.')
+    }
+
+    const reader = response.body.getReader()
+    let leftover = new Uint8Array(0)
+
+    let count = 0
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+
+      leftover = await this.processArray(leftover, value, keys, async () => {
+        count++
+        if (count % 1000 === 0) {
+          await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen
+        }
+      })
+    }
+    if (keys.size > 0) {
+      throw new Error(
+        'Items requested were not downloaded: ' + take(keys.values(), 10).join(',')
+      )
+    }
+  }
+
+  async processArray(
+    leftover: Uint8Array,
+    value: Uint8Array,
+    keys: Set<string>,
+    callback: () => Promise<void>
+  ): Promise<Uint8Array> {
+    //this concat will allocate a new array
+    const combined = this.concatUint8Arrays(leftover, value)
+    let start = 0
+
+    //subarray doesn't allocate
+    for (let i = 0; i < combined.length; i++) {
+      if (combined[i] === 0x0a) {
+        const line = combined.subarray(start, i) // line without \n
+        //strings are allocated here
+        const item = this.processLine(line)
+        this.#results?.add(item)
+        start = i + 1
+        await callback()
+        keys.delete(item.baseId)
+      }
+    }
+    return combined.subarray(start) // carry over remainder
+  }
+
+  processLine(line: Uint8Array): Item {
+    for (let i = 0; i < line.length; i++) {
+      if (line[i] === 0x09) {
+        //this is a tab
+        const baseId = this.#decoder.decode(line.subarray(0, i))
+        const json = line.subarray(i + 1)
+        const base = this.#decoder.decode(json)
+        const item = this.#processJson(baseId, base)
+        item.size = json.length
+        return item
+      }
+    }
+    throw new ObjectLoaderRuntimeError(
+      'Invalid line format: ' + this.#decoder.decode(line)
+    )
+  }
+
+  concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
+    const c = new Uint8Array(a.length + b.length)
+    c.set(a, 0)
+    c.set(b, a.length)
+    return c
+  }
+
+  async downloadSingle(): Promise<Item> {
+    const response = await this.#fetch(this.#requestUrlRootObj, {
+      headers: this.#headers
+    })
+    this.#validateResponse(response)
+    const responseText = await response.text()
+    const item = this.#processJson(this.#options.objectId, responseText)
+    item.size = 0
+    return item
+  }
+
+  #validateResponse(response: Response): void {
+    if (!response.ok) {
+      if ([401, 403].includes(response.status)) {
+        throw new ObjectLoaderRuntimeError('You do not have access!')
+      }
+      throw new ObjectLoaderRuntimeError(
+        `Failed to fetch objects: ${response.status} ${response.statusText})`
+      )
+    }
+  }
+}

package/src/operations/interfaces.ts

@@ -0,0 +1,18 @@
+import Queue from '../helpers/queue.js'
+import { Item } from '../types/types.js'
+
+export interface Downloader extends Queue<string> {
+  initializePool(params: {
+    results: Queue<Item>
+    total: number
+    maxDownloadBatchWait?: number
+  }): void
+  downloadSingle(): Promise<Item>
+  disposeAsync(): Promise<void>
+}
+
+export interface Database {
+  getAll(keys: string[]): Promise<(Item | undefined)[]>
+  cacheSaveBatch(params: { batch: Item[] }): Promise<void>
+  disposeAsync(): Promise<void>
+}