@speckle/objectloader2 2.24.0 → 2.25.0

package/src/operations/{__snapshots__ → downloaders/__snapshots__}/serverDownloader.spec.ts.snap

@@ -10,6 +10,7 @@ exports[`downloader > add extra header 1`] = `
     "speckle_type": "type",
   },
   "baseId": "id",
+  "size": 0,
 }
 `;
 
@@ -21,6 +22,36 @@ exports[`downloader > download batch of one 1`] = `
       "speckle_type": "type",
     },
     "baseId": "id",
+    "size": 33,
+  },
+]
+`;
+
+exports[`downloader > download batch of three 1`] = `
+[
+  {
+    "base": {
+      "id": "id1",
+      "speckle_type": "type",
+    },
+    "baseId": "id1",
+    "size": 34,
+  },
+  {
+    "base": {
+      "id": "id2",
+      "speckle_type": "type",
+    },
+    "baseId": "id2",
+    "size": 34,
+  },
+  {
+    "base": {
+      "id": "id3",
+      "speckle_type": "type",
+    },
+    "baseId": "id3",
+    "size": 34,
   },
 ]
 `;
@@ -33,6 +64,7 @@ exports[`downloader > download batch of two 1`] = `
       "speckle_type": "type",
     },
     "baseId": "id1",
+    "size": 34,
   },
   {
     "base": {
@@ -40,6 +72,7 @@ exports[`downloader > download batch of two 1`] = `
       "speckle_type": "type",
     },
     "baseId": "id2",
+    "size": 34,
   },
 ]
 `;
@@ -54,5 +87,6 @@ exports[`downloader > download single exists 1`] = `
     "speckle_type": "type",
   },
   "baseId": "id",
+  "size": 0,
 }
 `;

package/src/operations/{memoryDownloader.ts → downloaders/memoryDownloader.ts}

@@ -1,24 +1,25 @@
-import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
-import { Base, Item } from '../types/types.js'
-import { Downloader } from './interfaces.js'
+import Queue from '../../helpers/queue.js'
+import { Base, Item } from '../../types/types.js'
+import { Downloader } from '../interfaces.js'
 
 export class MemoryDownloader implements Downloader {
-  #items: Record<string, Base>
+  #items: Map<string, Base>
   #rootId: string
-  #results?: AsyncGeneratorQueue<Item>
+  #results?: Queue<Item>
 
-  constructor(
-    rootId: string,
-    items: Record<string, Base>,
-    results?: AsyncGeneratorQueue<Item>
-  ) {
+  constructor(rootId: string, items: Map<string, Base>) {
     this.#rootId = rootId
     this.#items = items
-    this.#results = results
   }
-  initializePool(): void {}
+  initializePool(params: {
+    results: Queue<Item>
+    total: number
+    maxDownloadBatchWait?: number
+  }): void {
+    this.#results = params.results
+  }
   downloadSingle(): Promise<Item> {
-    const root = this.#items[this.#rootId]
+    const root = this.#items.get(this.#rootId)
     if (root) {
       return Promise.resolve({ baseId: this.#rootId, base: root })
     }
@@ -28,7 +29,7 @@ export class MemoryDownloader implements Downloader {
     return Promise.resolve()
   }
   add(id: string): void {
-    const base = this.#items[id]
+    const base = this.#items.get(id)
     if (base) {
       this.#results?.add({ baseId: id, base })
       return

package/src/operations/{serverDownloader.spec.ts → downloaders/serverDownloader.spec.ts}

@@ -1,42 +1,33 @@
 import { describe, expect, test } from 'vitest'
 import createFetchMock from 'vitest-fetch-mock'
 import { vi } from 'vitest'
-import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
-import { Item } from '../types/types.js'
-import { Cache } from './interfaces.js'
+import { Item } from '../../types/types.js'
 import ServerDownloader from './serverDownloader.js'
+import { MemoryPump } from '../../helpers/memoryPump.js'
 
 describe('downloader', () => {
   test('download batch of one', async () => {
     const fetchMocker = createFetchMock(vi)
     const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
     fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
-    const results = new AsyncGeneratorQueue()
-    const db = {
-      async add(): Promise<void> {
-        return Promise.resolve()
-      }
-    } as unknown as Cache
+    const pump = new MemoryPump()
     const downloader = new ServerDownloader({
-      database: db,
-      results,
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
       objectId: 'objectId',
       token: 'token',
-
       fetch: fetchMocker
     })
-    downloader.initializePool({ total: 1, maxDownloadBatchWait: 200 })
+    downloader.initializePool({ results: pump, total: 1, maxDownloadBatchWait: 200 })
     downloader.add('id')
     await downloader.disposeAsync()
-    results.dispose()
     const r = []
-    for await (const x of results.consume()) {
+    for await (const x of pump.gather([i.baseId])) {
       r.push(x)
     }
 
     expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
   })
 
   test('download batch of two', async () => {
@@ -46,15 +37,9 @@ describe('downloader', () => {
     fetchMocker.mockResponseOnce(
       'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
     )
-    const results = new AsyncGeneratorQueue()
-    const db = {
-      async add(): Promise<void> {
-        return Promise.resolve()
-      }
-    } as unknown as Cache
+
+    const pump = new MemoryPump()
     const downloader = new ServerDownloader({
-      database: db,
-      results,
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
       objectId: 'objectId',
@@ -62,16 +47,55 @@ describe('downloader', () => {
 
       fetch: fetchMocker
     })
-    downloader.initializePool({ total: 2, maxDownloadBatchWait: 200 })
-    downloader.add('id')
+    downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 })
+    downloader.add('id1')
+    downloader.add('id2')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of pump.gather([i1.baseId, i2.baseId])) {
+      r.push(x)
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('download batch of three', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', speckle_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id2', speckle_type: 'type' } }
+    const i3: Item = { baseId: 'id3', base: { id: 'id3', speckle_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' +
+        JSON.stringify(i1.base) +
+        '\nid2\t' +
+        JSON.stringify(i2.base) +
+        '\nid3\t' +
+        JSON.stringify(i3.base) +
+        '\n'
+    )
+
+    const pump = new MemoryPump()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    downloader.initializePool({ results: pump, total: 3, maxDownloadBatchWait: 200 })
+    downloader.add('id1')
+    downloader.add('id2')
+    downloader.add('id3')
     await downloader.disposeAsync()
-    results.dispose()
     const r = []
-    for await (const x of results.consume()) {
+    for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) {
       r.push(x)
     }
 
     expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
   })
 
   test('download single exists', async () => {
@@ -81,15 +105,7 @@ describe('downloader', () => {
       base: { id: 'id', speckle_type: 'type', __closure: { childIds: 1 } }
     }
     fetchMocker.mockResponseOnce(JSON.stringify(i.base))
-    const results = new AsyncGeneratorQueue()
-    const db = {
-      async add(): Promise<void> {
-        return Promise.resolve()
-      }
-    } as unknown as Cache
     const downloader = new ServerDownloader({
-      database: db,
-      results,
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
       objectId: i.baseId,
@@ -99,6 +115,7 @@ describe('downloader', () => {
     })
     const x = await downloader.downloadSingle()
     expect(x).toMatchSnapshot()
+    await downloader.disposeAsync()
   })
 
   test('add extra header', async () => {
@@ -111,17 +128,9 @@ describe('downloader', () => {
       (req) => req.headers.get('x-test') === 'asdf',
       JSON.stringify(i.base)
     )
-    const results = new AsyncGeneratorQueue()
-    const db = {
-      async add(): Promise<void> {
-        return Promise.resolve()
-      }
-    } as unknown as Cache
     const headers = new Headers()
     headers.set('x-test', 'asdf')
     const downloader = new ServerDownloader({
-      database: db,
-      results,
       serverUrl: 'http://speckle.test',
       headers,
       streamId: 'streamId',
@@ -132,5 +141,21 @@ describe('downloader', () => {
     })
     const x = await downloader.downloadSingle()
     expect(x).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('can dispose used', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const headers = new Headers()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://speckle.test',
+      headers,
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    await downloader.disposeAsync()
   })
 })

package/src/operations/{serverDownloader.ts → downloaders/serverDownloader.ts}

@@ -1,22 +1,33 @@
-import BatchedPool from '../helpers/batchedPool.js'
-import Queue from '../helpers/queue.js'
-import { ObjectLoaderRuntimeError } from '../types/errors.js'
-import { Fetcher, isBase, Item } from '../types/types.js'
-import { Downloader } from './interfaces.js'
-import { BaseDownloadOptions } from './options.js'
+import BatchedPool from '../../helpers/batchedPool.js'
+import Queue from '../../helpers/queue.js'
+import { ObjectLoaderRuntimeError } from '../../types/errors.js'
+import { Fetcher, isBase, Item, take } from '../../types/types.js'
+import { Downloader } from '../interfaces.js'
+
+export interface ServerDownloaderOptions {
+  serverUrl: string
+  streamId: string
+  objectId: string
+  token?: string
+  headers?: Headers
+  fetch?: Fetcher
+}
 
 export default class ServerDownloader implements Downloader {
   #requestUrlRootObj: string
   #requestUrlChildren: string
   #headers: HeadersInit
-  #options: BaseDownloadOptions
+  #options: ServerDownloaderOptions
   #fetch: Fetcher
+  #results?: Queue<Item>
 
   #downloadQueue?: BatchedPool<string>
+  #decoder = new TextDecoder()
 
-  constructor(options: BaseDownloadOptions) {
+  constructor(options: ServerDownloaderOptions) {
     this.#options = options
-    this.#fetch = options.fetch ?? ((...args) => globalThis.fetch(...args))
+    this.#fetch =
+      options.fetch ?? ((...args): Promise<Response> => globalThis.fetch(...args))
 
     this.#headers = {}
     if (options.headers) {
@@ -45,17 +56,21 @@ export default class ServerDownloader implements Downloader {
     return [10000, 30000, 10000, 1000]
   }
 
-  initializePool(params: { total: number; maxDownloadBatchWait?: number }) {
-    const { total } = params
+  initializePool(params: {
+    results: Queue<Item>
+    total: number
+    maxDownloadBatchWait?: number
+  }): void {
+    const { results, total } = params
+    this.#results = results
     this.#downloadQueue = new BatchedPool<string>({
       concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
       maxWaitTime: params.maxDownloadBatchWait,
-      processFunction: (batch: string[]) =>
+      processFunction: (batch: string[]): Promise<void> =>
         this.downloadBatch({
           batch,
           url: this.#requestUrlChildren,
-          headers: this.#headers,
-          results: this.#options.results
+          headers: this.#headers
         })
     })
   }
@@ -73,7 +88,6 @@ export default class ServerDownloader implements Downloader {
 
   async disposeAsync(): Promise<void> {
     await this.#downloadQueue?.disposeAsync()
-    await this.#getPool().disposeAsync()
   }
 
   #processJson(baseId: string, unparsedBase: string): Item {
@@ -94,9 +108,9 @@ export default class ServerDownloader implements Downloader {
     batch: string[]
     url: string
     headers: HeadersInit
-    results: Queue<Item>
   }): Promise<void> {
-    const { batch, url, headers, results } = params
+    const { batch, url, headers } = params
+    const keys = new Set<string>(batch)
     const response = await this.#fetch(url, {
       method: 'POST',
       headers: { ...headers, 'Content-Type': 'application/json' },
@@ -109,35 +123,74 @@ export default class ServerDownloader implements Downloader {
     }
 
     const reader = response.body.getReader()
-    const decoder = new TextDecoder()
-    let buffer = '' // Temporary buffer to store incoming chunks
+    let leftover = new Uint8Array(0)
 
     let count = 0
     while (true) {
       const { done, value } = await reader.read()
       if (done) break
-      // Decode the chunk and add to buffer
-      buffer += decoder.decode(value, { stream: true })
-
-      // Try to process JSON objects from the buffer
-      let boundary = buffer.indexOf('\n')
-      while (boundary !== -1) {
-        const jsonString = buffer.slice(0, boundary)
-        buffer = buffer.slice(boundary + 1)
-        boundary = buffer.indexOf('\n')
-        if (jsonString) {
-          const pieces = jsonString.split('\t')
-          const [id, unparsedObj] = pieces
-          const item = this.#processJson(id, unparsedObj)
-          await this.#options.database.add(item)
-          results.add(item)
-          count++
-          if (count % 1000 === 0) {
-            await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen
-          }
+
+      leftover = await this.processArray(leftover, value, keys, async () => {
+        count++
+        if (count % 1000 === 0) {
+          await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen
         }
+      })
+    }
+    if (keys.size > 0) {
+      throw new Error(
+        'Items requested were not downloaded: ' + take(keys.values(), 10).join(',')
+      )
+    }
+  }
+
+  async processArray(
+    leftover: Uint8Array,
+    value: Uint8Array,
+    keys: Set<string>,
+    callback: () => Promise<void>
+  ): Promise<Uint8Array> {
+    //this concat will allocate a new array
+    const combined = this.concatUint8Arrays(leftover, value)
+    let start = 0
+
+    //subarray doesn't allocate
+    for (let i = 0; i < combined.length; i++) {
+      if (combined[i] === 0x0a) {
+        const line = combined.subarray(start, i) // line without \n
+        //strings are allocated here
+        const item = this.processLine(line)
+        this.#results?.add(item)
+        start = i + 1
+        await callback()
+        keys.delete(item.baseId)
       }
     }
+    return combined.subarray(start) // carry over remainder
+  }
+
+  processLine(line: Uint8Array): Item {
+    for (let i = 0; i < line.length; i++) {
+      if (line[i] === 0x09) {
+        //this is a tab
+        const baseId = this.#decoder.decode(line.subarray(0, i))
+        const json = line.subarray(i + 1)
+        const base = this.#decoder.decode(json)
+        const item = this.#processJson(baseId, base)
+        item.size = json.length
+        return item
+      }
+    }
+    throw new ObjectLoaderRuntimeError(
+      'Invalid line format: ' + this.#decoder.decode(line)
+    )
+  }
+
+  concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
+    const c = new Uint8Array(a.length + b.length)
+    c.set(a, 0)
+    c.set(b, a.length)
+    return c
   }
 
   async downloadSingle(): Promise<Item> {
@@ -147,6 +200,7 @@ export default class ServerDownloader implements Downloader {
     this.#validateResponse(response)
     const responseText = await response.text()
     const item = this.#processJson(this.#options.objectId, responseText)
+    item.size = 0
     return item
   }
 

package/src/operations/interfaces.ts

@@ -1,20 +1,19 @@
 import Queue from '../helpers/queue.js'
 import { Item } from '../types/types.js'
 
-export interface Cache {
-  getItem(params: { id: string }): Promise<Item | undefined>
-  processItems(params: {
-    ids: string[]
-    foundItems: Queue<Item>
-    notFoundItems: Queue<string>
-  }): Promise<void>
-
-  add(item: Item): Promise<void>
+export interface Downloader extends Queue<string> {
+  initializePool(params: {
+    results: Queue<Item>
+    total: number
+    maxDownloadBatchWait?: number
+  }): void
+  downloadSingle(): Promise<Item>
   disposeAsync(): Promise<void>
 }
 
-export interface Downloader extends Queue<string> {
-  initializePool(params: { total: number }): void
-  downloadSingle(): Promise<Item>
+export interface Database {
+  getAll(keys: string[]): Promise<(Item | undefined)[]>
+  getItem(params: { id: string }): Promise<Item | undefined>
+  cacheSaveBatch(params: { batch: Item[] }): Promise<void>
   disposeAsync(): Promise<void>
 }