@speckle/objectloader2 2.23.12

package/src/operations/interfaces.ts

@@ -0,0 +1,20 @@
+import Queue from '../helpers/queue.js'
+import { Item } from '../types/types.js'
+
+export interface Cache {
+  getItem(params: { id: string }): Promise<Item | undefined>
+  processItems(params: {
+    ids: string[]
+    foundItems: Queue<Item>
+    notFoundItems: Queue<string>
+  }): Promise<void>
+
+  add(item: Item): Promise<void>
+  disposeAsync(): Promise<void>
+}
+
+export interface Downloader extends Queue<string> {
+  initializePool(params: { total: number }): void
+  downloadSingle(): Promise<Item>
+  disposeAsync(): Promise<void>
+}

package/src/operations/objectLoader2.spec.ts

@@ -0,0 +1,161 @@
+import { describe, expect, test } from 'vitest'
+import ObjectLoader2 from './objectLoader2.js'
+import { Base, Item } from '../types/types.js'
+import { Cache, Downloader } from './interfaces.js'
+import Queue from '../helpers/queue.js'
+
+describe('objectloader2', () => {
+  test('can get a root object from cache', async () => {
+    const root = { baseId: 'baseId' } as unknown as Item
+    const cache = {
+      getItem(params: { id: string }): Promise<Item> {
+        expect(params.id).toBe(root.baseId)
+        return Promise.resolve(root)
+      }
+    } as Cache
+    const downloader = {} as Downloader
+    const loader = new ObjectLoader2({
+      serverUrl: 'a',
+      streamId: 'b',
+      objectId: root.baseId,
+      cache,
+      downloader
+    })
+    const x = await loader.getRootObject()
+    expect(x).toBe(root)
+  })
+
+  test('can get a root object from downloader', async () => {
+    const root = { baseId: 'baseId' } as unknown as Item
+    const cache = {
+      getItem(params: { id: string }): Promise<Item | undefined> {
+        expect(params.id).toBe(root.baseId)
+        return Promise.resolve<Item | undefined>(undefined)
+      },
+      add(item: Item): Promise<void> {
+        expect(item).toBe(root)
+        return Promise.resolve()
+      }
+    } as Cache
+    const downloader = {
+      downloadSingle(): Promise<Item> {
+        return Promise.resolve(root)
+      }
+    } as Downloader
+    const loader = new ObjectLoader2({
+      serverUrl: 'a',
+      streamId: 'b',
+      objectId: root.baseId,
+      cache,
+      downloader
+    })
+    const x = await loader.getRootObject()
+    expect(x).toBe(root)
+  })
+
+  test('can get single object from cache using iterator', async () => {
+    const rootId = 'baseId'
+    const rootBase: Base = { id: 'baseId' }
+    const root = { baseId: rootId, base: rootBase } as unknown as Item
+    const cache = {
+      getItem(params: { id: string }): Promise<Item | undefined> {
+        expect(params.id).toBe(rootId)
+        return Promise.resolve(root)
+      }
+    } as Cache
+    const downloader = {} as Downloader
+    const loader = new ObjectLoader2({
+      serverUrl: 'a',
+      streamId: 'b',
+      objectId: rootId,
+      cache,
+      downloader
+    })
+    const r = []
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+
+    expect(r.length).toBe(1)
+    expect(r[0]).toBe(rootBase)
+  })
+
+  test('can get root/child object from cache using iterator', async () => {
+    const child1Base = { id: 'child1Id' }
+    const child1 = { baseId: 'child1Id', base: child1Base } as unknown as Item
+
+    const rootId = 'rootId'
+    const rootBase: Base = { id: 'rootId', __closure: { child1Id: 100 } }
+    const root = {
+      baseId: rootId,
+      base: rootBase
+    } as unknown as Item
+
+    const cache = {
+      getItem(params: { id: string }): Promise<Item | undefined> {
+        expect(params.id).toBe(root.baseId)
+        return Promise.resolve(root)
+      },
+      processItems(params: {
+        ids: string[]
+        foundItems: Queue<Item>
+
+        notFoundItems: Queue<string>
+      }): Promise<void> {
+        expect(params.ids.length).toBe(1)
+        expect(params.ids[0]).toBe(child1.baseId)
+        params.foundItems.add(child1)
+        return Promise.resolve()
+      },
+      disposeAsync(): Promise<void> {
+        return Promise.resolve()
+      }
+    } as Cache
+    const downloader = {
+      initializePool(params: { total: number }): void {
+        expect(params.total).toBe(1)
+      },
+      disposeAsync(): Promise<void> {
+        return Promise.resolve()
+      }
+    } as Downloader
+    const loader = new ObjectLoader2({
+      serverUrl: 'a',
+      streamId: 'b',
+      objectId: root.baseId,
+      cache,
+      downloader
+    })
+    const r = []
+    for await (const x of loader.getObjectIterator()) {
+      r.push(x)
+    }
+
+    expect(r.length).toBe(2)
+    expect(r[0]).toBe(rootBase)
+    expect(r[1]).toBe(child1Base)
+  })
+
+  test('add extra header', async () => {
+    const root = { baseId: 'baseId' } as unknown as Item
+    const cache = {
+      getItem(params: { id: string }): Promise<Item> {
+        expect(params.id).toBe(root.baseId)
+        return Promise.resolve(root)
+      }
+    } as Cache
+    const downloader = {} as Downloader
+    const headers = new Headers()
+    headers.set('x-test', 'asdf')
+    const loader = new ObjectLoader2({
+      serverUrl: 'a',
+      streamId: 'b',
+      objectId: root.baseId,
+      headers,
+      cache,
+      downloader
+    })
+    const x = await loader.getRootObject()
+    expect(x).toBe(root)
+  })
+})

package/src/operations/objectLoader2.ts

@@ -0,0 +1,120 @@
+import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
+import { Cache, Downloader } from './interfaces.js'
+import IndexedDatabase from './indexedDatabase.js'
+import ServerDownloader from './serverDownloader.js'
+import { CustomLogger, Base, Item } from '../types/types.js'
+import { ObjectLoader2Options } from './options.js'
+import { DeferredBase } from '../helpers/deferredBase.js'
+
+export default class ObjectLoader2 {
+  #objectId: string
+
+  #logger: CustomLogger
+
+  #database: Cache
+  #downloader: Downloader
+
+  #gathered: AsyncGeneratorQueue<Item>
+
+  #buffer: DeferredBase[] = []
+
+  constructor(options: ObjectLoader2Options) {
+    this.#objectId = options.objectId
+
+    this.#logger = options.logger || console.log
+    this.#gathered = new AsyncGeneratorQueue()
+    this.#database =
+      options.cache ||
+      new IndexedDatabase({
+        logger: this.#logger,
+        maxCacheReadSize: 10000,
+        maxCacheWriteSize: 5000,
+        indexedDB: options.indexedDB,
+        keyRange: options.keyRange
+      })
+    this.#downloader =
+      options.downloader ||
+      new ServerDownloader({
+        database: this.#database,
+        results: this.#gathered,
+        serverUrl: options.serverUrl,
+        streamId: options.streamId,
+        objectId: this.#objectId,
+        token: options.token,
+        headers: options.headers
+      })
+  }
+
+  async disposeAsync(): Promise<void> {
+    await Promise.all([
+      this.#database.disposeAsync(),
+      this.#downloader.disposeAsync(),
+      this.#gathered.dispose()
+    ])
+  }
+
+  async getRootObject(): Promise<Item | undefined> {
+    const cachedRootObject = await this.#database.getItem({ id: this.#objectId })
+    if (cachedRootObject) {
+      return cachedRootObject
+    }
+    const rootItem = await this.#downloader.downloadSingle()
+
+    await this.#database.add(rootItem)
+    return rootItem
+  }
+
+  async getObject(params: { id: string }): Promise<Base> {
+    const item = await this.#database.getItem({ id: params.id })
+    if (item) {
+      return item.base
+    }
+    const deferredBase = this.#buffer.find((x) => x.id === params.id)
+    if (deferredBase) {
+      return await deferredBase.promise
+    }
+    const d = new DeferredBase(params.id)
+    this.#buffer.push(d)
+    return d
+  }
+
+  async getTotalObjectCount() {
+    const rootObj = await this.getRootObject()
+    const totalChildrenCount = Object.keys(rootObj?.base.__closure || {}).length
+    return totalChildrenCount + 1 //count the root
+  }
+
+  async *getObjectIterator(): AsyncGenerator<Base> {
+    const rootItem = await this.getRootObject()
+    if (rootItem === undefined) {
+      this.#logger('No root object found!')
+      return
+    }
+    yield rootItem.base
+    if (!rootItem.base.__closure) return
+
+    const children = Object.keys(rootItem.base.__closure)
+    const total = children.length
+    this.#downloader.initializePool({ total })
+    const processPromise = this.#database.processItems({
+      ids: children,
+      foundItems: this.#gathered,
+      notFoundItems: this.#downloader
+    })
+    let count = 0
+    for await (const item of this.#gathered.consume()) {
+      const deferredIndex = this.#buffer.findIndex((x) => x.id === item.baseId)
+      if (deferredIndex !== -1) {
+        const deferredBase = this.#buffer[deferredIndex]
+        deferredBase.resolve(item.base)
+        this.#buffer.splice(deferredIndex, 1)
+      }
+      yield item.base
+      count++
+      if (count >= total) {
+        await this.disposeAsync()
+      }
+    }
+    await processPromise
+  }
+}

package/src/operations/options.ts

@@ -0,0 +1,41 @@
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import Queue from '../helpers/queue.js'
+import { CustomLogger, Fetcher, Item } from '../types/types.js'
+import { Cache, Downloader } from './interfaces.js'
+
+export interface ObjectLoader2Options {
+  keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
+  indexedDB?: IDBFactory
+  serverUrl: string
+  streamId: string
+  objectId: string
+  token?: string
+  logger?: CustomLogger
+  headers?: Headers
+  cache?: Cache
+  downloader?: Downloader
+}
+export interface BaseDatabaseOptions {
+  logger?: CustomLogger
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+  maxCacheReadSize?: number
+  maxCacheWriteSize?: number
+  maxCacheBatchWriteWait?: number
+}
+
+export interface BaseDownloadOptions {
+  serverUrl: string
+  streamId: string
+  objectId: string
+  token?: string
+  headers?: Headers
+
+  fetch?: Fetcher
+  database: Cache
+  results: Queue<Item>
+}

package/src/operations/serverDownloader.spec.ts

@@ -0,0 +1,133 @@
+import { describe, expect, test } from 'vitest'
+import createFetchMock from 'vitest-fetch-mock'
+import { vi } from 'vitest'
+import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
+import { Item } from '../types/types.js'
+import { Cache } from './interfaces.js'
+import ServerDownloader from './serverDownloader.js'
+
+describe('downloader', () => {
+  test('download batch of one', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = { baseId: 'id', base: { id: 'id' } }
+    fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
+    const results = new AsyncGeneratorQueue<Item>()
+    const db = {
+      async add(): Promise<void> {
+        return Promise.resolve()
+      }
+    } as unknown as Cache
+    const downloader = new ServerDownloader({
+      database: db,
+      results,
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    downloader.initializePool({ total: 1, maxDownloadBatchWait: 200 })
+    downloader.add('id')
+    await downloader.disposeAsync()
+    results.dispose()
+    const r = []
+    for await (const x of results.consume()) {
+      r.push(x)
+    }
+
+    expect(r.length).toBe(1)
+    expect(JSON.stringify(r[0])).toBe(JSON.stringify(i))
+  })
+
+  test('download batch of two', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id2' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
+    )
+    const results = new AsyncGeneratorQueue<Item>()
+    const db = {
+      async add(): Promise<void> {
+        return Promise.resolve()
+      }
+    } as unknown as Cache
+    const downloader = new ServerDownloader({
+      database: db,
+      results,
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    downloader.initializePool({ total: 2, maxDownloadBatchWait: 200 })
+    downloader.add('id')
+    await downloader.disposeAsync()
+    results.dispose()
+    const r = []
+    for await (const x of results.consume()) {
+      r.push(x)
+    }
+
+    expect(r.length).toBe(2)
+    expect(JSON.stringify(r[0])).toBe(JSON.stringify(i1))
+    expect(JSON.stringify(r[1])).toBe(JSON.stringify(i2))
+  })
+
+  test('download single exists', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = { baseId: 'id', base: { id: 'id', __closure: { childIds: 1 } } }
+    fetchMocker.mockResponseOnce(JSON.stringify(i.base))
+    const results = new AsyncGeneratorQueue<Item>()
+    const db = {
+      async add(): Promise<void> {
+        return Promise.resolve()
+      }
+    } as unknown as Cache
+    const downloader = new ServerDownloader({
+      database: db,
+      results,
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    const x = await downloader.downloadSingle()
+    expect(JSON.stringify(x)).toBe(JSON.stringify(i))
+  })
+
+  test('add extra header', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = { baseId: 'id', base: { id: 'id', __closure: { childIds: 1 } } }
+    fetchMocker.mockResponseIf(
+      (req) => req.headers.get('x-test') === 'asdf',
+      JSON.stringify(i.base)
+    )
+    const results = new AsyncGeneratorQueue<Item>()
+    const db = {
+      async add(): Promise<void> {
+        return Promise.resolve()
+      }
+    } as unknown as Cache
+    const headers = new Headers()
+    headers.set('x-test', 'asdf')
+    const downloader = new ServerDownloader({
+      database: db,
+      results,
+      serverUrl: 'http://speckle.test',
+      headers,
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    const x = await downloader.downloadSingle()
+    expect(JSON.stringify(x)).toBe(JSON.stringify(i))
+  })
+})

package/src/operations/serverDownloader.ts

@@ -0,0 +1,163 @@
+import BatchedPool from '../helpers/batchedPool.js'
+import Queue from '../helpers/queue.js'
+import { ObjectLoaderRuntimeError } from '../types/errors.js'
+import { Fetcher, isBase, Item } from '../types/types.js'
+import { Downloader } from './interfaces.js'
+import { BaseDownloadOptions } from './options.js'
+
+export default class ServerDownloader implements Downloader {
+  #requestUrlRootObj: string
+  #requestUrlChildren: string
+  #headers: HeadersInit
+  #options: BaseDownloadOptions
+  #fetch: Fetcher
+
+  #downloadQueue?: BatchedPool<string>
+
+  constructor(options: BaseDownloadOptions) {
+    this.#options = options
+    this.#fetch = options.fetch ?? ((...args) => globalThis.fetch(...args))
+
+    this.#headers = {}
+    if (options.headers) {
+      for (const header of options.headers.entries()) {
+        this.#headers[header[0]] = header[1]
+      }
+    }
+    this.#headers['Accept'] = `text/plain`
+
+    if (this.#options.token) {
+      this.#headers['Authorization'] = `Bearer ${this.#options.token}`
+    }
+    this.#requestUrlChildren = `${this.#options.serverUrl}/api/getobjects/${
+      this.#options.streamId
+    }`
+    this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${
+      this.#options.streamId
+    }/${this.#options.objectId}/single`
+  }
+
+  #getDownloadCountAndSizes(total: number): number[] {
+    if (total <= 50) {
+      return [total]
+    }
+
+    return [10000, 30000, 10000, 1000]
+  }
+
+  initializePool(params: { total: number; maxDownloadBatchWait?: number }) {
+    const { total } = params
+    this.#downloadQueue = new BatchedPool<string>({
+      concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
+      maxWaitTime: params.maxDownloadBatchWait,
+      processFunction: (batch: string[]) =>
+        this.downloadBatch({
+          batch,
+          url: this.#requestUrlChildren,
+          headers: this.#headers,
+          results: this.#options.results
+        })
+    })
+  }
+
+  #getPool(): BatchedPool<string> {
+    if (this.#downloadQueue) {
+      return this.#downloadQueue
+    }
+    throw new Error('Download pool is not initialized')
+  }
+
+  add(id: string): void {
+    this.#getPool().add(id)
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#downloadQueue?.disposeAsync()
+    await this.#getPool().disposeAsync()
+  }
+
+  #processJson(baseId: string, unparsedBase: string): Item {
+    let base: unknown
+    try {
+      base = JSON.parse(unparsedBase)
+    } catch (e: unknown) {
+      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
+    }
+    if (isBase(base)) {
+      return { baseId, base }
+    } else {
+      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+    }
+  }
+
+  async downloadBatch(params: {
+    batch: string[]
+    url: string
+    headers: HeadersInit
+    results: Queue<Item>
+  }): Promise<void> {
+    const { batch, url, headers, results } = params
+    const response = await this.#fetch(url, {
+      method: 'POST',
+      headers: { ...headers, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ objects: JSON.stringify(batch) })
+    })
+
+    this.#validateResponse(response)
+    if (!response.body) {
+      throw new Error('ReadableStream not supported or response has no body.')
+    }
+
+    const reader = response.body.getReader()
+    const decoder = new TextDecoder()
+    let buffer = '' // Temporary buffer to store incoming chunks
+
+    let count = 0
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      // Decode the chunk and add to buffer
+      buffer += decoder.decode(value, { stream: true })
+
+      // Try to process JSON objects from the buffer
+      let boundary = buffer.indexOf('\n')
+      while (boundary !== -1) {
+        const jsonString = buffer.slice(0, boundary)
+        buffer = buffer.slice(boundary + 1)
+        boundary = buffer.indexOf('\n')
+        if (jsonString) {
+          const pieces = jsonString.split('\t')
+          const [id, unparsedObj] = pieces
+          const item = this.#processJson(id, unparsedObj)
+          await this.#options.database.add(item)
+          results.add(item)
+          count++
+          if (count % 1000 === 0) {
+            await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen
+          }
+        }
+      }
+    }
+  }
+
+  async downloadSingle(): Promise<Item> {
+    const response = await this.#fetch(this.#requestUrlRootObj, {
+      headers: this.#headers
+    })
+    this.#validateResponse(response)
+    const responseText = await response.text()
+    const item = this.#processJson(this.#options.objectId, responseText)
+    return item
+  }
+
+  #validateResponse(response: Response): void {
+    if (!response.ok) {
+      if ([401, 403].includes(response.status)) {
+        throw new ObjectLoaderRuntimeError('You do not have access!')
+      }
+      throw new ObjectLoaderRuntimeError(
+        `Failed to fetch objects: ${response.status} ${response.statusText})`
+      )
+    }
+  }
+}

package/src/test/e2e.spec.ts

@@ -0,0 +1,36 @@
+import { describe, test, expect } from 'vitest'
+import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+import ObjectLoader2 from '../operations/objectLoader2.js'
+import { Base } from '../types/types.js'
+
+describe('e2e', () => {
+  test('download small model', async () => {
+    // Revit sample house (good for bim-like stuff with many display meshes)
+    //const resource = 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+    const loader = new ObjectLoader2({
+      serverUrl: 'https://app.speckle.systems',
+      streamId: 'da9e320dad',
+      objectId: '31d10c0cea569a1e26809658ed27e281',
+      indexedDB: new IDBFactory(),
+      keyRange: IDBKeyRange
+    })
+
+    const getObjectPromise = loader.getObject({
+      id: '1708a78e057e8115f924c620ba686db6'
+    })
+
+    const bases: Base[] = []
+    for await (const obj of loader.getObjectIterator()) {
+      bases.push(obj)
+    }
+
+    expect(await loader.getTotalObjectCount()).toBe(1328)
+    expect(bases.length).toBe(1328)
+    const base = await getObjectPromise
+    expect(base).toBeDefined()
+    expect(base.id).toBe('1708a78e057e8115f924c620ba686db6')
+    const base2 = await loader.getObject({ id: '3841e3cbc45d52c47bc2f1b7b0ad4eb9' })
+    expect(base2).toBeDefined()
+    expect(base2.id).toBe('3841e3cbc45d52c47bc2f1b7b0ad4eb9')
+  }, 10000)
+})

package/src/types/errors.ts

@@ -0,0 +1,25 @@
+/**
+ * Base ObjectLoader error
+ */
+class BaseError extends Error {
+  /**
+   * Default message if none is passed
+   */
+  static defaultMessage = 'Unexpected error occurred'
+
+  /**
+   * @param {string} [message]
+   */
+  constructor(message: string) {
+    message ||= new.target.defaultMessage
+    super(message)
+  }
+}
+
+export class ObjectLoaderConfigurationError extends BaseError {
+  static defaultMessage = 'Object loader configured incorrectly!'
+}
+
+export class ObjectLoaderRuntimeError extends BaseError {
+  static defaultMessage = 'Object loader encountered a runtime problem!'
+}