@speckle/objectloader2 2.25.4 → 2.25.8

package/src/{operations/downloaders → core/stages}/serverDownloader.spec.ts

@@ -3,14 +3,14 @@ import createFetchMock from 'vitest-fetch-mock'
 import { vi } from 'vitest'
 import { Item } from '../../types/types.js'
 import ServerDownloader from './serverDownloader.js'
-import { MemoryPump } from '../../helpers/memoryPump.js'
+import AsyncGeneratorQueue from '../../queues/asyncGeneratorQueue.js'
 
 describe('downloader', () => {
   test('download batch of one', async () => {
     const fetchMocker = createFetchMock(vi)
     const i: Item = { baseId: 'id', base: { id: 'id', speckle_type: 'type' } }
     fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
-    const pump = new MemoryPump()
+    const gathered = new AsyncGeneratorQueue<Item>()
     const downloader = new ServerDownloader({
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
@@ -18,12 +18,18 @@ describe('downloader', () => {
       token: 'token',
       fetch: fetchMocker
     })
-    downloader.initializePool({ results: pump, total: 1, maxDownloadBatchWait: 200 })
+    downloader.initializePool({
+      results: gathered,
+      total: 1,
+      maxDownloadBatchWait: 200
+    })
     downloader.add('id')
-    await downloader.disposeAsync()
     const r = []
-    for await (const x of pump.gather([i.baseId])) {
+    for await (const x of gathered.consume()) {
       r.push(x)
+      if (r.length >= 1) {
+        break
+      }
     }
 
     expect(r).toMatchSnapshot()
@@ -38,7 +44,7 @@ describe('downloader', () => {
       'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
     )
 
-    const pump = new MemoryPump()
+    const gathered = new AsyncGeneratorQueue<Item>()
     const downloader = new ServerDownloader({
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
@@ -47,13 +53,20 @@ describe('downloader', () => {
 
       fetch: fetchMocker
     })
-    downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 })
+    downloader.initializePool({
+      results: gathered,
+      total: 2,
+      maxDownloadBatchWait: 200
+    })
     downloader.add('id1')
     downloader.add('id2')
     await downloader.disposeAsync()
     const r = []
-    for await (const x of pump.gather([i1.baseId, i2.baseId])) {
+    for await (const x of gathered.consume()) {
       r.push(x)
+      if (r.length >= 2) {
+        break
+      }
     }
 
     expect(r).toMatchSnapshot()
@@ -75,7 +88,7 @@ describe('downloader', () => {
         '\n'
     )
 
-    const pump = new MemoryPump()
+    const gathered = new AsyncGeneratorQueue<Item>()
     const downloader = new ServerDownloader({
       serverUrl: 'http://speckle.test',
       streamId: 'streamId',
@@ -84,20 +97,98 @@ describe('downloader', () => {
 
       fetch: fetchMocker
     })
-    downloader.initializePool({ results: pump, total: 3, maxDownloadBatchWait: 200 })
+    downloader.initializePool({
+      results: gathered,
+      total: 3,
+      maxDownloadBatchWait: 200
+    })
     downloader.add('id1')
     downloader.add('id2')
     downloader.add('id3')
     await downloader.disposeAsync()
     const r = []
-    for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) {
+    for await (const x of gathered.consume()) {
       r.push(x)
+      if (r.length >= 3) {
+        break
+      }
     }
 
     expect(r).toMatchSnapshot()
     await downloader.disposeAsync()
   })
 
+  test('download batch of three with Objects.Other.RawEncoding', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', speckle_type: 'type' } }
+    const i2: Item = {
+      baseId: 'id2',
+      base: { id: 'id2', speckle_type: 'Objects.Other.RawEncoding' }
+    }
+    const i3: Item = { baseId: 'id3', base: { id: 'id3', speckle_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' +
+        JSON.stringify(i1.base) +
+        '\nid2\t' +
+        JSON.stringify(i2.base) +
+        '\nid3\t' +
+        JSON.stringify(i3.base) +
+        '\n'
+    )
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    downloader.initializePool({
+      results: gathered,
+      total: 3,
+      maxDownloadBatchWait: 200
+    })
+    downloader.add('id1')
+    downloader.add('id2')
+    downloader.add('id3')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of gathered.consume()) {
+      r.push(x)
+      if (r.length >= 3) {
+        break
+      }
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test("download Objects.Other.RawEncoding doesn't exist", async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = {
+      baseId: 'id',
+      base: {
+        id: 'id',
+        speckle_type: 'Objects.Other.RawEncoding',
+        __closure: { childIds: 1 }
+      }
+    }
+    fetchMocker.mockResponseOnce(JSON.stringify(i.base))
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://speckle.test',
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker
+    })
+    await expect(downloader.downloadSingle()).rejects.toThrow()
+    await downloader.disposeAsync()
+  })
+
   test('download single exists', async () => {
     const fetchMocker = createFetchMock(vi)
     const i: Item = {

package/src/{operations/downloaders → core/stages}/serverDownloader.ts

@@ -1,7 +1,8 @@
-import BatchedPool from '../../helpers/batchedPool.js'
-import Queue from '../../helpers/queue.js'
+import BatchedPool from '../../queues/batchedPool.js'
+import Queue from '../../queues/queue.js'
 import { ObjectLoaderRuntimeError } from '../../types/errors.js'
-import { Fetcher, isBase, Item, take } from '../../types/types.js'
+import { Fetcher, indexOf, isBase, take } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
 import { Downloader } from '../interfaces.js'
 
 export interface ServerDownloaderOptions {
@@ -13,6 +14,8 @@ export interface ServerDownloaderOptions {
   fetch?: Fetcher
 }
 
+const MAX_SAFARI_DECODE_BYTES = 2 * 1024 * 1024 * 1024 - 1024 * 1024 // 2GB minus a margin
+
 export default class ServerDownloader implements Downloader {
   #requestUrlRootObj: string
   #requestUrlChildren: string
@@ -20,9 +23,14 @@ export default class ServerDownloader implements Downloader {
   #options: ServerDownloaderOptions
   #fetch: Fetcher
   #results?: Queue<Item>
+  #total?: number
 
   #downloadQueue?: BatchedPool<string>
-  #decoder = new TextDecoder()
+  #decoder = new TextDecoder('utf-8', { fatal: true })
+  #decodedBytesCount = 0
+
+  #rawString: string = 'Objects.Other.RawEncoding'
+  #rawEncoding: Uint8Array
 
   constructor(options: ServerDownloaderOptions) {
     this.#options = options
@@ -46,6 +54,9 @@ export default class ServerDownloader implements Downloader {
     this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${
       this.#options.streamId
     }/${this.#options.objectId}/single`
+
+    const encoder = new TextEncoder()
+    this.#rawEncoding = encoder.encode(this.#rawString)
   }
 
   #getDownloadCountAndSizes(total: number): number[] {
@@ -53,7 +64,7 @@ export default class ServerDownloader implements Downloader {
       return [total]
     }
 
-    return [10000, 30000, 10000, 1000]
+    return [10000, 25000, 10000, 1000]
   }
 
   initializePool(params: {
@@ -63,6 +74,7 @@ export default class ServerDownloader implements Downloader {
   }): void {
     const { results, total } = params
     this.#results = results
+    this.#total = total
     this.#downloadQueue = new BatchedPool<string>({
       concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
       maxWaitTime: params.maxDownloadBatchWait,
@@ -86,22 +98,26 @@ export default class ServerDownloader implements Downloader {
     this.#getPool().add(id)
   }
 
-  async disposeAsync(): Promise<void> {
-    await this.#downloadQueue?.disposeAsync()
-  }
+  /*
+  This is the most frequently reported and confirmed reason for this error in Safari. There's a known bug in WebKit (Safari's rendering engine) where TextDecoder can fail or throw a RangeError after decoding around 2GB of data. Chrome and other browsers handle much larger amounts of data without this specific limitation.
 
-  #processJson(baseId: string, unparsedBase: string): Item {
-    let base: unknown
-    try {
-      base = JSON.parse(unparsedBase)
-    } catch (e: unknown) {
-      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
-    }
-    if (isBase(base)) {
-      return { baseId, base }
-    } else {
-      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+Why it happens: It seems to be an internal memory or indexing limitation within Safari's TextDecoder implementation. After a certain threshold of data has been processed by a TextDecoder instance, it starts throwing this error.
+
+Chrome's behavior: Chrome generally handles larger data sizes without this specific RangeError. It might become slow or run out of general memory, but not typically with this specific error.
+  */
+  decodeChunk(chunkBuffer: Uint8Array): string {
+    if (this.#decodedBytesCount + chunkBuffer.byteLength > MAX_SAFARI_DECODE_BYTES) {
+      // Safari is approaching its limit, create a new decoder
+      this.#decoder = new TextDecoder('utf-8', { fatal: true })
+      this.#decodedBytesCount = 0 // Reset counter for the new decoder
     }
+    const decodedText = this.#decoder.decode(chunkBuffer)
+    this.#decodedBytesCount += chunkBuffer.byteLength
+    return decodedText
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#downloadQueue?.disposeAsync()
   }
 
   async downloadBatch(params: {
@@ -130,7 +146,7 @@ export default class ServerDownloader implements Downloader {
       const { done, value } = await reader.read()
       if (done) break
 
-      leftover = await this.processArray(leftover, value, keys, async () => {
+      leftover = await this.#processArray(leftover, value, keys, async () => {
         count++
         if (count % 1000 === 0) {
           await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen
@@ -142,16 +158,20 @@ export default class ServerDownloader implements Downloader {
         'Items requested were not downloaded: ' + take(keys.values(), 10).join(',')
       )
     }
+    count += keys.size // count the leftovers
+    if (count >= this.#total!) {
+      await this.#results?.disposeAsync() // mark the queue as done
+    }
   }
 
-  async processArray(
+  async #processArray(
     leftover: Uint8Array,
     value: Uint8Array,
     keys: Set<string>,
     callback: () => Promise<void>
   ): Promise<Uint8Array> {
     //this concat will allocate a new array
-    const combined = this.concatUint8Arrays(leftover, value)
+    const combined = this.#concatUint8Arrays(leftover, value)
     let start = 0
 
     //subarray doesn't allocate
@@ -159,34 +179,65 @@ export default class ServerDownloader implements Downloader {
       if (combined[i] === 0x0a) {
         const line = combined.subarray(start, i) // line without \n
         //strings are allocated here
-        const item = this.processLine(line)
-        this.#results?.add(item)
+        const item = this.#processLine(line)
         start = i + 1
         await callback()
         keys.delete(item.baseId)
+        this.#results?.add(item)
       }
     }
     return combined.subarray(start) // carry over remainder
   }
 
-  processLine(line: Uint8Array): Item {
+  #processLine(line: Uint8Array): Item {
     for (let i = 0; i < line.length; i++) {
       if (line[i] === 0x09) {
         //this is a tab
-        const baseId = this.#decoder.decode(line.subarray(0, i))
-        const json = line.subarray(i + 1)
-        const base = this.#decoder.decode(json)
+        const baseId = this.decodeChunk(line.subarray(0, i))
+        const jsonBytes = line.subarray(i + 1)
+
+        if (!this.#isValidBytes(jsonBytes)) {
+          return { baseId, base: undefined }
+        }
+        const base = this.decodeChunk(jsonBytes)
         const item = this.#processJson(baseId, base)
-        item.size = json.length
+        item.size = jsonBytes.length
         return item
       }
     }
     throw new ObjectLoaderRuntimeError(
-      'Invalid line format: ' + this.#decoder.decode(line)
+      'Invalid line format in response: ' + this.decodeChunk(line)
     )
   }
 
-  concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
+  #processJson(baseId: string, unparsedBase: string): Item {
+    let base: unknown
+    try {
+      base = JSON.parse(unparsedBase)
+    } catch (e: unknown) {
+      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
+    }
+    if (isBase(base)) {
+      return { baseId, base }
+    } else {
+      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+    }
+  }
+  #isValidString(json: string): boolean {
+    if (!json.includes(this.#rawString)) {
+      return true
+    }
+    return false
+  }
+
+  #isValidBytes(json: Uint8Array): boolean {
+    if (indexOf(json, this.#rawEncoding) === -1) {
+      return true
+    }
+    return false
+  }
+
+  #concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
     const c = new Uint8Array(a.length + b.length)
     c.set(a, 0)
     c.set(b, a.length)
@@ -199,7 +250,13 @@ export default class ServerDownloader implements Downloader {
     })
     this.#validateResponse(response)
     const responseText = await response.text()
+    if (!this.#isValidString(responseText)) {
+      throw new ObjectLoaderRuntimeError('Invalid base object')
+    }
     const item = this.#processJson(this.#options.objectId, responseText)
+    if (!item.base) {
+      throw new ObjectLoaderRuntimeError('Invalid base object')
+    }
     item.size = 0
     return item
   }

package/src/{operations → core}/traverser.ts

@@ -1,4 +1,5 @@
-import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js'
+import { isScalar, isBase, isReference } from '../types/functions.js'
+import { Base, DataChunk } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 
 export type ProgressStage = 'download' | 'construction'

package/src/deferment/MemoryCache.test.ts

@@ -0,0 +1,187 @@
+import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { MemoryCache, MemoryCacheItem, MemoryCacheOptions } from './MemoryCache.js'
+import { Item, Base } from '../types/types.js'
+import { CustomLogger } from '../types/functions.js'
+
+const logger: CustomLogger = () => {
+  // console.log(message, ...optionalParams)
+}
+
+const defaultOptions: MemoryCacheOptions = {
+  maxSizeInMb: 10,
+  ttlms: 1000
+}
+
+const createItem = (id: string, size: number, referencedIds: string[] = []): Item => {
+  const base: Base & { data?: unknown } = {
+    id,
+    // eslint-disable-next-line camelcase
+    speckle_type: 'Base'
+  }
+
+  if (referencedIds.length > 0) {
+    base.data = referencedIds.map((referencedId) => ({
+      // eslint-disable-next-line camelcase
+      speckle_type: 'reference',
+      referencedId
+    }))
+  }
+
+  return {
+    baseId: id,
+    base,
+    size
+  }
+}
+
+describe('MemoryCacheItem', () => {
+  it('should correctly determine if it is expired', () => {
+    const item = createItem('1', 100)
+    const now = Date.now()
+    const cacheItem = new MemoryCacheItem(item, now + 1000)
+    expect(cacheItem.isExpired(now + 500)).toBe(false)
+    expect(cacheItem.isExpired(now + 1500)).toBe(true)
+  })
+
+  it('should update its access time', () => {
+    const item = createItem('1', 100)
+    const now = Date.now()
+    const cacheItem = new MemoryCacheItem(item, now + 1000)
+    cacheItem.setAccess(now + 500, 1000)
+    expect(cacheItem.isExpired(now + 1000)).toBe(false)
+    expect(cacheItem.isExpired(now + 1600)).toBe(true)
+  })
+
+  it('should return the correct item', () => {
+    const item = createItem('1', 100)
+    const cacheItem = new MemoryCacheItem(item, Date.now() + 1000)
+    expect(cacheItem.getItem()).toEqual(item)
+  })
+
+  it('should be done if expired', () => {
+    const item = createItem('1', 100)
+    const now = Date.now()
+    const cacheItem = new MemoryCacheItem(item, now + 1000)
+    expect(cacheItem.done(now + 1500)).toBe(true)
+    expect(cacheItem.done(now + 500)).toBe(false)
+  })
+})
+
+describe('MemoryCache', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('should add and get an item', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    const item = createItem('1', 100)
+    const requestItem = vi.fn()
+
+    cache.add(item, requestItem)
+    const retrieved = cache.get('1')
+
+    expect(retrieved).toEqual(item)
+    expect(requestItem).not.toHaveBeenCalled()
+  })
+
+  it('should update expiry on get', () => {
+    const cache = new MemoryCache({ ...defaultOptions, ttlms: 100 }, logger)
+    const item = createItem('1', 100)
+    cache.add(item, vi.fn())
+
+    vi.advanceTimersByTime(50)
+    const retrieved = cache.get('1')
+    expect(retrieved).toBeDefined()
+
+    vi.advanceTimersByTime(80)
+    const retrievedAgain = cache.get('1')
+    expect(retrievedAgain).toBeDefined()
+  })
+
+  it('should return undefined for non-existent item', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    expect(cache.get('non-existent')).toBeUndefined()
+  })
+
+  it('should scan for references and request missing ones', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    const requestItem = vi.fn()
+    const item = createItem('1', 100, ['2', '3'])
+
+    cache.add(createItem('3', 50), vi.fn()) // pre-cache one of the references
+    cache.scanForReferences(item.base, requestItem)
+
+    expect(requestItem).toHaveBeenCalledTimes(1)
+    expect(requestItem).toHaveBeenCalledWith('2')
+    expect(requestItem).not.toHaveBeenCalledWith('3')
+  })
+
+  it('should clean up expired items when size exceeds max', () => {
+    const options = { maxSizeInMb: 0, ttlms: 100 } // 100 bytes
+    const cache = new MemoryCache(options, logger)
+    const requestItem = vi.fn()
+
+    const item1 = createItem('1', 60)
+    const item2 = createItem('2', 60)
+
+    const now = 1
+    cache.add(item1, requestItem, now - 200)
+    cache.cleanCache(now)
+    cache.add(item2, requestItem, now + 100)
+
+    expect(cache.get('1')).toBeUndefined()
+    expect(cache.get('2')).toBeDefined()
+  })
+
+  it('should not clean up expired items if they have references', () => {
+    const options = { maxSizeInMb: 0.0001, ttlms: 100 } // 100 bytes
+    const cache = new MemoryCache(options, logger)
+    const requestItem = vi.fn()
+
+    const item1 = createItem('1', 60)
+    const item2 = createItem('2', 60, ['1'])
+
+    cache.add(item1, requestItem)
+    vi.advanceTimersByTime(110) // item1 expires
+    cache.add(item2, requestItem) // item2 adds a reference to item1
+
+    vi.advanceTimersByTime(100)
+
+    expect(cache.get('1')).toBeDefined()
+    expect(cache.get('2')).toBeDefined()
+  })
+
+  it('compareMaybeBasesByReferences should sort correctly', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    const requestItem = vi.fn()
+
+    const item1 = createItem('1', 10)
+    const item2 = createItem('2', 10)
+    const item3 = createItem('3', 10, ['1', '2'])
+    const item4 = createItem('4', 10, ['2'])
+
+    cache.add(item1, requestItem)
+    cache.add(item2, requestItem)
+    cache.add(item3, requestItem)
+    cache.add(item4, requestItem)
+
+    expect(cache.compareMaybeBasesByReferences('1', '2')).toBe(-1)
+    expect(cache.compareMaybeBasesByReferences('2', '1')).toBe(1)
+    expect(cache.compareMaybeBasesByReferences('1', '1')).toBe(0)
+    expect(cache.compareMaybeBasesByReferences('1', '5')).toBe(1)
+    expect(cache.compareMaybeBasesByReferences('5', '1')).toBe(-1)
+    expect(cache.compareMaybeBasesByReferences('5', '6')).toBe(0)
+  })
+
+  it('should throw when used after dispose', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    cache.dispose()
+    const item = createItem('1', 100)
+    expect(() => cache.add(item, vi.fn())).toThrow('MemoryCache is disposed')
+    expect(() => cache.get('1')).toThrow('MemoryCache is disposed')
+  })
+})