@speckle/objectloader2 2.25.7 → 2.25.9

package/src/{operations/downloaders → core/stages}/serverDownloader.ts

@@ -1,7 +1,8 @@
-import BatchedPool from '../../helpers/batchedPool.js'
-import Queue from '../../helpers/queue.js'
+import BatchedPool from '../../queues/batchedPool.js'
+import Queue from '../../queues/queue.js'
 import { ObjectLoaderRuntimeError } from '../../types/errors.js'
-import { Fetcher, isBase, Item, take } from '../../types/types.js'
+import { Fetcher, indexOf, isBase, take } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
 import { Downloader } from '../interfaces.js'
 
 export interface ServerDownloaderOptions {
@@ -13,6 +14,8 @@ export interface ServerDownloaderOptions {
   fetch?: Fetcher
 }
 
+const MAX_SAFARI_DECODE_BYTES = 2 * 1024 * 1024 * 1024 - 1024 * 1024 // 2GB minus a margin
+
 export default class ServerDownloader implements Downloader {
   #requestUrlRootObj: string
   #requestUrlChildren: string
@@ -20,9 +23,14 @@ export default class ServerDownloader implements Downloader {
   #options: ServerDownloaderOptions
   #fetch: Fetcher
   #results?: Queue<Item>
+  #total?: number
 
   #downloadQueue?: BatchedPool<string>
-  #decoder = new TextDecoder()
+  #decoder = new TextDecoder('utf-8', { fatal: true })
+  #decodedBytesCount = 0
+
+  #rawString: string = 'Objects.Other.RawEncoding'
+  #rawEncoding: Uint8Array
 
   constructor(options: ServerDownloaderOptions) {
     this.#options = options
@@ -46,6 +54,9 @@ export default class ServerDownloader implements Downloader {
     this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${
       this.#options.streamId
     }/${this.#options.objectId}/single`
+
+    const encoder = new TextEncoder()
+    this.#rawEncoding = encoder.encode(this.#rawString)
   }
 
   #getDownloadCountAndSizes(total: number): number[] {
@@ -63,6 +74,7 @@ export default class ServerDownloader implements Downloader {
   }): void {
     const { results, total } = params
     this.#results = results
+    this.#total = total
     this.#downloadQueue = new BatchedPool<string>({
       concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
       maxWaitTime: params.maxDownloadBatchWait,
@@ -86,22 +98,26 @@ export default class ServerDownloader implements Downloader {
     this.#getPool().add(id)
   }
 
-  async disposeAsync(): Promise<void> {
-    await this.#downloadQueue?.disposeAsync()
-  }
+  /*
+  This is the most frequently reported and confirmed reason for this error in Safari. There's a known bug in WebKit (Safari's rendering engine) where TextDecoder can fail or throw a RangeError after decoding around 2GB of data. Chrome and other browsers handle much larger amounts of data without this specific limitation.
 
-  #processJson(baseId: string, unparsedBase: string): Item {
-    let base: unknown
-    try {
-      base = JSON.parse(unparsedBase)
-    } catch (e: unknown) {
-      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
-    }
-    if (isBase(base)) {
-      return { baseId, base }
-    } else {
-      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+Why it happens: It seems to be an internal memory or indexing limitation within Safari's TextDecoder implementation. After a certain threshold of data has been processed by a TextDecoder instance, it starts throwing this error.
+
+Chrome's behavior: Chrome generally handles larger data sizes without this specific RangeError. It might become slow or run out of general memory, but not typically with this specific error.
+  */
+  decodeChunk(chunkBuffer: Uint8Array): string {
+    if (this.#decodedBytesCount + chunkBuffer.byteLength > MAX_SAFARI_DECODE_BYTES) {
+      // Safari is approaching its limit, create a new decoder
+      this.#decoder = new TextDecoder('utf-8', { fatal: true })
+      this.#decodedBytesCount = 0 // Reset counter for the new decoder
     }
+    const decodedText = this.#decoder.decode(chunkBuffer)
+    this.#decodedBytesCount += chunkBuffer.byteLength
+    return decodedText
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#downloadQueue?.disposeAsync()
   }
 
   async downloadBatch(params: {
@@ -130,7 +146,7 @@ export default class ServerDownloader implements Downloader {
       const { done, value } = await reader.read()
       if (done) break
 
-      leftover = await this.processArray(leftover, value, keys, async () => {
+      leftover = await this.#processArray(leftover, value, keys, async () => {
         count++
         if (count % 1000 === 0) {
           await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen
@@ -142,16 +158,20 @@ export default class ServerDownloader implements Downloader {
         'Items requested were not downloaded: ' + take(keys.values(), 10).join(',')
       )
     }
+    count += keys.size // count the leftovers
+    if (count >= this.#total!) {
+      await this.#results?.disposeAsync() // mark the queue as done
+    }
   }
 
-  async processArray(
+  async #processArray(
     leftover: Uint8Array,
     value: Uint8Array,
     keys: Set<string>,
     callback: () => Promise<void>
   ): Promise<Uint8Array> {
     //this concat will allocate a new array
-    const combined = this.concatUint8Arrays(leftover, value)
+    const combined = this.#concatUint8Arrays(leftover, value)
     let start = 0
 
     //subarray doesn't allocate
@@ -159,34 +179,65 @@ export default class ServerDownloader implements Downloader {
       if (combined[i] === 0x0a) {
         const line = combined.subarray(start, i) // line without \n
         //strings are allocated here
-        const item = this.processLine(line)
-        this.#results?.add(item)
+        const item = this.#processLine(line)
         start = i + 1
         await callback()
         keys.delete(item.baseId)
+        this.#results?.add(item)
       }
     }
     return combined.subarray(start) // carry over remainder
   }
 
-  processLine(line: Uint8Array): Item {
+  #processLine(line: Uint8Array): Item {
     for (let i = 0; i < line.length; i++) {
       if (line[i] === 0x09) {
         //this is a tab
-        const baseId = this.#decoder.decode(line.subarray(0, i))
-        const json = line.subarray(i + 1)
-        const base = this.#decoder.decode(json)
+        const baseId = this.decodeChunk(line.subarray(0, i))
+        const jsonBytes = line.subarray(i + 1)
+
+        if (!this.#isValidBytes(jsonBytes)) {
+          return { baseId, base: undefined }
+        }
+        const base = this.decodeChunk(jsonBytes)
         const item = this.#processJson(baseId, base)
-        item.size = json.length
+        item.size = jsonBytes.length
         return item
       }
     }
     throw new ObjectLoaderRuntimeError(
-      'Invalid line format: ' + this.#decoder.decode(line)
+      'Invalid line format in response: ' + this.decodeChunk(line)
     )
   }
 
-  concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
+  #processJson(baseId: string, unparsedBase: string): Item {
+    let base: unknown
+    try {
+      base = JSON.parse(unparsedBase)
+    } catch (e: unknown) {
+      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
+    }
+    if (isBase(base)) {
+      return { baseId, base }
+    } else {
+      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+    }
+  }
+  #isValidString(json: string): boolean {
+    if (!json.includes(this.#rawString)) {
+      return true
+    }
+    return false
+  }
+
+  #isValidBytes(json: Uint8Array): boolean {
+    if (indexOf(json, this.#rawEncoding) === -1) {
+      return true
+    }
+    return false
+  }
+
+  #concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
     const c = new Uint8Array(a.length + b.length)
     c.set(a, 0)
     c.set(b, a.length)
@@ -199,7 +250,13 @@ export default class ServerDownloader implements Downloader {
     })
     this.#validateResponse(response)
     const responseText = await response.text()
+    if (!this.#isValidString(responseText)) {
+      throw new ObjectLoaderRuntimeError('Invalid base object')
+    }
     const item = this.#processJson(this.#options.objectId, responseText)
+    if (!item.base) {
+      throw new ObjectLoaderRuntimeError('Invalid base object')
+    }
     item.size = 0
     return item
   }

package/src/{operations → core}/traverser.ts

@@ -1,4 +1,5 @@
-import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js'
+import { isScalar, isBase, isReference } from '../types/functions.js'
+import { Base, DataChunk } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 
 export type ProgressStage = 'download' | 'construction'

package/src/deferment/MemoryCache.test.ts

@@ -0,0 +1,187 @@
+import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { MemoryCache, MemoryCacheItem, MemoryCacheOptions } from './MemoryCache.js'
+import { Item, Base } from '../types/types.js'
+import { CustomLogger } from '../types/functions.js'
+
+const logger: CustomLogger = () => {
+  // console.log(message, ...optionalParams)
+}
+
+const defaultOptions: MemoryCacheOptions = {
+  maxSizeInMb: 10,
+  ttlms: 1000
+}
+
+const createItem = (id: string, size: number, referencedIds: string[] = []): Item => {
+  const base: Base & { data?: unknown } = {
+    id,
+    // eslint-disable-next-line camelcase
+    speckle_type: 'Base'
+  }
+
+  if (referencedIds.length > 0) {
+    base.data = referencedIds.map((referencedId) => ({
+      // eslint-disable-next-line camelcase
+      speckle_type: 'reference',
+      referencedId
+    }))
+  }
+
+  return {
+    baseId: id,
+    base,
+    size
+  }
+}
+
+describe('MemoryCacheItem', () => {
+  it('should correctly determine if it is expired', () => {
+    const item = createItem('1', 100)
+    const now = Date.now()
+    const cacheItem = new MemoryCacheItem(item, now + 1000)
+    expect(cacheItem.isExpired(now + 500)).toBe(false)
+    expect(cacheItem.isExpired(now + 1500)).toBe(true)
+  })
+
+  it('should update its access time', () => {
+    const item = createItem('1', 100)
+    const now = Date.now()
+    const cacheItem = new MemoryCacheItem(item, now + 1000)
+    cacheItem.setAccess(now + 500, 1000)
+    expect(cacheItem.isExpired(now + 1000)).toBe(false)
+    expect(cacheItem.isExpired(now + 1600)).toBe(true)
+  })
+
+  it('should return the correct item', () => {
+    const item = createItem('1', 100)
+    const cacheItem = new MemoryCacheItem(item, Date.now() + 1000)
+    expect(cacheItem.getItem()).toEqual(item)
+  })
+
+  it('should be done if expired', () => {
+    const item = createItem('1', 100)
+    const now = Date.now()
+    const cacheItem = new MemoryCacheItem(item, now + 1000)
+    expect(cacheItem.done(now + 1500)).toBe(true)
+    expect(cacheItem.done(now + 500)).toBe(false)
+  })
+})
+
+describe('MemoryCache', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('should add and get an item', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    const item = createItem('1', 100)
+    const requestItem = vi.fn()
+
+    cache.add(item, requestItem)
+    const retrieved = cache.get('1')
+
+    expect(retrieved).toEqual(item)
+    expect(requestItem).not.toHaveBeenCalled()
+  })
+
+  it('should update expiry on get', () => {
+    const cache = new MemoryCache({ ...defaultOptions, ttlms: 100 }, logger)
+    const item = createItem('1', 100)
+    cache.add(item, vi.fn())
+
+    vi.advanceTimersByTime(50)
+    const retrieved = cache.get('1')
+    expect(retrieved).toBeDefined()
+
+    vi.advanceTimersByTime(80)
+    const retrievedAgain = cache.get('1')
+    expect(retrievedAgain).toBeDefined()
+  })
+
+  it('should return undefined for non-existent item', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    expect(cache.get('non-existent')).toBeUndefined()
+  })
+
+  it('should scan for references and request missing ones', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    const requestItem = vi.fn()
+    const item = createItem('1', 100, ['2', '3'])
+
+    cache.add(createItem('3', 50), vi.fn()) // pre-cache one of the references
+    cache.scanForReferences(item.base, requestItem)
+
+    expect(requestItem).toHaveBeenCalledTimes(1)
+    expect(requestItem).toHaveBeenCalledWith('2')
+    expect(requestItem).not.toHaveBeenCalledWith('3')
+  })
+
+  it('should clean up expired items when size exceeds max', () => {
+    const options = { maxSizeInMb: 0, ttlms: 100 } // 100 bytes
+    const cache = new MemoryCache(options, logger)
+    const requestItem = vi.fn()
+
+    const item1 = createItem('1', 60)
+    const item2 = createItem('2', 60)
+
+    const now = 1
+    cache.add(item1, requestItem, now - 200)
+    cache.cleanCache(now)
+    cache.add(item2, requestItem, now + 100)
+
+    expect(cache.get('1')).toBeUndefined()
+    expect(cache.get('2')).toBeDefined()
+  })
+
+  it('should not clean up expired items if they have references', () => {
+    const options = { maxSizeInMb: 0.0001, ttlms: 100 } // 100 bytes
+    const cache = new MemoryCache(options, logger)
+    const requestItem = vi.fn()
+
+    const item1 = createItem('1', 60)
+    const item2 = createItem('2', 60, ['1'])
+
+    cache.add(item1, requestItem)
+    vi.advanceTimersByTime(110) // item1 expires
+    cache.add(item2, requestItem) // item2 adds a reference to item1
+
+    vi.advanceTimersByTime(100)
+
+    expect(cache.get('1')).toBeDefined()
+    expect(cache.get('2')).toBeDefined()
+  })
+
+  it('compareMaybeBasesByReferences should sort correctly', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    const requestItem = vi.fn()
+
+    const item1 = createItem('1', 10)
+    const item2 = createItem('2', 10)
+    const item3 = createItem('3', 10, ['1', '2'])
+    const item4 = createItem('4', 10, ['2'])
+
+    cache.add(item1, requestItem)
+    cache.add(item2, requestItem)
+    cache.add(item3, requestItem)
+    cache.add(item4, requestItem)
+
+    expect(cache.compareMaybeBasesByReferences('1', '2')).toBe(-1)
+    expect(cache.compareMaybeBasesByReferences('2', '1')).toBe(1)
+    expect(cache.compareMaybeBasesByReferences('1', '1')).toBe(0)
+    expect(cache.compareMaybeBasesByReferences('1', '5')).toBe(1)
+    expect(cache.compareMaybeBasesByReferences('5', '1')).toBe(-1)
+    expect(cache.compareMaybeBasesByReferences('5', '6')).toBe(0)
+  })
+
+  it('should throw when used after dispose', () => {
+    const cache = new MemoryCache(defaultOptions, logger)
+    cache.dispose()
+    const item = createItem('1', 100)
+    expect(() => cache.add(item, vi.fn())).toThrow('MemoryCache is disposed')
+    expect(() => cache.get('1')).toThrow('MemoryCache is disposed')
+  })
+})

package/src/deferment/MemoryCache.ts

@@ -0,0 +1,189 @@
+import { CustomLogger } from '../types/functions.js'
+import { Item } from '../types/types.js'
+
+export interface MemoryCacheOptions {
+  maxSizeInMb: number
+  ttlms: number
+}
+
+export class MemoryCacheItem {
+  private item: Item
+  private expiresAt: number // Timestamp in ms
+
+  constructor(item: Item, expiresAt: number) {
+    this.item = item
+    this.expiresAt = expiresAt
+  }
+
+  isExpired(now: number): boolean {
+    return now > this.expiresAt
+  }
+  setAccess(now: number, ttl: number): void {
+    this.expiresAt = now + ttl
+  }
+
+  getItem(): Item {
+    return this.item
+  }
+
+  done(now: number): boolean {
+    if (this.isExpired(now)) {
+      return true
+    }
+    return false
+  }
+}
+
+export class MemoryCache {
+  private isGathered: Map<string, boolean> = new Map()
+  private references: Map<string, number> = new Map()
+  private cache: Map<string, MemoryCacheItem> = new Map()
+
+  private options: MemoryCacheOptions
+  private logger: CustomLogger
+  private disposed = false
+  private currentSize = 0
+  private timer?: ReturnType<typeof setTimeout>
+
+  constructor(options: MemoryCacheOptions, logger: CustomLogger) {
+    this.options = options
+    this.logger = logger
+    this.resetGlobalTimer()
+  }
+
+  add(item: Item, requestItem: (id: string) => void, testNow?: number): void {
+    if (this.disposed) throw new Error('MemoryCache is disposed')
+    this.currentSize += item.size || 0
+    this.cache.set(
+      item.baseId,
+      new MemoryCacheItem(item, (testNow || this.now()) + this.options.ttlms)
+    )
+
+    if (!this.isGathered.has(item.baseId)) {
+      this.isGathered.set(item.baseId, true)
+      this.scanForReferences(item.base!, requestItem)
+    }
+  }
+
+  get(id: string): Item | undefined {
+    if (this.disposed) throw new Error('MemoryCache is disposed')
+    const item = this.cache.get(id)
+    if (item) {
+      item.setAccess(this.now(), this.options.ttlms)
+      return item.getItem()
+    }
+    return undefined
+  }
+
+  scanForReferences(data: unknown, requestItem: (id: string) => void): void {
+    const scan = (item: unknown): void => {
+      // Stop if the item is null or not an object (i.e., primitive)
+      if (item === null || typeof item !== 'object') {
+        return
+      }
+
+      // If it's an array, scan each element
+      if (Array.isArray(item)) {
+        for (const element of item) {
+          scan(element)
+        }
+        return
+      }
+
+      // If it's an object, scan its properties
+      for (const key in item) {
+        if (Object.prototype.hasOwnProperty.call(item, key)) {
+          // We found the target property!
+          if (key === 'referencedId') {
+            const value = (item as { referencedId: unknown }).referencedId
+            // Ensure the value is a string before adding it
+            if (typeof value === 'string') {
+              this.references.set(value, (this.references.get(value) || 0) + 1)
+              if (!this.cache.has(value)) {
+                requestItem(value)
+              }
+            }
+          }
+
+          // Continue scanning deeper into the object's properties
+          scan((item as Record<string, unknown>)[key])
+        }
+      }
+    }
+
+    scan(data)
+  }
+
+  private resetGlobalTimer(): void {
+    const run = (): void => {
+      this.cleanCache()
+      this.timer = setTimeout(run, this.options.ttlms)
+    }
+    this.timer = setTimeout(run, this.options.ttlms)
+  }
+
+  private now(): number {
+    return Date.now()
+  }
+
+  cleanCache(testNow?: number): void {
+    const maxSizeBytes = this.options.maxSizeInMb * 1024 * 1024
+    if (this.currentSize < maxSizeBytes) {
+      this.logger(
+        `cache size (${this.currentSize} < ${maxSizeBytes}) is ok, no need to clean`
+      )
+      return
+    }
+    const now = testNow || this.now()
+    let cleaned = 0
+    const start = performance.now()
+    for (const deferredBase of Array.from(this.cache.values())
+      .filter((x) => x.isExpired(now))
+      .sort((a, b) =>
+        this.compareMaybeBasesByReferences(a.getItem().baseId, b.getItem().baseId)
+      )) {
+      if (deferredBase.done(now)) {
+        const id = deferredBase.getItem().baseId
+        const referenceCount = this.references.get(id) || 0
+        if (referenceCount > 0) {
+          // Skip eviction for items with reference counts greater than 0,
+          // as they are still in use and should not be removed from the cache.
+          continue
+        }
+        this.currentSize -= deferredBase.getItem().size || 0
+        this.cache.delete(id)
+        cleaned++
+        if (this.currentSize < maxSizeBytes) {
+          break
+        }
+      }
+    }
+    this.logger(
+      `cleaned cache: cleaned ${cleaned}, cached ${this.cache.size}, time ${
+        performance.now() - start
+      }`
+    )
+    return
+  }
+
+  compareMaybeBasesByReferences(id1: string, id2: string): number {
+    const a = this.references.get(id1)
+    const b = this.references.get(id2)
+    if (a === undefined && b === undefined) return 0
+    if (a === undefined) return -1
+    if (b === undefined) return 1
+    return a - b
+  }
+
+  dispose(): void {
+    if (this.disposed) return
+    this.disposed = true
+    if (this.timer) {
+      clearTimeout(this.timer)
+      this.timer = undefined
+    }
+    this.cache.clear()
+    this.isGathered.clear()
+    this.references.clear()
+  }
+}