@synchronized-console/objectloader2 2.25.7 → 2.31.14

package/src/core/stages/serverDownloader.spec.ts

@@ -0,0 +1,382 @@
+import { describe, expect, test } from 'vitest'
+import createFetchMock from 'vitest-fetch-mock'
+import { vi } from 'vitest'
+import { Item } from '../../types/types.js'
+import ServerDownloader from './serverDownloader.js'
+import AsyncGeneratorQueue from '../../queues/asyncGeneratorQueue.js'
+
+describe('downloader', () => {
+  test('download batch of one', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = { baseId: 'id', base: { id: 'id', synchronized_type: 'type' } }
+    fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    downloader.initialize({
+      results: gathered,
+      total: 1,
+      maxDownloadBatchWait: 200
+    })
+    downloader.add('id')
+    const r = []
+    for await (const x of gathered.consume()) {
+      r.push(x)
+      if (r.length >= 1) {
+        break
+      }
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('download batch of two', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', synchronized_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id2', synchronized_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
+    )
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    downloader.initialize({
+      results: gathered,
+      total: 2,
+      maxDownloadBatchWait: 200
+    })
+    downloader.add('id1')
+    downloader.add('id2')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of gathered.consume()) {
+      r.push(x)
+      if (r.length >= 2) {
+        break
+      }
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('download batch of three', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', synchronized_type: 'type' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id2', synchronized_type: 'type' } }
+    const i3: Item = { baseId: 'id3', base: { id: 'id3', synchronized_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' +
+        JSON.stringify(i1.base) +
+        '\nid2\t' +
+        JSON.stringify(i2.base) +
+        '\nid3\t' +
+        JSON.stringify(i3.base) +
+        '\n'
+    )
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    downloader.initialize({
+      results: gathered,
+      total: 3,
+      maxDownloadBatchWait: 200
+    })
+    downloader.add('id1')
+    downloader.add('id2')
+    downloader.add('id3')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of gathered.consume()) {
+      r.push(x)
+      if (r.length >= 3) {
+        break
+      }
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('download batch of three with Objects.Other.RawEncoding', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i1: Item = { baseId: 'id1', base: { id: 'id1', synchronized_type: 'type' } }
+    const i2: Item = {
+      baseId: 'id2',
+      base: { id: 'id2', synchronized_type: 'Objects.Other.RawEncoding' }
+    }
+    const i3: Item = { baseId: 'id3', base: { id: 'id3', synchronized_type: 'type' } }
+    fetchMocker.mockResponseOnce(
+      'id1\t' +
+        JSON.stringify(i1.base) +
+        '\nid2\t' +
+        JSON.stringify(i2.base) +
+        '\nid3\t' +
+        JSON.stringify(i3.base) +
+        '\n'
+    )
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    downloader.initialize({
+      results: gathered,
+      total: 3,
+      maxDownloadBatchWait: 200
+    })
+    downloader.add('id1')
+    downloader.add('id2')
+    downloader.add('id3')
+    await downloader.disposeAsync()
+    const r = []
+    for await (const x of gathered.consume()) {
+      r.push(x)
+      if (r.length >= 3) {
+        break
+      }
+    }
+
+    expect(r).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test("download Objects.Other.RawEncoding doesn't exist", async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = {
+      baseId: 'id',
+      base: {
+        id: 'id',
+        synchronized_type: 'Objects.Other.RawEncoding',
+        __closure: { childIds: 1 }
+      }
+    }
+    fetchMocker.mockResponseOnce(JSON.stringify(i.base))
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    await expect(downloader.downloadSingle()).rejects.toThrow()
+    await downloader.disposeAsync()
+  })
+
+  test('download single exists', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = {
+      baseId: 'id',
+      base: { id: 'id', synchronized_type: 'type', __closure: { childIds: 1 } }
+    }
+    fetchMocker.mockResponseOnce(JSON.stringify(i.base))
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    const x = await downloader.downloadSingle()
+    expect(x).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('add extra header', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const i: Item = {
+      baseId: 'id',
+      base: { id: 'id', synchronized_type: 'type', __closure: { childIds: 1 } }
+    }
+    fetchMocker.mockResponseIf(
+      (req) => req.headers.get('x-test') === 'asdf',
+      JSON.stringify(i.base)
+    )
+    const headers = new Headers()
+    headers.set('x-test', 'asdf')
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      headers,
+      streamId: 'streamId',
+      objectId: i.baseId,
+      token: 'token',
+
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    const x = await downloader.downloadSingle()
+    expect(x).toMatchSnapshot()
+    await downloader.disposeAsync()
+  })
+
+  test('can dispose used', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const headers = new Headers()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      headers,
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'token',
+
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+    await downloader.disposeAsync()
+  })
+
+  test('nothing is frozen when validateResponse returns 403', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+    // Mock a 403 Forbidden response
+    fetchMocker.mockResponseOnce('', { status: 403, statusText: 'Forbidden' })
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'invalid-token',
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+
+    try {
+      downloader.initialize({
+        results: gathered,
+        total: 2,
+        maxDownloadBatchWait: 100
+      })
+
+      // Add items to trigger batch processing
+      downloader.add('id1')
+      downloader.add('id2')
+
+      // Wait for the batch to be processed and fail with 403
+      await new Promise((resolve) => setTimeout(resolve, 200))
+
+      // Verify that the error was logged (indicating the batch processing failed)
+      expect(consoleErrorSpy).toHaveBeenCalledWith(
+        'Batch processing failed:',
+        expect.any(Error)
+      )
+
+      // The key test: verify we can still dispose the downloader properly
+      // This ensures the system isn't frozen and can clean up resources
+      const disposePromise = downloader.disposeAsync()
+
+      // Add a timeout to ensure disposal doesn't hang indefinitely
+      const timeoutPromise = new Promise((_, reject) => {
+        setTimeout(() => reject(new Error('Disposal timed out')), 5000)
+      })
+
+      // This should complete without timing out or throwing
+      await Promise.race([disposePromise, timeoutPromise])
+
+      // Additional verification: the batching queue should be marked as disposed
+      // We can't directly access the private field, but we can verify disposal completed
+      expect(true).toBe(true) // If we reach here, disposal succeeded
+    } finally {
+      consoleErrorSpy.mockRestore()
+    }
+  })
+
+  test('system remains functional after 403 error and can be properly cleaned up', async () => {
+    const fetchMocker = createFetchMock(vi)
+    const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+    // First call returns 403, subsequent calls should not be made due to queue disposal
+    fetchMocker.mockResponseOnce('', { status: 403, statusText: 'Forbidden' })
+
+    const gathered = new AsyncGeneratorQueue<Item>()
+    const downloader = new ServerDownloader({
+      serverUrl: 'http://synchronized.test',
+      streamId: 'streamId',
+      objectId: 'objectId',
+      token: 'invalid-token',
+      fetch: fetchMocker,
+      logger: (): void => {}
+    })
+
+    try {
+      downloader.initialize({
+        results: gathered,
+        total: 5,
+        maxDownloadBatchWait: 50
+      })
+
+      // Add first batch that will trigger the 403 error
+      downloader.add('id1')
+      downloader.add('id2')
+
+      // Wait for first batch to fail
+      await new Promise((resolve) => setTimeout(resolve, 100))
+
+      // Verify error was logged
+      expect(consoleErrorSpy).toHaveBeenCalledWith(
+        'Batch processing failed:',
+        expect.any(Error)
+      )
+
+      // Try to add more items after the failure
+      // These should be ignored since the queue is now disposed
+      downloader.add('id3')
+      downloader.add('id4')
+      downloader.add('id5')
+
+      // Wait a bit more to ensure no additional processing attempts
+      await new Promise((resolve) => setTimeout(resolve, 100))
+
+      // Note: The batching queue might make multiple attempts before disposal
+      // The key is that disposal should still work regardless of how many calls were made
+      expect(fetchMocker).toHaveBeenCalled()
+
+      // Critical test: disposal should complete without hanging
+      const start = Date.now()
+      await downloader.disposeAsync()
+      const elapsed = Date.now() - start
+
+      // Disposal should be quick (under 1 second) and not hang
+      expect(elapsed).toBeLessThan(1000)
+
+      // Verify that the results queue can also be disposed properly
+      await gathered.disposeAsync()
+    } finally {
+      consoleErrorSpy.mockRestore()
+    }
+  })
+})

package/src/{operations/downloaders → core/stages}/serverDownloader.ts

@@ -1,7 +1,8 @@
-import BatchedPool from '../../helpers/batchedPool.js'
-import Queue from '../../helpers/queue.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
 import { ObjectLoaderRuntimeError } from '../../types/errors.js'
-import { Fetcher, isBase, Item, take } from '../../types/types.js'
+import { CustomLogger, Fetcher, indexOf, isBase, take } from '../../types/functions.js'
+import { Item, ObjectAttributeMask } from '../../types/types.js'
 import { Downloader } from '../interfaces.js'
 
 export interface ServerDownloaderOptions {
@@ -10,9 +11,14 @@ export interface ServerDownloaderOptions {
   objectId: string
   token?: string
   headers?: Headers
+  logger: CustomLogger
   fetch?: Fetcher
+  attributeMask?: ObjectAttributeMask
+  objectTypeMask?: string[] // Temporary until server filters are elevated and ready
 }
 
+const MAX_SAFARI_DECODE_BYTES = 2 * 1024 * 1024 * 1024 - 1024 * 1024 // 2GB minus a margin
+
 export default class ServerDownloader implements Downloader {
   #requestUrlRootObj: string
   #requestUrlChildren: string
@@ -20,12 +26,19 @@ export default class ServerDownloader implements Downloader {
   #options: ServerDownloaderOptions
   #fetch: Fetcher
   #results?: Queue<Item>
+  #total?: number
+  #logger: CustomLogger
+
+  #downloadQueue?: BatchingQueue<string>
+  #decoder = new TextDecoder('utf-8', { fatal: true })
+  #decodedBytesCount = 0
 
-  #downloadQueue?: BatchedPool<string>
-  #decoder = new TextDecoder()
+  #rawStrings: Array<string> = []
+  #rawEncodings: Array<Uint8Array> = []
 
   constructor(options: ServerDownloaderOptions) {
     this.#options = options
+    this.#logger = options.logger
     this.#fetch =
       options.fetch ?? ((...args): Promise<Response> => globalThis.fetch(...args))
 
@@ -40,32 +53,34 @@ export default class ServerDownloader implements Downloader {
     if (this.#options.token) {
       this.#headers['Authorization'] = `Bearer ${this.#options.token}`
     }
-    this.#requestUrlChildren = `${this.#options.serverUrl}/api/getobjects/${
+    this.#requestUrlChildren = `${this.#options.serverUrl}/api/v2/projects/${
       this.#options.streamId
-    }`
+    }/object-stream/`
+
     this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${
       this.#options.streamId
     }/${this.#options.objectId}/single`
-  }
 
-  #getDownloadCountAndSizes(total: number): number[] {
-    if (total <= 50) {
-      return [total]
+    const encoder = new TextEncoder()
+    if (options.objectTypeMask) {
+      this.#rawStrings.push(...options.objectTypeMask)
+      this.#rawEncodings.push(
+        ...options.objectTypeMask.map((val) => encoder.encode(val))
+      )
     }
-
-    return [10000, 25000, 10000, 1000]
   }
 
-  initializePool(params: {
+  initialize(params: {
     results: Queue<Item>
     total: number
     maxDownloadBatchWait?: number
   }): void {
-    const { results, total } = params
+    const { results, total, maxDownloadBatchWait } = params
     this.#results = results
-    this.#downloadQueue = new BatchedPool<string>({
-      concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
-      maxWaitTime: params.maxDownloadBatchWait,
+    this.#total = total
+    this.#downloadQueue = new BatchingQueue<string>({
+      batchSize: 15000, // 15k is a good number for most cases
+      maxWaitTime: maxDownloadBatchWait ?? 1000, // 1 second
       processFunction: (batch: string[]): Promise<void> =>
         this.downloadBatch({
           batch,
@@ -75,46 +90,47 @@ export default class ServerDownloader implements Downloader {
     })
   }
 
-  #getPool(): BatchedPool<string> {
-    if (this.#downloadQueue) {
-      return this.#downloadQueue
-    }
-    throw new Error('Download pool is not initialized')
+  add(id: string): void {
+    this.#downloadQueue?.add(id, id)
   }
 
-  add(id: string): void {
-    this.#getPool().add(id)
+  /*
+  This is the most frequently reported and confirmed reason for this error in Safari. There's a known bug in WebKit (Safari's rendering engine) where TextDecoder can fail or throw a RangeError after decoding around 2GB of data. Chrome and other browsers handle much larger amounts of data without this specific limitation.
+
+Why it happens: It seems to be an internal memory or indexing limitation within Safari's TextDecoder implementation. After a certain threshold of data has been processed by a TextDecoder instance, it starts throwing this error.
+
+Chrome's behavior: Chrome generally handles larger data sizes without this specific RangeError. It might become slow or run out of general memory, but not typically with this specific error.
+  */
+  decodeChunk(chunkBuffer: Uint8Array): string {
+    if (this.#decodedBytesCount + chunkBuffer.byteLength > MAX_SAFARI_DECODE_BYTES) {
+      // Safari is approaching its limit, create a new decoder
+      this.#decoder = new TextDecoder('utf-8', { fatal: true })
+      this.#decodedBytesCount = 0 // Reset counter for the new decoder
+    }
+    const decodedText = this.#decoder.decode(chunkBuffer)
+    this.#decodedBytesCount += chunkBuffer.byteLength
+    return decodedText
   }
 
   async disposeAsync(): Promise<void> {
     await this.#downloadQueue?.disposeAsync()
   }
 
-  #processJson(baseId: string, unparsedBase: string): Item {
-    let base: unknown
-    try {
-      base = JSON.parse(unparsedBase)
-    } catch (e: unknown) {
-      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
-    }
-    if (isBase(base)) {
-      return { baseId, base }
-    } else {
-      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
-    }
-  }
-
   async downloadBatch(params: {
     batch: string[]
     url: string
     headers: HeadersInit
   }): Promise<void> {
     const { batch, url, headers } = params
+
+    const start = performance.now()
+    this.#logger(`Downloading batch of ${batch.length} items...`)
+    const attributeMask = this.#options.attributeMask
     const keys = new Set<string>(batch)
     const response = await this.#fetch(url, {
       method: 'POST',
       headers: { ...headers, 'Content-Type': 'application/json' },
-      body: JSON.stringify({ objects: JSON.stringify(batch) })
+      body: JSON.stringify({ objectIds: batch, attributeMask })
     })
 
     this.#validateResponse(response)
@@ -130,7 +146,7 @@ export default class ServerDownloader implements Downloader {
       const { done, value } = await reader.read()
       if (done) break
 
-      leftover = await this.processArray(leftover, value, keys, async () => {
+      leftover = await this.#processArray(leftover, value, keys, async () => {
         count++
         if (count % 1000 === 0) {
           await new Promise((resolve) => setTimeout(resolve, 100)) //allow other stuff to happen
@@ -142,16 +158,23 @@ export default class ServerDownloader implements Downloader {
         'Items requested were not downloaded: ' + take(keys.values(), 10).join(',')
       )
     }
+    count += keys.size // count the leftovers
+    if (count >= this.#total!) {
+      await this.#results?.disposeAsync() // mark the queue as done
+    }
+    this.#logger(
+      `Downloaded batch of ${batch.length} items in ${performance.now() - start}ms`
+    )
   }
 
-  async processArray(
+  async #processArray(
     leftover: Uint8Array,
     value: Uint8Array,
     keys: Set<string>,
     callback: () => Promise<void>
   ): Promise<Uint8Array> {
     //this concat will allocate a new array
-    const combined = this.concatUint8Arrays(leftover, value)
+    const combined = this.#concatUint8Arrays(leftover, value)
     let start = 0
 
     //subarray doesn't allocate
@@ -159,34 +182,68 @@ export default class ServerDownloader implements Downloader {
       if (combined[i] === 0x0a) {
         const line = combined.subarray(start, i) // line without \n
         //strings are allocated here
-        const item = this.processLine(line)
-        this.#results?.add(item)
+        const item = this.#processLine(line)
         start = i + 1
         await callback()
         keys.delete(item.baseId)
+        this.#results?.add(item)
       }
     }
     return combined.subarray(start) // carry over remainder
   }
 
-  processLine(line: Uint8Array): Item {
+  #processLine(line: Uint8Array): Item {
     for (let i = 0; i < line.length; i++) {
       if (line[i] === 0x09) {
         //this is a tab
-        const baseId = this.#decoder.decode(line.subarray(0, i))
-        const json = line.subarray(i + 1)
-        const base = this.#decoder.decode(json)
+        const baseId = this.decodeChunk(line.subarray(0, i))
+        const jsonBytes = line.subarray(i + 1)
+
+        if (!this.#isValidBytes(jsonBytes)) {
+          return { baseId, base: undefined }
+        }
+        const base = this.decodeChunk(jsonBytes)
         const item = this.#processJson(baseId, base)
-        item.size = json.length
+        item.size = jsonBytes.length
         return item
       }
     }
     throw new ObjectLoaderRuntimeError(
-      'Invalid line format: ' + this.#decoder.decode(line)
+      'Invalid line format in response: ' + this.decodeChunk(line)
     )
   }
 
-  concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
+  #processJson(baseId: string, unparsedBase: string): Item {
+    let base: unknown
+    try {
+      base = JSON.parse(unparsedBase)
+    } catch (e: unknown) {
+      throw new Error(`Error parsing object ${baseId}: ${(e as Error).message}`)
+    }
+    if (isBase(base)) {
+      return { baseId, base }
+    } else {
+      throw new ObjectLoaderRuntimeError(`${baseId} is not a base`)
+    }
+  }
+
+  #isValidString(json: string): boolean {
+    for (const rawString of this.#rawStrings)
+      if (json.includes(rawString)) {
+        return false
+      }
+    return true
+  }
+
+  #isValidBytes(json: Uint8Array): boolean {
+    for (const rawEncoding of this.#rawEncodings)
+      if (indexOf(json, rawEncoding) !== -1) {
+        return false
+      }
+    return true
+  }
+
+  #concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array {
     const c = new Uint8Array(a.length + b.length)
     c.set(a, 0)
     c.set(b, a.length)
@@ -199,7 +256,13 @@ export default class ServerDownloader implements Downloader {
     })
     this.#validateResponse(response)
     const responseText = await response.text()
+    if (!this.#isValidString(responseText)) {
+      throw new ObjectLoaderRuntimeError('Invalid base object')
+    }
     const item = this.#processJson(this.#options.objectId, responseText)
+    if (!item.base) {
+      throw new ObjectLoaderRuntimeError('Invalid base object')
+    }
     item.size = 0
     return item
   }

package/src/{operations → core}/traverser.ts

@@ -1,4 +1,5 @@
-import { Base, DataChunk, isBase, isReference, isScalar } from '../types/types.js'
+import { isScalar, isBase, isReference } from '../types/functions.js'
+import { Base, DataChunk } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 
 export type ProgressStage = 'download' | 'construction'