@synchronized-console/objectloader2 2.25.7 → 2.31.14

package/src/core/stages/cacheWriter.spec.ts

@@ -0,0 +1,213 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { Item } from '../../types/types.js'
+import { CacheWriter } from './cacheWriter.js'
+import { DefermentManager } from '../../deferment/defermentManager.js'
+import { Database } from '../interfaces.js'
+import { MemoryCache, MemoryCacheOptions } from '../../deferment/MemoryCache.js'
+import { CacheOptions } from '../options.js'
+import { CustomLogger } from '../../types/functions.js'
+
+// Mock implementations
+class MockDatabase implements Database {
+  savedItems: Item[] = []
+
+  getAll(/* keys */): Promise<(Item | undefined)[]> {
+    return Promise.resolve([])
+  }
+
+  putAll(batch: Item[]): Promise<void> {
+    this.savedItems.push(...batch)
+    return Promise.resolve()
+  }
+
+  dispose(): void {
+    this.savedItems = []
+  }
+}
+
+describe('CacheWriter', () => {
+  let database: MockDatabase
+  let defermentManager: DefermentManager
+  let cacheWriter: CacheWriter
+  let logger: CustomLogger
+  let requestItemMock: (id: string) => void
+  let options: CacheOptions
+  let memoryCache: MemoryCache
+
+  beforeEach(() => {
+    database = new MockDatabase()
+    logger = vi.fn() as CustomLogger
+    const memoryCacheOptions: MemoryCacheOptions = {
+      maxSizeInMb: 100,
+      ttlms: 60000
+    }
+    memoryCache = new MemoryCache(memoryCacheOptions, logger)
+    defermentManager = new DefermentManager(logger, memoryCache)
+    requestItemMock = vi.fn()
+
+    options = {
+      maxCacheWriteSize: 10,
+      maxCacheBatchWriteWait: 50,
+      maxCacheReadSize: 10,
+      maxCacheBatchReadWait: 50,
+      maxWriteQueueSize: 100
+    }
+
+    cacheWriter = new CacheWriter(
+      database,
+      logger,
+      defermentManager,
+      options,
+      requestItemMock
+    )
+  })
+
+  afterEach(async () => {
+    await cacheWriter.disposeAsync()
+  })
+
+  it('should write items to the database', async () => {
+    const item: Item = {
+      baseId: 'test-id',
+      base: { id: 'test-id', synchronized_type: 'test-type' }
+    }
+
+    cacheWriter.add(item)
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(1)
+    expect(database.savedItems[0].baseId).toBe('test-id')
+    expect(database.savedItems[0].base).toEqual({
+      id: 'test-id',
+      synchronized_type: 'test-type'
+    })
+  })
+
+  it('should write multiple items in a batch', async () => {
+    const items = [
+      {
+        baseId: 'item1',
+        base: { id: 'item1', synchronized_type: 'test-type' }
+      },
+      {
+        baseId: 'item2',
+        base: { id: 'item2', synchronized_type: 'test-type' }
+      },
+      {
+        baseId: 'item3',
+        base: { id: 'item3', synchronized_type: 'test-type' }
+      }
+    ]
+
+    items.forEach((item) => cacheWriter.add(item))
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(3)
+    expect(database.savedItems.map((item) => item.baseId)).toEqual([
+      'item1',
+      'item2',
+      'item3'
+    ])
+  })
+
+  it('should call undefer on the defermentManager', () => {
+    const spy = vi.spyOn(defermentManager, 'undefer')
+    const item: Item = {
+      baseId: 'test-id',
+      base: { id: 'test-id', synchronized_type: 'test-type' }
+    }
+
+    cacheWriter.add(item)
+
+    expect(spy).toHaveBeenCalledWith(item, requestItemMock)
+  })
+
+  it('should handle items with id but no base', async () => {
+    const item: Item = {
+      baseId: 'test-id-no-base'
+      // No base property
+    }
+
+    // Adding an item with no base should not throw an error
+    expect(() => cacheWriter.add(item)).not.toThrow()
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, options.maxCacheBatchWriteWait + 10)
+    )
+
+    expect(database.savedItems).toHaveLength(1)
+    expect(database.savedItems[0].baseId).toBe('test-id-no-base')
+    expect(database.savedItems[0].base).toBeUndefined()
+  })
+
+  it('should process items in batches according to maxCacheWriteSize', async () => {
+    const spy = vi.spyOn(database, 'putAll')
+    const smallBatchOptions: CacheOptions = {
+      ...options,
+      maxCacheWriteSize: 2, // Set small batch size
+      maxCacheBatchWriteWait: 100
+    }
+
+    const smallBatchCacheWriter = new CacheWriter(
+      database,
+      logger,
+      defermentManager,
+      smallBatchOptions,
+      requestItemMock
+    )
+
+    // Add 5 items
+    const items = Array.from({ length: 5 }, (_, i) => ({
+      baseId: `batch-item-${i}`,
+      base: { id: `batch-item-${i}`, synchronized_type: 'test-type' }
+    }))
+
+    items.forEach((item) => smallBatchCacheWriter.add(item))
+
+    // Wait for batch queue to process
+    await new Promise((resolve) =>
+      setTimeout(resolve, smallBatchOptions.maxCacheBatchWriteWait * 2 + 50)
+    )
+
+    // With batch size 2, we expect 3 calls to saveBatch: 2 items + 2 items + 1 item
+    expect(spy).toHaveBeenCalledTimes(3)
+
+    await smallBatchCacheWriter.disposeAsync()
+  })
+
+  it('should be disposed correctly', async () => {
+    expect(cacheWriter.isDisposed).toBe(false)
+    await cacheWriter.disposeAsync()
+    expect(cacheWriter.isDisposed).toBe(true)
+  })
+
+  it('should write directly with writeAll method', async () => {
+    const items = [
+      {
+        baseId: 'direct1',
+        base: { id: 'direct1', synchronized_type: 'test-type' }
+      },
+      {
+        baseId: 'direct2',
+        base: { id: 'direct2', synchronized_type: 'test-type' }
+      }
+    ]
+
+    await cacheWriter.writeAll(items)
+
+    expect(database.savedItems).toHaveLength(2)
+    expect(database.savedItems.map((item) => item.baseId)).toEqual([
+      'direct1',
+      'direct2'
+    ])
+  })
+})

package/src/core/stages/cacheWriter.ts

@@ -0,0 +1,64 @@
+import { Deferment } from '../../deferment/defermentManager.js'
+import BatchingQueue from '../../queues/batchingQueue.js'
+import Queue from '../../queues/queue.js'
+import { CustomLogger } from '../../types/functions.js'
+import { Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+import { CacheOptions } from '../options.js'
+
+export class CacheWriter implements Queue<Item> {
+  #writeQueue: BatchingQueue<Item> | undefined
+  #database: Database
+  #deferment: Deferment
+  #requestItem: (id: string) => void
+  #logger: CustomLogger
+  #options: CacheOptions
+  #disposed = false
+
+  constructor(
+    database: Database,
+    logger: CustomLogger,
+    deferment: Deferment,
+    options: CacheOptions,
+    requestItem: (id: string) => void
+  ) {
+    this.#database = database
+    this.#options = options
+    this.#logger = logger
+    this.#deferment = deferment
+    this.#requestItem = requestItem
+  }
+
+  add(item: Item): void {
+    if (!this.#writeQueue) {
+      this.#writeQueue = new BatchingQueue({
+        batchSize: this.#options.maxCacheWriteSize,
+        maxWaitTime: this.#options.maxCacheBatchWriteWait,
+        processFunction: async (batch: Item[]): Promise<void> => {
+          await this.writeAll(batch)
+        }
+      })
+    }
+    this.#writeQueue.add(item.baseId, item)
+    this.#deferment.undefer(item, this.#requestItem)
+  }
+
+  async writeAll(items: Item[]): Promise<void> {
+    const start = performance.now()
+    await this.#database.putAll(items)
+    this.#logger(
+      `writeBatch: wrote ${items.length}, time ${
+        performance.now() - start
+      } ms left ${this.#writeQueue?.count()}`
+    )
+  }
+
+  async disposeAsync(): Promise<void> {
+    this.#disposed = true
+    await this.#writeQueue?.disposeAsync()
+  }
+
+  get isDisposed(): boolean {
+    return this.#disposed
+  }
+}

package/src/{operations/databases → core/stages}/indexedDatabase.spec.ts

@@ -1,10 +1,14 @@
 import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
-import IndexedDatabase, { IndexedDatabaseOptions } from './indexedDatabase.js'
-import { Item } from '../../types/types.js'
+
+import { Base, Item } from '../../types/types.js'
+import { IndexedDatabase, IndexedDatabaseOptions } from './indexedDatabase.js'
 
 // Mock Item
-const defaultItem = (id: string): Item => ({ baseId: id, item: { foo: 'bar' } })
+const defaultItem = (id: string): Item => ({
+  baseId: id,
+  base: { foo: 'bar' } as unknown as Base
+})
 
 describe('IndexedDatabase', () => {
   let db: IndexedDatabase
@@ -15,19 +19,19 @@ describe('IndexedDatabase', () => {
     db = new IndexedDatabase(options)
   })
 
-  afterEach(async () => {
-    await db.disposeAsync()
+  afterEach(() => {
+    db.dispose()
   })
 
   it('should add and get multiple items', async () => {
     const items = [defaultItem('id1'), defaultItem('id2')]
-    await db.cacheSaveBatch({ batch: items })
+    await db.putAll(items)
     const result = await db.getAll(['id1', 'id2'])
     expect(result).toMatchSnapshot()
     expect(result).toEqual(items)
   })
 
-  it('should dispose without error', async () => {
-    await expect(db.disposeAsync()).resolves.not.toThrow()
+  it('should dispose without error', () => {
+    expect(() => db.dispose()).not.toThrow()
   })
 })

package/src/core/stages/indexedDatabase.ts

@@ -0,0 +1,178 @@
+/* eslint-disable @typescript-eslint/no-base-to-string */
+/* eslint-disable @typescript-eslint/prefer-promise-reject-errors */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+import { isSafari } from '@synchronized-console/shared'
+import { Item } from '../../types/types.js'
+import { Database } from '../interfaces.js'
+
+/**
+ * A wrapper class for IndexedDB to simplify common database operations.
+ */
+export interface IndexedDatabaseOptions {
+  indexedDB?: IDBFactory
+  keyRange?: {
+    bound: Function
+    lowerBound: Function
+    upperBound: Function
+  }
+}
+export class IndexedDatabase implements Database {
+  #options: IndexedDatabaseOptions
+
+  #db: IDBDatabase | undefined = undefined
+  readonly #dbName: string = 'synchronized-cache'
+  readonly #storeName: string = 'cache'
+
+  constructor(options: IndexedDatabaseOptions) {
+    this.#options = options
+  }
+
+  /**
+   * Initializes the database connection and creates the object store if needed.
+   * This must be called before any other database operations.
+   */
+  async init(): Promise<void> {
+    if (this.#db) return
+    await this.#safariFix()
+    return this.#openDatabase()
+  }
+
+  /**
+   * Opens the database, and if there's an error, deletes the database and tries again.
+   */
+  async #openDatabase(): Promise<void> {
+    const idb = this.#options.indexedDB ?? indexedDB
+
+    return new Promise((resolve, reject) => {
+      const request = idb.open(this.#dbName, 1)
+
+      request.onerror = (): any => {
+        console.warn(
+          `Failed to open database: ${this.#dbName}, deleting and trying again`
+        )
+        // Delete the database and try again
+        const deleteRequest = idb.deleteDatabase(this.#dbName)
+        deleteRequest.onsuccess = (): any => {
+          // Try opening again after deletion
+          void this.#openDatabase().then(resolve).catch(reject)
+        }
+        deleteRequest.onerror = (): any => {
+          reject(`Failed to delete and reopen database: ${this.#dbName}`)
+        }
+      }
+
+      request.onupgradeneeded = (event): any => {
+        const db = (event.target as IDBOpenDBRequest).result
+        if (db.objectStoreNames.contains(this.#storeName)) {
+          db.deleteObjectStore(this.#storeName)
+        }
+        db.createObjectStore(this.#storeName, { keyPath: 'baseId' })
+      }
+
+      request.onsuccess = (event): any => {
+        this.#db = (event.target as IDBOpenDBRequest).result
+        resolve()
+      }
+    })
+  }
+
+  #getDB(): IDBDatabase {
+    if (!this.#db) {
+      throw new Error('Database not initialized. Call init() first.')
+    }
+    return this.#db
+  }
+
+  /**
+   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
+   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+   */
+  async #safariFix(): Promise<void> {
+    // No point putting other browsers or older versions of Safari through this mess.
+    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
+
+    let intervalId: ReturnType<typeof setInterval>
+
+    return new Promise<void>((resolve: () => void) => {
+      const tryIdb = (): Promise<IDBDatabaseInfo[]> | undefined =>
+        this.#options.indexedDB?.databases().finally(resolve)
+      intervalId = setInterval(() => {
+        void tryIdb()
+      }, 100)
+      void tryIdb()
+    }).finally(() => clearInterval(intervalId))
+  }
+
+  /**
+   * Inserts or updates an array of items in a single transaction.
+   * @param data The array of items to insert.
+   */
+  async putAll(data: Item[]): Promise<void> {
+    await this.init() // Ensure the database is initialized
+    return new Promise((resolve, reject) => {
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readwrite', {
+          durability: 'relaxed'
+        })
+        const store = transaction.objectStore(this.#storeName)
+
+        transaction.onerror = (): any => {
+          reject(`Transaction error: ${transaction.error}`)
+        }
+        transaction.oncomplete = (): any => {
+          resolve()
+        }
+
+        data.forEach((item) => store.put(item))
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+
+  /**
+   * Retrieves an array of items from the object store based on their IDs.
+   * @param ids The array of IDs to retrieve.
+   */
+  async getAll(ids: string[]): Promise<(Item | undefined)[]> {
+    await this.init() // Ensure the database is initialized
+    return new Promise((resolve, reject) => {
+      if (ids.length === 0) {
+        return resolve([])
+      }
+      try {
+        const transaction = this.#getDB().transaction(this.#storeName, 'readonly', {
+          durability: 'relaxed'
+        })
+        const store = transaction.objectStore(this.#storeName)
+        const promises: Promise<Item | undefined>[] = []
+
+        for (const id of ids) {
+          promises.push(
+            new Promise((resolveGet, rejectGet) => {
+              const request = store.get(id)
+              request.onerror = (): void =>
+                rejectGet(`Request error for id ${id}: ${request.error}`)
+              // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+              request.onsuccess = (): void => resolveGet(request.result)
+            })
+          )
+        }
+
+        Promise.all(promises)
+          .then((results) => {
+            resolve(results)
+          })
+          .catch(reject)
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
+  dispose(): void {
+    if (!this.#db) return
+    this.#db.close()
+    this.#db = undefined
+  }
+}

package/src/{operations/databases → core/stages/memory}/memoryDatabase.spec.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect, beforeEach } from 'vitest'
+import { Item, Base } from '../../../types/types.js'
 import { MemoryDatabase } from './memoryDatabase.js'
-import { Base, Item } from '../../types/types.js'
 
 const makeItem = (id: string, foo = 'bar'): Item => ({
   baseId: id,
@@ -21,14 +21,14 @@ describe('MemoryDatabase', () => {
 
   it('should add and retrieve a single item', async () => {
     const item = makeItem('id1')
-    await db.cacheSaveBatch({ batch: [item] })
+    await db.putAll([item])
     const result = await db.getAll(['id1'])
     expect(result).toEqual([item])
   })
 
   it('should add and retrieve multiple items', async () => {
     const items = [makeItem('id1'), makeItem('id2', 'baz')]
-    await db.cacheSaveBatch({ batch: items })
+    await db.putAll(items)
     const result = await db.getAll(['id1', 'id2'])
     expect(result).toEqual(items)
   })
@@ -36,13 +36,13 @@ describe('MemoryDatabase', () => {
   it('should overwrite items with the same key', async () => {
     const item1 = makeItem('id1', 'foo')
     const item2 = makeItem('id1', 'bar')
-    await db.cacheSaveBatch({ batch: [item1] })
-    await db.cacheSaveBatch({ batch: [item2] })
+    await db.putAll([item1])
+    await db.putAll([item2])
     const result = await db.getAll(['id1'])
     expect(result).toEqual([item2])
   })
 
-  it('disposeAsync should resolve', async () => {
-    await expect(db.disposeAsync()).resolves.not.toThrow()
+  it('dispose should not throw', () => {
+    db.dispose()
   })
 })

package/src/{operations/databases → core/stages/memory}/memoryDatabase.ts

@@ -1,6 +1,6 @@
-import { Base, Item } from '../../types/types.js'
-import { Database } from '../interfaces.js'
-import { MemoryDatabaseOptions } from '../options.js'
+import { Base, Item } from '../../../types/types.js'
+import { Database } from '../../interfaces.js'
+import { MemoryDatabaseOptions } from '../../options.js'
 
 export class MemoryDatabase implements Database {
   private items: Map<string, Base>
@@ -22,14 +22,17 @@ export class MemoryDatabase implements Database {
     return Promise.resolve(found)
   }
 
-  cacheSaveBatch({ batch }: { batch: Item[] }): Promise<void> {
+  putAll(batch: Item[]): Promise<void> {
     for (const item of batch) {
+      if (!item.baseId || !item.base) {
+        throw new Error('Item must have a baseId and base')
+      }
       this.items.set(item.baseId, item.base)
     }
     return Promise.resolve()
   }
 
-  disposeAsync(): Promise<void> {
-    return Promise.resolve()
+  dispose(): void {
+    this.items.clear()
   }
 }

package/src/{operations/downloaders → core/stages/memory}/memoryDownloader.spec.ts

@@ -1,8 +1,8 @@
 import { describe, it, expect, beforeEach } from 'vitest'
 import { MemoryDownloader } from './memoryDownloader.js'
-import { Base, Item } from '../../types/types.js'
-import Queue from '../../helpers/queue.js'
-import BufferQueue from '../../helpers/bufferQueue.js'
+import BufferQueue from '../../../queues/bufferQueue.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
 
 const makeBase = (foo: string): Base => ({ foo } as unknown as Base)
 
@@ -33,13 +33,13 @@ describe('MemoryDownloader', () => {
   })
 
   it('should add found item to results queue', () => {
-    downloader.initializePool({ results, total: 2 })
+    downloader.initialize({ results, total: 2 })
     downloader.add('id2')
     expect(items).toMatchSnapshot()
   })
 
   it('should throw if added item is missing', () => {
-    downloader.initializePool({ results, total: 2 })
+    downloader.initialize({ results, total: 2 })
     expect(() => downloader.add('missing')).toThrow()
   })
 

package/src/{operations/downloaders → core/stages/memory}/memoryDownloader.ts

@@ -1,6 +1,6 @@
-import Queue from '../../helpers/queue.js'
-import { Base, Item } from '../../types/types.js'
-import { Downloader } from '../interfaces.js'
+import Queue from '../../../queues/queue.js'
+import { Base, Item } from '../../../types/types.js'
+import { Downloader } from '../../interfaces.js'
 
 export class MemoryDownloader implements Downloader {
   #items: Map<string, Base>
@@ -11,7 +11,7 @@ export class MemoryDownloader implements Downloader {
     this.#rootId = rootId
     this.#items = items
   }
-  initializePool(params: {
+  initialize(params: {
     results: Queue<Item>
     total: number
     maxDownloadBatchWait?: number