@synchronized-console/objectloader2 2.25.7 → 2.31.14

package/src/operations/downloaders/serverDownloader.spec.ts

@@ -1,161 +0,0 @@
-import { describe, expect, test } from 'vitest'
-import createFetchMock from 'vitest-fetch-mock'
-import { vi } from 'vitest'
-import { Item } from '../../types/types.js'
-import ServerDownloader from './serverDownloader.js'
-import { MemoryPump } from '../../helpers/memoryPump.js'
-
-describe('downloader', () => {
-  test('download batch of one', async () => {
-    const fetchMocker = createFetchMock(vi)
-    const i: Item = { baseId: 'id', base: { id: 'id', synchronized_type: 'type' } }
-    fetchMocker.mockResponseOnce('id\t' + JSON.stringify(i.base) + '\n')
-    const pump = new MemoryPump()
-    const downloader = new ServerDownloader({
-      serverUrl: 'http://synchronized.test',
-      streamId: 'streamId',
-      objectId: 'objectId',
-      token: 'token',
-      fetch: fetchMocker
-    })
-    downloader.initializePool({ results: pump, total: 1, maxDownloadBatchWait: 200 })
-    downloader.add('id')
-    await downloader.disposeAsync()
-    const r = []
-    for await (const x of pump.gather([i.baseId])) {
-      r.push(x)
-    }
-
-    expect(r).toMatchSnapshot()
-    await downloader.disposeAsync()
-  })
-
-  test('download batch of two', async () => {
-    const fetchMocker = createFetchMock(vi)
-    const i1: Item = { baseId: 'id1', base: { id: 'id1', synchronized_type: 'type' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id2', synchronized_type: 'type' } }
-    fetchMocker.mockResponseOnce(
-      'id1\t' + JSON.stringify(i1.base) + '\nid2\t' + JSON.stringify(i2.base) + '\n'
-    )
-
-    const pump = new MemoryPump()
-    const downloader = new ServerDownloader({
-      serverUrl: 'http://synchronized.test',
-      streamId: 'streamId',
-      objectId: 'objectId',
-      token: 'token',
-
-      fetch: fetchMocker
-    })
-    downloader.initializePool({ results: pump, total: 2, maxDownloadBatchWait: 200 })
-    downloader.add('id1')
-    downloader.add('id2')
-    await downloader.disposeAsync()
-    const r = []
-    for await (const x of pump.gather([i1.baseId, i2.baseId])) {
-      r.push(x)
-    }
-
-    expect(r).toMatchSnapshot()
-    await downloader.disposeAsync()
-  })
-
-  test('download batch of three', async () => {
-    const fetchMocker = createFetchMock(vi)
-    const i1: Item = { baseId: 'id1', base: { id: 'id1', synchronized_type: 'type' } }
-    const i2: Item = { baseId: 'id2', base: { id: 'id2', synchronized_type: 'type' } }
-    const i3: Item = { baseId: 'id3', base: { id: 'id3', synchronized_type: 'type' } }
-    fetchMocker.mockResponseOnce(
-      'id1\t' +
-        JSON.stringify(i1.base) +
-        '\nid2\t' +
-        JSON.stringify(i2.base) +
-        '\nid3\t' +
-        JSON.stringify(i3.base) +
-        '\n'
-    )
-
-    const pump = new MemoryPump()
-    const downloader = new ServerDownloader({
-      serverUrl: 'http://synchronized.test',
-      streamId: 'streamId',
-      objectId: 'objectId',
-      token: 'token',
-
-      fetch: fetchMocker
-    })
-    downloader.initializePool({ results: pump, total: 3, maxDownloadBatchWait: 200 })
-    downloader.add('id1')
-    downloader.add('id2')
-    downloader.add('id3')
-    await downloader.disposeAsync()
-    const r = []
-    for await (const x of pump.gather([i1.baseId, i2.baseId, i3.baseId])) {
-      r.push(x)
-    }
-
-    expect(r).toMatchSnapshot()
-    await downloader.disposeAsync()
-  })
-
-  test('download single exists', async () => {
-    const fetchMocker = createFetchMock(vi)
-    const i: Item = {
-      baseId: 'id',
-      base: { id: 'id', synchronized_type: 'type', __closure: { childIds: 1 } }
-    }
-    fetchMocker.mockResponseOnce(JSON.stringify(i.base))
-    const downloader = new ServerDownloader({
-      serverUrl: 'http://synchronized.test',
-      streamId: 'streamId',
-      objectId: i.baseId,
-      token: 'token',
-
-      fetch: fetchMocker
-    })
-    const x = await downloader.downloadSingle()
-    expect(x).toMatchSnapshot()
-    await downloader.disposeAsync()
-  })
-
-  test('add extra header', async () => {
-    const fetchMocker = createFetchMock(vi)
-    const i: Item = {
-      baseId: 'id',
-      base: { id: 'id', synchronized_type: 'type', __closure: { childIds: 1 } }
-    }
-    fetchMocker.mockResponseIf(
-      (req) => req.headers.get('x-test') === 'asdf',
-      JSON.stringify(i.base)
-    )
-    const headers = new Headers()
-    headers.set('x-test', 'asdf')
-    const downloader = new ServerDownloader({
-      serverUrl: 'http://synchronized.test',
-      headers,
-      streamId: 'streamId',
-      objectId: i.baseId,
-      token: 'token',
-
-      fetch: fetchMocker
-    })
-    const x = await downloader.downloadSingle()
-    expect(x).toMatchSnapshot()
-    await downloader.disposeAsync()
-  })
-
-  test('can dispose used', async () => {
-    const fetchMocker = createFetchMock(vi)
-    const headers = new Headers()
-    const downloader = new ServerDownloader({
-      serverUrl: 'http://synchronized.test',
-      headers,
-      streamId: 'streamId',
-      objectId: 'objectId',
-      token: 'token',
-
-      fetch: fetchMocker
-    })
-    await downloader.disposeAsync()
-  })
-})

package/src/operations/objectLoader2.ts

@@ -1,119 +0,0 @@
-import AsyncGeneratorQueue from '../helpers/asyncGeneratorQueue.js'
-import { Downloader, Database } from './interfaces.js'
-import { CustomLogger, Base, Item } from '../types/types.js'
-import { CacheOptions, ObjectLoader2Options } from './options.js'
-import { DefermentManager } from '../helpers/defermentManager.js'
-import { CacheReader } from '../helpers/cacheReader.js'
-import { CachePump } from '../helpers/cachePump.js'
-import AggregateQueue from '../helpers/aggregateQueue.js'
-import { ObjectLoader2Factory } from './objectLoader2Factory.js'
-
-export class ObjectLoader2 {
-  #rootId: string
-
-  #logger: CustomLogger
-
-  #database: Database
-  #downloader: Downloader
-  #pump: CachePump
-  #cache: CacheReader
-
-  #deferments: DefermentManager
-
-  #gathered: AsyncGeneratorQueue<Item>
-
-  #root?: Item = undefined
-
-  constructor(options: ObjectLoader2Options) {
-    this.#rootId = options.rootId
-    this.#logger = options.logger || console.log
-
-    const cacheOptions: CacheOptions = {
-      logger: this.#logger,
-      maxCacheReadSize: 10_000,
-      maxCacheWriteSize: 10_000,
-      maxWriteQueueSize: 40_000,
-      maxCacheBatchWriteWait: 3_000,
-      maxCacheBatchReadWait: 3_000
-    }
-
-    this.#gathered = new AsyncGeneratorQueue()
-    this.#database = options.database
-    this.#deferments = new DefermentManager({
-      maxSizeInMb: 2_000, // 2 GBs
-      ttlms: 15_000, // 15 seconds
-      logger: this.#logger
-    })
-    this.#cache = new CacheReader(this.#database, this.#deferments, cacheOptions)
-    this.#pump = new CachePump(
-      this.#database,
-      this.#gathered,
-      this.#deferments,
-      cacheOptions
-    )
-    this.#downloader = options.downloader
-  }
-
-  async disposeAsync(): Promise<void> {
-    await Promise.all([
-      this.#downloader.disposeAsync(),
-      this.#cache.disposeAsync(),
-      this.#pump.disposeAsync()
-    ])
-    this.#deferments.dispose()
-  }
-
-  async getRootObject(): Promise<Item | undefined> {
-    if (!this.#root) {
-      this.#root = (await this.#database.getAll([this.#rootId]))[0]
-      if (!this.#root) {
-        this.#root = await this.#downloader.downloadSingle()
-      }
-    }
-    return this.#root
-  }
-
-  async getObject(params: { id: string }): Promise<Base> {
-    return await this.#cache.getObject({ id: params.id })
-  }
-
-  async getTotalObjectCount(): Promise<number> {
-    const rootObj = await this.getRootObject()
-    const totalChildrenCount = Object.keys(rootObj?.base.__closure || {}).length
-    return totalChildrenCount + 1 //count the root
-  }
-
-  async *getObjectIterator(): AsyncGenerator<Base> {
-    const rootItem = await this.getRootObject()
-    if (rootItem === undefined) {
-      this.#logger('No root object found!')
-      return
-    }
-    //only for root
-    this.#pump.add(rootItem)
-    yield rootItem.base
-    if (!rootItem.base.__closure) return
-
-    //sort the closures by their values descending
-    const sortedClosures = Object.entries(rootItem.base.__closure).sort(
-      (a, b) => b[1] - a[1]
-    )
-    const children = sortedClosures.map((x) => x[0])
-    const total = children.length
-    this.#downloader.initializePool({
-      results: new AggregateQueue(this.#gathered, this.#pump),
-      total
-    })
-    for await (const item of this.#pump.gather(children, this.#downloader)) {
-      yield item.base
-    }
-  }
-
-  static createFromObjects(objects: Base[]): ObjectLoader2 {
-    return ObjectLoader2Factory.createFromObjects(objects)
-  }
-
-  static createFromJSON(json: string): ObjectLoader2 {
-    return ObjectLoader2Factory.createFromJSON(json)
-  }
-}

package/src/operations/objectLoader2Factory.ts

@@ -1,78 +0,0 @@
-import { Base, CustomLogger } from '../types/types.js'
-import IndexedDatabase from './databases/indexedDatabase.js'
-import { MemoryDatabase } from './databases/memoryDatabase.js'
-import { MemoryDownloader } from './downloaders/memoryDownloader.js'
-import ServerDownloader from './downloaders/serverDownloader.js'
-import { ObjectLoader2 } from './objectLoader2.js'
-
-export interface ObjectLoader2FactoryOptions {
-  useMemoryCache?: boolean
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
-  keyRange?: { bound: Function; lowerBound: Function; upperBound: Function }
-  indexedDB?: IDBFactory
-  logger?: CustomLogger
-}
-
-export class ObjectLoader2Factory {
-  static createFromObjects(objects: Base[]): ObjectLoader2 {
-    const root = objects[0]
-    const records: Map<string, Base> = new Map<string, Base>()
-    objects.forEach((element) => {
-      records.set(element.id, element)
-    })
-    const loader = new ObjectLoader2({
-      rootId: root.id,
-      database: new MemoryDatabase({ items: records }),
-      downloader: new MemoryDownloader(root.id, records)
-    })
-    return loader
-  }
-
-  static createFromJSON(json: string): ObjectLoader2 {
-    const jsonObj = JSON.parse(json) as Base[]
-    return this.createFromObjects(jsonObj)
-  }
-
-  static createFromUrl(params: {
-    serverUrl: string
-    streamId: string
-    objectId: string
-    token?: string
-    headers?: Headers
-    options?: ObjectLoader2FactoryOptions
-  }): ObjectLoader2 {
-    let loader: ObjectLoader2
-    if (params.options?.useMemoryCache) {
-      loader = new ObjectLoader2({
-        rootId: params.objectId,
-        downloader: new ServerDownloader({
-          serverUrl: params.serverUrl,
-          streamId: params.streamId,
-          objectId: params.objectId,
-          token: params.token,
-          headers: params.headers
-        }),
-        database: new MemoryDatabase({
-          items: new Map<string, Base>()
-        })
-      })
-    } else {
-      loader = new ObjectLoader2({
-        rootId: params.objectId,
-        downloader: new ServerDownloader({
-          serverUrl: params.serverUrl,
-          streamId: params.streamId,
-          objectId: params.objectId,
-          token: params.token,
-          headers: params.headers
-        }),
-        database: new IndexedDatabase({
-          logger: params.options?.logger,
-          indexedDB: params.options?.indexedDB,
-          keyRange: params.options?.keyRange
-        })
-      })
-    }
-    return loader
-  }
-}

package/src/{operations → core}/traverser.spec.ts

@@ -1,7 +1,7 @@
 import { describe, expect, test } from 'vitest'
-import { Base } from '../types/types.js'
 import { ObjectLoader2 } from './objectLoader2.js'
 import Traverser from './traverser.js'
+import { Base } from '../types/types.js'
 
 describe('Traverser', () => {
   test('root and two children with referenceId', async () => {