@speckle/objectloader2 2.23.12

package/.tshy/build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "../src",
+    "module": "nodenext",
+    "moduleResolution": "nodenext"
+  }
+}

package/.tshy/commonjs.json

@@ -0,0 +1,17 @@
+{
+  "extends": "./build.json",
+  "include": [
+    "../src/**/*.ts",
+    "../src/**/*.cts",
+    "../src/**/*.tsx",
+    "../src/**/*.json"
+  ],
+  "exclude": [
+    "../**/*.spec.ts",
+    "../src/**/*.mts",
+    "../src/package.json"
+  ],
+  "compilerOptions": {
+    "outDir": "../.tshy-build/commonjs"
+  }
+}

package/.tshy/esm.json

@@ -0,0 +1,16 @@
+{
+  "extends": "./build.json",
+  "include": [
+    "../src/**/*.ts",
+    "../src/**/*.mts",
+    "../src/**/*.tsx",
+    "../src/**/*.json"
+  ],
+  "exclude": [
+    "../**/*.spec.ts",
+    "../src/package.json"
+  ],
+  "compilerOptions": {
+    "outDir": "../.tshy-build/esm"
+  }
+}

package/dist/commonjs/index.js

@@ -0,0 +1,8 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const objectLoader2_js_1 = __importDefault(require("./operations/objectLoader2.js"));
+exports.default = objectLoader2_js_1.default;
+//# sourceMappingURL=index.js.map

package/dist/esm/index.js

@@ -0,0 +1,3 @@
+import ObjectLoader2 from './operations/objectLoader2.js';
+export default ObjectLoader2;
+//# sourceMappingURL=index.js.map

package/eslint.config.mjs

@@ -0,0 +1,57 @@
+import { baseConfigs, globals, getESMDirname } from '../../eslint.config.mjs'
+import tseslint from 'typescript-eslint'
+
+/**
+ * @type {Array<import('eslint').Linter.FlatConfig>}
+ */
+const configs = [
+  ...baseConfigs,
+  {
+    files: ['examples/browser/**/*.{ts,js}'],
+    languageOptions: {
+      globals: {
+        ...globals.browser
+      }
+    }
+  },
+  ...tseslint.configs.recommendedTypeChecked.map((c) => ({
+    ...c,
+    files: [...(c.files || []), '**/*.ts', '**/*.d.ts']
+  })),
+  {
+    files: ['**/*.ts', '**/*.d.ts'],
+    languageOptions: {
+      parserOptions: {
+        tsconfigRootDir: getESMDirname(import.meta.url),
+        projectService: {
+          allowDefaultProject: ['*.ts']
+        }
+      }
+    },
+    rules: {
+      '@typescript-eslint/restrict-template-expressions': 'off'
+    }
+  },
+  {
+    files: ['**/*.d.ts'],
+    rules: {
+      '@typescript-eslint/no-explicit-any': 'off'
+    }
+  },
+  {
+    files: ['vite.config.ts'],
+    languageOptions: {
+      globals: {
+        ...globals.node
+      }
+    }
+  },
+  {
+    files: ['**/*.spec.ts'],
+    rules: {
+      '@typescript-eslint/no-unused-expressions': 'off'
+    }
+  }
+]
+
+export default configs

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "@speckle/objectloader2",
+  "version": "2.23.12",
+  "description": "This is an updated objectloader for the Speckle viewer written in typescript",
+  "main": "./dist/commonjs/index.js",
+  "module": "./dist/esm/index.js",
+  "homepage": "https://speckle.systems",
+  "types": "./dist/commonjs/index.d.ts",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/specklesystems/speckle-server.git",
+    "directory": "packages/objectloader2"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "lint:eslint": "eslint .",
+    "lint:tsc": "tsc --noEmit",
+    "lint": "yarn lint:eslint && yarn lint:tsc",
+    "lint:ci": "yarn lint:tsc",
+    "build": "NODE_ENV=production tshy",
+    "dev": "tshy --watch",
+    "test": "vitest",
+    "test:single-run": "vitest run"
+  },
+  "keywords": [
+    "speckle",
+    "aec",
+    "speckle api"
+  ],
+  "author": "AEC Systems",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@speckle/shared": "^2.23.12",
+    "dexie": "^4.0.11"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.17.5",
+    "@types/lodash-es": "^4.17.6",
+    "@typescript-eslint/eslint-plugin": "^7.12.0",
+    "@typescript-eslint/parser": "^7.12.0",
+    "@vitest/ui": "^3.0.9",
+    "eslint": "^9.4.0",
+    "eslint-config-prettier": "^9.1.0",
+    "fake-indexeddb": "^6.0.0",
+    "prettier": "^3.3.2",
+    "tshy": "^3.0.2",
+    "typescript": "^5.2.2",
+    "vitest": "^3.0.7",
+    "vitest-fetch-mock": "^0.4.5"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": {
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/commonjs/index.d.ts",
+        "default": "./dist/commonjs/index.js"
+      }
+    }
+  },
+  "tshy": {
+    "exclude": [
+      "**/*.spec.ts"
+    ],
+    "exports": {
+      "./package.json": "./package.json",
+      ".": "./src/index.ts"
+    }
+  }
+}

package/readme.md

@@ -0,0 +1,42 @@
+# objectloader2 for the Speckle viewer
+
+This is an updated object loader for the Speckle viewer rewritten in Typescript.
+
+The main aim for the objectloader is:
+
+- download Speckle objects as JSON
+- cache in IndexedDB so the same objects aren't downloaded twice
+- give data to the viewer, as returned by the `getObjectIterator` generator, as soon as possible
+- do the above as concurrently as a browser allows
+
+## Architecture
+
+To achieve increased concurrency, the different phases of the objectloader are divided into pools of workers with queues to feed them.
+
+```mermaid
+flowchart TD
+  start(Root Commit)
+  getIds(Parse Root to get all IDs)
+  cached{Cached?}
+  download(Download IDs)
+  save(Write to Cache)
+  load(Load from Cache)
+  generate(Generate to Viewer!)
+
+  start --> getIds
+  getIds --> cached
+  cached -->|Yes| load
+  cached -->|No| download
+  load --> generate
+  download --> generate
+  download --> save
+
+```
+
+From the list of IDs, they are moved to a queue to be begin checking the cache from a pool of readers.
+
+Results are then sent to the viewer, if found, else they're send to the download queue.
+
+The download queue is a batching mechanism that gets what is available, up to a limit or a timeout. The results are parsed and given to the generator and written to another queue.
+
+The write cache queue is processed with a single writer to the indexeddb.

package/src/helpers/asyncGeneratorQueue.ts

@@ -0,0 +1,35 @@
+import Queue from './queue.js'
+
+export default class AsyncGeneratorQueue<T> implements Queue<T> {
+  #buffer: T[] = []
+  #resolveQueue: ((value: T) => void)[] = []
+  #finished = false
+
+  add(value: T): void {
+    if (this.#resolveQueue.length > 0) {
+      // If there's a pending consumer, resolve immediately
+      const resolve = this.#resolveQueue.shift()!
+      resolve(value)
+    } else {
+      // Otherwise, add to the buffer
+      this.#buffer.push(value)
+    }
+  }
+
+  async *consume(): AsyncGenerator<T> {
+    while (
+      !this.#finished ||
+      this.#resolveQueue.length > 0 ||
+      this.#buffer.length > 0
+    ) {
+      if (this.#buffer.length > 0) {
+        yield this.#buffer.shift()! // Yield available values
+      } else {
+        yield await new Promise<T>((resolve) => this.#resolveQueue.push(resolve))
+      }
+    }
+  }
+  dispose(): void {
+    this.#finished = true
+  }
+}

package/src/helpers/batchedPool.ts

@@ -0,0 +1,60 @@
+export default class BatchedPool<T> {
+  #queue: T[] = []
+  #concurrencyAndSizes: number[]
+  #processFunction: (batch: T[]) => Promise<void>
+
+  #baseInterval: number
+
+  #processingLoop: Promise<void>
+  #finished = false
+
+  constructor(params: {
+    concurrencyAndSizes: number[]
+    maxWaitTime?: number
+    processFunction: (batch: T[]) => Promise<void>
+  }) {
+    this.#concurrencyAndSizes = params.concurrencyAndSizes
+    this.#baseInterval = Math.min(params.maxWaitTime ?? 200, 200) // Initial batch time (ms)
+    this.#processFunction = params.processFunction
+    this.#processingLoop = this.#loop()
+  }
+
+  add(item: T): void {
+    this.#queue.push(item)
+  }
+
+  getBatch(batchSize: number): T[] {
+    return this.#queue.splice(0, Math.min(batchSize, this.#queue.length))
+  }
+
+  async #runWorker(batchSize: number) {
+    while (!this.#finished || this.#queue.length > 0) {
+      if (this.#queue.length > 0) {
+        const batch = this.getBatch(batchSize)
+        try {
+          await this.#processFunction(batch)
+        } catch (e) {
+          console.error(e)
+        }
+      }
+      await this.#delay(this.#baseInterval)
+    }
+  }
+
+  async disposeAsync(): Promise<void> {
+    this.#finished = true
+    await this.#processingLoop
+  }
+
+  async #loop(): Promise<void> {
+    // Initialize workers
+    const workers = Array.from(this.#concurrencyAndSizes, (batchSize: number) =>
+      this.#runWorker(batchSize)
+    )
+    await Promise.all(workers)
+  }
+
+  #delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms))
+  }
+}

package/src/helpers/batchingQueue.ts

@@ -0,0 +1,77 @@
+import Queue from './queue.js'
+
+export default class BatchingQueue<T> implements Queue<T> {
+  #queue: T[] = []
+  #batchSize: number
+  #processFunction: (batch: T[]) => Promise<void>
+
+  #baseInterval: number
+  #minInterval: number
+  #maxInterval: number
+
+  #processingLoop: Promise<void>
+  #finished = false
+
+  constructor(params: {
+    batchSize: number
+    maxWaitTime?: number
+    processFunction: (batch: T[]) => Promise<void>
+  }) {
+    this.#batchSize = params.batchSize
+    this.#baseInterval = Math.min(params.maxWaitTime ?? 200, 200) // Initial batch time (ms)
+    this.#minInterval = Math.min(params.maxWaitTime ?? 100, 100) // Minimum batch time
+    this.#maxInterval = Math.min(params.maxWaitTime ?? 3000, 3000) // Maximum batch time
+    this.#processFunction = params.processFunction
+    this.#processingLoop = this.#loop()
+  }
+
+  async disposeAsync(): Promise<void> {
+    this.#finished = true
+    await this.#processingLoop
+  }
+
+  add(item: T): void {
+    this.#queue.push(item)
+  }
+
+  count(): number {
+    return this.#queue.length
+  }
+
+  #getBatch(batchSize: number): T[] {
+    return this.#queue.splice(0, Math.min(batchSize, this.#queue.length))
+  }
+
+  async #loop(): Promise<void> {
+    let interval = this.#baseInterval
+    while (!this.#finished || this.#queue.length > 0) {
+      const startTime = performance.now()
+      if (this.#queue.length > 0) {
+        const batch = this.#getBatch(this.#batchSize)
+        //console.log('running with queue size of ' + this.#queue.length)
+        await this.#processFunction(batch)
+      }
+      if (this.#queue.length < this.#batchSize / 2) {
+        //refigure interval
+        const endTime = performance.now()
+        const duration = endTime - startTime
+        if (duration > interval) {
+          interval = Math.min(interval * 1.5, this.#maxInterval) // Increase if slow or empty
+        } else {
+          interval = Math.max(interval * 0.8, this.#minInterval) // Decrease if fast
+        }
+        /*console.log(
+          'queue is waiting ' +
+            interval / 1000 +
+            ' with queue size of ' +
+            this.#queue.length
+        )*/
+        await this.#delay(interval)
+      }
+    }
+  }
+
+  #delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms))
+  }
+}

package/src/helpers/bufferQueue.ts

@@ -0,0 +1,12 @@
+import Queue from './queue.js'
+
+export default class BufferQueue<T> implements Queue<T> {
+  #buffer: T[] = []
+  add(value: T): void {
+    this.#buffer.push(value)
+  }
+
+  values(): T[] {
+    return this.#buffer
+  }
+}

package/src/helpers/deferredBase.ts

@@ -0,0 +1,17 @@
+import { Base } from '../types/types.js'
+
+export class DeferredBase {
+  promise: Promise<Base>
+  resolve!: (value: Base) => void
+  reject!: (reason?: Error) => void
+
+  readonly id: string
+
+  constructor(id: string) {
+    this.id = id
+    this.promise = new Promise<Base>((resolve, reject) => {
+      this.resolve = resolve
+      this.reject = reject
+    })
+  }
+}

package/src/helpers/queue.ts

@@ -0,0 +1,3 @@
+export default interface Queue<T> {
+  add(value: T): void
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+import ObjectLoader2 from './operations/objectLoader2.js'
+
+export default ObjectLoader2

package/src/operations/indexedDatabase.spec.ts

@@ -0,0 +1,69 @@
+import { describe, expect, test } from 'vitest'
+import IndexedDatabase from './indexedDatabase.js'
+import { IDBFactory, IDBKeyRange } from 'fake-indexeddb'
+import { Item } from '../types/types.js'
+import BufferQueue from '../helpers/bufferQueue.js'
+
+describe('database cache', () => {
+  test('write single item to queue use getItem', async () => {
+    const i: Item = { baseId: 'id', base: { id: 'id' } }
+    const database = new IndexedDatabase({
+      indexedDB: new IDBFactory(),
+      keyRange: IDBKeyRange,
+      maxCacheBatchWriteWait: 200
+    })
+    await database.add(i)
+    await database.disposeAsync()
+
+    const x = await database.getItem({ id: 'id' })
+    expect(x).toBeDefined()
+    expect(JSON.stringify(x)).toBe(JSON.stringify(i))
+  })
+
+  test('write two items to queue use getItem', async () => {
+    const i1: Item = { baseId: 'id1', base: { id: 'id' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id' } }
+    const database = new IndexedDatabase({
+      indexedDB: new IDBFactory(),
+      keyRange: IDBKeyRange
+    })
+    await database.add(i1)
+    await database.add(i2)
+    await database.disposeAsync()
+
+    const x1 = await database.getItem({ id: i1.baseId })
+    expect(x1).toBeDefined()
+    expect(JSON.stringify(x1)).toBe(JSON.stringify(i1))
+
+    const x2 = await database.getItem({ id: i2.baseId })
+    expect(x2).toBeDefined()
+    expect(JSON.stringify(x2)).toBe(JSON.stringify(i2))
+  })
+
+  test('write two items to queue use getItem', async () => {
+    const i1: Item = { baseId: 'id1', base: { id: 'id' } }
+    const i2: Item = { baseId: 'id2', base: { id: 'id' } }
+    const database = new IndexedDatabase({
+      indexedDB: new IDBFactory(),
+      keyRange: IDBKeyRange
+    })
+    await database.add(i1)
+    await database.add(i2)
+    await database.disposeAsync()
+
+    const foundItems = new BufferQueue<Item>()
+    const notFoundItems = new BufferQueue<string>()
+
+    await database.processItems({
+      ids: [i1.baseId, i2.baseId],
+      foundItems,
+      notFoundItems
+    })
+
+    expect(foundItems.values().length).toBe(2)
+    expect(JSON.stringify(foundItems.values()[0])).toBe(JSON.stringify(i1))
+    expect(JSON.stringify(foundItems.values()[1])).toBe(JSON.stringify(i2))
+
+    expect(notFoundItems.values().length).toBe(0)
+  })
+})

package/src/operations/indexedDatabase.ts

@@ -0,0 +1,167 @@
+import BatchingQueue from '../helpers/batchingQueue.js'
+import Queue from '../helpers/queue.js'
+import { CustomLogger, Item } from '../types/types.js'
+import { isSafari } from '@speckle/shared'
+import { BaseDatabaseOptions } from './options.js'
+import { Cache } from './interfaces.js'
+import { Dexie, DexieOptions, Table } from 'dexie'
+
+class ObjectStore extends Dexie {
+  static #databaseName: string = 'speckle-cache'
+  objects!: Table<Item, string> // Table type: <entity, primaryKey>
+
+  constructor(options: DexieOptions) {
+    super(ObjectStore.#databaseName, options)
+
+    this.version(1).stores({
+      objects: 'baseId, item' // baseId is primary key
+    })
+  }
+}
+
+export default class IndexedDatabase implements Cache {
+  #options: BaseDatabaseOptions
+  #logger: CustomLogger
+
+  #cacheDB?: ObjectStore
+
+  #writeQueue: BatchingQueue<Item> | undefined
+
+  // #count: number = 0
+
+  constructor(options: BaseDatabaseOptions) {
+    this.#options = {
+      ...{
+        maxCacheReadSize: 10000,
+        maxCacheBatchWriteWait: 1000
+      },
+      ...options
+    }
+    this.#logger = options.logger || (() => {})
+  }
+
+  async add(item: Item): Promise<void> {
+    if (!this.#writeQueue) {
+      await this.#setupCacheDb()
+      this.#writeQueue = new BatchingQueue<Item>({
+        batchSize: this.#options.maxCacheWriteSize ?? 10000,
+        maxWaitTime: this.#options.maxCacheBatchWriteWait,
+        processFunction: (batch: Item[]) =>
+          this.#cacheSaveBatch({ batch, cacheDB: this.#cacheDB! })
+      })
+    }
+    this.#writeQueue.add(item)
+  }
+
+  async disposeAsync(): Promise<void> {
+    await this.#writeQueue?.disposeAsync()
+  }
+
+  async #openDatabase(): Promise<ObjectStore> {
+    const db = new ObjectStore({
+      indexedDB: this.#options.indexedDB ?? globalThis.indexedDB,
+      IDBKeyRange: this.#options.keyRange ?? IDBKeyRange,
+      chromeTransactionDurability: 'relaxed'
+    })
+    await db.open()
+    return db
+  }
+
+  async #setupCacheDb(): Promise<void> {
+    if (this.#cacheDB !== undefined) {
+      return
+    }
+
+    // Initialize
+    await this.#safariFix()
+    this.#cacheDB = await this.#openDatabase()
+  }
+
+  async processItems(params: {
+    ids: string[]
+    foundItems: Queue<Item>
+    notFoundItems: Queue<string>
+  }): Promise<void> {
+    const { ids, foundItems, notFoundItems } = params
+    await this.#setupCacheDb()
+    const maxCacheReadSize = this.#options.maxCacheReadSize ?? 10000
+
+    for (let i = 0; i < ids.length; ) {
+      if ((this.#writeQueue?.count() ?? 0) > maxCacheReadSize * 2) {
+        this.#logger(
+          'pausing reads (# in write queue: ' + this.#writeQueue?.count() + ')'
+        )
+        await new Promise((resolve) => setTimeout(resolve, 1000)) // Pause for 1 second, protects against out of memory
+        continue
+      }
+      const batch = ids.slice(i, i + maxCacheReadSize)
+      //   const x = this.#count
+      //   this.#count++
+      //  const startTime = performance.now()
+      // this.#logger('Start read ' + x + ' ' + batch.length)
+
+      //faster than BulkGet with dexie
+      await this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
+        const gets = batch.map((key) => this.#cacheDB!.objects.get(key))
+        const cachedData = await Promise.all(gets)
+        for (let i = 0; i < cachedData.length; i++) {
+          if (cachedData[i]) {
+            foundItems.add(cachedData[i]!)
+          } else {
+            notFoundItems.add(batch[i])
+          }
+        }
+      })
+      // const endTime = performance.now()
+      // const duration = endTime - startTime
+      // this.#logger('Read batch ' + x + ' ' + batch.length + ' ' + duration / 1000)
+
+      // interate down here to help with pausing
+      i += maxCacheReadSize
+    }
+  }
+
+  async getItem(params: { id: string }): Promise<Item | undefined> {
+    const { id } = params
+    await this.#setupCacheDb()
+
+    return this.#cacheDB!.transaction('r', this.#cacheDB!.objects, async () => {
+      return await this.#cacheDB?.objects.get(id)
+    })
+  }
+
+  async #cacheSaveBatch(params: {
+    batch: Item[]
+    cacheDB: ObjectStore
+  }): Promise<void> {
+    const { batch, cacheDB } = params
+    //const x = this.#count
+    //this.#count++
+
+    // const startTime = performance.now()
+    //  this.#logger('Start save ' + x + ' ' + batch.length)
+    await cacheDB.objects.bulkPut(batch)
+    // const endTime = performance.now()
+    // const duration = endTime - startTime
+    //this.#logger('Saved batch ' + x + ' ' + batch.length + ' ' + duration / 1000)
+  }
+
+  /**
+   * Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
+   * @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+   */
+  async #safariFix(): Promise<void> {
+    // No point putting other browsers or older versions of Safari through this mess.
+    if (!isSafari() || !this.#options.indexedDB?.databases) return Promise.resolve()
+
+    let intervalId: ReturnType<typeof setInterval>
+
+    return new Promise<void>((resolve: () => void) => {
+      const tryIdb = () => this.#options.indexedDB?.databases().finally(resolve)
+      intervalId = setInterval(() => {
+        void tryIdb()
+      }, 100)
+      void tryIdb()
+    }).finally(() => clearInterval(intervalId))
+  }
+}