@speckle/objectloader 2.1.1 → 2.4.0

package/examples/node/script.js

@@ -1 +1,24 @@
-// NOTE: This lib is not working in node, because node-fetch returns node-native readable streams - we need a workaround first. 
+// Since Node v<18 does not provide fetch, we need to pass it in the options object. Note that fetch must return a WHATWG compliant stream, so cross-fetch won't work, but node/undici's implementation will.
+
+import { fetch } from 'undici'
+import ObjectLoader from '../../index.js'
+
+let loader = new ObjectLoader({
+  serverUrl:"https://latest.speckle.dev",
+  streamId:"3ed8357f29",
+  objectId:"0408ab9caaa2ebefb2dd7f1f671e7555",
+  options:{ enableCaching: false, excludeProps:[], fetch }
+})
+
+
+let loadData = async function loadData() {
+
+  let obj = await loader.getAndConstructObject((e) =>{
+    console.log(e) // log progress!
+  })
+
+  console.log('Done!')
+  console.log( obj )
+}
+
+loadData()

package/index.js

@@ -1,34 +1,39 @@
 /**
- * Simple client that streams object info from a Speckle Server. 
+ * Simple client that streams object info from a Speckle Server.
  * TODO: Object construction progress reporting is weird.
  */
 
 
-
 export default class ObjectLoader {
 
   /**
    * Creates a new object loader instance.
-   * @param {*} param0 
+   * @param {*} param0
    */
-  constructor( { serverUrl, streamId, token, objectId, options = { fullyTraverseArrays: false, excludeProps: [ ] } } ) {
+  constructor( { serverUrl, streamId, token, objectId, options = { enableCaching: true, fullyTraverseArrays: false, excludeProps: [ ], fetch:null } } ) {
     this.INTERVAL_MS = 20
     this.TIMEOUT_MS = 180000 // three mins
 
     this.serverUrl = serverUrl || window.location.origin
     this.streamId = streamId
     this.objectId = objectId
-    this.token = token || localStorage.getItem( 'AuthToken' )
+    console.log('Object loader constructor called!')
+    try {
+      this.token = token || localStorage.getItem( 'AuthToken' )
+    } catch (error) {
+        // Accessing localStorage may throw when executing on sandboxed document, ignore.
+    }
 
     this.headers = {
       'Accept': 'text/plain'
     }
 
-    if( token ) {
+    if( this.token ) {
       this.headers['Authorization'] = `Bearer ${this.token}`
     }
 
-    this.requestUrl = `${this.serverUrl}/objects/${this.streamId}/${this.objectId}`
+    this.requestUrlRootObj = `${this.serverUrl}/objects/${this.streamId}/${this.objectId}/single`
+    this.requestUrlChildren = `${this.serverUrl}/api/getobjects/${this.streamId}`
     this.promises = []
     this.intervals = {}
     this.buffer = []
@@ -36,6 +41,35 @@ export default class ObjectLoader {
     this.totalChildrenCount = 0
     this.traversedReferencesCount = 0
     this.options = options
+    this.options.numConnections = this.options.numConnections || 4
+
+    this.cacheDB = null
+
+    this.lastAsyncPause = Date.now()
+    this.existingAsyncPause = null
+
+    // we can't simply bind fetch to this.fetch, so instead we have to do some acrobatics: https://stackoverflow.com/questions/69337187/uncaught-in-promise-typeerror-failed-to-execute-fetch-on-workerglobalscope#comment124731316_69337187
+    this.preferredFetch = options.fetch
+    this.fetch = function(...args) {
+      let currentFetch = this.preferredFetch || fetch
+      return currentFetch(...args)
+    }
+
+  }
+
+  async asyncPause() {
+    // Don't freeze the UI
+    // while ( this.existingAsyncPause ) {
+    //   await this.existingAsyncPause
+    // }
+    if ( Date.now() - this.lastAsyncPause >= 100 ) {
+      this.lastAsyncPause = Date.now()
+      this.existingAsyncPause = new Promise( resolve => setTimeout( resolve, 0 ) )
+      await this.existingAsyncPause
+      this.existingAsyncPause = null
+      if (Date.now() - this.lastAsyncPause > 500) console.log("Loader Event loop lag: ", Date.now() - this.lastAsyncPause)
+    }
+
   }
 
   dispose() {
@@ -45,25 +79,25 @@ export default class ObjectLoader {
 
   /**
    * Use this method to receive and construct the object. It will return the full, de-referenced and de-chunked original object.
-   * @param {*} onProgress 
-   * @returns 
+   * @param {*} onProgress
+   * @returns
    */
   async getAndConstructObject( onProgress ) {
-    
+
     ;( await this.downloadObjectsInBuffer( onProgress ) ) // Fire and forget; PS: semicolon of doom
-    
+
     let rootObject = await this.getObject( this.objectId )
     return this.traverseAndConstruct( rootObject, onProgress )
   }
 
   /**
    * Internal function used to download all the objects in a local buffer.
-   * @param {*} onProgress 
+   * @param {*} onProgress
    */
   async downloadObjectsInBuffer( onProgress ) {
     let first = true
     let downloadNum = 0
-    
+
     for await ( let obj of this.getObjectIterator() ) {
       if( first ) {
         this.totalChildrenCount = obj.totalChildrenCount
@@ -78,9 +112,9 @@ export default class ObjectLoader {
 
   /**
    * Internal function used to recursively traverse an object and populate its references and dechunk any arrays.
-   * @param {*} obj 
-   * @param {*} onProgress 
-   * @returns 
+   * @param {*} obj
+   * @param {*} onProgress
+   * @returns
    */
   async traverseAndConstruct( obj, onProgress ) {
     if( !obj ) return
@@ -91,20 +125,20 @@ export default class ObjectLoader {
       let arr = []
       for ( let element of obj ) {
         if ( typeof element !== 'object' && ! this.options.fullyTraverseArrays ) return obj
-        
+
         // Dereference element if needed
         let deRef = element.referencedId ? await this.getObject( element.referencedId ) : element
-        if( element.referencedId && onProgress ) onProgress( { stage: 'construction', current: ++this.traversedReferencesCount > this.totalChildrenCount ? this.totalChildrenCount : this.traversedReferencesCount, total: this.totalChildrenCount } ) 
-        
+        if( element.referencedId && onProgress ) onProgress( { stage: 'construction', current: ++this.traversedReferencesCount > this.totalChildrenCount ? this.totalChildrenCount : this.traversedReferencesCount, total: this.totalChildrenCount } )
+
         // Push the traversed object in the array
         arr.push( await this.traverseAndConstruct( deRef, onProgress ) )
       }
-      
+
       // De-chunk
-      if( arr[0]?.speckle_type?.toLowerCase().includes('datachunk') ) { 
+      if( arr[0]?.speckle_type?.toLowerCase().includes('datachunk') ) {
         return arr.reduce( ( prev, curr ) => prev.concat( curr.data ), [] )
       }
-      
+
       return arr
      }
 
@@ -113,11 +147,11 @@ export default class ObjectLoader {
     for( let ignoredProp of this.options.excludeProps ) {
       delete obj[ ignoredProp ]
     }
-    
+
     // 2) Iterate through obj
-    for( let prop in obj ) {      
-      if( typeof obj[prop] !== 'object' ) continue // leave alone primitive props
-      
+    for( let prop in obj ) {
+      if( typeof obj[prop] !== 'object' || obj[prop] === null ) continue // leave alone primitive props
+
       if( obj[prop].referencedId ) {
         obj[prop] = await this.getObject( obj[prop].referencedId )
         if( onProgress ) onProgress( { stage: 'construction', current: ++this.traversedReferencesCount > this.totalChildrenCount ? this.totalChildrenCount : this.traversedReferencesCount, total: this.totalChildrenCount } )
@@ -131,8 +165,8 @@ export default class ObjectLoader {
 
   /**
    * Internal function. Returns a promise that is resolved when the object id is loaded into the internal buffer.
-   * @param {*} id 
-   * @returns 
+   * @param {*} id
+   * @returns
    */
   async getObject( id ){
     if ( this.buffer[id] ) return this.buffer[id]
@@ -172,11 +206,15 @@ export default class ObjectLoader {
   }
 
   async * getObjectIterator(  ) {
+    let t0 = Date.now()
+    let count = 0
     for await ( let line of this.getRawObjectIterator() ) {
       let { id, obj } = this.processLine( line )
       this.buffer[ id ] = obj
+      count += 1
       yield obj
     }
+    console.log(`Loaded ${count} objects in: ${(Date.now() - t0) / 1000}`)
   }
 
   processLine( chunk ) {
@@ -185,31 +223,258 @@ export default class ObjectLoader {
   }
 
   async * getRawObjectIterator() {
-    const decoder = new TextDecoder()
-    const response = await fetch( this.requestUrl, { headers: this.headers } )
-    const reader = response.body.getReader()
-    let { value: chunk, done: readerDone } = await reader.read()
-    chunk = chunk ? decoder.decode( chunk ) : ''
+    let tSTART = Date.now()
+
+    if ( this.options.enableCaching && window.indexedDB && this.cacheDB === null) {
+      await safariFix()
+      let idbOpenRequest = indexedDB.open('speckle-object-cache', 1)
+      idbOpenRequest.onupgradeneeded = () => idbOpenRequest.result.createObjectStore('objects');
+      this.cacheDB = await this.promisifyIdbRequest( idbOpenRequest )
+    }
+
+    const rootObjJson = await this.getRawRootObject()
+    // console.log("Root in: ", Date.now() - tSTART)
+
+    yield `${this.objectId}\t${rootObjJson}`
+
+    const rootObj = JSON.parse(rootObjJson)
+    if ( !rootObj.__closure ) return
+
+    let childrenIds = Object.keys(rootObj.__closure).sort( (a, b) => rootObj.__closure[a] - rootObj.__closure[b] )
+    if ( childrenIds.length === 0 ) return
+
+    let splitHttpRequests = []
+
+    if ( childrenIds.length > 50 ) {
+      // split into 5%, 15%, 40%, 40% (5% for the high priority children: the ones with lower minDepth)
+      let splitBeforeCacheCheck = [ [], [], [], [] ]
+      let crtChildIndex = 0
+
+      for ( ; crtChildIndex < 0.05 * childrenIds.length; crtChildIndex++ ) {
+        splitBeforeCacheCheck[0].push( childrenIds[ crtChildIndex ] )
+      }
+      for ( ; crtChildIndex < 0.2 * childrenIds.length; crtChildIndex++ ) {
+        splitBeforeCacheCheck[1].push( childrenIds[ crtChildIndex ] )
+      }
+      for ( ; crtChildIndex < 0.6 * childrenIds.length; crtChildIndex++ ) {
+        splitBeforeCacheCheck[2].push( childrenIds[ crtChildIndex ] )
+      }
+      for ( ; crtChildIndex < childrenIds.length; crtChildIndex++ ) {
+        splitBeforeCacheCheck[3].push( childrenIds[ crtChildIndex ] )
+      }
+
+
+      console.log("Cache check for: ", splitBeforeCacheCheck)
+
+      let newChildren = []
+      let nextCachePromise = this.cacheGetObjects( splitBeforeCacheCheck[ 0 ] )
+
+      for ( let i = 0; i < 4; i++ ) {
+        let cachedObjects = await nextCachePromise
+        if ( i < 3 ) nextCachePromise = this.cacheGetObjects( splitBeforeCacheCheck[ i + 1 ] )
+
+        let sortedCachedKeys = Object.keys(cachedObjects).sort( (a, b) => rootObj.__closure[a] - rootObj.__closure[b] )
+        for ( let id of sortedCachedKeys ) {
+          yield `${id}\t${cachedObjects[ id ]}`
+        }
+        let newChildrenForBatch = splitBeforeCacheCheck[i].filter( id => !( id in cachedObjects ) )
+        newChildren.push( ...newChildrenForBatch )
+      }
+
+      if ( newChildren.length === 0 ) return
+
+      if ( newChildren.length <= 50 ) {
+        // we have almost all of children in the cache. do only 1 requests for the remaining new children
+        splitHttpRequests.push( newChildren )
+      } else {
+        // we now set up the batches for 4 http requests, starting from `newChildren` (already sorted by priority)
+        splitHttpRequests = [ [], [], [], [] ]
+        crtChildIndex = 0
+
+        for ( ; crtChildIndex < 0.05 * newChildren.length; crtChildIndex++ ) {
+          splitHttpRequests[0].push( newChildren[ crtChildIndex ] )
+        }
+        for ( ; crtChildIndex < 0.2 * newChildren.length; crtChildIndex++ ) {
+          splitHttpRequests[1].push( newChildren[ crtChildIndex ] )
+        }
+        for ( ; crtChildIndex < 0.6 * newChildren.length; crtChildIndex++ ) {
+          splitHttpRequests[2].push( newChildren[ crtChildIndex ] )
+        }
+        for ( ; crtChildIndex < newChildren.length; crtChildIndex++ ) {
+          splitHttpRequests[3].push( newChildren[ crtChildIndex ] )
+        }
+      }
+
+    } else {
+      // small object with <= 50 children. check cache and make only 1 request
+      const cachedObjects = await this.cacheGetObjects( childrenIds )
+      let sortedCachedKeys = Object.keys(cachedObjects).sort( (a, b) => rootObj.__closure[a] - rootObj.__closure[b] )
+      for ( let id of sortedCachedKeys ) {
+        yield `${id}\t${cachedObjects[ id ]}`
+      }
+      childrenIds = childrenIds.filter(id => !( id in cachedObjects ) )
+      if ( childrenIds.length === 0 ) return
+
+      // only 1 http request with the remaining children ( <= 50 )
+      splitHttpRequests.push( childrenIds )
+    }
+
+    // Starting http requests for batches in `splitHttpRequests`
 
-    let re = /\r\n|\n|\r/gm
-    let startIndex = 0
+    const decoders = []
+    const readers = []
+    const readPromisses = []
+    const startIndexes = []
+    const readBuffers = []
+    const finishedRequests = []
+
+    for (let i = 0; i < splitHttpRequests.length; i++) {
+      decoders.push(new TextDecoder())
+      readers.push( null )
+      readPromisses.push( null )
+      startIndexes.push( 0 )
+      readBuffers.push( '' )
+      finishedRequests.push( false )
+
+      this.fetch(
+        this.requestUrlChildren,
+        {
+          method: 'POST',
+          headers: { ...this.headers, 'Content-Type': 'application/json' },
+          body: JSON.stringify( { objects: JSON.stringify( splitHttpRequests[i] ) } )
+        }
+      ).then( crtResponse => {
+        let crtReader = crtResponse.body.getReader()
+        readers[i] = crtReader
+        let crtReadPromise = crtReader.read().then(x => { x.reqId = i; return x })
+        readPromisses[i] = crtReadPromise
+      })
+    }
 
     while ( true ) {
-      let result = re.exec( chunk )
-      if ( !result ) {
-        if ( readerDone ) break
-        let remainder = chunk.substr( startIndex ) 
-        ;( { value: chunk, done: readerDone } = await reader.read() ) // PS: semicolon of doom
-        chunk = remainder + ( chunk ? decoder.decode( chunk ) : '' )
-        startIndex = re.lastIndex = 0
+      let validReadPromises = readPromisses.filter(x => x != null)
+      if ( validReadPromises.length === 0 ) {
+        // Check if all requests finished
+        if ( finishedRequests.every(x => x) ) {
+          break
+        }
+        // Sleep 10 ms
+        await new Promise( ( resolve ) => {
+          setTimeout( resolve, 10 )
+        } )
+        continue
+      }
+
+      // Wait for data on any running request
+      let data = await Promise.any( validReadPromises )
+      let { value: crtDataChunk, done: readerDone, reqId } = data
+      finishedRequests[ reqId ] = readerDone
+
+      // Replace read promise on this request with a new `read` call
+      if ( !readerDone ) {
+         let crtReadPromise = readers[ reqId ].read().then(x => { x.reqId = reqId; return x })
+         readPromisses[ reqId ] = crtReadPromise
+      } else {
+        // This request finished. "Flush any non-newline-terminated text"
+        if ( readBuffers[ reqId ].length > 0 ) {
+          yield readBuffers[ reqId ]
+          readBuffers[ reqId ] = ''
+        }
+        // no other read calls for this request
+        readPromisses[ reqId ] = null
+      }
+
+      if ( !crtDataChunk )
         continue
+
+      crtDataChunk = decoders[ reqId ].decode( crtDataChunk )
+      let unprocessedText = readBuffers[ reqId ] + crtDataChunk
+      let unprocessedLines = unprocessedText.split(/\r\n|\n|\r/)
+      let remainderText = unprocessedLines.pop()
+      readBuffers[ reqId ] = remainderText
+
+      for ( let line of unprocessedLines ) {
+        yield line
       }
-      yield chunk.substring( startIndex, result.index )
-      startIndex = re.lastIndex
+      this.cacheStoreObjects(unprocessedLines)
     }
+  }
 
-    if ( startIndex < chunk.length ) {
-      yield chunk.substr( startIndex )
+  async getRawRootObject() {
+    const cachedRootObject = await this.cacheGetObjects( [ this.objectId ] )
+    if ( cachedRootObject[ this.objectId ] )
+      return cachedRootObject[ this.objectId ]
+    const response = await this.fetch( this.requestUrlRootObj, { headers: this.headers } )
+    const responseText = await response.text()
+    this.cacheStoreObjects( [ `${this.objectId}\t${responseText}` ] )
+    return responseText
+  }
+
+  promisifyIdbRequest(request) {
+    return new Promise((resolve, reject) => {
+      request.oncomplete = request.onsuccess = () => resolve(request.result);
+      request.onabort = request.onerror = () => reject(request.error);
+    })
+  }
+
+  async cacheGetObjects(ids) {
+    if ( !this.options.enableCaching || !window.indexedDB ) {
+      return {}
     }
+
+    let ret = {}
+
+    for (let i = 0; i < ids.length; i += 500) {
+      let idsChunk = ids.slice(i, i + 500)
+      let t0 = Date.now()
+
+      let store = this.cacheDB.transaction('objects', 'readonly').objectStore('objects')
+      let idbChildrenPromises = idsChunk.map( id => this.promisifyIdbRequest( store.get( id ) ).then( data => ( { id, data } ) ) )
+      let cachedData = await Promise.all(idbChildrenPromises)
+
+      // console.log("Cache check for : ", idsChunk.length, Date.now() - t0)
+
+      for ( let cachedObj of cachedData ) {
+        if ( !cachedObj.data ) // non-existent objects are retrieved with `undefined` data
+          continue
+        ret[ cachedObj.id ] = cachedObj.data
+      }
+    }
+
+    return ret
   }
+
+  cacheStoreObjects(objects) {
+    if ( !this.options.enableCaching || !window.indexedDB ) {
+      return {}
+    }
+
+    let store = this.cacheDB.transaction('objects', 'readwrite').objectStore('objects')
+    for ( let obj of objects ) {
+      let idAndData = obj.split( '\t' )
+      store.put(idAndData[1], idAndData[0])
+    }
+
+    return this.promisifyIdbRequest( store.transaction )
+  }
+}
+
+
+// Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
+function safariFix() {
+  const isSafari =
+    !navigator.userAgentData &&
+    /Safari\//.test(navigator.userAgent) &&
+    !/Chrom(e|ium)\//.test(navigator.userAgent)
+
+  // No point putting other browsers or older versions of Safari through this mess.
+  if (!isSafari || !indexedDB.databases) return Promise.resolve()
+
+  let intervalId
+
+  return new Promise( ( resolve ) => {
+    const tryIdb = () => indexedDB.databases().finally(resolve)
+    intervalId = setInterval(tryIdb, 100)
+    tryIdb()
+  }).finally( () => clearInterval(intervalId) )
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@speckle/objectloader",
-  "version": "2.1.1",
+  "version": "2.4.0",
   "description": "Simple API helper to stream in objects from the Speckle Server.",
   "main": "index.js",
   "homepage": "https://speckle.systems",
@@ -15,5 +15,8 @@
   },
   "keywords": ["speckle", "aec", "speckle api"],
   "author": "AEC Systems",
-  "license": "Apache-2.0"
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "undici": "^4.14.1"
+  }
 }

package/readme.md

@@ -10,9 +10,11 @@ Comprehensive developer and user documentation can be found in our:
 
 ## Getting started
 
-This is a small utility class that helps you stream an object and all its sub-components from the Speckle Server API. It is inteded to be used in contexts where you want to "download" the whole object, or iteratively traverse its whole tree.
+This is a small utility class that helps you stream an object and all its sub-components from the Speckle Server API. It is intended to be used in contexts where you want to "download" the whole object, or iteratively traverse its whole tree.
 
-Here's a sample way on how to use it, pfilfered from the [3d viewer package](../viewer):
+### In the browser
+
+Here's a sample way on how to use it, pilfered from the [3d viewer package](../viewer):
 
 ```js
 
@@ -52,6 +54,19 @@ let loader = new ObjectLoader( {
 
 let obj = await loader.getAndConstructObject( ( e ) => console.log( 'Progress', e ) )
 
+### On the server
+
+Since Node.js does not yet support the [`fetch API`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch), you'll need to provide your own `fetch` function in the options object. Note that `fetch` must return a [Web Stream](https://nodejs.org/api/webstreams.html), so [node-fetch](https://github.com/node-fetch/node-fetch) won't work, but [node/undici's](https://undici.nodejs.org/) implementation will.
+
+```js
+import { fetch } from 'undici'
+
+let loader = new ObjectLoader({
+  serverUrl: 'https://latest.speckle.dev',
+  streamId: '3ed8357f29',
+  objectId: '0408ab9caaa2ebefb2dd7f1f671e7555',
+  options: { enableCaching: false, excludeProps: [], fetch },
+})
 ```
 
 ## Community