@synchronized-console/objectloader2 2.25.7 → 2.31.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commonjs/{operations → core}/interfaces.d.ts +5 -7
- package/dist/commonjs/core/interfaces.d.ts.map +1 -0
- package/dist/commonjs/core/interfaces.js.map +1 -0
- package/dist/commonjs/{operations → core}/objectLoader2.d.ts +1 -1
- package/dist/commonjs/core/objectLoader2.d.ts.map +1 -0
- package/dist/commonjs/core/objectLoader2.js +120 -0
- package/dist/commonjs/core/objectLoader2.js.map +1 -0
- package/dist/{esm/operations → commonjs/core}/objectLoader2Factory.d.ts +10 -2
- package/dist/commonjs/core/objectLoader2Factory.d.ts.map +1 -0
- package/dist/commonjs/core/objectLoader2Factory.js +86 -0
- package/dist/commonjs/core/objectLoader2Factory.js.map +1 -0
- package/dist/commonjs/core/objectLoader2Factory.test.d.ts +2 -0
- package/dist/commonjs/core/objectLoader2Factory.test.d.ts.map +1 -0
- package/dist/commonjs/core/objectLoader2Factory.test.js +106 -0
- package/dist/commonjs/core/objectLoader2Factory.test.js.map +1 -0
- package/dist/{esm/operations → commonjs/core}/options.d.ts +4 -1
- package/dist/commonjs/core/options.d.ts.map +1 -0
- package/dist/commonjs/{operations → core}/options.js.map +1 -1
- package/dist/commonjs/core/stages/cacheReader.d.ts +18 -0
- package/dist/commonjs/core/stages/cacheReader.d.ts.map +1 -0
- package/dist/commonjs/core/stages/cacheReader.js +72 -0
- package/dist/commonjs/core/stages/cacheReader.js.map +1 -0
- package/dist/commonjs/core/stages/cacheWriter.d.ts +15 -0
- package/dist/commonjs/core/stages/cacheWriter.d.ts.map +1 -0
- package/dist/commonjs/core/stages/cacheWriter.js +50 -0
- package/dist/commonjs/core/stages/cacheWriter.js.map +1 -0
- package/dist/commonjs/core/stages/indexedDatabase.d.ts +34 -0
- package/dist/commonjs/core/stages/indexedDatabase.d.ts.map +1 -0
- package/dist/commonjs/core/stages/indexedDatabase.js +150 -0
- package/dist/commonjs/core/stages/indexedDatabase.js.map +1 -0
- package/dist/commonjs/core/stages/memory/memoryDatabase.d.ts +11 -0
- package/dist/commonjs/core/stages/memory/memoryDatabase.d.ts.map +1 -0
- package/dist/commonjs/{operations/databases → core/stages/memory}/memoryDatabase.js +6 -3
- package/dist/commonjs/core/stages/memory/memoryDatabase.js.map +1 -0
- package/dist/commonjs/{operations/downloaders → core/stages/memory}/memoryDownloader.d.ts +4 -4
- package/dist/commonjs/core/stages/memory/memoryDownloader.d.ts.map +1 -0
- package/dist/commonjs/{operations/downloaders → core/stages/memory}/memoryDownloader.js +1 -1
- package/dist/commonjs/core/stages/memory/memoryDownloader.js.map +1 -0
- package/dist/{esm/operations/downloaders → commonjs/core/stages}/serverDownloader.d.ts +8 -6
- package/dist/commonjs/core/stages/serverDownloader.d.ts.map +1 -0
- package/dist/commonjs/{operations/downloaders → core/stages}/serverDownloader.js +99 -50
- package/dist/commonjs/core/stages/serverDownloader.js.map +1 -0
- package/dist/commonjs/core/traverser.d.ts.map +1 -0
- package/dist/commonjs/{operations → core}/traverser.js +7 -7
- package/dist/commonjs/core/traverser.js.map +1 -0
- package/dist/commonjs/deferment/MemoryCache.d.ts +35 -0
- package/dist/commonjs/deferment/MemoryCache.d.ts.map +1 -0
- package/dist/commonjs/deferment/MemoryCache.js +162 -0
- package/dist/commonjs/deferment/MemoryCache.js.map +1 -0
- package/dist/commonjs/deferment/MemoryCache.test.d.ts +2 -0
- package/dist/commonjs/deferment/MemoryCache.test.d.ts.map +1 -0
- package/dist/commonjs/deferment/MemoryCache.test.js +153 -0
- package/dist/commonjs/deferment/MemoryCache.test.js.map +1 -0
- package/dist/commonjs/deferment/defermentManager.d.ts +32 -0
- package/dist/commonjs/deferment/defermentManager.d.ts.map +1 -0
- package/dist/commonjs/deferment/defermentManager.js +85 -0
- package/dist/commonjs/deferment/defermentManager.js.map +1 -0
- package/dist/commonjs/deferment/defermentManager.test.d.ts +2 -0
- package/dist/commonjs/deferment/defermentManager.test.d.ts.map +1 -0
- package/dist/commonjs/deferment/defermentManager.test.js +193 -0
- package/dist/commonjs/deferment/defermentManager.test.js.map +1 -0
- package/dist/commonjs/deferment/deferredBase.d.ts +12 -0
- package/dist/commonjs/deferment/deferredBase.d.ts.map +1 -0
- package/dist/commonjs/deferment/deferredBase.js +27 -0
- package/dist/commonjs/deferment/deferredBase.js.map +1 -0
- package/dist/commonjs/index.d.ts +4 -2
- package/dist/commonjs/index.d.ts.map +1 -1
- package/dist/commonjs/index.js +6 -3
- package/dist/commonjs/index.js.map +1 -1
- package/dist/commonjs/{helpers → queues}/aggregateQueue.d.ts +2 -1
- package/dist/commonjs/queues/aggregateQueue.d.ts.map +1 -0
- package/dist/commonjs/{helpers → queues}/aggregateQueue.js +4 -0
- package/dist/commonjs/queues/aggregateQueue.js.map +1 -0
- package/dist/{esm/helpers → commonjs/queues}/asyncGeneratorQueue.d.ts +1 -1
- package/dist/commonjs/queues/asyncGeneratorQueue.d.ts.map +1 -0
- package/dist/commonjs/{helpers → queues}/asyncGeneratorQueue.js +2 -1
- package/dist/commonjs/queues/asyncGeneratorQueue.js.map +1 -0
- package/dist/commonjs/{helpers → queues}/batchingQueue.d.ts +4 -1
- package/dist/commonjs/queues/batchingQueue.d.ts.map +1 -0
- package/dist/commonjs/queues/batchingQueue.dispose.test.d.ts +2 -0
- package/dist/commonjs/queues/batchingQueue.dispose.test.d.ts.map +1 -0
- package/dist/commonjs/queues/batchingQueue.dispose.test.js +62 -0
- package/dist/commonjs/queues/batchingQueue.dispose.test.js.map +1 -0
- package/dist/commonjs/queues/batchingQueue.js +128 -0
- package/dist/commonjs/queues/batchingQueue.js.map +1 -0
- package/dist/commonjs/queues/batchingQueue.test.d.ts +2 -0
- package/dist/commonjs/queues/batchingQueue.test.d.ts.map +1 -0
- package/dist/commonjs/queues/batchingQueue.test.js +236 -0
- package/dist/commonjs/queues/batchingQueue.test.js.map +1 -0
- package/dist/{esm/helpers → commonjs/queues}/bufferQueue.d.ts +1 -0
- package/dist/commonjs/queues/bufferQueue.d.ts.map +1 -0
- package/dist/commonjs/{helpers → queues}/bufferQueue.js +3 -0
- package/dist/commonjs/queues/bufferQueue.js.map +1 -0
- package/dist/commonjs/{helpers → queues}/keyedQueue.d.ts +1 -0
- package/dist/commonjs/queues/keyedQueue.d.ts.map +1 -0
- package/dist/commonjs/{helpers → queues}/keyedQueue.js +11 -0
- package/dist/commonjs/queues/keyedQueue.js.map +1 -0
- package/dist/commonjs/queues/keyedQueue.test.d.ts +2 -0
- package/dist/commonjs/queues/keyedQueue.test.d.ts.map +1 -0
- package/dist/commonjs/queues/keyedQueue.test.js +118 -0
- package/dist/commonjs/queues/keyedQueue.test.js.map +1 -0
- package/dist/commonjs/queues/queue.d.ts +5 -0
- package/dist/commonjs/queues/queue.d.ts.map +1 -0
- package/dist/commonjs/{helpers → queues}/queue.js.map +1 -1
- package/dist/commonjs/types/functions.d.ts +21 -0
- package/dist/commonjs/types/functions.d.ts.map +1 -0
- package/dist/commonjs/types/functions.js +92 -0
- package/dist/commonjs/types/functions.js.map +1 -0
- package/dist/commonjs/types/types.d.ts +6 -7
- package/dist/commonjs/types/types.d.ts.map +1 -1
- package/dist/commonjs/types/types.js +0 -36
- package/dist/commonjs/types/types.js.map +1 -1
- package/dist/esm/{operations → core}/interfaces.d.ts +5 -7
- package/dist/esm/core/interfaces.d.ts.map +1 -0
- package/dist/esm/core/interfaces.js.map +1 -0
- package/dist/esm/{operations → core}/objectLoader2.d.ts +1 -1
- package/dist/esm/core/objectLoader2.d.ts.map +1 -0
- package/dist/esm/core/objectLoader2.js +113 -0
- package/dist/esm/core/objectLoader2.js.map +1 -0
- package/dist/{commonjs/operations → esm/core}/objectLoader2Factory.d.ts +10 -2
- package/dist/esm/core/objectLoader2Factory.d.ts.map +1 -0
- package/dist/esm/core/objectLoader2Factory.js +79 -0
- package/dist/esm/core/objectLoader2Factory.js.map +1 -0
- package/dist/esm/core/objectLoader2Factory.test.d.ts +2 -0
- package/dist/esm/core/objectLoader2Factory.test.d.ts.map +1 -0
- package/dist/esm/core/objectLoader2Factory.test.js +104 -0
- package/dist/esm/core/objectLoader2Factory.test.js.map +1 -0
- package/dist/{commonjs/operations → esm/core}/options.d.ts +4 -1
- package/dist/esm/core/options.d.ts.map +1 -0
- package/dist/esm/{operations → core}/options.js.map +1 -1
- package/dist/esm/core/stages/cacheReader.d.ts +18 -0
- package/dist/esm/core/stages/cacheReader.d.ts.map +1 -0
- package/dist/esm/core/stages/cacheReader.js +65 -0
- package/dist/esm/core/stages/cacheReader.js.map +1 -0
- package/dist/esm/core/stages/cacheWriter.d.ts +15 -0
- package/dist/esm/core/stages/cacheWriter.d.ts.map +1 -0
- package/dist/esm/core/stages/cacheWriter.js +43 -0
- package/dist/esm/core/stages/cacheWriter.js.map +1 -0
- package/dist/esm/core/stages/indexedDatabase.d.ts +34 -0
- package/dist/esm/core/stages/indexedDatabase.d.ts.map +1 -0
- package/dist/esm/core/stages/indexedDatabase.js +146 -0
- package/dist/esm/core/stages/indexedDatabase.js.map +1 -0
- package/dist/esm/core/stages/memory/memoryDatabase.d.ts +11 -0
- package/dist/esm/core/stages/memory/memoryDatabase.d.ts.map +1 -0
- package/dist/esm/{operations/databases → core/stages/memory}/memoryDatabase.js +6 -3
- package/dist/esm/core/stages/memory/memoryDatabase.js.map +1 -0
- package/dist/esm/{operations/downloaders → core/stages/memory}/memoryDownloader.d.ts +4 -4
- package/dist/esm/core/stages/memory/memoryDownloader.d.ts.map +1 -0
- package/dist/esm/{operations/downloaders → core/stages/memory}/memoryDownloader.js +1 -1
- package/dist/esm/core/stages/memory/memoryDownloader.js.map +1 -0
- package/dist/{commonjs/operations/downloaders → esm/core/stages}/serverDownloader.d.ts +8 -6
- package/dist/esm/core/stages/serverDownloader.d.ts.map +1 -0
- package/dist/esm/{operations/downloaders → core/stages}/serverDownloader.js +98 -49
- package/dist/esm/core/stages/serverDownloader.js.map +1 -0
- package/dist/esm/core/traverser.d.ts.map +1 -0
- package/dist/esm/{operations → core}/traverser.js +1 -1
- package/dist/esm/core/traverser.js.map +1 -0
- package/dist/esm/deferment/MemoryCache.d.ts +35 -0
- package/dist/esm/deferment/MemoryCache.d.ts.map +1 -0
- package/dist/esm/deferment/MemoryCache.js +157 -0
- package/dist/esm/deferment/MemoryCache.js.map +1 -0
- package/dist/esm/deferment/MemoryCache.test.d.ts +2 -0
- package/dist/esm/deferment/MemoryCache.test.d.ts.map +1 -0
- package/dist/esm/deferment/MemoryCache.test.js +151 -0
- package/dist/esm/deferment/MemoryCache.test.js.map +1 -0
- package/dist/esm/deferment/defermentManager.d.ts +32 -0
- package/dist/esm/deferment/defermentManager.d.ts.map +1 -0
- package/dist/esm/deferment/defermentManager.js +80 -0
- package/dist/esm/deferment/defermentManager.js.map +1 -0
- package/dist/esm/deferment/defermentManager.test.d.ts +2 -0
- package/dist/esm/deferment/defermentManager.test.d.ts.map +1 -0
- package/dist/esm/deferment/defermentManager.test.js +191 -0
- package/dist/esm/deferment/defermentManager.test.js.map +1 -0
- package/dist/esm/deferment/deferredBase.d.ts +12 -0
- package/dist/esm/deferment/deferredBase.d.ts.map +1 -0
- package/dist/esm/deferment/deferredBase.js +23 -0
- package/dist/esm/deferment/deferredBase.js.map +1 -0
- package/dist/esm/index.d.ts +4 -2
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +3 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/{helpers → queues}/aggregateQueue.d.ts +2 -1
- package/dist/esm/queues/aggregateQueue.d.ts.map +1 -0
- package/dist/esm/{helpers → queues}/aggregateQueue.js +4 -0
- package/dist/esm/queues/aggregateQueue.js.map +1 -0
- package/dist/{commonjs/helpers → esm/queues}/asyncGeneratorQueue.d.ts +1 -1
- package/dist/esm/queues/asyncGeneratorQueue.d.ts.map +1 -0
- package/dist/esm/{helpers → queues}/asyncGeneratorQueue.js +2 -1
- package/dist/esm/queues/asyncGeneratorQueue.js.map +1 -0
- package/dist/esm/{helpers → queues}/batchingQueue.d.ts +4 -1
- package/dist/esm/queues/batchingQueue.d.ts.map +1 -0
- package/dist/esm/queues/batchingQueue.dispose.test.d.ts +2 -0
- package/dist/esm/queues/batchingQueue.dispose.test.d.ts.map +1 -0
- package/dist/esm/queues/batchingQueue.dispose.test.js +57 -0
- package/dist/esm/queues/batchingQueue.dispose.test.js.map +1 -0
- package/dist/esm/queues/batchingQueue.js +122 -0
- package/dist/esm/queues/batchingQueue.js.map +1 -0
- package/dist/esm/queues/batchingQueue.test.d.ts +2 -0
- package/dist/esm/queues/batchingQueue.test.d.ts.map +1 -0
- package/dist/esm/queues/batchingQueue.test.js +231 -0
- package/dist/esm/queues/batchingQueue.test.js.map +1 -0
- package/dist/{commonjs/helpers → esm/queues}/bufferQueue.d.ts +1 -0
- package/dist/esm/queues/bufferQueue.d.ts.map +1 -0
- package/dist/esm/{helpers → queues}/bufferQueue.js +3 -0
- package/dist/esm/queues/bufferQueue.js.map +1 -0
- package/dist/esm/{helpers → queues}/keyedQueue.d.ts +1 -0
- package/dist/esm/queues/keyedQueue.d.ts.map +1 -0
- package/dist/esm/{helpers → queues}/keyedQueue.js +11 -0
- package/dist/esm/queues/keyedQueue.js.map +1 -0
- package/dist/esm/queues/keyedQueue.test.d.ts +2 -0
- package/dist/esm/queues/keyedQueue.test.d.ts.map +1 -0
- package/dist/esm/queues/keyedQueue.test.js +113 -0
- package/dist/esm/queues/keyedQueue.test.js.map +1 -0
- package/dist/esm/queues/queue.d.ts +5 -0
- package/dist/esm/queues/queue.d.ts.map +1 -0
- package/dist/esm/{helpers → queues}/queue.js.map +1 -1
- package/dist/esm/types/functions.d.ts +21 -0
- package/dist/esm/types/functions.d.ts.map +1 -0
- package/dist/esm/types/functions.js +83 -0
- package/dist/esm/types/functions.js.map +1 -0
- package/dist/esm/types/types.d.ts +6 -7
- package/dist/esm/types/types.d.ts.map +1 -1
- package/dist/esm/types/types.js +1 -32
- package/dist/esm/types/types.js.map +1 -1
- package/package.json +5 -3
- package/src/{operations → core}/interfaces.ts +5 -5
- package/src/{operations → core}/objectLoader2.spec.ts +31 -11
- package/src/core/objectLoader2.ts +152 -0
- package/src/core/objectLoader2Factory.test.ts +135 -0
- package/src/core/objectLoader2Factory.ts +113 -0
- package/src/{operations → core}/options.ts +4 -1
- package/src/{operations/databases → core/stages}/__snapshots__/indexedDatabase.spec.ts.snap +4 -4
- package/src/{operations/downloaders → core/stages}/__snapshots__/serverDownloader.spec.ts.snap +25 -0
- package/src/{helpers → core/stages}/cacheReader.spec.ts +12 -7
- package/src/core/stages/cacheReader.ts +88 -0
- package/src/core/stages/cacheWriter.spec.ts +213 -0
- package/src/core/stages/cacheWriter.ts +64 -0
- package/src/{operations/databases → core/stages}/indexedDatabase.spec.ts +12 -8
- package/src/core/stages/indexedDatabase.ts +178 -0
- package/src/{operations/databases → core/stages/memory}/memoryDatabase.spec.ts +7 -7
- package/src/{operations/databases → core/stages/memory}/memoryDatabase.ts +9 -6
- package/src/{operations/downloaders → core/stages/memory}/memoryDownloader.spec.ts +5 -5
- package/src/{operations/downloaders → core/stages/memory}/memoryDownloader.ts +4 -4
- package/src/core/stages/serverDownloader.spec.ts +382 -0
- package/src/{operations/downloaders → core/stages}/serverDownloader.ts +115 -52
- package/src/{operations → core}/traverser.ts +2 -1
- package/src/deferment/MemoryCache.test.ts +187 -0
- package/src/deferment/MemoryCache.ts +189 -0
- package/src/deferment/defermentManager.test.ts +225 -0
- package/src/deferment/defermentManager.ts +95 -0
- package/src/deferment/deferredBase.ts +29 -0
- package/src/index.ts +4 -2
- package/src/{helpers → queues}/aggregateQueue.ts +5 -1
- package/src/{helpers → queues}/asyncGeneratorQueue.ts +2 -1
- package/src/queues/batchingQueue.dispose.test.ts +72 -0
- package/src/queues/batchingQueue.test.ts +270 -0
- package/src/queues/batchingQueue.ts +140 -0
- package/src/{helpers → queues}/bufferQueue.ts +3 -0
- package/src/queues/keyedQueue.test.ts +146 -0
- package/src/{helpers → queues}/keyedQueue.ts +12 -0
- package/src/{helpers → queues}/queue.ts +1 -0
- package/src/test/e2e.spec.ts +2 -2
- package/src/types/functions.spec.ts +155 -0
- package/src/types/functions.ts +116 -0
- package/src/types/types.ts +5 -50
- package/dist/commonjs/helpers/aggregateQueue.d.ts.map +0 -1
- package/dist/commonjs/helpers/aggregateQueue.js.map +0 -1
- package/dist/commonjs/helpers/asyncGeneratorQueue.d.ts.map +0 -1
- package/dist/commonjs/helpers/asyncGeneratorQueue.js.map +0 -1
- package/dist/commonjs/helpers/batchedPool.d.ts +0 -12
- package/dist/commonjs/helpers/batchedPool.d.ts.map +0 -1
- package/dist/commonjs/helpers/batchedPool.js +0 -45
- package/dist/commonjs/helpers/batchedPool.js.map +0 -1
- package/dist/commonjs/helpers/batchingQueue.d.ts.map +0 -1
- package/dist/commonjs/helpers/batchingQueue.js +0 -77
- package/dist/commonjs/helpers/batchingQueue.js.map +0 -1
- package/dist/commonjs/helpers/bufferQueue.d.ts.map +0 -1
- package/dist/commonjs/helpers/bufferQueue.js.map +0 -1
- package/dist/commonjs/helpers/cachePump.d.ts +0 -22
- package/dist/commonjs/helpers/cachePump.d.ts.map +0 -1
- package/dist/commonjs/helpers/cachePump.js +0 -86
- package/dist/commonjs/helpers/cachePump.js.map +0 -1
- package/dist/commonjs/helpers/cacheReader.d.ts +0 -14
- package/dist/commonjs/helpers/cacheReader.d.ts.map +0 -1
- package/dist/commonjs/helpers/cacheReader.js +0 -58
- package/dist/commonjs/helpers/cacheReader.js.map +0 -1
- package/dist/commonjs/helpers/defermentManager.d.ts +0 -28
- package/dist/commonjs/helpers/defermentManager.d.ts.map +0 -1
- package/dist/commonjs/helpers/defermentManager.js +0 -150
- package/dist/commonjs/helpers/defermentManager.js.map +0 -1
- package/dist/commonjs/helpers/deferredBase.d.ts +0 -19
- package/dist/commonjs/helpers/deferredBase.d.ts.map +0 -1
- package/dist/commonjs/helpers/deferredBase.js +0 -51
- package/dist/commonjs/helpers/deferredBase.js.map +0 -1
- package/dist/commonjs/helpers/keyedQueue.d.ts.map +0 -1
- package/dist/commonjs/helpers/keyedQueue.js.map +0 -1
- package/dist/commonjs/helpers/memoryPump.d.ts +0 -15
- package/dist/commonjs/helpers/memoryPump.d.ts.map +0 -1
- package/dist/commonjs/helpers/memoryPump.js +0 -34
- package/dist/commonjs/helpers/memoryPump.js.map +0 -1
- package/dist/commonjs/helpers/pump.d.ts +0 -8
- package/dist/commonjs/helpers/pump.d.ts.map +0 -1
- package/dist/commonjs/helpers/pump.js +0 -3
- package/dist/commonjs/helpers/pump.js.map +0 -1
- package/dist/commonjs/helpers/queue.d.ts +0 -4
- package/dist/commonjs/helpers/queue.d.ts.map +0 -1
- package/dist/commonjs/operations/databases/indexedDatabase.d.ts +0 -27
- package/dist/commonjs/operations/databases/indexedDatabase.d.ts.map +0 -1
- package/dist/commonjs/operations/databases/indexedDatabase.js +0 -98
- package/dist/commonjs/operations/databases/indexedDatabase.js.map +0 -1
- package/dist/commonjs/operations/databases/memoryDatabase.d.ts +0 -13
- package/dist/commonjs/operations/databases/memoryDatabase.d.ts.map +0 -1
- package/dist/commonjs/operations/databases/memoryDatabase.js.map +0 -1
- package/dist/commonjs/operations/downloaders/memoryDownloader.d.ts.map +0 -1
- package/dist/commonjs/operations/downloaders/memoryDownloader.js.map +0 -1
- package/dist/commonjs/operations/downloaders/serverDownloader.d.ts.map +0 -1
- package/dist/commonjs/operations/downloaders/serverDownloader.js.map +0 -1
- package/dist/commonjs/operations/interfaces.d.ts.map +0 -1
- package/dist/commonjs/operations/interfaces.js.map +0 -1
- package/dist/commonjs/operations/objectLoader2.d.ts.map +0 -1
- package/dist/commonjs/operations/objectLoader2.js +0 -101
- package/dist/commonjs/operations/objectLoader2.js.map +0 -1
- package/dist/commonjs/operations/objectLoader2Factory.d.ts.map +0 -1
- package/dist/commonjs/operations/objectLoader2Factory.js +0 -68
- package/dist/commonjs/operations/objectLoader2Factory.js.map +0 -1
- package/dist/commonjs/operations/options.d.ts.map +0 -1
- package/dist/commonjs/operations/traverser.d.ts.map +0 -1
- package/dist/commonjs/operations/traverser.js.map +0 -1
- package/dist/esm/helpers/aggregateQueue.d.ts.map +0 -1
- package/dist/esm/helpers/aggregateQueue.js.map +0 -1
- package/dist/esm/helpers/asyncGeneratorQueue.d.ts.map +0 -1
- package/dist/esm/helpers/asyncGeneratorQueue.js.map +0 -1
- package/dist/esm/helpers/batchedPool.d.ts +0 -12
- package/dist/esm/helpers/batchedPool.d.ts.map +0 -1
- package/dist/esm/helpers/batchedPool.js +0 -42
- package/dist/esm/helpers/batchedPool.js.map +0 -1
- package/dist/esm/helpers/batchingQueue.d.ts.map +0 -1
- package/dist/esm/helpers/batchingQueue.js +0 -71
- package/dist/esm/helpers/batchingQueue.js.map +0 -1
- package/dist/esm/helpers/bufferQueue.d.ts.map +0 -1
- package/dist/esm/helpers/bufferQueue.js.map +0 -1
- package/dist/esm/helpers/cachePump.d.ts +0 -22
- package/dist/esm/helpers/cachePump.d.ts.map +0 -1
- package/dist/esm/helpers/cachePump.js +0 -79
- package/dist/esm/helpers/cachePump.js.map +0 -1
- package/dist/esm/helpers/cacheReader.d.ts +0 -14
- package/dist/esm/helpers/cacheReader.d.ts.map +0 -1
- package/dist/esm/helpers/cacheReader.js +0 -51
- package/dist/esm/helpers/cacheReader.js.map +0 -1
- package/dist/esm/helpers/defermentManager.d.ts +0 -28
- package/dist/esm/helpers/defermentManager.d.ts.map +0 -1
- package/dist/esm/helpers/defermentManager.js +0 -146
- package/dist/esm/helpers/defermentManager.js.map +0 -1
- package/dist/esm/helpers/deferredBase.d.ts +0 -19
- package/dist/esm/helpers/deferredBase.d.ts.map +0 -1
- package/dist/esm/helpers/deferredBase.js +0 -47
- package/dist/esm/helpers/deferredBase.js.map +0 -1
- package/dist/esm/helpers/keyedQueue.d.ts.map +0 -1
- package/dist/esm/helpers/keyedQueue.js.map +0 -1
- package/dist/esm/helpers/memoryPump.d.ts +0 -15
- package/dist/esm/helpers/memoryPump.d.ts.map +0 -1
- package/dist/esm/helpers/memoryPump.js +0 -30
- package/dist/esm/helpers/memoryPump.js.map +0 -1
- package/dist/esm/helpers/pump.d.ts +0 -8
- package/dist/esm/helpers/pump.d.ts.map +0 -1
- package/dist/esm/helpers/pump.js +0 -2
- package/dist/esm/helpers/pump.js.map +0 -1
- package/dist/esm/helpers/queue.d.ts +0 -4
- package/dist/esm/helpers/queue.d.ts.map +0 -1
- package/dist/esm/operations/databases/indexedDatabase.d.ts +0 -27
- package/dist/esm/operations/databases/indexedDatabase.d.ts.map +0 -1
- package/dist/esm/operations/databases/indexedDatabase.js +0 -93
- package/dist/esm/operations/databases/indexedDatabase.js.map +0 -1
- package/dist/esm/operations/databases/memoryDatabase.d.ts +0 -13
- package/dist/esm/operations/databases/memoryDatabase.d.ts.map +0 -1
- package/dist/esm/operations/databases/memoryDatabase.js.map +0 -1
- package/dist/esm/operations/downloaders/memoryDownloader.d.ts.map +0 -1
- package/dist/esm/operations/downloaders/memoryDownloader.js.map +0 -1
- package/dist/esm/operations/downloaders/serverDownloader.d.ts.map +0 -1
- package/dist/esm/operations/downloaders/serverDownloader.js.map +0 -1
- package/dist/esm/operations/interfaces.d.ts.map +0 -1
- package/dist/esm/operations/interfaces.js.map +0 -1
- package/dist/esm/operations/objectLoader2.d.ts.map +0 -1
- package/dist/esm/operations/objectLoader2.js +0 -94
- package/dist/esm/operations/objectLoader2.js.map +0 -1
- package/dist/esm/operations/objectLoader2Factory.d.ts.map +0 -1
- package/dist/esm/operations/objectLoader2Factory.js +0 -61
- package/dist/esm/operations/objectLoader2Factory.js.map +0 -1
- package/dist/esm/operations/options.d.ts.map +0 -1
- package/dist/esm/operations/traverser.d.ts.map +0 -1
- package/dist/esm/operations/traverser.js.map +0 -1
- package/src/helpers/__snapshots__/cachePump.spec.ts.snap +0 -31
- package/src/helpers/__snapshots__/defermentManager.spec.ts.snap +0 -8
- package/src/helpers/batchedPool.ts +0 -56
- package/src/helpers/batchingQueue.ts +0 -85
- package/src/helpers/cachePump.disposal.spec.ts +0 -51
- package/src/helpers/cachePump.spec.ts +0 -104
- package/src/helpers/cachePump.ts +0 -107
- package/src/helpers/cacheReader.ts +0 -64
- package/src/helpers/defermentManager.defermentTotals.spec.ts +0 -53
- package/src/helpers/defermentManager.disposal.spec.ts +0 -28
- package/src/helpers/defermentManager.spec.ts +0 -37
- package/src/helpers/defermentManager.ts +0 -160
- package/src/helpers/deferredBase.ts +0 -55
- package/src/helpers/memoryPump.ts +0 -40
- package/src/helpers/pump.ts +0 -8
- package/src/operations/databases/indexedDatabase.ts +0 -126
- package/src/operations/downloaders/serverDownloader.spec.ts +0 -161
- package/src/operations/objectLoader2.ts +0 -119
- package/src/operations/objectLoader2Factory.ts +0 -78
- package/dist/commonjs/{operations → core}/interfaces.js +0 -0
- package/dist/commonjs/{operations → core}/options.js +0 -0
- package/dist/commonjs/{operations → core}/traverser.d.ts +0 -0
- package/dist/commonjs/{helpers → queues}/queue.js +0 -0
- package/dist/esm/{operations → core}/interfaces.js +0 -0
- package/dist/esm/{operations → core}/options.js +0 -0
- package/dist/esm/{operations → core}/traverser.d.ts +0 -0
- package/dist/esm/{helpers → queues}/queue.js +0 -0
- package/src/{operations → core}/__snapshots__/objectLoader2.spec.ts.snap +0 -0
- package/src/{operations → core}/__snapshots__/traverser.spec.ts.snap +0 -0
- package/src/{helpers → core/stages}/__snapshots__/cacheReader.spec.ts.snap +0 -0
- package/src/{operations/downloaders → core/stages/memory}/__snapshots__/memoryDownloader.spec.ts.snap +0 -0
- package/src/{operations → core}/traverser.spec.ts +1 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cacheWriter.js","sourceRoot":"","sources":["../../../../src/core/stages/cacheWriter.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,+BAA+B,CAAA;AAOzD,MAAM,OAAO,WAAW;IACtB,WAAW,CAAiC;IAC5C,SAAS,CAAU;IACnB,UAAU,CAAW;IACrB,YAAY,CAAsB;IAClC,OAAO,CAAc;IACrB,QAAQ,CAAc;IACtB,SAAS,GAAG,KAAK,CAAA;IAEjB,YACE,QAAkB,EAClB,MAAoB,EACpB,SAAoB,EACpB,OAAqB,EACrB,WAAiC;QAEjC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;QAC3B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAA;IACjC,CAAC;IAED,GAAG,CAAC,IAAU;QACZ,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,GAAG,IAAI,aAAa,CAAC;gBACnC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,iBAAiB;gBAC1C,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,sBAAsB;gBACjD,eAAe,EAAE,KAAK,EAAE,KAAa,EAAiB,EAAE;oBACtD,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;gBAC5B,CAAC;aACF,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QACvC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;IAClD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa;QAC1B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QAC/B,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAClC,IAAI,CAAC,OAAO,CACV,qBAAqB,KAAK,CAAC,MAAM,UAC/B,WAAW,CAAC,GAAG,EAAE,GAAG,KACtB,YAAY,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CACxC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,MAAM,IAAI,CAAC,WAAW,EAAE,YAAY,EAAE,CAAA;IACxC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;CACF"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Item } from '../../types/types.js';
|
|
2
|
+
import { Database } from '../interfaces.js';
|
|
3
|
+
/**
|
|
4
|
+
* A wrapper class for IndexedDB to simplify common database operations.
|
|
5
|
+
*/
|
|
6
|
+
export interface IndexedDatabaseOptions {
|
|
7
|
+
indexedDB?: IDBFactory;
|
|
8
|
+
keyRange?: {
|
|
9
|
+
bound: Function;
|
|
10
|
+
lowerBound: Function;
|
|
11
|
+
upperBound: Function;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export declare class IndexedDatabase implements Database {
|
|
15
|
+
#private;
|
|
16
|
+
constructor(options: IndexedDatabaseOptions);
|
|
17
|
+
/**
|
|
18
|
+
* Initializes the database connection and creates the object store if needed.
|
|
19
|
+
* This must be called before any other database operations.
|
|
20
|
+
*/
|
|
21
|
+
init(): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Inserts or updates an array of items in a single transaction.
|
|
24
|
+
* @param data The array of items to insert.
|
|
25
|
+
*/
|
|
26
|
+
putAll(data: Item[]): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Retrieves an array of items from the object store based on their IDs.
|
|
29
|
+
* @param ids The array of IDs to retrieve.
|
|
30
|
+
*/
|
|
31
|
+
getAll(ids: string[]): Promise<(Item | undefined)[]>;
|
|
32
|
+
dispose(): void;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=indexedDatabase.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexedDatabase.d.ts","sourceRoot":"","sources":["../../../../src/core/stages/indexedDatabase.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAE3C;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,UAAU,CAAA;IACtB,QAAQ,CAAC,EAAE;QACT,KAAK,EAAE,QAAQ,CAAA;QACf,UAAU,EAAE,QAAQ,CAAA;QACpB,UAAU,EAAE,QAAQ,CAAA;KACrB,CAAA;CACF;AACD,qBAAa,eAAgB,YAAW,QAAQ;;gBAOlC,OAAO,EAAE,sBAAsB;IAI3C;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwE3B;;;OAGG;IACG,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBzC;;;OAGG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC;IAmC1D,OAAO,IAAI,IAAI;CAKhB"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-base-to-string */
|
|
2
|
+
/* eslint-disable @typescript-eslint/prefer-promise-reject-errors */
|
|
3
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
4
|
+
/* eslint-disable @typescript-eslint/no-unsafe-function-type */
|
|
5
|
+
import { isSafari } from '@synchronized-console/shared';
|
|
6
|
+
export class IndexedDatabase {
|
|
7
|
+
#options;
|
|
8
|
+
#db = undefined;
|
|
9
|
+
#dbName = 'synchronized-cache';
|
|
10
|
+
#storeName = 'cache';
|
|
11
|
+
constructor(options) {
|
|
12
|
+
this.#options = options;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Initializes the database connection and creates the object store if needed.
|
|
16
|
+
* This must be called before any other database operations.
|
|
17
|
+
*/
|
|
18
|
+
async init() {
|
|
19
|
+
if (this.#db)
|
|
20
|
+
return;
|
|
21
|
+
await this.#safariFix();
|
|
22
|
+
return this.#openDatabase();
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Opens the database, and if there's an error, deletes the database and tries again.
|
|
26
|
+
*/
|
|
27
|
+
async #openDatabase() {
|
|
28
|
+
const idb = this.#options.indexedDB ?? indexedDB;
|
|
29
|
+
return new Promise((resolve, reject) => {
|
|
30
|
+
const request = idb.open(this.#dbName, 1);
|
|
31
|
+
request.onerror = () => {
|
|
32
|
+
console.warn(`Failed to open database: ${this.#dbName}, deleting and trying again`);
|
|
33
|
+
// Delete the database and try again
|
|
34
|
+
const deleteRequest = idb.deleteDatabase(this.#dbName);
|
|
35
|
+
deleteRequest.onsuccess = () => {
|
|
36
|
+
// Try opening again after deletion
|
|
37
|
+
void this.#openDatabase().then(resolve).catch(reject);
|
|
38
|
+
};
|
|
39
|
+
deleteRequest.onerror = () => {
|
|
40
|
+
reject(`Failed to delete and reopen database: ${this.#dbName}`);
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
request.onupgradeneeded = (event) => {
|
|
44
|
+
const db = event.target.result;
|
|
45
|
+
if (db.objectStoreNames.contains(this.#storeName)) {
|
|
46
|
+
db.deleteObjectStore(this.#storeName);
|
|
47
|
+
}
|
|
48
|
+
db.createObjectStore(this.#storeName, { keyPath: 'baseId' });
|
|
49
|
+
};
|
|
50
|
+
request.onsuccess = (event) => {
|
|
51
|
+
this.#db = event.target.result;
|
|
52
|
+
resolve();
|
|
53
|
+
};
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
#getDB() {
|
|
57
|
+
if (!this.#db) {
|
|
58
|
+
throw new Error('Database not initialized. Call init() first.');
|
|
59
|
+
}
|
|
60
|
+
return this.#db;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Fixes a Safari bug where IndexedDB requests get lost and never resolve - invoke before you use IndexedDB
|
|
64
|
+
* @link Credits and more info: https://github.com/jakearchibald/safari-14-idb-fix
|
|
65
|
+
*/
|
|
66
|
+
async #safariFix() {
|
|
67
|
+
// No point putting other browsers or older versions of Safari through this mess.
|
|
68
|
+
if (!isSafari() || !this.#options.indexedDB?.databases)
|
|
69
|
+
return Promise.resolve();
|
|
70
|
+
let intervalId;
|
|
71
|
+
return new Promise((resolve) => {
|
|
72
|
+
const tryIdb = () => this.#options.indexedDB?.databases().finally(resolve);
|
|
73
|
+
intervalId = setInterval(() => {
|
|
74
|
+
void tryIdb();
|
|
75
|
+
}, 100);
|
|
76
|
+
void tryIdb();
|
|
77
|
+
}).finally(() => clearInterval(intervalId));
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Inserts or updates an array of items in a single transaction.
|
|
81
|
+
* @param data The array of items to insert.
|
|
82
|
+
*/
|
|
83
|
+
async putAll(data) {
|
|
84
|
+
await this.init(); // Ensure the database is initialized
|
|
85
|
+
return new Promise((resolve, reject) => {
|
|
86
|
+
try {
|
|
87
|
+
const transaction = this.#getDB().transaction(this.#storeName, 'readwrite', {
|
|
88
|
+
durability: 'relaxed'
|
|
89
|
+
});
|
|
90
|
+
const store = transaction.objectStore(this.#storeName);
|
|
91
|
+
transaction.onerror = () => {
|
|
92
|
+
reject(`Transaction error: ${transaction.error}`);
|
|
93
|
+
};
|
|
94
|
+
transaction.oncomplete = () => {
|
|
95
|
+
resolve();
|
|
96
|
+
};
|
|
97
|
+
data.forEach((item) => store.put(item));
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
reject(error);
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Retrieves an array of items from the object store based on their IDs.
|
|
106
|
+
* @param ids The array of IDs to retrieve.
|
|
107
|
+
*/
|
|
108
|
+
async getAll(ids) {
|
|
109
|
+
await this.init(); // Ensure the database is initialized
|
|
110
|
+
return new Promise((resolve, reject) => {
|
|
111
|
+
if (ids.length === 0) {
|
|
112
|
+
return resolve([]);
|
|
113
|
+
}
|
|
114
|
+
try {
|
|
115
|
+
const transaction = this.#getDB().transaction(this.#storeName, 'readonly', {
|
|
116
|
+
durability: 'relaxed'
|
|
117
|
+
});
|
|
118
|
+
const store = transaction.objectStore(this.#storeName);
|
|
119
|
+
const promises = [];
|
|
120
|
+
for (const id of ids) {
|
|
121
|
+
promises.push(new Promise((resolveGet, rejectGet) => {
|
|
122
|
+
const request = store.get(id);
|
|
123
|
+
request.onerror = () => rejectGet(`Request error for id ${id}: ${request.error}`);
|
|
124
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
|
125
|
+
request.onsuccess = () => resolveGet(request.result);
|
|
126
|
+
}));
|
|
127
|
+
}
|
|
128
|
+
Promise.all(promises)
|
|
129
|
+
.then((results) => {
|
|
130
|
+
resolve(results);
|
|
131
|
+
})
|
|
132
|
+
.catch(reject);
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
reject(error);
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
dispose() {
|
|
140
|
+
if (!this.#db)
|
|
141
|
+
return;
|
|
142
|
+
this.#db.close();
|
|
143
|
+
this.#db = undefined;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
//# sourceMappingURL=indexedDatabase.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexedDatabase.js","sourceRoot":"","sources":["../../../../src/core/stages/indexedDatabase.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,oEAAoE;AACpE,uDAAuD;AACvD,+DAA+D;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAe1C,MAAM,OAAO,eAAe;IAC1B,QAAQ,CAAwB;IAEhC,GAAG,GAA4B,SAAS,CAAA;IAC/B,OAAO,GAAW,eAAe,CAAA;IACjC,UAAU,GAAW,OAAO,CAAA;IAErC,YAAY,OAA+B;QACzC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;IACzB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,GAAG;YAAE,OAAM;QACpB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACvB,OAAO,IAAI,CAAC,aAAa,EAAE,CAAA;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,SAAS,CAAA;QAEhD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YAEzC,OAAO,CAAC,OAAO,GAAG,GAAQ,EAAE;gBAC1B,OAAO,CAAC,IAAI,CACV,4BAA4B,IAAI,CAAC,OAAO,6BAA6B,CACtE,CAAA;gBACD,oCAAoC;gBACpC,MAAM,aAAa,GAAG,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBACtD,aAAa,CAAC,SAAS,GAAG,GAAQ,EAAE;oBAClC,mCAAmC;oBACnC,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;gBACvD,CAAC,CAAA;gBACD,aAAa,CAAC,OAAO,GAAG,GAAQ,EAAE;oBAChC,MAAM,CAAC,yCAAyC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;gBACjE,CAAC,CAAA;YACH,CAAC,CAAA;YAED,OAAO,CAAC,eAAe,GAAG,CAAC,KAAK,EAAO,EAAE;gBACvC,MAAM,EAAE,GAAI,KAAK,CAAC,MAA2B,CAAC,MAAM,CAAA;gBACpD,IAAI,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAClD,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBACvC,CAAC;gBACD,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAA;YAC9D,CAAC,CAAA;YAED,OAAO,CAAC,SAAS,GAAG,CAAC,KAAK,EAAO,EAAE;gBACjC,IAAI,CAAC,GAAG,GAAI,KAAK,CAAC,MAA2B,CAAC,MAAM,CAAA;gBACpD,OAAO,EAAE,CAAA;YACX,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;QACjE,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAA;IACjB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,iFAAiF;QACjF,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QAEhF,IAAI,UAA0C,CAAA;QAE9C,OAAO,IAAI,OAAO,CAAO,CAAC,OAAmB,EAAE,EAAE;YAC/C,MAAM,MAAM,GAAG,GAA2C,EAAE,CAC1D,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACvD,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC5B,KAAK,MAAM,EAAE,CAAA;YACf,CAAC,EAAE,GAAG,CAAC,CAAA;YACP,KAAK,MAAM,EAAE,CAAA;QACf,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAA;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,IAAY;QACvB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA,CAAC,qCAAqC;QACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE;oBAC1E,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAA;gBACF,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBAEtD,WAAW,CAAC,OAAO,GAAG,GAAQ,EAAE;oBAC9B,MAAM,CAAC,sBAAsB,WAAW,CAAC,KAAK,EAAE,CAAC,CAAA;gBACnD,CAAC,CAAA;gBACD,WAAW,CAAC,UAAU,GAAG,GAAQ,EAAE;oBACjC,OAAO,EAAE,CAAA;gBACX,CAAC,CAAA;gBAED,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;YACzC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAA;YACf,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,GAAa;QACxB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA,CAAC,qCAAqC;QACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,OAAO,CAAC,EAAE,CAAC,CAAA;YACpB,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE;oBACzE,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAA;gBACF,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBACtD,MAAM,QAAQ,GAAgC,EAAE,CAAA;gBAEhD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;oBACrB,QAAQ,CAAC,IAAI,CACX,IAAI,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;wBACpC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;wBAC7B,OAAO,CAAC,OAAO,GAAG,GAAS,EAAE,CAC3B,SAAS,CAAC,wBAAwB,EAAE,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;wBAC3D,iEAAiE;wBACjE,OAAO,CAAC,SAAS,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;oBAC5D,CAAC,CAAC,CACH,CAAA;gBACH,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;qBAClB,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;oBAChB,OAAO,CAAC,OAAO,CAAC,CAAA;gBAClB,CAAC,CAAC;qBACD,KAAK,CAAC,MAAM,CAAC,CAAA;YAClB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAA;YACf,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IACD,OAAO;QACL,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAM;QACrB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAA;QAChB,IAAI,CAAC,GAAG,GAAG,SAAS,CAAA;IACtB,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Item } from '../../../types/types.js';
|
|
2
|
+
import { Database } from '../../interfaces.js';
|
|
3
|
+
import { MemoryDatabaseOptions } from '../../options.js';
|
|
4
|
+
export declare class MemoryDatabase implements Database {
|
|
5
|
+
private items;
|
|
6
|
+
constructor(options?: MemoryDatabaseOptions);
|
|
7
|
+
getAll(keys: string[]): Promise<(Item | undefined)[]>;
|
|
8
|
+
putAll(batch: Item[]): Promise<void>;
|
|
9
|
+
dispose(): void;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=memoryDatabase.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memoryDatabase.d.ts","sourceRoot":"","sources":["../../../../../src/core/stages/memory/memoryDatabase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,IAAI,EAAE,MAAM,yBAAyB,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AAExD,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,KAAK,CAAmB;gBAEpB,OAAO,CAAC,EAAE,qBAAqB;IAI3C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC;IAarD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAUpC,OAAO,IAAI,IAAI;CAGhB"}
|
|
@@ -16,14 +16,17 @@ export class MemoryDatabase {
|
|
|
16
16
|
}
|
|
17
17
|
return Promise.resolve(found);
|
|
18
18
|
}
|
|
19
|
-
|
|
19
|
+
putAll(batch) {
|
|
20
20
|
for (const item of batch) {
|
|
21
|
+
if (!item.baseId || !item.base) {
|
|
22
|
+
throw new Error('Item must have a baseId and base');
|
|
23
|
+
}
|
|
21
24
|
this.items.set(item.baseId, item.base);
|
|
22
25
|
}
|
|
23
26
|
return Promise.resolve();
|
|
24
27
|
}
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
dispose() {
|
|
29
|
+
this.items.clear();
|
|
27
30
|
}
|
|
28
31
|
}
|
|
29
32
|
//# sourceMappingURL=memoryDatabase.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memoryDatabase.js","sourceRoot":"","sources":["../../../../../src/core/stages/memory/memoryDatabase.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,cAAc;IACjB,KAAK,CAAmB;IAEhC,YAAY,OAA+B;QACzC,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,GAAG,EAAgB,CAAA;IACxD,CAAC;IAED,MAAM,CAAC,IAAc;QACnB,MAAM,KAAK,GAAyB,EAAE,CAAA;QACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAChC,IAAI,IAAI,EAAE,CAAC;gBACT,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACvB,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,MAAM,CAAC,KAAa;QAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;YACrD,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IAED,OAAO;QACL,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;CACF"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import Queue from '
|
|
2
|
-
import { Base, Item } from '
|
|
3
|
-
import { Downloader } from '
|
|
1
|
+
import Queue from '../../../queues/queue.js';
|
|
2
|
+
import { Base, Item } from '../../../types/types.js';
|
|
3
|
+
import { Downloader } from '../../interfaces.js';
|
|
4
4
|
export declare class MemoryDownloader implements Downloader {
|
|
5
5
|
#private;
|
|
6
6
|
constructor(rootId: string, items: Map<string, Base>);
|
|
7
|
-
|
|
7
|
+
initialize(params: {
|
|
8
8
|
results: Queue<Item>;
|
|
9
9
|
total: number;
|
|
10
10
|
maxDownloadBatchWait?: number;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memoryDownloader.d.ts","sourceRoot":"","sources":["../../../../../src/core/stages/memory/memoryDownloader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,0BAA0B,CAAA;AAC5C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAA;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,qBAAa,gBAAiB,YAAW,UAAU;;gBAKrC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC;IAIpD,UAAU,CAAC,MAAM,EAAE;QACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;QACpB,KAAK,EAAE,MAAM,CAAA;QACb,oBAAoB,CAAC,EAAE,MAAM,CAAA;KAC9B,GAAG,IAAI;IAGR,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAO/B,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAG7B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;CAQtB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memoryDownloader.js","sourceRoot":"","sources":["../../../../../src/core/stages/memory/memoryDownloader.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,gBAAgB;IAC3B,MAAM,CAAmB;IACzB,OAAO,CAAQ;IACf,QAAQ,CAAc;IAEtB,YAAY,MAAc,EAAE,KAAwB;QAClD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;IACrB,CAAC;IACD,UAAU,CAAC,MAIV;QACC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAA;IAChC,CAAC;IACD,cAAc;QACZ,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC1C,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC,CAAA;IAC7D,CAAC;IACD,YAAY;QACV,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IACD,GAAG,CAAC,EAAU;QACZ,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAA;YACxC,OAAM;QACR,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;IAC5C,CAAC;CACF"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import Queue from '../../
|
|
2
|
-
import {
|
|
1
|
+
import Queue from '../../queues/queue.js';
|
|
2
|
+
import { CustomLogger, Fetcher } from '../../types/functions.js';
|
|
3
|
+
import { Item, ObjectAttributeMask } from '../../types/types.js';
|
|
3
4
|
import { Downloader } from '../interfaces.js';
|
|
4
5
|
export interface ServerDownloaderOptions {
|
|
5
6
|
serverUrl: string;
|
|
@@ -7,26 +8,27 @@ export interface ServerDownloaderOptions {
|
|
|
7
8
|
objectId: string;
|
|
8
9
|
token?: string;
|
|
9
10
|
headers?: Headers;
|
|
11
|
+
logger: CustomLogger;
|
|
10
12
|
fetch?: Fetcher;
|
|
13
|
+
attributeMask?: ObjectAttributeMask;
|
|
14
|
+
objectTypeMask?: string[];
|
|
11
15
|
}
|
|
12
16
|
export default class ServerDownloader implements Downloader {
|
|
13
17
|
#private;
|
|
14
18
|
constructor(options: ServerDownloaderOptions);
|
|
15
|
-
|
|
19
|
+
initialize(params: {
|
|
16
20
|
results: Queue<Item>;
|
|
17
21
|
total: number;
|
|
18
22
|
maxDownloadBatchWait?: number;
|
|
19
23
|
}): void;
|
|
20
24
|
add(id: string): void;
|
|
25
|
+
decodeChunk(chunkBuffer: Uint8Array): string;
|
|
21
26
|
disposeAsync(): Promise<void>;
|
|
22
27
|
downloadBatch(params: {
|
|
23
28
|
batch: string[];
|
|
24
29
|
url: string;
|
|
25
30
|
headers: HeadersInit;
|
|
26
31
|
}): Promise<void>;
|
|
27
|
-
processArray(leftover: Uint8Array, value: Uint8Array, keys: Set<string>, callback: () => Promise<void>): Promise<Uint8Array>;
|
|
28
|
-
processLine(line: Uint8Array): Item;
|
|
29
|
-
concatUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array;
|
|
30
32
|
downloadSingle(): Promise<Item>;
|
|
31
33
|
}
|
|
32
34
|
//# sourceMappingURL=serverDownloader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serverDownloader.d.ts","sourceRoot":"","sources":["../../../../src/core/stages/serverDownloader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,uBAAuB,CAAA;AAEzC,OAAO,EAAE,YAAY,EAAE,OAAO,EAAyB,MAAM,0BAA0B,CAAA;AACvF,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAChE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAE7C,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,aAAa,CAAC,EAAE,mBAAmB,CAAA;IACnC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;CAC1B;AAID,MAAM,CAAC,OAAO,OAAO,gBAAiB,YAAW,UAAU;;gBAiB7C,OAAO,EAAE,uBAAuB;IAkC5C,UAAU,CAAC,MAAM,EAAE;QACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;QACpB,KAAK,EAAE,MAAM,CAAA;QACb,oBAAoB,CAAC,EAAE,MAAM,CAAA;KAC9B,GAAG,IAAI;IAgBR,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAWrB,WAAW,CAAC,WAAW,EAAE,UAAU,GAAG,MAAM;IAWtC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B,aAAa,CAAC,MAAM,EAAE;QAC1B,KAAK,EAAE,MAAM,EAAE,CAAA;QACf,GAAG,EAAE,MAAM,CAAA;QACX,OAAO,EAAE,WAAW,CAAA;KACrB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkIX,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BtC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import
|
|
1
|
+
import BatchingQueue from '../../queues/batchingQueue.js';
|
|
2
2
|
import { ObjectLoaderRuntimeError } from '../../types/errors.js';
|
|
3
|
-
import { isBase, take } from '../../types/
|
|
3
|
+
import { indexOf, isBase, take } from '../../types/functions.js';
|
|
4
|
+
const MAX_SAFARI_DECODE_BYTES = 2 * 1024 * 1024 * 1024 - 1024 * 1024; // 2GB minus a margin
|
|
4
5
|
export default class ServerDownloader {
|
|
5
6
|
#requestUrlRootObj;
|
|
6
7
|
#requestUrlChildren;
|
|
@@ -8,10 +9,16 @@ export default class ServerDownloader {
|
|
|
8
9
|
#options;
|
|
9
10
|
#fetch;
|
|
10
11
|
#results;
|
|
12
|
+
#total;
|
|
13
|
+
#logger;
|
|
11
14
|
#downloadQueue;
|
|
12
|
-
#decoder = new TextDecoder();
|
|
15
|
+
#decoder = new TextDecoder('utf-8', { fatal: true });
|
|
16
|
+
#decodedBytesCount = 0;
|
|
17
|
+
#rawStrings = [];
|
|
18
|
+
#rawEncodings = [];
|
|
13
19
|
constructor(options) {
|
|
14
20
|
this.#options = options;
|
|
21
|
+
this.#logger = options.logger;
|
|
15
22
|
this.#fetch =
|
|
16
23
|
options.fetch ?? ((...args) => globalThis.fetch(...args));
|
|
17
24
|
this.#headers = {};
|
|
@@ -24,21 +31,21 @@ export default class ServerDownloader {
|
|
|
24
31
|
if (this.#options.token) {
|
|
25
32
|
this.#headers['Authorization'] = `Bearer ${this.#options.token}`;
|
|
26
33
|
}
|
|
27
|
-
this.#requestUrlChildren = `${this.#options.serverUrl}/api/
|
|
34
|
+
this.#requestUrlChildren = `${this.#options.serverUrl}/api/v2/projects/${this.#options.streamId}/object-stream/`;
|
|
28
35
|
this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${this.#options.streamId}/${this.#options.objectId}/single`;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
36
|
+
const encoder = new TextEncoder();
|
|
37
|
+
if (options.objectTypeMask) {
|
|
38
|
+
this.#rawStrings.push(...options.objectTypeMask);
|
|
39
|
+
this.#rawEncodings.push(...options.objectTypeMask.map((val) => encoder.encode(val)));
|
|
33
40
|
}
|
|
34
|
-
return [10000, 25000, 10000, 1000];
|
|
35
41
|
}
|
|
36
|
-
|
|
37
|
-
const { results, total } = params;
|
|
42
|
+
initialize(params) {
|
|
43
|
+
const { results, total, maxDownloadBatchWait } = params;
|
|
38
44
|
this.#results = results;
|
|
39
|
-
this.#
|
|
40
|
-
|
|
41
|
-
|
|
45
|
+
this.#total = total;
|
|
46
|
+
this.#downloadQueue = new BatchingQueue({
|
|
47
|
+
batchSize: 15000, // 15k is a good number for most cases
|
|
48
|
+
maxWaitTime: maxDownloadBatchWait ?? 1000, // 1 second
|
|
42
49
|
processFunction: (batch) => this.downloadBatch({
|
|
43
50
|
batch,
|
|
44
51
|
url: this.#requestUrlChildren,
|
|
@@ -46,40 +53,39 @@ export default class ServerDownloader {
|
|
|
46
53
|
})
|
|
47
54
|
});
|
|
48
55
|
}
|
|
49
|
-
#getPool() {
|
|
50
|
-
if (this.#downloadQueue) {
|
|
51
|
-
return this.#downloadQueue;
|
|
52
|
-
}
|
|
53
|
-
throw new Error('Download pool is not initialized');
|
|
54
|
-
}
|
|
55
56
|
add(id) {
|
|
56
|
-
this.#
|
|
57
|
+
this.#downloadQueue?.add(id, id);
|
|
58
|
+
}
|
|
59
|
+
/*
|
|
60
|
+
This is the most frequently reported and confirmed reason for this error in Safari. There's a known bug in WebKit (Safari's rendering engine) where TextDecoder can fail or throw a RangeError after decoding around 2GB of data. Chrome and other browsers handle much larger amounts of data without this specific limitation.
|
|
61
|
+
|
|
62
|
+
Why it happens: It seems to be an internal memory or indexing limitation within Safari's TextDecoder implementation. After a certain threshold of data has been processed by a TextDecoder instance, it starts throwing this error.
|
|
63
|
+
|
|
64
|
+
Chrome's behavior: Chrome generally handles larger data sizes without this specific RangeError. It might become slow or run out of general memory, but not typically with this specific error.
|
|
65
|
+
*/
|
|
66
|
+
decodeChunk(chunkBuffer) {
|
|
67
|
+
if (this.#decodedBytesCount + chunkBuffer.byteLength > MAX_SAFARI_DECODE_BYTES) {
|
|
68
|
+
// Safari is approaching its limit, create a new decoder
|
|
69
|
+
this.#decoder = new TextDecoder('utf-8', { fatal: true });
|
|
70
|
+
this.#decodedBytesCount = 0; // Reset counter for the new decoder
|
|
71
|
+
}
|
|
72
|
+
const decodedText = this.#decoder.decode(chunkBuffer);
|
|
73
|
+
this.#decodedBytesCount += chunkBuffer.byteLength;
|
|
74
|
+
return decodedText;
|
|
57
75
|
}
|
|
58
76
|
async disposeAsync() {
|
|
59
77
|
await this.#downloadQueue?.disposeAsync();
|
|
60
78
|
}
|
|
61
|
-
#processJson(baseId, unparsedBase) {
|
|
62
|
-
let base;
|
|
63
|
-
try {
|
|
64
|
-
base = JSON.parse(unparsedBase);
|
|
65
|
-
}
|
|
66
|
-
catch (e) {
|
|
67
|
-
throw new Error(`Error parsing object ${baseId}: ${e.message}`);
|
|
68
|
-
}
|
|
69
|
-
if (isBase(base)) {
|
|
70
|
-
return { baseId, base };
|
|
71
|
-
}
|
|
72
|
-
else {
|
|
73
|
-
throw new ObjectLoaderRuntimeError(`${baseId} is not a base`);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
79
|
async downloadBatch(params) {
|
|
77
80
|
const { batch, url, headers } = params;
|
|
81
|
+
const start = performance.now();
|
|
82
|
+
this.#logger(`Downloading batch of ${batch.length} items...`);
|
|
83
|
+
const attributeMask = this.#options.attributeMask;
|
|
78
84
|
const keys = new Set(batch);
|
|
79
85
|
const response = await this.#fetch(url, {
|
|
80
86
|
method: 'POST',
|
|
81
87
|
headers: { ...headers, 'Content-Type': 'application/json' },
|
|
82
|
-
body: JSON.stringify({
|
|
88
|
+
body: JSON.stringify({ objectIds: batch, attributeMask })
|
|
83
89
|
});
|
|
84
90
|
this.#validateResponse(response);
|
|
85
91
|
if (!response.body) {
|
|
@@ -92,7 +98,7 @@ export default class ServerDownloader {
|
|
|
92
98
|
const { done, value } = await reader.read();
|
|
93
99
|
if (done)
|
|
94
100
|
break;
|
|
95
|
-
leftover = await this
|
|
101
|
+
leftover = await this.#processArray(leftover, value, keys, async () => {
|
|
96
102
|
count++;
|
|
97
103
|
if (count % 1000 === 0) {
|
|
98
104
|
await new Promise((resolve) => setTimeout(resolve, 100)); //allow other stuff to happen
|
|
@@ -102,40 +108,77 @@ export default class ServerDownloader {
|
|
|
102
108
|
if (keys.size > 0) {
|
|
103
109
|
throw new Error('Items requested were not downloaded: ' + take(keys.values(), 10).join(','));
|
|
104
110
|
}
|
|
111
|
+
count += keys.size; // count the leftovers
|
|
112
|
+
if (count >= this.#total) {
|
|
113
|
+
await this.#results?.disposeAsync(); // mark the queue as done
|
|
114
|
+
}
|
|
115
|
+
this.#logger(`Downloaded batch of ${batch.length} items in ${performance.now() - start}ms`);
|
|
105
116
|
}
|
|
106
|
-
async processArray(leftover, value, keys, callback) {
|
|
117
|
+
async #processArray(leftover, value, keys, callback) {
|
|
107
118
|
//this concat will allocate a new array
|
|
108
|
-
const combined = this
|
|
119
|
+
const combined = this.#concatUint8Arrays(leftover, value);
|
|
109
120
|
let start = 0;
|
|
110
121
|
//subarray doesn't allocate
|
|
111
122
|
for (let i = 0; i < combined.length; i++) {
|
|
112
123
|
if (combined[i] === 0x0a) {
|
|
113
124
|
const line = combined.subarray(start, i); // line without \n
|
|
114
125
|
//strings are allocated here
|
|
115
|
-
const item = this
|
|
116
|
-
this.#results?.add(item);
|
|
126
|
+
const item = this.#processLine(line);
|
|
117
127
|
start = i + 1;
|
|
118
128
|
await callback();
|
|
119
129
|
keys.delete(item.baseId);
|
|
130
|
+
this.#results?.add(item);
|
|
120
131
|
}
|
|
121
132
|
}
|
|
122
133
|
return combined.subarray(start); // carry over remainder
|
|
123
134
|
}
|
|
124
|
-
processLine(line) {
|
|
135
|
+
#processLine(line) {
|
|
125
136
|
for (let i = 0; i < line.length; i++) {
|
|
126
137
|
if (line[i] === 0x09) {
|
|
127
138
|
//this is a tab
|
|
128
|
-
const baseId = this
|
|
129
|
-
const
|
|
130
|
-
|
|
139
|
+
const baseId = this.decodeChunk(line.subarray(0, i));
|
|
140
|
+
const jsonBytes = line.subarray(i + 1);
|
|
141
|
+
if (!this.#isValidBytes(jsonBytes)) {
|
|
142
|
+
return { baseId, base: undefined };
|
|
143
|
+
}
|
|
144
|
+
const base = this.decodeChunk(jsonBytes);
|
|
131
145
|
const item = this.#processJson(baseId, base);
|
|
132
|
-
item.size =
|
|
146
|
+
item.size = jsonBytes.length;
|
|
133
147
|
return item;
|
|
134
148
|
}
|
|
135
149
|
}
|
|
136
|
-
throw new ObjectLoaderRuntimeError('Invalid line format: ' + this
|
|
150
|
+
throw new ObjectLoaderRuntimeError('Invalid line format in response: ' + this.decodeChunk(line));
|
|
151
|
+
}
|
|
152
|
+
#processJson(baseId, unparsedBase) {
|
|
153
|
+
let base;
|
|
154
|
+
try {
|
|
155
|
+
base = JSON.parse(unparsedBase);
|
|
156
|
+
}
|
|
157
|
+
catch (e) {
|
|
158
|
+
throw new Error(`Error parsing object ${baseId}: ${e.message}`);
|
|
159
|
+
}
|
|
160
|
+
if (isBase(base)) {
|
|
161
|
+
return { baseId, base };
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
throw new ObjectLoaderRuntimeError(`${baseId} is not a base`);
|
|
165
|
+
}
|
|
137
166
|
}
|
|
138
|
-
|
|
167
|
+
#isValidString(json) {
|
|
168
|
+
for (const rawString of this.#rawStrings)
|
|
169
|
+
if (json.includes(rawString)) {
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
return true;
|
|
173
|
+
}
|
|
174
|
+
#isValidBytes(json) {
|
|
175
|
+
for (const rawEncoding of this.#rawEncodings)
|
|
176
|
+
if (indexOf(json, rawEncoding) !== -1) {
|
|
177
|
+
return false;
|
|
178
|
+
}
|
|
179
|
+
return true;
|
|
180
|
+
}
|
|
181
|
+
#concatUint8Arrays(a, b) {
|
|
139
182
|
const c = new Uint8Array(a.length + b.length);
|
|
140
183
|
c.set(a, 0);
|
|
141
184
|
c.set(b, a.length);
|
|
@@ -147,7 +190,13 @@ export default class ServerDownloader {
|
|
|
147
190
|
});
|
|
148
191
|
this.#validateResponse(response);
|
|
149
192
|
const responseText = await response.text();
|
|
193
|
+
if (!this.#isValidString(responseText)) {
|
|
194
|
+
throw new ObjectLoaderRuntimeError('Invalid base object');
|
|
195
|
+
}
|
|
150
196
|
const item = this.#processJson(this.#options.objectId, responseText);
|
|
197
|
+
if (!item.base) {
|
|
198
|
+
throw new ObjectLoaderRuntimeError('Invalid base object');
|
|
199
|
+
}
|
|
151
200
|
item.size = 0;
|
|
152
201
|
return item;
|
|
153
202
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serverDownloader.js","sourceRoot":"","sources":["../../../../src/core/stages/serverDownloader.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,MAAM,+BAA+B,CAAA;AAEzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAA;AAChE,OAAO,EAAyB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAA;AAgBvF,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAA,CAAC,qBAAqB;AAE1F,MAAM,CAAC,OAAO,OAAO,gBAAgB;IACnC,kBAAkB,CAAQ;IAC1B,mBAAmB,CAAQ;IAC3B,QAAQ,CAAa;IACrB,QAAQ,CAAyB;IACjC,MAAM,CAAS;IACf,QAAQ,CAAc;IACtB,MAAM,CAAS;IACf,OAAO,CAAc;IAErB,cAAc,CAAwB;IACtC,QAAQ,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACpD,kBAAkB,GAAG,CAAC,CAAA;IAEtB,WAAW,GAAkB,EAAE,CAAA;IAC/B,aAAa,GAAsB,EAAE,CAAA;IAErC,YAAY,OAAgC;QAC1C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAA;QAC7B,IAAI,CAAC,MAAM;YACT,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,EAAqB,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9E,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;QAClB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC/C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;YACtC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,YAAY,CAAA;QAEtC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,UAAU,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QAClE,CAAC;QACD,IAAI,CAAC,mBAAmB,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,oBACnD,IAAI,CAAC,QAAQ,CAAC,QAChB,iBAAiB,CAAA;QAEjB,IAAI,CAAC,kBAAkB,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,YAClD,IAAI,CAAC,QAAQ,CAAC,QAChB,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,SAAS,CAAA;QAEnC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;YAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CACrB,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAC5D,CAAA;QACH,CAAC;IACH,CAAC;IAED,UAAU,CAAC,MAIV;QACC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE,GAAG,MAAM,CAAA;QACvD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;QACnB,IAAI,CAAC,cAAc,GAAG,IAAI,aAAa,CAAS;YAC9C,SAAS,EAAE,KAAK,EAAE,sCAAsC;YACxD,WAAW,EAAE,oBAAoB,IAAI,IAAI,EAAE,WAAW;YACtD,eAAe,EAAE,CAAC,KAAe,EAAiB,EAAE,CAClD,IAAI,CAAC,aAAa,CAAC;gBACjB,KAAK;gBACL,GAAG,EAAE,IAAI,CAAC,mBAAmB;gBAC7B,OAAO,EAAE,IAAI,CAAC,QAAQ;aACvB,CAAC;SACL,CAAC,CAAA;IACJ,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;IAClC,CAAC;IAED;;;;;;MAME;IACF,WAAW,CAAC,WAAuB;QACjC,IAAI,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC,UAAU,GAAG,uBAAuB,EAAE,CAAC;YAC/E,wDAAwD;YACxD,IAAI,CAAC,QAAQ,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YACzD,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA,CAAC,oCAAoC;QAClE,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QACrD,IAAI,CAAC,kBAAkB,IAAI,WAAW,CAAC,UAAU,CAAA;QACjD,OAAO,WAAW,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,cAAc,EAAE,YAAY,EAAE,CAAA;IAC3C,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAInB;QACC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,MAAM,CAAA;QAEtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QAC/B,IAAI,CAAC,OAAO,CAAC,wBAAwB,KAAK,CAAC,MAAM,WAAW,CAAC,CAAA;QAC7D,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAA;QACjD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,KAAK,CAAC,CAAA;QACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE;YACtC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC3D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC;SAC1D,CAAC,CAAA;QAEF,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAChC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;QAC1E,CAAC;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAA;QACxC,IAAI,QAAQ,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAC3C,IAAI,IAAI;gBAAE,MAAK;YAEf,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,EAAE;gBACpE,KAAK,EAAE,CAAA;gBACP,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA,CAAC,6BAA6B;gBACxF,CAAC;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,uCAAuC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAC5E,CAAA;QACH,CAAC;QACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAA,CAAC,sBAAsB;QACzC,IAAI,KAAK,IAAI,IAAI,CAAC,MAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAA,CAAC,yBAAyB;QAC/D,CAAC;QACD,IAAI,CAAC,OAAO,CACV,uBAAuB,KAAK,CAAC,MAAM,aAAa,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAC9E,CAAA;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,QAAoB,EACpB,KAAiB,EACjB,IAAiB,EACjB,QAA6B;QAE7B,uCAAuC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QACzD,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,2BAA2B;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;gBAC3D,4BAA4B;gBAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;gBACpC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;gBACb,MAAM,QAAQ,EAAE,CAAA;gBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACxB,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;YAC1B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA,CAAC,uBAAuB;IACzD,CAAC;IAED,YAAY,CAAC,IAAgB;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACrB,eAAe;gBACf,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;gBACpD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAEtC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;oBACnC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;gBACpC,CAAC;gBACD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;gBACxC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;gBAC5C,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC,MAAM,CAAA;gBAC5B,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QACD,MAAM,IAAI,wBAAwB,CAChC,mCAAmC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAC7D,CAAA;IACH,CAAC;IAED,YAAY,CAAC,MAAc,EAAE,YAAoB;QAC/C,IAAI,IAAa,CAAA;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;QACjC,CAAC;QAAC,OAAO,CAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,KAAM,CAAW,CAAC,OAAO,EAAE,CAAC,CAAA;QAC5E,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,wBAAwB,CAAC,GAAG,MAAM,gBAAgB,CAAC,CAAA;QAC/D,CAAC;IACH,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,WAAW;YACtC,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,OAAO,KAAK,CAAA;YACd,CAAC;QACH,OAAO,IAAI,CAAA;IACb,CAAC;IAED,aAAa,CAAC,IAAgB;QAC5B,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,aAAa;YAC1C,IAAI,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtC,OAAO,KAAK,CAAA;YACd,CAAC;QACH,OAAO,IAAI,CAAA;IACb,CAAC;IAED,kBAAkB,CAAC,CAAa,EAAE,CAAa;QAC7C,MAAM,CAAC,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7C,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACX,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAA;QAClB,OAAO,CAAC,CAAA;IACV,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE;YAC1D,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC,CAAA;QACF,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAChC,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAC1C,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,wBAAwB,CAAC,qBAAqB,CAAC,CAAA;QAC3D,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;QACpE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,wBAAwB,CAAC,qBAAqB,CAAC,CAAA;QAC3D,CAAC;QACD,IAAI,CAAC,IAAI,GAAG,CAAC,CAAA;QACb,OAAO,IAAI,CAAA;IACb,CAAC;IAED,iBAAiB,CAAC,QAAkB;QAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,wBAAwB,CAAC,yBAAyB,CAAC,CAAA;YAC/D,CAAC;YACD,MAAM,IAAI,wBAAwB,CAChC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,CACtE,CAAA;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"traverser.d.ts","sourceRoot":"","sources":["../../../src/core/traverser.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAa,MAAM,mBAAmB,CAAA;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAElD,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,cAAc,CAAA;AACvD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE;IAC3B,KAAK,EAAE,aAAa,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;CACd,KAAK,IAAI,CAAA;AAEV,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;CACxB;AAED,MAAM,CAAC,OAAO,OAAO,SAAS;;gBAOhB,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,gBAAgB;IAKvD,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAehD,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAe5E,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;CAuDvE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"traverser.js","sourceRoot":"","sources":["../../../src/core/traverser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AAerE,MAAM,CAAC,OAAO,OAAO,SAAS;IAC5B,OAAO,CAAe;IACtB,QAAQ,CAAkB;IAE1B,mBAAmB,GAAG,CAAC,CAAA;IACvB,yBAAyB,GAAG,CAAC,CAAA;IAE7B,YAAY,MAAqB,EAAE,OAA0B;QAC3D,IAAI,CAAC,QAAQ,GAAG,OAAO,IAAI,EAAE,CAAA;QAC7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAuB;QACpC,IAAI,kBAAkB,GAA8B,SAAS,CAAA;QAC7D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACxB,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;QAED,IAAI,kBAAkB,EAAE,CAAC;YACvB,OAAO,MAAM,kBAAkB,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAqB,EAAE,UAAuB;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACrB,IAAI,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAC5B,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;YACtD,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAChC,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,EACvD,UAAU,CACX,CAAA;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAAU,EAAE,UAAuB;QACpD,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;YAC3D,OAAQ,IAAc,CAAC,WAAW,CAAC,CAAA;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACvC,MAAM,QAAQ,GAAoB,EAAE,CAAA;YACpC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,CACpE,CAAA;YACH,CAAC;YACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC7B,CAAC;QACD,OAAQ,IAAc,CAAC,WAAW,CAAC,CAAA;QAEnC,WAAW;QACX,IAAI,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,IAAiB,CAAA;YAC/B,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;YAClD,CAAC;QACH,CAAC;QAED,aAAa;QACb,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,KAAK,WAAW;gBAAE,SAAQ;YAClC,IAAI,IAAI,KAAK,aAAa;gBAAE,SAAQ;YACpC,IAAI,IAAI,KAAK,cAAc;gBAAE,SAAQ;YACrC,IAAI,IAAI,KAAK,MAAM;gBAAE,SAAQ;YAC7B,MAAM,QAAQ,GAAI,IAA2C,CAAC,IAAI,CAAC,CAAA;YACnE,IAAI,QAAQ,CAAC,QAAQ,CAAC;gBAAE,SAAQ;YAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAC/C,CAAC;iBAAM,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,YAAY,CACrB,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC,EAC3D,UAAU,CACX,CAAA;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAChD,CAAC;QACH,CAAC;QACD,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC;gBACT,KAAK,EAAE,cAAc;gBACrB,OAAO,EACL,EAAE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,mBAAmB;oBACzD,CAAC,CAAC,IAAI,CAAC,mBAAmB;oBAC1B,CAAC,CAAC,IAAI,CAAC,yBAAyB;gBACpC,KAAK,EAAE,IAAI,CAAC,mBAAmB;aAChC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;CACF"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { CustomLogger } from '../types/functions.js';
|
|
2
|
+
import { Item } from '../types/types.js';
|
|
3
|
+
export interface MemoryCacheOptions {
|
|
4
|
+
maxSizeInMb: number;
|
|
5
|
+
ttlms: number;
|
|
6
|
+
}
|
|
7
|
+
export declare class MemoryCacheItem {
|
|
8
|
+
private item;
|
|
9
|
+
private expiresAt;
|
|
10
|
+
constructor(item: Item, expiresAt: number);
|
|
11
|
+
isExpired(now: number): boolean;
|
|
12
|
+
setAccess(now: number, ttl: number): void;
|
|
13
|
+
getItem(): Item;
|
|
14
|
+
done(now: number): boolean;
|
|
15
|
+
}
|
|
16
|
+
export declare class MemoryCache {
|
|
17
|
+
private isGathered;
|
|
18
|
+
private references;
|
|
19
|
+
private cache;
|
|
20
|
+
private options;
|
|
21
|
+
private logger;
|
|
22
|
+
private disposed;
|
|
23
|
+
private currentSize;
|
|
24
|
+
private timer?;
|
|
25
|
+
constructor(options: MemoryCacheOptions, logger: CustomLogger);
|
|
26
|
+
add(item: Item, requestItem: (id: string) => void, testNow?: number): void;
|
|
27
|
+
get(id: string): Item | undefined;
|
|
28
|
+
scanForReferences(data: unknown, requestItem: (id: string) => void): void;
|
|
29
|
+
private resetGlobalTimer;
|
|
30
|
+
private now;
|
|
31
|
+
cleanCache(testNow?: number): void;
|
|
32
|
+
compareMaybeBasesByReferences(id1: string, id2: string): number;
|
|
33
|
+
dispose(): void;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=MemoryCache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MemoryCache.d.ts","sourceRoot":"","sources":["../../../src/deferment/MemoryCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAExC,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,IAAI,CAAM;IAClB,OAAO,CAAC,SAAS,CAAQ;gBAEb,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM;IAKzC,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAG/B,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAIzC,OAAO,IAAI,IAAI;IAIf,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;CAM3B;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,UAAU,CAAiC;IACnD,OAAO,CAAC,KAAK,CAA0C;IAEvD,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,WAAW,CAAI;IACvB,OAAO,CAAC,KAAK,CAAC,CAA+B;gBAEjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,YAAY;IAM7D,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;IAc1E,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAUjC,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAuCzE,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,GAAG;IAIX,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;IAwClC,6BAA6B,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM;IAS/D,OAAO,IAAI,IAAI;CAWhB"}
|