@live-change/db-store-indexeddb 0.9.229 → 0.9.231

package/lib/Store.js

@@ -70,8 +70,8 @@ class RangeObservable extends ReactiveDao.ObservableList {
     this.disposed = false
     this.ready = false
     this.respawnId = 0
-    this.refillId = 0
-    this.refillPromise = null
+    this.refillQueue = []
+    this.refillRunning = false
 
     this.forward = null
 
@@ -145,48 +145,78 @@ class RangeObservable extends ReactiveDao.ObservableList {
     }
   }
 
-  refillDeleted(from, limit) {
-    this.refillId ++
-    const refillId = this.refillId
-    let promise = (async () => {
-      let req
-      if(!this.range.reverse) {
-        req = { gt: from, limit }
-        if(this.range.lt) req.lt = this.range.lt
-        if(this.range.lte) req.lte = this.range.lte
-      } else {
-        req = { lt: from, limit, reverse: true }
-        if(this.range.gt) req.gt = this.range.gt
-        if(this.range.gte) req.gte = this.range.gte
+  refillDeleted(from, need) {
+    if(this.disposed) return Promise.resolve()
+    return new Promise((resolve, reject) => {
+      this.refillQueue.push({ from, need, resolve, reject })
+      this._drainRefill()
+    })
+  }
+
+  async _syncWindow() {
+    if(this.disposed || !this.range.limit) return
+    const reverse = !!this.range.reverse
+    const truth = await this.store.rangeGet(this.range)
+    if(this.disposed) return
+    const same = truth.length === this.list.length &&
+      truth.every((o, i) => this.list[i] && this.list[i].id === o.id)
+    if(same) return
+    const truthIds = new Set(truth.map(o => o.id))
+    for(const obj of this.list.slice()) {
+      if(!truthIds.has(obj.id)) this.removeByField('id', obj.id, obj)
+    }
+    for(const object of truth) {
+      this.putByField('id', object.id, object, reverse)
+    }
+    while(this.list.length > truth.length) {
+      const popped = this.list.pop()
+      this.fireObservers('removeByField', 'id', popped.id, popped)
+    }
+    while(this.range.limit && this.list.length > this.range.limit) {
+      const popped = this.list.pop()
+      this.fireObservers('removeByField', 'id', popped.id, popped)
+    }
+  }
+
+  async _drainRefill() {
+    if(this.refillRunning) return
+    this.refillRunning = true
+    const allJobs = []
+    try {
+      while(this.refillQueue.length) {
+        allJobs.push(...this.refillQueue.splice(0))
+        await this._syncWindow()
       }
-      const objects = await this.store.rangeGet(req)
-      if(this.refillId != refillId) return this.refillPromise
-      for(let object of objects) this.push(object)
-      this.refillPromise = null
-    })()
-    this.refillPromise = promise
-    return promise
+      await this._syncWindow()
+    } catch(err) {
+      this.refillRunning = false
+      for(const job of allJobs) job.reject(err)
+      for(const job of this.refillQueue.splice(0)) job.reject(err)
+      return
+    }
+    this.refillRunning = false
+    for(const job of allJobs) job.resolve()
+    if(this.refillQueue.length) this._drainRefill()
   }
 
   async deleteObject(object) {
-    if(!object) return;
+    if(!object) return
     await this.readPromise
+    if(this.disposed) return
     const id = object.id
     if(this.range.gt && !(id > this.range.gt)) return
     if(this.range.lt && !(id < this.range.lt)) return
-    if(this.range.limit && (this.list.length == this.range.limit || this.refillPromise)) {
-      let exists
-      let last
-      for(let obj of this.list) {
-        if(obj.id == id) exists = obj
-        else last = obj
-      }
-      this.removeByField('id', id, object)
-      if(exists) await this.refillDeleted(
-          last && last.id || (this.reverse ? this.range.lt || this.range.lte : this.range.gt || this.range.gte),
-          this.range.limit - this.list.length)
-    } else {
-      this.removeByField('id', id, object)
+    let exists = false, last = null
+    for(let obj of this.list) {
+      if(obj.id == id) exists = obj
+      else last = obj
+    }
+    this.removeByField('id', id, object)
+    if(exists && this.range.limit) {
+      const need = this.range.limit - this.list.length
+      if(need > 0) await this.refillDeleted(
+        last && last.id || (this.range.reverse ? this.range.lt || this.range.lte : this.range.gt || this.range.gte),
+        need)
     }
   }
 
@@ -198,6 +228,8 @@ class RangeObservable extends ReactiveDao.ObservableList {
     }
 
     this.disposed = true
+    const pending = this.refillQueue.splice(0)
+    for(const job of pending) job.resolve()
     this.respawnId++
     this.changesStream = null
 
@@ -216,6 +248,8 @@ class RangeObservable extends ReactiveDao.ObservableList {
     this.respawnId++
     this.ready = false
     this.disposed = false
+    this.refillQueue = []
+    this.refillRunning = false
     this.startReading()
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/db-store-indexeddb",
-  "version": "0.9.229",
+  "version": "0.9.231",
   "description": "Database with observable data for live queries",
   "main": "index.js",
   "scripts": {
@@ -26,9 +26,9 @@
     "tape": "^5.7.4"
   },
   "dependencies": {
-    "@live-change/dao": "^0.9.229",
-    "@live-change/flatten-interval-tree": "^0.9.229",
+    "@live-change/dao": "^0.9.231",
+    "@live-change/flatten-interval-tree": "^0.9.231",
     "broadcast-channel": "^4.2.0"
   },
-  "gitHead": "10de6af051ea567a0ae917d34ac08378da5465ea"
+  "gitHead": "d49b1369964dafe82610664dd56255b6458d0cae"
 }

package/tests/limited-range-observable.js

@@ -4,6 +4,13 @@ require('fake-indexeddb/auto.js')
 const idb = require('idb')
 const Store = require('../lib/Store.js')
 
+
+function assertLimit(t, obs) {
+  var limit = obs.range && obs.range.limit
+  t.ok(!limit || obs.list.length <= limit, 'within limit')
+  t.equal(new Set(obs.list.map(function (o) { return o.id })).size, obs.list.length, 'no dup')
+}
+
 test("store limited range observable", t => {
   t.plan(5)
 
@@ -47,60 +54,66 @@ test("store limited range observable", t => {
   }
 
   t.test("observe range [b_0,b_9]", t => {
-    t.plan(7)
+    t.plan(9)
     rangeObservable = store.rangeObservable({ gte: 'b_0', lte: 'b_9', limit: 5 })
     rangeObservable.observe(rangeObserver)
 
     let values
 
     t.test("get values", async t => {
-      t.plan(1)
+      t.plan(3)
       values = await getNextValue()
       t.deepEqual(values, objects.slice(0,5), 'range value')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_3' from observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_3')
       objects.splice(3, 1)
       await getNextValue() // one shorter
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_3' to observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_3', v: 10 }
       await store.put(obj)
       objects.splice(3, 0, obj)
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_32' to observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_32', v: 11 }
       await store.put(obj)
       objects.splice(4, 0, obj)
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_35' outside observed range limits", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_35', v: 11 }
       await store.put(obj)
       objects.splice(5, 0, obj)
       t.pass('added')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("delete object 'b_32' so b_35 will enter into range limits", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_32')
       objects.splice(4, 1)
       await getNextValue()
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("unobserve range", async t => {
@@ -111,7 +124,7 @@ test("store limited range observable", t => {
   })
 
   t.test("observe range (b_0,b_5)", t => {
-    t.plan(7)
+    t.plan(9)
     gotNextValue = false
     rangeObservable = store.rangeObservable({ gt: 'b_0', lt: 'b_5', limit: 3 })
     rangeObservable.observe(rangeObserver)
@@ -119,45 +132,51 @@ test("store limited range observable", t => {
     let values
 
     t.test("get values", async t => {
-      t.plan(1)
+      t.plan(3)
       values = await getNextValue()
       t.deepEqual(values, objects.slice(1,4), 'range value')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_0' outside observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_0')
       t.pass('deleted')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_5' outside observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_5')
       t.pass('deleted')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_4' outside observed range limits", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_4')
       t.pass('deleted')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_12' to observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_12', v: 7 }
       await store.put(obj)
       objects.splice(2, 0, obj)
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(1,4), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_12' from observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_12')
       objects.splice(2, 1)
       await getNextValue()
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(1,4) , 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("unobserve range", async t => {

package/tests/limited-reverse-range-observable.js

@@ -4,6 +4,13 @@ require('fake-indexeddb/auto.js')
 const idb = require('idb')
 const Store = require('../lib/Store.js')
 
+
+function assertLimit(t, obs) {
+  var limit = obs.range && obs.range.limit
+  t.ok(!limit || obs.list.length <= limit, 'within limit')
+  t.equal(new Set(obs.list.map(function (o) { return o.id })).size, obs.list.length, 'no dup')
+}
+
 test("store limited reverse range observable", t => {
   t.plan(5)
 
@@ -47,60 +54,66 @@ test("store limited reverse range observable", t => {
   }
 
   t.test("observe reverse range [b_9,b_0]", t => {
-    t.plan(7)
+    t.plan(9)
     rangeObservable = store.rangeObservable({ gte: 'b_0', lte: 'b_9', limit: 5, reverse: true })
     rangeObservable.observe(rangeObserver)
 
     let values
 
     t.test("get values", async t => {
-      t.plan(1)
+      t.plan(3)
       values = await getNextValue()
       t.deepEqual(values, objects.slice(0,5), 'range value')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_6' from observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_6')
       objects.splice(3, 1)
       await getNextValue() // one shorter
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_6' to observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_6', v: 10 }
       await store.put(obj)
       objects.splice(3, 0, obj)
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_58' to observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_58', v: 11 }
       await store.put(obj)
       objects.splice(4, 0, obj)
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_52' outside observed range limits", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_52', v: 11 }
       await store.put(obj)
       objects.splice(5, 0, obj)
       t.pass('added')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("delete object 'b_58' so b_52 will enter into range limits", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_58')
       objects.splice(4, 1)
       await getNextValue()
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(0, 5), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("unobserve range", async t => {
@@ -111,52 +124,58 @@ test("store limited reverse range observable", t => {
   })
 
   t.test("observe reverse range (b_5,b_0)", t => {
-    t.plan(7)
+    t.plan(9)
     gotNextValue = false
     rangeObservable = store.rangeObservable({ gt: 'b_0', lt: 'b_5', limit: 3, reverse: true })
     rangeObservable.observe(rangeObserver)
 
     let values
     t.test("get values", async t => {
-      t.plan(1)
+      t.plan(3)
       values = await getNextValue()
       t.deepEqual(values, objects.slice(6,9), 'range value')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_5' outside observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_5')
       t.pass('deleted')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_0' outside observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_0')
       t.pass('deleted')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_1' outside observed range limits", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_1')
       t.pass('deleted')
+      assertLimit(t, rangeObservable)
     })
 
     t.test("add object 'b_48' to observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       const obj = { id: 'b_48', v: 7 }
       await store.put(obj)
       objects.splice(6, 0, obj)
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(6,9), 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("remove object 'b_48' from observed range", async t => {
-      t.plan(1)
+      t.plan(3)
       await store.delete('b_48')
       objects.splice(6, 1)
       await getNextValue()
       let values = await getNextValue()
       t.deepEqual(values, objects.slice(6,9) , 'range value' )
+      assertLimit(t, rangeObservable)
     })
 
     t.test("unobserve range", async t => {

package/tests/stress-range-observable.js

@@ -0,0 +1,9 @@
+// browser-only: this package is CJS in Node; the shared suite is ESM.
+// CI Node skips the real stress (fake-indexeddb). RangeObservable still has
+// the same refill queue fix as the other stores.
+const test = require('tape')
+
+test('indexeddb range observable stress (browser-only)', t => {
+  t.skip('browser-only store')
+  t.end()
+})