@live-change/db-store-localstorage 0.6.0

package/index.js

@@ -0,0 +1 @@
+module.exports = require('./lib/Store.js')

package/lib/Store.js

@@ -0,0 +1,416 @@
+const IntervalTree = require('@live-change/interval-tree').default
+const ReactiveDao = require("@live-change/dao")
+const { BroadcastChannel, createLeaderElection } = require('broadcast-channel')
+const storage = require('./storage.js')
+
+class ObjectObservable extends ReactiveDao.ObservableValue {
+  constructor(store, key) {
+    super()
+    this.store = store
+    this.key = key
+
+    this.disposed = false
+    this.ready = false
+    this.respawnId = 0
+
+    this.forward = null
+
+    this.readPromise = this.startReading()
+  }
+
+  async startReading() {
+    this.store.objectObservables.set(this.key, this)
+    this.value = await this.store.objectGet(this.key)
+    this.fireObservers('set', this.value)
+  }
+
+  async set(value) {
+    await this.readPromise
+    this.value = value
+    this.fireObservers('set', this.value)
+  }
+
+  dispose() {
+    if(this.forward) {
+      this.forward.unobserve(this)
+      this.forward = null
+      return
+    }
+
+    this.disposed = true
+    this.respawnId++
+    if(this.changesStream) this.changesStream.close()
+    this.changesStream = null
+
+    this.store.objectObservables.delete(this.key)
+  }
+
+  respawn() {
+    const existingObservable = this.store.objectObservables.get(this.key)
+    if(existingObservable) {
+      this.forward = existingObservable
+      this.forward.observe(this)
+      return
+    }
+
+    this.respawnId++
+    if(this.changesStream) this.changesStream.close()
+    this.ready = false
+    this.disposed = false
+    this.startReading()
+  }
+}
+
+class RangeObservable extends ReactiveDao.ObservableList {
+  constructor(store, range) {
+    super()
+    this.store = store
+    this.range = range
+
+    this.disposed = false
+    this.ready = false
+    this.respawnId = 0
+    this.refillId = 0
+    this.refillPromise = null
+
+    this.forward = null
+
+    this.rangeKey = JSON.stringify(this.range)
+    this.rangeDescr = [ this.range.gt || this.range.gte || '', this.range.lt || this.range.lte || '\xFF\xFF\xFF\xFF']
+
+    this.readPromise = this.startReading()
+  }
+
+  async startReading() {
+    this.store.rangeObservables.set(this.rangeKey, this)
+    const treeInsert = this.rangeDescr
+    const inserted = this.store.rangeObservablesTree.insert(treeInsert, this)
+    if(this.store.rangeObservablesTree.search([this.low, this.high]).length == 0) {
+      console.error("TREE NOT WORKING")
+      console.log("INSERTED", JSON.stringify(treeInsert),
+          "TO TREE", this.store.rangeObservablesTree)
+      console.log("FOUND", this.store.rangeObservablesTree.search(this.rangeDescr))
+      console.log("ALL RECORDS", this.store.rangeObservablesTree.search(['', '\xFF\xFF\xFF\xFF']))
+      process.exit(1)
+    }
+    this.set(await this.store.rangeGet(this.range))
+  }
+
+  async putObject(object, oldObject) {
+    await this.readPromise
+    const id = object.id
+    if(this.range.gt && !(id > this.range.gt)) return
+    if(this.range.lt && !(id < this.range.lt)) return
+    if(!this.range.reverse) {
+      if(this.range.limit && this.list.length == this.range.limit) {
+        for(let i = 0, l = this.list.length; i < l; i++) {
+          if(this.list[i].id == id) {
+            this.list.splice(i, 1, object)
+            this.fireObservers('putByField', 'id', id, object, false, oldObject)
+            return
+          } else if(this.list[i].id > id) {
+            this.list.splice(i, 0, object)
+            this.fireObservers('putByField', 'id', id, object, false, oldObject)
+            const popped = this.list.pop()
+            this.fireObservers('removeByField', 'id', popped.id, popped)
+            return
+          }
+        }
+      } else {
+        this.putByField('id', object.id, object, false, oldObject)
+      }
+    } else {
+      if(this.range.limit && this.list.length == this.range.limit) {
+        for(let i = this.list.length-1; i >= 0; i--) {
+          if(this.list[i].id == id) {
+            this.list.splice(i, 1, object)
+            this.fireObservers('putByField', 'id', id, object, true, oldObject)
+            return
+          } else if(this.list[i].id > id) {
+            if(i == this.list.length - 1) return // last element is bigger, do nothing
+            this.list.splice(i + 1, 0, object)
+            this.fireObservers('putByField', 'id', id, object, true, oldObject)
+            const popped = this.list.pop()
+            this.fireObservers('removeByField', 'id', popped.id, popped)
+            return
+          }
+        }
+        this.list.splice(0, 0, object)
+        this.fireObservers('putByField', 'id', id, object, true)
+        const popped = this.list.pop()
+        this.fireObservers('removeByField', 'id', popped.id, popped)
+      } else {
+        this.putByField('id', id, object, true, oldObject)
+      }
+    }
+  }
+
+  refillDeleted(from, limit) {
+    this.refillId ++
+    const refillId = this.refillId
+    let promise = (async () => {
+      let req
+      if(!this.range.reverse) {
+        req = { gt: from, limit }
+        if(this.range.lt) req.lt = this.range.lt
+        if(this.range.lte) req.lte = this.range.lte
+      } else {
+        req = { lt: from, limit, reverse: true }
+        if(this.range.gt) req.gt = this.range.gt
+        if(this.range.gte) req.gte = this.range.gte
+      }
+      const objects = await this.store.rangeGet(req)
+      if(this.refillId != refillId) return this.refillPromise
+      for(let object of objects) this.push(object)
+      this.refillPromise = null
+    })()
+    this.refillPromise = promise
+    return promise
+  }
+
+  async deleteObject(object) {
+    if(!object) return;
+    await this.readPromise
+    const id = object.id
+    if(this.range.gt && !(id > this.range.gt)) return
+    if(this.range.lt && !(id < this.range.lt)) return
+    if(this.range.limit && (this.list.length == this.range.limit || this.refillPromise)) {
+      let exists
+      let last
+      for(let obj of this.list) {
+        if(obj.id == id) exists = obj
+        else last = obj
+      }
+      this.removeByField('id', id, object)
+      if(exists) await this.refillDeleted(
+          last && last.id || (this.reverse ? this.range.lt || this.range.lte : this.range.gt || this.range.gte),
+          this.range.limit - this.list.length)
+    } else {
+      this.removeByField('id', id, object)
+    }
+  }
+
+  dispose() {
+    if(this.forward) {
+      this.forward.unobserve(this)
+      this.forward = null
+      return
+    }
+
+    this.disposed = true
+    this.respawnId++
+    this.changesStream = null
+
+    this.store.rangeObservables.delete(this.rangeKey)
+    let removed = this.store.rangeObservablesTree.remove(this.rangeDescr, this)
+  }
+
+  respawn() {
+    const existingObservable = this.store.rangeObservables.get(JSON.stringify(this.range))
+    if(existingObservable) {
+      this.forward = existingObservable
+      this.forward.observe(this)
+      return
+    }
+
+    this.respawnId++
+    this.ready = false
+    this.disposed = false
+    this.startReading()
+  }
+}
+
+class Store {
+  constructor(dbName, storeName, type) {
+    if(!dbName) throw new Error("dbName argument is required")
+    if(!storeName) throw new Error("storeName argument is required")
+    if(!type) throw new Error("type argument is required")
+
+    this.dbName = dbName
+    this.storeName = storeName
+
+    this.prefix = `lcdb/${dbName}/${storeName}/`
+
+    this.finished = false
+
+    this.storage = storage[type]
+    if(!this.storage) throw new Error("Unknown storage type: " + type)
+
+    this.channel = null
+
+    this.objectObservables = new Map()
+    this.rangeObservables = new Map()
+    this.rangeObservablesTree = new IntervalTree()
+  }
+
+  async openChannel() {
+    this.channel = new BroadcastChannel(`lc-db-${this.dbName}-${this.storeName}`, {
+      idb: {
+        onclose: () => {
+          if(this.finished) return
+          this.channel.close()
+          this.openChannel()
+        }
+      }
+    })
+    this.channel.onmessage = message => this.handleChannelMessage(message)
+  }
+  async open() {
+    await this.openChannel()
+  }
+  async close() {
+    this.finished = true
+    await this.channel.close()
+  }
+
+  async deleteDb() {
+    if(!this.finished) await this.close()
+    const all = await this.storage.getKeys()
+    const own = all.filter(key => key.startsWith(this.prefix))
+    await this.storage.remove(own)
+  }
+
+  async handleChannelMessage(message) {
+    console.log("handleChannelMessage", message)
+    switch(message.type) {
+      case 'put' : {
+        const { object, oldObject } = message
+        const id = object?.id || oldObject?.id
+        if(typeof id != 'string') throw new Error(`ID is not string: ${JSON.stringify(id)}`)
+        const objectObservable = this.objectObservables.get(id)
+        if(objectObservable) objectObservable.set(object, oldObject)
+        const rangeObservables = this.rangeObservablesTree.search([id, id])
+        for(const rangeObservable of rangeObservables) {
+          rangeObservable.putObject(object, oldObject)
+        }
+      } break
+      case 'delete' : {
+        const { object } = message
+        const id = object?.id
+        if(typeof id != 'string') throw new Error(`ID is not string: ${JSON.stringify(id)}`)
+        const objectObservable = this.objectObservables.get(id)
+        if(objectObservable) objectObservable.set(null)
+        const rangeObservables = this.rangeObservablesTree.search([id, id])
+        for(const rangeObservable of rangeObservables) {
+          rangeObservable.deleteObject(object)
+        }
+      } break
+      default:
+        throw new Error("unknown message type " + message.type + ' in message ' + JSON.stringify(message))
+    }
+  }
+
+  async objectGet(id) {
+    if(!id) throw new Error("key is required")
+    if(typeof id != 'string') throw new Error(`ID is not string: ${JSON.stringify(id)}`)
+    return JSON.parse(await this.storage.getItem(this.prefix + id) || 'null')
+  }
+
+  objectObservable(key) {
+    let observable = this.objectObservables.get(key)
+    if(observable) return observable
+    observable = new ObjectObservable(this, key)
+    return observable
+  }
+
+  async rangeGet(range) {
+    if(!range) throw new Error("range not defined")
+    const { gt, gte, lt, lte, limit, reverse } = range
+    const all = (await this.storage.getKeys())
+      .filter(key => key.startsWith(this.prefix))
+      .sort()
+    if(range.reverse) all.reverse()
+    const keys = all.filter(key => {
+      const id = key.slice(this.prefix.length)
+      if(gt && !(id > gt)) return false
+      if(gte && !(id >= gte)) return false
+      if(lt && !(id < lt)) return false
+      if(lte && !(id <= lte)) return false
+      return true
+    }).slice(0, limit)
+    const objects = (await this.storage.getValues(keys))
+      .map(json => JSON.parse(json))
+      .sort((a, b) => (a.id > b.id) ? 1 : ((b.id > a.id) ? -1 : 0))
+    if(range.reverse) objects.reverse()
+    return objects
+  }
+
+  rangeObservable(range) {
+    let observable = this.rangeObservables.get(JSON.stringify(range))
+    if(observable) return observable
+    observable = new RangeObservable(this, range)
+    return observable
+  }
+
+  async countGet(range) {
+    if(!range) throw new Error("range not defined")
+    const { gt, gte, lt, lte, limit, reverse } = range
+    const all = await this.storage.getKeys()
+    const keys = all.filter(key => {
+      if(!key.startsWith(this.prefix)) return false
+      const id = key.slice(this.prefix.length)
+      if(gt && !(id > gt)) return false
+      if(gte && !(id >= gte)) return false
+      if(lt && !(id < lt)) return false
+      if(lte && !(id <= lte)) return false
+      return true
+    }).slice(0, limit)
+    return keys.length
+  }
+
+  countObservable(range) {
+    let observable = this.countObservables.get(JSON.stringify(range))
+    if(observable) return observable
+    observable = new CountObservable(this, range)
+    return observable
+  }
+
+  async rangeDelete(range) {
+    if(!range) throw new Error("range not defined")
+    const { gt, gte, lt, lte, limit, reverse } = range
+    const all = await this.storage.getKeys()
+    const keys = all.filter(key => {
+      if(!key.startsWith(this.prefix)) return false
+      const id = key.slice(this.prefix.length)
+      if(gt && !(id > gt)) return false
+      if(gte && !(id >= gte)) return false
+      if(lt && !(id < lt)) return false
+      if(lte && !(id <= lte)) return false
+      return true
+    }).slice(0, limit)
+    await this.storage.delete(keys)
+    return { count: keys.length, last: keys[keys.length - 1] }
+  }
+
+  async put(object) {
+    const id = object.id
+    if(typeof id != 'string') throw new Error(`ID is not string: ${JSON.stringify(id)}`)
+    const oldObject = JSON.parse(await this.storage.getItem(this.prefix + id) || 'null')
+    await this.storage.setItem(this.prefix + id, JSON.stringify(object))
+    const objectObservable = this.objectObservables.get(id)
+    if(objectObservable) objectObservable.set(object, oldObject)
+    const rangeObservables = this.rangeObservablesTree.search([id, id])
+    for(const rangeObservable of rangeObservables) {
+      rangeObservable.putObject(object, oldObject)
+    }
+    this.channel.postMessage({ type: "put", object, oldObject })
+    return oldObject
+  }
+
+  async delete(id) {
+    if(typeof id != 'string') throw new Error(`ID is not string: ${JSON.stringify(id)}`)
+    const object = JSON.parse(await this.storage.getItem(this.prefix + id))
+    await this.storage.removeItem(this.prefix + id)
+    const objectObservable = this.objectObservables.get(id)
+    if(objectObservable) objectObservable.set(null)
+    const rangeObservables = this.rangeObservablesTree.search([id, id])
+    for(const rangeObservable of rangeObservables) {
+      rangeObservable.deleteObject(object)
+    }
+    this.channel.postMessage({ type: "delete", object  })
+    return object
+  }
+
+}
+
+module.exports = Store

package/lib/storage.js

@@ -0,0 +1,158 @@
+let local
+let session
+
+function useWebStorage(storage) {
+  return {
+    setItem: async (key, value) => Promise.resolve(storage.setItem(key, JSON.stringify(value))),
+    getItem: async (key) => {
+      const json = storage.getItem(key)
+      return Promise.resolve(json && JSON.parse(json))
+    },
+    removeItem: async (key) => Promise.resolve(storage.removeItem(key)),
+    clear: async () => Promise.resolve(storage.clear()),
+    set: async (keys) => {
+      for(let key in keys) {
+        storage.setItem(key, JSON.stringify(keys[key]))
+      }
+      return Promise.resolve()
+    },
+    get: async (keys) => {
+      if(!keys) {
+        const result = {}
+        for(let i = 0; i<storage.length; i++) {
+          const key = storage.key(i)
+          result[key] = JSON.parse(storage.getItem(key))
+        }
+        return Promise.resolve(result)
+      }
+      const result = {}
+      const keysList = typeof keys == 'string' ? keys : (Array.isArray(keys) ? keys : Object.keys(keys))
+      for(let key of keysList) {
+        const item = storage.getItem(key)
+        result[key] = (item && JSON.parse(item)) ?? (typeof keys == 'object' ? keys[key] : undefined)
+      }
+      return Promise.resolve(result)
+    },
+    getValues: async (keys) => {
+      if(!keys) {
+        const result = new Array(storage.length)
+        for(let i = 0; i<storage.length; i++) {
+          const key = storage.key(i)
+          result[i] = JSON.parse(storage.getItem(key))
+        }
+        return Promise.resolve(result)
+      }
+      const result = new Array(keys.length)
+      for(let i = 0; i<keys.length; i++) {
+        const key = keys[i]
+        const item = storage.getItem(key)
+        result[i] = (item && JSON.parse(item)) ?? (typeof keys == 'object' ? keys[key] : undefined)
+      }
+      return Promise.resolve(result)
+    },
+    getKeys: async () => {
+      const result = []
+      for(let i = 0; i<storage.length; i++) {
+        result.push(storage.key(i))
+      }
+      return Promise.resolve(result)
+    },
+    remove: async (keys) => {
+      for(let key of keys) {
+        storage.removeItem(key)
+      }
+      return Promise.resolve()
+    }
+  }
+}
+
+function useExtensionStorage(storage) {
+  return {
+    set: async (keys) => storage.set(keys),
+    get: async (keys) => storage.get(keys),
+    getValues: async (keys) => Object.values(await storage.get(keys)),
+    getKeys: async () => Object.keys(await storage.get()),
+    remove: async (keys) => storage.remove(keys),
+    clear: async () => storage.clear(),
+    setItem: async (key, value) => storage.set({ [key]: value }),
+    getItem: async (key) => storage.get(key)[key],
+    removeItem: async (key) => storage.remove(key),
+  }
+}
+
+function useTestStorage() {
+  const storage = new Map()
+  return {
+    setItem: async (key, value) => Promise.resolve(storage.set(key, value)),
+    getItem: async (key) => Promise.resolve(storage.get(key)),
+    removeItem: async (key) => Promise.resolve(storage.delete(key)),
+    clear: async () => Promise.resolve(storage.clear()),
+    set: async (keys) => {
+      for(let key in keys) {
+        storage.set(key, keys[key])
+      }
+      return Promise.resolve()
+    },
+    get: async (keys) => {
+      if(!keys) {
+        const result = {}
+        for(let [key, value] of storage) {
+          result[key] = value
+        }
+        return Promise.resolve(result)
+      }
+      const result = {}
+      const keysList = typeof keys == 'string' ? keys : (Array.isArray(keys) ? keys : Object.keys(keys))
+      for(let key of keysList) {
+        result[key] = storage.get(key) ?? (typeof keys == 'object' ? keys[key] : undefined)
+      }
+      return Promise.resolve(result)
+    },
+    getValues: async (keys) => {
+      if(!keys) {
+        const result = new Array(storage.size)
+        let i = 0
+        for(let value of storage.values()) {
+          result[i++] = value
+        }
+        return Promise.resolve(result)
+      }
+      const result = new Array(keys.length)
+      for(let i = 0; i<keys.length; i++) {
+        const key = keys[i]
+        result[i] = storage.get(key) ?? (typeof keys == 'object' ? keys[key] : undefined)
+      }
+      return Promise.resolve(result)
+    },
+    getKeys: async () => Promise.resolve(Array.from(storage.keys())),
+    remove: async (keys) => {
+      for(let key of keys) {
+        storage.delete(key)
+      }
+      return Promise.resolve()
+    }
+  }
+}
+
+if(typeof window == 'undefined') {
+  local = useTestStorage()
+  session = useTestStorage()
+} else {
+  let isExtension = false
+  if (typeof browser !== "undefined") {
+    isExtension = browser.extension
+  } {
+    if (typeof chrome !== "undefined") {
+      window.browser = chrome.extension
+    }
+  }
+  if (isExtension) {
+    local = useExtensionStorage(browser.storage.local)
+    session = useExtensionStorage(browser.storage.session)
+  } else {
+    local = useWebStorage(window.localStorage)
+    session = useWebStorage(window.sessionStorage)
+  }
+}
+
+module.exports = { local, session }

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@live-change/db-store-localstorage",
+  "version": "0.6.0",
+  "description": "Database with observable data for live queries",
+  "main": "index.js",
+  "scripts": {
+    "test": "NODE_ENV=test tape tests/*"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/live-change/live-change-db.git"
+  },
+  "author": {
+    "email": "michal@laszczewski.com",
+    "name": "Michał Łaszczewski",
+    "url": "https://www.viamage.com/"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/live-change/live-change-db/issues"
+  },
+  "homepage": "https://github.com/live-change/live-change-db",
+  "devDependencies": {
+    "fake-indexeddb": "^3.1.3",
+    "idb": "^6.1.4",
+    "tape": "^5.3.2"
+  },
+  "dependencies": {
+    "@live-change/dao": "0.5.14",
+    "@live-change/interval-tree": "^1.0.12",
+    "broadcast-channel": "^4.2.0"
+  },
+  "gitHead": "358bc9b508a6e446b9286a01e0ceb0e77c1dec28"
+}

package/tests/broadcast-object-changes.js

@@ -0,0 +1,80 @@
+const test = require('tape')
+
+require('fake-indexeddb/auto.js')
+const idb = require('idb')
+const Store = require('../lib/Store.js')
+
+test("store broadcast object changes", t => {
+  t.plan(3)
+
+  let writeStore
+  let readStore
+
+  t.test("create stores", async t => {
+    t.plan(2)
+    readStore = new Store('test-broadcast-object-changes', 'test', 'local')
+    await readStore.open()
+    t.pass('read store created')
+    writeStore = new Store('test-broadcast-object-changes', 'test', 'local')
+    await writeStore.open()
+    t.pass('write store created')
+  })
+
+  let nextValueResolve
+  let gotNextValue
+  const getNextValue = () => {
+    if(gotNextValue) {
+      gotNextValue = false
+      return objectObservable.value
+    }
+    return new Promise((resolve, reject) => nextValueResolve = resolve)
+  }
+
+  let objectObservable
+  const objectObserver = (signal, value, ...rest) => {
+    console.log("SIGNAL", signal, value, ...rest)
+    if(nextValueResolve) {
+      nextValueResolve(value)
+    } else {
+      gotNextValue = true
+    }
+  }
+
+  t.test('observe object A', t => {
+    t.plan(3)
+    objectObservable = readStore.objectObservable('A')
+    objectObservable.observe(objectObserver)
+
+    let value
+
+    t.test('get value', async t => {
+      t.plan(1)
+      value = await getNextValue()
+      t.deepEqual(value, null, 'found null')
+    })
+
+    t.test("add object A", async t => {
+      t.plan(1)
+      await writeStore.put({ id: 'A', a: 1 })
+      let value = await getNextValue()
+      t.deepEqual(value, { id: 'A', a: 1 } , 'found object' )
+    })
+
+    t.test("delete object A", async t => {
+      t.plan(1)
+      await writeStore.delete('A')
+      let value = await getNextValue()
+      t.deepEqual(value, null , 'found null' )
+    })
+  })
+
+  t.test("close database", async t => {
+    t.plan(2)
+    await readStore.close()
+    t.pass('read store closed')
+    await writeStore.close()
+    t.pass('write store closed')
+    t.end()
+  })
+
+})