@live-change/db-store-observable-db 0.1.0 → 0.5.2

package/.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

package/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/db-store-observable-db.iml" filepath="$PROJECT_DIR$/db-store-observable-db.iml" />
+    </modules>
+  </component>
+</project>

package/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

package/db-store-observable-db.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

package/lib/Store.js

@@ -1,6 +1,8 @@
 const ReactiveDao = require("@live-change/dao")
 const WebSocket = require('ws')
 
+const debug = false
+
 const opCodes = {
   Ping : 0,
   Pong : 1,
@@ -50,13 +52,14 @@ const rangeFlags = {
 
 const resultPutFlags = {
   Found: 0x1,
-  Created:  0x2,
+  Created: 0x2,
   Last: 0x4
-};
+}
 
 class DatabaseConnection {
   constructor(url) {
     this.url = url
+    this.creatingStores = new Map()
     this.openingStores = new Map()
     this.openStores = []
     this.websocket = null
@@ -69,11 +72,14 @@ class DatabaseConnection {
   connect() {
     this.websocket = new WebSocket(this.url)
     this.websocket.binaryType = 'nodebuffer'
-    this.websocket.on('close', () => {
+    this.websocket.on('close', (reason) => {
+      console.error("WEBSOCKET CLOSED", reason)
+      process.exit(1)
       this.openedStores = []
     })
-    this.websocket.on('error', () => {
-
+    this.websocket.on('error', (error) => {
+      console.error("WEBSOCKET ERROR", error)
+      process.exit(1)
     })
     this.websocket.on('message', (message) => {
       if(typeof message != 'string') { // Buffer
@@ -91,10 +97,11 @@ class DatabaseConnection {
           //console.log("RESPONSE TO", requestId, request, "IS", message)
           if(request) {
             if(!request.handlePacket(message)) {
+              if(debug) console.log("REQUEST FINISHED", requestId)
               this.waitingRequests.delete(requestId)
             }
           } else {
-            console.log("unknown response", requestId)
+            if(debug) console.log("unknown response", requestId, message)
           }
         }
       } else { // text
@@ -112,6 +119,12 @@ class DatabaseConnection {
         }
       }
     })
+    setInterval(() => {
+      const packet = Buffer.alloc(1+8)
+      packet.writeUInt8(opCodes.Ping, 0)
+      packet.writeFloatBE(Date.now(), 1)
+      this.websocket.send(packet)
+    }, 2000)
   }
 
   close() {
@@ -121,6 +134,7 @@ class DatabaseConnection {
 
   addRequest(request) {
     this.waitingRequests.set(request.id, request)
+    if(debug) console.log("ADD REQUEST", request.id,"AT STATE", this.websocket.readyState)
     if(this.websocket.readyState == 1) { // 1 => open
       this.websocket.send(request.requestPacket)
     }
@@ -204,6 +218,7 @@ class DatabaseConnection {
     nameBuffer.copy(packet, 1+4+1)
     packet.writeUInt16BE(settingsBuffer.length, 1+4+1+nameBuffer.length)
     settingsBuffer.copy(packet, 1+4+1+nameBuffer.length+2)
+    if(debug) console.log("SEND CREATE DB", packet)
     return this.requestOkError(packet)
   }
 
@@ -217,18 +232,34 @@ class DatabaseConnection {
   }
 
   async createStore(databaseName, storeName) {
+    const storeKey = JSON.stringify([databaseName, storeName])
+    const createPromise = this.creatingStores.get(storeKey)
+    console.log("CREATING STORE PROMISE", createPromise)
+    if(createPromise) return createPromise
+    console.log("CREATING STORE", storeKey)
     const databaseNameBuffer = Buffer.from(databaseName)
     const storeNameBuffer = Buffer.from(storeName)
-    const packet = Buffer.alloc(1+4+1+databaseNameBuffer.length+1+storeNameBuffer.length)
+    const packet = Buffer.alloc(1 + 4 + 1 + databaseNameBuffer.length + 1 + storeNameBuffer.length)
     packet.writeUInt8(opCodes.CreateStore, 0)
-    packet.writeUInt8(databaseNameBuffer.length, 1+4)
-    databaseNameBuffer.copy(packet, 1+4+1)
-    packet.writeUInt8(storeNameBuffer.length, 1+4+1+databaseNameBuffer.length)
-    storeNameBuffer.copy(packet, 1+4+1+databaseNameBuffer.length+1)
-    return this.requestOkError(packet)
+    packet.writeUInt8(databaseNameBuffer.length, 1 + 4)
+    databaseNameBuffer.copy(packet, 1 + 4 + 1)
+    packet.writeUInt8(storeNameBuffer.length, 1 + 4 + 1 + databaseNameBuffer.length)
+    storeNameBuffer.copy(packet, 1 + 4 + 1 + databaseNameBuffer.length + 1)
+    const promise = this.requestOkError(packet)
+    this.creatingStores.set(storeKey, promise)
+    promise.then(() => {
+      this.creatingStores.delete(storeKey)
+      console.log("CREATED STORE", storeKey)
+    })
+    return promise
   }
 
   async deleteStore(databaseName, storeName) {
+    if(debug) console.log("DELETE STORE", databaseName, storeName)
+    for(let i = 0; i < this.openStores.length; i++) {
+      if(this.openStores[i] && this.openStores[i].storeName == storeName
+          && this.openStores[i].databaseName == databaseName) this.openStore[i] = null
+    }
     const databaseNameBuffer = Buffer.from(databaseName)
     const storeNameBuffer = Buffer.from(storeName)
     const packet = Buffer.alloc(1+4+1+databaseNameBuffer.length+1+storeNameBuffer.length)
@@ -245,25 +276,40 @@ class DatabaseConnection {
       if(this.openStores[i] && this.openStores[i].storeName == storeName
           && this.openStores[i].databaseName == databaseName) return i
     }
-    const currentPromise = this.openingStores.get(JSON.stringify(databaseName, storeName))
+    const storeKey = JSON.stringify([databaseName, storeName])
+    console.log("OPEN STORE!", storeKey)
+    const createPromise = this.creatingStores.get(storeKey)
+    if(createPromise) await createPromise
+    const currentPromise = this.openingStores.get(storeKey)
     if(currentPromise) return currentPromise
-    const openStoreId = this.openStores.length
-    const databaseNameBuffer = Buffer.from(databaseName)
-    const storeNameBuffer = Buffer.from(storeName)
-    const packet = Buffer.alloc(1+4+1+databaseNameBuffer.length+1+storeNameBuffer.length)
-    packet.writeUInt8(opCodes.OpenStore, 0)
-    packet.writeUInt8(databaseNameBuffer.length, 1+4)
-    databaseNameBuffer.copy(packet, 1+4+1)
-    packet.writeUInt8(storeNameBuffer.length, 1+4+1+databaseNameBuffer.length)
-    storeNameBuffer.copy(packet, 1+4+1+databaseNameBuffer.length+1)
-    this.openStores.push(null)
-    return this.requestOkError(packet).then(ok => {
-      this.openStores[openStoreId] = { storeName, databaseName }
-      return openStoreId
-    }).catch(error => {
-      this.openStores[openStoreId] = null
-      throw error
+    console.log("DO OPEN STORE!", storeKey)
+    const open = async () => {
+      const openStoreId = this.openStores.length
+      const databaseNameBuffer = Buffer.from(databaseName)
+      const storeNameBuffer = Buffer.from(storeName)
+      const packet = Buffer.alloc(1 + 4 + 1 + databaseNameBuffer.length + 1 + storeNameBuffer.length)
+      packet.writeUInt8(opCodes.OpenStore, 0)
+      packet.writeUInt8(databaseNameBuffer.length, 1 + 4)
+      databaseNameBuffer.copy(packet, 1 + 4 + 1)
+      packet.writeUInt8(storeNameBuffer.length, 1 + 4 + 1 + databaseNameBuffer.length)
+      storeNameBuffer.copy(packet, 1 + 4 + 1 + databaseNameBuffer.length + 1)
+      this.openStores.push(null)
+      return this.requestOkError(packet).then(ok => {
+        this.openStores[openStoreId] = { storeName, databaseName }
+        return openStoreId
+      }).catch(error => {
+        console.error("OPEN STORE FAILED!", error)
+        this.openStores[openStoreId] = null
+        throw error
+      })
+    }
+    const openPromise = open()
+    this.openingStores.set(storeKey, openPromise)
+    openPromise.then(() => {
+      console.log("STORE OPENED!", storeKey)
+      this.openingStores.delete(storeKey)
     })
+    return openPromise
   }
 
   async closeStore(databaseName, storeName) {
@@ -297,7 +343,7 @@ class DatabaseConnection {
     packet.writeUInt32BE(storeId, 1+4)
     packet.writeUInt16BE(keyBuffer.length, 1+4+4)
     keyBuffer.copy(packet, 1+4+4+2)
-    console.log("PUT VALUE", value, "BUF", valueBuffer)
+    if(debug) console.log("PUT", databaseName, "/", storeName , key, "VALUE", value, "BUF")
     packet.writeUInt32BE(valueBuffer.length, 1+4+4+2+keyBuffer.length)
     valueBuffer.copy(packet, 1+4+4+2+keyBuffer.length+4)
     return this.requestSingleResult(packet)
@@ -366,10 +412,11 @@ class DatabaseConnection {
       //console.log("HANDLE PACKET", packet)
       const opcode = packet.readInt8(0)
       if(opcode == opCodes.ResultPut) {
-        const keySize = packet.readUInt16BE(5)
-        const key = packet.toString('utf8', 7, 7 + keySize)
-        const valueSize = packet.readUInt32BE(7 + keySize)
-        const value = packet.toString('utf8', 7 + keySize + 4, 7 + keySize + 4 + valueSize)
+        const flags = packet.readUInt8(5) // ignore flags
+        const keySize = packet.readUInt16BE(6)
+        const key = packet.toString('utf8', 8, 8 + keySize)
+        const valueSize = packet.readUInt32BE(8 + keySize)
+        const value = packet.toString('utf8', 8 + keySize + 4, 8 + keySize + 4 + valueSize)
         results.push({ key, value })
         return true // need more data
       } else if(opcode == opCodes.ResultsDone) {
@@ -431,8 +478,9 @@ class DatabaseConnection {
     packet.writeUInt32BE(storeId, 1+4)
     packet.writeUInt16BE(keyBuffer.length, 1+4+4)
     keyBuffer.copy(packet, 1+4+4+2)
-    return this.rawRequest(packet, (packet) => {
-      //console.log("HANDLE PACKET", packet)
+    if(debug) console.log("OBSERVE KEY", key)
+    const requestId = this.rawRequest(packet, (packet) => {
+      if(debug) console.log("HANDLE OBSERVE PACKET", packet)
       const opcode = packet.readInt8(0)
       if(opcode == opCodes.Result) {
         return onValue(packet.toString('utf8', 5))
@@ -441,19 +489,24 @@ class DatabaseConnection {
       } else if(opcode == opCodes.Error) {
         const error = packet.toString('utf8', 5)
         onError(error)
+        return true
       } else {
         console.error("Unknown opcode", opCodeToString[opcode])
         onError("Unknown opcode: " + opCodeToString[opcode])
       }
+      process.exit(1)
       return false// finish waiting
     })
+    //console.log("OBSERVE", key, "=>", requestId)
+    return requestId
   }
 
   async observeRange(databaseName, storeName, range, onResultPut, onChanges, onError) {
     const storeId = await this.openStore(databaseName, storeName)
     const packet = this.rangePacket(storeId, opCodes.ObserveRange, range)
+    if(debug) console.log("OBSERVE RANGE", databaseName, storeName, range)
     return this.rawRequest(packet, (packet) => {
-      console.log("OBSERVE RANGE HANDLE PACKET", packet)
+      if(debug) console.log("OBSERVE RANGE HANDLE PACKET", packet)
       const opcode = packet.readInt8(0)
       if(opcode == opCodes.ResultPut) {
         const flags = packet.readUInt8(5)
@@ -485,7 +538,7 @@ class DatabaseConnection {
     const storeId = await this.openStore(databaseName, storeName)
     const packet = this.rangePacket(storeId, opCodes.ObserveCount, range)
     return this.rawRequest(packet, (packet) => {
-      console.log("OBSERVE RANGE HANDLE PACKET", packet)
+      if(debug) console.log("OBSERVE RANGE HANDLE PACKET", packet)
       const opcode = packet.readInt8(0)
       if(opcode == opCodes.ResultCount) {
         const count = packet.readUInt32BE(5)
@@ -502,10 +555,11 @@ class DatabaseConnection {
   }
 
   async unobserve(requestId) {
-    console.log("UNOBSERVE", requestId, this.waitingRequests)
+    //console.log("UNOBSERVE", requestId)
     if(!this.waitingRequests.get(requestId)) {
       throw new Error("unobserve of not observed! "+requestId)
     }
+    if(debug) console.error("REQUEST UNOBSERVED", requestId)
     this.waitingRequests.delete(requestId)
     const packet = Buffer.alloc(1+4)
     packet.writeUInt8(opCodes.Unobserve, 0)
@@ -538,7 +592,7 @@ class Store {
     if(!id) throw new Error("id is required")
     const observableValue = new ReactiveDao.ObservableValue()
     const onValue = (value) => {
-      console.log("OBJ", id, "ON VALUE", value)
+      if(debug) console.log("OBJ", id, "ON VALUE", value)
       observableValue.set(value && { id, ...JSON.parse(value) })
       return !observableValue.isDisposed()
     }
@@ -550,7 +604,9 @@ class Store {
     const oldDispose = observableValue.dispose
     const oldRespawn = observableValue.respawn
     observableValue.dispose = () => {
-      this.connection.unobserve(observableValue.requestId)
+      //console.trace("DISPOSE!")
+      //console.log("DISPOSE", observableValue.requestId, this.disposed)
+      //if(!this.disposed) this.connection.unobserve(observableValue.requestId)
       oldDispose.apply(this)
     }
     observableValue.respawn = async () => {
@@ -577,6 +633,10 @@ class Store {
     let results = []
     let changesMode = false
     const onResultPut = (found, last, key, value) => {
+      if(observableList.isDisposed()) {
+        console.error("PUT WHEN DISPOSED!", range, found, last, key, value)
+        return
+      }
       if(changesMode) {
         if(found) {
           if(last) {
@@ -596,18 +656,22 @@ class Store {
       return !observableList.isDisposed()
     }
     const onChanges = () => {
+      if(observableList.isDisposed()) console.error("CHANGES WHEN DISPOSED!")
       changesMode = true
       observableList.set(results)
-      results = null
+      results = []
       return !observableList.isDisposed()
     }
     const oldDispose = observableList.dispose
     const oldRespawn = observableList.respawn
     observableList.dispose = () => {
       this.connection.unobserve(observableList.requestId)
+      //console.log("UNOBSERVE STATE", observableList.list)
+      //console.trace("UNOBSERVE")
       oldDispose.apply(this)
     }
     observableList.respawn = async () => {
+      changesMode = false
       observableList.requestId = await this.connection.observeRange(this.databaseName, this.storeName, range,
           onResultPut, onChanges)
       oldRespawn.apply(this)
@@ -622,19 +686,20 @@ class Store {
   countObservable(range) {
     const observableValue = new ReactiveDao.ObservableValue()
     const onValue = (value) => {
-      console.log("COUNT", range, "ON VALUE", value)
+      if(debug) console.log("COUNT", range, "ON VALUE", value)
       observableValue.set(value)
       return !observableValue.isDisposed()
     }
     const onError = (error) => {
-      console.log("COUNT", range, "ON ERROR", error)
+      if(debug) console.log("COUNT", range, "ON ERROR", error)
       observableValue.error(error)
       return !observableValue.isDisposed()
     }
     const oldDispose = observableValue.dispose
     const oldRespawn = observableValue.respawn
     observableValue.dispose = () => {
-      this.connection.unobserve(observableValue.requestId)
+      if(observableValue.requestId >= 0) this.connection.unobserve(observableValue.requestId)
+      observableValue.requestId = -1
       oldDispose.apply(this)
     }
     observableValue.respawn = async () => {
@@ -662,14 +727,17 @@ class Store {
   async put(object) {
     const id = object.id
     if(typeof id != 'string') throw new Error(`ID is not string: ${JSON.stringify(id)}`)
+    if(debug) console.log("PUT!!!", object)
     if(!id) throw new Error("ID must not be empty string!")
-    return this.connection.put(this.databaseName, this.storeName, id,
+    const putResult = await this.connection.put(this.databaseName, this.storeName, id,
         JSON.stringify({ ...object, id: undefined }))
+    return putResult ? { id, ...JSON.parse(putResult) } : null
   }
 
   async delete(id) {
     if(!id) throw new Error("ID must not be empty string!")
-    return this.connection.delete(this.databaseName, this.storeName, id)
+    const deleteResult = await this.connection.delete(this.databaseName, this.storeName, id)
+    return deleteResult ? { id, ...JSON.parse(deleteResult) } : null
   }
 
   async close() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/db-store-observable-db",
-  "version": "0.1.0",
+  "version": "0.5.2",
   "description": "Database backend protocol",
   "main": "index.js",
   "scripts": {
@@ -28,5 +28,6 @@
     "bufferutil": "^4.0.3",
     "utf-8-validate": "^5.0.4",
     "ws": "^7.4.5"
-  }
+  },
+  "gitHead": "af756efb8db9ba90d79774786201cffe9c2162cf"
 }

package/tests/store-non-reactive.js

@@ -23,7 +23,7 @@ test("store non-reactive properties", t => {
   })
 
   t.test("non reactive operations", async t => {
-    t.plan(24)
+    t.plan(25)
 
     t.test("put value", async t => {
       t.plan(1)
@@ -78,6 +78,13 @@ test("store non-reactive properties", t => {
       t.deepEqual(values, 3, 'range count' )
     })
 
+    t.test("count all", async t => {
+      t.plan(1)
+      let values = await store.countGet({ })
+      t.deepEqual(values, 3, 'full count' )
+    })
+
+
     t.test("get reverse range [c,a]", async t => {
       t.plan(1)
       let values = await store.rangeGet({ gte: 'a', lte: 'c', reverse: true })