@live-change/db 0.3.59 → 0.3.63

package/lib/ChangeStream.js

@@ -5,25 +5,29 @@ class ChangeStream {
     throw new Error("abstract method - not implemented")
   }
   to(output) {
-    this.onChange((obj, oldObj, id, timestamp) => output.change(obj, oldObj, id, timestamp))
+    return this.onChange((obj, oldObj, id, timestamp) => output.change(obj, oldObj, id, timestamp))
   }
   filter(func) {
     const pipe = new ChangeStreamPipe()
-    this.onChange((obj, oldObj, id, timestamp) =>
+    const observerPromise = this.onChange((obj, oldObj, id, timestamp) =>
         pipe.change(obj && func(obj) ? obj : null, oldObj && func(oldObj) ? oldObj : null, id, timestamp))
+    pipe.master = this
+    pipe.observerPromise = observerPromise
     return pipe
   }
   map(func) {
     const pipe = new ChangeStreamPipe()
-    this.onChange((obj, oldObj, id, timestamp) =>
+    const observerPromise = this.onChange((obj, oldObj, id, timestamp) =>
         pipe.change(obj && func(obj), oldObj && func(oldObj), id, timestamp))
+    pipe.master = this
+    pipe.observerPromise = observerPromise
     return pipe
   }
   indexBy(func) {
     const pipe = new ChangeStreamPipe()
-    this.onChange((obj, oldObj, id, timestamp) => {
+    const observerPromise = this.onChange((obj, oldObj, id, timestamp) => {
       const indList = obj && func(obj)
-      const oldIndList = oldObj && func(obj)
+      const oldIndList = oldObj && func(oldObj)
       const ind = indList && indList.map(v => JSON.stringify(v)).join(':')+'_'+id
       const oldInd = oldIndList && oldIndList.map(v => JSON.stringify(v)).join(':')+'_'+id
       if(ind == oldInd) return // no index change, ignore
@@ -34,6 +38,8 @@ class ChangeStream {
         pipe.change(null, { id: oldInd, to: id }, oldInd, timestamp)
       }
     })
+    pipe.master = this
+    pipe.observerPromise = observerPromise
     return pipe
   }
 }
@@ -45,6 +51,14 @@ class ChangeStreamPipe extends ChangeStream {
   }
   onChange(cb) {
     this.callbacks.push(cb)
+    return cb
+  }
+  async unobserve(cb) {
+    const cbIndex = this.callbacks.indexOf(cb)
+    if(cbIndex == -1) throw new Error("observer not found")
+    if(this.callbacks.length == 0) {
+      this.master.unobservePromise = await this.observerPromise
+    }
   }
   async change(obj, oldObj, id, timestamp) {
     for(const callback of this.callbacks) await callback(obj, oldObj, id, timestamp)

package/lib/Database.js

@@ -8,7 +8,7 @@ const getRandomValues = require('get-random-values')
 const ReactiveDao = require("@live-change/dao")
 
 class Database {
-  constructor(config, storeFactory, saveConfig, deleteStore, name) {
+  constructor(config, storeFactory, saveConfig, deleteStore, name, createScriptContext) {
     this.name = name
     this.config = {
       tables: {},
@@ -23,6 +23,7 @@ class Database {
     this.tables = new Map()
     this.logs = new Map()
     this.indexes = new Map()
+    this.createScriptContext = createScriptContext
 
     this.configObservable = new ReactiveDao.ObservableValue(JSON.parse(JSON.stringify(this.config)))
     this.tablesListObservable = new ReactiveDao.ObservableList(Object.keys(this.config.tables))
@@ -92,7 +93,7 @@ class Database {
     this.tablesListObservable.remove(name)
     this.tables.delete(name)
   }
-  
+
   renameTable(name, newName) {
     if(this.config.tables[newName]) throw new Error(`Table ${newName} already exists`)
     const table = this.table(name)
@@ -226,25 +227,38 @@ class Database {
     return summary
   }
 
+  async openIndex(name) {
+    let config = this.config.indexes[name]
+    if(!config) {
+      console.log("INDEX", name, "NOT EXISTS - WAITING!")
+      await new Promise(r => setTimeout(r, 500))
+      config = this.config.indexes[name]
+    }
+    let index, code
+    if(!config) throw new Error(`Index ${name} not found`)
+    code = config.code
+    const params = config.parameters
+    index = new Index(this, name, code, params, config)
+    try {
+      console.log("STARTING INDEX", name)
+      await index.startIndex()
+      console.log("STARTED INDEX", name)
+    } catch(error) {
+      console.error("INDEX", name, "ERROR", error, "CODE:\n", code)
+      console.error("DELETING INDEX", name)
+      delete this.config.indexes[name]
+      this.indexesListObservable.remove(name)
+      if(this.onAutoRemoveIndex && config) this.onAutoRemoveIndex(name, config.uid)
+      await this.saveConfig(this.config)
+      throw error
+    }
+    this.indexes.set(name, index)
+    return index
+  }
   async index(name) {
     let index = this.indexes.get(name)
     if(!index) {
-      const config = this.config.indexes[name]
-      if(!config) throw new Error(`Index ${name} not found`)
-      const code = config.code
-      const params = config.parameters
-      index = new Index(this, name, code, params, config)
-      try {
-        await index.startIndex()
-      } catch(error) {
-        console.error("INDEX", name, "ERROR", error, "CODE:\n", index.code)
-        console.error("DELETING INDEX", name)
-        delete this.config.indexes[name]
-        this.indexesListObservable.remove(name)
-        if(this.onAutoRemoveIndex) this.onAutoRemoveIndex(name, config.uid)
-        await this.saveConfig(this.config)
-        throw error
-      }
+      index = this.openIndex(name)
       this.indexes.set(name, index)
     }
     return index

package/lib/Index.js

@@ -1,7 +1,6 @@
 const IntervalTree = require('node-interval-tree').default
 const ReactiveDao = require("@live-change/dao")
 const Table = require('./Table.js')
-const ScriptContext = require('./ScriptContext.js')
 const queryGet = require('./queryGet.js')
 const profileLog = require('./profileLog.js')
 const queryObservable = require('./queryObservable.js')
@@ -64,6 +63,7 @@ class TableReader extends ChangeStream {
     this.opLogReader = opLogReader
     this.prefix = prefix
     this.table = table
+    Promise.resolve(this.table).then(t=> {if(!t) throw new Error("TABLE NOT FOUND!!!")})
     this.isLog = isLog
     this.objectReaders = new Map()
     this.rangeReaders = new Map()
@@ -86,7 +86,7 @@ class TableReader extends ChangeStream {
         firstFull = 0
       }
     }, 1000)*/
-/*    if(this.prefix == 'table_triggers'){
+/*    //if(this.prefix == 'table_triggers'){
       let triggersDataJson = ""
       setInterval(() => {
         if(JSON.stringify(this.opLogObservable && this.opLogObservable.list) != triggersDataJson) {
@@ -222,7 +222,7 @@ class TableReader extends ChangeStream {
     }
   }
   async readTo(endKey) {
-    //console.log("RT", endKey, "IN", this.opLogBuffer)
+    //if(this.prefix == 'table_triggers') console.log("RT", endKey, "IN", this.opLogBuffer)
     let lastKey = null
     while(this.opLogBuffer[0] && this.opLogBuffer[0].id <= endKey) {
       const next = this.opLogBuffer.shift()
@@ -231,7 +231,7 @@ class TableReader extends ChangeStream {
         await this.change(next, null, next, next.id)
       } else {
         const op = next.operation
-        //console.log("HANDLE OP LOG OPERATION", next)
+        //if(this.prefix == 'table_triggers') console.log("HANDLE OP LOG OPERATION", next)
         if(op) {
           if(op.type == 'put') {
             await this.change(op.object, op.oldObject, op.object.id, next.id)
@@ -299,15 +299,16 @@ class OpLogReader {
 
   handleSignal() {
     if(this.readingMore) {
-      //console.log("STORE SIGNAL")
+      //if(this.indexName == 'triggers_new') console.log("STORE SIGNAL")
       this.gotSignals = true
     } else {
-      //console.log("READ MORE ON SIGNAL")
+      //if(this.indexName == 'triggers_new') console.log("READ MORE ON SIGNAL")
       this.readMore()
     }
   }
   async readMore() {
     this.readingMore = true
+    //if(this.indexName == 'triggers_new') console.log("READING TRIGGERS STARTED!")
     do {
       while(true) {
         this.gotSignals = false
@@ -317,9 +318,9 @@ class OpLogReader {
         let possibleNextKeys = await Promise.all(
             this.tableReaders.map(async tr => ({ reader: tr, key: await tr.nextKey() }))
         )
-        //console.log("GOT NEXT KEYS")
+        //if(this.indexName == 'triggers_new') console.log("GOT NEXT KEYS")
         if(this.disposed) return
-        //console.log("POSSIBLE NEXT KEYS", possibleNextKeys.map(({key, reader}) => [reader.prefix, key]))
+        //if(this.indexName == 'triggers_new') console.log("POSSIBLE NEXT KEYS", possibleNextKeys.map(({key, reader}) => [reader.prefix, key]))
         if(possibleNextKeys.length == 0) { /// It could happen when oplog is cleared
           return
         }
@@ -329,9 +330,9 @@ class OpLogReader {
             next = possibleKey
           }
         }
-        //console.log("NEXT KEY", next && next.reader && next.reader.prefix, next && next.key)
+        //if(this.indexName == 'triggers_new') console.log("NEXT KEY", next && next.reader && next.reader.prefix, next && next.key)
         const lastKey = '\xFF\xFF\xFF\xFF'
-        //console.log("NEXT", !!next, "KEY", next && next.key, lastKey)
+        //if(this.indexName == 'triggers_new') console.log("NEXT", !!next, "KEY", next && next.key, lastKey)
         if(!next || next.key == lastKey) break // nothing to read
         let otherReaderNext = null
         for(const possibleKey of possibleNextKeys) {
@@ -339,31 +340,38 @@ class OpLogReader {
               && (!otherReaderNext || possibleKey.key < otherReaderNext.key))
             otherReaderNext = possibleKey
         }
-        //console.log("OTHER READ NEXT", otherReaderNext && otherReaderNext.reader && otherReaderNext.reader.prefix,
-        //   otherReaderNext && otherReaderNext.key)
-        const readEnd = (otherReaderNext && otherReaderNext.key) // Read to next other reader key
+        //if(this.indexName == 'triggers_new')
+        //  console.log("OTHER READ NEXT", otherReaderNext && otherReaderNext.reader && otherReaderNext.reader.prefix,
+        //    otherReaderNext && otherReaderNext.key)
+        let readEnd = (otherReaderNext && otherReaderNext.key) // Read to next other reader key
             || (((''+(now - 1))).padStart(16, '0'))+':' // or to current timestamp
+        if(readEnd < next) {
+          readEnd = next+'\xff'
+        }
 
         if((next.key||'') < this.currentKey) {
           //debugger
           console.error("time travel", next.key, this.currentKey)
           //process.exit(1) /// TODO: do something about it!
         }
-        //console.log("CKN", this.currentKey, '=>', next.key)
+        //if(this.indexName == 'triggers_new') console.log("CKN", this.currentKey, '=>', next.key)
         this.currentKey = next.key
+        //if(this.indexName == 'triggers_new') console.log("READ TO", readEnd)
         const readKey = await next.reader.readTo(readEnd)
+        //if(this.indexName == 'triggers_new') console.log("READED")
         if(readKey) {
           if((readKey||'') < this.currentKey) {
             //debugger
             console.error("time travel", readKey, this.currentKey)
             //process.exit(1) /// TODO: do something about it!
           }
-          //console.log("CKR", this.currentKey, '=>', readKey)
+          //if(this.indexName == 'triggers_new') console.log("CKR", this.currentKey, '=>', readKey)
           this.currentKey = readKey
         }
       }
     } while(this.gotSignals)
     this.readingMore = false
+    //if(this.indexName == 'triggers_new') console.log("READING TRIGGERS FINISHED!")
   }
   dispose() {
     this.disposed = true
@@ -393,7 +401,7 @@ class IndexWriter {
     this.index.update(id, ops)
   }
   change(obj, oldObj) {
-    //console.log("INDEX WRITE", obj, oldObj)
+    //if(this.index.name == 'triggers_new') console.log("INDEX WRITE", obj, oldObj)
     if(obj) {
       if(oldObj && oldObj.id != obj.id) {
         this.index.delete(oldObj.id)
@@ -430,7 +438,7 @@ class Index extends Table {
   }
   async startIndex() {
     console.log("EXECUTING INDEX CODE", this.name)
-    this.scriptContext = new ScriptContext({
+    this.scriptContext = this.database.createScriptContext({
       /// TODO: script available routines
     })
     const queryFunction = this.scriptContext.run(this.code,`userCode:${this.database.name}/indexes/${this.name}`)

package/lib/ScriptContext.js

@@ -35,6 +35,7 @@ const defaultContext = {
     return new ChangeStreamPipe()
   },
   'performance': require('perf_hooks').performance,
+  constructor: null
 }
 
 const filenameRE = /scriptFile:(\d+):(\d+)\)$/g
@@ -47,9 +48,9 @@ class ScriptContext {
     }
     this.context = vm.createContext({ ...defaultContext, ...context, ...userContext })
 /*    vm.runInContext(`
-    (function() {      
-      const allowed = ${JSON.stringify(nativeGlobals.concat(Object.keys(userContext)))} 
-      const keys = Object.getOwnPropertyNames(this)    
+    (function() {
+      const allowed = ${JSON.stringify(nativeGlobals.concat(Object.keys(userContext)))}
+      const keys = Object.getOwnPropertyNames(this)
       keys.forEach((key) => {
         const item = this[key]
         if(!item) return

package/lib/queryObservable.js

@@ -91,7 +91,10 @@ class Reader extends ChangeStream {
   }
   async unobserve(observer) {
     const index = this.#observers.indexOf(observer)
-    if(index == -1) throw new Error("observer not found")
+    if(index == -1) {
+      console.error("OBSERVER NOT FOUND", observer)
+      throw new Error("observer not found")
+    }
     this.#observers.splice(index, 1)
     ;(await this.#observable).unobserve(observer)
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/db",
-  "version": "0.3.59",
+  "version": "0.3.63",
   "description": "Database with observable data for live queries",
   "main": "index.js",
   "scripts": {
@@ -21,24 +21,24 @@
   },
   "homepage": "https://github.com/live-change/db",
   "devDependencies": {
-    "@live-change/db-store-level": "^0.1.9",
+    "@live-change/db-store-level": "^0.1.14",
     "@live-change/db-store-lmdb": "^0.1.13",
     "encoding-down": "^6.3.0",
     "level-rocksdb": "^4.0.0",
     "leveldown": "^5.6.0",
     "levelup": "^4.4.0",
     "memdown": "^5.1.0",
+    "minimist": ">=1.2.3",
     "node-lmdb": "^0.8.0",
     "rimraf": "^3.0.2",
     "rocksdb": "^4.1.0",
-    "sockjs": "^0.3.20",
+    "sockjs": "^0.3.21",
     "subleveldown": "^4.1.4",
     "tape": "^4.13.3",
-    "websocket-extensions": ">=0.1.4",
-    "minimist": ">=1.2.3"
+    "websocket-extensions": ">=0.1.4"
   },
   "dependencies": {
-    "@live-change/dao": "^0.2.34",
+    "@live-change/dao": "^0.3.3",
     "get-random-values": "^1.2.2",
     "node-interval-tree": "^1.3.3"
   }