@live-change/framework 0.4.40 → 0.4.44

package/lib/App.js

@@ -9,19 +9,14 @@ const Service = require("./runtime/Service.js")
 
 const profileLog = require("./utils/profileLog.js")
 
-const RTCMSDao = require("./runtime/Dao.js")
+const Dao = require("./runtime/Dao.js")
 const ApiServer = require("./runtime/ApiServer.js")
 const SessionApiServer = require("./runtime/SessionApiServer.js")
 
-const { Client: ElasticSearch } = require('@elastic/elasticsearch')
-
-const AnalyticsWriter = require('./utils/AnalyticsWriter.js')
-
 const reverseRelationProcessor = require("./processors/reverseRelation.js")
 const indexListProcessor = require("./processors/indexList.js")
 const crudGenerator = require("./processors/crudGenerator.js")
 const draftGenerator = require("./processors/draftGenerator.js")
-const searchIndex = require("./processors/searchIndex.js")
 const daoPathView = require("./processors/daoPathView.js")
 const fetchView = require("./processors/fetchView.js")
 const accessControl = require("./processors/accessControl.js")
@@ -29,11 +24,14 @@ const autoValidation = require("./processors/autoValidation.js")
 const indexCode = require("./processors/indexCode.js")
 
 const databaseUpdater = require("./updaters/database.js")
-const elasticSearchUpdater = require("./updaters/elasticsearch.js")
 
 const accessControlFilter = require("./clientSideFilters/accessControlFilter.js")
 const clientSideFilter = require("./clientSideFilters/clientSideFilter.js")
 
+const commandExecutor = require("./processes/commandExecutor.js")
+const triggerExecutor = require("./processes/triggerExecutor.js")
+const eventListener = require('./processes/eventListener.js')
+
 const utils = require('./utils.js')
 
 class App {
@@ -46,26 +44,28 @@ class App {
     this.requestTimeout = (+env.DB_REQUEST_TIMEOUT) || 10*1000
 
     this.defaultProcessors = [
-        crudGenerator,
-        draftGenerator,
-        reverseRelationProcessor,
-        indexListProcessor,
-        daoPathView,
-        fetchView,
-        accessControl,
-        autoValidation,
-        indexCode
+      crudGenerator,
+      draftGenerator,
+      reverseRelationProcessor,
+      indexListProcessor,
+      daoPathView,
+      fetchView,
+      accessControl,
+      autoValidation,
+      indexCode
     ]
-    if(env.SEARCH_INDEX_PREFIX) this.defaultProcessors.push(searchIndex)
     this.defaultUpdaters = [
-        databaseUpdater
+      databaseUpdater
     ]
-    if(env.SEARCH_INDEX_PREFIX) this.defaultUpdaters.push(elasticSearchUpdater)
     this.defaultClientSideFilters = [
-        accessControlFilter,
-        clientSideFilter
+      accessControlFilter,
+      clientSideFilter
+    ]
+    this.defaultProcesses = [
+      commandExecutor,
+      triggerExecutor,
+      eventListener
     ]
-    this.defaultPath = "."
     const dbDao = new ReactiveDao(process.cwd()+' '+process.argv.join(' '), {
       remoteUrl: env.DB_URL || "http://localhost:9417/api/ws",
       protocols: {
@@ -98,22 +98,6 @@ class App {
     this.databaseName = env.DB_NAME || 'test'
   }
 
-  connectToSearch() {
-    if(!this.env.SEARCH_INDEX_PREFIX) throw new Error("ElasticSearch not configured")
-    if(this.search) return this.search
-    this.searchIndexPrefix = this.env.SEARCH_INDEX_PREFIX
-    this.search = new ElasticSearch({ node: this.env.SEARCH_URL || 'http://localhost:9200' })
-    //this.search.info(console.log)
-    return this.search
-  }
-
-  connectToAnalytics() {
-    if(!this.env.ANALYTICS_INDEX_PREFIX) throw new Error("ElasticSearch analytics not configured")
-    if(this.analytics) return this.analytics
-    this.analytics = new AnalyticsWriter(this.env.ANALYTICS_INDEX_PREFIX)
-    return this.analytics
-  }
-
   createServiceDefinition( definition ) {
     const config = this.config && this.config.services && this.config.services[definition.name]
     return new ServiceDefinition({ ...definition, config })
@@ -162,7 +146,8 @@ class App {
     await this.profileLog.end(profileOp)
   }
 
-  async startService( serviceDefinition, config ) {
+  async startService( serviceDefinition, config = {}) {
+    if(!config.processes) config.processes = this.defaultProcesses
     console.log("Starting service", serviceDefinition.name, "!")
     const profileOp = await this.profileLog.begin({
       operation: "startService", serviceName: serviceDefinition.name, config
@@ -170,14 +155,14 @@ class App {
     if(!(serviceDefinition instanceof ServiceDefinition))
       serviceDefinition = new ServiceDefinition(serviceDefinition)
     let service = new Service(serviceDefinition, this)
-    await service.start(config || {})
+    await service.start(config)
     console.log("service started", serviceDefinition.name, "!")
     await this.profileLog.end(profileOp)
     return service
   }
 
   async createReactiveDao( config, clientData ) {
-    return new RTCMSDao(config, clientData)
+    return new Dao(config, clientData)
   }
 
   async createApiServer( config ) {

package/lib/processes/commandExecutor.js

@@ -0,0 +1,74 @@
+const KeyBasedExecutionQueues = require('../utils/KeyBasedExecutionQueues.js')
+const CommandQueue = require('../utils/CommandQueue.js')
+const SingleEmitQueue = require('../utils/SingleEmitQueue.js')
+const SplitEmitQueue = require('../utils/SplitEmitQueue.js')
+
+async function startCommandExecutor(service, config) {
+  if(!config.runCommands) return
+
+  service.keyBasedExecutionQueues = service.keyBasedExecutionQueues || new KeyBasedExecutionQueues(r => r.key)
+  
+  service.commandQueue = new CommandQueue(service.dao, service.databaseName,
+      service.app.splitCommands ? `${service.name}_commands` : 'commands', service.name)
+  for (let actionName in service.actions) {
+    const action = service.actions[actionName]
+    if (action.definition.queuedBy) {
+      const queuedBy = action.definition.queuedBy
+      const keyFunction = typeof queuedBy == 'function' ? queuedBy : (
+          Array.isArray(queuedBy) ? (c) => JSON.stringify(queuedBy.map(k => c[k])) :
+              (c) => JSON.stringify(c[queuedBy]))
+      service.commandQueue.addCommandHandler(actionName, async (command) => {
+        const profileOp = await service.profileLog.begin({
+          operation: 'queueCommand', commandType: actionName,
+          commandId: command.id, client: command.client
+        })
+        const reportFinished = action.definition.waitForEvents ? 'command_' + command.id : undefined
+        const flags = {commandId: command.id, reportFinished}
+        const emit = service.app.splitEvents
+            ? new SplitEmitQueue(service, flags)
+            : new SingleEmitQueue(service, flags)
+        const routine = () => service.profileLog.profile({
+          operation: 'runCommand', commandType: actionName,
+          commandId: command.id, client: command.client
+        }, async () => {
+          const result = await service.app.assertTime('command ' + action.definition.name,
+              action.definition.timeout || 10000,
+              () => action.runCommand(command, (...args) => emit.emit(...args)), command)
+          const events = await emit.commit()
+          if (action.definition.waitForEvents)
+            await service.app.waitForEvents(reportFinished, events, action.definition.waitForEvents)
+          return result
+        })
+        routine.key = keyFunction(command)
+        const promise = service.keyBasedExecutionQueues.queue(routine)
+        await service.profileLog.endPromise(profileOp, promise)
+        return promise
+      })
+    } else {
+      service.commandQueue.addCommandHandler(actionName,
+          (command) => service.profileLog.profile({
+            operation: 'runCommand', commandType: actionName,
+            commandId: command.id, client: command.client
+          }, async () => {
+            const reportFinished = action.definition.waitForEvents ? 'command_' + command.id : undefined
+            const flags = {commandId: command.id, reportFinished}
+            const emit = service.app.splitEvents
+                ? new SplitEmitQueue(service, flags)
+                : new SingleEmitQueue(service, flags)
+            const result = await service.app.assertTime('command ' + action.definition.name,
+                action.definition.timeout || 10000,
+                () => action.runCommand(command, (...args) => emit.emit(...args)), command)
+            const events = await emit.commit()
+            if (action.definition.waitForEvents)
+              await service.app.waitForEvents(reportFinished, events, action.definition.waitForEvents)
+            return result
+          })
+      )
+
+    }
+  }
+
+  service.commandQueue.start()
+}
+
+module.exports = startCommandExecutor

package/lib/processes/eventListener.js

@@ -0,0 +1,40 @@
+const EventSourcing = require('../utils/EventSourcing.js')
+
+async function startEventListener(service, config) {
+  if(!config.handleEvents) return
+    
+  if(service.app.splitEvents) {
+    service.eventSourcing = new EventSourcing(service.dao, service.databaseName,
+        'events_'+service.name, service.name,
+        { filter: (event) => event.service == service.name })
+  } else {
+    service.eventSourcing = new EventSourcing(service.dao, service.databaseName,
+        'events', service.name,
+        { filter: (event) => event.service == service.name })
+  }
+
+
+  for (let eventName in service.events) {
+    const event = service.events[eventName]
+    service.eventSourcing.addEventHandler(eventName, async (ev, bucket) => {
+      return await service.profileLog.profile({ operation: "handleEvent", eventName, id: ev.id,
+            bucketId: bucket.id, triggerId: bucket.triggerId, commandId: bucket.commandId },
+          () => {
+            console.log("EXECUTING EVENT", ev)
+            return event.execute(ev, bucket)
+          }
+      )
+    })
+    service.eventSourcing.onBucketEnd = async (bucket, handledEvents) => {
+      if(bucket.reportFinished && handledEvents.length > 0) {
+        await service.dao.request(['database', 'update'], service.databaseName, 'eventReports', bucket.reportFinished,[
+          { op: "mergeSets", property: 'finished', values: handledEvents.map(ev => ({ id: ev.id, type: ev.type })) }
+        ])
+      }
+    }
+  }
+
+  service.eventSourcing.start()
+}
+
+module.exports = startEventListener

package/lib/processes/triggerExecutor.js

@@ -0,0 +1,93 @@
+const KeyBasedExecutionQueues = require('../utils/KeyBasedExecutionQueues.js')
+const CommandQueue = require('../utils/CommandQueue.js')
+const SingleEmitQueue = require('../utils/SingleEmitQueue.js')
+const SplitEmitQueue = require('../utils/SplitEmitQueue.js')
+
+async function startTriggerExecutor(service, config) {
+  if(!config.runCommands) return
+  
+  service.keyBasedExecutionQueues = service.keyBasedExecutionQueues || new KeyBasedExecutionQueues(r => r.key)
+
+  await service.dao.request(['database', 'createTable'], service.databaseName, 'triggerRoutes').catch(e => 'ok')
+
+  service.triggerQueue = new CommandQueue(service.dao, service.databaseName,
+      service.app.splitTriggers ? `${service.name}_triggers` : 'triggers', service.name )
+  for (let triggerName in service.triggers) {
+    const trigger = service.triggers[triggerName]
+    await service.dao.request(['database', 'put'], service.databaseName, 'triggerRoutes',
+        { id: triggerName + '=>' + service.name, trigger: triggerName, service: service.name })
+    if(trigger.definition.queuedBy) {
+      const queuedBy = trigger.definition.queuedBy
+      const keyFunction = typeof queuedBy == 'function' ? queuedBy : (
+          Array.isArray(queuedBy) ? (c) => JSON.stringify(queuedBy.map(k=>c[k])) :
+              (c) => JSON.stringify(c[queuedBy]) )
+      service.triggerQueue.addCommandHandler(triggerName, async (trig) => {
+        const profileOp = await service.profileLog.begin({ operation: 'queueTrigger', triggerType: triggerName,
+          triggerId: trig.id, by: trig.by })
+        console.log("QUEUED TRIGGER STARTED", trig)
+        const reportFinished = trigger.definition.waitForEvents ? 'trigger_'+trig.id : undefined
+        const flags = { triggerId: trig.id, reportFinished }
+        const emit = service.app.splitEvents
+            ? new SplitEmitQueue(service, flags)
+            : new SingleEmitQueue(service, flags)
+        const routine = () => service.profileLog.profile({ operation: 'runTrigger', triggerType: triggerName,
+          commandId: trig.id, by: trig.by }, async () => {
+          let result
+          try {
+            console.log("TRIGGERED!!", trig)
+            result = await service.app.assertTime('trigger '+trigger.definition.name,
+                trigger.definition.timeout || 10000,
+                () => trigger.execute(trig, (...args) => emit.emit(...args)), trig)
+            console.log("TRIGGER DONE!", trig)
+          } catch (e) {
+            console.error(`TRIGGER ${triggerName} ERROR`, e.stack)
+            throw e
+          }
+          const events = await emit.commit()
+          if(trigger.definition.waitForEvents)
+            await service.app.waitForEvents(reportFinished, events, trigger.definition.waitForEvents)
+          return result
+        })
+        try {
+          routine.key = keyFunction(trig)
+        } catch(e) {
+          console.error("QUEUE KEY FUNCTION ERROR", e)
+        }
+        console.log("TRIGGER QUEUE KEY", routine.key)
+        const promise = service.keyBasedExecutionQueues.queue(routine)
+        await service.profileLog.endPromise(profileOp, promise)
+        return promise
+      })
+    } else {
+      service.triggerQueue.addCommandHandler(triggerName,
+          (trig) => service.profileLog.profile({ operation: 'runTrigger', triggerType: triggerName,
+            commandId: trig.id, by: trig.by }, async () => {
+            console.log("NOT QUEUED TRIGGER STARTED", trig)
+            const reportFinished = trigger.definition.waitForEvents ? 'trigger_'+trig.id : undefined
+            const flags = { triggerId: trig.id, reportFinished }
+            const emit = service.app.splitEvents
+                ? new SplitEmitQueue(service, flags)
+                : new SingleEmitQueue(service, flags)
+            let result
+            try {
+              result = await service.app.assertTime('trigger '+trigger.definition.name,
+                  trigger.definition.timeout || 10000,
+                  () => trigger.execute(trig, (...args) => emit.emit(...args)), trig)
+              console.log("TRIGGER DONE!", trig)
+            } catch (e) {
+              console.error(`TRIGGER ${triggerName} ERROR`, e.stack)
+              throw e
+            }
+            const events = await emit.commit()
+            if(trigger.definition.waitForEvents)
+              await service.app.waitForEvents(reportFinished, events, trigger.definition.waitForEvents)
+            return result
+          })
+      )
+    }
+  }
+
+  service.triggerQueue.start()
+}
+
+module.exports = startTriggerExecutor

package/lib/runtime/Service.js

@@ -5,12 +5,6 @@ const View = require("./View.js")
 const Action = require("./Action.js")
 const EventHandler = require("./EventHandler.js")
 const TriggerHandler = require("./TriggerHandler.js")
-const SearchIndexer = require("./SearchIndexer.js")
-const ReactiveDao = require("@live-change/dao")
-
-const EventSourcing = require('../utils/EventSourcing.js')
-const CommandQueue = require('../utils/CommandQueue.js')
-const KeyBasedExecutionQueues = require('../utils/KeyBasedExecutionQueues.js')
 
 class Service {
 
@@ -79,11 +73,7 @@ class Service {
     //console.log("DEFN", this.definition)
     //console.log("DEFN JSON", JSON.stringify(this.definition.toJSON(), null, "  "))
 
-    let promises = []
-    if(config.runCommands) promises.push(this.startCommandExecutor())
-    if(config.handleEvents) promises.push(this.startEventListener())
-    if(config.indexSearch) promises.push(this.startSearchIndexer())
-
+    let promises = config.processes.map(proc => proc(this, config))
     await Promise.all(promises)
 
     //if(config.startEventListener) this.startEventListener()
@@ -99,335 +89,7 @@ class Service {
     return this.app.triggerService(service, data)
   }
 
-  async startEventListener() {
-    if(this.app.splitEvents) {
-      this.eventSourcing = new EventSourcing(this.dao, this.databaseName,
-          'events_'+this.name, this.name,
-          { filter: (event) => event.service == this.name })
-    } else {
-      this.eventSourcing = new EventSourcing(this.dao, this.databaseName,
-          'events', this.name,
-          { filter: (event) => event.service == this.name })
-    }
-
-
-    for (let eventName in this.events) {
-      const event = this.events[eventName]
-      this.eventSourcing.addEventHandler(eventName, async (ev, bucket) => {
-        return await this.profileLog.profile({ operation: "handleEvent", eventName, id: ev.id,
-          bucketId: bucket.id, triggerId: bucket.triggerId, commandId: bucket.commandId },
-            () => {
-              console.log("EXECUTING EVENT", ev)
-              return event.execute(ev, bucket)
-            }
-        )
-      })
-      this.eventSourcing.onBucketEnd = async (bucket, handledEvents) => {
-        if(bucket.reportFinished && handledEvents.length > 0) {
-          await this.dao.request(['database', 'update'], this.databaseName, 'eventReports', bucket.reportFinished,[
-            { op: "mergeSets", property: 'finished', values: handledEvents.map(ev => ({ id: ev.id, type: ev.type })) }
-          ])
-        }
-      }
-    }
-
-    this.eventSourcing.start()
-  }
-
-  async startCommandExecutor() {
-    this.commandQueue = new CommandQueue(this.dao, this.databaseName,
-        this.app.splitCommands ? `${this.name}_commands` : 'commands', this.name)
-    this.keyBasedCommandQueues = new KeyBasedExecutionQueues(r => r.key)
-    for (let actionName in this.actions) {
-      const action = this.actions[actionName]
-      if(action.definition.queuedBy) {
-        const queuedBy = action.definition.queuedBy
-        const keyFunction = typeof queuedBy == 'function' ? queuedBy : (
-            Array.isArray(queuedBy) ? (c) => JSON.stringify(queuedBy.map(k=>c[k])) :
-                (c) => JSON.stringify(c[queuedBy]) )
-        this.commandQueue.addCommandHandler(actionName, async (command) => {
-          const profileOp = await this.profileLog.begin({ operation: 'queueCommand', commandType: actionName,
-            commandId: command.id, client: command.client })
-          const reportFinished = action.definition.waitForEvents ? 'command_'+command.id : undefined
-          const flags = { commandId: command.id, reportFinished }
-          const emit = this.app.splitEvents
-              ? new SplitEmitQueue(this, flags)
-              : new SingleEmitQueue(this, flags)
-          const routine = () => this.profileLog.profile({ operation: 'runCommand', commandType: actionName,
-            commandId: command.id, client: command.client }, async () => {
-              const result = await this.app.assertTime('command '+action.definition.name,
-                  action.definition.timeout || 10000,
-                  () => action.runCommand(command, (...args) => emit.emit(...args)), command)
-              const events = await emit.commit()
-              if(action.definition.waitForEvents)
-                await this.app.waitForEvents(reportFinished, events, action.definition.waitForEvents)
-              return result
-            })
-          routine.key = keyFunction(command)
-          const promise = this.keyBasedCommandQueues.queue(routine)
-          await this.profileLog.endPromise(profileOp, promise)
-          return promise
-        })
-      } else {
-        this.commandQueue.addCommandHandler(actionName,
-            (command) => this.profileLog.profile({ operation: 'runCommand', commandType: actionName,
-              commandId: command.id, client: command.client }, async () => {
-            const reportFinished = action.definition.waitForEvents ? 'command_'+command.id : undefined
-            const flags = { commandId: command.id, reportFinished }
-            const emit = this.app.splitEvents
-                ? new SplitEmitQueue(this, flags)
-                : new SingleEmitQueue(this, flags)
-            const result = await this.app.assertTime('command '+action.definition.name,
-                action.definition.timeout || 10000,
-                () => action.runCommand(command, (...args) => emit.emit(...args)), command)
-            const events = await emit.commit()
-            if(action.definition.waitForEvents)
-              await this.app.waitForEvents(reportFinished, events, action.definition.waitForEvents)
-            return result
-          })
-        )
-
-      }
-    }
-
-    await this.dao.request(['database', 'createTable'], this.databaseName, 'triggerRoutes').catch(e => 'ok')
-
-    this.triggerQueue = new CommandQueue(this.dao, this.databaseName,
-        this.app.splitTriggers ? `${this.name}_triggers` : 'triggers', this.name )
-    this.keyBasedTriggerQueues = new KeyBasedExecutionQueues(r => r.key)
-    for (let triggerName in this.triggers) {
-      const trigger = this.triggers[triggerName]
-      await this.dao.request(['database', 'put'], this.databaseName, 'triggerRoutes',
-          { id: triggerName + '=>' + this.name, trigger: triggerName, service: this.name })
-      if(trigger.definition.queuedBy) {
-        const queuedBy = trigger.definition.queuedBy
-        const keyFunction = typeof queuedBy == 'function' ? queuedBy : (
-            Array.isArray(queuedBy) ? (c) => JSON.stringify(queuedBy.map(k=>c[k])) :
-                (c) => JSON.stringify(c[queuedBy]) )
-        this.triggerQueue.addCommandHandler(triggerName, async (trig) => {
-          const profileOp = await this.profileLog.begin({ operation: 'queueTrigger', triggerType: triggerName,
-            triggerId: trig.id, by: trig.by })
-          console.log("QUEUED TRIGGER STARTED", trig)
-          const reportFinished = trigger.definition.waitForEvents ? 'trigger_'+trig.id : undefined
-          const flags = { triggerId: trig.id, reportFinished }
-          const emit = this.app.splitEvents
-              ? new SplitEmitQueue(this, flags)
-              : new SingleEmitQueue(this, flags)
-          const routine = () => this.profileLog.profile({ operation: 'runTrigger', triggerType: triggerName,
-            commandId: trig.id, by: trig.by }, async () => {
-              let result
-              try {
-                console.log("TRIGGERED!!", trig)
-                result = await this.app.assertTime('trigger '+trigger.definition.name,
-                    trigger.definition.timeout || 10000,
-                    () => trigger.execute(trig, (...args) => emit.emit(...args)), trig)
-                console.log("TRIGGER DONE!", trig)
-              } catch (e) {
-                console.error(`TRIGGER ${triggerName} ERROR`, e.stack)
-                throw e
-              }
-              const events = await emit.commit()
-              if(trigger.definition.waitForEvents)
-                await this.app.waitForEvents(reportFinished, events, trigger.definition.waitForEvents)
-              return result
-            })
-          try {
-            routine.key = keyFunction(trig)
-          } catch(e) {
-            console.error("QUEUE KEY FUNCTION ERROR", e)
-          }
-          console.log("TRIGGER QUEUE KEY", routine.key)
-          const promise = this.keyBasedTriggerQueues.queue(routine)
-          await this.profileLog.endPromise(profileOp, promise)
-          return promise
-        })
-      } else {
-        this.triggerQueue.addCommandHandler(triggerName,
-        (trig) => this.profileLog.profile({ operation: 'runTrigger', triggerType: triggerName,
-          commandId: trig.id, by: trig.by }, async () => {
-            console.log("NOT QUEUED TRIGGER STARTED", trig)
-            const reportFinished = trigger.definition.waitForEvents ? 'trigger_'+trig.id : undefined
-            const flags = { triggerId: trig.id, reportFinished }
-            const emit = this.app.splitEvents
-                ? new SplitEmitQueue(this, flags)
-                : new SingleEmitQueue(this, flags)
-            let result
-            try {
-              result = await this.app.assertTime('trigger '+trigger.definition.name,
-                  trigger.definition.timeout || 10000,
-                  () => trigger.execute(trig, (...args) => emit.emit(...args)), trig)
-              console.log("TRIGGER DONE!", trig)
-            } catch (e) {
-              console.error(`TRIGGER ${triggerName} ERROR`, e.stack)
-              throw e
-            }
-            const events = await emit.commit()
-            if(trigger.definition.waitForEvents)
-              await this.app.waitForEvents(reportFinished, events, trigger.definition.waitForEvents)
-            return result
-          })
-        )
-      }
-    }
-
-    this.commandQueue.start()
-    this.triggerQueue.start()
-  }
-
-  async startSearchIndexer() {
-    let anyIndex = false
-    for(const name in this.models) if(this.models[name].definition.searchIndex) anyIndex = true
-    for(const name in this.indexes) if(this.indexes[name].definition.searchIndex) anyIndex = true
-    if(!anyIndex) {
-      console.log("not starting search indexer - nothing to index!")
-      return
-    }
-    console.log("starting search indexer!")
-    await this.dao.request(['database', 'createTable'], this.databaseName, 'searchIndexes').catch(e => 'ok')
-
-    this.searchIndexers = []
-
-    const elasticsearch = this.app.connectToSearch()
-
-    for(const modelName in this.models) {
-      const model = this.models[modelName]
-      const indexName = model.definition.searchIndex
-      if(!indexName) continue
-      const indexer = new SearchIndexer(
-          this.dao, this.databaseName, 'Table', model.tableName, elasticsearch, indexName, model.definition
-      )
-      this.searchIndexers.push(indexer)
-    }
-
-    for(const indexName in this.indexes) {
-      const index = this.indexes[indexName]
-      const indexName = index.definition.searchIndex
-      if(!indexName) continue
-      const indexer = new SearchIndexer(
-          this.dao, this.databaseName, 'Index', model.tableName, elasticsearch, indexName, index.definition
-      )
-      this.searchIndexers.push(indexer)
-    }
-
-    const promises = []
-    for(const searchIndexer of this.searchIndexers) {
-      promises.push(this.profileLog.profile({
-        operation: "startIndexer", serviceName: this.name, indexName: searchIndexer.indexName
-      }, () => searchIndexer.start()))
-    }
-    await Promise.all(promises)
-    console.log("search indexer started!")
-  }
-
 }
 
-class SplitEmitQueue {
-  constructor(service, flags = {}) {
-    this.service = service
-    this.flags = flags
-    this.emittedEvents = new Map()
-    this.commited = false
-  }
-
-  emit(service, event) {
-    if(!event) {
-      event = service
-      if(Array.isArray(event)) {
-        let hasServices = false
-        for(let ev of event) {
-          if(ev.service) hasServices = true
-        }
-        if(hasServices) {
-          for(let ev of event) {
-            this.emit(ev)
-          }
-          return
-        }
-      } else {
-        service = event.service || this.service.name
-      }
-    }
-    let events
-    if(!this.commited) {
-      events = this.emittedEvents.get(service)
-      if(!events) {
-        events = []
-        this.emittedEvents.set(service, events)
-      }
-    } else {
-      events = []
-    }
-    if(Array.isArray(event)) {
-      for(let ev of event) ev.service = service
-      events.push(...event)
-    } else {
-      event.service = service
-      events.push(event)
-    }
-    if(this.commited) {
-      if(events.length == 0) return
-      this.service.dao.request(['database', 'putLog'], this.service.databaseName,
-          this.service.name+'_events', { type: 'bucket', events, ...this.flags })
-    }
-  }
-
-  async commit() {
-    let promises = []
-    this.commited = true
-    if(this.emittedEvents.length == 0) return []
-    let allEvents = []
-    for(const [service, events] of this.emittedEvents.keys()) {
-      promises.push(this.service.dao.request(['database', 'putLog'], this.service.databaseName,
-          this.service.name+'_events', { type: 'bucket', events, ...this.flags }))
-      allEvents.push(...events)
-    }
-    await Promise.all(promises)
-    return allEvents
-  }
-}
-
-class SingleEmitQueue {
-  constructor(service, flags = {}) {
-    this.service = service
-    this.flags = flags
-    this.emittedEvents = []
-    this.commited = false
-  }
-
-  emit(service, event) {
-    if(!event) {
-      event = service
-      service = this.service.name
-    }
-    let events
-    if(!this.commited) {
-      events = this.emittedEvents
-    } else {
-      events = []
-    }
-    if(Array.isArray(event)) {
-      for(let ev of event) if(!ev.service) ev.service = service
-      events.push(...event)
-    } else {
-      if(!event.service) event.service = service
-      events.push(event)
-    }
-    if(this.commited) {
-      if(events.length == 0) return
-      this.service.dao.request(['database', 'putLog'], this.service.databaseName,
-          'events', { type: 'bucket', events, ...this.flags })
-    }
-  }
-
-  async commit() {
-    this.commited = true
-    if(this.emittedEvents.length == 0) return []
-    await this.service.dao.request(['database', 'putLog'], this.service.databaseName,
-          'events', { type: 'bucket', events: this.emittedEvents, ...this.flags })
-    return this.emittedEvents
-  }
-}
-
-
 
 module.exports = Service

package/lib/utils/ProfileLogFilesystemWriter.js

@@ -0,0 +1,40 @@
+const fs = require('fs')
+const os = require('os')
+const { once } = require('events')
+
+class ProfileLogFilesystemWriter {
+  constructor(path) {
+    this.profileLogStream = null
+    this.profileLogStreamDrainPromise = null
+
+    if(path) {
+      this.profileLogStream = fs.createWriteStream(path)
+    } else if(process.env.PROFILE_LOG_PATH) {
+      const dateString = new Date().toISOString().slice(0, -1).replace(/[T:\\.-]/gi, '_')
+      const serviceName = process.cwd().split('/').pop()
+      const hostname = os.hostname()
+      const username = os.userInfo().username
+
+      const logPath = process.env.PROFILE_LOG_PATH + serviceName +
+          '@' + username + '@' + hostname + '@' + dateString + '.prof.log'
+      this.profileLogStream = fs.createWriteStream(logPath)
+    }
+  }
+
+  async write(operation) {
+    if(!this.profileLogStream) return;
+    const msg = {
+      time: (new Date()).toISOString(),
+      ...operation
+    }
+    if(!this.profileLogStream.write(JSON.stringify(msg)+'\n')) {
+      if(!this.profileLogStreamDrainPromise) {
+        this.profileLogStreamDrainPromise = once(this.profileLogStream, 'drain')
+      }
+      await this.profileLogStreamDrainPromise
+      this.profileLogStreamDrainPromise = null
+    }
+  }
+}
+
+module.exports = ProfileLogFilesystemWriter

package/lib/utils/SingleEmitQueue.js

@@ -0,0 +1,44 @@
+
+class SingleEmitQueue {
+  constructor(service, flags = {}) {
+    this.service = service
+    this.flags = flags
+    this.emittedEvents = []
+    this.commited = false
+  }
+
+  emit(service, event) {
+    if(!event) {
+      event = service
+      service = this.service.name
+    }
+    let events
+    if(!this.commited) {
+      events = this.emittedEvents
+    } else {
+      events = []
+    }
+    if(Array.isArray(event)) {
+      for(let ev of event) if(!ev.service) ev.service = service
+      events.push(...event)
+    } else {
+      if(!event.service) event.service = service
+      events.push(event)
+    }
+    if(this.commited) {
+      if(events.length == 0) return
+      this.service.dao.request(['database', 'putLog'], this.service.databaseName,
+          'events', { type: 'bucket', events, ...this.flags })
+    }
+  }
+
+  async commit() {
+    this.commited = true
+    if(this.emittedEvents.length == 0) return []
+    await this.service.dao.request(['database', 'putLog'], this.service.databaseName,
+          'events', { type: 'bucket', events: this.emittedEvents, ...this.flags })
+    return this.emittedEvents
+  }
+}
+
+module.exports = SingleEmitQueue

package/lib/utils/SplitEmitQueue.js

@@ -0,0 +1,67 @@
+
+class SplitEmitQueue {
+  constructor(service, flags = {}) {
+    this.service = service
+    this.flags = flags
+    this.emittedEvents = new Map()
+    this.commited = false
+  }
+
+  emit(service, event) {
+    if(!event) {
+      event = service
+      if(Array.isArray(event)) {
+        let hasServices = false
+        for(let ev of event) {
+          if(ev.service) hasServices = true
+        }
+        if(hasServices) {
+          for(let ev of event) {
+            this.emit(ev)
+          }
+          return
+        }
+      } else {
+        service = event.service || this.service.name
+      }
+    }
+    let events
+    if(!this.commited) {
+      events = this.emittedEvents.get(service)
+      if(!events) {
+        events = []
+        this.emittedEvents.set(service, events)
+      }
+    } else {
+      events = []
+    }
+    if(Array.isArray(event)) {
+      for(let ev of event) ev.service = service
+      events.push(...event)
+    } else {
+      event.service = service
+      events.push(event)
+    }
+    if(this.commited) {
+      if(events.length == 0) return
+      this.service.dao.request(['database', 'putLog'], this.service.databaseName,
+          this.service.name+'_events', { type: 'bucket', events, ...this.flags })
+    }
+  }
+
+  async commit() {
+    let promises = []
+    this.commited = true
+    if(this.emittedEvents.length == 0) return []
+    let allEvents = []
+    for(const [service, events] of this.emittedEvents.keys()) {
+      promises.push(this.service.dao.request(['database', 'putLog'], this.service.databaseName,
+          this.service.name+'_events', { type: 'bucket', events, ...this.flags }))
+      allEvents.push(...events)
+    }
+    await Promise.all(promises)
+    return allEvents
+  }
+}
+
+module.exports = SplitEmitQueue

package/lib/utils/profileLog.js

@@ -1,40 +1,10 @@
-const fs = require('fs')
-const os = require('os')
-const { once } = require('events')
-
 class ProfileLog {
-  constructor(path) {
-    this.profileLogStream = null
-    this.profileLogStreamDrainPromise = null
-
-    if(process.env.PROFILE_LOG_PATH) {
-      const dateString = new Date().toISOString().slice(0, -1).replace(/[T:\\.-]/gi, '_')
-      const serviceName = process.cwd().split('/').pop()
-      const hostname = os.hostname()
-      const username = os.userInfo().username
-
-      const logPath = process.env.PROFILE_LOG_PATH + serviceName +
-          '@' + username + '@' + hostname + '@' + dateString + '.prof.log'
-      this.profileLogStream = fs.createWriteStream(logPath)
-    }
-    if(path) {
-      this.profileLogStream = fs.createWriteStream(path)
-    }
+  constructor() {
+    this.writers = []
   }
 
   async log(operation) {
-    if(!this.profileLogStream) return;
-    const msg = {
-      time: (new Date()).toISOString(),
-      ...operation
-    }
-    if(!this.profileLogStream.write(JSON.stringify(msg)+'\n')) {
-      if(!this.profileLogStreamDrainPromise) {
-        this.profileLogStreamDrainPromise = once(this.profileLogStream, 'drain')
-      }
-      await this.profileLogStreamDrainPromise
-      this.profileLogStreamDrainPromise = null
-    }
+    await Promise.all(this.writers.map(writer => writer.write(operation)))
   }
 
   async begin(operation) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/framework",
-  "version": "0.4.40",
+  "version": "0.4.44",
   "description": "Live Change Framework - ultimate solution for real time mobile/web apps",
   "main": "index.js",
   "scripts": {
@@ -41,8 +41,5 @@
     "subleveldown": "^5.0.1",
     "tape": "^5.3.1",
     "websocket": "^1.0.34"
-  },
-  "dependencies": {
-    "@elastic/elasticsearch": "7.13.0"
   }
 }

package/lib/processors/searchIndex.js

@@ -1,21 +0,0 @@
-const utils = require("../utils.js")
-
-module.exports = async function(service, app) {
-  const search = await app.connectToSearch()
-  const generateIndexName = (modelName) => {
-    return (app.searchIndexPrefix+service.name+"_"+modelName).toLowerCase()
-  }
-
-  for(let modelName in service.models) {
-    const model = service.models[modelName]
-    if (!model.search) continue
-    const searchIndex = generateIndexName(modelName)
-    model.searchIndex = searchIndex
-  }
-  for(let indexName in service.indexes) {
-    const index = service.models[indexName]
-    if (!index.search) continue
-    const searchIndex = generateIndexName(indexName)
-    model.searchIndex = searchIndex
-  }
-}

package/lib/updaters/elasticsearch.js

@@ -1,293 +0,0 @@
-const { typeName } = require("../utils.js")
-const SearchIndexer = require("../runtime/SearchIndexer.js")
-
-function generatePropertyMapping(property, search) {
-  //console.log("GENERATE PROPERTY MAPPING", property)
-  let options = search ? JSON.parse(JSON.stringify(search)) : {}
-  if(property.search === false) options.enabled = false
-  if(!options.type) {
-    switch(typeName(property.type)) {
-      case "String": options.type = "text"; break;
-      case "Number": options.type = "double"; break;
-      case "Date": options.type = "date"; break;
-      case "Boolean": options.type = "boolean"; break;
-      case "Array": options.type = "array"; break;
-      case "Object": options.type = "object"; break;
-      default: options.type = "keyword"
-    }
-  }
-  //console.log("GENERATED PROPERTY MAPPING", property, ":", options)
-  return options
-}
-
-function generatePropertyMappings(property, propName) {
-  const bindings = property.search
-      ? (Array.isArray(property.search) ? property.search : [ property.search ])
-      : [null]
-  const mappings = {}
-  for(const binding of bindings) {
-    const options = generatePropertyMapping(property, binding)
-    //console.log("OPTIONS", options)
-    if(options.type == 'object' && !options.properties) {
-      options.properties = {}
-      for(let propName in property.properties) {
-        const mappings = generatePropertyMappings(property.properties[propName], propName)
-        for(let key in mappings) options.properties[key] = mappings[key]
-      }
-      options.include_in_root = true
-    }
-    if(options.type == 'array') {
-      if(typeName(property.of) != "Object") {
-        return generatePropertyMappings(property.of, propName)
-      } else {
-        options.type = 'nested'
-      }
-    }
-    delete options.name
-    mappings[(binding && binding.name) || propName] = options
-  }
-  //console.log("PROPERTY MAPPINGS", propName, mappings)
-  return mappings
-}
-
-function generateMetadata(model) {
-  let properties = {}
-  for(const propName in model.properties) {
-    const mappings = generatePropertyMappings(model.properties[propName], propName)
-    for(let key in mappings) properties[key] = mappings[key]
-  }
-  let settings = (typeof model.search == 'object') ? model.search.settings : undefined
-  return {
-    settings,
-    mappings: {
-      _source: {
-        enabled: true
-      },
-      properties: {
-        id: { type: "keyword", index: false },
-        ...properties
-      }
-    }
-  }
-}
-
-async function updateElasticSearch(changes, service, app, force) {
-
-  const generateIndexName = (modelName) => {
-    return (app.searchIndexPrefix+service.name+"_"+modelName).toLowerCase()
-  }
-
-  const generateTableName = (modelName) => {
-    return service.name+"_"+modelName
-  }
-
-  console.log("ELASTICSEARCH UPDATER")
-
-  let changesByModel = new Map()
-  const addChange = function(modelName, change) {
-    let changes = changesByModel.get(modelName)
-    if(!changes) changesByModel.set(modelName, [change])
-      else changes.push(change)
-  }
-
-  /// Group by model
-  for(let change of changes) {
-    switch (change.operation) {
-      case "createModel": addChange(change.model.name, change); break
-      case "renameModel": addChange(change.from, change); break
-      case "deleteModel": addChange(change.name, change); break
-      case "createProperty":
-      case "renameProperty":
-      case "deleteProperty":
-      case "changePropertyType":
-      case "searchEnabled":
-      case "searchDisabled":
-      case "searchUpdated":
-      case "changePropertySearch": addChange(change.model, change); break
-      default:
-    }
-  }
-
-  const search = await app.connectToSearch()
-
-  async function getCurrentAlias(modelName) {
-    let alias = await search.indices.getAlias({name: generateIndexName(modelName) })
-    //console.log("GOT ALIAS", Object.keys(alias.body)[0])
-    return Object.keys(alias.body)[0]
-  }
-
-  async function setPropertyDefaultValue(currentAlias, propertyName, defaultValue) {
-    const req = {
-      index: currentAlias,
-      body: {
-        query: {
-          match_all: {}
-        },
-        script: {
-          source: `ctx._source.${propertyName} = ${JSON.stringify(defaultValue)}`,
-          lang: 'painless'
-        }
-      },
-      conflicts: 'proceed'
-    }
-    console.log("UPDATE BY QUERY", req)
-    await search.updateByQuery(req).catch(error => {
-      console.error("FIELD UPDATE ERROR", error.meta.body.error, error.meta.body.failures)
-      throw error
-    })
-  }
-
-  for(let [model, changes] of changesByModel.entries()) {
-    let definition = service.models[model]
-    for(let change of changes) {
-      if(!definition.search && change.operation!='searchDisabled' && change.operation!='deleteModel') return
-      switch (change.operation) {
-        case "createModel": {
-          if (changes.length != 1) {
-            console.error("Bad model operations set", changes)
-            throw new Error("createModel prohibits other operations for model")
-          }
-          const index =  generateIndexName(change.model.name) + '_1'
-          const metadata = generateMetadata(service.models[change.model.name])
-          console.log("INDEX", index)
-          console.log("METADATA", JSON.stringify(metadata,null, "  "))
-          await search.indices.create({
-            index,
-            body: metadata
-          })
-          await search.indices.putAlias({
-            name: generateIndexName(change.model.name),
-            index: generateIndexName(change.model.name) + '_1',
-          })
-        } break
-        case "searchEnabled": {
-          const index =  generateIndexName(change.model) + '_1'
-          const metadata = generateMetadata(service.models[change.model])
-          console.log("INDEX", index)
-          console.log("METADATA", JSON.stringify(metadata,null, "  "))
-          await search.indices.create({
-            index,
-            body: metadata
-          })
-          await search.indices.putAlias({
-            name: generateIndexName(change.model),
-            index,
-          })
-        } break
-        case "deleteModel":
-          if(changes.length != 1) {
-            console.error("Bad model operations set", changes)
-            throw new Error("deleteModel prohibits other operations for model")
-          } /// NO BREAK!
-        case "searchDisabled": {
-          console.log("SEARCH DISABLED")
-          const indexName = generateIndexName(change.model)
-          const currentAlias = await getCurrentAlias(change.model).catch(e=>null)
-          console.log("DELETE INDEX", currentAlias, "AND ALIAS", indexName)
-          if(currentAlias) await search.indices.delete({ index: currentAlias }).catch(e=>{})
-          await search.indices.deleteAlias({ name: indexName }).catch(e=>{})
-          await app.dao.request(['database', 'delete'], app.databaseName, 'searchIndexes', indexName)
-        } break
-        case "renameModel": {
-          const newAlias = generateIndexName(change.to) + '_1'
-          await search.indices.create({
-            name: newAlias,
-            body: generateMetadata(service.models[change.to])
-          })
-          await search.indices.putAlias({
-            name: generateIndexName(change.to),
-            index: newAlias
-          })
-          const currentAlias = await getCurrentAlias(change.from)
-          await search.reindex({ body: {
-            source: { index: currentAlias },
-            dest: { index: newAlias }
-          }})
-          await search.indices.delete({ name: currentAlias })
-          await search.indices.deleteAlias({ name: generateIndexName(change.from) })
-        } break
-        default:
-      }
-    }
-
-    let reindex = false
-    for(let change of changes) {
-      switch (change.operation) {
-        case "renameProperty":
-        case "deleteProperty":
-        case "changePropertyType":
-        case "changePropertySearch":
-        case "searchUpdated":
-          reindex = true;
-          break;
-        default:
-      }
-    }
-
-
-    if(reindex) {
-      try {
-        const currentAlias = await getCurrentAlias(model)
-        const currentVersion = +currentAlias.slice(currentAlias.lastIndexOf("_") + 1)
-        const newVersion = currentVersion + 1
-        const newAlias = generateIndexName(model) + "_" + newVersion
-        const metadata = generateMetadata(service.models[model])
-        console.log("METADATA", JSON.stringify(metadata, null, "  "))
-        await search.indices.create({
-          index: newAlias,
-          body: metadata
-        })
-
-        for(let change of changes) { /// Create properties before reindex
-          if(change.operation == 'createProperty')
-            if(typeof change.property.defaultValue != 'undefined')
-              await setPropertyDefaultValue(currentAlias, change.name, change.property.defaultValue)
-        }
-
-        /*await search.reindex({
-          body: {
-            source: { index: currentAlias },
-            dest: { index: newAlias }
-          }
-        })*/
-
-        const indexer = new SearchIndexer(app.dao, app.databaseName, 'Table',
-            generateTableName(model), search, newAlias, service.models[model])
-
-        await indexer.copyAll()
-
-        await search.indices.putAlias({
-          name: generateIndexName(model),
-          index: newAlias
-        })
-        await search.indices.delete({ index: currentAlias })
-      } catch(error) {
-        if(error.meta) console.error("REINDEXING ERROR", JSON.stringify(error.meta))
-          else console.error("REINDEXING ERROR", error)
-        throw error
-      }
-    } else {
-      for(let change of changes) {
-        switch (change.operation) {
-          case "createProperty": {
-            let properties = {}
-            properties[change.name] = generatePropertyMapping(change.property)
-            const currentAlias = await getCurrentAlias(change.model)
-            await search.indices.putMapping({
-              index: currentAlias,
-              body: {properties}
-            }).catch(error => {
-              console.error('ES ERROR', error.meta.body.error)
-              throw error
-            })
-            if(typeof change.property.defaultValue != 'undefined')
-              await setPropertyDefaultValue(currentAlias, change.name, change.property.defaultValue)
-          } break
-        }
-      }
-    }
-  }
-
-}
-
-module.exports = updateElasticSearch

package/lib/utils/AnalyticsWriter.js

@@ -1,79 +0,0 @@
-const { Client: ElasticSearch } = require('@elastic/elasticsearch')
-
-class AnalyticsWriter {
-
-  constructor(indexPrefix) {
-    this.dbPromise = null
-    this.currentIndex = null
-    this.isWriting = false
-    this.indexPrefix = indexPrefix
-    this.queue = []
-  }
-
-  initDb(index) {
-    if(this.currentIndex == index && this.dbPromise) return this.dbPromise
-    this.dbPromise = new Promise(async (resolve, reject) => {
-      const db = new ElasticSearch({ node: process.env.ANALYTICS_URL || 'http://localhost:9200' })
-      this.currentIndex = index
-      await db.indices.create({
-        index: this.currentIndex,
-        body: {
-          mappings: {
-            properties: {
-              timestamp: {
-                type: "date"
-              },
-              clientTS: {
-                type: "date"
-              }
-            }
-          }
-        }
-      }).catch(err => {
-        if(err.meta.body) {
-          console.error("ES ERR: ", err.meta.body)
-          if(err.meta.body.error.type == 'resource_already_exists_exception') return db
-        } else {
-          console.error("ES ERROR: ", err)
-        }
-        throw err
-      })
-      db.info(console.log)
-      resolve(db)
-    })
-    return this.dbPromise
-  }
-
-  saveEvents(events) {
-    this.queue = this.queue.concat(events)
-    this.writeEvents()
-  }
-
-  writeEvents() {
-    if(this.isWriting) return
-    if(this.queue.length == 0) return
-    this.isWriting = true
-    const index = this.indexPrefix+(new Date()).toISOString().slice(0, 7)
-    const data = this.queue.slice()
-    this.queue = []
-    let operations = new Array(data.length*2)
-    for(let i = 0; i < data.length; i++) {
-      operations[i * 2] = { index: { } }
-      operations[i * 2 + 1] = data[i]
-    }
-    this.initDb(index).then(db => db.bulk({
-      index,
-      body: operations
-    })).then(result => {
-      this.isWriting = false
-      if(this.queue.length > 0) setTimeout(() => this.writeEvents(), 10)
-    }).catch(error => {
-      console.error("COULD NOT WRITE EVENTS to ES!", error)
-      this.queue = data.concat(this.queue)
-      this.isWriting = false
-    })
-  }
-
-}
-
-module.exports = AnalyticsWriter