@live-change/security-service 0.2.5 → 0.2.19

package/ban.js

@@ -1,6 +1,6 @@
 const app = require("@live-change/framework").app()
 const definition = require('./definition.js')
-const { getClientKeysStrings } = require('./utils.js')
+const { getClientKeysStrings, multiKeyIndexQuery, fastMultiKeyIndexQuery } = require('./utils.js')
 const lcp = require('@live-change/pattern')
 
 const banProperties = {
@@ -49,7 +49,10 @@ const Ban = definition.model({
     },
     actionBans: {
       function: async function(input, output) {
-        const values = (ban) => {
+        function prefixes(ban) {
+          //output.debug("BAN", ban)
+          if(!ban.keys) return []
+          if(!ban.actions) return []
           const v = ban.keys.length
           const w = ban.actions.length
           let res = new Array(v * w)
@@ -59,26 +62,89 @@ const Ban = definition.model({
               res[i * v + j] = `${ban.actions[j]}:${key.key}:${key.value}`
             }
           }
+          //output.debug("BAN ACTIONS", res)
           return res
         }
-        await input.table("securityService_Ban").onChange((obj, oldObj) => {
+        function indexObject(prefix, obj) {
+          //output.debug("BAN", obj)
+          return {
+            id: prefix+'_'+obj.id,
+            to: obj.id,
+            type: obj.type,
+            expire: obj.expire,
+            actions: obj.actions
+          }
+        }
+        await input.table("security_Ban").onChange((obj, oldObj) => {
           if(obj && oldObj) {
-            let pointers = obj && new Set(values(obj))
-            let oldPointers = oldObj && new Set(values(oldObj))
+            //output.debug("CHANGE!", obj, oldObj)
+            const pointers = obj && new Set(prefixes(obj))
+            const oldPointers = oldObj && new Set(prefixes(oldObj))
             for(let pointer of pointers) {
-              if(!!oldPointers.has(pointer)) output.change({ id: pointer+'_'+obj.id, to: obj.id }, null)
+              if(!!oldPointers.has(pointer)) output.change(indexObject(pointer, obj), null)
             }
             for(let pointer of oldPointers) {
-              if(!!pointers.has(pointer)) output.change(null, { id: pointer+'_'+obj.id, to: obj.id })
+              if(!!pointers.has(pointer)) output.change(null, indexObject(pointer, oldObj))
             }
           } else if(obj) {
-            values(obj).forEach(v => output.change({ id: v+'_'+obj.id, to: obj.id }, null))
+            //output.debug("CREATE!", obj, oldObj)
+            prefixes(obj).forEach(v => output.change(indexObject(v, obj), null))
           } else if(oldObj) {
-            values(oldObj).forEach(v => output.change(null, { id: v+'_'+obj.id, to: obj.id }))
+            //output.debug("DELETE!", obj, oldObj)
+            prefixes(oldObj).forEach(v => output.change(null, indexObject(v, oldObj)))
           }
         })
       }
-    }
+    },
+    actionBansByType: {
+      function: async function(input, output) {
+        function prefixes(ban) {
+          //output.debug("BAN", ban)
+          if(!ban.keys) return []
+          if(!ban.actions) return []
+          const v = ban.keys.length
+          const w = ban.actions.length
+          let res = new Array(v * w)
+          for(let i = 0; i < v; i++) {
+            for(let j = 0; j < w; j++) {
+              const key = ban.keys[i]
+              res[i * v + j] = `${ban.actions[j]}:${key.key}:${key.value}:${ban.type}:${ban.expire}`
+            }
+          }
+          output.debug("BAN PREFIXES", res)
+          return res
+        }
+        function indexObject(prefix, obj) {
+          //output.debug("BAN", obj)
+          return {
+            id: prefix+'_'+obj.id,
+            to: obj.id,
+            type: obj.type,
+            expire: obj.expire,
+            actions: obj.actions
+          }
+        }
+        await input.table("security_Ban").onChange((obj, oldObj) => {
+          if(obj && oldObj) {
+            //output.debug("CHANGE!", obj, oldObj)
+            let pointers = obj && new Set(prefixes(obj))
+            let oldPointers = oldObj && new Set(prefixes(oldObj))
+            for(let pointer of pointers) {
+              if(!!oldPointers.has(pointer)) output.change(indexObject(pointer, obj), null)
+            }
+            for(let pointer of oldPointers) {
+              if(!!pointers.has(pointer)) output.change(null, indexObject(pointer, oldObj))
+            }
+          } else if(obj) {
+            //output.debug("CREATE!", obj, oldObj)
+            prefixes(obj).forEach(v => output.change(indexObject(v, obj), null))
+          } else if(oldObj) {
+            //output.debug("DELETE!", obj, oldObj)
+            prefixes(oldObj).forEach(v => output.change(null, indexObject(v, oldObj)))
+          }
+        })
+      }
+    },
   }
 })
 
@@ -101,23 +167,87 @@ definition.view({
   properties: {},
   daoPath(params, { client, service }) {
     const keys = getClientKeysStrings(client)
-    return multiKeyIndexQuery(keys, 'Ban_bans')
+    return multiKeyIndexQuery(keys, 'security_Ban_bans', Ban.tableName)
   },
 })
 
 definition.view({
-  name: "myActionBans",
+  name: "myActionsBans",
   properties: {
-    action: {
+    actions: {
       type: String
     }
   },
-  daoPath({ action }, { client, service }) {
-    const keys = getClientKeysStrings(client, action + ':')
-    return multiKeyIndexQuery(keys, 'Ban_actionBans')
+  daoPath({ actions }, { client, service }) {
+    const keys = []
+    for(const action of actions) {
+      keys.push(...getClientKeysStrings(client, action + ':'))
+    }
+    return fastMultiKeyIndexQuery(keys, 'security_Ban_actionBans', Ban.tableName)
   },
 })
 
+definition.view({
+  name: "myActionsBansByTypes",
+  properties: {
+    actions: {
+      type: Array,
+      of: {
+        type: String
+      }
+    },
+    types: {
+      type: Array,
+      of: {
+        type: String
+      }
+    }
+  },
+  daoPath({ actions, types }, { client, service }) {
+    const keys = []
+    for(const type of types) {
+      for (const action of actions) {
+        keys.push(...getClientKeysStrings(client, action + ':',':' + type))
+      }
+    }
+    console.log("BAN KEYS", keys)
+    return ['database', 'query', app.databaseName, `(${
+        async (input, output, { keys, indexName }) => {
+          function mapper(obj) {
+            if(!obj) return null
+            const { id, to, ...safeObj } = obj
+            return { ...safeObj, id: to }
+          }
+          function onIndexChange(obj, oldObj) {
+            output.change(mapper(obj), mapper(oldObj))
+          }
+          await Promise.all(keys.map(async (encodedKey) => {
+            const range = {
+              gte: encodedKey,
+              lte: encodedKey + "\xFF",
+              reverse: true,
+              limit: 1 // only last expire
+            }
+            await (await input.index(indexName)).range(range).onChange(onIndexChange)
+          }))
+        }
+    })`, { keys, indexName: 'security_Ban_actionBansByType' }]
+  },
+})
+
+/*function getBanPrefixes(keys, actions) {
+  const v = keys.length
+  const w = actions.length
+  let res = new Array(v * w)
+  for(let i = 0; i < v; i++) {
+    for(let j = 0; j < w; j++) {
+      const key = keys[i]
+      res[i * v + j] = `${actions[j]}:${key.key}:${key.value}`
+    }
+  }
+  return res
+}*/
+
 definition.trigger({
   name: "securityActionBan",
   properties: {
@@ -136,7 +266,6 @@ definition.trigger({
     }
   },
   async execute({ keys, event, ban: { actions, expire, type } }, { service }, emit) {
-    const ban = app.generateUid()
 
     console.log("SECURITY BAN!", arguments[0])
 
@@ -152,23 +281,25 @@ definition.trigger({
     console.log("BAN KEYS", banKeys)
     console.log("BAN EXPIRE", banExpire)
 
-    // service.trigger({
-    //   type: 'createTimer',
-    //   timer: {
-    //     timestamp: banExpire.getTime() + 1000,
-    //     service: 'security',
-    //     trigger: {
-    //       type: 'removeExpiredBan',
-    //       ban
-    //     }
-    //   }
-    // })
-
-    // emit({
-    //   type: "banCreated",
-    //   ban,
-    //   data: { actions, banKeys, expire, type }
-    // })
+    const ban = app.generateUid()
+
+    service.trigger({
+      type: 'createTimer',
+      timer: {
+        timestamp: banExpire.getTime() + 1000,
+        service: 'security',
+        trigger: {
+          type: 'removeExpiredBan',
+          ban
+        }
+      }
+    })
+
+    emit({
+      type: "banCreated",
+      ban,
+      data: { actions, keys: banKeys, expire: banExpire, type }
+    })
   }
 })
 
@@ -177,7 +308,8 @@ definition.trigger({
   properties: {
     ...banProperties
   },
-  async execute({ ban }, {client, service}, emit) {
+  async execute({ ban }, { client, service }, emit) {
+    console.log("REMOVE EXPIRED BAN", ban)
     emit({
       type: "banRemoved",
       ban

package/definition.js

@@ -1,8 +1,7 @@
 const app = require("@live-change/framework").app()
 
 const definition = app.createServiceDefinition({
-  name: "securityService"
+  name: "security"
 })
-const config = definition.config
 
 module.exports = definition

package/event.js

@@ -1,18 +1,18 @@
 const crypto = require('crypto')
 const app = require("@live-change/framework").app()
 const definition = require('./definition.js')
-const { getClientKeysObject } = require('./utils.js')
+const { getClientKeysObject, getClientKeysStrings} = require('./utils.js')
 
 const lcp = require('@live-change/pattern')
 const lcpDb = require('@live-change/pattern-db')
 const { request } = require('http')
 
-const securityPatterns = definition.config.patterns
-const relationsStore = lcpDb.relationsStore(app.dao, app.databaseName, 'securityService_relations')
+const securityPatterns = definition.config?.patterns ?? []
+const relationsStore = lcpDb.relationsStore(app.dao, app.databaseName, 'security_relations')
 lcp.prepareModelForLive(securityPatterns)
 //console.log("SECURITY PATTERNS", securityPatterns)
 
-const securityCounters = definition.config.counters
+const securityCounters = definition.config?.counters ?? []
 
 definition.beforeStart(service => {
   relationsStore.createTable()
@@ -23,22 +23,10 @@ const eventProperties = {
     type: String
   },
   keys: {
-    type: Array,
-    of: {
-      type: Object,
-      properties: {
-        key: {
-          type: {
-            String
-          }
-        },
-        value: {
-          type: {
-            String
-          }
-        }
-      }
-    }
+    type: Object
+  },
+  properties: {
+    type: Object
   },
   timestamp: {
     type: Date
@@ -51,8 +39,21 @@ const Event = definition.model({
     ...eventProperties
   },
   indexes: {
-    byTypeAndTimestamp: {
-      property: ['type', 'timestamp']
+    byKeyTypeAndTimestamp: {
+      function: async function(input, output) {
+        async function spread(event) {
+          for(const key in event.keys) {
+            output.put({
+              id: `${key}:${JSON.stringify(event.keys[key])}:${event.type}:${event.timestamp}_${event.id}`,
+              type: event.type,
+              timestamp: event.timestamp
+            })
+          }
+        }
+        await input.table('security_Event').onChange(
+          (obj, oldObj) => spread(obj)
+        )
+      }
     }
   }
 })
@@ -140,30 +141,42 @@ async function processSecurityCounters(event, timestamp, service) {
   for(const counter of counters) {
     const duration = lcp.parseDuration(counter.duration)
     for(const eventType of counter.match) {
-      const request = counterEventsRequests.find(req => req.type == eventType)
-      if(request) {
-        request.max = Math.max(request.max, counter.max)
-        request.duration = Math.max(request.duration, duration)
-      } else {
-        counterEventsRequests.push({
-          type: eventType,
-          max: counter.max,
-          duration
-        })
+      for(const key of counter.keys) {
+        const request = counterEventsRequests.find(req => req.type == eventType && req.key == key)
+        if(request) {
+          request.max = Math.max(request.max, counter.max)
+          request.duration = Math.max(request.duration, duration)
+        } else {
+          counterEventsRequests.push({
+            type: eventType,
+            key,
+            max: counter.max,
+            duration
+          })
+        }
       }
     }
   }
-  console.log("COUNTER EVENTS REQUESTS", counterEventsRequests)
-  const counterEvents = await Promise.all(counterEventsRequests.map(async request => ({
-    type: request.type,
-    events: await Event.indexRangeGet('byTypeAndTimestamp',{
-      gt: `"${request.type}":"${(new Date(now - request.duration - 1000)).toISOString()}"`,
-      lt: `"${request.type}":\xFF`,
+  const indexName = Event.tableName+'_byKeyTypeAndTimestamp'
+  //console.log("ALL COUNTER EVENTS REQUESTS", counterEventsRequests)
+  const counterEvents = await Promise.all(counterEventsRequests.map(async request => {
+    const back = (new Date(now - request.duration - 1000)).toISOString()
+    const prefix = `${request.key}:${JSON.stringify(event.keys[request.key])}:${request.type}:`
+    console.log("{", prefix)
+    const range = {
+      gt: prefix + back, // time limited
+      lt: prefix + '\xFF',
       reverse: true,
       limit: request.max
-    })
-  })))
-  console.log("COUNTER EVENTS", counterEvents)
+    }
+    console.log("R", range)
+    const events = await app.dao.get(['database', 'indexRange', service.databaseName, indexName, range])
+    console.log("RR", events)
+    return {
+      type: request.type,
+      events
+    }
+  }))
   for(const counter of counters) {
     const duration = lcp.parseDuration(counter.duration)
     const fromTime = (new Date(now - duration)).toISOString()
@@ -177,7 +190,7 @@ async function processSecurityCounters(event, timestamp, service) {
       }
     }
     if(count + 1 > counter.max) { // +1 for the new event, not added yet
-      console.log("COUNTER FIRE", counter)
+      //console.log("COUNTER FIRE", counter)
       actions.push(...counter.actions)
     }
   }
@@ -189,24 +202,14 @@ definition.trigger({
   properties: {
     event: {
       type: Object,
-      properties: {
-        type: {
-          type: String
-        },
-        keys: {
-          type: Object
-        },
-        properties: {
-          type: Object
-        }
-      }
+      properties: eventProperties
     },
     client: {
       type: Object
     }
   },
   async execute({ event, client, timestamp }, { service }, emit) {
-    console.log("SECURITY EVENT TRIGGERED", arguments[0])
+    //console.log("SECURITY EVENT TRIGGERED", arguments[0])
     event.id = event.id || app.generateUid()
     event.time = event.time || timestamp
     const time = (typeof event.time == 'number') ? event.time : new Date(event.time).getTime()
@@ -215,7 +218,7 @@ definition.trigger({
       event.keys = { ...getClientKeysObject(client), ...event.keys }
     }
 
-    console.log("PROCESS EVENT", event)
+    //console.log("PROCESS EVENT", event)
     let [
       { newRelations, canceledRelations, actions: patternsActions },
       { actions: countersActions }
@@ -229,7 +232,7 @@ definition.trigger({
 
     for(const action of actions) {
       const actionTypeUpperCase = action.type[0].toUpperCase() + action.type.slice(1)
-      console.log("ACTION", JSON.stringify(action, null, '  '))
+      //console.log("ACTION", JSON.stringify(action, null, '  '))
       promises.push(service.trigger({
         ...action,
         event,
@@ -252,3 +255,158 @@ definition.trigger({
   }
 })
 
+definition.view({
+  name: 'myCountersState',
+  properties: {
+    events: {
+      type: String
+    }
+  },
+  daoPath({ events }, { client, service }) {
+    const keys = getClientKeysObject(client)
+    const eventRequests = []
+    for(const eventType of events) {
+      for(const counter of securityCounters) {
+        if(!counter.visible) continue
+        if(!counter.match.includes(eventType)) continue
+        const duration = lcp.parseDuration(counter.duration)
+        for(const keyName in keys) {
+          const keyValue = keys[keyName]
+          if(keyValue === undefined) continue
+          eventRequests.push({
+            eventType,
+            counter: counter.id,
+            duration,
+            keyName,
+            keyValue,
+            max: counter.max
+          })
+        }
+      }
+    }
+    return ['database', 'query', service.databaseName, `(${
+      async (input, output, { eventRequests, indexName } ) => {
+        const index = await input.index(indexName)
+        
+        class Request {
+          constructor(eventRequest) {
+            this.eventRequest = eventRequest
+            
+            this.id = eventRequest.counter + ':' + eventRequest.keyName
+            this.count = 0
+            this.max = this.eventRequest.max
+            this.remaining = this.max + 1
+            
+            this.prefix = `${eventRequest.keyName}:${JSON.stringify(eventRequest.keyValue)}:${eventRequest.eventType}:` 
+            this.range = undefined 
+            this.indexRange = undefined
+            this.fromTime = 0
+            this.value = undefined
+            this.observer = undefined
+            this.oldest = undefined
+            this.expire = Infinity
+            
+            this.loading = false
+          }
+          async refresh() {
+            this.loading = true
+            const newFromTime = Date.now() - this.eventRequest.duration
+            //output.debug("REFRESH", newFromTime, this.fromTime, newFromTime - this.fromTime)
+            if(Math.abs(newFromTime - this.fromTime) > 5000) { // refresh only when time is different by 5s
+              //output.debug("OBSERVER REFRESH!")
+              this.fromTime = newFromTime
+              const newRange = {
+                gt: this.prefix + (new Date(this.fromTime)).toISOString(), // time limited
+                lt: this.prefix + '\xFF',
+                reverse: true,
+                limit: this.eventRequest.max
+              }
+              this.range = newRange
+              //output.debug("NEW RANGE", this.range)
+              if(this.observer) {
+                this.indexRange.unobserve(this.observer)
+              }
+              //output.debug("OBSERVE RANGE!", this.range)
+              this.indexRange = index.range(this.range)
+              
+              this.observer = await this.indexRange.onChange(async (ind, oldInd) => {
+                //output.debug("RANGE CHANGE!", newRange, ind)
+                if(!this.loading) {
+                  await this.refresh()
+                }
+              })
+            }
+            const newValue = await this.indexRange.get()
+            this.loading = false
+            if(JSON.stringify(this.value) != JSON.stringify(newValue)) {
+              this.value = newValue
+              if(this.value) {
+                this.count = this.value.length
+                this.oldest = this.value[this.count - 1]
+                this.remaining = this.max - this.count + 1
+                this.expire = this.oldest
+                    ? (new Date(this.oldest.timestamp)).getTime() + this.eventRequest.duration
+                    : Infinity
+              }
+              await recompute()
+            }
+          }
+        }
+        
+        const requests = new Array(eventRequests.length)
+
+        let currentTimeout = null
+        let expire = Infinity
+        async function recompute() {
+          //output.debug("RECOMPUTE?")
+          for(const request of requests) {
+            if(request.value === undefined) return // no recompute until all readed
+          }
+          //output.debug("RECOMPUTE!")
+          let firstExpire = Infinity
+          let firstExpireRequest = null
+          for(const request of requests) {
+            //console.log("REQUEST EXPIRE", request.prefix, request.expire)
+            if(request.expire < firstExpire) {
+              firstExpire = request.expire
+              firstExpireRequest = request
+            }
+            const { id, count, remaining, max, oldest, value } = request
+            output.change({
+              id,
+              count, remaining, max, 
+              oldest: oldest?.timestamp,
+              expire: request.expire ? new Date(request.expire) : null,
+              //value
+            })
+          }
+          //console.log("FIRST EXPIRE", firstExpire)
+          //console.log("LAST EXPIRE", expire)
+          if(firstExpire != expire) {
+            expire = firstExpire
+            if(currentTimeout) { // clears timeout
+              currentTimeout()
+            }
+            expire += 1000 // additional delay
+            //console.log("SET TIMEOUT", new Date(expire))
+            currentTimeout = output.timeout(expire, () => {
+              //output.debug("EXPIRE TIMEOUT!", expire)
+              firstExpireRequest.refresh()
+            }) // time parameter can be number or date or iso
+          }
+        }
+        
+        if(eventRequests.length == 0) return
+        for(const i in eventRequests) {
+          const eventRequest = eventRequests[i]
+          const request = new Request(eventRequest)
+          requests[i] = request
+          //output.debug("REQUEST", eventRequest, '=>', request)
+        }
+        await Promise.all(requests.map(async request => {
+          await request.refresh()
+        }))
+      }
+    })`, { eventRequests, indexName: Event.tableName + '_byKeyTypeAndTimestamp' }]
+  }
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/security-service",
-  "version": "0.2.5",
+  "version": "0.2.19",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -26,5 +26,5 @@
     "@live-change/pattern-db": "^0.2.2",
     "nodemailer": "^6.7.2"
   },
-  "gitHead": "7691357c7569a18373668691eb6a81a9e161194d"
+  "gitHead": "c2833a9660374ee17dd3def99fd1cdf47f6b1531"
 }

package/utils.js

@@ -1,9 +1,10 @@
+const app = require("@live-change/framework").app()
 const definition = require('./definition.js')
 
-const clientKeys = definition.config.clientKeys
+const clientKeys = definition.config?.clientKeys ?? []
 
-function multiKeyIndexQuery(keys, indexName) {
-  return ['database', 'query', database, `(${
+function multiKeyIndexQuery(keys, indexName, tableName) {
+  return ['database', 'query', app.databaseName, `(${
     async (input, output, { keys, indexName, tableName }) => {
       const objectStates = new Map()
       async function mapper(res) {
@@ -55,16 +56,38 @@ function multiKeyIndexQuery(keys, indexName) {
         await (await input.index(indexName)).range(range).onChange(onIndexChange)
       }))
     }
-  })`, { keys, indexName, tableName: Ban.tableName }]
+  })`, { keys, indexName, tableName }]
 }
 
-function getClientKeysStrings(client, prefix = '') {
+function fastMultiKeyIndexQuery(keys, indexName) {
+  return ['database', 'query', app.databaseName, `(${
+      async (input, output, { keys, indexName }) => {
+        function mapper(obj) {
+          if(!obj) return null
+          const { id, to, ...safeObj } = obj
+          return { ...safeObj, id: to }
+        }
+        function onIndexChange(obj, oldObj) {
+          output.change(mapper(obj), mapper(oldObj))
+        }
+        await Promise.all(keys.map(async (encodedKey) => {
+          const range = {
+            gte: encodedKey + '_',
+            lte: encodedKey + "_\xFF\xFF\xFF\xFF"
+          }
+          await (await input.index(indexName)).range(range).onChange(onIndexChange)
+        }))
+      }
+  })`, { keys, indexName}]
+}
+
+function getClientKeysStrings(client, prefix = '', suffix = '') {
   if(clientKeys) {
-    return clientKeys(client).map(k => prefix + k.key + ':' + k.value)
+    return clientKeys(client).filter(k => k.value).map(k => prefix + k.key + ':' + k.value + suffix)
   } else {
     const keys = []
     for(let key in client) {
-      keys.push(prefix + key + ':' + client[key])
+      if(client[key]) keys.push(prefix + key + ':' + client[key] + suffix)
     }
     return keys
   }
@@ -82,4 +105,4 @@ function getClientKeysObject(client, prefix = '') {
   }
 }
 
-module.exports = { multiKeyIndexQuery, getClientKeysStrings, getClientKeysObject }
+module.exports = { multiKeyIndexQuery, fastMultiKeyIndexQuery, getClientKeysStrings, getClientKeysObject }