@live-change/pattern-db 0.2.1 → 0.8.2

package/LICENSE.md

@@ -0,0 +1,11 @@
+Copyright 2019-2024 Michał Łaszczewski
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/lib/relations-store.js

@@ -1,56 +1,80 @@
 const lcp = require("@live-change/pattern")
+const crypto = require("crypto")
 
 function relationsStore(dao, database, table) {
 
   async function createTable() {
-    dao.request(['database', 'createTable'], database, table).catch(e => 'ok')
-    await this.connection.request(['database', 'createIndex'], database, table+'_eventTypeAndKeys', `(${
-        async function(input, output) {
-          const mapper = (obj) => ({ id: obj.eventType+'_'+obj.keys+'_'+obj.id, to: obj.id })
-          await input.table('users').onChange((obj, oldObj) =>
-              output.change(obj && mapper(obj), oldObj && mapper(oldObj)) )
-        }
-    })`, { tableName: this.tableName }).catch(e => 'ok')
-    await this.connection.request(['database', 'createIndex'], database, table+'_sourceRelation', `(${
-        async function(input, output) {
-          const mapper = (obj) => ({ id: obj.source+'_'+obj.relation+'_'+obj.id, to: obj.id })
-          await input.table('users').onChange((obj, oldObj) =>
-              output.change(obj && mapper(obj), oldObj && mapper(oldObj)) )
-        }
-    })`, { tableName: this.tableName }).catch(e => 'ok')
+    console.log("CREATE TABLE", table)
+    try {
+      await dao.request(['database', 'createTable'], database, table)
+    } catch(e) {
+      // console.error(e)
+    }
+    console.log("CREATE INDEX", table+'_eventTypeAndKeys')
+    try {
+      await dao.request(['database', 'createIndex'], database, table+'_eventTypeAndKeys', `(${
+          async function(input, output, { table }) {
+            const mapper = (obj) => ({ id: obj.eventType+'_'+obj.keys+'_'+obj.id, to: obj.id })
+            await input.table(table).onChange((obj, oldObj) =>
+                output.change(obj && mapper(obj), oldObj && mapper(oldObj)) )
+          }
+      })`, { table })
+    } catch(e) {
+      // console.error(e)
+    }
+    console.log("CREATE INDEX", table+'_sourceRelation')
+    try {
+      await dao.request(['database', 'createIndex'], database, table+'_sourceRelation', `(${
+          async function(input, output, { table }) {
+            const mapper = (obj) => ({ id: obj.source+'_'+obj.relation+'_'+obj.id, to: obj.id })
+            await input.table(table).onChange((obj, oldObj) =>
+                output.change(obj && mapper(obj), oldObj && mapper(oldObj)) )
+          }
+      })`, { table })
+    } catch(e) {
+     // console.error(e)
+    }
   }
 
   async function getRelations(type, keys) {
+    console.log("GET RELATIONS", type, keys)
     let keysList = Object.keys(keys).map(k => [k, keys[k]]).filter(([a,b]) => !!b)
     keysList.sort((a,b) => a[0] == b[0] ? 0 : (a[0] > b[0] ? 1 : -1))
     let keySets = lcp.allCombinations(keysList)
-    let promises = new Array(keySets.length)
+    const promises = new Array(keySets.length)
     for(let i = 0; i < keySets.length; i++) {
       const ks = keySets[i]
-      return dao.get(['database', 'query', database, `(${
+      // console.log("CHECKING KEYSET", ks)
+      // const tableData = await dao.get(['database', 'tableRange', database, table, {}])
+      // const indexData = await dao.get(['database', 'indexRange', database, table+'_eventTypeAndKeys', {}])
+      // console.log("IN TABLE:", tableData)
+      // console.log("IN INDEX:", indexData)
+      promises[i] = dao.get(['database', 'query', database, `(${
           async (input, output, { table, type, ks }) => {
             const mapper = async (res) => input.table(table).object(res.to).get()
             await input.index(table + "_eventTypeAndKeys").range({
               gte: type + '_' + ks + '_',
               lte: type + '_' + ks + "_\xFF\xFF\xFF\xFF"
             }).onChange(async (obj, oldObj) => {
-              output.change(obj && await mapper(obj), oldObj && await mapper(oldObj))
+              await output.change(obj && await mapper(obj), oldObj && await mapper(oldObj))
             })
           }
       })`, { table, type, ks}])
-
-      promises[i] = dao.get(['database', 'indexRange', database, table+'_eventTypeAndKeys', {
-        gte: type+'_'+ks+'_',
-        lte: type+'_'+ks+"_\xFF\xFF\xFF\xFF"
-      }])
     }
-    const results = (await Promise.all(promises)).reduce((a,b) => a.concat(b), [])
-    return results.map(obj => obj.to)
+    const queryResults = await Promise.all(promises)
+    const results = queryResults.flat().map(res => {
+      const keys = {}
+      for(let kd of res.keys) {
+        keys[kd[0]] = kd[1]
+      }
+      return { ...res, keys }
+    })
+    return results
   }
 
   const relationOperations = new Map() // need to queue operations on keys
 
-  function queueRelationChange(type, keysList, relationId, changeFun) {
+  async function queueRelationChange(type, keysList, relationId, changeFun) {
     const rKey = JSON.stringify([type, keysList, relationId])
     let op = relationOperations.get(rKey)
     return new Promise(async (resolve, reject) => {
@@ -75,15 +99,19 @@ function relationsStore(dao, database, table) {
                   output.change(obj && await mapper(obj), oldObj && await mapper(oldObj))
                 })
               }
-          })`, { table, type, ks}])
-          let currentRelation = relations.find(rel => rel.relation == relationId)
+          })`, { table, type, ks: keysList}])
+          let foundRelation = relations.find(rel => rel.relation == relationId)
+          //console.log("FOUND RELATION", foundRelation)
+          let currentRelation = foundRelation
           while (op.changes.length > 0) {
             for (const change of op.changes) currentRelation = change(currentRelation)
             op.changes = []
+            //console.log("CHANGED RELATION", currentRelation)
             if (currentRelation) {
+              if(!currentRelation.id) currentRelation.id = crypto.randomBytes(16).toString("hex")
+              console.log("PUT RELATION", currentRelation)
               await dao.request(['database', 'put', database, table, currentRelation])
-            } else {
-              await dao.request(['database', 'delete', database, table, currentRelation.id])
+              //console.log("RELATION WRITTEN!")
             }
           }
           relationOperations.delete(rKey)
@@ -99,26 +127,29 @@ function relationsStore(dao, database, table) {
     let promises = []
     let keysList = Object.keys(relation.keys).map(k => [k, relation.keys[k]]).filter(([a,b]) => !!b)
     keysList.sort((a,b) => a[0] == b[0] ? 0 : (a[0] > b[0] ? 1 : -1))
+    //console.log("RELATION SAVE!")
     for(const type of relation.eventTypes) {
       promises.push(
-          queueRelationChange(type, keysList, relation.relation, (currentRelation) => {
-            if(currentRelation) {
-              currentRelation.prev.push(...relation.prev)
-              if(mark) mark(currentRelation)
-              return currentRelation
-            } else {
-              const r = { ...relation, eventType: type, keys: keysList }
-              if(mark) mark(r)
-              return r
-            }
-          })
+        queueRelationChange(type, keysList, relation.relation, (currentRelation) => {
+          if(currentRelation) {
+            currentRelation.prev.push(...relation.prev)
+            if(mark) mark(currentRelation)
+            return currentRelation
+          } else {
+            const r = { ...relation, eventType: type, keys: keysList }
+            if(mark) mark(r)
+            return r
+          }
+        })
       )
     }
     await Promise.all(promises)
+    //console.log("RELATION SAVED!")
   }
 
   async function removeRelation(relation) {
-    return dao.get(['database', 'query', database, `(${
+    //console.log("REMOVE RELATION", relation)
+    return dao.request(['database', 'query'], database, `(${
         async (input, output, { relation, table }) =>
             await input.index(table+"_sourceRelation").range({
               gte: relation.source+'_'+relation.relation+'_',
@@ -126,7 +157,7 @@ function relationsStore(dao, database, table) {
             }).onChange((obj, oldObj) => {
               output.table(table).delete(obj.to)
             })
-    })`, { table, relation }])
+    })`, { table, relation })
   }
 
   return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/pattern-db",
-  "version": "0.2.1",
+  "version": "0.8.2",
   "description": "Realtime usage pattern analysis - database storage backend",
   "main": "index.js",
   "scripts": {
@@ -21,9 +21,10 @@
   },
   "homepage": "https://github.com/live-change/pattern-db",
   "devDependencies": {
-    "tape": "^4.13.0"
+    "tape": "^5.7.4"
   },
   "dependencies": {
-    "@live-change/pattern": "^0.1.0"
-  }
+    "@live-change/pattern": "^0.8.2"
+  },
+  "gitHead": "53b8efc8ec7f5c1c4af33077d8fb4a8a5580f1d9"
 }

package/tests/chain-with-expire.test.js

@@ -0,0 +1,143 @@
+const test = require('tape')
+const testDb = require('./testDb.js')
+const lcp = require('@live-change/pattern')
+const { relationsStore } = require('../lib/relations-store.js')
+
+let model
+
+test("compile fail2ban chain with expire", (t) => {
+
+  t.plan(2)
+
+  let { model: compiled, last } = lcp.chain([
+    "failed-login",
+    { eq: "ip", expire: "2m" },
+    "failed-login"
+  ])
+
+  compiled.elements[last].actions = [ 'ban' ]
+
+  console.log(JSON.stringify(compiled, null, '  '))
+
+  model = compiled
+
+  t.deepEqual(model, {
+    "elements": {
+      "failed-login": {
+        "type": "failed-login",
+        "id": "failed-login",
+        "prev": [],
+        "next": [
+          "failed-login/ip@[ip|wait:2m]/failed-login",
+          "failed-login/wait:2m@[ip|wait:2m]"
+        ]
+      },
+      "failed-login/ip|wait:2m/failed-login": {
+        "type": "failed-login",
+        "id": "failed-login/ip|wait:2m/failed-login",
+        "prev": [
+          "failed-login/ip@[ip|wait:2m]/failed-login",
+          "failed-login/wait:2m@[ip|wait:2m]"
+        ],
+        "next": [],
+        "actions": ['ban']
+      }
+    },
+    "relations": {
+      "failed-login/ip@[ip|wait:2m]/failed-login": {
+        "eq": [
+          {
+            "prev": "ip",
+            "next": "ip"
+          }
+        ],
+        "id": "failed-login/ip@[ip|wait:2m]/failed-login",
+        "cancel": [
+          "failed-login/wait:2m@[ip|wait:2m]"
+        ],
+        "prev": [
+          "failed-login"
+        ],
+        "next": [
+          "failed-login/ip|wait:2m/failed-login"
+        ]
+      },
+      "failed-login/wait:2m@[ip|wait:2m]": {
+        "id": "failed-login/wait:2m@[ip|wait:2m]",
+        "wait": "2m",
+        "cancel": [
+          "failed-login/ip@[ip|wait:2m]/failed-login",
+          "failed-login/wait:2m@[ip|wait:2m]"
+        ],
+        "prev": [
+          "failed-login"
+        ],
+        "next": []
+      }
+    }
+  })
+
+  t.test('live processor', async (t) => {
+    t.plan(4)
+
+    const db = await testDb()
+    const store = relationsStore(db, 'test', 'relations')
+    await store.createTable()
+
+    lcp.prepareModelForLive(model)
+    const processor = new lcp.LiveProcessor(model, store)
+    const ip = (Math.random()*1000).toFixed()
+    let time = 0
+
+    t.test('push first event', async (t) => {
+      t.plan(1)
+      const actions = await processor.processEvent({ id: 1, type: 'failed-login', keys: { ip }, time }, 0)
+      console.log("ACTIONS", actions)
+      const relations = await store.getRelations('failed-login', { ip })
+      //console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(relations.length > 0) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+    })
+
+    t.test('wait 2.5m for expire', async (t) => {
+      t.plan(1)
+      time += 2.5 * 60 * 1000
+      const actions = await processor.processTime(time)
+      console.log("ACTIONS", actions)
+      const relations = await store.getRelations('failed-login', { ip })
+      if(relations.length == 0) {
+        t.pass('expired')
+      } else {
+        t.fail('still exists')
+      }
+    })
+
+    t.test('push second event', async (t) => {
+      t.plan(1)
+      time += 1000
+      const actions = await processor.processEvent({ id: 2, type: 'failed-login', keys: { ip }, time }, 0)
+      console.log("ACTIONS", actions)
+      const relations = await store.getRelations('failed-login', { ip })
+      if(relations.length > 0) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+    })
+
+    t.test('push third event', async (t) => {
+      t.plan(1)
+      time += 1000
+      const actions = await processor.processEvent({ id:3, type: 'failed-login', keys: { ip }, time }, 0)
+      console.log("ACTIONS", actions)
+      t.deepEqual(actions, ['ban'], 'actions match')
+    })
+
+  })
+
+})
+
+

package/tests/linked-chain-with-expire.test.js

@@ -0,0 +1,145 @@
+const test = require('tape')
+const testDb = require('./testDb.js')
+const lcp = require('@live-change/pattern')
+const { relationsStore } = require('../lib/relations-store.js')
+let model
+
+test("compile fail2ban chain with expire", (t) => {
+
+  t.plan(2)
+
+  let { model: compiled } =
+      lcp.first({ id: "first-failed-attempt", type: "failed-login" })
+      .link({ eq: "ip", expire: "2m" },
+          lcp.first({ id: "second-failed-attempt", type: "failed-login", actions: ['ban'] })
+      )
+
+  console.log(JSON.stringify(compiled, null, '  '))
+
+  model = compiled
+
+  t.deepEqual(model, {
+        "elements": {
+          "first-failed-attempt": {
+            "id": "first-failed-attempt",
+            "type": "failed-login",
+            "prev": [],
+            "next": [
+              "first-failed-attempt/ip@[ip|wait:2m]/second-failed-attempt",
+              "first-failed-attempt/wait:2m@[ip|wait:2m]"
+            ]
+          },
+          "second-failed-attempt": {
+            "id": "second-failed-attempt",
+            "type": "failed-login",
+            "actions": [
+              "ban"
+            ],
+            "prev": [
+              "first-failed-attempt/ip@[ip|wait:2m]/second-failed-attempt"
+            ],
+            "next": []
+          }
+        },
+        "relations": {
+          "first-failed-attempt/ip@[ip|wait:2m]/second-failed-attempt": {
+            "eq": [
+              {
+                "prev": "ip",
+                "next": "ip"
+              }
+            ],
+            "id": "first-failed-attempt/ip@[ip|wait:2m]/second-failed-attempt",
+            "cancel": [
+              "first-failed-attempt/wait:2m@[ip|wait:2m]"
+            ],
+            "prev": [
+              "first-failed-attempt"
+            ],
+            "next": [
+              "second-failed-attempt"
+            ]
+          },
+          "first-failed-attempt/wait:2m@[ip|wait:2m]": {
+            "id": "first-failed-attempt/wait:2m@[ip|wait:2m]",
+            "wait": "2m",
+            "cancel": [
+              "first-failed-attempt/ip@[ip|wait:2m]/second-failed-attempt",
+              "first-failed-attempt/wait:2m@[ip|wait:2m]"
+            ],
+            "prev": [
+              "first-failed-attempt"
+            ],
+            "next": []
+          }
+        }
+      }, "compiled ok")
+
+  t.test('live processor', async (t) => {
+    t.plan(4)
+
+    const db = await testDb()
+    const store = relationsStore(db, 'test', 'relations')
+    await store.createTable()
+
+    lcp.prepareModelForLive(model)
+    const processor = new lcp.LiveProcessor(model, store)
+    const ip = (Math.random()*1000).toFixed()
+
+    let time = 0
+
+    t.test('push first event', async (t) => {
+      t.plan(1)
+      const actions = await processor.processEvent({ type: 'failed-login', keys: { ip }, time }, 0)
+      console.log("ACTIONS", actions)
+      const relations = await store.getRelations('failed-login', { ip })
+      //console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(relations.length > 0) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+    })
+
+    t.test('wait 2.5m for expire', async (t) => {
+      t.plan(1)
+      time += 2.5 * 60 * 1000
+      const actions = await processor.processTime(time)
+      console.log("ACTIONS", actions)
+      const relations = await store.getRelations('failed-login', { ip })
+      //console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(relations.length == 0) {
+        t.pass('expired')
+      } else {
+        t.fail('still exists')
+      }
+    })
+
+    t.test('push second event', async (t) => {
+      t.plan(1)
+      time += 1000
+      const actions = await processor.processEvent({ type: 'failed-login', keys: { ip }, time }, 0)
+      console.log("ACTIONS", actions)
+      const relations = await store.getRelations('failed-login', { ip })
+      //console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(relations.length > 0) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+    })
+
+    t.test('push third event', async (t) => {
+      t.plan(1)
+      time += 1000
+      const actions = await processor.processEvent({ type: 'failed-login', keys: { ip }, time }, 0)
+      console.log("ACTIONS", actions)
+      t.deepEqual(actions, ['ban'],'actions match')
+    })
+
+  })
+
+
+})
+
+

package/tests/linked-simple-chain.test.js

@@ -0,0 +1,124 @@
+const test = require('tape')
+const testDb = require('./testDb.js')
+const lcp = require('@live-change/pattern')
+const { relationsStore } = require('../lib/relations-store.js')
+
+let model
+
+test("compile simple chain", (t) => {
+
+  t.plan(2)
+
+  let { model: compiled } =
+      lcp.first({ id: 'visitor', type: 'enter-website' })
+          .link("sessionId", lcp.first({ id: 'started-registration', type: 'start-register' }))
+          .link("userId", lcp.first({ id: 'registered', type: 'finish-register' }))
+
+  console.log(JSON.stringify(compiled, null, '  '))
+
+  model = compiled
+
+  t.deepEqual(model, {
+    "elements": {
+      "visitor": {
+        "id": "visitor",
+        "type": "enter-website",
+        "prev": [],
+        "next": [
+          "visitor/sessionId/started-registration"
+        ]
+      },
+      "started-registration": {
+        "id": "started-registration",
+        "type": "start-register",
+        "prev": [
+          "visitor/sessionId/started-registration"
+        ],
+        "next": [
+          "started-registration/userId/registered"
+        ]
+      },
+      "registered": {
+        "id": "registered",
+        "type": "finish-register",
+        "prev": [
+          "started-registration/userId/registered"
+        ],
+        "next": []
+      }
+    },
+    "relations": {
+      "visitor/sessionId/started-registration": {
+        "eq": [
+          {
+            "prev": "sessionId",
+            "next": "sessionId"
+          }
+        ],
+        "id": "visitor/sessionId/started-registration",
+        "prev": [
+          "visitor"
+        ],
+        "next": [
+          "started-registration"
+        ]
+      },
+      "started-registration/userId/registered": {
+        "eq": [
+          {
+            "prev": "userId",
+            "next": "userId"
+          }
+        ],
+        "id": "started-registration/userId/registered",
+        "prev": [
+          "started-registration"
+        ],
+        "next": [
+          "registered"
+        ]
+      }
+    }
+  }, 'model compiled')
+
+  t.test('live processor', async (t) => {
+    t.plan(2)
+
+    const db = await testDb()
+    const store = relationsStore(db, 'test', 'relations')
+    await store.createTable()
+
+    lcp.prepareModelForLive(model)
+    const processor = new lcp.LiveProcessor(model, store)
+    const sessionId = (Math.random()*1000).toFixed()
+    const userId = (Math.random()*1000).toFixed()
+
+    t.test('push first event', async (t) => {
+      t.plan(1)
+      await processor.processEvent({ type: 'enter-website', keys: { sessionId }, time: 0 })
+      const relations = await store.getRelations('start-register', { sessionId })
+      //console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(relations.length > 0) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+    })
+
+    t.test('push second event', async (t) => {
+      t.plan(1)
+      await processor.processEvent({ type: 'start-register', keys: { sessionId, userId }, time: 100 })
+      const relations = await store.getRelations('finish-register', { userId })
+      //console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(relations.length > 0) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+    })
+
+  })
+
+})
+
+