@live-change/pattern 0.2.0 → 0.2.1

package/index.js

@@ -5,5 +5,6 @@ module.exports = {
   ...require('./lib/graph.js'),
   ...require('./lib/combinations.js'),
   ...require('./lib/duration.js'),
-  ...require('./lib/relations-store.js')
+  ...require('./lib/relations-store.js'),
+  ...require('./lib/match.js'),
 }

package/lib/graph.js

@@ -5,13 +5,13 @@ function updateGraph(event, transitions, aggregation, addToNode, addToRelation)
   for(const transition of transitions) {
     if(transition.relation && mark) mark(transition.relation)
     const ids = aggregation.nodes(transition, event.keys)
+    console.log("TR", transition, 'KEYS', event.keys, 'IDS', ids)
     for(let id of ids) {
       addToNode(id, transition, event, !transition.relation)
       if (transition.relation != null) {
         for(const prev of transition.relation.prev) {
-          //console.log("PREV", prev, transition.relation)
           const prevIds = aggregation.nodes(prev, transition.relation && transition.relation.keys)
-          for (const prevId of prevIds) {
+          for(const prevId of prevIds) {
             addToRelation(id, prevId, transition, event, true)
             addToRelation(prevId, id, transition, event, false)
           }
@@ -69,10 +69,10 @@ const agg = {
     relationEq: (transition, otherId) => (rel) => rel.to == otherId,
   },
   nodeElementDepth: {
-    nodes: ({ element, relation }, keys) => [element + ':' + (relation ? relation.depth : 0)],
+    nodes: ({ element, relation }, keys) => [element + ':' + (relation?.depth ?? 0)],
     mark: (event) =>
         (rel) => rel.depth = (rel.prev && rel.prev.length)
-            ? rel.prev.reduce((a, b) => Math.max(a, b.relation ? b.relation.depth : 0), 0) + 1
+            ? rel.prev.reduce((a, b) => Math.max(a, b?.relation?.depth ?? 0), 0) + 1
             : 0
   },
   nodeElement: {
@@ -106,30 +106,32 @@ class SummaryGraphProcessor extends LiveProcessor {
     this.removeCanceledRelations(canceledRelations)
 
     await updateGraph(event, transitions, this.aggregation,
-        (id, transition, event, start) => {
-          let node = this.graph.get(id)
-          if (!node) {
-            node = {
-              id,
-              prev: [],
-              next: [],
-              start: false
-            }
-            this.graph.set(id, node)
+      (id, transition, event, start) => {
+        let node = this.graph.get(id)
+        if (!node) {
+          node = {
+            id,
+            prev: [],
+            next: [],
+            start: false
           }
-          this.aggregation.addToNode(node, transition, event)
-          if(start) node.start = true
-        },
-        (fromId, toId, transition, event, prev) => {
-          let node = this.graph.get(fromId)
-          const list = (prev ? node.prev : node.next)
-          let rel = list.find(this.aggregation.relationEq(transition, toId))
-          if(!rel) {
-            rel = this.aggregation.relationFactory(transition, toId)
-            list.push(rel)
-          }
-          this.aggregation.addToRelation(rel, transition, event)
+          this.graph.set(id, node)
+        }
+        this.aggregation.addToNode(node, transition, event)
+        if(start) node.start = true
+      },
+      (fromId, toId, transition, event, prev) => {
+        let node = this.graph.get(fromId)
+        if(!node) console.log("NODE", node, fromId)
+        if(!node) throw new Error('node '+fromId+' not found when adding relation')
+        const list = (prev ? node.prev : node.next)
+        let rel = list.find(this.aggregation.relationEq(transition, toId))
+        if(!rel) {
+          rel = this.aggregation.relationFactory(transition, toId)
+          list.push(rel)
         }
+        this.aggregation.addToRelation(rel, transition, event)
+      }
     )
   }
 }

package/lib/live.js

@@ -1,4 +1,5 @@
 const { parseDuration } = require('./duration.js')
+const { eventElementMatch } = require('./match.js')
 
 function relationDescriptors(model, source, now, relation, keys, prev, withCancel = true) {
   let descriptors = []
@@ -45,8 +46,9 @@ function findDescriptor(that, list) {
 }
 
 async function processEvent(event, model, getRelationsByEvent) {
-  const { type, keys } = event
+  const { type, keys, properties } = event
   const nextRelations = await getRelationsByEvent(type, keys)
+  //console.log("EVENT", event, "NEXT RELATIONS", nextRelations)
 
   let newRelations = []
   let canceledRelations = []
@@ -58,7 +60,7 @@ async function processEvent(event, model, getRelationsByEvent) {
     if(findDescriptor(relationDescriptor, canceledRelations)) continue
     const relationModel = model.relations[relationDescriptor.relation]
     //console.log("NEXT RELATION MODEL", relationModel)
-    const nextElements = relationModel.next.filter(id => model.elements[id].type == type)
+    const nextElements = relationModel.next.filter(id => eventElementMatch(event, model.elements[id]))
     //console.log("NEXT ELEMENTS", nextElements)
     if (nextElements.length > 0) {
       if(relationModel.cancel) {
@@ -83,6 +85,7 @@ async function processEvent(event, model, getRelationsByEvent) {
   if(newElements) {
     for (const element of newElements) {
       const elementModel = model.elements[element]
+      if(!eventElementMatch(event, elementModel)) continue
       //console.log("NEW ELEMENT", element, elementModel)
       let transition = { element, relation: null }
       transitions.push(transition)
@@ -161,9 +164,10 @@ class LiveProcessor {
 
     const { newRelations, canceledRelations, actions } = changes
 
-   /* console.log("PROCESSING RESULTS:")
+    /*
+    console.log("PROCESSING RESULTS:")
     console.log("NR: ", newRelations)
-    console.log("CR: ", canceledRelations)*/
+    console.log("CR: ", canceledRelations)//*/
 
     await this.applyChanges(changes, event)
     return actions
@@ -197,8 +201,10 @@ class LiveProcessor {
   }
 
   async applyChanges({ newRelations, canceledRelations }, event) {
-    this.addNewRelations(newRelations)
-    this.removeCanceledRelations(canceledRelations)
+    await Promise.all([
+      this.addNewRelations(newRelations),
+      this.removeCanceledRelations(canceledRelations)
+    ])
   }
 }
 

package/lib/match.js

@@ -0,0 +1,27 @@
+function matchProperties(match, properties) {
+  for(const key in match) {
+    const value = match[key]
+    const property = properties[key]
+    if(typeof value == 'object') {
+      if(typeof property != 'object') return false
+      if(!matchProperties(value, property)) return false
+    } else {
+      if(value != property) return false
+    }
+  }
+  return true
+}
+
+function eventElementMatch(event, element) {
+  console.log("CHECK ELEMENT", element)
+  if(element.type != event.type) return false
+  if(element.match) {
+    console.log("MATCH PROPERTIES", event, element)
+    if(!event.properties) return false
+    if(!matchProperties(element.match, event.properties)) return false
+  }
+  console.log("ELEMENT FOUND", event, element)
+  return true
+}
+
+module.exports = { eventElementMatch, matchProperties }

package/lib/relations-store.js

@@ -18,7 +18,7 @@ function relationsStore() {
   let eventRelations = new Map() // By event and key and value
   let eventRelationsBySource = new Map()
 
-  async function getRelations(type, keys) {
+  async function getRelations(type, keys, properties) {
     const keyIds = eventRelationKeys(type, keys)
     const found = keyIds.map(key => eventRelations.get(key) || []).reduce((a,b) => a.concat(b), [])
     if(found.length == 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/pattern",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Realtime usage pattern analysis for security, analytics and gamification purposes",
   "main": "index.js",
   "scripts": {
@@ -10,11 +10,7 @@
     "type": "git",
     "url": "git+https://github.com/live-change/pattern.git"
   },
-  "author": {
-    "email": "m8@em8.pl",
-    "name": "Michał Łaszczewski",
-    "url": "https://www.viamage.com/"
-  },
+  "author": "Michał Łaszczewski <m8@em8.pl> (https://www.viamage.com/)",
   "license": "MIT",
   "bugs": {
     "url": "https://github.com/live-change/pattern/issues"
@@ -26,5 +22,10 @@
     "d3-path-arrows": "^0.4.0",
     "d3-sankey-circular": "^0.34.0",
     "tape": "^4.13.0"
-  }
+  },
+  "directories": {
+    "lib": "lib",
+    "test": "tests"
+  },
+  "keywords": []
 }

package/tests/{chain-with-expire.js → chain-with-expire.test.js}

@@ -1,5 +1,5 @@
 const test = require('tape')
-const rp = require('../index.js')
+const lcp = require('../index.js')
 
 let model
 
@@ -7,7 +7,7 @@ test("compile fail2ban chain with expire", (t) => {
 
   t.plan(2)
 
-  let { model: compiled, last } = rp.chain([
+  let { model: compiled, last } = lcp.chain([
     "failed-login",
     { eq: "ip", expire: "2m" },
     "failed-login"
@@ -78,8 +78,8 @@ test("compile fail2ban chain with expire", (t) => {
   t.test('live processor', (t) => {
     t.plan(4)
 
-    rp.prepareModelForLive(model)
-    const processor = new rp.LiveProcessor(model, rp.relationsStore())
+    lcp.prepareModelForLive(model)
+    const processor = new lcp.LiveProcessor(model, lcp.relationsStore())
     const ip = (Math.random()*1000).toFixed()
     let time = 0
 
@@ -89,6 +89,8 @@ test("compile fail2ban chain with expire", (t) => {
       console.log("EVT", processor.store.eventRelations)
       console.log("TO", processor.timeouts)
       console.log("ACTIONS", actions)
+      const relations = await processor.store.getRelations('failed-login', { ip })
+      console.log("RELATIONS", JSON.stringify(relations, null, "  "))
       if(processor.store.eventRelations.get(`["failed-login",[["ip","${ip}"]]]`)) t.pass('processed')
       else t.fail('no reaction')
     })

package/tests/chain-with-filter.test.js

@@ -0,0 +1,192 @@
+const test = require('tape')
+const lcp = require('../index.js')
+
+let model
+
+test("compile fail2ban chain with expire", (t) => {
+
+  t.plan(3)
+
+  let { model: compiled, last } = lcp.chain([
+    { type: "failed-authentication", match: { secretType: 'code' } },
+    "ip",
+    { type: "failed-authentication", match: { secretType: 'code' } }
+  ])
+
+  compiled.elements[last].actions = [ 'ban' ]
+
+  console.log(JSON.stringify(compiled, null, '  '))
+
+  model = compiled
+
+  t.deepEqual(model, {
+    "elements": {
+      "failed-authentication": {
+        "type": "failed-authentication",
+        "match": {
+          "secretType": "code"
+        },
+        "id": "failed-authentication",
+        "prev": [],
+        "next": [
+          "failed-authentication/ip/failed-authentication"
+        ]
+      },
+      "failed-authentication/ip/failed-authentication": {
+        "type": "failed-authentication",
+        "match": {
+          "secretType": "code"
+        },
+        "id": "failed-authentication/ip/failed-authentication",
+        "prev": [
+          "failed-authentication/ip/failed-authentication"
+        ],
+        "next": [],
+        "actions": [
+          "ban"
+        ]
+      }
+    },
+    "relations": {
+      "failed-authentication/ip/failed-authentication": {
+        "eq": [
+          {
+            "prev": "ip",
+            "next": "ip"
+          }
+        ],
+        "id": "failed-authentication/ip/failed-authentication",
+        "prev": [
+          "failed-authentication"
+        ],
+        "next": [
+          "failed-authentication/ip/failed-authentication"
+        ]
+      }
+    }
+  })
+
+  t.test('live processor with second event filtered', (t) => {
+    t.plan(3)
+
+    lcp.prepareModelForLive(model)
+    const processor = new lcp.LiveProcessor(model, lcp.relationsStore())
+    const ip = (Math.random() * 1000).toFixed()
+    let time = 0
+
+    t.test('push first event', async (t) => {
+      t.plan(1)
+      const actions = await processor.processEvent({
+        id: 1, type: 'failed-authentication', keys: { ip }, properties: { secretType: 'code' }, time
+      }, 0)
+      console.log("EVT", processor.store.eventRelations)
+      console.log("TO", processor.timeouts)
+      console.log("ACTIONS", actions)
+      const relations = await processor.store.getRelations('failed-authentication', { ip })
+      console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(processor.store.eventRelations.get(`["failed-authentication",[["ip","${ip}"]]]`)) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+    })
+
+    t.test('push second event', async (t) => {
+      t.plan(2)
+      time += 1000
+      const actions = await processor.processEvent({
+        id: 2, type: 'failed-authentication', keys: { ip }, properties: { secretType: 'link' }, time
+      }, 0)
+      console.log("EVT", processor.store.eventRelations)
+      console.log("TO", processor.timeouts)
+      const relations = await processor.store.getRelations('failed-authentication', { ip })
+      console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      console.log("ACTIONS", actions)
+      if(actions.length > 0) {
+        t.fail('actions where it should be filtered')
+      } else {
+        t.pass('event filtered')
+      }
+      if(processor.store.eventRelations.get(`["failed-authentication",[["ip","${ip}"]]]`)) {
+        t.pass('previous event exist')
+      } else {
+        t.fail('previous event not exist')
+      }
+    })
+
+    t.test('push third event', async (t) => {
+      t.plan(1)
+      time += 1000
+      const actions = await processor.processEvent({
+        id: 3, type: 'failed-authentication', keys: { ip }, properties: { secretType: 'code' },  time
+      }, 0)
+      console.log("ACTIONS", actions)
+      t.deepEqual(actions, ['ban'], 'actions match')
+    })
+
+  })
+
+  t.test('live processor with first event filtered', (t) => {
+    t.plan(3)
+
+    lcp.prepareModelForLive(model)
+    const processor = new lcp.LiveProcessor(model, lcp.relationsStore())
+    const ip = (Math.random() * 1000).toFixed()
+    let time = 0
+
+    t.test('push first event', async (t) => {
+      t.plan(1)
+      const actions = await processor.processEvent({
+        id: 1, type: 'failed-authentication', keys: { ip }, properties: { secretType: 'link' }, time
+      }, 0)
+      console.log("EVT", processor.store.eventRelations)
+      console.log("TO", processor.timeouts)
+      console.log("ACTIONS", actions)
+      const relations = await processor.store.getRelations('failed-authentication', { ip })
+      console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(!processor.store.eventRelations.get(`["failed-authentication",[["ip","${ip}"]]]`)) {
+        t.pass('filtered')
+      } else {
+        t.fail('processed')
+      }
+    })
+
+    t.test('push second event', async (t) => {
+      t.plan(2)
+      time += 1000
+      const actions = await processor.processEvent({
+        id: 1, type: 'failed-authentication', keys: { ip }, properties: { secretType: 'code' }, time
+      }, 0)
+      console.log("EVT", processor.store.eventRelations)
+      console.log("TO", processor.timeouts)
+      console.log("ACTIONS", actions)
+      const relations = await processor.store.getRelations('failed-authentication', { ip })
+      console.log("RELATIONS", JSON.stringify(relations, null, "  "))
+      if(processor.store.eventRelations.get(`["failed-authentication",[["ip","${ip}"]]]`)) {
+        t.pass('processed')
+      } else {
+        t.fail('no reaction')
+      }
+      console.log("ACTIONS", actions)
+      if(actions.length > 0) {
+        t.fail('actions where it should be filtered')
+      } else {
+        t.pass('no action')
+      }
+    })
+
+    t.test('push third event', async (t) => {
+      t.plan(1)
+      time += 1000
+      const actions = await processor.processEvent({
+        id: 3, type: 'failed-authentication', keys: { ip }, properties: { secretType: 'code' },  time
+      }, 0)
+      console.log("ACTIONS", actions)
+      t.deepEqual(actions, ['ban'], 'actions match')
+    })
+
+  })
+
+})
+
+

package/tests/{viral-loop.js → viral-loop.test.js}

@@ -4,7 +4,7 @@ const svg = require('./svg.js')
 
 let model
 
-test("simple chain", (t) => {
+test("viral loop", (t) => {
   t.plan(4)
 
   t.test('compile', (t) => {

package/.idea/$CACHE_FILE$

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="NodePackageJsonFileManager">
-    <packageJsonPaths />
-  </component>
-</project>

package/.idea/$PRODUCT_WORKSPACE_FILE$

@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="masterDetails">
-    <states>
-      <state key="ProjectJDKs.UI">
-        <settings>
-          <last-edited>10</last-edited>
-          <splitter-proportions>
-            <option name="proportions">
-              <list>
-                <option value="0.2" />
-              </list>
-            </option>
-          </splitter-proportions>
-        </settings>
-      </state>
-    </states>
-  </component>
-</project>

package/.idea/codeStyles/Project.xml

@@ -1,42 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <code_scheme name="Project" version="173">
-    <option name="OTHER_INDENT_OPTIONS">
-      <value>
-        <option name="INDENT_SIZE" value="2" />
-        <option name="TAB_SIZE" value="2" />
-      </value>
-    </option>
-    <codeStyleSettings language="CSS">
-      <indentOptions>
-        <option name="INDENT_SIZE" value="2" />
-        <option name="CONTINUATION_INDENT_SIZE" value="4" />
-        <option name="TAB_SIZE" value="2" />
-      </indentOptions>
-    </codeStyleSettings>
-    <codeStyleSettings language="HTML">
-      <indentOptions>
-        <option name="INDENT_SIZE" value="2" />
-        <option name="CONTINUATION_INDENT_SIZE" value="4" />
-        <option name="TAB_SIZE" value="2" />
-      </indentOptions>
-    </codeStyleSettings>
-    <codeStyleSettings language="JSON">
-      <indentOptions>
-        <option name="CONTINUATION_INDENT_SIZE" value="4" />
-        <option name="TAB_SIZE" value="2" />
-      </indentOptions>
-    </codeStyleSettings>
-    <codeStyleSettings language="JavaScript">
-      <indentOptions>
-        <option name="INDENT_SIZE" value="2" />
-        <option name="TAB_SIZE" value="2" />
-      </indentOptions>
-    </codeStyleSettings>
-    <codeStyleSettings language="SCSS">
-      <indentOptions>
-        <option name="CONTINUATION_INDENT_SIZE" value="4" />
-        <option name="TAB_SIZE" value="2" />
-      </indentOptions>
-    </codeStyleSettings>
-  </code_scheme>
-</component>

package/.idea/codeStyles/codeStyleConfig.xml

@@ -1,5 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <state>
-    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
-  </state>
-</component>

package/.idea/misc.xml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="JavaScriptSettings">
-    <option name="languageLevel" value="ES6" />
-  </component>
-  <component name="ProjectRootManager">
-    <output url="file://$PROJECT_DIR$/out" />
-  </component>
-</project>

package/.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/reactive-pattern.iml" filepath="$PROJECT_DIR$/reactive-pattern.iml" />
-    </modules>
-  </component>
-</project>

package/.idea/workspace.xml

@@ -1,83 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ChangeListManager">
-    <list default="true" id="4bcde310-3a88-4ac7-bcec-b705edb985bb" name="Default Changelist" comment="" />
-    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
-    <option name="SHOW_DIALOG" value="false" />
-    <option name="HIGHLIGHT_CONFLICTS" value="true" />
-    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
-    <option name="LAST_RESOLUTION" value="IGNORE" />
-  </component>
-  <component name="CodeStyleSettingsInfer">
-    <option name="done" value="true" />
-  </component>
-  <component name="ComposerSettings">
-    <execution>
-      <executable />
-    </execution>
-  </component>
-  <component name="FileTemplateManagerImpl">
-    <option name="RECENT_TEMPLATES">
-      <list>
-        <option value="JavaScript File" />
-      </list>
-    </option>
-  </component>
-  <component name="JsFlowSettings">
-    <service-enabled>true</service-enabled>
-    <exe-path />
-    <other-services-enabled>true</other-services-enabled>
-    <auto-save>true</auto-save>
-  </component>
-  <component name="ProjectCodeStyleSettingsMigration">
-    <option name="version" value="1" />
-  </component>
-  <component name="ProjectId" id="1UEIRU0fnQMKUQE3vydpnEMLQQ0" />
-  <component name="PropertiesComponent">
-    <property name="WebServerToolWindowFactoryState" value="false" />
-    <property name="javascript.nodejs.core.library.configured.version" value="10.17.0" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
-    <property name="nodejs_package_manager_path" value="npm" />
-    <property name="settings.editor.selected.configurable" value="Settings.JavaScript" />
-  </component>
-  <component name="RecentsManager">
-    <key name="MoveFile.RECENT_KEYS">
-      <recent name="$PROJECT_DIR$/.." />
-    </key>
-    <key name="CopyFile.RECENT_KEYS">
-      <recent name="$PROJECT_DIR$/tests" />
-    </key>
-  </component>
-  <component name="RunDashboard">
-    <option name="ruleStates">
-      <list>
-        <RuleState>
-          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
-        </RuleState>
-        <RuleState>
-          <option name="name" value="StatusDashboardGroupingRule" />
-        </RuleState>
-      </list>
-    </option>
-  </component>
-  <component name="SvnConfiguration">
-    <configuration />
-  </component>
-  <component name="TaskManager">
-    <task active="true" id="Default" summary="Default task">
-      <changelist id="4bcde310-3a88-4ac7-bcec-b705edb985bb" name="Default Changelist" comment="" />
-      <created>1574915319580</created>
-      <option name="number" value="Default" />
-      <option name="presentableId" value="Default" />
-      <updated>1574915319580</updated>
-      <workItem from="1574915325078" duration="15213000" />
-      <workItem from="1575015431540" duration="5716000" />
-      <workItem from="1575024741938" duration="3246000" />
-      <workItem from="1578040657122" duration="69000" />
-    </task>
-    <servers />
-  </component>
-  <component name="TypeScriptGeneratedFilesManager">
-    <option name="version" value="1" />
-  </component>
-</project>

package/reactive-pattern.iml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="WEB_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$USER_HOME$/IdeaProjects/reactive-pattern" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>