midas-edge 0.2.1 → 0.3.2

data/lib/midas/detector.rb

@@ -11,18 +11,7 @@ module Midas
     end
 
     def fit_predict(x)
-      threshold = @threshold || Float::NAN
-      result =
-        if x.is_a?(String)
-          _fit_predict_file(x, @rows, @buckets, @alpha, threshold, @relations, @directed, @seed)
-        else
-          x = Numo::Int32.cast(x) unless x.is_a?(Numo::NArray)
-          x = x.cast_to(Numo::Int32) unless x.is_a?(Numo::Int32)
-          raise ArgumentError, "Bad shape: #{x.shape}" unless x.rank == 2 && x.shape[1] == 3
-          _fit_predict_str(x.to_binary, @rows, @buckets, @alpha, threshold, @relations, @directed, @seed)
-        end
-
-      Numo::SFloat.from_binary(result)
+      _fit_predict(x, @rows, @buckets, @alpha, @threshold || Float::NAN, @relations, @directed, @seed)
     end
   end
 end

data/lib/midas/version.rb

@@ -1,3 +1,3 @@
 module Midas
-  VERSION = "0.2.1"
+  VERSION = "0.3.2"
 end

data/vendor/MIDAS/README.md

@@ -4,15 +4,15 @@
   <a href="https://aaai.org/Conferences/AAAI-20/">
     <img src="http://img.shields.io/badge/AAAI-2020-red.svg">
   </a>
-  <a href="https://www.comp.nus.edu.sg/~sbhatia/assets/pdf/midas.pdf"><img src="http://img.shields.io/badge/Paper-PDF-brightgreen.svg"></a>
+  <a href="https://arxiv.org/pdf/2009.08452.pdf"><img src="http://img.shields.io/badge/Paper-PDF-brightgreen.svg"></a>
   <a href="https://www.comp.nus.edu.sg/~sbhatia/assets/pdf/midasslides.pdf">
       <img src="http://img.shields.io/badge/Slides-PDF-ff9e18.svg">
   </a>
   <a href="https://youtu.be/Bd4PyLCHrto">
     <img src="http://img.shields.io/badge/Talk-Youtube-ff69b4.svg">
   </a>
-  <a href="https://www.kdnuggets.com/2020/04/midas-new-baseline-anomaly-detection-graphs.html"> 
-    <img src="https://img.shields.io/badge/Press-KDnuggets-orange.svg">
+  <a href="https://www.youtube.com/watch?v=DPmN-uPW8qU"> 
+    <img src="https://img.shields.io/badge/Overview-Youtube-orange.svg">
   </a>
   <a href="https://github.com/bhatiasiddharth/MIDAS/blob/master/LICENSE">
     <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg">
@@ -21,8 +21,8 @@
 
 C++ implementation of
 
-- Real-time Streaming Anomaly Detection in Dynamic Graphs. *Siddharth Bhatia, Rui Liu, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. (Under Review)
-- [MIDAS: Microcluster-Based Detector of Anomalies in Edge Streams](asset/Conference.pdf). *Siddharth Bhatia, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. AAAI 2020.
+- [Real-time Streaming Anomaly Detection in Dynamic Graphs](https://arxiv.org/pdf/2009.08452.pdf). *Siddharth Bhatia, Rui Liu, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. (Under Review)
+- [MIDAS: Microcluster-Based Detector of Anomalies in Edge Streams](https://arxiv.org/pdf/1911.04464.pdf). *Siddharth Bhatia, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. AAAI 2020.
 
 The old implementation is in another branch `OldImplementation`, it should be considered as being archived and will hardly receive feature updates.
 
@@ -30,13 +30,20 @@ The old implementation is in another branch `OldImplementation`, it should be co
 
 ## Table of Contents
 
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
 - [Features](#features)
 - [Demo](#demo)
 - [Customization](#customization)
-- [Online Articles](#online-articles)
-- [MIDAS in other Languages](#midas-in-other-languages)
+- [Other Files](#other-files)
+- [In Other Languages](#in-other-languages)
+- [Online Coverage](#online-coverage)
 - [Citation](#citation)
 
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
 ## Features
 
 - Finds Anomalies in Dynamic/Time-Evolving Graph: (Intrusion Detection, Fake Ratings, Financial Fraud)
@@ -45,7 +52,7 @@ The old implementation is in another branch `OldImplementation`, it should be co
 - Constant Memory (independent of graph size)
 - Constant Update Time (real-time anomaly detection to minimize harm)
 - Up to 55% more accurate and 929 times faster than the state of the art approaches
-- Some experiments are performed on the following datasets: 
+- Experiments are performed using the following datasets: 
   - [DARPA](https://www.ll.mit.edu/r-d/datasets/1998-darpa-intrusion-detection-evaluation-dataset)
   - [TwitterWorldCup2014](http://odds.cs.stonybrook.edu/twitterworldcup2014-dataset)
   - [TwitterSecurity](http://odds.cs.stonybrook.edu/twittersecurity-dataset)
@@ -56,31 +63,51 @@ If you use Windows:
 
 1. Open a Visual Studio developer command prompt, we want their toolchain
 1. `cd` to the project root `MIDAS/`
-1. `cmake -DCMAKE_BUILD_TYPE=Release -G "NMake Makefiles" -S . -B build/release`
+1. `cmake -DCMAKE_BUILD_TYPE=Release -GNinja -S . -B build/release`
 1. `cmake --build build/release --target Demo`
-1. `cd` to `MIDAS/build/release/src`
+1. `cd` to `MIDAS/build/release/`
 1. `.\Demo.exe`
 
-If you use Linux/macOS systems:
+If you use Linux/macOS:
 
 1. Open a terminal
 1. `cd` to the project root `MIDAS/`
 1. `cmake -DCMAKE_BUILD_TYPE=Release -S . -B build/release`
 1. `cmake --build build/release --target Demo`
-1. `cd` to `MIDAS/build/release/src`
+1. `cd` to `MIDAS/build/release/`
 1. `./Demo`
 
-The demo runs on `MIDAS/data/DARPA/darpa_processed.csv`, which has 4.5M records, with the filtering core.
+The demo runs on `MIDAS/data/DARPA/darpa_processed.csv`, which has 4.5M records, with the filtering core (MIDAS-F).
 
 The scores will be exported to `MIDAS/temp/Score.txt`, higher means more anomalous.
 
-All file paths are absolute and "hardcoded" by CMake, but it's suggested NOT to run by double-click on the executable file.
+All file paths are absolute and "hardcoded" by CMake, but it's suggested NOT to run by double clicking on the executable file.
+
+### Requirements
+
+Core
+- C++11
+- C++ standard libraries
+
+Demo
+- Python 3 (`MIDAS/util/EvaluateScore.py`)
+    - `pandas`: I/O 
+    - `scikit-learn`: Compute ROC-AUC
+
+Experiment
+- (Optional) Intel TBB: Parallelization
+- (Optional) OpenMP: Parallelization
+
+Other python utility scripts
+- Python 3
+    - `pandas`
+    - `scikit-learn`
 
 ## Customization
 
 ### Switch Cores
 
-Cores are instantiated at `MIDAS/example/Demo.cpp:64-66`, uncomment the chosen one.
+Cores are instantiated at `MIDAS/example/Demo.cpp:67-69`, uncomment the chosen one.
 
 ### Custom Dataset + `Demo.cpp`
 
@@ -89,48 +116,96 @@ You need to prepare three files:
 - Meta file
   - Only includes an integer `N`, the number of records in the dataset
   - Use its path for `pathMeta`
+  - E.g. `MIDAS/data/DARPA/darpa_shape.txt`
 - Data file
   - A header-less csv format file of shape `[N,3]`
   - Columns are sources, destinations, timestamps
   - Use its path for `pathData`
+  - E.g. `MIDAS/data/DARPA/darpa_processed.csv`
 - Label file
   - A header-less csv format file of shape `[N,1]`
   - The corresponding label for data records
     - 0 means normal record
     - 1 means anomalous record
-  - Use its path for `pathGroundTruth` 
+  - Use its path for `pathGroundTruth`
+  - E.g. `MIDAS/data/DARPA/darpa_ground_truth.csv`
 
 ### Custom Dataset + Custom Runner
 
-1. Include the header `MIDAS/CPU/NormalCore.hpp`, `MIDAS/CPU/RelationalCore.hpp` or `MIDAS/CPU/FilteringCore.hpp`
+1. Include the header `MIDAS/src/NormalCore.hpp`, `MIDAS/src/RelationalCore.hpp` or `MIDAS/src/FilteringCore.hpp`
 1. Instantiate cores with required parameters
-1. Call `operator()` on individual data records, it returns the anomaly score for the input record.
+1. Call `operator()` on individual data records, it returns the anomaly score for the input record
+
+## Other Files
+
+### `example/`
+
+#### `Experiment.cpp`
+
+The code we used for experiments.   
+It will try to use Intel TBB or OpenMP for parallelization.  
+You should comment all but only one runner function call in the `main()` as most results are exported to `MIDAS/temp/Experiiment.csv` together with many intermediate files.
+
+#### `Reproducible.cpp`
 
-## Online Articles
+Similar to `Demo.cpp`, but with all random parameters hardcoded and always produce the same result.  
+It's for other developers and us to test if the implementation in other languages can produce acceptable results.  
 
-1. KDnuggets: [Introducing MIDAS: A New Baseline for Anomaly Detection in Graphs](https://www.kdnuggets.com/2020/04/midas-new-baseline-anomaly-detection-graphs.html)
-2. Towards Data Science: [Controlling Fake News using Graphs and Statistics](https://towardsdatascience.com/controlling-fake-news-using-graphs-and-statistics-31ed116a986f)
-2. Towards Data Science: [Anomaly detection in dynamic graphs using MIDAS](https://towardsdatascience.com/anomaly-detection-in-dynamic-graphs-using-midas-e4f8d0b1db45)
-4. Towards AI: [Anomaly Detection with MIDAS](https://medium.com/towards-artificial-intelligence/anomaly-detection-with-midas-2735a2e6dce8)
-5. [AIhub Interview](https://aihub.org/2020/05/01/interview-with-siddharth-bhatia-a-new-approach-for-anomaly-detection/)
+### `util/`
 
-## MIDAS in Other Languages
+`DeleteTempFile.py`, `EvaluateScore.py` and `ReproduceROC.py` will show their usage and a short description when executed without any argument.
 
-1. [Golang](https://github.com/steve0hh/midas) by [Steve Tan](https://github.com/steve0hh)
-2. [Ruby](https://github.com/ankane/midas) by [Andrew Kane](https://github.com/ankane)
-3. [Rust](https://github.com/scooter-dangle/midas_rs) by [Scott Steele](https://github.com/scooter-dangle)
-4. [R](https://github.com/pteridin/MIDASwrappeR) by [Tobias Heidler](https://github.com/pteridin)
-5. [Python](https://github.com/ritesh99rakesh/pyMIDAS) by [Ritesh Kumar](https://github.com/ritesh99rakesh)
+#### `PreprocessData.py`
+
+The code to process the raw dataset into an easy-to-read format.
+Datasets are always assumed to be in a folder in `MIDAS/data/`.    
+It can process the following dataset(s)
+
+- `DARPA/darpa_original.csv` -> `DARPA/darpa_processed.csv`, `DARPA/darpa_ground_truth.csv`, `DARPA/darpa_shape.txt`
+
+## In Other Languages
+
+1. Python: [Rui Liu's MIDAS.Python](https://github.com/liurui39660/MIDAS.Python), [Ritesh Kumar's pyMIDAS](https://github.com/ritesh99rakesh/pyMIDAS)
+1. Golang: [Steve Tan's midas](https://github.com/steve0hh/midas)
+1. Ruby: [Andrew Kane's midas](https://github.com/ankane/midas)
+1. Rust: [Scott Steele's midas_rs](https://github.com/scooter-dangle/midas_rs)
+1. R: [Tobias Heidler's MIDASwrappeR](https://github.com/pteridin/MIDASwrappeR)
+1. Java: [Joshua Tokle's MIDAS-Java](https://github.com/jotok/MIDAS-Java)
+1. Julia: [Ashrya Agrawal's MIDAS.jl](https://github.com/ashryaagr/MIDAS.jl)
+
+## Online Coverage
+
+1. [ACM TechNews](https://technews.acm.org/archives.cfm?fo=2020-05-may/may-06-2020.html)
+1. [AIhub](https://aihub.org/2020/05/01/interview-with-siddharth-bhatia-a-new-approach-for-anomaly-detection/)
+1. [Hacker News](https://news.ycombinator.com/item?id=22802604)
+1. [KDnuggets](https://www.kdnuggets.com/2020/04/midas-new-baseline-anomaly-detection-graphs.html)
+1. [Microsoft](https://techcommunity.microsoft.com/t5/azure-sentinel/announcing-the-azure-sentinel-hackathon-winners/ba-p/1548240)
+1. [Towards Data Science](https://towardsdatascience.com/controlling-fake-news-using-graphs-and-statistics-31ed116a986f)
 
 ## Citation
 
-If you use this code for your research, please consider citing our paper.
+If you use this code for your research, please consider citing our arXiv preprint
+
+```bibtex
+@misc{bhatia2020realtime,
+    title={Real-Time Streaming Anomaly Detection in Dynamic Graphs},
+    author={Siddharth Bhatia and Rui Liu and Bryan Hooi and Minji Yoon and Kijung Shin and Christos Faloutsos},
+    year={2020},
+    eprint={2009.08452},
+    archivePrefix={arXiv},
+    primaryClass={cs.LG}
+}
 
 ```
+
+or our AAAI paper
+
+
+```bibtex
 @inproceedings{bhatia2020midas,
     title="MIDAS: Microcluster-Based Detector of Anomalies in Edge Streams",
     author="Siddharth {Bhatia} and Bryan {Hooi} and Minji {Yoon} and Kijung {Shin} and Christos {Faloutsos}",
     booktitle="AAAI 2020 : The Thirty-Fourth AAAI Conference on Artificial Intelligence",
     year="2020"
 }
-```
+```

data/vendor/MIDAS/src/{EdgeHash.hpp → CountMinSketch.hpp}

@@ -19,7 +19,7 @@
 #include <algorithm>
 
 namespace MIDAS {
-struct EdgeHash {
+struct CountMinSketch {
 	// Fields
 	// --------------------------------------------------------------------------------
 
@@ -33,10 +33,10 @@ struct EdgeHash {
 	// Methods
 	// --------------------------------------------------------------------------------
 
-	EdgeHash() = delete;
-	EdgeHash& operator=(const EdgeHash& b) = delete;
+	CountMinSketch() = delete;
+	CountMinSketch& operator=(const CountMinSketch& b) = delete;
 
-	EdgeHash(int numRow, int numColumn):
+	CountMinSketch(int numRow, int numColumn):
 		r(numRow),
 		c(numColumn),
 		lenData(r * c),
@@ -50,7 +50,7 @@ struct EdgeHash {
 		std::fill(data, data + lenData, 0);
 	}
 
-	EdgeHash(const EdgeHash& b):
+	CountMinSketch(const CountMinSketch& b):
 		r(b.r),
 		c(b.c),
 		lenData(b.lenData),
@@ -62,7 +62,7 @@ struct EdgeHash {
 		std::copy(b.data, b.data + lenData, data);
 	}
 
-	~EdgeHash() {
+	~CountMinSketch() {
 		delete[] param1;
 		delete[] param2;
 		delete[] data;
@@ -73,10 +73,11 @@ struct EdgeHash {
 	}
 
 	void MultiplyAll(float by) const {
-		std::for_each(data, data + lenData, [&](float& a) { a *= by; }); // Magic of vectorization
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			data[i] *= by;
 	}
 
-	void Hash(int a, int b, int* indexOut) const {
+	void Hash(int* indexOut, int a, int b = 0) const {
 		for (int i = 0; i < r; i++) {
 			indexOut[i] = ((a + m * b) * param1[i] + param2[i]) % c;
 			indexOut[i] += i * c + (indexOut[i] < 0 ? c : 0);
@@ -90,10 +91,10 @@ struct EdgeHash {
 		return least;
 	}
 
-	float Assign(const int* index, float to) const {
+	float Assign(const int* index, float with) const {
 		for (int i = 0; i < r; i++)
-			data[index[i]] = to;
-		return to;
+			data[index[i]] = with;
+		return with;
 	}
 
 	void Add(const int* index, float by = 1) const {

data/vendor/MIDAS/src/FilteringCore.hpp

@@ -17,26 +17,28 @@
 #pragma once
 
 #include <cmath>
-#include <algorithm>
 
-#include "EdgeHash.hpp"
-#include "NodeHash.hpp"
+#include "CountMinSketch.hpp"
 
 namespace MIDAS {
 struct FilteringCore {
 	const float threshold;
-	int timestampCurrent = 1;
+	int timestamp = 1;
 	const float factor;
-	int* const indexEdge; // Pre-compute the index to-be-modified, thanks to the same structure of CMSs
+	const int lenData;
+	int* const indexEdge; // Pre-compute the index to-be-modified, thanks to the Same-Layout Assumption
 	int* const indexSource;
 	int* const indexDestination;
-	EdgeHash numCurrentEdge, numTotalEdge, scoreEdge;
-	NodeHash numCurrentSource, numTotalSource, scoreSource;
-	NodeHash numCurrentDestination, numTotalDestination, scoreDestination;
+	CountMinSketch numCurrentEdge, numTotalEdge, scoreEdge;
+	CountMinSketch numCurrentSource, numTotalSource, scoreSource;
+	CountMinSketch numCurrentDestination, numTotalDestination, scoreDestination;
+	float timestampReciprocal = 0;
+	bool* const shouldMerge;
 
 	FilteringCore(int numRow, int numColumn, float threshold, float factor = 0.5):
 		threshold(threshold),
 		factor(factor),
+		lenData(numRow * numColumn), // I assume all CMSs have same size, but Same-Layout Assumption is not that strict
 		indexEdge(new int[numRow]),
 		indexSource(new int[numRow]),
 		indexDestination(new int[numRow]),
@@ -48,42 +50,43 @@ struct FilteringCore {
 		scoreSource(numCurrentSource),
 		numCurrentDestination(numRow, numColumn),
 		numTotalDestination(numCurrentDestination),
-		scoreDestination(numCurrentDestination) { }
+		scoreDestination(numCurrentDestination),
+		shouldMerge(new bool[numRow * numColumn]) { }
 
 	virtual ~FilteringCore() {
 		delete[] indexEdge;
 		delete[] indexSource;
 		delete[] indexDestination;
+		delete[] shouldMerge;
 	}
 
 	static float ComputeScore(float a, float s, float t) {
 		return s == 0 ? 0 : pow(a + s - a * t, 2) / (s * (t - 1)); // If t == 1, then s == 0, so no need to check twice
 	}
 
+	void ConditionalMerge(const float* current, float* total, const float* score) const {
+		for (int i = 0; i < lenData; i++)
+			shouldMerge[i] = score[i] < threshold;
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			total[i] += shouldMerge[i] * current[i] + (true - shouldMerge[i]) * total[i] * timestampReciprocal;
+	}
+
 	float operator()(int source, int destination, int timestamp) {
-		if (timestamp > timestampCurrent) {
-			for (int i = 0; i < numCurrentEdge.lenData; i++)
-				numTotalEdge.data[i] += scoreEdge.data[i] < threshold ?
-					numCurrentEdge.data[i] : timestampCurrent - 1 ?
-						numTotalEdge.data[i] / (timestampCurrent - 1) : 0;
-			for (int i = 0; i < numCurrentSource.lenData; i++)
-				numTotalSource.data[i] += scoreSource.data[i] < threshold ?
-					numCurrentSource.data[i] : timestampCurrent - 1 ?
-						numTotalSource.data[i] / (timestampCurrent - 1) : 0;
-			for (int i = 0; i < numCurrentDestination.lenData; i++)
-				numTotalDestination.data[i] += scoreDestination.data[i] < threshold ?
-					numCurrentDestination.data[i] : timestampCurrent - 1 ?
-						numTotalDestination.data[i] / (timestampCurrent - 1) : 0;
+		if (this->timestamp < timestamp) {
+			ConditionalMerge(numCurrentEdge.data, numTotalEdge.data, scoreEdge.data);
+			ConditionalMerge(numCurrentSource.data, numTotalSource.data, scoreSource.data);
+			ConditionalMerge(numCurrentDestination.data, numTotalDestination.data, scoreDestination.data);
 			numCurrentEdge.MultiplyAll(factor);
 			numCurrentSource.MultiplyAll(factor);
 			numCurrentDestination.MultiplyAll(factor);
-			timestampCurrent = timestamp;
+			timestampReciprocal = 1.f / (timestamp - 1); // So I can skip an if-statement
+			this->timestamp = timestamp;
 		}
-		numCurrentEdge.Hash(source, destination, indexEdge);
+		numCurrentEdge.Hash(indexEdge, source, destination);
 		numCurrentEdge.Add(indexEdge);
-		numCurrentSource.Hash(source, indexSource);
+		numCurrentSource.Hash(indexSource, source);
 		numCurrentSource.Add(indexSource);
-		numCurrentDestination.Hash(destination, indexDestination);
+		numCurrentDestination.Hash(indexDestination, destination);
 		numCurrentDestination.Add(indexDestination);
 		return std::max({
 			scoreEdge.Assign(indexEdge, ComputeScore(numCurrentEdge(indexEdge), numTotalEdge(indexEdge), timestamp)),

data/vendor/MIDAS/src/NormalCore.hpp

@@ -18,13 +18,13 @@
 
 #include <cmath>
 
-#include "EdgeHash.hpp"
+#include "CountMinSketch.hpp"
 
 namespace MIDAS {
 struct NormalCore {
-	int timestampCurrent = 1;
+	int timestamp = 1;
 	int* const index; // Pre-compute the index to-be-modified, thanks to the same structure of CMSs
-	EdgeHash numCurrent, numTotal;
+	CountMinSketch numCurrent, numTotal;
 
 	NormalCore(int numRow, int numColumn):
 		index(new int[numRow]),
@@ -40,11 +40,11 @@ struct NormalCore {
 	}
 
 	float operator()(int source, int destination, int timestamp) {
-		if (timestamp > timestampCurrent) {
+		if (this->timestamp < timestamp) {
 			numCurrent.ClearAll();
-			timestampCurrent = timestamp;
+			this->timestamp = timestamp;
 		}
-		numCurrent.Hash(source, destination, index);
+		numCurrent.Hash(index, source, destination);
 		numCurrent.Add(index);
 		numTotal.Add(index);
 		return ComputeScore(numCurrent(index), numTotal(index), timestamp);