midas-edge 0.2.0 → 0.3.1

data/lib/midas/detector.rb

@@ -11,18 +11,7 @@ module Midas
     end
 
     def fit_predict(x)
-      threshold = @threshold || Float::NAN
-      result =
-        if x.is_a?(String)
-          _fit_predict_file(x, @rows, @buckets, @alpha, threshold, @relations, @directed, @seed)
-        else
-          x = Numo::Int32.cast(x) unless x.is_a?(Numo::NArray)
-          x = x.cast_to(Numo::Int32) unless x.is_a?(Numo::Int32)
-          raise ArgumentError, "Bad shape: #{x.shape}" unless x.rank == 2 && x.shape[1] == 3
-          _fit_predict_str(x.to_binary, @rows, @buckets, @alpha, threshold, @relations, @directed, @seed)
-        end
-
-      Numo::SFloat.from_binary(result)
+      _fit_predict(x, @rows, @buckets, @alpha, @threshold || Float::NAN, @relations, @directed, @seed)
     end
   end
 end

data/lib/midas/version.rb

@@ -1,3 +1,3 @@
 module Midas
-  VERSION = "0.2.0"
+  VERSION = "0.3.1"
 end

data/vendor/MIDAS/README.md

@@ -4,15 +4,15 @@
   <a href="https://aaai.org/Conferences/AAAI-20/">
     <img src="http://img.shields.io/badge/AAAI-2020-red.svg">
   </a>
-  <a href="https://www.comp.nus.edu.sg/~sbhatia/assets/pdf/midas.pdf"><img src="http://img.shields.io/badge/Paper-PDF-brightgreen.svg"></a>
+  <a href="https://arxiv.org/pdf/2009.08452.pdf"><img src="http://img.shields.io/badge/Paper-PDF-brightgreen.svg"></a>
   <a href="https://www.comp.nus.edu.sg/~sbhatia/assets/pdf/midasslides.pdf">
       <img src="http://img.shields.io/badge/Slides-PDF-ff9e18.svg">
   </a>
   <a href="https://youtu.be/Bd4PyLCHrto">
     <img src="http://img.shields.io/badge/Talk-Youtube-ff69b4.svg">
   </a>
-  <a href="https://www.kdnuggets.com/2020/04/midas-new-baseline-anomaly-detection-graphs.html"> 
-    <img src="https://img.shields.io/badge/Press-KDnuggets-orange.svg">
+  <a href="https://www.youtube.com/watch?v=DPmN-uPW8qU"> 
+    <img src="https://img.shields.io/badge/Overview-Youtube-orange.svg">
   </a>
   <a href="https://github.com/bhatiasiddharth/MIDAS/blob/master/LICENSE">
     <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg">
@@ -21,8 +21,8 @@
 
 C++ implementation of
 
-- Real-time Streaming Anomaly Detection in Dynamic Graphs. *Siddharth Bhatia, Rui Liu, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. (Under Review)
-- [MIDAS: Microcluster-Based Detector of Anomalies in Edge Streams](asset/Conference.pdf). *Siddharth Bhatia, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. AAAI 2020.
+- [Real-time Streaming Anomaly Detection in Dynamic Graphs](https://arxiv.org/pdf/2009.08452.pdf). *Siddharth Bhatia, Rui Liu, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. (Under Review)
+- [MIDAS: Microcluster-Based Detector of Anomalies in Edge Streams](https://arxiv.org/pdf/1911.04464.pdf). *Siddharth Bhatia, Bryan Hooi, Minji Yoon, Kijung Shin, Christos Faloutsos*. AAAI 2020.
 
 The old implementation is in another branch `OldImplementation`, it should be considered as being archived and will hardly receive feature updates.
 
@@ -30,13 +30,20 @@ The old implementation is in another branch `OldImplementation`, it should be co
 
 ## Table of Contents
 
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
 - [Features](#features)
 - [Demo](#demo)
 - [Customization](#customization)
-- [Online Articles](#online-articles)
-- [MIDAS in other Languages](#midas-in-other-languages)
+- [Other Files](#other-files)
+- [In Other Languages](#in-other-languages)
+- [Online Coverage](#online-coverage)
 - [Citation](#citation)
 
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
 ## Features
 
 - Finds Anomalies in Dynamic/Time-Evolving Graph: (Intrusion Detection, Fake Ratings, Financial Fraud)
@@ -45,7 +52,7 @@ The old implementation is in another branch `OldImplementation`, it should be co
 - Constant Memory (independent of graph size)
 - Constant Update Time (real-time anomaly detection to minimize harm)
 - Up to 55% more accurate and 929 times faster than the state of the art approaches
-- Some experiments are performed on the following datasets: 
+- Experiments are performed using the following datasets: 
   - [DARPA](https://www.ll.mit.edu/r-d/datasets/1998-darpa-intrusion-detection-evaluation-dataset)
   - [TwitterWorldCup2014](http://odds.cs.stonybrook.edu/twitterworldcup2014-dataset)
   - [TwitterSecurity](http://odds.cs.stonybrook.edu/twittersecurity-dataset)
@@ -56,31 +63,51 @@ If you use Windows:
 
 1. Open a Visual Studio developer command prompt, we want their toolchain
 1. `cd` to the project root `MIDAS/`
-1. `cmake -DCMAKE_BUILD_TYPE=Release -G "NMake Makefiles" -S . -B build/release`
+1. `cmake -DCMAKE_BUILD_TYPE=Release -GNinja -S . -B build/release`
 1. `cmake --build build/release --target Demo`
-1. `cd` to `MIDAS/build/release/src`
+1. `cd` to `MIDAS/build/release/`
 1. `.\Demo.exe`
 
-If you use Linux/macOS systems:
+If you use Linux/macOS:
 
 1. Open a terminal
 1. `cd` to the project root `MIDAS/`
 1. `cmake -DCMAKE_BUILD_TYPE=Release -S . -B build/release`
 1. `cmake --build build/release --target Demo`
-1. `cd` to `MIDAS/build/release/src`
+1. `cd` to `MIDAS/build/release/`
 1. `./Demo`
 
-The demo runs on `MIDAS/data/DARPA/darpa_processed.csv`, which has 4.5M records, with the filtering core.
+The demo runs on `MIDAS/data/DARPA/darpa_processed.csv`, which has 4.5M records, with the filtering core (MIDAS-F).
 
 The scores will be exported to `MIDAS/temp/Score.txt`, higher means more anomalous.
 
-All file paths are absolute and "hardcoded" by CMake, but it's suggested NOT to run by double-click on the executable file.
+All file paths are absolute and "hardcoded" by CMake, but it's suggested NOT to run by double clicking on the executable file.
+
+### Requirements
+
+Core
+- C++11
+- C++ standard libraries
+
+Demo
+- Python 3 (`MIDAS/util/EvaluateScore.py`)
+    - `pandas`: I/O 
+    - `scikit-learn`: Compute ROC-AUC
+
+Experiment
+- (Optional) Intel TBB: Parallelization
+- (Optional) OpenMP: Parallelization
+
+Other python utility scripts
+- Python 3
+    - `pandas`
+    - `scikit-learn`
 
 ## Customization
 
 ### Switch Cores
 
-Cores are instantiated at `MIDAS/example/Demo.cpp:64-66`, uncomment the chosen one.
+Cores are instantiated at `MIDAS/example/Demo.cpp:67-69`, uncomment the chosen one.
 
 ### Custom Dataset + `Demo.cpp`
 
@@ -89,48 +116,96 @@ You need to prepare three files:
 - Meta file
   - Only includes an integer `N`, the number of records in the dataset
   - Use its path for `pathMeta`
+  - E.g. `MIDAS/data/DARPA/darpa_shape.txt`
 - Data file
   - A header-less csv format file of shape `[N,3]`
   - Columns are sources, destinations, timestamps
   - Use its path for `pathData`
+  - E.g. `MIDAS/data/DARPA/darpa_processed.csv`
 - Label file
   - A header-less csv format file of shape `[N,1]`
   - The corresponding label for data records
     - 0 means normal record
     - 1 means anomalous record
-  - Use its path for `pathGroundTruth` 
+  - Use its path for `pathGroundTruth`
+  - E.g. `MIDAS/data/DARPA/darpa_ground_truth.csv`
 
 ### Custom Dataset + Custom Runner
 
-1. Include the header `MIDAS/CPU/NormalCore.hpp`, `MIDAS/CPU/RelationalCore.hpp` or `MIDAS/CPU/FilteringCore.hpp`
+1. Include the header `MIDAS/src/NormalCore.hpp`, `MIDAS/src/RelationalCore.hpp` or `MIDAS/src/FilteringCore.hpp`
 1. Instantiate cores with required parameters
-1. Call `operator()` on individual data records, it returns the anomaly score for the input record.
+1. Call `operator()` on individual data records, it returns the anomaly score for the input record
+
+## Other Files
+
+### `example/`
+
+#### `Experiment.cpp`
+
+The code we used for experiments.   
+It will try to use Intel TBB or OpenMP for parallelization.  
+You should comment all but only one runner function call in the `main()` as most results are exported to `MIDAS/temp/Experiiment.csv` together with many intermediate files.
+
+#### `Reproducible.cpp`
 
-## Online Articles
+Similar to `Demo.cpp`, but with all random parameters hardcoded and always produce the same result.  
+It's for other developers and us to test if the implementation in other languages can produce acceptable results.  
 
-1. KDnuggets: [Introducing MIDAS: A New Baseline for Anomaly Detection in Graphs](https://www.kdnuggets.com/2020/04/midas-new-baseline-anomaly-detection-graphs.html)
-2. Towards Data Science: [Controlling Fake News using Graphs and Statistics](https://towardsdatascience.com/controlling-fake-news-using-graphs-and-statistics-31ed116a986f)
-2. Towards Data Science: [Anomaly detection in dynamic graphs using MIDAS](https://towardsdatascience.com/anomaly-detection-in-dynamic-graphs-using-midas-e4f8d0b1db45)
-4. Towards AI: [Anomaly Detection with MIDAS](https://medium.com/towards-artificial-intelligence/anomaly-detection-with-midas-2735a2e6dce8)
-5. [AIhub Interview](https://aihub.org/2020/05/01/interview-with-siddharth-bhatia-a-new-approach-for-anomaly-detection/)
+### `util/`
 
-## MIDAS in Other Languages
+`DeleteTempFile.py`, `EvaluateScore.py` and `ReproduceROC.py` will show their usage and a short description when executed without any argument.
 
-1. [Golang](https://github.com/steve0hh/midas) by [Steve Tan](https://github.com/steve0hh)
-2. [Ruby](https://github.com/ankane/midas) by [Andrew Kane](https://github.com/ankane)
-3. [Rust](https://github.com/scooter-dangle/midas_rs) by [Scott Steele](https://github.com/scooter-dangle)
-4. [R](https://github.com/pteridin/MIDASwrappeR) by [Tobias Heidler](https://github.com/pteridin)
-5. [Python](https://github.com/ritesh99rakesh/pyMIDAS) by [Ritesh Kumar](https://github.com/ritesh99rakesh)
+#### `PreprocessData.py`
+
+The code to process the raw dataset into an easy-to-read format.
+Datasets are always assumed to be in a folder in `MIDAS/data/`.    
+It can process the following dataset(s)
+
+- `DARPA/darpa_original.csv` -> `DARPA/darpa_processed.csv`, `DARPA/darpa_ground_truth.csv`, `DARPA/darpa_shape.txt`
+
+## In Other Languages
+
+1. Python: [Rui Liu's MIDAS.Python](https://github.com/liurui39660/MIDAS.Python), [Ritesh Kumar's pyMIDAS](https://github.com/ritesh99rakesh/pyMIDAS)
+1. Golang: [Steve Tan's midas](https://github.com/steve0hh/midas)
+1. Ruby: [Andrew Kane's midas](https://github.com/ankane/midas)
+1. Rust: [Scott Steele's midas_rs](https://github.com/scooter-dangle/midas_rs)
+1. R: [Tobias Heidler's MIDASwrappeR](https://github.com/pteridin/MIDASwrappeR)
+1. Java: [Joshua Tokle's MIDAS-Java](https://github.com/jotok/MIDAS-Java)
+1. Julia: [Ashrya Agrawal's MIDAS.jl](https://github.com/ashryaagr/MIDAS.jl)
+
+## Online Coverage
+
+1. [ACM TechNews](https://technews.acm.org/archives.cfm?fo=2020-05-may/may-06-2020.html)
+1. [AIhub](https://aihub.org/2020/05/01/interview-with-siddharth-bhatia-a-new-approach-for-anomaly-detection/)
+1. [Hacker News](https://news.ycombinator.com/item?id=22802604)
+1. [KDnuggets](https://www.kdnuggets.com/2020/04/midas-new-baseline-anomaly-detection-graphs.html)
+1. [Microsoft](https://techcommunity.microsoft.com/t5/azure-sentinel/announcing-the-azure-sentinel-hackathon-winners/ba-p/1548240)
+1. [Towards Data Science](https://towardsdatascience.com/controlling-fake-news-using-graphs-and-statistics-31ed116a986f)
 
 ## Citation
 
-If you use this code for your research, please consider citing our paper.
+If you use this code for your research, please consider citing our arXiv preprint
+
+```bibtex
+@misc{bhatia2020realtime,
+    title={Real-Time Streaming Anomaly Detection in Dynamic Graphs},
+    author={Siddharth Bhatia and Rui Liu and Bryan Hooi and Minji Yoon and Kijung Shin and Christos Faloutsos},
+    year={2020},
+    eprint={2009.08452},
+    archivePrefix={arXiv},
+    primaryClass={cs.LG}
+}
 
 ```
+
+or our AAAI paper
+
+
+```bibtex
 @inproceedings{bhatia2020midas,
     title="MIDAS: Microcluster-Based Detector of Anomalies in Edge Streams",
     author="Siddharth {Bhatia} and Bryan {Hooi} and Minji {Yoon} and Kijung {Shin} and Christos {Faloutsos}",
     booktitle="AAAI 2020 : The Thirty-Fourth AAAI Conference on Artificial Intelligence",
     year="2020"
 }
-```
+```

data/vendor/MIDAS/src/CountMinSketch.hpp

@@ -0,0 +1,105 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <algorithm>
+
+namespace MIDAS {
+struct CountMinSketch {
+	// Fields
+	// --------------------------------------------------------------------------------
+
+	const int r, c, m = 104729; // Yes, a magic number, I just pick a random prime
+	const int lenData;
+	int* const param1;
+	int* const param2;
+	float* const data;
+	constexpr static float infinity = std::numeric_limits<float>::infinity();
+
+	// Methods
+	// --------------------------------------------------------------------------------
+
+	CountMinSketch() = delete;
+	CountMinSketch& operator=(const CountMinSketch& b) = delete;
+
+	CountMinSketch(int numRow, int numColumn):
+		r(numRow),
+		c(numColumn),
+		lenData(r * c),
+		param1(new int[r]),
+		param2(new int[r]),
+		data(new float[lenData]) {
+		for (int i = 0; i < r; i++) {
+			param1[i] = rand() + 1; // ×0 is not a good idea, see Hash()
+			param2[i] = rand();
+		}
+		std::fill(data, data + lenData, 0);
+	}
+
+	CountMinSketch(const CountMinSketch& b):
+		r(b.r),
+		c(b.c),
+		lenData(b.lenData),
+		param1(new int[r]),
+		param2(new int[r]),
+		data(new float[lenData]) {
+		std::copy(b.param1, b.param1 + r, param1);
+		std::copy(b.param2, b.param2 + r, param2);
+		std::copy(b.data, b.data + lenData, data);
+	}
+
+	~CountMinSketch() {
+		delete[] param1;
+		delete[] param2;
+		delete[] data;
+	}
+
+	void ClearAll(float with = 0) const {
+		std::fill(data, data + lenData, with);
+	}
+
+	void MultiplyAll(float by) const {
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			data[i] *= by;
+	}
+
+	void Hash(int* indexOut, int a, int b = 0) const {
+		for (int i = 0; i < r; i++) {
+			indexOut[i] = ((a + m * b) * param1[i] + param2[i]) % c;
+			indexOut[i] += i * c + (indexOut[i] < 0 ? c : 0);
+		}
+	}
+
+	float operator()(const int* index) const {
+		float least = infinity;
+		for (int i = 0; i < r; i++)
+			least = std::min(least, data[index[i]]);
+		return least;
+	}
+
+	float Assign(const int* index, float with) const {
+		for (int i = 0; i < r; i++)
+			data[index[i]] = with;
+		return with;
+	}
+
+	void Add(const int* index, float by = 1) const {
+		for (int i = 0; i < r; i++)
+			data[index[i]] += by;
+	}
+};
+}

data/vendor/MIDAS/src/FilteringCore.hpp

@@ -0,0 +1,98 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <cmath>
+
+#include "CountMinSketch.hpp"
+
+namespace MIDAS {
+struct FilteringCore {
+	const float threshold;
+	int timestamp = 1;
+	const float factor;
+	const int lenData;
+	int* const indexEdge; // Pre-compute the index to-be-modified, thanks to the Same-Layout Assumption
+	int* const indexSource;
+	int* const indexDestination;
+	CountMinSketch numCurrentEdge, numTotalEdge, scoreEdge;
+	CountMinSketch numCurrentSource, numTotalSource, scoreSource;
+	CountMinSketch numCurrentDestination, numTotalDestination, scoreDestination;
+	float timestampReciprocal = 0;
+	bool* const shouldMerge;
+
+	FilteringCore(int numRow, int numColumn, float threshold, float factor = 0.5):
+		threshold(threshold),
+		factor(factor),
+		lenData(numRow * numColumn), // I assume all CMSs have same size, but Same-Layout Assumption is not that strict
+		indexEdge(new int[numRow]),
+		indexSource(new int[numRow]),
+		indexDestination(new int[numRow]),
+		numCurrentEdge(numRow, numColumn),
+		numTotalEdge(numCurrentEdge),
+		scoreEdge(numCurrentEdge),
+		numCurrentSource(numRow, numColumn),
+		numTotalSource(numCurrentSource),
+		scoreSource(numCurrentSource),
+		numCurrentDestination(numRow, numColumn),
+		numTotalDestination(numCurrentDestination),
+		scoreDestination(numCurrentDestination),
+		shouldMerge(new bool[numRow * numColumn]) { }
+
+	virtual ~FilteringCore() {
+		delete[] indexEdge;
+		delete[] indexSource;
+		delete[] indexDestination;
+		delete[] shouldMerge;
+	}
+
+	static float ComputeScore(float a, float s, float t) {
+		return s == 0 ? 0 : pow(a + s - a * t, 2) / (s * (t - 1)); // If t == 1, then s == 0, so no need to check twice
+	}
+
+	void ConditionalMerge(const float* current, float* total, const float* score) const {
+		for (int i = 0; i < lenData; i++)
+			shouldMerge[i] = score[i] < threshold;
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			total[i] += shouldMerge[i] * current[i] + (true - shouldMerge[i]) * total[i] * timestampReciprocal;
+	}
+
+	float operator()(int source, int destination, int timestamp) {
+		if (this->timestamp < timestamp) {
+			ConditionalMerge(numCurrentEdge.data, numTotalEdge.data, scoreEdge.data);
+			ConditionalMerge(numCurrentSource.data, numTotalSource.data, scoreSource.data);
+			ConditionalMerge(numCurrentDestination.data, numTotalDestination.data, scoreDestination.data);
+			numCurrentEdge.MultiplyAll(factor);
+			numCurrentSource.MultiplyAll(factor);
+			numCurrentDestination.MultiplyAll(factor);
+			timestampReciprocal = 1.f / (timestamp - 1); // So I can skip an if-statement
+			this->timestamp = timestamp;
+		}
+		numCurrentEdge.Hash(indexEdge, source, destination);
+		numCurrentEdge.Add(indexEdge);
+		numCurrentSource.Hash(indexSource, source);
+		numCurrentSource.Add(indexSource);
+		numCurrentDestination.Hash(indexDestination, destination);
+		numCurrentDestination.Add(indexDestination);
+		return std::max({
+			scoreEdge.Assign(indexEdge, ComputeScore(numCurrentEdge(indexEdge), numTotalEdge(indexEdge), timestamp)),
+			scoreSource.Assign(indexSource, ComputeScore(numCurrentSource(indexSource), numTotalSource(indexSource), timestamp)),
+			scoreDestination.Assign(indexDestination, ComputeScore(numCurrentDestination(indexDestination), numTotalDestination(indexDestination), timestamp)),
+		});
+	}
+};
+}