midas-edge 0.1.0 → 0.2.3

data/vendor/MIDAS/src/CountMinSketch.hpp

@@ -0,0 +1,105 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <algorithm>
+
+namespace MIDAS {
+struct CountMinSketch {
+	// Fields
+	// --------------------------------------------------------------------------------
+
+	const int r, c, m = 104729; // Yes, a magic number, I just pick a random prime
+	const int lenData;
+	int* const param1;
+	int* const param2;
+	float* const data;
+	constexpr static float infinity = std::numeric_limits<float>::infinity();
+
+	// Methods
+	// --------------------------------------------------------------------------------
+
+	CountMinSketch() = delete;
+	CountMinSketch& operator=(const CountMinSketch& b) = delete;
+
+	CountMinSketch(int numRow, int numColumn):
+		r(numRow),
+		c(numColumn),
+		lenData(r * c),
+		param1(new int[r]),
+		param2(new int[r]),
+		data(new float[lenData]) {
+		for (int i = 0; i < r; i++) {
+			param1[i] = rand() + 1; // ×0 is not a good idea, see Hash()
+			param2[i] = rand();
+		}
+		std::fill(data, data + lenData, 0);
+	}
+
+	CountMinSketch(const CountMinSketch& b):
+		r(b.r),
+		c(b.c),
+		lenData(b.lenData),
+		param1(new int[r]),
+		param2(new int[r]),
+		data(new float[lenData]) {
+		std::copy(b.param1, b.param1 + r, param1);
+		std::copy(b.param2, b.param2 + r, param2);
+		std::copy(b.data, b.data + lenData, data);
+	}
+
+	~CountMinSketch() {
+		delete[] param1;
+		delete[] param2;
+		delete[] data;
+	}
+
+	void ClearAll(float with = 0) const {
+		std::fill(data, data + lenData, with);
+	}
+
+	void MultiplyAll(float by) const {
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			data[i] *= by;
+	}
+
+	void Hash(int* indexOut, int a, int b = 0) const {
+		for (int i = 0; i < r; i++) {
+			indexOut[i] = ((a + m * b) * param1[i] + param2[i]) % c;
+			indexOut[i] += i * c + (indexOut[i] < 0 ? c : 0);
+		}
+	}
+
+	float operator()(const int* index) const {
+		float least = infinity;
+		for (int i = 0; i < r; i++)
+			least = std::min(least, data[index[i]]);
+		return least;
+	}
+
+	float Assign(const int* index, float with) const {
+		for (int i = 0; i < r; i++)
+			data[index[i]] = with;
+		return with;
+	}
+
+	void Add(const int* index, float by = 1) const {
+		for (int i = 0; i < r; i++)
+			data[index[i]] += by;
+	}
+};
+}

data/vendor/MIDAS/src/FilteringCore.hpp

@@ -0,0 +1,98 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <cmath>
+
+#include "CountMinSketch.hpp"
+
+namespace MIDAS {
+struct FilteringCore {
+	const float threshold;
+	int timestamp = 1;
+	const float factor;
+	const int lenData;
+	int* const indexEdge; // Pre-compute the index to-be-modified, thanks to the Same-Layout Assumption
+	int* const indexSource;
+	int* const indexDestination;
+	CountMinSketch numCurrentEdge, numTotalEdge, scoreEdge;
+	CountMinSketch numCurrentSource, numTotalSource, scoreSource;
+	CountMinSketch numCurrentDestination, numTotalDestination, scoreDestination;
+	float timestampReciprocal = 0;
+	bool* const shouldMerge;
+
+	FilteringCore(int numRow, int numColumn, float threshold, float factor = 0.5):
+		threshold(threshold),
+		factor(factor),
+		lenData(numRow * numColumn), // I assume all CMSs have same size, but Same-Layout Assumption is not that strict
+		indexEdge(new int[numRow]),
+		indexSource(new int[numRow]),
+		indexDestination(new int[numRow]),
+		numCurrentEdge(numRow, numColumn),
+		numTotalEdge(numCurrentEdge),
+		scoreEdge(numCurrentEdge),
+		numCurrentSource(numRow, numColumn),
+		numTotalSource(numCurrentSource),
+		scoreSource(numCurrentSource),
+		numCurrentDestination(numRow, numColumn),
+		numTotalDestination(numCurrentDestination),
+		scoreDestination(numCurrentDestination),
+		shouldMerge(new bool[numRow * numColumn]) { }
+
+	virtual ~FilteringCore() {
+		delete[] indexEdge;
+		delete[] indexSource;
+		delete[] indexDestination;
+		delete[] shouldMerge;
+	}
+
+	static float ComputeScore(float a, float s, float t) {
+		return s == 0 ? 0 : pow(a + s - a * t, 2) / (s * (t - 1)); // If t == 1, then s == 0, so no need to check twice
+	}
+
+	void ConditionalMerge(const float* current, float* total, const float* score) const {
+		for (int i = 0; i < lenData; i++)
+			shouldMerge[i] = score[i] < threshold;
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			total[i] += shouldMerge[i] * current[i] + (true - shouldMerge[i]) * total[i] * timestampReciprocal;
+	}
+
+	float operator()(int source, int destination, int timestamp) {
+		if (this->timestamp < timestamp) {
+			ConditionalMerge(numCurrentEdge.data, numTotalEdge.data, scoreEdge.data);
+			ConditionalMerge(numCurrentSource.data, numTotalSource.data, scoreSource.data);
+			ConditionalMerge(numCurrentDestination.data, numTotalDestination.data, scoreDestination.data);
+			numCurrentEdge.MultiplyAll(factor);
+			numCurrentSource.MultiplyAll(factor);
+			numCurrentDestination.MultiplyAll(factor);
+			timestampReciprocal = 1.f / (timestamp - 1); // So I can skip an if-statement
+			this->timestamp = timestamp;
+		}
+		numCurrentEdge.Hash(indexEdge, source, destination);
+		numCurrentEdge.Add(indexEdge);
+		numCurrentSource.Hash(indexSource, source);
+		numCurrentSource.Add(indexSource);
+		numCurrentDestination.Hash(indexDestination, destination);
+		numCurrentDestination.Add(indexDestination);
+		return std::max({
+			scoreEdge.Assign(indexEdge, ComputeScore(numCurrentEdge(indexEdge), numTotalEdge(indexEdge), timestamp)),
+			scoreSource.Assign(indexSource, ComputeScore(numCurrentSource(indexSource), numTotalSource(indexSource), timestamp)),
+			scoreDestination.Assign(indexDestination, ComputeScore(numCurrentDestination(indexDestination), numTotalDestination(indexDestination), timestamp)),
+		});
+	}
+};
+}

data/vendor/MIDAS/src/NormalCore.hpp

@@ -0,0 +1,53 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <cmath>
+
+#include "CountMinSketch.hpp"
+
+namespace MIDAS {
+struct NormalCore {
+	int timestamp = 1;
+	int* const index; // Pre-compute the index to-be-modified, thanks to the same structure of CMSs
+	CountMinSketch numCurrent, numTotal;
+
+	NormalCore(int numRow, int numColumn):
+		index(new int[numRow]),
+		numCurrent(numRow, numColumn),
+		numTotal(numCurrent) { }
+
+	virtual ~NormalCore() {
+		delete[] index;
+	}
+
+	static float ComputeScore(float a, float s, float t) {
+		return s == 0 || t - 1 == 0 ? 0 : pow((a - s / t) * t, 2) / (s * (t - 1));
+	}
+
+	float operator()(int source, int destination, int timestamp) {
+		if (this->timestamp < timestamp) {
+			numCurrent.ClearAll();
+			this->timestamp = timestamp;
+		}
+		numCurrent.Hash(index, source, destination);
+		numCurrent.Add(index);
+		numTotal.Add(index);
+		return ComputeScore(numCurrent(index), numTotal(index), timestamp);
+	}
+};
+}

data/vendor/MIDAS/src/RelationalCore.hpp

@@ -0,0 +1,79 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <cmath>
+
+#include "CountMinSketch.hpp"
+
+namespace MIDAS {
+struct RelationalCore {
+	int timestamp = 1;
+	const float factor;
+	int* const indexEdge; // Pre-compute the index to-be-modified, thanks to the same structure of CMSs
+	int* const indexSource;
+	int* const indexDestination;
+	CountMinSketch numCurrentEdge, numTotalEdge;
+	CountMinSketch numCurrentSource, numTotalSource;
+	CountMinSketch numCurrentDestination, numTotalDestination;
+
+	RelationalCore(int numRow, int numColumn, float factor = 0.5):
+		factor(factor),
+		indexEdge(new int[numRow]),
+		indexSource(new int[numRow]),
+		indexDestination(new int[numRow]),
+		numCurrentEdge(numRow, numColumn),
+		numTotalEdge(numCurrentEdge),
+		numCurrentSource(numRow, numColumn),
+		numTotalSource(numCurrentSource),
+		numCurrentDestination(numRow, numColumn),
+		numTotalDestination(numCurrentDestination) { }
+
+	virtual ~RelationalCore() {
+		delete[] indexEdge;
+		delete[] indexSource;
+		delete[] indexDestination;
+	}
+
+	static float ComputeScore(float a, float s, float t) {
+		return s == 0 || t - 1 == 0 ? 0 : pow((a - s / t) * t, 2) / (s * (t - 1));
+	}
+
+	float operator()(int source, int destination, int timestamp) {
+		if (this->timestamp < timestamp) {
+			numCurrentEdge.MultiplyAll(factor);
+			numCurrentSource.MultiplyAll(factor);
+			numCurrentDestination.MultiplyAll(factor);
+			this->timestamp = timestamp;
+		}
+		numCurrentEdge.Hash(indexEdge, source, destination);
+		numCurrentEdge.Add(indexEdge);
+		numTotalEdge.Add(indexEdge);
+		numCurrentSource.Hash(indexSource, source);
+		numCurrentSource.Add(indexSource);
+		numTotalSource.Add(indexSource);
+		numCurrentDestination.Hash(indexDestination, destination);
+		numCurrentDestination.Add(indexDestination);
+		numTotalDestination.Add(indexDestination);
+		return std::max({
+			ComputeScore(numCurrentEdge(indexEdge), numTotalEdge(indexEdge), timestamp),
+			ComputeScore(numCurrentSource(indexSource), numTotalSource(indexSource), timestamp),
+			ComputeScore(numCurrentDestination(indexDestination), numTotalDestination(indexDestination), timestamp),
+		});
+	}
+};
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: midas-edge
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.3
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-18 00:00:00.000000000 Z
+date: 2020-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
-description: 
+description:
 email: andrew@chartkick.com
 executables: []
 extensions:
@@ -109,23 +109,18 @@ files:
 - ext/midas/extconf.rb
 - lib/midas-edge.rb
 - lib/midas/detector.rb
-- lib/midas/ext.bundle
 - lib/midas/version.rb
 - vendor/MIDAS/LICENSE
 - vendor/MIDAS/README.md
-- vendor/MIDAS/anom.cpp
-- vendor/MIDAS/anom.hpp
-- vendor/MIDAS/argparse.hpp
-- vendor/MIDAS/edgehash.cpp
-- vendor/MIDAS/edgehash.hpp
-- vendor/MIDAS/main.cpp
-- vendor/MIDAS/nodehash.cpp
-- vendor/MIDAS/nodehash.hpp
+- vendor/MIDAS/src/CountMinSketch.hpp
+- vendor/MIDAS/src/FilteringCore.hpp
+- vendor/MIDAS/src/NormalCore.hpp
+- vendor/MIDAS/src/RelationalCore.hpp
 homepage: https://github.com/ankane/midas
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -140,8 +135,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Edge stream anomaly detection for Ruby
 test_files: []

data/vendor/MIDAS/anom.cpp

@@ -1,88 +0,0 @@
-#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y))
-#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y))
-
-#include <iostream>
-#include <math.h>
-#include <algorithm>
-#include <vector>
-#include "anom.hpp"
-#include "edgehash.hpp"
-#include "nodehash.hpp"
-
-vector<double>* midas(vector<int>& src, vector<int>& dst, vector<int>& times, int num_rows, int num_buckets)
-{
-    int m = *max_element(src.begin(), src.end());
-    Edgehash cur_count(num_rows, num_buckets, m);
-    Edgehash total_count(num_rows, num_buckets, m);
-    vector<double>* anom_score = new vector<double>(src.size());
-    int cur_t = 1, size = src.size(), cur_src, cur_dst;
-    double cur_mean, sqerr, cur_score;
-    for (int i = 0; i < size; i++) {
-
-        if (i == 0 || times[i] > cur_t) {
-            cur_count.clear();
-            cur_t = times[i];
-        }
-
-        cur_src = src[i];
-        cur_dst = dst[i];
-        cur_count.insert(cur_src, cur_dst, 1);
-        total_count.insert(cur_src, cur_dst, 1);
-        cur_mean = total_count.get_count(cur_src, cur_dst) / cur_t;
-        sqerr = pow(cur_count.get_count(cur_src, cur_dst) - cur_mean, 2);
-        if (cur_t == 1) cur_score = 0;
-        else cur_score = sqerr / cur_mean + sqerr / (cur_mean * (cur_t - 1));
-        (*anom_score)[i] = cur_score;
-    }
-
-    return anom_score;
-}
-
-double counts_to_anom(double tot, double cur, int cur_t)
-{
-    double cur_mean = tot / cur_t;
-    double sqerr = pow(MAX(0, cur - cur_mean), 2);
-    return sqerr / cur_mean + sqerr / (cur_mean * MAX(1, cur_t - 1));
-}
-
-vector<double>* midasR(vector<int>& src, vector<int>& dst, vector<int>& times, int num_rows, int num_buckets, double factor)
-{
-    int m = *max_element(src.begin(), src.end());
-    Edgehash cur_count(num_rows, num_buckets, m);
-    Edgehash total_count(num_rows, num_buckets, m);
-    Nodehash src_score(num_rows, num_buckets);
-    Nodehash dst_score(num_rows, num_buckets);
-    Nodehash src_total(num_rows, num_buckets);
-    Nodehash dst_total(num_rows, num_buckets);
-    vector<double>* anom_score = new vector<double>(src.size());
-    int cur_t = 1, size = src.size(), cur_src, cur_dst;
-    double cur_score, cur_score_src, cur_score_dst, combined_score;
-
-    for (int i = 0; i < size; i++) {
-    	
-        if (i == 0 || times[i] > cur_t) {
-            cur_count.lower(factor);
-            src_score.lower(factor);
-            dst_score.lower(factor);
-            cur_t = times[i];
-        }
-
-        cur_src = src[i];
-        cur_dst = dst[i];
-        cur_count.insert(cur_src, cur_dst, 1);
-        total_count.insert(cur_src, cur_dst, 1);
-        src_score.insert(cur_src, 1);
-        dst_score.insert(cur_dst, 1);
-        src_total.insert(cur_src, 1);
-        dst_total.insert(cur_dst, 1);
-        cur_score = counts_to_anom(total_count.get_count(cur_src, cur_dst), cur_count.get_count(cur_src, cur_dst), cur_t);
-        cur_score_src = counts_to_anom(src_total.get_count(cur_src), src_score.get_count(cur_src), cur_t);
-        cur_score_dst = counts_to_anom(dst_total.get_count(cur_dst), dst_score.get_count(cur_dst), cur_t);
-        //combined_score = MAX(cur_score_src, cur_score_dst) + cur_score;
-        //combined_score = cur_score_src + cur_score_dst + cur_score;
-        combined_score = MAX(MAX(cur_score_src, cur_score_dst), cur_score);
-        (*anom_score)[i] = log(1 + combined_score);
-    }
-
-    return anom_score;
-}