midas-edge 0.1.1 → 0.3.0

data/vendor/MIDAS/src/CountMinSketch.hpp

@@ -0,0 +1,105 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <algorithm>
+
+namespace MIDAS {
+struct CountMinSketch {
+	// Fields
+	// --------------------------------------------------------------------------------
+
+	const int r, c, m = 104729; // Yes, a magic number, I just pick a random prime
+	const int lenData;
+	int* const param1;
+	int* const param2;
+	float* const data;
+	constexpr static float infinity = std::numeric_limits<float>::infinity();
+
+	// Methods
+	// --------------------------------------------------------------------------------
+
+	CountMinSketch() = delete;
+	CountMinSketch& operator=(const CountMinSketch& b) = delete;
+
+	CountMinSketch(int numRow, int numColumn):
+		r(numRow),
+		c(numColumn),
+		lenData(r * c),
+		param1(new int[r]),
+		param2(new int[r]),
+		data(new float[lenData]) {
+		for (int i = 0; i < r; i++) {
+			param1[i] = rand() + 1; // ×0 is not a good idea, see Hash()
+			param2[i] = rand();
+		}
+		std::fill(data, data + lenData, 0);
+	}
+
+	CountMinSketch(const CountMinSketch& b):
+		r(b.r),
+		c(b.c),
+		lenData(b.lenData),
+		param1(new int[r]),
+		param2(new int[r]),
+		data(new float[lenData]) {
+		std::copy(b.param1, b.param1 + r, param1);
+		std::copy(b.param2, b.param2 + r, param2);
+		std::copy(b.data, b.data + lenData, data);
+	}
+
+	~CountMinSketch() {
+		delete[] param1;
+		delete[] param2;
+		delete[] data;
+	}
+
+	void ClearAll(float with = 0) const {
+		std::fill(data, data + lenData, with);
+	}
+
+	void MultiplyAll(float by) const {
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			data[i] *= by;
+	}
+
+	void Hash(int* indexOut, int a, int b = 0) const {
+		for (int i = 0; i < r; i++) {
+			indexOut[i] = ((a + m * b) * param1[i] + param2[i]) % c;
+			indexOut[i] += i * c + (indexOut[i] < 0 ? c : 0);
+		}
+	}
+
+	float operator()(const int* index) const {
+		float least = infinity;
+		for (int i = 0; i < r; i++)
+			least = std::min(least, data[index[i]]);
+		return least;
+	}
+
+	float Assign(const int* index, float with) const {
+		for (int i = 0; i < r; i++)
+			data[index[i]] = with;
+		return with;
+	}
+
+	void Add(const int* index, float by = 1) const {
+		for (int i = 0; i < r; i++)
+			data[index[i]] += by;
+	}
+};
+}

data/vendor/MIDAS/src/FilteringCore.hpp

@@ -0,0 +1,98 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <cmath>
+
+#include "CountMinSketch.hpp"
+
+namespace MIDAS {
+struct FilteringCore {
+	const float threshold;
+	int timestamp = 1;
+	const float factor;
+	const int lenData;
+	int* const indexEdge; // Pre-compute the index to-be-modified, thanks to the Same-Layout Assumption
+	int* const indexSource;
+	int* const indexDestination;
+	CountMinSketch numCurrentEdge, numTotalEdge, scoreEdge;
+	CountMinSketch numCurrentSource, numTotalSource, scoreSource;
+	CountMinSketch numCurrentDestination, numTotalDestination, scoreDestination;
+	float timestampReciprocal = 0;
+	bool* const shouldMerge;
+
+	FilteringCore(int numRow, int numColumn, float threshold, float factor = 0.5):
+		threshold(threshold),
+		factor(factor),
+		lenData(numRow * numColumn), // I assume all CMSs have same size, but Same-Layout Assumption is not that strict
+		indexEdge(new int[numRow]),
+		indexSource(new int[numRow]),
+		indexDestination(new int[numRow]),
+		numCurrentEdge(numRow, numColumn),
+		numTotalEdge(numCurrentEdge),
+		scoreEdge(numCurrentEdge),
+		numCurrentSource(numRow, numColumn),
+		numTotalSource(numCurrentSource),
+		scoreSource(numCurrentSource),
+		numCurrentDestination(numRow, numColumn),
+		numTotalDestination(numCurrentDestination),
+		scoreDestination(numCurrentDestination),
+		shouldMerge(new bool[numRow * numColumn]) { }
+
+	virtual ~FilteringCore() {
+		delete[] indexEdge;
+		delete[] indexSource;
+		delete[] indexDestination;
+		delete[] shouldMerge;
+	}
+
+	static float ComputeScore(float a, float s, float t) {
+		return s == 0 ? 0 : pow(a + s - a * t, 2) / (s * (t - 1)); // If t == 1, then s == 0, so no need to check twice
+	}
+
+	void ConditionalMerge(const float* current, float* total, const float* score) const {
+		for (int i = 0; i < lenData; i++)
+			shouldMerge[i] = score[i] < threshold;
+		for (int i = 0, I = lenData; i < I; i++) // Vectorization
+			total[i] += shouldMerge[i] * current[i] + (true - shouldMerge[i]) * total[i] * timestampReciprocal;
+	}
+
+	float operator()(int source, int destination, int timestamp) {
+		if (this->timestamp < timestamp) {
+			ConditionalMerge(numCurrentEdge.data, numTotalEdge.data, scoreEdge.data);
+			ConditionalMerge(numCurrentSource.data, numTotalSource.data, scoreSource.data);
+			ConditionalMerge(numCurrentDestination.data, numTotalDestination.data, scoreDestination.data);
+			numCurrentEdge.MultiplyAll(factor);
+			numCurrentSource.MultiplyAll(factor);
+			numCurrentDestination.MultiplyAll(factor);
+			timestampReciprocal = 1.f / (timestamp - 1); // So I can skip an if-statement
+			this->timestamp = timestamp;
+		}
+		numCurrentEdge.Hash(indexEdge, source, destination);
+		numCurrentEdge.Add(indexEdge);
+		numCurrentSource.Hash(indexSource, source);
+		numCurrentSource.Add(indexSource);
+		numCurrentDestination.Hash(indexDestination, destination);
+		numCurrentDestination.Add(indexDestination);
+		return std::max({
+			scoreEdge.Assign(indexEdge, ComputeScore(numCurrentEdge(indexEdge), numTotalEdge(indexEdge), timestamp)),
+			scoreSource.Assign(indexSource, ComputeScore(numCurrentSource(indexSource), numTotalSource(indexSource), timestamp)),
+			scoreDestination.Assign(indexDestination, ComputeScore(numCurrentDestination(indexDestination), numTotalDestination(indexDestination), timestamp)),
+		});
+	}
+};
+}

data/vendor/MIDAS/src/NormalCore.hpp

@@ -0,0 +1,53 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <cmath>
+
+#include "CountMinSketch.hpp"
+
+namespace MIDAS {
+struct NormalCore {
+	int timestamp = 1;
+	int* const index; // Pre-compute the index to-be-modified, thanks to the same structure of CMSs
+	CountMinSketch numCurrent, numTotal;
+
+	NormalCore(int numRow, int numColumn):
+		index(new int[numRow]),
+		numCurrent(numRow, numColumn),
+		numTotal(numCurrent) { }
+
+	virtual ~NormalCore() {
+		delete[] index;
+	}
+
+	static float ComputeScore(float a, float s, float t) {
+		return s == 0 || t - 1 == 0 ? 0 : pow((a - s / t) * t, 2) / (s * (t - 1));
+	}
+
+	float operator()(int source, int destination, int timestamp) {
+		if (this->timestamp < timestamp) {
+			numCurrent.ClearAll();
+			this->timestamp = timestamp;
+		}
+		numCurrent.Hash(index, source, destination);
+		numCurrent.Add(index);
+		numTotal.Add(index);
+		return ComputeScore(numCurrent(index), numTotal(index), timestamp);
+	}
+};
+}

data/vendor/MIDAS/src/RelationalCore.hpp

@@ -0,0 +1,79 @@
+// -----------------------------------------------------------------------------
+// Copyright 2020 Rui Liu (liurui39660) and Siddharth Bhatia (bhatiasiddharth)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// -----------------------------------------------------------------------------
+
+#pragma once
+
+#include <cmath>
+
+#include "CountMinSketch.hpp"
+
+namespace MIDAS {
+struct RelationalCore {
+	int timestamp = 1;
+	const float factor;
+	int* const indexEdge; // Pre-compute the index to-be-modified, thanks to the same structure of CMSs
+	int* const indexSource;
+	int* const indexDestination;
+	CountMinSketch numCurrentEdge, numTotalEdge;
+	CountMinSketch numCurrentSource, numTotalSource;
+	CountMinSketch numCurrentDestination, numTotalDestination;
+
+	RelationalCore(int numRow, int numColumn, float factor = 0.5):
+		factor(factor),
+		indexEdge(new int[numRow]),
+		indexSource(new int[numRow]),
+		indexDestination(new int[numRow]),
+		numCurrentEdge(numRow, numColumn),
+		numTotalEdge(numCurrentEdge),
+		numCurrentSource(numRow, numColumn),
+		numTotalSource(numCurrentSource),
+		numCurrentDestination(numRow, numColumn),
+		numTotalDestination(numCurrentDestination) { }
+
+	virtual ~RelationalCore() {
+		delete[] indexEdge;
+		delete[] indexSource;
+		delete[] indexDestination;
+	}
+
+	static float ComputeScore(float a, float s, float t) {
+		return s == 0 || t - 1 == 0 ? 0 : pow((a - s / t) * t, 2) / (s * (t - 1));
+	}
+
+	float operator()(int source, int destination, int timestamp) {
+		if (this->timestamp < timestamp) {
+			numCurrentEdge.MultiplyAll(factor);
+			numCurrentSource.MultiplyAll(factor);
+			numCurrentDestination.MultiplyAll(factor);
+			this->timestamp = timestamp;
+		}
+		numCurrentEdge.Hash(indexEdge, source, destination);
+		numCurrentEdge.Add(indexEdge);
+		numTotalEdge.Add(indexEdge);
+		numCurrentSource.Hash(indexSource, source);
+		numCurrentSource.Add(indexSource);
+		numTotalSource.Add(indexSource);
+		numCurrentDestination.Hash(indexDestination, destination);
+		numCurrentDestination.Add(indexDestination);
+		numTotalDestination.Add(indexDestination);
+		return std::max({
+			ComputeScore(numCurrentEdge(indexEdge), numTotalEdge(indexEdge), timestamp),
+			ComputeScore(numCurrentSource(indexSource), numTotalSource(indexSource), timestamp),
+			ComputeScore(numCurrentDestination(indexDestination), numTotalDestination(indexDestination), timestamp),
+		});
+	}
+};
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: midas-edge
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-19 00:00:00.000000000 Z
+date: 2021-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: 4.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: 4.0.2
 - !ruby/object:Gem::Dependency
   name: numo-narray
   requirement: !ruby/object:Gem::Requirement
@@ -38,64 +38,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake-compiler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-description: 
-email: andrew@chartkick.com
+description:
+email: andrew@ankane.org
 executables: []
 extensions:
 - ext/midas/extconf.rb
@@ -112,19 +56,15 @@ files:
 - lib/midas/version.rb
 - vendor/MIDAS/LICENSE
 - vendor/MIDAS/README.md
-- vendor/MIDAS/anom.cpp
-- vendor/MIDAS/anom.hpp
-- vendor/MIDAS/argparse.hpp
-- vendor/MIDAS/edgehash.cpp
-- vendor/MIDAS/edgehash.hpp
-- vendor/MIDAS/main.cpp
-- vendor/MIDAS/nodehash.cpp
-- vendor/MIDAS/nodehash.hpp
+- vendor/MIDAS/src/CountMinSketch.hpp
+- vendor/MIDAS/src/FilteringCore.hpp
+- vendor/MIDAS/src/NormalCore.hpp
+- vendor/MIDAS/src/RelationalCore.hpp
 homepage: https://github.com/ankane/midas
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -132,15 +72,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Edge stream anomaly detection for Ruby
 test_files: []