anomaly_detection 0.1.0

data/NOTICE.txt

@@ -0,0 +1,15 @@
+Copyright (C) 2015 Twitter, Inc and other contributors
+Copyright (C) 2021 Andrew Kane
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.

data/README.md

@@ -0,0 +1,89 @@
+# AnomalyDetection.rb
+
+:fire: [AnomalyDetection](https://github.com/twitter/AnomalyDetection) for Ruby
+
+Learn [how it works](https://blog.twitter.com/engineering/en_us/a/2015/introducing-practical-and-robust-anomaly-detection-in-a-time-series)
+
+[![Build Status](https://github.com/ankane/AnomalyDetection.rb/workflows/build/badge.svg?branch=master)](https://github.com/ankane/AnomalyDetection.rb/actions)
+
+## Installation
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'anomaly_detection'
+```
+
+## Getting Started
+
+Detect anomalies in a time series
+
+```ruby
+series = {
+  Date.parse("2020-01-01") => 100,
+  Date.parse("2020-01-02") => 150,
+  Date.parse("2020-01-03") => 136,
+  # ...
+}
+
+AnomalyDetection.detect(series, period: 7)
+```
+
+Works great with [Groupdate](https://github.com/ankane/groupdate)
+
+```ruby
+series = User.group_by_day(:created_at).count
+AnomalyDetection.detect(series, period: 7)
+```
+
+Series can also be an array without times (the index is returned)
+
+```ruby
+series = [100, 150, 136, ...]
+AnomalyDetection.detect(series, period: 7)
+```
+
+## Options
+
+Pass options
+
+```ruby
+AnomalyDetection.detect(
+  series,
+  period: 7,            # number of observations in a single period
+  alpha: 0.05,          # level of statistical significance
+  max_anoms: 0.1,       # maximum number of anomalies as percent of data
+  direction: "both"     # pos, neg, or both
+)
+```
+
+## Credits
+
+This library was ported from the [AnomalyDetection](https://github.com/twitter/AnomalyDetection) R package and is available under the same license. It uses [cdflib](https://people.sc.fsu.edu/~jburkardt/cpp_src/cdflib/cdflib.html) for the quantile function.
+
+## References
+
+- [Automatic Anomaly Detection in the Cloud Via Statistical Learning](https://arxiv.org/abs/1704.07706)
+
+## History
+
+View the [changelog](https://github.com/ankane/AnomalyDetection.rb/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/AnomalyDetection.rb/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/AnomalyDetection.rb/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/AnomalyDetection.rb.git
+cd AnomalyDetection.rb
+bundle install
+bundle exec rake compile
+bundle exec rake test
+```

data/ext/anomaly_detection/anomaly_detection.cpp

@@ -0,0 +1,156 @@
+#include <cassert>
+#include <iostream>
+#include <iterator>
+#include <vector>
+
+#include "cdflib.hpp"
+#include "stl.hpp"
+
+float median(const std::vector<float>& data) {
+    std::vector<float> sorted(data);
+    std::sort(sorted.begin(), sorted.end());
+    return (sorted[(sorted.size() - 1) / 2] + sorted[sorted.size() / 2]) / 2.0;
+}
+
+float mad(const std::vector<float>& data) {
+    auto med = median(data);
+    std::vector<float> res;
+    res.reserve(data.size());
+    for (auto v : data) {
+        res.push_back(fabs(v - med));
+    }
+    return 1.4826 * median(res);
+}
+
+float qt(double p, double df) {
+    int which = 2;
+    double q = 1 - p;
+    double t;
+    int status;
+    double bound;
+    cdft(&which, &p, &q, &t, &df, &status, &bound);
+
+    if (status != 0) {
+        throw std::invalid_argument("Bad status");
+    }
+    return t;
+}
+
+std::vector<size_t> detect_anoms(const std::vector<float>& data, int num_obs_per_period, float k, float alpha, bool one_tail, bool upper_tail) {
+    auto num_obs = data.size();
+
+    // Check to make sure we have at least two periods worth of data for anomaly context
+    assert(num_obs >= num_obs_per_period * 2);
+
+    // Handle NANs
+    auto nan = std::count_if(data.begin(), data.end(), [](const auto& value) { return std::isnan(value); });
+    if (nan > 0) {
+        throw std::invalid_argument("Data contains NANs");
+    }
+
+    // Decompose data. This returns a univarite remainder which will be used for anomaly detection. Optionally, we might NOT decompose.
+    auto seasonal_length = data.size() * 10 + 1;
+    auto data_decomp = stl::params().robust(true).seasonal_length(seasonal_length).fit(data, num_obs_per_period);
+
+    auto seasonal = data_decomp.seasonal;
+    auto med = median(data);
+    std::vector<float> data2;
+    data2.reserve(data.size());
+    for (auto i = 0; i < data.size(); i++) {
+        data2.push_back(data[i] - seasonal[i] - med);
+    }
+
+    auto max_outliers = (size_t) num_obs * k;
+    assert(max_outliers > 0);
+
+    auto n = data2.size();
+
+    std::vector<size_t> r_idx;
+
+    std::vector<size_t> indexes;
+    indexes.reserve(data2.size());
+    for (auto i = 0; i < data2.size(); i++) {
+        indexes.push_back(i);
+    }
+
+    // Compute test statistic until r=max_outliers values have been removed from the sample
+    for (auto i = 1; i <= max_outliers; i++) {
+        // TODO Improve performance between loop iterations
+        auto ma = median(data2);
+        std::vector<float> ares;
+        ares.reserve(data2.size());
+        if (one_tail) {
+            if (upper_tail) {
+                for (auto v : data2) {
+                    ares.push_back(v - ma);
+                }
+            } else {
+                for (auto v : data2) {
+                    ares.push_back(ma - v);
+                }
+            }
+        } else {
+            for (auto v : data2) {
+                ares.push_back(fabs(v - ma));
+            }
+        }
+
+        // Protect against constant time series
+        auto data_sigma = mad(data2);
+        if (data_sigma == 0.0) {
+            break;
+        }
+
+        auto iter = std::max_element(ares.begin(), ares.end());
+        auto r_idx_i = std::distance(ares.begin(), iter);
+        auto r_idx_i2 = indexes[r_idx_i];
+
+        // Only need to take sigma of r for performance
+        auto r = ares[r_idx_i] / data_sigma;
+
+        // TODO Swap to last position and delete
+        data2.erase(data2.begin() + r_idx_i);
+        indexes.erase(indexes.begin() + r_idx_i);
+
+        // Compute critical value
+        float p;
+        if (one_tail) {
+            p = 1.0 - alpha / (n - i + 1);
+        } else {
+            p = 1.0 - alpha / (2.0 * (n - i + 1));
+        }
+
+        auto t = qt(p, n - i - 1);
+        auto lam = t * (n - i) / sqrt(((n - i - 1) + powf(t, 2.0)) * (n - i + 1));
+
+        if (r > lam) {
+            r_idx.push_back(r_idx_i2);
+        } else {
+            break;
+        }
+    }
+
+    // Sort like R version
+    std::sort(r_idx.begin(), r_idx.end());
+
+    return r_idx;
+}
+
+std::vector<size_t> anomalies(const std::vector<float>& x, int period, float k, float alpha, const std::string& direction) {
+    bool one_tail;
+    bool upper_tail;
+    if (direction == "pos") {
+        one_tail = true;
+        upper_tail = true;
+    } else if (direction == "neg") {
+        one_tail = true;
+        upper_tail = false;
+    } else if (direction == "both") {
+        one_tail = false;
+        upper_tail = true; // not used
+    } else {
+        throw std::invalid_argument("Bad direction");
+    }
+
+    return detect_anoms(x, period, k, alpha, one_tail, upper_tail);
+}