umappp 0.1.2

data/vendor/kmeans/MiniBatch.hpp

@@ -0,0 +1,269 @@
+#ifndef KMEANS_MINIBATCH_HPP
+#define KMEANS_MINIBATCH_HPP
+
+#include <vector>
+#include <algorithm>
+#include <numeric>
+#include <cstdint>
+#include <stdexcept>
+#include <limits>
+#include <random>
+
+#include "Base.hpp"
+#include "Details.hpp"
+#include "QuickSearch.hpp"
+#include "compute_wcss.hpp"
+#include "is_edge_case.hpp"
+#include "random.hpp"
+
+/**
+ * @file MiniBatch.hpp
+ *
+ * @brief Implements the mini-batch algorithm for k-means clustering.
+ */
+
+namespace kmeans {
+
+/**
+ * @brief Implements the mini-batch algorithm for k-means clustering.
+ *
+ * The mini-batch approach is similar to Lloyd's algorithm in that it runs through a set of observations, assigns each to the closest centroid, updates the centroids and repeats.
+ * The key difference is that each iteration is performed with a random subset of observations (i.e., a "mini-batch"), instead of the full set of observations.
+ * This reduces computational time at the cost of some solution quality.
+ * In theory, it can also reduce memory usage, though this is not particularly relevant here as we are already assuming that the data can be fully stored in memory.
+ * 
+ * The update procedure for a cluster's centroid involves adjusting the coordinates by the assigned observations in the mini-batch.
+ * The resulting vector can be interpreted as the mean of all observations that have ever been sampled (possibly multiple times) to that cluster.
+ * Thus, the magnitude of the updates will decrease in later iterations as the relative effect of newly sampled points is reduced.
+ * This ensures that the centroids will stabilize at a sufficiently large number of iterations.
+ *
+ * We may stop the algorithm before the maximum number of iterations if only a few observations are reassigned at each iteration. 
+ * Specifically, every \f$h\f$ iterations, we compute the proportion of sampled observations for each cluster in the past \f$h\f$ mini-batches that were reassigned to/from that cluster.
+ * If this proportion is less than some threshold \f$p\f$ for all clusters, we consider that the algorithm has converged.
+ *
+ * @tparam DATA_t Floating-point type for the data and centroids.
+ * @tparam CLUSTER_t Integer type for the cluster assignments.
+ * @tparam INDEX_t Integer type for the observation index.
+ */
+template<typename DATA_t = double, typename CLUSTER_t = int, typename INDEX_t = int>
+class MiniBatch : public Refine<DATA_t, CLUSTER_t, INDEX_t> {
+public:
+    /** 
+     * @brief Default parameter values for `MiniBatch`.
+     */
+    struct Defaults {
+        /** 
+         * See `MiniBatch::set_max_iterations()`.
+         */
+        static constexpr int max_iterations = 100;
+
+        /** 
+         * See `MiniBatch::set_batch_size()`.
+         */
+        static constexpr INDEX_t batch_size = 500;
+
+        /** 
+         * See `MiniBatch::set_max_change_proportion()`.
+         */
+        static constexpr double max_change_proportion = 0.01;
+
+        /** 
+         * See `MiniBatch::set_convergence_history()`.
+         */
+        static constexpr int convergence_history = 10;
+
+        /** 
+         * See `MiniBatch::set_seed()`.
+         */
+        static constexpr uint64_t seed = 1234567890;
+    };
+
+private:
+    int maxiter = Defaults::max_iterations;
+
+    INDEX_t batch_size = Defaults::batch_size;
+
+    int history = Defaults::convergence_history;
+
+    double max_change = Defaults::max_change_proportion;
+
+    uint64_t seed = Defaults::seed;
+public:
+    /**
+     * @param i Maximum number of iterations.
+     * More iterations increase the opportunity for convergence at the cost of more computational time.
+     *
+     * @return A reference to this `MiniBatch` object.
+     */
+    MiniBatch& set_max_iterations(int i = Defaults::max_iterations) {
+        maxiter = i;
+        return *this;
+    }
+
+    /**
+     * @param s Number of observations in the mini-batch.
+     * Larger numbers improve quality at the cost of computational time.
+     *
+     * @return A reference to this `MiniBatch` object.
+     */
+    MiniBatch& set_batch_size(INDEX_t s = Defaults::batch_size) {
+        batch_size = s;
+        return *this;
+    }
+
+    /**
+     * @param p Maximum proportion of observations in each cluster that can be reassigned when checking for convergence (i.e., \f$p\f$).
+     * Lower values improve the quality of the result at the cost of computational time.
+     *
+     * @return A reference to this `MiniBatch` object.
+     */
+    MiniBatch& set_max_change_proportion(double p = Defaults::max_change_proportion) {
+        max_change = p;
+        return *this;
+    }
+
+    /**
+     * @param h Number of iterations to remember when checking for convergence (i.e., \f$h\f$).
+     * Larger values improve the quality of the result at the cost of computational time.
+     *
+     * @return A reference to this `MiniBatch` object.
+     */
+    MiniBatch& set_convergence_history(int h = Defaults::convergence_history) {
+        history = h;
+        return *this;
+    }
+
+    /** 
+     * @param s Seed to use for PRNG when sampling observations to use in each mini-batch.
+     *
+     * @return A reference to this `MiniBatch` object.
+     */
+    MiniBatch& set_seed(uint64_t s = Defaults::seed) {
+        seed = s;
+        return *this;
+    }
+
+public:
+    /**
+     * @param ndim Number of dimensions.
+     * @param nobs Number of observations.
+     * @param[in] data Pointer to a `ndim`-by-`nobs` array where columns are observations and rows are dimensions. 
+     * Data should be stored in column-major order.
+     * @param ncenters Number of cluster centers.
+     * @param[in, out] centers Pointer to a `ndim`-by-`ncenters` array where columns are cluster centers and rows are dimensions. 
+     * On input, this should contain the initial centroid locations for each cluster.
+     * Data should be stored in column-major order.
+     * On output, this will contain the final centroid locations for each cluster.
+     * @param[out] clusters Pointer to an array of length `nobs`.
+     * On output, this will contain the cluster assignment for each observation.
+     *
+     * @return `centers` and `clusters` are filled, and a `Details` object is returned containing clustering statistics.
+     * If `ncenters > nobs`, only the first `nobs` columns of the `centers` array will be filled.
+     */
+    Details<DATA_t, INDEX_t> run(int ndim, INDEX_t nobs, const DATA_t* data, CLUSTER_t ncenters, DATA_t* centers, CLUSTER_t* clusters) {
+        if (is_edge_case(nobs, ncenters)) {
+            return process_edge_case(ndim, nobs, data, ncenters, centers, clusters);
+        }
+
+        int iter = 0, status = 0;
+        std::vector<INDEX_t> total_sampled(ncenters);
+        std::vector<CLUSTER_t> previous(nobs);
+        std::vector<INDEX_t> last_changed(ncenters), last_sampled(ncenters);
+
+        auto actual_batch_size = std::min(batch_size, nobs);
+        int last_updated = 0;
+        std::mt19937_64 eng(seed);
+
+        for (iter = 1; iter <= maxiter; ++iter) {
+            auto chosen = sample_without_replacement(nobs, actual_batch_size, eng);
+            if (iter > 1) {
+                for (auto o : chosen) {
+                    previous[o] = clusters[o];
+                }
+            }
+
+            QuickSearch<DATA_t, CLUSTER_t> index(ndim, ncenters, centers);
+            #pragma omp parallel for
+            for (size_t i = 0; i < chosen.size(); ++i) {
+                clusters[chosen[i]] = index.find(data + chosen[i] * ndim);
+            }
+
+            // Updating the means for each cluster.
+            for (auto o : chosen) {
+                const auto c = clusters[o];
+                auto& n = total_sampled[c];
+                ++n;
+
+                auto ocopy = data + o * ndim;
+                auto ccopy = centers + c * ndim;
+                for (int d = 0; d < ndim; ++d, ++ocopy, ++ccopy) {
+                    (*ccopy) += (*ocopy - *ccopy)/n;
+                }
+            }
+
+            // Checking for updates.
+            if (iter != 1) {
+                for (auto o : chosen) {
+                    ++(last_sampled[previous[o]]);
+                    if (previous[o] != clusters[o]) {
+                        ++(last_sampled[clusters[o]]);
+                        ++(last_changed[previous[o]]);
+                        ++(last_changed[clusters[o]]);
+                    }
+                }
+
+                if (iter % history == 1) {
+                    bool too_many_changes = false;
+                    for (CLUSTER_t c = 0; c < ncenters; ++c) {
+                        if (static_cast<double>(last_changed[c]) >= last_sampled[c] * max_change) {
+                            too_many_changes = true;
+                            break;
+                        }
+                    }
+
+                    if (!too_many_changes) {
+                        break;
+                    }
+                    std::fill(last_sampled.begin(), last_sampled.end(), 0);
+                    std::fill(last_changed.begin(), last_changed.end(), 0);
+                }
+            }
+        }
+
+        if (iter == maxiter + 1) {
+            status = 2;
+        }
+
+        // Run through all observations to make sure they have the latest cluster assignments.
+        QuickSearch<DATA_t, CLUSTER_t> index(ndim, ncenters, centers);
+        #pragma omp parallel for
+        for (INDEX_t o = 0; o < nobs; ++o) {
+            clusters[o] = index.find(data + o * ndim);
+        }
+
+        std::fill(total_sampled.begin(), total_sampled.end(), 0);
+        for (INDEX_t o = 0; o < nobs; ++o) {
+            ++total_sampled[clusters[o]];
+        }
+
+        for (auto c : total_sampled) {
+            if (c == 0) {
+                status = 1;
+                break;
+            }
+        }
+
+        compute_centroids(ndim, nobs, data, ncenters, centers, clusters, total_sampled);
+
+        return Details<DATA_t, INDEX_t>(
+            std::move(total_sampled),
+            compute_wcss(ndim, nobs, data, ncenters, centers, clusters),
+            iter, 
+            status
+        );
+    }
+};
+
+}
+
+#endif

data/vendor/kmeans/QuickSearch.hpp

@@ -0,0 +1,179 @@
+#ifndef KMEANS_QUICKSEARCH_HPP
+#define KMEANS_QUICKSEARCH_HPP
+
+#include <vector>
+#include <random>
+#include <limits>
+#include <cmath>
+#include <tuple>
+#include <iostream>
+
+namespace kmeans {
+
+/* Adapted from http://stevehanov.ca/blog/index.php?id=130 */
+template<typename DATA_t = double, typename CLUSTER_t = int> 
+class QuickSearch {
+private:
+    int num_dim;
+    CLUSTER_t num_obs;
+    const DATA_t* reference;
+
+    static DATA_t normalize(DATA_t x) {
+        return std::sqrt(x);
+    }
+
+    static DATA_t raw_distance(const DATA_t* x, const DATA_t* y, int ndim) {
+        DATA_t output = 0;
+        for (int i = 0; i < ndim; ++i, ++x, ++y) {
+            output += (*x - *y) * (*x - *y);
+        }
+        return output;
+    }
+
+private:
+    typedef int NodeIndex_t;
+    static const NodeIndex_t LEAF_MARKER=-1;
+
+    // Single node of a VP tree (has a point and radius; left children are closer to point than the radius)
+    struct Node {
+        DATA_t threshold;  // radius 
+        CLUSTER_t index; // original index of current vantage point 
+        NodeIndex_t left;  // node index of the next vantage point for all children closer than 'threshold' from the current vantage point
+        NodeIndex_t right; // node index of the next vantage point for all children further than 'threshold' from the current vantage point
+        Node(NodeIndex_t i=0) : threshold(0), index(i), left(LEAF_MARKER), right(LEAF_MARKER) {}
+    };
+    std::vector<Node> nodes;
+
+    typedef std::tuple<CLUSTER_t, const DATA_t*, DATA_t> DataPoint; // internal distances computed using "DATA_t" type, even if output is returned with DISTANCE_t.
+
+    template<class SAMPLER>
+    NodeIndex_t buildFromPoints(NodeIndex_t lower, NodeIndex_t upper, std::vector<DataPoint>& items, SAMPLER& rng) {
+        if (upper == lower) {     // indicates that we're done here!
+            return LEAF_MARKER;
+        }
+
+        NodeIndex_t pos = nodes.size();
+        nodes.resize(pos + 1);
+        Node& node=nodes.back();
+            
+        int gap = upper - lower;
+        if (gap > 1) {      // if we did not arrive at leaf yet
+
+            /* Choose an arbitrary point and move it to the start of the [lower, upper)
+             * interval in 'items'; this is our new vantage point.
+             * 
+             * Yes, I know that the modulo method does not provide strictly
+             * uniform values but statistical correctness doesn't really matter
+             * here... but reproducibility across platforms does matter, and
+             * std::uniform_int_distribution is implementation-dependent!
+             */
+            NodeIndex_t i = static_cast<NodeIndex_t>(rng() % gap + lower);
+            std::swap(items[lower], items[i]);
+            const auto& vantage = items[lower];
+
+            // Compute distances to the new vantage point.
+            const DATA_t* ref = std::get<1>(vantage);
+            for (size_t i = lower + 1; i < upper; ++i) {
+                const DATA_t* loc = std::get<1>(items[i]);
+                std::get<2>(items[i]) = raw_distance(ref, loc, num_dim);
+            }
+
+            // Partition around the median distance from the vantage point.
+            NodeIndex_t median = lower + gap/2;
+            std::nth_element(items.begin() + lower + 1, items.begin() + median, items.begin() + upper,
+                [&](const DataPoint& left, const DataPoint& right) -> bool {
+                    return std::get<2>(left) < std::get<2>(right);
+                }
+            );
+           
+            // Threshold of the new node will be the distance to the median
+            node.threshold = normalize(std::get<2>(items[median]));
+
+            // Recursively build tree
+            node.index = std::get<0>(vantage);
+            node.left = buildFromPoints(lower + 1, median, items, rng);
+            node.right = buildFromPoints(median, upper, items, rng);
+        } else {
+            node.index = std::get<0>(items[lower]);
+        }
+        
+        return pos;
+    }
+
+private:
+    template<typename INPUT_t>
+    void search_nn(NodeIndex_t curnode_index, const INPUT_t* target, CLUSTER_t& closest, DATA_t& tau) const { 
+        if (curnode_index == LEAF_MARKER) { // indicates that we're done here
+            return;
+        }
+        
+        // Compute distance between target and current node
+        const auto& curnode=nodes[curnode_index];
+        DATA_t dist = normalize(raw_distance(reference + curnode.index * num_dim, target, num_dim));
+
+        // If current node within radius tau
+        if (dist < tau) {
+            closest = curnode.index;
+            tau = dist;
+        }
+
+        // Return if we arrived at a leaf
+        if (curnode.left == LEAF_MARKER && curnode.right == LEAF_MARKER) {
+            return;
+        }
+        
+        // If the target lies within the radius of ball
+        if (dist < curnode.threshold) {
+            if (dist - tau <= curnode.threshold) {         // if there can still be neighbors inside the ball, recursively search left child first
+                search_nn(curnode.left, target, closest, tau);
+            }
+            
+            if (dist + tau >= curnode.threshold) {         // if there can still be neighbors outside the ball, recursively search right child
+                search_nn(curnode.right, target, closest, tau);
+            }
+        
+        // If the target lies outsize the radius of the ball
+        } else {
+            if (dist + tau >= curnode.threshold) {         // if there can still be neighbors outside the ball, recursively search right child first
+                search_nn(curnode.right, target, closest, tau);
+            }
+            
+            if (dist - tau <= curnode.threshold) {         // if there can still be neighbors inside the ball, recursively search left child
+                search_nn(curnode.left, target, closest, tau);
+            }
+        }
+    }
+
+public:
+    QuickSearch(CLUSTER_t ndim, CLUSTER_t nobs, const DATA_t* vals) : num_dim(ndim), num_obs(nobs), reference(vals) {
+        std::vector<DataPoint> items;
+        items.reserve(num_obs);
+        auto ptr = vals;
+        for (CLUSTER_t i = 0; i < num_obs; ++i, ptr += num_dim) {
+            items.push_back(DataPoint(i, ptr, 0));
+        }
+
+        nodes.reserve(num_obs);
+        std::mt19937_64 rand(1234567890); // seed doesn't really matter, we don't need statistical correctness here.
+        buildFromPoints(0, num_obs, items, rand);
+        return;
+    }
+
+    CLUSTER_t find(const DATA_t* query) const {
+        DATA_t tau = std::numeric_limits<DATA_t>::max();
+        CLUSTER_t closest = 0;
+        search_nn(0, query, closest, tau);
+        return closest;
+    }
+
+    std::pair<CLUSTER_t, DATA_t> find_with_distance(const DATA_t* query) const {
+        DATA_t tau = std::numeric_limits<DATA_t>::max();
+        CLUSTER_t closest = 0;
+        search_nn(0, query, closest, tau);
+        return std::make_pair(closest, tau);
+    }
+};
+
+}
+
+#endif

data/vendor/kmeans/compute_centroids.hpp

@@ -0,0 +1,32 @@
+#ifndef KMEANS_COMPUTE_CENTROIDS_HPP
+#define KMEANS_COMPUTE_CENTROIDS_HPP
+
+#include <algorithm>
+
+namespace kmeans {
+
+template<typename DATA_t = double, typename INDEX_t = int, typename CLUSTER_t = int, class V>
+void compute_centroids(int ndim, INDEX_t nobs, const DATA_t* data, CLUSTER_t ncenters, DATA_t* centers, const CLUSTER_t* clusters, const V& sizes) {
+    std::fill(centers, centers + ndim * ncenters, 0);
+
+    for (INDEX_t obs = 0; obs < nobs; ++obs) {
+        auto copy = centers + clusters[obs] * ndim;
+        auto mine = data + obs * ndim;
+        for (int dim = 0; dim < ndim; ++dim, ++copy, ++mine) {
+            *copy += *mine;
+        }
+    }
+
+    for (CLUSTER_t cen = 0; cen < ncenters; ++cen) {
+        if (sizes[cen]) {
+            auto curcenter = centers + cen * ndim;
+            for (int dim = 0; dim < ndim; ++dim, ++curcenter) {
+                *curcenter /= sizes[cen];
+            }
+        }
+    }
+}
+
+}
+
+#endif

data/vendor/kmeans/compute_wcss.hpp

@@ -0,0 +1,27 @@
+#ifndef KMEANS_COMPUTE_WCSS_HPP
+#define KMEANS_COMPUTE_WCSS_HPP
+
+#include <vector>
+
+namespace kmeans {
+
+template<typename DATA_t = double, typename INDEX_t = int, typename CLUSTER_t = int>
+std::vector<DATA_t> compute_wcss(int ndim, INDEX_t nobs, const DATA_t* data, CLUSTER_t ncenters, const DATA_t* centers, const CLUSTER_t* clusters) {
+    std::vector<DATA_t> wcss(ncenters);
+    for (INDEX_t obs = 0; obs < nobs; ++obs) {
+        auto cen = clusters[obs];
+        auto curcenter = centers + cen * ndim;
+        auto& curwcss = wcss[cen];
+
+        auto curdata = data + obs * ndim;
+        for (int dim = 0; dim < ndim; ++dim, ++curcenter, ++curdata) {
+            curwcss += (*curdata - *curcenter) * (*curdata - *curcenter);
+        }
+    }
+
+    return wcss;
+}
+
+}
+
+#endif

data/vendor/kmeans/is_edge_case.hpp

@@ -0,0 +1,42 @@
+#ifndef KMEANS_IS_EDGE_CASE_HPP
+#define KMEANS_IS_EDGE_CASE_HPP
+
+#include <numeric>
+#include <algorithm>
+#include "compute_wcss.hpp"
+#include "compute_centroids.hpp"
+
+namespace kmeans {
+
+template<typename INDEX_t = int, typename CLUSTER_t = int>
+bool is_edge_case(INDEX_t nobs, CLUSTER_t ncenters) {
+    return (ncenters <= 1 || static_cast<INDEX_t>(ncenters) >= nobs);
+}
+
+template<typename DATA_t = double, typename INDEX_t = int, typename CLUSTER_t = int>
+Details<DATA_t, INDEX_t> process_edge_case(int ndim, INDEX_t nobs, const DATA_t* data, CLUSTER_t ncenters, DATA_t* centers, CLUSTER_t* clusters) {
+    if (ncenters == 1) {
+        // All points in cluster 0.
+        std::fill(clusters, clusters + nobs, 0);
+        std::vector<INDEX_t> sizes(1, nobs);
+        compute_centroids(ndim, nobs, data, ncenters, centers, clusters, sizes);
+        auto wcss = compute_wcss(ndim, nobs, data, ncenters, centers, clusters);
+        return Details(std::move(sizes), std::move(wcss), 0, 0);
+
+    } else if (ncenters >= nobs) {
+        // Special case, each observation is a center.
+        std::iota(clusters, clusters + nobs, 0);
+        std::vector<INDEX_t> sizes(ncenters);
+        std::fill(sizes.begin(), sizes.begin() + nobs, 1);
+        compute_centroids(ndim, nobs, data, ncenters, centers, clusters, sizes);
+        auto wcss = compute_wcss(ndim, nobs, data, ncenters, centers, clusters);
+        return Details(std::move(sizes), std::move(wcss), 0, (ncenters > nobs ? 3 : 0));
+
+    } else { //i.e., ncenters == 0, provided is_edge_case is true.
+        return Details<DATA_t, INDEX_t>(0, 3);
+    }
+}
+
+}
+
+#endif

data/vendor/kmeans/random.hpp

@@ -0,0 +1,55 @@
+#ifndef KMEANS_RANDOM_HPP
+#define KMEANS_RANDOM_HPP
+
+#include <vector>
+#include <numeric>
+#include "aarand/aarand.hpp"
+
+namespace kmeans {
+
+template<typename T = int, class ENGINE>
+std::vector<T> sample_without_replacement(T population, size_t choose, ENGINE& eng) {
+    std::vector<T> sofar;
+
+    if (choose >= population) {
+        sofar.resize(population);
+        std::iota(sofar.begin(), sofar.end(), 0);
+    } else {
+        sofar.reserve(choose);
+        T traversed = 0;
+
+        while (sofar.size() < choose) {
+            if (static_cast<double>(choose - sofar.size()) > static_cast<double>(population - traversed) * aarand::standard_uniform(eng)) {
+                sofar.push_back(traversed);
+            }
+            ++traversed;
+        }
+    }
+
+    return sofar;
+}
+
+template<typename DATA_t, typename INDEX_t, class ENGINE>
+INDEX_t weighted_sample(const std::vector<DATA_t>& cumulative, const std::vector<DATA_t>& mindist, INDEX_t nobs, ENGINE& eng) {
+    auto total = cumulative.back();
+    INDEX_t chosen_id = 0;
+
+    do {
+        const DATA_t sampled_weight = total * aarand::standard_uniform(eng);
+        chosen_id = std::lower_bound(cumulative.begin(), cumulative.end(), sampled_weight) - cumulative.begin();
+
+        // We wrap this in a do/while to defend against edge cases where
+        // ties are chosen. The most obvious of these is when you get a
+        // `sampled_weight` of zero _and_ there exists a bunch of zeros at
+        // the start of `cumulative`. One could also get unexpected ties
+        // from limited precision in floating point comparisons, so we'll
+        // just be safe and implement a loop here, in the same vein as
+        // uniform01.
+    } while (chosen_id == nobs || mindist[chosen_id] == 0);
+
+    return chosen_id;
+}
+
+}
+
+#endif