ckmeans 0.1.2 → 1.0.0.rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72a2d84963628565eb94d962dc73aa1230f7af8c6948dc6713b0f9582d1bb401
-  data.tar.gz: ec3cb0c09eaaf38147d1a7dfd1f772b960905f336b801aade402d6f795329d27
+  metadata.gz: 0c032b968f4f996b50ea2d63b1624eccd2d6dd4ff4922042143ada4200664216
+  data.tar.gz: c8b220a8ebe08b2aebc78cb7c5e347e4d734627aac6ad61da298dc22ee30e884
 SHA512:
-  metadata.gz: ceb63e72327d2f3a00aee2c23c7ba8bd63f835d7b0132ce0785f9636b550a79ea4827eadfeb9a6d95c36e9b046ae5005325adeaf2cdd1689444d8f7af181bbc1
-  data.tar.gz: 0d7d7ca2c942ecb238c7d26cc601d64187ae97218bdd59aeb6ca067d11f08472725e6b3e8ca9021be88d63aa4ae45a5d629a5f8966a3731b7941aa6c0a205619
+  metadata.gz: 0fa159e921f89ba73ca478476903ae3a07893ea8cdff2b86ec25605f22f2864c75eb8763c5daf50657b194ed216dec69cfe99e405035f2b83b540e3e5c5c2599
+  data.tar.gz: 99f4ea7b2db58fb076325a4b3cd3b5866e3ad35d48c3d89521555b22e14720ac37e422a3ad258c402494cc6c2896a9a48bab4c126d4660912f29caa4a393a28b

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2025-03-31 15:04:58 UTC using RuboCop version 1.75.1.
+# on 2025-04-17 07:09:28 UTC using RuboCop version 1.75.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -11,48 +11,35 @@
 Metrics/AbcSize:
   Max: 95
 
-# Offense count: 3
+# Offense count: 2
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 # AllowedMethods: refine
 Metrics/BlockLength:
-  Max: 112
+  Max: 41
 
 # Offense count: 3
 # Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/CyclomaticComplexity:
   Max: 10
 
-# Offense count: 5
+# Offense count: 6
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 48
 
-# Offense count: 5
-# Configuration parameters: CountKeywordArgs, MaxOptionalParameters.
-Metrics/ParameterLists:
-  Max: 9
-
 # Offense count: 3
 # Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/PerceivedComplexity:
   Max: 13
 
-# Offense count: 13
+# Offense count: 12
 # Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
 # AllowedNames: as, at, by, cc, db, id, if, in, io, ip, of, on, os, pp, to
 Naming/MethodParameterName:
   Exclude:
     - 'lib/ckmeans/clusterer.rb'
 
-# Offense count: 1
-# Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers, AllowedPatterns.
-# SupportedStyles: snake_case, normalcase, non_integer
-# AllowedIdentifiers: TLS1_1, TLS1_2, capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339, x86_64
-Naming/VariableNumber:
-  Exclude:
-    - 'lib/ckmeans/clusterer.rb'
-
-# Offense count: 6
+# Offense count: 5
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: EnforcedStyle, AllowedMethods, AllowedPatterns.
 # SupportedStyles: predicate, comparison

data/Rakefile

@@ -2,6 +2,12 @@
 
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "rake/extensiontask"
+
+Rake::ExtensionTask.new("extensions") do |ext|
+  ext.lib_dir = "lib/ckmeans"
+  ext.ext_dir = "ext/ckmeans"
+end
 
 RSpec::Core::RakeTask.new(:spec)
 
@@ -9,4 +15,4 @@ require "rubocop/rake_task"
 
 RuboCop::RakeTask.new
 
-task default: %i[spec rubocop]
+task default: %i[compile spec rubocop]

data/ext/ckmeans/extconf.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "mkmf"
+
+create_makefile("ckmeans/extensions")

data/ext/ckmeans/extensions.c

@@ -0,0 +1,704 @@
+#include <stdio.h>
+#include <assert.h>
+#include <math.h>
+#include "ruby.h"
+
+typedef struct Arena {
+    uint32_t capacity;
+    uint32_t offset;
+    uint8_t  *buffer;
+} Arena;
+
+typedef struct MatrixF {
+    uint32_t ncols;
+    uint32_t nrows;
+    long double *values;
+} MatrixF;
+
+typedef struct MatrixI {
+    uint32_t ncols;
+    uint32_t nrows;
+    uint32_t *values;
+} MatrixI;
+
+typedef struct VectorF {
+    uint32_t nvalues;
+    long double *values;
+} VectorF;
+
+typedef struct VectorI {
+    uint32_t nvalues;
+    uint32_t *values;
+} VectorI;
+
+typedef struct State {
+    uint32_t xcount;
+    uint32_t kmin;
+    uint32_t kmax;
+    bool     apply_deviation;
+    Arena   *arena;
+    VectorF *xsorted;
+    MatrixF *cost;
+    MatrixI *splits;
+    VectorF *xsum;
+    VectorF *xsumsq;
+} State;
+
+typedef struct RowParams {
+    uint32_t row;
+    uint32_t imin;
+    uint32_t imax;
+    uint32_t istep;
+} RowParams;
+
+typedef struct {
+    long double mean;
+    long double variance;
+} SegmentStats;
+
+VALUE        rb_ckmeans_sorted_group_sizes(VALUE self);
+
+Arena       *arena_create(uint32_t);
+void        *arena_alloc(Arena*, uint32_t);
+void         arena_rewind(Arena*);
+void         arena_destroy(Arena*);
+
+MatrixF     *matrix_create_f(Arena*, uint32_t, uint32_t);
+MatrixI     *matrix_create_i(Arena*, uint32_t, uint32_t);
+void         matrix_set_f(MatrixF*, uint32_t, uint32_t, long double value);
+long double  matrix_get_f(MatrixF*, uint32_t, uint32_t);
+void         matrix_inspect_f(MatrixF*);
+void         matrix_set_i(MatrixI*, uint32_t, uint32_t, uint32_t value);
+uint32_t     matrix_get_i(MatrixI*, uint32_t, uint32_t);
+void         matrix_inspect_i(MatrixI*);
+
+VectorF     *vector_create_f(Arena*, uint32_t);
+void         vector_set_f(VectorF*, uint32_t offset, long double value);
+long double  vector_get_f(VectorF*, uint32_t offset);
+long double  vector_get_diff_f(VectorF*, uint32_t, uint32_t);
+void         vector_inspect_f(VectorF*);
+VectorI     *vector_create_i(Arena*, uint32_t);
+VectorI     *vector_dup_i(VectorI*, Arena*);
+void         vector_set_i(VectorI*, uint32_t offset, uint32_t value);
+uint32_t     vector_get_i(VectorI*, uint32_t offset);
+void         vector_downsize_i(VectorI*, uint32_t);
+void         vector_inspect_i(VectorI*);
+
+long double  dissimilarity(uint32_t, uint32_t, VectorF*, VectorF*);
+void         fill_row(State, uint32_t, uint32_t, uint32_t);
+void         smawk(State, RowParams, VectorI*);
+void         find_min_from_candidates(State, RowParams, VectorI*);
+VectorI     *prune_candidates(State, RowParams, VectorI*);
+void         fill_even_positions(State, RowParams, VectorI*);
+SegmentStats shifted_data_variance(VectorF*, uint32_t, uint32_t);
+VectorI     *backtrack_sizes(State, uint32_t);
+uint32_t     find_koptimal(State);
+
+void Init_extensions(void) {
+    VALUE ckmeans_module = rb_const_get(rb_cObject, rb_intern("Ckmeans"));
+    VALUE clusterer_class = rb_const_get(ckmeans_module, rb_intern("Clusterer"));
+
+    rb_define_private_method(clusterer_class, "sorted_group_sizes", rb_ckmeans_sorted_group_sizes, 0);
+}
+
+# define ARENA_MIN_CAPACITY 1024
+# define ALLOCATION_FACTOR 20
+# define PIx2 (M_PI * 2.0)
+
+VALUE rb_ckmeans_sorted_group_sizes(VALUE self) {
+    VALUE rb_xcount              = rb_ivar_get(self, rb_intern("@xcount"));
+    VALUE rb_kmin                = rb_ivar_get(self, rb_intern("@kmin"));
+    VALUE rb_kmax                = rb_ivar_get(self, rb_intern("@kmax"));
+    VALUE rb_xsorted             = rb_ivar_get(self, rb_intern("@xsorted"));
+    VALUE rb_apply_bic_deviation = rb_ivar_get(self, rb_intern("@apply_bic_deviation"));
+    uint32_t xcount              = NUM2UINT(rb_xcount);
+    uint32_t kmin                = NUM2UINT(rb_kmin);
+    uint32_t kmax                = NUM2UINT(rb_kmax);
+    bool apply_deviation         = RTEST(rb_apply_bic_deviation);
+    Arena *arena                 = arena_create(sizeof(int) * xcount * kmax * ALLOCATION_FACTOR);
+
+    if (arena == NULL) {
+        return Qnil;
+    }
+
+    MatrixF *cost    = matrix_create_f(arena, kmax, xcount);
+    MatrixI *splits  = matrix_create_i(arena, kmax, xcount);
+    VectorF *xsorted = vector_create_f(arena, xcount);
+    /* TODO: pack sums into one vector of pairs */
+    VectorF *xsum    = vector_create_f(arena, xcount);
+    VectorF *xsumsq  = vector_create_f(arena, xcount);
+
+    for (uint32_t i = 0; i < xcount; i++) {
+        long double xi = NUM2DBL(rb_ary_entry(rb_xsorted, i));
+        vector_set_f(xsorted, i, xi);
+    }
+
+    State state = {
+        .arena           = arena,
+        .xcount          = xcount,
+        .kmin            = kmin,
+        .kmax            = kmax,
+        .apply_deviation = apply_deviation,
+        .xsorted         = xsorted,
+        .cost            = cost,
+        .splits          = splits,
+        .xsum            = xsum,
+        .xsumsq          = xsumsq
+    };
+
+
+    long double shift        = vector_get_f(xsorted, xcount / 2);
+    long double diff_initial = vector_get_f(xsorted, 0) - shift;
+
+    vector_set_f(xsum, 0, diff_initial);
+    vector_set_f(xsumsq, 0, diff_initial * diff_initial);
+
+    for (uint32_t i = 1; i < xcount; i++) {
+        long double xi          = vector_get_f(xsorted, i);
+        long double xsum_prev   = vector_get_f(xsum, i - 1);
+        long double xsumsq_prev = vector_get_f(xsumsq, i - 1);
+        long double diff        = xi - shift;
+
+        vector_set_f(xsum, i, xsum_prev + diff);
+        vector_set_f(xsumsq, i, xsumsq_prev + diff * diff);
+        matrix_set_f(cost, 0, i, dissimilarity(0, i, xsum, xsumsq));
+        matrix_set_i(splits, 0, i, 0);
+    }
+
+    for (uint32_t q = 1; q <= kmax - 1; q++) {
+        uint32_t imin = (q < kmax - 1) ? ((q > 1) ? q : 1) : xcount - 1;
+        fill_row(state, q, imin, xcount - 1);
+    }
+
+    uint32_t koptimal = find_koptimal(state);
+
+    VectorI *sizes = backtrack_sizes(state, koptimal);
+
+    /* printf("XSORTED \t"); vector_inspect_f(xsorted); */
+    /* printf("K OPTIMAL: %lld\n", koptimal); */
+    /* printf("SIZES \t"); vector_inspect_i(sizes); */
+    /* printf("FINAL COST\n"); matrix_inspect_f(cost); */
+    /* printf("FINAL SPLITS\n"); matrix_inspect_i(splits); */
+
+    VALUE response = rb_ary_new2(sizes->nvalues);
+    for (uint32_t i = 0; i < sizes->nvalues; i++) {
+        VALUE size = LONG2NUM(vector_get_i(sizes, i));
+        rb_ary_store(response, i, size);
+    }
+
+    arena_destroy(arena);
+
+    return response;
+}
+
+uint32_t find_koptimal(State state)
+{
+    uint32_t kmin           = state.kmin;
+    uint32_t kmax           = state.kmax;
+    uint32_t xcount         = state.xcount;
+    uint32_t kopt           = kmin;
+    uint32_t xindex_max     = state.xcount - 1;
+    VectorF *xsorted       = state.xsorted;
+    long double x0         = vector_get_f(xsorted, 0);
+    long double xn         = vector_get_f(xsorted, xindex_max);
+    long double max_bic    = 0.0;
+    long double adjustment = state.apply_deviation ? 0.0 : 1.0;
+
+    for (uint32_t k = kmin; k <= kmax; k++) {
+        uint32_t index_right, index_left = 0;
+        long double bin_left, bin_right, loglikelihood = 0.0;
+        VectorI *sizes = backtrack_sizes(state, k);
+
+        for (uint32_t kb = 0; kb < k; kb++) {
+            uint32_t npoints   = vector_get_i(sizes, kb);
+            index_right        = index_left + npoints - 1;
+            long double xleft  = vector_get_f(xsorted, index_left);
+            long double xright = vector_get_f(xsorted, index_right);
+            bin_left           = xleft;
+            bin_right          = xright;
+
+            if (xleft == xright) {
+                bin_left  = index_left == 0
+                    ? x0
+                    : (vector_get_f(xsorted, index_left - 1) + xleft) / 2;
+                bin_right = index_right < xindex_max
+                    ? (xright + vector_get_f(xsorted, index_right + 1)) / 2
+                    : xn;
+            }
+
+            long double bin_width = bin_right - bin_left;
+            SegmentStats stats    = shifted_data_variance(xsorted, index_left, index_right);
+            long double mean      = stats.mean;
+            long double variance  = stats.variance;
+
+            if (variance > 0) {
+                for (uint32_t i = index_left; i <= index_right; i++) {
+                    long double xi = vector_get_f(xsorted, i);
+                    loglikelihood += -(xi - mean) * (xi - mean) / (2.0 * variance);
+                }
+                loglikelihood += npoints * (
+                    (log(npoints / (long double) xcount) * adjustment) -
+                    (0.5 * log(PIx2 * variance))
+                );
+            } else {
+                loglikelihood += npoints * log(1.0 / bin_width / xcount);
+            }
+
+            index_left = index_right + 1;
+        }
+
+        long double bic = (2.0 * loglikelihood) - (((3 * k) - 1) * log((long double) xcount));
+
+        if (k == kmin) {
+            max_bic = bic;
+            kopt = kmin;
+        } else if (bic > max_bic) {
+            max_bic = bic;
+            kopt = k;
+        }
+    }
+
+    return kopt;
+}
+
+VectorI *backtrack_sizes(State state, uint32_t k)
+{
+    MatrixI *splits = state.splits;
+    VectorI *sizes  = vector_create_i(state.arena, k);
+    uint32_t xcount = state.xcount;
+    uint32_t right  = xcount - 1;
+    uint32_t left   = 0;
+
+    // Common case works with `i` remaining unsigned and unconditional assignment of the next `left` and `right`
+    for (uint32_t i = k - 1; i > 0; i--, right = left - 1) {
+        left = matrix_get_i(splits, i, right);
+        vector_set_i(sizes, i, right - left + 1);
+    }
+    left = matrix_get_i(splits, 0, right);
+    vector_set_i(sizes, 0, right - left + 1);
+
+    return sizes;
+}
+
+SegmentStats shifted_data_variance(VectorF *xsorted, uint32_t left, uint32_t right)
+{
+    const uint32_t n    = right - left + 1;
+    long double sum    = 0.0;
+    long double sumsq  = 0.0;
+    SegmentStats stats = { .mean = 0.0, .variance = 0.0 };
+
+    if (right >= left) {
+        const long double median = vector_get_f(xsorted, (left + right) / 2);
+
+        for (uint32_t i = left; i <= right; i++) {
+            const long double sumi = vector_get_f(xsorted, i) - median;
+
+            sum   += sumi;
+            sumsq += sumi * sumi;
+        }
+
+        stats.mean = (sum / n) + median;
+        if (n > 1) {
+            stats.variance = (sumsq - (sum * sum / n)) / (n - 1);
+        }
+    }
+
+    return stats;
+}
+
+void fill_row(State state, uint32_t q, uint32_t imin, uint32_t imax) {
+    uint32_t size = imax - q + 1;
+    VectorI *split_candidates = vector_create_i(state.arena, size);
+    for (uint32_t i = 0; i < size; i++) {
+        vector_set_i(split_candidates, i, q + i);
+    }
+    RowParams rparams = { .row = q, .imin = imin, .imax = imax, .istep = 1 };
+    smawk(state, rparams, split_candidates);
+}
+
+void smawk(State state, RowParams rparams, VectorI *split_candidates) {
+    const uint32_t imin  = rparams.imin;
+    const uint32_t imax  = rparams.imax;
+    const uint32_t istep = rparams.istep;
+
+    if ((imax - imin) <= (0 * istep)) {
+        find_min_from_candidates(state, rparams, split_candidates);
+    } else {
+        VectorI *odd_candidates = prune_candidates(state, rparams, split_candidates);
+        /* printf("PRUNED\t"); vector_inspect_i(odd_candidates); */
+        uint32_t istepx2         = istep * 2;
+        uint32_t imin_odd        = imin + istep;
+        uint32_t imax_odd        = imin_odd + ((imax - imin_odd) / istepx2 * istepx2);
+        RowParams rparams_odd   = { .row = rparams.row, .imin = imin_odd, .imax = imax_odd, .istep = istepx2 };
+
+        smawk(state, rparams_odd, odd_candidates);
+        fill_even_positions(state, rparams, split_candidates);
+    }
+}
+
+void fill_even_positions(State state, RowParams rparams, VectorI *split_candidates)
+{
+    uint32_t row     = rparams.row;
+    uint32_t imin    = rparams.imin;
+    uint32_t imax    = rparams.imax;
+    uint32_t istep   = rparams.istep;
+    uint32_t n       = split_candidates->nvalues;
+    uint32_t istepx2 = istep * 2;
+    uint32_t jl      = vector_get_i(split_candidates, 0);
+    VectorF *xsum   = state.xsum;
+    VectorF *xsumsq = state.xsumsq;
+    MatrixI *splits = state.splits;
+
+    for (uint32_t i = imin, r = 0; i <= imax; i += istepx2) {
+        while (vector_get_i(split_candidates, r) < jl) r++;
+
+        uint32_t rcandidate     = vector_get_i(split_candidates, r);
+        uint32_t cost_base_row = row - 1;
+        uint32_t cost_base_col = rcandidate - 1;
+        long double cost      =
+            matrix_get_f(state.cost, cost_base_row, cost_base_col) + dissimilarity(rcandidate, i, xsum, xsumsq);
+
+        matrix_set_f(state.cost, row, i, cost);
+        matrix_set_i(state.splits, row, i, rcandidate);
+
+        uint32_t jh         =
+            (i + istep) <= imax
+            ? matrix_get_i(splits, row, i + istep)
+            : vector_get_i(split_candidates, n - 1);
+
+        uint32_t jmax       = jh < i ? jh : i;
+        long double sjimin = dissimilarity(jmax, i, xsum, xsumsq);
+
+        for (++r; r < n && vector_get_i(split_candidates, r) <= jmax; r++) {
+            uint32_t jabs = vector_get_i(split_candidates, r);
+
+            if (jabs > i) break;
+            if (jabs < matrix_get_i(splits, row - 1, i)) continue;
+
+            long double cost_base = matrix_get_f(state.cost, row - 1, jabs  - 1);
+            long double sj        = cost_base + dissimilarity(jabs, i, xsum, xsumsq);
+            long double cost_prev = matrix_get_f(state.cost, row, i);
+
+            if (sj <= cost_prev) {
+                matrix_set_f(state.cost, row, i, sj);
+                matrix_set_i(state.splits, row, i, jabs);
+            } else if (cost_base + sjimin > cost_prev) {
+                break;
+            }
+        }
+
+        r--;
+        jl = jh;
+    }
+}
+
+void find_min_from_candidates(State state, RowParams rparams, VectorI *split_candidates)
+{
+    const uint32_t row     = rparams.row;
+    const uint32_t imin    = rparams.imin;
+    const uint32_t imax    = rparams.imax;
+    const uint32_t istep   = rparams.istep;
+    MatrixF *const cost   = state.cost;
+    MatrixI *const splits = state.splits;
+
+    uint32_t optimal_split_idx_prev = 0;
+
+    for (uint32_t i = imin; i <= imax; i += istep)
+    {
+        const uint32_t optimal_split_idx = optimal_split_idx_prev;
+        const uint32_t optimal_split     = vector_get_i(split_candidates, optimal_split_idx);
+        const uint32_t cost_prev         = matrix_get_f(cost, row - 1, optimal_split - 1);
+        const long double added_cost    = dissimilarity(optimal_split, i, state.xsum, state.xsumsq);
+
+        matrix_set_f(cost, row, i, cost_prev + added_cost);
+        matrix_set_i(splits, row, i, optimal_split);
+
+        for (uint32_t r = optimal_split_idx + 1; r < split_candidates->nvalues; r++)
+        {
+            uint32_t split = vector_get_i(split_candidates, r);
+
+            if (split < matrix_get_i(splits, row - 1, i)) continue;
+            if (split > i) break;
+
+            long double split_cost =
+                matrix_get_f(cost, row - 1, split - 1) + dissimilarity(split, i, state.xsum, state.xsumsq);
+
+            if (split_cost > matrix_get_f(cost, row, i)) continue;
+
+            matrix_set_f(cost, row, i, split_cost);
+            matrix_set_i(splits, row, i, split);
+            optimal_split_idx_prev = r;
+        }
+    }
+}
+
+VectorI *prune_candidates(State state, RowParams rparams, VectorI *split_candidates)
+{
+    uint32_t n = ((rparams.imax - rparams.imin) / rparams.istep) + 1;
+    uint32_t m = split_candidates->nvalues;
+
+    if (n >= m) return split_candidates;
+
+    uint32_t left    = -1;
+    uint32_t right   = 0;
+    VectorI *pruned = vector_dup_i(split_candidates, state.arena);
+
+    while (m > n)
+    {
+        uint32_t p         = left + 1;
+        uint32_t i         = rparams.imin + p * rparams.istep;
+        uint32_t j         = vector_get_i(pruned, right);
+        uint32_t jnext     = vector_get_i(pruned, right + 1);
+        long double sl    =
+            matrix_get_f(state.cost, rparams.row - 1, j - 1) + dissimilarity(j, i, state.xsum, state.xsumsq);
+        long double snext =
+            matrix_get_f(state.cost, rparams.row - 1, jnext - 1) + dissimilarity(jnext, i, state.xsum, state.xsumsq);
+
+        if ((sl < snext) && (p < n - 1)) {
+            left++;
+            right++;
+            vector_set_i(pruned, left, j);
+        } else if ((sl < snext) && (p == n - 1)) {
+            right++;
+            m--;
+            vector_set_i(pruned, right, j);
+        } else {
+            if (p > 0) {
+                /* TODO: extract `vector_setcpy_T` */
+                vector_set_i(pruned, right, vector_get_i(pruned, left));
+                left--;
+            } else {
+                right++;
+            }
+
+            m--;
+        }
+    }
+
+    for (uint32_t i = left + 1; i < m; i++) {
+        /* TODO: extract `vector_setcpy_T` */
+        vector_set_i(pruned, i, vector_get_i(pruned, right++));
+    }
+
+    vector_downsize_i(pruned, m);
+
+    return pruned;
+}
+
+long double dissimilarity(uint32_t j, uint32_t i, VectorF *xsum, VectorF *xsumsq) {
+    long double sji = 0.0;
+
+    if (j >= i) return sji;
+
+    if (j > 0) {
+        /* TODO: looks more like `segment_delta` */
+        long double segment_sum = vector_get_diff_f(xsum, i, j - 1);
+        uint32_t segment_size    = i - j + 1;
+        sji                     = vector_get_diff_f(xsumsq, i, j - 1) - (segment_sum * segment_sum / segment_size);
+    } else {
+        long double xsumi = vector_get_f(xsum, i);
+        sji               = vector_get_f(xsumsq, i) - (xsumi * xsumi / (i + 1));
+    }
+
+    return (sji > 0) ? sji : 0.0;
+}
+
+VectorF *vector_create_f(Arena *arena, uint32_t nvalues) {
+    VectorF *v;
+
+    v          = arena_alloc(arena, sizeof(*v));
+    v->values  = arena_alloc(arena, sizeof(*(v->values)) * nvalues);
+    v->nvalues = nvalues;
+
+    return v;
+}
+
+VectorI *vector_create_i(Arena *arena, uint32_t nvalues) {
+    VectorI *v;
+
+    v          = arena_alloc(arena, sizeof(*v));
+    v->values  = arena_alloc(arena, sizeof(*(v->values)) * nvalues);
+    v->nvalues = nvalues;
+
+    return v;
+}
+
+VectorI *vector_dup_i(VectorI *v, Arena *arena)
+{
+    VectorI *vdup = vector_create_i(arena, v->nvalues);
+
+    /* TODO: use one memcpy call */
+    for (uint32_t i = 0; i < v->nvalues; i++) {
+        vector_set_i(vdup, i, vector_get_i(v, i));
+    }
+
+    return vdup;
+}
+
+void vector_set_f(VectorF *v, uint32_t offset, long double value) {
+    assert(offset < v->nvalues && "[vector_set_f] element index should be less than nvalues");
+
+    *(v->values + offset) = value;
+}
+
+void vector_set_i(VectorI *v, uint32_t offset, uint32_t value) {
+    assert(offset < v->nvalues && "[vector_set_i] element index should be less than nvalues");
+
+    *(v->values + offset) = value;
+}
+
+uint32_t vector_get_i(VectorI *v, uint32_t offset) {
+    assert(offset < v->nvalues && "[vector_get_i] element index should be less than nvalues");
+
+    return *(v->values + offset);
+}
+
+void vector_downsize_i(VectorI *v, uint32_t new_size) {
+    v->nvalues = new_size;
+}
+
+void vector_inspect_i(VectorI *v) {
+    for (uint32_t i = 0; i < v->nvalues - 1; i++)
+        printf("%u, ", vector_get_i(v, i));
+    printf("%u\n", vector_get_i(v, v->nvalues - 1));
+}
+
+long double vector_get_f(VectorF *v, uint32_t offset) {
+    assert(offset < v->nvalues && "[vector_get_f] element index should be less than nvalues");
+
+    return *(v->values + offset);
+}
+
+long double vector_get_diff_f(VectorF *v, uint32_t i, uint32_t j) {
+    assert(i < v->nvalues && "[vector_get_diff_f] i should be less than nvalues");
+    assert(j < v->nvalues && "[vector_get_diff_f] j should be less than nvalues");
+
+    return *(v->values + i) - *(v->values + j);
+}
+
+void vector_inspect_f(VectorF *v) {
+    for (uint32_t i = 0; i < v->nvalues - 1; i++)
+        printf("%Lf, ", vector_get_f(v, i));
+    printf("%Lf\n", vector_get_f(v, v->nvalues - 1));
+}
+
+MatrixF *matrix_create_f(Arena *arena, uint32_t nrows, uint32_t ncols) {
+    MatrixF *m;
+
+    m         = arena_alloc(arena, sizeof(*m));
+    m->values = arena_alloc(arena, sizeof(*(m->values)) * ncols * nrows);
+    m->ncols  = ncols;
+    m->nrows  = nrows;
+
+    return m;
+}
+
+MatrixI *matrix_create_i(Arena *arena, uint32_t nrows, uint32_t ncols) {
+    MatrixI *m;
+
+    m         = arena_alloc(arena, sizeof(*m));
+    m->values = arena_alloc(arena, sizeof(*(m->values)) * ncols * nrows);
+    m->ncols  = ncols;
+    m->nrows  = nrows;
+
+    return m;
+}
+
+void matrix_set_f(MatrixF *m, uint32_t i, uint32_t j, long double value) {
+    assert(i < m->nrows && "[matrix_set_f] row offset should be less than nrows");
+    assert(j < m->cols &&  "[matrix_set_f] col offset should be less than ncols");
+
+    uint32_t offset = i * m->ncols + j;
+    *(m->values + offset) = value;
+}
+
+long double matrix_get_f(MatrixF *m, uint32_t i, uint32_t j) {
+    assert(i < m->nrows && "[matrix_get_f] row offset should be less than nrows");
+    assert(j < m->cols &&  "[matrix_get_f] col offset should be less than ncols");
+
+    uint32_t offset = i * m->ncols + j;
+    return *(m->values + offset);
+}
+
+void matrix_inspect_f(MatrixF *m) {
+    for (uint32_t i = 0; i < m->nrows; i++) {
+        for (uint32_t j = 0; j < m->ncols - 1; j++) {
+            long double value = matrix_get_f(m, i, j);
+
+            printf("%Lf, ", value);
+        }
+        printf("%Lf\n", matrix_get_f(m, i, m->ncols - 1));
+    }
+}
+
+void matrix_inspect_i(MatrixI *m) {
+    for (uint32_t i = 0; i < m->nrows; i++) {
+        for (uint32_t j = 0; j < m->ncols - 1; j++)
+            printf("%u, ", matrix_get_i(m, i, j));
+        printf("%u\n", matrix_get_i(m, i, m->ncols - 1));
+    }
+}
+
+void matrix_set_i(MatrixI *m, uint32_t i, uint32_t j, uint32_t value) {
+    assert(i < m->nrows && "[matrix_set_i] row offset should be less than nrows");
+    assert(j < m->cols &&  "[matrix_set_i] col offset should be less than ncols");
+
+    uint32_t offset = i * m->ncols + j;
+    *(m->values + offset) = value;
+}
+
+uint32_t matrix_get_i(MatrixI *m, uint32_t i, uint32_t j) {
+    assert(i < m->nrows && "[matrix_get_i] row offset should be less than nrows");
+    assert(j < m->cols &&  "[matrix_get_i] col offset should be less than ncols");
+
+    uint32_t offset = i * m->ncols + j;
+    return *(m->values + offset);
+}
+
+Arena *arena_create(uint32_t capacity) {
+    if (capacity < ARENA_MIN_CAPACITY) {
+        capacity = ARENA_MIN_CAPACITY;
+    }
+
+    Arena *arena;
+
+    arena = malloc(sizeof(*arena));
+    if (!arena) {
+        printf("Failed to allocate arena\n");
+        return NULL;
+    }
+
+    arena->buffer = calloc(1, capacity);
+    if (!arena->buffer) {
+        printf("Failed to allocate arena\n");
+        free(arena);
+        return NULL;
+    }
+
+    arena->capacity = capacity;
+    arena->offset   = 0;
+
+    printf("[Arena Created] Capacity: %u, offset: %u\n", arena->capacity, arena->offset);
+
+    return arena;
+}
+
+void *arena_alloc(Arena *arena, uint32_t size) {
+    size = (size + 7) & ~7;
+
+    if (arena->offset + size > arena->capacity) {
+        printf("Arena Out Of Memory\n");
+        return NULL;
+    }
+
+    void *ptr = arena->buffer + arena->offset;
+    arena->offset += size;
+
+    return ptr;
+}
+
+void arena_destroy(Arena *arena) {
+    printf("[Arena Destroy] Capacity: %u, offset: %u, left: %u\n", arena->capacity, arena->offset, arena->capacity - arena->offset);
+    free(arena->buffer);
+    free(arena);
+}

data/lib/ckmeans/clusterer.rb

@@ -2,8 +2,6 @@
 
 module Ckmeans
   class Clusterer # rubocop:disable Style/Documentation, Metrics/ClassLength
-    attr_reader :xcount, :xsorted, :kmin, :kmax, :smat, :jmat, :kestimate
-
     PI_DOUBLE = Math::PI * 2
 
     def initialize(entries, kmin, kmax = kmin, kestimate = :regular)
@@ -12,85 +10,94 @@ module Ckmeans
       raise ArgumentError, "Minimum cluster count is bigger than element count" if kmin > @xcount
       raise ArgumentError, "Maximum cluster count is bigger than element count" if kmax > @xcount
 
-      @kmin = kmin
-      @unique_xcount = entries.uniq.size
-      @kmax = [@unique_xcount, kmax].min
-      @xsorted = entries.sort
-      @kestimate = kestimate
+      @kmin                = kmin
+      @unique_xcount       = entries.uniq.size
+      @kmax                = [@unique_xcount, kmax].min
+      @xsorted_original    = entries.sort
+      @xsorted             = @xsorted_original.map(&:to_f)
+      @apply_bic_deviation = kestimate == :sensitive
     end
 
     def clusters
       @clusters ||=
         if @unique_xcount <= 1
-          [xsorted]
+          [@xsorted_original]
         else
-          @smat = Array.new(kmax) { Array.new(xcount) { 0.0 } }
-          @jmat = Array.new(kmax) { Array.new(xcount) { 0 } }
-
-          kappa = kmax
-          n = xcount
-          xsum = Array.new(n)
-          xsumsq = Array.new(n)
-          shift = xsorted[n / 2]
-          xsum[0] = xsorted[0] - shift
+          sorted_group_sizes.each_with_object([]) do |size, groups|
+            groups << @xsorted_original.shift(size)
+          end
+
+=begin # rubocop:disable Style/BlockComments
+          @cost   = Array.new(kmax) { Array.new(xcount) { 0.0 } }
+          @splits = Array.new(kmax) { Array.new(xcount) { 0 } }
+          @xsum   = Array.new(xcount)
+          @xsumsq = Array.new(xcount)
+
+          shift     = xsorted[xcount / 2]
+          xsum[0]   = xsorted[0].to_f - shift
           xsumsq[0] = xsum[0]**2
-          1.upto(n - 1) do |i|
-            xsum[i] = xsum[i - 1] + xsorted[i] - shift
-            xsumsq[i] = xsumsq[i - 1] + ((xsorted[i] - shift) * (xsorted[i] - shift))
-            smat[0][i] = dissim(0, i, xsum, xsumsq)
-            jmat[0][i] = 0
+
+          1.upto(xcount - 1) do |i|
+            xf = xsorted[i].to_f
+            xsum[i]      = xsum[i - 1] + xf - shift
+            xsumsq[i]    = xsumsq[i - 1] + ((xf - shift) * (xf - shift))
+            cost[0][i]   = dissim(0, i)
+            splits[0][i] = 0
           end
 
-          kappa_dec = kappa - 1
-          1.upto(kappa_dec) do |q|
-            imin = q < kappa_dec ? [1, q].max : n - 1
-            fill_row(q, imin, n - 1, xsum, xsumsq)
+          kmax_idx = kmax - 1
+          1.upto(kmax_idx) do |q|
+            imin = q < kmax_idx ? [1, q].max : xcount - 1
+            fill_row(q, imin, xcount - 1)
           end
 
           kopt = koptimal
 
-          results = []
-          backtrack(kopt) do |q, left, right|
-            results[q] = xsorted[left..right]
+          puts "RB COST\n", cost.map(&:inspect)
+          puts "RB SPLITS\n", splits.map(&:inspect)
+          puts "RB K OPTIMAL: #{kopt}"
+
+          backtrack(kopt).each_with_object(Array.new(kopt)) do |(q, left, right), res|
+            res[q] = xsorted[left..right]
           end
-          results
+=end
         end
     end
 
     private
 
-    def koptimal
-      kopt = kmin
-      n = xcount
-      max_bic = 0.0
+    attr_reader :cost, :splits, :xsum, :xsumsq, :xcount, :xsorted, :kmin, :kmax
 
-      # Deviation from BIC formula to favor smaller clusters
-      adjustment = kestimate == :sensitive ? 0.0 : 1.0
+    def koptimal # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      kopt       = kmin
+      n          = xcount
+      max_bic    = 0.0
+      adjustment = kestimate == :sensitive ? 0.0 : 1.0 # Deviation from BIC formula to favor smaller clusters
 
       kmin.upto(kmax) do |k|
-        sizes = Array.new(k)
-        backtrack(k) { |q, left, right| sizes[q] = right - left + 1 }
-        index_left = 0
-        index_right = nil
+        sizes = backtrack(k).each_with_object(Array.new(k)) { |(q, left, right), sz| sz[q] = right - left + 1 }
+
+        index_left    = 0
+        index_right   = nil
         loglikelihood = 0.0
-        bin_left = nil
-        bin_right = nil
+        bin_left      = nil
+        bin_right     = nil
 
         k.times do |kb|
           num_points_in_bin = sizes[kb]
           index_right = index_left + num_points_in_bin - 1
 
           if xsorted[index_left] < xsorted[index_right]
-            bin_left = xsorted[index_left]
+            bin_left  = xsorted[index_left]
             bin_right = xsorted[index_right]
           elsif xsorted[index_left] == xsorted[index_right]
-            bin_left = index_left == 0 ? xsorted[0] : (xsorted[index_left - 1] + xsorted[index_left]) / 2.0
+            bin_left  = index_left == 0 ? xsorted[0] : (xsorted[index_left - 1] + xsorted[index_left]) / 2.0
             bin_right = index_right < n - 1 ? (xsorted[index_right] + xsorted[index_right + 1]) / 2.0 : xsorted[n - 1]
           else
             raise "ERROR: binLeft > binRight"
           end
 
-          bin_width = bin_right - bin_left
+          bin_width = bin_right.to_f - bin_left
 
           mean, variance = shifted_data_variance(index_left, index_right)
 
@@ -112,10 +119,10 @@ module Ckmeans
 
         if k == kmin
           max_bic = bic
-          kopt = kmin
+          kopt    = kmin
         elsif bic > max_bic
           max_bic = bic
-          kopt = k
+          kopt    = k
         end
       end
 
@@ -123,22 +130,22 @@ module Ckmeans
     end
 
     def shifted_data_variance(ileft, iright)
-      sum = 0.0
-      sumsq = 0.0
-      mean = 0.0
+      sum      = 0.0
+      sumsq    = 0.0
+      mean     = 0.0
       variance = 0.0
-      n = iright - ileft + 1
+      n        = iright - ileft + 1
 
       if iright >= ileft
-        median = xsorted[(ileft + iright) / 2]
+        median = xsorted[(ileft + iright) / 2].to_f
 
         ileft.upto(iright) do |i|
-          sumi = xsorted[i] - median
-          sum += sumi
+          sumi   = xsorted[i] - median
+          sum   += sumi
           sumsq += sumi**2
         end
 
-        mean = (sum / n) + median
+        mean     = (sum / n) + median
         variance = (sumsq - (sum * sum / n)) / (n - 1) if n > 1
       end
 
@@ -146,12 +153,13 @@ module Ckmeans
     end
 
     def backtrack(k)
-      n = jmat[0].size
-      right = n - 1
-      left = nil
+      return to_enum(__method__, k) unless block_given?
+
+      right = xcount - 1
+      left  = nil
 
       (k - 1).downto(0) do |q|
-        left = jmat[q][right]
+        left = splits[q][right]
 
         yield q, left, right
 
@@ -159,7 +167,7 @@ module Ckmeans
       end
     end
 
-    def dissim(j, i, xsum, xsumsq)
+    def dissim(j, i)
       return 0.0 if j >= i
 
       sji =
@@ -174,80 +182,82 @@ module Ckmeans
       [0, sji].max
     end
 
-    def fill_row(q, imin, imax, xsum, xsumsq)
+    def fill_row(q, imin, imax)
       size = imax - q + 1
 
       js = Array.new(size) { |i| q + i }
-      smawk(imin, imax, 1, q, js, xsum, xsumsq)
+      smawk(imin, imax, 1, q, js)
     end
 
-    def smawk(imin, imax, istep, q, js, xsum, xsumsq)
+    def smawk(imin, imax, istep, q, js)
       if (imax - imin) <= (0 * istep)
-        find_min_from_candidates(q, imin, imax, istep, js, xsum, xsumsq)
+        find_min_from_candidates(q, imin, imax, istep, js)
       else
-        js_odd = js_reduced(imin, imax, istep, q, js, xsum, xsumsq)
+        js_odd = prune_candidates(imin, imax, istep, q, js)
+        # puts "Pruned: #{js_odd.inspect}"
         istepx2 = istep * 2
         imin_odd = imin + istep
         imax_odd = imin_odd + ((imax - imin_odd) / istepx2 * istepx2)
-        smawk(imin_odd, imax_odd, istepx2, q, js_odd, xsum, xsumsq)
-        fill_even_positions(imin, imax, istep, q, js, smat, jmat, xsum, xsumsq)
+        smawk(imin_odd, imax_odd, istepx2, q, js_odd)
+        fill_even_positions(imin, imax, istep, q, js)
       end
     end
 
-    def find_min_from_candidates(q, imin, imax, istep, js, xsum, xsumsq)
-      rmin_prev = 0
+    def find_min_from_candidates(q, imin, imax, istep, js)
+      optimal_split_index_prev = 0
 
       (imin..imax).step(istep) do |i|
-        rmin = rmin_prev
-        smat[q][i] = smat[q - 1][js[rmin] - 1] + dissim(js[rmin], i, xsum, xsumsq)
-        jmat[q][i] = js[rmin]
+        optimal_split_index = optimal_split_index_prev
+        optimal_split       = js[optimal_split_index]
+        cost[q][i]          = cost[q - 1][optimal_split - 1] + dissim(optimal_split, i)
+        splits[q][i]        = optimal_split
 
-        ((rmin + 1)...js.size).each do |r|
-          jabs = js[r]
+        ((optimal_split_index + 1)...js.size).each do |split_index|
+          jabs = js[split_index]
 
-          next if jabs < jmat[q - 1][i]
+          next if jabs < splits[q - 1][i]
           break if jabs > i
 
-          sj = smat[q - 1][jabs - 1] + dissim(jabs, i, xsum, xsumsq)
+          sj = cost[q - 1][jabs - 1] + dissim(jabs, i)
 
-          next unless sj <= smat[q][i]
+          next unless sj <= cost[q][i]
 
-          smat[q][i] = sj
-          jmat[q][i] = js[r]
-          rmin_prev = r
+          cost[q][i]               = sj
+          splits[q][i]             = js[split_index]
+          optimal_split_index_prev = split_index
         end
       end
     end
 
-    def js_reduced(imin, imax, istep, q, js, xsum, xsumsq)
+    def prune_candidates(imin, imax, istep, q, js)
       n = ((imax - imin) / istep) + 1
       m = js.size
 
       return js if n >= m
 
-      js_red = js.dup
+      pruned = js.dup
       left = -1
       right = 0
 
       while m > n
-        p = left + 1
-        i = imin + (p * istep)
-        j = js_red[right]
-        sl = smat[q - 1][j - 1] + dissim(j, i, xsum, xsumsq)
-        jplus1 = js_red[right + 1]
-        splus1 = smat[q - 1][jplus1 - 1] + dissim(jplus1, i, xsum, xsumsq)
-
-        if (sl < splus1) && (p < n - 1)
+        p     = left + 1
+        i     = imin + (p * istep)
+        j     = pruned[right]
+        jnext = pruned[right + 1]
+        sl    = cost[q - 1][j - 1] + dissim(j, i)
+        snext = cost[q - 1][jnext - 1] + dissim(jnext, i)
+
+        if (sl < snext) && (p < n - 1)
           left += 1
-          js_red[left] = j
+          pruned[left] = j
           right += 1
-        elsif (sl < splus1) && (p == n - 1)
+        elsif (sl < snext) && (p == n - 1)
           right += 1
-          js_red[right] = j
+          pruned[right] = j
           m -= 1
         else
           if p > 0
-            js_red[right] = js_red[left]
+            pruned[right] = pruned[left]
             left -= 1
           else
             right += 1
@@ -258,15 +268,15 @@ module Ckmeans
       end
 
       ((left + 1)...m).each do |r|
-        js_red[r] = js_red[right]
+        pruned[r] = pruned[right]
         right += 1
       end
 
-      js_red.slice!(m..-1) if js_red.size > m
-      js_red
+      pruned.slice!(m..-1) if pruned.size > m
+      pruned
     end
 
-    def fill_even_positions(imin, imax, istep, q, js, smat, jmat, xsum, xsumsq)
+    def fill_even_positions(imin, imax, istep, q, js)
       n = js.size
       istepx2 = istep * 2
       jl = js[0]
@@ -276,11 +286,11 @@ module Ckmeans
       while i <= imax
         r += 1 while js[r] < jl
 
-        smat[q][i] = smat[q - 1][js[r] - 1] + dissim(js[r], i, xsum, xsumsq)
-        jmat[q][i] = js[r]
-        jh         = ((i + istep) <= imax ? jmat[q][i + istep] : js[n - 1]).to_i
-        jmax       = [jh, i].min.to_i
-        sjimin     = dissim(jmax, i, xsum, xsumsq)
+        cost[q][i]   = cost[q - 1][js[r] - 1] + dissim(js[r], i)
+        splits[q][i] = js[r]
+        jh           = (i + istep) <= imax ? splits[q][i + istep] : js[n - 1]
+        jmax         = [jh, i].min
+        sjimin       = dissim(jmax, i)
 
         r += 1
         while r < n && js[r] <= jmax
@@ -288,18 +298,18 @@ module Ckmeans
 
           break if jabs > i
 
-          if jabs < jmat[q - 1][i]
+          if jabs < splits[q - 1][i]
             r += 1
             next
           end
 
-          s  = dissim(jabs, i, xsum, xsumsq)
-          sj = smat[q - 1][jabs - 1] + s
+          cost_base = cost[q - 1][jabs - 1]
+          sj        = cost_base + dissim(jabs, i)
 
-          if sj <= smat[q][i]
-            smat[q][i] = sj
-            jmat[q][i] = js[r]
-          elsif smat[q - 1][jabs - 1] + sjimin > smat[q][i]
+          if sj <= cost[q][i]
+            cost[q][i]   = sj
+            splits[q][i] = jabs
+          elsif cost_base + sjimin > cost[q][i]
             break
           end
 
@@ -314,3 +324,5 @@ module Ckmeans
     end
   end
 end
+
+require "ckmeans/extensions"

data/lib/ckmeans/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ckmeans
-  VERSION = "0.1.2"
+  VERSION = "1.0.0.rc"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: ckmeans
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 1.0.0.rc
 platform: ruby
 authors:
 - Vlad Lebedev
 bindir: exe
 cert_chain: []
-date: 2025-03-31 00:00:00.000000000 Z
+date: 2025-04-22 00:00:00.000000000 Z
 dependencies: []
 description: Repeatable clustering of unidimensional data
 email:
@@ -24,6 +24,8 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- ext/ckmeans/extconf.rb
+- ext/ckmeans/extensions.c
 - lib/ckmeans.rb
 - lib/ckmeans/clusterer.rb
 - lib/ckmeans/version.rb