fast_statistics 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 77ab0c69365318bdd1f68d562c0460ed573d12bf1301542e65fe7ff204f70118
+  data.tar.gz: ff64565f990ea2d6f77c52afbb63d7271f25737744be191146d5935d69727a30
+SHA512:
+  metadata.gz: 898331717252505cff2e972ee2fb380709b070209197bf56374fe3c398aa906945c71a543f435d493aa5f5c5f7e7c65100979772f56bd5d28eb68967788fa6d3
+  data.tar.gz: e725e1d41c7749b8bc14eca78e46f574d477e12ddce3c8cd09817af82b4581ca3dcf24f2f2e2ea06128c2c65fe906ba365c38c7e911d20210b474b698065a01b

data/.gitignore

@@ -0,0 +1,22 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+Gemfile.lock
+
+*.so
+*.o
+compile_flags.txt
+compile_commands.json
+ext/fast_statistics/Makefile
+ext/fast_statistics/mkmf.log
+.clang-format
+.cache
+
+squiggles.txt
+README.md.html

data/.rspec

@@ -0,0 +1,3 @@
+--require spec_helper
+--format documentation
+--fail-fast

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.5
+  - 2.6
+  - 2.7

data/Gemfile

@@ -0,0 +1,16 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fast_statistics.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+gem "rake-compiler", "~> 1.1"
+gem "rspec", "~> 3.0"
+
+# Dependencies for running benchmarks 
+gem "benchmark-ips"
+gem "terminal-table"
+gem "descriptive_statistics"
+gem "ruby_native_statistics"
+gem "numo-narray"
+gem "nmatrix"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Martin Nyaga
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,164 @@
+# FastStatistics
+![Build Status](https://travis-ci.com/Martin-Nyaga/fast_statistics.svg?branch=master)
+
+A high performance native ruby extension (written in C++) for computation of
+descriptive statistics.
+
+## Overview
+This gem provides fast computation of descriptive statistics (min, max, mean,
+median, 1st and 3rd quartiles, population standard deviation) for a multivariate
+dataset (represented as a 2D array) in ruby.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fast_statistics'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install fast_statistics
+
+## Usage
+
+Given you have some multivariate (2-dimensional) data:
+```ruby
+data = [
+  [0.6269, 0.3783, 0.1477, 0.2374],
+  [0.4209, 0.1055, 0.8000, 0.2023],
+  [0.1124, 0.1021, 0.1936, 0.8566],
+  [0.6454, 0.5362, 0.4567, 0.8309],
+  [0.4828, 0.1572, 0.5706, 0.4085],
+  [0.5594, 0.0979, 0.4078, 0.5885],
+  [0.8659, 0.5346, 0.5566, 0.6166],
+  [0.7256, 0.5841, 0.8546, 0.3918]
+]
+```
+
+You can compute descriptive statistics for all the inner arrays as follows:
+
+```ruby
+require "fast_statistics"
+
+FastStatistics::Array2D.new(data).descriptive_statistics
+# Result: 
+#
+# [{:min=>0.1477,
+#   :max=>0.6269,
+#   :mean=>0.347575,
+#   :median=>0.30785,
+#   :q1=>0.214975,
+#   :q3=>0.44045,
+#   :standard_deviation=>0.18100761551658537},
+#  {:min=>0.1055,
+#   :max=>0.8,
+#   :mean=>0.38217500000000004,
+#   :median=>0.3116,
+#   :q1=>0.1781,
+#   :q3=>0.515675,
+#   :standard_deviation=>0.26691825878909076},
+#  ...,
+#  {:min=>0.3918,
+#   :max=>0.8546,
+#   :mean=>0.639025,
+#   :median=>0.6548499999999999,
+#   :q1=>0.536025,
+#   :q3=>0.75785,
+#   :standard_deviation=>0.1718318709523935}]
+```
+
+## Benchmarks
+
+Some alternatives compared are:
+- [descriptive_statistics](https://github.com/thirtysixthspan/descriptive_statistics)
+- [ruby-native-statistics](https://github.com/corybuecker/ruby-native-statistics)
+- [Numo::NArray](https://github.com/ruby-numo/numo-narray)
+- Hand-written ruby (using the same algorithm implemented in C++ in this gem)
+
+Benchmarked on my machine (8th gen i7, sse2), this gem is **~11x**
+faster than an optimal algorithm in hand-written ruby, and **~4.7x** faster than
+the next fastest available native ruby extension (that I tested).
+
+You can reivew the benchmark implementations at `benchmark/benchmark.rb` and run the
+benchmark with `rake benchmark`. 
+
+Results:
+```
+Comparing calculated statistics with 10 values for 8 variables...
+Test passed, results are equal to 6 decimal places!
+
+Benchmarking with 100,000 values for 12 variables...
+Warming up --------------------------------------
+descriptive_statistics   1.000  i/100ms
+           Custom ruby   1.000  i/100ms
+                narray   1.000  i/100ms
+ruby_native_statistics   1.000  i/100ms
+        FastStatistics   3.000  i/100ms
+Calculating -------------------------------------
+descriptive_statistics   0.473  (± 0.0%) i/s -      3.000  in   6.354555s
+           Custom ruby   2.518  (± 0.0%) i/s -     13.000  in   5.169084s
+                narray   4.231  (± 0.0%) i/s -     22.000  in   5.210299s
+ruby_native_statistics   5.962  (± 0.0%) i/s -     30.000  in   5.041869s
+        FastStatistics   28.417  (±10.6%) i/s -    141.000  in   5.012229s
+
+Comparison:
+        FastStatistics:   28.4 i/s
+ruby_native_statistics:    6.0 i/s - 4.77x  (± 0.00) slower
+                narray:    4.2 i/s - 6.72x  (± 0.00) slower
+           Custom ruby:    2.5 i/s - 11.29x  (± 0.00) slower
+descriptive_statistics:    0.5 i/s - 60.09x  (± 0.00) slower
+```
+
+## Background & Implementation 
+
+The inspiration for this gem was a use-case in an analytics ruby application,
+where we frequently had to compute descriptive statistics for fairly large
+multivariate datasets. Calculations in ruby were not fast enough, so I
+first explored performing the computations natively in [this
+repository](https://github.com/Martin-Nyaga/ruby-ffi-simd). The results were
+promising, so I decided to package it as a ruby gem.
+
+The following factors combined help this gem achieve high performance compared
+to available native alternatives and hand-written computations in ruby:
+
+- It is written in C++ and so can leverage the speed of native execution. 
+- It minimises the number of operations by calculating the statistics in as few
+  operations as possible (1 sort + 2 loops). Most native alternatives don't
+  provide a built in way to get all these statistics at once. Instead, they only
+  provide APIs where you make single calls for individual statistics. Through
+  such an API, building this set of summary statistics typically ends up looping
+  through the data more times than is necessary.
+- This gem uses explicit 128-bit-wide SIMD intrinsics (on platforms where they
+  are available) to parallelize computations for 2 variables at the same time
+  where possible, giving an additional speed advantage while still being single
+  threaded.
+
+That said, there are some limitations in the current implementation:
+- The variables in the 2D array must all have the same number of data points
+  (inner arrays must have the same length) and contain only numbers (i.e. no
+  `nil` awareness is present).
+- There is currently no API to calculate single statistics (although this may be
+  made available in the future).
+
+This is an early release and should be considered unstable, at least until I'm
+confident in the stability & performance in a real world application setting
+(let me know in [the Welcome discussion
+thread](https://github.com/Martin-Nyaga/fast_statistics/discussions/1) if you
+use it!). I'm also not really an expert in C++, so reviews & suggestions are
+welcome.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/Martin-Nyaga/fast_statistics.
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,36 @@
+require "bundler/gem_tasks"
+require "rake/extensiontask"
+
+task :default => :spec
+
+# Compile
+Rake::ExtensionTask.new "fast_statistics" do |ext|
+  ext.lib_dir = "lib/fast_statistics"
+end
+
+# Rspec
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec, [:spec] => [:clean, :compile])
+rescue LoadError
+end
+
+# Benchmark
+task :benchmark => [:clean, :compile] do 
+  require_relative "./benchmark/benchmark"
+  bench = DescriptiveStatsBenchmark
+  bench.compare_results!
+  bench.benchmark_ips!
+end
+
+task :profile => [:clean, :compile] do
+  require "fast_statistics"
+  $stdout.sync = true
+
+  variables = 12
+  length = 100_000
+  data = (0..(variables - 1)).map { (0..(length - 1)).map { rand } }
+  FastStatistics::Array2D.new(data, dtype: :float).mean.to_a
+  puts
+  puts
+end

data/benchmark/base.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "benchmark"
+require "benchmark/ips"
+require "terminal-table"
+
+class BaseBenchmark
+  class << self
+    @@tests = []
+
+    def benchmark(name, &block)
+      @@tests.push(TestCase.new(name, block))
+    end
+
+    def tests
+      @@tests
+    end
+
+    def compare_results!(data_points: 10, precision: 6)
+      data = generate_data(data_points)
+      puts("Comparing calculated statistics with #{format_number(data_points)} values for #{data.length} variables...")
+
+      test_results = tests.map { |test| test.run(data) }
+
+      # Uncomment to print results
+      # test_results.zip(tests) do |results, test|
+      #   print_results(test.name, results, precision)
+      # end
+      if assert_values_within_delta(test_results, 10 ** -precision)
+        puts("Test passed, results are equal to #{precision} decimal places!")
+        puts
+      end
+
+    rescue TestFailure => e
+      puts("Test results did not match!")
+      exit(1)
+    end
+
+    def benchmark!(data_points: 100_000, variables_count: 12)
+      data = generate_data(data_points, variables_count)
+      puts("Benchmarking with #{format_number(data_points)} values for #{data.length} variables...")
+
+      ::Benchmark.bmbm do |x|
+        tests.each do |test|
+          x.report(test.name) do
+            test.run(data)
+          end
+        end
+      end
+    end
+
+    def benchmark_ips!(data_points: 100_000, variables_count: 12)
+      data = generate_data(data_points, variables_count)
+      puts("Benchmarking with #{format_number(data_points)} values for #{data.length} variables...")
+
+      ::Benchmark.ips do |x|
+        tests.each do |test|
+          x.report(test.name) do
+            test.run(data)
+          end
+
+          x.compare!
+        end
+      end
+    end
+
+    private
+
+    def generate_data(length, variables = 8)
+      data = (0..(variables - 1)).map { (0..(length - 1)).map { rand } }
+    end
+
+    def print_results(title, results, precision)
+      headers = results[0].keys
+      values = results.map { |r| r.values.map { |v| "%.#{precision}f" % v } }
+      table = Terminal::Table.new(headings: headers, rows: values)
+      puts(title + ":")
+      puts(table)
+    end
+
+    def assert_values_within_delta(values, delta)
+      values.combination(2).each do |expected, actual|
+        unless expected.length == actual.length
+          raise TestFailure, "Results don't match!"
+        end
+
+        expected.each_with_index do |expected_result, i|
+          actual_result = actual[i]
+
+          if actual_result.is_a?(Hash) && expected_result.is_a?(Hash)
+            expected_result.each do |k, _v|
+              assert_in_delta(actual_result[k], expected_result[k], delta)
+            end
+          else
+
+            assert_in_delta(actual_result, expected_result, delta)
+          end
+        end
+      end
+
+      true
+    end
+
+    def assert_in_delta(expected, actual, delta)
+      unless (expected - actual).abs < delta
+        raise TestFailure, "Results don't match!"
+      end
+
+      true
+    end
+
+    def format_number(number)
+      number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse
+    end
+
+    TestCase = Struct.new(:name, :block) do
+      def run(data)
+        block.call(data)
+      end
+    end
+
+    class TestFailure < StandardError
+    end
+  end
+end

data/benchmark/benchmark.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+
+require_relative "./base"
+require_relative "./helpers"
+
+require "fast_statistics"
+require "descriptive_statistics/safe"
+require "ruby_native_statistics"
+require "numo/narray"
+
+class DescriptiveStatsBenchmark < BaseBenchmark
+  class << self
+    include Helpers
+  end
+
+  benchmark "descriptive_statistics" do |data|
+    data.map do |arr|
+      stats = DescriptiveStatistics::Stats.new(arr)
+      {
+        mean: stats.mean,
+        min: stats.min,
+        max: stats.max,
+        median: stats.median,
+        q1: stats.percentile(25),
+        q3: stats.percentile(75),
+        standard_deviation: stats.standard_deviation
+      }
+    end
+  end
+
+  benchmark "Custom ruby" do |data|
+    data.map do |arr|
+      arr.sort!
+
+      min = arr.first
+      max = arr.last
+      length = arr.length
+      median = percentile(50, arr, length)
+      q1 = percentile(25, arr, length)
+      q3 = percentile(75, arr, length)
+      sum = arr.inject(0) { |sum, x| sum + x}
+
+      mean = sum / length
+      variance = arr.inject(0) { |var, x| var += ((x - mean) ** 2) / length }
+      standard_deviation = Math.sqrt(variance)
+      {
+        mean: mean,
+        min: min,
+        max: max,
+        median: median,
+        q1: q1,
+        q3: q3,
+        standard_deviation: standard_deviation
+      }
+    end
+  end
+
+  benchmark "narray" do |data|
+    data.map do |arr|
+      narr = Numo::DFloat[arr]
+      narr.sort
+      min = narr[0]
+      length = arr.length
+      max = narr[length - 1]
+      median = percentile(50, narr, length)
+      q1 = percentile(25, narr, length)
+      q3 = percentile(75, narr, length)
+      mean = narr.mean
+      variance = 0
+      narr.each { |x| variance += ((x - mean) ** 2) / length }
+      standard_deviation = Math.sqrt(variance)
+      {
+        mean: mean,
+        min: min,
+        max: max,
+        median: median,
+        q1: q1,
+        q3: q3,
+        standard_deviation: standard_deviation
+      }
+    end
+  end
+
+  benchmark "ruby_native_statistics" do |data|
+    data.map do |arr|
+      {
+        mean: arr.mean,
+        min: arr.min,
+        max: arr.max,
+        median: arr.median,
+        q1: arr.percentile(0.25),
+        q3: arr.percentile(0.75),
+        standard_deviation: arr.stdevp
+      }
+    end
+  end
+
+  benchmark "FastStatistics" do |data|
+    FastStatistics::Array2D.new(data).descriptive_statistics
+  end
+end

data/benchmark/helpers.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Helpers
+  def percentile(p, arr, len)
+    return arr[len - 1] if p == 100
+    rank = p / 100.0 * (len - 1)
+    lower = arr[rank.floor]
+    upper = arr[rank.floor + 1]
+    lower + (upper - lower) * (rank - rank.floor)
+  end
+end
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fast_statistics"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/rake

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ext/fast_statistics/array_2d.cpp

@@ -0,0 +1,188 @@
+#include "array_2d.h"
+#include "debug.h"
+
+namespace array_2d
+{
+
+DFloat::~DFloat()
+{
+  free(entries);
+  delete[] stats;
+}
+
+DFloat::DFloat(VALUE arrays)
+{
+  cols = rb_array_len(arrays);
+  rows = rb_array_len(rb_ary_entry(arrays, 0));
+  entries = (double*)malloc(cols * rows * sizeof(double));
+  stats = NULL;
+
+  for (int j = 0; j < cols; j++) {
+    for (int i = 0; i < rows; i++) {
+      entries[j * rows + i] = (double)NUM2DBL(rb_ary_entry(rb_ary_entry(arrays, j), i));
+    }
+  }
+}
+
+inline void
+DFloat::sort(double* col)
+{
+  std::sort(col, col + rows);
+}
+
+inline double
+DFloat::percentile(double* col, double pct)
+{
+  if (pct == 1.0) return col[rows - 1];
+  double rank = pct * (double)(rows - 1);
+  int floored_rank = floor(rank);
+  double lower = col[floored_rank];
+  double upper = col[floored_rank + 1];
+  return lower + (upper - lower) * (rank - floored_rank);
+}
+
+inline double
+DFloat::sum(double* col)
+{
+  double sum = 0.0;
+  for (int row = 0; row < rows; row++) {
+    sum += col[row];
+  }
+  return sum;
+}
+
+inline double
+DFloat::standard_deviation(double* col, double mean)
+{
+  double variance = 0.0f;
+  for (int i = 0; i < rows; i++) {
+    double value = col[i];
+    double deviation = value - mean;
+    double sqr_deviation = deviation * deviation;
+    variance += (sqr_deviation / (double)rows);
+  }
+  double result = sqrt(variance);
+  return result;
+}
+
+Stats*
+DFloat::descriptive_statistics()
+{
+  stats = new Stats[cols];
+
+  for (int col = 0; col < cols; col++) {
+    Stats var_stats;
+    double* col_arr = base_ptr(col);
+
+    sort(col_arr);
+
+    var_stats.min = col_arr[0];
+    var_stats.max = col_arr[rows - 1];
+    var_stats.median = percentile(col_arr, 0.5);
+    var_stats.q1 = percentile(col_arr, 0.25);
+    var_stats.q3 = percentile(col_arr, 0.75);
+    double total = sum(col_arr);
+    var_stats.mean = total / (double)rows;
+    var_stats.standard_deviation = standard_deviation(col_arr, var_stats.mean);
+
+    stats[col] = var_stats;
+  }
+
+  return stats;
+}
+
+#ifdef HAVE_XMMINTRIN_H
+inline double
+DFloat::safe_entry(int col, int row)
+{
+  if (col < cols) {
+    return *(base_ptr(col) + row);
+  } else {
+    return 0;
+  }
+}
+
+inline void
+DFloat::sort_columns(int start_col, int pack_size)
+{
+  for (int i = 0; i < pack_size; i++) {
+    if ((start_col + i) < cols) {
+      double* col_arr = base_ptr(start_col + i);
+      sort(col_arr);
+    }
+  }
+}
+
+inline __m128d
+DFloat::pack(int start_col, int row)
+{
+  __m128d packed = _mm_set_pd(safe_entry(start_col + 1, row), safe_entry(start_col + 0, row));
+  return packed;
+}
+
+inline __m128d
+DFloat::percentile_packed(int start_col, float pct)
+{
+  if (pct == 1.0) return pack(start_col, rows - 1);
+  double rank = pct * (double)(rows - 1);
+  int floored_rank = floor(rank);
+  __m128d lower = pack(start_col, floored_rank);
+  __m128d upper = pack(start_col, floored_rank + 1);
+  __m128d upper_minus_lower = _mm_sub_pd(upper, lower);
+  __m128d rank_minus_floored_rank = _mm_sub_pd(_mm_set_pd1(rank), _mm_set_pd1((float)floored_rank));
+  return _mm_add_pd(lower, _mm_mul_pd(upper_minus_lower, rank_minus_floored_rank));
+}
+
+Stats*
+DFloat::descriptive_statistics_packed()
+{
+  stats = new Stats[cols];
+  const int simd_pack_size = 2;
+
+  __m128d lengths = _mm_set_pd1((double)rows);
+  for (int col = 0; col < cols; col += simd_pack_size) {
+    sort_columns(col, simd_pack_size);
+
+    __m128d mins = pack(col, 0);
+    __m128d maxes = pack(col, rows - 1);
+    __m128d sums = _mm_setzero_pd();
+    for (int row_index = 0; row_index < rows; row_index++) {
+      __m128d packed = pack(col, row_index);
+      sums = _mm_add_pd(sums, packed);
+    }
+    __m128d means = _mm_div_pd(sums, lengths);
+
+    __m128d medians = percentile_packed(col, 0.5f);
+    __m128d q1s = percentile_packed(col, 0.25f);
+    __m128d q3s = percentile_packed(col, 0.75f);
+
+    __m128d variances = _mm_setzero_pd();
+    for (int row_index = 0; row_index < rows; row_index++) {
+      __m128d packed = pack(col, row_index);
+      __m128d deviation = _mm_sub_pd(packed, means);
+      __m128d sqr_deviation = _mm_mul_pd(deviation, deviation);
+      variances = _mm_add_pd(variances, _mm_div_pd(sqr_deviation, lengths));
+    }
+    __m128d stdevs = _mm_sqrt_pd(variances);
+
+    for (int simd_slot_index = 0; simd_slot_index < simd_pack_size; simd_slot_index++) {
+      if ((col + simd_slot_index) < cols) {
+        Stats var_stats;
+        var_stats.min = MM_GET_INDEX(mins, simd_slot_index);
+        var_stats.max = MM_GET_INDEX(maxes, simd_slot_index);
+        var_stats.mean = MM_GET_INDEX(means, simd_slot_index);
+        var_stats.median = MM_GET_INDEX(medians, simd_slot_index);
+        var_stats.q1 = MM_GET_INDEX(q1s, simd_slot_index);
+        var_stats.q3 = MM_GET_INDEX(q3s, simd_slot_index);
+        var_stats.standard_deviation = MM_GET_INDEX(stdevs, simd_slot_index);
+
+        stats[col + simd_slot_index] = var_stats;
+      }
+    }
+  }
+
+  return stats;
+}
+
+#endif
+} // namespace array_2d

data/ext/fast_statistics/array_2d.h

@@ -0,0 +1,60 @@
+#ifndef ARRAY_2D_H
+#define ARRAY_2D_H
+#include <algorithm>
+#include "ruby.h"
+
+#ifdef HAVE_XMMINTRIN_H
+#include <xmmintrin.h>
+#define MM_GET_INDEX(packed, index) *(((double*)&packed) + index);
+#endif
+
+namespace array_2d
+{
+
+struct Stats {
+  double min;
+  double max;
+  double mean;
+  double median;
+  double q1;
+  double q3;
+  double standard_deviation;
+
+  Stats()
+  {
+    min = 0.0, max = 0.0, mean = 0.0, median = 0.0, q1 = 0.0, q3 = 0.0, standard_deviation = 0.0;
+  };
+};
+
+class DFloat
+{
+  inline double* base_ptr(int col) { return entries + (col * rows); }
+  inline void sort(double* col);
+  inline double percentile(double* col, double pct);
+  inline double sum(double* col);
+  inline double standard_deviation(double* col, double mean);
+
+#ifdef HAVE_XMMINTRIN_H
+  inline double safe_entry(int col, int row);
+  inline void sort_columns(int start_col, int pack_size);
+  inline __m128d percentile_packed(int start_col, float pct);
+  inline __m128d pack(int start_col, int row);
+#endif
+
+public:
+  int cols;
+  int rows;
+  double* entries;
+  Stats* stats;
+
+  DFloat(VALUE ruby_arr);
+  ~DFloat();
+
+  Stats* descriptive_statistics();
+
+#ifdef HAVE_XMMINTRIN_H
+  Stats* descriptive_statistics_packed();
+#endif
+};
+} // namespace array_2d
+#endif

data/ext/fast_statistics/debug.h

@@ -0,0 +1,36 @@
+#ifdef DEBUG
+#ifndef DEBUG_H
+#define DEBUG_H
+
+#include <x86intrin.h>
+#include <stdio.h>
+#include <string.h>
+
+#define CONCAT_(a, b) a##b
+#define CONCAT(a,b) CONCAT_(a, b)
+#define PROFILE DebugTimer CONCAT(Timer, __COUNTER__)(__func__, __LINE__);
+
+struct DebugTimer {
+  char* name;
+  unsigned long long counter;
+  
+  DebugTimer(const char* function_name, int line_number) {
+    name = (char*)malloc(200 * sizeof(char));
+
+    strcpy(name, function_name);
+    strcat(name, "_");
+    snprintf(name + strlen(name), 4, "%d", line_number);
+
+    counter = __rdtsc();
+  }
+
+  ~DebugTimer() {
+    printf("\n%30s:\t %-10llu", name, __rdtsc() - counter);
+    free(name);
+    fflush(stdout);
+  }
+};
+#endif
+#else
+#define PROFILE();
+#endif

data/ext/fast_statistics/extconf.rb

@@ -0,0 +1,21 @@
+require "mkmf"
+
+# Enable debug compile using DEBUG env var
+if ENV["DEBUG"]
+  puts "Compiling in debug mode..."
+  CONFIG["debugflags"] = "-g"
+  CONFIG["optflags"] = "-O0"
+  $defs << "-DDEBUG"
+end
+
+# Disable warnings
+[
+  / -Wdeclaration-after-statement/,
+  / -Wimplicit-int/,
+  / -Wimplicit-function-declaration/,
+].each do |flag|
+  CONFIG["warnflags"].slice!(flag)
+end
+
+have_header("xmmintrin.h")
+create_makefile("fast_statistics/fast_statistics")

data/ext/fast_statistics/fast_statistics.cpp

@@ -0,0 +1,190 @@
+#include "fast_statistics.h"
+#include "array_2d.h"
+
+using namespace array_2d;
+
+static VALUE mFastStatistics;
+static VALUE cArray2D;
+
+// Helper
+VALUE
+build_results_hashes(Stats* stats, int num_variables)
+{
+  VALUE a_results = rb_ary_new();
+
+  VALUE s_min = rb_sym("min");
+  VALUE s_max = rb_sym("max");
+  VALUE s_mean = rb_sym("mean");
+  VALUE s_median = rb_sym("median");
+  VALUE s_q1 = rb_sym("q1");
+  VALUE s_q3 = rb_sym("q3");
+  VALUE s_standard_deviation = rb_sym("standard_deviation");
+
+  for (int i = 0; i < num_variables; i++) {
+    VALUE h_result = rb_hash_new();
+    Stats var_stats = stats[i];
+
+    rb_hash_aset(h_result, s_min, DBL2NUM(var_stats.min));
+    rb_hash_aset(h_result, s_max, DBL2NUM(var_stats.max));
+    rb_hash_aset(h_result, s_mean, DBL2NUM(var_stats.mean));
+    rb_hash_aset(h_result, s_median, DBL2NUM(var_stats.median));
+    rb_hash_aset(h_result, s_q1, DBL2NUM(var_stats.q1));
+    rb_hash_aset(h_result, s_q3, DBL2NUM(var_stats.q3));
+    rb_hash_aset(h_result, s_standard_deviation, DBL2NUM(var_stats.standard_deviation));
+
+    rb_ary_push(a_results, h_result);
+  }
+
+  return a_results;
+}
+
+// Common
+void
+free_wrapped_array(void* array)
+{
+  ((DFloat*)array)->~DFloat();
+}
+
+size_t
+wrapped_array_size(const void* data)
+{
+  DFloat* array = (DFloat*)data;
+  size_t size = sizeof(array->entries) + sizeof(*array);
+  return size;
+}
+
+static rb_data_type_t dfloat_wrapper = [] {
+  rb_data_type_t wrapper{};
+  wrapper.wrap_struct_name = "dfloat";
+  wrapper.function = { dmark : NULL, dfree : free_wrapped_array, dsize : wrapped_array_size };
+  wrapper.data = NULL;
+  wrapper.flags = RUBY_TYPED_FREE_IMMEDIATELY;
+  return wrapper;
+}();
+
+VALUE
+cArray2D_alloc(VALUE self)
+{
+  void* dfloat = (void*)malloc(sizeof(void*));
+
+  return TypedData_Wrap_Struct(self, &dfloat_wrapper, dfloat);
+}
+
+inline bool
+cArray2D_check_array_args(VALUE arrays)
+{
+  if (TYPE(arrays) == T_ARRAY && TYPE(rb_ary_entry(arrays, 0)) == T_ARRAY) {
+    return true;
+  } else {
+    Check_Type(arrays, T_ARRAY);
+    Check_Type(rb_ary_entry(arrays, 0), T_ARRAY);
+    return false;
+  }
+}
+
+//{{{ Unpacked
+VALUE
+simd_disabled(VALUE self)
+{
+  return Qfalse;
+}
+
+/*
+ * def initialize(arrays)
+ */
+VALUE
+cArray2D_initialize_unpacked(VALUE self, VALUE arrays)
+{
+  // Typecheck 2d array
+  if (cArray2D_check_array_args(arrays)) {
+    // Initialize dfloat structure to store Dfloat in type wrapper
+    void* dfloat;
+    UNWRAP_DFLOAT(self, dfloat);
+    new (dfloat) DFloat(arrays);
+  }
+  return self;
+}
+
+/*
+ * Unpacked descriptive statistics
+ *
+ * def descriptive_statistics
+ */
+VALUE
+cArray2D_descriptive_statistics_unpacked(VALUE self)
+{
+  void* dfloat_untyped;
+  UNWRAP_DFLOAT(self, dfloat_untyped);
+
+  DFloat* dfloat = ((DFloat*)dfloat_untyped);
+  Stats* stats = dfloat->descriptive_statistics();
+  return build_results_hashes(stats, dfloat->cols);
+}
+//}}}
+
+// Packed
+#ifdef HAVE_XMMINTRIN_H
+extern "C" VALUE
+simd_enabled(VALUE self)
+{
+  return Qtrue;
+}
+
+/*
+ * def initialize(arrays)
+ */
+VALUE
+cArray2D_initialize_packed(VALUE self, VALUE arrays)
+{
+  // Typecheck 2d array
+  if (cArray2D_check_array_args(arrays)) {
+    // Initialize dfloat structure to store Dfloat in type wrapper
+    void* dfloat;
+    UNWRAP_DFLOAT(self, dfloat);
+    new (dfloat) DFloat(arrays);
+  }
+  return self;
+}
+
+/*
+ * Packed descriptive statistics
+ *
+ * def descriptive_statistics
+ */
+VALUE
+cArray2D_descriptive_statistics_packed(VALUE self)
+{
+  void* dfloat_untyped;
+  UNWRAP_DFLOAT(self, dfloat_untyped);
+
+  DFloat* dfloat = ((DFloat*)dfloat_untyped);
+  Stats* stats = dfloat->descriptive_statistics_packed();
+  return build_results_hashes(stats, dfloat->cols);
+}
+#endif
+
+extern "C" void
+Init_fast_statistics(void)
+{
+  mFastStatistics = rb_define_module("FastStatistics");
+  cArray2D = rb_define_class_under(mFastStatistics, "Array2D", rb_cData);
+  rb_define_alloc_func(cArray2D, cArray2D_alloc);
+
+#ifdef HAVE_XMMINTRIN_H
+  rb_define_singleton_method(mFastStatistics, "simd_enabled?", RUBY_METHOD_FUNC(simd_enabled), 0);
+  rb_define_method(cArray2D, "initialize", RUBY_METHOD_FUNC(cArray2D_initialize_packed), 1);
+  rb_define_method(
+    cArray2D,
+    "descriptive_statistics",
+    RUBY_METHOD_FUNC(cArray2D_descriptive_statistics_packed),
+    0);
+#else
+  rb_define_singleton_method(mFastStatistics, "simd_enabled?", RUBY_METHOD_FUNC(simd_disabled), 0);
+  rb_define_method(cArray2D, "initialize", RUBY_METHOD_FUNC(cArray2D_initialize_unpacked), 1);
+  rb_define_method(
+    cArray2D,
+    "descriptive_statistics",
+    RUBY_METHOD_FUNC(cArray2D_descriptive_statistics_unpacked),
+    0);
+#endif
+}

data/ext/fast_statistics/fast_statistics.h

@@ -0,0 +1,14 @@
+#ifndef FAST_STATISTICS_H
+#define FAST_STATISTICS_H
+
+#include <ruby.h>
+#ifdef HAVE_XMMINTRIN_H
+#include <xmmintrin.h>
+#endif
+
+#include "debug.h"
+
+#define rb_sym(str) ID2SYM(rb_intern(str))
+#define UNWRAP_DFLOAT(obj, var) TypedData_Get_Struct((obj), void*, &dfloat_wrapper, (var));
+
+#endif

data/fast_statistics.gemspec

@@ -0,0 +1,27 @@
+require_relative 'lib/fast_statistics/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fast_statistics"
+  spec.version       = FastStatistics::VERSION
+  spec.authors       = ["Martin Nyaga"]
+  spec.email         = ["martin@martinnyaga.com"]
+
+  spec.summary       = %q{Fast computation of descriptive statistics in ruby}
+  spec.description   = %q{Fast computation of descriptive statistics in ruby using native code and SIMD}
+  spec.homepage      = "https://github.com/martin-nyaga/ruby-ffi-simd"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/martin-nyaga/fast_statistics"
+  spec.metadata["changelog_uri"] = "https://github.com/martin-nyaga/fast_statistics"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ["lib"]
+
+  spec.extensions = %w[ext/fast_statistics/extconf.rb]
+end

data/lib/fast_statistics.rb

@@ -0,0 +1,5 @@
+require "fast_statistics/version"
+require "fast_statistics/fast_statistics"
+
+module FastStatistics
+end

data/lib/fast_statistics/version.rb

@@ -0,0 +1,3 @@
+module FastStatistics
+  VERSION = "0.1.1"
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: fast_statistics
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Martin Nyaga
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-03-23 00:00:00.000000000 Z
+dependencies: []
+description: Fast computation of descriptive statistics in ruby using native code
+  and SIMD
+email:
+- martin@martinnyaga.com
+executables: []
+extensions:
+- ext/fast_statistics/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark/base.rb
+- benchmark/benchmark.rb
+- benchmark/helpers.rb
+- bin/console
+- bin/rake
+- bin/rspec
+- bin/setup
+- ext/fast_statistics/array_2d.cpp
+- ext/fast_statistics/array_2d.h
+- ext/fast_statistics/debug.h
+- ext/fast_statistics/extconf.rb
+- ext/fast_statistics/fast_statistics.cpp
+- ext/fast_statistics/fast_statistics.h
+- fast_statistics.gemspec
+- lib/fast_statistics.rb
+- lib/fast_statistics/version.rb
+homepage: https://github.com/martin-nyaga/ruby-ffi-simd
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/martin-nyaga/ruby-ffi-simd
+  source_code_uri: https://github.com/martin-nyaga/fast_statistics
+  changelog_uri: https://github.com/martin-nyaga/fast_statistics
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Fast computation of descriptive statistics in ruby
+test_files: []