spatial_stats 0.2.2 → 1.0.4

data/ext/spatial_stats/csr_matrix.h

@@ -0,0 +1,34 @@
+#ifndef CSR_MATRIX
+#define CSR_MATRIX
+
+typedef struct csr_matrix
+{
+    char init;
+    int n;
+    int nnz;
+    double *values;
+    int *col_index;
+    int *row_index;
+} csr_matrix;
+
+void csr_matrix_free(void *mat);
+size_t csr_matrix_memsize(const void *ptr);
+
+// ruby VALUE for csr_matrix
+static const rb_data_type_t csr_matrix_type = {
+    "SpatialStats::Weights::CSRMatrix",
+    {NULL, csr_matrix_free, csr_matrix_memsize},
+    0,
+    0,
+    RUBY_TYPED_FREE_IMMEDIATELY};
+
+void mat_to_sparse(csr_matrix *csr, VALUE data, VALUE keys, VALUE num_rows);
+VALUE csr_matrix_alloc(VALUE self);
+VALUE csr_matrix_initialize(VALUE self, VALUE data, VALUE num_rows);
+VALUE csr_matrix_values(VALUE self);
+VALUE csr_matrix_col_index(VALUE self);
+VALUE csr_matrix_row_index(VALUE self);
+VALUE csr_matrix_mulvec(VALUE self, VALUE vec);
+VALUE csr_matrix_dot_row(VALUE self, VALUE vec, VALUE row);
+VALUE csr_matrix_coordinates(VALUE self);
+#endif

data/ext/spatial_stats/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+
+create_header
+create_makefile 'spatial_stats/spatial_stats'

data/ext/spatial_stats/spatial_stats.c

@@ -0,0 +1,32 @@
+#include <ruby.h>
+#include "csr_matrix.h"
+
+/**
+ * Document-class: SpatialStats::Weights::CSRMatrix
+ * 
+ * CSRMatrix partially implements a compressed sparse row matrix to perform
+ * spatial lag and other calculations. This will generally be used 
+ * to store the weights of an observation set.
+ * 
+ * @see https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format) 
+ * 
+ */
+void Init_spatial_stats()
+{
+    VALUE spatial_stats_mod = rb_define_module("SpatialStats");
+    VALUE weights_mod = rb_define_module_under(spatial_stats_mod, "Weights");
+    VALUE csr_matrix_class = rb_define_class_under(weights_mod, "CSRMatrix", rb_cData);
+
+    rb_define_alloc_func(csr_matrix_class, csr_matrix_alloc);
+    rb_define_method(csr_matrix_class, "initialize", csr_matrix_initialize, 2);
+    rb_define_method(csr_matrix_class, "values", csr_matrix_values, 0);
+    rb_define_method(csr_matrix_class, "col_index", csr_matrix_col_index, 0);
+    rb_define_method(csr_matrix_class, "row_index", csr_matrix_row_index, 0);
+    rb_define_method(csr_matrix_class, "mulvec", csr_matrix_mulvec, 1);
+    rb_define_method(csr_matrix_class, "dot_row", csr_matrix_dot_row, 2);
+    rb_define_method(csr_matrix_class, "coordinates", csr_matrix_coordinates, 0);
+
+    rb_define_attr(csr_matrix_class, "m", 1, 0);
+    rb_define_attr(csr_matrix_class, "n", 1, 0);
+    rb_define_attr(csr_matrix_class, "nnz", 1, 0);
+}

data/lib/spatial_stats.rb

@@ -6,6 +6,7 @@ require 'spatial_stats/global'
 require 'spatial_stats/local'
 require 'spatial_stats/narray_ext'
 require 'spatial_stats/queries'
+require 'spatial_stats/spatial_stats' # c extensions
 require 'spatial_stats/utils'
 require 'spatial_stats/weights'
 

data/lib/spatial_stats/global/bivariate_moran.rb

@@ -19,9 +19,25 @@ module SpatialStats
         @scope = scope
         @x_field = x_field
         @y_field = y_field
-        @weights = weights
+        @weights = weights.standardize
+      end
+
+      ##
+      # A new instance of BivariateMoran, from vector and weights.
+      #
+      # @param [Array] x observations of dataset
+      # @param [WeightsMatrix] weights to define relationships between observations
+      #
+      # @return [BivariateMoran]
+      def self.from_observations(x, y, weights)
+        n = weights.n
+        raise ArgumentError, 'Data size != weights.n' if x.size != n || y.size != n
+
+        instance = new(nil, nil, nil, weights.standardize)
+        instance.x = x
+        instance.y = y
+        instance
       end
-      attr_writer :x, :y
 
       ##
       # Computes the global spatial correlation of x against spatially lagged
@@ -29,8 +45,7 @@ module SpatialStats
       #
       # @return [Float]
       def stat
-        w = @weights.standardized
-        y_lag = SpatialStats::Utils::Lag.neighbor_sum(w, y)
+        y_lag = SpatialStats::Utils::Lag.neighbor_sum(weights, y)
         numerator = 0
         x.each_with_index do |xi, idx|
           numerator += xi * y_lag[idx]
@@ -55,19 +70,20 @@ module SpatialStats
       #
       # @return [Float]
       def variance
-        n = @weights.n
-        wij = @weights.full
-        w = wij.sum
+        n = weights.n
+        w_sum = n.to_f
         e = expectation
 
-        s1 = s1_calc(n, wij)
-        s2 = s2_calc(n, wij)
+        wij = weights.sparse.coordinates
+
+        s1 = s1_calc(wij)
+        s2 = s2_calc(n, wij, weights.sparse.row_index)
         s3 = s3_calc(n, x)
 
-        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w**2)
-        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w**2)
+        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w_sum**2)
+        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w_sum**2)
 
-        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w**2)
+        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w_sum**2)
         var_right = e**2
         var_left - var_right
       end
@@ -88,6 +104,19 @@ module SpatialStats
         mc_bv(permutations, seed)
       end
 
+      ##
+      # Summary of the statistic. Computes +stat+ and +mc+ and returns the values
+      # in a hash.
+      #
+      # @param [Integer] permutations to run. Last digit should be 9 to produce round numbers.
+      # @param [Integer] seed used in random number generator for shuffles.
+      #
+      # @return [Hash]
+      def summary(permutations = 99, seed = nil)
+        p_val = mc(permutations, seed)
+        { stat: stat, p: p_val }
+      end
+
       ##
       # Standardized variables queried from +x_field+.
       #
@@ -108,32 +137,41 @@ module SpatialStats
 
       private
 
+      def stat_mc(perms)
+        x_arr = Numo::DFloat.cast(x)
+        lag = w.dot(perms.transpose)
+        x_arr.dot(lag) / (x_arr**2).sum
+      end
+
       def s3_calc(n, zs)
         numerator = (1.0 / n) * zs.sum { |v| v**4 }
         denominator = ((1.0 / n) * zs.sum { |v| v**2 })**2
         numerator / denominator
       end
 
-      def s2_calc(n, wij)
+      def s2_calc(n, wij, row_index)
         s2 = 0
-        (0..n - 1).each do |i|
+        wij_arr = wij.to_a # for row slicing
+        (0..n - 1).each do |idx|
+          row = wij_arr[row_index[idx]..(row_index[idx + 1] - 1)]
           left_term = 0
           right_term = 0
-          (0..n - 1).each do |j|
-            left_term += wij[i, j]
-            right_term += wij[j, i]
+
+          row.each do |coords, val|
+            left_term += val
+            right_term += wij[coords.reverse] || 0
           end
           s2 += (left_term + right_term)**2
         end
         s2
       end
 
-      def s1_calc(n, wij)
+      def s1_calc(wij)
         s1 = 0
-        (0..n - 1).each do |i|
-          (0..n - 1).each do |j|
-            s1 += (wij[i, j] + wij[j, i])**2
-          end
+        wij.each do |coords, val|
+          # (wij + wji)**2
+          wji = wij[coords.reverse] || 0
+          s1 += (val + wji)**2
         end
         s1 / 2
       end

data/lib/spatial_stats/global/moran.rb

@@ -18,7 +18,6 @@ module SpatialStats
       def initialize(scope, field, weights)
         super(scope, field, weights)
       end
-      attr_writer :x
 
       ##
       # Computes the global spatial autocorrelation of x against a spatially
@@ -29,8 +28,7 @@ module SpatialStats
         # compute's Moran's I. numerator is sum of zi * spatial lag of zi
         # denominator is sum of zi**2.
         # have to use row-standardized weights
-        w = @weights.standardized
-        z_lag = SpatialStats::Utils::Lag.neighbor_sum(w, z)
+        z_lag = SpatialStats::Utils::Lag.neighbor_sum(weights, z)
         numerator = 0
         z.each_with_index do |zi, j|
           row_sum = zi * z_lag[j]
@@ -49,7 +47,7 @@ module SpatialStats
       # @return [Float]
       def expectation
         # -1/(n-1)
-        -1.0 / (@weights.n - 1)
+        -1.0 / (weights.n - 1)
       end
 
       ##
@@ -58,19 +56,20 @@ module SpatialStats
       #
       # @return [Float]
       def variance
-        n = @weights.n
-        wij = @weights.full
-        w = wij.sum
+        n = weights.n
+        w_sum = n # standardized weights
         e = expectation
 
-        s1 = s1_calc(n, wij)
-        s2 = s2_calc(n, wij)
+        wij = weights.sparse.coordinates
+
+        s1 = s1_calc(wij)
+        s2 = s2_calc(n, wij, weights.sparse.row_index)
         s3 = s3_calc(n, z)
 
-        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w**2)
-        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w**2)
+        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w_sum**2)
+        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w_sum**2)
 
-        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w**2)
+        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w_sum**2)
         var_right = e**2
         var_left - var_right
       end
@@ -92,58 +91,66 @@ module SpatialStats
       end
 
       ##
-      # Values of the +field+ queried from the +scope+
+      # Summary of the statistic. Computes +stat+ and +mc+ and returns the values
+      # in a hash.
       #
-      # @return [Array]
-      def x
-        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
-      end
-
-      # TODO: remove these last 2 methods and just standardize x.
-      ##
-      # Mean of x
+      # @param [Integer] permutations to run. Last digit should be 9 to produce round numbers.
+      # @param [Integer] seed used in random number generator for shuffles.
       #
-      # @return [Float]
-      def zbar
-        x.sum / x.size
+      # @return [Hash]
+      def summary(permutations = 99, seed = nil)
+        p_val = mc(permutations, seed)
+        { stat: stat, p: p_val }
       end
 
       ##
-      # Array of xi - zbar for i: [0:n-1]
+      # Values of the +field+ queried from the +scope+
       #
       # @return [Array]
-      def z
-        x.map { |val| val - zbar }
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
+                                               .standardize
       end
+      alias z x
 
       private
 
+      def stat_mc(perms)
+        z_arr = Numo::DFloat.cast(z)
+        lag = w.dot(perms.transpose)
+        z_arr.dot(lag) / (z_arr**2).sum
+      end
+
       def s3_calc(n, zs)
         numerator = (1.0 / n) * zs.sum { |v| v**4 }
         denominator = ((1.0 / n) * zs.sum { |v| v**2 })**2
         numerator / denominator
       end
 
-      def s2_calc(n, wij)
+      # use row_index to take slices of wij
+      def s2_calc(n, wij, row_index)
         s2 = 0
+        wij_arr = wij.to_a # for row slicing
         (0..n - 1).each do |idx|
+          row = wij_arr[row_index[idx]..(row_index[idx + 1] - 1)]
           left_term = 0
           right_term = 0
-          (0..n - 1).each do |j|
-            left_term += wij[idx, j]
-            right_term += wij[j, idx]
+
+          row.each do |coords, val|
+            left_term += val
+            right_term += wij[coords.reverse] || 0
           end
           s2 += (left_term + right_term)**2
         end
         s2
       end
 
-      def s1_calc(n, wij)
+      def s1_calc(wij)
         s1 = 0
-        (0..n - 1).each do |idx|
-          (0..n - 1).each do |j|
-            s1 += (wij[idx, j] + wij[j, idx])**2
-          end
+        wij.each do |coords, val|
+          # (wij + wji)**2
+          wji = wij[coords.reverse] || 0
+          s1 += (val + wji)**2
         end
         s1 / 2
       end

data/lib/spatial_stats/global/stat.rb

@@ -11,10 +11,25 @@ module SpatialStats
       def initialize(scope, field, weights)
         @scope = scope
         @field = field
-        @weights = weights
+        @weights = weights.standardize
       end
       attr_accessor :scope, :field, :weights
 
+      ##
+      # A new instance of Stat, from vector and weights.
+      #
+      # @param [Array] x observations of dataset
+      # @param [WeightsMatrix] weights to define relationships between observations
+      #
+      # @return [Stat]
+      def self.from_observations(x, weights)
+        raise ArgumentError, 'Data size != weights.n' if x.size != weights.n
+
+        instance = new(nil, nil, weights.standardize)
+        instance.x = x
+        instance
+      end
+
       def stat
         raise NotImplementedError, 'method stat not defined'
       end
@@ -39,27 +54,36 @@ module SpatialStats
         (stat - expectation) / Math.sqrt(variance)
       end
 
+      def x=(values)
+        @x = values.standardize
+      end
+      alias z= x=
+
+      def y=(values)
+        @y = values.standardize
+      end
+
       def mc(permutations, seed)
         rng = gen_rng(seed)
         shuffles = []
         permutations.times do
           shuffles << x.shuffle(random: rng)
         end
+        shuffles = Numo::DFloat.cast(shuffles)
+
         # r is the number of equal to or more extreme samples
         # one sided
-        stat_orig = stat
-        r = 0
-        shuffles.each do |shuffle|
-          klass = self.class.new(@scope, @field, @weights)
-          klass.x = shuffle
-
-          # https://geodacenter.github.io/glossary.html#ppvalue
-          if stat_orig.positive?
-            r += 1 if klass.stat >= stat_orig
-          else
-            r += 1 if klass.stat <= stat_orig
-          end
-        end
+        stat_orig = stat.round(5)
+        # r = 0
+
+        # compute new stat values
+        stat_new = stat_mc(shuffles)
+
+        r = if stat_orig.positive?
+              (stat_new >= stat_orig).count
+            else
+              (stat_new <= stat_orig).count
+            end
 
         (r + 1.0) / (permutations + 1.0)
       end
@@ -71,27 +95,31 @@ module SpatialStats
         permutations.times do
           shuffles << y.shuffle(random: rng)
         end
+        shuffles = Numo::DFloat.cast(shuffles)
 
         # r is the number of equal to or more extreme samples
-        stat_orig = stat
-        r = 0
-        shuffles.each do |shuffle|
-          klass = self.class.new(@scope, @x_field, @y_field, @weights)
-          klass.x = x
-          klass.y = shuffle
-
-          if stat_orig.positive?
-            r += 1 if klass.stat >= stat_orig
-          else
-            r += 1 if klass.stat <= stat_orig
-          end
-        end
+        stat_orig = stat.round(5)
+        stat_new = stat_mc(shuffles)
+
+        r = if stat_orig.positive?
+              (stat_new >= stat_orig).count
+            else
+              (stat_new <= stat_orig).count
+            end
 
         (r + 1.0) / (permutations + 1.0)
       end
 
       private
 
+      def stat_mc(_shuffles)
+        raise NotImplementedError, 'private method stat_mc not defined'
+      end
+
+      def w
+        @w ||= weights.dense
+      end
+
       def gen_rng(seed)
         if seed
           Random.new(seed)