spatial_stats 0.2.2 → 1.0.0

data/lib/spatial_stats/local/moran.rb

@@ -20,6 +20,7 @@ module SpatialStats
       def initialize(scope, field, weights)
         super(scope, field, weights)
       end
+      attr_writer :x
 
       ##
       # Computes the local indicator of spatial autocorrelation (lisa) for
@@ -55,12 +56,11 @@ module SpatialStats
       # @return [Array] of variances for each observation
       def variance
         # formula is A - B - (E[I])**2
-        wt = w.row_standardized
         exp = expectation
 
         vars = []
-        a_terms = a_calc(wt)
-        b_terms = b_calc(wt)
+        a_terms = a_calc
+        b_terms = b_calc
 
         a_terms.each_with_index do |a_term, idx|
           vars << (a_term - b_terms[idx] - (exp**2))
@@ -68,6 +68,21 @@ module SpatialStats
         vars
       end
 
+      ##
+      # Computes the groups each observation belongs to.
+      # Potential groups for Moran's I are:
+      # [HH] High-High
+      # [HL] High-Low
+      # [LH] Low-High
+      # [LL] Low-Low
+      #
+      # This is the same as the +#quads+ method in the +Stat+ class.
+      #
+      # @return [Array] groups for each observation
+      def groups
+        quads
+      end
+
       ##
       # Values of the +field+ queried from the +scope+
       #
@@ -85,7 +100,7 @@ module SpatialStats
       def z_lag
         # w is already row_standardized, so we are using
         # neighbor sum instead of neighbor_average to save cost
-        @z_lag ||= SpatialStats::Utils::Lag.neighbor_sum(w, z)
+        @z_lag ||= SpatialStats::Utils::Lag.neighbor_sum(weights, z)
       end
 
       private
@@ -102,6 +117,15 @@ module SpatialStats
         z[idx] * z_lag_i
       end
 
+      def mc_observation_calc(stat_i_orig, stat_i_new, _permutations)
+        # Since moran can be positive or negative, go by this definition
+        if stat_i_orig.positive?
+          (stat_i_new >= stat_i_orig).count
+        else
+          (stat_i_new <= stat_i_orig).count
+        end
+      end
+
       def si2
         # @si2 ||= z.sample_variance
         # we standardize so sample_variance is 1
@@ -109,20 +133,27 @@ module SpatialStats
       end
 
       # https://pro.arcgis.com/en/pro-app/tool-reference/spatial-statistics/h-local-morans-i-additional-math.htm
-      def a_calc(wt)
-        n = wt.shape[0]
+      # TODO: sparse
+      def a_calc
+        n = weights.n
         b2i = b2i_calc
+
+        wts = weights.sparse.values
+        row_index = weights.sparse.row_index
+
         a_terms = []
 
         (0..n - 1).each do |idx|
-          sigma_term = wt[idx, true].to_a.sum { |v| v**2 }
+          row_range = row_index[idx]..(row_index[idx + 1] - 1)
+          wt = wts[row_range]
+          sigma_term = wt.sum { |v| v**2 }
           a_terms << (n - b2i) * sigma_term / (n - 1)
         end
         a_terms
       end
 
-      def b_calc(wt)
-        n = wt.shape[0]
+      def b_calc
+        n = weights.n
         b2i = b2i_calc
         b_terms = []
 

data/lib/spatial_stats/local/multivariate_geary.rb

@@ -23,7 +23,7 @@ module SpatialStats
       def initialize(scope, fields, weights)
         @scope = scope
         @fields = fields
-        @weights = weights
+        @weights = weights.standardize
       end
       attr_accessor :scope, :fields, :weights
 
@@ -67,22 +67,24 @@ module SpatialStats
         stat_orig = stat
         rs = [0] * n
 
-        ws = neighbor_weights
+        row_index = weights.sparse.row_index
+        ws = weights.sparse.values
 
         idx = 0
         while idx < n
           stat_i_orig = stat_orig[idx]
-          wi = Numo::DFloat.cast(ws[idx])
+
+          row_range = row_index[idx]..(row_index[idx + 1] - 1)
+          if row_range.size.zero?
+            rs[idx] = permutations
+            idx += 1
+            next
+          end
+          wi = Numo::DFloat.cast(ws[row_range])
 
           # for each field, compute the C value at that index.
           stat_i_new = mc_i(wi, shuffles[idx], idx)
-
-          rs[idx] = if stat_i_orig.positive?
-                      (stat_i_new >= stat_i_orig).count
-                    else
-                      (stat_i_new <= stat_i_orig).count
-                    end
-
+          rs[idx] = mc_observation_calc(stat_i_orig, stat_i_new, permutations)
           idx += 1
         end
 
@@ -91,6 +93,10 @@ module SpatialStats
         end
       end
 
+      def groups
+        raise NotImplementedError, 'groups not implemented'
+      end
+
       private
 
       def mc_i(wi, perms, idx)
@@ -108,6 +114,19 @@ module SpatialStats
         cs.mean(0)
       end
 
+      def mc_observation_calc(stat_i_orig, stat_i_new, _permutations)
+        # Geary cannot be negative, so we have to use this technique from
+        # GeoDa to determine p values. Note I slightly modified it to be inclusive
+        # on both tails not just the lower tail.
+        # https://github.com/GeoDaCenter/geoda/blob/master/Explore/LocalGearyCoordinator.cpp#L981        mean = stat_i_new.mean
+        mean = stat_i_new.mean
+        if stat_i_orig <= mean
+          (stat_i_new <= stat_i_orig).count
+        else
+          (stat_i_new >= stat_i_orig).count
+        end
+      end
+
       def field_data
         @field_data ||= fields.map do |field|
           SpatialStats::Queries::Variables.query_field(@scope, field)

data/lib/spatial_stats/local/stat.rb

@@ -12,7 +12,7 @@ module SpatialStats
       def initialize(scope, field, weights)
         @scope = scope
         @field = field
-        @weights = weights
+        @weights = weights.standardize
       end
       attr_accessor :scope, :field, :weights
 
@@ -62,17 +62,20 @@ module SpatialStats
         # need to get k for max_neighbors
         # and wc for cardinalities of each item
         # this returns an array of length n with
-        # (permutations x neighborz) Numo Arrays.
+        # (permutations x neighbors) Numo Arrays.
         # This helps reduce computation time because
         # we are only dealing with neighbors for each
         # entry not the entire list of permutations for each entry.
         n_1 = weights.n - 1
 
+        sparse = weights.sparse
+        row_index = sparse.row_index
+
         # weight counts
-        wc = [0] * weights.n
+        wc = Array.new(weights.n)
         k = 0
         (0..n_1).each do |idx|
-          wc[idx] = (w[idx, true] > 0).count
+          wc[idx] = row_index[idx + 1] - row_index[idx]
         end
 
         k = wc.max + 1
@@ -112,26 +115,43 @@ module SpatialStats
         # of the entire set. This will be done for each item.
         rng = gen_rng(seed)
         shuffles = crand(x, permutations, rng)
+
         n = weights.n
         # r is the number of equal to or more extreme samples
         stat_orig = stat
         rs = [0] * n
 
-        ws = neighbor_weights
+        row_index = weights.sparse.row_index
+        ws = weights.sparse.values
 
         idx = 0
         while idx < n
+          # need to truncate because floats from
+          # c in sparse matrix are inconsistent with
+          # dfloats
           stat_i_orig = stat_orig[idx]
 
-          wi = Numo::DFloat.cast(ws[idx])
+          # account for case where there are no neighbors
+          # the way Numo handles negative ranges, it returns the max
+          # so there will be a len 0 z array being multiplied by a
+          # max_neighbor width permutation matrix.
+          # Need to skip.
+          row_range = row_index[idx]..(row_index[idx + 1] - 1)
+          if row_range.size.zero?
+            rs[idx] = permutations
+            idx += 1
+            next
+          end
+          wi = Numo::DFloat.cast(ws[row_range])
           stat_i_new = mc_i(wi, shuffles[idx], idx)
 
-          rs[idx] = if stat_i_orig.positive?
-                      (stat_i_new >= stat_i_orig).count
-                    else
-                      (stat_i_new <= stat_i_orig).count
-                    end
-
+          rs[idx] = mc_observation_calc(stat_i_orig, stat_i_new,
+                                        permutations)
+          # rs[idx] = if stat_i_orig.positive?
+          #             (stat_i_new >= stat_i_orig).count
+          #           else
+          #             (stat_i_new <= stat_i_orig).count
+          #           end
           idx += 1
         end
 
@@ -160,19 +180,30 @@ module SpatialStats
         stat_orig = stat
         rs = [0] * n
 
-        ws = neighbor_weights
+        row_index = weights.sparse.row_index
+        ws = weights.sparse.values
 
         idx = 0
         while idx < n
           stat_i_orig = stat_orig[idx]
-          wi = Numo::DFloat.cast(ws[idx])
+
+          row_range = row_index[idx]..(row_index[idx + 1] - 1)
+          if row_range.size.zero?
+            rs[idx] = permutations
+            idx += 1
+            next
+          end
+          wi = Numo::DFloat.cast(ws[row_range])
+
           stat_i_new = mc_i(wi, shuffles[idx], idx)
 
-          rs[idx] = if stat_i_orig.positive?
-                      (stat_i_new >= stat_i_orig).count
-                    else
-                      (stat_i_new <= stat_i_orig).count
-                    end
+          rs[idx] = mc_observation_calc(stat_i_orig, stat_i_new,
+                                        permutations)
+          # if stat_i_orig.positive?
+          #             (stat_i_new >= stat_i_orig).count
+          #           else
+          #             (stat_i_new <= stat_i_orig).count
+          #           end
 
           idx += 1
         end
@@ -200,8 +231,7 @@ module SpatialStats
       # @return [Array] of labels
       def quads
         # https://github.com/pysal/esda/blob/master/esda/moran.py#L925
-        w = @weights.full
-        z_lag = SpatialStats::Utils::Lag.neighbor_average(w, z)
+        z_lag = SpatialStats::Utils::Lag.neighbor_average(weights, z)
         zp = z.map(&:positive?)
         lp = z_lag.map(&:positive?)
 
@@ -221,6 +251,22 @@ module SpatialStats
         end
       end
 
+      ##
+      # Summary of the statistic. Computes +stat+, +mc+, and +groups+ then returns the values
+      # in a hash array.
+      #
+      # @param [Integer] permutations to run. Last digit should be 9 to produce round numbers.
+      # @param [Integer] seed used in random number generator for shuffles.
+      #
+      # @return [Array]
+      def summary(permutations = 99, seed = nil)
+        p_vals = mc(permutations, seed)
+        data = weights.keys.zip(stat, p_vals, groups)
+        data.map do |row|
+          { key: row[0], stat: row[1], p: row[2], group: row[3] }
+        end
+      end
+
       private
 
       def stat_i
@@ -231,8 +277,12 @@ module SpatialStats
         raise NotImplementedError, 'method mc_i not defined'
       end
 
+      def mc_observation_calc(_stat_i_orig, _stat_i_new, _permutations)
+        raise NotImplementedError, 'method mc_observation_calc not defined'
+      end
+
       def w
-        weights.standardized
+        @w ||= weights.dense
       end
 
       def gen_rng(seed = nil)
@@ -242,20 +292,6 @@ module SpatialStats
           Random.new
         end
       end
-
-      def neighbor_weights
-        # record the non-zero weights in variable length arrays for each
-        # row in the weights table
-        ws = [[]] * weights.n
-        (0..weights.n - 1).each do |idx|
-          neighbors = []
-          w[idx, true].each do |wij|
-            neighbors << wij if wij != 0
-          end
-          ws[idx] = neighbors
-        end
-        ws
-      end
     end
   end
 end

data/lib/spatial_stats/narray_ext.rb

@@ -12,11 +12,11 @@ module Numo
     #
     # @ example
     #
-    #   Numo::DFloat [[0, 1, 1], [1, 1, 1]].row_standardized
+    #   Numo::DFloat [[0, 1, 1], [1, 1, 1]].row_standardize
     #   Numo::DFloat [[0, 0.5, 0.5], [0.33333, 0.33333, 0.33333]]
     #
     # @return [Numo::NArray]
-    def row_standardized
+    def row_standardize
       # every row will sum up to 1, or if they are all 0, do nothing
       standardized = each_over_axis.map do |row|
         sum = row.sum
@@ -38,16 +38,16 @@ module Numo
     #
     # @ example
     #
-    #   Numo::DFloat [[0, 1, 0], [1, 0, 1], [0, 1, 0]].windowed
+    #   Numo::DFloat [[0, 1, 0], [1, 0, 1], [0, 1, 0]].window
     #   Numo::DFloat [[1, 1, 0], [1, 1, 1], [0, 1, 1]]
     #
     # @ example
     #   # Input will be equivalent to output in this case
-    #   Numo::DFloat [[1, 1, 0], [1, 0, 1], [0, 1, 0]].windowed
+    #   Numo::DFloat [[1, 1, 0], [1, 0, 1], [0, 1, 0]].window
     #   Numo::DFloat [[1, 1, 0], [1, 0, 1], [0, 1, 0]]
     #
     # @return [Numo::NArray]
-    def windowed
+    def window
       # in windowed calculations, the diagonal is set to 1
       # if trace (sum of diag) is 0, add it, else return input
       if trace.zero?

data/lib/spatial_stats/utils/lag.rb

@@ -11,36 +11,36 @@ module SpatialStats
       # Dot product of the row_standardized input matrix
       # by the input vector, variables.
       #
-      # @param [Numo::NArray] matrix 2-D square matrix.
+      # @param [WeightsMatrix] matrix holding target weights.
       # @param [Array] variables vector multiplying the matrix
       #
       # @return [Array] resultant vector
       def self.neighbor_average(matrix, variables)
-        matrix = matrix.row_standardized
+        matrix = matrix.standardize
         neighbor_sum(matrix, variables)
       end
 
       ##
       # Dot product of the input matrix by the input vector, variables.
       #
-      # @param [Numo::NArray] matrix 2-D square matrix.
+      # @param [WeightsMatrix] matrix holding target weights.
       # @param [Array] variables vector multiplying the matrix
       #
       # @return [Array] resultant vector
       def self.neighbor_sum(matrix, variables)
-        matrix.dot(variables).to_a
+        matrix.sparse.mulvec(variables)
       end
 
       ##
-      # Dot product of the input windowed, row standardizd matrix by
+      # Dot product of the input windowed, row standardized matrix by
       # the input vector, variables.
       #
-      # @param [Numo::NArray] matrix 2-D square matrix.
+      # @param [WeightsMatrix] matrix holding target weights.
       # @param [Array] variables vector multiplying the matrix
       #
       # @return [Array] resultant vector
       def self.window_average(matrix, variables)
-        matrix = matrix.windowed.row_standardized
+        matrix = matrix.window.standardize
         window_sum(matrix, variables)
       end
 
@@ -48,13 +48,13 @@ module SpatialStats
       # Dot product of the input windowed matrix by
       # the input vector, variables.
       #
-      # @param [Numo::NArray] matrix 2-D square matrix.
+      # @param [WeightsMatrix] matrix holding target weights.
       # @param [Array] variables vector multiplying the matrix
       #
       # @return [Array] resultant vector
       def self.window_sum(matrix, variables)
-        matrix = matrix.windowed
-        matrix.dot(variables).to_a
+        matrix = matrix.window
+        matrix.sparse.mulvec(variables)
       end
     end
   end

data/lib/spatial_stats/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SpatialStats
-  VERSION = '0.2.2'
+  VERSION = '1.0.0'
 end

data/lib/spatial_stats/weights/contiguous.rb

@@ -15,21 +15,26 @@ module SpatialStats
       #
       # @return [WeightsMatrix]
       def self.rook(scope, field)
-        p_key = scope.primary_key
-        keys = scope.pluck(p_key).sort
-
         neighbors = SpatialStats::Queries::Weights
                     .rook_contiguity_neighbors(scope, field)
 
+        # get keys to make sure we have consistent dimensions when
+        # some entries don't have neighbors.
+        # define a new hash that has all the keys from scope
+        keys = SpatialStats::Queries::Variables.query_field(scope, scope.klass.primary_key)
+
         neighbors = neighbors.group_by(&:i_id)
+        missing_neighbors = Hash[(keys - neighbors.keys).map { |key| [key, []] }]
+        neighbors = neighbors.merge(missing_neighbors)
+
         weights = neighbors.transform_values do |value|
           value.map do |neighbor|
-            hash = neighbor.as_json(only: [:j_id]).symbolize_keys
+            hash = { id: neighbor[:j_id] }
             hash[:weight] = 1
             hash
           end
         end
-        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+        SpatialStats::Weights::WeightsMatrix.new(weights)
       end
 
       ##
@@ -40,21 +45,26 @@ module SpatialStats
       #
       # @return [WeightsMatrix]
       def self.queen(scope, field)
-        p_key = scope.primary_key
-        keys = scope.pluck(p_key).sort
-
         neighbors = SpatialStats::Queries::Weights
                     .queen_contiguity_neighbors(scope, field)
 
+        # get keys to make sure we have consistent dimensions when
+        # some entries don't have neighbors.
+        # define a new hash that has all the keys from scope
+        keys = SpatialStats::Queries::Variables.query_field(scope, scope.klass.primary_key)
+
         neighbors = neighbors.group_by(&:i_id)
+        missing_neighbors = Hash[(keys - neighbors.keys).map { |key| [key, []] }]
+        neighbors = neighbors.merge(missing_neighbors)
+
         weights = neighbors.transform_values do |value|
           value.map do |neighbor|
-            hash = neighbor.as_json(only: [:j_id]).symbolize_keys
+            hash = { id: neighbor[:j_id] }
             hash[:weight] = 1
             hash
           end
         end
-        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+        SpatialStats::Weights::WeightsMatrix.new(weights)
       end
     end
   end

data/lib/spatial_stats/weights/distant.rb

@@ -15,21 +15,26 @@ module SpatialStats
       #
       # @return [WeightsMatrix]
       def self.distance_band(scope, field, bandwidth)
-        p_key = scope.primary_key
-        keys = scope.pluck(p_key).sort
-
         neighbors = SpatialStats::Queries::Weights
                     .distance_band_neighbors(scope, field, bandwidth)
 
+        # get keys to make sure we have consistent dimensions when
+        # some entries don't have neighbors.
+        # define a new hash that has all the keys from scope
+        keys = SpatialStats::Queries::Variables.query_field(scope, scope.klass.primary_key)
+
         neighbors = neighbors.group_by(&:i_id)
+        missing_neighbors = Hash[(keys - neighbors.keys).map { |key| [key, []] }]
+        neighbors = neighbors.merge(missing_neighbors)
+
         weights = neighbors.transform_values do |value|
           value.map do |neighbor|
-            hash = neighbor.as_json(only: [:j_id]).symbolize_keys
+            hash = { id: neighbor[:j_id] }
             hash[:weight] = 1
             hash
           end
         end
-        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+        SpatialStats::Weights::WeightsMatrix.new(weights)
       end
 
       ##
@@ -41,21 +46,26 @@ module SpatialStats
       #
       # @return [WeightsMatrix]
       def self.knn(scope, field, k)
-        p_key = scope.primary_key
-        keys = scope.pluck(p_key).sort
-
         neighbors = SpatialStats::Queries::Weights
                     .knn(scope, field, k)
 
+        # get keys to make sure we have consistent dimensions when
+        # some entries don't have neighbors.
+        # define a new hash that has all the keys from scope
+        keys = SpatialStats::Queries::Variables.query_field(scope, scope.klass.primary_key)
+
         neighbors = neighbors.group_by(&:i_id)
+        missing_neighbors = Hash[(keys - neighbors.keys).map { |key| [key, []] }]
+        neighbors = neighbors.merge(missing_neighbors)
+
         weights = neighbors.transform_values do |value|
           value.map do |neighbor|
-            hash = neighbor.as_json(only: [:j_id]).symbolize_keys
+            hash = { id: neighbor[:j_id] }
             hash[:weight] = 1
             hash
           end
         end
-        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+        SpatialStats::Weights::WeightsMatrix.new(weights)
       end
 
       ##
@@ -68,20 +78,24 @@ module SpatialStats
       #
       # @return [WeightsMatrix]
       def self.idw_band(scope, field, bandwidth, alpha = 1)
-        p_key = scope.primary_key
-        keys = scope.pluck(p_key).sort
-
         neighbors = SpatialStats::Queries::Weights
                     .idw_band(scope, field, bandwidth, alpha)
+
+        # get keys to make sure we have consistent dimensions when
+        # some entries don't have neighbors.
+        # define a new hash that has all the keys from scope
+        keys = SpatialStats::Queries::Variables.query_field(scope, scope.klass.primary_key)
         neighbors = neighbors.group_by { |pair| pair[:i_id] }
+        missing_neighbors = Hash[(keys - neighbors.keys).map { |key| [key, []] }]
+        neighbors = neighbors.merge(missing_neighbors)
 
         # only keep j_id and weight
         weights = neighbors.transform_values do |value|
           value.map do |neighbor|
-            { weight: neighbor[:weight], j_id: neighbor[:j_id] }
+            { weight: neighbor[:weight], id: neighbor[:j_id] }
           end
         end
-        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+        SpatialStats::Weights::WeightsMatrix.new(weights)
       end
 
       ##
@@ -94,20 +108,24 @@ module SpatialStats
       #
       # @return [WeightsMatrix]
       def self.idw_knn(scope, field, k, alpha = 1)
-        p_key = scope.primary_key
-        keys = scope.pluck(p_key).sort
-
         neighbors = SpatialStats::Queries::Weights
                     .idw_knn(scope, field, k, alpha)
+
+        # get keys to make sure we have consistent dimensions when
+        # some entries don't have neighbors.
+        # define a new hash that has all the keys from scope
+        keys = SpatialStats::Queries::Variables.query_field(scope, scope.klass.primary_key)
         neighbors = neighbors.group_by { |pair| pair[:i_id] }
+        missing_neighbors = Hash[(keys - neighbors.keys).map { |key| [key, []] }]
+        neighbors = neighbors.merge(missing_neighbors)
 
         # only keep j_id and weight
         weights = neighbors.transform_values do |value|
           value.map do |neighbor|
-            { weight: neighbor[:weight], j_id: neighbor[:j_id] }
+            { weight: neighbor[:weight], id: neighbor[:j_id] }
           end
         end
-        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+        SpatialStats::Weights::WeightsMatrix.new(weights)
       end
     end
   end