spatial_stats 0.2.2 → 1.0.0

data/lib/spatial_stats/global/bivariate_moran.rb

@@ -19,7 +19,7 @@ module SpatialStats
         @scope = scope
         @x_field = x_field
         @y_field = y_field
-        @weights = weights
+        @weights = weights.standardize
       end
       attr_writer :x, :y
 
@@ -29,8 +29,7 @@ module SpatialStats
       #
       # @return [Float]
       def stat
-        w = @weights.standardized
-        y_lag = SpatialStats::Utils::Lag.neighbor_sum(w, y)
+        y_lag = SpatialStats::Utils::Lag.neighbor_sum(weights, y)
         numerator = 0
         x.each_with_index do |xi, idx|
           numerator += xi * y_lag[idx]
@@ -55,19 +54,20 @@ module SpatialStats
       #
       # @return [Float]
       def variance
-        n = @weights.n
-        wij = @weights.full
-        w = wij.sum
+        n = weights.n
+        w_sum = n.to_f
         e = expectation
 
-        s1 = s1_calc(n, wij)
-        s2 = s2_calc(n, wij)
+        wij = weights.sparse.coordinates
+
+        s1 = s1_calc(wij)
+        s2 = s2_calc(n, wij, weights.sparse.row_index)
         s3 = s3_calc(n, x)
 
-        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w**2)
-        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w**2)
+        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w_sum**2)
+        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w_sum**2)
 
-        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w**2)
+        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w_sum**2)
         var_right = e**2
         var_left - var_right
       end
@@ -88,6 +88,19 @@ module SpatialStats
         mc_bv(permutations, seed)
       end
 
+      ##
+      # Summary of the statistic. Computes +stat+ and +mc+ and returns the values
+      # in a hash.
+      #
+      # @param [Integer] permutations to run. Last digit should be 9 to produce round numbers.
+      # @param [Integer] seed used in random number generator for shuffles.
+      #
+      # @return [Hash]
+      def summary(permutations = 99, seed = nil)
+        p_val = mc(permutations, seed)
+        { stat: stat, p: p_val }
+      end
+
       ##
       # Standardized variables queried from +x_field+.
       #
@@ -108,32 +121,41 @@ module SpatialStats
 
       private
 
+      def stat_mc(perms)
+        x_arr = Numo::DFloat.cast(x)
+        lag = w.dot(perms.transpose)
+        x_arr.dot(lag) / (x_arr**2).sum
+      end
+
       def s3_calc(n, zs)
         numerator = (1.0 / n) * zs.sum { |v| v**4 }
         denominator = ((1.0 / n) * zs.sum { |v| v**2 })**2
         numerator / denominator
       end
 
-      def s2_calc(n, wij)
+      def s2_calc(n, wij, row_index)
         s2 = 0
-        (0..n - 1).each do |i|
+        wij_arr = wij.to_a # for row slicing
+        (0..n - 1).each do |idx|
+          row = wij_arr[row_index[idx]..(row_index[idx + 1] - 1)]
           left_term = 0
           right_term = 0
-          (0..n - 1).each do |j|
-            left_term += wij[i, j]
-            right_term += wij[j, i]
+
+          row.each do |coords, val|
+            left_term += val
+            right_term += wij[coords.reverse] || 0
           end
           s2 += (left_term + right_term)**2
         end
         s2
       end
 
-      def s1_calc(n, wij)
+      def s1_calc(wij)
         s1 = 0
-        (0..n - 1).each do |i|
-          (0..n - 1).each do |j|
-            s1 += (wij[i, j] + wij[j, i])**2
-          end
+        wij.each do |coords, val|
+          # (wij + wji)**2
+          wji = wij[coords.reverse] || 0
+          s1 += (val + wji)**2
         end
         s1 / 2
       end

data/lib/spatial_stats/global/moran.rb

@@ -29,8 +29,7 @@ module SpatialStats
         # compute's Moran's I. numerator is sum of zi * spatial lag of zi
         # denominator is sum of zi**2.
         # have to use row-standardized weights
-        w = @weights.standardized
-        z_lag = SpatialStats::Utils::Lag.neighbor_sum(w, z)
+        z_lag = SpatialStats::Utils::Lag.neighbor_sum(weights, z)
         numerator = 0
         z.each_with_index do |zi, j|
           row_sum = zi * z_lag[j]
@@ -49,7 +48,7 @@ module SpatialStats
       # @return [Float]
       def expectation
         # -1/(n-1)
-        -1.0 / (@weights.n - 1)
+        -1.0 / (weights.n - 1)
       end
 
       ##
@@ -58,19 +57,20 @@ module SpatialStats
       #
       # @return [Float]
       def variance
-        n = @weights.n
-        wij = @weights.full
-        w = wij.sum
+        n = weights.n
+        w_sum = n # standardized weights
         e = expectation
 
-        s1 = s1_calc(n, wij)
-        s2 = s2_calc(n, wij)
+        wij = weights.sparse.coordinates
+
+        s1 = s1_calc(wij)
+        s2 = s2_calc(n, wij, weights.sparse.row_index)
         s3 = s3_calc(n, z)
 
-        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w**2)
-        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w**2)
+        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w_sum**2)
+        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w_sum**2)
 
-        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w**2)
+        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w_sum**2)
         var_right = e**2
         var_left - var_right
       end
@@ -92,58 +92,66 @@ module SpatialStats
       end
 
       ##
-      # Values of the +field+ queried from the +scope+
+      # Summary of the statistic. Computes +stat+ and +mc+ and returns the values
+      # in a hash.
       #
-      # @return [Array]
-      def x
-        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
-      end
-
-      # TODO: remove these last 2 methods and just standardize x.
-      ##
-      # Mean of x
+      # @param [Integer] permutations to run. Last digit should be 9 to produce round numbers.
+      # @param [Integer] seed used in random number generator for shuffles.
       #
-      # @return [Float]
-      def zbar
-        x.sum / x.size
+      # @return [Hash]
+      def summary(permutations = 99, seed = nil)
+        p_val = mc(permutations, seed)
+        { stat: stat, p: p_val }
       end
 
       ##
-      # Array of xi - zbar for i: [0:n-1]
+      # Values of the +field+ queried from the +scope+
       #
       # @return [Array]
-      def z
-        x.map { |val| val - zbar }
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
+                                               .standardize
       end
+      alias z x
 
       private
 
+      def stat_mc(perms)
+        z_arr = Numo::DFloat.cast(z)
+        lag = w.dot(perms.transpose)
+        z_arr.dot(lag) / (z_arr**2).sum
+      end
+
       def s3_calc(n, zs)
         numerator = (1.0 / n) * zs.sum { |v| v**4 }
         denominator = ((1.0 / n) * zs.sum { |v| v**2 })**2
         numerator / denominator
       end
 
-      def s2_calc(n, wij)
+      # use row_index to take slices of wij
+      def s2_calc(n, wij, row_index)
         s2 = 0
+        wij_arr = wij.to_a # for row slicing
         (0..n - 1).each do |idx|
+          row = wij_arr[row_index[idx]..(row_index[idx + 1] - 1)]
           left_term = 0
           right_term = 0
-          (0..n - 1).each do |j|
-            left_term += wij[idx, j]
-            right_term += wij[j, idx]
+
+          row.each do |coords, val|
+            left_term += val
+            right_term += wij[coords.reverse] || 0
           end
           s2 += (left_term + right_term)**2
         end
         s2
       end
 
-      def s1_calc(n, wij)
+      def s1_calc(wij)
         s1 = 0
-        (0..n - 1).each do |idx|
-          (0..n - 1).each do |j|
-            s1 += (wij[idx, j] + wij[j, idx])**2
-          end
+        wij.each do |coords, val|
+          # (wij + wji)**2
+          wji = wij[coords.reverse] || 0
+          s1 += (val + wji)**2
         end
         s1 / 2
       end

data/lib/spatial_stats/global/stat.rb

@@ -11,7 +11,7 @@ module SpatialStats
       def initialize(scope, field, weights)
         @scope = scope
         @field = field
-        @weights = weights
+        @weights = weights.standardize
       end
       attr_accessor :scope, :field, :weights
 
@@ -45,21 +45,21 @@ module SpatialStats
         permutations.times do
           shuffles << x.shuffle(random: rng)
         end
+        shuffles = Numo::DFloat.cast(shuffles)
+
         # r is the number of equal to or more extreme samples
         # one sided
-        stat_orig = stat
-        r = 0
-        shuffles.each do |shuffle|
-          klass = self.class.new(@scope, @field, @weights)
-          klass.x = shuffle
-
-          # https://geodacenter.github.io/glossary.html#ppvalue
-          if stat_orig.positive?
-            r += 1 if klass.stat >= stat_orig
-          else
-            r += 1 if klass.stat <= stat_orig
-          end
-        end
+        stat_orig = stat.round(5)
+        # r = 0
+
+        # compute new stat values
+        stat_new = stat_mc(shuffles)
+
+        r = if stat_orig.positive?
+              (stat_new >= stat_orig).count
+            else
+              (stat_new <= stat_orig).count
+            end
 
         (r + 1.0) / (permutations + 1.0)
       end
@@ -71,27 +71,31 @@ module SpatialStats
         permutations.times do
           shuffles << y.shuffle(random: rng)
         end
+        shuffles = Numo::DFloat.cast(shuffles)
 
         # r is the number of equal to or more extreme samples
-        stat_orig = stat
-        r = 0
-        shuffles.each do |shuffle|
-          klass = self.class.new(@scope, @x_field, @y_field, @weights)
-          klass.x = x
-          klass.y = shuffle
-
-          if stat_orig.positive?
-            r += 1 if klass.stat >= stat_orig
-          else
-            r += 1 if klass.stat <= stat_orig
-          end
-        end
+        stat_orig = stat.round(5)
+        stat_new = stat_mc(shuffles)
+
+        r = if stat_orig.positive?
+              (stat_new >= stat_orig).count
+            else
+              (stat_new <= stat_orig).count
+            end
 
         (r + 1.0) / (permutations + 1.0)
       end
 
       private
 
+      def stat_mc(_shuffles)
+        raise NotImplementedError, 'private method stat_mc not defined'
+      end
+
+      def w
+        @w ||= weights.dense
+      end
+
       def gen_rng(seed)
         if seed
           Random.new(seed)

data/lib/spatial_stats/local/bivariate_moran.rb

@@ -19,9 +19,10 @@ module SpatialStats
         @scope = scope
         @x_field = x_field
         @y_field = y_field
-        @weights = weights
+        @weights = weights.standardize
       end
       attr_accessor :scope, :x_field, :y_field, :weights
+      attr_writer :x, :y
 
       ##
       # Computes the local indicator of spatial correlation for
@@ -62,6 +63,61 @@ module SpatialStats
         mc_bv(permutations, seed)
       end
 
+      ##
+      # Determines what quadrant an observation is in. Based on its value
+      # compared to its neighbors. This does not work for all stats, since
+      # it requires that values be negative.
+      #
+      # In a standardized array of z, high values are values greater than 0
+      # and it's neighbors are determined by the spatial lag and if that is
+      # positive then it's neighbors would be high, low otherwise.
+      #
+      # Quadrants are:
+      # [HH] a high value surrounded by other high values
+      # [LH] a low value surrounded by high values
+      # [LL] a low value surrounded by low values
+      # [HL] a high value surrounded by low values
+      #
+      # @return [Array] of labels
+      def quads
+        # https://github.com/pysal/esda/blob/master/esda/moran.py#L925
+        z_lag = SpatialStats::Utils::Lag.neighbor_average(weights, y)
+        zp = x.map(&:positive?)
+        lp = z_lag.map(&:positive?)
+
+        # hh = zp & lp
+        # lh = zp ^ true & lp
+        # ll = zp ^ true & lp ^ true
+        # hl = zp next to lp ^ true
+        hh = zp.each_with_index.map { |v, idx| v & lp[idx] }
+        lh = zp.each_with_index.map { |v, idx| (v ^ true) & lp[idx] }
+        ll = zp.each_with_index.map { |v, idx| (v ^ true) & (lp[idx] ^ true) }
+        hl = zp.each_with_index.map { |v, idx| v & (lp[idx] ^ true) }
+
+        # now zip lists and map them to proper terms
+        quad_terms = %w[HH LH LL HL]
+        hh.zip(lh, ll, hl).map do |feature|
+          quad_terms[feature.index(true)]
+        end
+      end
+      alias groups quads
+
+      ##
+      # Summary of the statistic. Computes +stat+, +mc+, and +groups+ then returns the values
+      # in a hash array.
+      #
+      # @param [Integer] permutations to run. Last digit should be 9 to produce round numbers.
+      # @param [Integer] seed used in random number generator for shuffles.
+      #
+      # @return [Array]
+      def summary(permutations = 99, seed = nil)
+        p_vals = mc(permutations, seed)
+        data = weights.keys.zip(stat, p_vals, groups)
+        data.map do |row|
+          { key: row[0], stat: row[1], p: row[2], group: row[3] }
+        end
+      end
+
       def x
         @x ||= SpatialStats::Queries::Variables.query_field(@scope, @x_field)
                                                .standardize
@@ -79,8 +135,17 @@ module SpatialStats
         x[idx] * y_lag_i
       end
 
+      def mc_observation_calc(stat_i_orig, stat_i_new, _permutations)
+        # Since moran can be positive or negative, go by this definition
+        if stat_i_orig.positive?
+          (stat_i_new >= stat_i_orig).count
+        else
+          (stat_i_new <= stat_i_orig).count
+        end
+      end
+
       def y_lag
-        @y_lag ||= SpatialStats::Utils::Lag.neighbor_sum(w, y)
+        @y_lag ||= SpatialStats::Utils::Lag.neighbor_sum(weights, y)
       end
     end
   end

data/lib/spatial_stats/local/geary.rb

@@ -18,6 +18,7 @@ module SpatialStats
       def initialize(scope, field, weights)
         super(scope, field, weights)
       end
+      attr_writer :x
 
       ##
       # Computes Geary's C for every observation in the +scoe+.
@@ -32,6 +33,25 @@ module SpatialStats
       end
       alias c stat
 
+      ##
+      # Computes the groups each observation belongs to.
+      # Potential groups for Geary's C are:
+      # [HH] High-High
+      # [LL] Low-Low
+      # [N] Negative - Group traditionally for HL and LH, but since the difference is squared they are in the same group.
+      #
+      #
+      # @return [Array] groups for each observation
+      def groups
+        quads.map do |quad|
+          if %w[HL LH].include?(quad)
+            'N'
+          else
+            quad
+          end
+        end
+      end
+
       ##
       # Values of the +field+ queried from the +scope+
       #
@@ -45,9 +65,11 @@ module SpatialStats
       private
 
       def stat_i(idx)
-        zs = Numo::DFloat.cast(z)
-        zi = (z[idx] - zs)**2
-        (w[idx, true] * zi).sum
+        # TODO: maybe don't even use stat_i
+        # just form all of the modified zs and then
+        # pass it to a loop of mulvec all implemented in c ext
+        zi = z.map { |val| (z[idx] - val)**2 }
+        weights.sparse.dot_row(zi, idx)
       end
 
       def mc_i(wi, perms, idx)
@@ -55,8 +77,17 @@ module SpatialStats
         (wi * zi).sum(1)
       end
 
-      def w
-        @w ||= weights.full.row_standardized
+      def mc_observation_calc(stat_i_orig, stat_i_new, _permutations)
+        # Geary cannot be negative, so we have to use this technique from
+        # GeoDa to determine p values. Note I slightly modified it to be inclusive
+        # on both tails not just the lower tail.
+        # https://github.com/GeoDaCenter/geoda/blob/master/Explore/LocalGearyCoordinator.cpp#L981        mean = stat_i_new.mean
+        mean = stat_i_new.mean
+        if stat_i_orig <= mean
+          (stat_i_new <= stat_i_orig).count
+        else
+          (stat_i_new >= stat_i_orig).count
+        end
       end
     end
   end

data/lib/spatial_stats/local/getis_ord.rb

@@ -14,13 +14,18 @@ module SpatialStats
       # @param [ActiveRecord::Relation] scope
       # @param [Symbol, String] field to query from scope
       # @param [WeightsMatrix] weights to define relationship between observations in scope
+      # @param [Boolean] star to preset if star will be true or false. Will be calculated otherwise.
       #
       # @return [GetisOrd]
       def initialize(scope, field, weights, star = nil)
-        super(scope, field, weights)
+        @scope = scope
+        @field = field
+        @weights = weights
         @star = star
+        calc_weights
       end
       attr_accessor :star
+      attr_writer :x
 
       ##
       # Computes the G or G* statistic for every observation in x.
@@ -33,6 +38,25 @@ module SpatialStats
       end
       alias g stat
 
+      ##
+      # Computes the groups each observation belongs to.
+      # Potential groups for G are:
+      # [H] High
+      # [L] Low
+      #
+      # Group is high when standardized z is positive, low otherwise.
+      #
+      # @return [Array] groups for each observation
+      def groups
+        z.standardize.map do |val|
+          if val.positive?
+            'H'
+          else
+            'L'
+          end
+        end
+      end
+
       ##
       # Values of the +field+ queried from the +scope+
       #
@@ -50,7 +74,7 @@ module SpatialStats
       # @return [Boolean] of star
       def star?
         if @star.nil?
-          @star = weights.full.trace.positive?
+          @star = weights.dense.trace.positive?
         else
           @star
         end
@@ -67,25 +91,29 @@ module SpatialStats
         x_lag_i / denominators[idx]
       end
 
-      def w
-        @w ||= begin
-          if star?
-            weights.full.windowed.row_standardized
-          else
-            weights.standardized
-          end
+      def mc_observation_calc(stat_i_orig, stat_i_new, permutations)
+        # GetisOrd cannot be negative, so we have to use this technique from
+        # ESDA to determine if we should select p or 1-p.
+        # https://github.com/pysal/esda/blob/master/esda/getisord.py#L388
+        num_larger = (stat_i_new >= stat_i_orig).count
+        is_low = (permutations - num_larger) < num_larger
+        if is_low
+          permutations - num_larger
+        else
+          num_larger
         end
       end
 
+      def calc_weights
+        @weights = if star?
+                     weights.window.standardize
+                   else
+                     weights.standardize
+                   end
+      end
+
       def z_lag
-        # window if star is true
-        @z_lag ||= begin
-          if star?
-            SpatialStats::Utils::Lag.window_sum(w, x)
-          else
-            SpatialStats::Utils::Lag.neighbor_sum(w, x)
-          end
-        end
+        @z_lag ||= SpatialStats::Utils::Lag.neighbor_sum(weights, x)
       end
       alias x_lag z_lag