spatial_stats 0.1.0

data/lib/spatial_stats/local/moran.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+# https://pro.arcgis.com/en/pro-app/tool-reference/spatial-statistics/h-how-cluster-and-outlier-analysis-anselin-local-m.htm
+# For now, instead of doing neighbor's variance (Si**2), I'm going to use
+# the total sample variance. This is how GeoDa does it, but is different
+# than arcgis. This shouldn't affect the expectation and variance of I.
+module SpatialStats
+  module Local
+    class Moran < Stat
+      def initialize(scope, field, weights)
+        super(scope, field, weights)
+        @scope = scope
+        @field = field
+        @weights = weights
+      end
+      attr_writer :x, :z_lag
+
+      def i
+        z.each_with_index.map do |_z_val, idx|
+          i_i(idx)
+        end
+      end
+
+      def i_i(idx)
+        # method to compute i at a single index.
+        # this is important for permutation testing
+        # because for each test we only want the result from
+        # 1 index not the entire set, so this will save lots of
+        # computations.
+        sum_term = z_lag[idx]
+        (z[idx] / si2) * sum_term
+      end
+
+      def expectation
+        # since we are using row standardized weights, the expectation
+        # will just be -1/(n-1) for all items. Otherwise, it would be
+        # a vector where the sum of the weights for each row is the numerator
+        # in the equation.
+        -1.0 / (@weights.n - 1)
+      end
+
+      def variance
+        # formula is A - B - (E[I])**2
+        wt = w.row_standardized
+        exp = expectation
+
+        vars = []
+        a_terms = a_calc(wt)
+        b_terms = b_calc(wt)
+
+        a_terms.each_with_index do |a_term, idx|
+          vars << (a_term - b_terms[idx] - (exp**2))
+        end
+        vars
+      end
+
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
+                                               .standardize
+      end
+      alias z x
+
+      def z_lag
+        # can't memoize yet because of mc testing
+        # w is already row_standardized, so we are using
+        # neighbor sum instead of neighbor_average to save cost
+        @z_lag ||= SpatialStats::Utils::Lag.neighbor_sum(w, z)
+      end
+
+      private
+
+      def si2
+        # @si2 ||= z.sample_variance
+        # we standardize so sample_variance is 1
+        1.0
+      end
+
+      # https://pro.arcgis.com/en/pro-app/tool-reference/spatial-statistics/h-local-morans-i-additional-math.htm
+      def a_calc(wt)
+        n = wt.shape[0]
+        b2i = b2i_calc
+        a_terms = []
+
+        (0..n - 1).each do |idx|
+          sigma_term = wt[idx, true].to_a.sum { |v| v**2 }
+          a_terms << (n - b2i) * sigma_term / (n - 1)
+        end
+        a_terms
+      end
+
+      def b_calc(wt)
+        n = wt.shape[0]
+        b2i = b2i_calc
+        b_terms = []
+
+        # technically, the formula is Sigma k (sigma h (wik * wih))
+        # since we use row standardized matricies, this is always 1
+        # for each row
+        # this also means that all b_terms will be the same.
+        sigma_term = 1.0
+        b_terms << sigma_term * (2 * b2i - n) / ((n - 1) * (n - 2))
+        b_terms * n
+      end
+
+      def b2i_calc
+        numerator = z.sum { |v| v**4 }
+        denominator = z.sum { |v| v**2 }
+        numerator / (denominator**2)
+      end
+    end
+  end
+end

data/lib/spatial_stats/local/multivariate_geary.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Local
+    class MultivariateGeary < Stat
+      def initialize(scope, fields, weights)
+        @scope = scope
+        @fields = fields
+        @weights = weights
+      end
+      attr_accessor :scope, :fields, :weights
+
+      def i
+        m = fields.size
+        gearys = fields.map do |field|
+          Geary.new(scope, field, weights).i
+        end
+        gearys.transpose.map { |x| x.reduce(:+) / m }
+      end
+
+      def mc(permutations = 99, seed = nil)
+        # in this case, one tuple of vals is held constant, then
+        # the rest are shuffled, so for crand we will pass in an arr
+        # of indices, which will return a list of new orders for the fields.
+        # They will then be shuffled corresponding to the new indices.
+        rng = gen_rng(seed)
+        n = w.shape[0]
+        indices = (0..(n - 1)).to_a
+        shuffles = crand(indices, permutations, rng)
+
+        i_orig = i
+        rs = [0] * i_orig.size
+        shuffles.each_with_index do |perms, idx|
+          ii_orig = i_orig[idx]
+          perms.each do |perm|
+            # essentially reimplement i here, but only use i_i
+            m = fields.size
+            gearys = fields.each_with_index.map do |field, field_idx|
+              geary = Geary.new(scope, field, weights)
+              geary.x = field_data[field_idx].values_at(*perm)
+              geary.i_i(idx)
+            end
+            ii_new = gearys.sum { |x| x / m }
+
+            if ii_orig.positive?
+              rs[idx] += 1 if ii_new >= ii_orig
+            else
+              rs[idx] += 1 if ii_new <= ii_orig
+            end
+          end
+        end
+
+        rs.map do |ri|
+          (ri + 1.0) / (permutations + 1.0)
+        end
+      end
+
+      private
+
+      def field_data
+        @field_data ||= fields.map do |field|
+          SpatialStats::Queries::Variables.query_field(@scope, field)
+                                          .standardize
+        end
+      end
+    end
+  end
+end

data/lib/spatial_stats/local/stat.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Local
+    class Stat
+      # Base class for local stats
+      def initialize(scope, field, weights)
+        @scope = scope
+        @field = field
+        @weights = weights
+      end
+      attr_accessor :scope, :field, :weights
+
+      def i
+        raise NotImplementedError, 'method i not defined'
+      end
+
+      def i_i(_idx)
+        raise NotImplementedError, 'method i_i not defined'
+      end
+
+      def expectation
+        raise NotImplementedError, 'method expectation not implemented'
+      end
+
+      def variance
+        raise NotImplementedError, 'method variance not implemented'
+      end
+
+      def z_score
+        numerators = i.map { |v| v - expectation }
+        denominators = variance.map { |v| Math.sqrt(v) }
+        numerators.each_with_index.map do |numerator, idx|
+          numerator / denominators[idx]
+        end
+      end
+
+      def crand(arr, permutations, rng)
+        # conditional randomization method
+        # will generate an n x permutations array of arrays.
+        # For each n, i will be held the same and the values around it will
+        # be permutated.
+        arr.each_with_index.map do |xi, idx|
+          tmp_arr = arr.dup
+          tmp_arr.delete_at(idx)
+          permutations.times.map do
+            perm = tmp_arr.shuffle(random: rng)
+            perm.insert(idx, xi)
+          end
+        end
+      end
+
+      # def crandi(arr, permutations, rng)
+      #   n = @weights.n
+      #   lisas = Numo::DFloat.zeros([n, permutations])
+
+      #   ids = (0..n - 1).to_a
+      #   rids = permutations.times.map do
+      #     ids.shuffle(random: rng)
+      #   end
+      #   p rids
+
+      #   (0..n - 1).each do |idx|
+      #     idsi = ids.dup
+      #     idsi.delete_at(idx)
+      #     ids.shuffle!(random: rng)
+      #     tmp = arr[idsi[rids[]]]
+      #   end
+      # end
+
+      def mc(permutations = 99, seed = nil)
+        # For local tests, we need to shuffle the values
+        # but for each item, hold its value in place and shuffle
+        # its neighbors. Then we will only test for that item instead
+        # of the entire set. This will be done for each item.
+        rng = gen_rng(seed)
+        shuffles = crand(x, permutations, rng)
+
+        # r is the number of equal to or more extreme samples
+        i_orig = i
+        rs = [0] * i_orig.size
+
+        # For each shuffle, we only need the spatially lagged variable
+        # at one index, but it needs to be an array of length n.
+        # Store a zeros array that can be mutated or duplicated and the
+        # lagged variable at idx will only be set there.
+        lagged = [0] * i_orig.size
+
+        shuffles.each_with_index do |perms, idx|
+          ii_orig = i_orig[idx]
+          wi = w[idx, true] # current weight row
+          perms.each do |perm|
+            stat = self.class.new(scope, field, weights)
+            stat.x = perm
+
+            # avoids computing lag for entire data set
+            # when we only care about one entry
+            lagged_var = wi.dot(perm)
+            z_lag = lagged.dup
+            z_lag[idx] = lagged_var
+            stat.z_lag = z_lag
+
+            ii_new = stat.i_i(idx)
+
+            # https://geodacenter.github.io/glossary.html#ppvalue
+            # NOTE: this is inconsistent with the output from GeoDa
+            # for local permutation tests, they seem to use greater than
+            # not greater than or equal to. I'm going to go by the definition
+            # in the glossary for now.
+            if ii_orig.positive?
+              rs[idx] += 1 if ii_new >= ii_orig
+            else
+              rs[idx] += 1 if ii_new <= ii_orig
+            end
+          end
+        end
+
+        rs.map do |ri|
+          (ri + 1.0) / (permutations + 1.0)
+        end
+      end
+
+      def mc_bv(permutations, seed)
+        rng = gen_rng(seed)
+        shuffles = crand(y, permutations, rng)
+
+        # r is the number of equal to or more extreme samples
+        i_orig = i
+        rs = [0] * i_orig.size
+        shuffles.each_with_index do |perms, idx|
+          ii_orig = i_orig[idx]
+          perms.each do |perm|
+            stat = self.class.new(@scope, @x_field, @y_field, @weights)
+            stat.x = x
+            stat.y = perm
+            ii_new = stat.i_i(idx)
+
+            if ii_orig.positive?
+              rs[idx] += 1 if ii_new >= ii_orig
+            else
+              rs[idx] += 1 if ii_new <= ii_orig
+            end
+          end
+        end
+
+        rs.map do |ri|
+          (ri + 1.0) / (permutations + 1.0)
+        end
+      end
+
+      def quads
+        # https://github.com/pysal/esda/blob/master/esda/moran.py#L925
+        w = @weights.full
+        z_lag = SpatialStats::Utils::Lag.neighbor_average(w, z)
+        zp = z.map(&:positive?)
+        lp = z_lag.map(&:positive?)
+
+        # hh = zp & lp
+        # lh = zp ^ true & lp
+        # ll = zp ^ true & lp ^ true
+        # hl = zp next to lp ^ true
+        hh = zp.each_with_index.map { |v, idx| v & lp[idx] }
+        lh = zp.each_with_index.map { |v, idx| (v ^ true) & lp[idx] }
+        ll = zp.each_with_index.map { |v, idx| (v ^ true) & (lp[idx] ^ true) }
+        hl = zp.each_with_index.map { |v, idx| v & (lp[idx] ^ true) }
+
+        # now zip lists and map them to proper terms
+        quad_terms = %w[HH LH LL HL]
+        hh.zip(lh, ll, hl).map do |feature|
+          quad_terms[feature.index(true)]
+        end
+      end
+
+      private
+
+      def w
+        weights.standardized
+      end
+
+      def gen_rng(seed = nil)
+        if seed
+          Random.new(seed)
+        else
+          Random.new
+        end
+      end
+    end
+  end
+end

data/lib/spatial_stats/local.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'spatial_stats/local/stat'
+require 'spatial_stats/local/bivariate_moran'
+require 'spatial_stats/local/g'
+require 'spatial_stats/local/geary'
+require 'spatial_stats/local/moran'
+require 'spatial_stats/local/multivariate_geary'

data/lib/spatial_stats/narray_ext.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'numo/narray'
+
+module Numo
+  class NArray
+    def row_standardized
+      # every row will sum up to 1, or if they are all 0, do nothing
+      standardized = each_over_axis.map do |row|
+        sum = row.sum
+        if sum.zero?
+          # for some reason, we have to do this instead of just returning
+          # row. If row is returned, it is cast as [0,0,0] => [0,1,0] for
+          # example.
+          self.class.zeros(row.size)
+        else
+          row / sum
+        end
+      end
+      self.class.cast(standardized)
+    end
+
+    def windowed
+      # in windowed calculations, the diagonal is set to 1
+      # if trace (sum of diag) is 0, add it, else return input
+      if trace.zero?
+        self + self.class.eye(shape[0])
+      else
+        self
+      end
+    end
+  end
+end

data/lib/spatial_stats/queries/variables.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Queries
+    module Variables
+      # Module to query for the desired variable from the given scope
+      # and include the primary keys so that the weights matrix
+      # will know that its keys will match up with the variables.
+      def self.query_field(scope, field)
+        klass = scope.klass
+        column = ActiveRecord::Base.connection.quote_column_name(field)
+        primary_key = klass.quoted_primary_key
+        variables = klass.find_by_sql([<<-SQL, scope: scope])
+          WITH scope as (:scope)
+          SELECT scope.#{column} as field FROM scope
+          ORDER BY scope.#{primary_key} ASC
+        SQL
+        variables.map(&:field)
+      end
+    end
+  end
+end

data/lib/spatial_stats/queries/weights.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Queries
+    # This provides PostGIS queries for calculating weights/neighbors
+    # of spatial data sets
+    module Weights
+      def self.idw_knn(scope, column, n, alpha)
+        klass = scope.klass
+        column = ActiveRecord::Base.connection.quote_column_name(column)
+        primary_key = klass.quoted_primary_key
+        neighbors = klass.find_by_sql([<<-SQL, scope: scope, n: n])
+          WITH scope as (:scope)
+          SELECT neighbors.*
+          FROM scope AS a
+            CROSS JOIN LATERAL (
+            SELECT a.#{primary_key} as i_id, b.#{primary_key} as j_id,
+            ST_Distance(a.#{column}, b.#{column}) as distance
+            FROM scope as b
+            WHERE a.#{primary_key} <> b.#{primary_key}
+            ORDER BY a.#{column} <-> b.#{column}
+            LIMIT :n
+          ) AS neighbors
+        SQL
+
+        # if the lowest distance is <1, then we need to scale
+        # every distance by the factor that makes the lowest 1
+        min_dist = neighbors.map(&:distance).min
+        scale = if min_dist < 1
+                  1 / min_dist
+                else
+                  1
+                end
+
+        neighbors.map do |neighbor|
+          # formula is 1/(d^alpha)
+          weight = 1.0 / ((scale * neighbor.distance)**alpha)
+          hash = neighbor.as_json.symbolize_keys
+          hash[:weight] = weight
+          hash
+        end
+      end
+
+      def self.idw_band(scope, column, bandwidth, alpha = 1)
+        klass = scope.klass
+        column = ActiveRecord::Base.connection.quote_column_name(column)
+        primary_key = klass.quoted_primary_key
+        neighbors = klass.find_by_sql([<<-SQL, scope: scope, bandwidth: bandwidth])
+          WITH neighbors AS (
+            WITH scope AS (:scope)
+            SELECT a.#{primary_key} as i_id, b.#{primary_key} as j_id,
+            ST_DWithin(a.#{column}, b.#{column}, :bandwidth) as is_neighbor,
+            ST_Distance(a.#{column}, b.#{column}) as distance
+            FROM scope as a, scope as b
+            ORDER BY i_id
+          )
+          SELECT * FROM neighbors WHERE is_neighbor = 't' AND i_id <> j_id
+        SQL
+
+        # if the lowest distance is <1, then we need to scale
+        # every distance by the factor that makes the lowest 1
+        min_dist = neighbors.map(&:distance).min
+        scale = if min_dist < 1
+                  1 / min_dist
+                else
+                  1
+                end
+
+        neighbors.map do |neighbor|
+          # formula is 1/(d^alpha)
+          weight = 1.0 / ((scale * neighbor.distance)**alpha)
+          hash = neighbor.as_json.symbolize_keys
+          hash[:weight] = weight
+          hash
+        end
+      end
+
+      def self.knn(scope, column, n)
+        klass = scope.klass
+        column = ActiveRecord::Base.connection.quote_column_name(column)
+        primary_key = klass.quoted_primary_key
+        klass.find_by_sql([<<-SQL, scope: scope, n: n])
+          WITH scope as (:scope)
+          SELECT neighbors.*
+          FROM scope AS a
+            CROSS JOIN LATERAL (
+            SELECT a.#{primary_key} as i_id, b.#{primary_key} as j_id
+            FROM scope as b
+            WHERE a.#{primary_key} <> b.#{primary_key}
+            ORDER BY a.#{column} <-> b.#{column}
+            LIMIT :n
+          ) AS neighbors
+        SQL
+      end
+
+      def self.distance_band_neighbors(scope, column, bandwidth)
+        klass = scope.klass
+        column = ActiveRecord::Base.connection.quote_column_name(column)
+        primary_key = klass.quoted_primary_key
+        klass.find_by_sql([<<-SQL, scope: scope, distance: bandwidth])
+          WITH neighbors AS (
+            WITH scope AS (:scope)
+            SELECT a.#{primary_key} as i_id, b.#{primary_key} as j_id,
+            ST_DWithin(a.#{column}, b.#{column}, :distance) as is_neighbor
+            FROM scope as a, scope as b
+            ORDER BY i_id
+          )
+          SELECT * FROM neighbors WHERE is_neighbor = 't' AND i_id <> j_id
+        SQL
+      end
+
+      # DE-9IM queen contiguiety = F***T****
+      def self.queen_contiguity_neighbors(scope, column)
+        _contiguity_neighbors(scope, column, 'F***T****')
+      end
+
+      def self.rook_contiguity_neighbors(scope, column)
+        _contiguity_neighbors(scope, column, 'F***1****')
+      end
+
+      def self._contiguity_neighbors(scope, column, pattern)
+        klass = scope.klass
+        column = ActiveRecord::Base.connection.quote_column_name(column)
+        primary_key = klass.quoted_primary_key
+        klass.find_by_sql([<<-SQL, scope: scope])
+          WITH neighbors AS (
+            WITH scope AS (:scope)
+            SELECT a.#{primary_key} as i_id, b.#{primary_key} as j_id,
+            ST_RELATE(a.#{column}, b.#{column}, \'#{pattern}\') as is_neighbor
+            FROM scope as a, scope as b
+            ORDER BY i_id
+          )
+          SELECT * FROM neighbors WHERE is_neighbor = 't'
+        SQL
+      end
+    end
+  end
+end

data/lib/spatial_stats/queries.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'spatial_stats/queries/variables'
+require 'spatial_stats/queries/weights'

data/lib/spatial_stats/railtie.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  class Railtie < ::Rails::Railtie
+  end
+end

data/lib/spatial_stats/utils/lag.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'numo/narray'
+module SpatialStats
+  module Utils
+    module Lag
+      # module for computing spatially lagged variables
+      # from a weights matrix and variable array
+      def self.neighbor_average(matrix, variables)
+        matrix = matrix.row_standardized
+        neighbor_sum(matrix, variables)
+      end
+
+      def self.neighbor_sum(matrix, variables)
+        matrix.dot(variables).to_a
+      end
+
+      def self.window_average(matrix, variables)
+        matrix = matrix.windowed.row_standardized
+        window_sum(matrix, variables)
+      end
+
+      def self.window_sum(matrix, variables)
+        matrix = matrix.windowed
+        matrix.dot(variables).to_a
+      end
+    end
+  end
+end

data/lib/spatial_stats/utils.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'spatial_stats/utils/lag'

data/lib/spatial_stats/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  VERSION = '0.1.0'
+end

data/lib/spatial_stats/weights/contiguous.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Weights
+    module Contiguous
+      def self.rook(scope, field)
+        p_key = scope.primary_key
+        keys = scope.pluck(p_key).sort
+
+        neighbors = SpatialStats::Queries::Weights
+                    .rook_contiguity_neighbors(scope, field)
+
+        neighbors = neighbors.group_by(&:i_id)
+        weights = neighbors.transform_values do |value|
+          value.map do |neighbor|
+            hash = neighbor.as_json(only: [:j_id]).symbolize_keys
+            hash[:weight] = 1
+            hash
+          end
+        end
+        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+      end
+
+      def self.queen(scope, field)
+        p_key = scope.primary_key
+        keys = scope.pluck(p_key).sort
+
+        neighbors = SpatialStats::Queries::Weights
+                    .queen_contiguity_neighbors(scope, field)
+
+        neighbors = neighbors.group_by(&:i_id)
+        weights = neighbors.transform_values do |value|
+          value.map do |neighbor|
+            hash = neighbor.as_json(only: [:j_id]).symbolize_keys
+            hash[:weight] = 1
+            hash
+          end
+        end
+        SpatialStats::Weights::WeightsMatrix.new(keys, weights)
+      end
+    end
+  end
+end