spatial_stats 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 229947cadddad49771285a051c87a3797166fd4407a24fb27c9b2e3ccabe709c
+  data.tar.gz: a4e255eea9ceb986ae3078939cac4f7bc766fc3304801531095883091ceb1c79
+SHA512:
+  metadata.gz: d8dff7610402b8867f463d5e0c6a1b4d4ca00ce3fffc0047c2f470f71d71c4856e1534ead05ceaca3fef6d060a46e0e2151e7ac27d988625255bba3063f4bb35
+  data.tar.gz: ab930c04d462cc5ff9da4479078ee14ffdfe4e657430176d9d4583322077f87b433115b98edcfbed6268bd1a24204fdd13ef8808e0d31bfa485e9a3790a1d116

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2020 Keith Doggett
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# SpatialStats
+
+Short description and motivation.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'spatial_stats'
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install spatial_stats
+```
+
+## Usage
+
+How to use my plugin.
+
+## Contributing
+
+Once cloned, run the following commands to setup the test database.
+
+```sh
+cd ./spatial_stats
+bundle install
+cd test/dummy
+rake db:create
+rake db:migrate
+```
+
+If you are getting an error, you may need to set the following environment variables.
+
+```
+$PGUSER # default "postgres"
+$PGPASSWORD # default ""
+$PGHOST # default "127.0.0.1"
+$PGPORT # default "5432"
+$PGDATABASE # default "spatial_stats_test"
+```
+
+If the dummy app is setup correctly, run the following:
+
+```
+cd ../..
+rake
+```
+
+This will run the tests. If they all pass, then your environment is setup correctly.
+
+Note: It is recommended to have GEOS installed and linked to RGeo. You can test this by running the following:
+
+```
+cd test/dummy
+rails c
+
+RGeo::Geos.supported?
+# => true
+```
+
+## TODO
+
+- Memoize expensive functions within classes
+- Make star a parameter to getis-ord class
+
+## Future Work
+
+#### General
+
+- Refactor stats to inherit an abstract class.
+
+#### Weights
+
+- Add Kernel based weighting.
+
+#### Utils
+
+- Rate smoothing
+- Bayes smoothing
+
+#### Local
+
+- Join Count Statistic
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'SpatialStats'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+  t.warning = false # shut up annoying warnings
+end
+
+task default: :test

data/lib/spatial_stats/enumerable_ext.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Enumerable
+  def standardize
+    # standardize is (variable - mean)/stdev
+    m = mean
+    std = Math.sqrt(sample_variance)
+    map { |v| (v - m) / std }
+  end
+
+  def mean
+    sum / size.to_f
+  end
+
+  def sample_variance
+    m = mean
+    numerator = sum { |v| (v - m)**2 }
+    numerator / (size - 1).to_f
+  end
+end

data/lib/spatial_stats/global/bivariate_moran.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# https://geodacenter.github.io/workbook/5b_global_adv/lab5b.html
+module SpatialStats
+  module Global
+    class BivariateMoran < Stat
+      def initialize(scope, x_field, y_field, weights)
+        @scope = scope
+        @x_field = x_field
+        @y_field = y_field
+        @weights = weights
+      end
+      attr_writer :x, :y
+
+      def i
+        w = @weights.standardized
+        y_lag = SpatialStats::Utils::Lag.neighbor_sum(w, y)
+        numerator = 0
+        x.each_with_index do |xi, idx|
+          numerator += xi * y_lag[idx]
+        end
+
+        denominator = x.sum { |xi| xi**2 }
+        numerator / denominator
+      end
+
+      def expectation
+        -1.0 / (@weights.n - 1)
+      end
+
+      def variance
+        # https://en.wikipedia.org/wiki/Moran%27s_I#Expected_value
+        n = @weights.n
+        wij = @weights.full
+        w = wij.sum
+        e = expectation
+
+        s1 = s1_calc(n, wij)
+        s2 = s2_calc(n, wij)
+        s3 = s3_calc(n, x)
+
+        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w**2)
+        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w**2)
+
+        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w**2)
+        var_right = e**2
+        var_left - var_right
+      end
+
+      def mc(permutations = 99, seed = nil)
+        # call super monte carlo for multivariate
+        mc_bv(permutations, seed)
+      end
+
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @x_field)
+                                               .standardize
+      end
+
+      def y
+        @y ||= SpatialStats::Queries::Variables.query_field(@scope, @y_field)
+                                               .standardize
+      end
+
+      private
+
+      def s3_calc(n, zs)
+        numerator = (1.0 / n) * zs.sum { |v| v**4 }
+        denominator = ((1.0 / n) * zs.sum { |v| v**2 })**2
+        numerator / denominator
+      end
+
+      def s2_calc(n, wij)
+        s2 = 0
+        (0..n - 1).each do |i|
+          left_term = 0
+          right_term = 0
+          (0..n - 1).each do |j|
+            left_term += wij[i, j]
+            right_term += wij[j, i]
+          end
+          s2 += (left_term + right_term)**2
+        end
+        s2
+      end
+
+      def s1_calc(n, wij)
+        s1 = 0
+        (0..n - 1).each do |i|
+          (0..n - 1).each do |j|
+            s1 += (wij[i, j] + wij[j, i])**2
+          end
+        end
+        s1 / 2
+      end
+    end
+  end
+end

data/lib/spatial_stats/global/moran.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Global
+    class Moran < Stat
+      def initialize(scope, field, weights)
+        super(scope, field, weights)
+      end
+      attr_writer :x
+
+      def i
+        # compute's Moran's I. numerator is sum of zi * spatial lag of zi
+        # denominator is sum of zi**2.
+        # have to use row-standardized
+        @i ||= begin
+          w = @weights.standardized
+          z_lag = SpatialStats::Utils::Lag.neighbor_sum(w, z)
+          numerator = 0
+          z.each_with_index do |zi, j|
+            row_sum = zi * z_lag[j]
+            numerator += row_sum
+          end
+
+          denominator = z.sum { |zi| zi**2 }
+          numerator / denominator
+        end
+      end
+
+      def expectation
+        # -1/(n-1)
+        -1.0 / (@weights.n - 1)
+      end
+
+      def variance
+        # https://en.wikipedia.org/wiki/Moran%27s_I#Expected_value
+        n = @weights.n
+        wij = @weights.full
+        w = wij.sum
+        e = expectation
+
+        s1 = s1_calc(n, wij)
+        s2 = s2_calc(n, wij)
+        s3 = s3_calc(n, z)
+
+        s4 = (n**2 - 3 * n + 3) * s1 - n * s2 + 3 * (w**2)
+        s5 = (n**2 - n) * s1 - 2 * n * s2 + 6 * (w**2)
+
+        var_left = (n * s4 - s3 * s5) / ((n - 1) * (n - 2) * (n - 3) * w**2)
+        var_right = e**2
+        var_left - var_right
+      end
+
+      def mc(permutations = 99, seed = nil)
+        super(permutations, seed)
+      end
+
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
+      end
+
+      def zbar
+        x.sum / x.size
+      end
+
+      def z
+        x.map { |val| val - zbar }
+      end
+
+      private
+
+      def s3_calc(n, zs)
+        numerator = (1.0 / n) * zs.sum { |v| v**4 }
+        denominator = ((1.0 / n) * zs.sum { |v| v**2 })**2
+        numerator / denominator
+      end
+
+      def s2_calc(n, wij)
+        s2 = 0
+        (0..n - 1).each do |i|
+          left_term = 0
+          right_term = 0
+          (0..n - 1).each do |j|
+            left_term += wij[i, j]
+            right_term += wij[j, i]
+          end
+          s2 += (left_term + right_term)**2
+        end
+        s2
+      end
+
+      def s1_calc(n, wij)
+        s1 = 0
+        (0..n - 1).each do |i|
+          (0..n - 1).each do |j|
+            s1 += (wij[i, j] + wij[j, i])**2
+          end
+        end
+        s1 / 2
+      end
+    end
+  end
+end

data/lib/spatial_stats/global/stat.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Global
+    class Stat
+      # Base class for global stats
+      def initialize(scope, field, weights)
+        @scope = scope
+        @field = field
+        @weights = weights
+      end
+      attr_accessor :scope, :field, :weights
+
+      def i
+        raise NotImplementedError, 'method i not defined'
+      end
+
+      def expectation
+        raise NotImplementedError, 'method expectation not implemented'
+      end
+
+      def variance
+        raise NotImplementedError, 'method variance not implemented'
+      end
+
+      def z_score
+        (i - expectation) / Math.sqrt(variance)
+      end
+
+      def mc(permutations, seed)
+        rng = gen_rng(seed)
+        shuffles = []
+        permutations.times do
+          shuffles << x.shuffle(random: rng)
+        end
+        # r is the number of equal to or more extreme samples
+        # one sided
+        i_orig = i
+        r = 0
+        shuffles.each do |shuffle|
+          stat = self.class.new(@scope, @field, @weights)
+          stat.x = shuffle
+
+          # https://geodacenter.github.io/glossary.html#ppvalue
+          if i_orig.positive?
+            r += 1 if stat.i >= i_orig
+          else
+            r += 1 if stat.i <= i_orig
+          end
+        end
+
+        (r + 1.0) / (permutations + 1.0)
+      end
+
+      def mc_bv(permutations, seed)
+        # in multivariate, hold x and shuffle y
+        rng = gen_rng(seed)
+        shuffles = []
+        permutations.times do
+          shuffles << y.shuffle(random: rng)
+        end
+
+        # r is the number of equal to or more extreme samples
+        i_orig = i
+        r = 0
+        shuffles.each do |shuffle|
+          stat = self.class.new(@scope, @x_field, @y_field, @weights)
+          stat.x = x
+          stat.y = shuffle
+
+          if i_orig.positive?
+            r += 1 if stat.i >= i_orig
+          else
+            r += 1 if stat.i <= i_orig
+          end
+        end
+
+        (r + 1.0) / (permutations + 1.0)
+      end
+
+      private
+
+      def gen_rng(seed)
+        if seed
+          Random.new(seed)
+        else
+          Random.new
+        end
+      end
+    end
+  end
+end

data/lib/spatial_stats/global.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'spatial_stats/global/stat'
+require 'spatial_stats/global/bivariate_moran'
+require 'spatial_stats/global/moran'

data/lib/spatial_stats/local/bivariate_moran.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Local
+    class BivariateMoran < Stat
+      def initialize(scope, x_field, y_field, weights)
+        @scope = scope
+        @x_field = x_field
+        @y_field = y_field
+        @weights = weights
+      end
+      attr_accessor :scope, :x_field, :y_field, :weights
+      attr_writer :x, :y
+
+      def i
+        x.each_with_index.map do |_xi, idx|
+          i_i(idx)
+        end
+      end
+
+      def i_i(idx)
+        x[idx] * y_lag[idx]
+      end
+
+      def mc(permutations = 99, seed = nil)
+        mc_bv(permutations, seed)
+      end
+
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @x_field)
+                                               .standardize
+      end
+
+      def y
+        @y ||= SpatialStats::Queries::Variables.query_field(@scope, @y_field)
+                                               .standardize
+      end
+
+      private
+
+      def y_lag
+        @y_lag ||= SpatialStats::Utils::Lag.neighbor_sum(w, y)
+      end
+    end
+  end
+end

data/lib/spatial_stats/local/g.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Local
+    class G < Stat
+      def initialize(scope, field, weights, star = false)
+        super(scope, field, weights)
+        @star = star
+      end
+      attr_accessor :star
+      attr_writer :x, :z_lag
+
+      def i
+        x.each_with_index.map do |_x_val, idx|
+          i_i(idx)
+        end
+      end
+
+      def i_i(idx)
+        x_lag[idx] / denominators[idx]
+      end
+
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
+      end
+      alias z x
+
+      def star?
+        @star ||= weights.full.trace.positive?
+      end
+
+      private
+
+      def w
+        @w ||= begin
+          if star?
+            # TODO: try to fix this because it will still likely be a
+            # bottleneck in mc testing
+            weights.full.windowed.row_standardized
+          else
+            weights.standardized
+          end
+        end
+      end
+
+      def z_lag
+        # window if star is true
+        @z_lag ||= begin
+          if star?
+            SpatialStats::Utils::Lag.window_sum(w, x)
+          else
+            SpatialStats::Utils::Lag.neighbor_sum(w, x)
+          end
+        end
+      end
+      alias x_lag z_lag
+
+      def denominators
+        @denominators ||= begin
+          n = w.shape[0]
+          if star?
+            [x.sum] * n
+          else
+            # add everything but i
+            (0..n - 1).each.map do |idx|
+              terms = x.dup
+              terms.delete_at(idx)
+              terms.sum
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spatial_stats/local/geary.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module SpatialStats
+  module Local
+    class Geary < Stat
+      def initialize(scope, field, weights)
+        super(scope, field, weights)
+      end
+      attr_writer :x
+
+      def i
+        z.each_with_index.map do |_zi, idx|
+          i_i(idx)
+        end
+      end
+
+      def i_i(idx)
+        n = w.shape[0]
+        sum = 0
+        (0..n - 1).each do |j|
+          sum += w[idx, j] * ((z[idx] - z[j])**2)
+        end
+        sum
+      end
+
+      def x
+        @x ||= SpatialStats::Queries::Variables.query_field(@scope, @field)
+                                               .standardize
+      end
+      alias z x
+
+      def mc(permutations = 99, seed = nil)
+        # For local tests, we need to shuffle the values
+        # but for each item, hold its value in place and shuffle
+        # its neighbors. Then we will only test for that item instead
+        # of the entire set. This will be done for each item.
+        rng = gen_rng(seed)
+        shuffles = crand(x, permutations, rng)
+
+        # r is the number of equal to or more extreme samples
+        i_orig = i
+        rs = [0] * i_orig.size
+
+        shuffles.each_with_index do |perms, idx|
+          ii_orig = i_orig[idx]
+          perms.each do |perm|
+            stat = self.class.new(scope, field, weights)
+            stat.x = perm
+            ii_new = stat.i_i(idx)
+
+            # https://geodacenter.github.io/glossary.html#ppvalue
+            # NOTE: this is inconsistent with the output from GeoDa
+            # for local permutation tests, they seem to use greater than
+            # not greater than or equal to. I'm going to go by the definition
+            # in the glossary for now.
+            if ii_orig.positive?
+              rs[idx] += 1 if ii_new >= ii_orig
+            else
+              rs[idx] += 1 if ii_new <= ii_orig
+            end
+          end
+        end
+
+        rs.map do |ri|
+          (ri + 1.0) / (permutations + 1.0)
+        end
+      end
+
+      private
+
+      def w
+        @w ||= weights.full.row_standardized
+      end
+    end
+  end
+end