spatial_stats 0.2.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c754e131546cc4aa0bcb8ba0cadc2dc1230d5e6c08a29f142e3e3692ef8da02d
-  data.tar.gz: 05a6a60a79bc29f93586a53f6c60856b885e97331f34b3791f4ea76a1af8dd4a
+  metadata.gz: '08787d83d402843b9d711fd88cdc5f22cbdacfb7a960d8bdb185591f10215f85'
+  data.tar.gz: 96241b1ef7099ce6371f24dee56258121e0d6686ec746ac0411c4ed8ccaf5cad
 SHA512:
-  metadata.gz: 1ff203ee049060a41843cc463bcac36db59ab2b697dbef7a6a671fc70b316e66f5e65d1bb40d3a4a3604f68e9697330b26c505c3ca3304beacd5b2ced8b1b2be
-  data.tar.gz: dc66e81605de0b33871376858a097294d0f7f85a6332c8c35c1ff92b8af41e81a58a5a79e8c28589f2b25a34f783c642c89575fc1333404d13d17b6bf003ed12
+  metadata.gz: 4ec63da930afeae2b68e3f821e82c8c18f18b11ffa3296ad34e56706b54db11170e5319133d84e9440360e984776cdc5adbd6193d7de252f6f618136a431696b
+  data.tar.gz: 13964eb6a903e5ba0881b0ea2dfeab5e10fcfe51722d9e8a3342f2084200a60bc046d7375e29f651a5267d04a7b9b3a571f26b1ecf13ebf1c3fa2a3211ac6d8c

data/README.md

@@ -64,14 +64,14 @@ Example: Define WeightsMatrix and get the matrix in row_standardized format.
 
 ```ruby
 weights = {
-    1 => [{ j_id: 2, weight: 1 }, { j_id: 4, weight: 1 }],
-    2 => [{ j_id: 1, weight: 1 }],
-    3 => [{ j_id: 4, weight: 1 }],
-    4 => [{ j_id: 1, weight: 1 }, { j_id: 3, weight: 1 }]
+    1 => [{ id: 2, weight: 1 }, { id: 4, weight: 1 }],
+    2 => [{ id: 1, weight: 1 }],
+    3 => [{ id: 4, weight: 1 }],
+    4 => [{ id: 1, weight: 1 }, { id: 3, weight: 1 }]
 }
 keys = weights.keys
 wm = SpatialStats::Weights::WeightsMatrix.new(keys, weights)
-#  => #<SpatialStats::Weights::WeightsMatrix:0x0000561e205677c0 @keys=[1, 2, 3, 4], @weights={1=>[{:j_id=>2, :weight=>1}, {:j_id=>4, :weight=>1}], 2=>[{:j_id=>1, :weight=>1}], 3=>[{:j_id=>4, :weight=>1}], 4=>[{:j_id=>1, :weight=>1}, {:j_id=>3, :weight=>1}]}, @n=4>
+#  => #<SpatialStats::Weights::WeightsMatrix:0x0000561e205677c0 @keys=[1, 2, 3, 4], @weights={1=>[{:id=>2, :weight=>1}, {:id=>4, :weight=>1}], 2=>[{:id=>1, :weight=>1}], 3=>[{:id=>4, :weight=>1}], 4=>[{:id=>1, :weight=>1}, {:id=>3, :weight=>1}]}, @n=4>
 
 wm.standardized
 #  => Numo::DFloat#shape=[4,4]
@@ -118,6 +118,26 @@ moran.i
 # => 0.834
 ```
 
+#### Compute Moran's I without Querying Data
+
+To calculate the statistic by using an array of data and not querying a database field. The order of the data must correspond to the order of `weights.keys`.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+
+field = nil
+moran = SpatialStats::Global::Moran.new(scope, field, weights)
+# => <SpatialStats::Global::Moran>
+
+# important to standardize the data!
+data = [1,2,3,4,5,6].standardize
+moran.x = data
+
+moran.stat
+# => 0.521
+```
+
 #### Compute Moran's I Z-Score
 
 ```ruby
@@ -144,6 +164,20 @@ moran.mc(999, 123_456)
 # => 0.003
 ```
 
+#### Get Summary of Permutation Test
+
+All stat classes have the `summary` method which takes `permutations` and `seed` as its parameters. `summary` runs `stat` and `mc` then combines the results into a hash.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+moran = SpatialStats::Global::Moran.new(scope, :avg_income, weights)
+# => <SpatialStats::Global::Moran>
+
+moran.summary(999, 123_456)
+# => {stat: 0.834, p: 0.003}
+```
+
 ### Local Stats
 
 Local stats compute a value each observation in the dataset, like how similar its neighbors are to itself. Local stats operate similarly to global stats, except that almost every operation will return an array of length `n` where `n` is the number of observations in the dataset.
@@ -165,6 +199,26 @@ moran.i
 # => [0.888, 0.675, 0.2345, -0.987, -0.42, ...]
 ```
 
+#### Compute Moran's I without Querying Data
+
+To calculate the statistic by using an array of data and not querying a database field. The order of the data must correspond to the order of `weights.keys`.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+
+field = nil
+moran = SpatialStats::Local::Moran.new(scope, field, weights)
+# => <SpatialStats::Local::Moran>
+
+# important to standardize the data!
+data = [1,2,3,4,5,6].standardize
+moran.x = data
+
+moran.stat
+# => [0.521, 0.123, -0.432, -0.56,. ...]
+```
+
 #### Compute Moran's I Z-Scores
 
 Note: Many classes do not have a variance or expectation method implemented and this will raise a `NotImplementedError`.
@@ -193,6 +247,20 @@ moran.mc(999, 123_456)
 # => [0.24, 0.13, 0.53, 0.023, 0.65, ...]
 ```
 
+#### Get Summary of Permutation Test
+
+All stat classes have the `summary` method which takes `permutations` and `seed` as its parameters. `summary` runs `stat`, `mc`, and `groups` then combines the results into a hash array indexed by `weight.keys`.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+moran = SpatialStats::Local::Moran.new(scope, :avg_income, weights)
+# => <SpatialStats::Local::Moran>
+
+moran.summary(999, 123_456)
+# => [{key: 1, stat: 0.521, p: 0.24, group: 'HH'}, ...]
+```
+
 ## Contributing
 
 Once cloned, run the following commands to setup the test database.
@@ -240,16 +308,20 @@ RGeo::Geos.supported?
 - ~~Make star a parameter to getis-ord class~~
 - ~~Add examples/usage to docs~~
 - ~~Create RDocs~~
-- Refactor Global Moran and BVMoran
-- Support non-numeric keys in WeightsMatrix/General refactor
-- Write SparseMatrix C ext
+- ~~Refactor Global Moran and BVMoran~~
+- ~~Support non-numeric keys in WeightsMatrix/General refactor~~
+- ~~Write SparseMatrix C ext~~
+- ~~Change instances of `standardized` and `windowed` to `standardize` and `window`, respectively.~~
+- ~~Add `positive` and `negative` groups for `GetisOrd` and `Geary`, similar to how `#quads` is implemented.~~
+- ~~Add `#summary` method to statistics that will combine stat vals with p-vals, and quads or hot/cold spot info.~~
+- ~~Add ability to assign `x` or `z` on stat classes so users are not forced to query data to input it into models. Add example to README.~~
 
 ## Future Work
 
 #### General
 
 - ~~Refactor stats to inherit an abstract class.~~
-- Change WeightsMatrix class and Stat classes to utilize sparse matrix methods.
+- ~~Change WeightsMatrix class and Stat classes to utilize sparse matrix methods.~~
 - Split into two separate gems spatial_stats and spatial_stats-activerecord
 
 #### Weights
@@ -261,7 +333,7 @@ RGeo::Geos.supported?
 - Rate smoothing
 - Bayes smoothing
 
-### Global
+#### Global
 
 - Geary class
 - GetisOrd class
@@ -270,7 +342,7 @@ RGeo::Geos.supported?
 
 - Join Count Statistic
 
-### PPA
+#### PPA
 
 - Add descriptive stat methods for point clusters.
 

data/Rakefile

@@ -18,6 +18,12 @@ end
 
 require 'bundler/gem_tasks'
 
+require 'rake/extensiontask'
+
+Rake::ExtensionTask.new 'spatial_stats' do |ext|
+  ext.lib_dir = 'lib/spatial_stats'
+end
+
 require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |t|
@@ -27,4 +33,5 @@ Rake::TestTask.new(:test) do |t|
   t.warning = false # shut up annoying warnings
 end
 
+task test: :compile
 task default: :test

data/ext/spatial_stats/csr_matrix.c

@@ -0,0 +1,365 @@
+#include <ruby.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "csr_matrix.h"
+
+void csr_matrix_free(void *mat)
+{
+    csr_matrix *csr = (csr_matrix *)mat;
+
+    if (csr->init == 1)
+    {
+        free(csr->values);
+        free(csr->col_index);
+        free(csr->row_index);
+    }
+    free(mat);
+}
+
+size_t csr_matrix_memsize(const void *ptr)
+{
+    const csr_matrix *csr = (const csr_matrix *)ptr;
+    return sizeof(*csr);
+}
+
+VALUE csr_matrix_alloc(VALUE self)
+{
+    csr_matrix *csr = malloc(sizeof(csr_matrix));
+    return TypedData_Wrap_Struct(self, &csr_matrix_type, csr);
+}
+
+void mat_to_sparse(csr_matrix *csr, VALUE data, VALUE num_rows, VALUE num_cols)
+{
+    int nnz = 0;
+    int m = NUM2INT(num_rows);
+    int n = NUM2INT(num_cols);
+
+    double *values;
+    int *col_index;
+    int *row_index;
+
+    int nz_idx;
+    double entry;
+
+    int i;
+    int j;
+    int index;
+
+    // first get number non zero count so we can alloc values and col_index
+    for (i = 0; i < m; i++)
+    {
+        for (j = 0; j < n; j++)
+        {
+            index = i * n + j;
+            if (NUM2DBL(rb_ary_entry(data, index)) != 0)
+            {
+                nnz++;
+            }
+        }
+    }
+
+    values = malloc(sizeof(double) * nnz);
+    col_index = malloc(sizeof(int) * nnz);
+    row_index = malloc(sizeof(int) * (m + 1));
+
+    // for every non-zero, record value, column and then get values per row
+    nz_idx = 0;
+    for (i = 0; i < m; i++)
+    {
+        row_index[i] = nz_idx;
+        for (j = 0; j < n; j++)
+        {
+            index = i * n + j;
+            entry = NUM2DBL(rb_ary_entry(data, index));
+            if (entry != 0)
+            {
+                values[nz_idx] = entry;
+                col_index[nz_idx] = j;
+                nz_idx++;
+            }
+        }
+    }
+    row_index[m] = nnz;
+
+    csr->m = m;
+    csr->n = n;
+    csr->nnz = nnz;
+    csr->values = values;
+    csr->col_index = col_index;
+    csr->row_index = row_index;
+    csr->init = 1;
+}
+
+/**
+ *  A new instance of CSRMatrix.
+ *  Uses a 1-D representation of a 2-D array as input.
+ *  @example
+ *      dummy_data = [
+ *                      [0, 1, 2]
+ *                      [3, 4, 5],
+ *                      [6, 7, 8]   
+ *                   ]
+ *      num_rows = 3
+ *      num_cols = 3
+ *      data = dummy_data.flatten
+ *      # => [0, 1, 2, 3, 4, 5, 6, 7, 8]
+ *      
+ *      csr = CSRMatrix.new(data, num_rows, num_cols)
+ * 
+ *  @param [Array] data in 1-D format
+ *  @param [Integer] num_rows in the 2-D representation
+ *  @param [Integer] num_cols in the 2-D representation
+ *  
+ *  @return [CSRMatrix]
+ */
+VALUE csr_matrix_initialize(VALUE self, VALUE data, VALUE num_rows, VALUE num_cols)
+{
+
+    csr_matrix *csr;
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+    csr->init = 0;
+
+    Check_Type(data, T_ARRAY);
+    Check_Type(num_rows, T_FIXNUM);
+    Check_Type(num_cols, T_FIXNUM);
+
+    // check dimensions are correct
+    if (NUM2INT(num_rows) * NUM2INT(num_cols) != rb_array_len(data))
+    {
+        rb_raise(rb_eArgError, "n_rows * n_cols != data.size, check your dimensions");
+    }
+
+    mat_to_sparse(csr, data, num_rows, num_cols);
+
+    rb_iv_set(self, "@m", num_rows);
+    rb_iv_set(self, "@n", num_cols);
+    rb_iv_set(self, "@nnz", INT2NUM(csr->nnz));
+
+    return self;
+}
+
+/**
+ *  Non-zero values in the matrix.
+ *  
+ *  @return [Array] of the non-zero values.
+ */
+VALUE csr_matrix_values(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_ary_new_capa(csr->nnz);
+    for (i = 0; i < csr->nnz; i++)
+    {
+        rb_ary_store(result, i, DBL2NUM(csr->values[i]));
+    }
+
+    return result;
+}
+
+/**
+ *  Column indices of the non-zero values.
+ *  
+ *  @return [Array] of the column indices.
+ */
+VALUE csr_matrix_col_index(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_ary_new_capa(csr->nnz);
+    for (i = 0; i < csr->nnz; i++)
+    {
+        rb_ary_store(result, i, INT2NUM(csr->col_index[i]));
+    }
+
+    return result;
+}
+
+/**
+ *  Row indices of the non-zero values. Represents the start index
+ *  of values in a row. For example [0,2,3] would represent a matrix
+ *  with 2 rows, the first containing 2 non-zero values and the second
+ *  containing 1. Length is num_rows + 1.
+ * 
+ *  Used for row slicing operations.
+ *  
+ *  @return [Array] of the row indices.
+ */
+VALUE csr_matrix_row_index(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_ary_new_capa(csr->m + 1);
+    for (i = 0; i <= csr->m; i++)
+    {
+        rb_ary_store(result, i, INT2NUM(csr->row_index[i]));
+    }
+
+    return result;
+}
+
+/**
+ *  Multiply matrix by the input vector.
+ *  
+ *  @see https://github.com/scipy/scipy/blob/53fac7a1d8a81d48be757632ad285b6fc76529ba/scipy/sparse/sparsetools/csr.h#L1120
+ *  
+ *  @param [Array] vec of length n. 
+ * 
+ *  @return [Array] of the result of the multiplication.
+ */
+VALUE csr_matrix_mulvec(VALUE self, VALUE vec)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+    int jj;
+    double tmp;
+
+    Check_Type(vec, T_ARRAY);
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    if (rb_array_len(vec) != csr->n)
+    {
+        rb_raise(rb_eArgError, "Dimension Mismatch CSRMatrix.n != vec.size");
+    }
+
+    result = rb_ary_new_capa(csr->m);
+
+    // float *vals = (float *)DATA_PTR(result);
+
+    for (i = 0; i < csr->m; i++)
+    {
+        tmp = 0;
+        for (jj = csr->row_index[i]; jj < csr->row_index[i + 1]; jj++)
+        {
+            tmp += csr->values[jj] * NUM2DBL(rb_ary_entry(vec, csr->col_index[jj]));
+        }
+        rb_ary_store(result, i, DBL2NUM(tmp));
+    }
+
+    return result;
+}
+
+/**
+ *  Compute the dot product of the given row with the input vector.
+ *  Equivalent to +mulvec(vec)[row]+.
+ *  
+ *  @param [Array] vec of length n. 
+ *  @param [Integer] row of the dot product.
+ * 
+ *  @return [Float] of the result of the dot product.
+ */
+VALUE csr_matrix_dot_row(VALUE self, VALUE vec, VALUE row)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+    int jj;
+    double tmp;
+
+    Check_Type(vec, T_ARRAY);
+    Check_Type(row, T_FIXNUM);
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    if (rb_array_len(vec) != csr->n)
+    {
+        rb_raise(rb_eArgError, "Dimension Mismatch CSRMatrix.n != vec.size");
+    }
+
+    i = NUM2INT(row);
+    if (!(i >= 0 && i < csr->m))
+    {
+        rb_raise(rb_eArgError, "Index Error row_idx >= m or idx < 0");
+    }
+
+    tmp = 0;
+    for (jj = csr->row_index[i]; jj < csr->row_index[i + 1]; jj++)
+    {
+        tmp += csr->values[jj] * NUM2DBL(rb_ary_entry(vec, csr->col_index[jj]));
+    }
+
+    result = DBL2NUM(tmp);
+    return result;
+}
+
+/** 
+ *  A hash representation of the matrix with coordinates as keys.
+ *  @example
+ *      data = [
+ *              [0, 1, 0]
+ *              [0, 0, 0],
+ *              [1, 0, 1]   
+ *             ]
+ *      num_rows = 3
+ *      num_cols = 3
+ *      data = data.flatten!
+ *      csr = CSRMatrix.new(data, num_rows, num_cols)
+ *  
+ *      csr.coordinates
+ *      # => {
+ *              [0,1] => 1,
+ *              [2,0] => 1,
+ *              [2,2] => 1
+ *           }
+ *  
+ *  @return [Hash]
+ */
+VALUE csr_matrix_coordinates(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+    int k;
+
+    VALUE key;
+    VALUE val;
+    int row_end;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_hash_new();
+
+    // iterate through every value in the matrix and assign it's coordinates
+    // [x,y] as the key to the hash, with the value as the value.
+    // Use i to keep track of what row we are on.
+    i = 0;
+    row_end = csr->row_index[1];
+    for (k = 0; k < csr->nnz; k++)
+    {
+        if (k == row_end)
+        {
+            i++;
+            row_end = csr->row_index[i + 1];
+        }
+
+        // store i,j coordinates j is col_index[k]
+        key = rb_ary_new_capa(2);
+        rb_ary_store(key, 0, INT2NUM(i));
+        rb_ary_store(key, 1, INT2NUM(csr->col_index[k]));
+
+        val = DBL2NUM(csr->values[k]);
+
+        rb_hash_aset(result, key, val);
+    }
+
+    return result;
+}

data/ext/spatial_stats/csr_matrix.h

@@ -0,0 +1,35 @@
+#ifndef CSR_MATRIX
+#define CSR_MATRIX
+
+typedef struct csr_matrix
+{
+    char init;
+    int m;
+    int n;
+    int nnz;
+    double *values;
+    int *col_index;
+    int *row_index;
+} csr_matrix;
+
+void csr_matrix_free(void *mat);
+size_t csr_matrix_memsize(const void *ptr);
+
+// ruby VALUE for csr_matrix
+static const rb_data_type_t csr_matrix_type = {
+    "SpatialStats::Weights::CSRMatrix",
+    {NULL, csr_matrix_free, csr_matrix_memsize},
+    0,
+    0,
+    RUBY_TYPED_FREE_IMMEDIATELY};
+
+void mat_to_sparse(csr_matrix *csr, VALUE data, VALUE num_rows, VALUE num_cols);
+VALUE csr_matrix_alloc(VALUE self);
+VALUE csr_matrix_initialize(VALUE self, VALUE data, VALUE num_rows, VALUE num_cols);
+VALUE csr_matrix_values(VALUE self);
+VALUE csr_matrix_col_index(VALUE self);
+VALUE csr_matrix_row_index(VALUE self);
+VALUE csr_matrix_mulvec(VALUE self, VALUE vec);
+VALUE csr_matrix_dot_row(VALUE self, VALUE vec, VALUE row);
+VALUE csr_matrix_coordinates(VALUE self);
+#endif

data/ext/spatial_stats/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+
+create_header
+create_makefile 'spatial_stats/spatial_stats'

data/ext/spatial_stats/spatial_stats.c

@@ -0,0 +1,32 @@
+#include <ruby.h>
+#include "csr_matrix.h"
+
+/**
+ * Document-class: SpatialStats::Weights::CSRMatrix
+ * 
+ * CSRMatrix partially implements a compressed sparse row matrix to perform
+ * spatial lag and other calculations. This will generally be used 
+ * to store the weights of an observation set.
+ * 
+ * @see https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format) 
+ * 
+ */
+void Init_spatial_stats()
+{
+    VALUE spatial_stats_mod = rb_define_module("SpatialStats");
+    VALUE weights_mod = rb_define_module_under(spatial_stats_mod, "Weights");
+    VALUE csr_matrix_class = rb_define_class_under(weights_mod, "CSRMatrix", rb_cData);
+
+    rb_define_alloc_func(csr_matrix_class, csr_matrix_alloc);
+    rb_define_method(csr_matrix_class, "initialize", csr_matrix_initialize, 3);
+    rb_define_method(csr_matrix_class, "values", csr_matrix_values, 0);
+    rb_define_method(csr_matrix_class, "col_index", csr_matrix_col_index, 0);
+    rb_define_method(csr_matrix_class, "row_index", csr_matrix_row_index, 0);
+    rb_define_method(csr_matrix_class, "mulvec", csr_matrix_mulvec, 1);
+    rb_define_method(csr_matrix_class, "dot_row", csr_matrix_dot_row, 2);
+    rb_define_method(csr_matrix_class, "coordinates", csr_matrix_coordinates, 0);
+
+    rb_define_attr(csr_matrix_class, "m", 1, 0);
+    rb_define_attr(csr_matrix_class, "n", 1, 0);
+    rb_define_attr(csr_matrix_class, "nnz", 1, 0);
+}