spatial_stats 0.2.2 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c754e131546cc4aa0bcb8ba0cadc2dc1230d5e6c08a29f142e3e3692ef8da02d
-  data.tar.gz: 05a6a60a79bc29f93586a53f6c60856b885e97331f34b3791f4ea76a1af8dd4a
+  metadata.gz: 053a1b234d54d2231e35c7eb74547b086f7415fc96190bc7d3afb25f51dd879e
+  data.tar.gz: a85d682009559812fb20695c5bea8f2978acf49a12d0b3df2506d58ced8c5e77
 SHA512:
-  metadata.gz: 1ff203ee049060a41843cc463bcac36db59ab2b697dbef7a6a671fc70b316e66f5e65d1bb40d3a4a3604f68e9697330b26c505c3ca3304beacd5b2ced8b1b2be
-  data.tar.gz: dc66e81605de0b33871376858a097294d0f7f85a6332c8c35c1ff92b8af41e81a58a5a79e8c28589f2b25a34f783c642c89575fc1333404d13d17b6bf003ed12
+  metadata.gz: 868ec3b50f4b1b9fc261d6dd94439bba62a08d4c496a48d164c41590d1cd5a088c87de682a7cb96bcf4dbe0d534b5cf35e136e39d42c505ba8cc1f4c3569a3d4
+  data.tar.gz: b767cc6ed7f22ce4096965a6172f77d21a348d51bd84dc3b96f0d51bf16e2f4f92617e21e9bcda1a36eeda2cd5c07c51452dd55d9e5a62567e206bf7a06c8fe0

data/README.md

@@ -1,8 +1,12 @@
+![Spatial Stats](/assets/ruby.svg)
+
 [![Build Status](https://travis-ci.com/keithdoggett/spatial_stats.svg?branch=master)](https://travis-ci.com/keithdoggett/spatial_stats)
 
+[Docs](https://keithdoggett.github.io/spatial_stats)
+
 # SpatialStats
 
-SpatialStats is an ActiveRecord plugin that utilizes PostGIS and Ruby to compute weights/statistics of spatial data sets in Rails Apps.
+SpatialStats is an ActiveRecord/Rails plugin that utilizes PostGIS to compute weights/statistics of spatial data sets in Rails Apps.
 
 ## Installation
 
@@ -58,42 +62,50 @@ weights = SpatialStats::Weights::Distant.idw_knn(scope, :geom, 5)
 
 Weight matrices can be defined by a hash that describes each key's neighbor and weight.
 
-Note: Currently, the keys must be numeric.
-
 Example: Define WeightsMatrix and get the matrix in row_standardized format.
 
 ```ruby
 weights = {
-    1 => [{ j_id: 2, weight: 1 }, { j_id: 4, weight: 1 }],
-    2 => [{ j_id: 1, weight: 1 }],
-    3 => [{ j_id: 4, weight: 1 }],
-    4 => [{ j_id: 1, weight: 1 }, { j_id: 3, weight: 1 }]
+    1 => [{ id: 2, weight: 1 }, { id: 4, weight: 1 }],
+    2 => [{ id: 1, weight: 1 }],
+    3 => [{ id: 4, weight: 1 }],
+    4 => [{ id: 1, weight: 1 }, { id: 3, weight: 1 }]
 }
 keys = weights.keys
-wm = SpatialStats::Weights::WeightsMatrix.new(keys, weights)
-#  => #<SpatialStats::Weights::WeightsMatrix:0x0000561e205677c0 @keys=[1, 2, 3, 4], @weights={1=>[{:j_id=>2, :weight=>1}, {:j_id=>4, :weight=>1}], 2=>[{:j_id=>1, :weight=>1}], 3=>[{:j_id=>4, :weight=>1}], 4=>[{:j_id=>1, :weight=>1}, {:j_id=>3, :weight=>1}]}, @n=4>
-
-wm.standardized
-#  => Numo::DFloat#shape=[4,4]
-#[[0, 0.5, 0, 0.5],
-# [1, 0, 0, 0],
-# [0, 0, 0, 1],
-# [0.5, 0, 0.5, 0]]
+wm = SpatialStats::Weights::WeightsMatrix.new(weights)
+#  => #<SpatialStats::Weights::WeightsMatrix:0x0000561e205677c0 @keys=[1, 2, 3, 4], @weights={1=>[{:id=>2, :weight=>1}, {:id=>4, :weight=>1}], 2=>[{:id=>1, :weight=>1}], 3=>[{:id=>4, :weight=>1}], 4=>[{:id=>1, :weight=>1}, {:id=>3, :weight=>1}]}, @n=4>
+
+wm = wm.standardize
+#  => #<SpatialStats::Weights::WeightsMatrix:0x0000561e205677c0 @keys=[1, 2, 3, 4], @weights={1=>[{:id=>2, :weight=>0.5}, {:id=>4, :weight=>0.5}], 2=>[{:id=>1, :weight=>1}], 3=>[{:id=>4, :weight=>1}], 4=>[{:id=>1, :weight=>0.5}, {:id=>3, :weight=>0.5}]}, @n=4>
+
+wm.dense
+# => Numo::DFloat[
+#    [0, 0.5, 0, 0.5],
+#    [1, 0, 0, 0],
+#    [0, 0, 0, 1],
+#    [0.5, 0, 0.5, 0]
+#   ]
+
+wm.sparse
+# => #<SpatialStats::Weights::CSRMatrix @m=4, @n=4, @nnz=6>
 ```
 
 ### Lagged Variables
 
-Spatially lagged variables can be computed with a 2-D n x n `Numo::NArray` and 1-D vector (`Array` or `Numo::NArray`).
+Spatially lagged variables can be computed with weights matrix and 1-D vector (`Array`).
 
 #### Compute a Lagged Variable
 
 ```ruby
-w = Numo::DFloat[[0, 0.5, 0, 0.5],
-                 [1, 0, 0, 0],
-                 [0, 0, 0, 1],
-                 [0.5, 0, 0.5, 0]]
+weights = {
+    1 => [{ id: 2, weight: 1 }, { id: 4, weight: 1 }],
+    2 => [{ id: 1, weight: 1 }],
+    3 => [{ id: 4, weight: 1 }],
+    4 => [{ id: 1, weight: 1 }, { id: 3, weight: 1 }]
+}
+wm = SpatialStats::Weights::WeightsMatrix.new(weights).standardize
 vec = [1, 2, 3, 4]
-lagged_var = SpatialStats::Utils::Lag.neighbor_sum(w, vec)
+lagged_var = SpatialStats::Utils::Lag.neighbor_sum(wm, vec)
 # => [3.0, 1.0, 4.0, 2.0]
 ```
 
@@ -118,6 +130,26 @@ moran.i
 # => 0.834
 ```
 
+#### Compute Moran's I without Querying Data
+
+To calculate the statistic by using an array of data and not querying a database field. The order of the data must correspond to the order of `weights.keys`.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+
+field = nil
+moran = SpatialStats::Global::Moran.new(scope, field, weights)
+# => <SpatialStats::Global::Moran>
+
+# data is automatically standardized on input
+data = [1,2,3,4,5,6]
+moran.x = data
+
+moran.stat
+# => 0.521
+```
+
 #### Compute Moran's I Z-Score
 
 ```ruby
@@ -144,6 +176,20 @@ moran.mc(999, 123_456)
 # => 0.003
 ```
 
+#### Get Summary of Permutation Test
+
+All stat classes have the `summary` method which takes `permutations` and `seed` as its parameters. `summary` runs `stat` and `mc` then combines the results into a hash.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+moran = SpatialStats::Global::Moran.new(scope, :avg_income, weights)
+# => <SpatialStats::Global::Moran>
+
+moran.summary(999, 123_456)
+# => {stat: 0.834, p: 0.003}
+```
+
 ### Local Stats
 
 Local stats compute a value each observation in the dataset, like how similar its neighbors are to itself. Local stats operate similarly to global stats, except that almost every operation will return an array of length `n` where `n` is the number of observations in the dataset.
@@ -165,6 +211,26 @@ moran.i
 # => [0.888, 0.675, 0.2345, -0.987, -0.42, ...]
 ```
 
+#### Compute Moran's I without Querying Data
+
+To calculate the statistic by using an array of data and not querying a database field. The order of the data must correspond to the order of `weights.keys`.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+
+field = nil
+moran = SpatialStats::Local::Moran.new(scope, field, weights)
+# => <SpatialStats::Local::Moran>
+
+# data is automatically standardized on input
+data = [1,2,3,4,5,6]
+moran.x = data
+
+moran.stat
+# => [0.521, 0.123, -0.432, -0.56,. ...]
+```
+
 #### Compute Moran's I Z-Scores
 
 Note: Many classes do not have a variance or expectation method implemented and this will raise a `NotImplementedError`.
@@ -193,6 +259,20 @@ moran.mc(999, 123_456)
 # => [0.24, 0.13, 0.53, 0.023, 0.65, ...]
 ```
 
+#### Get Summary of Permutation Test
+
+All stat classes have the `summary` method which takes `permutations` and `seed` as its parameters. `summary` runs `stat`, `mc`, and `groups` then combines the results into a hash array indexed by `weight.keys`.
+
+```ruby
+scope = County.all
+weights = SpatialStats::Weights::Contiguous.rook(scope, :geom)
+moran = SpatialStats::Local::Moran.new(scope, :avg_income, weights)
+# => <SpatialStats::Local::Moran>
+
+moran.summary(999, 123_456)
+# => [{key: 1, stat: 0.521, p: 0.24, group: 'HH'}, ...]
+```
+
 ## Contributing
 
 Once cloned, run the following commands to setup the test database.
@@ -234,46 +314,37 @@ RGeo::Geos.supported?
 # => true
 ```
 
-## TODO
-
-- ~~Memoize expensive functions within classes~~
-- ~~Make star a parameter to getis-ord class~~
-- ~~Add examples/usage to docs~~
-- ~~Create RDocs~~
-- Refactor Global Moran and BVMoran
-- Support non-numeric keys in WeightsMatrix/General refactor
-- Write SparseMatrix C ext
-
-## Future Work
-
-#### General
-
-- ~~Refactor stats to inherit an abstract class.~~
-- Change WeightsMatrix class and Stat classes to utilize sparse matrix methods.
-- Split into two separate gems spatial_stats and spatial_stats-activerecord
-
-#### Weights
-
-- Add Kernel based weighting
-
-#### Utils
+## Path Forward
 
-- Rate smoothing
-- Bayes smoothing
+Summaries of milestones for v1.x and v2.0. These lists are subject to change. If you have an additional feature you want to see for either milestone, open up an issue or PR.
 
-### Global
+### v1.x
 
-- Geary class
-- GetisOrd class
+1. Global Measurements
+   - `Geary`'s C
+   - `GetisOrd`
+2. Local Measurements
+   - `Join Count`
+3. Utilities
+   - Add support for .gal/.swm file imports
+   - Add support for Rate variables
+   - Add support for Bayes smoothing
+   - ~Add support for Bonferroni Bounds and FDR~
+4. General
+   - ~Add new stat constructors that only rely on a weights matrix and data vector~
+   - Point Pattern Analysis Module
+   - Regression Module
 
-#### Local
+### v2.0
 
-- Join Count Statistic
+- Break gem into core `spatial_stats` that will not include queries module and `spatial_stats-activerecord`. This will remove the dependency on rails for the core gem.
+- Create `spatial_stats-import/geojson/shp` gem that will allow importing files and generating a `WeightsMatrix`. Will likely rely on `RGeo` or another spatial lib.
 
-### PPA
+### Other TODOs
 
-- Add descriptive stat methods for point clusters.
+- Update Docs to show `from_observation` when version is bumped
+- Refactor `MultivariateGeary` so that it can be used without `activerecord` by adding `from_observations` and supporting methods.
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+The gem is available as open source under the terms of the [BSD-3-Clause](https://opensource.org/licenses/BSD-3-Clause).

data/Rakefile

@@ -18,6 +18,12 @@ end
 
 require 'bundler/gem_tasks'
 
+require 'rake/extensiontask'
+
+Rake::ExtensionTask.new 'spatial_stats' do |ext|
+  ext.lib_dir = 'lib/spatial_stats'
+end
+
 require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |t|
@@ -27,4 +33,5 @@ Rake::TestTask.new(:test) do |t|
   t.warning = false # shut up annoying warnings
 end
 
+task test: :compile
 task default: :test

data/ext/spatial_stats/csr_matrix.c

@@ -0,0 +1,380 @@
+#include <ruby.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "csr_matrix.h"
+
+void csr_matrix_free(void *mat)
+{
+    csr_matrix *csr = (csr_matrix *)mat;
+
+    if (csr->init == 1)
+    {
+        free(csr->values);
+        free(csr->col_index);
+        free(csr->row_index);
+    }
+    free(mat);
+}
+
+size_t csr_matrix_memsize(const void *ptr)
+{
+    const csr_matrix *csr = (const csr_matrix *)ptr;
+    return sizeof(*csr);
+}
+
+VALUE csr_matrix_alloc(VALUE self)
+{
+    csr_matrix *csr = malloc(sizeof(csr_matrix));
+    return TypedData_Wrap_Struct(self, &csr_matrix_type, csr);
+}
+
+void mat_to_sparse(csr_matrix *csr, VALUE data, VALUE keys, VALUE num_rows)
+{
+
+    int nnz = 0;
+    int n = NUM2INT(num_rows);
+    int m;
+
+    VALUE key;
+    VALUE row;
+    VALUE entry;
+    VALUE key_lookup = rb_hash_new();
+    VALUE weight_sym = ID2SYM(rb_intern("weight"));
+    VALUE id_sym = ID2SYM(rb_intern("id"));
+
+    double *values;
+    int *col_index;
+    int *row_index;
+
+    int nz_idx;
+    double weight;
+
+    int i;
+    int j;
+
+    // first get number non zero count so we can alloc values and col_index
+    for (i = 0; i < n; i++)
+    {
+        key = rb_ary_entry(keys, i);
+
+        // set lookup index for this key
+        rb_hash_aset(key_lookup, key, INT2NUM(i));
+
+        // check the value of this row is actually an array
+        // if it is, add array len to nnz
+        row = rb_hash_aref(data, key);
+        Check_Type(row, T_ARRAY);
+        nnz += rb_array_len(row);
+    }
+
+    values = malloc(sizeof(double) * nnz);
+    col_index = malloc(sizeof(int) * nnz);
+    row_index = malloc(sizeof(int) * (n + 1));
+
+    // for every row, work through each hash
+    // in each hash, add the weight to values and get col_index
+    // by looking at the key_lookup of id.
+    // Row index will be computed by adding len of each row and updating array.
+    nz_idx = 0;
+    for (i = 0; i < n; i++)
+    {
+        row_index[i] = nz_idx;
+
+        key = rb_ary_entry(keys, i);
+        row = rb_hash_aref(data, key);
+        m = rb_array_len(row);
+
+        for (j = 0; j < m; j++)
+        {
+            entry = rb_ary_entry(row, j);
+            Check_Type(entry, T_HASH);
+
+            key = rb_hash_aref(entry, id_sym);
+            weight = NUM2DBL(rb_hash_aref(entry, weight_sym));
+
+            // assign the nnz the weight
+            // get index in the keys array of key from lookup table
+            values[nz_idx] = weight;
+            col_index[nz_idx] = NUM2INT(rb_hash_aref(key_lookup, key));
+            nz_idx++;
+        }
+    }
+    row_index[n] = nnz;
+
+    csr->n = n;
+    csr->nnz = nnz;
+    csr->values = values;
+    csr->col_index = col_index;
+    csr->row_index = row_index;
+    csr->init = 1;
+}
+
+/**
+ *  A new instance of CSRMatrix.
+ *  Uses a Dictionary of Keys (DOK) as input to represent a square matrix.
+ *  @example
+ *      weights = {
+ *          'a' => [{ id: 'c', weight: 1 }],
+ *          'b' => [{ id: 'b', weight: 1 }],
+ *          'c' => [{ id: 'a', weight: 1 }]
+ *      }
+ *      num_rows = 3
+ *      
+ *      csr = CSRMatrix.new(data, num_rows)
+ * 
+ *  @param [Array] data in 1-D format
+ *  @param [Integer] num_rows in the 2-D representation
+ *  
+ *  @return [CSRMatrix]
+ */
+VALUE csr_matrix_initialize(VALUE self, VALUE data, VALUE num_rows)
+{
+    VALUE keys;
+    csr_matrix *csr;
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+    csr->init = 0;
+
+    Check_Type(data, T_HASH);
+    Check_Type(num_rows, T_FIXNUM);
+
+    keys = rb_funcall(data, rb_intern("keys"), 0);
+
+    // check dimensions are correct
+    if (NUM2INT(num_rows) != rb_array_len(keys))
+    {
+        rb_raise(rb_eArgError, "n_rows != keys.size, check your dimensions");
+    }
+
+    mat_to_sparse(csr, data, keys, num_rows);
+
+    rb_iv_set(self, "@n", num_rows);
+    rb_iv_set(self, "@nnz", INT2NUM(csr->nnz));
+
+    return self;
+}
+
+/**
+ *  Non-zero values in the matrix.
+ *  
+ *  @return [Array] of the non-zero values.
+ */
+VALUE csr_matrix_values(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_ary_new_capa(csr->nnz);
+    for (i = 0; i < csr->nnz; i++)
+    {
+        rb_ary_store(result, i, DBL2NUM(csr->values[i]));
+    }
+
+    return result;
+}
+
+/**
+ *  Column indices of the non-zero values.
+ *  
+ *  @return [Array] of the column indices.
+ */
+VALUE csr_matrix_col_index(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_ary_new_capa(csr->nnz);
+    for (i = 0; i < csr->nnz; i++)
+    {
+        rb_ary_store(result, i, INT2NUM(csr->col_index[i]));
+    }
+
+    return result;
+}
+
+/**
+ *  Row indices of the non-zero values. Represents the start index
+ *  of values in a row. For example [0,2,3] would represent a matrix
+ *  with 2 rows, the first containing 2 non-zero values and the second
+ *  containing 1. Length is num_rows + 1.
+ * 
+ *  Used for row slicing operations.
+ *  
+ *  @return [Array] of the row indices.
+ */
+VALUE csr_matrix_row_index(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_ary_new_capa(csr->n + 1);
+    for (i = 0; i <= csr->n; i++)
+    {
+        rb_ary_store(result, i, INT2NUM(csr->row_index[i]));
+    }
+
+    return result;
+}
+
+/**
+ *  Multiply matrix by the input vector.
+ *  
+ *  @see https://github.com/scipy/scipy/blob/53fac7a1d8a81d48be757632ad285b6fc76529ba/scipy/sparse/sparsetools/csr.h#L1120
+ *  
+ *  @param [Array] vec of length n. 
+ * 
+ *  @return [Array] of the result of the multiplication.
+ */
+VALUE csr_matrix_mulvec(VALUE self, VALUE vec)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+    int jj;
+    double tmp;
+
+    Check_Type(vec, T_ARRAY);
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    if (rb_array_len(vec) != csr->n)
+    {
+        rb_raise(rb_eArgError, "Dimension Mismatch CSRMatrix.n != vec.size");
+    }
+
+    result = rb_ary_new_capa(csr->n);
+
+    // float *vals = (float *)DATA_PTR(result);
+
+    for (i = 0; i < csr->n; i++)
+    {
+        tmp = 0;
+        for (jj = csr->row_index[i]; jj < csr->row_index[i + 1]; jj++)
+        {
+            tmp += csr->values[jj] * NUM2DBL(rb_ary_entry(vec, csr->col_index[jj]));
+        }
+        rb_ary_store(result, i, DBL2NUM(tmp));
+    }
+
+    return result;
+}
+
+/**
+ *  Compute the dot product of the given row with the input vector.
+ *  Equivalent to +mulvec(vec)[row]+.
+ *  
+ *  @param [Array] vec of length n. 
+ *  @param [Integer] row of the dot product.
+ * 
+ *  @return [Float] of the result of the dot product.
+ */
+VALUE csr_matrix_dot_row(VALUE self, VALUE vec, VALUE row)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+    int jj;
+    double tmp;
+
+    Check_Type(vec, T_ARRAY);
+    Check_Type(row, T_FIXNUM);
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    if (rb_array_len(vec) != csr->n)
+    {
+        rb_raise(rb_eArgError, "Dimension Mismatch CSRMatrix.n != vec.size");
+    }
+
+    i = NUM2INT(row);
+    if (!(i >= 0 && i < csr->n))
+    {
+        rb_raise(rb_eArgError, "Index Error row_idx >= m or idx < 0");
+    }
+
+    tmp = 0;
+    for (jj = csr->row_index[i]; jj < csr->row_index[i + 1]; jj++)
+    {
+        tmp += csr->values[jj] * NUM2DBL(rb_ary_entry(vec, csr->col_index[jj]));
+    }
+
+    result = DBL2NUM(tmp);
+    return result;
+}
+
+/** 
+ *  A hash representation of the matrix with coordinates as keys.
+ *  @example
+ *      data = [
+ *              [0, 1, 0]
+ *              [0, 0, 0],
+ *              [1, 0, 1]   
+ *             ]
+ *      num_rows = 3
+ *      num_cols = 3
+ *      data = data.flatten!
+ *      csr = CSRMatrix.new(data, num_rows, num_cols)
+ *  
+ *      csr.coordinates
+ *      # => {
+ *              [0,1] => 1,
+ *              [2,0] => 1,
+ *              [2,2] => 1
+ *           }
+ *  
+ *  @return [Hash]
+ */
+VALUE csr_matrix_coordinates(VALUE self)
+{
+    csr_matrix *csr;
+    VALUE result;
+
+    int i;
+    int k;
+
+    VALUE key;
+    VALUE val;
+    int row_end;
+
+    TypedData_Get_Struct(self, csr_matrix, &csr_matrix_type, csr);
+
+    result = rb_hash_new();
+
+    // iterate through every value in the matrix and assign it's coordinates
+    // [x,y] as the key to the hash, with the value as the value.
+    // Use i to keep track of what row we are on.
+    i = 0;
+    row_end = csr->row_index[1];
+    for (k = 0; k < csr->nnz; k++)
+    {
+        if (k == row_end)
+        {
+            i++;
+            row_end = csr->row_index[i + 1];
+        }
+
+        // store i,j coordinates j is col_index[k]
+        key = rb_ary_new_capa(2);
+        rb_ary_store(key, 0, INT2NUM(i));
+        rb_ary_store(key, 1, INT2NUM(csr->col_index[k]));
+
+        val = DBL2NUM(csr->values[k]);
+
+        rb_hash_aset(result, key, val);
+    }
+
+    return result;
+}