spatial_features 2.14.0 → 2.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0aa9e8be0d78cabc65695247e3000bcc2d69ce75aeffe1333b0dd5ce423eb562
-  data.tar.gz: 587dc7a4be41c38acd66de43864ed3b8b51f22186996fcdd2c906c01ee97102e
+  metadata.gz: e0bc8fc084675d82dfe1414f599b32d22b3b4435525246580c7cd2af1f5e3afc
+  data.tar.gz: 78b7e7fa4b2e3fe9f46931c4b5a0bcf14c3f518a9e813e48fcb3985ac1bd171f
 SHA512:
-  metadata.gz: 75293b351a8bfa078b1eac5c02a7d5dcce3e6eaa15a1779e660d0ca1c8a1cf93be2ad30190946e8934f51301c8535cfda4a75d0af2d86d051ffdd79646f170eb
-  data.tar.gz: a725a65315d15a5885bd475dc6575969823cac93bc865f4dca3f583e602342a7b40a1afef2478264e74f29a79d44be8fb3528648144b10fe9d4c0a3d8f5858ab
+  metadata.gz: 1fe304ec107f4d9ca8537a4a665c118bb20a1365b2bee3a58d88bd2ccc65758a4906a812a09632ad779e2df60b715c100967e62bfb97e23c012e96a7d031f958
+  data.tar.gz: ab611750f2a507937f0f3a62b120c619b7e2b3a9fda837657eeb844bd0e02911d3fd5419a779f60ed0d20bdee6b387e4fb2b9b51e814216b1a0a82bce11852b5

data/app/models/abstract_feature.rb

@@ -1,6 +1,9 @@
 class AbstractFeature < ActiveRecord::Base
   self.table_name = 'features'
 
+  class_attribute :automatically_cache_derivatives
+  self.automatically_cache_derivatives = true
+
   class_attribute :lowres_simplification
   self.lowres_simplification = 2 # Threshold in meters
 
@@ -12,15 +15,21 @@ class AbstractFeature < ActiveRecord::Base
 
   before_validation :sanitize_feature_type
   validates_presence_of :geog
-  validate :validate_geometry
-  before_save :sanitize
-  after_save :cache_derivatives, :if => :saved_change_to_geog?
+  validate :validate_geometry, if: :will_save_change_to_geog?
+  before_save :sanitize, if: :will_save_change_to_geog?
+  after_save :cache_derivatives, :if => [:automatically_cache_derivatives?, :saved_change_to_geog?]
 
   def self.cache_key
     result = connection.select_one(all.select('max(id) AS max, count(*) AS count').to_sql)
     "#{result['max']}-#{result['count']}"
   end
 
+  # for Rails >= 5 ActiveRecord collections we override the collection_cache_key
+  # to prevent Rails doing its default query on `updated_at`
+  def self.collection_cache_key(_collection, _timestamp_column)
+    self.cache_key
+  end
+
   def self.with_metadata(k, v)
     if k.present? && v.present?
       where('metadata->? = ?', k, v)
@@ -82,6 +91,14 @@ class AbstractFeature < ActiveRecord::Base
     return envelope_json.values_at(0,2)
   end
 
+  def self.without_caching_derivatives(&block)
+    old = automatically_cache_derivatives
+    self.automatically_cache_derivatives = false
+    block.call
+  ensure
+    self.automatically_cache_derivatives = old
+  end
+
   def self.cache_derivatives(options = {})
     update_all <<-SQL.squish
       geom         = ST_Transform(geog::geometry, #{detect_srid('geom')}),
@@ -106,10 +123,17 @@ class AbstractFeature < ActiveRecord::Base
     SQL
   end
 
-  def self.geojson(lowres: false, precision: 6, properties: {}, srid: 4326) # default srid is 4326 so output is Google Maps compatible
-    column = lowres ? "ST_Transform(geom_lowres, #{srid})" : 'geog'
-    properties_sql = <<~SQL if properties.present?
-      , 'properties', json_build_object(#{properties.map {|k,v| "'#{k}',#{v}" }.join(',') })
+  def self.geojson(lowres: false, precision: 6, properties: true, srid: 4326, centroids: false) # default srid is 4326 so output is Google Maps compatible
+    if centroids
+      column = 'centroid'
+    elsif lowres
+      column = "ST_Transform(geom_lowres, #{srid})"
+    else
+      column = 'geog'
+    end
+
+    properties_sql = <<~SQL if properties
+      , 'properties', hstore_to_json(metadata)
     SQL
 
     sql = <<~SQL
@@ -132,7 +156,7 @@ class AbstractFeature < ActiveRecord::Base
   end
 
   def cache_derivatives(*args)
-    self.class.where(:id => self.id).cache_derivatives(*args)
+    self.class.default_scoped.where(:id => self.id).cache_derivatives(*args)
   end
 
   def kml(options = {})

data/app/models/feature.rb

@@ -11,6 +11,16 @@ class Feature < AbstractFeature
 
   after_save :refresh_aggregate, if: :automatically_refresh_aggregate?
 
+  # Features are used for display so we also cache their KML representation
+  def self.cache_derivatives(options = {})
+    super
+    update_all <<-SQL.squish
+      kml          = ST_AsKML(geog, 6),
+      kml_lowres   = ST_AsKML(geom_lowres, #{options.fetch(:lowres_precision, lowres_precision)}),
+      kml_centroid = ST_AsKML(centroid)
+    SQL
+  end
+
   def self.defer_aggregate_refresh(&block)
     start_at = Feature.maximum(:id).to_i + 1
     output = without_aggregate_refresh(&block)
@@ -30,9 +40,10 @@ class Feature < AbstractFeature
 
   def self.refresh_aggregates
     # Find one feature from each spatial model and trigger the aggregate feature refresh
-    ids = select('MAX(id)')
-            .where.not(:spatial_model_type => nil, :spatial_model_id => nil)
+    ids = where.not(:spatial_model_type => nil)
+            .where.not(:spatial_model_id => nil)
             .group('spatial_model_type, spatial_model_id')
+            .pluck('MAX(id)')
 
     # Unscope so that newly built AggregateFeatures get their type column set correctly
     AbstractFeature.unscoped { where(:id => ids).find_each(&:refresh_aggregate) }
@@ -49,14 +60,4 @@ class Feature < AbstractFeature
     # this field blank.
     spatial_model_id? && automatically_refresh_aggregate && saved_change_to_geog?
   end
-
-  # Features are used for display so we also cache their KML representation
-  def self.cache_derivatives(options = {})
-    super
-    update_all <<-SQL.squish
-      kml          = ST_AsKML(geog, 6),
-      kml_lowres   = ST_AsKML(geom_lowres, #{options.fetch(:lowres_precision, lowres_precision)}),
-      kml_centroid = ST_AsKML(centroid)
-    SQL
-  end
 end

data/lib/spatial_features.rb

@@ -12,6 +12,7 @@ require 'spatial_features/controller_helpers/spatial_extensions'
 require 'spatial_features/download'
 require 'spatial_features/unzip'
 require 'spatial_features/utils'
+require 'spatial_features/validation'
 
 require 'spatial_features/has_spatial_features'
 require 'spatial_features/has_spatial_features/queued_spatial_processing'

data/lib/spatial_features/controller_helpers/spatial_extensions.rb

@@ -6,8 +6,6 @@ module SpatialExtensions
       model.failed_feature_update_jobs.destroy_all
       model.delay_update_features!
     end
-
-    redirect_to :back
   end
 
   def abstract_proximity_action(scope, target, distance, &block)

data/lib/spatial_features/download.rb

@@ -12,7 +12,7 @@ module SpatialFeatures
     def self.open(file, unzip: nil, **unzip_options)
       file = Kernel.open(file)
       file = normalize_file(file) if file.is_a?(StringIO)
-      if Unzip.is_zip?(file)
+      if unzip && Unzip.is_zip?(file)
         file = find_in_zip(file, find: unzip, **unzip_options)
       end
       return file
@@ -26,6 +26,12 @@ module SpatialFeatures
       end
     end
 
+    def self.entries(file)
+      file = Kernel.open(file)
+      file = normalize_file(file) if file.is_a?(StringIO)
+      Unzip.entries(file)
+    end
+
     def self.find_in_zip(file, find:, **unzip_options)
       return File.open(Unzip.paths(file, :find => find, **unzip_options))
     end

data/lib/spatial_features/has_spatial_features.rb

@@ -88,16 +88,16 @@ module SpatialFeatures
 
     # Returns true if the model stores a hash of the features so we don't need to process the features if they haven't changed
     def has_spatial_features_hash?
-      column_names.include? 'features_hash'
+      owner_class_has_loaded_column?('features_hash')
     end
 
     # Returns true if the model stores a cache of the features area
     def has_features_area?
-      column_names.include? 'features_area'
+      owner_class_has_loaded_column?('features_area')
     end
 
     def area_in_square_meters
-      features.area_in_square_meters
+      features.area
     end
 
     private
@@ -160,6 +160,12 @@ module SpatialFeatures
       scope = scope.where(:spatial_model_id => other) unless Utils.class_of(other) == other
       return scope
     end
+
+    def owner_class_has_loaded_column?(column_name)
+      return false unless connected?
+      return false unless table_exists?
+      column_names.include? column_name
+    end
   end
 
   module InstanceMethods
@@ -202,7 +208,7 @@ module SpatialFeatures
     end
 
     def features_area_in_square_meters
-      @features_area_in_square_meters ||= features.area
+      aggregate_feature&.area
     end
 
     def total_intersection_area_in_square_meters(other)

data/lib/spatial_features/has_spatial_features/feature_import.rb

@@ -79,14 +79,24 @@ module SpatialFeatures
     end
 
     def import_features(imports, skip_invalid)
-      self.features.delete_all
+      features.delete_all
       valid, invalid = Feature.defer_aggregate_refresh do
-        imports.flat_map(&:features).partition do |feature|
-          feature.spatial_model = self
-          feature.save
+        Feature.without_caching_derivatives do
+          imports.flat_map(&:features).partition do |feature|
+            feature.spatial_model = self
+            feature.save
+          end
         end
       end
 
+      if persisted?
+        features.reset # Reset the association cache because we've updated the features
+        features.cache_derivatives
+      else
+        self.features = valid # Assign the features so when we save this record we update the foreign key on the features
+        Feature.where(id: features).cache_derivatives
+      end
+
       errors = imports.flat_map(&:errors)
       invalid.each do |feature|
         errors << "Feature #{feature.name}: #{feature.errors.full_messages.to_sentence}"
@@ -98,7 +108,7 @@ module SpatialFeatures
         raise ImportError, "Error updating #{self.class} #{self.id}. #{errors.to_sentence}"
       end
 
-      self.features = valid
+      return features
     end
 
     def features_cache_key_matches?(cache_key)

data/lib/spatial_features/importers/shapefile.rb

@@ -12,7 +12,7 @@ module SpatialFeatures
       end
 
       def cache_key
-        @cache_key ||= Digest::MD5.hexdigest(features.to_json)
+        @cache_key ||= Digest::MD5.file(archive).to_s
       end
 
       private
@@ -49,13 +49,27 @@ module SpatialFeatures
         SQL
       end
 
+
       def file
-        @file ||= Download.open(@data, unzip: /\.shp$/, downcase: true)
+        @file ||= begin
+          validate_file!
+          Download.open(archive, unzip: /\.shp$/, downcase: true)
+        end
+      end
+
+      def validate_file!
+        return unless Unzip.is_zip?(archive)
+        Validation.validate_shapefile_archive!(Download.entries(archive), default_proj4_projection: default_proj4_projection)
+      end
+
+      def archive
+        @archive ||= Download.open(@data)
       end
     end
 
     # ERRORS
     class IndeterminateShapefileProjection < SpatialFeatures::ImportError; end
     class IncompleteShapefileArchive < SpatialFeatures::ImportError; end
+    class InvalidShapefileArchive < SpatialFeatures::ImportError; end
   end
 end

data/lib/spatial_features/validation.rb

@@ -0,0 +1,55 @@
+module SpatialFeatures
+  module Validation
+    # SHP file must come first
+    REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS = %w[shp shx dbf prj].freeze
+
+    # Check if a shapefile archive includes the required component files, otherwise
+    # raise an exception.
+    #
+    # @param [Zip::File] zip_file                 A Zip::File object
+    # @param [String] default_proj4_projection    Optional, if supplied we don't raise an exception when we're missing a .PRJ file
+    # @param [Boolean] allow_generic_zip_files    When true, we skip validation entirely if the archive does not contain a .SHP file
+    def self.validate_shapefile_archive!(zip_file, default_proj4_projection: nil, allow_generic_zip_files: false)
+      zip_file_entries = zip_file.entries.each_with_object({}) do |f, obj|
+        ext = File.extname(f.name).downcase[1..-1]
+        next unless ext
+
+        if ext.casecmp?("shp") && obj.key?(ext)
+          raise ::SpatialFeatures::Importers::InvalidShapefileArchive, "Zip files that contain multiple Shapefiles are not supported. Please separate each Shapefile into its own zip file."
+        end
+
+        obj[ext] = File.basename(f.name, '.*')
+      end
+
+      shapefile_basename = zip_file_entries["shp"]
+      unless shapefile_basename
+        # not a shapefile archive but we don't care
+        return if allow_generic_zip_files
+
+        raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a SHP file"
+      end
+
+      REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS[1..-1].each do |ext|
+        ext_basename = zip_file_entries[ext]
+        next if ext_basename&.casecmp?(shapefile_basename)
+
+        case ext
+        when "prj"
+          # special case for missing projection files to allow using default_proj4_projection
+          next if default_proj4_projection
+
+          raise ::SpatialFeatures::Importers::IndeterminateShapefileProjection, "Shapefile archive is missing a projection file: #{expected_component_path(shapefile_basename, ext)}"
+        else
+          # for all un-handled cases of missing files raise the more generic error
+          raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a required file: #{expected_component_path(shapefile_basename, ext)}"
+        end
+      end
+
+      true
+    end
+
+    def self.expected_component_path(basename, ext)
+      "#{basename}.#{ext}"
+    end
+  end
+end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "2.14.0"
+  VERSION = "2.16.0"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 2.14.0
+  version: 2.16.0
 platform: ruby
 authors:
 - Ryan Wallace
 - Nicholas Jakobsen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-17 00:00:00.000000000 Z
+date: 2021-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -20,7 +20,7 @@ dependencies:
         version: '4.2'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
         version: '4.2'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: delayed_job_active_record
   requirement: !ruby/object:Gem::Requirement
@@ -63,16 +63,16 @@ dependencies:
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -107,14 +107,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -166,6 +166,7 @@ files:
 - lib/spatial_features/uncached_result.rb
 - lib/spatial_features/unzip.rb
 - lib/spatial_features/utils.rb
+- lib/spatial_features/validation.rb
 - lib/spatial_features/venn_polygons.rb
 - lib/spatial_features/version.rb
 - lib/tasks/spatial_features_tasks.rake
@@ -173,7 +174,7 @@ homepage: https://github.com/culturecode/spatial_features
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -188,8 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Adds spatial methods to a model.
 test_files: []