spatial_features 2.14.1 → 2.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f96f62ce16dc4da95bc8a7f236461c6a50f45ab0c68e4dcf1a7842d944fc2452
-  data.tar.gz: b76f8341b8014bbe2072caa6a186116d820285879a4704f11622ae0e7c8f6bf7
+  metadata.gz: 79fb559903943c11893b5d338f5c8030971f23051cf3611a326d0e013720f5e5
+  data.tar.gz: 1311241026ec5dc76a90e04f8fcc9e93c92965875f0967e1b27a0e1f64694ab3
 SHA512:
-  metadata.gz: f6f032f1db3ce25b0c30f9db6483e2ccfa39be32509fd5967fb75a8b88db54e61b5903e3b74fac7ef420e2fe1127687dac821295d29c13ff8b4c95106ce98bc2
-  data.tar.gz: 0dd720b90186edecbb1a81b9112796d3738154f5b12207a46e1db532a23ca9a61c3e919042c88e27cdb70453669f60f3e4d1bb17af7567a86d858a08a0bc9bed
+  metadata.gz: 8080e32e08c262fb3c44c6a228b0c599a1030ec52f9daea7862a06a98aa72b46dad77fb73127d5d40e6e999549ea44ebde0b77e826b1776e92eefe6b4a14898c
+  data.tar.gz: 3d591754fdc5a1ddb92146d805074aa5dc4bc65f5261eb0c169546f886b5dee784de7f24f56829a359641ae778960ffd54923c6b2535261d2e1194e0c9db0859

data/app/models/abstract_feature.rb

@@ -1,6 +1,9 @@
 class AbstractFeature < ActiveRecord::Base
   self.table_name = 'features'
 
+  class_attribute :automatically_cache_derivatives
+  self.automatically_cache_derivatives = true
+
   class_attribute :lowres_simplification
   self.lowres_simplification = 2 # Threshold in meters
 
@@ -12,15 +15,21 @@ class AbstractFeature < ActiveRecord::Base
 
   before_validation :sanitize_feature_type
   validates_presence_of :geog
-  validate :validate_geometry
-  before_save :sanitize
-  after_save :cache_derivatives, :if => :saved_change_to_geog?
+  validate :validate_geometry, if: :will_save_change_to_geog?
+  before_save :sanitize, if: :will_save_change_to_geog?
+  after_save :cache_derivatives, :if => [:automatically_cache_derivatives?, :saved_change_to_geog?]
 
   def self.cache_key
     result = connection.select_one(all.select('max(id) AS max, count(*) AS count').to_sql)
     "#{result['max']}-#{result['count']}"
   end
 
+  # for Rails >= 5 ActiveRecord collections we override the collection_cache_key
+  # to prevent Rails doing its default query on `updated_at`
+  def self.collection_cache_key(_collection, _timestamp_column)
+    self.cache_key
+  end
+
   def self.with_metadata(k, v)
     if k.present? && v.present?
       where('metadata->? = ?', k, v)
@@ -82,6 +91,14 @@ class AbstractFeature < ActiveRecord::Base
     return envelope_json.values_at(0,2)
   end
 
+  def self.without_caching_derivatives(&block)
+    old = automatically_cache_derivatives
+    self.automatically_cache_derivatives = false
+    block.call
+  ensure
+    self.automatically_cache_derivatives = old
+  end
+
   def self.cache_derivatives(options = {})
     update_all <<-SQL.squish
       geom         = ST_Transform(geog::geometry, #{detect_srid('geom')}),
@@ -132,7 +149,7 @@ class AbstractFeature < ActiveRecord::Base
   end
 
   def cache_derivatives(*args)
-    self.class.where(:id => self.id).cache_derivatives(*args)
+    self.class.default_scoped.where(:id => self.id).cache_derivatives(*args)
   end
 
   def kml(options = {})

data/app/models/feature.rb

@@ -11,6 +11,16 @@ class Feature < AbstractFeature
 
   after_save :refresh_aggregate, if: :automatically_refresh_aggregate?
 
+  # Features are used for display so we also cache their KML representation
+  def self.cache_derivatives(options = {})
+    super
+    update_all <<-SQL.squish
+      kml          = ST_AsKML(geog, 6),
+      kml_lowres   = ST_AsKML(geom_lowres, #{options.fetch(:lowres_precision, lowres_precision)}),
+      kml_centroid = ST_AsKML(centroid)
+    SQL
+  end
+
   def self.defer_aggregate_refresh(&block)
     start_at = Feature.maximum(:id).to_i + 1
     output = without_aggregate_refresh(&block)
@@ -30,9 +40,10 @@ class Feature < AbstractFeature
 
   def self.refresh_aggregates
     # Find one feature from each spatial model and trigger the aggregate feature refresh
-    ids = select('MAX(id)')
-            .where.not(:spatial_model_type => nil, :spatial_model_id => nil)
+    ids = where.not(:spatial_model_type => nil)
+            .where.not(:spatial_model_id => nil)
             .group('spatial_model_type, spatial_model_id')
+            .pluck('MAX(id)')
 
     # Unscope so that newly built AggregateFeatures get their type column set correctly
     AbstractFeature.unscoped { where(:id => ids).find_each(&:refresh_aggregate) }
@@ -49,14 +60,4 @@ class Feature < AbstractFeature
     # this field blank.
     spatial_model_id? && automatically_refresh_aggregate && saved_change_to_geog?
   end
-
-  # Features are used for display so we also cache their KML representation
-  def self.cache_derivatives(options = {})
-    super
-    update_all <<-SQL.squish
-      kml          = ST_AsKML(geog, 6),
-      kml_lowres   = ST_AsKML(geom_lowres, #{options.fetch(:lowres_precision, lowres_precision)}),
-      kml_centroid = ST_AsKML(centroid)
-    SQL
-  end
 end

data/lib/spatial_features.rb

@@ -12,6 +12,7 @@ require 'spatial_features/controller_helpers/spatial_extensions'
 require 'spatial_features/download'
 require 'spatial_features/unzip'
 require 'spatial_features/utils'
+require 'spatial_features/validation'
 
 require 'spatial_features/has_spatial_features'
 require 'spatial_features/has_spatial_features/queued_spatial_processing'

data/lib/spatial_features/controller_helpers/spatial_extensions.rb

@@ -6,8 +6,6 @@ module SpatialExtensions
       model.failed_feature_update_jobs.destroy_all
       model.delay_update_features!
     end
-
-    redirect_to :back
   end
 
   def abstract_proximity_action(scope, target, distance, &block)

data/lib/spatial_features/download.rb

@@ -12,7 +12,7 @@ module SpatialFeatures
     def self.open(file, unzip: nil, **unzip_options)
       file = Kernel.open(file)
       file = normalize_file(file) if file.is_a?(StringIO)
-      if Unzip.is_zip?(file)
+      if unzip && Unzip.is_zip?(file)
         file = find_in_zip(file, find: unzip, **unzip_options)
       end
       return file
@@ -26,6 +26,12 @@ module SpatialFeatures
       end
     end
 
+    def self.entries(file)
+      file = Kernel.open(file)
+      file = normalize_file(file) if file.is_a?(StringIO)
+      Unzip.entries(file)
+    end
+
     def self.find_in_zip(file, find:, **unzip_options)
       return File.open(Unzip.paths(file, :find => find, **unzip_options))
     end

data/lib/spatial_features/has_spatial_features.rb

@@ -88,16 +88,16 @@ module SpatialFeatures
 
     # Returns true if the model stores a hash of the features so we don't need to process the features if they haven't changed
     def has_spatial_features_hash?
-      column_names.include? 'features_hash'
+      owner_class_has_loaded_column?('features_hash')
     end
 
     # Returns true if the model stores a cache of the features area
     def has_features_area?
-      column_names.include? 'features_area'
+      owner_class_has_loaded_column?('features_area')
     end
 
     def area_in_square_meters
-      features.area_in_square_meters
+      features.area
     end
 
     private
@@ -160,6 +160,12 @@ module SpatialFeatures
       scope = scope.where(:spatial_model_id => other) unless Utils.class_of(other) == other
       return scope
     end
+
+    def owner_class_has_loaded_column?(column_name)
+      return false unless connected?
+      return false unless table_exists?
+      column_names.include? column_name
+    end
   end
 
   module InstanceMethods
@@ -202,7 +208,7 @@ module SpatialFeatures
     end
 
     def features_area_in_square_meters
-      @features_area_in_square_meters ||= features.area
+      aggregate_feature&.area
     end
 
     def total_intersection_area_in_square_meters(other)

data/lib/spatial_features/has_spatial_features/feature_import.rb

@@ -79,14 +79,24 @@ module SpatialFeatures
     end
 
     def import_features(imports, skip_invalid)
-      self.features.delete_all
+      features.delete_all
       valid, invalid = Feature.defer_aggregate_refresh do
-        imports.flat_map(&:features).partition do |feature|
-          feature.spatial_model = self
-          feature.save
+        Feature.without_caching_derivatives do
+          imports.flat_map(&:features).partition do |feature|
+            feature.spatial_model = self
+            feature.save
+          end
         end
       end
 
+      if persisted?
+        features.reset # Reset the association cache because we've updated the features
+        features.cache_derivatives
+      else
+        self.features = valid # Assign the features so when we save this record we update the foreign key on the features
+        Feature.where(id: features).cache_derivatives
+      end
+
       errors = imports.flat_map(&:errors)
       invalid.each do |feature|
         errors << "Feature #{feature.name}: #{feature.errors.full_messages.to_sentence}"
@@ -98,7 +108,7 @@ module SpatialFeatures
         raise ImportError, "Error updating #{self.class} #{self.id}. #{errors.to_sentence}"
       end
 
-      self.features = valid
+      return features
     end
 
     def features_cache_key_matches?(cache_key)

data/lib/spatial_features/importers/shapefile.rb

@@ -12,7 +12,7 @@ module SpatialFeatures
       end
 
       def cache_key
-        @cache_key ||= Digest::MD5.hexdigest(features.to_json)
+        @cache_key ||= Digest::MD5.file(archive).to_s
       end
 
       private
@@ -49,13 +49,27 @@ module SpatialFeatures
         SQL
       end
 
+
       def file
-        @file ||= Download.open(@data, unzip: /\.shp$/, downcase: true)
+        @file ||= begin
+          validate_file!
+          Download.open(archive, unzip: /\.shp$/, downcase: true)
+        end
+      end
+
+      def validate_file!
+        return unless Unzip.is_zip?(archive)
+        Validation.validate_shapefile_archive!(Download.entries(archive), default_proj4_projection: default_proj4_projection)
+      end
+
+      def archive
+        @archive ||= Download.open(@data)
       end
     end
 
     # ERRORS
     class IndeterminateShapefileProjection < SpatialFeatures::ImportError; end
     class IncompleteShapefileArchive < SpatialFeatures::ImportError; end
+    class InvalidShapefileArchive < SpatialFeatures::ImportError; end
   end
 end

data/lib/spatial_features/validation.rb

@@ -0,0 +1,55 @@
+module SpatialFeatures
+  module Validation
+    # SHP file must come first
+    REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS = %w[shp shx dbf prj].freeze
+
+    # Check if a shapefile archive includes the required component files, otherwise
+    # raise an exception.
+    #
+    # @param [Zip::File] zip_file                 A Zip::File object
+    # @param [String] default_proj4_projection    Optional, if supplied we don't raise an exception when we're missing a .PRJ file
+    # @param [Boolean] allow_generic_zip_files    When true, we skip validation entirely if the archive does not contain a .SHP file
+    def self.validate_shapefile_archive!(zip_file, default_proj4_projection: nil, allow_generic_zip_files: false)
+      zip_file_entries = zip_file.entries.each_with_object({}) do |f, obj|
+        ext = File.extname(f.name).downcase[1..-1]
+        next unless ext
+
+        if ext.casecmp?("shp") && obj.key?(ext)
+          raise ::SpatialFeatures::Importers::InvalidShapefileArchive, "Zip files that contain multiple Shapefiles are not supported. Please separate each Shapefile into its own zip file."
+        end
+
+        obj[ext] = File.basename(f.name, '.*')
+      end
+
+      shapefile_basename = zip_file_entries["shp"]
+      unless shapefile_basename
+        # not a shapefile archive but we don't care
+        return if allow_generic_zip_files
+
+        raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a SHP file"
+      end
+
+      REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS[1..-1].each do |ext|
+        ext_basename = zip_file_entries[ext]
+        next if ext_basename&.casecmp?(shapefile_basename)
+
+        case ext
+        when "prj"
+          # special case for missing projection files to allow using default_proj4_projection
+          next if default_proj4_projection
+
+          raise ::SpatialFeatures::Importers::IndeterminateShapefileProjection, "Shapefile archive is missing a projection file: #{expected_component_path(shapefile_basename, ext)}"
+        else
+          # for all un-handled cases of missing files raise the more generic error
+          raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a required file: #{expected_component_path(shapefile_basename, ext)}"
+        end
+      end
+
+      true
+    end
+
+    def self.expected_component_path(basename, ext)
+      "#{basename}.#{ext}"
+    end
+  end
+end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "2.14.1"
+  VERSION = "2.15.0"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 2.14.1
+  version: 2.15.0
 platform: ruby
 authors:
 - Ryan Wallace
 - Nicholas Jakobsen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-15 00:00:00.000000000 Z
+date: 2021-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -20,7 +20,7 @@ dependencies:
         version: '4.2'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
         version: '4.2'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: delayed_job_active_record
   requirement: !ruby/object:Gem::Requirement
@@ -107,14 +107,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -166,6 +166,7 @@ files:
 - lib/spatial_features/uncached_result.rb
 - lib/spatial_features/unzip.rb
 - lib/spatial_features/utils.rb
+- lib/spatial_features/validation.rb
 - lib/spatial_features/venn_polygons.rb
 - lib/spatial_features/version.rb
 - lib/tasks/spatial_features_tasks.rake
@@ -173,7 +174,7 @@ homepage: https://github.com/culturecode/spatial_features
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -188,8 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Adds spatial methods to a model.
 test_files: []