spatial_features 3.4.2 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c6d46bc910c82f59f8dd93bbd1acebd4c9f16ea7b549615ba11fd873a586b54
-  data.tar.gz: 3d0b3682100055904fc403d872b9ed368abb4c19ebe1154c8e057f1105a3a25f
+  metadata.gz: 58ed4738a69dce88104f9ad09ad32dc9f6ebe8819e6bd4bd87c30ea7bde58c70
+  data.tar.gz: b70f376893db22e801ce2b1535c26390f02a754a6884938713ad292cac7432da
 SHA512:
-  metadata.gz: d80268d841701617edb70c129f94bba4ada269c850c56cc3a50fd93f97846b19c295aaeb004a05f64cae07cf392a6bb51ef13d922db593ce850729a22e5a240b
-  data.tar.gz: 895b642f76622a38f3651a95225c1308a6de2652c1b9dd262e8c0f2ca3778a0a91860c30a6a11d801ad01ee3f4e90744ca92ed34b548ba86c57010bb59fc1433
+  metadata.gz: 9e0aa77934a5e00d891bb22adc72e3779267193035deaad0e236ced895c0565c06a6d4b81218e2d2708a493f9913fefdec10cac6e64b40d105cb0e0518f57ec4
+  data.tar.gz: 9456cac927abb58f4710cbbc6366e5f906c93460e68013be9bd48cef90340d2ccf3d43d29e8ac8dff1752fc1c3950030aa494fbaffb3ca91e8ffc3ee179d8876

data/README.md

@@ -200,6 +200,23 @@ add_index :features, :source_identifier
 MyModel.update_features!(:force => true) # Force an `update_features!` will populate the source_identifier column.
 ```
 
+## Upgrading From 3.4 to 3.5
+The gem now relies on virtual columns to set a number of derived column values.
+```ruby
+  change_table :features do |t|
+    t.remove :tilegeom, :feature_type, :centroid, :area, :north, :east, :south, :west
+
+    t.virtual :tilegeom, :type => 'geometry(Geometry,3857)', as: "ST_Transform(geom, 3857)", stored: true, :index => { :using => :gist }
+    t.virtual :feature_type, :type => :string, as: "CASE GeometryType(geog) WHEN 'POLYGON' THEN 'polygon' WHEN 'MULTIPOLYGON' THEN 'polygon' WHEN 'GEOMETRYCOLLECTION' THEN 'polygon' WHEN 'LINESTRING' THEN 'line' WHEN 'MULTILINESTRING' THEN 'line' WHEN 'POINT' THEN 'point' WHEN 'MULTIPOINT' THEN 'point' END", stored: true, :index => true
+    t.virtual :centroid, :type => :geography, as: "ST_PointOnSurface(geog::geometry)", stored: true
+    t.virtual :area, :type => :decimal, as: "ST_Area(geog)", stored: true
+    t.virtual :north, :type => :decimal, as: "ST_YMax(geog::geometry)", stored: true
+    t.virtual :east, :type => :decimal, as: "ST_XMax(geog::geometry)", stored: true
+    t.virtual :south, :type => :decimal, as: "ST_YMin(geog::geometry)", stored: true
+    t.virtual :west, :type => :decimal, as: "ST_XMin(geog::geometry)", stored: true
+  end
+```
+
 ## Testing
 
 Create a postgres database:

data/app/models/abstract_feature.rb

@@ -13,20 +13,19 @@ class AbstractFeature < ActiveRecord::Base
 
   FEATURE_TYPES = %w(polygon point line)
 
-  before_validation :sanitize_feature_type
   validates_presence_of :geog
   validate :validate_geometry, if: :will_save_change_to_geog?
   before_save :sanitize, if: :will_save_change_to_geog?
   after_save :cache_derivatives, :if => [:automatically_cache_derivatives?, :saved_change_to_geog?]
 
-  # for Rails >= 5 ActiveRecord collections we override the collection_cache_key
-  # to prevent Rails doing its default query on `updated_at`
-  def self.collection_cache_key(_collection, _timestamp_column)
-    self.cache_key
+  def self.cache_key
+    collection_cache_key
   end
 
-  def self.cache_key
-    "#{maximum(:id)}-#{count}"
+  # for Rails >= 5 ActiveRecord collections we override the collection_cache_key
+  # to prevent Rails doing its default query on `updated_at`
+  def self.collection_cache_key(collection = all, *)
+    "#{collection.maximum(:id)}-#{collection.count}"
   end
 
   def self.with_metadata(k, v)
@@ -113,13 +112,7 @@ class AbstractFeature < ActiveRecord::Base
 
   def self.cache_derivatives(options = {})
     update_all <<-SQL.squish
-      geom         = ST_Transform(geog::geometry, #{detect_srid('geom')}),
-      north        = ST_YMax(geog::geometry),
-      east         = ST_XMax(geog::geometry),
-      south        = ST_YMin(geog::geometry),
-      west         = ST_XMin(geog::geometry),
-      area         = ST_Area(geog),
-      centroid     = ST_PointOnSurface(geog::geometry)
+      geom         = ST_Transform(geog::geometry, #{detect_srid('geom')})
     SQL
 
     invalid('geom').update_all <<-SQL.squish
@@ -127,8 +120,7 @@ class AbstractFeature < ActiveRecord::Base
     SQL
 
     update_all <<-SQL.squish
-      geom_lowres  = ST_SimplifyPreserveTopology(geom, #{options.fetch(:lowres_simplification, lowres_simplification)}),
-      tilegeom     = ST_Transform(geom, 3857)
+      geom_lowres  = ST_SimplifyPreserveTopology(geom, #{options.fetch(:lowres_simplification, lowres_simplification)})
     SQL
 
     invalid('geom_lowres').update_all <<-SQL.squish
@@ -286,10 +278,6 @@ class AbstractFeature < ActiveRecord::Base
     return error.fetch('invalid_geometry_message') if error
   end
 
-  def sanitize_feature_type
-    self.feature_type = FEATURE_TYPES.find {|type| self.feature_type.to_s.strip.downcase.include?(type) }
-  end
-
   def sanitize_input_for_sql(input)
     self.class.send(:sanitize_sql_for_conditions, input)
   end

data/app/models/feature.rb

@@ -9,7 +9,7 @@ class Feature < AbstractFeature
 
   has_one :aggregate_feature, lambda { |feature| where(:spatial_model_type => feature.spatial_model_type) }, :foreign_key => :spatial_model_id, :primary_key => :spatial_model_id
 
-  validates_inclusion_of :feature_type, :in => FEATURE_TYPES
+  scope :source_identifier, lambda {|source_identifier| where(:source_identifier => source_identifier) if source_identifier.present? }
 
   before_save :truncate_name
 

data/lib/spatial_features/controller_helpers/spatial_extensions.rb

@@ -1,10 +1,10 @@
 module SpatialExtensions
   private
 
-  def abstract_refresh_geometry_action(models)
+  def abstract_refresh_geometry_action(models, **update_options)
     Array.wrap(models).each do |model|
       model.failed_feature_update_jobs.destroy_all
-      model.delay_update_features!
+      model.delay_update_features!(**update_options)
     end
   end
 

data/lib/spatial_features/has_spatial_features/feature_import.rb

@@ -54,6 +54,10 @@ module SpatialFeatures
       if skip_invalid
         Rails.logger.warn "Error updating #{self.class} #{self.id}. #{e.message}"
         return nil
+      elsif ENCODING_ERROR.match?(e.message)
+        raise ImportEncodingError,
+              "One or more features you are trying to import has text encoded in an un-supported format (#{e.message})",
+              e.backtrace
       else
         raise ImportError, e.message, e.backtrace
       end
@@ -151,5 +155,5 @@ module SpatialFeatures
     end
   end
 
-  class ImportError < StandardError; end
+  ENCODING_ERROR = /invalid byte sequence/i.freeze
 end

data/lib/spatial_features/has_spatial_features/queued_spatial_processing.rb

@@ -1,17 +1,27 @@
 module SpatialFeatures
   module QueuedSpatialProcessing
     extend ActiveSupport::Concern
+    mattr_accessor :priority_offset, default: 0 # Offsets the queued priority of spatial tasks. Lower numbers run with higher priority
 
-    def queue_update_spatial_cache(*args)
-      queue_spatial_task('update_spatial_cache', *args)
+    def self.update_cached_status(record, method_name, state)
+      return unless record.has_attribute?(:spatial_processing_status_cache)
+
+      cache = record.spatial_processing_status_cache || {}
+      cache[method_name] = state
+      record.spatial_processing_status_cache = cache
+      record.update_column(:spatial_processing_status_cache, cache) if record.will_save_change_to_spatial_processing_status_cache?
+    end
+
+    def queue_update_spatial_cache(*args, priority: priority_offset + 1, **kwargs)
+      queue_spatial_task('update_spatial_cache', *args, priority:, **kwargs)
     end
 
-    def delay_update_features!(*args)
-      queue_spatial_task('update_features!', *args)
+    def delay_update_features!(*args, priority: priority_offset + 0, **kwargs)
+      queue_spatial_task('update_features!', *args, priority:, **kwargs)
     end
 
-    def updating_features?
-      case spatial_processing_status(:update_features!)
+    def updating_features?(**options)
+      case spatial_processing_status(:update_features!, **options)
       when :queued, :processing
         true
       else
@@ -23,18 +33,37 @@ module SpatialFeatures
       spatial_processing_status(:update_features!) == :failure
     end
 
-    def spatial_processing_status(method_name)
+    def spatial_processing_status(method_name, use_cache: true)
       if has_attribute?(:spatial_processing_status_cache)
+        update_spatial_processing_status(method_name) unless use_cache
         spatial_processing_status_cache[method_name.to_s]&.to_sym
       end
     end
 
+    def update_spatial_processing_status(method_name)
+      latest_job = spatial_processing_jobs(method_name).last
+
+      if !latest_job
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, nil)
+      elsif latest_job.failed_at?
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, :failure)
+      elsif latest_job.locked_at?
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, :processing)
+      else
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, :queued)
+      end
+    end
+
     def feature_update_error
       (failed_feature_update_jobs.first.try(:last_error) || '').split("\n").first
     end
 
     def running_feature_update_jobs
-      spatial_processing_jobs('update_features!').where(failed_at: nil)
+      spatial_processing_jobs('update_features!').where(failed_at: nil).where.not(locked_at: nil)
+    end
+
+    def queued_feature_update_jobs
+      spatial_processing_jobs('update_features!').where(failed_at: nil, locked_at: nil)
     end
 
     def failed_feature_update_jobs
@@ -47,8 +76,9 @@ module SpatialFeatures
 
     private
 
-    def queue_spatial_task(method_name, *args)
-      Delayed::Job.enqueue SpatialProcessingJob.new(self, method_name, *args), :queue => spatial_processing_queue_name + method_name
+    def queue_spatial_task(method_name, *args, priority: 1, **kwargs)
+      # NOTE: We pass kwargs as an arg because Delayed::Job does not support separation of positional and keyword arguments in Ruby 3.0. Instead we perform manual extraction in `perform`.
+      Delayed::Job.enqueue SpatialProcessingJob.new(self, method_name, *args, kwargs), :queue => spatial_processing_queue_name + method_name, priority:
     end
 
     def spatial_processing_queue_name
@@ -68,9 +98,15 @@ module SpatialFeatures
         update_cached_status(:queued)
       end
 
+      def before(job)
+        ids = running_jobs.where.not(:id => job.id).pluck(:id)
+        raise "Already processing delayed jobs in this spatial queue: Delayed::Job #{ids.to_sentence}." if ids.present?
+      end
+
       def perform
         update_cached_status(:processing)
-        @record.send(@method_name, *@args)
+        options = @args.extract_options!
+        @record.send(@method_name, *@args, **options)
       end
 
       def success(job)
@@ -88,11 +124,13 @@ module SpatialFeatures
       private
 
       def update_cached_status(state)
-        if @record.has_attribute?(:spatial_processing_status_cache)
-          cache = @record.spatial_processing_status_cache || {}
-          cache[@method_name] = state
-          @record.update_column(:spatial_processing_status_cache, cache)
-        end
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(@record, @method_name, state)
+      end
+
+      def running_jobs
+        @record.spatial_processing_jobs
+          .where(:locked_at => Delayed::Worker.max_run_time.ago..Time.current)
+          .where(:failed_at => nil)
       end
     end
   end

data/lib/spatial_features/has_spatial_features.rb

@@ -29,7 +29,7 @@ module SpatialFeatures
         delegate :has_spatial_features_hash?, :has_features_area?, :to => self
       end
 
-      self.spatial_features_options = self.spatial_features_options.merge(options)
+      self.spatial_features_options = self.spatial_features_options.deep_merge(options)
     end
   end
 

data/lib/spatial_features/importers/base.rb

@@ -66,5 +66,6 @@ module SpatialFeatures
   # EXCEPTIONS
 
   class ImportError < StandardError; end
+  class ImportEncodingError < ImportError; end
   class EmptyImportError < ImportError; end
 end

data/lib/spatial_features/importers/kml.rb

@@ -36,7 +36,7 @@ module SpatialFeatures
 
             importable_image_paths = images_from_metadata(metadata)
 
-            yield OpenStruct.new(:feature_type => sql_type, :geog => geog, :name => name, :metadata => metadata, :importable_image_paths => importable_image_paths)
+            yield OpenStruct.new(:geog => geog, :name => name, :metadata => metadata, :importable_image_paths => importable_image_paths)
           end
         end
       end

data/lib/spatial_features/importers/shapefile.rb

@@ -6,6 +6,8 @@ module SpatialFeatures
     class Shapefile < Base
       class_attribute :default_proj4_projection
 
+      PROJ4_4326 = '+proj=longlat +datum=WGS84 +no_defs'.freeze
+
       def initialize(data, proj4: nil, **options)
         super(data, **options)
         @proj4 = proj4
@@ -25,15 +27,10 @@ module SpatialFeatures
 
       private
 
-      def build_features
-        validate_shapefile!
-        super
-      end
-
-      def each_record(&block)
-        RGeo::Shapefile::Reader.open(file.path) do |records|
+      def each_record
+        open_shapefile(archive) do |records, proj4|
           records.each do |record|
-            yield OpenStruct.new data_from_wkt(record.geometry.as_text, proj4_projection).merge(:metadata => record.attributes) if record.geometry.present?
+            yield OpenStruct.new data_from_record(record, proj4) if record.geometry.present?
           end
         end
       rescue Errno::ENOENT => e
@@ -45,26 +42,61 @@ module SpatialFeatures
         end
       end
 
-      def proj4_projection
-        @proj4 ||= proj4_from_file || default_proj4_projection || raise(IndeterminateShapefileProjection, 'Could not determine shapefile projection. Check that `gdalsrsinfo` is installed.')
+      def data_from_record(record, proj4 = nil)
+        geometry = record.geometry
+        wkt = geometry.as_text
+        data = { :metadata => record.attributes }
+
+        if proj4 == PROJ4_4326
+          data[:geog] = wkt
+        else
+          data[:geog] = ActiveRecord::Base.connection.select_value <<-SQL
+            SELECT ST_Transform(ST_GeomFromText('#{wkt}'), '#{proj4}', 4326) AS geog
+          SQL
+        end
+
+        return data
       end
 
-      def proj4_from_file
-        # Sanitize: "'+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs '\n" and lately
-        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs \n" to
-        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs"
-        `gdalsrsinfo "#{file.path}" -o proj4`.strip.remove(/^'|'$/).presence
+      def open_shapefile(file, &block)
+        # the individual SHP file for processing (automatically extracted from a ZIP archive if necessary)
+        file = possible_shp_files.first if Unzip.is_zip?(file)
+        projected_file = project_to_4326(file.path)
+        file = projected_file || file
+        validate_shapefile!(file.path)
+        proj4 = proj4_projection(file.path)
+
+        RGeo::Shapefile::Reader.open(file.path) do |records| # Fall back to unprojected geometry if projection fails
+          block.call records, proj4
+        end
+      ensure
+        if projected_file
+          projected_file.close
+          ::File.delete(projected_file)
+        end
+      end
+
+      def proj4_projection(file_path)
+        proj4_from_file(file_path) || default_proj4_projection || raise(IndeterminateShapefileProjection, 'Could not determine shapefile projection. Check that `gdalsrsinfo` is installed.')
       end
 
-      def data_from_wkt(wkt, proj4)
-        ActiveRecord::Base.connection.select_one <<-SQL
-          SELECT ST_Transform(ST_GeomFromText('#{wkt}'), '#{proj4}', 4326) AS geog, GeometryType(ST_GeomFromText('#{wkt}')) AS feature_type
-        SQL
+      def validate_shapefile!(file_path)
+        Validation.validate_shapefile!(::File.open(file_path), default_proj4_projection: default_proj4_projection)
       end
 
-      # the individual SHP file for processing (automatically extracted from a ZIP archive if necessary)
-      def file
-        @file ||= Unzip.is_zip?(archive) ? possible_shp_files.first : archive
+      # Use OGR2OGR to reproject into EPSG:4326 so we can skip the reprojection step per-feature
+      def project_to_4326(file_path)
+        output_path = Tempfile.create([::File.basename(file_path, '.shp') + '_epsg_4326_', '.shp']) { |file| file.path }
+        return unless (proj4 = proj4_from_file(file_path))
+        return unless system("ogr2ogr -s_srs '#{proj4}' -t_srs EPSG:4326 '#{output_path}' '#{file_path}'")
+        return ::File.open(output_path)
+      end
+
+      def proj4_from_file(file_path)
+        # Sanitize: "'+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs '\n" and lately
+        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs \n" to
+        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs"
+        `gdalsrsinfo "#{file_path}" -o proj4`.strip.remove(/^'|'$/).presence
       end
 
       # a zip archive may contain multiple SHP files
@@ -76,10 +108,6 @@ module SpatialFeatures
         end
       end
 
-      def validate_shapefile!
-        Validation.validate_shapefile!(file, default_proj4_projection: default_proj4_projection)
-      end
-
       def archive
         @archive ||= Download.open(@data)
       end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "3.4.2"
+  VERSION = "3.5.1"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 3.4.2
+  version: 3.5.1
 platform: ruby
 authors:
 - Ryan Wallace
 - Nicholas Jakobsen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-26 00:00:00.000000000 Z
+date: 2024-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -51,14 +51,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: rgeo-geojson
   requirement: !ruby/object:Gem::Requirement
@@ -101,6 +101,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -188,7 +208,7 @@ homepage: https://github.com/culturecode/spatial_features
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -203,8 +223,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.23
-signing_key: 
+rubygems_version: 3.5.1
+signing_key:
 specification_version: 4
 summary: Adds spatial methods to a model.
 test_files: []