spatial_features 3.4.2 → 3.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c6d46bc910c82f59f8dd93bbd1acebd4c9f16ea7b549615ba11fd873a586b54
-  data.tar.gz: 3d0b3682100055904fc403d872b9ed368abb4c19ebe1154c8e057f1105a3a25f
+  metadata.gz: 9407d9a0f72fe25d714ee781b2902f387c77837c8deb3b1c1df8cdaa7be97b35
+  data.tar.gz: af6a1f4386fcb85adb7b0c17925f13ff11f0730fd4fe407cdbf1c8efdcfbbdee
 SHA512:
-  metadata.gz: d80268d841701617edb70c129f94bba4ada269c850c56cc3a50fd93f97846b19c295aaeb004a05f64cae07cf392a6bb51ef13d922db593ce850729a22e5a240b
-  data.tar.gz: 895b642f76622a38f3651a95225c1308a6de2652c1b9dd262e8c0f2ca3778a0a91860c30a6a11d801ad01ee3f4e90744ca92ed34b548ba86c57010bb59fc1433
+  metadata.gz: 22acdcc935004caaf2783679b0e0126ddfe679cd7f51c29f0eb6f358746c9cfb01c67dd5f67db4bfa26f975aec7da783d6e3ad45312e292081677ff194761552
+  data.tar.gz: 84fd0dcfd2a158bd7612df2da14b6124bc9b5a4b668779057037cf1c17a4be03eda6641ee9204d3eef696766e446c3764fcb6976ea3387ad660e8fef63285c13

data/app/models/abstract_feature.rb

@@ -19,14 +19,14 @@ class AbstractFeature < ActiveRecord::Base
   before_save :sanitize, if: :will_save_change_to_geog?
   after_save :cache_derivatives, :if => [:automatically_cache_derivatives?, :saved_change_to_geog?]
 
-  # for Rails >= 5 ActiveRecord collections we override the collection_cache_key
-  # to prevent Rails doing its default query on `updated_at`
-  def self.collection_cache_key(_collection, _timestamp_column)
-    self.cache_key
+  def self.cache_key
+    collection_cache_key
   end
 
-  def self.cache_key
-    "#{maximum(:id)}-#{count}"
+  # for Rails >= 5 ActiveRecord collections we override the collection_cache_key
+  # to prevent Rails doing its default query on `updated_at`
+  def self.collection_cache_key(collection = all, *)
+    "#{collection.maximum(:id)}-#{collection.count}"
   end
 
   def self.with_metadata(k, v)

data/app/models/feature.rb

@@ -9,6 +9,8 @@ class Feature < AbstractFeature
 
   has_one :aggregate_feature, lambda { |feature| where(:spatial_model_type => feature.spatial_model_type) }, :foreign_key => :spatial_model_id, :primary_key => :spatial_model_id
 
+  scope :source_identifier, lambda {|source_identifier| where(:source_identifier => source_identifier) if source_identifier.present? }
+
   validates_inclusion_of :feature_type, :in => FEATURE_TYPES
 
   before_save :truncate_name

data/lib/spatial_features/controller_helpers/spatial_extensions.rb

@@ -1,10 +1,10 @@
 module SpatialExtensions
   private
 
-  def abstract_refresh_geometry_action(models)
+  def abstract_refresh_geometry_action(models, **update_options)
     Array.wrap(models).each do |model|
       model.failed_feature_update_jobs.destroy_all
-      model.delay_update_features!
+      model.delay_update_features!(**update_options)
     end
   end
 

data/lib/spatial_features/has_spatial_features/feature_import.rb

@@ -54,6 +54,10 @@ module SpatialFeatures
       if skip_invalid
         Rails.logger.warn "Error updating #{self.class} #{self.id}. #{e.message}"
         return nil
+      elsif ENCODING_ERROR.match?(e.message)
+        raise ImportEncodingError,
+              "One or more features you are trying to import has text encoded in an un-supported format (#{e.message})",
+              e.backtrace
       else
         raise ImportError, e.message, e.backtrace
       end
@@ -151,5 +155,5 @@ module SpatialFeatures
     end
   end
 
-  class ImportError < StandardError; end
+  ENCODING_ERROR = /invalid byte sequence/i.freeze
 end

data/lib/spatial_features/has_spatial_features/queued_spatial_processing.rb

@@ -2,16 +2,25 @@ module SpatialFeatures
   module QueuedSpatialProcessing
     extend ActiveSupport::Concern
 
-    def queue_update_spatial_cache(*args)
-      queue_spatial_task('update_spatial_cache', *args)
+    def self.update_cached_status(record, method_name, state)
+      return unless record.has_attribute?(:spatial_processing_status_cache)
+
+      cache = record.spatial_processing_status_cache || {}
+      cache[method_name] = state
+      record.spatial_processing_status_cache = cache
+      record.update_column(:spatial_processing_status_cache, cache) if record.will_save_change_to_spatial_processing_status_cache?
+    end
+
+    def queue_update_spatial_cache(*args, **kwargs)
+      queue_spatial_task('update_spatial_cache', *args, **kwargs)
     end
 
-    def delay_update_features!(*args)
-      queue_spatial_task('update_features!', *args)
+    def delay_update_features!(*args, **kwargs)
+      queue_spatial_task('update_features!', *args, **kwargs)
     end
 
-    def updating_features?
-      case spatial_processing_status(:update_features!)
+    def updating_features?(**options)
+      case spatial_processing_status(:update_features!, **options)
       when :queued, :processing
         true
       else
@@ -23,18 +32,37 @@ module SpatialFeatures
       spatial_processing_status(:update_features!) == :failure
     end
 
-    def spatial_processing_status(method_name)
+    def spatial_processing_status(method_name, use_cache: true)
       if has_attribute?(:spatial_processing_status_cache)
+        update_spatial_processing_status(method_name) unless use_cache
         spatial_processing_status_cache[method_name.to_s]&.to_sym
       end
     end
 
+    def update_spatial_processing_status(method_name)
+      latest_job = spatial_processing_jobs(method_name).last
+
+      if !latest_job
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, nil)
+      elsif latest_job.failed_at?
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, :failure)
+      elsif latest_job.locked_at?
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, :processing)
+      else
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(self, method_name, :queued)
+      end
+    end
+
     def feature_update_error
       (failed_feature_update_jobs.first.try(:last_error) || '').split("\n").first
     end
 
     def running_feature_update_jobs
-      spatial_processing_jobs('update_features!').where(failed_at: nil)
+      spatial_processing_jobs('update_features!').where(failed_at: nil).where.not(locked_at: nil)
+    end
+
+    def queued_feature_update_jobs
+      spatial_processing_jobs('update_features!').where(failed_at: nil, locked_at: nil)
     end
 
     def failed_feature_update_jobs
@@ -47,8 +75,9 @@ module SpatialFeatures
 
     private
 
-    def queue_spatial_task(method_name, *args)
-      Delayed::Job.enqueue SpatialProcessingJob.new(self, method_name, *args), :queue => spatial_processing_queue_name + method_name
+    def queue_spatial_task(method_name, *args, priority: 1, **kwargs)
+      # NOTE: We pass kwargs as an arg because Delayed::Job does not support separation of positional and keyword arguments in Ruby 3.0. Instead we perform manual extraction in `perform`.
+      Delayed::Job.enqueue SpatialProcessingJob.new(self, method_name, *args, kwargs), :queue => spatial_processing_queue_name + method_name, :priority => priority
     end
 
     def spatial_processing_queue_name
@@ -68,9 +97,15 @@ module SpatialFeatures
         update_cached_status(:queued)
       end
 
+      def before(job)
+        ids = running_jobs.where.not(:id => job.id).pluck(:id)
+        raise "Already processing delayed jobs in this spatial queue: Delayed::Job #{ids.to_sentence}." if ids.present?
+      end
+
       def perform
         update_cached_status(:processing)
-        @record.send(@method_name, *@args)
+        options = @args.extract_options!
+        @record.send(@method_name, *@args, **options)
       end
 
       def success(job)
@@ -88,11 +123,13 @@ module SpatialFeatures
       private
 
       def update_cached_status(state)
-        if @record.has_attribute?(:spatial_processing_status_cache)
-          cache = @record.spatial_processing_status_cache || {}
-          cache[@method_name] = state
-          @record.update_column(:spatial_processing_status_cache, cache)
-        end
+        SpatialFeatures::QueuedSpatialProcessing.update_cached_status(@record, @method_name, state)
+      end
+
+      def running_jobs
+        @record.spatial_processing_jobs
+          .where(:locked_at => Delayed::Worker.max_run_time.ago..Time.current)
+          .where(:failed_at => nil)
       end
     end
   end

data/lib/spatial_features/has_spatial_features.rb

@@ -29,7 +29,7 @@ module SpatialFeatures
         delegate :has_spatial_features_hash?, :has_features_area?, :to => self
       end
 
-      self.spatial_features_options = self.spatial_features_options.merge(options)
+      self.spatial_features_options = self.spatial_features_options.deep_merge(options)
     end
   end
 

data/lib/spatial_features/importers/base.rb

@@ -66,5 +66,6 @@ module SpatialFeatures
   # EXCEPTIONS
 
   class ImportError < StandardError; end
+  class ImportEncodingError < ImportError; end
   class EmptyImportError < ImportError; end
 end

data/lib/spatial_features/importers/shapefile.rb

@@ -6,6 +6,9 @@ module SpatialFeatures
     class Shapefile < Base
       class_attribute :default_proj4_projection
 
+      FEATURE_TYPE_FOR_DIMENSION = { 0 => 'point', 1 => 'line', 2 => 'polygon' }.freeze
+      PROJ4_4326 = '+proj=longlat +datum=WGS84 +no_defs'.freeze
+
       def initialize(data, proj4: nil, **options)
         super(data, **options)
         @proj4 = proj4
@@ -25,15 +28,10 @@ module SpatialFeatures
 
       private
 
-      def build_features
-        validate_shapefile!
-        super
-      end
-
-      def each_record(&block)
-        RGeo::Shapefile::Reader.open(file.path) do |records|
+      def each_record
+        open_shapefile(archive) do |records, proj4|
           records.each do |record|
-            yield OpenStruct.new data_from_wkt(record.geometry.as_text, proj4_projection).merge(:metadata => record.attributes) if record.geometry.present?
+            yield OpenStruct.new data_from_record(record, proj4) if record.geometry.present?
           end
         end
       rescue Errno::ENOENT => e
@@ -45,26 +43,61 @@ module SpatialFeatures
         end
       end
 
-      def proj4_projection
-        @proj4 ||= proj4_from_file || default_proj4_projection || raise(IndeterminateShapefileProjection, 'Could not determine shapefile projection. Check that `gdalsrsinfo` is installed.')
+      def data_from_record(record, proj4 = nil)
+        geometry = record.geometry
+        wkt = geometry.as_text
+        data = { :metadata => record.attributes, feature_type: FEATURE_TYPE_FOR_DIMENSION.fetch(geometry.dimension) }
+
+        if proj4 == PROJ4_4326
+          data[:geog] = wkt
+        else
+          data[:geog] = ActiveRecord::Base.connection.select_value <<-SQL
+            SELECT ST_Transform(ST_GeomFromText('#{wkt}'), '#{proj4}', 4326) AS geog
+          SQL
+        end
+
+        return data
       end
 
-      def proj4_from_file
-        # Sanitize: "'+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs '\n" and lately
-        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs \n" to
-        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs"
-        `gdalsrsinfo "#{file.path}" -o proj4`.strip.remove(/^'|'$/).presence
+      def open_shapefile(file, &block)
+        # the individual SHP file for processing (automatically extracted from a ZIP archive if necessary)
+        file = possible_shp_files.first if Unzip.is_zip?(file)
+        projected_file = project_to_4326(file.path)
+        file = projected_file || file
+        validate_shapefile!(file.path)
+        proj4 = proj4_projection(file.path)
+
+        RGeo::Shapefile::Reader.open(file.path) do |records| # Fall back to unprojected geometry if projection fails
+          block.call records, proj4
+        end
+      ensure
+        if projected_file
+          projected_file.close
+          ::File.delete(projected_file)
+        end
+      end
+
+      def proj4_projection(file_path)
+        proj4_from_file(file_path) || default_proj4_projection || raise(IndeterminateShapefileProjection, 'Could not determine shapefile projection. Check that `gdalsrsinfo` is installed.')
       end
 
-      def data_from_wkt(wkt, proj4)
-        ActiveRecord::Base.connection.select_one <<-SQL
-          SELECT ST_Transform(ST_GeomFromText('#{wkt}'), '#{proj4}', 4326) AS geog, GeometryType(ST_GeomFromText('#{wkt}')) AS feature_type
-        SQL
+      def validate_shapefile!(file_path)
+        Validation.validate_shapefile!(::File.open(file_path), default_proj4_projection: default_proj4_projection)
       end
 
-      # the individual SHP file for processing (automatically extracted from a ZIP archive if necessary)
-      def file
-        @file ||= Unzip.is_zip?(archive) ? possible_shp_files.first : archive
+      # Use OGR2OGR to reproject into EPSG:4326 so we can skip the reprojection step per-feature
+      def project_to_4326(file_path)
+        output_path = Tempfile.create([::File.basename(file_path, '.shp') + '_epsg_4326_', '.shp']) { |file| file.path }
+        return unless (proj4 = proj4_from_file(file_path))
+        return unless system("ogr2ogr -s_srs '#{proj4}' -t_srs EPSG:4326 '#{output_path}' '#{file_path}'")
+        return ::File.open(output_path)
+      end
+
+      def proj4_from_file(file_path)
+        # Sanitize: "'+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs '\n" and lately
+        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs \n" to
+        #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs"
+        `gdalsrsinfo "#{file_path}" -o proj4`.strip.remove(/^'|'$/).presence
       end
 
       # a zip archive may contain multiple SHP files
@@ -76,10 +109,6 @@ module SpatialFeatures
         end
       end
 
-      def validate_shapefile!
-        Validation.validate_shapefile!(file, default_proj4_projection: default_proj4_projection)
-      end
-
       def archive
         @archive ||= Download.open(@data)
       end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "3.4.2"
+  VERSION = "3.4.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 3.4.2
+  version: 3.4.8
 platform: ruby
 authors:
 - Ryan Wallace
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-26 00:00:00.000000000 Z
+date: 2024-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails