spatial_features 1.7.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 670d368c7bf08ca4ff1280a2d61760ad5a391a03
-  data.tar.gz: 707999939635257129e3f8e354c2c73e6ed9dc8a
+  metadata.gz: ee690ccc750978af8db4b018a4564dcad10085cc
+  data.tar.gz: cb9e15cdee984d7c7398f68ec95af49a973a0571
 SHA512:
-  metadata.gz: 748760b027e02b3832319a26056ac2b2898ff2eb7db19de2819f1e04f67c95fbf8a8ad240051e4dc515ecfc503f6289cbaf5a90ebb615e7fda98630675180f15
-  data.tar.gz: fccaf84adbfc1bccd73acd83bb73552c2f4261beee9adbb508b24aaf9667ffdaa0d2f2ae0a543da4dab06c96f237562f32e01a05a1458ad6d153315d7a61dd99
+  metadata.gz: d1816cccfc59bb3d8ad6dfd19467a734e0889fd658224d19099e0c7266dd30b16ecc6ff5c305c557649fe4278c3c0dcab9be4ee8f10d5aa1db37103c60a119d1
+  data.tar.gz: 97f3e7e7eff1d3feac000560651944bd391d5533055a3131b29ad39901ee9442d23d70854f2b10c796006129cd38f8e336e2a5f73d53cffbcb642d3ac43ad5ca

data/README.md

@@ -48,3 +48,33 @@ execute("
   );
 ")
 ```
+
+## Usage
+
+In your model
+
+```ruby
+class Location < ActiveRecord::Base
+  has_spatial_features
+end
+
+Person.new(:features => [Feature.new(:geog => 'some binary PostGIS Geography string')])
+```
+
+### Import
+
+You can specify multiple import sources for geometry. Each key is a method that returns the data for the Importer, and
+each value is the Importer to use to parse the data. See each Importer for more details.
+```ruby
+class Location < ActiveRecord::Base
+  has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File' }
+
+  def remote_kml_url
+    "www.test.com/kml/#{id}.kml"
+  end
+
+  def file
+    File.open('local/files/my_kml')
+  end
+end
+```

data/app/models/feature.rb

@@ -1,10 +1,16 @@
 class Feature < ActiveRecord::Base
   belongs_to :spatial_model, :polymorphic => :true
 
+  attr_writer :make_valid
+
+  FEATURE_TYPES = %w(polygon point line)
+
   before_validation :sanitize_feature_type
   validates_presence_of :geog
   validate :geometry_is_valid
-  validates_inclusion_of :feature_type, :in => ['polygon', 'point', 'line']
+  validates_inclusion_of :feature_type, :in => FEATURE_TYPES
+  before_save :sanitize
+  before_save :make_valid, if: :make_valid?
   after_save :cache_derivatives
 
   def self.with_metadata(k, v)
@@ -98,8 +104,21 @@ class Feature < ActiveRecord::Base
     return geometry
   end
 
+  def make_valid?
+    @make_valid
+  end
+
   private
 
+  def make_valid
+    self.geog = ActiveRecord::Base.connection.select_value("SELECT ST_CollectionExtract(ST_MakeValid('#{sanitize}'),3)")
+  end
+
+  # Use ST_Force_2D to discard z-coordinates that cause failures later in the process
+  def sanitize
+    self.geog = ActiveRecord::Base.connection.select_value("SELECT ST_Force2D('#{geog}')")
+  end
+
   def self.detect_srid(column_name)
     connection.select_value("SELECT Find_SRID('public', '#{table_name}', '#{column_name}')")
   end
@@ -116,7 +135,7 @@ class Feature < ActiveRecord::Base
   end
 
   def sanitize_feature_type
-    self.feature_type = self.feature_type.to_s.strip.downcase
+    self.feature_type = FEATURE_TYPES.find {|type| self.feature_type.to_s.strip.downcase.include?(type) }
   end
 
   def sanitize_input_for_sql(input)

data/lib/spatial_features/download.rb

@@ -0,0 +1,32 @@
+require 'open-uri'
+
+module SpatialFeatures
+  module Download
+    # file can be a url, path, or file, any of which can return be a zipped archive
+    def self.read(file, unzip: nil)
+      file = open(file, unzip: unzip)
+      path = ::File.path(file)
+      return ::File.read(path)
+    end
+
+    def self.open(file, unzip: nil)
+      file = Kernel.open(file)
+      file = normalize_file(file) if file.is_a?(StringIO)
+      file = find_in_zip(file, unzip) if Unzip.is_zip?(file)
+      return file
+    end
+
+    def self.normalize_file(file)
+      Tempfile.new.tap do |temp|
+        temp.binmode
+        temp.write(file.read)
+        temp.rewind
+      end
+    end
+
+    def self.find_in_zip(file, unzip)
+      raise "Must specify an :unzip option if opening a zip file. e.g. open(file, :find => '.shp')" unless unzip
+      return File.open(Unzip.paths(file, :find => unzip))
+    end
+  end
+end

data/lib/spatial_features/has_spatial_features/delayed_feature_import.rb

@@ -0,0 +1,36 @@
+module SpatialFeatures
+  module DelayedFeatureImport
+    include FeatureImport
+
+    def queue_feature_update!(options = {})
+      job = UpdateFeaturesJob.new(options.merge :spatial_model_type => self.class, :spatial_model_id => self.id)
+      Delayed::Job.enqueue(job, :queue => delayed_jobs_queue_name)
+    end
+
+    def updating_features?
+      running_feature_update_jobs.exists?
+    end
+
+    def feature_update_error
+      (failed_feature_update_jobs.first.try(:last_error) || '').split("\n").first
+    end
+
+    def running_feature_update_jobs
+      feature_update_jobs.where(failed_at: nil)
+    end
+
+    def failed_feature_update_jobs
+      feature_update_jobs.where.not(failed_at: nil)
+    end
+
+    def feature_update_jobs
+      Delayed::Job.where(queue: delayed_jobs_queue_name)
+    end
+
+    private
+
+    def delayed_jobs_queue_name
+      "#{self.class}/#{self.id}/update_features"
+    end
+  end
+end

data/lib/spatial_features/has_spatial_features/feature_import.rb

@@ -0,0 +1,64 @@
+require 'digest/md5'
+
+module SpatialFeatures
+  module FeatureImport
+    def update_features!(skip_invalid: false, options: {})
+      options = options.reverse_merge(spatial_features_options).reverse_merge(:import => {})
+
+      ActiveRecord::Base.transaction do
+        imports = spatial_feature_imports(options[:import], options[:make_valid])
+        cache_key = Digest::MD5.hexdigest(imports.collect(&:cache_key).join)
+
+        return if features_cache_key_matches?(cache_key)
+
+        import_features(imports)
+        validate_features!(imports, skip_invalid)
+        set_features_cache_key(cache_key)
+
+        return true
+      end
+    end
+
+    private
+
+    def spatial_feature_imports(import_options, make_valid)
+      import_options.collect do |data_method, importer_name|
+        data = send(data_method)
+        "SpatialFeatures::Importers::#{importer_name}".constantize.new(data, :make_valid => make_valid) if data.present?
+      end.compact
+    end
+
+    def import_features(imports)
+      self.features.destroy_all
+      self.features = imports.flat_map(&:features)
+    end
+
+    def validate_features!(imports, skip_invalid = false)
+      invalid = features.select {|feature| feature.errors.present? }
+      features.destroy(invalid)
+
+      return if skip_invalid
+
+      errors = imports.flat_map(&:errors)
+      invalid.each do |feature|
+        errors << "Feature #{feature.name}: #{feature.errors.full_messages.to_sentence}"
+      end
+
+      if errors.present?
+        raise ImportError, "Error updating #{self.class} #{self.id}. #{errors.to_sentence}"
+      end
+    end
+
+    def features_cache_key_matches?(cache_key)
+      has_spatial_features_hash? && cache_key == features_hash
+    end
+
+    def set_features_cache_key(cache_key)
+      return unless has_spatial_features_hash?
+      self.features_hash = cache_key
+      update_column(:features_hash, cache_key) unless new_record?
+    end
+  end
+
+  class ImportError < StandardError; end
+end

data/lib/spatial_features/has_spatial_features.rb

@@ -1,29 +1,41 @@
 module SpatialFeatures
   module ActMethod
     def has_spatial_features(options = {})
-      extend ClassMethods
-      include InstanceMethods
+      unless acts_like?(:spatial_features)
+        extend ClassMethods
+        include InstanceMethods
+        include DelayedFeatureImport
 
-      has_many :features, lambda { extending FeaturesAssociationExtensions }, :as => :spatial_model, :dependent => :delete_all
+        class_attribute :spatial_features_options
+        self.spatial_features_options = {}
 
-      scope :with_features, lambda { joins(:features).uniq }
-      scope :without_features, lambda { joins("LEFT OUTER JOIN features ON features.spatial_model_type = '#{name}' AND features.spatial_model_id = #{table_name}.id").where("features.id IS NULL") }
+        has_many :features, lambda { extending FeaturesAssociationExtensions }, :as => :spatial_model, :dependent => :delete_all
 
-      scope :with_spatial_cache, lambda {|klass| joins(:spatial_cache).where(:spatial_caches => { :intersection_model_type =>  klass }).uniq }
-      scope :without_spatial_cache, lambda {|klass| joins("LEFT OUTER JOIN #{SpatialCache.table_name} ON spatial_model_id = #{table_name}.id AND spatial_model_type = '#{name}' and intersection_model_type = '#{klass}'").where('spatial_model_id IS NULL') }
-      scope :with_stale_spatial_cache, lambda { joins(:spatial_cache).where("#{table_name}.features_hash != spatial_caches.features_hash").uniq } if has_spatial_features_hash?
+        scope :with_features, lambda { joins(:features).uniq }
+        scope :without_features, lambda { joins("LEFT OUTER JOIN features ON features.spatial_model_type = '#{name}' AND features.spatial_model_id = #{table_name}.id").where("features.id IS NULL") }
 
-      has_many :spatial_cache, :as => :spatial_model, :dependent => :delete_all
-      has_many :model_a_spatial_proximities, :as => :model_a, :class_name => 'SpatialProximity', :dependent => :delete_all
-      has_many :model_b_spatial_proximities, :as => :model_b, :class_name => 'SpatialProximity', :dependent => :delete_all
+        scope :with_spatial_cache, lambda {|klass| joins(:spatial_cache).where(:spatial_caches => { :intersection_model_type =>  klass }).uniq }
+        scope :without_spatial_cache, lambda {|klass| joins("LEFT OUTER JOIN #{SpatialCache.table_name} ON spatial_model_id = #{table_name}.id AND spatial_model_type = '#{name}' and intersection_model_type = '#{klass}'").where('spatial_model_id IS NULL') }
+        scope :with_stale_spatial_cache, lambda { joins(:spatial_cache).where("#{table_name}.features_hash != spatial_caches.features_hash").uniq } if has_spatial_features_hash?
 
-      after_save :update_features_area, :if => :features_hash_changed? if has_features_area? && has_spatial_features_hash?
+        has_many :spatial_cache, :as => :spatial_model, :dependent => :delete_all
+        has_many :model_a_spatial_proximities, :as => :model_a, :class_name => 'SpatialProximity', :dependent => :delete_all
+        has_many :model_b_spatial_proximities, :as => :model_b, :class_name => 'SpatialProximity', :dependent => :delete_all
 
-      delegate :has_spatial_features_hash?, :has_features_area?, :to => self
+        after_save :update_features_area, :if => :features_hash_changed? if has_features_area? && has_spatial_features_hash?
+
+        delegate :has_spatial_features_hash?, :has_features_area?, :to => self
+      end
+
+      self.spatial_features_options = self.spatial_features_options.merge(options)
     end
   end
 
   module ClassMethods
+    def acts_like_spatial_features?
+      true
+    end
+
     # Add methods to generate cache keys for a record or all records of this class
     # NOTE: features are never updated, only deleted and created, therefore we can
     # tell if they have changed by finding the maximum id and count instead of needing timestamps
@@ -164,7 +176,7 @@ module SpatialFeatures
   end
 
   module InstanceMethods
-    def has_spatial_features?
+    def acts_like_spatial_features?
       true
     end
 

data/lib/spatial_features/importers/base.rb

@@ -0,0 +1,51 @@
+require 'digest/md5'
+
+module SpatialFeatures
+  module Importers
+    class Base
+      attr_reader :errors
+
+      def initialize(data, make_valid: false)
+        @make_valid = make_valid
+        @data = data
+        @errors = []
+      end
+
+      def features
+        @features ||= build_features
+      end
+
+      def cache_key
+        @cache_key ||= Digest::MD5.hexdigest(@data)
+      end
+
+      private
+
+      def build_features
+        new_features = []
+
+        each_record do |record|
+          begin
+            new_features << build_feature(record)
+          rescue => e
+            @errors << e.message
+          end
+        end
+
+        return new_features
+      end
+
+      def each_record(&block)
+        raise NotImplementedError, 'Subclasses should implement this method and yield objects that can be passed to #build_feature'
+      end
+
+      def build_feature(record)
+        Feature.new(:name => record.name, :metadata => record.metadata, :feature_type => record.feature_type, :geog => record.geog, :make_valid => @make_valid)
+      end
+    end
+  end
+
+  # EXCEPTIONS
+
+  class ImportError < StandardError; end
+end

data/lib/spatial_features/importers/file.rb

@@ -0,0 +1,21 @@
+require 'open-uri'
+
+module SpatialFeatures
+  module Importers
+    class File < SimpleDelegator
+      def initialize(data, *args)
+        file = Download.open(data, unzip: %w(.kml .shp))
+
+        if file.path.end_with? '.kml'
+          __setobj__(KMLFile.new(file, *args))
+
+        elsif file.path.end_with? '.shp'
+          __setobj__(Shapefile.new(file, *args))
+
+        else
+          raise ImportError, "Could not import file. Supported formats are KMZ, KML, and zipped ArcGIS shapefiles"
+        end
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/geomark.rb

@@ -0,0 +1,15 @@
+module SpatialFeatures
+  module Importers
+    class Geomark < KMLFile
+      def initialize(geomark, *args)
+        super geomark_url(geomark), *args
+      end
+
+      private
+
+      def geomark_url(geomark)
+        "http://apps.gov.bc.ca/pub/geomark/geomarks/#{geomark}/parts.kml?srid=4326"
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/kml.rb

@@ -0,0 +1,26 @@
+require 'ostruct'
+
+module SpatialFeatures
+  module Importers
+    class KML < Base
+      private
+
+      def each_record(&block)
+        Nokogiri::XML(@data).css('Placemark').each do |placemark|
+          name = placemark.css('name').text
+          metadata = {:description => placemark.css('description').text}
+
+          {'Polygon' => 'POLYGON', 'LineString' => 'LINE', 'Point' => 'POINT'}.each do |kml_type, sql_type|
+            placemark.css(kml_type).each do |placemark|
+              yield OpenStruct.new(:feature_type => sql_type, :geog => geom_from_kml(placemark), :name => name, :metadata => metadata)
+            end
+          end
+        end
+      end
+
+      def geom_from_kml(kml)
+        ActiveRecord::Base.connection.select_value("SELECT ST_GeomFromKML('#{kml}')")
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/kml_file.rb

@@ -0,0 +1,13 @@
+module SpatialFeatures
+  module Importers
+    class KMLFile < KML
+      def initialize(path_or_url, *args)
+        super Download.read(path_or_url, unzip: '.kml'), *args
+
+      rescue SocketError, Errno::ECONNREFUSED, OpenURI::HTTPError
+        url = URI(path_or_url)
+        raise ImportError, "KML server is not responding. Ensure server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/kml_file_arcgis.rb

@@ -0,0 +1,27 @@
+require 'ostruct'
+
+module SpatialFeatures
+  module Importers
+    class KMLFileArcGIS < KMLFile
+      def initialize(data, *args)
+        super
+
+      rescue SocketError, Errno::ECONNREFUSED
+        url = URI(data)
+        raise ImportError, "ArcGIS Server is not responding. Ensure ArcGIS Server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
+      rescue OpenURI::HTTPError
+        raise ImportError, "ArcGIS Map Service not found. Ensure ArcGIS Server is running and accessible at #{path_or_url}."
+      end
+
+      private
+
+      # ArcGIS includes metadata as an html table in the description
+      def each_record(&block)
+        super do |record|
+          record.metadata = Hash[Nokogiri::XML(record.metadata[:description]).css('td').collect(&:text).each_slice(2).to_a]
+          yield record
+        end
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/shapefile.rb

@@ -0,0 +1,34 @@
+require 'ostruct'
+require 'digest/md5'
+
+module SpatialFeatures
+  module Importers
+    class Shapefile < Base
+      def cache_key
+        @cache_key ||= Digest::MD5.hexdigest(features.to_json)
+      end
+
+      private
+
+      def each_record(&block)
+        file = Download.open(@data, unzip: '.shp')
+        proj4 = proj4_from_file(file)
+        RGeo::Shapefile::Reader.open(file.path) do |records|
+          records.each do |record|
+            yield OpenStruct.new data_from_wkt(record.geometry.as_text, proj4).merge(:metadata => record.attributes)
+          end
+        end
+      end
+
+      def proj4_from_file(file)
+        `gdalsrsinfo "#{file.path}" -o proj4`[/'(.+)'/,1] # Sanitize "'+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs '\n"
+      end
+
+      def data_from_wkt(wkt, proj4)
+        ActiveRecord::Base.connection.select_one <<-SQL
+          SELECT ST_Transform(ST_GeomFromText('#{wkt}'), '#{proj4}', 4326) AS geog, GeometryType(ST_GeomFromText('#{wkt}')) AS feature_type
+        SQL
+      end
+    end
+  end
+end

data/lib/spatial_features/unzip.rb

@@ -0,0 +1,37 @@
+module SpatialFeatures
+  module Unzip
+    def self.paths(file_path, find: nil)
+      dir = Dir.mktmpdir
+      paths = []
+
+      entries(file_path).each do |entry|
+        path = "#{dir}/#{entry.name}"
+        entry.extract(path)
+        paths << path
+      end
+
+      if find = Array.wrap(find).presence
+        paths = paths.detect {|path| find.any? {|pattern| path.include?(pattern) } }
+        raise(ImportError, "No file matched #{find}") unless paths.present?
+      end
+
+      return paths
+    end
+
+    def self.names(file_path)
+      entries(file_path).collect(&:name)
+    end
+
+    def self.entries(file_path)
+      Zip::File.open(File.path(file_path))
+    end
+
+    def self.is_zip?(file)
+      zip = file.readline.start_with?('PK')
+      file.rewind
+      return zip
+    rescue EOFError
+      return false
+    end
+  end
+end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "1.7.1"
+  VERSION = "2.0.0"
 end

data/lib/spatial_features/workers/{arcgis_update_features_job.rb → update_features_job.rb}

@@ -1,4 +1,4 @@
-class ArcGISUpdateFeaturesJob < Struct.new(:options)
+class UpdateFeaturesJob < Struct.new(:options)
   def perform
     model = options[:spatial_model_type].find(options[:spatial_model_id])
 
@@ -22,7 +22,7 @@ class ArcGISUpdateFeaturesJob < Struct.new(:options)
     normalized_messages = []
 
     if message =~ /invalid KML representation/
-      normalized_messages += invalid_kml_reason(message).presence || ["invalid KML being generated by ArcGIS."]
+      normalized_messages += invalid_kml_reason(message).presence || ["KML importer received invalid geometry."]
     end
 
     if message =~ /Self-intersection/

data/lib/spatial_features.rb

@@ -1,13 +1,28 @@
+# Gems
+require 'rgeo/shapefile'
+require 'nokogiri'
+require 'zip'
+
 # LIB
 require 'spatial_features/caching'
 require 'spatial_features/venn_polygons'
-require 'spatial_features/has_spatial_features'
+require 'spatial_features/controller_helpers/spatial_extensions'
+require 'spatial_features/download'
+require 'spatial_features/unzip'
 
-require 'spatial_features/import/arcgis_kmz_features'
+require 'spatial_features/has_spatial_features'
+require 'spatial_features/has_spatial_features/feature_import'
+require 'spatial_features/has_spatial_features/delayed_feature_import'
 
-require 'spatial_features/controller_helpers/spatial_extensions'
+require 'spatial_features/importers/base'
+require 'spatial_features/importers/file'
+require 'spatial_features/importers/kml'
+require 'spatial_features/importers/kml_file'
+require 'spatial_features/importers/kml_file_arcgis'
+require 'spatial_features/importers/geomark'
+require 'spatial_features/importers/shapefile'
 
-require 'spatial_features/workers/arcgis_update_features_job'
+require 'spatial_features/workers/update_features_job'
 
 require 'spatial_features/engine'
 
@@ -19,3 +34,6 @@ end
 
 # Load the act method
 ActiveRecord::Base.send :extend, SpatialFeatures::ActMethod
+
+# Suppress date warnings when unzipping KMZ saved by Google Earth, see https://github.com/rubyzip/rubyzip/issues/112
+Zip.warn_invalid_date = false

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Ryan Wallace
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-12 00:00:00.000000000 Z
+date: 2016-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -18,9 +18,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,23 +25,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.0
 - !ruby/object:Gem::Dependency
   name: delayed_job_active_record
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.0.3
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: rgeo-shapefile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.0.3
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -63,16 +99,16 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
 description: Adds spatial methods to a model.
 email:
 - contact@culturecode.ca
@@ -90,12 +126,22 @@ files:
 - lib/spatial_features.rb
 - lib/spatial_features/caching.rb
 - lib/spatial_features/controller_helpers/spatial_extensions.rb
+- lib/spatial_features/download.rb
 - lib/spatial_features/engine.rb
 - lib/spatial_features/has_spatial_features.rb
-- lib/spatial_features/import/arcgis_kmz_features.rb
+- lib/spatial_features/has_spatial_features/delayed_feature_import.rb
+- lib/spatial_features/has_spatial_features/feature_import.rb
+- lib/spatial_features/importers/base.rb
+- lib/spatial_features/importers/file.rb
+- lib/spatial_features/importers/geomark.rb
+- lib/spatial_features/importers/kml.rb
+- lib/spatial_features/importers/kml_file.rb
+- lib/spatial_features/importers/kml_file_arcgis.rb
+- lib/spatial_features/importers/shapefile.rb
+- lib/spatial_features/unzip.rb
 - lib/spatial_features/venn_polygons.rb
 - lib/spatial_features/version.rb
-- lib/spatial_features/workers/arcgis_update_features_job.rb
+- lib/spatial_features/workers/update_features_job.rb
 - lib/tasks/spatial_features_tasks.rake
 homepage: https://github.com/culturecode/spatial_features
 licenses:
@@ -117,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Adds spatial methods to a model.

data/lib/spatial_features/import/arcgis_kmz_features.rb

@@ -1,144 +0,0 @@
-module ArcGISKmzFeatures
-  require 'open-uri'
-  require 'digest/md5'
-
-  def update_features!(options = {})
-    @skip_invalid = options[:skip_invalid]
-    @make_valid = options[:make_valid]
-    @feature_error_messages = []
-    kml_array = []
-    cache_kml = ''
-
-    Array(arcgis_kmz_url).each do |url|
-      kml_array << open_kmz_url(url)
-      cache_kml << kml_array.last.to_s
-    end
-
-    if has_spatial_features_hash?
-      new_features_hash = Digest::MD5.hexdigest(cache_kml) if cache_kml.present?
-
-      if new_features_hash != self.features_hash
-        replace_features(kml_array)
-        update_attributes(:features_hash => new_features_hash)
-      else
-        return false
-      end
-    else
-      replace_features(kml_array)
-    end
-
-    return true
-  end
-
-  def queue_feature_update!(options = {})
-    Delayed::Job.enqueue ArcGISUpdateFeaturesJob.new(options.merge :spatial_model_type => self.class, :spatial_model_id => self.id), :queue => delayed_jobs_queue_name
-  end
-
-  def updating_features?
-    running_feature_update_jobs.exists?
-  end
-
-  def feature_update_error
-    (failed_feature_update_jobs.first.try(:last_error) || '').split("\n").first
-  end
-
-  def running_feature_update_jobs
-    feature_update_jobs.where(failed_at: nil)
-  end
-
-  def failed_feature_update_jobs
-    feature_update_jobs.where.not(failed_at: nil)
-  end
-
-  def feature_update_jobs
-    Delayed::Job.where(queue: delayed_jobs_queue_name)
-  end
-
-  private
-
-  def delayed_jobs_queue_name
-    "#{self.class}/#{self.id}/update_features"
-  end
-
-  def replace_features(kml_array)
-    new_features = []
-    kml_array.each {|kml| new_features.concat build_features(kml) }
-
-    ActiveRecord::Base.transaction do
-      self.features.destroy_all
-      new_features.each(&:save)
-      self.clear_association_cache # clear_association_cache so after_feature_update knows about the new features
-
-      @feature_error_messages.concat new_features.collect {|feature| "Feature #{feature.name}: #{feature.errors.full_messages.to_sentence}" if feature.errors.present? }.compact.flatten
-      if @feature_error_messages.present? && !@skip_invalid
-        raise UpdateError, "Error updating #{self.class} #{self.id}. #{@feature_error_messages.to_sentence}"
-      end
-    end
-  end
-
-  def build_features(kml)
-    new_type_features = []
-
-    extract_kml_features(kml) do |feature_type, feature, name, metadata|
-      begin
-        new_type_features << build_feature(feature_type, name, metadata, build_geom(feature))
-      rescue => e
-        @feature_error_messages << e.message
-      end
-    end
-
-    return new_type_features
-  end
-
-  # Use ST_Force_2D to discard z-coordinates that cause failures later in the process
-  def build_geom(feature)
-    if make_valid?
-      geom = ActiveRecord::Base.connection.select_value("SELECT ST_CollectionExtract(ST_MakeValid(ST_Force_2D(ST_GeomFromKML('#{feature}'))),3)")
-    else
-      geom = ActiveRecord::Base.connection.select_value("SELECT ST_Force_2D(ST_GeomFromKML('#{feature}'))")
-    end
-  end
-
-  def extract_kml_features(kml, &block)
-    Nokogiri::XML(kml).css('Placemark').each do |placemark|
-      name = placemark.css('name').text
-      metadata = Hash[Nokogiri::XML(placemark.css('description').text).css('td').collect(&:text).each_slice(2).to_a]
-
-      {'Polygon' => 'POLYGON', 'LineString' => 'LINE', 'Point' => 'POINT'}.each do |kml_type, sql_type|
-        placemark.css(kml_type).each do |feature|
-          yield sql_type, feature, name, metadata
-        end
-      end
-    end
-  end
-
-  def build_feature(feature_type, name, metadata, geom)
-    Feature.new(:spatial_model => self, :name => name, :metadata => metadata, :feature_type => feature_type, :geog => geom)
-  end
-
-  def open_kmz_url(url)
-    url = URI(url)
-
-    Zip::InputStream.open(open(url)) do |io|
-      while (entry = io.get_next_entry)
-        return io.read if entry.name.downcase == 'doc.kml'
-      end
-    end
-
-    return nil
-
-  rescue SocketError, Errno::ECONNREFUSED
-    raise UpdateError, "ArcGIS Server is not responding. Ensure ArcGIS Server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
-  rescue OpenURI::HTTPError
-    raise UpdateError, "ArcGIS Map Service not found. Ensure ArcGIS Server is running and accessible at #{url}."
-  rescue => e
-    raise UpdateError, e.message
-  end
-
-  # Can be overridden to use PostGIS to force geometry to be valid
-  def make_valid?
-    !!@make_valid
-  end
-
-  class UpdateError < StandardError; end
-end