spatial_features 2.18.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41b8326969b8e58e74200ad16194cbc2e6307ba9a1e1b81cc1bc0339ead5ecd5
-  data.tar.gz: aeee4a1a92da17454c61f853cd6cb9917b727dbc4a5bc8095cc74d17846642f5
+  metadata.gz: 6d3fe76f0ce82837de428e401ce4f6a20a84b97acb2085374a8bf8fe1daba5c8
+  data.tar.gz: fed058553913c07b27838424dc07b1b63677f28798fae7d5534b55de7c1aa713
 SHA512:
-  metadata.gz: 4f793c92a7c6018655bf8cd20f77f0b7b3b106cd8922fede6090c7c3e8513df966fedddbd1f1e5614a9b14f41b47fd3658e0b8db642cd7287138b611b398735a
-  data.tar.gz: 4b965732b239b2fde43d1449fe31862255d03c5e84f34011707cf26ffffd3b9775646712f754863305f1375e3cb64136bef028558c40dc0f823d58db69408eed
+  metadata.gz: 0f6aebd3553502127a91f68d5f692bd6311a8c7af0aca22bccc54cec712aa8371696b0f8e19c0c7aaffa9e98c1ee809d478612a5f7c2441452a202bc9f53280a
+  data.tar.gz: fcf4f4eb0fc85f82ff726ec950936b9c45bdd9aaf85eb06067acac71b250185b6640e89f3536e5e5e1759b93efbee66e855460964ba112aa4f26a77cefae0abe

data/README.md

@@ -30,10 +30,9 @@ Adds spatial methods to a model.
         name character varying(255),
         feature_type character varying(255),
         geog geography,
-        geom geometry(Geometry,3005),
-        geom_lowres geometry(Geometry,3005),
-        kml text,
-        kml_lowres text,
+        geom geometry(Geometry,4326),
+        geom_lowres geometry(Geometry,4326),
+        tilegeom geometry(Geometry,3857),
         metadata hstore,
         area double precision,
         north numeric(9,6),
@@ -41,7 +40,6 @@ Adds spatial methods to a model.
         south numeric(9,6),
         west numeric(9,6),
         centroid geography,
-        kml_centroid text
     );
 
     CREATE SEQUENCE features_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1;
@@ -53,6 +51,7 @@ Adds spatial methods to a model.
     CREATE INDEX index_features_on_spatial_model_id_and_spatial_model_type ON features USING btree (spatial_model_id, spatial_model_type);
     CREATE INDEX index_features_on_geom ON features USING gist (geom);
     CREATE INDEX index_features_on_geom_lowres ON features USING gist (geom_lowres);
+    CREATE INDEX index_features_on_tilegeom ON features USING gist (tilegeom);
 
     CREATE TABLE spatial_caches (
         id integer NOT NULL,
@@ -109,8 +108,17 @@ Person.new(:features => [Feature.new(:geog => 'some binary PostGIS Geography str
 You can specify multiple import sources for geometry. Each key is a method that returns the data for the Importer, and
 each value is the Importer to use to parse the data. See each Importer for more details.
 ```ruby
+def ImageImporter
+  def self.call(feature, image_paths)
+    image_paths.each do |pathname|
+      # ...
+    end
+  end
+end
+
 class Location < ActiveRecord::Base
-  has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File', :geojson => 'GeoJSON' }
+  has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File', :geojson => 'ESRIGeoJSON' },
+                       :image_handlers => ['ImageImporter']
 
   def remote_kml_url
     "www.test.com/kml/#{id}.kml"
@@ -154,3 +162,22 @@ Feature.reset_column_information
 AbstractFeature.update_all(:type => 'Feature')
 Feature.refresh_aggregates
 ```
+
+## Upgrading From 2.8.x to 3.0
+Cached KML layers are no longer generated as Mapbox Vector Tile is the primary expected output. The columns can be left
+in place or you can remove the KML cache columns.
+
+```ruby
+remove_column :features, :kml
+remove_column :features, :kml_lowres
+remove_column :features, :kml_centroid
+```
+
+A new `tilegeom` column has been added to support MVT tile output, which is now the preferred map output format instead
+of GeoJSON or KML. It keeps memory usage low and is fast to generate.
+
+```ruby
+add_column :features, :tilegeom, :geometry
+add_index :features, :tilegeom, :using => :gist
+Feature.update_all('tilegeom = ST_Transform(geom, 3857)')
+```

data/app/models/abstract_feature.rb

@@ -117,7 +117,8 @@ class AbstractFeature < ActiveRecord::Base
     SQL
 
     update_all <<-SQL.squish
-      geom_lowres  = ST_SimplifyPreserveTopology(geom, #{options.fetch(:lowres_simplification, lowres_simplification)})
+      geom_lowres  = ST_SimplifyPreserveTopology(geom, #{options.fetch(:lowres_simplification, lowres_simplification)}),
+      tilegeom     = ST_Transform(geom, 3857)
     SQL
 
     invalid('geom_lowres').update_all <<-SQL.squish
@@ -125,7 +126,44 @@ class AbstractFeature < ActiveRecord::Base
     SQL
   end
 
-  def self.geojson(lowres: false, precision: 6, properties: true, srid: 4326, centroids: false) # default srid is 4326 so output is Google Maps compatible
+  def self.mvt(*args)
+    select_sql = mvt_sql(*args)
+
+    # Result is a hex string representing the desired binary output so we need to convert it to binary
+    result = SpatialFeatures::Utils.select_db_value(select_sql)
+    result.remove!(/^\\x/)
+    result = result.scan(/../).map(&:hex).pack('c*')
+
+    return result
+  end
+
+  def self.mvt_sql(tile_x, tile_y, zoom, properties: true, centroids: false, metadata: {}, scope: nil)
+    if centroids
+      column = 'ST_Transform(centroid::geometry, 3857)' # MVT works in SRID 3857
+    else
+      column = 'tilegeom'
+    end
+
+    subquery = select(:id)
+                .select("ST_AsMVTGeom(#{column}, ST_TileEnvelope(#{zoom}, #{tile_x}, #{tile_y}), extent => 4096, buffer => 64) AS geom")
+                .where("#{column} && ST_TileEnvelope(#{zoom}, #{tile_x}, #{tile_y}, margin => (64.0 / 4096))")
+                .order(:id)
+
+    # Merge additional scopes in to allow joins and other columns to be included in the feature output
+    subquery = subquery.merge(scope) unless scope.nil?
+
+    # Add metadata
+    metadata.each do |column, value|
+      subquery = subquery.select("#{value} AS #{column}")
+    end
+
+    select_sql = <<~SQL
+      SELECT ST_AsMVT(mvtgeom.*, 'default', 4096, 'geom', 'id') AS mvt
+      FROM (#{subquery.to_sql}) mvtgeom
+    SQL
+  end
+
+  def self.geojson(lowres: false, precision: 6, properties: true, srid: 4326, centroids: false, features_only: false, include_record_identifiers: false) # default srid is 4326 so output is Google Maps compatible
     if centroids
       column = 'centroid'
     elsif lowres
@@ -134,27 +172,52 @@ class AbstractFeature < ActiveRecord::Base
       column = 'geog'
     end
 
-    properties_sql = <<~SQL if properties
-      , 'properties', hstore_to_json(metadata)
+    properties_sql = []
+
+    if include_record_identifiers
+      properties_sql << "hstore(ARRAY['feature_name', name::varchar, 'feature_id', id::varchar, 'spatial_model_type', spatial_model_type::varchar, 'spatial_model_id', spatial_model_id::varchar])"
+    end
+
+    if properties
+      properties_sql << "metadata"
+    end
+
+    if properties.is_a?(Hash)
+      properties_sql << <<~SQL
+        hstore(ARRAY[#{properties.flatten.map {|e| "'#{e.to_s}'" }.join(',')}])
+      SQL
+    end
+
+    properties_sql = <<~SQL if properties_sql.present?
+      , 'properties', hstore_to_json(#{properties_sql.join(' || ')})
     SQL
 
     sql = <<~SQL
+      json_agg(
+        json_build_object(
+          'type', 'Feature',
+          'geometry', ST_AsGeoJSON(#{column}, #{precision})::json
+          #{properties_sql}
+        )
+      )
+    SQL
+
+    sql = <<~SQL unless features_only
       json_build_object(
         'type', 'FeatureCollection',
-        'features', json_agg(
-          json_build_object(
-            'type', 'Feature',
-            'geometry', ST_AsGeoJSON(#{column}, #{precision})::json
-            #{properties_sql}
-          )
-        )
+        'features', #{sql}
       )
     SQL
     SpatialFeatures::Utils.select_db_value(all.select(sql))
   end
 
-  def feature_bounds
-    slice(:north, :east, :south, :west)
+  def self.bounds
+    values = pluck('MAX(north) AS north, MAX(east) AS east, MIN(south) AS south, MIN(west) AS west').first
+    [:north, :east, :south, :west].zip(values).to_h.with_indifferent_access.transform_values!(&:to_f) if values&.compact.present?
+  end
+
+  def bounds
+    slice(:north, :east, :south, :west).with_indifferent_access.transform_values!(&:to_f)
   end
 
   def cache_derivatives(*args)
@@ -162,9 +225,8 @@ class AbstractFeature < ActiveRecord::Base
   end
 
   def kml(options = {})
-    geometry = options[:lowres] ? kml_lowres : super()
-    geometry = "<MultiGeometry>#{geometry}#{kml_centroid}</MultiGeometry>" if options[:centroid]
-    return geometry
+    column = options[:lowres] ? 'geom_lowres' : 'geog'
+    return SpatialFeatures::Utils.select_db_value(self.class.where(:id => id).select("ST_AsKML(#{column}, 6)"))
   end
 
   def geojson(*args)
@@ -210,7 +272,7 @@ class AbstractFeature < ActiveRecord::Base
 
   def geometry_validation_message
     klass = self.class.base_class # Use the base class because we don't want to have to include a type column in our select
-    error = klass.connection.select_one(klass.unscoped.invalid.from("(SELECT '#{sanitize_input_for_sql(self.geog)}'::geometry AS geog) #{klass.table_name}"))
+    error = klass.connection.select_one(klass.unscoped.invalid.from("(SELECT '#{sanitize_input_for_sql(self.geog)}'::geography::geometry AS geog) #{klass.table_name}")) # Ensure we cast to geography because the geog attribute value may not have been coerced to geography yet, so we want it to apply the +-180/90 bounds to any odd geometry that will happen when we save to the database
     return error.fetch('invalid_geometry_message') if error
   end
 

data/app/models/aggregate_feature.rb

@@ -4,7 +4,11 @@ class AggregateFeature < AbstractFeature
   has_many :features, lambda { |aggregate| where(:spatial_model_type => aggregate.spatial_model_type) }, :foreign_key => :spatial_model_id, :primary_key => :spatial_model_id
 
   # Aggregate the features for the spatial model into a single feature
-  def refresh
+  before_validation :set_geog, :on => :create, :unless => :geog?
+
+  private
+
+  def set_geog
     feature_array_sql = <<~SQL
       ARRAY[
         (#{features.select('ST_UNION(ST_CollectionExtract(geog::geometry, 1))').to_sql}),
@@ -19,6 +23,5 @@ class AggregateFeature < AbstractFeature
       FROM (SELECT unnest(#{feature_array_sql})) AS features
       WHERE NOT ST_IsEmpty(unnest)
     SQL
-    self.save!
   end
 end

data/app/models/feature.rb

@@ -1,3 +1,5 @@
+require_dependency SpatialFeatures::Engine.root.join('app/models/abstract_feature')
+
 class Feature < AbstractFeature
   class_attribute :automatically_refresh_aggregate
   self.automatically_refresh_aggregate = true
@@ -13,15 +15,7 @@ class Feature < AbstractFeature
 
   after_save :refresh_aggregate, if: :automatically_refresh_aggregate?
 
-  # Features are used for display so we also cache their KML representation
-  def self.cache_derivatives(options = {})
-    super
-    update_all <<-SQL.squish
-      kml          = ST_AsKML(geog, 6),
-      kml_lowres   = ST_AsKML(geom_lowres, #{options.fetch(:lowres_precision, lowres_precision)}),
-      kml_centroid = ST_AsKML(centroid)
-    SQL
-  end
+  attr_accessor :importable_image_paths # :nodoc:
 
   def self.defer_aggregate_refresh(&block)
     start_at = Feature.maximum(:id).to_i + 1
@@ -52,8 +46,8 @@ class Feature < AbstractFeature
   end
 
   def refresh_aggregate
-    build_aggregate_feature unless aggregate_feature&.persisted?
-    aggregate_feature.refresh
+    aggregate_feature&.destroy # Destroy the existing aggregate feature to ensure its cache key changes when it is refreshed
+    create_aggregate_feature!
   end
 
   def automatically_refresh_aggregate?

data/config/initializers/mime_types.rb

@@ -1 +1,4 @@
 Mime::Type.register "application/vnd.geo+json", :geojson
+Mime::Type.register "application/vnd.mapbox-vector-tile", :mvt
+Mime::Type.register "application/vnd.google-earth.kml+xml", :kml
+Mime::Type.register "application/vnd.google-earth.kmz", :kmz

data/lib/spatial_features/controller_helpers/spatial_extensions.rb

@@ -12,11 +12,13 @@ module SpatialExtensions
     @nearby_records = scope_for_search(scope).within_buffer(target, distance, :distance => true, :intersection_area => true).order('distance_in_meters ASC, intersection_area_in_square_meters DESC, id ASC')
     @target = target
 
-    yield if block_given?
-
-    respond_to do |format|
-      format.html { render :template => 'shared/spatial/feature_proximity', :layout => false }
-      format.kml { render :template => 'shared/spatial/feature_proximity' }
+    if block_given?
+      block.call(@nearby_records)
+    else
+      respond_to do |format|
+        format.html { render :template => 'shared/spatial/feature_proximity', :layout => false }
+        format.kml { render :template => 'shared/spatial/feature_proximity' }
+      end
     end
   end
 
@@ -25,10 +27,12 @@ module SpatialExtensions
     @klass = klass_for_search(scope)
     @target = target
 
-    yield if block_given?
-
-    respond_to do |format|
-      format.kml { render :template => 'shared/spatial/feature_venn_polygons' }
+    if block_given?
+      block.call(@venn_polygons)
+    else
+      respond_to do |format|
+        format.kml { render :template => 'shared/spatial/feature_venn_polygons' }
+      end
     end
   end
 
@@ -37,6 +41,12 @@ module SpatialExtensions
   end
 
   def scope_for_search(scope)
-    params.key?(:ids) ? scope.where(:id => params[:ids]) : scope
+    if params.key?(:ids)
+      ids = params[:ids]
+      ids = ids.split(/\D/) if ids.is_a?(String)
+      scope.where(:id => ids)
+    else
+      scope
+    end
   end
 end

data/lib/spatial_features/has_spatial_features/feature_import.rb

@@ -8,7 +8,7 @@ module SpatialFeatures
     included do
       extend ActiveModel::Callbacks
       define_model_callbacks :update_features
-      spatial_features_options.reverse_merge!(:import => {}, spatial_cache: [])
+      spatial_features_options.reverse_merge!(:import => {}, :spatial_cache => [], :image_handlers => [])
     end
 
     module ClassMethods
@@ -42,6 +42,13 @@ module SpatialFeatures
 
         return true
       end
+    rescue StandardError => e
+      if skip_invalid
+        Rails.logger.warn "Error updating #{self.class} #{self.id}. #{e.message}"
+        return nil
+      else
+        raise ImportError, e.message, e.backtrace
+      end
     end
 
     def update_features_cache_key(cache_key)
@@ -78,13 +85,30 @@ module SpatialFeatures
       "SpatialFeatures::Importers::#{importer_name}".constantize
     end
 
+    def handle_images(feature)
+      return if feature.importable_image_paths.nil? || feature.importable_image_paths.empty?
+
+      Array(spatial_features_options[:image_handlers]).each do |image_handler|
+        image_handler_from_name(image_handler).call(feature, feature.importable_image_paths)
+      end
+    end
+
+    def image_handler_from_name(handler_name)
+      handler_name.to_s.constantize
+    end
+
     def import_features(imports, skip_invalid)
       features.delete_all
       valid, invalid = Feature.defer_aggregate_refresh do
         Feature.without_caching_derivatives do
           imports.flat_map(&:features).partition do |feature|
             feature.spatial_model = self
-            feature.save
+            if feature.save
+              handle_images(feature)
+              true
+            else
+              false
+            end
           end
         end
       end

data/lib/spatial_features/has_spatial_features/queued_spatial_processing.rb

@@ -41,8 +41,8 @@ module SpatialFeatures
       spatial_processing_jobs('update_features!').where.not(failed_at: nil)
     end
 
-    def spatial_processing_jobs(suffix = nil)
-      Delayed::Job.where(:queue => "#{spatial_processing_queue_name}#{suffix}")
+    def spatial_processing_jobs(method_name = nil)
+      Delayed::Job.where('queue LIKE ?', "#{spatial_processing_queue_name}#{method_name}%")
     end
 
     private
@@ -77,7 +77,7 @@ module SpatialFeatures
         update_cached_status(:success)
       end
 
-      def error(job)
+      def error(job, exception)
         update_cached_status(:failure)
       end
 

data/lib/spatial_features/has_spatial_features.rb

@@ -15,6 +15,8 @@ module SpatialFeatures
 
         scope :with_features, lambda { joins(:features).uniq }
         scope :without_features, lambda { joins("LEFT OUTER JOIN features ON features.spatial_model_type = '#{Utils.base_class(name)}' AND features.spatial_model_id = #{table_name}.id").where("features.id IS NULL") }
+        scope :include_bounds, lambda { SQLHelpers.append_select(joins(:aggregate_feature), :north, :east, :south, :west) }
+        scope :include_area, lambda { SQLHelpers.append_select(joins(:aggregate_feature), :area) }
 
         scope :with_spatial_cache, lambda {|klass| joins(:spatial_caches).where(:spatial_caches => { :intersection_model_type =>  Utils.class_name_with_ancestors(klass) }).uniq }
         scope :without_spatial_cache, lambda {|klass| joins("LEFT OUTER JOIN #{SpatialCache.table_name} ON #{SpatialCache.table_name}.spatial_model_id = #{table_name}.id AND #{SpatialCache.table_name}.spatial_model_type = '#{Utils.base_class(name)}' and intersection_model_type IN ('#{Utils.class_name_with_ancestors(klass).join("','") }')").where("#{SpatialCache.table_name}.spatial_model_id IS NULL") }
@@ -31,6 +33,16 @@ module SpatialFeatures
     end
   end
 
+  module SQLHelpers
+    # Add select fields without replacing the implicit `table_name.*`
+    def self.append_select(scope, *fields)
+      if(!scope.select_values.any?)
+        fields.unshift(scope.arel_table[Arel.star])
+      end
+      scope.select(*fields)
+    end
+  end
+
   module ClassMethods
     def acts_like_spatial_features?
       true
@@ -44,7 +56,7 @@ module SpatialFeatures
       within_buffer(other, 0, options)
     end
 
-    def within_buffer(other, buffer_in_meters = 0, options = {})
+    def within_buffer(other, buffer_in_meters = 0, **options)
       return none if other.is_a?(ActiveRecord::Base) && other.new_record?
 
       # Cache only works on single records, not scopes.
@@ -68,6 +80,10 @@ module SpatialFeatures
       features.points
     end
 
+    def bounds
+      aggregate_features.bounds
+    end
+
     def features
       type = base_class.to_s # Rails stores polymorphic foreign keys as the base class
       if all == unscoped
@@ -97,24 +113,22 @@ module SpatialFeatures
     end
 
     def area_in_square_meters
-      features.area
+      abstract_features.area
     end
 
     private
 
     def cached_within_buffer_scope(other, buffer_in_meters, options)
       options = options.reverse_merge(:columns => "#{table_name}.*")
-
-      # Don't use the cache if it doesn't exist
-      unless other.class.unscoped { other.spatial_cache_for?(Utils.class_of(self), buffer_in_meters) } # Unscope so if we're checking for same class intersections the scope doesn't affect this lookup
-        return none.extending(UncachedResult)
-      end
-
       scope = cached_spatial_join(other)
       scope = scope.select(options[:columns])
       scope = scope.where("spatial_proximities.distance_in_meters <= ?", buffer_in_meters) if buffer_in_meters
       scope = scope.select("spatial_proximities.distance_in_meters") if options[:distance]
       scope = scope.select("spatial_proximities.intersection_area_in_square_meters") if options[:intersection_area]
+
+      # Don't use the cache if it doesn't exist, but make sure we keep the same select columns in case chained scopes reference them, e.g. order by
+      scope = scope.none.extending(UncachedResult) unless other.class.unscoped { other.spatial_cache_for?(Utils.class_of(self), buffer_in_meters) } # Unscope so if we're checking for same class intersections the scope doesn't affect this lookup
+
       return scope
     end
 
@@ -174,7 +188,7 @@ module SpatialFeatures
     end
 
     def features_cache_key
-      "#{self.class.name}/#{id}-#{has_spatial_features_hash? ? features_hash : aggregate_feature.cache_key}"
+      "#{self.class.name}/#{id}-#{has_spatial_features_hash? ? features_hash : (aggregate_feature || features).cache_key}"
     end
 
     def polygons?
@@ -202,12 +216,15 @@ module SpatialFeatures
     end
 
     def bounds
-      if association(:aggregate_feature).loaded?
-        aggregate_feature&.feature_bounds
-      else
-        result = aggregate_features.pluck(:north, :east, :south, :west).first
-        [:north, :east, :south, :west].zip(result.map {|bound| BigDecimal(bound) }).to_h.with_indifferent_access if result
-      end
+      @bounds ||=
+        if has_attribute?(:north) && has_attribute?(:east) && has_attribute?(:south) && has_attribute?(:west)
+          slice(:north, :east, :south, :west).with_indifferent_access.transform_values!(&:to_f)
+        elsif association(:aggregate_feature).loaded?
+          # Aggregate features can be very large and take a while to load. Avoid loading one just to load the bounds.
+          aggregate_feature&.bounds
+        else
+          aggregate_features.bounds
+        end
     end
 
     def total_intersection_area_percentage(klass)
@@ -217,11 +234,14 @@ module SpatialFeatures
     end
 
     def features_area_in_square_meters
-      if association(:aggregate_feature).loaded?
-        aggregate_feature&.area
-      else
-        aggregate_features.pluck(:area).first
-      end
+      @features_area_in_square_meters ||=
+        if has_attribute?(:area)
+          area
+        elsif association(:aggregate_feature).loaded?
+          aggregate_feature&.area
+        else
+          aggregate_features.pluck(:area).first
+        end
     end
 
     def total_intersection_area_in_square_meters(other)

data/lib/spatial_features/importers/base.rb

@@ -46,7 +46,8 @@ module SpatialFeatures
       end
 
       def build_feature(record)
-        Feature.new(:name => record.name, :metadata => record.metadata, :feature_type => record.feature_type, :geog => record.geog, :make_valid => @make_valid)
+        importable_image_paths = record.importable_image_paths if record.respond_to?(:importable_image_paths)
+        Feature.new(:name => record.name, :metadata => record.metadata, :feature_type => record.feature_type, :geog => record.geog, :importable_image_paths => importable_image_paths, :make_valid => @make_valid)
       end
     end
   end

data/lib/spatial_features/importers/esri_geo_json.rb

@@ -0,0 +1,27 @@
+require 'ostruct'
+require 'digest/md5'
+require 'spatial_features/importers/geo_json'
+
+module SpatialFeatures
+  module Importers
+    class ESRIGeoJSON < GeoJSON
+      def parsed_geojson
+        @parsed_geojson ||= JSON.parse(geojson)
+      end
+
+      def geojson
+        @geojson ||= esri_json_to_geojson(@data)
+      end
+
+      private
+
+      def esri_json_to_geojson(url)
+        if URI.parse(url).relative?
+          `ogr2ogr -t_srs EPSG:4326 -f GeoJSON /dev/stdout "#{url}"` # It is a local file path
+        else
+          `ogr2ogr -t_srs EPSG:4326 -f GeoJSON /dev/stdout "#{url}" OGRGeoJSON`
+        end
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/file.rb

@@ -3,14 +3,16 @@ require 'open-uri'
 module SpatialFeatures
   module Importers
     class File < SimpleDelegator
-      INVALID_ARCHIVE = "Archive did not contain a .kml or .shp file. Supported formats are KMZ, KML, and zipped ArcGIS shapefiles.".freeze
+      INVALID_ARCHIVE = "Archive did not contain a .kml, .shp, .json, or .geojson file.".freeze
+      SUPPORTED_FORMATS = "Supported formats are KMZ, KML, zipped ArcGIS shapefiles, ESRI JSON, and GeoJSON.".freeze
 
+      FILE_PATTERNS = [/\.kml$/, /\.shp$/, /\.json$/, /\.geojson$/]
       def self.create_all(data, **options)
-        Download.open_each(data, unzip: [/\.kml$/, /\.shp$/], downcase: true).map do |file|
+        Download.open_each(data, unzip: FILE_PATTERNS, downcase: true).map do |file|
           new(data, **options, current_file: file)
         end
       rescue Unzip::PathNotFound
-        raise ImportError, INVALID_ARCHIVE
+        raise ImportError, INVALID_ARCHIVE + " " + SUPPORTED_FORMATS
       end
 
       # The File importer may be initialized multiple times by `::create_all` if it
@@ -21,7 +23,7 @@ module SpatialFeatures
       # If no `current_file` is passed then we just take the first valid file that we find.
       def initialize(data, *args, current_file: nil, **options)
         begin
-          current_file ||= Download.open_each(data, unzip: [/\.kml$/, /\.shp$/], downcase: true).first
+          current_file ||= Download.open_each(data, unzip: FILE_PATTERNS, downcase: true).first
         rescue Unzip::PathNotFound
           raise ImportError, INVALID_ARCHIVE
         end
@@ -31,8 +33,10 @@ module SpatialFeatures
           __setobj__(KMLFile.new(current_file, *args, **options))
         when '.shp'
           __setobj__(Shapefile.new(current_file, *args, **options))
+        when '.json', '.geojson'
+          __setobj__(ESRIGeoJSON.new(current_file.path, *args, **options))
         else
-          raise ImportError, "Could not import file. Supported formats are KMZ, KML, and zipped ArcGIS shapefiles"
+          raise ImportError, "Could not import file. " + SUPPORTED_FORMATS
         end
       end
     end

data/lib/spatial_features/importers/{geojson.rb → geo_json.rb}

@@ -5,15 +5,13 @@ module SpatialFeatures
   module Importers
     class GeoJSON < Base
       def cache_key
-        @cache_key ||= Digest::MD5.hexdigest(@data.to_json)
+        @cache_key ||= Digest::MD5.hexdigest(geojson)
       end
 
       private
 
       def each_record(&block)
-        return unless @data
-
-        @data.fetch('features', []).each do |record|
+        parsed_geojson.fetch('features', []).each do |record|
           metadata = record['properties'] || {}
           name = metadata.delete('name')
           yield OpenStruct.new(
@@ -24,6 +22,14 @@ module SpatialFeatures
           )
         end
       end
+
+      def parsed_geojson
+        @parsed_geojson ||= (@data.is_a?(String) ? JSON.parse(@data) : @data) || {}
+      end
+
+      def geojson
+        @geojson ||= @data.is_a?(String) ? @data : @data.to_json
+      end
     end
   end
 end

data/lib/spatial_features/importers/kml.rb

@@ -3,6 +3,14 @@ require 'ostruct'
 module SpatialFeatures
   module Importers
     class KML < Base
+      # <SimpleData name> keys that may contain <img> tags
+      IMAGE_METADATA_KEYS = %w[pdfmaps_photos].freeze
+
+      def initialize(data, base_dir: nil, **args)
+        @base_dir = base_dir
+        super data, **args
+      end
+
       private
 
       def each_record(&block)
@@ -18,10 +26,11 @@ module SpatialFeatures
             next if blank_feature?(feature)
 
             geog = geom_from_kml(feature)
-
             next if geog.blank?
 
-            yield OpenStruct.new(:feature_type => sql_type, :geog => geog, :name => name, :metadata => metadata)
+            importable_image_paths = images_from_metadata(metadata)
+
+            yield OpenStruct.new(:feature_type => sql_type, :geog => geog, :name => name, :metadata => metadata, :importable_image_paths => importable_image_paths)
           end
         end
       end
@@ -32,17 +41,35 @@ module SpatialFeatures
 
       def geom_from_kml(kml)
         geom = nil
+        conn = nil
 
         # Do query in a new thread so we use a new connection (if the query fails it will poison the transaction of the current connection)
+        #
+        # We manually checkout a new connection since Rails re-uses DB connections across threads.
         Thread.new do
-          geom = SpatialFeatures::Utils.select_db_value("SELECT ST_GeomFromKML(#{ActiveRecord::Base.connection.quote(kml.to_s)})")
+          conn = ActiveRecord::Base.connection_pool.checkout
+          geom = conn.select_value("SELECT ST_GeomFromKML(#{conn.quote(kml.to_s)})")
         rescue ActiveRecord::StatementInvalid => e # Discard Invalid KML features
           geom = nil
+        ensure
+          ActiveRecord::Base.connection_pool.checkin(conn) if conn
         end.join
 
         return geom
       end
 
+      def images_from_metadata(metadata)
+        IMAGE_METADATA_KEYS.flat_map do |key|
+          images = metadata.delete(key)
+          next unless images
+
+          Nokogiri::HTML.fragment(images).css("img").map do |img|
+            next unless (src = img["src"])
+            @base_dir.join(src.downcase)
+          end
+        end.compact
+      end
+
       def extract_metadata(placemark)
         metadata = {}
         metadata.merge! extract_table(placemark)

data/lib/spatial_features/importers/kml_file.rb

@@ -1,13 +1,17 @@
 module SpatialFeatures
   module Importers
     class KMLFile < KML
-      def initialize(path_or_url, *args)
-        super Download.read(path_or_url, unzip: '.kml'), *args
-
+      def initialize(path_or_url, **options)
+        path = Download.open_each(path_or_url, unzip: [/\.kml$/], downcase: true).first
+        super ::File.read(path), base_dir: Pathname.new(path).dirname, **options
       rescue SocketError, Errno::ECONNREFUSED, OpenURI::HTTPError
         url = URI(path_or_url)
         raise ImportError, "KML server is not responding. Ensure server is running and accessible at #{[url.scheme, "//#{url.host}", url.port].select(&:present?).join(':')}."
       end
+
+      def cache_key
+        @cache_key ||= Digest::MD5.hexdigest(@data)
+      end
     end
   end
 end

data/lib/spatial_features/unzip.rb

@@ -2,6 +2,9 @@ require 'fileutils'
 
 module SpatialFeatures
   module Unzip
+    # paths containing '__macosx' or beginning with a '.'
+    IGNORED_ENTRY_PATHS = /(\A|\/)(__macosx|\.)/i.freeze
+
     def self.paths(file_path, find: nil, **extract_options)
       paths = extract(file_path, **extract_options)
 
@@ -16,6 +19,7 @@ module SpatialFeatures
     def self.extract(file_path, output_dir = Dir.mktmpdir, downcase: false)
       [].tap do |paths|
         entries(file_path).each do |entry|
+          next if entry.name =~ IGNORED_ENTRY_PATHS
           output_filename = entry.name
           output_filename = output_filename.downcase if downcase
           path = "#{output_dir}/#{output_filename}"

data/lib/spatial_features/utils.rb

@@ -59,7 +59,7 @@ module SpatialFeatures
 
     # Convert a hash of GeoJSON data into a PostGIS geometry object
     def geom_from_json(geometry)
-      select_db_value("SELECT ST_GeomFromGeoJSON('#{geometry.to_json}')")
+      RGeo::GeoJSON.decode(geometry.to_json).as_text
     end
   end
 end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "2.18.0"
+  VERSION = "3.0.0"
 end

data/lib/spatial_features.rb

@@ -1,6 +1,7 @@
 # Gems
 require 'delayed_job_active_record'
 require 'rgeo/shapefile'
+require 'rgeo/geo_json'
 require 'nokogiri'
 require 'zip'
 
@@ -20,11 +21,11 @@ require 'spatial_features/has_spatial_features/feature_import'
 
 require 'spatial_features/importers/base'
 require 'spatial_features/importers/file'
-require 'spatial_features/importers/geojson'
+require 'spatial_features/importers/geo_json'
+require 'spatial_features/importers/esri_geo_json'
 require 'spatial_features/importers/kml'
 require 'spatial_features/importers/kml_file'
 require 'spatial_features/importers/kml_file_arcgis'
-require 'spatial_features/importers/json_arcgis'
 require 'spatial_features/importers/geomark'
 require 'spatial_features/importers/shapefile'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 2.18.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Ryan Wallace
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-22 00:00:00.000000000 Z
+date: 2022-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -60,47 +60,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: rgeo-geojson
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 2.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 2.1.1
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: chroma
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -144,7 +144,6 @@ files:
 - app/models/feature.rb
 - app/models/spatial_cache.rb
 - app/models/spatial_proximity.rb
-- config/initializers/chroma_serializers.rb
 - config/initializers/mime_types.rb
 - config/initializers/register_oids.rb
 - lib/spatial_features.rb
@@ -156,10 +155,10 @@ files:
 - lib/spatial_features/has_spatial_features/feature_import.rb
 - lib/spatial_features/has_spatial_features/queued_spatial_processing.rb
 - lib/spatial_features/importers/base.rb
+- lib/spatial_features/importers/esri_geo_json.rb
 - lib/spatial_features/importers/file.rb
-- lib/spatial_features/importers/geojson.rb
+- lib/spatial_features/importers/geo_json.rb
 - lib/spatial_features/importers/geomark.rb
-- lib/spatial_features/importers/json_arcgis.rb
 - lib/spatial_features/importers/kml.rb
 - lib/spatial_features/importers/kml_file.rb
 - lib/spatial_features/importers/kml_file_arcgis.rb
@@ -190,7 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubyforge_project: 
+rubygems_version: 2.7.6.3
 signing_key: 
 specification_version: 4
 summary: Adds spatial methods to a model.

data/config/initializers/chroma_serializers.rb

@@ -1,15 +0,0 @@
-module Chroma
-  class Color
-    module Serializers
-      # Google's Fusion Table colouring expects the alpha value in the last position, not the first
-      def to_ft_hex
-        [
-         to_2char_hex(@rgb.r),
-         to_2char_hex(@rgb.g),
-         to_2char_hex(@rgb.b),
-         to_2char_hex(alpha * 255)
-        ].join('')
-      end
-    end
-  end
-end

data/lib/spatial_features/importers/json_arcgis.rb

@@ -1,29 +0,0 @@
-require 'ostruct'
-require 'digest/md5'
-
-module SpatialFeatures
-  module Importers
-    class JsonArcGIS < Base
-      def cache_key
-        @cache_key ||= Digest::MD5.hexdigest(features.to_json)
-      end
-
-      private
-
-      def each_record(&block)
-        json = esri_json_to_geojson(@data)
-        json['features'].each do |record|
-          yield OpenStruct.new(
-            :feature_type => record['geometry']['type'],
-            :geog => SpatialFeatures::Utils.geom_from_json(record['geometry']),
-            :metadata => record['properties']
-          )
-        end
-      end
-
-      def esri_json_to_geojson(url)
-        JSON.parse(`ogr2ogr -f GeoJSON /dev/stdout "#{url}" OGRGeoJSON`)
-      end
-    end
-  end
-end