spatial_features 2.17.2 → 2.17.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86e67798202370af90674a888afbf1ac4f7847fa24385e1457ebe14e3ff6d4bc
-  data.tar.gz: 75e5c07bb3dc7c3f969f36919d7a468046fec0fcb3683e28f2c862dce9bcee11
+  metadata.gz: e0a8fb7f4886f07f02b8b110f7975d91538324d6cf2c5a70b3b247dfb6719c98
+  data.tar.gz: 15c681c5de6fccb10a515c8ff1f8a423a7c9fa1c1fded8955c2072dd76f5d4f2
 SHA512:
-  metadata.gz: fc4d1aeb966cc2623bd8d324e6ef1bf1fc8e3b4f4bdffddaf123f30e76d827461b0b2277b141ae57a501fd26dc47105aa289771c70f302d1119f5d7270d9ff96
-  data.tar.gz: 41a82aebeef10ff3445348a0859e2db72f42e4b5041fd27b8b72d1368b12d4294e8ec9b270bcf570280f28b23819ed3ad80b066803c0997eca9777f361a778e2
+  metadata.gz: 0a9224909db6dd85bf20cf9b9561e78cdc12f094423c86b0280f0e2a96e2dad9ae24030c4a4014b448f566e232b7dd37620d2a8890c411af97487a0047261346
+  data.tar.gz: 740174a0882bc82edd053e4adf31318c6afc51860a66a3d50068e3c106cc8ac9351061f9d4990c652c5f41704815e2d5b8c8ef3d6b7aaa47525d9663f01acce4

data/README.md

@@ -110,7 +110,7 @@ You can specify multiple import sources for geometry. Each key is a method that
 each value is the Importer to use to parse the data. See each Importer for more details.
 ```ruby
 class Location < ActiveRecord::Base
-  has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File' }
+  has_spatial_features :import => { :remote_kml_url => 'KMLFile', :file => 'File', :geojson => 'GeoJSON' }
 
   def remote_kml_url
     "www.test.com/kml/#{id}.kml"
@@ -119,6 +119,10 @@ class Location < ActiveRecord::Base
   def file
     File.open('local/files/my_kml')
   end
+
+  def geojson
+    { "type" => "FeatureCollection", "features" => [] }
+  end
 end
 ```
 

data/app/models/abstract_feature.rb

@@ -56,7 +56,7 @@ class AbstractFeature < ActiveRecord::Base
 
   def self.area_in_square_meters(geom = 'geom_lowres')
     current_scope = all.polygons
-    unscoped { connection.select_value(select("ST_Area(ST_Union(#{geom}))").from(current_scope, :features)).to_f }
+    unscoped { SpatialFeatures::Utils.select_db_value(select("ST_Area(ST_Union(#{geom}))").from(current_scope, :features)).to_f }
   end
 
   def self.total_intersection_area_in_square_meters(other_features, geom = 'geom_lowres')
@@ -66,7 +66,7 @@ class AbstractFeature < ActiveRecord::Base
     query = base_class.unscoped.select('ST_Area(ST_Intersection(ST_Union(features.geom), ST_Union(other_features.geom)))')
                     .from(scope, "features")
                     .joins("INNER JOIN (#{other_scope.to_sql}) AS other_features ON ST_Intersects(features.geom, other_features.geom)")
-    return connection.select_value(query).to_f
+    return SpatialFeatures::Utils.select_db_value(query).to_f
   end
 
   def self.intersecting(other)
@@ -147,7 +147,7 @@ class AbstractFeature < ActiveRecord::Base
         )
       )
     SQL
-    connection.select_value(all.select(sql))
+    SpatialFeatures::Utils.select_db_value(all.select(sql))
   end
 
   def feature_bounds
@@ -175,17 +175,17 @@ class AbstractFeature < ActiveRecord::Base
   private
 
   def make_valid
-    self.geog = ActiveRecord::Base.connection.select_value("SELECT ST_Buffer('#{sanitize}', 0)")
+    self.geog = SpatialFeatures::Utils.select_db_value("SELECT ST_Buffer('#{sanitize}', 0)")
   end
 
   # Use ST_Force2D to discard z-coordinates that cause failures later in the process
   def sanitize
-    self.geog = ActiveRecord::Base.connection.select_value("SELECT ST_Force2D('#{geog}')")
+    self.geog = SpatialFeatures::Utils.select_db_value("SELECT ST_Force2D('#{geog}')")
   end
 
   SRID_CACHE = {}
   def self.detect_srid(column_name)
-    SRID_CACHE[column_name] ||= connection.select_value("SELECT Find_SRID('public', '#{table_name}', '#{column_name}')")
+    SRID_CACHE[column_name] ||= SpatialFeatures::Utils.select_db_value("SELECT Find_SRID('public', '#{table_name}', '#{column_name}')")
   end
 
   def self.join_other_features(other)

data/app/models/aggregate_feature.rb

@@ -14,7 +14,7 @@ class AggregateFeature < AbstractFeature
     SQL
 
     # Remove empty features so ST_COLLECT doesn't choke. This seems to be a difference between PostGIS 2.x and 3.x
-    self.geog = ActiveRecord::Base.connection.select_value <<~SQL
+    self.geog = SpatialFeatures::Utils.select_db_value <<~SQL
       SELECT COALESCE(ST_Collect(unnest)::geography, ST_GeogFromText('MULTIPOLYGON EMPTY'))
       FROM (SELECT unnest(#{feature_array_sql})) AS features
       WHERE NOT ST_IsEmpty(unnest)

data/app/models/feature.rb

@@ -9,6 +9,8 @@ class Feature < AbstractFeature
 
   validates_inclusion_of :feature_type, :in => FEATURE_TYPES
 
+  before_save :truncate_name
+
   after_save :refresh_aggregate, if: :automatically_refresh_aggregate?
 
   # Features are used for display so we also cache their KML representation
@@ -60,4 +62,12 @@ class Feature < AbstractFeature
     # this field blank.
     spatial_model_id? && automatically_refresh_aggregate && saved_change_to_geog?
   end
+
+  private
+
+  def truncate_name
+    return unless name?
+    col_size = Feature.columns_hash["name"]&.limit || 255
+    self.name = name.to_s.truncate(col_size)
+  end
 end

data/lib/spatial_features/download.rb

@@ -4,20 +4,30 @@ module SpatialFeatures
   module Download
     # file can be a url, path, or file, any of which can return be a zipped archive
     def self.read(file, unzip: nil, **unzip_options)
-      file = open(file, unzip: unzip, **unzip_options)
+      file = Download.open_each(file, unzip: unzip, **unzip_options).first
       path = ::File.path(file)
       return ::File.read(path)
     end
 
-    def self.open(file, unzip: nil, **unzip_options)
+    # file can be a url, path, or file, any of which can return be a zipped archive
+    def self.open(file)
       file = Kernel.open(file)
       file = normalize_file(file) if file.is_a?(StringIO)
-      if unzip && Unzip.is_zip?(file)
-        file = find_in_zip(file, find: unzip, **unzip_options)
-      end
       return file
     end
 
+    # file can be a url, path, or file, any of which can return be a zipped archive
+    def self.open_each(file, unzip: nil, **unzip_options)
+      file = Download.open(file)
+      files = if unzip && Unzip.is_zip?(file)
+        find_in_zip(file, find: unzip, **unzip_options)
+      else
+        [file]
+      end
+
+      return files.map { |f| File.open(f) }
+    end
+
     def self.normalize_file(file)
       Tempfile.new.tap do |temp|
         temp.binmode
@@ -33,7 +43,7 @@ module SpatialFeatures
     end
 
     def self.find_in_zip(file, find:, **unzip_options)
-      return File.open(Unzip.paths(file, :find => find, **unzip_options))
+      Unzip.paths(file, :find => find, **unzip_options)
     end
   end
 end

data/lib/spatial_features/has_spatial_features/feature_import.rb

@@ -68,8 +68,8 @@ module SpatialFeatures
 
     def spatial_feature_imports(import_options, make_valid)
       import_options.flat_map do |data_method, importer_name|
-        Array.wrap(send(data_method)).map do |data|
-          spatial_importer_from_name(importer_name).new(data, :make_valid => make_valid) if data.present?
+        Array.wrap(send(data_method)).flat_map do |data|
+          spatial_importer_from_name(importer_name).create_all(data, :make_valid => make_valid) if data.present?
         end
       end.compact
     end

data/lib/spatial_features/importers/base.rb

@@ -19,6 +19,12 @@ module SpatialFeatures
         @cache_key ||= Digest::MD5.hexdigest(@data)
       end
 
+      # factory method that should always be used instead of `new` when creating importers
+      # returns an array of Importer::* objects
+      def self.create_all(data, **options)
+        [new(data, **options)]
+      end
+
       private
 
       def build_features

data/lib/spatial_features/importers/file.rb

@@ -3,18 +3,34 @@ require 'open-uri'
 module SpatialFeatures
   module Importers
     class File < SimpleDelegator
-      def initialize(data, *args)
+      INVALID_ARCHIVE = "Archive did not contain a .kml or .shp file. Supported formats are KMZ, KML, and zipped ArcGIS shapefiles.".freeze
+
+      def self.create_all(data, **options)
+        Download.open_each(data, unzip: [/\.kml$/, /\.shp$/], downcase: true).map do |file|
+          new(data, **options, current_file: file)
+        end
+      rescue Unzip::PathNotFound
+        raise ImportError, INVALID_ARCHIVE
+      end
+
+      # The File importer may be initialized multiple times by `::create_all` if it
+      # receives ZIP data containing multiple KML or SHP files. We use `current_file`
+      # to distinguish which file in the archive is currently being
+      # processed.
+      #
+      # If no `current_file` is passed then we just take the first valid file that we find.
+      def initialize(data, *args, current_file: nil, **options)
         begin
-          file = Download.open(data, unzip: [/\.kml$/, /\.shp$/], downcase: true)
+          current_file ||= Download.open_each(data, unzip: [/\.kml$/, /\.shp$/], downcase: true).first
         rescue Unzip::PathNotFound
-          raise ImportError, "Archive did not contain a .kml or .shp file. Supported formats are KMZ, KML, and zipped ArcGIS shapefiles."
+          raise ImportError, INVALID_ARCHIVE
         end
 
-        case ::File.extname(file.path.downcase)
+        case ::File.extname(current_file.path.downcase)
         when '.kml'
-          __setobj__(KMLFile.new(file, *args))
+          __setobj__(KMLFile.new(current_file, *args, **options))
         when '.shp'
-          __setobj__(Shapefile.new(file, *args))
+          __setobj__(Shapefile.new(current_file, *args, **options))
         else
           raise ImportError, "Could not import file. Supported formats are KMZ, KML, and zipped ArcGIS shapefiles"
         end

data/lib/spatial_features/importers/geojson.rb

@@ -0,0 +1,29 @@
+require 'ostruct'
+require 'digest/md5'
+
+module SpatialFeatures
+  module Importers
+    class GeoJSON < Base
+      def cache_key
+        @cache_key ||= Digest::MD5.hexdigest(@data.to_json)
+      end
+
+      private
+
+      def each_record(&block)
+        return unless @data
+
+        @data.fetch('features', []).each do |record|
+          metadata = record['properties'] || {}
+          name = metadata.delete('name')
+          yield OpenStruct.new(
+            :feature_type => record['geometry']['type'],
+            :geog => SpatialFeatures::Utils.geom_from_json(record['geometry']),
+            :name => name,
+            :metadata => metadata
+          )
+        end
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/json_arcgis.rb

@@ -15,7 +15,7 @@ module SpatialFeatures
         json['features'].each do |record|
           yield OpenStruct.new(
             :feature_type => record['geometry']['type'],
-            :geog => geom_from_json(record['geometry']),
+            :geog => SpatialFeatures::Utils.geom_from_json(record['geometry']),
             :metadata => record['properties']
           )
         end
@@ -24,10 +24,6 @@ module SpatialFeatures
       def esri_json_to_geojson(url)
         JSON.parse(`ogr2ogr -f GeoJSON /dev/stdout "#{url}" OGRGeoJSON`)
       end
-
-      def geom_from_json(geometry)
-        ActiveRecord::Base.connection.select_value("SELECT ST_GeomFromGeoJSON('#{geometry.to_json}')")
-      end
     end
   end
 end

data/lib/spatial_features/importers/kml.rb

@@ -35,7 +35,7 @@ module SpatialFeatures
 
         # Do query in a new thread so we use a new connection (if the query fails it will poison the transaction of the current connection)
         Thread.new do
-          geom = ActiveRecord::Base.connection.select_value("SELECT ST_GeomFromKML(#{ActiveRecord::Base.connection.quote(kml.to_s)})")
+          geom = SpatialFeatures::Utils.select_db_value("SELECT ST_GeomFromKML(#{ActiveRecord::Base.connection.quote(kml.to_s)})")
         rescue ActiveRecord::StatementInvalid => e # Discard Invalid KML features
           geom = nil
         end.join

data/lib/spatial_features/importers/shapefile.rb

@@ -7,7 +7,7 @@ module SpatialFeatures
       class_attribute :default_proj4_projection
 
       def initialize(data, *args, proj4: nil, **options)
-        super(data, *args, **options)
+        super(data, **options)
         @proj4 = proj4
       end
 
@@ -15,8 +15,21 @@ module SpatialFeatures
         @cache_key ||= Digest::MD5.file(archive).to_s
       end
 
+      def self.create_all(data, **options)
+        Download.open_each(data, unzip: [/\.shp$/], downcase: true).map do |file|
+          new(file, **options)
+        end
+      rescue Unzip::PathNotFound
+        raise ImportError, INVALID_ARCHIVE
+      end
+
       private
 
+      def build_features
+        validate_shapefile!
+        super
+      end
+
       def each_record(&block)
         RGeo::Shapefile::Reader.open(file.path) do |records|
           records.each do |record|
@@ -49,17 +62,22 @@ module SpatialFeatures
         SQL
       end
 
-
+      # the individual SHP file for processing (automatically extracted from a ZIP archive if necessary)
       def file
-        @file ||= begin
-          validate_file!
-          Download.open(archive, unzip: /\.shp$/, downcase: true)
+        @file ||= Unzip.is_zip?(archive) ? possible_shp_files.first : archive
+      end
+
+      # a zip archive may contain multiple SHP files
+      def possible_shp_files
+        @possible_shp_files ||= begin
+          Download.open_each(archive, unzip: /\.shp$/, downcase: true)
+        rescue Unzip::PathNotFound
+          raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a SHP file"
         end
       end
 
-      def validate_file!
-        return unless Unzip.is_zip?(archive)
-        Validation.validate_shapefile_archive!(Download.entries(archive), default_proj4_projection: default_proj4_projection)
+      def validate_shapefile!
+        Validation.validate_shapefile!(file, default_proj4_projection: default_proj4_projection)
       end
 
       def archive

data/lib/spatial_features/unzip.rb

@@ -6,11 +6,11 @@ module SpatialFeatures
       paths = extract(file_path, **extract_options)
 
       if find = Array.wrap(find).presence
-        paths = paths.detect {|path| find.any? {|pattern| path.index(pattern) } }
-        raise(PathNotFound, "Archive did not contain a file matching #{find}") unless paths.present?
+        paths = paths.select {|path| find.any? {|pattern| path.index(pattern) } }
+        raise(PathNotFound, "Archive did not contain a file matching #{find}") if paths.empty?
       end
 
-      return paths
+      return Array(paths)
     end
 
     def self.extract(file_path, output_dir = Dir.mktmpdir, downcase: false)

data/lib/spatial_features/utils.rb

@@ -52,5 +52,14 @@ module SpatialFeatures
         object.unscope(:select).select(:id).to_sql
       end
     end
+
+    def select_db_value(query)
+      ActiveRecord::Base.connection.select_value(query)
+    end
+
+    # Convert a hash of GeoJSON data into a PostGIS geometry object
+    def geom_from_json(geometry)
+      select_db_value("SELECT ST_GeomFromGeoJSON('#{geometry.to_json}')")
+    end
   end
 end

data/lib/spatial_features/validation.rb

@@ -1,55 +1,49 @@
 module SpatialFeatures
   module Validation
-    # SHP file must come first
     REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS = %w[shp shx dbf prj].freeze
 
-    # Check if a shapefile archive includes the required component files, otherwise
-    # raise an exception.
-    #
-    # @param [Zip::File] zip_file                 A Zip::File object
-    # @param [String] default_proj4_projection    Optional, if supplied we don't raise an exception when we're missing a .PRJ file
-    # @param [Boolean] allow_generic_zip_files    When true, we skip validation entirely if the archive does not contain a .SHP file
-    def self.validate_shapefile_archive!(zip_file, default_proj4_projection: nil, allow_generic_zip_files: false)
-      zip_file_entries = zip_file.entries.each_with_object({}) do |f, obj|
-        ext = File.extname(f.name).downcase[1..-1]
-        next unless ext
-
-        if ext.casecmp?("shp") && obj.key?(ext)
-          raise ::SpatialFeatures::Importers::InvalidShapefileArchive, "Zip files that contain multiple Shapefiles are not supported. Please separate each Shapefile into its own zip file."
+    class << self
+      # Check if a shapefile includes the required component files, otherwise
+      # raise an exception.
+      #
+      # This validation operates by checking sibling files in the same directory,
+      # similar to how `rgeo-shapefile` validates SHP  files.
+      #
+      # @param [File] shp_file                      A File object
+      # @param [String] default_proj4_projection    Optional, if supplied we don't raise an exception when we're missing a .PRJ file
+      def validate_shapefile!(shp_file, default_proj4_projection: nil)
+        basename = File.basename(shp_file.path, '.*')
+        path = shp_file.path.to_s.sub(/\.shp$/i, "")
+
+        required_extensions = REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS
+        required_extensions -= ['prj'] if default_proj4_projection
+
+        required_extensions.each do |ext|
+          component_path = "#{path}.#{ext}"
+          next if ::File.file?(component_path) && ::File.readable?(component_path)
+
+          case ext
+            when "prj"
+              raise ::SpatialFeatures::Importers::IndeterminateShapefileProjection, "Shapefile archive is missing a projection file: #{File.basename(component_path)}"
+            else
+              raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a required file: #{File.basename(component_path)}"
+            end
         end
 
-        obj[ext] = File.basename(f.name, '.*')
+        true
       end
 
-      shapefile_basename = zip_file_entries["shp"]
-      unless shapefile_basename
-        # not a shapefile archive but we don't care
-        return if allow_generic_zip_files
-
-        raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a SHP file"
-      end
-
-      REQUIRED_SHAPEFILE_COMPONENT_EXTENSIONS[1..-1].each do |ext|
-        ext_basename = zip_file_entries[ext]
-        next if ext_basename&.casecmp?(shapefile_basename)
-
-        case ext
-        when "prj"
-          # special case for missing projection files to allow using default_proj4_projection
-          next if default_proj4_projection
-
-          raise ::SpatialFeatures::Importers::IndeterminateShapefileProjection, "Shapefile archive is missing a projection file: #{expected_component_path(shapefile_basename, ext)}"
-        else
-          # for all un-handled cases of missing files raise the more generic error
-          raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a required file: #{expected_component_path(shapefile_basename, ext)}"
+      # Validation helper that takes examines an entire ZIP file
+      #
+      # Useful for validating before persisting records but not used internally
+      def validate_shapefile_archive!(path, default_proj4_projection: nil, allow_generic_zip_files: false)
+        Download.open_each(path, unzip: /\.shp$/, downcase: true).each do |shp_file|
+          validate_shapefile!(shp_file, default_proj4_projection: default_proj4_projection)
         end
+      rescue Unzip::PathNotFound
+        raise ::SpatialFeatures::Importers::IncompleteShapefileArchive, "Shapefile archive is missing a SHP file" \
+          unless allow_generic_zip_files
       end
-
-      true
-    end
-
-    def self.expected_component_path(basename, ext)
-      "#{basename}.#{ext}"
     end
   end
 end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "2.17.2"
+  VERSION = "2.17.3"
 end

data/lib/spatial_features.rb

@@ -20,6 +20,7 @@ require 'spatial_features/has_spatial_features/feature_import'
 
 require 'spatial_features/importers/base'
 require 'spatial_features/importers/file'
+require 'spatial_features/importers/geojson'
 require 'spatial_features/importers/kml'
 require 'spatial_features/importers/kml_file'
 require 'spatial_features/importers/kml_file_arcgis'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 2.17.2
+  version: 2.17.3
 platform: ruby
 authors:
 - Ryan Wallace
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-06 00:00:00.000000000 Z
+date: 2021-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -157,6 +157,7 @@ files:
 - lib/spatial_features/has_spatial_features/queued_spatial_processing.rb
 - lib/spatial_features/importers/base.rb
 - lib/spatial_features/importers/file.rb
+- lib/spatial_features/importers/geojson.rb
 - lib/spatial_features/importers/geomark.rb
 - lib/spatial_features/importers/json_arcgis.rb
 - lib/spatial_features/importers/kml.rb