spatial_features 3.11.1 → 3.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41ad7ed6a989f41b717e103a1f800a60c776ecdf772d3be54dd65128ac84882c
-  data.tar.gz: 6b741f6c962678db2edf963fdd53a87756c657476159f882af36a0bbb1685798
+  metadata.gz: 992823b144d43f8a6c13f32ce0fbd71bb6bea5bafb1aadb7840767f18d2a8d9f
+  data.tar.gz: 45397ed799db69896d94a48e8c3958ab7026420587e9dd1a4755fe57a650132f
 SHA512:
-  metadata.gz: da7049b48402b343a62045dd9910c334240ebfc9d02f0738c5896d3a0033330b4293badfa5e39b9837c34204d3fb221116a79810a812ffa41b3cbf05c0784367
-  data.tar.gz: 5bf14c630adc5bf960a106e0d1ae17b4490770e62dadbcce24dfeaf0cc711ac0e77ef218266d764744a847e66e7bcd0a22cb81e153c7cf2cba1d176d2a3dc266
+  metadata.gz: 9a9c533b9af654d339344dee13b62f14becf6e2c20fdd57868bfe5c185a5bede4657d130a6c9049d0557b274c1ed2a4a7cd70a8b826c3548824ed4b7bf7df76c
+  data.tar.gz: 7a9d9ecdd4f65b01e2cc3c0558ca3699205ecf783617e2adcd4958ca2639dd8240a99b883f1567199ce34ca5a3aad4d5bee4804902c9dddc3cf7e38690015fa5

data/Rakefile

@@ -16,3 +16,5 @@ RSpec::Core::RakeTask.new(:spec) do |task|
 end
 
 task :default => :spec
+
+Dir['tasks/*.rake'].each { |f| load f }

data/lib/spatial_features/download.rb

@@ -1,42 +1,93 @@
+require 'net/http'
 require 'open-uri'
+require 'openssl'
 
 module SpatialFeatures
   module Download
-    # file can be a url, path, or file, any of which can return be a zipped archive
-    def self.open(file)
-      file = URI.open(file)
-      file = normalize_file(file) if file.is_a?(StringIO)
-      return file
-    end
+    REMOTE_URL = %r{\Ahttps?://}i.freeze
+
+    # Seconds to wait for a remote source, applied both to establishing the connection and to
+    # each read. Without it a hung server holds an import worker open indefinitely.
+    mattr_accessor :timeout
+    self.timeout = 60
 
-    # file can be a url, path, or file, any of which can return be a zipped archive
-    def self.open_each(path_or_url, unzip: nil, **unzip_options)
-      file = Download.open(path_or_url)
-      files = if unzip && Unzip.is_zip?(file)
-        find_in_zip(file, find: unzip, **unzip_options)
-      else
-        [file]
+    # Errors meaning a remote source could not be read at all, as opposed to being read and
+    # found unusable. `OpenURI::HTTPError` covers the 4xx and 5xx replies.
+    UNREACHABLE_ERRORS = [::OpenURI::HTTPError, ::SocketError, ::SystemCallError,
+                          ::Net::OpenTimeout, ::Net::ReadTimeout, ::OpenSSL::SSL::SSLError].freeze
+
+    class << self
+      # Returns an open File for `file`, which may be a URL, a path, or a File. The content may
+      # be a zipped archive; `::open_each` unwraps one.
+      #
+      # @raise [SpatialFeatures::ImportError] when a remote source cannot be reached.
+      def open(file)
+        file = fetch(file)
+        file = normalize_file(file) if file.is_a?(StringIO)
+        return file
       end
 
-      return files.map { |f| File.open(f) }
-    end
+      # Returns the body of `path_or_url` as a String, without writing it to disk.
+      #
+      # @raise [SpatialFeatures::ImportError] when a remote source cannot be reached.
+      def read(path_or_url)
+        fetch(path_or_url).read
+      end
+
+      # Returns an open File for each source in `path_or_url`, unwrapping an archive when
+      # `unzip` is given a pattern its entries can match.
+      def open_each(path_or_url, unzip: nil, **unzip_options)
+        file = Download.open(path_or_url)
+        files = if unzip && Unzip.is_zip?(file)
+          find_in_zip(file, find: unzip, **unzip_options)
+        else
+          [file]
+        end
 
-    def self.normalize_file(file)
-      Tempfile.new.tap do |temp|
-        temp.binmode
-        temp.write(file.read)
-        temp.rewind
+        return files.map { |f| File.open(f) }
       end
-    end
 
-    def self.entries(file)
-      file = Kernel.open(file)
-      file = normalize_file(file) if file.is_a?(StringIO)
-      Unzip.entries(file)
-    end
+      def normalize_file(file)
+        Tempfile.new.tap do |temp|
+          temp.binmode
+          temp.write(file.read)
+          temp.rewind
+        end
+      end
+
+      # Returns the entries of the archive at `file` without extracting them.
+      def entries(file)
+        file = fetch(file)
+        file = normalize_file(file) if file.is_a?(StringIO)
+        Unzip.entries(file)
+      end
+
+      def find_in_zip(file, find:, **unzip_options)
+        Unzip.paths(file, find: find, **unzip_options)
+      end
+
+      private
 
-    def self.find_in_zip(file, find:, **unzip_options)
-      Unzip.paths(file, find: find, **unzip_options)
+      # Returns an IO for `file`: fetched over the network when it is a remote URL, opened from
+      # disk when it is any other String, and left to open itself otherwise.
+      #
+      # @note A local path goes to `File.open`, never `URI.open`. `URI.open` hands anything
+      #   that is not a URL to `Kernel#open`, which runs the name as a command when it begins
+      #   with a pipe.
+      # @note Timeouts and the unreachable rescue apply only to a remote URL. `URI.open`
+      #   rejects the timeouts when handed an already open file, and `Errno::ENOENT` for a
+      #   local path is a `SystemCallError` that callers turn into a message for the person who
+      #   uploaded the file, without naming the path the server looked in.
+      def fetch(file)
+        return URI.open(file) unless file.is_a?(String)
+        return File.open(file) unless file.match?(REMOTE_URL)
+
+        begin
+          URI.open(file, :open_timeout => timeout, :read_timeout => timeout)
+        rescue *UNREACHABLE_ERRORS => e
+          raise SpatialFeatures::ImportError, "This source could not be reached. #{e.message}"
+        end
+      end
     end
   end
 end

data/lib/spatial_features/gdal.rb

@@ -0,0 +1,42 @@
+require 'open3'
+
+module SpatialFeatures
+  # Runs the GDAL command line tools.
+  #
+  # Every argument reaches the tool as a single argv entry, so a path or a projection taken
+  # from an uploaded archive arrives as one string rather than as shell syntax.
+  module GDAL
+    class << self
+      # Returns what `tool` wrote to standard output.
+      #
+      # @param tool [String] the executable name, such as `ogr2ogr`.
+      # @param args [Array<String>] one argv entry each.
+      # @return [String] the output, empty when the tool wrote nothing.
+      # @raise [Errno::ENOENT] when the tool is not installed.
+      def capture(tool, *args)
+        Open3.capture2(*argv(tool, args)).first
+      end
+
+      # Runs `tool` for its exit status rather than its output.
+      #
+      # @param tool [String] the executable name, such as `ogr2ogr`.
+      # @param args [Array<String>] one argv entry each.
+      # @return [Boolean] true when the tool exited successfully.
+      # @raise [Errno::ENOENT] when the tool is not installed.
+      def run(tool, *args)
+        system(*argv(tool, args))
+      end
+
+      private
+
+      # Returns the argv to spawn `tool` with.
+      #
+      # @note The command is a two element array so that Ruby spawns the executable directly.
+      #   Both `system` and `Open3` fall back to a shell when handed a lone string, which
+      #   would put every argument here back in reach of shell parsing.
+      def argv(tool, args)
+        [[tool.to_s, tool.to_s], *args.map(&:to_s)]
+      end
+    end
+  end
+end

data/lib/spatial_features/importers/esri_geo_json.rb

@@ -1,9 +1,18 @@
 require 'digest/md5'
+require 'json'
+require 'tempfile'
 require 'spatial_features/importers/geo_json'
 
 module SpatialFeatures
   module Importers
     class ESRIGeoJSON < GeoJSON
+      # How many pages to walk before concluding the endpoint is ignoring `resultOffset` and
+      # serving the same features over and over. A service capped at the usual 1000 or 2000
+      # features per page would have to hold half a million features to reach this
+      # legitimately. Raise it for a layer that genuinely holds more.
+      class_attribute :max_pages
+      self.max_pages = 500
+
       def parsed_geojson
         @parsed_geojson ||= JSON.parse(geojson)
       end
@@ -14,12 +23,115 @@ module SpatialFeatures
 
       private
 
-      def esri_json_to_geojson(url)
-        if URI.parse(url).relative?
-          `ogr2ogr -t_srs EPSG:4326 -f GeoJSON /dev/stdout "#{url}"` # It is a local file path
-        else
-          `ogr2ogr -t_srs EPSG:4326 -f GeoJSON /dev/stdout "#{url}" OGRGeoJSON`
+      # Returns the layer as GeoJSON. A relative path is read from disk; anything else is
+      # downloaded first, so OGR is always given a local file.
+      #
+      # @param path_or_url [String] a local file path, or the URL of an ArcGIS query endpoint.
+      # @return [String] a GeoJSON FeatureCollection in EPSG:4326.
+      # @note Downloading rather than passing the URL to OGR is what makes an endpoint offering
+      #   only HTTPS 1.1 readable. GDAL's curl client gets an empty reply from those servers,
+      #   which surfaces as `ERROR 1: Empty reply from server`.
+      def esri_json_to_geojson(path_or_url)
+        return ogr2ogr_to_geojson(path_or_url) if URI.parse(path_or_url).relative?
+
+        with_downloaded_file(path_or_url) do |path|
+          ogr2ogr_to_geojson(path)
+        end
+      end
+
+      # Returns the GeoJSON OGR reads out of the file at `path`, reprojected to EPSG:4326.
+      # OGR selects its driver by inspecting the content, so the file may hold either GeoJSON
+      # or ESRI JSON.
+      #
+      # @note No layer name is passed. OGR names a local file's layer after its basename, and
+      #   naming a layer that does not exist fails the read.
+      def ogr2ogr_to_geojson(path)
+        GDAL.capture('ogr2ogr', '-t_srs', 'EPSG:4326', '-f', 'GeoJSON', '/dev/stdout', path)
+      end
+
+      # Downloads the query into a tempfile and yields its path, removing it afterwards.
+      def with_downloaded_file(url)
+        Tempfile.create(['esri_geojson', '.json']) do |tempfile|
+          tempfile.binmode
+          download_paginated(url, tempfile)
+          tempfile.close
+          return yield(tempfile.path)
+        end
+      end
+
+      # Walks the query's pages with `resultOffset` and writes them to `io` as one collection.
+      # ArcGIS endpoints cap each response at the service's `maxRecordCount`, commonly 1000 or
+      # 2000 features, and set `exceededTransferLimit` while more results are waiting.
+      #
+      # @note Raises `SpatialFeatures::ImportError` once `max_pages` requests have been made
+      #   and the endpoint still reports more, since a server that ignores `resultOffset`
+      #   otherwise repeats its first page until the process runs out of memory.
+      def download_paginated(url, io)
+        combined = nil
+        offset = 0
+        pages = 0
+
+        loop do
+          page = fetch_page(paginated_url(url, offset))
+          page_features = page['features'] || []
+
+          if combined.nil?
+            combined = page
+          else
+            combined['features'].concat(page_features)
+          end
+
+          break if page_features.empty? || !exceeded_transfer_limit?(page)
+
+          pages += 1
+          if pages >= max_pages
+            raise SpatialFeatures::ImportError,
+                  "This layer was still reporting more features after #{max_pages} requests. " \
+                  "The server may be ignoring the `resultOffset` parameter."
+          end
+
+          offset += page_features.length
         end
+
+        if combined
+          combined.delete('exceededTransferLimit')
+          combined['properties']&.delete('exceededTransferLimit')
+          io.write(JSON.dump(combined))
+        end
+      end
+
+      # Returns one page of the query.
+      #
+      # @return [Hash] the parsed response body.
+      # @raise [SpatialFeatures::ImportError] when the endpoint cannot be reached, or answers
+      #   with a body that is not JSON. An endpoint that rejects a query replies with HTML or
+      #   an error document, which the parse failure alone does not convey.
+      def fetch_page(url)
+        body = Download.read(url)
+        JSON.parse(body)
+      rescue JSON::ParserError
+        raise SpatialFeatures::ImportError,
+              "This layer did not return map data. The server replied with #{body.to_s[0, 100].inspect}."
+      end
+
+      # Returns true while the service reports that more features are waiting. Services set the
+      # flag at the top level or under `properties` depending on the response format.
+      def exceeded_transfer_limit?(page)
+        page['exceededTransferLimit'] || page.dig('properties', 'exceededTransferLimit')
+      end
+
+      # Returns `url` with `resultOffset` set to `offset`, replacing any the caller supplied.
+      # Returns it unchanged for the first page, so a service that does not paginate is asked
+      # exactly what the caller asked for.
+      def paginated_url(url, offset)
+        return url if offset.zero?
+
+        uri = URI.parse(url)
+        params = URI.decode_www_form(uri.query || '')
+        params.reject! { |key, _| key == 'resultOffset' }
+        params << ['resultOffset', offset.to_s]
+        uri.query = URI.encode_www_form(params)
+        uri.to_s
       end
     end
   end

data/lib/spatial_features/importers/shapefile.rb

@@ -55,8 +55,11 @@ module SpatialFeatures
         if proj4 == PROJ4_4326
           data[:geog] = wkt
         else
-          data[:geog] = ActiveRecord::Base.connection.select_value <<-SQL
-            SELECT ST_Transform(ST_GeomFromText('#{wkt}'), '#{proj4}', 4326) AS geog
+          # `proj4` is read out of the uploaded archive's .prj, so it is quoted before it
+          # reaches the statement.
+          conn = ActiveRecord::Base.connection
+          data[:geog] = conn.select_value <<-SQL
+            SELECT ST_Transform(ST_GeomFromText(#{conn.quote(wkt)}), #{conn.quote(proj4)}, 4326) AS geog
           SQL
         end
 
@@ -90,18 +93,27 @@ module SpatialFeatures
       end
 
       # Use OGR2OGR to reproject into EPSG:4326 so we can skip the reprojection step per-feature
+      #
+      # @note Both the projection and the path come from the uploaded archive, so they are
+      #   passed as an argv list and no shell parses them. Assembling a command string here
+      #   would let an uploader run commands of their choosing.
       def project_to_4326(file_path)
         output_path = Tempfile.create([::File.basename(file_path, '.shp') + '_epsg_4326_', '.shp']) { |file| file.path }
         return unless (proj4 = proj4_from_file(file_path))
-        return unless system("ogr2ogr -s_srs '#{proj4}' -t_srs EPSG:4326 '#{output_path}' '#{file_path}'")
+        return unless GDAL.run('ogr2ogr', '-s_srs', proj4, '-t_srs', 'EPSG:4326', output_path, file_path)
         return ::File.open(output_path)
       end
 
+      # Returns the PROJ.4 projection string GDAL reads out of the file's .prj, or nil when
+      # it can't determine one. Returns nil when `gdalsrsinfo` is not installed, which
+      # `proj4_projection` reports.
       def proj4_from_file(file_path)
         # Sanitize: "'+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs '\n" and lately
         #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs \n" to
         #           "+proj=utm +zone=11 +datum=NAD83 +units=m +no_defs"
-        `gdalsrsinfo "#{file_path}" -o proj4`.strip.remove(/^'|'$/).presence
+        GDAL.capture('gdalsrsinfo', file_path, '-o', 'proj4').strip.remove(/^'|'$/).presence
+      rescue Errno::ENOENT
+        nil
       end
 
       # a zip archive may contain multiple SHP files

data/lib/spatial_features/unzip.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'pathname'
 
 module SpatialFeatures
   module Unzip
@@ -51,16 +52,26 @@ module SpatialFeatures
       end
     end
 
+    # Extracts the archive's entries and returns their paths, skipping entries that a name
+    # cannot place inside `tmpdir`.
+    #
+    # @param tmpdir [String] where to extract to. Must already exist, since the destination
+    #   is resolved before anything is written. Defaults to a fresh temporary directory.
     def self.extract(file_path, tmpdir: nil, downcase: false)
       tmpdir ||= Dir.mktmpdir
+      root = Pathname.new(tmpdir).realpath
+
       [].tap do |paths|
         entries(file_path).each do |entry|
           next if entry.name =~ IGNORED_ENTRY_PATHS
+          next if entry.symlink?
 
           output_filename = entry.name
           output_filename = output_filename.downcase if downcase
 
-          path = "#{tmpdir}/#{output_filename}"
+          path = contained_path(root, output_filename)
+          next unless path
+
           directory = File.dirname(path)
           basename = File.basename(path)
 
@@ -72,6 +83,24 @@ module SpatialFeatures
       end
     end
 
+    # Returns where an entry of this name lands under `root`, or nil when the name would place
+    # it outside. Entry names are stored in the archive verbatim, so they can carry `..`
+    # segments that climb out of the directory we extract into.
+    #
+    # @note `root` must already be a real path, and `::extract` skips symlink entries, so
+    #   nothing beneath it is a symlink. `cleanpath` resolves `..` lexically, so a symlink
+    #   under `root` would let an entry name walk back out of the directory unnoticed.
+    def self.contained_path(root, output_filename)
+      path = root.join(output_filename).cleanpath
+
+      return unless path.to_s.start_with?("#{root}#{File::SEPARATOR}")
+
+      # `cleanpath` drops the trailing separator that marks a directory entry, which
+      # `PathNotFound#extensions` reads to tell directories from the files it reports.
+      output_filename.end_with?(File::SEPARATOR) ? "#{path}#{File::SEPARATOR}" : path.to_s
+    end
+    private_class_method :contained_path
+
     def self.names(file_path)
       entries(file_path).collect(&:name)
     end

data/lib/spatial_features/version.rb

@@ -1,3 +1,3 @@
 module SpatialFeatures
-  VERSION = "3.11.1"
+  VERSION = "3.12.0"
 end

data/lib/spatial_features.rb

@@ -10,6 +10,7 @@ require 'spatial_features/caching'
 require 'spatial_features/uncached_result'
 require 'spatial_features/venn_polygons'
 require 'spatial_features/controller_helpers/spatial_extensions'
+require 'spatial_features/gdal'
 require 'spatial_features/download'
 require 'spatial_features/unzip'
 require 'spatial_features/utils'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spatial_features
 version: !ruby/object:Gem::Version
-  version: 3.11.1
+  version: 3.12.0
 platform: ruby
 authors:
 - Ryan Wallace
@@ -198,6 +198,7 @@ files:
 - lib/spatial_features/controller_helpers/spatial_extensions.rb
 - lib/spatial_features/download.rb
 - lib/spatial_features/engine.rb
+- lib/spatial_features/gdal.rb
 - lib/spatial_features/has_spatial_features.rb
 - lib/spatial_features/has_spatial_features/feature_import.rb
 - lib/spatial_features/has_spatial_features/queued_spatial_processing.rb
@@ -236,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.5
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Adds spatial methods to a model.
 test_files: []